1/*
2 * Copyright (c) 2010-2013, 2016-2019 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2001-2005 The Regents of The University of Michigan
15 * All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 *
40 * Authors: Ali Saidi
41 * Nathan Binkert
42 * Steve Reinhardt
43 */
44
45#include "arch/arm/tlb.hh"
46
47#include <memory>
48#include <string>
49#include <vector>
50
51#include "arch/arm/faults.hh"
52#include "arch/arm/pagetable.hh"
53#include "arch/arm/stage2_lookup.hh"
54#include "arch/arm/stage2_mmu.hh"
55#include "arch/arm/system.hh"
56#include "arch/arm/table_walker.hh"
57#include "arch/arm/utility.hh"
58#include "arch/generic/mmapped_ipr.hh"
59#include "base/inifile.hh"
60#include "base/str.hh"
61#include "base/trace.hh"
62#include "cpu/base.hh"
63#include "cpu/thread_context.hh"
64#include "debug/Checkpoint.hh"
65#include "debug/TLB.hh"
66#include "debug/TLBVerbose.hh"
67#include "mem/page_table.hh"
68#include "mem/request.hh"
69#include "params/ArmTLB.hh"
70#include "sim/full_system.hh"
71#include "sim/process.hh"
72
73using namespace std;
74using namespace ArmISA;
75
76TLB::TLB(const ArmTLBParams *p)
77 : BaseTLB(p), table(new TlbEntry[p->size]), size(p->size),
78 isStage2(p->is_stage2), stage2Req(false), stage2DescReq(false), _attr(0),
79 directToStage2(false), tableWalker(p->walker), stage2Tlb(NULL),
80 stage2Mmu(NULL), test(nullptr), rangeMRU(1),
81 aarch64(false), aarch64EL(EL0), isPriv(false), isSecure(false),
82 isHyp(false), asid(0), vmid(0), hcr(0), dacr(0),
83 miscRegValid(false), miscRegContext(0), curTranType(NormalTran)
84{
85 const ArmSystem *sys = dynamic_cast<const ArmSystem *>(p->sys);
86
87 tableWalker->setTlb(this);
88
89 // Cache system-level properties
90 haveLPAE = tableWalker->haveLPAE();
91 haveVirtualization = tableWalker->haveVirtualization();
92 haveLargeAsid64 = tableWalker->haveLargeAsid64();
93
94 if (sys)
95 m5opRange = sys->m5opRange();
96}
97
98TLB::~TLB()
99{
100 delete[] table;
101}
102
103void
104TLB::init()
105{
106 if (stage2Mmu && !isStage2)
107 stage2Tlb = stage2Mmu->stage2Tlb();
108}
109
110void
111TLB::setMMU(Stage2MMU *m, MasterID master_id)
112{
113 stage2Mmu = m;
114 tableWalker->setMMU(m, master_id);
115}
116
117bool
118TLB::translateFunctional(ThreadContext *tc, Addr va, Addr &pa)
119{
120 updateMiscReg(tc);
121
122 if (directToStage2) {
123 assert(stage2Tlb);
124 return stage2Tlb->translateFunctional(tc, va, pa);
125 }
126
127 TlbEntry *e = lookup(va, asid, vmid, isHyp, isSecure, true, false,
128 aarch64 ? aarch64EL : EL1);
129 if (!e)
130 return false;
131 pa = e->pAddr(va);
132 return true;
133}
134
135Fault
136TLB::finalizePhysical(const RequestPtr &req,
137 ThreadContext *tc, Mode mode) const
138{
139 const Addr paddr = req->getPaddr();
140
141 if (m5opRange.contains(paddr)) {
142 req->setFlags(Request::MMAPPED_IPR | Request::GENERIC_IPR);
143 req->setPaddr(GenericISA::iprAddressPseudoInst(
144 (paddr >> 8) & 0xFF,
145 paddr & 0xFF));
146 }
147
148 return NoFault;
149}
150
151TlbEntry*
152TLB::lookup(Addr va, uint16_t asn, uint8_t vmid, bool hyp, bool secure,
153 bool functional, bool ignore_asn, ExceptionLevel target_el)
154{
155
156 TlbEntry *retval = NULL;
157
158 // Maintaining LRU array
159 int x = 0;
160 while (retval == NULL && x < size) {
161 if ((!ignore_asn && table[x].match(va, asn, vmid, hyp, secure, false,
162 target_el)) ||
163 (ignore_asn && table[x].match(va, vmid, hyp, secure, target_el))) {
164 // We only move the hit entry ahead when the position is higher
165 // than rangeMRU
166 if (x > rangeMRU && !functional) {
167 TlbEntry tmp_entry = table[x];
168 for (int i = x; i > 0; i--)
169 table[i] = table[i - 1];
170 table[0] = tmp_entry;
171 retval = &table[0];
172 } else {
173 retval = &table[x];
174 }
175 break;
176 }
177 ++x;
178 }
179
180 DPRINTF(TLBVerbose, "Lookup %#x, asn %#x -> %s vmn 0x%x hyp %d secure %d "
181 "ppn %#x size: %#x pa: %#x ap:%d ns:%d nstid:%d g:%d asid: %d "
182 "el: %d\n",
183 va, asn, retval ? "hit" : "miss", vmid, hyp, secure,
184 retval ? retval->pfn : 0, retval ? retval->size : 0,
185 retval ? retval->pAddr(va) : 0, retval ? retval->ap : 0,
186 retval ? retval->ns : 0, retval ? retval->nstid : 0,
187 retval ? retval->global : 0, retval ? retval->asid : 0,
188 retval ? retval->el : 0);
189
190 return retval;
191}
192
193// insert a new TLB entry
194void
195TLB::insert(Addr addr, TlbEntry &entry)
196{
197 DPRINTF(TLB, "Inserting entry into TLB with pfn:%#x size:%#x vpn: %#x"
198 " asid:%d vmid:%d N:%d global:%d valid:%d nc:%d xn:%d"
199 " ap:%#x domain:%#x ns:%d nstid:%d isHyp:%d\n", entry.pfn,
200 entry.size, entry.vpn, entry.asid, entry.vmid, entry.N,
201 entry.global, entry.valid, entry.nonCacheable, entry.xn,
202 entry.ap, static_cast<uint8_t>(entry.domain), entry.ns, entry.nstid,
203 entry.isHyp);
204
205 if (table[size - 1].valid)
206 DPRINTF(TLB, " - Replacing Valid entry %#x, asn %d vmn %d ppn %#x "
207 "size: %#x ap:%d ns:%d nstid:%d g:%d isHyp:%d el: %d\n",
208 table[size-1].vpn << table[size-1].N, table[size-1].asid,
209 table[size-1].vmid, table[size-1].pfn << table[size-1].N,
210 table[size-1].size, table[size-1].ap, table[size-1].ns,
211 table[size-1].nstid, table[size-1].global, table[size-1].isHyp,
212 table[size-1].el);
213
214 //inserting to MRU position and evicting the LRU one
215
216 for (int i = size - 1; i > 0; --i)
217 table[i] = table[i-1];
218 table[0] = entry;
219
220 inserts++;
221 ppRefills->notify(1);
222}
223
224void
225TLB::printTlb() const
226{
227 int x = 0;
228 TlbEntry *te;
229 DPRINTF(TLB, "Current TLB contents:\n");
230 while (x < size) {
231 te = &table[x];
232 if (te->valid)
233 DPRINTF(TLB, " * %s\n", te->print());
234 ++x;
235 }
236}
237
238void
239TLB::flushAllSecurity(bool secure_lookup, ExceptionLevel target_el,
240 bool ignore_el)
241{
242 DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n",
243 (secure_lookup ? "secure" : "non-secure"));
244 int x = 0;
245 TlbEntry *te;
246 while (x < size) {
247 te = &table[x];
248 const bool el_match = ignore_el ?
249 true : te->checkELMatch(target_el);
250
251 if (te->valid && secure_lookup == !te->nstid &&
252 (te->vmid == vmid || secure_lookup) && el_match) {
253
254 DPRINTF(TLB, " - %s\n", te->print());
255 te->valid = false;
256 flushedEntries++;
257 }
258 ++x;
259 }
260
261 flushTlb++;
262
263 // If there's a second stage TLB (and we're not it) then flush it as well
264 // if we're currently in hyp mode
265 if (!isStage2 && isHyp) {
266 stage2Tlb->flushAllSecurity(secure_lookup, EL1, true);
267 }
268}
269
270void
271TLB::flushAllNs(ExceptionLevel target_el, bool ignore_el)
272{
273 bool hyp = target_el == EL2;
274
275 DPRINTF(TLB, "Flushing all NS TLB entries (%s lookup)\n",
276 (hyp ? "hyp" : "non-hyp"));
277 int x = 0;
278 TlbEntry *te;
279 while (x < size) {
280 te = &table[x];
281 const bool el_match = ignore_el ?
282 true : te->checkELMatch(target_el);
283
284 if (te->valid && te->nstid && te->isHyp == hyp && el_match) {
285
286 DPRINTF(TLB, " - %s\n", te->print());
287 flushedEntries++;
288 te->valid = false;
289 }
290 ++x;
291 }
292
293 flushTlb++;
294
295 // If there's a second stage TLB (and we're not it) then flush it as well
296 if (!isStage2 && !hyp) {
297 stage2Tlb->flushAllNs(EL1, true);
298 }
299}
300
301void
302TLB::flushMvaAsid(Addr mva, uint64_t asn, bool secure_lookup,
303 ExceptionLevel target_el)
304{
305 DPRINTF(TLB, "Flushing TLB entries with mva: %#x, asid: %#x "
306 "(%s lookup)\n", mva, asn, (secure_lookup ?
307 "secure" : "non-secure"));
308 _flushMva(mva, asn, secure_lookup, false, target_el);
309 flushTlbMvaAsid++;
310}
311
312void
313TLB::flushAsid(uint64_t asn, bool secure_lookup, ExceptionLevel target_el)
314{
315 DPRINTF(TLB, "Flushing TLB entries with asid: %#x (%s lookup)\n", asn,
316 (secure_lookup ? "secure" : "non-secure"));
317
318 int x = 0 ;
319 TlbEntry *te;
320
321 while (x < size) {
322 te = &table[x];
323 if (te->valid && te->asid == asn && secure_lookup == !te->nstid &&
324 (te->vmid == vmid || secure_lookup) &&
325 te->checkELMatch(target_el)) {
326
327 te->valid = false;
328 DPRINTF(TLB, " - %s\n", te->print());
329 flushedEntries++;
330 }
331 ++x;
332 }
333 flushTlbAsid++;
334}
335
336void
337TLB::flushMva(Addr mva, bool secure_lookup, ExceptionLevel target_el)
338{
339 DPRINTF(TLB, "Flushing TLB entries with mva: %#x (%s lookup)\n", mva,
340 (secure_lookup ? "secure" : "non-secure"));
341 _flushMva(mva, 0xbeef, secure_lookup, true, target_el);
342 flushTlbMva++;
343}
344
345void
346TLB::_flushMva(Addr mva, uint64_t asn, bool secure_lookup,
347 bool ignore_asn, ExceptionLevel target_el)
348{
349 TlbEntry *te;
350 // D5.7.2: Sign-extend address to 64 bits
351 mva = sext<56>(mva);
352
353 bool hyp = target_el == EL2;
354
355 te = lookup(mva, asn, vmid, hyp, secure_lookup, false, ignore_asn,
356 target_el);
357 while (te != NULL) {
358 if (secure_lookup == !te->nstid) {
359 DPRINTF(TLB, " - %s\n", te->print());
360 te->valid = false;
361 flushedEntries++;
362 }
363 te = lookup(mva, asn, vmid, hyp, secure_lookup, false, ignore_asn,
364 target_el);
365 }
366}
367
368void
369TLB::flushIpaVmid(Addr ipa, bool secure_lookup, ExceptionLevel target_el)
370{
371 assert(!isStage2);
372 stage2Tlb->_flushMva(ipa, 0xbeef, secure_lookup, true, target_el);
373}
374
375void
376TLB::drainResume()
377{
378 // We might have unserialized something or switched CPUs, so make
379 // sure to re-read the misc regs.
380 miscRegValid = false;
381}
382
383void
384TLB::takeOverFrom(BaseTLB *_otlb)
385{
386 TLB *otlb = dynamic_cast<TLB*>(_otlb);
387 /* Make sure we actually have a valid type */
388 if (otlb) {
389 _attr = otlb->_attr;
390 haveLPAE = otlb->haveLPAE;
391 directToStage2 = otlb->directToStage2;
392 stage2Req = otlb->stage2Req;
393 stage2DescReq = otlb->stage2DescReq;
394
395 /* Sync the stage2 MMU if they exist in both
396 * the old CPU and the new
397 */
398 if (!isStage2 &&
399 stage2Tlb && otlb->stage2Tlb) {
400 stage2Tlb->takeOverFrom(otlb->stage2Tlb);
401 }
402 } else {
403 panic("Incompatible TLB type!");
404 }
405}
406
407void
408TLB::serialize(CheckpointOut &cp) const
409{
410 DPRINTF(Checkpoint, "Serializing Arm TLB\n");
411
412 SERIALIZE_SCALAR(_attr);
413 SERIALIZE_SCALAR(haveLPAE);
414 SERIALIZE_SCALAR(directToStage2);
415 SERIALIZE_SCALAR(stage2Req);
416 SERIALIZE_SCALAR(stage2DescReq);
417
418 int num_entries = size;
419 SERIALIZE_SCALAR(num_entries);
420 for (int i = 0; i < size; i++)
421 table[i].serializeSection(cp, csprintf("TlbEntry%d", i));
422}
423
424void
425TLB::unserialize(CheckpointIn &cp)
426{
427 DPRINTF(Checkpoint, "Unserializing Arm TLB\n");
428
429 UNSERIALIZE_SCALAR(_attr);
430 UNSERIALIZE_SCALAR(haveLPAE);
431 UNSERIALIZE_SCALAR(directToStage2);
432 UNSERIALIZE_SCALAR(stage2Req);
433 UNSERIALIZE_SCALAR(stage2DescReq);
434
435 int num_entries;
436 UNSERIALIZE_SCALAR(num_entries);
437 for (int i = 0; i < min(size, num_entries); i++)
438 table[i].unserializeSection(cp, csprintf("TlbEntry%d", i));
439}
440
441void
442TLB::regStats()
443{
444 BaseTLB::regStats();
445 instHits
446 .name(name() + ".inst_hits")
447 .desc("ITB inst hits")
448 ;
449
450 instMisses
451 .name(name() + ".inst_misses")
452 .desc("ITB inst misses")
453 ;
454
455 instAccesses
456 .name(name() + ".inst_accesses")
457 .desc("ITB inst accesses")
458 ;
459
460 readHits
461 .name(name() + ".read_hits")
462 .desc("DTB read hits")
463 ;
464
465 readMisses
466 .name(name() + ".read_misses")
467 .desc("DTB read misses")
468 ;
469
470 readAccesses
471 .name(name() + ".read_accesses")
472 .desc("DTB read accesses")
473 ;
474
475 writeHits
476 .name(name() + ".write_hits")
477 .desc("DTB write hits")
478 ;
479
480 writeMisses
481 .name(name() + ".write_misses")
482 .desc("DTB write misses")
483 ;
484
485 writeAccesses
486 .name(name() + ".write_accesses")
487 .desc("DTB write accesses")
488 ;
489
490 hits
491 .name(name() + ".hits")
492 .desc("DTB hits")
493 ;
494
495 misses
496 .name(name() + ".misses")
497 .desc("DTB misses")
498 ;
499
500 accesses
501 .name(name() + ".accesses")
502 .desc("DTB accesses")
503 ;
504
505 flushTlb
506 .name(name() + ".flush_tlb")
507 .desc("Number of times complete TLB was flushed")
508 ;
509
510 flushTlbMva
511 .name(name() + ".flush_tlb_mva")
512 .desc("Number of times TLB was flushed by MVA")
513 ;
514
515 flushTlbMvaAsid
516 .name(name() + ".flush_tlb_mva_asid")
517 .desc("Number of times TLB was flushed by MVA & ASID")
518 ;
519
520 flushTlbAsid
521 .name(name() + ".flush_tlb_asid")
522 .desc("Number of times TLB was flushed by ASID")
523 ;
524
525 flushedEntries
526 .name(name() + ".flush_entries")
527 .desc("Number of entries that have been flushed from TLB")
528 ;
529
530 alignFaults
531 .name(name() + ".align_faults")
532 .desc("Number of TLB faults due to alignment restrictions")
533 ;
534
535 prefetchFaults
536 .name(name() + ".prefetch_faults")
537 .desc("Number of TLB faults due to prefetch")
538 ;
539
540 domainFaults
541 .name(name() + ".domain_faults")
542 .desc("Number of TLB faults due to domain restrictions")
543 ;
544
545 permsFaults
546 .name(name() + ".perms_faults")
547 .desc("Number of TLB faults due to permissions restrictions")
548 ;
549
550 instAccesses = instHits + instMisses;
551 readAccesses = readHits + readMisses;
552 writeAccesses = writeHits + writeMisses;
553 hits = readHits + writeHits + instHits;
554 misses = readMisses + writeMisses + instMisses;
555 accesses = readAccesses + writeAccesses + instAccesses;
556}
557
558void
559TLB::regProbePoints()
560{
561 ppRefills.reset(new ProbePoints::PMU(getProbeManager(), "Refills"));
562}
563
564Fault
565TLB::translateSe(const RequestPtr &req, ThreadContext *tc, Mode mode,
566 Translation *translation, bool &delay, bool timing)
567{
568 updateMiscReg(tc);
569 Addr vaddr_tainted = req->getVaddr();
570 Addr vaddr = 0;
571 if (aarch64)
572 vaddr = purifyTaggedAddr(vaddr_tainted, tc, aarch64EL, ttbcr);
573 else
574 vaddr = vaddr_tainted;
575 Request::Flags flags = req->getFlags();
576
577 bool is_fetch = (mode == Execute);
578 bool is_write = (mode == Write);
579
580 if (!is_fetch) {
581 assert(flags & MustBeOne || req->isPrefetch());
582 if (sctlr.a || !(flags & AllowUnaligned)) {
583 if (vaddr & mask(flags & AlignmentMask)) {
584 // LPAE is always disabled in SE mode
585 return std::make_shared<DataAbort>(
586 vaddr_tainted,
587 TlbEntry::DomainType::NoAccess, is_write,
588 ArmFault::AlignmentFault, isStage2,
589 ArmFault::VmsaTran);
590 }
591 }
592 }
593
594 Addr paddr;
595 Process *p = tc->getProcessPtr();
596
597 if (!p->pTable->translate(vaddr, paddr))
598 return std::make_shared<GenericPageTableFault>(vaddr_tainted);
599 req->setPaddr(paddr);
600
601 return finalizePhysical(req, tc, mode);
602}
603
604Fault
605TLB::checkPermissions(TlbEntry *te, const RequestPtr &req, Mode mode)
606{
607 // a data cache maintenance instruction that operates by MVA does
608 // not generate a Data Abort exeception due to a Permission fault
609 if (req->isCacheMaintenance()) {
610 return NoFault;
611 }
612
613 Addr vaddr = req->getVaddr(); // 32-bit don't have to purify
614 Request::Flags flags = req->getFlags();
615 bool is_fetch = (mode == Execute);
616 bool is_write = (mode == Write);
617 bool is_priv = isPriv && !(flags & UserMode);
618
619 // Get the translation type from the actuall table entry
620 ArmFault::TranMethod tranMethod = te->longDescFormat ? ArmFault::LpaeTran
621 : ArmFault::VmsaTran;
622
623 // If this is the second stage of translation and the request is for a
624 // stage 1 page table walk then we need to check the HCR.PTW bit. This
625 // allows us to generate a fault if the request targets an area marked
626 // as a device or strongly ordered.
627 if (isStage2 && req->isPTWalk() && hcr.ptw &&
628 (te->mtype != TlbEntry::MemoryType::Normal)) {
629 return std::make_shared<DataAbort>(
630 vaddr, te->domain, is_write,
631 ArmFault::PermissionLL + te->lookupLevel,
632 isStage2, tranMethod);
633 }
634
635 // Generate an alignment fault for unaligned data accesses to device or
636 // strongly ordered memory
637 if (!is_fetch) {
638 if (te->mtype != TlbEntry::MemoryType::Normal) {
639 if (vaddr & mask(flags & AlignmentMask)) {
640 alignFaults++;
641 return std::make_shared<DataAbort>(
642 vaddr, TlbEntry::DomainType::NoAccess, is_write,
643 ArmFault::AlignmentFault, isStage2,
644 tranMethod);
645 }
646 }
647 }
648
649 if (te->nonCacheable) {
650 // Prevent prefetching from I/O devices.
651 if (req->isPrefetch()) {
652 // Here we can safely use the fault status for the short
653 // desc. format in all cases
654 return std::make_shared<PrefetchAbort>(
655 vaddr, ArmFault::PrefetchUncacheable,
656 isStage2, tranMethod);
657 }
658 }
659
660 if (!te->longDescFormat) {
661 switch ((dacr >> (static_cast<uint8_t>(te->domain) * 2)) & 0x3) {
662 case 0:
663 domainFaults++;
664 DPRINTF(TLB, "TLB Fault: Data abort on domain. DACR: %#x"
665 " domain: %#x write:%d\n", dacr,
666 static_cast<uint8_t>(te->domain), is_write);
667 if (is_fetch) {
668 // Use PC value instead of vaddr because vaddr might
669 // be aligned to cache line and should not be the
670 // address reported in FAR
671 return std::make_shared<PrefetchAbort>(
672 req->getPC(),
673 ArmFault::DomainLL + te->lookupLevel,
674 isStage2, tranMethod);
675 } else
676 return std::make_shared<DataAbort>(
677 vaddr, te->domain, is_write,
678 ArmFault::DomainLL + te->lookupLevel,
679 isStage2, tranMethod);
680 case 1:
681 // Continue with permissions check
682 break;
683 case 2:
684 panic("UNPRED domain\n");
685 case 3:
686 return NoFault;
687 }
688 }
689
690 // The 'ap' variable is AP[2:0] or {AP[2,1],1b'0}, i.e. always three bits
691 uint8_t ap = te->longDescFormat ? te->ap << 1 : te->ap;
692 uint8_t hap = te->hap;
693
694 if (sctlr.afe == 1 || te->longDescFormat)
695 ap |= 1;
696
697 bool abt;
698 bool isWritable = true;
699 // If this is a stage 2 access (eg for reading stage 1 page table entries)
700 // then don't perform the AP permissions check, we stil do the HAP check
701 // below.
702 if (isStage2) {
703 abt = false;
704 } else {
705 switch (ap) {
706 case 0:
707 DPRINTF(TLB, "Access permissions 0, checking rs:%#x\n",
708 (int)sctlr.rs);
709 if (!sctlr.xp) {
710 switch ((int)sctlr.rs) {
711 case 2:
712 abt = is_write;
713 break;
714 case 1:
715 abt = is_write || !is_priv;
716 break;
717 case 0:
718 case 3:
719 default:
720 abt = true;
721 break;
722 }
723 } else {
724 abt = true;
725 }
726 break;
727 case 1:
728 abt = !is_priv;
729 break;
730 case 2:
731 abt = !is_priv && is_write;
732 isWritable = is_priv;
733 break;
734 case 3:
735 abt = false;
736 break;
737 case 4:
738 panic("UNPRED premissions\n");
739 case 5:
740 abt = !is_priv || is_write;
741 isWritable = false;
742 break;
743 case 6:
744 case 7:
745 abt = is_write;
746 isWritable = false;
747 break;
748 default:
749 panic("Unknown permissions %#x\n", ap);
750 }
751 }
752
753 bool hapAbt = is_write ? !(hap & 2) : !(hap & 1);
754 bool xn = te->xn || (isWritable && sctlr.wxn) ||
755 (ap == 3 && sctlr.uwxn && is_priv);
756 if (is_fetch && (abt || xn ||
757 (te->longDescFormat && te->pxn && is_priv) ||
758 (isSecure && te->ns && scr.sif))) {
759 permsFaults++;
760 DPRINTF(TLB, "TLB Fault: Prefetch abort on permission check. AP:%d "
761 "priv:%d write:%d ns:%d sif:%d sctlr.afe: %d \n",
762 ap, is_priv, is_write, te->ns, scr.sif,sctlr.afe);
763 // Use PC value instead of vaddr because vaddr might be aligned to
764 // cache line and should not be the address reported in FAR
765 return std::make_shared<PrefetchAbort>(
766 req->getPC(),
767 ArmFault::PermissionLL + te->lookupLevel,
768 isStage2, tranMethod);
769 } else if (abt | hapAbt) {
770 permsFaults++;
771 DPRINTF(TLB, "TLB Fault: Data abort on permission check. AP:%d priv:%d"
772 " write:%d\n", ap, is_priv, is_write);
773 return std::make_shared<DataAbort>(
774 vaddr, te->domain, is_write,
775 ArmFault::PermissionLL + te->lookupLevel,
776 isStage2 | !abt, tranMethod);
777 }
778 return NoFault;
779}
780
781
782Fault
783TLB::checkPermissions64(TlbEntry *te, const RequestPtr &req, Mode mode,
784 ThreadContext *tc)
785{
786 assert(aarch64);
787
788 // A data cache maintenance instruction that operates by VA does
789 // not generate a Permission fault unless:
790 // * It is a data cache invalidate (dc ivac) which requires write
791 // permissions to the VA, or
792 // * It is executed from EL0
793 if (req->isCacheClean() && aarch64EL != EL0 && !isStage2) {
794 return NoFault;
795 }
796
797 Addr vaddr_tainted = req->getVaddr();
798 Addr vaddr = purifyTaggedAddr(vaddr_tainted, tc, aarch64EL, ttbcr);
799
800 Request::Flags flags = req->getFlags();
801 bool is_fetch = (mode == Execute);
802 // Cache clean operations require read permissions to the specified VA
803 bool is_write = !req->isCacheClean() && mode == Write;
804 bool is_priv M5_VAR_USED = isPriv && !(flags & UserMode);
805
806 updateMiscReg(tc, curTranType);
807
808 // If this is the second stage of translation and the request is for a
809 // stage 1 page table walk then we need to check the HCR.PTW bit. This
810 // allows us to generate a fault if the request targets an area marked
811 // as a device or strongly ordered.
812 if (isStage2 && req->isPTWalk() && hcr.ptw &&
813 (te->mtype != TlbEntry::MemoryType::Normal)) {
814 return std::make_shared<DataAbort>(
815 vaddr_tainted, te->domain, is_write,
816 ArmFault::PermissionLL + te->lookupLevel,
817 isStage2, ArmFault::LpaeTran);
818 }
819
820 // Generate an alignment fault for unaligned accesses to device or
821 // strongly ordered memory
822 if (!is_fetch) {
823 if (te->mtype != TlbEntry::MemoryType::Normal) {
824 if (vaddr & mask(flags & AlignmentMask)) {
825 alignFaults++;
826 return std::make_shared<DataAbort>(
827 vaddr_tainted,
828 TlbEntry::DomainType::NoAccess, is_write,
829 ArmFault::AlignmentFault, isStage2,
830 ArmFault::LpaeTran);
831 }
832 }
833 }
834
835 if (te->nonCacheable) {
836 // Prevent prefetching from I/O devices.
837 if (req->isPrefetch()) {
838 // Here we can safely use the fault status for the short
839 // desc. format in all cases
840 return std::make_shared<PrefetchAbort>(
841 vaddr_tainted,
842 ArmFault::PrefetchUncacheable,
843 isStage2, ArmFault::LpaeTran);
844 }
845 }
846
847 uint8_t ap = 0x3 & (te->ap); // 2-bit access protection field
848 bool grant = false;
849
850 uint8_t xn = te->xn;
851 uint8_t pxn = te->pxn;
852 bool r = !is_write && !is_fetch;
853 bool w = is_write;
854 bool x = is_fetch;
855 DPRINTF(TLBVerbose, "Checking permissions: ap:%d, xn:%d, pxn:%d, r:%d, "
856 "w:%d, x:%d\n", ap, xn, pxn, r, w, x);
857
858 if (isStage2) {
859 assert(ArmSystem::haveVirtualization(tc) && aarch64EL != EL2);
860 // In stage 2 we use the hypervisor access permission bits.
861 // The following permissions are described in ARM DDI 0487A.f
862 // D4-1802
863 uint8_t hap = 0x3 & te->hap;
864 if (is_fetch) {
865 // sctlr.wxn overrides the xn bit
866 grant = !sctlr.wxn && !xn;
867 } else if (is_write) {
868 grant = hap & 0x2;
869 } else { // is_read
870 grant = hap & 0x1;
871 }
872 } else {
873 switch (aarch64EL) {
874 case EL0:
875 {
876 uint8_t perm = (ap << 2) | (xn << 1) | pxn;
877 switch (perm) {
878 case 0:
879 case 1:
880 case 8:
881 case 9:
882 grant = x;
883 break;
884 case 4:
885 case 5:
886 grant = r || w || (x && !sctlr.wxn);
887 break;
888 case 6:
889 case 7:
890 grant = r || w;
891 break;
892 case 12:
893 case 13:
894 grant = r || x;
895 break;
896 case 14:
897 case 15:
898 grant = r;
899 break;
900 default:
901 grant = false;
902 }
903 }
904 break;
905 case EL1:
906 {
907 if (checkPAN(tc, ap, req, mode)) {
908 grant = false;
909 break;
910 }
911
912 uint8_t perm = (ap << 2) | (xn << 1) | pxn;
913 switch (perm) {
914 case 0:
915 case 2:
916 grant = r || w || (x && !sctlr.wxn);
917 break;
918 case 1:
919 case 3:
920 case 4:
921 case 5:
922 case 6:
923 case 7:
924 // regions that are writeable at EL0 should not be
925 // executable at EL1
926 grant = r || w;
927 break;
928 case 8:
929 case 10:
930 case 12:
931 case 14:
932 grant = r || x;
933 break;
934 case 9:
935 case 11:
936 case 13:
937 case 15:
938 grant = r;
939 break;
940 default:
941 grant = false;
942 }
943 }
944 break;
945 case EL2:
946 if (checkPAN(tc, ap, req, mode)) {
947 grant = false;
948 break;
949 }
950 M5_FALLTHROUGH;
951 case EL3:
952 {
953 uint8_t perm = (ap & 0x2) | xn;
954 switch (perm) {
955 case 0:
956 grant = r || w || (x && !sctlr.wxn) ;
957 break;
958 case 1:
959 grant = r || w;
960 break;
961 case 2:
962 grant = r || x;
963 break;
964 case 3:
965 grant = r;
966 break;
967 default:
968 grant = false;
969 }
970 }
971 break;
972 }
973 }
974
975 if (!grant) {
976 if (is_fetch) {
977 permsFaults++;
978 DPRINTF(TLB, "TLB Fault: Prefetch abort on permission check. "
979 "AP:%d priv:%d write:%d ns:%d sif:%d "
980 "sctlr.afe: %d\n",
981 ap, is_priv, is_write, te->ns, scr.sif, sctlr.afe);
982 // Use PC value instead of vaddr because vaddr might be aligned to
983 // cache line and should not be the address reported in FAR
984 return std::make_shared<PrefetchAbort>(
985 req->getPC(),
986 ArmFault::PermissionLL + te->lookupLevel,
987 isStage2, ArmFault::LpaeTran);
988 } else {
989 permsFaults++;
990 DPRINTF(TLB, "TLB Fault: Data abort on permission check. AP:%d "
991 "priv:%d write:%d\n", ap, is_priv, is_write);
992 return std::make_shared<DataAbort>(
993 vaddr_tainted, te->domain, is_write,
994 ArmFault::PermissionLL + te->lookupLevel,
995 isStage2, ArmFault::LpaeTran);
996 }
997 }
998
999 return NoFault;
1000}
1001
1002bool
1003TLB::checkPAN(ThreadContext *tc, uint8_t ap, const RequestPtr &req, Mode mode)
1004{
1005 // The PAN bit has no effect on:
1006 // 1) Instruction accesses.
1007 // 2) Data Cache instructions other than DC ZVA
1008 // 3) Address translation instructions, other than ATS1E1RP and
1009 // ATS1E1WP when ARMv8.2-ATS1E1 is implemented. (Unimplemented in
1010 // gem5)
1011 // 4) Unprivileged instructions (Unimplemented in gem5)
1012 AA64MMFR1 mmfr1 = tc->readMiscReg(MISCREG_ID_AA64MMFR1_EL1);
1013 if (mmfr1.pan && cpsr.pan && (ap & 0x1) && mode != Execute &&
1014 (!req->isCacheMaintenance() ||
1015 (req->getFlags() & Request::CACHE_BLOCK_ZERO))) {
1016 return true;
1017 } else {
1018 return false;
1019 }
1020}
1021
1022Fault
1023TLB::translateFs(const RequestPtr &req, ThreadContext *tc, Mode mode,
1024 Translation *translation, bool &delay, bool timing,
1025 TLB::ArmTranslationType tranType, bool functional)
1026{
1027 // No such thing as a functional timing access
1028 assert(!(timing && functional));
1029
1030 updateMiscReg(tc, tranType);
1031
1032 Addr vaddr_tainted = req->getVaddr();
1033 Addr vaddr = 0;
1034 if (aarch64)
1035 vaddr = purifyTaggedAddr(vaddr_tainted, tc, aarch64EL, ttbcr);
1036 else
1037 vaddr = vaddr_tainted;
1038 Request::Flags flags = req->getFlags();
1039
1040 bool is_fetch = (mode == Execute);
1041 bool is_write = (mode == Write);
1042 bool long_desc_format = aarch64 || longDescFormatInUse(tc);
1043 ArmFault::TranMethod tranMethod = long_desc_format ? ArmFault::LpaeTran
1044 : ArmFault::VmsaTran;
1045
1046 req->setAsid(asid);
1047
1048 DPRINTF(TLBVerbose, "CPSR is priv:%d UserMode:%d secure:%d S1S2NsTran:%d\n",
1049 isPriv, flags & UserMode, isSecure, tranType & S1S2NsTran);
1050
1051 DPRINTF(TLB, "translateFs addr %#x, mode %d, st2 %d, scr %#x sctlr %#x "
1052 "flags %#lx tranType 0x%x\n", vaddr_tainted, mode, isStage2,
1053 scr, sctlr, flags, tranType);
1054
1055 if ((req->isInstFetch() && (!sctlr.i)) ||
1056 ((!req->isInstFetch()) && (!sctlr.c))){
1057 if (!req->isCacheMaintenance()) {
1058 req->setFlags(Request::UNCACHEABLE);
1059 }
1060 req->setFlags(Request::STRICT_ORDER);
1061 }
1062 if (!is_fetch) {
1063 assert(flags & MustBeOne || req->isPrefetch());
1064 if (sctlr.a || !(flags & AllowUnaligned)) {
1065 if (vaddr & mask(flags & AlignmentMask)) {
1066 alignFaults++;
1067 return std::make_shared<DataAbort>(
1068 vaddr_tainted,
1069 TlbEntry::DomainType::NoAccess, is_write,
1070 ArmFault::AlignmentFault, isStage2,
1071 tranMethod);
1072 }
1073 }
1074 }
1075
1076 // If guest MMU is off or hcr.vm=0 go straight to stage2
1077 if ((isStage2 && !hcr.vm) || (!isStage2 && !sctlr.m)) {
1078
1079 req->setPaddr(vaddr);
1080 // When the MMU is off the security attribute corresponds to the
1081 // security state of the processor
1082 if (isSecure)
1083 req->setFlags(Request::SECURE);
1084
1085 // @todo: double check this (ARM ARM issue C B3.2.1)
1086 if (long_desc_format || sctlr.tre == 0 || nmrr.ir0 == 0 ||
1087 nmrr.or0 == 0 || prrr.tr0 != 0x2) {
1088 if (!req->isCacheMaintenance()) {
1089 req->setFlags(Request::UNCACHEABLE);
1090 }
1091 req->setFlags(Request::STRICT_ORDER);
1092 }
1093
1094 // Set memory attributes
1095 TlbEntry temp_te;
1096 temp_te.ns = !isSecure;
1097 if (isStage2 || hcr.dc == 0 || isSecure ||
1098 (isHyp && !(tranType & S1CTran))) {
1099
1100 temp_te.mtype = is_fetch ? TlbEntry::MemoryType::Normal
1101 : TlbEntry::MemoryType::StronglyOrdered;
1102 temp_te.innerAttrs = 0x0;
1103 temp_te.outerAttrs = 0x0;
1104 temp_te.shareable = true;
1105 temp_te.outerShareable = true;
1106 } else {
1107 temp_te.mtype = TlbEntry::MemoryType::Normal;
1108 temp_te.innerAttrs = 0x3;
1109 temp_te.outerAttrs = 0x3;
1110 temp_te.shareable = false;
1111 temp_te.outerShareable = false;
1112 }
1113 temp_te.setAttributes(long_desc_format);
1114 DPRINTF(TLBVerbose, "(No MMU) setting memory attributes: shareable: "
1115 "%d, innerAttrs: %d, outerAttrs: %d, isStage2: %d\n",
1116 temp_te.shareable, temp_te.innerAttrs, temp_te.outerAttrs,
1117 isStage2);
1118 setAttr(temp_te.attributes);
1119
1120 return testTranslation(req, mode, TlbEntry::DomainType::NoAccess);
1121 }
1122
1123 DPRINTF(TLBVerbose, "Translating %s=%#x context=%d\n",
1124 isStage2 ? "IPA" : "VA", vaddr_tainted, asid);
1125 // Translation enabled
1126
1127 TlbEntry *te = NULL;
1128 TlbEntry mergeTe;
1129 Fault fault = getResultTe(&te, req, tc, mode, translation, timing,
1130 functional, &mergeTe);
1131 // only proceed if we have a valid table entry
1132 if ((te == NULL) && (fault == NoFault)) delay = true;
1133
1134 // If we have the table entry transfer some of the attributes to the
1135 // request that triggered the translation
1136 if (te != NULL) {
1137 // Set memory attributes
1138 DPRINTF(TLBVerbose,
1139 "Setting memory attributes: shareable: %d, innerAttrs: %d, "
1140 "outerAttrs: %d, mtype: %d, isStage2: %d\n",
1141 te->shareable, te->innerAttrs, te->outerAttrs,
1142 static_cast<uint8_t>(te->mtype), isStage2);
1143 setAttr(te->attributes);
1144
1145 if (te->nonCacheable && !req->isCacheMaintenance())
1146 req->setFlags(Request::UNCACHEABLE);
1147
1148 // Require requests to be ordered if the request goes to
1149 // strongly ordered or device memory (i.e., anything other
1150 // than normal memory requires strict order).
1151 if (te->mtype != TlbEntry::MemoryType::Normal)
1152 req->setFlags(Request::STRICT_ORDER);
1153
1154 Addr pa = te->pAddr(vaddr);
1155 req->setPaddr(pa);
1156
1157 if (isSecure && !te->ns) {
1158 req->setFlags(Request::SECURE);
1159 }
1160 if ((!is_fetch) && (vaddr & mask(flags & AlignmentMask)) &&
1161 (te->mtype != TlbEntry::MemoryType::Normal)) {
1162 // Unaligned accesses to Device memory should always cause an
1163 // abort regardless of sctlr.a
1164 alignFaults++;
1165 return std::make_shared<DataAbort>(
1166 vaddr_tainted,
1167 TlbEntry::DomainType::NoAccess, is_write,
1168 ArmFault::AlignmentFault, isStage2,
1169 tranMethod);
1170 }
1171
1172 // Check for a trickbox generated address fault
1173 if (fault == NoFault)
1174 fault = testTranslation(req, mode, te->domain);
1175 }
1176
1177 if (fault == NoFault) {
1178 // Don't try to finalize a physical address unless the
1179 // translation has completed (i.e., there is a table entry).
1180 return te ? finalizePhysical(req, tc, mode) : NoFault;
1181 } else {
1182 return fault;
1183 }
1184}
1185
1186Fault
1187TLB::translateAtomic(const RequestPtr &req, ThreadContext *tc, Mode mode,
1188 TLB::ArmTranslationType tranType)
1189{
1190 updateMiscReg(tc, tranType);
1191
1192 if (directToStage2) {
1193 assert(stage2Tlb);
1194 return stage2Tlb->translateAtomic(req, tc, mode, tranType);
1195 }
1196
1197 bool delay = false;
1198 Fault fault;
1199 if (FullSystem)
1200 fault = translateFs(req, tc, mode, NULL, delay, false, tranType);
1201 else
1202 fault = translateSe(req, tc, mode, NULL, delay, false);
1203 assert(!delay);
1204 return fault;
1205}
1206
1207Fault
1208TLB::translateFunctional(const RequestPtr &req, ThreadContext *tc, Mode mode,
1209 TLB::ArmTranslationType tranType)
1210{
1211 updateMiscReg(tc, tranType);
1212
1213 if (directToStage2) {
1214 assert(stage2Tlb);
1215 return stage2Tlb->translateFunctional(req, tc, mode, tranType);
1216 }
1217
1218 bool delay = false;
1219 Fault fault;
1220 if (FullSystem)
1221 fault = translateFs(req, tc, mode, NULL, delay, false, tranType, true);
1222 else
1223 fault = translateSe(req, tc, mode, NULL, delay, false);
1224 assert(!delay);
1225 return fault;
1226}
1227
1228void
1229TLB::translateTiming(const RequestPtr &req, ThreadContext *tc,
1230 Translation *translation, Mode mode, TLB::ArmTranslationType tranType)
1231{
1232 updateMiscReg(tc, tranType);
1233
1234 if (directToStage2) {
1235 assert(stage2Tlb);
1236 stage2Tlb->translateTiming(req, tc, translation, mode, tranType);
1237 return;
1238 }
1239
1240 assert(translation);
1241
1242 translateComplete(req, tc, translation, mode, tranType, isStage2);
1243}
1244
1245Fault
1246TLB::translateComplete(const RequestPtr &req, ThreadContext *tc,
1247 Translation *translation, Mode mode, TLB::ArmTranslationType tranType,
1248 bool callFromS2)
1249{
1250 bool delay = false;
1251 Fault fault;
1252 if (FullSystem)
1253 fault = translateFs(req, tc, mode, translation, delay, true, tranType);
1254 else
1255 fault = translateSe(req, tc, mode, translation, delay, true);
1256 DPRINTF(TLBVerbose, "Translation returning delay=%d fault=%d\n", delay, fault !=
1257 NoFault);
1258 // If we have a translation, and we're not in the middle of doing a stage
1259 // 2 translation tell the translation that we've either finished or its
1260 // going to take a while. By not doing this when we're in the middle of a
1261 // stage 2 translation we prevent marking the translation as delayed twice,
1262 // one when the translation starts and again when the stage 1 translation
1263 // completes.
1264 if (translation && (callFromS2 || !stage2Req || req->hasPaddr() || fault != NoFault)) {
1265 if (!delay)
1266 translation->finish(fault, req, tc, mode);
1267 else
1268 translation->markDelayed();
1269 }
1270 return fault;
1271}
1272
1273Port *
1274TLB::getTableWalkerPort()
1275{
1276 return &stage2Mmu->getDMAPort();
1277}
1278
1279void
1280TLB::updateMiscReg(ThreadContext *tc, ArmTranslationType tranType)
1281{
1282 // check if the regs have changed, or the translation mode is different.
1283 // NOTE: the tran type doesn't affect stage 2 TLB's as they only handle
1284 // one type of translation anyway
1285 if (miscRegValid && miscRegContext == tc->contextId() &&
1286 ((tranType == curTranType) || isStage2)) {
1287 return;
1288 }
1289
1290 DPRINTF(TLBVerbose, "TLB variables changed!\n");
1291 cpsr = tc->readMiscReg(MISCREG_CPSR);
1292
1293 // Dependencies: SCR/SCR_EL3, CPSR
1294 isSecure = inSecureState(tc) &&
1295 !(tranType & HypMode) && !(tranType & S1S2NsTran);
1296
1297 aarch64EL = tranTypeEL(cpsr, tranType);
1298 aarch64 = isStage2 ?
1299 ELIs64(tc, EL2) :
1300 ELIs64(tc, aarch64EL == EL0 ? EL1 : aarch64EL);
1301
1302 if (aarch64) { // AArch64
1303 // determine EL we need to translate in
1304 switch (aarch64EL) {
1305 case EL0:
1306 case EL1:
1307 {
1308 sctlr = tc->readMiscReg(MISCREG_SCTLR_EL1);
1309 ttbcr = tc->readMiscReg(MISCREG_TCR_EL1);
1310 uint64_t ttbr_asid = ttbcr.a1 ?
1311 tc->readMiscReg(MISCREG_TTBR1_EL1) :
1312 tc->readMiscReg(MISCREG_TTBR0_EL1);
1313 asid = bits(ttbr_asid,
1314 (haveLargeAsid64 && ttbcr.as) ? 63 : 55, 48);
1315 }
1316 break;
1317 case EL2:
1318 sctlr = tc->readMiscReg(MISCREG_SCTLR_EL2);
1319 ttbcr = tc->readMiscReg(MISCREG_TCR_EL2);
1320 asid = -1;
1321 break;
1322 case EL3:
1323 sctlr = tc->readMiscReg(MISCREG_SCTLR_EL3);
1324 ttbcr = tc->readMiscReg(MISCREG_TCR_EL3);
1325 asid = -1;
1326 break;
1327 }
1328 hcr = tc->readMiscReg(MISCREG_HCR_EL2);
1329 scr = tc->readMiscReg(MISCREG_SCR_EL3);
1330 isPriv = aarch64EL != EL0;
1331 if (haveVirtualization) {
1332 vmid = bits(tc->readMiscReg(MISCREG_VTTBR_EL2), 55, 48);
1333 isHyp = aarch64EL == EL2;
1334 isHyp |= tranType & HypMode;
1335 isHyp &= (tranType & S1S2NsTran) == 0;
1336 isHyp &= (tranType & S1CTran) == 0;
1337 // Work out if we should skip the first stage of translation and go
1338 // directly to stage 2. This value is cached so we don't have to
1339 // compute it for every translation.
1340 stage2Req = isStage2 ||
1341 (hcr.vm && !isHyp && !isSecure &&
1342 !(tranType & S1CTran) && (aarch64EL < EL2) &&
1343 !(tranType & S1E1Tran)); // <--- FIX THIS HACK
1344 stage2DescReq = isStage2 || (hcr.vm && !isHyp && !isSecure &&
1345 (aarch64EL < EL2));
1346 directToStage2 = !isStage2 && stage2Req && !sctlr.m;
1347 } else {
1348 vmid = 0;
1349 isHyp = false;
1350 directToStage2 = false;
1351 stage2Req = false;
1352 stage2DescReq = false;
1353 }
1354 } else { // AArch32
1355 sctlr = tc->readMiscReg(snsBankedIndex(MISCREG_SCTLR, tc,
1356 !isSecure));
1357 ttbcr = tc->readMiscReg(snsBankedIndex(MISCREG_TTBCR, tc,
1358 !isSecure));
1359 scr = tc->readMiscReg(MISCREG_SCR);
1360 isPriv = cpsr.mode != MODE_USER;
1361 if (longDescFormatInUse(tc)) {
1362 uint64_t ttbr_asid = tc->readMiscReg(
1363 snsBankedIndex(ttbcr.a1 ? MISCREG_TTBR1 :
1364 MISCREG_TTBR0,
1365 tc, !isSecure));
1366 asid = bits(ttbr_asid, 55, 48);
1367 } else { // Short-descriptor translation table format in use
1368 CONTEXTIDR context_id = tc->readMiscReg(snsBankedIndex(
1369 MISCREG_CONTEXTIDR, tc,!isSecure));
1370 asid = context_id.asid;
1371 }
1372 prrr = tc->readMiscReg(snsBankedIndex(MISCREG_PRRR, tc,
1373 !isSecure));
1374 nmrr = tc->readMiscReg(snsBankedIndex(MISCREG_NMRR, tc,
1375 !isSecure));
1376 dacr = tc->readMiscReg(snsBankedIndex(MISCREG_DACR, tc,
1377 !isSecure));
1378 hcr = tc->readMiscReg(MISCREG_HCR);
1379
1380 if (haveVirtualization) {
1381 vmid = bits(tc->readMiscReg(MISCREG_VTTBR), 55, 48);
1382 isHyp = cpsr.mode == MODE_HYP;
1383 isHyp |= tranType & HypMode;
1384 isHyp &= (tranType & S1S2NsTran) == 0;
1385 isHyp &= (tranType & S1CTran) == 0;
1386 if (isHyp) {
1387 sctlr = tc->readMiscReg(MISCREG_HSCTLR);
1388 }
1389 // Work out if we should skip the first stage of translation and go
1390 // directly to stage 2. This value is cached so we don't have to
1391 // compute it for every translation.
1392 stage2Req = hcr.vm && !isStage2 && !isHyp && !isSecure &&
1393 !(tranType & S1CTran);
1394 stage2DescReq = hcr.vm && !isStage2 && !isHyp && !isSecure;
1395 directToStage2 = stage2Req && !sctlr.m;
1396 } else {
1397 vmid = 0;
1398 stage2Req = false;
1399 isHyp = false;
1400 directToStage2 = false;
1401 stage2DescReq = false;
1402 }
1403 }
1404 miscRegValid = true;
1405 miscRegContext = tc->contextId();
1406 curTranType = tranType;
1407}
1408
1409ExceptionLevel
1410TLB::tranTypeEL(CPSR cpsr, ArmTranslationType type)
1411{
1412 switch (type) {
1413 case S1E0Tran:
1414 case S12E0Tran:
1415 return EL0;
1416
1417 case S1E1Tran:
1418 case S12E1Tran:
1419 return EL1;
1420
1421 case S1E2Tran:
1422 return EL2;
1423
1424 case S1E3Tran:
1425 return EL3;
1426
1427 case NormalTran:
1428 case S1CTran:
1429 case S1S2NsTran:
1430 case HypMode:
| 1/*
2 * Copyright (c) 2010-2013, 2016-2019 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Copyright (c) 2001-2005 The Regents of The University of Michigan
15 * All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions are
19 * met: redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer;
21 * redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution;
24 * neither the name of the copyright holders nor the names of its
25 * contributors may be used to endorse or promote products derived from
26 * this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 *
40 * Authors: Ali Saidi
41 * Nathan Binkert
42 * Steve Reinhardt
43 */
44
45#include "arch/arm/tlb.hh"
46
47#include <memory>
48#include <string>
49#include <vector>
50
51#include "arch/arm/faults.hh"
52#include "arch/arm/pagetable.hh"
53#include "arch/arm/stage2_lookup.hh"
54#include "arch/arm/stage2_mmu.hh"
55#include "arch/arm/system.hh"
56#include "arch/arm/table_walker.hh"
57#include "arch/arm/utility.hh"
58#include "arch/generic/mmapped_ipr.hh"
59#include "base/inifile.hh"
60#include "base/str.hh"
61#include "base/trace.hh"
62#include "cpu/base.hh"
63#include "cpu/thread_context.hh"
64#include "debug/Checkpoint.hh"
65#include "debug/TLB.hh"
66#include "debug/TLBVerbose.hh"
67#include "mem/page_table.hh"
68#include "mem/request.hh"
69#include "params/ArmTLB.hh"
70#include "sim/full_system.hh"
71#include "sim/process.hh"
72
73using namespace std;
74using namespace ArmISA;
75
76TLB::TLB(const ArmTLBParams *p)
77 : BaseTLB(p), table(new TlbEntry[p->size]), size(p->size),
78 isStage2(p->is_stage2), stage2Req(false), stage2DescReq(false), _attr(0),
79 directToStage2(false), tableWalker(p->walker), stage2Tlb(NULL),
80 stage2Mmu(NULL), test(nullptr), rangeMRU(1),
81 aarch64(false), aarch64EL(EL0), isPriv(false), isSecure(false),
82 isHyp(false), asid(0), vmid(0), hcr(0), dacr(0),
83 miscRegValid(false), miscRegContext(0), curTranType(NormalTran)
84{
85 const ArmSystem *sys = dynamic_cast<const ArmSystem *>(p->sys);
86
87 tableWalker->setTlb(this);
88
89 // Cache system-level properties
90 haveLPAE = tableWalker->haveLPAE();
91 haveVirtualization = tableWalker->haveVirtualization();
92 haveLargeAsid64 = tableWalker->haveLargeAsid64();
93
94 if (sys)
95 m5opRange = sys->m5opRange();
96}
97
98TLB::~TLB()
99{
100 delete[] table;
101}
102
103void
104TLB::init()
105{
106 if (stage2Mmu && !isStage2)
107 stage2Tlb = stage2Mmu->stage2Tlb();
108}
109
110void
111TLB::setMMU(Stage2MMU *m, MasterID master_id)
112{
113 stage2Mmu = m;
114 tableWalker->setMMU(m, master_id);
115}
116
117bool
118TLB::translateFunctional(ThreadContext *tc, Addr va, Addr &pa)
119{
120 updateMiscReg(tc);
121
122 if (directToStage2) {
123 assert(stage2Tlb);
124 return stage2Tlb->translateFunctional(tc, va, pa);
125 }
126
127 TlbEntry *e = lookup(va, asid, vmid, isHyp, isSecure, true, false,
128 aarch64 ? aarch64EL : EL1);
129 if (!e)
130 return false;
131 pa = e->pAddr(va);
132 return true;
133}
134
135Fault
136TLB::finalizePhysical(const RequestPtr &req,
137 ThreadContext *tc, Mode mode) const
138{
139 const Addr paddr = req->getPaddr();
140
141 if (m5opRange.contains(paddr)) {
142 req->setFlags(Request::MMAPPED_IPR | Request::GENERIC_IPR);
143 req->setPaddr(GenericISA::iprAddressPseudoInst(
144 (paddr >> 8) & 0xFF,
145 paddr & 0xFF));
146 }
147
148 return NoFault;
149}
150
151TlbEntry*
152TLB::lookup(Addr va, uint16_t asn, uint8_t vmid, bool hyp, bool secure,
153 bool functional, bool ignore_asn, ExceptionLevel target_el)
154{
155
156 TlbEntry *retval = NULL;
157
158 // Maintaining LRU array
159 int x = 0;
160 while (retval == NULL && x < size) {
161 if ((!ignore_asn && table[x].match(va, asn, vmid, hyp, secure, false,
162 target_el)) ||
163 (ignore_asn && table[x].match(va, vmid, hyp, secure, target_el))) {
164 // We only move the hit entry ahead when the position is higher
165 // than rangeMRU
166 if (x > rangeMRU && !functional) {
167 TlbEntry tmp_entry = table[x];
168 for (int i = x; i > 0; i--)
169 table[i] = table[i - 1];
170 table[0] = tmp_entry;
171 retval = &table[0];
172 } else {
173 retval = &table[x];
174 }
175 break;
176 }
177 ++x;
178 }
179
180 DPRINTF(TLBVerbose, "Lookup %#x, asn %#x -> %s vmn 0x%x hyp %d secure %d "
181 "ppn %#x size: %#x pa: %#x ap:%d ns:%d nstid:%d g:%d asid: %d "
182 "el: %d\n",
183 va, asn, retval ? "hit" : "miss", vmid, hyp, secure,
184 retval ? retval->pfn : 0, retval ? retval->size : 0,
185 retval ? retval->pAddr(va) : 0, retval ? retval->ap : 0,
186 retval ? retval->ns : 0, retval ? retval->nstid : 0,
187 retval ? retval->global : 0, retval ? retval->asid : 0,
188 retval ? retval->el : 0);
189
190 return retval;
191}
192
193// insert a new TLB entry
194void
195TLB::insert(Addr addr, TlbEntry &entry)
196{
197 DPRINTF(TLB, "Inserting entry into TLB with pfn:%#x size:%#x vpn: %#x"
198 " asid:%d vmid:%d N:%d global:%d valid:%d nc:%d xn:%d"
199 " ap:%#x domain:%#x ns:%d nstid:%d isHyp:%d\n", entry.pfn,
200 entry.size, entry.vpn, entry.asid, entry.vmid, entry.N,
201 entry.global, entry.valid, entry.nonCacheable, entry.xn,
202 entry.ap, static_cast<uint8_t>(entry.domain), entry.ns, entry.nstid,
203 entry.isHyp);
204
205 if (table[size - 1].valid)
206 DPRINTF(TLB, " - Replacing Valid entry %#x, asn %d vmn %d ppn %#x "
207 "size: %#x ap:%d ns:%d nstid:%d g:%d isHyp:%d el: %d\n",
208 table[size-1].vpn << table[size-1].N, table[size-1].asid,
209 table[size-1].vmid, table[size-1].pfn << table[size-1].N,
210 table[size-1].size, table[size-1].ap, table[size-1].ns,
211 table[size-1].nstid, table[size-1].global, table[size-1].isHyp,
212 table[size-1].el);
213
214 //inserting to MRU position and evicting the LRU one
215
216 for (int i = size - 1; i > 0; --i)
217 table[i] = table[i-1];
218 table[0] = entry;
219
220 inserts++;
221 ppRefills->notify(1);
222}
223
224void
225TLB::printTlb() const
226{
227 int x = 0;
228 TlbEntry *te;
229 DPRINTF(TLB, "Current TLB contents:\n");
230 while (x < size) {
231 te = &table[x];
232 if (te->valid)
233 DPRINTF(TLB, " * %s\n", te->print());
234 ++x;
235 }
236}
237
238void
239TLB::flushAllSecurity(bool secure_lookup, ExceptionLevel target_el,
240 bool ignore_el)
241{
242 DPRINTF(TLB, "Flushing all TLB entries (%s lookup)\n",
243 (secure_lookup ? "secure" : "non-secure"));
244 int x = 0;
245 TlbEntry *te;
246 while (x < size) {
247 te = &table[x];
248 const bool el_match = ignore_el ?
249 true : te->checkELMatch(target_el);
250
251 if (te->valid && secure_lookup == !te->nstid &&
252 (te->vmid == vmid || secure_lookup) && el_match) {
253
254 DPRINTF(TLB, " - %s\n", te->print());
255 te->valid = false;
256 flushedEntries++;
257 }
258 ++x;
259 }
260
261 flushTlb++;
262
263 // If there's a second stage TLB (and we're not it) then flush it as well
264 // if we're currently in hyp mode
265 if (!isStage2 && isHyp) {
266 stage2Tlb->flushAllSecurity(secure_lookup, EL1, true);
267 }
268}
269
270void
271TLB::flushAllNs(ExceptionLevel target_el, bool ignore_el)
272{
273 bool hyp = target_el == EL2;
274
275 DPRINTF(TLB, "Flushing all NS TLB entries (%s lookup)\n",
276 (hyp ? "hyp" : "non-hyp"));
277 int x = 0;
278 TlbEntry *te;
279 while (x < size) {
280 te = &table[x];
281 const bool el_match = ignore_el ?
282 true : te->checkELMatch(target_el);
283
284 if (te->valid && te->nstid && te->isHyp == hyp && el_match) {
285
286 DPRINTF(TLB, " - %s\n", te->print());
287 flushedEntries++;
288 te->valid = false;
289 }
290 ++x;
291 }
292
293 flushTlb++;
294
295 // If there's a second stage TLB (and we're not it) then flush it as well
296 if (!isStage2 && !hyp) {
297 stage2Tlb->flushAllNs(EL1, true);
298 }
299}
300
301void
302TLB::flushMvaAsid(Addr mva, uint64_t asn, bool secure_lookup,
303 ExceptionLevel target_el)
304{
305 DPRINTF(TLB, "Flushing TLB entries with mva: %#x, asid: %#x "
306 "(%s lookup)\n", mva, asn, (secure_lookup ?
307 "secure" : "non-secure"));
308 _flushMva(mva, asn, secure_lookup, false, target_el);
309 flushTlbMvaAsid++;
310}
311
312void
313TLB::flushAsid(uint64_t asn, bool secure_lookup, ExceptionLevel target_el)
314{
315 DPRINTF(TLB, "Flushing TLB entries with asid: %#x (%s lookup)\n", asn,
316 (secure_lookup ? "secure" : "non-secure"));
317
318 int x = 0 ;
319 TlbEntry *te;
320
321 while (x < size) {
322 te = &table[x];
323 if (te->valid && te->asid == asn && secure_lookup == !te->nstid &&
324 (te->vmid == vmid || secure_lookup) &&
325 te->checkELMatch(target_el)) {
326
327 te->valid = false;
328 DPRINTF(TLB, " - %s\n", te->print());
329 flushedEntries++;
330 }
331 ++x;
332 }
333 flushTlbAsid++;
334}
335
336void
337TLB::flushMva(Addr mva, bool secure_lookup, ExceptionLevel target_el)
338{
339 DPRINTF(TLB, "Flushing TLB entries with mva: %#x (%s lookup)\n", mva,
340 (secure_lookup ? "secure" : "non-secure"));
341 _flushMva(mva, 0xbeef, secure_lookup, true, target_el);
342 flushTlbMva++;
343}
344
345void
346TLB::_flushMva(Addr mva, uint64_t asn, bool secure_lookup,
347 bool ignore_asn, ExceptionLevel target_el)
348{
349 TlbEntry *te;
350 // D5.7.2: Sign-extend address to 64 bits
351 mva = sext<56>(mva);
352
353 bool hyp = target_el == EL2;
354
355 te = lookup(mva, asn, vmid, hyp, secure_lookup, false, ignore_asn,
356 target_el);
357 while (te != NULL) {
358 if (secure_lookup == !te->nstid) {
359 DPRINTF(TLB, " - %s\n", te->print());
360 te->valid = false;
361 flushedEntries++;
362 }
363 te = lookup(mva, asn, vmid, hyp, secure_lookup, false, ignore_asn,
364 target_el);
365 }
366}
367
368void
369TLB::flushIpaVmid(Addr ipa, bool secure_lookup, ExceptionLevel target_el)
370{
371 assert(!isStage2);
372 stage2Tlb->_flushMva(ipa, 0xbeef, secure_lookup, true, target_el);
373}
374
375void
376TLB::drainResume()
377{
378 // We might have unserialized something or switched CPUs, so make
379 // sure to re-read the misc regs.
380 miscRegValid = false;
381}
382
383void
384TLB::takeOverFrom(BaseTLB *_otlb)
385{
386 TLB *otlb = dynamic_cast<TLB*>(_otlb);
387 /* Make sure we actually have a valid type */
388 if (otlb) {
389 _attr = otlb->_attr;
390 haveLPAE = otlb->haveLPAE;
391 directToStage2 = otlb->directToStage2;
392 stage2Req = otlb->stage2Req;
393 stage2DescReq = otlb->stage2DescReq;
394
395 /* Sync the stage2 MMU if they exist in both
396 * the old CPU and the new
397 */
398 if (!isStage2 &&
399 stage2Tlb && otlb->stage2Tlb) {
400 stage2Tlb->takeOverFrom(otlb->stage2Tlb);
401 }
402 } else {
403 panic("Incompatible TLB type!");
404 }
405}
406
407void
408TLB::serialize(CheckpointOut &cp) const
409{
410 DPRINTF(Checkpoint, "Serializing Arm TLB\n");
411
412 SERIALIZE_SCALAR(_attr);
413 SERIALIZE_SCALAR(haveLPAE);
414 SERIALIZE_SCALAR(directToStage2);
415 SERIALIZE_SCALAR(stage2Req);
416 SERIALIZE_SCALAR(stage2DescReq);
417
418 int num_entries = size;
419 SERIALIZE_SCALAR(num_entries);
420 for (int i = 0; i < size; i++)
421 table[i].serializeSection(cp, csprintf("TlbEntry%d", i));
422}
423
424void
425TLB::unserialize(CheckpointIn &cp)
426{
427 DPRINTF(Checkpoint, "Unserializing Arm TLB\n");
428
429 UNSERIALIZE_SCALAR(_attr);
430 UNSERIALIZE_SCALAR(haveLPAE);
431 UNSERIALIZE_SCALAR(directToStage2);
432 UNSERIALIZE_SCALAR(stage2Req);
433 UNSERIALIZE_SCALAR(stage2DescReq);
434
435 int num_entries;
436 UNSERIALIZE_SCALAR(num_entries);
437 for (int i = 0; i < min(size, num_entries); i++)
438 table[i].unserializeSection(cp, csprintf("TlbEntry%d", i));
439}
440
441void
442TLB::regStats()
443{
444 BaseTLB::regStats();
445 instHits
446 .name(name() + ".inst_hits")
447 .desc("ITB inst hits")
448 ;
449
450 instMisses
451 .name(name() + ".inst_misses")
452 .desc("ITB inst misses")
453 ;
454
455 instAccesses
456 .name(name() + ".inst_accesses")
457 .desc("ITB inst accesses")
458 ;
459
460 readHits
461 .name(name() + ".read_hits")
462 .desc("DTB read hits")
463 ;
464
465 readMisses
466 .name(name() + ".read_misses")
467 .desc("DTB read misses")
468 ;
469
470 readAccesses
471 .name(name() + ".read_accesses")
472 .desc("DTB read accesses")
473 ;
474
475 writeHits
476 .name(name() + ".write_hits")
477 .desc("DTB write hits")
478 ;
479
480 writeMisses
481 .name(name() + ".write_misses")
482 .desc("DTB write misses")
483 ;
484
485 writeAccesses
486 .name(name() + ".write_accesses")
487 .desc("DTB write accesses")
488 ;
489
490 hits
491 .name(name() + ".hits")
492 .desc("DTB hits")
493 ;
494
495 misses
496 .name(name() + ".misses")
497 .desc("DTB misses")
498 ;
499
500 accesses
501 .name(name() + ".accesses")
502 .desc("DTB accesses")
503 ;
504
505 flushTlb
506 .name(name() + ".flush_tlb")
507 .desc("Number of times complete TLB was flushed")
508 ;
509
510 flushTlbMva
511 .name(name() + ".flush_tlb_mva")
512 .desc("Number of times TLB was flushed by MVA")
513 ;
514
515 flushTlbMvaAsid
516 .name(name() + ".flush_tlb_mva_asid")
517 .desc("Number of times TLB was flushed by MVA & ASID")
518 ;
519
520 flushTlbAsid
521 .name(name() + ".flush_tlb_asid")
522 .desc("Number of times TLB was flushed by ASID")
523 ;
524
525 flushedEntries
526 .name(name() + ".flush_entries")
527 .desc("Number of entries that have been flushed from TLB")
528 ;
529
530 alignFaults
531 .name(name() + ".align_faults")
532 .desc("Number of TLB faults due to alignment restrictions")
533 ;
534
535 prefetchFaults
536 .name(name() + ".prefetch_faults")
537 .desc("Number of TLB faults due to prefetch")
538 ;
539
540 domainFaults
541 .name(name() + ".domain_faults")
542 .desc("Number of TLB faults due to domain restrictions")
543 ;
544
545 permsFaults
546 .name(name() + ".perms_faults")
547 .desc("Number of TLB faults due to permissions restrictions")
548 ;
549
550 instAccesses = instHits + instMisses;
551 readAccesses = readHits + readMisses;
552 writeAccesses = writeHits + writeMisses;
553 hits = readHits + writeHits + instHits;
554 misses = readMisses + writeMisses + instMisses;
555 accesses = readAccesses + writeAccesses + instAccesses;
556}
557
558void
559TLB::regProbePoints()
560{
561 ppRefills.reset(new ProbePoints::PMU(getProbeManager(), "Refills"));
562}
563
564Fault
565TLB::translateSe(const RequestPtr &req, ThreadContext *tc, Mode mode,
566 Translation *translation, bool &delay, bool timing)
567{
568 updateMiscReg(tc);
569 Addr vaddr_tainted = req->getVaddr();
570 Addr vaddr = 0;
571 if (aarch64)
572 vaddr = purifyTaggedAddr(vaddr_tainted, tc, aarch64EL, ttbcr);
573 else
574 vaddr = vaddr_tainted;
575 Request::Flags flags = req->getFlags();
576
577 bool is_fetch = (mode == Execute);
578 bool is_write = (mode == Write);
579
580 if (!is_fetch) {
581 assert(flags & MustBeOne || req->isPrefetch());
582 if (sctlr.a || !(flags & AllowUnaligned)) {
583 if (vaddr & mask(flags & AlignmentMask)) {
584 // LPAE is always disabled in SE mode
585 return std::make_shared<DataAbort>(
586 vaddr_tainted,
587 TlbEntry::DomainType::NoAccess, is_write,
588 ArmFault::AlignmentFault, isStage2,
589 ArmFault::VmsaTran);
590 }
591 }
592 }
593
594 Addr paddr;
595 Process *p = tc->getProcessPtr();
596
597 if (!p->pTable->translate(vaddr, paddr))
598 return std::make_shared<GenericPageTableFault>(vaddr_tainted);
599 req->setPaddr(paddr);
600
601 return finalizePhysical(req, tc, mode);
602}
603
604Fault
605TLB::checkPermissions(TlbEntry *te, const RequestPtr &req, Mode mode)
606{
607 // a data cache maintenance instruction that operates by MVA does
608 // not generate a Data Abort exeception due to a Permission fault
609 if (req->isCacheMaintenance()) {
610 return NoFault;
611 }
612
613 Addr vaddr = req->getVaddr(); // 32-bit don't have to purify
614 Request::Flags flags = req->getFlags();
615 bool is_fetch = (mode == Execute);
616 bool is_write = (mode == Write);
617 bool is_priv = isPriv && !(flags & UserMode);
618
619 // Get the translation type from the actuall table entry
620 ArmFault::TranMethod tranMethod = te->longDescFormat ? ArmFault::LpaeTran
621 : ArmFault::VmsaTran;
622
623 // If this is the second stage of translation and the request is for a
624 // stage 1 page table walk then we need to check the HCR.PTW bit. This
625 // allows us to generate a fault if the request targets an area marked
626 // as a device or strongly ordered.
627 if (isStage2 && req->isPTWalk() && hcr.ptw &&
628 (te->mtype != TlbEntry::MemoryType::Normal)) {
629 return std::make_shared<DataAbort>(
630 vaddr, te->domain, is_write,
631 ArmFault::PermissionLL + te->lookupLevel,
632 isStage2, tranMethod);
633 }
634
635 // Generate an alignment fault for unaligned data accesses to device or
636 // strongly ordered memory
637 if (!is_fetch) {
638 if (te->mtype != TlbEntry::MemoryType::Normal) {
639 if (vaddr & mask(flags & AlignmentMask)) {
640 alignFaults++;
641 return std::make_shared<DataAbort>(
642 vaddr, TlbEntry::DomainType::NoAccess, is_write,
643 ArmFault::AlignmentFault, isStage2,
644 tranMethod);
645 }
646 }
647 }
648
649 if (te->nonCacheable) {
650 // Prevent prefetching from I/O devices.
651 if (req->isPrefetch()) {
652 // Here we can safely use the fault status for the short
653 // desc. format in all cases
654 return std::make_shared<PrefetchAbort>(
655 vaddr, ArmFault::PrefetchUncacheable,
656 isStage2, tranMethod);
657 }
658 }
659
660 if (!te->longDescFormat) {
661 switch ((dacr >> (static_cast<uint8_t>(te->domain) * 2)) & 0x3) {
662 case 0:
663 domainFaults++;
664 DPRINTF(TLB, "TLB Fault: Data abort on domain. DACR: %#x"
665 " domain: %#x write:%d\n", dacr,
666 static_cast<uint8_t>(te->domain), is_write);
667 if (is_fetch) {
668 // Use PC value instead of vaddr because vaddr might
669 // be aligned to cache line and should not be the
670 // address reported in FAR
671 return std::make_shared<PrefetchAbort>(
672 req->getPC(),
673 ArmFault::DomainLL + te->lookupLevel,
674 isStage2, tranMethod);
675 } else
676 return std::make_shared<DataAbort>(
677 vaddr, te->domain, is_write,
678 ArmFault::DomainLL + te->lookupLevel,
679 isStage2, tranMethod);
680 case 1:
681 // Continue with permissions check
682 break;
683 case 2:
684 panic("UNPRED domain\n");
685 case 3:
686 return NoFault;
687 }
688 }
689
690 // The 'ap' variable is AP[2:0] or {AP[2,1],1b'0}, i.e. always three bits
691 uint8_t ap = te->longDescFormat ? te->ap << 1 : te->ap;
692 uint8_t hap = te->hap;
693
694 if (sctlr.afe == 1 || te->longDescFormat)
695 ap |= 1;
696
697 bool abt;
698 bool isWritable = true;
699 // If this is a stage 2 access (eg for reading stage 1 page table entries)
700 // then don't perform the AP permissions check, we stil do the HAP check
701 // below.
702 if (isStage2) {
703 abt = false;
704 } else {
705 switch (ap) {
706 case 0:
707 DPRINTF(TLB, "Access permissions 0, checking rs:%#x\n",
708 (int)sctlr.rs);
709 if (!sctlr.xp) {
710 switch ((int)sctlr.rs) {
711 case 2:
712 abt = is_write;
713 break;
714 case 1:
715 abt = is_write || !is_priv;
716 break;
717 case 0:
718 case 3:
719 default:
720 abt = true;
721 break;
722 }
723 } else {
724 abt = true;
725 }
726 break;
727 case 1:
728 abt = !is_priv;
729 break;
730 case 2:
731 abt = !is_priv && is_write;
732 isWritable = is_priv;
733 break;
734 case 3:
735 abt = false;
736 break;
737 case 4:
738 panic("UNPRED premissions\n");
739 case 5:
740 abt = !is_priv || is_write;
741 isWritable = false;
742 break;
743 case 6:
744 case 7:
745 abt = is_write;
746 isWritable = false;
747 break;
748 default:
749 panic("Unknown permissions %#x\n", ap);
750 }
751 }
752
753 bool hapAbt = is_write ? !(hap & 2) : !(hap & 1);
754 bool xn = te->xn || (isWritable && sctlr.wxn) ||
755 (ap == 3 && sctlr.uwxn && is_priv);
756 if (is_fetch && (abt || xn ||
757 (te->longDescFormat && te->pxn && is_priv) ||
758 (isSecure && te->ns && scr.sif))) {
759 permsFaults++;
760 DPRINTF(TLB, "TLB Fault: Prefetch abort on permission check. AP:%d "
761 "priv:%d write:%d ns:%d sif:%d sctlr.afe: %d \n",
762 ap, is_priv, is_write, te->ns, scr.sif,sctlr.afe);
763 // Use PC value instead of vaddr because vaddr might be aligned to
764 // cache line and should not be the address reported in FAR
765 return std::make_shared<PrefetchAbort>(
766 req->getPC(),
767 ArmFault::PermissionLL + te->lookupLevel,
768 isStage2, tranMethod);
769 } else if (abt | hapAbt) {
770 permsFaults++;
771 DPRINTF(TLB, "TLB Fault: Data abort on permission check. AP:%d priv:%d"
772 " write:%d\n", ap, is_priv, is_write);
773 return std::make_shared<DataAbort>(
774 vaddr, te->domain, is_write,
775 ArmFault::PermissionLL + te->lookupLevel,
776 isStage2 | !abt, tranMethod);
777 }
778 return NoFault;
779}
780
781
782Fault
783TLB::checkPermissions64(TlbEntry *te, const RequestPtr &req, Mode mode,
784 ThreadContext *tc)
785{
786 assert(aarch64);
787
788 // A data cache maintenance instruction that operates by VA does
789 // not generate a Permission fault unless:
790 // * It is a data cache invalidate (dc ivac) which requires write
791 // permissions to the VA, or
792 // * It is executed from EL0
793 if (req->isCacheClean() && aarch64EL != EL0 && !isStage2) {
794 return NoFault;
795 }
796
797 Addr vaddr_tainted = req->getVaddr();
798 Addr vaddr = purifyTaggedAddr(vaddr_tainted, tc, aarch64EL, ttbcr);
799
800 Request::Flags flags = req->getFlags();
801 bool is_fetch = (mode == Execute);
802 // Cache clean operations require read permissions to the specified VA
803 bool is_write = !req->isCacheClean() && mode == Write;
804 bool is_priv M5_VAR_USED = isPriv && !(flags & UserMode);
805
806 updateMiscReg(tc, curTranType);
807
808 // If this is the second stage of translation and the request is for a
809 // stage 1 page table walk then we need to check the HCR.PTW bit. This
810 // allows us to generate a fault if the request targets an area marked
811 // as a device or strongly ordered.
812 if (isStage2 && req->isPTWalk() && hcr.ptw &&
813 (te->mtype != TlbEntry::MemoryType::Normal)) {
814 return std::make_shared<DataAbort>(
815 vaddr_tainted, te->domain, is_write,
816 ArmFault::PermissionLL + te->lookupLevel,
817 isStage2, ArmFault::LpaeTran);
818 }
819
820 // Generate an alignment fault for unaligned accesses to device or
821 // strongly ordered memory
822 if (!is_fetch) {
823 if (te->mtype != TlbEntry::MemoryType::Normal) {
824 if (vaddr & mask(flags & AlignmentMask)) {
825 alignFaults++;
826 return std::make_shared<DataAbort>(
827 vaddr_tainted,
828 TlbEntry::DomainType::NoAccess, is_write,
829 ArmFault::AlignmentFault, isStage2,
830 ArmFault::LpaeTran);
831 }
832 }
833 }
834
835 if (te->nonCacheable) {
836 // Prevent prefetching from I/O devices.
837 if (req->isPrefetch()) {
838 // Here we can safely use the fault status for the short
839 // desc. format in all cases
840 return std::make_shared<PrefetchAbort>(
841 vaddr_tainted,
842 ArmFault::PrefetchUncacheable,
843 isStage2, ArmFault::LpaeTran);
844 }
845 }
846
847 uint8_t ap = 0x3 & (te->ap); // 2-bit access protection field
848 bool grant = false;
849
850 uint8_t xn = te->xn;
851 uint8_t pxn = te->pxn;
852 bool r = !is_write && !is_fetch;
853 bool w = is_write;
854 bool x = is_fetch;
855 DPRINTF(TLBVerbose, "Checking permissions: ap:%d, xn:%d, pxn:%d, r:%d, "
856 "w:%d, x:%d\n", ap, xn, pxn, r, w, x);
857
858 if (isStage2) {
859 assert(ArmSystem::haveVirtualization(tc) && aarch64EL != EL2);
860 // In stage 2 we use the hypervisor access permission bits.
861 // The following permissions are described in ARM DDI 0487A.f
862 // D4-1802
863 uint8_t hap = 0x3 & te->hap;
864 if (is_fetch) {
865 // sctlr.wxn overrides the xn bit
866 grant = !sctlr.wxn && !xn;
867 } else if (is_write) {
868 grant = hap & 0x2;
869 } else { // is_read
870 grant = hap & 0x1;
871 }
872 } else {
873 switch (aarch64EL) {
874 case EL0:
875 {
876 uint8_t perm = (ap << 2) | (xn << 1) | pxn;
877 switch (perm) {
878 case 0:
879 case 1:
880 case 8:
881 case 9:
882 grant = x;
883 break;
884 case 4:
885 case 5:
886 grant = r || w || (x && !sctlr.wxn);
887 break;
888 case 6:
889 case 7:
890 grant = r || w;
891 break;
892 case 12:
893 case 13:
894 grant = r || x;
895 break;
896 case 14:
897 case 15:
898 grant = r;
899 break;
900 default:
901 grant = false;
902 }
903 }
904 break;
905 case EL1:
906 {
907 if (checkPAN(tc, ap, req, mode)) {
908 grant = false;
909 break;
910 }
911
912 uint8_t perm = (ap << 2) | (xn << 1) | pxn;
913 switch (perm) {
914 case 0:
915 case 2:
916 grant = r || w || (x && !sctlr.wxn);
917 break;
918 case 1:
919 case 3:
920 case 4:
921 case 5:
922 case 6:
923 case 7:
924 // regions that are writeable at EL0 should not be
925 // executable at EL1
926 grant = r || w;
927 break;
928 case 8:
929 case 10:
930 case 12:
931 case 14:
932 grant = r || x;
933 break;
934 case 9:
935 case 11:
936 case 13:
937 case 15:
938 grant = r;
939 break;
940 default:
941 grant = false;
942 }
943 }
944 break;
945 case EL2:
946 if (checkPAN(tc, ap, req, mode)) {
947 grant = false;
948 break;
949 }
950 M5_FALLTHROUGH;
951 case EL3:
952 {
953 uint8_t perm = (ap & 0x2) | xn;
954 switch (perm) {
955 case 0:
956 grant = r || w || (x && !sctlr.wxn) ;
957 break;
958 case 1:
959 grant = r || w;
960 break;
961 case 2:
962 grant = r || x;
963 break;
964 case 3:
965 grant = r;
966 break;
967 default:
968 grant = false;
969 }
970 }
971 break;
972 }
973 }
974
975 if (!grant) {
976 if (is_fetch) {
977 permsFaults++;
978 DPRINTF(TLB, "TLB Fault: Prefetch abort on permission check. "
979 "AP:%d priv:%d write:%d ns:%d sif:%d "
980 "sctlr.afe: %d\n",
981 ap, is_priv, is_write, te->ns, scr.sif, sctlr.afe);
982 // Use PC value instead of vaddr because vaddr might be aligned to
983 // cache line and should not be the address reported in FAR
984 return std::make_shared<PrefetchAbort>(
985 req->getPC(),
986 ArmFault::PermissionLL + te->lookupLevel,
987 isStage2, ArmFault::LpaeTran);
988 } else {
989 permsFaults++;
990 DPRINTF(TLB, "TLB Fault: Data abort on permission check. AP:%d "
991 "priv:%d write:%d\n", ap, is_priv, is_write);
992 return std::make_shared<DataAbort>(
993 vaddr_tainted, te->domain, is_write,
994 ArmFault::PermissionLL + te->lookupLevel,
995 isStage2, ArmFault::LpaeTran);
996 }
997 }
998
999 return NoFault;
1000}
1001
1002bool
1003TLB::checkPAN(ThreadContext *tc, uint8_t ap, const RequestPtr &req, Mode mode)
1004{
1005 // The PAN bit has no effect on:
1006 // 1) Instruction accesses.
1007 // 2) Data Cache instructions other than DC ZVA
1008 // 3) Address translation instructions, other than ATS1E1RP and
1009 // ATS1E1WP when ARMv8.2-ATS1E1 is implemented. (Unimplemented in
1010 // gem5)
1011 // 4) Unprivileged instructions (Unimplemented in gem5)
1012 AA64MMFR1 mmfr1 = tc->readMiscReg(MISCREG_ID_AA64MMFR1_EL1);
1013 if (mmfr1.pan && cpsr.pan && (ap & 0x1) && mode != Execute &&
1014 (!req->isCacheMaintenance() ||
1015 (req->getFlags() & Request::CACHE_BLOCK_ZERO))) {
1016 return true;
1017 } else {
1018 return false;
1019 }
1020}
1021
1022Fault
1023TLB::translateFs(const RequestPtr &req, ThreadContext *tc, Mode mode,
1024 Translation *translation, bool &delay, bool timing,
1025 TLB::ArmTranslationType tranType, bool functional)
1026{
1027 // No such thing as a functional timing access
1028 assert(!(timing && functional));
1029
1030 updateMiscReg(tc, tranType);
1031
1032 Addr vaddr_tainted = req->getVaddr();
1033 Addr vaddr = 0;
1034 if (aarch64)
1035 vaddr = purifyTaggedAddr(vaddr_tainted, tc, aarch64EL, ttbcr);
1036 else
1037 vaddr = vaddr_tainted;
1038 Request::Flags flags = req->getFlags();
1039
1040 bool is_fetch = (mode == Execute);
1041 bool is_write = (mode == Write);
1042 bool long_desc_format = aarch64 || longDescFormatInUse(tc);
1043 ArmFault::TranMethod tranMethod = long_desc_format ? ArmFault::LpaeTran
1044 : ArmFault::VmsaTran;
1045
1046 req->setAsid(asid);
1047
1048 DPRINTF(TLBVerbose, "CPSR is priv:%d UserMode:%d secure:%d S1S2NsTran:%d\n",
1049 isPriv, flags & UserMode, isSecure, tranType & S1S2NsTran);
1050
1051 DPRINTF(TLB, "translateFs addr %#x, mode %d, st2 %d, scr %#x sctlr %#x "
1052 "flags %#lx tranType 0x%x\n", vaddr_tainted, mode, isStage2,
1053 scr, sctlr, flags, tranType);
1054
1055 if ((req->isInstFetch() && (!sctlr.i)) ||
1056 ((!req->isInstFetch()) && (!sctlr.c))){
1057 if (!req->isCacheMaintenance()) {
1058 req->setFlags(Request::UNCACHEABLE);
1059 }
1060 req->setFlags(Request::STRICT_ORDER);
1061 }
1062 if (!is_fetch) {
1063 assert(flags & MustBeOne || req->isPrefetch());
1064 if (sctlr.a || !(flags & AllowUnaligned)) {
1065 if (vaddr & mask(flags & AlignmentMask)) {
1066 alignFaults++;
1067 return std::make_shared<DataAbort>(
1068 vaddr_tainted,
1069 TlbEntry::DomainType::NoAccess, is_write,
1070 ArmFault::AlignmentFault, isStage2,
1071 tranMethod);
1072 }
1073 }
1074 }
1075
1076 // If guest MMU is off or hcr.vm=0 go straight to stage2
1077 if ((isStage2 && !hcr.vm) || (!isStage2 && !sctlr.m)) {
1078
1079 req->setPaddr(vaddr);
1080 // When the MMU is off the security attribute corresponds to the
1081 // security state of the processor
1082 if (isSecure)
1083 req->setFlags(Request::SECURE);
1084
1085 // @todo: double check this (ARM ARM issue C B3.2.1)
1086 if (long_desc_format || sctlr.tre == 0 || nmrr.ir0 == 0 ||
1087 nmrr.or0 == 0 || prrr.tr0 != 0x2) {
1088 if (!req->isCacheMaintenance()) {
1089 req->setFlags(Request::UNCACHEABLE);
1090 }
1091 req->setFlags(Request::STRICT_ORDER);
1092 }
1093
1094 // Set memory attributes
1095 TlbEntry temp_te;
1096 temp_te.ns = !isSecure;
1097 if (isStage2 || hcr.dc == 0 || isSecure ||
1098 (isHyp && !(tranType & S1CTran))) {
1099
1100 temp_te.mtype = is_fetch ? TlbEntry::MemoryType::Normal
1101 : TlbEntry::MemoryType::StronglyOrdered;
1102 temp_te.innerAttrs = 0x0;
1103 temp_te.outerAttrs = 0x0;
1104 temp_te.shareable = true;
1105 temp_te.outerShareable = true;
1106 } else {
1107 temp_te.mtype = TlbEntry::MemoryType::Normal;
1108 temp_te.innerAttrs = 0x3;
1109 temp_te.outerAttrs = 0x3;
1110 temp_te.shareable = false;
1111 temp_te.outerShareable = false;
1112 }
1113 temp_te.setAttributes(long_desc_format);
1114 DPRINTF(TLBVerbose, "(No MMU) setting memory attributes: shareable: "
1115 "%d, innerAttrs: %d, outerAttrs: %d, isStage2: %d\n",
1116 temp_te.shareable, temp_te.innerAttrs, temp_te.outerAttrs,
1117 isStage2);
1118 setAttr(temp_te.attributes);
1119
1120 return testTranslation(req, mode, TlbEntry::DomainType::NoAccess);
1121 }
1122
1123 DPRINTF(TLBVerbose, "Translating %s=%#x context=%d\n",
1124 isStage2 ? "IPA" : "VA", vaddr_tainted, asid);
1125 // Translation enabled
1126
1127 TlbEntry *te = NULL;
1128 TlbEntry mergeTe;
1129 Fault fault = getResultTe(&te, req, tc, mode, translation, timing,
1130 functional, &mergeTe);
1131 // only proceed if we have a valid table entry
1132 if ((te == NULL) && (fault == NoFault)) delay = true;
1133
1134 // If we have the table entry transfer some of the attributes to the
1135 // request that triggered the translation
1136 if (te != NULL) {
1137 // Set memory attributes
1138 DPRINTF(TLBVerbose,
1139 "Setting memory attributes: shareable: %d, innerAttrs: %d, "
1140 "outerAttrs: %d, mtype: %d, isStage2: %d\n",
1141 te->shareable, te->innerAttrs, te->outerAttrs,
1142 static_cast<uint8_t>(te->mtype), isStage2);
1143 setAttr(te->attributes);
1144
1145 if (te->nonCacheable && !req->isCacheMaintenance())
1146 req->setFlags(Request::UNCACHEABLE);
1147
1148 // Require requests to be ordered if the request goes to
1149 // strongly ordered or device memory (i.e., anything other
1150 // than normal memory requires strict order).
1151 if (te->mtype != TlbEntry::MemoryType::Normal)
1152 req->setFlags(Request::STRICT_ORDER);
1153
1154 Addr pa = te->pAddr(vaddr);
1155 req->setPaddr(pa);
1156
1157 if (isSecure && !te->ns) {
1158 req->setFlags(Request::SECURE);
1159 }
1160 if ((!is_fetch) && (vaddr & mask(flags & AlignmentMask)) &&
1161 (te->mtype != TlbEntry::MemoryType::Normal)) {
1162 // Unaligned accesses to Device memory should always cause an
1163 // abort regardless of sctlr.a
1164 alignFaults++;
1165 return std::make_shared<DataAbort>(
1166 vaddr_tainted,
1167 TlbEntry::DomainType::NoAccess, is_write,
1168 ArmFault::AlignmentFault, isStage2,
1169 tranMethod);
1170 }
1171
1172 // Check for a trickbox generated address fault
1173 if (fault == NoFault)
1174 fault = testTranslation(req, mode, te->domain);
1175 }
1176
1177 if (fault == NoFault) {
1178 // Don't try to finalize a physical address unless the
1179 // translation has completed (i.e., there is a table entry).
1180 return te ? finalizePhysical(req, tc, mode) : NoFault;
1181 } else {
1182 return fault;
1183 }
1184}
1185
1186Fault
1187TLB::translateAtomic(const RequestPtr &req, ThreadContext *tc, Mode mode,
1188 TLB::ArmTranslationType tranType)
1189{
1190 updateMiscReg(tc, tranType);
1191
1192 if (directToStage2) {
1193 assert(stage2Tlb);
1194 return stage2Tlb->translateAtomic(req, tc, mode, tranType);
1195 }
1196
1197 bool delay = false;
1198 Fault fault;
1199 if (FullSystem)
1200 fault = translateFs(req, tc, mode, NULL, delay, false, tranType);
1201 else
1202 fault = translateSe(req, tc, mode, NULL, delay, false);
1203 assert(!delay);
1204 return fault;
1205}
1206
1207Fault
1208TLB::translateFunctional(const RequestPtr &req, ThreadContext *tc, Mode mode,
1209 TLB::ArmTranslationType tranType)
1210{
1211 updateMiscReg(tc, tranType);
1212
1213 if (directToStage2) {
1214 assert(stage2Tlb);
1215 return stage2Tlb->translateFunctional(req, tc, mode, tranType);
1216 }
1217
1218 bool delay = false;
1219 Fault fault;
1220 if (FullSystem)
1221 fault = translateFs(req, tc, mode, NULL, delay, false, tranType, true);
1222 else
1223 fault = translateSe(req, tc, mode, NULL, delay, false);
1224 assert(!delay);
1225 return fault;
1226}
1227
1228void
1229TLB::translateTiming(const RequestPtr &req, ThreadContext *tc,
1230 Translation *translation, Mode mode, TLB::ArmTranslationType tranType)
1231{
1232 updateMiscReg(tc, tranType);
1233
1234 if (directToStage2) {
1235 assert(stage2Tlb);
1236 stage2Tlb->translateTiming(req, tc, translation, mode, tranType);
1237 return;
1238 }
1239
1240 assert(translation);
1241
1242 translateComplete(req, tc, translation, mode, tranType, isStage2);
1243}
1244
1245Fault
1246TLB::translateComplete(const RequestPtr &req, ThreadContext *tc,
1247 Translation *translation, Mode mode, TLB::ArmTranslationType tranType,
1248 bool callFromS2)
1249{
1250 bool delay = false;
1251 Fault fault;
1252 if (FullSystem)
1253 fault = translateFs(req, tc, mode, translation, delay, true, tranType);
1254 else
1255 fault = translateSe(req, tc, mode, translation, delay, true);
1256 DPRINTF(TLBVerbose, "Translation returning delay=%d fault=%d\n", delay, fault !=
1257 NoFault);
1258 // If we have a translation, and we're not in the middle of doing a stage
1259 // 2 translation tell the translation that we've either finished or its
1260 // going to take a while. By not doing this when we're in the middle of a
1261 // stage 2 translation we prevent marking the translation as delayed twice,
1262 // one when the translation starts and again when the stage 1 translation
1263 // completes.
1264 if (translation && (callFromS2 || !stage2Req || req->hasPaddr() || fault != NoFault)) {
1265 if (!delay)
1266 translation->finish(fault, req, tc, mode);
1267 else
1268 translation->markDelayed();
1269 }
1270 return fault;
1271}
1272
1273Port *
1274TLB::getTableWalkerPort()
1275{
1276 return &stage2Mmu->getDMAPort();
1277}
1278
1279void
1280TLB::updateMiscReg(ThreadContext *tc, ArmTranslationType tranType)
1281{
1282 // check if the regs have changed, or the translation mode is different.
1283 // NOTE: the tran type doesn't affect stage 2 TLB's as they only handle
1284 // one type of translation anyway
1285 if (miscRegValid && miscRegContext == tc->contextId() &&
1286 ((tranType == curTranType) || isStage2)) {
1287 return;
1288 }
1289
1290 DPRINTF(TLBVerbose, "TLB variables changed!\n");
1291 cpsr = tc->readMiscReg(MISCREG_CPSR);
1292
1293 // Dependencies: SCR/SCR_EL3, CPSR
1294 isSecure = inSecureState(tc) &&
1295 !(tranType & HypMode) && !(tranType & S1S2NsTran);
1296
1297 aarch64EL = tranTypeEL(cpsr, tranType);
1298 aarch64 = isStage2 ?
1299 ELIs64(tc, EL2) :
1300 ELIs64(tc, aarch64EL == EL0 ? EL1 : aarch64EL);
1301
1302 if (aarch64) { // AArch64
1303 // determine EL we need to translate in
1304 switch (aarch64EL) {
1305 case EL0:
1306 case EL1:
1307 {
1308 sctlr = tc->readMiscReg(MISCREG_SCTLR_EL1);
1309 ttbcr = tc->readMiscReg(MISCREG_TCR_EL1);
1310 uint64_t ttbr_asid = ttbcr.a1 ?
1311 tc->readMiscReg(MISCREG_TTBR1_EL1) :
1312 tc->readMiscReg(MISCREG_TTBR0_EL1);
1313 asid = bits(ttbr_asid,
1314 (haveLargeAsid64 && ttbcr.as) ? 63 : 55, 48);
1315 }
1316 break;
1317 case EL2:
1318 sctlr = tc->readMiscReg(MISCREG_SCTLR_EL2);
1319 ttbcr = tc->readMiscReg(MISCREG_TCR_EL2);
1320 asid = -1;
1321 break;
1322 case EL3:
1323 sctlr = tc->readMiscReg(MISCREG_SCTLR_EL3);
1324 ttbcr = tc->readMiscReg(MISCREG_TCR_EL3);
1325 asid = -1;
1326 break;
1327 }
1328 hcr = tc->readMiscReg(MISCREG_HCR_EL2);
1329 scr = tc->readMiscReg(MISCREG_SCR_EL3);
1330 isPriv = aarch64EL != EL0;
1331 if (haveVirtualization) {
1332 vmid = bits(tc->readMiscReg(MISCREG_VTTBR_EL2), 55, 48);
1333 isHyp = aarch64EL == EL2;
1334 isHyp |= tranType & HypMode;
1335 isHyp &= (tranType & S1S2NsTran) == 0;
1336 isHyp &= (tranType & S1CTran) == 0;
1337 // Work out if we should skip the first stage of translation and go
1338 // directly to stage 2. This value is cached so we don't have to
1339 // compute it for every translation.
1340 stage2Req = isStage2 ||
1341 (hcr.vm && !isHyp && !isSecure &&
1342 !(tranType & S1CTran) && (aarch64EL < EL2) &&
1343 !(tranType & S1E1Tran)); // <--- FIX THIS HACK
1344 stage2DescReq = isStage2 || (hcr.vm && !isHyp && !isSecure &&
1345 (aarch64EL < EL2));
1346 directToStage2 = !isStage2 && stage2Req && !sctlr.m;
1347 } else {
1348 vmid = 0;
1349 isHyp = false;
1350 directToStage2 = false;
1351 stage2Req = false;
1352 stage2DescReq = false;
1353 }
1354 } else { // AArch32
1355 sctlr = tc->readMiscReg(snsBankedIndex(MISCREG_SCTLR, tc,
1356 !isSecure));
1357 ttbcr = tc->readMiscReg(snsBankedIndex(MISCREG_TTBCR, tc,
1358 !isSecure));
1359 scr = tc->readMiscReg(MISCREG_SCR);
1360 isPriv = cpsr.mode != MODE_USER;
1361 if (longDescFormatInUse(tc)) {
1362 uint64_t ttbr_asid = tc->readMiscReg(
1363 snsBankedIndex(ttbcr.a1 ? MISCREG_TTBR1 :
1364 MISCREG_TTBR0,
1365 tc, !isSecure));
1366 asid = bits(ttbr_asid, 55, 48);
1367 } else { // Short-descriptor translation table format in use
1368 CONTEXTIDR context_id = tc->readMiscReg(snsBankedIndex(
1369 MISCREG_CONTEXTIDR, tc,!isSecure));
1370 asid = context_id.asid;
1371 }
1372 prrr = tc->readMiscReg(snsBankedIndex(MISCREG_PRRR, tc,
1373 !isSecure));
1374 nmrr = tc->readMiscReg(snsBankedIndex(MISCREG_NMRR, tc,
1375 !isSecure));
1376 dacr = tc->readMiscReg(snsBankedIndex(MISCREG_DACR, tc,
1377 !isSecure));
1378 hcr = tc->readMiscReg(MISCREG_HCR);
1379
1380 if (haveVirtualization) {
1381 vmid = bits(tc->readMiscReg(MISCREG_VTTBR), 55, 48);
1382 isHyp = cpsr.mode == MODE_HYP;
1383 isHyp |= tranType & HypMode;
1384 isHyp &= (tranType & S1S2NsTran) == 0;
1385 isHyp &= (tranType & S1CTran) == 0;
1386 if (isHyp) {
1387 sctlr = tc->readMiscReg(MISCREG_HSCTLR);
1388 }
1389 // Work out if we should skip the first stage of translation and go
1390 // directly to stage 2. This value is cached so we don't have to
1391 // compute it for every translation.
1392 stage2Req = hcr.vm && !isStage2 && !isHyp && !isSecure &&
1393 !(tranType & S1CTran);
1394 stage2DescReq = hcr.vm && !isStage2 && !isHyp && !isSecure;
1395 directToStage2 = stage2Req && !sctlr.m;
1396 } else {
1397 vmid = 0;
1398 stage2Req = false;
1399 isHyp = false;
1400 directToStage2 = false;
1401 stage2DescReq = false;
1402 }
1403 }
1404 miscRegValid = true;
1405 miscRegContext = tc->contextId();
1406 curTranType = tranType;
1407}
1408
1409ExceptionLevel
1410TLB::tranTypeEL(CPSR cpsr, ArmTranslationType type)
1411{
1412 switch (type) {
1413 case S1E0Tran:
1414 case S12E0Tran:
1415 return EL0;
1416
1417 case S1E1Tran:
1418 case S12E1Tran:
1419 return EL1;
1420
1421 case S1E2Tran:
1422 return EL2;
1423
1424 case S1E3Tran:
1425 return EL3;
1426
1427 case NormalTran:
1428 case S1CTran:
1429 case S1S2NsTran:
1430 case HypMode:
|
1432
1433 default:
1434 panic("Unknown translation mode!\n");
1435 }
1436}
1437
1438Fault
1439TLB::getTE(TlbEntry **te, const RequestPtr &req, ThreadContext *tc, Mode mode,
1440 Translation *translation, bool timing, bool functional,
1441 bool is_secure, TLB::ArmTranslationType tranType)
1442{
1443 // In a 2-stage system, the IPA->PA translation can be started via this
1444 // call so make sure the miscRegs are correct.
1445 if (isStage2) {
1446 updateMiscReg(tc, tranType);
1447 }
1448 bool is_fetch = (mode == Execute);
1449 bool is_write = (mode == Write);
1450
1451 Addr vaddr_tainted = req->getVaddr();
1452 Addr vaddr = 0;
1453 ExceptionLevel target_el = aarch64 ? aarch64EL : EL1;
1454 if (aarch64) {
1455 vaddr = purifyTaggedAddr(vaddr_tainted, tc, target_el, ttbcr);
1456 } else {
1457 vaddr = vaddr_tainted;
1458 }
1459 *te = lookup(vaddr, asid, vmid, isHyp, is_secure, false, false, target_el);
1460 if (*te == NULL) {
1461 if (req->isPrefetch()) {
1462 // if the request is a prefetch don't attempt to fill the TLB or go
1463 // any further with the memory access (here we can safely use the
1464 // fault status for the short desc. format in all cases)
1465 prefetchFaults++;
1466 return std::make_shared<PrefetchAbort>(
1467 vaddr_tainted, ArmFault::PrefetchTLBMiss, isStage2);
1468 }
1469
1470 if (is_fetch)
1471 instMisses++;
1472 else if (is_write)
1473 writeMisses++;
1474 else
1475 readMisses++;
1476
1477 // start translation table walk, pass variables rather than
1478 // re-retreaving in table walker for speed
1479 DPRINTF(TLB, "TLB Miss: Starting hardware table walker for %#x(%d:%d)\n",
1480 vaddr_tainted, asid, vmid);
1481 Fault fault;
1482 fault = tableWalker->walk(req, tc, asid, vmid, isHyp, mode,
1483 translation, timing, functional, is_secure,
1484 tranType, stage2DescReq);
1485 // for timing mode, return and wait for table walk,
1486 if (timing || fault != NoFault) {
1487 return fault;
1488 }
1489
1490 *te = lookup(vaddr, asid, vmid, isHyp, is_secure, false, false, target_el);
1491 if (!*te)
1492 printTlb();
1493 assert(*te);
1494 } else {
1495 if (is_fetch)
1496 instHits++;
1497 else if (is_write)
1498 writeHits++;
1499 else
1500 readHits++;
1501 }
1502 return NoFault;
1503}
1504
1505Fault
1506TLB::getResultTe(TlbEntry **te, const RequestPtr &req,
1507 ThreadContext *tc, Mode mode,
1508 Translation *translation, bool timing, bool functional,
1509 TlbEntry *mergeTe)
1510{
1511 Fault fault;
1512
1513 if (isStage2) {
1514 // We are already in the stage 2 TLB. Grab the table entry for stage
1515 // 2 only. We are here because stage 1 translation is disabled.
1516 TlbEntry *s2Te = NULL;
1517 // Get the stage 2 table entry
1518 fault = getTE(&s2Te, req, tc, mode, translation, timing, functional,
1519 isSecure, curTranType);
1520 // Check permissions of stage 2
1521 if ((s2Te != NULL) && (fault == NoFault)) {
1522 if (aarch64)
1523 fault = checkPermissions64(s2Te, req, mode, tc);
1524 else
1525 fault = checkPermissions(s2Te, req, mode);
1526 }
1527 *te = s2Te;
1528 return fault;
1529 }
1530
1531 TlbEntry *s1Te = NULL;
1532
1533 Addr vaddr_tainted = req->getVaddr();
1534
1535 // Get the stage 1 table entry
1536 fault = getTE(&s1Te, req, tc, mode, translation, timing, functional,
1537 isSecure, curTranType);
1538 // only proceed if we have a valid table entry
1539 if ((s1Te != NULL) && (fault == NoFault)) {
1540 // Check stage 1 permissions before checking stage 2
1541 if (aarch64)
1542 fault = checkPermissions64(s1Te, req, mode, tc);
1543 else
1544 fault = checkPermissions(s1Te, req, mode);
1545 if (stage2Req & (fault == NoFault)) {
1546 Stage2LookUp *s2Lookup = new Stage2LookUp(this, stage2Tlb, *s1Te,
1547 req, translation, mode, timing, functional, curTranType);
1548 fault = s2Lookup->getTe(tc, mergeTe);
1549 if (s2Lookup->isComplete()) {
1550 *te = mergeTe;
1551 // We've finished with the lookup so delete it
1552 delete s2Lookup;
1553 } else {
1554 // The lookup hasn't completed, so we can't delete it now. We
1555 // get round this by asking the object to self delete when the
1556 // translation is complete.
1557 s2Lookup->setSelfDelete();
1558 }
1559 } else {
1560 // This case deals with an S1 hit (or bypass), followed by
1561 // an S2 hit-but-perms issue
1562 if (isStage2) {
1563 DPRINTF(TLBVerbose, "s2TLB: reqVa %#x, reqPa %#x, fault %p\n",
1564 vaddr_tainted, req->hasPaddr() ? req->getPaddr() : ~0, fault);
1565 if (fault != NoFault) {
1566 ArmFault *armFault = reinterpret_cast<ArmFault *>(fault.get());
1567 armFault->annotate(ArmFault::S1PTW, false);
1568 armFault->annotate(ArmFault::OVA, vaddr_tainted);
1569 }
1570 }
1571 *te = s1Te;
1572 }
1573 }
1574 return fault;
1575}
1576
1577void
1578TLB::setTestInterface(SimObject *_ti)
1579{
1580 if (!_ti) {
1581 test = nullptr;
1582 } else {
1583 TlbTestInterface *ti(dynamic_cast<TlbTestInterface *>(_ti));
1584 fatal_if(!ti, "%s is not a valid ARM TLB tester\n", _ti->name());
1585 test = ti;
1586 }
1587}
1588
1589Fault
1590TLB::testTranslation(const RequestPtr &req, Mode mode,
1591 TlbEntry::DomainType domain)
1592{
1593 if (!test || !req->hasSize() || req->getSize() == 0 ||
1594 req->isCacheMaintenance()) {
1595 return NoFault;
1596 } else {
1597 return test->translationCheck(req, isPriv, mode, domain);
1598 }
1599}
1600
1601Fault
1602TLB::testWalk(Addr pa, Addr size, Addr va, bool is_secure, Mode mode,
1603 TlbEntry::DomainType domain, LookupLevel lookup_level)
1604{
1605 if (!test) {
1606 return NoFault;
1607 } else {
1608 return test->walkCheck(pa, size, va, is_secure, isPriv, mode,
1609 domain, lookup_level);
1610 }
1611}
1612
1613
1614ArmISA::TLB *
1615ArmTLBParams::create()
1616{
1617 return new ArmISA::TLB(this);
1618}
| 1432
1433 default:
1434 panic("Unknown translation mode!\n");
1435 }
1436}
1437
1438Fault
1439TLB::getTE(TlbEntry **te, const RequestPtr &req, ThreadContext *tc, Mode mode,
1440 Translation *translation, bool timing, bool functional,
1441 bool is_secure, TLB::ArmTranslationType tranType)
1442{
1443 // In a 2-stage system, the IPA->PA translation can be started via this
1444 // call so make sure the miscRegs are correct.
1445 if (isStage2) {
1446 updateMiscReg(tc, tranType);
1447 }
1448 bool is_fetch = (mode == Execute);
1449 bool is_write = (mode == Write);
1450
1451 Addr vaddr_tainted = req->getVaddr();
1452 Addr vaddr = 0;
1453 ExceptionLevel target_el = aarch64 ? aarch64EL : EL1;
1454 if (aarch64) {
1455 vaddr = purifyTaggedAddr(vaddr_tainted, tc, target_el, ttbcr);
1456 } else {
1457 vaddr = vaddr_tainted;
1458 }
1459 *te = lookup(vaddr, asid, vmid, isHyp, is_secure, false, false, target_el);
1460 if (*te == NULL) {
1461 if (req->isPrefetch()) {
1462 // if the request is a prefetch don't attempt to fill the TLB or go
1463 // any further with the memory access (here we can safely use the
1464 // fault status for the short desc. format in all cases)
1465 prefetchFaults++;
1466 return std::make_shared<PrefetchAbort>(
1467 vaddr_tainted, ArmFault::PrefetchTLBMiss, isStage2);
1468 }
1469
1470 if (is_fetch)
1471 instMisses++;
1472 else if (is_write)
1473 writeMisses++;
1474 else
1475 readMisses++;
1476
1477 // start translation table walk, pass variables rather than
1478 // re-retreaving in table walker for speed
1479 DPRINTF(TLB, "TLB Miss: Starting hardware table walker for %#x(%d:%d)\n",
1480 vaddr_tainted, asid, vmid);
1481 Fault fault;
1482 fault = tableWalker->walk(req, tc, asid, vmid, isHyp, mode,
1483 translation, timing, functional, is_secure,
1484 tranType, stage2DescReq);
1485 // for timing mode, return and wait for table walk,
1486 if (timing || fault != NoFault) {
1487 return fault;
1488 }
1489
1490 *te = lookup(vaddr, asid, vmid, isHyp, is_secure, false, false, target_el);
1491 if (!*te)
1492 printTlb();
1493 assert(*te);
1494 } else {
1495 if (is_fetch)
1496 instHits++;
1497 else if (is_write)
1498 writeHits++;
1499 else
1500 readHits++;
1501 }
1502 return NoFault;
1503}
1504
1505Fault
1506TLB::getResultTe(TlbEntry **te, const RequestPtr &req,
1507 ThreadContext *tc, Mode mode,
1508 Translation *translation, bool timing, bool functional,
1509 TlbEntry *mergeTe)
1510{
1511 Fault fault;
1512
1513 if (isStage2) {
1514 // We are already in the stage 2 TLB. Grab the table entry for stage
1515 // 2 only. We are here because stage 1 translation is disabled.
1516 TlbEntry *s2Te = NULL;
1517 // Get the stage 2 table entry
1518 fault = getTE(&s2Te, req, tc, mode, translation, timing, functional,
1519 isSecure, curTranType);
1520 // Check permissions of stage 2
1521 if ((s2Te != NULL) && (fault == NoFault)) {
1522 if (aarch64)
1523 fault = checkPermissions64(s2Te, req, mode, tc);
1524 else
1525 fault = checkPermissions(s2Te, req, mode);
1526 }
1527 *te = s2Te;
1528 return fault;
1529 }
1530
1531 TlbEntry *s1Te = NULL;
1532
1533 Addr vaddr_tainted = req->getVaddr();
1534
1535 // Get the stage 1 table entry
1536 fault = getTE(&s1Te, req, tc, mode, translation, timing, functional,
1537 isSecure, curTranType);
1538 // only proceed if we have a valid table entry
1539 if ((s1Te != NULL) && (fault == NoFault)) {
1540 // Check stage 1 permissions before checking stage 2
1541 if (aarch64)
1542 fault = checkPermissions64(s1Te, req, mode, tc);
1543 else
1544 fault = checkPermissions(s1Te, req, mode);
1545 if (stage2Req & (fault == NoFault)) {
1546 Stage2LookUp *s2Lookup = new Stage2LookUp(this, stage2Tlb, *s1Te,
1547 req, translation, mode, timing, functional, curTranType);
1548 fault = s2Lookup->getTe(tc, mergeTe);
1549 if (s2Lookup->isComplete()) {
1550 *te = mergeTe;
1551 // We've finished with the lookup so delete it
1552 delete s2Lookup;
1553 } else {
1554 // The lookup hasn't completed, so we can't delete it now. We
1555 // get round this by asking the object to self delete when the
1556 // translation is complete.
1557 s2Lookup->setSelfDelete();
1558 }
1559 } else {
1560 // This case deals with an S1 hit (or bypass), followed by
1561 // an S2 hit-but-perms issue
1562 if (isStage2) {
1563 DPRINTF(TLBVerbose, "s2TLB: reqVa %#x, reqPa %#x, fault %p\n",
1564 vaddr_tainted, req->hasPaddr() ? req->getPaddr() : ~0, fault);
1565 if (fault != NoFault) {
1566 ArmFault *armFault = reinterpret_cast<ArmFault *>(fault.get());
1567 armFault->annotate(ArmFault::S1PTW, false);
1568 armFault->annotate(ArmFault::OVA, vaddr_tainted);
1569 }
1570 }
1571 *te = s1Te;
1572 }
1573 }
1574 return fault;
1575}
1576
1577void
1578TLB::setTestInterface(SimObject *_ti)
1579{
1580 if (!_ti) {
1581 test = nullptr;
1582 } else {
1583 TlbTestInterface *ti(dynamic_cast<TlbTestInterface *>(_ti));
1584 fatal_if(!ti, "%s is not a valid ARM TLB tester\n", _ti->name());
1585 test = ti;
1586 }
1587}
1588
1589Fault
1590TLB::testTranslation(const RequestPtr &req, Mode mode,
1591 TlbEntry::DomainType domain)
1592{
1593 if (!test || !req->hasSize() || req->getSize() == 0 ||
1594 req->isCacheMaintenance()) {
1595 return NoFault;
1596 } else {
1597 return test->translationCheck(req, isPriv, mode, domain);
1598 }
1599}
1600
1601Fault
1602TLB::testWalk(Addr pa, Addr size, Addr va, bool is_secure, Mode mode,
1603 TlbEntry::DomainType domain, LookupLevel lookup_level)
1604{
1605 if (!test) {
1606 return NoFault;
1607 } else {
1608 return test->walkCheck(pa, size, va, is_secure, isPriv, mode,
1609 domain, lookup_level);
1610 }
1611}
1612
1613
1614ArmISA::TLB *
1615ArmTLBParams::create()
1616{
1617 return new ArmISA::TLB(this);
1618}
|