table_walker.cc revision 11395:032bc62120eb 
1/*
2 * Copyright (c) 2010, 2012-2015 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder.  You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Ali Saidi
38 *          Giacomo Gabrielli
39 */
40#include "arch/arm/table_walker.hh"
41
42#include <memory>
43
44#include "arch/arm/faults.hh"
45#include "arch/arm/stage2_mmu.hh"
46#include "arch/arm/system.hh"
47#include "arch/arm/tlb.hh"
48#include "cpu/base.hh"
49#include "cpu/thread_context.hh"
50#include "debug/Checkpoint.hh"
51#include "debug/Drain.hh"
52#include "debug/TLB.hh"
53#include "debug/TLBVerbose.hh"
54#include "dev/dma_device.hh"
55#include "sim/system.hh"
56
57using namespace ArmISA;
58
59TableWalker::TableWalker(const Params *p)
60    : MemObject(p),
61      stage2Mmu(NULL), port(NULL), masterId(Request::invldMasterId),
62      isStage2(p->is_stage2), tlb(NULL),
63      currState(NULL), pending(false),
64      numSquashable(p->num_squash_per_cycle),
65      pendingReqs(0),
66      pendingChangeTick(curTick()),
67      doL1DescEvent(this), doL2DescEvent(this),
68      doL0LongDescEvent(this), doL1LongDescEvent(this), doL2LongDescEvent(this),
69      doL3LongDescEvent(this),
70      doProcessEvent(this)
71{
72    sctlr = 0;
73
74    // Cache system-level properties
75    if (FullSystem) {
76        ArmSystem *armSys = dynamic_cast<ArmSystem *>(p->sys);
77        assert(armSys);
78        haveSecurity = armSys->haveSecurity();
79        _haveLPAE = armSys->haveLPAE();
80        _haveVirtualization = armSys->haveVirtualization();
81        physAddrRange = armSys->physAddrRange();
82        _haveLargeAsid64 = armSys->haveLargeAsid64();
83    } else {
84        haveSecurity = _haveLPAE = _haveVirtualization = false;
85        _haveLargeAsid64 = false;
86        physAddrRange = 32;
87    }
88
89}
90
91TableWalker::~TableWalker()
92{
93    ;
94}
95
96void
97TableWalker::setMMU(Stage2MMU *m, MasterID master_id)
98{
99    stage2Mmu = m;
100    port = &m->getPort();
101    masterId = master_id;
102}
103
104void
105TableWalker::init()
106{
107    fatal_if(!stage2Mmu, "Table walker must have a valid stage-2 MMU\n");
108    fatal_if(!port, "Table walker must have a valid port\n");
109    fatal_if(!tlb, "Table walker must have a valid TLB\n");
110}
111
112BaseMasterPort&
113TableWalker::getMasterPort(const std::string &if_name, PortID idx)
114{
115    if (if_name == "port") {
116        if (!isStage2) {
117            return *port;
118        } else {
119            fatal("Cannot access table walker port through stage-two walker\n");
120        }
121    }
122    return MemObject::getMasterPort(if_name, idx);
123}
124
125TableWalker::WalkerState::WalkerState() :
126    tc(nullptr), aarch64(false), el(EL0), physAddrRange(0), req(nullptr),
127    asid(0), vmid(0), isHyp(false), transState(nullptr),
128    vaddr(0), vaddr_tainted(0), isWrite(false), isFetch(false), isSecure(false),
129    secureLookup(false), rwTable(false), userTable(false), xnTable(false),
130    pxnTable(false), stage2Req(false), doingStage2(false),
131    stage2Tran(nullptr), timing(false), functional(false),
132    mode(BaseTLB::Read), tranType(TLB::NormalTran), l2Desc(l1Desc),
133    delayed(false), tableWalker(nullptr)
134{
135}
136
137void
138TableWalker::completeDrain()
139{
140    if (drainState() == DrainState::Draining &&
141        stateQueues[L1].empty() && stateQueues[L2].empty() &&
142        pendingQueue.empty()) {
143
144        DPRINTF(Drain, "TableWalker done draining, processing drain event\n");
145        signalDrainDone();
146    }
147}
148
149DrainState
150TableWalker::drain()
151{
152    bool state_queues_not_empty = false;
153
154    for (int i = 0; i < MAX_LOOKUP_LEVELS; ++i) {
155        if (!stateQueues[i].empty()) {
156            state_queues_not_empty = true;
157            break;
158        }
159    }
160
161    if (state_queues_not_empty || pendingQueue.size()) {
162        DPRINTF(Drain, "TableWalker not drained\n");
163        return DrainState::Draining;
164    } else {
165        DPRINTF(Drain, "TableWalker free, no need to drain\n");
166        return DrainState::Drained;
167    }
168}
169
170void
171TableWalker::drainResume()
172{
173    if (params()->sys->isTimingMode() && currState) {
174        delete currState;
175        currState = NULL;
176        pendingChange();
177    }
178}
179
180Fault
181TableWalker::walk(RequestPtr _req, ThreadContext *_tc, uint16_t _asid,
182                  uint8_t _vmid, bool _isHyp, TLB::Mode _mode,
183                  TLB::Translation *_trans, bool _timing, bool _functional,
184                  bool secure, TLB::ArmTranslationType tranType)
185{
186    assert(!(_functional && _timing));
187    ++statWalks;
188
189    WalkerState *savedCurrState = NULL;
190
191    if (!currState && !_functional) {
192        // For atomic mode, a new WalkerState instance should be only created
193        // once per TLB. For timing mode, a new instance is generated for every
194        // TLB miss.
195        DPRINTF(TLBVerbose, "creating new instance of WalkerState\n");
196
197        currState = new WalkerState();
198        currState->tableWalker = this;
199    } else if (_functional) {
200        // If we are mixing functional mode with timing (or even
201        // atomic), we need to to be careful and clean up after
202        // ourselves to not risk getting into an inconsistent state.
203        DPRINTF(TLBVerbose, "creating functional instance of WalkerState\n");
204        savedCurrState = currState;
205        currState = new WalkerState();
206        currState->tableWalker = this;
207    } else if (_timing) {
208        // This is a translation that was completed and then faulted again
209        // because some underlying parameters that affect the translation
210        // changed out from under us (e.g. asid). It will either be a
211        // misprediction, in which case nothing will happen or we'll use
212        // this fault to re-execute the faulting instruction which should clean
213        // up everything.
214        if (currState->vaddr_tainted == _req->getVaddr()) {
215            ++statSquashedBefore;
216            return std::make_shared<ReExec>();
217        }
218    }
219    pendingChange();
220
221    currState->startTime = curTick();
222    currState->tc = _tc;
223    currState->aarch64 = opModeIs64(currOpMode(_tc));
224    currState->el = currEL(_tc);
225    currState->transState = _trans;
226    currState->req = _req;
227    currState->fault = NoFault;
228    currState->asid = _asid;
229    currState->vmid = _vmid;
230    currState->isHyp = _isHyp;
231    currState->timing = _timing;
232    currState->functional = _functional;
233    currState->mode = _mode;
234    currState->tranType = tranType;
235    currState->isSecure = secure;
236    currState->physAddrRange = physAddrRange;
237
238    /** @todo These should be cached or grabbed from cached copies in
239     the TLB, all these miscreg reads are expensive */
240    currState->vaddr_tainted = currState->req->getVaddr();
241    if (currState->aarch64)
242        currState->vaddr = purifyTaggedAddr(currState->vaddr_tainted,
243                                            currState->tc, currState->el);
244    else
245        currState->vaddr = currState->vaddr_tainted;
246
247    if (currState->aarch64) {
248        switch (currState->el) {
249          case EL0:
250          case EL1:
251            currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1);
252            currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL1);
253            break;
254          // @todo: uncomment this to enable Virtualization
255          // case EL2:
256          //   assert(haveVirtualization);
257          //   currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL2);
258          //   currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL2);
259          //   break;
260          case EL3:
261            assert(haveSecurity);
262            currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL3);
263            currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL3);
264            break;
265          default:
266            panic("Invalid exception level");
267            break;
268        }
269    } else {
270        currState->sctlr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
271            MISCREG_SCTLR, currState->tc, !currState->isSecure));
272        currState->ttbcr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
273            MISCREG_TTBCR, currState->tc, !currState->isSecure));
274        currState->htcr  = currState->tc->readMiscReg(MISCREG_HTCR);
275        currState->hcr   = currState->tc->readMiscReg(MISCREG_HCR);
276        currState->vtcr  = currState->tc->readMiscReg(MISCREG_VTCR);
277    }
278    sctlr = currState->sctlr;
279
280    currState->isFetch = (currState->mode == TLB::Execute);
281    currState->isWrite = (currState->mode == TLB::Write);
282
283    statRequestOrigin[REQUESTED][currState->isFetch]++;
284
285    // We only do a second stage of translation if we're not secure, or in
286    // hyp mode, the second stage MMU is enabled, and this table walker
287    // instance is the first stage.
288    currState->doingStage2 = false;
289    // @todo: for now disable this in AArch64 (HCR is not set)
290    currState->stage2Req = !currState->aarch64 && currState->hcr.vm &&
291                           !isStage2 && !currState->isSecure && !currState->isHyp;
292
293    bool long_desc_format = currState->aarch64 ||
294                            (_haveLPAE && currState->ttbcr.eae) ||
295                            _isHyp || isStage2;
296
297    if (long_desc_format) {
298        // Helper variables used for hierarchical permissions
299        currState->secureLookup = currState->isSecure;
300        currState->rwTable = true;
301        currState->userTable = true;
302        currState->xnTable = false;
303        currState->pxnTable = false;
304
305        ++statWalksLongDescriptor;
306    } else {
307        ++statWalksShortDescriptor;
308    }
309
310    if (!currState->timing) {
311        Fault fault = NoFault;
312        if (currState->aarch64)
313            fault = processWalkAArch64();
314        else if (long_desc_format)
315            fault = processWalkLPAE();
316        else
317            fault = processWalk();
318
319        // If this was a functional non-timing access restore state to
320        // how we found it.
321        if (currState->functional) {
322            delete currState;
323            currState = savedCurrState;
324        }
325        return fault;
326    }
327
328    if (pending || pendingQueue.size()) {
329        pendingQueue.push_back(currState);
330        currState = NULL;
331        pendingChange();
332    } else {
333        pending = true;
334        pendingChange();
335        if (currState->aarch64)
336            return processWalkAArch64();
337        else if (long_desc_format)
338            return processWalkLPAE();
339        else
340            return processWalk();
341    }
342
343    return NoFault;
344}
345
346void
347TableWalker::processWalkWrapper()
348{
349    assert(!currState);
350    assert(pendingQueue.size());
351    pendingChange();
352    currState = pendingQueue.front();
353
354    ExceptionLevel target_el = EL0;
355    if (currState->aarch64)
356        target_el = currEL(currState->tc);
357    else
358        target_el = EL1;
359
360    // Check if a previous walk filled this request already
361    // @TODO Should this always be the TLB or should we look in the stage2 TLB?
362    TlbEntry* te = tlb->lookup(currState->vaddr, currState->asid,
363            currState->vmid, currState->isHyp, currState->isSecure, true, false,
364            target_el);
365
366    // Check if we still need to have a walk for this request. If the requesting
367    // instruction has been squashed, or a previous walk has filled the TLB with
368    // a match, we just want to get rid of the walk. The latter could happen
369    // when there are multiple outstanding misses to a single page and a
370    // previous request has been successfully translated.
371    if (!currState->transState->squashed() && !te) {
372        // We've got a valid request, lets process it
373        pending = true;
374        pendingQueue.pop_front();
375        // Keep currState in case one of the processWalk... calls NULLs it
376        WalkerState *curr_state_copy = currState;
377        Fault f;
378        if (currState->aarch64)
379            f = processWalkAArch64();
380        else if ((_haveLPAE && currState->ttbcr.eae) || currState->isHyp || isStage2)
381            f = processWalkLPAE();
382        else
383            f = processWalk();
384
385        if (f != NoFault) {
386            curr_state_copy->transState->finish(f, curr_state_copy->req,
387                    curr_state_copy->tc, curr_state_copy->mode);
388
389            delete curr_state_copy;
390        }
391        return;
392    }
393
394
395    // If the instruction that we were translating for has been
396    // squashed we shouldn't bother.
397    unsigned num_squashed = 0;
398    ThreadContext *tc = currState->tc;
399    while ((num_squashed < numSquashable) && currState &&
400           (currState->transState->squashed() || te)) {
401        pendingQueue.pop_front();
402        num_squashed++;
403        statSquashedBefore++;
404
405        DPRINTF(TLB, "Squashing table walk for address %#x\n",
406                      currState->vaddr_tainted);
407
408        if (currState->transState->squashed()) {
409            // finish the translation which will delete the translation object
410            currState->transState->finish(
411                std::make_shared<UnimpFault>("Squashed Inst"),
412                currState->req, currState->tc, currState->mode);
413        } else {
414            // translate the request now that we know it will work
415            statWalkServiceTime.sample(curTick() - currState->startTime);
416            tlb->translateTiming(currState->req, currState->tc,
417                        currState->transState, currState->mode);
418
419        }
420
421        // delete the current request
422        delete currState;
423
424        // peak at the next one
425        if (pendingQueue.size()) {
426            currState = pendingQueue.front();
427            te = tlb->lookup(currState->vaddr, currState->asid,
428                currState->vmid, currState->isHyp, currState->isSecure, true,
429                false, target_el);
430        } else {
431            // Terminate the loop, nothing more to do
432            currState = NULL;
433        }
434    }
435    pendingChange();
436
437    // if we still have pending translations, schedule more work
438    nextWalk(tc);
439    currState = NULL;
440}
441
442Fault
443TableWalker::processWalk()
444{
445    Addr ttbr = 0;
446
447    // If translation isn't enabled, we shouldn't be here
448    assert(currState->sctlr.m || isStage2);
449
450    DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x, bits:%#x\n",
451            currState->vaddr_tainted, currState->ttbcr, mbits(currState->vaddr, 31,
452                                                      32 - currState->ttbcr.n));
453
454    statWalkWaitTime.sample(curTick() - currState->startTime);
455
456    if (currState->ttbcr.n == 0 || !mbits(currState->vaddr, 31,
457                                          32 - currState->ttbcr.n)) {
458        DPRINTF(TLB, " - Selecting TTBR0\n");
459        // Check if table walk is allowed when Security Extensions are enabled
460        if (haveSecurity && currState->ttbcr.pd0) {
461            if (currState->isFetch)
462                return std::make_shared<PrefetchAbort>(
463                    currState->vaddr_tainted,
464                    ArmFault::TranslationLL + L1,
465                    isStage2,
466                    ArmFault::VmsaTran);
467            else
468                return std::make_shared<DataAbort>(
469                    currState->vaddr_tainted,
470                    TlbEntry::DomainType::NoAccess, currState->isWrite,
471                    ArmFault::TranslationLL + L1, isStage2,
472                    ArmFault::VmsaTran);
473        }
474        ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
475            MISCREG_TTBR0, currState->tc, !currState->isSecure));
476    } else {
477        DPRINTF(TLB, " - Selecting TTBR1\n");
478        // Check if table walk is allowed when Security Extensions are enabled
479        if (haveSecurity && currState->ttbcr.pd1) {
480            if (currState->isFetch)
481                return std::make_shared<PrefetchAbort>(
482                    currState->vaddr_tainted,
483                    ArmFault::TranslationLL + L1,
484                    isStage2,
485                    ArmFault::VmsaTran);
486            else
487                return std::make_shared<DataAbort>(
488                    currState->vaddr_tainted,
489                    TlbEntry::DomainType::NoAccess, currState->isWrite,
490                    ArmFault::TranslationLL + L1, isStage2,
491                    ArmFault::VmsaTran);
492        }
493        ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
494            MISCREG_TTBR1, currState->tc, !currState->isSecure));
495        currState->ttbcr.n = 0;
496    }
497
498    Addr l1desc_addr = mbits(ttbr, 31, 14 - currState->ttbcr.n) |
499        (bits(currState->vaddr, 31 - currState->ttbcr.n, 20) << 2);
500    DPRINTF(TLB, " - Descriptor at address %#x (%s)\n", l1desc_addr,
501            currState->isSecure ? "s" : "ns");
502
503    // Trickbox address check
504    Fault f;
505    f = testWalk(l1desc_addr, sizeof(uint32_t),
506                 TlbEntry::DomainType::NoAccess, L1);
507    if (f) {
508        DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
509        if (currState->timing) {
510            pending = false;
511            nextWalk(currState->tc);
512            currState = NULL;
513        } else {
514            currState->tc = NULL;
515            currState->req = NULL;
516        }
517        return f;
518    }
519
520    Request::Flags flag = Request::PT_WALK;
521    if (currState->sctlr.c == 0) {
522        flag.set(Request::UNCACHEABLE);
523    }
524
525    if (currState->isSecure) {
526        flag.set(Request::SECURE);
527    }
528
529    bool delayed;
530    delayed = fetchDescriptor(l1desc_addr, (uint8_t*)&currState->l1Desc.data,
531                              sizeof(uint32_t), flag, L1, &doL1DescEvent,
532                              &TableWalker::doL1Descriptor);
533    if (!delayed) {
534       f = currState->fault;
535    }
536
537    return f;
538}
539
540Fault
541TableWalker::processWalkLPAE()
542{
543    Addr ttbr, ttbr0_max, ttbr1_min, desc_addr;
544    int tsz, n;
545    LookupLevel start_lookup_level = L1;
546
547    DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x\n",
548            currState->vaddr_tainted, currState->ttbcr);
549
550    statWalkWaitTime.sample(curTick() - currState->startTime);
551
552    Request::Flags flag = Request::PT_WALK;
553    if (currState->isSecure)
554        flag.set(Request::SECURE);
555
556    // work out which base address register to use, if in hyp mode we always
557    // use HTTBR
558    if (isStage2) {
559        DPRINTF(TLB, " - Selecting VTTBR (long-desc.)\n");
560        ttbr = currState->tc->readMiscReg(MISCREG_VTTBR);
561        tsz  = sext<4>(currState->vtcr.t0sz);
562        start_lookup_level = currState->vtcr.sl0 ? L1 : L2;
563    } else if (currState->isHyp) {
564        DPRINTF(TLB, " - Selecting HTTBR (long-desc.)\n");
565        ttbr = currState->tc->readMiscReg(MISCREG_HTTBR);
566        tsz  = currState->htcr.t0sz;
567    } else {
568        assert(_haveLPAE && currState->ttbcr.eae);
569
570        // Determine boundaries of TTBR0/1 regions
571        if (currState->ttbcr.t0sz)
572            ttbr0_max = (1ULL << (32 - currState->ttbcr.t0sz)) - 1;
573        else if (currState->ttbcr.t1sz)
574            ttbr0_max = (1ULL << 32) -
575                (1ULL << (32 - currState->ttbcr.t1sz)) - 1;
576        else
577            ttbr0_max = (1ULL << 32) - 1;
578        if (currState->ttbcr.t1sz)
579            ttbr1_min = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz));
580        else
581            ttbr1_min = (1ULL << (32 - currState->ttbcr.t0sz));
582
583        // The following code snippet selects the appropriate translation table base
584        // address (TTBR0 or TTBR1) and the appropriate starting lookup level
585        // depending on the address range supported by the translation table (ARM
586        // ARM issue C B3.6.4)
587        if (currState->vaddr <= ttbr0_max) {
588            DPRINTF(TLB, " - Selecting TTBR0 (long-desc.)\n");
589            // Check if table walk is allowed
590            if (currState->ttbcr.epd0) {
591                if (currState->isFetch)
592                    return std::make_shared<PrefetchAbort>(
593                        currState->vaddr_tainted,
594                        ArmFault::TranslationLL + L1,
595                        isStage2,
596                        ArmFault::LpaeTran);
597                else
598                    return std::make_shared<DataAbort>(
599                        currState->vaddr_tainted,
600                        TlbEntry::DomainType::NoAccess,
601                        currState->isWrite,
602                        ArmFault::TranslationLL + L1,
603                        isStage2,
604                        ArmFault::LpaeTran);
605            }
606            ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
607                MISCREG_TTBR0, currState->tc, !currState->isSecure));
608            tsz = currState->ttbcr.t0sz;
609            if (ttbr0_max < (1ULL << 30))  // Upper limit < 1 GB
610                start_lookup_level = L2;
611        } else if (currState->vaddr >= ttbr1_min) {
612            DPRINTF(TLB, " - Selecting TTBR1 (long-desc.)\n");
613            // Check if table walk is allowed
614            if (currState->ttbcr.epd1) {
615                if (currState->isFetch)
616                    return std::make_shared<PrefetchAbort>(
617                        currState->vaddr_tainted,
618                        ArmFault::TranslationLL + L1,
619                        isStage2,
620                        ArmFault::LpaeTran);
621                else
622                    return std::make_shared<DataAbort>(
623                        currState->vaddr_tainted,
624                        TlbEntry::DomainType::NoAccess,
625                        currState->isWrite,
626                        ArmFault::TranslationLL + L1,
627                        isStage2,
628                        ArmFault::LpaeTran);
629            }
630            ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
631                MISCREG_TTBR1, currState->tc, !currState->isSecure));
632            tsz = currState->ttbcr.t1sz;
633            if (ttbr1_min >= (1ULL << 31) + (1ULL << 30))  // Lower limit >= 3 GB
634                start_lookup_level = L2;
635        } else {
636            // Out of boundaries -> translation fault
637            if (currState->isFetch)
638                return std::make_shared<PrefetchAbort>(
639                    currState->vaddr_tainted,
640                    ArmFault::TranslationLL + L1,
641                    isStage2,
642                    ArmFault::LpaeTran);
643            else
644                return std::make_shared<DataAbort>(
645                    currState->vaddr_tainted,
646                    TlbEntry::DomainType::NoAccess,
647                    currState->isWrite, ArmFault::TranslationLL + L1,
648                    isStage2, ArmFault::LpaeTran);
649        }
650
651    }
652
653    // Perform lookup (ARM ARM issue C B3.6.6)
654    if (start_lookup_level == L1) {
655        n = 5 - tsz;
656        desc_addr = mbits(ttbr, 39, n) |
657            (bits(currState->vaddr, n + 26, 30) << 3);
658        DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
659                desc_addr, currState->isSecure ? "s" : "ns");
660    } else {
661        // Skip first-level lookup
662        n = (tsz >= 2 ? 14 - tsz : 12);
663        desc_addr = mbits(ttbr, 39, n) |
664            (bits(currState->vaddr, n + 17, 21) << 3);
665        DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
666                desc_addr, currState->isSecure ? "s" : "ns");
667    }
668
669    // Trickbox address check
670    Fault f = testWalk(desc_addr, sizeof(uint64_t),
671                       TlbEntry::DomainType::NoAccess, start_lookup_level);
672    if (f) {
673        DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
674        if (currState->timing) {
675            pending = false;
676            nextWalk(currState->tc);
677            currState = NULL;
678        } else {
679            currState->tc = NULL;
680            currState->req = NULL;
681        }
682        return f;
683    }
684
685    if (currState->sctlr.c == 0) {
686        flag.set(Request::UNCACHEABLE);
687    }
688
689    currState->longDesc.lookupLevel = start_lookup_level;
690    currState->longDesc.aarch64 = false;
691    currState->longDesc.grainSize = Grain4KB;
692
693    Event *event = start_lookup_level == L1 ? (Event *) &doL1LongDescEvent
694                                            : (Event *) &doL2LongDescEvent;
695
696    bool delayed = fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data,
697                                   sizeof(uint64_t), flag, start_lookup_level,
698                                   event, &TableWalker::doLongDescriptor);
699    if (!delayed) {
700        f = currState->fault;
701    }
702
703    return f;
704}
705
706unsigned
707TableWalker::adjustTableSizeAArch64(unsigned tsz)
708{
709    if (tsz < 25)
710        return 25;
711    if (tsz > 48)
712        return 48;
713    return tsz;
714}
715
716bool
717TableWalker::checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange)
718{
719    return (currPhysAddrRange != MaxPhysAddrRange &&
720            bits(addr, MaxPhysAddrRange - 1, currPhysAddrRange));
721}
722
723Fault
724TableWalker::processWalkAArch64()
725{
726    assert(currState->aarch64);
727
728    DPRINTF(TLB, "Beginning table walk for address %#llx, TCR: %#llx\n",
729            currState->vaddr_tainted, currState->tcr);
730
731    static const GrainSize GrainMapDefault[] =
732      { Grain4KB, Grain64KB, Grain16KB, ReservedGrain };
733    static const GrainSize GrainMap_EL1_tg1[] =
734      { ReservedGrain, Grain16KB, Grain4KB, Grain64KB };
735
736    statWalkWaitTime.sample(curTick() - currState->startTime);
737
738    // Determine TTBR, table size, granule size and phys. address range
739    Addr ttbr = 0;
740    int tsz = 0, ps = 0;
741    GrainSize tg = Grain4KB; // grain size computed from tg* field
742    bool fault = false;
743    switch (currState->el) {
744      case EL0:
745      case EL1:
746        switch (bits(currState->vaddr, 63,48)) {
747          case 0:
748            DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
749            ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL1);
750            tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
751            tg = GrainMapDefault[currState->tcr.tg0];
752            if (bits(currState->vaddr, 63, tsz) != 0x0 ||
753                currState->tcr.epd0)
754              fault = true;
755            break;
756          case 0xffff:
757            DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n");
758            ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL1);
759            tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz);
760            tg = GrainMap_EL1_tg1[currState->tcr.tg1];
761            if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) ||
762                currState->tcr.epd1)
763              fault = true;
764            break;
765          default:
766            // top two bytes must be all 0s or all 1s, else invalid addr
767            fault = true;
768        }
769        ps = currState->tcr.ips;
770        break;
771      case EL2:
772      case EL3:
773        switch(bits(currState->vaddr, 63,48)) {
774            case 0:
775                DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
776                if (currState->el == EL2)
777                    ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL2);
778                else
779                    ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL3);
780                tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
781                tg = GrainMapDefault[currState->tcr.tg0];
782                break;
783            default:
784                // invalid addr if top two bytes are not all 0s
785                fault = true;
786        }
787        ps = currState->tcr.ips;
788        break;
789    }
790
791    if (fault) {
792        Fault f;
793        if (currState->isFetch)
794            f =  std::make_shared<PrefetchAbort>(
795                currState->vaddr_tainted,
796                ArmFault::TranslationLL + L0, isStage2,
797                ArmFault::LpaeTran);
798        else
799            f = std::make_shared<DataAbort>(
800                currState->vaddr_tainted,
801                TlbEntry::DomainType::NoAccess,
802                currState->isWrite,
803                ArmFault::TranslationLL + L0,
804                isStage2, ArmFault::LpaeTran);
805
806        if (currState->timing) {
807            pending = false;
808            nextWalk(currState->tc);
809            currState = NULL;
810        } else {
811            currState->tc = NULL;
812            currState->req = NULL;
813        }
814        return f;
815
816    }
817
818    if (tg == ReservedGrain) {
819        warn_once("Reserved granule size requested; gem5's IMPLEMENTATION "
820                  "DEFINED behavior takes this to mean 4KB granules\n");
821        tg = Grain4KB;
822    }
823
824    int stride = tg - 3;
825    LookupLevel start_lookup_level = MAX_LOOKUP_LEVELS;
826
827    // Determine starting lookup level
828    // See aarch64/translation/walk in Appendix G: ARMv8 Pseudocode Library
829    // in ARM DDI 0487A.  These table values correspond to the cascading tests
830    // to compute the lookup level and are of the form
831    // (grain_size + N*stride), for N = {1, 2, 3}.
832    // A value of 64 will never succeed and a value of 0 will always succeed.
833    {
834        struct GrainMap {
835            GrainSize grain_size;
836            unsigned lookup_level_cutoff[MAX_LOOKUP_LEVELS];
837        };
838        static const GrainMap GM[] = {
839            { Grain4KB,  { 39, 30,  0, 0 } },
840            { Grain16KB, { 47, 36, 25, 0 } },
841            { Grain64KB, { 64, 42, 29, 0 } }
842        };
843
844        const unsigned *lookup = NULL; // points to a lookup_level_cutoff
845
846        for (unsigned i = 0; i < 3; ++i) { // choose entry of GM[]
847            if (tg == GM[i].grain_size) {
848                lookup = GM[i].lookup_level_cutoff;
849                break;
850            }
851        }
852        assert(lookup);
853
854        for (int L = L0; L != MAX_LOOKUP_LEVELS; ++L) {
855            if (tsz > lookup[L]) {
856                start_lookup_level = (LookupLevel) L;
857                break;
858            }
859        }
860        panic_if(start_lookup_level == MAX_LOOKUP_LEVELS,
861                 "Table walker couldn't find lookup level\n");
862    }
863
864    // Determine table base address
865    int base_addr_lo = 3 + tsz - stride * (3 - start_lookup_level) - tg;
866    Addr base_addr = mbits(ttbr, 47, base_addr_lo);
867
868    // Determine physical address size and raise an Address Size Fault if
869    // necessary
870    int pa_range = decodePhysAddrRange64(ps);
871    // Clamp to lower limit
872    if (pa_range > physAddrRange)
873        currState->physAddrRange = physAddrRange;
874    else
875        currState->physAddrRange = pa_range;
876    if (checkAddrSizeFaultAArch64(base_addr, currState->physAddrRange)) {
877        DPRINTF(TLB, "Address size fault before any lookup\n");
878        Fault f;
879        if (currState->isFetch)
880            f = std::make_shared<PrefetchAbort>(
881                currState->vaddr_tainted,
882                ArmFault::AddressSizeLL + start_lookup_level,
883                isStage2,
884                ArmFault::LpaeTran);
885        else
886            f = std::make_shared<DataAbort>(
887                currState->vaddr_tainted,
888                TlbEntry::DomainType::NoAccess,
889                currState->isWrite,
890                ArmFault::AddressSizeLL + start_lookup_level,
891                isStage2,
892                ArmFault::LpaeTran);
893
894
895        if (currState->timing) {
896            pending = false;
897            nextWalk(currState->tc);
898            currState = NULL;
899        } else {
900            currState->tc = NULL;
901            currState->req = NULL;
902        }
903        return f;
904
905   }
906
907    // Determine descriptor address
908    Addr desc_addr = base_addr |
909        (bits(currState->vaddr, tsz - 1,
910              stride * (3 - start_lookup_level) + tg) << 3);
911
912    // Trickbox address check
913    Fault f = testWalk(desc_addr, sizeof(uint64_t),
914                       TlbEntry::DomainType::NoAccess, start_lookup_level);
915    if (f) {
916        DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
917        if (currState->timing) {
918            pending = false;
919            nextWalk(currState->tc);
920            currState = NULL;
921        } else {
922            currState->tc = NULL;
923            currState->req = NULL;
924        }
925        return f;
926    }
927
928    Request::Flags flag = Request::PT_WALK;
929    if (currState->sctlr.c == 0) {
930        flag.set(Request::UNCACHEABLE);
931    }
932
933    if (currState->isSecure) {
934        flag.set(Request::SECURE);
935    }
936
937    currState->longDesc.lookupLevel = start_lookup_level;
938    currState->longDesc.aarch64 = true;
939    currState->longDesc.grainSize = tg;
940
941    if (currState->timing) {
942        Event *event;
943        switch (start_lookup_level) {
944          case L0:
945            event = (Event *) &doL0LongDescEvent;
946            break;
947          case L1:
948            event = (Event *) &doL1LongDescEvent;
949            break;
950          case L2:
951            event = (Event *) &doL2LongDescEvent;
952            break;
953          case L3:
954            event = (Event *) &doL3LongDescEvent;
955            break;
956          default:
957            panic("Invalid table lookup level");
958            break;
959        }
960        port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
961                       event, (uint8_t*) &currState->longDesc.data,
962                       currState->tc->getCpuPtr()->clockPeriod(), flag);
963        DPRINTF(TLBVerbose,
964                "Adding to walker fifo: queue size before adding: %d\n",
965                stateQueues[start_lookup_level].size());
966        stateQueues[start_lookup_level].push_back(currState);
967        currState = NULL;
968    } else if (!currState->functional) {
969        port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
970                       NULL, (uint8_t*) &currState->longDesc.data,
971                       currState->tc->getCpuPtr()->clockPeriod(), flag);
972        doLongDescriptor();
973        f = currState->fault;
974    } else {
975        RequestPtr req = new Request(desc_addr, sizeof(uint64_t), flag,
976                                     masterId);
977        PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
978        pkt->dataStatic((uint8_t*) &currState->longDesc.data);
979        port->sendFunctional(pkt);
980        doLongDescriptor();
981        delete req;
982        delete pkt;
983        f = currState->fault;
984    }
985
986    return f;
987}
988
989void
990TableWalker::memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr,
991                      uint8_t texcb, bool s)
992{
993    // Note: tc and sctlr local variables are hiding tc and sctrl class
994    // variables
995    DPRINTF(TLBVerbose, "memAttrs texcb:%d s:%d\n", texcb, s);
996    te.shareable = false; // default value
997    te.nonCacheable = false;
998    te.outerShareable = false;
999    if (sctlr.tre == 0 || ((sctlr.tre == 1) && (sctlr.m == 0))) {
1000        switch(texcb) {
1001          case 0: // Stongly-ordered
1002            te.nonCacheable = true;
1003            te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1004            te.shareable = true;
1005            te.innerAttrs = 1;
1006            te.outerAttrs = 0;
1007            break;
1008          case 1: // Shareable Device
1009            te.nonCacheable = true;
1010            te.mtype = TlbEntry::MemoryType::Device;
1011            te.shareable = true;
1012            te.innerAttrs = 3;
1013            te.outerAttrs = 0;
1014            break;
1015          case 2: // Outer and Inner Write-Through, no Write-Allocate
1016            te.mtype = TlbEntry::MemoryType::Normal;
1017            te.shareable = s;
1018            te.innerAttrs = 6;
1019            te.outerAttrs = bits(texcb, 1, 0);
1020            break;
1021          case 3: // Outer and Inner Write-Back, no Write-Allocate
1022            te.mtype = TlbEntry::MemoryType::Normal;
1023            te.shareable = s;
1024            te.innerAttrs = 7;
1025            te.outerAttrs = bits(texcb, 1, 0);
1026            break;
1027          case 4: // Outer and Inner Non-cacheable
1028            te.nonCacheable = true;
1029            te.mtype = TlbEntry::MemoryType::Normal;
1030            te.shareable = s;
1031            te.innerAttrs = 0;
1032            te.outerAttrs = bits(texcb, 1, 0);
1033            break;
1034          case 5: // Reserved
1035            panic("Reserved texcb value!\n");
1036            break;
1037          case 6: // Implementation Defined
1038            panic("Implementation-defined texcb value!\n");
1039            break;
1040          case 7: // Outer and Inner Write-Back, Write-Allocate
1041            te.mtype = TlbEntry::MemoryType::Normal;
1042            te.shareable = s;
1043            te.innerAttrs = 5;
1044            te.outerAttrs = 1;
1045            break;
1046          case 8: // Non-shareable Device
1047            te.nonCacheable = true;
1048            te.mtype = TlbEntry::MemoryType::Device;
1049            te.shareable = false;
1050            te.innerAttrs = 3;
1051            te.outerAttrs = 0;
1052            break;
1053          case 9 ... 15:  // Reserved
1054            panic("Reserved texcb value!\n");
1055            break;
1056          case 16 ... 31: // Cacheable Memory
1057            te.mtype = TlbEntry::MemoryType::Normal;
1058            te.shareable = s;
1059            if (bits(texcb, 1,0) == 0 || bits(texcb, 3,2) == 0)
1060                te.nonCacheable = true;
1061            te.innerAttrs = bits(texcb, 1, 0);
1062            te.outerAttrs = bits(texcb, 3, 2);
1063            break;
1064          default:
1065            panic("More than 32 states for 5 bits?\n");
1066        }
1067    } else {
1068        assert(tc);
1069        PRRR prrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_PRRR,
1070                                    currState->tc, !currState->isSecure));
1071        NMRR nmrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_NMRR,
1072                                    currState->tc, !currState->isSecure));
1073        DPRINTF(TLBVerbose, "memAttrs PRRR:%08x NMRR:%08x\n", prrr, nmrr);
1074        uint8_t curr_tr = 0, curr_ir = 0, curr_or = 0;
1075        switch(bits(texcb, 2,0)) {
1076          case 0:
1077            curr_tr = prrr.tr0;
1078            curr_ir = nmrr.ir0;
1079            curr_or = nmrr.or0;
1080            te.outerShareable = (prrr.nos0 == 0);
1081            break;
1082          case 1:
1083            curr_tr = prrr.tr1;
1084            curr_ir = nmrr.ir1;
1085            curr_or = nmrr.or1;
1086            te.outerShareable = (prrr.nos1 == 0);
1087            break;
1088          case 2:
1089            curr_tr = prrr.tr2;
1090            curr_ir = nmrr.ir2;
1091            curr_or = nmrr.or2;
1092            te.outerShareable = (prrr.nos2 == 0);
1093            break;
1094          case 3:
1095            curr_tr = prrr.tr3;
1096            curr_ir = nmrr.ir3;
1097            curr_or = nmrr.or3;
1098            te.outerShareable = (prrr.nos3 == 0);
1099            break;
1100          case 4:
1101            curr_tr = prrr.tr4;
1102            curr_ir = nmrr.ir4;
1103            curr_or = nmrr.or4;
1104            te.outerShareable = (prrr.nos4 == 0);
1105            break;
1106          case 5:
1107            curr_tr = prrr.tr5;
1108            curr_ir = nmrr.ir5;
1109            curr_or = nmrr.or5;
1110            te.outerShareable = (prrr.nos5 == 0);
1111            break;
1112          case 6:
1113            panic("Imp defined type\n");
1114          case 7:
1115            curr_tr = prrr.tr7;
1116            curr_ir = nmrr.ir7;
1117            curr_or = nmrr.or7;
1118            te.outerShareable = (prrr.nos7 == 0);
1119            break;
1120        }
1121
1122        switch(curr_tr) {
1123          case 0:
1124            DPRINTF(TLBVerbose, "StronglyOrdered\n");
1125            te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1126            te.nonCacheable = true;
1127            te.innerAttrs = 1;
1128            te.outerAttrs = 0;
1129            te.shareable = true;
1130            break;
1131          case 1:
1132            DPRINTF(TLBVerbose, "Device ds1:%d ds0:%d s:%d\n",
1133                    prrr.ds1, prrr.ds0, s);
1134            te.mtype = TlbEntry::MemoryType::Device;
1135            te.nonCacheable = true;
1136            te.innerAttrs = 3;
1137            te.outerAttrs = 0;
1138            if (prrr.ds1 && s)
1139                te.shareable = true;
1140            if (prrr.ds0 && !s)
1141                te.shareable = true;
1142            break;
1143          case 2:
1144            DPRINTF(TLBVerbose, "Normal ns1:%d ns0:%d s:%d\n",
1145                    prrr.ns1, prrr.ns0, s);
1146            te.mtype = TlbEntry::MemoryType::Normal;
1147            if (prrr.ns1 && s)
1148                te.shareable = true;
1149            if (prrr.ns0 && !s)
1150                te.shareable = true;
1151            break;
1152          case 3:
1153            panic("Reserved type");
1154        }
1155
1156        if (te.mtype == TlbEntry::MemoryType::Normal){
1157            switch(curr_ir) {
1158              case 0:
1159                te.nonCacheable = true;
1160                te.innerAttrs = 0;
1161                break;
1162              case 1:
1163                te.innerAttrs = 5;
1164                break;
1165              case 2:
1166                te.innerAttrs = 6;
1167                break;
1168              case 3:
1169                te.innerAttrs = 7;
1170                break;
1171            }
1172
1173            switch(curr_or) {
1174              case 0:
1175                te.nonCacheable = true;
1176                te.outerAttrs = 0;
1177                break;
1178              case 1:
1179                te.outerAttrs = 1;
1180                break;
1181              case 2:
1182                te.outerAttrs = 2;
1183                break;
1184              case 3:
1185                te.outerAttrs = 3;
1186                break;
1187            }
1188        }
1189    }
1190    DPRINTF(TLBVerbose, "memAttrs: shareable: %d, innerAttrs: %d, "
1191            "outerAttrs: %d\n",
1192            te.shareable, te.innerAttrs, te.outerAttrs);
1193    te.setAttributes(false);
1194}
1195
1196void
1197TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te,
1198    LongDescriptor &lDescriptor)
1199{
1200    assert(_haveLPAE);
1201
1202    uint8_t attr;
1203    uint8_t sh = lDescriptor.sh();
1204    // Different format and source of attributes if this is a stage 2
1205    // translation
1206    if (isStage2) {
1207        attr = lDescriptor.memAttr();
1208        uint8_t attr_3_2 = (attr >> 2) & 0x3;
1209        uint8_t attr_1_0 =  attr       & 0x3;
1210
1211        DPRINTF(TLBVerbose, "memAttrsLPAE MemAttr:%#x sh:%#x\n", attr, sh);
1212
1213        if (attr_3_2 == 0) {
1214            te.mtype        = attr_1_0 == 0 ? TlbEntry::MemoryType::StronglyOrdered
1215                                            : TlbEntry::MemoryType::Device;
1216            te.outerAttrs   = 0;
1217            te.innerAttrs   = attr_1_0 == 0 ? 1 : 3;
1218            te.nonCacheable = true;
1219        } else {
1220            te.mtype        = TlbEntry::MemoryType::Normal;
1221            te.outerAttrs   = attr_3_2 == 1 ? 0 :
1222                              attr_3_2 == 2 ? 2 : 1;
1223            te.innerAttrs   = attr_1_0 == 1 ? 0 :
1224                              attr_1_0 == 2 ? 6 : 5;
1225            te.nonCacheable = (attr_3_2 == 1) || (attr_1_0 == 1);
1226        }
1227    } else {
1228        uint8_t attrIndx = lDescriptor.attrIndx();
1229
1230        // LPAE always uses remapping of memory attributes, irrespective of the
1231        // value of SCTLR.TRE
1232        MiscRegIndex reg = attrIndx & 0x4 ? MISCREG_MAIR1 : MISCREG_MAIR0;
1233        int reg_as_int = flattenMiscRegNsBanked(reg, currState->tc,
1234                                                !currState->isSecure);
1235        uint32_t mair = currState->tc->readMiscReg(reg_as_int);
1236        attr = (mair >> (8 * (attrIndx % 4))) & 0xff;
1237        uint8_t attr_7_4 = bits(attr, 7, 4);
1238        uint8_t attr_3_0 = bits(attr, 3, 0);
1239        DPRINTF(TLBVerbose, "memAttrsLPAE AttrIndx:%#x sh:%#x, attr %#x\n", attrIndx, sh, attr);
1240
1241        // Note: the memory subsystem only cares about the 'cacheable' memory
1242        // attribute. The other attributes are only used to fill the PAR register
1243        // accordingly to provide the illusion of full support
1244        te.nonCacheable = false;
1245
1246        switch (attr_7_4) {
1247          case 0x0:
1248            // Strongly-ordered or Device memory
1249            if (attr_3_0 == 0x0)
1250                te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1251            else if (attr_3_0 == 0x4)
1252                te.mtype = TlbEntry::MemoryType::Device;
1253            else
1254                panic("Unpredictable behavior\n");
1255            te.nonCacheable = true;
1256            te.outerAttrs   = 0;
1257            break;
1258          case 0x4:
1259            // Normal memory, Outer Non-cacheable
1260            te.mtype = TlbEntry::MemoryType::Normal;
1261            te.outerAttrs = 0;
1262            if (attr_3_0 == 0x4)
1263                // Inner Non-cacheable
1264                te.nonCacheable = true;
1265            else if (attr_3_0 < 0x8)
1266                panic("Unpredictable behavior\n");
1267            break;
1268          case 0x8:
1269          case 0x9:
1270          case 0xa:
1271          case 0xb:
1272          case 0xc:
1273          case 0xd:
1274          case 0xe:
1275          case 0xf:
1276            if (attr_7_4 & 0x4) {
1277                te.outerAttrs = (attr_7_4 & 1) ? 1 : 3;
1278            } else {
1279                te.outerAttrs = 0x2;
1280            }
1281            // Normal memory, Outer Cacheable
1282            te.mtype = TlbEntry::MemoryType::Normal;
1283            if (attr_3_0 != 0x4 && attr_3_0 < 0x8)
1284                panic("Unpredictable behavior\n");
1285            break;
1286          default:
1287            panic("Unpredictable behavior\n");
1288            break;
1289        }
1290
1291        switch (attr_3_0) {
1292          case 0x0:
1293            te.innerAttrs = 0x1;
1294            break;
1295          case 0x4:
1296            te.innerAttrs = attr_7_4 == 0 ? 0x3 : 0;
1297            break;
1298          case 0x8:
1299          case 0x9:
1300          case 0xA:
1301          case 0xB:
1302            te.innerAttrs = 6;
1303            break;
1304          case 0xC:
1305          case 0xD:
1306          case 0xE:
1307          case 0xF:
1308            te.innerAttrs = attr_3_0 & 1 ? 0x5 : 0x7;
1309            break;
1310          default:
1311            panic("Unpredictable behavior\n");
1312            break;
1313        }
1314    }
1315
1316    te.outerShareable = sh == 2;
1317    te.shareable       = (sh & 0x2) ? true : false;
1318    te.setAttributes(true);
1319    te.attributes |= (uint64_t) attr << 56;
1320}
1321
1322void
1323TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, uint8_t attrIndx,
1324                             uint8_t sh)
1325{
1326    DPRINTF(TLBVerbose, "memAttrsAArch64 AttrIndx:%#x sh:%#x\n", attrIndx, sh);
1327
1328    // Select MAIR
1329    uint64_t mair;
1330    switch (currState->el) {
1331      case EL0:
1332      case EL1:
1333        mair = tc->readMiscReg(MISCREG_MAIR_EL1);
1334        break;
1335      case EL2:
1336        mair = tc->readMiscReg(MISCREG_MAIR_EL2);
1337        break;
1338      case EL3:
1339        mair = tc->readMiscReg(MISCREG_MAIR_EL3);
1340        break;
1341      default:
1342        panic("Invalid exception level");
1343        break;
1344    }
1345
1346    // Select attributes
1347    uint8_t attr = bits(mair, 8 * attrIndx + 7, 8 * attrIndx);
1348    uint8_t attr_lo = bits(attr, 3, 0);
1349    uint8_t attr_hi = bits(attr, 7, 4);
1350
1351    // Memory type
1352    te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal;
1353
1354    // Cacheability
1355    te.nonCacheable = false;
1356    if (te.mtype == TlbEntry::MemoryType::Device ||  // Device memory
1357        attr_hi == 0x8 ||  // Normal memory, Outer Non-cacheable
1358        attr_lo == 0x8) {  // Normal memory, Inner Non-cacheable
1359        te.nonCacheable = true;
1360    }
1361
1362    te.shareable       = sh == 2;
1363    te.outerShareable = (sh & 0x2) ? true : false;
1364    // Attributes formatted according to the 64-bit PAR
1365    te.attributes = ((uint64_t) attr << 56) |
1366        (1 << 11) |     // LPAE bit
1367        (te.ns << 9) |  // NS bit
1368        (sh << 7);
1369}
1370
1371void
1372TableWalker::doL1Descriptor()
1373{
1374    if (currState->fault != NoFault) {
1375        return;
1376    }
1377
1378    DPRINTF(TLB, "L1 descriptor for %#x is %#x\n",
1379            currState->vaddr_tainted, currState->l1Desc.data);
1380    TlbEntry te;
1381
1382    switch (currState->l1Desc.type()) {
1383      case L1Descriptor::Ignore:
1384      case L1Descriptor::Reserved:
1385        if (!currState->timing) {
1386            currState->tc = NULL;
1387            currState->req = NULL;
1388        }
1389        DPRINTF(TLB, "L1 Descriptor Reserved/Ignore, causing fault\n");
1390        if (currState->isFetch)
1391            currState->fault =
1392                std::make_shared<PrefetchAbort>(
1393                    currState->vaddr_tainted,
1394                    ArmFault::TranslationLL + L1,
1395                    isStage2,
1396                    ArmFault::VmsaTran);
1397        else
1398            currState->fault =
1399                std::make_shared<DataAbort>(
1400                    currState->vaddr_tainted,
1401                    TlbEntry::DomainType::NoAccess,
1402                    currState->isWrite,
1403                    ArmFault::TranslationLL + L1, isStage2,
1404                    ArmFault::VmsaTran);
1405        return;
1406      case L1Descriptor::Section:
1407        if (currState->sctlr.afe && bits(currState->l1Desc.ap(), 0) == 0) {
1408            /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is
1409              * enabled if set, do l1.Desc.setAp0() instead of generating
1410              * AccessFlag0
1411              */
1412
1413            currState->fault = std::make_shared<DataAbort>(
1414                currState->vaddr_tainted,
1415                currState->l1Desc.domain(),
1416                currState->isWrite,
1417                ArmFault::AccessFlagLL + L1,
1418                isStage2,
1419                ArmFault::VmsaTran);
1420        }
1421        if (currState->l1Desc.supersection()) {
1422            panic("Haven't implemented supersections\n");
1423        }
1424        insertTableEntry(currState->l1Desc, false);
1425        return;
1426      case L1Descriptor::PageTable:
1427        {
1428            Addr l2desc_addr;
1429            l2desc_addr = currState->l1Desc.l2Addr() |
1430                (bits(currState->vaddr, 19, 12) << 2);
1431            DPRINTF(TLB, "L1 descriptor points to page table at: %#x (%s)\n",
1432                    l2desc_addr, currState->isSecure ? "s" : "ns");
1433
1434            // Trickbox address check
1435            currState->fault = testWalk(l2desc_addr, sizeof(uint32_t),
1436                                        currState->l1Desc.domain(), L2);
1437
1438            if (currState->fault) {
1439                if (!currState->timing) {
1440                    currState->tc = NULL;
1441                    currState->req = NULL;
1442                }
1443                return;
1444            }
1445
1446            Request::Flags flag = Request::PT_WALK;
1447            if (currState->isSecure)
1448                flag.set(Request::SECURE);
1449
1450            bool delayed;
1451            delayed = fetchDescriptor(l2desc_addr,
1452                                      (uint8_t*)&currState->l2Desc.data,
1453                                      sizeof(uint32_t), flag, -1, &doL2DescEvent,
1454                                      &TableWalker::doL2Descriptor);
1455            if (delayed) {
1456                currState->delayed = true;
1457            }
1458
1459            return;
1460        }
1461      default:
1462        panic("A new type in a 2 bit field?\n");
1463    }
1464}
1465
1466void
1467TableWalker::doLongDescriptor()
1468{
1469    if (currState->fault != NoFault) {
1470        return;
1471    }
1472
1473    DPRINTF(TLB, "L%d descriptor for %#llx is %#llx (%s)\n",
1474            currState->longDesc.lookupLevel, currState->vaddr_tainted,
1475            currState->longDesc.data,
1476            currState->aarch64 ? "AArch64" : "long-desc.");
1477
1478    if ((currState->longDesc.type() == LongDescriptor::Block) ||
1479        (currState->longDesc.type() == LongDescriptor::Page)) {
1480        DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, pxn: %d, "
1481                "xn: %d, ap: %d, af: %d, type: %d\n",
1482                currState->longDesc.lookupLevel,
1483                currState->longDesc.data,
1484                currState->longDesc.pxn(),
1485                currState->longDesc.xn(),
1486                currState->longDesc.ap(),
1487                currState->longDesc.af(),
1488                currState->longDesc.type());
1489    } else {
1490        DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, type: %d\n",
1491                currState->longDesc.lookupLevel,
1492                currState->longDesc.data,
1493                currState->longDesc.type());
1494    }
1495
1496    TlbEntry te;
1497
1498    switch (currState->longDesc.type()) {
1499      case LongDescriptor::Invalid:
1500        if (!currState->timing) {
1501            currState->tc = NULL;
1502            currState->req = NULL;
1503        }
1504
1505        DPRINTF(TLB, "L%d descriptor Invalid, causing fault type %d\n",
1506                currState->longDesc.lookupLevel,
1507                ArmFault::TranslationLL + currState->longDesc.lookupLevel);
1508        if (currState->isFetch)
1509            currState->fault = std::make_shared<PrefetchAbort>(
1510                currState->vaddr_tainted,
1511                ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1512                isStage2,
1513                ArmFault::LpaeTran);
1514        else
1515            currState->fault = std::make_shared<DataAbort>(
1516                currState->vaddr_tainted,
1517                TlbEntry::DomainType::NoAccess,
1518                currState->isWrite,
1519                ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1520                isStage2,
1521                ArmFault::LpaeTran);
1522        return;
1523      case LongDescriptor::Block:
1524      case LongDescriptor::Page:
1525        {
1526            bool fault = false;
1527            bool aff = false;
1528            // Check for address size fault
1529            if (checkAddrSizeFaultAArch64(
1530                    mbits(currState->longDesc.data, MaxPhysAddrRange - 1,
1531                          currState->longDesc.offsetBits()),
1532                    currState->physAddrRange)) {
1533                fault = true;
1534                DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1535                        currState->longDesc.lookupLevel);
1536            // Check for access fault
1537            } else if (currState->longDesc.af() == 0) {
1538                fault = true;
1539                DPRINTF(TLB, "L%d descriptor causing Access Fault\n",
1540                        currState->longDesc.lookupLevel);
1541                aff = true;
1542            }
1543            if (fault) {
1544                if (currState->isFetch)
1545                    currState->fault = std::make_shared<PrefetchAbort>(
1546                        currState->vaddr_tainted,
1547                        (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1548                        currState->longDesc.lookupLevel,
1549                        isStage2,
1550                        ArmFault::LpaeTran);
1551                else
1552                    currState->fault = std::make_shared<DataAbort>(
1553                        currState->vaddr_tainted,
1554                        TlbEntry::DomainType::NoAccess, currState->isWrite,
1555                        (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1556                        currState->longDesc.lookupLevel,
1557                        isStage2,
1558                        ArmFault::LpaeTran);
1559            } else {
1560                insertTableEntry(currState->longDesc, true);
1561            }
1562        }
1563        return;
1564      case LongDescriptor::Table:
1565        {
1566            // Set hierarchical permission flags
1567            currState->secureLookup = currState->secureLookup &&
1568                currState->longDesc.secureTable();
1569            currState->rwTable = currState->rwTable &&
1570                currState->longDesc.rwTable();
1571            currState->userTable = currState->userTable &&
1572                currState->longDesc.userTable();
1573            currState->xnTable = currState->xnTable ||
1574                currState->longDesc.xnTable();
1575            currState->pxnTable = currState->pxnTable ||
1576                currState->longDesc.pxnTable();
1577
1578            // Set up next level lookup
1579            Addr next_desc_addr = currState->longDesc.nextDescAddr(
1580                currState->vaddr);
1581
1582            DPRINTF(TLB, "L%d descriptor points to L%d descriptor at: %#x (%s)\n",
1583                    currState->longDesc.lookupLevel,
1584                    currState->longDesc.lookupLevel + 1,
1585                    next_desc_addr,
1586                    currState->secureLookup ? "s" : "ns");
1587
1588            // Check for address size fault
1589            if (currState->aarch64 && checkAddrSizeFaultAArch64(
1590                    next_desc_addr, currState->physAddrRange)) {
1591                DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1592                        currState->longDesc.lookupLevel);
1593                if (currState->isFetch)
1594                    currState->fault = std::make_shared<PrefetchAbort>(
1595                        currState->vaddr_tainted,
1596                        ArmFault::AddressSizeLL
1597                        + currState->longDesc.lookupLevel,
1598                        isStage2,
1599                        ArmFault::LpaeTran);
1600                else
1601                    currState->fault = std::make_shared<DataAbort>(
1602                        currState->vaddr_tainted,
1603                        TlbEntry::DomainType::NoAccess, currState->isWrite,
1604                        ArmFault::AddressSizeLL
1605                        + currState->longDesc.lookupLevel,
1606                        isStage2,
1607                        ArmFault::LpaeTran);
1608                return;
1609            }
1610
1611            // Trickbox address check
1612            currState->fault = testWalk(
1613                next_desc_addr, sizeof(uint64_t), TlbEntry::DomainType::Client,
1614                toLookupLevel(currState->longDesc.lookupLevel +1));
1615
1616            if (currState->fault) {
1617                if (!currState->timing) {
1618                    currState->tc = NULL;
1619                    currState->req = NULL;
1620                }
1621                return;
1622            }
1623
1624            Request::Flags flag = Request::PT_WALK;
1625            if (currState->secureLookup)
1626                flag.set(Request::SECURE);
1627
1628            currState->longDesc.lookupLevel =
1629                (LookupLevel) (currState->longDesc.lookupLevel + 1);
1630            Event *event = NULL;
1631            switch (currState->longDesc.lookupLevel) {
1632              case L1:
1633                assert(currState->aarch64);
1634                event = &doL1LongDescEvent;
1635                break;
1636              case L2:
1637                event = &doL2LongDescEvent;
1638                break;
1639              case L3:
1640                event = &doL3LongDescEvent;
1641                break;
1642              default:
1643                panic("Wrong lookup level in table walk\n");
1644                break;
1645            }
1646
1647            bool delayed;
1648            delayed = fetchDescriptor(next_desc_addr, (uint8_t*)&currState->longDesc.data,
1649                                      sizeof(uint64_t), flag, -1, event,
1650                                      &TableWalker::doLongDescriptor);
1651            if (delayed) {
1652                 currState->delayed = true;
1653            }
1654        }
1655        return;
1656      default:
1657        panic("A new type in a 2 bit field?\n");
1658    }
1659}
1660
1661void
1662TableWalker::doL2Descriptor()
1663{
1664    if (currState->fault != NoFault) {
1665        return;
1666    }
1667
1668    DPRINTF(TLB, "L2 descriptor for %#x is %#x\n",
1669            currState->vaddr_tainted, currState->l2Desc.data);
1670    TlbEntry te;
1671
1672    if (currState->l2Desc.invalid()) {
1673        DPRINTF(TLB, "L2 descriptor invalid, causing fault\n");
1674        if (!currState->timing) {
1675            currState->tc = NULL;
1676            currState->req = NULL;
1677        }
1678        if (currState->isFetch)
1679            currState->fault = std::make_shared<PrefetchAbort>(
1680                    currState->vaddr_tainted,
1681                    ArmFault::TranslationLL + L2,
1682                    isStage2,
1683                    ArmFault::VmsaTran);
1684        else
1685            currState->fault = std::make_shared<DataAbort>(
1686                currState->vaddr_tainted, currState->l1Desc.domain(),
1687                currState->isWrite, ArmFault::TranslationLL + L2,
1688                isStage2,
1689                ArmFault::VmsaTran);
1690        return;
1691    }
1692
1693    if (currState->sctlr.afe && bits(currState->l2Desc.ap(), 0) == 0) {
1694        /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is enabled
1695          * if set, do l2.Desc.setAp0() instead of generating AccessFlag0
1696          */
1697         DPRINTF(TLB, "Generating access fault at L2, afe: %d, ap: %d\n",
1698                 currState->sctlr.afe, currState->l2Desc.ap());
1699
1700        currState->fault = std::make_shared<DataAbort>(
1701            currState->vaddr_tainted,
1702            TlbEntry::DomainType::NoAccess, currState->isWrite,
1703            ArmFault::AccessFlagLL + L2, isStage2,
1704            ArmFault::VmsaTran);
1705    }
1706
1707    insertTableEntry(currState->l2Desc, false);
1708}
1709
1710void
1711TableWalker::doL1DescriptorWrapper()
1712{
1713    currState = stateQueues[L1].front();
1714    currState->delayed = false;
1715    // if there's a stage2 translation object we don't need it any more
1716    if (currState->stage2Tran) {
1717        delete currState->stage2Tran;
1718        currState->stage2Tran = NULL;
1719    }
1720
1721
1722    DPRINTF(TLBVerbose, "L1 Desc object host addr: %p\n",&currState->l1Desc.data);
1723    DPRINTF(TLBVerbose, "L1 Desc object      data: %08x\n",currState->l1Desc.data);
1724
1725    DPRINTF(TLBVerbose, "calling doL1Descriptor for vaddr:%#x\n", currState->vaddr_tainted);
1726    doL1Descriptor();
1727
1728    stateQueues[L1].pop_front();
1729    // Check if fault was generated
1730    if (currState->fault != NoFault) {
1731        currState->transState->finish(currState->fault, currState->req,
1732                                      currState->tc, currState->mode);
1733        statWalksShortTerminatedAtLevel[0]++;
1734
1735        pending = false;
1736        nextWalk(currState->tc);
1737
1738        currState->req = NULL;
1739        currState->tc = NULL;
1740        currState->delayed = false;
1741        delete currState;
1742    }
1743    else if (!currState->delayed) {
1744        // delay is not set so there is no L2 to do
1745        // Don't finish the translation if a stage 2 look up is underway
1746        if (!currState->doingStage2) {
1747            statWalkServiceTime.sample(curTick() - currState->startTime);
1748            DPRINTF(TLBVerbose, "calling translateTiming again\n");
1749            currState->fault = tlb->translateTiming(currState->req, currState->tc,
1750                currState->transState, currState->mode);
1751            statWalksShortTerminatedAtLevel[0]++;
1752        }
1753
1754        pending = false;
1755        nextWalk(currState->tc);
1756
1757        currState->req = NULL;
1758        currState->tc = NULL;
1759        currState->delayed = false;
1760        delete currState;
1761    } else {
1762        // need to do L2 descriptor
1763        stateQueues[L2].push_back(currState);
1764    }
1765    currState = NULL;
1766}
1767
1768void
1769TableWalker::doL2DescriptorWrapper()
1770{
1771    currState = stateQueues[L2].front();
1772    assert(currState->delayed);
1773    // if there's a stage2 translation object we don't need it any more
1774    if (currState->stage2Tran) {
1775        delete currState->stage2Tran;
1776        currState->stage2Tran = NULL;
1777    }
1778
1779    DPRINTF(TLBVerbose, "calling doL2Descriptor for vaddr:%#x\n",
1780            currState->vaddr_tainted);
1781    doL2Descriptor();
1782
1783    // Check if fault was generated
1784    if (currState->fault != NoFault) {
1785        currState->transState->finish(currState->fault, currState->req,
1786                                      currState->tc, currState->mode);
1787        statWalksShortTerminatedAtLevel[1]++;
1788    }
1789    else {
1790        // Don't finish the translation if a stage 2 look up is underway
1791        if (!currState->doingStage2) {
1792            statWalkServiceTime.sample(curTick() - currState->startTime);
1793            DPRINTF(TLBVerbose, "calling translateTiming again\n");
1794            currState->fault = tlb->translateTiming(currState->req,
1795                currState->tc, currState->transState, currState->mode);
1796            statWalksShortTerminatedAtLevel[1]++;
1797        }
1798    }
1799
1800
1801    stateQueues[L2].pop_front();
1802    pending = false;
1803    nextWalk(currState->tc);
1804
1805    currState->req = NULL;
1806    currState->tc = NULL;
1807    currState->delayed = false;
1808
1809    delete currState;
1810    currState = NULL;
1811}
1812
1813void
1814TableWalker::doL0LongDescriptorWrapper()
1815{
1816    doLongDescriptorWrapper(L0);
1817}
1818
1819void
1820TableWalker::doL1LongDescriptorWrapper()
1821{
1822    doLongDescriptorWrapper(L1);
1823}
1824
1825void
1826TableWalker::doL2LongDescriptorWrapper()
1827{
1828    doLongDescriptorWrapper(L2);
1829}
1830
1831void
1832TableWalker::doL3LongDescriptorWrapper()
1833{
1834    doLongDescriptorWrapper(L3);
1835}
1836
1837void
1838TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level)
1839{
1840    currState = stateQueues[curr_lookup_level].front();
1841    assert(curr_lookup_level == currState->longDesc.lookupLevel);
1842    currState->delayed = false;
1843
1844    // if there's a stage2 translation object we don't need it any more
1845    if (currState->stage2Tran) {
1846        delete currState->stage2Tran;
1847        currState->stage2Tran = NULL;
1848    }
1849
1850    DPRINTF(TLBVerbose, "calling doLongDescriptor for vaddr:%#x\n",
1851            currState->vaddr_tainted);
1852    doLongDescriptor();
1853
1854    stateQueues[curr_lookup_level].pop_front();
1855
1856    if (currState->fault != NoFault) {
1857        // A fault was generated
1858        currState->transState->finish(currState->fault, currState->req,
1859                                      currState->tc, currState->mode);
1860
1861        pending = false;
1862        nextWalk(currState->tc);
1863
1864        currState->req = NULL;
1865        currState->tc = NULL;
1866        currState->delayed = false;
1867        delete currState;
1868    } else if (!currState->delayed) {
1869        // No additional lookups required
1870        // Don't finish the translation if a stage 2 look up is underway
1871        if (!currState->doingStage2) {
1872            DPRINTF(TLBVerbose, "calling translateTiming again\n");
1873            statWalkServiceTime.sample(curTick() - currState->startTime);
1874            currState->fault = tlb->translateTiming(currState->req, currState->tc,
1875                                                    currState->transState,
1876                                                    currState->mode);
1877            statWalksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++;
1878        }
1879
1880        pending = false;
1881        nextWalk(currState->tc);
1882
1883        currState->req = NULL;
1884        currState->tc = NULL;
1885        currState->delayed = false;
1886        delete currState;
1887    } else {
1888        if (curr_lookup_level >= MAX_LOOKUP_LEVELS - 1)
1889            panic("Max. number of lookups already reached in table walk\n");
1890        // Need to perform additional lookups
1891        stateQueues[currState->longDesc.lookupLevel].push_back(currState);
1892    }
1893    currState = NULL;
1894}
1895
1896
1897void
1898TableWalker::nextWalk(ThreadContext *tc)
1899{
1900    if (pendingQueue.size())
1901        schedule(doProcessEvent, clockEdge(Cycles(1)));
1902    else
1903        completeDrain();
1904}
1905
1906bool
1907TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes,
1908    Request::Flags flags, int queueIndex, Event *event,
1909    void (TableWalker::*doDescriptor)())
1910{
1911    bool isTiming = currState->timing;
1912
1913    // do the requests for the page table descriptors have to go through the
1914    // second stage MMU
1915    if (currState->stage2Req) {
1916        Fault fault;
1917        flags = flags | TLB::MustBeOne;
1918
1919        if (isTiming) {
1920            Stage2MMU::Stage2Translation *tran = new
1921                Stage2MMU::Stage2Translation(*stage2Mmu, data, event,
1922                                             currState->vaddr);
1923            currState->stage2Tran = tran;
1924            stage2Mmu->readDataTimed(currState->tc, descAddr, tran, numBytes,
1925                                     flags);
1926            fault = tran->fault;
1927        } else {
1928            fault = stage2Mmu->readDataUntimed(currState->tc,
1929                currState->vaddr, descAddr, data, numBytes, flags,
1930                currState->functional);
1931        }
1932
1933        if (fault != NoFault) {
1934            currState->fault = fault;
1935        }
1936        if (isTiming) {
1937            if (queueIndex >= 0) {
1938                DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1939                        stateQueues[queueIndex].size());
1940                stateQueues[queueIndex].push_back(currState);
1941                currState = NULL;
1942            }
1943        } else {
1944            (this->*doDescriptor)();
1945        }
1946    } else {
1947        if (isTiming) {
1948            port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, event, data,
1949                           currState->tc->getCpuPtr()->clockPeriod(),flags);
1950            if (queueIndex >= 0) {
1951                DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1952                        stateQueues[queueIndex].size());
1953                stateQueues[queueIndex].push_back(currState);
1954                currState = NULL;
1955            }
1956        } else if (!currState->functional) {
1957            port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, NULL, data,
1958                           currState->tc->getCpuPtr()->clockPeriod(), flags);
1959            (this->*doDescriptor)();
1960        } else {
1961            RequestPtr req = new Request(descAddr, numBytes, flags, masterId);
1962            req->taskId(ContextSwitchTaskId::DMA);
1963            PacketPtr  pkt = new Packet(req, MemCmd::ReadReq);
1964            pkt->dataStatic(data);
1965            port->sendFunctional(pkt);
1966            (this->*doDescriptor)();
1967            delete req;
1968            delete pkt;
1969        }
1970    }
1971    return (isTiming);
1972}
1973
1974void
1975TableWalker::insertTableEntry(DescriptorBase &descriptor, bool longDescriptor)
1976{
1977    TlbEntry te;
1978
1979    // Create and fill a new page table entry
1980    te.valid          = true;
1981    te.longDescFormat = longDescriptor;
1982    te.isHyp          = currState->isHyp;
1983    te.asid           = currState->asid;
1984    te.vmid           = currState->vmid;
1985    te.N              = descriptor.offsetBits();
1986    te.vpn            = currState->vaddr >> te.N;
1987    te.size           = (1<<te.N) - 1;
1988    te.pfn            = descriptor.pfn();
1989    te.domain         = descriptor.domain();
1990    te.lookupLevel    = descriptor.lookupLevel;
1991    te.ns             = !descriptor.secure(haveSecurity, currState) || isStage2;
1992    te.nstid          = !currState->isSecure;
1993    te.xn             = descriptor.xn();
1994    if (currState->aarch64)
1995        te.el         = currState->el;
1996    else
1997        te.el         = 1;
1998
1999    statPageSizes[pageSizeNtoStatBin(te.N)]++;
2000    statRequestOrigin[COMPLETED][currState->isFetch]++;
2001
2002    // ASID has no meaning for stage 2 TLB entries, so mark all stage 2 entries
2003    // as global
2004    te.global         = descriptor.global(currState) || isStage2;
2005    if (longDescriptor) {
2006        LongDescriptor lDescriptor =
2007            dynamic_cast<LongDescriptor &>(descriptor);
2008
2009        te.xn |= currState->xnTable;
2010        te.pxn = currState->pxnTable || lDescriptor.pxn();
2011        if (isStage2) {
2012            // this is actually the HAP field, but its stored in the same bit
2013            // possitions as the AP field in a stage 1 translation.
2014            te.hap = lDescriptor.ap();
2015        } else {
2016           te.ap = ((!currState->rwTable || descriptor.ap() >> 1) << 1) |
2017               (currState->userTable && (descriptor.ap() & 0x1));
2018        }
2019        if (currState->aarch64)
2020            memAttrsAArch64(currState->tc, te, currState->longDesc.attrIndx(),
2021                            currState->longDesc.sh());
2022        else
2023            memAttrsLPAE(currState->tc, te, lDescriptor);
2024    } else {
2025        te.ap = descriptor.ap();
2026        memAttrs(currState->tc, te, currState->sctlr, descriptor.texcb(),
2027                 descriptor.shareable());
2028    }
2029
2030    // Debug output
2031    DPRINTF(TLB, descriptor.dbgHeader().c_str());
2032    DPRINTF(TLB, " - N:%d pfn:%#x size:%#x global:%d valid:%d\n",
2033            te.N, te.pfn, te.size, te.global, te.valid);
2034    DPRINTF(TLB, " - vpn:%#x xn:%d pxn:%d ap:%d domain:%d asid:%d "
2035            "vmid:%d hyp:%d nc:%d ns:%d\n", te.vpn, te.xn, te.pxn,
2036            te.ap, static_cast<uint8_t>(te.domain), te.asid, te.vmid, te.isHyp,
2037            te.nonCacheable, te.ns);
2038    DPRINTF(TLB, " - domain from L%d desc:%d data:%#x\n",
2039            descriptor.lookupLevel, static_cast<uint8_t>(descriptor.domain()),
2040            descriptor.getRawData());
2041
2042    // Insert the entry into the TLB
2043    tlb->insert(currState->vaddr, te);
2044    if (!currState->timing) {
2045        currState->tc  = NULL;
2046        currState->req = NULL;
2047    }
2048}
2049
2050ArmISA::TableWalker *
2051ArmTableWalkerParams::create()
2052{
2053    return new ArmISA::TableWalker(this);
2054}
2055
2056LookupLevel
2057TableWalker::toLookupLevel(uint8_t lookup_level_as_int)
2058{
2059    switch (lookup_level_as_int) {
2060      case L1:
2061        return L1;
2062      case L2:
2063        return L2;
2064      case L3:
2065        return L3;
2066      default:
2067        panic("Invalid lookup level conversion");
2068    }
2069}
2070
2071/* this method keeps track of the table walker queue's residency, so
2072 * needs to be called whenever requests start and complete. */
2073void
2074TableWalker::pendingChange()
2075{
2076    unsigned n = pendingQueue.size();
2077    if ((currState != NULL) && (currState != pendingQueue.front())) {
2078        ++n;
2079    }
2080
2081    if (n != pendingReqs) {
2082        Tick now = curTick();
2083        statPendingWalks.sample(pendingReqs, now - pendingChangeTick);
2084        pendingReqs = n;
2085        pendingChangeTick = now;
2086    }
2087}
2088
2089Fault
2090TableWalker::testWalk(Addr pa, Addr size, TlbEntry::DomainType domain,
2091                      LookupLevel lookup_level)
2092{
2093    return tlb->testWalk(pa, size, currState->vaddr, currState->isSecure,
2094                         currState->mode, domain, lookup_level);
2095}
2096
2097
2098uint8_t
2099TableWalker::pageSizeNtoStatBin(uint8_t N)
2100{
2101    /* for statPageSizes */
2102    switch(N) {
2103        case 12: return 0; // 4K
2104        case 14: return 1; // 16K (using 16K granule in v8-64)
2105        case 16: return 2; // 64K
2106        case 20: return 3; // 1M
2107        case 21: return 4; // 2M-LPAE
2108        case 24: return 5; // 16M
2109        case 25: return 6; // 32M (using 16K granule in v8-64)
2110        case 29: return 7; // 512M (using 64K granule in v8-64)
2111        case 30: return 8; // 1G-LPAE
2112        default:
2113            panic("unknown page size");
2114            return 255;
2115    }
2116}
2117
2118void
2119TableWalker::regStats()
2120{
2121    statWalks
2122        .name(name() + ".walks")
2123        .desc("Table walker walks requested")
2124        ;
2125
2126    statWalksShortDescriptor
2127        .name(name() + ".walksShort")
2128        .desc("Table walker walks initiated with short descriptors")
2129        .flags(Stats::nozero)
2130        ;
2131
2132    statWalksLongDescriptor
2133        .name(name() + ".walksLong")
2134        .desc("Table walker walks initiated with long descriptors")
2135        .flags(Stats::nozero)
2136        ;
2137
2138    statWalksShortTerminatedAtLevel
2139        .init(2)
2140        .name(name() + ".walksShortTerminationLevel")
2141        .desc("Level at which table walker walks "
2142              "with short descriptors terminate")
2143        .flags(Stats::nozero)
2144        ;
2145    statWalksShortTerminatedAtLevel.subname(0, "Level1");
2146    statWalksShortTerminatedAtLevel.subname(1, "Level2");
2147
2148    statWalksLongTerminatedAtLevel
2149        .init(4)
2150        .name(name() + ".walksLongTerminationLevel")
2151        .desc("Level at which table walker walks "
2152              "with long descriptors terminate")
2153        .flags(Stats::nozero)
2154        ;
2155    statWalksLongTerminatedAtLevel.subname(0, "Level0");
2156    statWalksLongTerminatedAtLevel.subname(1, "Level1");
2157    statWalksLongTerminatedAtLevel.subname(2, "Level2");
2158    statWalksLongTerminatedAtLevel.subname(3, "Level3");
2159
2160    statSquashedBefore
2161        .name(name() + ".walksSquashedBefore")
2162        .desc("Table walks squashed before starting")
2163        .flags(Stats::nozero)
2164        ;
2165
2166    statSquashedAfter
2167        .name(name() + ".walksSquashedAfter")
2168        .desc("Table walks squashed after completion")
2169        .flags(Stats::nozero)
2170        ;
2171
2172    statWalkWaitTime
2173        .init(16)
2174        .name(name() + ".walkWaitTime")
2175        .desc("Table walker wait (enqueue to first request) latency")
2176        .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2177        ;
2178
2179    statWalkServiceTime
2180        .init(16)
2181        .name(name() + ".walkCompletionTime")
2182        .desc("Table walker service (enqueue to completion) latency")
2183        .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2184        ;
2185
2186    statPendingWalks
2187        .init(16)
2188        .name(name() + ".walksPending")
2189        .desc("Table walker pending requests distribution")
2190        .flags(Stats::pdf | Stats::dist | Stats::nozero | Stats::nonan)
2191        ;
2192
2193    statPageSizes // see DDI 0487A D4-1661
2194        .init(9)
2195        .name(name() + ".walkPageSizes")
2196        .desc("Table walker page sizes translated")
2197        .flags(Stats::total | Stats::pdf | Stats::dist | Stats::nozero)
2198        ;
2199    statPageSizes.subname(0, "4K");
2200    statPageSizes.subname(1, "16K");
2201    statPageSizes.subname(2, "64K");
2202    statPageSizes.subname(3, "1M");
2203    statPageSizes.subname(4, "2M");
2204    statPageSizes.subname(5, "16M");
2205    statPageSizes.subname(6, "32M");
2206    statPageSizes.subname(7, "512M");
2207    statPageSizes.subname(8, "1G");
2208
2209    statRequestOrigin
2210        .init(2,2) // Instruction/Data, requests/completed
2211        .name(name() + ".walkRequestOrigin")
2212        .desc("Table walker requests started/completed, data/inst")
2213        .flags(Stats::total)
2214        ;
2215    statRequestOrigin.subname(0,"Requested");
2216    statRequestOrigin.subname(1,"Completed");
2217    statRequestOrigin.ysubname(0,"Data");
2218    statRequestOrigin.ysubname(1,"Inst");
2219}
2220