1/*
| 1/*
|
2 * Copyright (c) 2010, 2012-2014 ARM Limited
| 2 * Copyright (c) 2010, 2012-2015 ARM Limited
|
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Ali Saidi
38 * Giacomo Gabrielli
39 */
40
41#include <memory>
42
43#include "arch/arm/faults.hh"
44#include "arch/arm/stage2_mmu.hh"
45#include "arch/arm/system.hh"
46#include "arch/arm/table_walker.hh"
47#include "arch/arm/tlb.hh"
48#include "cpu/base.hh"
49#include "cpu/thread_context.hh"
50#include "debug/Checkpoint.hh"
51#include "debug/Drain.hh"
52#include "debug/TLB.hh"
53#include "debug/TLBVerbose.hh"
54#include "sim/system.hh"
55
56using namespace ArmISA;
57
58TableWalker::TableWalker(const Params *p)
| 3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Ali Saidi
38 * Giacomo Gabrielli
39 */
40
41#include <memory>
42
43#include "arch/arm/faults.hh"
44#include "arch/arm/stage2_mmu.hh"
45#include "arch/arm/system.hh"
46#include "arch/arm/table_walker.hh"
47#include "arch/arm/tlb.hh"
48#include "cpu/base.hh"
49#include "cpu/thread_context.hh"
50#include "debug/Checkpoint.hh"
51#include "debug/Drain.hh"
52#include "debug/TLB.hh"
53#include "debug/TLBVerbose.hh"
54#include "sim/system.hh"
55
56using namespace ArmISA;
57
58TableWalker::TableWalker(const Params *p)
|
59 : MemObject(p), port(this, p->sys), drainManager(NULL),
60 stage2Mmu(NULL), isStage2(p->is_stage2), tlb(NULL),
61 currState(NULL), pending(false), masterId(p->sys->getMasterId(name())),
| 59 : MemObject(p), drainManager(NULL),
60 stage2Mmu(NULL), port(NULL), masterId(Request::invldMasterId),
61 isStage2(p->is_stage2), tlb(NULL),
62 currState(NULL), pending(false),
|
62 numSquashable(p->num_squash_per_cycle),
63 pendingReqs(0),
64 pendingChangeTick(curTick()),
65 doL1DescEvent(this), doL2DescEvent(this),
66 doL0LongDescEvent(this), doL1LongDescEvent(this), doL2LongDescEvent(this),
67 doL3LongDescEvent(this),
68 doProcessEvent(this)
69{
70 sctlr = 0;
71
72 // Cache system-level properties
73 if (FullSystem) {
| 63 numSquashable(p->num_squash_per_cycle),
64 pendingReqs(0),
65 pendingChangeTick(curTick()),
66 doL1DescEvent(this), doL2DescEvent(this),
67 doL0LongDescEvent(this), doL1LongDescEvent(this), doL2LongDescEvent(this),
68 doL3LongDescEvent(this),
69 doProcessEvent(this)
70{
71 sctlr = 0;
72
73 // Cache system-level properties
74 if (FullSystem) {
|
74 armSys = dynamic_cast(p->sys);
| 75 ArmSystem *armSys = dynamic_cast<ArmSystem *>(p->sys);
|
75 assert(armSys);
76 haveSecurity = armSys->haveSecurity();
77 _haveLPAE = armSys->haveLPAE();
78 _haveVirtualization = armSys->haveVirtualization();
79 physAddrRange = armSys->physAddrRange();
80 _haveLargeAsid64 = armSys->haveLargeAsid64();
81 } else {
| 76 assert(armSys);
77 haveSecurity = armSys->haveSecurity();
78 _haveLPAE = armSys->haveLPAE();
79 _haveVirtualization = armSys->haveVirtualization();
80 physAddrRange = armSys->physAddrRange();
81 _haveLargeAsid64 = armSys->haveLargeAsid64();
82 } else {
|
82 armSys = NULL;
| |
83 haveSecurity = _haveLPAE = _haveVirtualization = false;
84 _haveLargeAsid64 = false;
85 physAddrRange = 32;
86 }
87
88}
89
90TableWalker::~TableWalker()
91{
92 ;
93}
94
| 83 haveSecurity = _haveLPAE = _haveVirtualization = false;
84 _haveLargeAsid64 = false;
85 physAddrRange = 32;
86 }
87
88}
89
90TableWalker::~TableWalker()
91{
92 ;
93}
94
|
| 95void
96TableWalker::setMMU(Stage2MMU *m, MasterID master_id)
97{
98 stage2Mmu = m;
99 port = &m->getPort();
100 masterId = master_id;
101}
102
103void
104TableWalker::init()
105{
106 fatal_if(!stage2Mmu, "Table walker must have a valid stage-2 MMU\n");
107 fatal_if(!port, "Table walker must have a valid port\n");
108 fatal_if(!tlb, "Table walker must have a valid TLB\n");
109}
110
111BaseMasterPort&
112TableWalker::getMasterPort(const std::string &if_name, PortID idx)
113{
114 if (if_name == "port") {
115 if (!isStage2) {
116 return *port;
117 } else {
118 fatal("Cannot access table walker port through stage-two walker\n");
119 }
120 }
121 return MemObject::getMasterPort(if_name, idx);
122}
123
|
95TableWalker::WalkerState::WalkerState() :
96 tc(nullptr), aarch64(false), el(EL0), physAddrRange(0), req(nullptr),
97 asid(0), vmid(0), isHyp(false), transState(nullptr),
98 vaddr(0), vaddr_tainted(0), isWrite(false), isFetch(false), isSecure(false),
99 secureLookup(false), rwTable(false), userTable(false), xnTable(false),
100 pxnTable(false), stage2Req(false), doingStage2(false),
101 stage2Tran(nullptr), timing(false), functional(false),
102 mode(BaseTLB::Read), tranType(TLB::NormalTran), l2Desc(l1Desc),
103 delayed(false), tableWalker(nullptr)
104{
105}
106
107void
108TableWalker::completeDrain()
109{
110 if (drainManager && stateQueues[L1].empty() && stateQueues[L2].empty() &&
111 pendingQueue.empty()) {
112 setDrainState(Drainable::Drained);
113 DPRINTF(Drain, "TableWalker done draining, processing drain event\n");
114 drainManager->signalDrainDone();
115 drainManager = NULL;
116 }
117}
118
119unsigned int
120TableWalker::drain(DrainManager *dm)
121{
| 124TableWalker::WalkerState::WalkerState() :
125 tc(nullptr), aarch64(false), el(EL0), physAddrRange(0), req(nullptr),
126 asid(0), vmid(0), isHyp(false), transState(nullptr),
127 vaddr(0), vaddr_tainted(0), isWrite(false), isFetch(false), isSecure(false),
128 secureLookup(false), rwTable(false), userTable(false), xnTable(false),
129 pxnTable(false), stage2Req(false), doingStage2(false),
130 stage2Tran(nullptr), timing(false), functional(false),
131 mode(BaseTLB::Read), tranType(TLB::NormalTran), l2Desc(l1Desc),
132 delayed(false), tableWalker(nullptr)
133{
134}
135
136void
137TableWalker::completeDrain()
138{
139 if (drainManager && stateQueues[L1].empty() && stateQueues[L2].empty() &&
140 pendingQueue.empty()) {
141 setDrainState(Drainable::Drained);
142 DPRINTF(Drain, "TableWalker done draining, processing drain event\n");
143 drainManager->signalDrainDone();
144 drainManager = NULL;
145 }
146}
147
148unsigned int
149TableWalker::drain(DrainManager *dm)
150{
|
122 unsigned int count = port.drain(dm);
123
| |
124 bool state_queues_not_empty = false;
125
126 for (int i = 0; i < MAX_LOOKUP_LEVELS; ++i) {
127 if (!stateQueues[i].empty()) {
128 state_queues_not_empty = true;
129 break;
130 }
131 }
132
133 if (state_queues_not_empty || pendingQueue.size()) {
134 drainManager = dm;
135 setDrainState(Drainable::Draining);
136 DPRINTF(Drain, "TableWalker not drained\n");
137
138 // return port drain count plus the table walker itself needs to drain
| 151 bool state_queues_not_empty = false;
152
153 for (int i = 0; i < MAX_LOOKUP_LEVELS; ++i) {
154 if (!stateQueues[i].empty()) {
155 state_queues_not_empty = true;
156 break;
157 }
158 }
159
160 if (state_queues_not_empty || pendingQueue.size()) {
161 drainManager = dm;
162 setDrainState(Drainable::Draining);
163 DPRINTF(Drain, "TableWalker not drained\n");
164
165 // return port drain count plus the table walker itself needs to drain
|
139 return count + 1;
| 166 return 1;
|
140 } else {
141 setDrainState(Drainable::Drained);
142 DPRINTF(Drain, "TableWalker free, no need to drain\n");
143
144 // table walker is drained, but its ports may still need to be drained
| 167 } else {
168 setDrainState(Drainable::Drained);
169 DPRINTF(Drain, "TableWalker free, no need to drain\n");
170
171 // table walker is drained, but its ports may still need to be drained
|
145 return count;
| 172 return 0;
|
146 }
147}
148
149void
150TableWalker::drainResume()
151{
152 Drainable::drainResume();
153 if (params()->sys->isTimingMode() && currState) {
154 delete currState;
155 currState = NULL;
156 pendingChange();
157 }
158}
159
| 173 }
174}
175
176void
177TableWalker::drainResume()
178{
179 Drainable::drainResume();
180 if (params()->sys->isTimingMode() && currState) {
181 delete currState;
182 currState = NULL;
183 pendingChange();
184 }
185}
186
|
160BaseMasterPort&
161TableWalker::getMasterPort(const std::string &if_name, PortID idx)
162{
163 if (if_name == "port") {
164 return port;
165 }
166 return MemObject::getMasterPort(if_name, idx);
167}
168
| |
169Fault
170TableWalker::walk(RequestPtr _req, ThreadContext *_tc, uint16_t _asid,
171 uint8_t _vmid, bool _isHyp, TLB::Mode _mode,
172 TLB::Translation *_trans, bool _timing, bool _functional,
173 bool secure, TLB::ArmTranslationType tranType)
174{
175 assert(!(_functional && _timing));
176 ++statWalks;
177
178 WalkerState *savedCurrState = NULL;
179
180 if (!currState && !_functional) {
181 // For atomic mode, a new WalkerState instance should be only created
182 // once per TLB. For timing mode, a new instance is generated for every
183 // TLB miss.
184 DPRINTF(TLBVerbose, "creating new instance of WalkerState\n");
185
186 currState = new WalkerState();
187 currState->tableWalker = this;
188 } else if (_functional) {
189 // If we are mixing functional mode with timing (or even
190 // atomic), we need to to be careful and clean up after
191 // ourselves to not risk getting into an inconsistent state.
192 DPRINTF(TLBVerbose, "creating functional instance of WalkerState\n");
193 savedCurrState = currState;
194 currState = new WalkerState();
195 currState->tableWalker = this;
196 } else if (_timing) {
197 // This is a translation that was completed and then faulted again
198 // because some underlying parameters that affect the translation
199 // changed out from under us (e.g. asid). It will either be a
200 // misprediction, in which case nothing will happen or we'll use
201 // this fault to re-execute the faulting instruction which should clean
202 // up everything.
203 if (currState->vaddr_tainted == _req->getVaddr()) {
204 ++statSquashedBefore;
205 return std::make_shared<ReExec>();
206 }
207 }
208 pendingChange();
209
210 currState->startTime = curTick();
211 currState->tc = _tc;
212 currState->aarch64 = opModeIs64(currOpMode(_tc));
213 currState->el = currEL(_tc);
214 currState->transState = _trans;
215 currState->req = _req;
216 currState->fault = NoFault;
217 currState->asid = _asid;
218 currState->vmid = _vmid;
219 currState->isHyp = _isHyp;
220 currState->timing = _timing;
221 currState->functional = _functional;
222 currState->mode = _mode;
223 currState->tranType = tranType;
224 currState->isSecure = secure;
225 currState->physAddrRange = physAddrRange;
226
227 /** @todo These should be cached or grabbed from cached copies in
228 the TLB, all these miscreg reads are expensive */
229 currState->vaddr_tainted = currState->req->getVaddr();
230 if (currState->aarch64)
231 currState->vaddr = purifyTaggedAddr(currState->vaddr_tainted,
232 currState->tc, currState->el);
233 else
234 currState->vaddr = currState->vaddr_tainted;
235
236 if (currState->aarch64) {
237 switch (currState->el) {
238 case EL0:
239 case EL1:
240 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1);
241 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL1);
242 break;
243 // @todo: uncomment this to enable Virtualization
244 // case EL2:
245 // assert(haveVirtualization);
246 // currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL2);
247 // currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL2);
248 // break;
249 case EL3:
250 assert(haveSecurity);
251 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL3);
252 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL3);
253 break;
254 default:
255 panic("Invalid exception level");
256 break;
257 }
258 } else {
259 currState->sctlr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
260 MISCREG_SCTLR, currState->tc, !currState->isSecure));
261 currState->ttbcr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
262 MISCREG_TTBCR, currState->tc, !currState->isSecure));
263 currState->htcr = currState->tc->readMiscReg(MISCREG_HTCR);
264 currState->hcr = currState->tc->readMiscReg(MISCREG_HCR);
265 currState->vtcr = currState->tc->readMiscReg(MISCREG_VTCR);
266 }
267 sctlr = currState->sctlr;
268
269 currState->isFetch = (currState->mode == TLB::Execute);
270 currState->isWrite = (currState->mode == TLB::Write);
271
272 statRequestOrigin[REQUESTED][currState->isFetch]++;
273
274 // We only do a second stage of translation if we're not secure, or in
275 // hyp mode, the second stage MMU is enabled, and this table walker
276 // instance is the first stage.
277 currState->doingStage2 = false;
278 // @todo: for now disable this in AArch64 (HCR is not set)
279 currState->stage2Req = !currState->aarch64 && currState->hcr.vm &&
280 !isStage2 && !currState->isSecure && !currState->isHyp;
281
282 bool long_desc_format = currState->aarch64 ||
283 (_haveLPAE && currState->ttbcr.eae) ||
284 _isHyp || isStage2;
285
286 if (long_desc_format) {
287 // Helper variables used for hierarchical permissions
288 currState->secureLookup = currState->isSecure;
289 currState->rwTable = true;
290 currState->userTable = true;
291 currState->xnTable = false;
292 currState->pxnTable = false;
293
294 ++statWalksLongDescriptor;
295 } else {
296 ++statWalksShortDescriptor;
297 }
298
299 if (!currState->timing) {
300 Fault fault = NoFault;
301 if (currState->aarch64)
302 fault = processWalkAArch64();
303 else if (long_desc_format)
304 fault = processWalkLPAE();
305 else
306 fault = processWalk();
307
308 // If this was a functional non-timing access restore state to
309 // how we found it.
310 if (currState->functional) {
311 delete currState;
312 currState = savedCurrState;
313 }
314 return fault;
315 }
316
317 if (pending || pendingQueue.size()) {
318 pendingQueue.push_back(currState);
319 currState = NULL;
320 pendingChange();
321 } else {
322 pending = true;
323 pendingChange();
324 if (currState->aarch64)
325 return processWalkAArch64();
326 else if (long_desc_format)
327 return processWalkLPAE();
328 else
329 return processWalk();
330 }
331
332 return NoFault;
333}
334
335void
336TableWalker::processWalkWrapper()
337{
338 assert(!currState);
339 assert(pendingQueue.size());
340 pendingChange();
341 currState = pendingQueue.front();
342
343 ExceptionLevel target_el = EL0;
344 if (currState->aarch64)
345 target_el = currEL(currState->tc);
346 else
347 target_el = EL1;
348
349 // Check if a previous walk filled this request already
350 // @TODO Should this always be the TLB or should we look in the stage2 TLB?
351 TlbEntry* te = tlb->lookup(currState->vaddr, currState->asid,
352 currState->vmid, currState->isHyp, currState->isSecure, true, false,
353 target_el);
354
355 // Check if we still need to have a walk for this request. If the requesting
356 // instruction has been squashed, or a previous walk has filled the TLB with
357 // a match, we just want to get rid of the walk. The latter could happen
358 // when there are multiple outstanding misses to a single page and a
359 // previous request has been successfully translated.
360 if (!currState->transState->squashed() && !te) {
361 // We've got a valid request, lets process it
362 pending = true;
363 pendingQueue.pop_front();
364 // Keep currState in case one of the processWalk... calls NULLs it
365 WalkerState *curr_state_copy = currState;
366 Fault f;
367 if (currState->aarch64)
368 f = processWalkAArch64();
369 else if ((_haveLPAE && currState->ttbcr.eae) || currState->isHyp || isStage2)
370 f = processWalkLPAE();
371 else
372 f = processWalk();
373
374 if (f != NoFault) {
375 curr_state_copy->transState->finish(f, curr_state_copy->req,
376 curr_state_copy->tc, curr_state_copy->mode);
377
378 delete curr_state_copy;
379 }
380 return;
381 }
382
383
384 // If the instruction that we were translating for has been
385 // squashed we shouldn't bother.
386 unsigned num_squashed = 0;
387 ThreadContext *tc = currState->tc;
388 while ((num_squashed < numSquashable) && currState &&
389 (currState->transState->squashed() || te)) {
390 pendingQueue.pop_front();
391 num_squashed++;
392 statSquashedBefore++;
393
394 DPRINTF(TLB, "Squashing table walk for address %#x\n",
395 currState->vaddr_tainted);
396
397 if (currState->transState->squashed()) {
398 // finish the translation which will delete the translation object
399 currState->transState->finish(
400 std::make_shared<UnimpFault>("Squashed Inst"),
401 currState->req, currState->tc, currState->mode);
402 } else {
403 // translate the request now that we know it will work
404 statWalkServiceTime.sample(curTick() - currState->startTime);
405 tlb->translateTiming(currState->req, currState->tc,
406 currState->transState, currState->mode);
407
408 }
409
410 // delete the current request
411 delete currState;
412
413 // peak at the next one
414 if (pendingQueue.size()) {
415 currState = pendingQueue.front();
416 te = tlb->lookup(currState->vaddr, currState->asid,
417 currState->vmid, currState->isHyp, currState->isSecure, true,
418 false, target_el);
419 } else {
420 // Terminate the loop, nothing more to do
421 currState = NULL;
422 }
423 }
424 pendingChange();
425
426 // if we still have pending translations, schedule more work
427 nextWalk(tc);
428 currState = NULL;
429}
430
431Fault
432TableWalker::processWalk()
433{
434 Addr ttbr = 0;
435
436 // If translation isn't enabled, we shouldn't be here
437 assert(currState->sctlr.m || isStage2);
438
439 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x, bits:%#x\n",
440 currState->vaddr_tainted, currState->ttbcr, mbits(currState->vaddr, 31,
441 32 - currState->ttbcr.n));
442
443 statWalkWaitTime.sample(curTick() - currState->startTime);
444
445 if (currState->ttbcr.n == 0 || !mbits(currState->vaddr, 31,
446 32 - currState->ttbcr.n)) {
447 DPRINTF(TLB, " - Selecting TTBR0\n");
448 // Check if table walk is allowed when Security Extensions are enabled
449 if (haveSecurity && currState->ttbcr.pd0) {
450 if (currState->isFetch)
451 return std::make_shared<PrefetchAbort>(
452 currState->vaddr_tainted,
453 ArmFault::TranslationLL + L1,
454 isStage2,
455 ArmFault::VmsaTran);
456 else
457 return std::make_shared<DataAbort>(
458 currState->vaddr_tainted,
459 TlbEntry::DomainType::NoAccess, currState->isWrite,
460 ArmFault::TranslationLL + L1, isStage2,
461 ArmFault::VmsaTran);
462 }
463 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
464 MISCREG_TTBR0, currState->tc, !currState->isSecure));
465 } else {
466 DPRINTF(TLB, " - Selecting TTBR1\n");
467 // Check if table walk is allowed when Security Extensions are enabled
468 if (haveSecurity && currState->ttbcr.pd1) {
469 if (currState->isFetch)
470 return std::make_shared<PrefetchAbort>(
471 currState->vaddr_tainted,
472 ArmFault::TranslationLL + L1,
473 isStage2,
474 ArmFault::VmsaTran);
475 else
476 return std::make_shared<DataAbort>(
477 currState->vaddr_tainted,
478 TlbEntry::DomainType::NoAccess, currState->isWrite,
479 ArmFault::TranslationLL + L1, isStage2,
480 ArmFault::VmsaTran);
481 }
482 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
483 MISCREG_TTBR1, currState->tc, !currState->isSecure));
484 currState->ttbcr.n = 0;
485 }
486
487 Addr l1desc_addr = mbits(ttbr, 31, 14 - currState->ttbcr.n) |
488 (bits(currState->vaddr, 31 - currState->ttbcr.n, 20) << 2);
489 DPRINTF(TLB, " - Descriptor at address %#x (%s)\n", l1desc_addr,
490 currState->isSecure ? "s" : "ns");
491
492 // Trickbox address check
493 Fault f;
494 f = tlb->walkTrickBoxCheck(l1desc_addr, currState->isSecure,
495 currState->vaddr, sizeof(uint32_t), currState->isFetch,
496 currState->isWrite, TlbEntry::DomainType::NoAccess, L1);
497 if (f) {
498 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
499 if (currState->timing) {
500 pending = false;
501 nextWalk(currState->tc);
502 currState = NULL;
503 } else {
504 currState->tc = NULL;
505 currState->req = NULL;
506 }
507 return f;
508 }
509
510 Request::Flags flag = 0;
511 if (currState->sctlr.c == 0) {
512 flag = Request::UNCACHEABLE;
513 }
514
515 bool delayed;
516 delayed = fetchDescriptor(l1desc_addr, (uint8_t*)&currState->l1Desc.data,
517 sizeof(uint32_t), flag, L1, &doL1DescEvent,
518 &TableWalker::doL1Descriptor);
519 if (!delayed) {
520 f = currState->fault;
521 }
522
523 return f;
524}
525
526Fault
527TableWalker::processWalkLPAE()
528{
529 Addr ttbr, ttbr0_max, ttbr1_min, desc_addr;
530 int tsz, n;
531 LookupLevel start_lookup_level = L1;
532
533 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x\n",
534 currState->vaddr_tainted, currState->ttbcr);
535
536 statWalkWaitTime.sample(curTick() - currState->startTime);
537
538 Request::Flags flag = 0;
539 if (currState->isSecure)
540 flag.set(Request::SECURE);
541
542 // work out which base address register to use, if in hyp mode we always
543 // use HTTBR
544 if (isStage2) {
545 DPRINTF(TLB, " - Selecting VTTBR (long-desc.)\n");
546 ttbr = currState->tc->readMiscReg(MISCREG_VTTBR);
547 tsz = sext<4>(currState->vtcr.t0sz);
548 start_lookup_level = currState->vtcr.sl0 ? L1 : L2;
549 } else if (currState->isHyp) {
550 DPRINTF(TLB, " - Selecting HTTBR (long-desc.)\n");
551 ttbr = currState->tc->readMiscReg(MISCREG_HTTBR);
552 tsz = currState->htcr.t0sz;
553 } else {
554 assert(_haveLPAE && currState->ttbcr.eae);
555
556 // Determine boundaries of TTBR0/1 regions
557 if (currState->ttbcr.t0sz)
558 ttbr0_max = (1ULL << (32 - currState->ttbcr.t0sz)) - 1;
559 else if (currState->ttbcr.t1sz)
560 ttbr0_max = (1ULL << 32) -
561 (1ULL << (32 - currState->ttbcr.t1sz)) - 1;
562 else
563 ttbr0_max = (1ULL << 32) - 1;
564 if (currState->ttbcr.t1sz)
565 ttbr1_min = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz));
566 else
567 ttbr1_min = (1ULL << (32 - currState->ttbcr.t0sz));
568
569 // The following code snippet selects the appropriate translation table base
570 // address (TTBR0 or TTBR1) and the appropriate starting lookup level
571 // depending on the address range supported by the translation table (ARM
572 // ARM issue C B3.6.4)
573 if (currState->vaddr <= ttbr0_max) {
574 DPRINTF(TLB, " - Selecting TTBR0 (long-desc.)\n");
575 // Check if table walk is allowed
576 if (currState->ttbcr.epd0) {
577 if (currState->isFetch)
578 return std::make_shared<PrefetchAbort>(
579 currState->vaddr_tainted,
580 ArmFault::TranslationLL + L1,
581 isStage2,
582 ArmFault::LpaeTran);
583 else
584 return std::make_shared<DataAbort>(
585 currState->vaddr_tainted,
586 TlbEntry::DomainType::NoAccess,
587 currState->isWrite,
588 ArmFault::TranslationLL + L1,
589 isStage2,
590 ArmFault::LpaeTran);
591 }
592 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
593 MISCREG_TTBR0, currState->tc, !currState->isSecure));
594 tsz = currState->ttbcr.t0sz;
595 if (ttbr0_max < (1ULL << 30)) // Upper limit < 1 GB
596 start_lookup_level = L2;
597 } else if (currState->vaddr >= ttbr1_min) {
598 DPRINTF(TLB, " - Selecting TTBR1 (long-desc.)\n");
599 // Check if table walk is allowed
600 if (currState->ttbcr.epd1) {
601 if (currState->isFetch)
602 return std::make_shared<PrefetchAbort>(
603 currState->vaddr_tainted,
604 ArmFault::TranslationLL + L1,
605 isStage2,
606 ArmFault::LpaeTran);
607 else
608 return std::make_shared<DataAbort>(
609 currState->vaddr_tainted,
610 TlbEntry::DomainType::NoAccess,
611 currState->isWrite,
612 ArmFault::TranslationLL + L1,
613 isStage2,
614 ArmFault::LpaeTran);
615 }
616 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
617 MISCREG_TTBR1, currState->tc, !currState->isSecure));
618 tsz = currState->ttbcr.t1sz;
619 if (ttbr1_min >= (1ULL << 31) + (1ULL << 30)) // Lower limit >= 3 GB
620 start_lookup_level = L2;
621 } else {
622 // Out of boundaries -> translation fault
623 if (currState->isFetch)
624 return std::make_shared<PrefetchAbort>(
625 currState->vaddr_tainted,
626 ArmFault::TranslationLL + L1,
627 isStage2,
628 ArmFault::LpaeTran);
629 else
630 return std::make_shared<DataAbort>(
631 currState->vaddr_tainted,
632 TlbEntry::DomainType::NoAccess,
633 currState->isWrite, ArmFault::TranslationLL + L1,
634 isStage2, ArmFault::LpaeTran);
635 }
636
637 }
638
639 // Perform lookup (ARM ARM issue C B3.6.6)
640 if (start_lookup_level == L1) {
641 n = 5 - tsz;
642 desc_addr = mbits(ttbr, 39, n) |
643 (bits(currState->vaddr, n + 26, 30) << 3);
644 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
645 desc_addr, currState->isSecure ? "s" : "ns");
646 } else {
647 // Skip first-level lookup
648 n = (tsz >= 2 ? 14 - tsz : 12);
649 desc_addr = mbits(ttbr, 39, n) |
650 (bits(currState->vaddr, n + 17, 21) << 3);
651 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
652 desc_addr, currState->isSecure ? "s" : "ns");
653 }
654
655 // Trickbox address check
656 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
657 currState->vaddr, sizeof(uint64_t), currState->isFetch,
658 currState->isWrite, TlbEntry::DomainType::NoAccess,
659 start_lookup_level);
660 if (f) {
661 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
662 if (currState->timing) {
663 pending = false;
664 nextWalk(currState->tc);
665 currState = NULL;
666 } else {
667 currState->tc = NULL;
668 currState->req = NULL;
669 }
670 return f;
671 }
672
673 if (currState->sctlr.c == 0) {
674 flag = Request::UNCACHEABLE;
675 }
676
677 if (currState->isSecure)
678 flag.set(Request::SECURE);
679
680 currState->longDesc.lookupLevel = start_lookup_level;
681 currState->longDesc.aarch64 = false;
682 currState->longDesc.grainSize = Grain4KB;
683
684 Event *event = start_lookup_level == L1 ? (Event *) &doL1LongDescEvent
685 : (Event *) &doL2LongDescEvent;
686
687 bool delayed = fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data,
688 sizeof(uint64_t), flag, start_lookup_level,
689 event, &TableWalker::doLongDescriptor);
690 if (!delayed) {
691 f = currState->fault;
692 }
693
694 return f;
695}
696
697unsigned
698TableWalker::adjustTableSizeAArch64(unsigned tsz)
699{
700 if (tsz < 25)
701 return 25;
702 if (tsz > 48)
703 return 48;
704 return tsz;
705}
706
707bool
708TableWalker::checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange)
709{
710 return (currPhysAddrRange != MaxPhysAddrRange &&
711 bits(addr, MaxPhysAddrRange - 1, currPhysAddrRange));
712}
713
714Fault
715TableWalker::processWalkAArch64()
716{
717 assert(currState->aarch64);
718
719 DPRINTF(TLB, "Beginning table walk for address %#llx, TCR: %#llx\n",
720 currState->vaddr_tainted, currState->tcr);
721
722 static const GrainSize GrainMapDefault[] =
723 { Grain4KB, Grain64KB, Grain16KB, ReservedGrain };
724 static const GrainSize GrainMap_EL1_tg1[] =
725 { ReservedGrain, Grain16KB, Grain4KB, Grain64KB };
726
727 statWalkWaitTime.sample(curTick() - currState->startTime);
728
729 // Determine TTBR, table size, granule size and phys. address range
730 Addr ttbr = 0;
731 int tsz = 0, ps = 0;
732 GrainSize tg = Grain4KB; // grain size computed from tg* field
733 bool fault = false;
734 switch (currState->el) {
735 case EL0:
736 case EL1:
737 switch (bits(currState->vaddr, 63,48)) {
738 case 0:
739 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
740 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL1);
741 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
742 tg = GrainMapDefault[currState->tcr.tg0];
743 if (bits(currState->vaddr, 63, tsz) != 0x0 ||
744 currState->tcr.epd0)
745 fault = true;
746 break;
747 case 0xffff:
748 DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n");
749 ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL1);
750 tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz);
751 tg = GrainMap_EL1_tg1[currState->tcr.tg1];
752 if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) ||
753 currState->tcr.epd1)
754 fault = true;
755 break;
756 default:
757 // top two bytes must be all 0s or all 1s, else invalid addr
758 fault = true;
759 }
760 ps = currState->tcr.ips;
761 break;
762 case EL2:
763 case EL3:
764 switch(bits(currState->vaddr, 63,48)) {
765 case 0:
766 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
767 if (currState->el == EL2)
768 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL2);
769 else
770 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL3);
771 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
772 tg = GrainMapDefault[currState->tcr.tg0];
773 break;
774 default:
775 // invalid addr if top two bytes are not all 0s
776 fault = true;
777 }
778 ps = currState->tcr.ips;
779 break;
780 }
781
782 if (fault) {
783 Fault f;
784 if (currState->isFetch)
785 f = std::make_shared<PrefetchAbort>(
786 currState->vaddr_tainted,
787 ArmFault::TranslationLL + L0, isStage2,
788 ArmFault::LpaeTran);
789 else
790 f = std::make_shared<DataAbort>(
791 currState->vaddr_tainted,
792 TlbEntry::DomainType::NoAccess,
793 currState->isWrite,
794 ArmFault::TranslationLL + L0,
795 isStage2, ArmFault::LpaeTran);
796
797 if (currState->timing) {
798 pending = false;
799 nextWalk(currState->tc);
800 currState = NULL;
801 } else {
802 currState->tc = NULL;
803 currState->req = NULL;
804 }
805 return f;
806
807 }
808
809 if (tg == ReservedGrain) {
810 warn_once("Reserved granule size requested; gem5's IMPLEMENTATION "
811 "DEFINED behavior takes this to mean 4KB granules\n");
812 tg = Grain4KB;
813 }
814
815 int stride = tg - 3;
816 LookupLevel start_lookup_level = MAX_LOOKUP_LEVELS;
817
818 // Determine starting lookup level
819 // See aarch64/translation/walk in Appendix G: ARMv8 Pseudocode Library
820 // in ARM DDI 0487A. These table values correspond to the cascading tests
821 // to compute the lookup level and are of the form
822 // (grain_size + N*stride), for N = {1, 2, 3}.
823 // A value of 64 will never succeed and a value of 0 will always succeed.
824 {
825 struct GrainMap {
826 GrainSize grain_size;
827 unsigned lookup_level_cutoff[MAX_LOOKUP_LEVELS];
828 };
829 static const GrainMap GM[] = {
830 { Grain4KB, { 39, 30, 0, 0 } },
831 { Grain16KB, { 47, 36, 25, 0 } },
832 { Grain64KB, { 64, 42, 29, 0 } }
833 };
834
835 const unsigned *lookup = NULL; // points to a lookup_level_cutoff
836
837 for (unsigned i = 0; i < 3; ++i) { // choose entry of GM[]
838 if (tg == GM[i].grain_size) {
839 lookup = GM[i].lookup_level_cutoff;
840 break;
841 }
842 }
843 assert(lookup);
844
845 for (int L = L0; L != MAX_LOOKUP_LEVELS; ++L) {
846 if (tsz > lookup[L]) {
847 start_lookup_level = (LookupLevel) L;
848 break;
849 }
850 }
851 panic_if(start_lookup_level == MAX_LOOKUP_LEVELS,
852 "Table walker couldn't find lookup level\n");
853 }
854
855 // Determine table base address
856 int base_addr_lo = 3 + tsz - stride * (3 - start_lookup_level) - tg;
857 Addr base_addr = mbits(ttbr, 47, base_addr_lo);
858
859 // Determine physical address size and raise an Address Size Fault if
860 // necessary
861 int pa_range = decodePhysAddrRange64(ps);
862 // Clamp to lower limit
863 if (pa_range > physAddrRange)
864 currState->physAddrRange = physAddrRange;
865 else
866 currState->physAddrRange = pa_range;
867 if (checkAddrSizeFaultAArch64(base_addr, currState->physAddrRange)) {
868 DPRINTF(TLB, "Address size fault before any lookup\n");
869 Fault f;
870 if (currState->isFetch)
871 f = std::make_shared<PrefetchAbort>(
872 currState->vaddr_tainted,
873 ArmFault::AddressSizeLL + start_lookup_level,
874 isStage2,
875 ArmFault::LpaeTran);
876 else
877 f = std::make_shared<DataAbort>(
878 currState->vaddr_tainted,
879 TlbEntry::DomainType::NoAccess,
880 currState->isWrite,
881 ArmFault::AddressSizeLL + start_lookup_level,
882 isStage2,
883 ArmFault::LpaeTran);
884
885
886 if (currState->timing) {
887 pending = false;
888 nextWalk(currState->tc);
889 currState = NULL;
890 } else {
891 currState->tc = NULL;
892 currState->req = NULL;
893 }
894 return f;
895
896 }
897
898 // Determine descriptor address
899 Addr desc_addr = base_addr |
900 (bits(currState->vaddr, tsz - 1,
901 stride * (3 - start_lookup_level) + tg) << 3);
902
903 // Trickbox address check
904 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
905 currState->vaddr, sizeof(uint64_t), currState->isFetch,
906 currState->isWrite, TlbEntry::DomainType::NoAccess,
907 start_lookup_level);
908 if (f) {
909 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
910 if (currState->timing) {
911 pending = false;
912 nextWalk(currState->tc);
913 currState = NULL;
914 } else {
915 currState->tc = NULL;
916 currState->req = NULL;
917 }
918 return f;
919 }
920
921 Request::Flags flag = 0;
922 if (currState->sctlr.c == 0) {
923 flag = Request::UNCACHEABLE;
924 }
925
926 currState->longDesc.lookupLevel = start_lookup_level;
927 currState->longDesc.aarch64 = true;
928 currState->longDesc.grainSize = tg;
929
930 if (currState->timing) {
931 Event *event;
932 switch (start_lookup_level) {
933 case L0:
934 event = (Event *) &doL0LongDescEvent;
935 break;
936 case L1:
937 event = (Event *) &doL1LongDescEvent;
938 break;
939 case L2:
940 event = (Event *) &doL2LongDescEvent;
941 break;
942 case L3:
943 event = (Event *) &doL3LongDescEvent;
944 break;
945 default:
946 panic("Invalid table lookup level");
947 break;
948 }
| 187Fault
188TableWalker::walk(RequestPtr _req, ThreadContext *_tc, uint16_t _asid,
189 uint8_t _vmid, bool _isHyp, TLB::Mode _mode,
190 TLB::Translation *_trans, bool _timing, bool _functional,
191 bool secure, TLB::ArmTranslationType tranType)
192{
193 assert(!(_functional && _timing));
194 ++statWalks;
195
196 WalkerState *savedCurrState = NULL;
197
198 if (!currState && !_functional) {
199 // For atomic mode, a new WalkerState instance should be only created
200 // once per TLB. For timing mode, a new instance is generated for every
201 // TLB miss.
202 DPRINTF(TLBVerbose, "creating new instance of WalkerState\n");
203
204 currState = new WalkerState();
205 currState->tableWalker = this;
206 } else if (_functional) {
207 // If we are mixing functional mode with timing (or even
208 // atomic), we need to to be careful and clean up after
209 // ourselves to not risk getting into an inconsistent state.
210 DPRINTF(TLBVerbose, "creating functional instance of WalkerState\n");
211 savedCurrState = currState;
212 currState = new WalkerState();
213 currState->tableWalker = this;
214 } else if (_timing) {
215 // This is a translation that was completed and then faulted again
216 // because some underlying parameters that affect the translation
217 // changed out from under us (e.g. asid). It will either be a
218 // misprediction, in which case nothing will happen or we'll use
219 // this fault to re-execute the faulting instruction which should clean
220 // up everything.
221 if (currState->vaddr_tainted == _req->getVaddr()) {
222 ++statSquashedBefore;
223 return std::make_shared<ReExec>();
224 }
225 }
226 pendingChange();
227
228 currState->startTime = curTick();
229 currState->tc = _tc;
230 currState->aarch64 = opModeIs64(currOpMode(_tc));
231 currState->el = currEL(_tc);
232 currState->transState = _trans;
233 currState->req = _req;
234 currState->fault = NoFault;
235 currState->asid = _asid;
236 currState->vmid = _vmid;
237 currState->isHyp = _isHyp;
238 currState->timing = _timing;
239 currState->functional = _functional;
240 currState->mode = _mode;
241 currState->tranType = tranType;
242 currState->isSecure = secure;
243 currState->physAddrRange = physAddrRange;
244
245 /** @todo These should be cached or grabbed from cached copies in
246 the TLB, all these miscreg reads are expensive */
247 currState->vaddr_tainted = currState->req->getVaddr();
248 if (currState->aarch64)
249 currState->vaddr = purifyTaggedAddr(currState->vaddr_tainted,
250 currState->tc, currState->el);
251 else
252 currState->vaddr = currState->vaddr_tainted;
253
254 if (currState->aarch64) {
255 switch (currState->el) {
256 case EL0:
257 case EL1:
258 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL1);
259 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL1);
260 break;
261 // @todo: uncomment this to enable Virtualization
262 // case EL2:
263 // assert(haveVirtualization);
264 // currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL2);
265 // currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL2);
266 // break;
267 case EL3:
268 assert(haveSecurity);
269 currState->sctlr = currState->tc->readMiscReg(MISCREG_SCTLR_EL3);
270 currState->tcr = currState->tc->readMiscReg(MISCREG_TCR_EL3);
271 break;
272 default:
273 panic("Invalid exception level");
274 break;
275 }
276 } else {
277 currState->sctlr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
278 MISCREG_SCTLR, currState->tc, !currState->isSecure));
279 currState->ttbcr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
280 MISCREG_TTBCR, currState->tc, !currState->isSecure));
281 currState->htcr = currState->tc->readMiscReg(MISCREG_HTCR);
282 currState->hcr = currState->tc->readMiscReg(MISCREG_HCR);
283 currState->vtcr = currState->tc->readMiscReg(MISCREG_VTCR);
284 }
285 sctlr = currState->sctlr;
286
287 currState->isFetch = (currState->mode == TLB::Execute);
288 currState->isWrite = (currState->mode == TLB::Write);
289
290 statRequestOrigin[REQUESTED][currState->isFetch]++;
291
292 // We only do a second stage of translation if we're not secure, or in
293 // hyp mode, the second stage MMU is enabled, and this table walker
294 // instance is the first stage.
295 currState->doingStage2 = false;
296 // @todo: for now disable this in AArch64 (HCR is not set)
297 currState->stage2Req = !currState->aarch64 && currState->hcr.vm &&
298 !isStage2 && !currState->isSecure && !currState->isHyp;
299
300 bool long_desc_format = currState->aarch64 ||
301 (_haveLPAE && currState->ttbcr.eae) ||
302 _isHyp || isStage2;
303
304 if (long_desc_format) {
305 // Helper variables used for hierarchical permissions
306 currState->secureLookup = currState->isSecure;
307 currState->rwTable = true;
308 currState->userTable = true;
309 currState->xnTable = false;
310 currState->pxnTable = false;
311
312 ++statWalksLongDescriptor;
313 } else {
314 ++statWalksShortDescriptor;
315 }
316
317 if (!currState->timing) {
318 Fault fault = NoFault;
319 if (currState->aarch64)
320 fault = processWalkAArch64();
321 else if (long_desc_format)
322 fault = processWalkLPAE();
323 else
324 fault = processWalk();
325
326 // If this was a functional non-timing access restore state to
327 // how we found it.
328 if (currState->functional) {
329 delete currState;
330 currState = savedCurrState;
331 }
332 return fault;
333 }
334
335 if (pending || pendingQueue.size()) {
336 pendingQueue.push_back(currState);
337 currState = NULL;
338 pendingChange();
339 } else {
340 pending = true;
341 pendingChange();
342 if (currState->aarch64)
343 return processWalkAArch64();
344 else if (long_desc_format)
345 return processWalkLPAE();
346 else
347 return processWalk();
348 }
349
350 return NoFault;
351}
352
353void
354TableWalker::processWalkWrapper()
355{
356 assert(!currState);
357 assert(pendingQueue.size());
358 pendingChange();
359 currState = pendingQueue.front();
360
361 ExceptionLevel target_el = EL0;
362 if (currState->aarch64)
363 target_el = currEL(currState->tc);
364 else
365 target_el = EL1;
366
367 // Check if a previous walk filled this request already
368 // @TODO Should this always be the TLB or should we look in the stage2 TLB?
369 TlbEntry* te = tlb->lookup(currState->vaddr, currState->asid,
370 currState->vmid, currState->isHyp, currState->isSecure, true, false,
371 target_el);
372
373 // Check if we still need to have a walk for this request. If the requesting
374 // instruction has been squashed, or a previous walk has filled the TLB with
375 // a match, we just want to get rid of the walk. The latter could happen
376 // when there are multiple outstanding misses to a single page and a
377 // previous request has been successfully translated.
378 if (!currState->transState->squashed() && !te) {
379 // We've got a valid request, lets process it
380 pending = true;
381 pendingQueue.pop_front();
382 // Keep currState in case one of the processWalk... calls NULLs it
383 WalkerState *curr_state_copy = currState;
384 Fault f;
385 if (currState->aarch64)
386 f = processWalkAArch64();
387 else if ((_haveLPAE && currState->ttbcr.eae) || currState->isHyp || isStage2)
388 f = processWalkLPAE();
389 else
390 f = processWalk();
391
392 if (f != NoFault) {
393 curr_state_copy->transState->finish(f, curr_state_copy->req,
394 curr_state_copy->tc, curr_state_copy->mode);
395
396 delete curr_state_copy;
397 }
398 return;
399 }
400
401
402 // If the instruction that we were translating for has been
403 // squashed we shouldn't bother.
404 unsigned num_squashed = 0;
405 ThreadContext *tc = currState->tc;
406 while ((num_squashed < numSquashable) && currState &&
407 (currState->transState->squashed() || te)) {
408 pendingQueue.pop_front();
409 num_squashed++;
410 statSquashedBefore++;
411
412 DPRINTF(TLB, "Squashing table walk for address %#x\n",
413 currState->vaddr_tainted);
414
415 if (currState->transState->squashed()) {
416 // finish the translation which will delete the translation object
417 currState->transState->finish(
418 std::make_shared<UnimpFault>("Squashed Inst"),
419 currState->req, currState->tc, currState->mode);
420 } else {
421 // translate the request now that we know it will work
422 statWalkServiceTime.sample(curTick() - currState->startTime);
423 tlb->translateTiming(currState->req, currState->tc,
424 currState->transState, currState->mode);
425
426 }
427
428 // delete the current request
429 delete currState;
430
431 // peak at the next one
432 if (pendingQueue.size()) {
433 currState = pendingQueue.front();
434 te = tlb->lookup(currState->vaddr, currState->asid,
435 currState->vmid, currState->isHyp, currState->isSecure, true,
436 false, target_el);
437 } else {
438 // Terminate the loop, nothing more to do
439 currState = NULL;
440 }
441 }
442 pendingChange();
443
444 // if we still have pending translations, schedule more work
445 nextWalk(tc);
446 currState = NULL;
447}
448
449Fault
450TableWalker::processWalk()
451{
452 Addr ttbr = 0;
453
454 // If translation isn't enabled, we shouldn't be here
455 assert(currState->sctlr.m || isStage2);
456
457 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x, bits:%#x\n",
458 currState->vaddr_tainted, currState->ttbcr, mbits(currState->vaddr, 31,
459 32 - currState->ttbcr.n));
460
461 statWalkWaitTime.sample(curTick() - currState->startTime);
462
463 if (currState->ttbcr.n == 0 || !mbits(currState->vaddr, 31,
464 32 - currState->ttbcr.n)) {
465 DPRINTF(TLB, " - Selecting TTBR0\n");
466 // Check if table walk is allowed when Security Extensions are enabled
467 if (haveSecurity && currState->ttbcr.pd0) {
468 if (currState->isFetch)
469 return std::make_shared<PrefetchAbort>(
470 currState->vaddr_tainted,
471 ArmFault::TranslationLL + L1,
472 isStage2,
473 ArmFault::VmsaTran);
474 else
475 return std::make_shared<DataAbort>(
476 currState->vaddr_tainted,
477 TlbEntry::DomainType::NoAccess, currState->isWrite,
478 ArmFault::TranslationLL + L1, isStage2,
479 ArmFault::VmsaTran);
480 }
481 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
482 MISCREG_TTBR0, currState->tc, !currState->isSecure));
483 } else {
484 DPRINTF(TLB, " - Selecting TTBR1\n");
485 // Check if table walk is allowed when Security Extensions are enabled
486 if (haveSecurity && currState->ttbcr.pd1) {
487 if (currState->isFetch)
488 return std::make_shared<PrefetchAbort>(
489 currState->vaddr_tainted,
490 ArmFault::TranslationLL + L1,
491 isStage2,
492 ArmFault::VmsaTran);
493 else
494 return std::make_shared<DataAbort>(
495 currState->vaddr_tainted,
496 TlbEntry::DomainType::NoAccess, currState->isWrite,
497 ArmFault::TranslationLL + L1, isStage2,
498 ArmFault::VmsaTran);
499 }
500 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
501 MISCREG_TTBR1, currState->tc, !currState->isSecure));
502 currState->ttbcr.n = 0;
503 }
504
505 Addr l1desc_addr = mbits(ttbr, 31, 14 - currState->ttbcr.n) |
506 (bits(currState->vaddr, 31 - currState->ttbcr.n, 20) << 2);
507 DPRINTF(TLB, " - Descriptor at address %#x (%s)\n", l1desc_addr,
508 currState->isSecure ? "s" : "ns");
509
510 // Trickbox address check
511 Fault f;
512 f = tlb->walkTrickBoxCheck(l1desc_addr, currState->isSecure,
513 currState->vaddr, sizeof(uint32_t), currState->isFetch,
514 currState->isWrite, TlbEntry::DomainType::NoAccess, L1);
515 if (f) {
516 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
517 if (currState->timing) {
518 pending = false;
519 nextWalk(currState->tc);
520 currState = NULL;
521 } else {
522 currState->tc = NULL;
523 currState->req = NULL;
524 }
525 return f;
526 }
527
528 Request::Flags flag = 0;
529 if (currState->sctlr.c == 0) {
530 flag = Request::UNCACHEABLE;
531 }
532
533 bool delayed;
534 delayed = fetchDescriptor(l1desc_addr, (uint8_t*)&currState->l1Desc.data,
535 sizeof(uint32_t), flag, L1, &doL1DescEvent,
536 &TableWalker::doL1Descriptor);
537 if (!delayed) {
538 f = currState->fault;
539 }
540
541 return f;
542}
543
544Fault
545TableWalker::processWalkLPAE()
546{
547 Addr ttbr, ttbr0_max, ttbr1_min, desc_addr;
548 int tsz, n;
549 LookupLevel start_lookup_level = L1;
550
551 DPRINTF(TLB, "Beginning table walk for address %#x, TTBCR: %#x\n",
552 currState->vaddr_tainted, currState->ttbcr);
553
554 statWalkWaitTime.sample(curTick() - currState->startTime);
555
556 Request::Flags flag = 0;
557 if (currState->isSecure)
558 flag.set(Request::SECURE);
559
560 // work out which base address register to use, if in hyp mode we always
561 // use HTTBR
562 if (isStage2) {
563 DPRINTF(TLB, " - Selecting VTTBR (long-desc.)\n");
564 ttbr = currState->tc->readMiscReg(MISCREG_VTTBR);
565 tsz = sext<4>(currState->vtcr.t0sz);
566 start_lookup_level = currState->vtcr.sl0 ? L1 : L2;
567 } else if (currState->isHyp) {
568 DPRINTF(TLB, " - Selecting HTTBR (long-desc.)\n");
569 ttbr = currState->tc->readMiscReg(MISCREG_HTTBR);
570 tsz = currState->htcr.t0sz;
571 } else {
572 assert(_haveLPAE && currState->ttbcr.eae);
573
574 // Determine boundaries of TTBR0/1 regions
575 if (currState->ttbcr.t0sz)
576 ttbr0_max = (1ULL << (32 - currState->ttbcr.t0sz)) - 1;
577 else if (currState->ttbcr.t1sz)
578 ttbr0_max = (1ULL << 32) -
579 (1ULL << (32 - currState->ttbcr.t1sz)) - 1;
580 else
581 ttbr0_max = (1ULL << 32) - 1;
582 if (currState->ttbcr.t1sz)
583 ttbr1_min = (1ULL << 32) - (1ULL << (32 - currState->ttbcr.t1sz));
584 else
585 ttbr1_min = (1ULL << (32 - currState->ttbcr.t0sz));
586
587 // The following code snippet selects the appropriate translation table base
588 // address (TTBR0 or TTBR1) and the appropriate starting lookup level
589 // depending on the address range supported by the translation table (ARM
590 // ARM issue C B3.6.4)
591 if (currState->vaddr <= ttbr0_max) {
592 DPRINTF(TLB, " - Selecting TTBR0 (long-desc.)\n");
593 // Check if table walk is allowed
594 if (currState->ttbcr.epd0) {
595 if (currState->isFetch)
596 return std::make_shared<PrefetchAbort>(
597 currState->vaddr_tainted,
598 ArmFault::TranslationLL + L1,
599 isStage2,
600 ArmFault::LpaeTran);
601 else
602 return std::make_shared<DataAbort>(
603 currState->vaddr_tainted,
604 TlbEntry::DomainType::NoAccess,
605 currState->isWrite,
606 ArmFault::TranslationLL + L1,
607 isStage2,
608 ArmFault::LpaeTran);
609 }
610 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
611 MISCREG_TTBR0, currState->tc, !currState->isSecure));
612 tsz = currState->ttbcr.t0sz;
613 if (ttbr0_max < (1ULL << 30)) // Upper limit < 1 GB
614 start_lookup_level = L2;
615 } else if (currState->vaddr >= ttbr1_min) {
616 DPRINTF(TLB, " - Selecting TTBR1 (long-desc.)\n");
617 // Check if table walk is allowed
618 if (currState->ttbcr.epd1) {
619 if (currState->isFetch)
620 return std::make_shared<PrefetchAbort>(
621 currState->vaddr_tainted,
622 ArmFault::TranslationLL + L1,
623 isStage2,
624 ArmFault::LpaeTran);
625 else
626 return std::make_shared<DataAbort>(
627 currState->vaddr_tainted,
628 TlbEntry::DomainType::NoAccess,
629 currState->isWrite,
630 ArmFault::TranslationLL + L1,
631 isStage2,
632 ArmFault::LpaeTran);
633 }
634 ttbr = currState->tc->readMiscReg(flattenMiscRegNsBanked(
635 MISCREG_TTBR1, currState->tc, !currState->isSecure));
636 tsz = currState->ttbcr.t1sz;
637 if (ttbr1_min >= (1ULL << 31) + (1ULL << 30)) // Lower limit >= 3 GB
638 start_lookup_level = L2;
639 } else {
640 // Out of boundaries -> translation fault
641 if (currState->isFetch)
642 return std::make_shared<PrefetchAbort>(
643 currState->vaddr_tainted,
644 ArmFault::TranslationLL + L1,
645 isStage2,
646 ArmFault::LpaeTran);
647 else
648 return std::make_shared<DataAbort>(
649 currState->vaddr_tainted,
650 TlbEntry::DomainType::NoAccess,
651 currState->isWrite, ArmFault::TranslationLL + L1,
652 isStage2, ArmFault::LpaeTran);
653 }
654
655 }
656
657 // Perform lookup (ARM ARM issue C B3.6.6)
658 if (start_lookup_level == L1) {
659 n = 5 - tsz;
660 desc_addr = mbits(ttbr, 39, n) |
661 (bits(currState->vaddr, n + 26, 30) << 3);
662 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
663 desc_addr, currState->isSecure ? "s" : "ns");
664 } else {
665 // Skip first-level lookup
666 n = (tsz >= 2 ? 14 - tsz : 12);
667 desc_addr = mbits(ttbr, 39, n) |
668 (bits(currState->vaddr, n + 17, 21) << 3);
669 DPRINTF(TLB, " - Descriptor at address %#x (%s) (long-desc.)\n",
670 desc_addr, currState->isSecure ? "s" : "ns");
671 }
672
673 // Trickbox address check
674 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
675 currState->vaddr, sizeof(uint64_t), currState->isFetch,
676 currState->isWrite, TlbEntry::DomainType::NoAccess,
677 start_lookup_level);
678 if (f) {
679 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
680 if (currState->timing) {
681 pending = false;
682 nextWalk(currState->tc);
683 currState = NULL;
684 } else {
685 currState->tc = NULL;
686 currState->req = NULL;
687 }
688 return f;
689 }
690
691 if (currState->sctlr.c == 0) {
692 flag = Request::UNCACHEABLE;
693 }
694
695 if (currState->isSecure)
696 flag.set(Request::SECURE);
697
698 currState->longDesc.lookupLevel = start_lookup_level;
699 currState->longDesc.aarch64 = false;
700 currState->longDesc.grainSize = Grain4KB;
701
702 Event *event = start_lookup_level == L1 ? (Event *) &doL1LongDescEvent
703 : (Event *) &doL2LongDescEvent;
704
705 bool delayed = fetchDescriptor(desc_addr, (uint8_t*)&currState->longDesc.data,
706 sizeof(uint64_t), flag, start_lookup_level,
707 event, &TableWalker::doLongDescriptor);
708 if (!delayed) {
709 f = currState->fault;
710 }
711
712 return f;
713}
714
715unsigned
716TableWalker::adjustTableSizeAArch64(unsigned tsz)
717{
718 if (tsz < 25)
719 return 25;
720 if (tsz > 48)
721 return 48;
722 return tsz;
723}
724
725bool
726TableWalker::checkAddrSizeFaultAArch64(Addr addr, int currPhysAddrRange)
727{
728 return (currPhysAddrRange != MaxPhysAddrRange &&
729 bits(addr, MaxPhysAddrRange - 1, currPhysAddrRange));
730}
731
732Fault
733TableWalker::processWalkAArch64()
734{
735 assert(currState->aarch64);
736
737 DPRINTF(TLB, "Beginning table walk for address %#llx, TCR: %#llx\n",
738 currState->vaddr_tainted, currState->tcr);
739
740 static const GrainSize GrainMapDefault[] =
741 { Grain4KB, Grain64KB, Grain16KB, ReservedGrain };
742 static const GrainSize GrainMap_EL1_tg1[] =
743 { ReservedGrain, Grain16KB, Grain4KB, Grain64KB };
744
745 statWalkWaitTime.sample(curTick() - currState->startTime);
746
747 // Determine TTBR, table size, granule size and phys. address range
748 Addr ttbr = 0;
749 int tsz = 0, ps = 0;
750 GrainSize tg = Grain4KB; // grain size computed from tg* field
751 bool fault = false;
752 switch (currState->el) {
753 case EL0:
754 case EL1:
755 switch (bits(currState->vaddr, 63,48)) {
756 case 0:
757 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
758 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL1);
759 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
760 tg = GrainMapDefault[currState->tcr.tg0];
761 if (bits(currState->vaddr, 63, tsz) != 0x0 ||
762 currState->tcr.epd0)
763 fault = true;
764 break;
765 case 0xffff:
766 DPRINTF(TLB, " - Selecting TTBR1 (AArch64)\n");
767 ttbr = currState->tc->readMiscReg(MISCREG_TTBR1_EL1);
768 tsz = adjustTableSizeAArch64(64 - currState->tcr.t1sz);
769 tg = GrainMap_EL1_tg1[currState->tcr.tg1];
770 if (bits(currState->vaddr, 63, tsz) != mask(64-tsz) ||
771 currState->tcr.epd1)
772 fault = true;
773 break;
774 default:
775 // top two bytes must be all 0s or all 1s, else invalid addr
776 fault = true;
777 }
778 ps = currState->tcr.ips;
779 break;
780 case EL2:
781 case EL3:
782 switch(bits(currState->vaddr, 63,48)) {
783 case 0:
784 DPRINTF(TLB, " - Selecting TTBR0 (AArch64)\n");
785 if (currState->el == EL2)
786 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL2);
787 else
788 ttbr = currState->tc->readMiscReg(MISCREG_TTBR0_EL3);
789 tsz = adjustTableSizeAArch64(64 - currState->tcr.t0sz);
790 tg = GrainMapDefault[currState->tcr.tg0];
791 break;
792 default:
793 // invalid addr if top two bytes are not all 0s
794 fault = true;
795 }
796 ps = currState->tcr.ips;
797 break;
798 }
799
800 if (fault) {
801 Fault f;
802 if (currState->isFetch)
803 f = std::make_shared<PrefetchAbort>(
804 currState->vaddr_tainted,
805 ArmFault::TranslationLL + L0, isStage2,
806 ArmFault::LpaeTran);
807 else
808 f = std::make_shared<DataAbort>(
809 currState->vaddr_tainted,
810 TlbEntry::DomainType::NoAccess,
811 currState->isWrite,
812 ArmFault::TranslationLL + L0,
813 isStage2, ArmFault::LpaeTran);
814
815 if (currState->timing) {
816 pending = false;
817 nextWalk(currState->tc);
818 currState = NULL;
819 } else {
820 currState->tc = NULL;
821 currState->req = NULL;
822 }
823 return f;
824
825 }
826
827 if (tg == ReservedGrain) {
828 warn_once("Reserved granule size requested; gem5's IMPLEMENTATION "
829 "DEFINED behavior takes this to mean 4KB granules\n");
830 tg = Grain4KB;
831 }
832
833 int stride = tg - 3;
834 LookupLevel start_lookup_level = MAX_LOOKUP_LEVELS;
835
836 // Determine starting lookup level
837 // See aarch64/translation/walk in Appendix G: ARMv8 Pseudocode Library
838 // in ARM DDI 0487A. These table values correspond to the cascading tests
839 // to compute the lookup level and are of the form
840 // (grain_size + N*stride), for N = {1, 2, 3}.
841 // A value of 64 will never succeed and a value of 0 will always succeed.
842 {
843 struct GrainMap {
844 GrainSize grain_size;
845 unsigned lookup_level_cutoff[MAX_LOOKUP_LEVELS];
846 };
847 static const GrainMap GM[] = {
848 { Grain4KB, { 39, 30, 0, 0 } },
849 { Grain16KB, { 47, 36, 25, 0 } },
850 { Grain64KB, { 64, 42, 29, 0 } }
851 };
852
853 const unsigned *lookup = NULL; // points to a lookup_level_cutoff
854
855 for (unsigned i = 0; i < 3; ++i) { // choose entry of GM[]
856 if (tg == GM[i].grain_size) {
857 lookup = GM[i].lookup_level_cutoff;
858 break;
859 }
860 }
861 assert(lookup);
862
863 for (int L = L0; L != MAX_LOOKUP_LEVELS; ++L) {
864 if (tsz > lookup[L]) {
865 start_lookup_level = (LookupLevel) L;
866 break;
867 }
868 }
869 panic_if(start_lookup_level == MAX_LOOKUP_LEVELS,
870 "Table walker couldn't find lookup level\n");
871 }
872
873 // Determine table base address
874 int base_addr_lo = 3 + tsz - stride * (3 - start_lookup_level) - tg;
875 Addr base_addr = mbits(ttbr, 47, base_addr_lo);
876
877 // Determine physical address size and raise an Address Size Fault if
878 // necessary
879 int pa_range = decodePhysAddrRange64(ps);
880 // Clamp to lower limit
881 if (pa_range > physAddrRange)
882 currState->physAddrRange = physAddrRange;
883 else
884 currState->physAddrRange = pa_range;
885 if (checkAddrSizeFaultAArch64(base_addr, currState->physAddrRange)) {
886 DPRINTF(TLB, "Address size fault before any lookup\n");
887 Fault f;
888 if (currState->isFetch)
889 f = std::make_shared<PrefetchAbort>(
890 currState->vaddr_tainted,
891 ArmFault::AddressSizeLL + start_lookup_level,
892 isStage2,
893 ArmFault::LpaeTran);
894 else
895 f = std::make_shared<DataAbort>(
896 currState->vaddr_tainted,
897 TlbEntry::DomainType::NoAccess,
898 currState->isWrite,
899 ArmFault::AddressSizeLL + start_lookup_level,
900 isStage2,
901 ArmFault::LpaeTran);
902
903
904 if (currState->timing) {
905 pending = false;
906 nextWalk(currState->tc);
907 currState = NULL;
908 } else {
909 currState->tc = NULL;
910 currState->req = NULL;
911 }
912 return f;
913
914 }
915
916 // Determine descriptor address
917 Addr desc_addr = base_addr |
918 (bits(currState->vaddr, tsz - 1,
919 stride * (3 - start_lookup_level) + tg) << 3);
920
921 // Trickbox address check
922 Fault f = tlb->walkTrickBoxCheck(desc_addr, currState->isSecure,
923 currState->vaddr, sizeof(uint64_t), currState->isFetch,
924 currState->isWrite, TlbEntry::DomainType::NoAccess,
925 start_lookup_level);
926 if (f) {
927 DPRINTF(TLB, "Trickbox check caused fault on %#x\n", currState->vaddr_tainted);
928 if (currState->timing) {
929 pending = false;
930 nextWalk(currState->tc);
931 currState = NULL;
932 } else {
933 currState->tc = NULL;
934 currState->req = NULL;
935 }
936 return f;
937 }
938
939 Request::Flags flag = 0;
940 if (currState->sctlr.c == 0) {
941 flag = Request::UNCACHEABLE;
942 }
943
944 currState->longDesc.lookupLevel = start_lookup_level;
945 currState->longDesc.aarch64 = true;
946 currState->longDesc.grainSize = tg;
947
948 if (currState->timing) {
949 Event *event;
950 switch (start_lookup_level) {
951 case L0:
952 event = (Event *) &doL0LongDescEvent;
953 break;
954 case L1:
955 event = (Event *) &doL1LongDescEvent;
956 break;
957 case L2:
958 event = (Event *) &doL2LongDescEvent;
959 break;
960 case L3:
961 event = (Event *) &doL3LongDescEvent;
962 break;
963 default:
964 panic("Invalid table lookup level");
965 break;
966 }
|
949 port.dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
| 967 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
|
950 event, (uint8_t*) &currState->longDesc.data,
951 currState->tc->getCpuPtr()->clockPeriod(), flag);
952 DPRINTF(TLBVerbose,
953 "Adding to walker fifo: queue size before adding: %d\n",
954 stateQueues[start_lookup_level].size());
955 stateQueues[start_lookup_level].push_back(currState);
956 currState = NULL;
957 } else if (!currState->functional) {
| 968 event, (uint8_t*) &currState->longDesc.data,
969 currState->tc->getCpuPtr()->clockPeriod(), flag);
970 DPRINTF(TLBVerbose,
971 "Adding to walker fifo: queue size before adding: %d\n",
972 stateQueues[start_lookup_level].size());
973 stateQueues[start_lookup_level].push_back(currState);
974 currState = NULL;
975 } else if (!currState->functional) {
|
958 port.dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
| 976 port->dmaAction(MemCmd::ReadReq, desc_addr, sizeof(uint64_t),
|
959 NULL, (uint8_t*) &currState->longDesc.data,
960 currState->tc->getCpuPtr()->clockPeriod(), flag);
961 doLongDescriptor();
962 f = currState->fault;
963 } else {
964 RequestPtr req = new Request(desc_addr, sizeof(uint64_t), flag,
965 masterId);
966 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
967 pkt->dataStatic((uint8_t*) &currState->longDesc.data);
| 977 NULL, (uint8_t*) &currState->longDesc.data,
978 currState->tc->getCpuPtr()->clockPeriod(), flag);
979 doLongDescriptor();
980 f = currState->fault;
981 } else {
982 RequestPtr req = new Request(desc_addr, sizeof(uint64_t), flag,
983 masterId);
984 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
985 pkt->dataStatic((uint8_t*) &currState->longDesc.data);
|
968 port.sendFunctional(pkt);
| 986 port->sendFunctional(pkt);
|
969 doLongDescriptor();
970 delete req;
971 delete pkt;
972 f = currState->fault;
973 }
974
975 return f;
976}
977
978void
979TableWalker::memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr,
980 uint8_t texcb, bool s)
981{
982 // Note: tc and sctlr local variables are hiding tc and sctrl class
983 // variables
984 DPRINTF(TLBVerbose, "memAttrs texcb:%d s:%d\n", texcb, s);
985 te.shareable = false; // default value
986 te.nonCacheable = false;
987 te.outerShareable = false;
988 if (sctlr.tre == 0 || ((sctlr.tre == 1) && (sctlr.m == 0))) {
989 switch(texcb) {
990 case 0: // Stongly-ordered
991 te.nonCacheable = true;
992 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
993 te.shareable = true;
994 te.innerAttrs = 1;
995 te.outerAttrs = 0;
996 break;
997 case 1: // Shareable Device
998 te.nonCacheable = true;
999 te.mtype = TlbEntry::MemoryType::Device;
1000 te.shareable = true;
1001 te.innerAttrs = 3;
1002 te.outerAttrs = 0;
1003 break;
1004 case 2: // Outer and Inner Write-Through, no Write-Allocate
1005 te.mtype = TlbEntry::MemoryType::Normal;
1006 te.shareable = s;
1007 te.innerAttrs = 6;
1008 te.outerAttrs = bits(texcb, 1, 0);
1009 break;
1010 case 3: // Outer and Inner Write-Back, no Write-Allocate
1011 te.mtype = TlbEntry::MemoryType::Normal;
1012 te.shareable = s;
1013 te.innerAttrs = 7;
1014 te.outerAttrs = bits(texcb, 1, 0);
1015 break;
1016 case 4: // Outer and Inner Non-cacheable
1017 te.nonCacheable = true;
1018 te.mtype = TlbEntry::MemoryType::Normal;
1019 te.shareable = s;
1020 te.innerAttrs = 0;
1021 te.outerAttrs = bits(texcb, 1, 0);
1022 break;
1023 case 5: // Reserved
1024 panic("Reserved texcb value!\n");
1025 break;
1026 case 6: // Implementation Defined
1027 panic("Implementation-defined texcb value!\n");
1028 break;
1029 case 7: // Outer and Inner Write-Back, Write-Allocate
1030 te.mtype = TlbEntry::MemoryType::Normal;
1031 te.shareable = s;
1032 te.innerAttrs = 5;
1033 te.outerAttrs = 1;
1034 break;
1035 case 8: // Non-shareable Device
1036 te.nonCacheable = true;
1037 te.mtype = TlbEntry::MemoryType::Device;
1038 te.shareable = false;
1039 te.innerAttrs = 3;
1040 te.outerAttrs = 0;
1041 break;
1042 case 9 ... 15: // Reserved
1043 panic("Reserved texcb value!\n");
1044 break;
1045 case 16 ... 31: // Cacheable Memory
1046 te.mtype = TlbEntry::MemoryType::Normal;
1047 te.shareable = s;
1048 if (bits(texcb, 1,0) == 0 || bits(texcb, 3,2) == 0)
1049 te.nonCacheable = true;
1050 te.innerAttrs = bits(texcb, 1, 0);
1051 te.outerAttrs = bits(texcb, 3, 2);
1052 break;
1053 default:
1054 panic("More than 32 states for 5 bits?\n");
1055 }
1056 } else {
1057 assert(tc);
1058 PRRR prrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_PRRR,
1059 currState->tc, !currState->isSecure));
1060 NMRR nmrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_NMRR,
1061 currState->tc, !currState->isSecure));
1062 DPRINTF(TLBVerbose, "memAttrs PRRR:%08x NMRR:%08x\n", prrr, nmrr);
1063 uint8_t curr_tr = 0, curr_ir = 0, curr_or = 0;
1064 switch(bits(texcb, 2,0)) {
1065 case 0:
1066 curr_tr = prrr.tr0;
1067 curr_ir = nmrr.ir0;
1068 curr_or = nmrr.or0;
1069 te.outerShareable = (prrr.nos0 == 0);
1070 break;
1071 case 1:
1072 curr_tr = prrr.tr1;
1073 curr_ir = nmrr.ir1;
1074 curr_or = nmrr.or1;
1075 te.outerShareable = (prrr.nos1 == 0);
1076 break;
1077 case 2:
1078 curr_tr = prrr.tr2;
1079 curr_ir = nmrr.ir2;
1080 curr_or = nmrr.or2;
1081 te.outerShareable = (prrr.nos2 == 0);
1082 break;
1083 case 3:
1084 curr_tr = prrr.tr3;
1085 curr_ir = nmrr.ir3;
1086 curr_or = nmrr.or3;
1087 te.outerShareable = (prrr.nos3 == 0);
1088 break;
1089 case 4:
1090 curr_tr = prrr.tr4;
1091 curr_ir = nmrr.ir4;
1092 curr_or = nmrr.or4;
1093 te.outerShareable = (prrr.nos4 == 0);
1094 break;
1095 case 5:
1096 curr_tr = prrr.tr5;
1097 curr_ir = nmrr.ir5;
1098 curr_or = nmrr.or5;
1099 te.outerShareable = (prrr.nos5 == 0);
1100 break;
1101 case 6:
1102 panic("Imp defined type\n");
1103 case 7:
1104 curr_tr = prrr.tr7;
1105 curr_ir = nmrr.ir7;
1106 curr_or = nmrr.or7;
1107 te.outerShareable = (prrr.nos7 == 0);
1108 break;
1109 }
1110
1111 switch(curr_tr) {
1112 case 0:
1113 DPRINTF(TLBVerbose, "StronglyOrdered\n");
1114 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1115 te.nonCacheable = true;
1116 te.innerAttrs = 1;
1117 te.outerAttrs = 0;
1118 te.shareable = true;
1119 break;
1120 case 1:
1121 DPRINTF(TLBVerbose, "Device ds1:%d ds0:%d s:%d\n",
1122 prrr.ds1, prrr.ds0, s);
1123 te.mtype = TlbEntry::MemoryType::Device;
1124 te.nonCacheable = true;
1125 te.innerAttrs = 3;
1126 te.outerAttrs = 0;
1127 if (prrr.ds1 && s)
1128 te.shareable = true;
1129 if (prrr.ds0 && !s)
1130 te.shareable = true;
1131 break;
1132 case 2:
1133 DPRINTF(TLBVerbose, "Normal ns1:%d ns0:%d s:%d\n",
1134 prrr.ns1, prrr.ns0, s);
1135 te.mtype = TlbEntry::MemoryType::Normal;
1136 if (prrr.ns1 && s)
1137 te.shareable = true;
1138 if (prrr.ns0 && !s)
1139 te.shareable = true;
1140 break;
1141 case 3:
1142 panic("Reserved type");
1143 }
1144
1145 if (te.mtype == TlbEntry::MemoryType::Normal){
1146 switch(curr_ir) {
1147 case 0:
1148 te.nonCacheable = true;
1149 te.innerAttrs = 0;
1150 break;
1151 case 1:
1152 te.innerAttrs = 5;
1153 break;
1154 case 2:
1155 te.innerAttrs = 6;
1156 break;
1157 case 3:
1158 te.innerAttrs = 7;
1159 break;
1160 }
1161
1162 switch(curr_or) {
1163 case 0:
1164 te.nonCacheable = true;
1165 te.outerAttrs = 0;
1166 break;
1167 case 1:
1168 te.outerAttrs = 1;
1169 break;
1170 case 2:
1171 te.outerAttrs = 2;
1172 break;
1173 case 3:
1174 te.outerAttrs = 3;
1175 break;
1176 }
1177 }
1178 }
1179 DPRINTF(TLBVerbose, "memAttrs: shareable: %d, innerAttrs: %d, "
1180 "outerAttrs: %d\n",
1181 te.shareable, te.innerAttrs, te.outerAttrs);
1182 te.setAttributes(false);
1183}
1184
1185void
1186TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te,
1187 LongDescriptor &lDescriptor)
1188{
1189 assert(_haveLPAE);
1190
1191 uint8_t attr;
1192 uint8_t sh = lDescriptor.sh();
1193 // Different format and source of attributes if this is a stage 2
1194 // translation
1195 if (isStage2) {
1196 attr = lDescriptor.memAttr();
1197 uint8_t attr_3_2 = (attr >> 2) & 0x3;
1198 uint8_t attr_1_0 = attr & 0x3;
1199
1200 DPRINTF(TLBVerbose, "memAttrsLPAE MemAttr:%#x sh:%#x\n", attr, sh);
1201
1202 if (attr_3_2 == 0) {
1203 te.mtype = attr_1_0 == 0 ? TlbEntry::MemoryType::StronglyOrdered
1204 : TlbEntry::MemoryType::Device;
1205 te.outerAttrs = 0;
1206 te.innerAttrs = attr_1_0 == 0 ? 1 : 3;
1207 te.nonCacheable = true;
1208 } else {
1209 te.mtype = TlbEntry::MemoryType::Normal;
1210 te.outerAttrs = attr_3_2 == 1 ? 0 :
1211 attr_3_2 == 2 ? 2 : 1;
1212 te.innerAttrs = attr_1_0 == 1 ? 0 :
1213 attr_1_0 == 2 ? 6 : 5;
1214 te.nonCacheable = (attr_3_2 == 1) || (attr_1_0 == 1);
1215 }
1216 } else {
1217 uint8_t attrIndx = lDescriptor.attrIndx();
1218
1219 // LPAE always uses remapping of memory attributes, irrespective of the
1220 // value of SCTLR.TRE
1221 MiscRegIndex reg = attrIndx & 0x4 ? MISCREG_MAIR1 : MISCREG_MAIR0;
1222 int reg_as_int = flattenMiscRegNsBanked(reg, currState->tc,
1223 !currState->isSecure);
1224 uint32_t mair = currState->tc->readMiscReg(reg_as_int);
1225 attr = (mair >> (8 * (attrIndx % 4))) & 0xff;
1226 uint8_t attr_7_4 = bits(attr, 7, 4);
1227 uint8_t attr_3_0 = bits(attr, 3, 0);
1228 DPRINTF(TLBVerbose, "memAttrsLPAE AttrIndx:%#x sh:%#x, attr %#x\n", attrIndx, sh, attr);
1229
1230 // Note: the memory subsystem only cares about the 'cacheable' memory
1231 // attribute. The other attributes are only used to fill the PAR register
1232 // accordingly to provide the illusion of full support
1233 te.nonCacheable = false;
1234
1235 switch (attr_7_4) {
1236 case 0x0:
1237 // Strongly-ordered or Device memory
1238 if (attr_3_0 == 0x0)
1239 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1240 else if (attr_3_0 == 0x4)
1241 te.mtype = TlbEntry::MemoryType::Device;
1242 else
1243 panic("Unpredictable behavior\n");
1244 te.nonCacheable = true;
1245 te.outerAttrs = 0;
1246 break;
1247 case 0x4:
1248 // Normal memory, Outer Non-cacheable
1249 te.mtype = TlbEntry::MemoryType::Normal;
1250 te.outerAttrs = 0;
1251 if (attr_3_0 == 0x4)
1252 // Inner Non-cacheable
1253 te.nonCacheable = true;
1254 else if (attr_3_0 < 0x8)
1255 panic("Unpredictable behavior\n");
1256 break;
1257 case 0x8:
1258 case 0x9:
1259 case 0xa:
1260 case 0xb:
1261 case 0xc:
1262 case 0xd:
1263 case 0xe:
1264 case 0xf:
1265 if (attr_7_4 & 0x4) {
1266 te.outerAttrs = (attr_7_4 & 1) ? 1 : 3;
1267 } else {
1268 te.outerAttrs = 0x2;
1269 }
1270 // Normal memory, Outer Cacheable
1271 te.mtype = TlbEntry::MemoryType::Normal;
1272 if (attr_3_0 != 0x4 && attr_3_0 < 0x8)
1273 panic("Unpredictable behavior\n");
1274 break;
1275 default:
1276 panic("Unpredictable behavior\n");
1277 break;
1278 }
1279
1280 switch (attr_3_0) {
1281 case 0x0:
1282 te.innerAttrs = 0x1;
1283 break;
1284 case 0x4:
1285 te.innerAttrs = attr_7_4 == 0 ? 0x3 : 0;
1286 break;
1287 case 0x8:
1288 case 0x9:
1289 case 0xA:
1290 case 0xB:
1291 te.innerAttrs = 6;
1292 break;
1293 case 0xC:
1294 case 0xD:
1295 case 0xE:
1296 case 0xF:
1297 te.innerAttrs = attr_3_0 & 1 ? 0x5 : 0x7;
1298 break;
1299 default:
1300 panic("Unpredictable behavior\n");
1301 break;
1302 }
1303 }
1304
1305 te.outerShareable = sh == 2;
1306 te.shareable = (sh & 0x2) ? true : false;
1307 te.setAttributes(true);
1308 te.attributes |= (uint64_t) attr << 56;
1309}
1310
1311void
1312TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, uint8_t attrIndx,
1313 uint8_t sh)
1314{
1315 DPRINTF(TLBVerbose, "memAttrsAArch64 AttrIndx:%#x sh:%#x\n", attrIndx, sh);
1316
1317 // Select MAIR
1318 uint64_t mair;
1319 switch (currState->el) {
1320 case EL0:
1321 case EL1:
1322 mair = tc->readMiscReg(MISCREG_MAIR_EL1);
1323 break;
1324 case EL2:
1325 mair = tc->readMiscReg(MISCREG_MAIR_EL2);
1326 break;
1327 case EL3:
1328 mair = tc->readMiscReg(MISCREG_MAIR_EL3);
1329 break;
1330 default:
1331 panic("Invalid exception level");
1332 break;
1333 }
1334
1335 // Select attributes
1336 uint8_t attr = bits(mair, 8 * attrIndx + 7, 8 * attrIndx);
1337 uint8_t attr_lo = bits(attr, 3, 0);
1338 uint8_t attr_hi = bits(attr, 7, 4);
1339
1340 // Memory type
1341 te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal;
1342
1343 // Cacheability
1344 te.nonCacheable = false;
1345 if (te.mtype == TlbEntry::MemoryType::Device || // Device memory
1346 attr_hi == 0x8 || // Normal memory, Outer Non-cacheable
1347 attr_lo == 0x8) { // Normal memory, Inner Non-cacheable
1348 te.nonCacheable = true;
1349 }
1350
1351 te.shareable = sh == 2;
1352 te.outerShareable = (sh & 0x2) ? true : false;
1353 // Attributes formatted according to the 64-bit PAR
1354 te.attributes = ((uint64_t) attr << 56) |
1355 (1 << 11) | // LPAE bit
1356 (te.ns << 9) | // NS bit
1357 (sh << 7);
1358}
1359
1360void
1361TableWalker::doL1Descriptor()
1362{
1363 if (currState->fault != NoFault) {
1364 return;
1365 }
1366
1367 DPRINTF(TLB, "L1 descriptor for %#x is %#x\n",
1368 currState->vaddr_tainted, currState->l1Desc.data);
1369 TlbEntry te;
1370
1371 switch (currState->l1Desc.type()) {
1372 case L1Descriptor::Ignore:
1373 case L1Descriptor::Reserved:
1374 if (!currState->timing) {
1375 currState->tc = NULL;
1376 currState->req = NULL;
1377 }
1378 DPRINTF(TLB, "L1 Descriptor Reserved/Ignore, causing fault\n");
1379 if (currState->isFetch)
1380 currState->fault =
1381 std::make_shared<PrefetchAbort>(
1382 currState->vaddr_tainted,
1383 ArmFault::TranslationLL + L1,
1384 isStage2,
1385 ArmFault::VmsaTran);
1386 else
1387 currState->fault =
1388 std::make_shared<DataAbort>(
1389 currState->vaddr_tainted,
1390 TlbEntry::DomainType::NoAccess,
1391 currState->isWrite,
1392 ArmFault::TranslationLL + L1, isStage2,
1393 ArmFault::VmsaTran);
1394 return;
1395 case L1Descriptor::Section:
1396 if (currState->sctlr.afe && bits(currState->l1Desc.ap(), 0) == 0) {
1397 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is
1398 * enabled if set, do l1.Desc.setAp0() instead of generating
1399 * AccessFlag0
1400 */
1401
1402 currState->fault = std::make_shared<DataAbort>(
1403 currState->vaddr_tainted,
1404 currState->l1Desc.domain(),
1405 currState->isWrite,
1406 ArmFault::AccessFlagLL + L1,
1407 isStage2,
1408 ArmFault::VmsaTran);
1409 }
1410 if (currState->l1Desc.supersection()) {
1411 panic("Haven't implemented supersections\n");
1412 }
1413 insertTableEntry(currState->l1Desc, false);
1414 return;
1415 case L1Descriptor::PageTable:
1416 {
1417 Addr l2desc_addr;
1418 l2desc_addr = currState->l1Desc.l2Addr() |
1419 (bits(currState->vaddr, 19, 12) << 2);
1420 DPRINTF(TLB, "L1 descriptor points to page table at: %#x (%s)\n",
1421 l2desc_addr, currState->isSecure ? "s" : "ns");
1422
1423 // Trickbox address check
1424 currState->fault = tlb->walkTrickBoxCheck(
1425 l2desc_addr, currState->isSecure, currState->vaddr,
1426 sizeof(uint32_t), currState->isFetch, currState->isWrite,
1427 currState->l1Desc.domain(), L2);
1428
1429 if (currState->fault) {
1430 if (!currState->timing) {
1431 currState->tc = NULL;
1432 currState->req = NULL;
1433 }
1434 return;
1435 }
1436
1437 Request::Flags flag = 0;
1438 if (currState->isSecure)
1439 flag.set(Request::SECURE);
1440
1441 bool delayed;
1442 delayed = fetchDescriptor(l2desc_addr,
1443 (uint8_t*)&currState->l2Desc.data,
1444 sizeof(uint32_t), flag, -1, &doL2DescEvent,
1445 &TableWalker::doL2Descriptor);
1446 if (delayed) {
1447 currState->delayed = true;
1448 }
1449
1450 return;
1451 }
1452 default:
1453 panic("A new type in a 2 bit field?\n");
1454 }
1455}
1456
1457void
1458TableWalker::doLongDescriptor()
1459{
1460 if (currState->fault != NoFault) {
1461 return;
1462 }
1463
1464 DPRINTF(TLB, "L%d descriptor for %#llx is %#llx (%s)\n",
1465 currState->longDesc.lookupLevel, currState->vaddr_tainted,
1466 currState->longDesc.data,
1467 currState->aarch64 ? "AArch64" : "long-desc.");
1468
1469 if ((currState->longDesc.type() == LongDescriptor::Block) ||
1470 (currState->longDesc.type() == LongDescriptor::Page)) {
1471 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, pxn: %d, "
1472 "xn: %d, ap: %d, af: %d, type: %d\n",
1473 currState->longDesc.lookupLevel,
1474 currState->longDesc.data,
1475 currState->longDesc.pxn(),
1476 currState->longDesc.xn(),
1477 currState->longDesc.ap(),
1478 currState->longDesc.af(),
1479 currState->longDesc.type());
1480 } else {
1481 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, type: %d\n",
1482 currState->longDesc.lookupLevel,
1483 currState->longDesc.data,
1484 currState->longDesc.type());
1485 }
1486
1487 TlbEntry te;
1488
1489 switch (currState->longDesc.type()) {
1490 case LongDescriptor::Invalid:
1491 if (!currState->timing) {
1492 currState->tc = NULL;
1493 currState->req = NULL;
1494 }
1495
1496 DPRINTF(TLB, "L%d descriptor Invalid, causing fault type %d\n",
1497 currState->longDesc.lookupLevel,
1498 ArmFault::TranslationLL + currState->longDesc.lookupLevel);
1499 if (currState->isFetch)
1500 currState->fault = std::make_shared<PrefetchAbort>(
1501 currState->vaddr_tainted,
1502 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1503 isStage2,
1504 ArmFault::LpaeTran);
1505 else
1506 currState->fault = std::make_shared<DataAbort>(
1507 currState->vaddr_tainted,
1508 TlbEntry::DomainType::NoAccess,
1509 currState->isWrite,
1510 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1511 isStage2,
1512 ArmFault::LpaeTran);
1513 return;
1514 case LongDescriptor::Block:
1515 case LongDescriptor::Page:
1516 {
1517 bool fault = false;
1518 bool aff = false;
1519 // Check for address size fault
1520 if (checkAddrSizeFaultAArch64(
1521 mbits(currState->longDesc.data, MaxPhysAddrRange - 1,
1522 currState->longDesc.offsetBits()),
1523 currState->physAddrRange)) {
1524 fault = true;
1525 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1526 currState->longDesc.lookupLevel);
1527 // Check for access fault
1528 } else if (currState->longDesc.af() == 0) {
1529 fault = true;
1530 DPRINTF(TLB, "L%d descriptor causing Access Fault\n",
1531 currState->longDesc.lookupLevel);
1532 aff = true;
1533 }
1534 if (fault) {
1535 if (currState->isFetch)
1536 currState->fault = std::make_shared<PrefetchAbort>(
1537 currState->vaddr_tainted,
1538 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1539 currState->longDesc.lookupLevel,
1540 isStage2,
1541 ArmFault::LpaeTran);
1542 else
1543 currState->fault = std::make_shared<DataAbort>(
1544 currState->vaddr_tainted,
1545 TlbEntry::DomainType::NoAccess, currState->isWrite,
1546 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1547 currState->longDesc.lookupLevel,
1548 isStage2,
1549 ArmFault::LpaeTran);
1550 } else {
1551 insertTableEntry(currState->longDesc, true);
1552 }
1553 }
1554 return;
1555 case LongDescriptor::Table:
1556 {
1557 // Set hierarchical permission flags
1558 currState->secureLookup = currState->secureLookup &&
1559 currState->longDesc.secureTable();
1560 currState->rwTable = currState->rwTable &&
1561 currState->longDesc.rwTable();
1562 currState->userTable = currState->userTable &&
1563 currState->longDesc.userTable();
1564 currState->xnTable = currState->xnTable ||
1565 currState->longDesc.xnTable();
1566 currState->pxnTable = currState->pxnTable ||
1567 currState->longDesc.pxnTable();
1568
1569 // Set up next level lookup
1570 Addr next_desc_addr = currState->longDesc.nextDescAddr(
1571 currState->vaddr);
1572
1573 DPRINTF(TLB, "L%d descriptor points to L%d descriptor at: %#x (%s)\n",
1574 currState->longDesc.lookupLevel,
1575 currState->longDesc.lookupLevel + 1,
1576 next_desc_addr,
1577 currState->secureLookup ? "s" : "ns");
1578
1579 // Check for address size fault
1580 if (currState->aarch64 && checkAddrSizeFaultAArch64(
1581 next_desc_addr, currState->physAddrRange)) {
1582 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1583 currState->longDesc.lookupLevel);
1584 if (currState->isFetch)
1585 currState->fault = std::make_shared<PrefetchAbort>(
1586 currState->vaddr_tainted,
1587 ArmFault::AddressSizeLL
1588 + currState->longDesc.lookupLevel,
1589 isStage2,
1590 ArmFault::LpaeTran);
1591 else
1592 currState->fault = std::make_shared<DataAbort>(
1593 currState->vaddr_tainted,
1594 TlbEntry::DomainType::NoAccess, currState->isWrite,
1595 ArmFault::AddressSizeLL
1596 + currState->longDesc.lookupLevel,
1597 isStage2,
1598 ArmFault::LpaeTran);
1599 return;
1600 }
1601
1602 // Trickbox address check
1603 currState->fault = tlb->walkTrickBoxCheck(
1604 next_desc_addr, currState->vaddr,
1605 currState->vaddr, sizeof(uint64_t),
1606 currState->isFetch, currState->isWrite,
1607 TlbEntry::DomainType::Client,
1608 toLookupLevel(currState->longDesc.lookupLevel +1));
1609
1610 if (currState->fault) {
1611 if (!currState->timing) {
1612 currState->tc = NULL;
1613 currState->req = NULL;
1614 }
1615 return;
1616 }
1617
1618 Request::Flags flag = 0;
1619 if (currState->secureLookup)
1620 flag.set(Request::SECURE);
1621
1622 currState->longDesc.lookupLevel =
1623 (LookupLevel) (currState->longDesc.lookupLevel + 1);
1624 Event *event = NULL;
1625 switch (currState->longDesc.lookupLevel) {
1626 case L1:
1627 assert(currState->aarch64);
1628 event = &doL1LongDescEvent;
1629 break;
1630 case L2:
1631 event = &doL2LongDescEvent;
1632 break;
1633 case L3:
1634 event = &doL3LongDescEvent;
1635 break;
1636 default:
1637 panic("Wrong lookup level in table walk\n");
1638 break;
1639 }
1640
1641 bool delayed;
1642 delayed = fetchDescriptor(next_desc_addr, (uint8_t*)&currState->longDesc.data,
1643 sizeof(uint64_t), flag, -1, event,
1644 &TableWalker::doLongDescriptor);
1645 if (delayed) {
1646 currState->delayed = true;
1647 }
1648 }
1649 return;
1650 default:
1651 panic("A new type in a 2 bit field?\n");
1652 }
1653}
1654
1655void
1656TableWalker::doL2Descriptor()
1657{
1658 if (currState->fault != NoFault) {
1659 return;
1660 }
1661
1662 DPRINTF(TLB, "L2 descriptor for %#x is %#x\n",
1663 currState->vaddr_tainted, currState->l2Desc.data);
1664 TlbEntry te;
1665
1666 if (currState->l2Desc.invalid()) {
1667 DPRINTF(TLB, "L2 descriptor invalid, causing fault\n");
1668 if (!currState->timing) {
1669 currState->tc = NULL;
1670 currState->req = NULL;
1671 }
1672 if (currState->isFetch)
1673 currState->fault = std::make_shared<PrefetchAbort>(
1674 currState->vaddr_tainted,
1675 ArmFault::TranslationLL + L2,
1676 isStage2,
1677 ArmFault::VmsaTran);
1678 else
1679 currState->fault = std::make_shared<DataAbort>(
1680 currState->vaddr_tainted, currState->l1Desc.domain(),
1681 currState->isWrite, ArmFault::TranslationLL + L2,
1682 isStage2,
1683 ArmFault::VmsaTran);
1684 return;
1685 }
1686
1687 if (currState->sctlr.afe && bits(currState->l2Desc.ap(), 0) == 0) {
1688 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is enabled
1689 * if set, do l2.Desc.setAp0() instead of generating AccessFlag0
1690 */
1691 DPRINTF(TLB, "Generating access fault at L2, afe: %d, ap: %d\n",
1692 currState->sctlr.afe, currState->l2Desc.ap());
1693
1694 currState->fault = std::make_shared<DataAbort>(
1695 currState->vaddr_tainted,
1696 TlbEntry::DomainType::NoAccess, currState->isWrite,
1697 ArmFault::AccessFlagLL + L2, isStage2,
1698 ArmFault::VmsaTran);
1699 }
1700
1701 insertTableEntry(currState->l2Desc, false);
1702}
1703
1704void
1705TableWalker::doL1DescriptorWrapper()
1706{
1707 currState = stateQueues[L1].front();
1708 currState->delayed = false;
1709 // if there's a stage2 translation object we don't need it any more
1710 if (currState->stage2Tran) {
1711 delete currState->stage2Tran;
1712 currState->stage2Tran = NULL;
1713 }
1714
1715
1716 DPRINTF(TLBVerbose, "L1 Desc object host addr: %p\n",&currState->l1Desc.data);
1717 DPRINTF(TLBVerbose, "L1 Desc object data: %08x\n",currState->l1Desc.data);
1718
1719 DPRINTF(TLBVerbose, "calling doL1Descriptor for vaddr:%#x\n", currState->vaddr_tainted);
1720 doL1Descriptor();
1721
1722 stateQueues[L1].pop_front();
1723 // Check if fault was generated
1724 if (currState->fault != NoFault) {
1725 currState->transState->finish(currState->fault, currState->req,
1726 currState->tc, currState->mode);
1727 statWalksShortTerminatedAtLevel[0]++;
1728
1729 pending = false;
1730 nextWalk(currState->tc);
1731
1732 currState->req = NULL;
1733 currState->tc = NULL;
1734 currState->delayed = false;
1735 delete currState;
1736 }
1737 else if (!currState->delayed) {
1738 // delay is not set so there is no L2 to do
1739 // Don't finish the translation if a stage 2 look up is underway
1740 if (!currState->doingStage2) {
1741 statWalkServiceTime.sample(curTick() - currState->startTime);
1742 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1743 currState->fault = tlb->translateTiming(currState->req, currState->tc,
1744 currState->transState, currState->mode);
1745 statWalksShortTerminatedAtLevel[0]++;
1746 }
1747
1748 pending = false;
1749 nextWalk(currState->tc);
1750
1751 currState->req = NULL;
1752 currState->tc = NULL;
1753 currState->delayed = false;
1754 delete currState;
1755 } else {
1756 // need to do L2 descriptor
1757 stateQueues[L2].push_back(currState);
1758 }
1759 currState = NULL;
1760}
1761
1762void
1763TableWalker::doL2DescriptorWrapper()
1764{
1765 currState = stateQueues[L2].front();
1766 assert(currState->delayed);
1767 // if there's a stage2 translation object we don't need it any more
1768 if (currState->stage2Tran) {
1769 delete currState->stage2Tran;
1770 currState->stage2Tran = NULL;
1771 }
1772
1773 DPRINTF(TLBVerbose, "calling doL2Descriptor for vaddr:%#x\n",
1774 currState->vaddr_tainted);
1775 doL2Descriptor();
1776
1777 // Check if fault was generated
1778 if (currState->fault != NoFault) {
1779 currState->transState->finish(currState->fault, currState->req,
1780 currState->tc, currState->mode);
1781 statWalksShortTerminatedAtLevel[1]++;
1782 }
1783 else {
1784 // Don't finish the translation if a stage 2 look up is underway
1785 if (!currState->doingStage2) {
1786 statWalkServiceTime.sample(curTick() - currState->startTime);
1787 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1788 currState->fault = tlb->translateTiming(currState->req,
1789 currState->tc, currState->transState, currState->mode);
1790 statWalksShortTerminatedAtLevel[1]++;
1791 }
1792 }
1793
1794
1795 stateQueues[L2].pop_front();
1796 pending = false;
1797 nextWalk(currState->tc);
1798
1799 currState->req = NULL;
1800 currState->tc = NULL;
1801 currState->delayed = false;
1802
1803 delete currState;
1804 currState = NULL;
1805}
1806
1807void
1808TableWalker::doL0LongDescriptorWrapper()
1809{
1810 doLongDescriptorWrapper(L0);
1811}
1812
1813void
1814TableWalker::doL1LongDescriptorWrapper()
1815{
1816 doLongDescriptorWrapper(L1);
1817}
1818
1819void
1820TableWalker::doL2LongDescriptorWrapper()
1821{
1822 doLongDescriptorWrapper(L2);
1823}
1824
1825void
1826TableWalker::doL3LongDescriptorWrapper()
1827{
1828 doLongDescriptorWrapper(L3);
1829}
1830
1831void
1832TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level)
1833{
1834 currState = stateQueues[curr_lookup_level].front();
1835 assert(curr_lookup_level == currState->longDesc.lookupLevel);
1836 currState->delayed = false;
1837
1838 // if there's a stage2 translation object we don't need it any more
1839 if (currState->stage2Tran) {
1840 delete currState->stage2Tran;
1841 currState->stage2Tran = NULL;
1842 }
1843
1844 DPRINTF(TLBVerbose, "calling doLongDescriptor for vaddr:%#x\n",
1845 currState->vaddr_tainted);
1846 doLongDescriptor();
1847
1848 stateQueues[curr_lookup_level].pop_front();
1849
1850 if (currState->fault != NoFault) {
1851 // A fault was generated
1852 currState->transState->finish(currState->fault, currState->req,
1853 currState->tc, currState->mode);
1854
1855 pending = false;
1856 nextWalk(currState->tc);
1857
1858 currState->req = NULL;
1859 currState->tc = NULL;
1860 currState->delayed = false;
1861 delete currState;
1862 } else if (!currState->delayed) {
1863 // No additional lookups required
1864 // Don't finish the translation if a stage 2 look up is underway
1865 if (!currState->doingStage2) {
1866 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1867 statWalkServiceTime.sample(curTick() - currState->startTime);
1868 currState->fault = tlb->translateTiming(currState->req, currState->tc,
1869 currState->transState,
1870 currState->mode);
1871 statWalksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++;
1872 }
1873
1874 pending = false;
1875 nextWalk(currState->tc);
1876
1877 currState->req = NULL;
1878 currState->tc = NULL;
1879 currState->delayed = false;
1880 delete currState;
1881 } else {
1882 if (curr_lookup_level >= MAX_LOOKUP_LEVELS - 1)
1883 panic("Max. number of lookups already reached in table walk\n");
1884 // Need to perform additional lookups
1885 stateQueues[currState->longDesc.lookupLevel].push_back(currState);
1886 }
1887 currState = NULL;
1888}
1889
1890
1891void
1892TableWalker::nextWalk(ThreadContext *tc)
1893{
1894 if (pendingQueue.size())
1895 schedule(doProcessEvent, clockEdge(Cycles(1)));
1896 else
1897 completeDrain();
1898}
1899
1900bool
1901TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes,
1902 Request::Flags flags, int queueIndex, Event *event,
1903 void (TableWalker::*doDescriptor)())
1904{
1905 bool isTiming = currState->timing;
1906
1907 // do the requests for the page table descriptors have to go through the
1908 // second stage MMU
1909 if (currState->stage2Req) {
1910 Fault fault;
1911 flags = flags | TLB::MustBeOne;
1912
1913 if (isTiming) {
1914 Stage2MMU::Stage2Translation *tran = new
1915 Stage2MMU::Stage2Translation(*stage2Mmu, data, event,
1916 currState->vaddr);
1917 currState->stage2Tran = tran;
1918 stage2Mmu->readDataTimed(currState->tc, descAddr, tran, numBytes,
| 987 doLongDescriptor();
988 delete req;
989 delete pkt;
990 f = currState->fault;
991 }
992
993 return f;
994}
995
996void
997TableWalker::memAttrs(ThreadContext *tc, TlbEntry &te, SCTLR sctlr,
998 uint8_t texcb, bool s)
999{
1000 // Note: tc and sctlr local variables are hiding tc and sctrl class
1001 // variables
1002 DPRINTF(TLBVerbose, "memAttrs texcb:%d s:%d\n", texcb, s);
1003 te.shareable = false; // default value
1004 te.nonCacheable = false;
1005 te.outerShareable = false;
1006 if (sctlr.tre == 0 || ((sctlr.tre == 1) && (sctlr.m == 0))) {
1007 switch(texcb) {
1008 case 0: // Stongly-ordered
1009 te.nonCacheable = true;
1010 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1011 te.shareable = true;
1012 te.innerAttrs = 1;
1013 te.outerAttrs = 0;
1014 break;
1015 case 1: // Shareable Device
1016 te.nonCacheable = true;
1017 te.mtype = TlbEntry::MemoryType::Device;
1018 te.shareable = true;
1019 te.innerAttrs = 3;
1020 te.outerAttrs = 0;
1021 break;
1022 case 2: // Outer and Inner Write-Through, no Write-Allocate
1023 te.mtype = TlbEntry::MemoryType::Normal;
1024 te.shareable = s;
1025 te.innerAttrs = 6;
1026 te.outerAttrs = bits(texcb, 1, 0);
1027 break;
1028 case 3: // Outer and Inner Write-Back, no Write-Allocate
1029 te.mtype = TlbEntry::MemoryType::Normal;
1030 te.shareable = s;
1031 te.innerAttrs = 7;
1032 te.outerAttrs = bits(texcb, 1, 0);
1033 break;
1034 case 4: // Outer and Inner Non-cacheable
1035 te.nonCacheable = true;
1036 te.mtype = TlbEntry::MemoryType::Normal;
1037 te.shareable = s;
1038 te.innerAttrs = 0;
1039 te.outerAttrs = bits(texcb, 1, 0);
1040 break;
1041 case 5: // Reserved
1042 panic("Reserved texcb value!\n");
1043 break;
1044 case 6: // Implementation Defined
1045 panic("Implementation-defined texcb value!\n");
1046 break;
1047 case 7: // Outer and Inner Write-Back, Write-Allocate
1048 te.mtype = TlbEntry::MemoryType::Normal;
1049 te.shareable = s;
1050 te.innerAttrs = 5;
1051 te.outerAttrs = 1;
1052 break;
1053 case 8: // Non-shareable Device
1054 te.nonCacheable = true;
1055 te.mtype = TlbEntry::MemoryType::Device;
1056 te.shareable = false;
1057 te.innerAttrs = 3;
1058 te.outerAttrs = 0;
1059 break;
1060 case 9 ... 15: // Reserved
1061 panic("Reserved texcb value!\n");
1062 break;
1063 case 16 ... 31: // Cacheable Memory
1064 te.mtype = TlbEntry::MemoryType::Normal;
1065 te.shareable = s;
1066 if (bits(texcb, 1,0) == 0 || bits(texcb, 3,2) == 0)
1067 te.nonCacheable = true;
1068 te.innerAttrs = bits(texcb, 1, 0);
1069 te.outerAttrs = bits(texcb, 3, 2);
1070 break;
1071 default:
1072 panic("More than 32 states for 5 bits?\n");
1073 }
1074 } else {
1075 assert(tc);
1076 PRRR prrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_PRRR,
1077 currState->tc, !currState->isSecure));
1078 NMRR nmrr = tc->readMiscReg(flattenMiscRegNsBanked(MISCREG_NMRR,
1079 currState->tc, !currState->isSecure));
1080 DPRINTF(TLBVerbose, "memAttrs PRRR:%08x NMRR:%08x\n", prrr, nmrr);
1081 uint8_t curr_tr = 0, curr_ir = 0, curr_or = 0;
1082 switch(bits(texcb, 2,0)) {
1083 case 0:
1084 curr_tr = prrr.tr0;
1085 curr_ir = nmrr.ir0;
1086 curr_or = nmrr.or0;
1087 te.outerShareable = (prrr.nos0 == 0);
1088 break;
1089 case 1:
1090 curr_tr = prrr.tr1;
1091 curr_ir = nmrr.ir1;
1092 curr_or = nmrr.or1;
1093 te.outerShareable = (prrr.nos1 == 0);
1094 break;
1095 case 2:
1096 curr_tr = prrr.tr2;
1097 curr_ir = nmrr.ir2;
1098 curr_or = nmrr.or2;
1099 te.outerShareable = (prrr.nos2 == 0);
1100 break;
1101 case 3:
1102 curr_tr = prrr.tr3;
1103 curr_ir = nmrr.ir3;
1104 curr_or = nmrr.or3;
1105 te.outerShareable = (prrr.nos3 == 0);
1106 break;
1107 case 4:
1108 curr_tr = prrr.tr4;
1109 curr_ir = nmrr.ir4;
1110 curr_or = nmrr.or4;
1111 te.outerShareable = (prrr.nos4 == 0);
1112 break;
1113 case 5:
1114 curr_tr = prrr.tr5;
1115 curr_ir = nmrr.ir5;
1116 curr_or = nmrr.or5;
1117 te.outerShareable = (prrr.nos5 == 0);
1118 break;
1119 case 6:
1120 panic("Imp defined type\n");
1121 case 7:
1122 curr_tr = prrr.tr7;
1123 curr_ir = nmrr.ir7;
1124 curr_or = nmrr.or7;
1125 te.outerShareable = (prrr.nos7 == 0);
1126 break;
1127 }
1128
1129 switch(curr_tr) {
1130 case 0:
1131 DPRINTF(TLBVerbose, "StronglyOrdered\n");
1132 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1133 te.nonCacheable = true;
1134 te.innerAttrs = 1;
1135 te.outerAttrs = 0;
1136 te.shareable = true;
1137 break;
1138 case 1:
1139 DPRINTF(TLBVerbose, "Device ds1:%d ds0:%d s:%d\n",
1140 prrr.ds1, prrr.ds0, s);
1141 te.mtype = TlbEntry::MemoryType::Device;
1142 te.nonCacheable = true;
1143 te.innerAttrs = 3;
1144 te.outerAttrs = 0;
1145 if (prrr.ds1 && s)
1146 te.shareable = true;
1147 if (prrr.ds0 && !s)
1148 te.shareable = true;
1149 break;
1150 case 2:
1151 DPRINTF(TLBVerbose, "Normal ns1:%d ns0:%d s:%d\n",
1152 prrr.ns1, prrr.ns0, s);
1153 te.mtype = TlbEntry::MemoryType::Normal;
1154 if (prrr.ns1 && s)
1155 te.shareable = true;
1156 if (prrr.ns0 && !s)
1157 te.shareable = true;
1158 break;
1159 case 3:
1160 panic("Reserved type");
1161 }
1162
1163 if (te.mtype == TlbEntry::MemoryType::Normal){
1164 switch(curr_ir) {
1165 case 0:
1166 te.nonCacheable = true;
1167 te.innerAttrs = 0;
1168 break;
1169 case 1:
1170 te.innerAttrs = 5;
1171 break;
1172 case 2:
1173 te.innerAttrs = 6;
1174 break;
1175 case 3:
1176 te.innerAttrs = 7;
1177 break;
1178 }
1179
1180 switch(curr_or) {
1181 case 0:
1182 te.nonCacheable = true;
1183 te.outerAttrs = 0;
1184 break;
1185 case 1:
1186 te.outerAttrs = 1;
1187 break;
1188 case 2:
1189 te.outerAttrs = 2;
1190 break;
1191 case 3:
1192 te.outerAttrs = 3;
1193 break;
1194 }
1195 }
1196 }
1197 DPRINTF(TLBVerbose, "memAttrs: shareable: %d, innerAttrs: %d, "
1198 "outerAttrs: %d\n",
1199 te.shareable, te.innerAttrs, te.outerAttrs);
1200 te.setAttributes(false);
1201}
1202
1203void
1204TableWalker::memAttrsLPAE(ThreadContext *tc, TlbEntry &te,
1205 LongDescriptor &lDescriptor)
1206{
1207 assert(_haveLPAE);
1208
1209 uint8_t attr;
1210 uint8_t sh = lDescriptor.sh();
1211 // Different format and source of attributes if this is a stage 2
1212 // translation
1213 if (isStage2) {
1214 attr = lDescriptor.memAttr();
1215 uint8_t attr_3_2 = (attr >> 2) & 0x3;
1216 uint8_t attr_1_0 = attr & 0x3;
1217
1218 DPRINTF(TLBVerbose, "memAttrsLPAE MemAttr:%#x sh:%#x\n", attr, sh);
1219
1220 if (attr_3_2 == 0) {
1221 te.mtype = attr_1_0 == 0 ? TlbEntry::MemoryType::StronglyOrdered
1222 : TlbEntry::MemoryType::Device;
1223 te.outerAttrs = 0;
1224 te.innerAttrs = attr_1_0 == 0 ? 1 : 3;
1225 te.nonCacheable = true;
1226 } else {
1227 te.mtype = TlbEntry::MemoryType::Normal;
1228 te.outerAttrs = attr_3_2 == 1 ? 0 :
1229 attr_3_2 == 2 ? 2 : 1;
1230 te.innerAttrs = attr_1_0 == 1 ? 0 :
1231 attr_1_0 == 2 ? 6 : 5;
1232 te.nonCacheable = (attr_3_2 == 1) || (attr_1_0 == 1);
1233 }
1234 } else {
1235 uint8_t attrIndx = lDescriptor.attrIndx();
1236
1237 // LPAE always uses remapping of memory attributes, irrespective of the
1238 // value of SCTLR.TRE
1239 MiscRegIndex reg = attrIndx & 0x4 ? MISCREG_MAIR1 : MISCREG_MAIR0;
1240 int reg_as_int = flattenMiscRegNsBanked(reg, currState->tc,
1241 !currState->isSecure);
1242 uint32_t mair = currState->tc->readMiscReg(reg_as_int);
1243 attr = (mair >> (8 * (attrIndx % 4))) & 0xff;
1244 uint8_t attr_7_4 = bits(attr, 7, 4);
1245 uint8_t attr_3_0 = bits(attr, 3, 0);
1246 DPRINTF(TLBVerbose, "memAttrsLPAE AttrIndx:%#x sh:%#x, attr %#x\n", attrIndx, sh, attr);
1247
1248 // Note: the memory subsystem only cares about the 'cacheable' memory
1249 // attribute. The other attributes are only used to fill the PAR register
1250 // accordingly to provide the illusion of full support
1251 te.nonCacheable = false;
1252
1253 switch (attr_7_4) {
1254 case 0x0:
1255 // Strongly-ordered or Device memory
1256 if (attr_3_0 == 0x0)
1257 te.mtype = TlbEntry::MemoryType::StronglyOrdered;
1258 else if (attr_3_0 == 0x4)
1259 te.mtype = TlbEntry::MemoryType::Device;
1260 else
1261 panic("Unpredictable behavior\n");
1262 te.nonCacheable = true;
1263 te.outerAttrs = 0;
1264 break;
1265 case 0x4:
1266 // Normal memory, Outer Non-cacheable
1267 te.mtype = TlbEntry::MemoryType::Normal;
1268 te.outerAttrs = 0;
1269 if (attr_3_0 == 0x4)
1270 // Inner Non-cacheable
1271 te.nonCacheable = true;
1272 else if (attr_3_0 < 0x8)
1273 panic("Unpredictable behavior\n");
1274 break;
1275 case 0x8:
1276 case 0x9:
1277 case 0xa:
1278 case 0xb:
1279 case 0xc:
1280 case 0xd:
1281 case 0xe:
1282 case 0xf:
1283 if (attr_7_4 & 0x4) {
1284 te.outerAttrs = (attr_7_4 & 1) ? 1 : 3;
1285 } else {
1286 te.outerAttrs = 0x2;
1287 }
1288 // Normal memory, Outer Cacheable
1289 te.mtype = TlbEntry::MemoryType::Normal;
1290 if (attr_3_0 != 0x4 && attr_3_0 < 0x8)
1291 panic("Unpredictable behavior\n");
1292 break;
1293 default:
1294 panic("Unpredictable behavior\n");
1295 break;
1296 }
1297
1298 switch (attr_3_0) {
1299 case 0x0:
1300 te.innerAttrs = 0x1;
1301 break;
1302 case 0x4:
1303 te.innerAttrs = attr_7_4 == 0 ? 0x3 : 0;
1304 break;
1305 case 0x8:
1306 case 0x9:
1307 case 0xA:
1308 case 0xB:
1309 te.innerAttrs = 6;
1310 break;
1311 case 0xC:
1312 case 0xD:
1313 case 0xE:
1314 case 0xF:
1315 te.innerAttrs = attr_3_0 & 1 ? 0x5 : 0x7;
1316 break;
1317 default:
1318 panic("Unpredictable behavior\n");
1319 break;
1320 }
1321 }
1322
1323 te.outerShareable = sh == 2;
1324 te.shareable = (sh & 0x2) ? true : false;
1325 te.setAttributes(true);
1326 te.attributes |= (uint64_t) attr << 56;
1327}
1328
1329void
1330TableWalker::memAttrsAArch64(ThreadContext *tc, TlbEntry &te, uint8_t attrIndx,
1331 uint8_t sh)
1332{
1333 DPRINTF(TLBVerbose, "memAttrsAArch64 AttrIndx:%#x sh:%#x\n", attrIndx, sh);
1334
1335 // Select MAIR
1336 uint64_t mair;
1337 switch (currState->el) {
1338 case EL0:
1339 case EL1:
1340 mair = tc->readMiscReg(MISCREG_MAIR_EL1);
1341 break;
1342 case EL2:
1343 mair = tc->readMiscReg(MISCREG_MAIR_EL2);
1344 break;
1345 case EL3:
1346 mair = tc->readMiscReg(MISCREG_MAIR_EL3);
1347 break;
1348 default:
1349 panic("Invalid exception level");
1350 break;
1351 }
1352
1353 // Select attributes
1354 uint8_t attr = bits(mair, 8 * attrIndx + 7, 8 * attrIndx);
1355 uint8_t attr_lo = bits(attr, 3, 0);
1356 uint8_t attr_hi = bits(attr, 7, 4);
1357
1358 // Memory type
1359 te.mtype = attr_hi == 0 ? TlbEntry::MemoryType::Device : TlbEntry::MemoryType::Normal;
1360
1361 // Cacheability
1362 te.nonCacheable = false;
1363 if (te.mtype == TlbEntry::MemoryType::Device || // Device memory
1364 attr_hi == 0x8 || // Normal memory, Outer Non-cacheable
1365 attr_lo == 0x8) { // Normal memory, Inner Non-cacheable
1366 te.nonCacheable = true;
1367 }
1368
1369 te.shareable = sh == 2;
1370 te.outerShareable = (sh & 0x2) ? true : false;
1371 // Attributes formatted according to the 64-bit PAR
1372 te.attributes = ((uint64_t) attr << 56) |
1373 (1 << 11) | // LPAE bit
1374 (te.ns << 9) | // NS bit
1375 (sh << 7);
1376}
1377
1378void
1379TableWalker::doL1Descriptor()
1380{
1381 if (currState->fault != NoFault) {
1382 return;
1383 }
1384
1385 DPRINTF(TLB, "L1 descriptor for %#x is %#x\n",
1386 currState->vaddr_tainted, currState->l1Desc.data);
1387 TlbEntry te;
1388
1389 switch (currState->l1Desc.type()) {
1390 case L1Descriptor::Ignore:
1391 case L1Descriptor::Reserved:
1392 if (!currState->timing) {
1393 currState->tc = NULL;
1394 currState->req = NULL;
1395 }
1396 DPRINTF(TLB, "L1 Descriptor Reserved/Ignore, causing fault\n");
1397 if (currState->isFetch)
1398 currState->fault =
1399 std::make_shared<PrefetchAbort>(
1400 currState->vaddr_tainted,
1401 ArmFault::TranslationLL + L1,
1402 isStage2,
1403 ArmFault::VmsaTran);
1404 else
1405 currState->fault =
1406 std::make_shared<DataAbort>(
1407 currState->vaddr_tainted,
1408 TlbEntry::DomainType::NoAccess,
1409 currState->isWrite,
1410 ArmFault::TranslationLL + L1, isStage2,
1411 ArmFault::VmsaTran);
1412 return;
1413 case L1Descriptor::Section:
1414 if (currState->sctlr.afe && bits(currState->l1Desc.ap(), 0) == 0) {
1415 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is
1416 * enabled if set, do l1.Desc.setAp0() instead of generating
1417 * AccessFlag0
1418 */
1419
1420 currState->fault = std::make_shared<DataAbort>(
1421 currState->vaddr_tainted,
1422 currState->l1Desc.domain(),
1423 currState->isWrite,
1424 ArmFault::AccessFlagLL + L1,
1425 isStage2,
1426 ArmFault::VmsaTran);
1427 }
1428 if (currState->l1Desc.supersection()) {
1429 panic("Haven't implemented supersections\n");
1430 }
1431 insertTableEntry(currState->l1Desc, false);
1432 return;
1433 case L1Descriptor::PageTable:
1434 {
1435 Addr l2desc_addr;
1436 l2desc_addr = currState->l1Desc.l2Addr() |
1437 (bits(currState->vaddr, 19, 12) << 2);
1438 DPRINTF(TLB, "L1 descriptor points to page table at: %#x (%s)\n",
1439 l2desc_addr, currState->isSecure ? "s" : "ns");
1440
1441 // Trickbox address check
1442 currState->fault = tlb->walkTrickBoxCheck(
1443 l2desc_addr, currState->isSecure, currState->vaddr,
1444 sizeof(uint32_t), currState->isFetch, currState->isWrite,
1445 currState->l1Desc.domain(), L2);
1446
1447 if (currState->fault) {
1448 if (!currState->timing) {
1449 currState->tc = NULL;
1450 currState->req = NULL;
1451 }
1452 return;
1453 }
1454
1455 Request::Flags flag = 0;
1456 if (currState->isSecure)
1457 flag.set(Request::SECURE);
1458
1459 bool delayed;
1460 delayed = fetchDescriptor(l2desc_addr,
1461 (uint8_t*)&currState->l2Desc.data,
1462 sizeof(uint32_t), flag, -1, &doL2DescEvent,
1463 &TableWalker::doL2Descriptor);
1464 if (delayed) {
1465 currState->delayed = true;
1466 }
1467
1468 return;
1469 }
1470 default:
1471 panic("A new type in a 2 bit field?\n");
1472 }
1473}
1474
1475void
1476TableWalker::doLongDescriptor()
1477{
1478 if (currState->fault != NoFault) {
1479 return;
1480 }
1481
1482 DPRINTF(TLB, "L%d descriptor for %#llx is %#llx (%s)\n",
1483 currState->longDesc.lookupLevel, currState->vaddr_tainted,
1484 currState->longDesc.data,
1485 currState->aarch64 ? "AArch64" : "long-desc.");
1486
1487 if ((currState->longDesc.type() == LongDescriptor::Block) ||
1488 (currState->longDesc.type() == LongDescriptor::Page)) {
1489 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, pxn: %d, "
1490 "xn: %d, ap: %d, af: %d, type: %d\n",
1491 currState->longDesc.lookupLevel,
1492 currState->longDesc.data,
1493 currState->longDesc.pxn(),
1494 currState->longDesc.xn(),
1495 currState->longDesc.ap(),
1496 currState->longDesc.af(),
1497 currState->longDesc.type());
1498 } else {
1499 DPRINTF(TLBVerbose, "Analyzing L%d descriptor: %#llx, type: %d\n",
1500 currState->longDesc.lookupLevel,
1501 currState->longDesc.data,
1502 currState->longDesc.type());
1503 }
1504
1505 TlbEntry te;
1506
1507 switch (currState->longDesc.type()) {
1508 case LongDescriptor::Invalid:
1509 if (!currState->timing) {
1510 currState->tc = NULL;
1511 currState->req = NULL;
1512 }
1513
1514 DPRINTF(TLB, "L%d descriptor Invalid, causing fault type %d\n",
1515 currState->longDesc.lookupLevel,
1516 ArmFault::TranslationLL + currState->longDesc.lookupLevel);
1517 if (currState->isFetch)
1518 currState->fault = std::make_shared<PrefetchAbort>(
1519 currState->vaddr_tainted,
1520 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1521 isStage2,
1522 ArmFault::LpaeTran);
1523 else
1524 currState->fault = std::make_shared<DataAbort>(
1525 currState->vaddr_tainted,
1526 TlbEntry::DomainType::NoAccess,
1527 currState->isWrite,
1528 ArmFault::TranslationLL + currState->longDesc.lookupLevel,
1529 isStage2,
1530 ArmFault::LpaeTran);
1531 return;
1532 case LongDescriptor::Block:
1533 case LongDescriptor::Page:
1534 {
1535 bool fault = false;
1536 bool aff = false;
1537 // Check for address size fault
1538 if (checkAddrSizeFaultAArch64(
1539 mbits(currState->longDesc.data, MaxPhysAddrRange - 1,
1540 currState->longDesc.offsetBits()),
1541 currState->physAddrRange)) {
1542 fault = true;
1543 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1544 currState->longDesc.lookupLevel);
1545 // Check for access fault
1546 } else if (currState->longDesc.af() == 0) {
1547 fault = true;
1548 DPRINTF(TLB, "L%d descriptor causing Access Fault\n",
1549 currState->longDesc.lookupLevel);
1550 aff = true;
1551 }
1552 if (fault) {
1553 if (currState->isFetch)
1554 currState->fault = std::make_shared<PrefetchAbort>(
1555 currState->vaddr_tainted,
1556 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1557 currState->longDesc.lookupLevel,
1558 isStage2,
1559 ArmFault::LpaeTran);
1560 else
1561 currState->fault = std::make_shared<DataAbort>(
1562 currState->vaddr_tainted,
1563 TlbEntry::DomainType::NoAccess, currState->isWrite,
1564 (aff ? ArmFault::AccessFlagLL : ArmFault::AddressSizeLL) +
1565 currState->longDesc.lookupLevel,
1566 isStage2,
1567 ArmFault::LpaeTran);
1568 } else {
1569 insertTableEntry(currState->longDesc, true);
1570 }
1571 }
1572 return;
1573 case LongDescriptor::Table:
1574 {
1575 // Set hierarchical permission flags
1576 currState->secureLookup = currState->secureLookup &&
1577 currState->longDesc.secureTable();
1578 currState->rwTable = currState->rwTable &&
1579 currState->longDesc.rwTable();
1580 currState->userTable = currState->userTable &&
1581 currState->longDesc.userTable();
1582 currState->xnTable = currState->xnTable ||
1583 currState->longDesc.xnTable();
1584 currState->pxnTable = currState->pxnTable ||
1585 currState->longDesc.pxnTable();
1586
1587 // Set up next level lookup
1588 Addr next_desc_addr = currState->longDesc.nextDescAddr(
1589 currState->vaddr);
1590
1591 DPRINTF(TLB, "L%d descriptor points to L%d descriptor at: %#x (%s)\n",
1592 currState->longDesc.lookupLevel,
1593 currState->longDesc.lookupLevel + 1,
1594 next_desc_addr,
1595 currState->secureLookup ? "s" : "ns");
1596
1597 // Check for address size fault
1598 if (currState->aarch64 && checkAddrSizeFaultAArch64(
1599 next_desc_addr, currState->physAddrRange)) {
1600 DPRINTF(TLB, "L%d descriptor causing Address Size Fault\n",
1601 currState->longDesc.lookupLevel);
1602 if (currState->isFetch)
1603 currState->fault = std::make_shared<PrefetchAbort>(
1604 currState->vaddr_tainted,
1605 ArmFault::AddressSizeLL
1606 + currState->longDesc.lookupLevel,
1607 isStage2,
1608 ArmFault::LpaeTran);
1609 else
1610 currState->fault = std::make_shared<DataAbort>(
1611 currState->vaddr_tainted,
1612 TlbEntry::DomainType::NoAccess, currState->isWrite,
1613 ArmFault::AddressSizeLL
1614 + currState->longDesc.lookupLevel,
1615 isStage2,
1616 ArmFault::LpaeTran);
1617 return;
1618 }
1619
1620 // Trickbox address check
1621 currState->fault = tlb->walkTrickBoxCheck(
1622 next_desc_addr, currState->vaddr,
1623 currState->vaddr, sizeof(uint64_t),
1624 currState->isFetch, currState->isWrite,
1625 TlbEntry::DomainType::Client,
1626 toLookupLevel(currState->longDesc.lookupLevel +1));
1627
1628 if (currState->fault) {
1629 if (!currState->timing) {
1630 currState->tc = NULL;
1631 currState->req = NULL;
1632 }
1633 return;
1634 }
1635
1636 Request::Flags flag = 0;
1637 if (currState->secureLookup)
1638 flag.set(Request::SECURE);
1639
1640 currState->longDesc.lookupLevel =
1641 (LookupLevel) (currState->longDesc.lookupLevel + 1);
1642 Event *event = NULL;
1643 switch (currState->longDesc.lookupLevel) {
1644 case L1:
1645 assert(currState->aarch64);
1646 event = &doL1LongDescEvent;
1647 break;
1648 case L2:
1649 event = &doL2LongDescEvent;
1650 break;
1651 case L3:
1652 event = &doL3LongDescEvent;
1653 break;
1654 default:
1655 panic("Wrong lookup level in table walk\n");
1656 break;
1657 }
1658
1659 bool delayed;
1660 delayed = fetchDescriptor(next_desc_addr, (uint8_t*)&currState->longDesc.data,
1661 sizeof(uint64_t), flag, -1, event,
1662 &TableWalker::doLongDescriptor);
1663 if (delayed) {
1664 currState->delayed = true;
1665 }
1666 }
1667 return;
1668 default:
1669 panic("A new type in a 2 bit field?\n");
1670 }
1671}
1672
1673void
1674TableWalker::doL2Descriptor()
1675{
1676 if (currState->fault != NoFault) {
1677 return;
1678 }
1679
1680 DPRINTF(TLB, "L2 descriptor for %#x is %#x\n",
1681 currState->vaddr_tainted, currState->l2Desc.data);
1682 TlbEntry te;
1683
1684 if (currState->l2Desc.invalid()) {
1685 DPRINTF(TLB, "L2 descriptor invalid, causing fault\n");
1686 if (!currState->timing) {
1687 currState->tc = NULL;
1688 currState->req = NULL;
1689 }
1690 if (currState->isFetch)
1691 currState->fault = std::make_shared<PrefetchAbort>(
1692 currState->vaddr_tainted,
1693 ArmFault::TranslationLL + L2,
1694 isStage2,
1695 ArmFault::VmsaTran);
1696 else
1697 currState->fault = std::make_shared<DataAbort>(
1698 currState->vaddr_tainted, currState->l1Desc.domain(),
1699 currState->isWrite, ArmFault::TranslationLL + L2,
1700 isStage2,
1701 ArmFault::VmsaTran);
1702 return;
1703 }
1704
1705 if (currState->sctlr.afe && bits(currState->l2Desc.ap(), 0) == 0) {
1706 /** @todo: check sctlr.ha (bit[17]) if Hardware Access Flag is enabled
1707 * if set, do l2.Desc.setAp0() instead of generating AccessFlag0
1708 */
1709 DPRINTF(TLB, "Generating access fault at L2, afe: %d, ap: %d\n",
1710 currState->sctlr.afe, currState->l2Desc.ap());
1711
1712 currState->fault = std::make_shared<DataAbort>(
1713 currState->vaddr_tainted,
1714 TlbEntry::DomainType::NoAccess, currState->isWrite,
1715 ArmFault::AccessFlagLL + L2, isStage2,
1716 ArmFault::VmsaTran);
1717 }
1718
1719 insertTableEntry(currState->l2Desc, false);
1720}
1721
1722void
1723TableWalker::doL1DescriptorWrapper()
1724{
1725 currState = stateQueues[L1].front();
1726 currState->delayed = false;
1727 // if there's a stage2 translation object we don't need it any more
1728 if (currState->stage2Tran) {
1729 delete currState->stage2Tran;
1730 currState->stage2Tran = NULL;
1731 }
1732
1733
1734 DPRINTF(TLBVerbose, "L1 Desc object host addr: %p\n",&currState->l1Desc.data);
1735 DPRINTF(TLBVerbose, "L1 Desc object data: %08x\n",currState->l1Desc.data);
1736
1737 DPRINTF(TLBVerbose, "calling doL1Descriptor for vaddr:%#x\n", currState->vaddr_tainted);
1738 doL1Descriptor();
1739
1740 stateQueues[L1].pop_front();
1741 // Check if fault was generated
1742 if (currState->fault != NoFault) {
1743 currState->transState->finish(currState->fault, currState->req,
1744 currState->tc, currState->mode);
1745 statWalksShortTerminatedAtLevel[0]++;
1746
1747 pending = false;
1748 nextWalk(currState->tc);
1749
1750 currState->req = NULL;
1751 currState->tc = NULL;
1752 currState->delayed = false;
1753 delete currState;
1754 }
1755 else if (!currState->delayed) {
1756 // delay is not set so there is no L2 to do
1757 // Don't finish the translation if a stage 2 look up is underway
1758 if (!currState->doingStage2) {
1759 statWalkServiceTime.sample(curTick() - currState->startTime);
1760 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1761 currState->fault = tlb->translateTiming(currState->req, currState->tc,
1762 currState->transState, currState->mode);
1763 statWalksShortTerminatedAtLevel[0]++;
1764 }
1765
1766 pending = false;
1767 nextWalk(currState->tc);
1768
1769 currState->req = NULL;
1770 currState->tc = NULL;
1771 currState->delayed = false;
1772 delete currState;
1773 } else {
1774 // need to do L2 descriptor
1775 stateQueues[L2].push_back(currState);
1776 }
1777 currState = NULL;
1778}
1779
1780void
1781TableWalker::doL2DescriptorWrapper()
1782{
1783 currState = stateQueues[L2].front();
1784 assert(currState->delayed);
1785 // if there's a stage2 translation object we don't need it any more
1786 if (currState->stage2Tran) {
1787 delete currState->stage2Tran;
1788 currState->stage2Tran = NULL;
1789 }
1790
1791 DPRINTF(TLBVerbose, "calling doL2Descriptor for vaddr:%#x\n",
1792 currState->vaddr_tainted);
1793 doL2Descriptor();
1794
1795 // Check if fault was generated
1796 if (currState->fault != NoFault) {
1797 currState->transState->finish(currState->fault, currState->req,
1798 currState->tc, currState->mode);
1799 statWalksShortTerminatedAtLevel[1]++;
1800 }
1801 else {
1802 // Don't finish the translation if a stage 2 look up is underway
1803 if (!currState->doingStage2) {
1804 statWalkServiceTime.sample(curTick() - currState->startTime);
1805 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1806 currState->fault = tlb->translateTiming(currState->req,
1807 currState->tc, currState->transState, currState->mode);
1808 statWalksShortTerminatedAtLevel[1]++;
1809 }
1810 }
1811
1812
1813 stateQueues[L2].pop_front();
1814 pending = false;
1815 nextWalk(currState->tc);
1816
1817 currState->req = NULL;
1818 currState->tc = NULL;
1819 currState->delayed = false;
1820
1821 delete currState;
1822 currState = NULL;
1823}
1824
1825void
1826TableWalker::doL0LongDescriptorWrapper()
1827{
1828 doLongDescriptorWrapper(L0);
1829}
1830
1831void
1832TableWalker::doL1LongDescriptorWrapper()
1833{
1834 doLongDescriptorWrapper(L1);
1835}
1836
1837void
1838TableWalker::doL2LongDescriptorWrapper()
1839{
1840 doLongDescriptorWrapper(L2);
1841}
1842
1843void
1844TableWalker::doL3LongDescriptorWrapper()
1845{
1846 doLongDescriptorWrapper(L3);
1847}
1848
1849void
1850TableWalker::doLongDescriptorWrapper(LookupLevel curr_lookup_level)
1851{
1852 currState = stateQueues[curr_lookup_level].front();
1853 assert(curr_lookup_level == currState->longDesc.lookupLevel);
1854 currState->delayed = false;
1855
1856 // if there's a stage2 translation object we don't need it any more
1857 if (currState->stage2Tran) {
1858 delete currState->stage2Tran;
1859 currState->stage2Tran = NULL;
1860 }
1861
1862 DPRINTF(TLBVerbose, "calling doLongDescriptor for vaddr:%#x\n",
1863 currState->vaddr_tainted);
1864 doLongDescriptor();
1865
1866 stateQueues[curr_lookup_level].pop_front();
1867
1868 if (currState->fault != NoFault) {
1869 // A fault was generated
1870 currState->transState->finish(currState->fault, currState->req,
1871 currState->tc, currState->mode);
1872
1873 pending = false;
1874 nextWalk(currState->tc);
1875
1876 currState->req = NULL;
1877 currState->tc = NULL;
1878 currState->delayed = false;
1879 delete currState;
1880 } else if (!currState->delayed) {
1881 // No additional lookups required
1882 // Don't finish the translation if a stage 2 look up is underway
1883 if (!currState->doingStage2) {
1884 DPRINTF(TLBVerbose, "calling translateTiming again\n");
1885 statWalkServiceTime.sample(curTick() - currState->startTime);
1886 currState->fault = tlb->translateTiming(currState->req, currState->tc,
1887 currState->transState,
1888 currState->mode);
1889 statWalksLongTerminatedAtLevel[(unsigned) curr_lookup_level]++;
1890 }
1891
1892 pending = false;
1893 nextWalk(currState->tc);
1894
1895 currState->req = NULL;
1896 currState->tc = NULL;
1897 currState->delayed = false;
1898 delete currState;
1899 } else {
1900 if (curr_lookup_level >= MAX_LOOKUP_LEVELS - 1)
1901 panic("Max. number of lookups already reached in table walk\n");
1902 // Need to perform additional lookups
1903 stateQueues[currState->longDesc.lookupLevel].push_back(currState);
1904 }
1905 currState = NULL;
1906}
1907
1908
1909void
1910TableWalker::nextWalk(ThreadContext *tc)
1911{
1912 if (pendingQueue.size())
1913 schedule(doProcessEvent, clockEdge(Cycles(1)));
1914 else
1915 completeDrain();
1916}
1917
1918bool
1919TableWalker::fetchDescriptor(Addr descAddr, uint8_t *data, int numBytes,
1920 Request::Flags flags, int queueIndex, Event *event,
1921 void (TableWalker::*doDescriptor)())
1922{
1923 bool isTiming = currState->timing;
1924
1925 // do the requests for the page table descriptors have to go through the
1926 // second stage MMU
1927 if (currState->stage2Req) {
1928 Fault fault;
1929 flags = flags | TLB::MustBeOne;
1930
1931 if (isTiming) {
1932 Stage2MMU::Stage2Translation *tran = new
1933 Stage2MMU::Stage2Translation(*stage2Mmu, data, event,
1934 currState->vaddr);
1935 currState->stage2Tran = tran;
1936 stage2Mmu->readDataTimed(currState->tc, descAddr, tran, numBytes,
|
1919 flags, masterId);
| 1937 flags);
|
1920 fault = tran->fault;
1921 } else {
1922 fault = stage2Mmu->readDataUntimed(currState->tc,
| 1938 fault = tran->fault;
1939 } else {
1940 fault = stage2Mmu->readDataUntimed(currState->tc,
|
1923 currState->vaddr, descAddr, data, numBytes, flags, masterId,
| 1941 currState->vaddr, descAddr, data, numBytes, flags,
|
1924 currState->functional);
1925 }
1926
1927 if (fault != NoFault) {
1928 currState->fault = fault;
1929 }
1930 if (isTiming) {
1931 if (queueIndex >= 0) {
1932 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1933 stateQueues[queueIndex].size());
1934 stateQueues[queueIndex].push_back(currState);
1935 currState = NULL;
1936 }
1937 } else {
1938 (this->*doDescriptor)();
1939 }
1940 } else {
1941 if (isTiming) {
| 1942 currState->functional);
1943 }
1944
1945 if (fault != NoFault) {
1946 currState->fault = fault;
1947 }
1948 if (isTiming) {
1949 if (queueIndex >= 0) {
1950 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1951 stateQueues[queueIndex].size());
1952 stateQueues[queueIndex].push_back(currState);
1953 currState = NULL;
1954 }
1955 } else {
1956 (this->*doDescriptor)();
1957 }
1958 } else {
1959 if (isTiming) {
|
1942 port.dmaAction(MemCmd::ReadReq, descAddr, numBytes, event, data,
| 1960 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, event, data,
|
1943 currState->tc->getCpuPtr()->clockPeriod(),flags);
1944 if (queueIndex >= 0) {
1945 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1946 stateQueues[queueIndex].size());
1947 stateQueues[queueIndex].push_back(currState);
1948 currState = NULL;
1949 }
1950 } else if (!currState->functional) {
| 1961 currState->tc->getCpuPtr()->clockPeriod(),flags);
1962 if (queueIndex >= 0) {
1963 DPRINTF(TLBVerbose, "Adding to walker fifo: queue size before adding: %d\n",
1964 stateQueues[queueIndex].size());
1965 stateQueues[queueIndex].push_back(currState);
1966 currState = NULL;
1967 }
1968 } else if (!currState->functional) {
|
1951 port.dmaAction(MemCmd::ReadReq, descAddr, numBytes, NULL, data,
| 1969 port->dmaAction(MemCmd::ReadReq, descAddr, numBytes, NULL, data,
|
1952 currState->tc->getCpuPtr()->clockPeriod(), flags);
1953 (this->*doDescriptor)();
1954 } else {
1955 RequestPtr req = new Request(descAddr, numBytes, flags, masterId);
1956 req->taskId(ContextSwitchTaskId::DMA);
1957 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
1958 pkt->dataStatic(data);
| 1970 currState->tc->getCpuPtr()->clockPeriod(), flags);
1971 (this->*doDescriptor)();
1972 } else {
1973 RequestPtr req = new Request(descAddr, numBytes, flags, masterId);
1974 req->taskId(ContextSwitchTaskId::DMA);
1975 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
1976 pkt->dataStatic(data);
|
1959 port.sendFunctional(pkt);
| 1977 port->sendFunctional(pkt);
|
1960 (this->*doDescriptor)();
1961 delete req;
1962 delete pkt;
1963 }
1964 }
1965 return (isTiming);
1966}
1967
1968void
1969TableWalker::insertTableEntry(DescriptorBase &descriptor, bool longDescriptor)
1970{
1971 TlbEntry te;
1972
1973 // Create and fill a new page table entry
1974 te.valid = true;
1975 te.longDescFormat = longDescriptor;
1976 te.isHyp = currState->isHyp;
1977 te.asid = currState->asid;
1978 te.vmid = currState->vmid;
1979 te.N = descriptor.offsetBits();
1980 te.vpn = currState->vaddr >> te.N;
1981 te.size = (1<<te.N) - 1;
1982 te.pfn = descriptor.pfn();
1983 te.domain = descriptor.domain();
1984 te.lookupLevel = descriptor.lookupLevel;
1985 te.ns = !descriptor.secure(haveSecurity, currState) || isStage2;
1986 te.nstid = !currState->isSecure;
1987 te.xn = descriptor.xn();
1988 if (currState->aarch64)
1989 te.el = currState->el;
1990 else
1991 te.el = 1;
1992
1993 statPageSizes[pageSizeNtoStatBin(te.N)]++;
1994 statRequestOrigin[COMPLETED][currState->isFetch]++;
1995
1996 // ASID has no meaning for stage 2 TLB entries, so mark all stage 2 entries
1997 // as global
1998 te.global = descriptor.global(currState) || isStage2;
1999 if (longDescriptor) {
2000 LongDescriptor lDescriptor =
2001 dynamic_cast<LongDescriptor &>(descriptor);
2002
2003 te.xn |= currState->xnTable;
2004 te.pxn = currState->pxnTable || lDescriptor.pxn();
2005 if (isStage2) {
2006 // this is actually the HAP field, but its stored in the same bit
2007 // possitions as the AP field in a stage 1 translation.
2008 te.hap = lDescriptor.ap();
2009 } else {
2010 te.ap = ((!currState->rwTable || descriptor.ap() >> 1) << 1) |
2011 (currState->userTable && (descriptor.ap() & 0x1));
2012 }
2013 if (currState->aarch64)
2014 memAttrsAArch64(currState->tc, te, currState->longDesc.attrIndx(),
2015 currState->longDesc.sh());
2016 else
2017 memAttrsLPAE(currState->tc, te, lDescriptor);
2018 } else {
2019 te.ap = descriptor.ap();
2020 memAttrs(currState->tc, te, currState->sctlr, descriptor.texcb(),
2021 descriptor.shareable());
2022 }
2023
2024 // Debug output
2025 DPRINTF(TLB, descriptor.dbgHeader().c_str());
2026 DPRINTF(TLB, " - N:%d pfn:%#x size:%#x global:%d valid:%d\n",
2027 te.N, te.pfn, te.size, te.global, te.valid);
2028 DPRINTF(TLB, " - vpn:%#x xn:%d pxn:%d ap:%d domain:%d asid:%d "
2029 "vmid:%d hyp:%d nc:%d ns:%d\n", te.vpn, te.xn, te.pxn,
2030 te.ap, static_cast<uint8_t>(te.domain), te.asid, te.vmid, te.isHyp,
2031 te.nonCacheable, te.ns);
2032 DPRINTF(TLB, " - domain from L%d desc:%d data:%#x\n",
2033 descriptor.lookupLevel, static_cast<uint8_t>(descriptor.domain()),
2034 descriptor.getRawData());
2035
2036 // Insert the entry into the TLB
2037 tlb->insert(currState->vaddr, te);
2038 if (!currState->timing) {
2039 currState->tc = NULL;
2040 currState->req = NULL;
2041 }
2042}
2043
2044ArmISA::TableWalker *
2045ArmTableWalkerParams::create()
2046{
2047 return new ArmISA::TableWalker(this);
2048}
2049
2050LookupLevel
2051TableWalker::toLookupLevel(uint8_t lookup_level_as_int)
2052{
2053 switch (lookup_level_as_int) {
2054 case L1:
2055 return L1;
2056 case L2:
2057 return L2;
2058 case L3:
2059 return L3;
2060 default:
2061 panic("Invalid lookup level conversion");
2062 }
2063}
2064
2065/* this method keeps track of the table walker queue's residency, so
2066 * needs to be called whenever requests start and complete. */
2067void
2068TableWalker::pendingChange()
2069{
2070 unsigned n = pendingQueue.size();
2071 if ((currState != NULL) && (currState != pendingQueue.front())) {
2072 ++n;
2073 }
2074
2075 if (n != pendingReqs) {
2076 Tick now = curTick();
2077 statPendingWalks.sample(pendingReqs, now - pendingChangeTick);
2078 pendingReqs = n;
2079 pendingChangeTick = now;
2080 }
2081}
2082
2083uint8_t
2084TableWalker::pageSizeNtoStatBin(uint8_t N)
2085{
2086 /* for statPageSizes */
2087 switch(N) {
2088 case 12: return 0; // 4K
2089 case 14: return 1; // 16K (using 16K granule in v8-64)
2090 case 16: return 2; // 64K
2091 case 20: return 3; // 1M
2092 case 21: return 4; // 2M-LPAE
2093 case 24: return 5; // 16M
2094 case 25: return 6; // 32M (using 16K granule in v8-64)
2095 case 29: return 7; // 512M (using 64K granule in v8-64)
2096 case 30: return 8; // 1G-LPAE
2097 default:
2098 panic("unknown page size");
2099 return 255;
2100 }
2101}
2102
2103void
2104TableWalker::regStats()
2105{
2106 statWalks
2107 .name(name() + ".walks")
2108 .desc("Table walker walks requested")
2109 ;
2110
2111 statWalksShortDescriptor
2112 .name(name() + ".walksShort")
2113 .desc("Table walker walks initiated with short descriptors")
2114 .flags(Stats::nozero)
2115 ;
2116
2117 statWalksLongDescriptor
2118 .name(name() + ".walksLong")
2119 .desc("Table walker walks initiated with long descriptors")
2120 .flags(Stats::nozero)
2121 ;
2122
2123 statWalksShortTerminatedAtLevel
2124 .init(2)
2125 .name(name() + ".walksShortTerminationLevel")
2126 .desc("Level at which table walker walks "
2127 "with short descriptors terminate")
2128 .flags(Stats::nozero)
2129 ;
2130 statWalksShortTerminatedAtLevel.subname(0, "Level1");
2131 statWalksShortTerminatedAtLevel.subname(1, "Level2");
2132
2133 statWalksLongTerminatedAtLevel
2134 .init(4)
2135 .name(name() + ".walksLongTerminationLevel")
2136 .desc("Level at which table walker walks "
2137 "with long descriptors terminate")
2138 .flags(Stats::nozero)
2139 ;
2140 statWalksLongTerminatedAtLevel.subname(0, "Level0");
2141 statWalksLongTerminatedAtLevel.subname(1, "Level1");
2142 statWalksLongTerminatedAtLevel.subname(2, "Level2");
2143 statWalksLongTerminatedAtLevel.subname(3, "Level3");
2144
2145 statSquashedBefore
2146 .name(name() + ".walksSquashedBefore")
2147 .desc("Table walks squashed before starting")
2148 .flags(Stats::nozero)
2149 ;
2150
2151 statSquashedAfter
2152 .name(name() + ".walksSquashedAfter")
2153 .desc("Table walks squashed after completion")
2154 .flags(Stats::nozero)
2155 ;
2156
2157 statWalkWaitTime
2158 .init(16)
2159 .name(name() + ".walkWaitTime")
2160 .desc("Table walker wait (enqueue to first request) latency")
2161 .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2162 ;
2163
2164 statWalkServiceTime
2165 .init(16)
2166 .name(name() + ".walkCompletionTime")
2167 .desc("Table walker service (enqueue to completion) latency")
2168 .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2169 ;
2170
2171 statPendingWalks
2172 .init(16)
2173 .name(name() + ".walksPending")
2174 .desc("Table walker pending requests distribution")
2175 .flags(Stats::pdf | Stats::dist | Stats::nozero | Stats::nonan)
2176 ;
2177
2178 statPageSizes // see DDI 0487A D4-1661
2179 .init(9)
2180 .name(name() + ".walkPageSizes")
2181 .desc("Table walker page sizes translated")
2182 .flags(Stats::total | Stats::pdf | Stats::dist | Stats::nozero)
2183 ;
2184 statPageSizes.subname(0, "4K");
2185 statPageSizes.subname(1, "16K");
2186 statPageSizes.subname(2, "64K");
2187 statPageSizes.subname(3, "1M");
2188 statPageSizes.subname(4, "2M");
2189 statPageSizes.subname(5, "16M");
2190 statPageSizes.subname(6, "32M");
2191 statPageSizes.subname(7, "512M");
2192 statPageSizes.subname(8, "1G");
2193
2194 statRequestOrigin
2195 .init(2,2) // Instruction/Data, requests/completed
2196 .name(name() + ".walkRequestOrigin")
2197 .desc("Table walker requests started/completed, data/inst")
2198 .flags(Stats::total)
2199 ;
2200 statRequestOrigin.subname(0,"Requested");
2201 statRequestOrigin.subname(1,"Completed");
2202 statRequestOrigin.ysubname(0,"Data");
2203 statRequestOrigin.ysubname(1,"Inst");
2204}
| 1978 (this->*doDescriptor)();
1979 delete req;
1980 delete pkt;
1981 }
1982 }
1983 return (isTiming);
1984}
1985
1986void
1987TableWalker::insertTableEntry(DescriptorBase &descriptor, bool longDescriptor)
1988{
1989 TlbEntry te;
1990
1991 // Create and fill a new page table entry
1992 te.valid = true;
1993 te.longDescFormat = longDescriptor;
1994 te.isHyp = currState->isHyp;
1995 te.asid = currState->asid;
1996 te.vmid = currState->vmid;
1997 te.N = descriptor.offsetBits();
1998 te.vpn = currState->vaddr >> te.N;
1999 te.size = (1<<te.N) - 1;
2000 te.pfn = descriptor.pfn();
2001 te.domain = descriptor.domain();
2002 te.lookupLevel = descriptor.lookupLevel;
2003 te.ns = !descriptor.secure(haveSecurity, currState) || isStage2;
2004 te.nstid = !currState->isSecure;
2005 te.xn = descriptor.xn();
2006 if (currState->aarch64)
2007 te.el = currState->el;
2008 else
2009 te.el = 1;
2010
2011 statPageSizes[pageSizeNtoStatBin(te.N)]++;
2012 statRequestOrigin[COMPLETED][currState->isFetch]++;
2013
2014 // ASID has no meaning for stage 2 TLB entries, so mark all stage 2 entries
2015 // as global
2016 te.global = descriptor.global(currState) || isStage2;
2017 if (longDescriptor) {
2018 LongDescriptor lDescriptor =
2019 dynamic_cast<LongDescriptor &>(descriptor);
2020
2021 te.xn |= currState->xnTable;
2022 te.pxn = currState->pxnTable || lDescriptor.pxn();
2023 if (isStage2) {
2024 // this is actually the HAP field, but its stored in the same bit
2025 // possitions as the AP field in a stage 1 translation.
2026 te.hap = lDescriptor.ap();
2027 } else {
2028 te.ap = ((!currState->rwTable || descriptor.ap() >> 1) << 1) |
2029 (currState->userTable && (descriptor.ap() & 0x1));
2030 }
2031 if (currState->aarch64)
2032 memAttrsAArch64(currState->tc, te, currState->longDesc.attrIndx(),
2033 currState->longDesc.sh());
2034 else
2035 memAttrsLPAE(currState->tc, te, lDescriptor);
2036 } else {
2037 te.ap = descriptor.ap();
2038 memAttrs(currState->tc, te, currState->sctlr, descriptor.texcb(),
2039 descriptor.shareable());
2040 }
2041
2042 // Debug output
2043 DPRINTF(TLB, descriptor.dbgHeader().c_str());
2044 DPRINTF(TLB, " - N:%d pfn:%#x size:%#x global:%d valid:%d\n",
2045 te.N, te.pfn, te.size, te.global, te.valid);
2046 DPRINTF(TLB, " - vpn:%#x xn:%d pxn:%d ap:%d domain:%d asid:%d "
2047 "vmid:%d hyp:%d nc:%d ns:%d\n", te.vpn, te.xn, te.pxn,
2048 te.ap, static_cast<uint8_t>(te.domain), te.asid, te.vmid, te.isHyp,
2049 te.nonCacheable, te.ns);
2050 DPRINTF(TLB, " - domain from L%d desc:%d data:%#x\n",
2051 descriptor.lookupLevel, static_cast<uint8_t>(descriptor.domain()),
2052 descriptor.getRawData());
2053
2054 // Insert the entry into the TLB
2055 tlb->insert(currState->vaddr, te);
2056 if (!currState->timing) {
2057 currState->tc = NULL;
2058 currState->req = NULL;
2059 }
2060}
2061
2062ArmISA::TableWalker *
2063ArmTableWalkerParams::create()
2064{
2065 return new ArmISA::TableWalker(this);
2066}
2067
2068LookupLevel
2069TableWalker::toLookupLevel(uint8_t lookup_level_as_int)
2070{
2071 switch (lookup_level_as_int) {
2072 case L1:
2073 return L1;
2074 case L2:
2075 return L2;
2076 case L3:
2077 return L3;
2078 default:
2079 panic("Invalid lookup level conversion");
2080 }
2081}
2082
2083/* this method keeps track of the table walker queue's residency, so
2084 * needs to be called whenever requests start and complete. */
2085void
2086TableWalker::pendingChange()
2087{
2088 unsigned n = pendingQueue.size();
2089 if ((currState != NULL) && (currState != pendingQueue.front())) {
2090 ++n;
2091 }
2092
2093 if (n != pendingReqs) {
2094 Tick now = curTick();
2095 statPendingWalks.sample(pendingReqs, now - pendingChangeTick);
2096 pendingReqs = n;
2097 pendingChangeTick = now;
2098 }
2099}
2100
2101uint8_t
2102TableWalker::pageSizeNtoStatBin(uint8_t N)
2103{
2104 /* for statPageSizes */
2105 switch(N) {
2106 case 12: return 0; // 4K
2107 case 14: return 1; // 16K (using 16K granule in v8-64)
2108 case 16: return 2; // 64K
2109 case 20: return 3; // 1M
2110 case 21: return 4; // 2M-LPAE
2111 case 24: return 5; // 16M
2112 case 25: return 6; // 32M (using 16K granule in v8-64)
2113 case 29: return 7; // 512M (using 64K granule in v8-64)
2114 case 30: return 8; // 1G-LPAE
2115 default:
2116 panic("unknown page size");
2117 return 255;
2118 }
2119}
2120
2121void
2122TableWalker::regStats()
2123{
2124 statWalks
2125 .name(name() + ".walks")
2126 .desc("Table walker walks requested")
2127 ;
2128
2129 statWalksShortDescriptor
2130 .name(name() + ".walksShort")
2131 .desc("Table walker walks initiated with short descriptors")
2132 .flags(Stats::nozero)
2133 ;
2134
2135 statWalksLongDescriptor
2136 .name(name() + ".walksLong")
2137 .desc("Table walker walks initiated with long descriptors")
2138 .flags(Stats::nozero)
2139 ;
2140
2141 statWalksShortTerminatedAtLevel
2142 .init(2)
2143 .name(name() + ".walksShortTerminationLevel")
2144 .desc("Level at which table walker walks "
2145 "with short descriptors terminate")
2146 .flags(Stats::nozero)
2147 ;
2148 statWalksShortTerminatedAtLevel.subname(0, "Level1");
2149 statWalksShortTerminatedAtLevel.subname(1, "Level2");
2150
2151 statWalksLongTerminatedAtLevel
2152 .init(4)
2153 .name(name() + ".walksLongTerminationLevel")
2154 .desc("Level at which table walker walks "
2155 "with long descriptors terminate")
2156 .flags(Stats::nozero)
2157 ;
2158 statWalksLongTerminatedAtLevel.subname(0, "Level0");
2159 statWalksLongTerminatedAtLevel.subname(1, "Level1");
2160 statWalksLongTerminatedAtLevel.subname(2, "Level2");
2161 statWalksLongTerminatedAtLevel.subname(3, "Level3");
2162
2163 statSquashedBefore
2164 .name(name() + ".walksSquashedBefore")
2165 .desc("Table walks squashed before starting")
2166 .flags(Stats::nozero)
2167 ;
2168
2169 statSquashedAfter
2170 .name(name() + ".walksSquashedAfter")
2171 .desc("Table walks squashed after completion")
2172 .flags(Stats::nozero)
2173 ;
2174
2175 statWalkWaitTime
2176 .init(16)
2177 .name(name() + ".walkWaitTime")
2178 .desc("Table walker wait (enqueue to first request) latency")
2179 .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2180 ;
2181
2182 statWalkServiceTime
2183 .init(16)
2184 .name(name() + ".walkCompletionTime")
2185 .desc("Table walker service (enqueue to completion) latency")
2186 .flags(Stats::pdf | Stats::nozero | Stats::nonan)
2187 ;
2188
2189 statPendingWalks
2190 .init(16)
2191 .name(name() + ".walksPending")
2192 .desc("Table walker pending requests distribution")
2193 .flags(Stats::pdf | Stats::dist | Stats::nozero | Stats::nonan)
2194 ;
2195
2196 statPageSizes // see DDI 0487A D4-1661
2197 .init(9)
2198 .name(name() + ".walkPageSizes")
2199 .desc("Table walker page sizes translated")
2200 .flags(Stats::total | Stats::pdf | Stats::dist | Stats::nozero)
2201 ;
2202 statPageSizes.subname(0, "4K");
2203 statPageSizes.subname(1, "16K");
2204 statPageSizes.subname(2, "64K");
2205 statPageSizes.subname(3, "1M");
2206 statPageSizes.subname(4, "2M");
2207 statPageSizes.subname(5, "16M");
2208 statPageSizes.subname(6, "32M");
2209 statPageSizes.subname(7, "512M");
2210 statPageSizes.subname(8, "1G");
2211
2212 statRequestOrigin
2213 .init(2,2) // Instruction/Data, requests/completed
2214 .name(name() + ".walkRequestOrigin")
2215 .desc("Table walker requests started/completed, data/inst")
2216 .flags(Stats::total)
2217 ;
2218 statRequestOrigin.subname(0,"Requested");
2219 statRequestOrigin.subname(1,"Completed");
2220 statRequestOrigin.ysubname(0,"Data");
2221 statRequestOrigin.ysubname(1,"Inst");
2222}
|