1/*
2 * Copyright (c) 2013, 2018-2019 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Stan Czerniawski
38 */
39
40#include "dev/arm/smmu_v3.hh"
41
42#include <cstddef>
43#include <cstring>
44
45#include "base/bitfield.hh"
46#include "base/cast.hh"
47#include "base/logging.hh"
48#include "base/trace.hh"
49#include "base/types.hh"
50#include "debug/Checkpoint.hh"
51#include "debug/SMMUv3.hh"
52#include "dev/arm/smmu_v3_transl.hh"
53#include "mem/packet_access.hh"
54#include "sim/system.hh"
55
56SMMUv3::SMMUv3(SMMUv3Params *params) :
57 MemObject(params),
58 system(*params->system),
59 masterId(params->system->getMasterId(this)),
60 masterPort(name() + ".master", *this),
61 masterTableWalkPort(name() + ".master_walker", *this),
62 controlPort(name() + ".control", *this, params->reg_map),
63 tlb(params->tlb_entries, params->tlb_assoc, params->tlb_policy),
64 configCache(params->cfg_entries, params->cfg_assoc, params->cfg_policy),
65 ipaCache(params->ipa_entries, params->ipa_assoc, params->ipa_policy),
66 walkCache({ { params->walk_S1L0, params->walk_S1L1,
67 params->walk_S1L2, params->walk_S1L3,
68 params->walk_S2L0, params->walk_S2L1,
69 params->walk_S2L2, params->walk_S2L3 } },
70 params->walk_assoc, params->walk_policy),
71 tlbEnable(params->tlb_enable),
72 configCacheEnable(params->cfg_enable),
73 ipaCacheEnable(params->ipa_enable),
74 walkCacheEnable(params->walk_enable),
75 tableWalkPortEnable(false),
76 walkCacheNonfinalEnable(params->wc_nonfinal_enable),
77 walkCacheS1Levels(params->wc_s1_levels),
78 walkCacheS2Levels(params->wc_s2_levels),
79 masterPortWidth(params->master_port_width),
80 tlbSem(params->tlb_slots),
81 ifcSmmuSem(1),
82 smmuIfcSem(1),
83 configSem(params->cfg_slots),
84 ipaSem(params->ipa_slots),
85 walkSem(params->walk_slots),
86 masterPortSem(1),
87 transSem(params->xlate_slots),
88 ptwSem(params->ptw_slots),
89 cycleSem(1),
90 tlbLat(params->tlb_lat),
91 ifcSmmuLat(params->ifc_smmu_lat),
92 smmuIfcLat(params->smmu_ifc_lat),
93 configLat(params->cfg_lat),
94 ipaLat(params->ipa_lat),
95 walkLat(params->walk_lat),
96 slaveInterfaces(params->slave_interfaces),
97 commandExecutor(name() + ".cmd_exec", *this),
98 regsMap(params->reg_map),
99 processCommandsEvent(this)
100{
101 fatal_if(regsMap.size() != SMMU_REG_SIZE,
102 "Invalid register map size: %#x different than SMMU_REG_SIZE = %#x\n",
103 regsMap.size(), SMMU_REG_SIZE);
104
105 // Init smmu registers to 0
106 memset(®s, 0, sizeof(regs));
107
108 // Setup RO ID registers
109 regs.idr0 = params->smmu_idr0;
110 regs.idr1 = params->smmu_idr1;
111 regs.idr2 = params->smmu_idr2;
112 regs.idr3 = params->smmu_idr3;
113 regs.idr4 = params->smmu_idr4;
114 regs.idr5 = params->smmu_idr5;
115 regs.iidr = params->smmu_iidr;
116 regs.aidr = params->smmu_aidr;
117
118 // TODO: At the moment it possible to set the ID registers to hold
119 // any possible value. It would be nice to have a sanity check here
120 // at construction time in case some idx registers are programmed to
121 // store an unallowed values or if the are configuration conflicts.
122 warn("SMMUv3 IDx register values unchecked\n");
123
124 for (auto ifc : slaveInterfaces)
125 ifc->setSMMU(this);
126}
127
128bool
129SMMUv3::masterRecvTimingResp(PacketPtr pkt)
130{
131 DPRINTF(SMMUv3, "[t] master resp addr=%#x size=%#x\n",
132 pkt->getAddr(), pkt->getSize());
133
134 // @todo: We need to pay for this and not just zero it out
135 pkt->headerDelay = pkt->payloadDelay = 0;
136
137 SMMUProcess *proc =
138 safe_cast<SMMUProcess *>(pkt->popSenderState());
139
140 runProcessTiming(proc, pkt);
141
142 return true;
143}
144
145void
146SMMUv3::masterRecvReqRetry()
147{
148 assert(!packetsToRetry.empty());
149
150 while (!packetsToRetry.empty()) {
151 SMMUAction a = packetsToRetry.front();
152
153 assert(a.type==ACTION_SEND_REQ || a.type==ACTION_SEND_REQ_FINAL);
154
155 DPRINTF(SMMUv3, "[t] master retr addr=%#x size=%#x\n",
156 a.pkt->getAddr(), a.pkt->getSize());
157
158 if (!masterPort.sendTimingReq(a.pkt))
159 break;
160
161 packetsToRetry.pop();
162
163 /*
164 * ACTION_SEND_REQ_FINAL means that we have just forwarded the packet
165 * on the master interface; this means that we no longer hold on to
166 * that transaction and therefore can accept a new one.
167 * If the slave port was stalled then unstall it (send retry).
168 */
169 if (a.type == ACTION_SEND_REQ_FINAL)
170 scheduleSlaveRetries();
171 }
172}
173
174bool
175SMMUv3::masterTableWalkRecvTimingResp(PacketPtr pkt)
176{
177 DPRINTF(SMMUv3, "[t] master HWTW resp addr=%#x size=%#x\n",
178 pkt->getAddr(), pkt->getSize());
179
180 // @todo: We need to pay for this and not just zero it out
181 pkt->headerDelay = pkt->payloadDelay = 0;
182
183 SMMUProcess *proc =
184 safe_cast<SMMUProcess *>(pkt->popSenderState());
185
186 runProcessTiming(proc, pkt);
187
188 return true;
189}
190
191void
192SMMUv3::masterTableWalkRecvReqRetry()
193{
194 assert(tableWalkPortEnable);
195 assert(!packetsTableWalkToRetry.empty());
196
197 while (!packetsTableWalkToRetry.empty()) {
198 SMMUAction a = packetsTableWalkToRetry.front();
199
200 assert(a.type==ACTION_SEND_REQ);
201
202 DPRINTF(SMMUv3, "[t] master HWTW retr addr=%#x size=%#x\n",
203 a.pkt->getAddr(), a.pkt->getSize());
204
205 if (!masterTableWalkPort.sendTimingReq(a.pkt))
206 break;
207
208 packetsTableWalkToRetry.pop();
209 }
210}
211
212void
213SMMUv3::scheduleSlaveRetries()
214{
215 for (auto ifc : slaveInterfaces) {
216 ifc->scheduleDeviceRetry();
217 }
218}
219
220SMMUAction
221SMMUv3::runProcess(SMMUProcess *proc, PacketPtr pkt)
222{
223 if (system.isAtomicMode()) {
224 return runProcessAtomic(proc, pkt);
225 } else if (system.isTimingMode()) {
226 return runProcessTiming(proc, pkt);
227 } else {
228 panic("Not in timing or atomic mode!");
229 }
230}
231
232SMMUAction
233SMMUv3::runProcessAtomic(SMMUProcess *proc, PacketPtr pkt)
234{
235 SMMUAction action;
236 Tick delay = 0;
237 bool finished = false;
238
239 do {
240 action = proc->run(pkt);
241
242 switch (action.type) {
243 case ACTION_SEND_REQ:
244 // Send an MMU initiated request on the table walk port if it is
245 // enabled. Otherwise, fall through and handle same as the final
246 // ACTION_SEND_REQ_FINAL request.
247 if (tableWalkPortEnable) {
248 delay += masterTableWalkPort.sendAtomic(action.pkt);
249 pkt = action.pkt;
250 break;
251 }
252 M5_FALLTHROUGH;
253 case ACTION_SEND_REQ_FINAL:
254 delay += masterPort.sendAtomic(action.pkt);
255 pkt = action.pkt;
256 break;
257
258 case ACTION_SEND_RESP:
259 case ACTION_SEND_RESP_ATS:
260 case ACTION_SLEEP:
261 finished = true;
262 break;
263
264 case ACTION_DELAY:
265 delay += action.delay;
266 break;
267
268 case ACTION_TERMINATE:
269 panic("ACTION_TERMINATE in atomic mode\n");
270
271 default:
272 panic("Unknown action\n");
273 }
274 } while (!finished);
275
276 action.delay = delay;
277
278 return action;
279}
280
281SMMUAction
282SMMUv3::runProcessTiming(SMMUProcess *proc, PacketPtr pkt)
283{
284 SMMUAction action = proc->run(pkt);
285
286 switch (action.type) {
287 case ACTION_SEND_REQ:
288 // Send an MMU initiated request on the table walk port if it is
289 // enabled. Otherwise, fall through and handle same as the final
290 // ACTION_SEND_REQ_FINAL request.
291 if (tableWalkPortEnable) {
292 action.pkt->pushSenderState(proc);
293
294 DPRINTF(SMMUv3, "[t] master HWTW req addr=%#x size=%#x\n",
295 action.pkt->getAddr(), action.pkt->getSize());
296
297 if (packetsTableWalkToRetry.empty()
298 && masterTableWalkPort.sendTimingReq(action.pkt)) {
299 scheduleSlaveRetries();
300 } else {
301 DPRINTF(SMMUv3, "[t] master HWTW req needs retry,"
302 " qlen=%d\n", packetsTableWalkToRetry.size());
303 packetsTableWalkToRetry.push(action);
304 }
305
306 break;
307 }
308 M5_FALLTHROUGH;
309 case ACTION_SEND_REQ_FINAL:
310 action.pkt->pushSenderState(proc);
311
312 DPRINTF(SMMUv3, "[t] master req addr=%#x size=%#x\n",
313 action.pkt->getAddr(), action.pkt->getSize());
314
315 if (packetsToRetry.empty() && masterPort.sendTimingReq(action.pkt)) {
316 scheduleSlaveRetries();
317 } else {
318 DPRINTF(SMMUv3, "[t] master req needs retry, qlen=%d\n",
319 packetsToRetry.size());
320 packetsToRetry.push(action);
321 }
322
323 break;
324
325 case ACTION_SEND_RESP:
326 // @todo: We need to pay for this and not just zero it out
327 action.pkt->headerDelay = action.pkt->payloadDelay = 0;
328
329 DPRINTF(SMMUv3, "[t] slave resp addr=%#x size=%#x\n",
330 action.pkt->getAddr(),
331 action.pkt->getSize());
332
333 assert(action.ifc);
334 action.ifc->schedTimingResp(action.pkt);
335
336 delete proc;
337 break;
338
339 case ACTION_SEND_RESP_ATS:
340 // @todo: We need to pay for this and not just zero it out
341 action.pkt->headerDelay = action.pkt->payloadDelay = 0;
342
343 DPRINTF(SMMUv3, "[t] ATS slave resp addr=%#x size=%#x\n",
344 action.pkt->getAddr(), action.pkt->getSize());
345
346 assert(action.ifc);
347 action.ifc->schedAtsTimingResp(action.pkt);
348
349 delete proc;
350 break;
351
352 case ACTION_DELAY:
353 case ACTION_SLEEP:
354 break;
355
356 case ACTION_TERMINATE:
357 delete proc;
358 break;
359
360 default:
361 panic("Unknown action\n");
362 }
363
364 return action;
365}
366
367void
368SMMUv3::processCommands()
369{
370 DPRINTF(SMMUv3, "processCommands()\n");
371
372 if (system.isAtomicMode()) {
373 SMMUAction a = runProcessAtomic(&commandExecutor, NULL);
374 (void) a;
375 } else if (system.isTimingMode()) {
376 if (!commandExecutor.isBusy())
377 runProcessTiming(&commandExecutor, NULL);
378 } else {
379 panic("Not in timing or atomic mode!");
380 }
381}
382
383void
384SMMUv3::processCommand(const SMMUCommand &cmd)
385{
| 1/*
2 * Copyright (c) 2013, 2018-2019 ARM Limited
3 * All rights reserved
4 *
5 * The license below extends only to copyright in the software and shall
6 * not be construed as granting a license to any other intellectual
7 * property including but not limited to intellectual property relating
8 * to a hardware implementation of the functionality of the software
9 * licensed hereunder. You may use the software subject to the license
10 * terms below provided that you ensure that this notice is replicated
11 * unmodified and in its entirety in all distributions of the software,
12 * modified or unmodified, in source code or in binary form.
13 *
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions are
16 * met: redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer;
18 * redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution;
21 * neither the name of the copyright holders nor the names of its
22 * contributors may be used to endorse or promote products derived from
23 * this software without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 *
37 * Authors: Stan Czerniawski
38 */
39
40#include "dev/arm/smmu_v3.hh"
41
42#include <cstddef>
43#include <cstring>
44
45#include "base/bitfield.hh"
46#include "base/cast.hh"
47#include "base/logging.hh"
48#include "base/trace.hh"
49#include "base/types.hh"
50#include "debug/Checkpoint.hh"
51#include "debug/SMMUv3.hh"
52#include "dev/arm/smmu_v3_transl.hh"
53#include "mem/packet_access.hh"
54#include "sim/system.hh"
55
56SMMUv3::SMMUv3(SMMUv3Params *params) :
57 MemObject(params),
58 system(*params->system),
59 masterId(params->system->getMasterId(this)),
60 masterPort(name() + ".master", *this),
61 masterTableWalkPort(name() + ".master_walker", *this),
62 controlPort(name() + ".control", *this, params->reg_map),
63 tlb(params->tlb_entries, params->tlb_assoc, params->tlb_policy),
64 configCache(params->cfg_entries, params->cfg_assoc, params->cfg_policy),
65 ipaCache(params->ipa_entries, params->ipa_assoc, params->ipa_policy),
66 walkCache({ { params->walk_S1L0, params->walk_S1L1,
67 params->walk_S1L2, params->walk_S1L3,
68 params->walk_S2L0, params->walk_S2L1,
69 params->walk_S2L2, params->walk_S2L3 } },
70 params->walk_assoc, params->walk_policy),
71 tlbEnable(params->tlb_enable),
72 configCacheEnable(params->cfg_enable),
73 ipaCacheEnable(params->ipa_enable),
74 walkCacheEnable(params->walk_enable),
75 tableWalkPortEnable(false),
76 walkCacheNonfinalEnable(params->wc_nonfinal_enable),
77 walkCacheS1Levels(params->wc_s1_levels),
78 walkCacheS2Levels(params->wc_s2_levels),
79 masterPortWidth(params->master_port_width),
80 tlbSem(params->tlb_slots),
81 ifcSmmuSem(1),
82 smmuIfcSem(1),
83 configSem(params->cfg_slots),
84 ipaSem(params->ipa_slots),
85 walkSem(params->walk_slots),
86 masterPortSem(1),
87 transSem(params->xlate_slots),
88 ptwSem(params->ptw_slots),
89 cycleSem(1),
90 tlbLat(params->tlb_lat),
91 ifcSmmuLat(params->ifc_smmu_lat),
92 smmuIfcLat(params->smmu_ifc_lat),
93 configLat(params->cfg_lat),
94 ipaLat(params->ipa_lat),
95 walkLat(params->walk_lat),
96 slaveInterfaces(params->slave_interfaces),
97 commandExecutor(name() + ".cmd_exec", *this),
98 regsMap(params->reg_map),
99 processCommandsEvent(this)
100{
101 fatal_if(regsMap.size() != SMMU_REG_SIZE,
102 "Invalid register map size: %#x different than SMMU_REG_SIZE = %#x\n",
103 regsMap.size(), SMMU_REG_SIZE);
104
105 // Init smmu registers to 0
106 memset(®s, 0, sizeof(regs));
107
108 // Setup RO ID registers
109 regs.idr0 = params->smmu_idr0;
110 regs.idr1 = params->smmu_idr1;
111 regs.idr2 = params->smmu_idr2;
112 regs.idr3 = params->smmu_idr3;
113 regs.idr4 = params->smmu_idr4;
114 regs.idr5 = params->smmu_idr5;
115 regs.iidr = params->smmu_iidr;
116 regs.aidr = params->smmu_aidr;
117
118 // TODO: At the moment it possible to set the ID registers to hold
119 // any possible value. It would be nice to have a sanity check here
120 // at construction time in case some idx registers are programmed to
121 // store an unallowed values or if the are configuration conflicts.
122 warn("SMMUv3 IDx register values unchecked\n");
123
124 for (auto ifc : slaveInterfaces)
125 ifc->setSMMU(this);
126}
127
128bool
129SMMUv3::masterRecvTimingResp(PacketPtr pkt)
130{
131 DPRINTF(SMMUv3, "[t] master resp addr=%#x size=%#x\n",
132 pkt->getAddr(), pkt->getSize());
133
134 // @todo: We need to pay for this and not just zero it out
135 pkt->headerDelay = pkt->payloadDelay = 0;
136
137 SMMUProcess *proc =
138 safe_cast<SMMUProcess *>(pkt->popSenderState());
139
140 runProcessTiming(proc, pkt);
141
142 return true;
143}
144
145void
146SMMUv3::masterRecvReqRetry()
147{
148 assert(!packetsToRetry.empty());
149
150 while (!packetsToRetry.empty()) {
151 SMMUAction a = packetsToRetry.front();
152
153 assert(a.type==ACTION_SEND_REQ || a.type==ACTION_SEND_REQ_FINAL);
154
155 DPRINTF(SMMUv3, "[t] master retr addr=%#x size=%#x\n",
156 a.pkt->getAddr(), a.pkt->getSize());
157
158 if (!masterPort.sendTimingReq(a.pkt))
159 break;
160
161 packetsToRetry.pop();
162
163 /*
164 * ACTION_SEND_REQ_FINAL means that we have just forwarded the packet
165 * on the master interface; this means that we no longer hold on to
166 * that transaction and therefore can accept a new one.
167 * If the slave port was stalled then unstall it (send retry).
168 */
169 if (a.type == ACTION_SEND_REQ_FINAL)
170 scheduleSlaveRetries();
171 }
172}
173
174bool
175SMMUv3::masterTableWalkRecvTimingResp(PacketPtr pkt)
176{
177 DPRINTF(SMMUv3, "[t] master HWTW resp addr=%#x size=%#x\n",
178 pkt->getAddr(), pkt->getSize());
179
180 // @todo: We need to pay for this and not just zero it out
181 pkt->headerDelay = pkt->payloadDelay = 0;
182
183 SMMUProcess *proc =
184 safe_cast<SMMUProcess *>(pkt->popSenderState());
185
186 runProcessTiming(proc, pkt);
187
188 return true;
189}
190
191void
192SMMUv3::masterTableWalkRecvReqRetry()
193{
194 assert(tableWalkPortEnable);
195 assert(!packetsTableWalkToRetry.empty());
196
197 while (!packetsTableWalkToRetry.empty()) {
198 SMMUAction a = packetsTableWalkToRetry.front();
199
200 assert(a.type==ACTION_SEND_REQ);
201
202 DPRINTF(SMMUv3, "[t] master HWTW retr addr=%#x size=%#x\n",
203 a.pkt->getAddr(), a.pkt->getSize());
204
205 if (!masterTableWalkPort.sendTimingReq(a.pkt))
206 break;
207
208 packetsTableWalkToRetry.pop();
209 }
210}
211
212void
213SMMUv3::scheduleSlaveRetries()
214{
215 for (auto ifc : slaveInterfaces) {
216 ifc->scheduleDeviceRetry();
217 }
218}
219
220SMMUAction
221SMMUv3::runProcess(SMMUProcess *proc, PacketPtr pkt)
222{
223 if (system.isAtomicMode()) {
224 return runProcessAtomic(proc, pkt);
225 } else if (system.isTimingMode()) {
226 return runProcessTiming(proc, pkt);
227 } else {
228 panic("Not in timing or atomic mode!");
229 }
230}
231
232SMMUAction
233SMMUv3::runProcessAtomic(SMMUProcess *proc, PacketPtr pkt)
234{
235 SMMUAction action;
236 Tick delay = 0;
237 bool finished = false;
238
239 do {
240 action = proc->run(pkt);
241
242 switch (action.type) {
243 case ACTION_SEND_REQ:
244 // Send an MMU initiated request on the table walk port if it is
245 // enabled. Otherwise, fall through and handle same as the final
246 // ACTION_SEND_REQ_FINAL request.
247 if (tableWalkPortEnable) {
248 delay += masterTableWalkPort.sendAtomic(action.pkt);
249 pkt = action.pkt;
250 break;
251 }
252 M5_FALLTHROUGH;
253 case ACTION_SEND_REQ_FINAL:
254 delay += masterPort.sendAtomic(action.pkt);
255 pkt = action.pkt;
256 break;
257
258 case ACTION_SEND_RESP:
259 case ACTION_SEND_RESP_ATS:
260 case ACTION_SLEEP:
261 finished = true;
262 break;
263
264 case ACTION_DELAY:
265 delay += action.delay;
266 break;
267
268 case ACTION_TERMINATE:
269 panic("ACTION_TERMINATE in atomic mode\n");
270
271 default:
272 panic("Unknown action\n");
273 }
274 } while (!finished);
275
276 action.delay = delay;
277
278 return action;
279}
280
281SMMUAction
282SMMUv3::runProcessTiming(SMMUProcess *proc, PacketPtr pkt)
283{
284 SMMUAction action = proc->run(pkt);
285
286 switch (action.type) {
287 case ACTION_SEND_REQ:
288 // Send an MMU initiated request on the table walk port if it is
289 // enabled. Otherwise, fall through and handle same as the final
290 // ACTION_SEND_REQ_FINAL request.
291 if (tableWalkPortEnable) {
292 action.pkt->pushSenderState(proc);
293
294 DPRINTF(SMMUv3, "[t] master HWTW req addr=%#x size=%#x\n",
295 action.pkt->getAddr(), action.pkt->getSize());
296
297 if (packetsTableWalkToRetry.empty()
298 && masterTableWalkPort.sendTimingReq(action.pkt)) {
299 scheduleSlaveRetries();
300 } else {
301 DPRINTF(SMMUv3, "[t] master HWTW req needs retry,"
302 " qlen=%d\n", packetsTableWalkToRetry.size());
303 packetsTableWalkToRetry.push(action);
304 }
305
306 break;
307 }
308 M5_FALLTHROUGH;
309 case ACTION_SEND_REQ_FINAL:
310 action.pkt->pushSenderState(proc);
311
312 DPRINTF(SMMUv3, "[t] master req addr=%#x size=%#x\n",
313 action.pkt->getAddr(), action.pkt->getSize());
314
315 if (packetsToRetry.empty() && masterPort.sendTimingReq(action.pkt)) {
316 scheduleSlaveRetries();
317 } else {
318 DPRINTF(SMMUv3, "[t] master req needs retry, qlen=%d\n",
319 packetsToRetry.size());
320 packetsToRetry.push(action);
321 }
322
323 break;
324
325 case ACTION_SEND_RESP:
326 // @todo: We need to pay for this and not just zero it out
327 action.pkt->headerDelay = action.pkt->payloadDelay = 0;
328
329 DPRINTF(SMMUv3, "[t] slave resp addr=%#x size=%#x\n",
330 action.pkt->getAddr(),
331 action.pkt->getSize());
332
333 assert(action.ifc);
334 action.ifc->schedTimingResp(action.pkt);
335
336 delete proc;
337 break;
338
339 case ACTION_SEND_RESP_ATS:
340 // @todo: We need to pay for this and not just zero it out
341 action.pkt->headerDelay = action.pkt->payloadDelay = 0;
342
343 DPRINTF(SMMUv3, "[t] ATS slave resp addr=%#x size=%#x\n",
344 action.pkt->getAddr(), action.pkt->getSize());
345
346 assert(action.ifc);
347 action.ifc->schedAtsTimingResp(action.pkt);
348
349 delete proc;
350 break;
351
352 case ACTION_DELAY:
353 case ACTION_SLEEP:
354 break;
355
356 case ACTION_TERMINATE:
357 delete proc;
358 break;
359
360 default:
361 panic("Unknown action\n");
362 }
363
364 return action;
365}
366
367void
368SMMUv3::processCommands()
369{
370 DPRINTF(SMMUv3, "processCommands()\n");
371
372 if (system.isAtomicMode()) {
373 SMMUAction a = runProcessAtomic(&commandExecutor, NULL);
374 (void) a;
375 } else if (system.isTimingMode()) {
376 if (!commandExecutor.isBusy())
377 runProcessTiming(&commandExecutor, NULL);
378 } else {
379 panic("Not in timing or atomic mode!");
380 }
381}
382
383void
384SMMUv3::processCommand(const SMMUCommand &cmd)
385{
|
386 switch (cmd.type) {
| 386 switch (cmd.dw0.type) {
|
387 case CMD_PRF_CONFIG:
388 DPRINTF(SMMUv3, "CMD_PREFETCH_CONFIG - ignored\n");
389 break;
390
391 case CMD_PRF_ADDR:
392 DPRINTF(SMMUv3, "CMD_PREFETCH_ADDR - ignored\n");
393 break;
394
| 387 case CMD_PRF_CONFIG:
388 DPRINTF(SMMUv3, "CMD_PREFETCH_CONFIG - ignored\n");
389 break;
390
391 case CMD_PRF_ADDR:
392 DPRINTF(SMMUv3, "CMD_PREFETCH_ADDR - ignored\n");
393 break;
394
|
395 case CMD_INV_STE:
396 DPRINTF(SMMUv3, "CMD_INV_STE sid=%#x\n", cmd.data[0]);
397 configCache.invalidateSID(cmd.data[0]);
| 395 case CMD_CFGI_STE: {
396 DPRINTF(SMMUv3, "CMD_CFGI_STE sid=%#x\n", cmd.dw0.sid);
397 configCache.invalidateSID(cmd.dw0.sid);
|
398 break;
| 398 break;
|
| 399 }
|
399
| 400
|
400 case CMD_INV_CD:
401 DPRINTF(SMMUv3, "CMD_INV_CD sid=%#x ssid=%#x\n",
402 cmd.data[0], cmd.data[1]);
403 configCache.invalidateSSID(cmd.data[0], cmd.data[1]);
| 401 case CMD_CFGI_STE_RANGE: {
402 const auto range = cmd.dw1.range;
403 if (range == 31) {
404 // CMD_CFGI_ALL is an alias of CMD_CFGI_STE_RANGE with
405 // range = 31
406 DPRINTF(SMMUv3, "CMD_CFGI_ALL\n");
407 configCache.invalidateAll();
408 } else {
409 DPRINTF(SMMUv3, "CMD_CFGI_STE_RANGE\n");
410 const auto start_sid = cmd.dw0.sid & ~((1 << (range + 1)) - 1);
411 const auto end_sid = start_sid + (1 << (range + 1)) - 1;
412 for (auto sid = start_sid; sid <= end_sid; sid++)
413 configCache.invalidateSID(sid);
414 }
|
404 break;
| 415 break;
|
| 416 }
|
405
| 417
|
406 case CMD_INV_CD_ALL:
407 DPRINTF(SMMUv3, "CMD_INV_CD_ALL sid=%#x\n", cmd.data[0]);
408 configCache.invalidateSID(cmd.data[0]);
| 418 case CMD_CFGI_CD: {
419 DPRINTF(SMMUv3, "CMD_CFGI_CD sid=%#x ssid=%#x\n",
420 cmd.dw0.sid, cmd.dw0.ssid);
421 configCache.invalidateSSID(cmd.dw0.sid, cmd.dw0.ssid);
|
409 break;
| 422 break;
|
| 423 }
|
410
| 424
|
411 case CMD_INV_ALL:
412 DPRINTF(SMMUv3, "CMD_INV_ALL\n");
413 configCache.invalidateAll();
| 425 case CMD_CFGI_CD_ALL: {
426 DPRINTF(SMMUv3, "CMD_CFGI_CD_ALL sid=%#x\n", cmd.dw0.sid);
427 configCache.invalidateSID(cmd.dw0.sid);
|
414 break;
| 428 break;
|
| 429 }
|
415
| 430
|
416 case CMD_TLBI_ALL:
417 DPRINTF(SMMUv3, "CMD_TLBI_ALL\n");
| 431 case CMD_TLBI_NH_ALL: {
432 DPRINTF(SMMUv3, "CMD_TLBI_NH_ALL vmid=%#x\n", cmd.dw0.vmid);
|
418 for (auto slave_interface : slaveInterfaces) {
| 433 for (auto slave_interface : slaveInterfaces) {
|
419 slave_interface->microTLB->invalidateAll();
420 slave_interface->mainTLB->invalidateAll();
| 434 slave_interface->microTLB->invalidateVMID(cmd.dw0.vmid);
435 slave_interface->mainTLB->invalidateVMID(cmd.dw0.vmid);
|
421 }
| 436 }
|
422 tlb.invalidateAll();
423 ipaCache.invalidateAll();
424 walkCache.invalidateAll();
| 437 tlb.invalidateVMID(cmd.dw0.vmid);
438 walkCache.invalidateVMID(cmd.dw0.vmid);
|
425 break;
| 439 break;
|
| 440 }
|
426
| 441
|
427 case CMD_TLBI_ASID:
428 DPRINTF(SMMUv3, "CMD_TLBI_ASID asid=%#x vmid=%#x\n",
429 cmd.data[0], cmd.data[1]);
| 442 case CMD_TLBI_NH_ASID: {
443 DPRINTF(SMMUv3, "CMD_TLBI_NH_ASID asid=%#x vmid=%#x\n",
444 cmd.dw0.asid, cmd.dw0.vmid);
|
430 for (auto slave_interface : slaveInterfaces) {
431 slave_interface->microTLB->invalidateASID(
| 445 for (auto slave_interface : slaveInterfaces) {
446 slave_interface->microTLB->invalidateASID(
|
432 cmd.data[0], cmd.data[1]);
| 447 cmd.dw0.asid, cmd.dw0.vmid);
|
433 slave_interface->mainTLB->invalidateASID(
| 448 slave_interface->mainTLB->invalidateASID(
|
434 cmd.data[0], cmd.data[1]);
| 449 cmd.dw0.asid, cmd.dw0.vmid);
|
435 }
| 450 }
|
436 tlb.invalidateASID(cmd.data[0], cmd.data[1]);
437 walkCache.invalidateASID(cmd.data[0], cmd.data[1]);
| 451 tlb.invalidateASID(cmd.dw0.asid, cmd.dw0.vmid);
452 walkCache.invalidateASID(cmd.dw0.asid, cmd.dw0.vmid);
|
438 break;
| 453 break;
|
| 454 }
|
439
| 455
|
440 case CMD_TLBI_VAAL:
441 DPRINTF(SMMUv3, "CMD_TLBI_VAAL va=%#08x vmid=%#x\n",
442 cmd.data[0], cmd.data[1]);
| 456 case CMD_TLBI_NH_VAA: {
457 const Addr addr = cmd.addr();
458 DPRINTF(SMMUv3, "CMD_TLBI_NH_VAA va=%#08x vmid=%#x\n",
459 addr, cmd.dw0.vmid);
|
443 for (auto slave_interface : slaveInterfaces) {
444 slave_interface->microTLB->invalidateVAA(
| 460 for (auto slave_interface : slaveInterfaces) {
461 slave_interface->microTLB->invalidateVAA(
|
445 cmd.data[0], cmd.data[1]);
| 462 addr, cmd.dw0.vmid);
|
446 slave_interface->mainTLB->invalidateVAA(
| 463 slave_interface->mainTLB->invalidateVAA(
|
447 cmd.data[0], cmd.data[1]);
| 464 addr, cmd.dw0.vmid);
|
448 }
| 465 }
|
449 tlb.invalidateVAA(cmd.data[0], cmd.data[1]);
450 break;
| 466 tlb.invalidateVAA(addr, cmd.dw0.vmid);
|
451
| 467
|
452 case CMD_TLBI_VAA:
453 DPRINTF(SMMUv3, "CMD_TLBI_VAA va=%#08x vmid=%#x\n",
454 cmd.data[0], cmd.data[1]);
455 for (auto slave_interface : slaveInterfaces) {
456 slave_interface->microTLB->invalidateVAA(
457 cmd.data[0], cmd.data[1]);
458 slave_interface->mainTLB->invalidateVAA(
459 cmd.data[0], cmd.data[1]);
460 }
461 tlb.invalidateVAA(cmd.data[0], cmd.data[1]);
462 walkCache.invalidateVAA(cmd.data[0], cmd.data[1]);
| 468 if (!cmd.dw1.leaf)
469 walkCache.invalidateVAA(addr, cmd.dw0.vmid);
|
463 break;
| 470 break;
|
| 471 }
|
464
| 472
|
465 case CMD_TLBI_VAL:
466 DPRINTF(SMMUv3, "CMD_TLBI_VAL va=%#08x asid=%#x vmid=%#x\n",
467 cmd.data[0], cmd.data[1], cmd.data[2]);
| 473 case CMD_TLBI_NH_VA: {
474 const Addr addr = cmd.addr();
475 DPRINTF(SMMUv3, "CMD_TLBI_NH_VA va=%#08x asid=%#x vmid=%#x\n",
476 addr, cmd.dw0.asid, cmd.dw0.vmid);
|
468 for (auto slave_interface : slaveInterfaces) {
469 slave_interface->microTLB->invalidateVA(
| 477 for (auto slave_interface : slaveInterfaces) {
478 slave_interface->microTLB->invalidateVA(
|
470 cmd.data[0], cmd.data[1], cmd.data[2]);
| 479 addr, cmd.dw0.asid, cmd.dw0.vmid);
|
471 slave_interface->mainTLB->invalidateVA(
| 480 slave_interface->mainTLB->invalidateVA(
|
472 cmd.data[0], cmd.data[1], cmd.data[2]);
| 481 addr, cmd.dw0.asid, cmd.dw0.vmid);
|
473 }
| 482 }
|
474 tlb.invalidateVA(cmd.data[0], cmd.data[1], cmd.data[2]);
475 break;
| 483 tlb.invalidateVA(addr, cmd.dw0.asid, cmd.dw0.vmid);
|
476
| 484
|
477 case CMD_TLBI_VA:
478 DPRINTF(SMMUv3, "CMD_TLBI_VA va=%#08x asid=%#x vmid=%#x\n",
479 cmd.data[0], cmd.data[1], cmd.data[2]);
480 for (auto slave_interface : slaveInterfaces) {
481 slave_interface->microTLB->invalidateVA(
482 cmd.data[0], cmd.data[1], cmd.data[2]);
483 slave_interface->mainTLB->invalidateVA(
484 cmd.data[0], cmd.data[1], cmd.data[2]);
485 }
486 tlb.invalidateVA(cmd.data[0], cmd.data[1], cmd.data[2]);
487 walkCache.invalidateVA(cmd.data[0], cmd.data[1], cmd.data[2]);
| 485 if (!cmd.dw1.leaf)
486 walkCache.invalidateVA(addr, cmd.dw0.asid, cmd.dw0.vmid);
|
488 break;
| 487 break;
|
| 488 }
|
489
| 489
|
490 case CMD_TLBI_VM_IPAL:
491 DPRINTF(SMMUv3, "CMD_TLBI_VM_IPAL ipa=%#08x vmid=%#x\n",
492 cmd.data[0], cmd.data[1]);
| 490 case CMD_TLBI_S2_IPA: {
491 const Addr addr = cmd.addr();
492 DPRINTF(SMMUv3, "CMD_TLBI_S2_IPA ipa=%#08x vmid=%#x\n",
493 addr, cmd.dw0.vmid);
|
493 // This does not invalidate TLBs containing
494 // combined Stage1 + Stage2 translations, as per the spec.
| 494 // This does not invalidate TLBs containing
495 // combined Stage1 + Stage2 translations, as per the spec.
|
495 ipaCache.invalidateIPA(cmd.data[0], cmd.data[1]);
496 walkCache.invalidateVMID(cmd.data[1]);
| 496 ipaCache.invalidateIPA(addr, cmd.dw0.vmid);
497
498 if (!cmd.dw1.leaf)
499 walkCache.invalidateVMID(cmd.dw0.vmid);
|
497 break;
| 500 break;
|
| 501 }
|
498
| 502
|
499 case CMD_TLBI_VM_IPA:
500 DPRINTF(SMMUv3, "CMD_TLBI_VM_IPA ipa=%#08x vmid=%#x\n",
501 cmd.data[0], cmd.data[1]);
502 // This does not invalidate TLBs containing
503 // combined Stage1 + Stage2 translations, as per the spec.
504 ipaCache.invalidateIPA(cmd.data[0], cmd.data[1]);
505 walkCache.invalidateVMID(cmd.data[1]);
| 503 case CMD_TLBI_S12_VMALL: {
504 DPRINTF(SMMUv3, "CMD_TLBI_S12_VMALL vmid=%#x\n", cmd.dw0.vmid);
505 for (auto slave_interface : slaveInterfaces) {
506 slave_interface->microTLB->invalidateVMID(cmd.dw0.vmid);
507 slave_interface->mainTLB->invalidateVMID(cmd.dw0.vmid);
508 }
509 tlb.invalidateVMID(cmd.dw0.vmid);
510 ipaCache.invalidateVMID(cmd.dw0.vmid);
511 walkCache.invalidateVMID(cmd.dw0.vmid);
|
506 break;
| 512 break;
|
| 513 }
|
507
| 514
|
508 case CMD_TLBI_VM_S12:
509 DPRINTF(SMMUv3, "CMD_TLBI_VM_S12 vmid=%#x\n", cmd.data[0]);
| 515 case CMD_TLBI_NSNH_ALL: {
516 DPRINTF(SMMUv3, "CMD_TLBI_NSNH_ALL\n");
|
510 for (auto slave_interface : slaveInterfaces) {
| 517 for (auto slave_interface : slaveInterfaces) {
|
511 slave_interface->microTLB->invalidateVMID(cmd.data[0]);
512 slave_interface->mainTLB->invalidateVMID(cmd.data[0]);
| 518 slave_interface->microTLB->invalidateAll();
519 slave_interface->mainTLB->invalidateAll();
|
513 }
| 520 }
|
514 tlb.invalidateVMID(cmd.data[0]);
515 ipaCache.invalidateVMID(cmd.data[0]);
516 walkCache.invalidateVMID(cmd.data[0]);
| 521 tlb.invalidateAll();
522 ipaCache.invalidateAll();
523 walkCache.invalidateAll();
|
517 break;
| 524 break;
|
| 525 }
|
518
| 526
|
519 case CMD_RESUME_S:
520 DPRINTF(SMMUv3, "CMD_RESUME_S\n");
| 527 case CMD_RESUME:
528 DPRINTF(SMMUv3, "CMD_RESUME\n");
|
521 panic("resume unimplemented");
522 break;
523
524 default:
| 529 panic("resume unimplemented");
530 break;
531
532 default:
|
525 warn("Unimplemented command %#x\n", cmd.type);
| 533 warn("Unimplemented command %#x\n", cmd.dw0.type);
|
526 break;
527 }
528}
529
530const PageTableOps*
531SMMUv3::getPageTableOps(uint8_t trans_granule)
532{
533 static V8PageTableOps4k ptOps4k;
534 static V8PageTableOps16k ptOps16k;
535 static V8PageTableOps64k ptOps64k;
536
537 switch (trans_granule) {
538 case TRANS_GRANULE_4K: return &ptOps4k;
539 case TRANS_GRANULE_16K: return &ptOps16k;
540 case TRANS_GRANULE_64K: return &ptOps64k;
541 default:
542 panic("Unknown translation granule size %d", trans_granule);
543 }
544}
545
546Tick
547SMMUv3::readControl(PacketPtr pkt)
548{
549 DPRINTF(SMMUv3, "readControl: addr=%08x size=%d\n",
550 pkt->getAddr(), pkt->getSize());
551
552 int offset = pkt->getAddr() - regsMap.start();
553 assert(offset >= 0 && offset < SMMU_REG_SIZE);
554
555 if (inSecureBlock(offset)) {
556 warn("smmu: secure registers (0x%x) are not implemented\n",
557 offset);
558 }
559
560 auto reg_ptr = regs.data + offset;
561
562 switch (pkt->getSize()) {
563 case sizeof(uint32_t):
564 pkt->setLE<uint32_t>(*reinterpret_cast<uint32_t *>(reg_ptr));
565 break;
566 case sizeof(uint64_t):
567 pkt->setLE<uint64_t>(*reinterpret_cast<uint64_t *>(reg_ptr));
568 break;
569 default:
570 panic("smmu: unallowed access size: %d bytes\n", pkt->getSize());
571 break;
572 }
573
574 pkt->makeAtomicResponse();
575
576 return 0;
577}
578
579Tick
580SMMUv3::writeControl(PacketPtr pkt)
581{
582 int offset = pkt->getAddr() - regsMap.start();
583 assert(offset >= 0 && offset < SMMU_REG_SIZE);
584
585 DPRINTF(SMMUv3, "writeControl: addr=%08x size=%d data=%16x\n",
586 pkt->getAddr(), pkt->getSize(),
587 pkt->getSize() == sizeof(uint64_t) ?
588 pkt->getLE<uint64_t>() : pkt->getLE<uint32_t>());
589
590 switch (offset) {
591 case offsetof(SMMURegs, cr0):
592 assert(pkt->getSize() == sizeof(uint32_t));
593 regs.cr0 = regs.cr0ack = pkt->getLE<uint32_t>();
594 break;
595
596 case offsetof(SMMURegs, cr1):
597 case offsetof(SMMURegs, cr2):
598 case offsetof(SMMURegs, strtab_base_cfg):
599 case offsetof(SMMURegs, eventq_cons):
600 case offsetof(SMMURegs, eventq_irq_cfg1):
601 case offsetof(SMMURegs, priq_cons):
602 assert(pkt->getSize() == sizeof(uint32_t));
603 *reinterpret_cast<uint32_t *>(regs.data + offset) =
604 pkt->getLE<uint32_t>();
605 break;
606
607 case offsetof(SMMURegs, cmdq_cons):
608 assert(pkt->getSize() == sizeof(uint32_t));
609 if (regs.cr0 & CR0_CMDQEN_MASK) {
610 warn("CMDQ is enabled: ignoring write to CMDQ_CONS\n");
611 } else {
612 *reinterpret_cast<uint32_t *>(regs.data + offset) =
613 pkt->getLE<uint32_t>();
614 }
615 break;
616
617 case offsetof(SMMURegs, cmdq_prod):
618 assert(pkt->getSize() == sizeof(uint32_t));
619 *reinterpret_cast<uint32_t *>(regs.data + offset) =
620 pkt->getLE<uint32_t>();
621 schedule(processCommandsEvent, nextCycle());
622 break;
623
624 case offsetof(SMMURegs, strtab_base):
625 case offsetof(SMMURegs, eventq_irq_cfg0):
626 assert(pkt->getSize() == sizeof(uint64_t));
627 *reinterpret_cast<uint64_t *>(regs.data + offset) =
628 pkt->getLE<uint64_t>();
629 break;
630
631 case offsetof(SMMURegs, cmdq_base):
632 assert(pkt->getSize() == sizeof(uint64_t));
633 if (regs.cr0 & CR0_CMDQEN_MASK) {
634 warn("CMDQ is enabled: ignoring write to CMDQ_BASE\n");
635 } else {
636 *reinterpret_cast<uint64_t *>(regs.data + offset) =
637 pkt->getLE<uint64_t>();
638 regs.cmdq_cons = 0;
639 regs.cmdq_prod = 0;
640 }
641 break;
642
643 case offsetof(SMMURegs, eventq_base):
644 assert(pkt->getSize() == sizeof(uint64_t));
645 *reinterpret_cast<uint64_t *>(regs.data + offset) =
646 pkt->getLE<uint64_t>();
647 regs.eventq_cons = 0;
648 regs.eventq_prod = 0;
649 break;
650
651 case offsetof(SMMURegs, priq_base):
652 assert(pkt->getSize() == sizeof(uint64_t));
653 *reinterpret_cast<uint64_t *>(regs.data + offset) =
654 pkt->getLE<uint64_t>();
655 regs.priq_cons = 0;
656 regs.priq_prod = 0;
657 break;
658
659 default:
660 if (inSecureBlock(offset)) {
661 warn("smmu: secure registers (0x%x) are not implemented\n",
662 offset);
663 } else {
664 warn("smmu: write to read-only/undefined register at 0x%x\n",
665 offset);
666 }
667 }
668
669 pkt->makeAtomicResponse();
670
671 return 0;
672}
673
674bool
675SMMUv3::inSecureBlock(uint32_t offs) const
676{
677 if (offs >= offsetof(SMMURegs, _secure_regs) && offs < SMMU_SECURE_SZ)
678 return true;
679 else
680 return false;
681}
682
683void
684SMMUv3::init()
685{
686 // make sure both sides are connected and have the same block size
687 if (!masterPort.isConnected())
688 fatal("Master port is not connected.\n");
689
690 // If the second master port is connected for the table walks, enable
691 // the mode to send table walks through this port instead
692 if (masterTableWalkPort.isConnected())
693 tableWalkPortEnable = true;
694
695 // notify the master side of our address ranges
696 for (auto ifc : slaveInterfaces) {
697 ifc->sendRange();
698 }
699
700 if (controlPort.isConnected())
701 controlPort.sendRangeChange();
702}
703
704void
705SMMUv3::regStats()
706{
707 MemObject::regStats();
708
709 using namespace Stats;
710
711 for (size_t i = 0; i < slaveInterfaces.size(); i++) {
712 slaveInterfaces[i]->microTLB->regStats(
713 csprintf("%s.utlb%d", name(), i));
714 slaveInterfaces[i]->mainTLB->regStats(
715 csprintf("%s.maintlb%d", name(), i));
716 }
717
718 tlb.regStats(name() + ".tlb");
719 configCache.regStats(name() + ".cfg");
720 ipaCache.regStats(name() + ".ipa");
721 walkCache.regStats(name() + ".walk");
722
723 steL1Fetches
724 .name(name() + ".steL1Fetches")
725 .desc("STE L1 fetches")
726 .flags(pdf);
727
728 steFetches
729 .name(name() + ".steFetches")
730 .desc("STE fetches")
731 .flags(pdf);
732
733 cdL1Fetches
734 .name(name() + ".cdL1Fetches")
735 .desc("CD L1 fetches")
736 .flags(pdf);
737
738 cdFetches
739 .name(name() + ".cdFetches")
740 .desc("CD fetches")
741 .flags(pdf);
742
743 translationTimeDist
744 .init(0, 2000000, 2000)
745 .name(name() + ".translationTimeDist")
746 .desc("Time to translate address")
747 .flags(pdf);
748
749 ptwTimeDist
750 .init(0, 2000000, 2000)
751 .name(name() + ".ptwTimeDist")
752 .desc("Time to walk page tables")
753 .flags(pdf);
754}
755
756DrainState
757SMMUv3::drain()
758{
759 // Wait until the Command Executor is not busy
760 if (commandExecutor.isBusy()) {
761 return DrainState::Draining;
762 }
763 return DrainState::Drained;
764}
765
766void
767SMMUv3::serialize(CheckpointOut &cp) const
768{
769 DPRINTF(Checkpoint, "Serializing SMMUv3\n");
770
771 SERIALIZE_ARRAY(regs.data, sizeof(regs.data) / sizeof(regs.data[0]));
772}
773
774void
775SMMUv3::unserialize(CheckpointIn &cp)
776{
777 DPRINTF(Checkpoint, "Unserializing SMMUv3\n");
778
779 UNSERIALIZE_ARRAY(regs.data, sizeof(regs.data) / sizeof(regs.data[0]));
780}
781
782Port&
783SMMUv3::getPort(const std::string &name, PortID id)
784{
785 if (name == "master") {
786 return masterPort;
787 } else if (name == "master_walker") {
788 return masterTableWalkPort;
789 } else if (name == "control") {
790 return controlPort;
791 } else {
792 return MemObject::getPort(name, id);
793 }
794}
795
796SMMUv3*
797SMMUv3Params::create()
798{
799 return new SMMUv3(this);
800}
| 534 break;
535 }
536}
537
538const PageTableOps*
539SMMUv3::getPageTableOps(uint8_t trans_granule)
540{
541 static V8PageTableOps4k ptOps4k;
542 static V8PageTableOps16k ptOps16k;
543 static V8PageTableOps64k ptOps64k;
544
545 switch (trans_granule) {
546 case TRANS_GRANULE_4K: return &ptOps4k;
547 case TRANS_GRANULE_16K: return &ptOps16k;
548 case TRANS_GRANULE_64K: return &ptOps64k;
549 default:
550 panic("Unknown translation granule size %d", trans_granule);
551 }
552}
553
554Tick
555SMMUv3::readControl(PacketPtr pkt)
556{
557 DPRINTF(SMMUv3, "readControl: addr=%08x size=%d\n",
558 pkt->getAddr(), pkt->getSize());
559
560 int offset = pkt->getAddr() - regsMap.start();
561 assert(offset >= 0 && offset < SMMU_REG_SIZE);
562
563 if (inSecureBlock(offset)) {
564 warn("smmu: secure registers (0x%x) are not implemented\n",
565 offset);
566 }
567
568 auto reg_ptr = regs.data + offset;
569
570 switch (pkt->getSize()) {
571 case sizeof(uint32_t):
572 pkt->setLE<uint32_t>(*reinterpret_cast<uint32_t *>(reg_ptr));
573 break;
574 case sizeof(uint64_t):
575 pkt->setLE<uint64_t>(*reinterpret_cast<uint64_t *>(reg_ptr));
576 break;
577 default:
578 panic("smmu: unallowed access size: %d bytes\n", pkt->getSize());
579 break;
580 }
581
582 pkt->makeAtomicResponse();
583
584 return 0;
585}
586
587Tick
588SMMUv3::writeControl(PacketPtr pkt)
589{
590 int offset = pkt->getAddr() - regsMap.start();
591 assert(offset >= 0 && offset < SMMU_REG_SIZE);
592
593 DPRINTF(SMMUv3, "writeControl: addr=%08x size=%d data=%16x\n",
594 pkt->getAddr(), pkt->getSize(),
595 pkt->getSize() == sizeof(uint64_t) ?
596 pkt->getLE<uint64_t>() : pkt->getLE<uint32_t>());
597
598 switch (offset) {
599 case offsetof(SMMURegs, cr0):
600 assert(pkt->getSize() == sizeof(uint32_t));
601 regs.cr0 = regs.cr0ack = pkt->getLE<uint32_t>();
602 break;
603
604 case offsetof(SMMURegs, cr1):
605 case offsetof(SMMURegs, cr2):
606 case offsetof(SMMURegs, strtab_base_cfg):
607 case offsetof(SMMURegs, eventq_cons):
608 case offsetof(SMMURegs, eventq_irq_cfg1):
609 case offsetof(SMMURegs, priq_cons):
610 assert(pkt->getSize() == sizeof(uint32_t));
611 *reinterpret_cast<uint32_t *>(regs.data + offset) =
612 pkt->getLE<uint32_t>();
613 break;
614
615 case offsetof(SMMURegs, cmdq_cons):
616 assert(pkt->getSize() == sizeof(uint32_t));
617 if (regs.cr0 & CR0_CMDQEN_MASK) {
618 warn("CMDQ is enabled: ignoring write to CMDQ_CONS\n");
619 } else {
620 *reinterpret_cast<uint32_t *>(regs.data + offset) =
621 pkt->getLE<uint32_t>();
622 }
623 break;
624
625 case offsetof(SMMURegs, cmdq_prod):
626 assert(pkt->getSize() == sizeof(uint32_t));
627 *reinterpret_cast<uint32_t *>(regs.data + offset) =
628 pkt->getLE<uint32_t>();
629 schedule(processCommandsEvent, nextCycle());
630 break;
631
632 case offsetof(SMMURegs, strtab_base):
633 case offsetof(SMMURegs, eventq_irq_cfg0):
634 assert(pkt->getSize() == sizeof(uint64_t));
635 *reinterpret_cast<uint64_t *>(regs.data + offset) =
636 pkt->getLE<uint64_t>();
637 break;
638
639 case offsetof(SMMURegs, cmdq_base):
640 assert(pkt->getSize() == sizeof(uint64_t));
641 if (regs.cr0 & CR0_CMDQEN_MASK) {
642 warn("CMDQ is enabled: ignoring write to CMDQ_BASE\n");
643 } else {
644 *reinterpret_cast<uint64_t *>(regs.data + offset) =
645 pkt->getLE<uint64_t>();
646 regs.cmdq_cons = 0;
647 regs.cmdq_prod = 0;
648 }
649 break;
650
651 case offsetof(SMMURegs, eventq_base):
652 assert(pkt->getSize() == sizeof(uint64_t));
653 *reinterpret_cast<uint64_t *>(regs.data + offset) =
654 pkt->getLE<uint64_t>();
655 regs.eventq_cons = 0;
656 regs.eventq_prod = 0;
657 break;
658
659 case offsetof(SMMURegs, priq_base):
660 assert(pkt->getSize() == sizeof(uint64_t));
661 *reinterpret_cast<uint64_t *>(regs.data + offset) =
662 pkt->getLE<uint64_t>();
663 regs.priq_cons = 0;
664 regs.priq_prod = 0;
665 break;
666
667 default:
668 if (inSecureBlock(offset)) {
669 warn("smmu: secure registers (0x%x) are not implemented\n",
670 offset);
671 } else {
672 warn("smmu: write to read-only/undefined register at 0x%x\n",
673 offset);
674 }
675 }
676
677 pkt->makeAtomicResponse();
678
679 return 0;
680}
681
682bool
683SMMUv3::inSecureBlock(uint32_t offs) const
684{
685 if (offs >= offsetof(SMMURegs, _secure_regs) && offs < SMMU_SECURE_SZ)
686 return true;
687 else
688 return false;
689}
690
691void
692SMMUv3::init()
693{
694 // make sure both sides are connected and have the same block size
695 if (!masterPort.isConnected())
696 fatal("Master port is not connected.\n");
697
698 // If the second master port is connected for the table walks, enable
699 // the mode to send table walks through this port instead
700 if (masterTableWalkPort.isConnected())
701 tableWalkPortEnable = true;
702
703 // notify the master side of our address ranges
704 for (auto ifc : slaveInterfaces) {
705 ifc->sendRange();
706 }
707
708 if (controlPort.isConnected())
709 controlPort.sendRangeChange();
710}
711
712void
713SMMUv3::regStats()
714{
715 MemObject::regStats();
716
717 using namespace Stats;
718
719 for (size_t i = 0; i < slaveInterfaces.size(); i++) {
720 slaveInterfaces[i]->microTLB->regStats(
721 csprintf("%s.utlb%d", name(), i));
722 slaveInterfaces[i]->mainTLB->regStats(
723 csprintf("%s.maintlb%d", name(), i));
724 }
725
726 tlb.regStats(name() + ".tlb");
727 configCache.regStats(name() + ".cfg");
728 ipaCache.regStats(name() + ".ipa");
729 walkCache.regStats(name() + ".walk");
730
731 steL1Fetches
732 .name(name() + ".steL1Fetches")
733 .desc("STE L1 fetches")
734 .flags(pdf);
735
736 steFetches
737 .name(name() + ".steFetches")
738 .desc("STE fetches")
739 .flags(pdf);
740
741 cdL1Fetches
742 .name(name() + ".cdL1Fetches")
743 .desc("CD L1 fetches")
744 .flags(pdf);
745
746 cdFetches
747 .name(name() + ".cdFetches")
748 .desc("CD fetches")
749 .flags(pdf);
750
751 translationTimeDist
752 .init(0, 2000000, 2000)
753 .name(name() + ".translationTimeDist")
754 .desc("Time to translate address")
755 .flags(pdf);
756
757 ptwTimeDist
758 .init(0, 2000000, 2000)
759 .name(name() + ".ptwTimeDist")
760 .desc("Time to walk page tables")
761 .flags(pdf);
762}
763
764DrainState
765SMMUv3::drain()
766{
767 // Wait until the Command Executor is not busy
768 if (commandExecutor.isBusy()) {
769 return DrainState::Draining;
770 }
771 return DrainState::Drained;
772}
773
774void
775SMMUv3::serialize(CheckpointOut &cp) const
776{
777 DPRINTF(Checkpoint, "Serializing SMMUv3\n");
778
779 SERIALIZE_ARRAY(regs.data, sizeof(regs.data) / sizeof(regs.data[0]));
780}
781
782void
783SMMUv3::unserialize(CheckpointIn &cp)
784{
785 DPRINTF(Checkpoint, "Unserializing SMMUv3\n");
786
787 UNSERIALIZE_ARRAY(regs.data, sizeof(regs.data) / sizeof(regs.data[0]));
788}
789
790Port&
791SMMUv3::getPort(const std::string &name, PortID id)
792{
793 if (name == "master") {
794 return masterPort;
795 } else if (name == "master_walker") {
796 return masterTableWalkPort;
797 } else if (name == "control") {
798 return controlPort;
799 } else {
800 return MemObject::getPort(name, id);
801 }
802}
803
804SMMUv3*
805SMMUv3Params::create()
806{
807 return new SMMUv3(this);
808}
|