1// Copyright (c) 2017-2018 ARM Limited
2// All rights reserved
3//
4// The license below extends only to copyright in the software and shall
5// not be construed as granting a license to any other intellectual
6// property including but not limited to intellectual property relating
7// to a hardware implementation of the functionality of the software
8// licensed hereunder. You may use the software subject to the license
9// terms below provided that you ensure that this notice is replicated
10// unmodified and in its entirety in all distributions of the software,
11// modified or unmodified, in source code or in binary form.
12//
13// Redistribution and use in source and binary forms, with or without
14// modification, are permitted provided that the following conditions are
15// met: redistributions of source code must retain the above copyright
16// notice, this list of conditions and the following disclaimer;
17// redistributions in binary form must reproduce the above copyright
18// notice, this list of conditions and the following disclaimer in the
19// documentation and/or other materials provided with the distribution;
20// neither the name of the copyright holders nor the names of its
21// contributors may be used to endorse or promote products derived from
22// this software without specific prior written permission.
23//
24// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
27// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
30// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35//
36// Authors: Giacomo Gabrielli
37
38// @file Definition of SVE memory access instructions.
39
40output header {{
41
42 // Decodes SVE contiguous load instructions, scalar plus scalar form.
43 template <template <typename T1, typename T2> class Base>
44 StaticInstPtr
45 decodeSveContigLoadSSInsts(uint8_t dtype, ExtMachInst machInst,
46 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
47 IntRegIndex rm, bool firstFaulting)
48 {
49 const char* mn = firstFaulting ? "ldff1" : "ld1";
50 switch (dtype) {
51 case 0x0:
52 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
53 case 0x1:
54 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
55 case 0x2:
56 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
57 case 0x3:
58 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
59 case 0x4:
60 return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, rm);
61 case 0x5:
62 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
63 case 0x6:
64 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
65 case 0x7:
66 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
67 case 0x8:
68 return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, rm);
69 case 0x9:
70 return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, rm);
71 case 0xa:
72 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
73 case 0xb:
74 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
75 case 0xc:
76 return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, rm);
77 case 0xd:
78 return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, rm);
79 case 0xe:
80 return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, rm);
81 case 0xf:
82 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm);
83 }
84 return new Unknown64(machInst);
85 }
86
87 // Decodes SVE contiguous load instructions, scalar plus immediate form.
88 template <template <typename T1, typename T2> class Base>
89 StaticInstPtr
90 decodeSveContigLoadSIInsts(uint8_t dtype, ExtMachInst machInst,
91 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
92 uint64_t imm, bool nonFaulting,
93 bool replicate = false)
94 {
95 assert(!(nonFaulting && replicate));
96 const char* mn = replicate ? "ld1r" : (nonFaulting ? "ldnf1" : "ld1");
97 switch (dtype) {
98 case 0x0:
99 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
100 case 0x1:
101 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
102 case 0x2:
103 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
104 case 0x3:
105 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
106 case 0x4:
107 return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, imm);
108 case 0x5:
109 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
110 case 0x6:
111 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
112 case 0x7:
113 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
114 case 0x8:
115 return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, imm);
116 case 0x9:
117 return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, imm);
118 case 0xa:
119 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
120 case 0xb:
121 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
122 case 0xc:
123 return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, imm);
124 case 0xd:
125 return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, imm);
126 case 0xe:
127 return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, imm);
128 case 0xf:
129 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm);
130 }
131 return new Unknown64(machInst);
132 }
133
134 // Decodes SVE contiguous store instructions, scalar plus scalar form.
135 template <template <typename T1, typename T2> class Base>
136 StaticInstPtr
137 decodeSveContigStoreSSInsts(uint8_t dtype, ExtMachInst machInst,
138 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
139 IntRegIndex rm)
140 {
141 const char* mn = "st1";
142 switch (dtype) {
143 case 0x0:
144 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
145 case 0x1:
146 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
147 case 0x2:
148 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
149 case 0x3:
150 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
151 case 0x5:
152 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
153 case 0x6:
154 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
155 case 0x7:
156 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
157 case 0xa:
158 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
159 case 0xb:
160 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
161 case 0xf:
162 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm);
163 }
164 return new Unknown64(machInst);
165 }
166
167 // Decodes SVE contiguous store instructions, scalar plus immediate form.
168 template <template <typename T1, typename T2> class Base>
169 StaticInstPtr
170 decodeSveContigStoreSIInsts(uint8_t dtype, ExtMachInst machInst,
171 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
172 int8_t imm)
173 {
174 const char* mn = "st1";
175 switch (dtype) {
176 case 0x0:
177 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
178 case 0x1:
179 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
180 case 0x2:
181 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
182 case 0x3:
183 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
184 case 0x5:
185 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
186 case 0x6:
187 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
188 case 0x7:
189 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
190 case 0xa:
191 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
192 case 0xb:
193 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
194 case 0xf:
195 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm);
196 }
197 return new Unknown64(machInst);
198 }
199
200 // NOTE: SVE load-and-replicate instructions are decoded with
201 // decodeSveContigLoadSIInsts(...).
202
203}};
204
205output decoder {{
206
207 template <class etype>
208 StaticInstPtr
209 decodeSveStructLoadSIInstsByNReg(uint8_t esize, ExtMachInst machInst,
210 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
211 int64_t imm, int numregs)
212 {
213 static const char* nm[5][4] = {
214 { nullptr, nullptr, nullptr, nullptr},
215 { nullptr, nullptr, nullptr, nullptr},
216 { "ld2b", "ld2h", "ld2w", "ld2d" },
217 { "ld3b", "ld3h", "ld3w", "ld3d" },
218 { "ld4b", "ld4h", "ld4w", "ld4d" } };
219
220 switch (numregs) {
221 case 2:
222 return new SveLdStructSI<etype,
223 SveLoadRegImmMicroop,
224 SveDeIntrlv2Microop>(
225 nm[numregs][esize], machInst, MemReadOp,
226 zt, pg, xn, imm, numregs);
227 case 3:
228 return new SveLdStructSI<etype,
229 SveLoadRegImmMicroop,
230 SveDeIntrlv3Microop>(
231 nm[numregs][esize], machInst, MemReadOp,
232 zt, pg, xn, imm, numregs);
233 case 4:
234 return new SveLdStructSI<etype,
235 SveLoadRegImmMicroop,
236 SveDeIntrlv4Microop>(
237 nm[numregs][esize], machInst, MemReadOp,
238 zt, pg, xn, imm, numregs);
239 }
240 return new Unknown64(machInst);
241 }
242
243 StaticInstPtr
244 decodeSveStructLoadSIInsts(uint8_t esize, ExtMachInst machInst,
245 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
246 int64_t imm, int numregs)
247 {
248 switch (esize) {
249 case 0:
250 return decodeSveStructLoadSIInstsByNReg<uint8_t>(esize,
251 machInst, zt, pg, xn, imm, numregs);
252 case 1:
253 return decodeSveStructLoadSIInstsByNReg<uint16_t>(esize,
254 machInst, zt, pg, xn, imm, numregs);
255 case 2:
256 return decodeSveStructLoadSIInstsByNReg<uint32_t>(esize,
257 machInst, zt, pg, xn, imm, numregs);
258 case 3:
259 return decodeSveStructLoadSIInstsByNReg<uint64_t>(esize,
260 machInst, zt, pg, xn, imm, numregs);
261 }
262 return new Unknown64(machInst);
263 }
264
265 template <class etype>
266 StaticInstPtr
267 decodeSveStructStoreSIInstsByNReg(uint8_t esize, ExtMachInst machInst,
268 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
269 int64_t imm, int numregs)
270 {
271 static const char* nm[5][4] = {
272 { nullptr, nullptr, nullptr, nullptr},
273 { nullptr, nullptr, nullptr, nullptr},
274 { "st2b", "st2h", "st2w", "st2d" },
275 { "st3b", "st3h", "st3w", "st3d" },
276 { "st4b", "st4h", "st4w", "st4d" } };
277
278 switch (numregs) {
279 case 2:
280 return new SveStStructSI<etype,
281 SveStoreRegImmMicroop,
282 SveIntrlv2Microop>(
283 nm[numregs][esize], machInst, MemWriteOp,
284 zt, pg, xn, imm, numregs);
285 case 3:
286 return new SveStStructSI<etype,
287 SveStoreRegImmMicroop,
288 SveIntrlv3Microop>(
289 nm[numregs][esize], machInst, MemWriteOp,
290 zt, pg, xn, imm, numregs);
291 case 4:
292 return new SveStStructSI<etype,
293 SveStoreRegImmMicroop,
294 SveIntrlv4Microop>(
295 nm[numregs][esize], machInst, MemWriteOp,
296 zt, pg, xn, imm, numregs);
297 }
298 return new Unknown64(machInst);
299 }
300
301 StaticInstPtr
302 decodeSveStructStoreSIInsts(uint8_t esize, ExtMachInst machInst,
303 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
304 int64_t imm, int numregs)
305 {
306 switch (esize) {
307 case 0:
308 return decodeSveStructStoreSIInstsByNReg<uint8_t>(esize,
309 machInst, zt, pg, xn, imm, numregs);
310 case 1:
311 return decodeSveStructStoreSIInstsByNReg<uint16_t>(esize,
312 machInst, zt, pg, xn, imm, numregs);
313 case 2:
314 return decodeSveStructStoreSIInstsByNReg<uint32_t>(esize,
315 machInst, zt, pg, xn, imm, numregs);
316 case 3:
317 return decodeSveStructStoreSIInstsByNReg<uint64_t>(esize,
318 machInst, zt, pg, xn, imm, numregs);
319 }
320 return new Unknown64(machInst);
321 }
322
323 template <class etype>
324 StaticInstPtr
325 decodeSveStructLoadSSInstsByNReg(uint8_t esize, ExtMachInst machInst,
326 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
327 IntRegIndex xm, int numregs)
328 {
329 static const char* nm[5][4] = {
330 { nullptr, nullptr, nullptr, nullptr},
331 { nullptr, nullptr, nullptr, nullptr},
332 { "ld2b", "ld2h", "ld2w", "ld2d" },
333 { "ld3b", "ld3h", "ld3w", "ld3d" },
334 { "ld4b", "ld4h", "ld4w", "ld4d" } };
335
336 switch (numregs) {
337 case 2:
338 return new SveLdStructSS<etype,
339 SveLoadRegRegMicroop,
340 SveDeIntrlv2Microop>(
341 nm[numregs][esize], machInst, MemReadOp,
342 zt, pg, xn, xm, numregs);
343 case 3:
344 return new SveLdStructSS<etype,
345 SveLoadRegRegMicroop,
346 SveDeIntrlv3Microop>(
347 nm[numregs][esize], machInst, MemReadOp,
348 zt, pg, xn, xm, numregs);
349 case 4:
350 return new SveLdStructSS<etype,
351 SveLoadRegRegMicroop,
352 SveDeIntrlv4Microop>(
353 nm[numregs][esize], machInst, MemReadOp,
354 zt, pg, xn, xm, numregs);
355 }
356 return new Unknown64(machInst);
357 }
358
359 StaticInstPtr
360 decodeSveStructLoadSSInsts(uint8_t esize, ExtMachInst machInst,
361 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
362 IntRegIndex xm, int numregs)
363 {
364 switch (esize) {
365 case 0:
366 return decodeSveStructLoadSSInstsByNReg<uint8_t>(esize,
367 machInst, zt, pg, xn, xm, numregs);
368 case 1:
369 return decodeSveStructLoadSSInstsByNReg<uint16_t>(esize,
370 machInst, zt, pg, xn, xm, numregs);
371 case 2:
372 return decodeSveStructLoadSSInstsByNReg<uint32_t>(esize,
373 machInst, zt, pg, xn, xm, numregs);
374 case 3:
375 return decodeSveStructLoadSSInstsByNReg<uint64_t>(esize,
376 machInst, zt, pg, xn, xm, numregs);
377 }
378 return new Unknown64(machInst);
379 }
380
381 template <class etype>
382 StaticInstPtr
383 decodeSveStructStoreSSInstsByNReg(uint8_t esize, ExtMachInst machInst,
384 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
385 IntRegIndex xm, int numregs)
386 {
387 static const char* nm[5][4] = {
388 { nullptr, nullptr, nullptr, nullptr},
389 { nullptr, nullptr, nullptr, nullptr},
390 { "st2b", "st2h", "st2w", "st2d" },
391 { "st3b", "st3h", "st3w", "st3d" },
392 { "st4b", "st4h", "st4w", "st4d" } };
393
394 switch (numregs) {
395 case 2:
396 return new SveStStructSS<etype,
397 SveStoreRegRegMicroop,
398 SveIntrlv2Microop>(
399 nm[numregs][esize], machInst, MemWriteOp,
400 zt, pg, xn, xm, numregs);
401 case 3:
402 return new SveStStructSS<etype,
403 SveStoreRegRegMicroop,
404 SveIntrlv3Microop>(
405 nm[numregs][esize], machInst, MemWriteOp,
406 zt, pg, xn, xm, numregs);
407 case 4:
408 return new SveStStructSS<etype,
409 SveStoreRegRegMicroop,
410 SveIntrlv4Microop>(
411 nm[numregs][esize], machInst, MemWriteOp,
412 zt, pg, xn, xm, numregs);
413 }
414 return new Unknown64(machInst);
415 }
416
417 StaticInstPtr
418 decodeSveStructStoreSSInsts(uint8_t esize, ExtMachInst machInst,
419 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
420 IntRegIndex xm, int numregs)
421 {
422 switch (esize) {
423 case 0:
424 return decodeSveStructStoreSSInstsByNReg<uint8_t>(esize,
425 machInst, zt, pg, xn, xm, numregs);
426 case 1:
427 return decodeSveStructStoreSSInstsByNReg<uint16_t>(esize,
428 machInst, zt, pg, xn, xm, numregs);
429 case 2:
430 return decodeSveStructStoreSSInstsByNReg<uint32_t>(esize,
431 machInst, zt, pg, xn, xm, numregs);
432 case 3:
433 return decodeSveStructStoreSSInstsByNReg<uint64_t>(esize,
434 machInst, zt, pg, xn, xm, numregs);
435 }
436 return new Unknown64(machInst);
437 }
438
439 StaticInstPtr
440 decodeSveGatherLoadVIInsts(uint8_t dtype, ExtMachInst machInst,
441 IntRegIndex zt, IntRegIndex pg, IntRegIndex zn,
442 uint64_t imm, bool esizeIs32,
443 bool firstFault)
444 {
445 const char* mn = firstFault ? "ldff1" : "ld1";
446 switch (dtype) {
447 case 0x0:
448 if (esizeIs32) {
449 return new SveIndexedMemVI<int32_t, int8_t,
450 SveGatherLoadVIMicroop,
451 SveFirstFaultWritebackMicroop>(
452 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
453 } else {
454 return new SveIndexedMemVI<int64_t, int8_t,
455 SveGatherLoadVIMicroop,
456 SveFirstFaultWritebackMicroop>(
457 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
458 }
459 case 0x1:
460 if (esizeIs32) {
461 return new SveIndexedMemVI<uint32_t, uint8_t,
462 SveGatherLoadVIMicroop,
463 SveFirstFaultWritebackMicroop>(
464 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
465 } else {
466 return new SveIndexedMemVI<uint64_t, uint8_t,
467 SveGatherLoadVIMicroop,
468 SveFirstFaultWritebackMicroop>(
469 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
470 }
471 case 0x2:
472 if (esizeIs32) {
473 return new SveIndexedMemVI<int32_t, int16_t,
474 SveGatherLoadVIMicroop,
475 SveFirstFaultWritebackMicroop>(
476 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
477 } else {
478 return new SveIndexedMemVI<int64_t, int16_t,
479 SveGatherLoadVIMicroop,
480 SveFirstFaultWritebackMicroop>(
481 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
482 }
483 case 0x3:
484 if (esizeIs32) {
485 return new SveIndexedMemVI<uint32_t, uint16_t,
486 SveGatherLoadVIMicroop,
487 SveFirstFaultWritebackMicroop>(
488 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
489 } else {
490 return new SveIndexedMemVI<uint64_t, uint16_t,
491 SveGatherLoadVIMicroop,
492 SveFirstFaultWritebackMicroop>(
493 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
494 }
495 case 0x4:
496 if (esizeIs32) {
497 break;
498 } else {
499 return new SveIndexedMemVI<int64_t, int32_t,
500 SveGatherLoadVIMicroop,
501 SveFirstFaultWritebackMicroop>(
502 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
503 }
504 case 0x5:
505 if (esizeIs32) {
506 return new SveIndexedMemVI<uint32_t, uint32_t,
507 SveGatherLoadVIMicroop,
508 SveFirstFaultWritebackMicroop>(
509 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
510 } else {
511 return new SveIndexedMemVI<uint64_t, uint32_t,
512 SveGatherLoadVIMicroop,
513 SveFirstFaultWritebackMicroop>(
514 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
515 }
516 case 0x7:
517 if (esizeIs32) {
518 break;
519 } else {
520 return new SveIndexedMemVI<uint64_t, uint64_t,
521 SveGatherLoadVIMicroop,
522 SveFirstFaultWritebackMicroop>(
523 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
524 }
525 }
526 return new Unknown64(machInst);
527 }
528
529 StaticInstPtr
530 decodeSveGatherLoadSVInsts(uint8_t dtype, ExtMachInst machInst,
531 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
532 IntRegIndex zm, bool esizeIs32, bool offsetIs32,
533 bool offsetIsSigned, bool offsetIsScaled,
534 bool firstFault)
535 {
536 const char* mn = firstFault ? "ldff1" : "ld1";
537 switch (dtype) {
538 case 0x0:
539 if (esizeIs32) {
540 return new SveIndexedMemSV<int32_t, int8_t,
541 SveGatherLoadSVMicroop,
542 SveFirstFaultWritebackMicroop>(
543 mn, machInst, MemReadOp, zt, pg, rn, zm,
544 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
545 } else {
546 return new SveIndexedMemSV<int64_t, int8_t,
547 SveGatherLoadSVMicroop,
548 SveFirstFaultWritebackMicroop>(
549 mn, machInst, MemReadOp, zt, pg, rn, zm,
550 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
551 }
552 case 0x1:
553 if (esizeIs32) {
554 return new SveIndexedMemSV<uint32_t, uint8_t,
555 SveGatherLoadSVMicroop,
556 SveFirstFaultWritebackMicroop>(
557 mn, machInst, MemReadOp, zt, pg, rn, zm,
558 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
559 } else {
560 return new SveIndexedMemSV<uint64_t, uint8_t,
561 SveGatherLoadSVMicroop,
562 SveFirstFaultWritebackMicroop>(
563 mn, machInst, MemReadOp, zt, pg, rn, zm,
564 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
565 }
566 case 0x2:
567 if (esizeIs32) {
568 return new SveIndexedMemSV<int32_t, int16_t,
569 SveGatherLoadSVMicroop,
570 SveFirstFaultWritebackMicroop>(
571 mn, machInst, MemReadOp, zt, pg, rn, zm,
572 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
573 } else {
574 return new SveIndexedMemSV<int64_t, int16_t,
575 SveGatherLoadSVMicroop,
576 SveFirstFaultWritebackMicroop>(
577 mn, machInst, MemReadOp, zt, pg, rn, zm,
578 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
579 }
580 case 0x3:
581 if (esizeIs32) {
582 return new SveIndexedMemSV<uint32_t, uint16_t,
583 SveGatherLoadSVMicroop,
584 SveFirstFaultWritebackMicroop>(
585 mn, machInst, MemReadOp, zt, pg, rn, zm,
586 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
587 } else {
588 return new SveIndexedMemSV<uint64_t, uint16_t,
589 SveGatherLoadSVMicroop,
590 SveFirstFaultWritebackMicroop>(
591 mn, machInst, MemReadOp, zt, pg, rn, zm,
592 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
593 }
594 case 0x4:
595 if (esizeIs32) {
596 break;
597 } else {
598 return new SveIndexedMemSV<int64_t, int32_t,
599 SveGatherLoadSVMicroop,
600 SveFirstFaultWritebackMicroop>(
601 mn, machInst, MemReadOp, zt, pg, rn, zm,
602 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
603 }
604 case 0x5:
605 if (esizeIs32) {
606 return new SveIndexedMemSV<uint32_t, uint32_t,
607 SveGatherLoadSVMicroop,
608 SveFirstFaultWritebackMicroop>(
609 mn, machInst, MemReadOp, zt, pg, rn, zm,
610 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
611 } else {
612 return new SveIndexedMemSV<uint64_t, uint32_t,
613 SveGatherLoadSVMicroop,
614 SveFirstFaultWritebackMicroop>(
615 mn, machInst, MemReadOp, zt, pg, rn, zm,
616 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
617 }
618 case 0x7:
619 if (esizeIs32) {
620 break;
621 } else {
622 return new SveIndexedMemSV<uint64_t, uint64_t,
623 SveGatherLoadSVMicroop,
624 SveFirstFaultWritebackMicroop>(
625 mn, machInst, MemReadOp, zt, pg, rn, zm,
626 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
627 }
628 }
629 return new Unknown64(machInst);
630 }
631
632 StaticInstPtr
633 decodeSveScatterStoreVIInsts(uint8_t msz, ExtMachInst machInst,
634 IntRegIndex zt, IntRegIndex pg,
635 IntRegIndex zn, uint64_t imm,
636 bool esizeIs32)
637 {
638 const char* mn = "st1";
639 switch (msz) {
640 case 0x0:
641 if (esizeIs32) {
642 return new SveIndexedMemVI<uint32_t, uint8_t,
643 SveScatterStoreVIMicroop,
644 SveFirstFaultWritebackMicroop>(
645 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
646 } else {
647 return new SveIndexedMemVI<uint64_t, uint8_t,
648 SveScatterStoreVIMicroop,
649 SveFirstFaultWritebackMicroop>(
650 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
651 }
652 case 0x1:
653 if (esizeIs32) {
654 return new SveIndexedMemVI<uint32_t, uint16_t,
655 SveScatterStoreVIMicroop,
656 SveFirstFaultWritebackMicroop>(
657 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
658 } else {
659 return new SveIndexedMemVI<uint64_t, uint16_t,
660 SveScatterStoreVIMicroop,
661 SveFirstFaultWritebackMicroop>(
662 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
663 }
664 case 0x2:
665 if (esizeIs32) {
666 return new SveIndexedMemVI<uint32_t, uint32_t,
667 SveScatterStoreVIMicroop,
668 SveFirstFaultWritebackMicroop>(
669 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
670 } else {
671 return new SveIndexedMemVI<uint64_t, uint32_t,
672 SveScatterStoreVIMicroop,
673 SveFirstFaultWritebackMicroop>(
674 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
675 }
676 case 0x3:
677 if (esizeIs32) {
678 break;
679 } else {
680 return new SveIndexedMemVI<uint64_t, uint64_t,
681 SveScatterStoreVIMicroop,
682 SveFirstFaultWritebackMicroop>(
683 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
684 }
685 }
686 return new Unknown64(machInst);
687 }
688
689 StaticInstPtr
690 decodeSveScatterStoreSVInsts(uint8_t msz, ExtMachInst machInst,
691 IntRegIndex zt, IntRegIndex pg,
692 IntRegIndex rn, IntRegIndex zm,
693 bool esizeIs32, bool offsetIs32,
694 bool offsetIsSigned, bool offsetIsScaled)
695 {
696 const char* mn = "st1";
697 switch (msz) {
698 case 0x0:
699 if (esizeIs32) {
700 return new SveIndexedMemSV<uint32_t, uint8_t,
701 SveScatterStoreSVMicroop,
702 SveFirstFaultWritebackMicroop>(
703 mn, machInst, MemWriteOp, zt, pg, rn, zm,
704 offsetIs32, offsetIsSigned, offsetIsScaled, false);
705 } else {
706 return new SveIndexedMemSV<uint64_t, uint8_t,
707 SveScatterStoreSVMicroop,
708 SveFirstFaultWritebackMicroop>(
709 mn, machInst, MemWriteOp, zt, pg, rn, zm,
710 offsetIs32, offsetIsSigned, offsetIsScaled, false);
711 }
712 case 0x1:
713 if (esizeIs32) {
714 return new SveIndexedMemSV<uint32_t, uint16_t,
715 SveScatterStoreSVMicroop,
716 SveFirstFaultWritebackMicroop>(
717 mn, machInst, MemWriteOp, zt, pg, rn, zm,
718 offsetIs32, offsetIsSigned, offsetIsScaled, false);
719 } else {
720 return new SveIndexedMemSV<uint64_t, uint16_t,
721 SveScatterStoreSVMicroop,
722 SveFirstFaultWritebackMicroop>(
723 mn, machInst, MemWriteOp, zt, pg, rn, zm,
724 offsetIs32, offsetIsSigned, offsetIsScaled, false);
725 }
726 case 0x2:
727 if (esizeIs32) {
728 return new SveIndexedMemSV<uint32_t, uint32_t,
729 SveScatterStoreSVMicroop,
730 SveFirstFaultWritebackMicroop>(
731 mn, machInst, MemWriteOp, zt, pg, rn, zm,
732 offsetIs32, offsetIsSigned, offsetIsScaled, false);
733 } else {
734 return new SveIndexedMemSV<uint64_t, uint32_t,
735 SveScatterStoreSVMicroop,
736 SveFirstFaultWritebackMicroop>(
737 mn, machInst, MemWriteOp, zt, pg, rn, zm,
738 offsetIs32, offsetIsSigned, offsetIsScaled, false);
739 }
740 case 0x3:
741 if (esizeIs32) {
742 break;
743 } else {
744 return new SveIndexedMemSV<uint64_t, uint64_t,
745 SveScatterStoreSVMicroop,
746 SveFirstFaultWritebackMicroop>(
747 mn, machInst, MemWriteOp, zt, pg, rn, zm,
748 offsetIs32, offsetIsSigned, offsetIsScaled, false);
749 }
750 }
751 return new Unknown64(machInst);
752 }
753
754}};
755
756
757let {{
758
759 header_output = ''
760 exec_output = ''
761 decoders = { 'Generic': {} }
762
763 SPAlignmentCheckCode = '''
764 if (this->baseIsSP && bits(XBase, 3, 0) &&
765 SPAlignmentCheckEnabled(xc->tcBase())) {
766 return std::make_shared<SPAlignmentFault>();
767 }
768 '''
769
770 def emitSveMemFillSpill(isPred):
771 global header_output, exec_output, decoders
772 eaCode = SPAlignmentCheckCode + '''
773 int memAccessSize = %(memacc_size)s;
774 EA = XBase + ((int64_t) imm * %(memacc_size)s)''' % {
775 'memacc_size': 'eCount / 8' if isPred else 'eCount'}
776 loadRdEnableCode = '''
777 auto rdEn = std::vector<bool>();
778 '''
779 if isPred:
780 loadMemAccCode = '''
781 int index = 0;
782 uint8_t byte;
783 for (int i = 0; i < eCount / 8; i++) {
784 byte = memDataView[i];
785 for (int j = 0; j < 8; j++, index++) {
786 PDest_x[index] = (byte >> j) & 1;
787 }
788 }
789 '''
790 storeMemAccCode = '''
791 int index = 0;
792 uint8_t byte;
793 for (int i = 0; i < eCount / 8; i++) {
794 byte = 0;
795 for (int j = 0; j < 8; j++, index++) {
796 byte |= PDest_x[index] << j;
797 }
798 memDataView[i] = byte;
799 }
800 '''
801 storeWrEnableCode = '''
802 auto wrEn = std::vector<bool>(eCount / 8, true);
803 '''
804 else:
805 loadMemAccCode = '''
806 for (int i = 0; i < eCount; i++) {
807 AA64FpDest_x[i] = memDataView[i];
808 }
809 '''
810 storeMemAccCode = '''
811 for (int i = 0; i < eCount; i++) {
812 memDataView[i] = AA64FpDest_x[i];
813 }
814 '''
815 storeWrEnableCode = '''
816 auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
817 '''
818 loadIop = InstObjParams('ldr',
819 'SveLdrPred' if isPred else 'SveLdrVec',
820 'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill',
821 {'tpl_header': '',
822 'tpl_args': '',
823 'memacc_code': loadMemAccCode,
824 'ea_code' : sveEnabledCheckCode + eaCode,
825 'rden_code' : loadRdEnableCode,
826 'fault_code' : '',
827 'fa_code' : ''},
828 ['IsMemRef', 'IsLoad'])
829 storeIop = InstObjParams('str',
830 'SveStrPred' if isPred else 'SveStrVec',
831 'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill',
832 {'tpl_header': '',
833 'tpl_args': '',
834 'wren_code': storeWrEnableCode,
835 'memacc_code': storeMemAccCode,
836 'ea_code' : sveEnabledCheckCode + eaCode,
837 'fa_code' : ''},
838 ['IsMemRef', 'IsStore'])
839 header_output += SveMemFillSpillOpDeclare.subst(loadIop)
840 header_output += SveMemFillSpillOpDeclare.subst(storeIop)
841 exec_output += (
842 SveContigLoadExecute.subst(loadIop) +
843 SveContigLoadInitiateAcc.subst(loadIop) +
844 SveContigLoadCompleteAcc.subst(loadIop) +
845 SveContigStoreExecute.subst(storeIop) +
846 SveContigStoreInitiateAcc.subst(storeIop) +
847 SveContigStoreCompleteAcc.subst(storeIop))
848
849 loadTplArgs = (
850 ('uint8_t', 'uint8_t'),
851 ('uint16_t', 'uint8_t'),
852 ('uint32_t', 'uint8_t'),
853 ('uint64_t', 'uint8_t'),
854 ('int64_t', 'int32_t'),
855 ('uint16_t', 'uint16_t'),
856 ('uint32_t', 'uint16_t'),
857 ('uint64_t', 'uint16_t'),
858 ('int64_t', 'int16_t'),
859 ('int32_t', 'int16_t'),
860 ('uint32_t', 'uint32_t'),
861 ('uint64_t', 'uint32_t'),
862 ('int64_t', 'int8_t'),
863 ('int32_t', 'int8_t'),
864 ('int16_t', 'int8_t'),
865 ('uint64_t', 'uint64_t'),
866 )
867
868 storeTplArgs = (
869 ('uint8_t', 'uint8_t'),
870 ('uint16_t', 'uint8_t'),
871 ('uint32_t', 'uint8_t'),
872 ('uint64_t', 'uint8_t'),
873 ('uint16_t', 'uint16_t'),
874 ('uint32_t', 'uint16_t'),
875 ('uint64_t', 'uint16_t'),
876 ('uint32_t', 'uint32_t'),
877 ('uint64_t', 'uint32_t'),
878 ('uint64_t', 'uint64_t'),
879 )
880
881 gatherLoadTplArgs = (
882 ('int32_t', 'int8_t'),
883 ('int64_t', 'int8_t'),
884 ('uint32_t', 'uint8_t'),
885 ('uint64_t', 'uint8_t'),
886 ('int32_t', 'int16_t'),
887 ('int64_t', 'int16_t'),
888 ('uint32_t', 'uint16_t'),
889 ('uint64_t', 'uint16_t'),
890 ('int64_t', 'int32_t'),
891 ('uint32_t', 'uint32_t'),
892 ('uint64_t', 'uint32_t'),
893 ('uint64_t', 'uint64_t'),
894 )
895
896 scatterStoreTplArgs = (
897 ('uint32_t', 'uint8_t'),
898 ('uint64_t', 'uint8_t'),
899 ('uint32_t', 'uint16_t'),
900 ('uint64_t', 'uint16_t'),
901 ('uint32_t', 'uint32_t'),
902 ('uint64_t', 'uint32_t'),
903 ('uint64_t', 'uint64_t'),
904 )
905
906 # Generates definitions for SVE contiguous loads
907 def emitSveContigMemInsts(offsetIsImm):
908 global header_output, exec_output, decoders
909 # First-faulting instructions only have a scalar plus scalar form,
910 # while non-faulting instructions only a scalar plus immediate form, so
911 # `offsetIsImm` is used to determine which class of instructions is
912 # generated
913 firstFaulting = not offsetIsImm
914 tplHeader = 'template <class RegElemType, class MemElemType>'
915 tplArgs = '<RegElemType, MemElemType>'
916 eaCode = SPAlignmentCheckCode + '''
917 int memAccessSize = eCount * sizeof(MemElemType);
918 EA = XBase + '''
919 if offsetIsImm:
920 eaCode += '((int64_t) this->imm * eCount * sizeof(MemElemType))'
921 else:
922 eaCode += '(XOffset * sizeof(MemElemType));'
923 loadRdEnableCode = '''
924 auto rdEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
925 for (int i = 0; i < eCount; i++) {
926 if (!GpOp_x[i]) {
927 for (int j = 0; j < sizeof(MemElemType); j++) {
928 rdEn[sizeof(MemElemType) * i + j] = false;
929 }
930 }
931 }
932 '''
933 loadMemAccCode = '''
934 for (int i = 0; i < eCount; i++) {
935 if (GpOp_x[i]) {
936 AA64FpDest_x[i] = memDataView[i];
937 } else {
938 AA64FpDest_x[i] = 0;
939 }
940 }
941 '''
942 storeMemAccCode = '''
943 for (int i = 0; i < eCount; i++) {
944 if (GpOp_x[i]) {
945 memDataView[i] = AA64FpDest_x[i];
946 } else {
947 memDataView[i] = 0;
948 for (int j = 0; j < sizeof(MemElemType); j++) {
949 wrEn[sizeof(MemElemType) * i + j] = false;
950 }
951 }
952 }
953 '''
954 storeWrEnableCode = '''
955 auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
956 '''
957 ffrReadBackCode = '''
958 auto& firstFaultReg = Ffr;'''
959 fautlingLoadmemAccCode = '''
960 for (int i = 0; i < eCount; i++) {
961 if (GpOp_x[i] && firstFaultReg[i * sizeof(RegElemType)]) {
962 AA64FpDest_x[i] = memDataView[i];
963 } else {
964 AA64FpDest_x[i] = 0;
965 }
966 }
967 '''
968 nonFaultingCode = 'true ||'
969 faultCode = '''
970 Addr fault_addr;
971 if (fault == NoFault || getFaultVAddr(fault, fault_addr)) {
972 unsigned fault_elem_index;
973 if (fault != NoFault) {
974 assert(fault_addr >= EA);
975 fault_elem_index = (fault_addr - EA) / sizeof(MemElemType);
976 } else {
977 fault_elem_index = eCount + 1;
978 }
979 int first_active_index;
980 for (first_active_index = 0;
981 first_active_index < eCount && !(GpOp_x[first_active_index]);
982 first_active_index++);
983 if (%s first_active_index < fault_elem_index) {
984 for (int i = 0; i < eCount; i++) {
985 for (int j = 0; j < sizeof(RegElemType); j++) {
986 if (i < fault_elem_index) {
987 Ffr_ub[i * sizeof(RegElemType) + j] = FfrAux_x[i];
988 } else {
989 Ffr_ub[i * sizeof(RegElemType) + j] = 0;
990 }
991 }
992 }
993 fault = NoFault;
994 if (first_active_index >= fault_elem_index) {
995 // non-faulting load needs this
996 xc->setMemAccPredicate(false);
997 }
998 }
999 }
1000 ''' % ('' if firstFaulting else nonFaultingCode)
1001
1002 loadIop = InstObjParams('ld1',
1003 'SveContigLoadSI' if offsetIsImm else 'SveContigLoadSS',
1004 'SveContigMemSI' if offsetIsImm else 'SveContigMemSS',
1005 {'tpl_header': tplHeader,
1006 'tpl_args': tplArgs,
1007 'rden_code' : loadRdEnableCode,
1008 'memacc_code': loadMemAccCode,
1009 'ea_code' : sveEnabledCheckCode + eaCode,
1010 'fault_code' : '',
1011 'fa_code' : ''},
1012 ['IsMemRef', 'IsLoad'])
1013 storeIop = InstObjParams('st1',
1014 'SveContigStoreSI' if offsetIsImm else 'SveContigStoreSS',
1015 'SveContigMemSI' if offsetIsImm else 'SveContigMemSS',
1016 {'tpl_header': tplHeader,
1017 'tpl_args': tplArgs,
1018 'wren_code': storeWrEnableCode,
1019 'memacc_code': storeMemAccCode,
1020 'ea_code' : sveEnabledCheckCode + eaCode,
1021 'fa_code' : ''},
1022 ['IsMemRef', 'IsStore'])
1023 faultIop = InstObjParams('ldff1' if firstFaulting else 'ldnf1',
1024 'SveContigFFLoadSS' if firstFaulting else 'SveContigNFLoadSI',
1025 'SveContigMemSS' if firstFaulting else 'SveContigMemSI',
1026 {'tpl_header': tplHeader,
1027 'tpl_args': tplArgs,
1028 'rden_code' : loadRdEnableCode,
1029 'memacc_code': fautlingLoadmemAccCode,
1030 'ea_code' : sveEnabledCheckCode + eaCode,
1031 'fault_code' : faultCode,
1032 'fa_code' : ''},
1033 ['IsMemRef', 'IsLoad'])
1034 faultIop.snippets['memacc_code'] = (ffrReadBackCode +
1035 faultIop.snippets['memacc_code'])
1036 if offsetIsImm:
1037 header_output += SveContigMemSIOpDeclare.subst(loadIop)
1038 header_output += SveContigMemSIOpDeclare.subst(storeIop)
1039 header_output += SveContigMemSIOpDeclare.subst(faultIop)
1040 else:
1041 header_output += SveContigMemSSOpDeclare.subst(loadIop)
1042 header_output += SveContigMemSSOpDeclare.subst(storeIop)
1043 header_output += SveContigMemSSOpDeclare.subst(faultIop)
1044 exec_output += (
1045 SveContigLoadExecute.subst(loadIop) +
1046 SveContigLoadInitiateAcc.subst(loadIop) +
1047 SveContigLoadCompleteAcc.subst(loadIop) +
1048 SveContigStoreExecute.subst(storeIop) +
1049 SveContigStoreInitiateAcc.subst(storeIop) +
1050 SveContigStoreCompleteAcc.subst(storeIop) +
1051 SveContigLoadExecute.subst(faultIop) +
1052 SveContigLoadInitiateAcc.subst(faultIop) +
1053 SveContigLoadCompleteAcc.subst(faultIop))
1054
1055 for args in loadTplArgs:
1056 substDict = {'tpl_args': '<%s>' % ', '.join(args),
1057 'class_name': 'SveContigLoadSI' if offsetIsImm
1058 else 'SveContigLoadSS'}
1059 exec_output += SveContigMemExecDeclare.subst(substDict)
1060 for args in storeTplArgs:
1061 substDict = {'tpl_args': '<%s>' % ', '.join(args),
1062 'class_name': 'SveContigStoreSI' if offsetIsImm
1063 else 'SveContigStoreSS'}
1064 exec_output += SveContigMemExecDeclare.subst(substDict)
1065 for args in loadTplArgs:
1066 substDict = {'tpl_args': '<%s>' % ', '.join(args),
1067 'class_name': 'SveContigFFLoadSS' if firstFaulting
1068 else 'SveContigNFLoadSI'}
1069 exec_output += SveContigMemExecDeclare.subst(substDict)
1070
1071
1072 # Generates definitions for SVE load-and-replicate instructions
1073 def emitSveLoadAndRepl():
1074 global header_output, exec_output, decoders
1075 tplHeader = 'template <class RegElemType, class MemElemType>'
1076 tplArgs = '<RegElemType, MemElemType>'
1077 eaCode = SPAlignmentCheckCode + '''
1078 EA = XBase + imm * sizeof(MemElemType);'''
1079 memAccCode = '''
1080 for (int i = 0; i < eCount; i++) {
1081 if (GpOp_x[i]) {
1082 AA64FpDest_x[i] = memData;
1083 } else {
1084 AA64FpDest_x[i] = 0;
1085 }
1086 }
1087 '''
1088 iop = InstObjParams('ld1r',
1089 'SveLoadAndRepl',
1090 'SveContigMemSI',
1091 {'tpl_header': tplHeader,
1092 'tpl_args': tplArgs,
1093 'memacc_code': memAccCode,
1094 'ea_code' : sveEnabledCheckCode + eaCode,
1095 'fa_code' : ''},
1096 ['IsMemRef', 'IsLoad'])
1097 header_output += SveContigMemSIOpDeclare.subst(iop)
1098 exec_output += (
1099 SveLoadAndReplExecute.subst(iop) +
1100 SveLoadAndReplInitiateAcc.subst(iop) +
1101 SveLoadAndReplCompleteAcc.subst(iop))
1102 for args in loadTplArgs:
1103 substDict = {'tpl_args': '<%s>' % ', '.join(args),
1104 'class_name': 'SveLoadAndRepl'}
1105 exec_output += SveContigMemExecDeclare.subst(substDict)
1106
1107 class IndexedAddrForm:
1108 VEC_PLUS_IMM = 0
1109 SCA_PLUS_VEC = 1
1110
1111 # Generates definitions for the transfer microops of SVE indexed memory
1112 # operations (gather loads, scatter stores)
1113 def emitSveIndexedMemMicroops(indexed_addr_form):
1114 assert indexed_addr_form in (IndexedAddrForm.VEC_PLUS_IMM,
1115 IndexedAddrForm.SCA_PLUS_VEC)
1116 global header_output, exec_output, decoders
1117 tplHeader = 'template <class RegElemType, class MemElemType>'
1118 tplArgs = '<RegElemType, MemElemType>'
1119 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM:
1120 eaCode = '''
1121 EA = AA64FpBase_x[elemIndex] + imm * sizeof(MemElemType)'''
1122 else:
1123 eaCode = '''
1124 uint64_t offset = AA64FpOffset_x[elemIndex];
1125 if (offsetIs32) {
1126 offset &= (1ULL << 32) - 1;
1127 }
1128 if (offsetIsSigned) {
1129 offset = sext<32>(offset);
1130 }
1131 if (offsetIsScaled) {
1132 offset *= sizeof(MemElemType);
1133 }
1134 EA = XBase + offset'''
1135 loadMemAccCode = '''
1136 AA64FpDest_x[elemIndex] = memData;
1137 '''
1138 storeMemAccCode = '''
1139 memData = AA64FpDest_x[elemIndex];
1140 '''
1141 predCheckCode = 'GpOp_x[index]'
1142 faultStatusSetCode = 'PUreg0_x[elemIndex] = 1;'
1143 faultStatusResetCode = 'PUreg0_x[elemIndex] = 0;'
1144 loadIop = InstObjParams('ld1',
1145 ('SveGatherLoadVIMicroop'
1146 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM
1147 else 'SveGatherLoadSVMicroop'),
1148 'MicroOp',
1149 {'tpl_header': tplHeader,
1150 'tpl_args': tplArgs,
1151 'memacc_code': loadMemAccCode,
1152 'ea_code' : sveEnabledCheckCode + eaCode,
1153 'fault_status_set_code' : faultStatusSetCode,
1154 'fault_status_reset_code' : faultStatusResetCode,
1155 'pred_check_code' : predCheckCode,
1156 'fa_code' : ''},
1157 ['IsMicroop', 'IsMemRef', 'IsLoad'])
1158 storeIop = InstObjParams('st1',
1159 ('SveScatterStoreVIMicroop'
1160 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM
1161 else 'SveScatterStoreSVMicroop'),
1162 'MicroOp',
1163 {'tpl_header': tplHeader,
1164 'tpl_args': tplArgs,
1165 'memacc_code': storeMemAccCode,
1166 'ea_code' : sveEnabledCheckCode + eaCode,
1167 'pred_check_code' : predCheckCode,
1168 'fa_code' : ''},
1169 ['IsMicroop', 'IsMemRef', 'IsStore'])
1170 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM:
1171 header_output += SveIndexedMemVIMicroopDeclare.subst(loadIop)
1172 header_output += SveIndexedMemVIMicroopDeclare.subst(storeIop)
1173 else:
1174 header_output += SveIndexedMemSVMicroopDeclare.subst(loadIop)
1175 header_output += SveIndexedMemSVMicroopDeclare.subst(storeIop)
1176 exec_output += (
1177 SveGatherLoadMicroopExecute.subst(loadIop) +
1178 SveGatherLoadMicroopInitiateAcc.subst(loadIop) +
1179 SveGatherLoadMicroopCompleteAcc.subst(loadIop) +
1180 SveScatterStoreMicroopExecute.subst(storeIop) +
1181 SveScatterStoreMicroopInitiateAcc.subst(storeIop) +
1182 SveScatterStoreMicroopCompleteAcc.subst(storeIop))
1183 for args in gatherLoadTplArgs:
1184 substDict = {'tpl_args': '<%s>' % ', '.join(args),
1185 'class_name': (
1186 'SveGatherLoadVIMicroop'
1187 if indexed_addr_form == \
1188 IndexedAddrForm.VEC_PLUS_IMM
1189 else 'SveGatherLoadSVMicroop')}
1190 # TODO: this should become SveMemExecDeclare
1191 exec_output += SveContigMemExecDeclare.subst(substDict)
1192 for args in scatterStoreTplArgs:
1193 substDict = {'tpl_args': '<%s>' % ', '.join(args),
1194 'class_name': (
1195 'SveScatterStoreVIMicroop'
1196 if indexed_addr_form == \
1197 IndexedAddrForm.VEC_PLUS_IMM
1198 else 'SveScatterStoreSVMicroop')}
1199 # TODO: this should become SveMemExecDeclare
1200 exec_output += SveContigMemExecDeclare.subst(substDict)
1201
1202 firstFaultTplArgs = ('int32_t', 'int64_t', 'uint32_t', 'uint64_t')
1203
1204 def emitSveFirstFaultWritebackMicroop():
1205 global header_output, exec_output, decoders
1206 tplHeader = 'template <class RegElemType>'
1207 tplArgs = '<RegElemType>'
1208 faultStatusCheckCode = 'PUreg0_x[index]'
1209 firstFaultResetCode = '''
1210 for(int j = 0; j < sizeof(RegElemType); j++) {
1211 Ffr_ub[index * sizeof(RegElemType) + j] = 0;
1212 }
1213 '''
1214 firstFaultForwardCode = '''
1215 for(int j = 0; j < sizeof(RegElemType); j++) {
1216 Ffr_ub[index * sizeof(RegElemType) + j] = FfrAux_x[index];
1217 }
1218 '''
1219 iop = InstObjParams('ldff1',
1220 'SveFirstFaultWritebackMicroop',
1221 'MicroOp',
1222 {'tpl_header': tplHeader,
1223 'tpl_args': tplArgs,
1224 'fault_status_check_code' : faultStatusCheckCode,
1225 'first_fault_reset_code' : firstFaultResetCode,
1226 'first_fault_forward_code' : firstFaultForwardCode},
1227 ['IsMicroop'])
1228 header_output += SveFirstFaultWritebackMicroopDeclare.subst(iop)
1229 exec_output += SveFirstFaultWritebackMicroopExecute.subst(iop)
1230 for args in firstFaultTplArgs:
1231 substDict = {'targs': args,
1232 'class_name' : 'SveFirstFaultWritebackMicroop' }
1233 exec_output += SveOpExecDeclare.subst(substDict)
1234
1235 # Generates definitions for the first microop of SVE gather loads, required
1236 # to propagate the source vector register to the transfer microops
1237 def emitSveGatherLoadCpySrcVecMicroop():
1238 global header_output, exec_output, decoders
1239 code = sveEnabledCheckCode + '''
1240 unsigned eCount = ArmStaticInst::getCurSveVecLen<uint8_t>(
1241 xc->tcBase());
1242 for (unsigned i = 0; i < eCount; i++) {
1243 AA64FpUreg0_ub[i] = AA64FpOp1_ub[i];
1244 }'''
1245 iop = InstObjParams('ld1',
1246 'SveGatherLoadCpySrcVecMicroop',
1247 'MicroOp',
1248 {'code': code},
1249 ['IsMicroop'])
1250 header_output += SveGatherLoadCpySrcVecMicroopDeclare.subst(iop)
1251 exec_output += SveGatherLoadCpySrcVecMicroopExecute.subst(iop)
1252
1253 def emitSveInterleaveMicroop():
1254 global header_output, exec_output, decoders
1255 code2 = sveEnabledCheckCode + '''
1256 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1257 xc->tcBase());
1258 for (unsigned int i = 0; i < eCount; ++i) {
1259 unsigned int absIdx = regIndex * eCount + i;
1260 unsigned int srcIdx = absIdx / numRegs;
1261 unsigned int srcVec = absIdx % numRegs;
1262 if (srcVec == 0)
1263 AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx];
1264 else if (srcVec == 1)
1265 AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx];
1266 }'''
1267
1268 code3 = sveEnabledCheckCode + '''
1269 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1270 xc->tcBase());
1271 for (unsigned int i = 0; i < eCount; ++i) {
1272 unsigned int absIdx = regIndex * eCount + i;
1273 unsigned int srcIdx = absIdx / numRegs;
1274 unsigned int srcVec = absIdx % numRegs;
1275 if (srcVec == 0)
1276 AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx];
1277 else if (srcVec == 1)
1278 AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx];
1279 else if (srcVec == 2)
1280 AA64FpDest_x[i] = AA64FpOp1V2S_x[srcIdx];
1281 }'''
1282
1283 code4 = sveEnabledCheckCode + '''
1284 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1285 xc->tcBase());
1286 for (unsigned int i = 0; i < eCount; ++i) {
1287 unsigned int absIdx = regIndex * eCount + i;
1288 unsigned int srcIdx = absIdx / numRegs;
1289 unsigned int srcVec = absIdx % numRegs;
1290 if (srcVec == 0)
1291 AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx];
1292 else if (srcVec == 1)
1293 AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx];
1294 else if (srcVec == 2)
1295 AA64FpDest_x[i] = AA64FpOp1V2S_x[srcIdx];
1296 else if (srcVec == 3)
1297 AA64FpDest_x[i] = AA64FpOp1V3S_x[srcIdx];
1298 }'''
1299
1300 iop2 = InstObjParams('intrlv',
1301 'SveIntrlv2Microop',
1302 'MicroOp',
1303 {'code': code2},
1304 ['IsMicroop'])
1305 iop3 = InstObjParams('intrlv',
1306 'SveIntrlv3Microop',
1307 'MicroOp',
1308 {'code': code3},
1309 ['IsMicroop'])
1310 iop4 = InstObjParams('intrlv',
1311 'SveIntrlv4Microop',
1312 'MicroOp',
1313 {'code': code4},
1314 ['IsMicroop'])
1315 header_output += SveIntrlvMicroopDeclare.subst(iop2);
1316 header_output += SveIntrlvMicroopDeclare.subst(iop3);
1317 header_output += SveIntrlvMicroopDeclare.subst(iop4);
1318 exec_output += SveIntrlvMicroopExecute.subst(iop2);
1319 exec_output += SveIntrlvMicroopExecute.subst(iop3);
1320 exec_output += SveIntrlvMicroopExecute.subst(iop4);
1321 for type in ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'):
1322 for nreg in range(2,5):
1323 substDict = {'targs' : type,
1324 'class_name' : 'SveIntrlv' + str(nreg) + 'Microop'}
1325 exec_output += SveIntrlvMicroopExecDeclare.subst(substDict)
1326
1327 def emitSveDeInterleaveMicroop():
1328 global header_output, exec_output, decoders
1329 code2 = sveEnabledCheckCode + '''
1330 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1331 xc->tcBase());
1332 for (unsigned int i = 0; i < eCount; ++i) {
1333 unsigned int absIdx = (regIndex + numRegs * i);
1334 unsigned int srcIdx = absIdx % eCount;
1335 unsigned int srcVec = absIdx / eCount;
1336 if (srcVec == 0)
1337 AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx];
1338 else if(srcVec == 1)
1339 AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx];
1340 }'''
1341
1342 code3 = sveEnabledCheckCode + '''
1343 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1344 xc->tcBase());
1345 for (unsigned int i = 0; i < eCount; ++i) {
1346 unsigned int absIdx = (regIndex + numRegs * i);
1347 unsigned int srcIdx = absIdx % eCount;
1348 unsigned int srcVec = absIdx / eCount;
1349 if (srcVec == 0)
1350 AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx];
1351 else if(srcVec == 1)
1352 AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx];
1353 else if(srcVec == 2)
1354 AA64FpDest_x[i] = AA64IntrlvReg2_x[srcIdx];
1355 }'''
1356
1357 code4 = sveEnabledCheckCode + '''
1358 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1359 xc->tcBase());
1360 for (unsigned int i = 0; i < eCount; ++i) {
1361 unsigned int absIdx = (regIndex + numRegs * i);
1362 unsigned int srcIdx = absIdx % eCount;
1363 unsigned int srcVec = absIdx / eCount;
1364 if (srcVec == 0)
1365 AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx];
1366 else if(srcVec == 1)
1367 AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx];
1368 else if(srcVec == 2)
1369 AA64FpDest_x[i] = AA64IntrlvReg2_x[srcIdx];
1370 else if(srcVec == 3)
1371 AA64FpDest_x[i] = AA64IntrlvReg3_x[srcIdx];
1372 }'''
1373
1374 iop2 = InstObjParams('deintrlv',
1375 'SveDeIntrlv2Microop',
1376 'MicroOp',
1377 {'code': code2},
1378 ['IsMicroop'])
1379 iop3 = InstObjParams('deintrlv',
1380 'SveDeIntrlv3Microop',
1381 'MicroOp',
1382 {'code': code3},
1383 ['IsMicroop'])
1384 iop4 = InstObjParams('deintrlv',
1385 'SveDeIntrlv4Microop',
1386 'MicroOp',
1387 {'code': code4},
1388 ['IsMicroop'])
1389 header_output += SveDeIntrlvMicroopDeclare.subst(iop2);
1390 header_output += SveDeIntrlvMicroopDeclare.subst(iop3);
1391 header_output += SveDeIntrlvMicroopDeclare.subst(iop4);
1392 exec_output += SveIntrlvMicroopExecute.subst(iop2);
1393 exec_output += SveIntrlvMicroopExecute.subst(iop3);
1394 exec_output += SveIntrlvMicroopExecute.subst(iop4);
1395 for type in ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'):
1396 for nreg in range(2,5):
1397 substDict = {'targs' : type,
1398 'class_name' : 'SveDeIntrlv' + str(nreg) + 'Microop'}
1399 exec_output += SveIntrlvMicroopExecDeclare.subst(substDict)
1400
1401 # Generates definitions for SVE struct load/store microops
1402 def emitSveStructMemInsts(offsetIsImm):
1403 global header_output, exec_output, decoders
1404 eaCode = SPAlignmentCheckCode + '''
1405 int memAccessSize = eCount * sizeof(Element);
1406 EA = memAccessSize * regIndex + XBase + '''
1407 if offsetIsImm:
1408 eaCode += '((int64_t) this->imm * eCount * sizeof(Element))'
1409 else:
1410 eaCode += '(XOffset * sizeof(Element));'
1411 loadMemAccCode = '''
1412 for (int i = 0; i < eCount; i++) {
1413 int gpIdx = (regIndex * eCount + i) / numRegs;
1414 if (GpOp_x[gpIdx]) {
1415 AA64FpDest_x[i] = memDataView[i];
1416 } else {
1417 AA64FpDest_x[i] = 0;
1418 }
1419 }
1420 '''
1421 storeMemAccCode = '''
1422 for (int i = 0; i < eCount; i++) {
1423 int gpIdx = (regIndex * eCount + i) / numRegs;
1424 if (GpOp_x[gpIdx]) {
1425 memDataView[i] = AA64FpDest_x[i];
1426 } else {
1427 memDataView[i] = 0;
1428 for (int j = 0; j < sizeof(Element); j++) {
1429 wrEn[sizeof(Element) * i + j] = false;
1430 }
1431 }
1432 }
1433 '''
1434 storeWrEnableCode = '''
1435 auto wrEn = std::vector<bool>(sizeof(Element) * eCount, true);
1436 '''
1437 loadIop = InstObjParams('ldxx',
1438 'SveLoadRegImmMicroop' if offsetIsImm else 'SveLoadRegRegMicroop',
1439 'MicroOp',
1440 {'targs': 'Element',
1441 'memacc_code': loadMemAccCode,
1442 'ea_code' : sveEnabledCheckCode + eaCode,
1443 'fa_code' : ''},
1444 ['IsMemRef', 'IsLoad', 'IsMicroop'])
1445 storeIop = InstObjParams('stxx',
1446 'SveStoreRegImmMicroop' if offsetIsImm
1447 else 'SveStoreRegRegMicroop',
1448 'MicroOp',
1449 {'targs': 'Element',
1450 'wren_code': storeWrEnableCode,
1451 'memacc_code': storeMemAccCode,
1452 'ea_code' : sveEnabledCheckCode + eaCode,
1453 'fa_code' : ''},
1454 ['IsMemRef', 'IsStore', 'IsMicroop'])
1455 if offsetIsImm:
1456 header_output += SveStructMemSIMicroopDeclare.subst(loadIop)
1457 header_output += SveStructMemSIMicroopDeclare.subst(storeIop)
1458 else:
1459 header_output += SveStructMemSSMicroopDeclare.subst(loadIop)
1460 header_output += SveStructMemSSMicroopDeclare.subst(storeIop)
1461 exec_output += (
1462 SveStructLoadExecute.subst(loadIop) +
1463 SveStructLoadInitiateAcc.subst(loadIop) +
1464 SveStructLoadCompleteAcc.subst(loadIop) +
1465 SveStructStoreExecute.subst(storeIop) +
1466 SveStructStoreInitiateAcc.subst(storeIop) +
1467 SveStructStoreCompleteAcc.subst(storeIop))
1468 tplArgs = ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t')
1469 for type in tplArgs:
1470 substDict = {'targs': type,
1471 'class_name': 'SveLoadRegImmMicroop' if offsetIsImm
1472 else 'SveLoadRegRegMicroop'}
1473 exec_output += SveStructMemExecDeclare.subst(substDict)
1474 substDict['class_name'] = ('SveStoreRegImmMicroop' if offsetIsImm
1475 else 'SveStoreRegRegMicroop')
1476 exec_output += SveStructMemExecDeclare.subst(substDict)
1477
| 1// Copyright (c) 2017-2018 ARM Limited
2// All rights reserved
3//
4// The license below extends only to copyright in the software and shall
5// not be construed as granting a license to any other intellectual
6// property including but not limited to intellectual property relating
7// to a hardware implementation of the functionality of the software
8// licensed hereunder. You may use the software subject to the license
9// terms below provided that you ensure that this notice is replicated
10// unmodified and in its entirety in all distributions of the software,
11// modified or unmodified, in source code or in binary form.
12//
13// Redistribution and use in source and binary forms, with or without
14// modification, are permitted provided that the following conditions are
15// met: redistributions of source code must retain the above copyright
16// notice, this list of conditions and the following disclaimer;
17// redistributions in binary form must reproduce the above copyright
18// notice, this list of conditions and the following disclaimer in the
19// documentation and/or other materials provided with the distribution;
20// neither the name of the copyright holders nor the names of its
21// contributors may be used to endorse or promote products derived from
22// this software without specific prior written permission.
23//
24// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
27// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
30// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35//
36// Authors: Giacomo Gabrielli
37
38// @file Definition of SVE memory access instructions.
39
40output header {{
41
42 // Decodes SVE contiguous load instructions, scalar plus scalar form.
43 template <template <typename T1, typename T2> class Base>
44 StaticInstPtr
45 decodeSveContigLoadSSInsts(uint8_t dtype, ExtMachInst machInst,
46 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
47 IntRegIndex rm, bool firstFaulting)
48 {
49 const char* mn = firstFaulting ? "ldff1" : "ld1";
50 switch (dtype) {
51 case 0x0:
52 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
53 case 0x1:
54 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
55 case 0x2:
56 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
57 case 0x3:
58 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
59 case 0x4:
60 return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, rm);
61 case 0x5:
62 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
63 case 0x6:
64 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
65 case 0x7:
66 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
67 case 0x8:
68 return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, rm);
69 case 0x9:
70 return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, rm);
71 case 0xa:
72 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
73 case 0xb:
74 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
75 case 0xc:
76 return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, rm);
77 case 0xd:
78 return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, rm);
79 case 0xe:
80 return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, rm);
81 case 0xf:
82 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm);
83 }
84 return new Unknown64(machInst);
85 }
86
87 // Decodes SVE contiguous load instructions, scalar plus immediate form.
88 template <template <typename T1, typename T2> class Base>
89 StaticInstPtr
90 decodeSveContigLoadSIInsts(uint8_t dtype, ExtMachInst machInst,
91 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
92 uint64_t imm, bool nonFaulting,
93 bool replicate = false)
94 {
95 assert(!(nonFaulting && replicate));
96 const char* mn = replicate ? "ld1r" : (nonFaulting ? "ldnf1" : "ld1");
97 switch (dtype) {
98 case 0x0:
99 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
100 case 0x1:
101 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
102 case 0x2:
103 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
104 case 0x3:
105 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
106 case 0x4:
107 return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, imm);
108 case 0x5:
109 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
110 case 0x6:
111 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
112 case 0x7:
113 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
114 case 0x8:
115 return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, imm);
116 case 0x9:
117 return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, imm);
118 case 0xa:
119 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
120 case 0xb:
121 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
122 case 0xc:
123 return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, imm);
124 case 0xd:
125 return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, imm);
126 case 0xe:
127 return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, imm);
128 case 0xf:
129 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm);
130 }
131 return new Unknown64(machInst);
132 }
133
134 // Decodes SVE contiguous store instructions, scalar plus scalar form.
135 template <template <typename T1, typename T2> class Base>
136 StaticInstPtr
137 decodeSveContigStoreSSInsts(uint8_t dtype, ExtMachInst machInst,
138 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
139 IntRegIndex rm)
140 {
141 const char* mn = "st1";
142 switch (dtype) {
143 case 0x0:
144 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
145 case 0x1:
146 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
147 case 0x2:
148 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
149 case 0x3:
150 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
151 case 0x5:
152 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
153 case 0x6:
154 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
155 case 0x7:
156 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
157 case 0xa:
158 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
159 case 0xb:
160 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
161 case 0xf:
162 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm);
163 }
164 return new Unknown64(machInst);
165 }
166
167 // Decodes SVE contiguous store instructions, scalar plus immediate form.
168 template <template <typename T1, typename T2> class Base>
169 StaticInstPtr
170 decodeSveContigStoreSIInsts(uint8_t dtype, ExtMachInst machInst,
171 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
172 int8_t imm)
173 {
174 const char* mn = "st1";
175 switch (dtype) {
176 case 0x0:
177 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
178 case 0x1:
179 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
180 case 0x2:
181 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
182 case 0x3:
183 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
184 case 0x5:
185 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
186 case 0x6:
187 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
188 case 0x7:
189 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
190 case 0xa:
191 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
192 case 0xb:
193 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
194 case 0xf:
195 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm);
196 }
197 return new Unknown64(machInst);
198 }
199
200 // NOTE: SVE load-and-replicate instructions are decoded with
201 // decodeSveContigLoadSIInsts(...).
202
203}};
204
205output decoder {{
206
207 template <class etype>
208 StaticInstPtr
209 decodeSveStructLoadSIInstsByNReg(uint8_t esize, ExtMachInst machInst,
210 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
211 int64_t imm, int numregs)
212 {
213 static const char* nm[5][4] = {
214 { nullptr, nullptr, nullptr, nullptr},
215 { nullptr, nullptr, nullptr, nullptr},
216 { "ld2b", "ld2h", "ld2w", "ld2d" },
217 { "ld3b", "ld3h", "ld3w", "ld3d" },
218 { "ld4b", "ld4h", "ld4w", "ld4d" } };
219
220 switch (numregs) {
221 case 2:
222 return new SveLdStructSI<etype,
223 SveLoadRegImmMicroop,
224 SveDeIntrlv2Microop>(
225 nm[numregs][esize], machInst, MemReadOp,
226 zt, pg, xn, imm, numregs);
227 case 3:
228 return new SveLdStructSI<etype,
229 SveLoadRegImmMicroop,
230 SveDeIntrlv3Microop>(
231 nm[numregs][esize], machInst, MemReadOp,
232 zt, pg, xn, imm, numregs);
233 case 4:
234 return new SveLdStructSI<etype,
235 SveLoadRegImmMicroop,
236 SveDeIntrlv4Microop>(
237 nm[numregs][esize], machInst, MemReadOp,
238 zt, pg, xn, imm, numregs);
239 }
240 return new Unknown64(machInst);
241 }
242
243 StaticInstPtr
244 decodeSveStructLoadSIInsts(uint8_t esize, ExtMachInst machInst,
245 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
246 int64_t imm, int numregs)
247 {
248 switch (esize) {
249 case 0:
250 return decodeSveStructLoadSIInstsByNReg<uint8_t>(esize,
251 machInst, zt, pg, xn, imm, numregs);
252 case 1:
253 return decodeSveStructLoadSIInstsByNReg<uint16_t>(esize,
254 machInst, zt, pg, xn, imm, numregs);
255 case 2:
256 return decodeSveStructLoadSIInstsByNReg<uint32_t>(esize,
257 machInst, zt, pg, xn, imm, numregs);
258 case 3:
259 return decodeSveStructLoadSIInstsByNReg<uint64_t>(esize,
260 machInst, zt, pg, xn, imm, numregs);
261 }
262 return new Unknown64(machInst);
263 }
264
265 template <class etype>
266 StaticInstPtr
267 decodeSveStructStoreSIInstsByNReg(uint8_t esize, ExtMachInst machInst,
268 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
269 int64_t imm, int numregs)
270 {
271 static const char* nm[5][4] = {
272 { nullptr, nullptr, nullptr, nullptr},
273 { nullptr, nullptr, nullptr, nullptr},
274 { "st2b", "st2h", "st2w", "st2d" },
275 { "st3b", "st3h", "st3w", "st3d" },
276 { "st4b", "st4h", "st4w", "st4d" } };
277
278 switch (numregs) {
279 case 2:
280 return new SveStStructSI<etype,
281 SveStoreRegImmMicroop,
282 SveIntrlv2Microop>(
283 nm[numregs][esize], machInst, MemWriteOp,
284 zt, pg, xn, imm, numregs);
285 case 3:
286 return new SveStStructSI<etype,
287 SveStoreRegImmMicroop,
288 SveIntrlv3Microop>(
289 nm[numregs][esize], machInst, MemWriteOp,
290 zt, pg, xn, imm, numregs);
291 case 4:
292 return new SveStStructSI<etype,
293 SveStoreRegImmMicroop,
294 SveIntrlv4Microop>(
295 nm[numregs][esize], machInst, MemWriteOp,
296 zt, pg, xn, imm, numregs);
297 }
298 return new Unknown64(machInst);
299 }
300
301 StaticInstPtr
302 decodeSveStructStoreSIInsts(uint8_t esize, ExtMachInst machInst,
303 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
304 int64_t imm, int numregs)
305 {
306 switch (esize) {
307 case 0:
308 return decodeSveStructStoreSIInstsByNReg<uint8_t>(esize,
309 machInst, zt, pg, xn, imm, numregs);
310 case 1:
311 return decodeSveStructStoreSIInstsByNReg<uint16_t>(esize,
312 machInst, zt, pg, xn, imm, numregs);
313 case 2:
314 return decodeSveStructStoreSIInstsByNReg<uint32_t>(esize,
315 machInst, zt, pg, xn, imm, numregs);
316 case 3:
317 return decodeSveStructStoreSIInstsByNReg<uint64_t>(esize,
318 machInst, zt, pg, xn, imm, numregs);
319 }
320 return new Unknown64(machInst);
321 }
322
323 template <class etype>
324 StaticInstPtr
325 decodeSveStructLoadSSInstsByNReg(uint8_t esize, ExtMachInst machInst,
326 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
327 IntRegIndex xm, int numregs)
328 {
329 static const char* nm[5][4] = {
330 { nullptr, nullptr, nullptr, nullptr},
331 { nullptr, nullptr, nullptr, nullptr},
332 { "ld2b", "ld2h", "ld2w", "ld2d" },
333 { "ld3b", "ld3h", "ld3w", "ld3d" },
334 { "ld4b", "ld4h", "ld4w", "ld4d" } };
335
336 switch (numregs) {
337 case 2:
338 return new SveLdStructSS<etype,
339 SveLoadRegRegMicroop,
340 SveDeIntrlv2Microop>(
341 nm[numregs][esize], machInst, MemReadOp,
342 zt, pg, xn, xm, numregs);
343 case 3:
344 return new SveLdStructSS<etype,
345 SveLoadRegRegMicroop,
346 SveDeIntrlv3Microop>(
347 nm[numregs][esize], machInst, MemReadOp,
348 zt, pg, xn, xm, numregs);
349 case 4:
350 return new SveLdStructSS<etype,
351 SveLoadRegRegMicroop,
352 SveDeIntrlv4Microop>(
353 nm[numregs][esize], machInst, MemReadOp,
354 zt, pg, xn, xm, numregs);
355 }
356 return new Unknown64(machInst);
357 }
358
359 StaticInstPtr
360 decodeSveStructLoadSSInsts(uint8_t esize, ExtMachInst machInst,
361 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
362 IntRegIndex xm, int numregs)
363 {
364 switch (esize) {
365 case 0:
366 return decodeSveStructLoadSSInstsByNReg<uint8_t>(esize,
367 machInst, zt, pg, xn, xm, numregs);
368 case 1:
369 return decodeSveStructLoadSSInstsByNReg<uint16_t>(esize,
370 machInst, zt, pg, xn, xm, numregs);
371 case 2:
372 return decodeSveStructLoadSSInstsByNReg<uint32_t>(esize,
373 machInst, zt, pg, xn, xm, numregs);
374 case 3:
375 return decodeSveStructLoadSSInstsByNReg<uint64_t>(esize,
376 machInst, zt, pg, xn, xm, numregs);
377 }
378 return new Unknown64(machInst);
379 }
380
381 template <class etype>
382 StaticInstPtr
383 decodeSveStructStoreSSInstsByNReg(uint8_t esize, ExtMachInst machInst,
384 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
385 IntRegIndex xm, int numregs)
386 {
387 static const char* nm[5][4] = {
388 { nullptr, nullptr, nullptr, nullptr},
389 { nullptr, nullptr, nullptr, nullptr},
390 { "st2b", "st2h", "st2w", "st2d" },
391 { "st3b", "st3h", "st3w", "st3d" },
392 { "st4b", "st4h", "st4w", "st4d" } };
393
394 switch (numregs) {
395 case 2:
396 return new SveStStructSS<etype,
397 SveStoreRegRegMicroop,
398 SveIntrlv2Microop>(
399 nm[numregs][esize], machInst, MemWriteOp,
400 zt, pg, xn, xm, numregs);
401 case 3:
402 return new SveStStructSS<etype,
403 SveStoreRegRegMicroop,
404 SveIntrlv3Microop>(
405 nm[numregs][esize], machInst, MemWriteOp,
406 zt, pg, xn, xm, numregs);
407 case 4:
408 return new SveStStructSS<etype,
409 SveStoreRegRegMicroop,
410 SveIntrlv4Microop>(
411 nm[numregs][esize], machInst, MemWriteOp,
412 zt, pg, xn, xm, numregs);
413 }
414 return new Unknown64(machInst);
415 }
416
417 StaticInstPtr
418 decodeSveStructStoreSSInsts(uint8_t esize, ExtMachInst machInst,
419 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
420 IntRegIndex xm, int numregs)
421 {
422 switch (esize) {
423 case 0:
424 return decodeSveStructStoreSSInstsByNReg<uint8_t>(esize,
425 machInst, zt, pg, xn, xm, numregs);
426 case 1:
427 return decodeSveStructStoreSSInstsByNReg<uint16_t>(esize,
428 machInst, zt, pg, xn, xm, numregs);
429 case 2:
430 return decodeSveStructStoreSSInstsByNReg<uint32_t>(esize,
431 machInst, zt, pg, xn, xm, numregs);
432 case 3:
433 return decodeSveStructStoreSSInstsByNReg<uint64_t>(esize,
434 machInst, zt, pg, xn, xm, numregs);
435 }
436 return new Unknown64(machInst);
437 }
438
439 StaticInstPtr
440 decodeSveGatherLoadVIInsts(uint8_t dtype, ExtMachInst machInst,
441 IntRegIndex zt, IntRegIndex pg, IntRegIndex zn,
442 uint64_t imm, bool esizeIs32,
443 bool firstFault)
444 {
445 const char* mn = firstFault ? "ldff1" : "ld1";
446 switch (dtype) {
447 case 0x0:
448 if (esizeIs32) {
449 return new SveIndexedMemVI<int32_t, int8_t,
450 SveGatherLoadVIMicroop,
451 SveFirstFaultWritebackMicroop>(
452 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
453 } else {
454 return new SveIndexedMemVI<int64_t, int8_t,
455 SveGatherLoadVIMicroop,
456 SveFirstFaultWritebackMicroop>(
457 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
458 }
459 case 0x1:
460 if (esizeIs32) {
461 return new SveIndexedMemVI<uint32_t, uint8_t,
462 SveGatherLoadVIMicroop,
463 SveFirstFaultWritebackMicroop>(
464 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
465 } else {
466 return new SveIndexedMemVI<uint64_t, uint8_t,
467 SveGatherLoadVIMicroop,
468 SveFirstFaultWritebackMicroop>(
469 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
470 }
471 case 0x2:
472 if (esizeIs32) {
473 return new SveIndexedMemVI<int32_t, int16_t,
474 SveGatherLoadVIMicroop,
475 SveFirstFaultWritebackMicroop>(
476 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
477 } else {
478 return new SveIndexedMemVI<int64_t, int16_t,
479 SveGatherLoadVIMicroop,
480 SveFirstFaultWritebackMicroop>(
481 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
482 }
483 case 0x3:
484 if (esizeIs32) {
485 return new SveIndexedMemVI<uint32_t, uint16_t,
486 SveGatherLoadVIMicroop,
487 SveFirstFaultWritebackMicroop>(
488 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
489 } else {
490 return new SveIndexedMemVI<uint64_t, uint16_t,
491 SveGatherLoadVIMicroop,
492 SveFirstFaultWritebackMicroop>(
493 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
494 }
495 case 0x4:
496 if (esizeIs32) {
497 break;
498 } else {
499 return new SveIndexedMemVI<int64_t, int32_t,
500 SveGatherLoadVIMicroop,
501 SveFirstFaultWritebackMicroop>(
502 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
503 }
504 case 0x5:
505 if (esizeIs32) {
506 return new SveIndexedMemVI<uint32_t, uint32_t,
507 SveGatherLoadVIMicroop,
508 SveFirstFaultWritebackMicroop>(
509 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
510 } else {
511 return new SveIndexedMemVI<uint64_t, uint32_t,
512 SveGatherLoadVIMicroop,
513 SveFirstFaultWritebackMicroop>(
514 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
515 }
516 case 0x7:
517 if (esizeIs32) {
518 break;
519 } else {
520 return new SveIndexedMemVI<uint64_t, uint64_t,
521 SveGatherLoadVIMicroop,
522 SveFirstFaultWritebackMicroop>(
523 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
524 }
525 }
526 return new Unknown64(machInst);
527 }
528
529 StaticInstPtr
530 decodeSveGatherLoadSVInsts(uint8_t dtype, ExtMachInst machInst,
531 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
532 IntRegIndex zm, bool esizeIs32, bool offsetIs32,
533 bool offsetIsSigned, bool offsetIsScaled,
534 bool firstFault)
535 {
536 const char* mn = firstFault ? "ldff1" : "ld1";
537 switch (dtype) {
538 case 0x0:
539 if (esizeIs32) {
540 return new SveIndexedMemSV<int32_t, int8_t,
541 SveGatherLoadSVMicroop,
542 SveFirstFaultWritebackMicroop>(
543 mn, machInst, MemReadOp, zt, pg, rn, zm,
544 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
545 } else {
546 return new SveIndexedMemSV<int64_t, int8_t,
547 SveGatherLoadSVMicroop,
548 SveFirstFaultWritebackMicroop>(
549 mn, machInst, MemReadOp, zt, pg, rn, zm,
550 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
551 }
552 case 0x1:
553 if (esizeIs32) {
554 return new SveIndexedMemSV<uint32_t, uint8_t,
555 SveGatherLoadSVMicroop,
556 SveFirstFaultWritebackMicroop>(
557 mn, machInst, MemReadOp, zt, pg, rn, zm,
558 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
559 } else {
560 return new SveIndexedMemSV<uint64_t, uint8_t,
561 SveGatherLoadSVMicroop,
562 SveFirstFaultWritebackMicroop>(
563 mn, machInst, MemReadOp, zt, pg, rn, zm,
564 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
565 }
566 case 0x2:
567 if (esizeIs32) {
568 return new SveIndexedMemSV<int32_t, int16_t,
569 SveGatherLoadSVMicroop,
570 SveFirstFaultWritebackMicroop>(
571 mn, machInst, MemReadOp, zt, pg, rn, zm,
572 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
573 } else {
574 return new SveIndexedMemSV<int64_t, int16_t,
575 SveGatherLoadSVMicroop,
576 SveFirstFaultWritebackMicroop>(
577 mn, machInst, MemReadOp, zt, pg, rn, zm,
578 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
579 }
580 case 0x3:
581 if (esizeIs32) {
582 return new SveIndexedMemSV<uint32_t, uint16_t,
583 SveGatherLoadSVMicroop,
584 SveFirstFaultWritebackMicroop>(
585 mn, machInst, MemReadOp, zt, pg, rn, zm,
586 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
587 } else {
588 return new SveIndexedMemSV<uint64_t, uint16_t,
589 SveGatherLoadSVMicroop,
590 SveFirstFaultWritebackMicroop>(
591 mn, machInst, MemReadOp, zt, pg, rn, zm,
592 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
593 }
594 case 0x4:
595 if (esizeIs32) {
596 break;
597 } else {
598 return new SveIndexedMemSV<int64_t, int32_t,
599 SveGatherLoadSVMicroop,
600 SveFirstFaultWritebackMicroop>(
601 mn, machInst, MemReadOp, zt, pg, rn, zm,
602 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
603 }
604 case 0x5:
605 if (esizeIs32) {
606 return new SveIndexedMemSV<uint32_t, uint32_t,
607 SveGatherLoadSVMicroop,
608 SveFirstFaultWritebackMicroop>(
609 mn, machInst, MemReadOp, zt, pg, rn, zm,
610 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
611 } else {
612 return new SveIndexedMemSV<uint64_t, uint32_t,
613 SveGatherLoadSVMicroop,
614 SveFirstFaultWritebackMicroop>(
615 mn, machInst, MemReadOp, zt, pg, rn, zm,
616 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
617 }
618 case 0x7:
619 if (esizeIs32) {
620 break;
621 } else {
622 return new SveIndexedMemSV<uint64_t, uint64_t,
623 SveGatherLoadSVMicroop,
624 SveFirstFaultWritebackMicroop>(
625 mn, machInst, MemReadOp, zt, pg, rn, zm,
626 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
627 }
628 }
629 return new Unknown64(machInst);
630 }
631
632 StaticInstPtr
633 decodeSveScatterStoreVIInsts(uint8_t msz, ExtMachInst machInst,
634 IntRegIndex zt, IntRegIndex pg,
635 IntRegIndex zn, uint64_t imm,
636 bool esizeIs32)
637 {
638 const char* mn = "st1";
639 switch (msz) {
640 case 0x0:
641 if (esizeIs32) {
642 return new SveIndexedMemVI<uint32_t, uint8_t,
643 SveScatterStoreVIMicroop,
644 SveFirstFaultWritebackMicroop>(
645 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
646 } else {
647 return new SveIndexedMemVI<uint64_t, uint8_t,
648 SveScatterStoreVIMicroop,
649 SveFirstFaultWritebackMicroop>(
650 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
651 }
652 case 0x1:
653 if (esizeIs32) {
654 return new SveIndexedMemVI<uint32_t, uint16_t,
655 SveScatterStoreVIMicroop,
656 SveFirstFaultWritebackMicroop>(
657 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
658 } else {
659 return new SveIndexedMemVI<uint64_t, uint16_t,
660 SveScatterStoreVIMicroop,
661 SveFirstFaultWritebackMicroop>(
662 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
663 }
664 case 0x2:
665 if (esizeIs32) {
666 return new SveIndexedMemVI<uint32_t, uint32_t,
667 SveScatterStoreVIMicroop,
668 SveFirstFaultWritebackMicroop>(
669 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
670 } else {
671 return new SveIndexedMemVI<uint64_t, uint32_t,
672 SveScatterStoreVIMicroop,
673 SveFirstFaultWritebackMicroop>(
674 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
675 }
676 case 0x3:
677 if (esizeIs32) {
678 break;
679 } else {
680 return new SveIndexedMemVI<uint64_t, uint64_t,
681 SveScatterStoreVIMicroop,
682 SveFirstFaultWritebackMicroop>(
683 mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
684 }
685 }
686 return new Unknown64(machInst);
687 }
688
689 StaticInstPtr
690 decodeSveScatterStoreSVInsts(uint8_t msz, ExtMachInst machInst,
691 IntRegIndex zt, IntRegIndex pg,
692 IntRegIndex rn, IntRegIndex zm,
693 bool esizeIs32, bool offsetIs32,
694 bool offsetIsSigned, bool offsetIsScaled)
695 {
696 const char* mn = "st1";
697 switch (msz) {
698 case 0x0:
699 if (esizeIs32) {
700 return new SveIndexedMemSV<uint32_t, uint8_t,
701 SveScatterStoreSVMicroop,
702 SveFirstFaultWritebackMicroop>(
703 mn, machInst, MemWriteOp, zt, pg, rn, zm,
704 offsetIs32, offsetIsSigned, offsetIsScaled, false);
705 } else {
706 return new SveIndexedMemSV<uint64_t, uint8_t,
707 SveScatterStoreSVMicroop,
708 SveFirstFaultWritebackMicroop>(
709 mn, machInst, MemWriteOp, zt, pg, rn, zm,
710 offsetIs32, offsetIsSigned, offsetIsScaled, false);
711 }
712 case 0x1:
713 if (esizeIs32) {
714 return new SveIndexedMemSV<uint32_t, uint16_t,
715 SveScatterStoreSVMicroop,
716 SveFirstFaultWritebackMicroop>(
717 mn, machInst, MemWriteOp, zt, pg, rn, zm,
718 offsetIs32, offsetIsSigned, offsetIsScaled, false);
719 } else {
720 return new SveIndexedMemSV<uint64_t, uint16_t,
721 SveScatterStoreSVMicroop,
722 SveFirstFaultWritebackMicroop>(
723 mn, machInst, MemWriteOp, zt, pg, rn, zm,
724 offsetIs32, offsetIsSigned, offsetIsScaled, false);
725 }
726 case 0x2:
727 if (esizeIs32) {
728 return new SveIndexedMemSV<uint32_t, uint32_t,
729 SveScatterStoreSVMicroop,
730 SveFirstFaultWritebackMicroop>(
731 mn, machInst, MemWriteOp, zt, pg, rn, zm,
732 offsetIs32, offsetIsSigned, offsetIsScaled, false);
733 } else {
734 return new SveIndexedMemSV<uint64_t, uint32_t,
735 SveScatterStoreSVMicroop,
736 SveFirstFaultWritebackMicroop>(
737 mn, machInst, MemWriteOp, zt, pg, rn, zm,
738 offsetIs32, offsetIsSigned, offsetIsScaled, false);
739 }
740 case 0x3:
741 if (esizeIs32) {
742 break;
743 } else {
744 return new SveIndexedMemSV<uint64_t, uint64_t,
745 SveScatterStoreSVMicroop,
746 SveFirstFaultWritebackMicroop>(
747 mn, machInst, MemWriteOp, zt, pg, rn, zm,
748 offsetIs32, offsetIsSigned, offsetIsScaled, false);
749 }
750 }
751 return new Unknown64(machInst);
752 }
753
754}};
755
756
757let {{
758
759 header_output = ''
760 exec_output = ''
761 decoders = { 'Generic': {} }
762
763 SPAlignmentCheckCode = '''
764 if (this->baseIsSP && bits(XBase, 3, 0) &&
765 SPAlignmentCheckEnabled(xc->tcBase())) {
766 return std::make_shared<SPAlignmentFault>();
767 }
768 '''
769
770 def emitSveMemFillSpill(isPred):
771 global header_output, exec_output, decoders
772 eaCode = SPAlignmentCheckCode + '''
773 int memAccessSize = %(memacc_size)s;
774 EA = XBase + ((int64_t) imm * %(memacc_size)s)''' % {
775 'memacc_size': 'eCount / 8' if isPred else 'eCount'}
776 loadRdEnableCode = '''
777 auto rdEn = std::vector<bool>();
778 '''
779 if isPred:
780 loadMemAccCode = '''
781 int index = 0;
782 uint8_t byte;
783 for (int i = 0; i < eCount / 8; i++) {
784 byte = memDataView[i];
785 for (int j = 0; j < 8; j++, index++) {
786 PDest_x[index] = (byte >> j) & 1;
787 }
788 }
789 '''
790 storeMemAccCode = '''
791 int index = 0;
792 uint8_t byte;
793 for (int i = 0; i < eCount / 8; i++) {
794 byte = 0;
795 for (int j = 0; j < 8; j++, index++) {
796 byte |= PDest_x[index] << j;
797 }
798 memDataView[i] = byte;
799 }
800 '''
801 storeWrEnableCode = '''
802 auto wrEn = std::vector<bool>(eCount / 8, true);
803 '''
804 else:
805 loadMemAccCode = '''
806 for (int i = 0; i < eCount; i++) {
807 AA64FpDest_x[i] = memDataView[i];
808 }
809 '''
810 storeMemAccCode = '''
811 for (int i = 0; i < eCount; i++) {
812 memDataView[i] = AA64FpDest_x[i];
813 }
814 '''
815 storeWrEnableCode = '''
816 auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
817 '''
818 loadIop = InstObjParams('ldr',
819 'SveLdrPred' if isPred else 'SveLdrVec',
820 'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill',
821 {'tpl_header': '',
822 'tpl_args': '',
823 'memacc_code': loadMemAccCode,
824 'ea_code' : sveEnabledCheckCode + eaCode,
825 'rden_code' : loadRdEnableCode,
826 'fault_code' : '',
827 'fa_code' : ''},
828 ['IsMemRef', 'IsLoad'])
829 storeIop = InstObjParams('str',
830 'SveStrPred' if isPred else 'SveStrVec',
831 'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill',
832 {'tpl_header': '',
833 'tpl_args': '',
834 'wren_code': storeWrEnableCode,
835 'memacc_code': storeMemAccCode,
836 'ea_code' : sveEnabledCheckCode + eaCode,
837 'fa_code' : ''},
838 ['IsMemRef', 'IsStore'])
839 header_output += SveMemFillSpillOpDeclare.subst(loadIop)
840 header_output += SveMemFillSpillOpDeclare.subst(storeIop)
841 exec_output += (
842 SveContigLoadExecute.subst(loadIop) +
843 SveContigLoadInitiateAcc.subst(loadIop) +
844 SveContigLoadCompleteAcc.subst(loadIop) +
845 SveContigStoreExecute.subst(storeIop) +
846 SveContigStoreInitiateAcc.subst(storeIop) +
847 SveContigStoreCompleteAcc.subst(storeIop))
848
849 loadTplArgs = (
850 ('uint8_t', 'uint8_t'),
851 ('uint16_t', 'uint8_t'),
852 ('uint32_t', 'uint8_t'),
853 ('uint64_t', 'uint8_t'),
854 ('int64_t', 'int32_t'),
855 ('uint16_t', 'uint16_t'),
856 ('uint32_t', 'uint16_t'),
857 ('uint64_t', 'uint16_t'),
858 ('int64_t', 'int16_t'),
859 ('int32_t', 'int16_t'),
860 ('uint32_t', 'uint32_t'),
861 ('uint64_t', 'uint32_t'),
862 ('int64_t', 'int8_t'),
863 ('int32_t', 'int8_t'),
864 ('int16_t', 'int8_t'),
865 ('uint64_t', 'uint64_t'),
866 )
867
868 storeTplArgs = (
869 ('uint8_t', 'uint8_t'),
870 ('uint16_t', 'uint8_t'),
871 ('uint32_t', 'uint8_t'),
872 ('uint64_t', 'uint8_t'),
873 ('uint16_t', 'uint16_t'),
874 ('uint32_t', 'uint16_t'),
875 ('uint64_t', 'uint16_t'),
876 ('uint32_t', 'uint32_t'),
877 ('uint64_t', 'uint32_t'),
878 ('uint64_t', 'uint64_t'),
879 )
880
881 gatherLoadTplArgs = (
882 ('int32_t', 'int8_t'),
883 ('int64_t', 'int8_t'),
884 ('uint32_t', 'uint8_t'),
885 ('uint64_t', 'uint8_t'),
886 ('int32_t', 'int16_t'),
887 ('int64_t', 'int16_t'),
888 ('uint32_t', 'uint16_t'),
889 ('uint64_t', 'uint16_t'),
890 ('int64_t', 'int32_t'),
891 ('uint32_t', 'uint32_t'),
892 ('uint64_t', 'uint32_t'),
893 ('uint64_t', 'uint64_t'),
894 )
895
896 scatterStoreTplArgs = (
897 ('uint32_t', 'uint8_t'),
898 ('uint64_t', 'uint8_t'),
899 ('uint32_t', 'uint16_t'),
900 ('uint64_t', 'uint16_t'),
901 ('uint32_t', 'uint32_t'),
902 ('uint64_t', 'uint32_t'),
903 ('uint64_t', 'uint64_t'),
904 )
905
906 # Generates definitions for SVE contiguous loads
907 def emitSveContigMemInsts(offsetIsImm):
908 global header_output, exec_output, decoders
909 # First-faulting instructions only have a scalar plus scalar form,
910 # while non-faulting instructions only a scalar plus immediate form, so
911 # `offsetIsImm` is used to determine which class of instructions is
912 # generated
913 firstFaulting = not offsetIsImm
914 tplHeader = 'template <class RegElemType, class MemElemType>'
915 tplArgs = '<RegElemType, MemElemType>'
916 eaCode = SPAlignmentCheckCode + '''
917 int memAccessSize = eCount * sizeof(MemElemType);
918 EA = XBase + '''
919 if offsetIsImm:
920 eaCode += '((int64_t) this->imm * eCount * sizeof(MemElemType))'
921 else:
922 eaCode += '(XOffset * sizeof(MemElemType));'
923 loadRdEnableCode = '''
924 auto rdEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
925 for (int i = 0; i < eCount; i++) {
926 if (!GpOp_x[i]) {
927 for (int j = 0; j < sizeof(MemElemType); j++) {
928 rdEn[sizeof(MemElemType) * i + j] = false;
929 }
930 }
931 }
932 '''
933 loadMemAccCode = '''
934 for (int i = 0; i < eCount; i++) {
935 if (GpOp_x[i]) {
936 AA64FpDest_x[i] = memDataView[i];
937 } else {
938 AA64FpDest_x[i] = 0;
939 }
940 }
941 '''
942 storeMemAccCode = '''
943 for (int i = 0; i < eCount; i++) {
944 if (GpOp_x[i]) {
945 memDataView[i] = AA64FpDest_x[i];
946 } else {
947 memDataView[i] = 0;
948 for (int j = 0; j < sizeof(MemElemType); j++) {
949 wrEn[sizeof(MemElemType) * i + j] = false;
950 }
951 }
952 }
953 '''
954 storeWrEnableCode = '''
955 auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
956 '''
957 ffrReadBackCode = '''
958 auto& firstFaultReg = Ffr;'''
959 fautlingLoadmemAccCode = '''
960 for (int i = 0; i < eCount; i++) {
961 if (GpOp_x[i] && firstFaultReg[i * sizeof(RegElemType)]) {
962 AA64FpDest_x[i] = memDataView[i];
963 } else {
964 AA64FpDest_x[i] = 0;
965 }
966 }
967 '''
968 nonFaultingCode = 'true ||'
969 faultCode = '''
970 Addr fault_addr;
971 if (fault == NoFault || getFaultVAddr(fault, fault_addr)) {
972 unsigned fault_elem_index;
973 if (fault != NoFault) {
974 assert(fault_addr >= EA);
975 fault_elem_index = (fault_addr - EA) / sizeof(MemElemType);
976 } else {
977 fault_elem_index = eCount + 1;
978 }
979 int first_active_index;
980 for (first_active_index = 0;
981 first_active_index < eCount && !(GpOp_x[first_active_index]);
982 first_active_index++);
983 if (%s first_active_index < fault_elem_index) {
984 for (int i = 0; i < eCount; i++) {
985 for (int j = 0; j < sizeof(RegElemType); j++) {
986 if (i < fault_elem_index) {
987 Ffr_ub[i * sizeof(RegElemType) + j] = FfrAux_x[i];
988 } else {
989 Ffr_ub[i * sizeof(RegElemType) + j] = 0;
990 }
991 }
992 }
993 fault = NoFault;
994 if (first_active_index >= fault_elem_index) {
995 // non-faulting load needs this
996 xc->setMemAccPredicate(false);
997 }
998 }
999 }
1000 ''' % ('' if firstFaulting else nonFaultingCode)
1001
1002 loadIop = InstObjParams('ld1',
1003 'SveContigLoadSI' if offsetIsImm else 'SveContigLoadSS',
1004 'SveContigMemSI' if offsetIsImm else 'SveContigMemSS',
1005 {'tpl_header': tplHeader,
1006 'tpl_args': tplArgs,
1007 'rden_code' : loadRdEnableCode,
1008 'memacc_code': loadMemAccCode,
1009 'ea_code' : sveEnabledCheckCode + eaCode,
1010 'fault_code' : '',
1011 'fa_code' : ''},
1012 ['IsMemRef', 'IsLoad'])
1013 storeIop = InstObjParams('st1',
1014 'SveContigStoreSI' if offsetIsImm else 'SveContigStoreSS',
1015 'SveContigMemSI' if offsetIsImm else 'SveContigMemSS',
1016 {'tpl_header': tplHeader,
1017 'tpl_args': tplArgs,
1018 'wren_code': storeWrEnableCode,
1019 'memacc_code': storeMemAccCode,
1020 'ea_code' : sveEnabledCheckCode + eaCode,
1021 'fa_code' : ''},
1022 ['IsMemRef', 'IsStore'])
1023 faultIop = InstObjParams('ldff1' if firstFaulting else 'ldnf1',
1024 'SveContigFFLoadSS' if firstFaulting else 'SveContigNFLoadSI',
1025 'SveContigMemSS' if firstFaulting else 'SveContigMemSI',
1026 {'tpl_header': tplHeader,
1027 'tpl_args': tplArgs,
1028 'rden_code' : loadRdEnableCode,
1029 'memacc_code': fautlingLoadmemAccCode,
1030 'ea_code' : sveEnabledCheckCode + eaCode,
1031 'fault_code' : faultCode,
1032 'fa_code' : ''},
1033 ['IsMemRef', 'IsLoad'])
1034 faultIop.snippets['memacc_code'] = (ffrReadBackCode +
1035 faultIop.snippets['memacc_code'])
1036 if offsetIsImm:
1037 header_output += SveContigMemSIOpDeclare.subst(loadIop)
1038 header_output += SveContigMemSIOpDeclare.subst(storeIop)
1039 header_output += SveContigMemSIOpDeclare.subst(faultIop)
1040 else:
1041 header_output += SveContigMemSSOpDeclare.subst(loadIop)
1042 header_output += SveContigMemSSOpDeclare.subst(storeIop)
1043 header_output += SveContigMemSSOpDeclare.subst(faultIop)
1044 exec_output += (
1045 SveContigLoadExecute.subst(loadIop) +
1046 SveContigLoadInitiateAcc.subst(loadIop) +
1047 SveContigLoadCompleteAcc.subst(loadIop) +
1048 SveContigStoreExecute.subst(storeIop) +
1049 SveContigStoreInitiateAcc.subst(storeIop) +
1050 SveContigStoreCompleteAcc.subst(storeIop) +
1051 SveContigLoadExecute.subst(faultIop) +
1052 SveContigLoadInitiateAcc.subst(faultIop) +
1053 SveContigLoadCompleteAcc.subst(faultIop))
1054
1055 for args in loadTplArgs:
1056 substDict = {'tpl_args': '<%s>' % ', '.join(args),
1057 'class_name': 'SveContigLoadSI' if offsetIsImm
1058 else 'SveContigLoadSS'}
1059 exec_output += SveContigMemExecDeclare.subst(substDict)
1060 for args in storeTplArgs:
1061 substDict = {'tpl_args': '<%s>' % ', '.join(args),
1062 'class_name': 'SveContigStoreSI' if offsetIsImm
1063 else 'SveContigStoreSS'}
1064 exec_output += SveContigMemExecDeclare.subst(substDict)
1065 for args in loadTplArgs:
1066 substDict = {'tpl_args': '<%s>' % ', '.join(args),
1067 'class_name': 'SveContigFFLoadSS' if firstFaulting
1068 else 'SveContigNFLoadSI'}
1069 exec_output += SveContigMemExecDeclare.subst(substDict)
1070
1071
1072 # Generates definitions for SVE load-and-replicate instructions
1073 def emitSveLoadAndRepl():
1074 global header_output, exec_output, decoders
1075 tplHeader = 'template <class RegElemType, class MemElemType>'
1076 tplArgs = '<RegElemType, MemElemType>'
1077 eaCode = SPAlignmentCheckCode + '''
1078 EA = XBase + imm * sizeof(MemElemType);'''
1079 memAccCode = '''
1080 for (int i = 0; i < eCount; i++) {
1081 if (GpOp_x[i]) {
1082 AA64FpDest_x[i] = memData;
1083 } else {
1084 AA64FpDest_x[i] = 0;
1085 }
1086 }
1087 '''
1088 iop = InstObjParams('ld1r',
1089 'SveLoadAndRepl',
1090 'SveContigMemSI',
1091 {'tpl_header': tplHeader,
1092 'tpl_args': tplArgs,
1093 'memacc_code': memAccCode,
1094 'ea_code' : sveEnabledCheckCode + eaCode,
1095 'fa_code' : ''},
1096 ['IsMemRef', 'IsLoad'])
1097 header_output += SveContigMemSIOpDeclare.subst(iop)
1098 exec_output += (
1099 SveLoadAndReplExecute.subst(iop) +
1100 SveLoadAndReplInitiateAcc.subst(iop) +
1101 SveLoadAndReplCompleteAcc.subst(iop))
1102 for args in loadTplArgs:
1103 substDict = {'tpl_args': '<%s>' % ', '.join(args),
1104 'class_name': 'SveLoadAndRepl'}
1105 exec_output += SveContigMemExecDeclare.subst(substDict)
1106
1107 class IndexedAddrForm:
1108 VEC_PLUS_IMM = 0
1109 SCA_PLUS_VEC = 1
1110
1111 # Generates definitions for the transfer microops of SVE indexed memory
1112 # operations (gather loads, scatter stores)
1113 def emitSveIndexedMemMicroops(indexed_addr_form):
1114 assert indexed_addr_form in (IndexedAddrForm.VEC_PLUS_IMM,
1115 IndexedAddrForm.SCA_PLUS_VEC)
1116 global header_output, exec_output, decoders
1117 tplHeader = 'template <class RegElemType, class MemElemType>'
1118 tplArgs = '<RegElemType, MemElemType>'
1119 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM:
1120 eaCode = '''
1121 EA = AA64FpBase_x[elemIndex] + imm * sizeof(MemElemType)'''
1122 else:
1123 eaCode = '''
1124 uint64_t offset = AA64FpOffset_x[elemIndex];
1125 if (offsetIs32) {
1126 offset &= (1ULL << 32) - 1;
1127 }
1128 if (offsetIsSigned) {
1129 offset = sext<32>(offset);
1130 }
1131 if (offsetIsScaled) {
1132 offset *= sizeof(MemElemType);
1133 }
1134 EA = XBase + offset'''
1135 loadMemAccCode = '''
1136 AA64FpDest_x[elemIndex] = memData;
1137 '''
1138 storeMemAccCode = '''
1139 memData = AA64FpDest_x[elemIndex];
1140 '''
1141 predCheckCode = 'GpOp_x[index]'
1142 faultStatusSetCode = 'PUreg0_x[elemIndex] = 1;'
1143 faultStatusResetCode = 'PUreg0_x[elemIndex] = 0;'
1144 loadIop = InstObjParams('ld1',
1145 ('SveGatherLoadVIMicroop'
1146 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM
1147 else 'SveGatherLoadSVMicroop'),
1148 'MicroOp',
1149 {'tpl_header': tplHeader,
1150 'tpl_args': tplArgs,
1151 'memacc_code': loadMemAccCode,
1152 'ea_code' : sveEnabledCheckCode + eaCode,
1153 'fault_status_set_code' : faultStatusSetCode,
1154 'fault_status_reset_code' : faultStatusResetCode,
1155 'pred_check_code' : predCheckCode,
1156 'fa_code' : ''},
1157 ['IsMicroop', 'IsMemRef', 'IsLoad'])
1158 storeIop = InstObjParams('st1',
1159 ('SveScatterStoreVIMicroop'
1160 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM
1161 else 'SveScatterStoreSVMicroop'),
1162 'MicroOp',
1163 {'tpl_header': tplHeader,
1164 'tpl_args': tplArgs,
1165 'memacc_code': storeMemAccCode,
1166 'ea_code' : sveEnabledCheckCode + eaCode,
1167 'pred_check_code' : predCheckCode,
1168 'fa_code' : ''},
1169 ['IsMicroop', 'IsMemRef', 'IsStore'])
1170 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM:
1171 header_output += SveIndexedMemVIMicroopDeclare.subst(loadIop)
1172 header_output += SveIndexedMemVIMicroopDeclare.subst(storeIop)
1173 else:
1174 header_output += SveIndexedMemSVMicroopDeclare.subst(loadIop)
1175 header_output += SveIndexedMemSVMicroopDeclare.subst(storeIop)
1176 exec_output += (
1177 SveGatherLoadMicroopExecute.subst(loadIop) +
1178 SveGatherLoadMicroopInitiateAcc.subst(loadIop) +
1179 SveGatherLoadMicroopCompleteAcc.subst(loadIop) +
1180 SveScatterStoreMicroopExecute.subst(storeIop) +
1181 SveScatterStoreMicroopInitiateAcc.subst(storeIop) +
1182 SveScatterStoreMicroopCompleteAcc.subst(storeIop))
1183 for args in gatherLoadTplArgs:
1184 substDict = {'tpl_args': '<%s>' % ', '.join(args),
1185 'class_name': (
1186 'SveGatherLoadVIMicroop'
1187 if indexed_addr_form == \
1188 IndexedAddrForm.VEC_PLUS_IMM
1189 else 'SveGatherLoadSVMicroop')}
1190 # TODO: this should become SveMemExecDeclare
1191 exec_output += SveContigMemExecDeclare.subst(substDict)
1192 for args in scatterStoreTplArgs:
1193 substDict = {'tpl_args': '<%s>' % ', '.join(args),
1194 'class_name': (
1195 'SveScatterStoreVIMicroop'
1196 if indexed_addr_form == \
1197 IndexedAddrForm.VEC_PLUS_IMM
1198 else 'SveScatterStoreSVMicroop')}
1199 # TODO: this should become SveMemExecDeclare
1200 exec_output += SveContigMemExecDeclare.subst(substDict)
1201
1202 firstFaultTplArgs = ('int32_t', 'int64_t', 'uint32_t', 'uint64_t')
1203
1204 def emitSveFirstFaultWritebackMicroop():
1205 global header_output, exec_output, decoders
1206 tplHeader = 'template <class RegElemType>'
1207 tplArgs = '<RegElemType>'
1208 faultStatusCheckCode = 'PUreg0_x[index]'
1209 firstFaultResetCode = '''
1210 for(int j = 0; j < sizeof(RegElemType); j++) {
1211 Ffr_ub[index * sizeof(RegElemType) + j] = 0;
1212 }
1213 '''
1214 firstFaultForwardCode = '''
1215 for(int j = 0; j < sizeof(RegElemType); j++) {
1216 Ffr_ub[index * sizeof(RegElemType) + j] = FfrAux_x[index];
1217 }
1218 '''
1219 iop = InstObjParams('ldff1',
1220 'SveFirstFaultWritebackMicroop',
1221 'MicroOp',
1222 {'tpl_header': tplHeader,
1223 'tpl_args': tplArgs,
1224 'fault_status_check_code' : faultStatusCheckCode,
1225 'first_fault_reset_code' : firstFaultResetCode,
1226 'first_fault_forward_code' : firstFaultForwardCode},
1227 ['IsMicroop'])
1228 header_output += SveFirstFaultWritebackMicroopDeclare.subst(iop)
1229 exec_output += SveFirstFaultWritebackMicroopExecute.subst(iop)
1230 for args in firstFaultTplArgs:
1231 substDict = {'targs': args,
1232 'class_name' : 'SveFirstFaultWritebackMicroop' }
1233 exec_output += SveOpExecDeclare.subst(substDict)
1234
1235 # Generates definitions for the first microop of SVE gather loads, required
1236 # to propagate the source vector register to the transfer microops
1237 def emitSveGatherLoadCpySrcVecMicroop():
1238 global header_output, exec_output, decoders
1239 code = sveEnabledCheckCode + '''
1240 unsigned eCount = ArmStaticInst::getCurSveVecLen<uint8_t>(
1241 xc->tcBase());
1242 for (unsigned i = 0; i < eCount; i++) {
1243 AA64FpUreg0_ub[i] = AA64FpOp1_ub[i];
1244 }'''
1245 iop = InstObjParams('ld1',
1246 'SveGatherLoadCpySrcVecMicroop',
1247 'MicroOp',
1248 {'code': code},
1249 ['IsMicroop'])
1250 header_output += SveGatherLoadCpySrcVecMicroopDeclare.subst(iop)
1251 exec_output += SveGatherLoadCpySrcVecMicroopExecute.subst(iop)
1252
1253 def emitSveInterleaveMicroop():
1254 global header_output, exec_output, decoders
1255 code2 = sveEnabledCheckCode + '''
1256 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1257 xc->tcBase());
1258 for (unsigned int i = 0; i < eCount; ++i) {
1259 unsigned int absIdx = regIndex * eCount + i;
1260 unsigned int srcIdx = absIdx / numRegs;
1261 unsigned int srcVec = absIdx % numRegs;
1262 if (srcVec == 0)
1263 AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx];
1264 else if (srcVec == 1)
1265 AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx];
1266 }'''
1267
1268 code3 = sveEnabledCheckCode + '''
1269 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1270 xc->tcBase());
1271 for (unsigned int i = 0; i < eCount; ++i) {
1272 unsigned int absIdx = regIndex * eCount + i;
1273 unsigned int srcIdx = absIdx / numRegs;
1274 unsigned int srcVec = absIdx % numRegs;
1275 if (srcVec == 0)
1276 AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx];
1277 else if (srcVec == 1)
1278 AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx];
1279 else if (srcVec == 2)
1280 AA64FpDest_x[i] = AA64FpOp1V2S_x[srcIdx];
1281 }'''
1282
1283 code4 = sveEnabledCheckCode + '''
1284 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1285 xc->tcBase());
1286 for (unsigned int i = 0; i < eCount; ++i) {
1287 unsigned int absIdx = regIndex * eCount + i;
1288 unsigned int srcIdx = absIdx / numRegs;
1289 unsigned int srcVec = absIdx % numRegs;
1290 if (srcVec == 0)
1291 AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx];
1292 else if (srcVec == 1)
1293 AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx];
1294 else if (srcVec == 2)
1295 AA64FpDest_x[i] = AA64FpOp1V2S_x[srcIdx];
1296 else if (srcVec == 3)
1297 AA64FpDest_x[i] = AA64FpOp1V3S_x[srcIdx];
1298 }'''
1299
1300 iop2 = InstObjParams('intrlv',
1301 'SveIntrlv2Microop',
1302 'MicroOp',
1303 {'code': code2},
1304 ['IsMicroop'])
1305 iop3 = InstObjParams('intrlv',
1306 'SveIntrlv3Microop',
1307 'MicroOp',
1308 {'code': code3},
1309 ['IsMicroop'])
1310 iop4 = InstObjParams('intrlv',
1311 'SveIntrlv4Microop',
1312 'MicroOp',
1313 {'code': code4},
1314 ['IsMicroop'])
1315 header_output += SveIntrlvMicroopDeclare.subst(iop2);
1316 header_output += SveIntrlvMicroopDeclare.subst(iop3);
1317 header_output += SveIntrlvMicroopDeclare.subst(iop4);
1318 exec_output += SveIntrlvMicroopExecute.subst(iop2);
1319 exec_output += SveIntrlvMicroopExecute.subst(iop3);
1320 exec_output += SveIntrlvMicroopExecute.subst(iop4);
1321 for type in ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'):
1322 for nreg in range(2,5):
1323 substDict = {'targs' : type,
1324 'class_name' : 'SveIntrlv' + str(nreg) + 'Microop'}
1325 exec_output += SveIntrlvMicroopExecDeclare.subst(substDict)
1326
1327 def emitSveDeInterleaveMicroop():
1328 global header_output, exec_output, decoders
1329 code2 = sveEnabledCheckCode + '''
1330 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1331 xc->tcBase());
1332 for (unsigned int i = 0; i < eCount; ++i) {
1333 unsigned int absIdx = (regIndex + numRegs * i);
1334 unsigned int srcIdx = absIdx % eCount;
1335 unsigned int srcVec = absIdx / eCount;
1336 if (srcVec == 0)
1337 AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx];
1338 else if(srcVec == 1)
1339 AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx];
1340 }'''
1341
1342 code3 = sveEnabledCheckCode + '''
1343 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1344 xc->tcBase());
1345 for (unsigned int i = 0; i < eCount; ++i) {
1346 unsigned int absIdx = (regIndex + numRegs * i);
1347 unsigned int srcIdx = absIdx % eCount;
1348 unsigned int srcVec = absIdx / eCount;
1349 if (srcVec == 0)
1350 AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx];
1351 else if(srcVec == 1)
1352 AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx];
1353 else if(srcVec == 2)
1354 AA64FpDest_x[i] = AA64IntrlvReg2_x[srcIdx];
1355 }'''
1356
1357 code4 = sveEnabledCheckCode + '''
1358 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1359 xc->tcBase());
1360 for (unsigned int i = 0; i < eCount; ++i) {
1361 unsigned int absIdx = (regIndex + numRegs * i);
1362 unsigned int srcIdx = absIdx % eCount;
1363 unsigned int srcVec = absIdx / eCount;
1364 if (srcVec == 0)
1365 AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx];
1366 else if(srcVec == 1)
1367 AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx];
1368 else if(srcVec == 2)
1369 AA64FpDest_x[i] = AA64IntrlvReg2_x[srcIdx];
1370 else if(srcVec == 3)
1371 AA64FpDest_x[i] = AA64IntrlvReg3_x[srcIdx];
1372 }'''
1373
1374 iop2 = InstObjParams('deintrlv',
1375 'SveDeIntrlv2Microop',
1376 'MicroOp',
1377 {'code': code2},
1378 ['IsMicroop'])
1379 iop3 = InstObjParams('deintrlv',
1380 'SveDeIntrlv3Microop',
1381 'MicroOp',
1382 {'code': code3},
1383 ['IsMicroop'])
1384 iop4 = InstObjParams('deintrlv',
1385 'SveDeIntrlv4Microop',
1386 'MicroOp',
1387 {'code': code4},
1388 ['IsMicroop'])
1389 header_output += SveDeIntrlvMicroopDeclare.subst(iop2);
1390 header_output += SveDeIntrlvMicroopDeclare.subst(iop3);
1391 header_output += SveDeIntrlvMicroopDeclare.subst(iop4);
1392 exec_output += SveIntrlvMicroopExecute.subst(iop2);
1393 exec_output += SveIntrlvMicroopExecute.subst(iop3);
1394 exec_output += SveIntrlvMicroopExecute.subst(iop4);
1395 for type in ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'):
1396 for nreg in range(2,5):
1397 substDict = {'targs' : type,
1398 'class_name' : 'SveDeIntrlv' + str(nreg) + 'Microop'}
1399 exec_output += SveIntrlvMicroopExecDeclare.subst(substDict)
1400
1401 # Generates definitions for SVE struct load/store microops
1402 def emitSveStructMemInsts(offsetIsImm):
1403 global header_output, exec_output, decoders
1404 eaCode = SPAlignmentCheckCode + '''
1405 int memAccessSize = eCount * sizeof(Element);
1406 EA = memAccessSize * regIndex + XBase + '''
1407 if offsetIsImm:
1408 eaCode += '((int64_t) this->imm * eCount * sizeof(Element))'
1409 else:
1410 eaCode += '(XOffset * sizeof(Element));'
1411 loadMemAccCode = '''
1412 for (int i = 0; i < eCount; i++) {
1413 int gpIdx = (regIndex * eCount + i) / numRegs;
1414 if (GpOp_x[gpIdx]) {
1415 AA64FpDest_x[i] = memDataView[i];
1416 } else {
1417 AA64FpDest_x[i] = 0;
1418 }
1419 }
1420 '''
1421 storeMemAccCode = '''
1422 for (int i = 0; i < eCount; i++) {
1423 int gpIdx = (regIndex * eCount + i) / numRegs;
1424 if (GpOp_x[gpIdx]) {
1425 memDataView[i] = AA64FpDest_x[i];
1426 } else {
1427 memDataView[i] = 0;
1428 for (int j = 0; j < sizeof(Element); j++) {
1429 wrEn[sizeof(Element) * i + j] = false;
1430 }
1431 }
1432 }
1433 '''
1434 storeWrEnableCode = '''
1435 auto wrEn = std::vector<bool>(sizeof(Element) * eCount, true);
1436 '''
1437 loadIop = InstObjParams('ldxx',
1438 'SveLoadRegImmMicroop' if offsetIsImm else 'SveLoadRegRegMicroop',
1439 'MicroOp',
1440 {'targs': 'Element',
1441 'memacc_code': loadMemAccCode,
1442 'ea_code' : sveEnabledCheckCode + eaCode,
1443 'fa_code' : ''},
1444 ['IsMemRef', 'IsLoad', 'IsMicroop'])
1445 storeIop = InstObjParams('stxx',
1446 'SveStoreRegImmMicroop' if offsetIsImm
1447 else 'SveStoreRegRegMicroop',
1448 'MicroOp',
1449 {'targs': 'Element',
1450 'wren_code': storeWrEnableCode,
1451 'memacc_code': storeMemAccCode,
1452 'ea_code' : sveEnabledCheckCode + eaCode,
1453 'fa_code' : ''},
1454 ['IsMemRef', 'IsStore', 'IsMicroop'])
1455 if offsetIsImm:
1456 header_output += SveStructMemSIMicroopDeclare.subst(loadIop)
1457 header_output += SveStructMemSIMicroopDeclare.subst(storeIop)
1458 else:
1459 header_output += SveStructMemSSMicroopDeclare.subst(loadIop)
1460 header_output += SveStructMemSSMicroopDeclare.subst(storeIop)
1461 exec_output += (
1462 SveStructLoadExecute.subst(loadIop) +
1463 SveStructLoadInitiateAcc.subst(loadIop) +
1464 SveStructLoadCompleteAcc.subst(loadIop) +
1465 SveStructStoreExecute.subst(storeIop) +
1466 SveStructStoreInitiateAcc.subst(storeIop) +
1467 SveStructStoreCompleteAcc.subst(storeIop))
1468 tplArgs = ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t')
1469 for type in tplArgs:
1470 substDict = {'targs': type,
1471 'class_name': 'SveLoadRegImmMicroop' if offsetIsImm
1472 else 'SveLoadRegRegMicroop'}
1473 exec_output += SveStructMemExecDeclare.subst(substDict)
1474 substDict['class_name'] = ('SveStoreRegImmMicroop' if offsetIsImm
1475 else 'SveStoreRegRegMicroop')
1476 exec_output += SveStructMemExecDeclare.subst(substDict)
1477
|