1// Copyright (c) 2017-2018 ARM Limited 2// All rights reserved 3// 4// The license below extends only to copyright in the software and shall 5// not be construed as granting a license to any other intellectual 6// property including but not limited to intellectual property relating 7// to a hardware implementation of the functionality of the software 8// licensed hereunder. You may use the software subject to the license 9// terms below provided that you ensure that this notice is replicated 10// unmodified and in its entirety in all distributions of the software, 11// modified or unmodified, in source code or in binary form. 12// 13// Redistribution and use in source and binary forms, with or without 14// modification, are permitted provided that the following conditions are 15// met: redistributions of source code must retain the above copyright 16// notice, this list of conditions and the following disclaimer; 17// redistributions in binary form must reproduce the above copyright 18// notice, this list of conditions and the following disclaimer in the 19// documentation and/or other materials provided with the distribution; 20// neither the name of the copyright holders nor the names of its 21// contributors may be used to endorse or promote products derived from 22// this software without specific prior written permission. 23// 24// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 27// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 28// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 30// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 34// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35// 36// Authors: Giacomo Gabrielli 37 38// @file Definition of SVE memory access instructions. 39 40output header {{ 41 42 // Decodes SVE contiguous load instructions, scalar plus scalar form. 43 template <template <typename T1, typename T2> class Base> 44 StaticInstPtr 45 decodeSveContigLoadSSInsts(uint8_t dtype, ExtMachInst machInst, 46 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 47 IntRegIndex rm, bool firstFaulting) 48 { 49 const char* mn = firstFaulting ? "ldff1" : "ld1"; 50 switch (dtype) { 51 case 0x0: 52 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 53 case 0x1: 54 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 55 case 0x2: 56 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 57 case 0x3: 58 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 59 case 0x4: 60 return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, rm); 61 case 0x5: 62 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 63 case 0x6: 64 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 65 case 0x7: 66 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 67 case 0x8: 68 return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, rm); 69 case 0x9: 70 return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, rm); 71 case 0xa: 72 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm); 73 case 0xb: 74 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm); 75 case 0xc: 76 return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, rm); 77 case 0xd: 78 return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, rm); 79 case 0xe: 80 return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, rm); 81 case 0xf: 82 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm); 83 } 84 return new Unknown64(machInst); 85 } 86 87 // Decodes SVE contiguous load instructions, scalar plus immediate form. 88 template <template <typename T1, typename T2> class Base> 89 StaticInstPtr 90 decodeSveContigLoadSIInsts(uint8_t dtype, ExtMachInst machInst, 91 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 92 uint64_t imm, bool nonFaulting, 93 bool replicate = false) 94 { 95 assert(!(nonFaulting && replicate)); 96 const char* mn = replicate ? "ld1r" : (nonFaulting ? "ldnf1" : "ld1"); 97 switch (dtype) { 98 case 0x0: 99 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 100 case 0x1: 101 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 102 case 0x2: 103 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 104 case 0x3: 105 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 106 case 0x4: 107 return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, imm); 108 case 0x5: 109 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 110 case 0x6: 111 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 112 case 0x7: 113 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 114 case 0x8: 115 return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, imm); 116 case 0x9: 117 return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, imm); 118 case 0xa: 119 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm); 120 case 0xb: 121 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm); 122 case 0xc: 123 return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, imm); 124 case 0xd: 125 return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, imm); 126 case 0xe: 127 return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, imm); 128 case 0xf: 129 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm); 130 } 131 return new Unknown64(machInst); 132 } 133 134 // Decodes SVE contiguous store instructions, scalar plus scalar form. 135 template <template <typename T1, typename T2> class Base> 136 StaticInstPtr 137 decodeSveContigStoreSSInsts(uint8_t dtype, ExtMachInst machInst, 138 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 139 IntRegIndex rm) 140 { 141 const char* mn = "st1"; 142 switch (dtype) { 143 case 0x0: 144 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 145 case 0x1: 146 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 147 case 0x2: 148 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 149 case 0x3: 150 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 151 case 0x5: 152 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 153 case 0x6: 154 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 155 case 0x7: 156 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 157 case 0xa: 158 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm); 159 case 0xb: 160 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm); 161 case 0xf: 162 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm); 163 } 164 return new Unknown64(machInst); 165 } 166 167 // Decodes SVE contiguous store instructions, scalar plus immediate form. 168 template <template <typename T1, typename T2> class Base> 169 StaticInstPtr 170 decodeSveContigStoreSIInsts(uint8_t dtype, ExtMachInst machInst, 171 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 172 int8_t imm) 173 { 174 const char* mn = "st1"; 175 switch (dtype) { 176 case 0x0: 177 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 178 case 0x1: 179 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 180 case 0x2: 181 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 182 case 0x3: 183 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 184 case 0x5: 185 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 186 case 0x6: 187 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 188 case 0x7: 189 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 190 case 0xa: 191 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm); 192 case 0xb: 193 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm); 194 case 0xf: 195 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm); 196 } 197 return new Unknown64(machInst); 198 } 199 200 // NOTE: SVE load-and-replicate instructions are decoded with 201 // decodeSveContigLoadSIInsts(...). 202 203}}; 204 205output decoder {{ 206 207 template <class etype> 208 StaticInstPtr 209 decodeSveStructLoadSIInstsByNReg(uint8_t esize, ExtMachInst machInst, 210 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 211 int64_t imm, int numregs) 212 { 213 static const char* nm[5][4] = { 214 { nullptr, nullptr, nullptr, nullptr}, 215 { nullptr, nullptr, nullptr, nullptr}, 216 { "ld2b", "ld2h", "ld2w", "ld2d" }, 217 { "ld3b", "ld3h", "ld3w", "ld3d" }, 218 { "ld4b", "ld4h", "ld4w", "ld4d" } }; 219 220 switch (numregs) { 221 case 2: 222 return new SveLdStructSI<etype, 223 SveLoadRegImmMicroop, 224 SveDeIntrlv2Microop>( 225 nm[numregs][esize], machInst, MemReadOp, 226 zt, pg, xn, imm, numregs); 227 case 3: 228 return new SveLdStructSI<etype, 229 SveLoadRegImmMicroop, 230 SveDeIntrlv3Microop>( 231 nm[numregs][esize], machInst, MemReadOp, 232 zt, pg, xn, imm, numregs); 233 case 4: 234 return new SveLdStructSI<etype, 235 SveLoadRegImmMicroop, 236 SveDeIntrlv4Microop>( 237 nm[numregs][esize], machInst, MemReadOp, 238 zt, pg, xn, imm, numregs); 239 } 240 return new Unknown64(machInst); 241 } 242 243 StaticInstPtr 244 decodeSveStructLoadSIInsts(uint8_t esize, ExtMachInst machInst, 245 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 246 int64_t imm, int numregs) 247 { 248 switch (esize) { 249 case 0: 250 return decodeSveStructLoadSIInstsByNReg<uint8_t>(esize, 251 machInst, zt, pg, xn, imm, numregs); 252 case 1: 253 return decodeSveStructLoadSIInstsByNReg<uint16_t>(esize, 254 machInst, zt, pg, xn, imm, numregs); 255 case 2: 256 return decodeSveStructLoadSIInstsByNReg<uint32_t>(esize, 257 machInst, zt, pg, xn, imm, numregs); 258 case 3: 259 return decodeSveStructLoadSIInstsByNReg<uint64_t>(esize, 260 machInst, zt, pg, xn, imm, numregs); 261 } 262 return new Unknown64(machInst); 263 } 264 265 template <class etype> 266 StaticInstPtr 267 decodeSveStructStoreSIInstsByNReg(uint8_t esize, ExtMachInst machInst, 268 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 269 int64_t imm, int numregs) 270 { 271 static const char* nm[5][4] = { 272 { nullptr, nullptr, nullptr, nullptr}, 273 { nullptr, nullptr, nullptr, nullptr}, 274 { "st2b", "st2h", "st2w", "st2d" }, 275 { "st3b", "st3h", "st3w", "st3d" }, 276 { "st4b", "st4h", "st4w", "st4d" } }; 277 278 switch (numregs) { 279 case 2: 280 return new SveStStructSI<etype, 281 SveStoreRegImmMicroop, 282 SveIntrlv2Microop>( 283 nm[numregs][esize], machInst, MemWriteOp, 284 zt, pg, xn, imm, numregs); 285 case 3: 286 return new SveStStructSI<etype, 287 SveStoreRegImmMicroop, 288 SveIntrlv3Microop>( 289 nm[numregs][esize], machInst, MemWriteOp, 290 zt, pg, xn, imm, numregs); 291 case 4: 292 return new SveStStructSI<etype, 293 SveStoreRegImmMicroop, 294 SveIntrlv4Microop>( 295 nm[numregs][esize], machInst, MemWriteOp, 296 zt, pg, xn, imm, numregs); 297 } 298 return new Unknown64(machInst); 299 } 300 301 StaticInstPtr 302 decodeSveStructStoreSIInsts(uint8_t esize, ExtMachInst machInst, 303 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 304 int64_t imm, int numregs) 305 { 306 switch (esize) { 307 case 0: 308 return decodeSveStructStoreSIInstsByNReg<uint8_t>(esize, 309 machInst, zt, pg, xn, imm, numregs); 310 case 1: 311 return decodeSveStructStoreSIInstsByNReg<uint16_t>(esize, 312 machInst, zt, pg, xn, imm, numregs); 313 case 2: 314 return decodeSveStructStoreSIInstsByNReg<uint32_t>(esize, 315 machInst, zt, pg, xn, imm, numregs); 316 case 3: 317 return decodeSveStructStoreSIInstsByNReg<uint64_t>(esize, 318 machInst, zt, pg, xn, imm, numregs); 319 } 320 return new Unknown64(machInst); 321 } 322 323 template <class etype> 324 StaticInstPtr 325 decodeSveStructLoadSSInstsByNReg(uint8_t esize, ExtMachInst machInst, 326 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 327 IntRegIndex xm, int numregs) 328 { 329 static const char* nm[5][4] = { 330 { nullptr, nullptr, nullptr, nullptr}, 331 { nullptr, nullptr, nullptr, nullptr}, 332 { "ld2b", "ld2h", "ld2w", "ld2d" }, 333 { "ld3b", "ld3h", "ld3w", "ld3d" }, 334 { "ld4b", "ld4h", "ld4w", "ld4d" } }; 335 336 switch (numregs) { 337 case 2: 338 return new SveLdStructSS<etype, 339 SveLoadRegRegMicroop, 340 SveDeIntrlv2Microop>( 341 nm[numregs][esize], machInst, MemReadOp, 342 zt, pg, xn, xm, numregs); 343 case 3: 344 return new SveLdStructSS<etype, 345 SveLoadRegRegMicroop, 346 SveDeIntrlv3Microop>( 347 nm[numregs][esize], machInst, MemReadOp, 348 zt, pg, xn, xm, numregs); 349 case 4: 350 return new SveLdStructSS<etype, 351 SveLoadRegRegMicroop, 352 SveDeIntrlv4Microop>( 353 nm[numregs][esize], machInst, MemReadOp, 354 zt, pg, xn, xm, numregs); 355 } 356 return new Unknown64(machInst); 357 } 358 359 StaticInstPtr 360 decodeSveStructLoadSSInsts(uint8_t esize, ExtMachInst machInst, 361 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 362 IntRegIndex xm, int numregs) 363 { 364 switch (esize) { 365 case 0: 366 return decodeSveStructLoadSSInstsByNReg<uint8_t>(esize, 367 machInst, zt, pg, xn, xm, numregs); 368 case 1: 369 return decodeSveStructLoadSSInstsByNReg<uint16_t>(esize, 370 machInst, zt, pg, xn, xm, numregs); 371 case 2: 372 return decodeSveStructLoadSSInstsByNReg<uint32_t>(esize, 373 machInst, zt, pg, xn, xm, numregs); 374 case 3: 375 return decodeSveStructLoadSSInstsByNReg<uint64_t>(esize, 376 machInst, zt, pg, xn, xm, numregs); 377 } 378 return new Unknown64(machInst); 379 } 380 381 template <class etype> 382 StaticInstPtr 383 decodeSveStructStoreSSInstsByNReg(uint8_t esize, ExtMachInst machInst, 384 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 385 IntRegIndex xm, int numregs) 386 { 387 static const char* nm[5][4] = { 388 { nullptr, nullptr, nullptr, nullptr}, 389 { nullptr, nullptr, nullptr, nullptr}, 390 { "st2b", "st2h", "st2w", "st2d" }, 391 { "st3b", "st3h", "st3w", "st3d" }, 392 { "st4b", "st4h", "st4w", "st4d" } }; 393 394 switch (numregs) { 395 case 2: 396 return new SveStStructSS<etype, 397 SveStoreRegRegMicroop, 398 SveIntrlv2Microop>( 399 nm[numregs][esize], machInst, MemWriteOp, 400 zt, pg, xn, xm, numregs); 401 case 3: 402 return new SveStStructSS<etype, 403 SveStoreRegRegMicroop, 404 SveIntrlv3Microop>( 405 nm[numregs][esize], machInst, MemWriteOp, 406 zt, pg, xn, xm, numregs); 407 case 4: 408 return new SveStStructSS<etype, 409 SveStoreRegRegMicroop, 410 SveIntrlv4Microop>( 411 nm[numregs][esize], machInst, MemWriteOp, 412 zt, pg, xn, xm, numregs); 413 } 414 return new Unknown64(machInst); 415 } 416 417 StaticInstPtr 418 decodeSveStructStoreSSInsts(uint8_t esize, ExtMachInst machInst, 419 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 420 IntRegIndex xm, int numregs) 421 { 422 switch (esize) { 423 case 0: 424 return decodeSveStructStoreSSInstsByNReg<uint8_t>(esize, 425 machInst, zt, pg, xn, xm, numregs); 426 case 1: 427 return decodeSveStructStoreSSInstsByNReg<uint16_t>(esize, 428 machInst, zt, pg, xn, xm, numregs); 429 case 2: 430 return decodeSveStructStoreSSInstsByNReg<uint32_t>(esize, 431 machInst, zt, pg, xn, xm, numregs); 432 case 3: 433 return decodeSveStructStoreSSInstsByNReg<uint64_t>(esize, 434 machInst, zt, pg, xn, xm, numregs); 435 } 436 return new Unknown64(machInst); 437 } 438 439 StaticInstPtr 440 decodeSveGatherLoadVIInsts(uint8_t dtype, ExtMachInst machInst, 441 IntRegIndex zt, IntRegIndex pg, IntRegIndex zn, 442 uint64_t imm, bool esizeIs32, 443 bool firstFault) 444 { 445 const char* mn = firstFault ? "ldff1" : "ld1"; 446 switch (dtype) { 447 case 0x0: 448 if (esizeIs32) { 449 return new SveIndexedMemVI<int32_t, int8_t, 450 SveGatherLoadVIMicroop, 451 SveFirstFaultWritebackMicroop>( 452 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 453 } else { 454 return new SveIndexedMemVI<int64_t, int8_t, 455 SveGatherLoadVIMicroop, 456 SveFirstFaultWritebackMicroop>( 457 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 458 } 459 case 0x1: 460 if (esizeIs32) { 461 return new SveIndexedMemVI<uint32_t, uint8_t, 462 SveGatherLoadVIMicroop, 463 SveFirstFaultWritebackMicroop>( 464 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 465 } else { 466 return new SveIndexedMemVI<uint64_t, uint8_t, 467 SveGatherLoadVIMicroop, 468 SveFirstFaultWritebackMicroop>( 469 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 470 } 471 case 0x2: 472 if (esizeIs32) { 473 return new SveIndexedMemVI<int32_t, int16_t, 474 SveGatherLoadVIMicroop, 475 SveFirstFaultWritebackMicroop>( 476 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 477 } else { 478 return new SveIndexedMemVI<int64_t, int16_t, 479 SveGatherLoadVIMicroop, 480 SveFirstFaultWritebackMicroop>( 481 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 482 } 483 case 0x3: 484 if (esizeIs32) { 485 return new SveIndexedMemVI<uint32_t, uint16_t, 486 SveGatherLoadVIMicroop, 487 SveFirstFaultWritebackMicroop>( 488 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 489 } else { 490 return new SveIndexedMemVI<uint64_t, uint16_t, 491 SveGatherLoadVIMicroop, 492 SveFirstFaultWritebackMicroop>( 493 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 494 } 495 case 0x4: 496 if (esizeIs32) { 497 break; 498 } else { 499 return new SveIndexedMemVI<int64_t, int32_t, 500 SveGatherLoadVIMicroop, 501 SveFirstFaultWritebackMicroop>( 502 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 503 } 504 case 0x5: 505 if (esizeIs32) { 506 return new SveIndexedMemVI<uint32_t, uint32_t, 507 SveGatherLoadVIMicroop, 508 SveFirstFaultWritebackMicroop>( 509 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 510 } else { 511 return new SveIndexedMemVI<uint64_t, uint32_t, 512 SveGatherLoadVIMicroop, 513 SveFirstFaultWritebackMicroop>( 514 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 515 } 516 case 0x7: 517 if (esizeIs32) { 518 break; 519 } else { 520 return new SveIndexedMemVI<uint64_t, uint64_t, 521 SveGatherLoadVIMicroop, 522 SveFirstFaultWritebackMicroop>( 523 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 524 } 525 } 526 return new Unknown64(machInst); 527 } 528 529 StaticInstPtr 530 decodeSveGatherLoadSVInsts(uint8_t dtype, ExtMachInst machInst, 531 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 532 IntRegIndex zm, bool esizeIs32, bool offsetIs32, 533 bool offsetIsSigned, bool offsetIsScaled, 534 bool firstFault) 535 { 536 const char* mn = firstFault ? "ldff1" : "ld1"; 537 switch (dtype) { 538 case 0x0: 539 if (esizeIs32) { 540 return new SveIndexedMemSV<int32_t, int8_t, 541 SveGatherLoadSVMicroop, 542 SveFirstFaultWritebackMicroop>( 543 mn, machInst, MemReadOp, zt, pg, rn, zm, 544 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 545 } else { 546 return new SveIndexedMemSV<int64_t, int8_t, 547 SveGatherLoadSVMicroop, 548 SveFirstFaultWritebackMicroop>( 549 mn, machInst, MemReadOp, zt, pg, rn, zm, 550 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 551 } 552 case 0x1: 553 if (esizeIs32) { 554 return new SveIndexedMemSV<uint32_t, uint8_t, 555 SveGatherLoadSVMicroop, 556 SveFirstFaultWritebackMicroop>( 557 mn, machInst, MemReadOp, zt, pg, rn, zm, 558 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 559 } else { 560 return new SveIndexedMemSV<uint64_t, uint8_t, 561 SveGatherLoadSVMicroop, 562 SveFirstFaultWritebackMicroop>( 563 mn, machInst, MemReadOp, zt, pg, rn, zm, 564 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 565 } 566 case 0x2: 567 if (esizeIs32) { 568 return new SveIndexedMemSV<int32_t, int16_t, 569 SveGatherLoadSVMicroop, 570 SveFirstFaultWritebackMicroop>( 571 mn, machInst, MemReadOp, zt, pg, rn, zm, 572 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 573 } else { 574 return new SveIndexedMemSV<int64_t, int16_t, 575 SveGatherLoadSVMicroop, 576 SveFirstFaultWritebackMicroop>( 577 mn, machInst, MemReadOp, zt, pg, rn, zm, 578 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 579 } 580 case 0x3: 581 if (esizeIs32) { 582 return new SveIndexedMemSV<uint32_t, uint16_t, 583 SveGatherLoadSVMicroop, 584 SveFirstFaultWritebackMicroop>( 585 mn, machInst, MemReadOp, zt, pg, rn, zm, 586 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 587 } else { 588 return new SveIndexedMemSV<uint64_t, uint16_t, 589 SveGatherLoadSVMicroop, 590 SveFirstFaultWritebackMicroop>( 591 mn, machInst, MemReadOp, zt, pg, rn, zm, 592 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 593 } 594 case 0x4: 595 if (esizeIs32) { 596 break; 597 } else { 598 return new SveIndexedMemSV<int64_t, int32_t, 599 SveGatherLoadSVMicroop, 600 SveFirstFaultWritebackMicroop>( 601 mn, machInst, MemReadOp, zt, pg, rn, zm, 602 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 603 } 604 case 0x5: 605 if (esizeIs32) { 606 return new SveIndexedMemSV<uint32_t, uint32_t, 607 SveGatherLoadSVMicroop, 608 SveFirstFaultWritebackMicroop>( 609 mn, machInst, MemReadOp, zt, pg, rn, zm, 610 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 611 } else { 612 return new SveIndexedMemSV<uint64_t, uint32_t, 613 SveGatherLoadSVMicroop, 614 SveFirstFaultWritebackMicroop>( 615 mn, machInst, MemReadOp, zt, pg, rn, zm, 616 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 617 } 618 case 0x7: 619 if (esizeIs32) { 620 break; 621 } else { 622 return new SveIndexedMemSV<uint64_t, uint64_t, 623 SveGatherLoadSVMicroop, 624 SveFirstFaultWritebackMicroop>( 625 mn, machInst, MemReadOp, zt, pg, rn, zm, 626 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 627 } 628 } 629 return new Unknown64(machInst); 630 } 631 632 StaticInstPtr 633 decodeSveScatterStoreVIInsts(uint8_t msz, ExtMachInst machInst, 634 IntRegIndex zt, IntRegIndex pg, 635 IntRegIndex zn, uint64_t imm, 636 bool esizeIs32) 637 { 638 const char* mn = "st1"; 639 switch (msz) { 640 case 0x0: 641 if (esizeIs32) { 642 return new SveIndexedMemVI<uint32_t, uint8_t, 643 SveScatterStoreVIMicroop, 644 SveFirstFaultWritebackMicroop>( 645 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 646 } else { 647 return new SveIndexedMemVI<uint64_t, uint8_t, 648 SveScatterStoreVIMicroop, 649 SveFirstFaultWritebackMicroop>( 650 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 651 } 652 case 0x1: 653 if (esizeIs32) { 654 return new SveIndexedMemVI<uint32_t, uint16_t, 655 SveScatterStoreVIMicroop, 656 SveFirstFaultWritebackMicroop>( 657 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 658 } else { 659 return new SveIndexedMemVI<uint64_t, uint16_t, 660 SveScatterStoreVIMicroop, 661 SveFirstFaultWritebackMicroop>( 662 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 663 } 664 case 0x2: 665 if (esizeIs32) { 666 return new SveIndexedMemVI<uint32_t, uint32_t, 667 SveScatterStoreVIMicroop, 668 SveFirstFaultWritebackMicroop>( 669 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 670 } else { 671 return new SveIndexedMemVI<uint64_t, uint32_t, 672 SveScatterStoreVIMicroop, 673 SveFirstFaultWritebackMicroop>( 674 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 675 } 676 case 0x3: 677 if (esizeIs32) { 678 break; 679 } else { 680 return new SveIndexedMemVI<uint64_t, uint64_t, 681 SveScatterStoreVIMicroop, 682 SveFirstFaultWritebackMicroop>( 683 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 684 } 685 } 686 return new Unknown64(machInst); 687 } 688 689 StaticInstPtr 690 decodeSveScatterStoreSVInsts(uint8_t msz, ExtMachInst machInst, 691 IntRegIndex zt, IntRegIndex pg, 692 IntRegIndex rn, IntRegIndex zm, 693 bool esizeIs32, bool offsetIs32, 694 bool offsetIsSigned, bool offsetIsScaled) 695 { 696 const char* mn = "st1"; 697 switch (msz) { 698 case 0x0: 699 if (esizeIs32) { 700 return new SveIndexedMemSV<uint32_t, uint8_t, 701 SveScatterStoreSVMicroop, 702 SveFirstFaultWritebackMicroop>( 703 mn, machInst, MemWriteOp, zt, pg, rn, zm, 704 offsetIs32, offsetIsSigned, offsetIsScaled, false); 705 } else { 706 return new SveIndexedMemSV<uint64_t, uint8_t, 707 SveScatterStoreSVMicroop, 708 SveFirstFaultWritebackMicroop>( 709 mn, machInst, MemWriteOp, zt, pg, rn, zm, 710 offsetIs32, offsetIsSigned, offsetIsScaled, false); 711 } 712 case 0x1: 713 if (esizeIs32) { 714 return new SveIndexedMemSV<uint32_t, uint16_t, 715 SveScatterStoreSVMicroop, 716 SveFirstFaultWritebackMicroop>( 717 mn, machInst, MemWriteOp, zt, pg, rn, zm, 718 offsetIs32, offsetIsSigned, offsetIsScaled, false); 719 } else { 720 return new SveIndexedMemSV<uint64_t, uint16_t, 721 SveScatterStoreSVMicroop, 722 SveFirstFaultWritebackMicroop>( 723 mn, machInst, MemWriteOp, zt, pg, rn, zm, 724 offsetIs32, offsetIsSigned, offsetIsScaled, false); 725 } 726 case 0x2: 727 if (esizeIs32) { 728 return new SveIndexedMemSV<uint32_t, uint32_t, 729 SveScatterStoreSVMicroop, 730 SveFirstFaultWritebackMicroop>( 731 mn, machInst, MemWriteOp, zt, pg, rn, zm, 732 offsetIs32, offsetIsSigned, offsetIsScaled, false); 733 } else { 734 return new SveIndexedMemSV<uint64_t, uint32_t, 735 SveScatterStoreSVMicroop, 736 SveFirstFaultWritebackMicroop>( 737 mn, machInst, MemWriteOp, zt, pg, rn, zm, 738 offsetIs32, offsetIsSigned, offsetIsScaled, false); 739 } 740 case 0x3: 741 if (esizeIs32) { 742 break; 743 } else { 744 return new SveIndexedMemSV<uint64_t, uint64_t, 745 SveScatterStoreSVMicroop, 746 SveFirstFaultWritebackMicroop>( 747 mn, machInst, MemWriteOp, zt, pg, rn, zm, 748 offsetIs32, offsetIsSigned, offsetIsScaled, false); 749 } 750 } 751 return new Unknown64(machInst); 752 } 753 754}}; 755 756 757let {{ 758 759 header_output = '' 760 exec_output = '' 761 decoders = { 'Generic': {} } 762 763 SPAlignmentCheckCode = ''' 764 if (this->baseIsSP && bits(XBase, 3, 0) && 765 SPAlignmentCheckEnabled(xc->tcBase())) { 766 return std::make_shared<SPAlignmentFault>(); 767 } 768 ''' 769 770 def emitSveMemFillSpill(isPred): 771 global header_output, exec_output, decoders 772 eaCode = SPAlignmentCheckCode + ''' 773 int memAccessSize = %(memacc_size)s; 774 EA = XBase + ((int64_t) imm * %(memacc_size)s)''' % { 775 'memacc_size': 'eCount / 8' if isPred else 'eCount'} 776 loadRdEnableCode = ''' 777 auto rdEn = std::vector<bool>(); 778 ''' 779 if isPred: 780 loadMemAccCode = ''' 781 int index = 0; 782 uint8_t byte; 783 for (int i = 0; i < eCount / 8; i++) { 784 byte = memDataView[i]; 785 for (int j = 0; j < 8; j++, index++) { 786 PDest_x[index] = (byte >> j) & 1; 787 } 788 } 789 ''' 790 storeMemAccCode = ''' 791 int index = 0; 792 uint8_t byte; 793 for (int i = 0; i < eCount / 8; i++) { 794 byte = 0; 795 for (int j = 0; j < 8; j++, index++) { 796 byte |= PDest_x[index] << j; 797 } 798 memDataView[i] = byte; 799 } 800 ''' 801 storeWrEnableCode = ''' 802 auto wrEn = std::vector<bool>(eCount / 8, true); 803 ''' 804 else: 805 loadMemAccCode = ''' 806 for (int i = 0; i < eCount; i++) { 807 AA64FpDest_x[i] = memDataView[i]; 808 } 809 ''' 810 storeMemAccCode = ''' 811 for (int i = 0; i < eCount; i++) { 812 memDataView[i] = AA64FpDest_x[i]; 813 } 814 ''' 815 storeWrEnableCode = ''' 816 auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true); 817 ''' 818 loadIop = InstObjParams('ldr', 819 'SveLdrPred' if isPred else 'SveLdrVec', 820 'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill', 821 {'tpl_header': '', 822 'tpl_args': '', 823 'memacc_code': loadMemAccCode, 824 'ea_code' : sveEnabledCheckCode + eaCode, 825 'rden_code' : loadRdEnableCode, 826 'fault_code' : '', 827 'fa_code' : ''}, 828 ['IsMemRef', 'IsLoad']) 829 storeIop = InstObjParams('str', 830 'SveStrPred' if isPred else 'SveStrVec', 831 'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill', 832 {'tpl_header': '', 833 'tpl_args': '', 834 'wren_code': storeWrEnableCode, 835 'memacc_code': storeMemAccCode, 836 'ea_code' : sveEnabledCheckCode + eaCode, 837 'fa_code' : ''}, 838 ['IsMemRef', 'IsStore']) 839 header_output += SveMemFillSpillOpDeclare.subst(loadIop) 840 header_output += SveMemFillSpillOpDeclare.subst(storeIop) 841 exec_output += ( 842 SveContigLoadExecute.subst(loadIop) + 843 SveContigLoadInitiateAcc.subst(loadIop) + 844 SveContigLoadCompleteAcc.subst(loadIop) + 845 SveContigStoreExecute.subst(storeIop) + 846 SveContigStoreInitiateAcc.subst(storeIop) + 847 SveContigStoreCompleteAcc.subst(storeIop)) 848 849 loadTplArgs = ( 850 ('uint8_t', 'uint8_t'), 851 ('uint16_t', 'uint8_t'), 852 ('uint32_t', 'uint8_t'), 853 ('uint64_t', 'uint8_t'), 854 ('int64_t', 'int32_t'), 855 ('uint16_t', 'uint16_t'), 856 ('uint32_t', 'uint16_t'), 857 ('uint64_t', 'uint16_t'), 858 ('int64_t', 'int16_t'), 859 ('int32_t', 'int16_t'), 860 ('uint32_t', 'uint32_t'), 861 ('uint64_t', 'uint32_t'), 862 ('int64_t', 'int8_t'), 863 ('int32_t', 'int8_t'), 864 ('int16_t', 'int8_t'), 865 ('uint64_t', 'uint64_t'), 866 ) 867 868 storeTplArgs = ( 869 ('uint8_t', 'uint8_t'), 870 ('uint16_t', 'uint8_t'), 871 ('uint32_t', 'uint8_t'), 872 ('uint64_t', 'uint8_t'), 873 ('uint16_t', 'uint16_t'), 874 ('uint32_t', 'uint16_t'), 875 ('uint64_t', 'uint16_t'), 876 ('uint32_t', 'uint32_t'), 877 ('uint64_t', 'uint32_t'), 878 ('uint64_t', 'uint64_t'), 879 ) 880 881 gatherLoadTplArgs = ( 882 ('int32_t', 'int8_t'), 883 ('int64_t', 'int8_t'), 884 ('uint32_t', 'uint8_t'), 885 ('uint64_t', 'uint8_t'), 886 ('int32_t', 'int16_t'), 887 ('int64_t', 'int16_t'), 888 ('uint32_t', 'uint16_t'), 889 ('uint64_t', 'uint16_t'), 890 ('int64_t', 'int32_t'), 891 ('uint32_t', 'uint32_t'), 892 ('uint64_t', 'uint32_t'), 893 ('uint64_t', 'uint64_t'), 894 ) 895 896 scatterStoreTplArgs = ( 897 ('uint32_t', 'uint8_t'), 898 ('uint64_t', 'uint8_t'), 899 ('uint32_t', 'uint16_t'), 900 ('uint64_t', 'uint16_t'), 901 ('uint32_t', 'uint32_t'), 902 ('uint64_t', 'uint32_t'), 903 ('uint64_t', 'uint64_t'), 904 ) 905 906 # Generates definitions for SVE contiguous loads 907 def emitSveContigMemInsts(offsetIsImm): 908 global header_output, exec_output, decoders 909 # First-faulting instructions only have a scalar plus scalar form, 910 # while non-faulting instructions only a scalar plus immediate form, so 911 # `offsetIsImm` is used to determine which class of instructions is 912 # generated 913 firstFaulting = not offsetIsImm 914 tplHeader = 'template <class RegElemType, class MemElemType>' 915 tplArgs = '<RegElemType, MemElemType>' 916 eaCode = SPAlignmentCheckCode + ''' 917 int memAccessSize = eCount * sizeof(MemElemType); 918 EA = XBase + ''' 919 if offsetIsImm: 920 eaCode += '((int64_t) this->imm * eCount * sizeof(MemElemType))' 921 else: 922 eaCode += '(XOffset * sizeof(MemElemType));' 923 loadRdEnableCode = ''' 924 auto rdEn = std::vector<bool>(sizeof(MemElemType) * eCount, true); 925 for (int i = 0; i < eCount; i++) { 926 if (!GpOp_x[i]) { 927 for (int j = 0; j < sizeof(MemElemType); j++) { 928 rdEn[sizeof(MemElemType) * i + j] = false; 929 } 930 } 931 } 932 ''' 933 loadMemAccCode = ''' 934 for (int i = 0; i < eCount; i++) { 935 if (GpOp_x[i]) { 936 AA64FpDest_x[i] = memDataView[i]; 937 } else { 938 AA64FpDest_x[i] = 0; 939 } 940 } 941 ''' 942 storeMemAccCode = ''' 943 for (int i = 0; i < eCount; i++) { 944 if (GpOp_x[i]) { 945 memDataView[i] = AA64FpDest_x[i]; 946 } else { 947 memDataView[i] = 0; 948 for (int j = 0; j < sizeof(MemElemType); j++) { 949 wrEn[sizeof(MemElemType) * i + j] = false; 950 } 951 } 952 } 953 ''' 954 storeWrEnableCode = ''' 955 auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true); 956 ''' 957 ffrReadBackCode = ''' 958 auto& firstFaultReg = Ffr;''' 959 fautlingLoadmemAccCode = ''' 960 for (int i = 0; i < eCount; i++) { 961 if (GpOp_x[i] && firstFaultReg[i * sizeof(RegElemType)]) { 962 AA64FpDest_x[i] = memDataView[i]; 963 } else { 964 AA64FpDest_x[i] = 0; 965 } 966 } 967 ''' 968 nonFaultingCode = 'true ||' 969 faultCode = ''' 970 Addr fault_addr; 971 if (fault == NoFault || getFaultVAddr(fault, fault_addr)) { 972 unsigned fault_elem_index; 973 if (fault != NoFault) { 974 assert(fault_addr >= EA); 975 fault_elem_index = (fault_addr - EA) / sizeof(MemElemType); 976 } else { 977 fault_elem_index = eCount + 1; 978 } 979 int first_active_index; 980 for (first_active_index = 0; 981 first_active_index < eCount && !(GpOp_x[first_active_index]); 982 first_active_index++); 983 if (%s first_active_index < fault_elem_index) { 984 for (int i = 0; i < eCount; i++) { 985 for (int j = 0; j < sizeof(RegElemType); j++) { 986 if (i < fault_elem_index) { 987 Ffr_ub[i * sizeof(RegElemType) + j] = FfrAux_x[i]; 988 } else { 989 Ffr_ub[i * sizeof(RegElemType) + j] = 0; 990 } 991 } 992 } 993 fault = NoFault; 994 if (first_active_index >= fault_elem_index) { 995 // non-faulting load needs this 996 xc->setMemAccPredicate(false); 997 } 998 } 999 } 1000 ''' % ('' if firstFaulting else nonFaultingCode) 1001 1002 loadIop = InstObjParams('ld1', 1003 'SveContigLoadSI' if offsetIsImm else 'SveContigLoadSS', 1004 'SveContigMemSI' if offsetIsImm else 'SveContigMemSS', 1005 {'tpl_header': tplHeader, 1006 'tpl_args': tplArgs, 1007 'rden_code' : loadRdEnableCode, 1008 'memacc_code': loadMemAccCode, 1009 'ea_code' : sveEnabledCheckCode + eaCode, 1010 'fault_code' : '', 1011 'fa_code' : ''}, 1012 ['IsMemRef', 'IsLoad']) 1013 storeIop = InstObjParams('st1', 1014 'SveContigStoreSI' if offsetIsImm else 'SveContigStoreSS', 1015 'SveContigMemSI' if offsetIsImm else 'SveContigMemSS', 1016 {'tpl_header': tplHeader, 1017 'tpl_args': tplArgs, 1018 'wren_code': storeWrEnableCode, 1019 'memacc_code': storeMemAccCode, 1020 'ea_code' : sveEnabledCheckCode + eaCode, 1021 'fa_code' : ''}, 1022 ['IsMemRef', 'IsStore']) 1023 faultIop = InstObjParams('ldff1' if firstFaulting else 'ldnf1', 1024 'SveContigFFLoadSS' if firstFaulting else 'SveContigNFLoadSI', 1025 'SveContigMemSS' if firstFaulting else 'SveContigMemSI', 1026 {'tpl_header': tplHeader, 1027 'tpl_args': tplArgs, 1028 'rden_code' : loadRdEnableCode, 1029 'memacc_code': fautlingLoadmemAccCode, 1030 'ea_code' : sveEnabledCheckCode + eaCode, 1031 'fault_code' : faultCode, 1032 'fa_code' : ''}, 1033 ['IsMemRef', 'IsLoad']) 1034 faultIop.snippets['memacc_code'] = (ffrReadBackCode + 1035 faultIop.snippets['memacc_code']) 1036 if offsetIsImm: 1037 header_output += SveContigMemSIOpDeclare.subst(loadIop) 1038 header_output += SveContigMemSIOpDeclare.subst(storeIop) 1039 header_output += SveContigMemSIOpDeclare.subst(faultIop) 1040 else: 1041 header_output += SveContigMemSSOpDeclare.subst(loadIop) 1042 header_output += SveContigMemSSOpDeclare.subst(storeIop) 1043 header_output += SveContigMemSSOpDeclare.subst(faultIop) 1044 exec_output += ( 1045 SveContigLoadExecute.subst(loadIop) + 1046 SveContigLoadInitiateAcc.subst(loadIop) + 1047 SveContigLoadCompleteAcc.subst(loadIop) + 1048 SveContigStoreExecute.subst(storeIop) + 1049 SveContigStoreInitiateAcc.subst(storeIop) + 1050 SveContigStoreCompleteAcc.subst(storeIop) + 1051 SveContigLoadExecute.subst(faultIop) + 1052 SveContigLoadInitiateAcc.subst(faultIop) + 1053 SveContigLoadCompleteAcc.subst(faultIop)) 1054 1055 for args in loadTplArgs: 1056 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1057 'class_name': 'SveContigLoadSI' if offsetIsImm 1058 else 'SveContigLoadSS'} 1059 exec_output += SveContigMemExecDeclare.subst(substDict) 1060 for args in storeTplArgs: 1061 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1062 'class_name': 'SveContigStoreSI' if offsetIsImm 1063 else 'SveContigStoreSS'} 1064 exec_output += SveContigMemExecDeclare.subst(substDict) 1065 for args in loadTplArgs: 1066 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1067 'class_name': 'SveContigFFLoadSS' if firstFaulting 1068 else 'SveContigNFLoadSI'} 1069 exec_output += SveContigMemExecDeclare.subst(substDict) 1070 1071 1072 # Generates definitions for SVE load-and-replicate instructions 1073 def emitSveLoadAndRepl(): 1074 global header_output, exec_output, decoders 1075 tplHeader = 'template <class RegElemType, class MemElemType>' 1076 tplArgs = '<RegElemType, MemElemType>' 1077 eaCode = SPAlignmentCheckCode + ''' 1078 EA = XBase + imm * sizeof(MemElemType);''' 1079 memAccCode = ''' 1080 for (int i = 0; i < eCount; i++) { 1081 if (GpOp_x[i]) { 1082 AA64FpDest_x[i] = memData; 1083 } else { 1084 AA64FpDest_x[i] = 0; 1085 } 1086 } 1087 ''' 1088 iop = InstObjParams('ld1r', 1089 'SveLoadAndRepl', 1090 'SveContigMemSI', 1091 {'tpl_header': tplHeader, 1092 'tpl_args': tplArgs, 1093 'memacc_code': memAccCode, 1094 'ea_code' : sveEnabledCheckCode + eaCode, 1095 'fa_code' : ''}, 1096 ['IsMemRef', 'IsLoad']) 1097 header_output += SveContigMemSIOpDeclare.subst(iop) 1098 exec_output += ( 1099 SveLoadAndReplExecute.subst(iop) + 1100 SveLoadAndReplInitiateAcc.subst(iop) + 1101 SveLoadAndReplCompleteAcc.subst(iop)) 1102 for args in loadTplArgs: 1103 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1104 'class_name': 'SveLoadAndRepl'} 1105 exec_output += SveContigMemExecDeclare.subst(substDict) 1106 1107 class IndexedAddrForm: 1108 VEC_PLUS_IMM = 0 1109 SCA_PLUS_VEC = 1 1110 1111 # Generates definitions for the transfer microops of SVE indexed memory 1112 # operations (gather loads, scatter stores) 1113 def emitSveIndexedMemMicroops(indexed_addr_form): 1114 assert indexed_addr_form in (IndexedAddrForm.VEC_PLUS_IMM, 1115 IndexedAddrForm.SCA_PLUS_VEC) 1116 global header_output, exec_output, decoders 1117 tplHeader = 'template <class RegElemType, class MemElemType>' 1118 tplArgs = '<RegElemType, MemElemType>' 1119 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM: 1120 eaCode = ''' 1121 EA = AA64FpBase_x[elemIndex] + imm * sizeof(MemElemType)''' 1122 else: 1123 eaCode = ''' 1124 uint64_t offset = AA64FpOffset_x[elemIndex]; 1125 if (offsetIs32) { 1126 offset &= (1ULL << 32) - 1; 1127 } 1128 if (offsetIsSigned) { 1129 offset = sext<32>(offset); 1130 } 1131 if (offsetIsScaled) { 1132 offset *= sizeof(MemElemType); 1133 } 1134 EA = XBase + offset''' 1135 loadMemAccCode = ''' 1136 AA64FpDest_x[elemIndex] = memData; 1137 ''' 1138 storeMemAccCode = ''' 1139 memData = AA64FpDest_x[elemIndex]; 1140 ''' 1141 predCheckCode = 'GpOp_x[index]' 1142 faultStatusSetCode = 'PUreg0_x[elemIndex] = 1;' 1143 faultStatusResetCode = 'PUreg0_x[elemIndex] = 0;' 1144 loadIop = InstObjParams('ld1', 1145 ('SveGatherLoadVIMicroop' 1146 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM 1147 else 'SveGatherLoadSVMicroop'), 1148 'MicroOp', 1149 {'tpl_header': tplHeader, 1150 'tpl_args': tplArgs, 1151 'memacc_code': loadMemAccCode, 1152 'ea_code' : sveEnabledCheckCode + eaCode, 1153 'fault_status_set_code' : faultStatusSetCode, 1154 'fault_status_reset_code' : faultStatusResetCode, 1155 'pred_check_code' : predCheckCode, 1156 'fa_code' : ''}, 1157 ['IsMicroop', 'IsMemRef', 'IsLoad']) 1158 storeIop = InstObjParams('st1', 1159 ('SveScatterStoreVIMicroop' 1160 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM 1161 else 'SveScatterStoreSVMicroop'), 1162 'MicroOp', 1163 {'tpl_header': tplHeader, 1164 'tpl_args': tplArgs, 1165 'memacc_code': storeMemAccCode, 1166 'ea_code' : sveEnabledCheckCode + eaCode, 1167 'pred_check_code' : predCheckCode, 1168 'fa_code' : ''}, 1169 ['IsMicroop', 'IsMemRef', 'IsStore']) 1170 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM: 1171 header_output += SveIndexedMemVIMicroopDeclare.subst(loadIop) 1172 header_output += SveIndexedMemVIMicroopDeclare.subst(storeIop) 1173 else: 1174 header_output += SveIndexedMemSVMicroopDeclare.subst(loadIop) 1175 header_output += SveIndexedMemSVMicroopDeclare.subst(storeIop) 1176 exec_output += ( 1177 SveGatherLoadMicroopExecute.subst(loadIop) + 1178 SveGatherLoadMicroopInitiateAcc.subst(loadIop) + 1179 SveGatherLoadMicroopCompleteAcc.subst(loadIop) + 1180 SveScatterStoreMicroopExecute.subst(storeIop) + 1181 SveScatterStoreMicroopInitiateAcc.subst(storeIop) + 1182 SveScatterStoreMicroopCompleteAcc.subst(storeIop)) 1183 for args in gatherLoadTplArgs: 1184 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1185 'class_name': ( 1186 'SveGatherLoadVIMicroop' 1187 if indexed_addr_form == \ 1188 IndexedAddrForm.VEC_PLUS_IMM 1189 else 'SveGatherLoadSVMicroop')} 1190 # TODO: this should become SveMemExecDeclare 1191 exec_output += SveContigMemExecDeclare.subst(substDict) 1192 for args in scatterStoreTplArgs: 1193 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1194 'class_name': ( 1195 'SveScatterStoreVIMicroop' 1196 if indexed_addr_form == \ 1197 IndexedAddrForm.VEC_PLUS_IMM 1198 else 'SveScatterStoreSVMicroop')} 1199 # TODO: this should become SveMemExecDeclare 1200 exec_output += SveContigMemExecDeclare.subst(substDict) 1201 1202 firstFaultTplArgs = ('int32_t', 'int64_t', 'uint32_t', 'uint64_t') 1203 1204 def emitSveFirstFaultWritebackMicroop(): 1205 global header_output, exec_output, decoders 1206 tplHeader = 'template <class RegElemType>' 1207 tplArgs = '<RegElemType>' 1208 faultStatusCheckCode = 'PUreg0_x[index]' 1209 firstFaultResetCode = ''' 1210 for(int j = 0; j < sizeof(RegElemType); j++) { 1211 Ffr_ub[index * sizeof(RegElemType) + j] = 0; 1212 } 1213 ''' 1214 firstFaultForwardCode = ''' 1215 for(int j = 0; j < sizeof(RegElemType); j++) { 1216 Ffr_ub[index * sizeof(RegElemType) + j] = FfrAux_x[index]; 1217 } 1218 ''' 1219 iop = InstObjParams('ldff1', 1220 'SveFirstFaultWritebackMicroop', 1221 'MicroOp', 1222 {'tpl_header': tplHeader, 1223 'tpl_args': tplArgs, 1224 'fault_status_check_code' : faultStatusCheckCode, 1225 'first_fault_reset_code' : firstFaultResetCode, 1226 'first_fault_forward_code' : firstFaultForwardCode}, 1227 ['IsMicroop']) 1228 header_output += SveFirstFaultWritebackMicroopDeclare.subst(iop) 1229 exec_output += SveFirstFaultWritebackMicroopExecute.subst(iop) 1230 for args in firstFaultTplArgs: 1231 substDict = {'targs': args, 1232 'class_name' : 'SveFirstFaultWritebackMicroop' } 1233 exec_output += SveOpExecDeclare.subst(substDict) 1234 1235 # Generates definitions for the first microop of SVE gather loads, required 1236 # to propagate the source vector register to the transfer microops 1237 def emitSveGatherLoadCpySrcVecMicroop(): 1238 global header_output, exec_output, decoders 1239 code = sveEnabledCheckCode + ''' 1240 unsigned eCount = ArmStaticInst::getCurSveVecLen<uint8_t>( 1241 xc->tcBase()); 1242 for (unsigned i = 0; i < eCount; i++) { 1243 AA64FpUreg0_ub[i] = AA64FpOp1_ub[i]; 1244 }''' 1245 iop = InstObjParams('ld1', 1246 'SveGatherLoadCpySrcVecMicroop', 1247 'MicroOp', 1248 {'code': code}, 1249 ['IsMicroop']) 1250 header_output += SveGatherLoadCpySrcVecMicroopDeclare.subst(iop) 1251 exec_output += SveGatherLoadCpySrcVecMicroopExecute.subst(iop) 1252 1253 def emitSveInterleaveMicroop(): 1254 global header_output, exec_output, decoders 1255 code2 = sveEnabledCheckCode + ''' 1256 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>( 1257 xc->tcBase()); 1258 for (unsigned int i = 0; i < eCount; ++i) { 1259 unsigned int absIdx = regIndex * eCount + i; 1260 unsigned int srcIdx = absIdx / numRegs; 1261 unsigned int srcVec = absIdx % numRegs; 1262 if (srcVec == 0) 1263 AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx]; 1264 else if (srcVec == 1) 1265 AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx]; 1266 }''' 1267 1268 code3 = sveEnabledCheckCode + ''' 1269 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>( 1270 xc->tcBase()); 1271 for (unsigned int i = 0; i < eCount; ++i) { 1272 unsigned int absIdx = regIndex * eCount + i; 1273 unsigned int srcIdx = absIdx / numRegs; 1274 unsigned int srcVec = absIdx % numRegs; 1275 if (srcVec == 0) 1276 AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx]; 1277 else if (srcVec == 1) 1278 AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx]; 1279 else if (srcVec == 2) 1280 AA64FpDest_x[i] = AA64FpOp1V2S_x[srcIdx]; 1281 }''' 1282 1283 code4 = sveEnabledCheckCode + ''' 1284 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>( 1285 xc->tcBase()); 1286 for (unsigned int i = 0; i < eCount; ++i) { 1287 unsigned int absIdx = regIndex * eCount + i; 1288 unsigned int srcIdx = absIdx / numRegs; 1289 unsigned int srcVec = absIdx % numRegs; 1290 if (srcVec == 0) 1291 AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx]; 1292 else if (srcVec == 1) 1293 AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx]; 1294 else if (srcVec == 2) 1295 AA64FpDest_x[i] = AA64FpOp1V2S_x[srcIdx]; 1296 else if (srcVec == 3) 1297 AA64FpDest_x[i] = AA64FpOp1V3S_x[srcIdx]; 1298 }''' 1299 1300 iop2 = InstObjParams('intrlv', 1301 'SveIntrlv2Microop', 1302 'MicroOp', 1303 {'code': code2}, 1304 ['IsMicroop']) 1305 iop3 = InstObjParams('intrlv', 1306 'SveIntrlv3Microop', 1307 'MicroOp', 1308 {'code': code3}, 1309 ['IsMicroop']) 1310 iop4 = InstObjParams('intrlv', 1311 'SveIntrlv4Microop', 1312 'MicroOp', 1313 {'code': code4}, 1314 ['IsMicroop']) 1315 header_output += SveIntrlvMicroopDeclare.subst(iop2); 1316 header_output += SveIntrlvMicroopDeclare.subst(iop3); 1317 header_output += SveIntrlvMicroopDeclare.subst(iop4); 1318 exec_output += SveIntrlvMicroopExecute.subst(iop2); 1319 exec_output += SveIntrlvMicroopExecute.subst(iop3); 1320 exec_output += SveIntrlvMicroopExecute.subst(iop4); 1321 for type in ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'): 1322 for nreg in range(2,5): 1323 substDict = {'targs' : type, 1324 'class_name' : 'SveIntrlv' + str(nreg) + 'Microop'} 1325 exec_output += SveIntrlvMicroopExecDeclare.subst(substDict) 1326 1327 def emitSveDeInterleaveMicroop(): 1328 global header_output, exec_output, decoders 1329 code2 = sveEnabledCheckCode + ''' 1330 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>( 1331 xc->tcBase()); 1332 for (unsigned int i = 0; i < eCount; ++i) { 1333 unsigned int absIdx = (regIndex + numRegs * i); 1334 unsigned int srcIdx = absIdx % eCount; 1335 unsigned int srcVec = absIdx / eCount; 1336 if (srcVec == 0) 1337 AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx]; 1338 else if(srcVec == 1) 1339 AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx]; 1340 }''' 1341 1342 code3 = sveEnabledCheckCode + ''' 1343 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>( 1344 xc->tcBase()); 1345 for (unsigned int i = 0; i < eCount; ++i) { 1346 unsigned int absIdx = (regIndex + numRegs * i); 1347 unsigned int srcIdx = absIdx % eCount; 1348 unsigned int srcVec = absIdx / eCount; 1349 if (srcVec == 0) 1350 AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx]; 1351 else if(srcVec == 1) 1352 AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx]; 1353 else if(srcVec == 2) 1354 AA64FpDest_x[i] = AA64IntrlvReg2_x[srcIdx]; 1355 }''' 1356 1357 code4 = sveEnabledCheckCode + ''' 1358 unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>( 1359 xc->tcBase()); 1360 for (unsigned int i = 0; i < eCount; ++i) { 1361 unsigned int absIdx = (regIndex + numRegs * i); 1362 unsigned int srcIdx = absIdx % eCount; 1363 unsigned int srcVec = absIdx / eCount; 1364 if (srcVec == 0) 1365 AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx]; 1366 else if(srcVec == 1) 1367 AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx]; 1368 else if(srcVec == 2) 1369 AA64FpDest_x[i] = AA64IntrlvReg2_x[srcIdx]; 1370 else if(srcVec == 3) 1371 AA64FpDest_x[i] = AA64IntrlvReg3_x[srcIdx]; 1372 }''' 1373 1374 iop2 = InstObjParams('deintrlv', 1375 'SveDeIntrlv2Microop', 1376 'MicroOp', 1377 {'code': code2}, 1378 ['IsMicroop']) 1379 iop3 = InstObjParams('deintrlv', 1380 'SveDeIntrlv3Microop', 1381 'MicroOp', 1382 {'code': code3}, 1383 ['IsMicroop']) 1384 iop4 = InstObjParams('deintrlv', 1385 'SveDeIntrlv4Microop', 1386 'MicroOp', 1387 {'code': code4}, 1388 ['IsMicroop']) 1389 header_output += SveDeIntrlvMicroopDeclare.subst(iop2); 1390 header_output += SveDeIntrlvMicroopDeclare.subst(iop3); 1391 header_output += SveDeIntrlvMicroopDeclare.subst(iop4); 1392 exec_output += SveIntrlvMicroopExecute.subst(iop2); 1393 exec_output += SveIntrlvMicroopExecute.subst(iop3); 1394 exec_output += SveIntrlvMicroopExecute.subst(iop4); 1395 for type in ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'): 1396 for nreg in range(2,5): 1397 substDict = {'targs' : type, 1398 'class_name' : 'SveDeIntrlv' + str(nreg) + 'Microop'} 1399 exec_output += SveIntrlvMicroopExecDeclare.subst(substDict) 1400 1401 # Generates definitions for SVE struct load/store microops 1402 def emitSveStructMemInsts(offsetIsImm): 1403 global header_output, exec_output, decoders 1404 eaCode = SPAlignmentCheckCode + ''' 1405 int memAccessSize = eCount * sizeof(Element); 1406 EA = memAccessSize * regIndex + XBase + ''' 1407 if offsetIsImm: 1408 eaCode += '((int64_t) this->imm * eCount * sizeof(Element))' 1409 else: 1410 eaCode += '(XOffset * sizeof(Element));' 1411 loadMemAccCode = ''' 1412 for (int i = 0; i < eCount; i++) { 1413 int gpIdx = (regIndex * eCount + i) / numRegs; 1414 if (GpOp_x[gpIdx]) { 1415 AA64FpDest_x[i] = memDataView[i]; 1416 } else { 1417 AA64FpDest_x[i] = 0; 1418 } 1419 } 1420 ''' 1421 storeMemAccCode = ''' 1422 for (int i = 0; i < eCount; i++) { 1423 int gpIdx = (regIndex * eCount + i) / numRegs; 1424 if (GpOp_x[gpIdx]) { 1425 memDataView[i] = AA64FpDest_x[i]; 1426 } else { 1427 memDataView[i] = 0; 1428 for (int j = 0; j < sizeof(Element); j++) { 1429 wrEn[sizeof(Element) * i + j] = false; 1430 } 1431 } 1432 } 1433 ''' 1434 storeWrEnableCode = ''' 1435 auto wrEn = std::vector<bool>(sizeof(Element) * eCount, true); 1436 ''' 1437 loadIop = InstObjParams('ldxx', 1438 'SveLoadRegImmMicroop' if offsetIsImm else 'SveLoadRegRegMicroop', 1439 'MicroOp', 1440 {'targs': 'Element', 1441 'memacc_code': loadMemAccCode, 1442 'ea_code' : sveEnabledCheckCode + eaCode, 1443 'fa_code' : ''}, 1444 ['IsMemRef', 'IsLoad', 'IsMicroop']) 1445 storeIop = InstObjParams('stxx', 1446 'SveStoreRegImmMicroop' if offsetIsImm 1447 else 'SveStoreRegRegMicroop', 1448 'MicroOp', 1449 {'targs': 'Element', 1450 'wren_code': storeWrEnableCode, 1451 'memacc_code': storeMemAccCode, 1452 'ea_code' : sveEnabledCheckCode + eaCode, 1453 'fa_code' : ''}, 1454 ['IsMemRef', 'IsStore', 'IsMicroop']) 1455 if offsetIsImm: 1456 header_output += SveStructMemSIMicroopDeclare.subst(loadIop) 1457 header_output += SveStructMemSIMicroopDeclare.subst(storeIop) 1458 else: 1459 header_output += SveStructMemSSMicroopDeclare.subst(loadIop) 1460 header_output += SveStructMemSSMicroopDeclare.subst(storeIop) 1461 exec_output += ( 1462 SveStructLoadExecute.subst(loadIop) + 1463 SveStructLoadInitiateAcc.subst(loadIop) + 1464 SveStructLoadCompleteAcc.subst(loadIop) + 1465 SveStructStoreExecute.subst(storeIop) + 1466 SveStructStoreInitiateAcc.subst(storeIop) + 1467 SveStructStoreCompleteAcc.subst(storeIop)) 1468 tplArgs = ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t') 1469 for type in tplArgs: 1470 substDict = {'targs': type, 1471 'class_name': 'SveLoadRegImmMicroop' if offsetIsImm 1472 else 'SveLoadRegRegMicroop'} 1473 exec_output += SveStructMemExecDeclare.subst(substDict) 1474 substDict['class_name'] = ('SveStoreRegImmMicroop' if offsetIsImm 1475 else 'SveStoreRegRegMicroop') 1476 exec_output += SveStructMemExecDeclare.subst(substDict) 1477
|