2// All rights reserved 3// 4// The license below extends only to copyright in the software and shall 5// not be construed as granting a license to any other intellectual 6// property including but not limited to intellectual property relating 7// to a hardware implementation of the functionality of the software 8// licensed hereunder. You may use the software subject to the license 9// terms below provided that you ensure that this notice is replicated 10// unmodified and in its entirety in all distributions of the software, 11// modified or unmodified, in source code or in binary form. 12// 13// Redistribution and use in source and binary forms, with or without 14// modification, are permitted provided that the following conditions are 15// met: redistributions of source code must retain the above copyright 16// notice, this list of conditions and the following disclaimer; 17// redistributions in binary form must reproduce the above copyright 18// notice, this list of conditions and the following disclaimer in the 19// documentation and/or other materials provided with the distribution; 20// neither the name of the copyright holders nor the names of its 21// contributors may be used to endorse or promote products derived from 22// this software without specific prior written permission. 23// 24// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 27// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 28// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 30// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 34// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35// 36// Authors: Giacomo Gabrielli 37 38// @file Definition of SVE memory access instructions. 39 40output header {{ 41 42 // Decodes SVE contiguous load instructions, scalar plus scalar form. 43 template <template <typename T1, typename T2> class Base> 44 StaticInstPtr 45 decodeSveContigLoadSSInsts(uint8_t dtype, ExtMachInst machInst, 46 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 47 IntRegIndex rm, bool firstFaulting) 48 { 49 const char* mn = firstFaulting ? "ldff1" : "ld1"; 50 switch (dtype) { 51 case 0x0: 52 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 53 case 0x1: 54 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 55 case 0x2: 56 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 57 case 0x3: 58 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 59 case 0x4: 60 return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, rm); 61 case 0x5: 62 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 63 case 0x6: 64 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 65 case 0x7: 66 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 67 case 0x8: 68 return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, rm); 69 case 0x9: 70 return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, rm); 71 case 0xa: 72 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm); 73 case 0xb: 74 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm); 75 case 0xc: 76 return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, rm); 77 case 0xd: 78 return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, rm); 79 case 0xe: 80 return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, rm); 81 case 0xf: 82 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm); 83 } 84 return new Unknown64(machInst); 85 } 86 87 // Decodes SVE contiguous load instructions, scalar plus immediate form. 88 template <template <typename T1, typename T2> class Base> 89 StaticInstPtr 90 decodeSveContigLoadSIInsts(uint8_t dtype, ExtMachInst machInst, 91 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 92 uint64_t imm, bool nonFaulting, 93 bool replicate = false) 94 { 95 assert(!(nonFaulting && replicate)); 96 const char* mn = replicate ? "ld1r" : (nonFaulting ? "ldnf1" : "ld1"); 97 switch (dtype) { 98 case 0x0: 99 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 100 case 0x1: 101 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 102 case 0x2: 103 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 104 case 0x3: 105 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 106 case 0x4: 107 return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, imm); 108 case 0x5: 109 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 110 case 0x6: 111 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 112 case 0x7: 113 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 114 case 0x8: 115 return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, imm); 116 case 0x9: 117 return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, imm); 118 case 0xa: 119 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm); 120 case 0xb: 121 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm); 122 case 0xc: 123 return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, imm); 124 case 0xd: 125 return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, imm); 126 case 0xe: 127 return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, imm); 128 case 0xf: 129 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm); 130 } 131 return new Unknown64(machInst); 132 } 133 134 // Decodes SVE contiguous store instructions, scalar plus scalar form. 135 template <template <typename T1, typename T2> class Base> 136 StaticInstPtr 137 decodeSveContigStoreSSInsts(uint8_t dtype, ExtMachInst machInst, 138 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 139 IntRegIndex rm) 140 { 141 const char* mn = "st1"; 142 switch (dtype) { 143 case 0x0: 144 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 145 case 0x1: 146 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 147 case 0x2: 148 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 149 case 0x3: 150 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 151 case 0x5: 152 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 153 case 0x6: 154 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 155 case 0x7: 156 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 157 case 0xa: 158 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm); 159 case 0xb: 160 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm); 161 case 0xf: 162 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm); 163 } 164 return new Unknown64(machInst); 165 } 166 167 // Decodes SVE contiguous store instructions, scalar plus immediate form. 168 template <template <typename T1, typename T2> class Base> 169 StaticInstPtr 170 decodeSveContigStoreSIInsts(uint8_t dtype, ExtMachInst machInst, 171 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 172 int8_t imm) 173 { 174 const char* mn = "st1"; 175 switch (dtype) { 176 case 0x0: 177 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 178 case 0x1: 179 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 180 case 0x2: 181 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 182 case 0x3: 183 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 184 case 0x5: 185 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 186 case 0x6: 187 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 188 case 0x7: 189 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 190 case 0xa: 191 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm); 192 case 0xb: 193 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm); 194 case 0xf: 195 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm); 196 } 197 return new Unknown64(machInst); 198 } 199 200 // NOTE: SVE load-and-replicate instructions are decoded with 201 // decodeSveContigLoadSIInsts(...). 202 203}}; 204 205output decoder {{ 206 207 template <class etype> 208 StaticInstPtr 209 decodeSveStructLoadSIInstsByNReg(uint8_t esize, ExtMachInst machInst, 210 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 211 int64_t imm, int numregs) 212 { 213 static const char* nm[5][4] = { 214 { nullptr, nullptr, nullptr, nullptr}, 215 { nullptr, nullptr, nullptr, nullptr}, 216 { "ld2b", "ld2h", "ld2w", "ld2d" }, 217 { "ld3b", "ld3h", "ld3w", "ld3d" }, 218 { "ld4b", "ld4h", "ld4w", "ld4d" } }; 219 220 switch (numregs) { 221 case 2: 222 return new SveLdStructSI<etype, 223 SveLoadRegImmMicroop, 224 SveDeIntrlv2Microop>( 225 nm[numregs][esize], machInst, MemReadOp, 226 zt, pg, xn, imm, numregs); 227 case 3: 228 return new SveLdStructSI<etype, 229 SveLoadRegImmMicroop, 230 SveDeIntrlv3Microop>( 231 nm[numregs][esize], machInst, MemReadOp, 232 zt, pg, xn, imm, numregs); 233 case 4: 234 return new SveLdStructSI<etype, 235 SveLoadRegImmMicroop, 236 SveDeIntrlv4Microop>( 237 nm[numregs][esize], machInst, MemReadOp, 238 zt, pg, xn, imm, numregs); 239 } 240 return new Unknown64(machInst); 241 } 242 243 StaticInstPtr 244 decodeSveStructLoadSIInsts(uint8_t esize, ExtMachInst machInst, 245 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 246 int64_t imm, int numregs) 247 { 248 switch (esize) { 249 case 0: 250 return decodeSveStructLoadSIInstsByNReg<uint8_t>(esize, 251 machInst, zt, pg, xn, imm, numregs); 252 case 1: 253 return decodeSveStructLoadSIInstsByNReg<uint16_t>(esize, 254 machInst, zt, pg, xn, imm, numregs); 255 case 2: 256 return decodeSveStructLoadSIInstsByNReg<uint32_t>(esize, 257 machInst, zt, pg, xn, imm, numregs); 258 case 3: 259 return decodeSveStructLoadSIInstsByNReg<uint64_t>(esize, 260 machInst, zt, pg, xn, imm, numregs); 261 } 262 return new Unknown64(machInst); 263 } 264 265 template <class etype> 266 StaticInstPtr 267 decodeSveStructStoreSIInstsByNReg(uint8_t esize, ExtMachInst machInst, 268 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 269 int64_t imm, int numregs) 270 { 271 static const char* nm[5][4] = { 272 { nullptr, nullptr, nullptr, nullptr}, 273 { nullptr, nullptr, nullptr, nullptr}, 274 { "st2b", "st2h", "st2w", "st2d" }, 275 { "st3b", "st3h", "st3w", "st3d" }, 276 { "st4b", "st4h", "st4w", "st4d" } }; 277 278 switch (numregs) { 279 case 2: 280 return new SveStStructSI<etype, 281 SveStoreRegImmMicroop, 282 SveIntrlv2Microop>( 283 nm[numregs][esize], machInst, MemWriteOp, 284 zt, pg, xn, imm, numregs); 285 case 3: 286 return new SveStStructSI<etype, 287 SveStoreRegImmMicroop, 288 SveIntrlv3Microop>( 289 nm[numregs][esize], machInst, MemWriteOp, 290 zt, pg, xn, imm, numregs); 291 case 4: 292 return new SveStStructSI<etype, 293 SveStoreRegImmMicroop, 294 SveIntrlv4Microop>( 295 nm[numregs][esize], machInst, MemWriteOp, 296 zt, pg, xn, imm, numregs); 297 } 298 return new Unknown64(machInst); 299 } 300 301 StaticInstPtr 302 decodeSveStructStoreSIInsts(uint8_t esize, ExtMachInst machInst, 303 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 304 int64_t imm, int numregs) 305 { 306 switch (esize) { 307 case 0: 308 return decodeSveStructStoreSIInstsByNReg<uint8_t>(esize, 309 machInst, zt, pg, xn, imm, numregs); 310 case 1: 311 return decodeSveStructStoreSIInstsByNReg<uint16_t>(esize, 312 machInst, zt, pg, xn, imm, numregs); 313 case 2: 314 return decodeSveStructStoreSIInstsByNReg<uint32_t>(esize, 315 machInst, zt, pg, xn, imm, numregs); 316 case 3: 317 return decodeSveStructStoreSIInstsByNReg<uint64_t>(esize, 318 machInst, zt, pg, xn, imm, numregs); 319 } 320 return new Unknown64(machInst); 321 } 322 323 template <class etype> 324 StaticInstPtr 325 decodeSveStructLoadSSInstsByNReg(uint8_t esize, ExtMachInst machInst, 326 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 327 IntRegIndex xm, int numregs) 328 { 329 static const char* nm[5][4] = { 330 { nullptr, nullptr, nullptr, nullptr}, 331 { nullptr, nullptr, nullptr, nullptr}, 332 { "ld2b", "ld2h", "ld2w", "ld2d" }, 333 { "ld3b", "ld3h", "ld3w", "ld3d" }, 334 { "ld4b", "ld4h", "ld4w", "ld4d" } }; 335 336 switch (numregs) { 337 case 2: 338 return new SveLdStructSS<etype, 339 SveLoadRegRegMicroop, 340 SveDeIntrlv2Microop>( 341 nm[numregs][esize], machInst, MemReadOp, 342 zt, pg, xn, xm, numregs); 343 case 3: 344 return new SveLdStructSS<etype, 345 SveLoadRegRegMicroop, 346 SveDeIntrlv3Microop>( 347 nm[numregs][esize], machInst, MemReadOp, 348 zt, pg, xn, xm, numregs); 349 case 4: 350 return new SveLdStructSS<etype, 351 SveLoadRegRegMicroop, 352 SveDeIntrlv4Microop>( 353 nm[numregs][esize], machInst, MemReadOp, 354 zt, pg, xn, xm, numregs); 355 } 356 return new Unknown64(machInst); 357 } 358 359 StaticInstPtr 360 decodeSveStructLoadSSInsts(uint8_t esize, ExtMachInst machInst, 361 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 362 IntRegIndex xm, int numregs) 363 { 364 switch (esize) { 365 case 0: 366 return decodeSveStructLoadSSInstsByNReg<uint8_t>(esize, 367 machInst, zt, pg, xn, xm, numregs); 368 case 1: 369 return decodeSveStructLoadSSInstsByNReg<uint16_t>(esize, 370 machInst, zt, pg, xn, xm, numregs); 371 case 2: 372 return decodeSveStructLoadSSInstsByNReg<uint32_t>(esize, 373 machInst, zt, pg, xn, xm, numregs); 374 case 3: 375 return decodeSveStructLoadSSInstsByNReg<uint64_t>(esize, 376 machInst, zt, pg, xn, xm, numregs); 377 } 378 return new Unknown64(machInst); 379 } 380 381 template <class etype> 382 StaticInstPtr 383 decodeSveStructStoreSSInstsByNReg(uint8_t esize, ExtMachInst machInst, 384 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 385 IntRegIndex xm, int numregs) 386 { 387 static const char* nm[5][4] = { 388 { nullptr, nullptr, nullptr, nullptr}, 389 { nullptr, nullptr, nullptr, nullptr}, 390 { "st2b", "st2h", "st2w", "st2d" }, 391 { "st3b", "st3h", "st3w", "st3d" }, 392 { "st4b", "st4h", "st4w", "st4d" } }; 393 394 switch (numregs) { 395 case 2: 396 return new SveStStructSS<etype, 397 SveStoreRegRegMicroop, 398 SveIntrlv2Microop>( 399 nm[numregs][esize], machInst, MemWriteOp, 400 zt, pg, xn, xm, numregs); 401 case 3: 402 return new SveStStructSS<etype, 403 SveStoreRegRegMicroop, 404 SveIntrlv3Microop>( 405 nm[numregs][esize], machInst, MemWriteOp, 406 zt, pg, xn, xm, numregs); 407 case 4: 408 return new SveStStructSS<etype, 409 SveStoreRegRegMicroop, 410 SveIntrlv4Microop>( 411 nm[numregs][esize], machInst, MemWriteOp, 412 zt, pg, xn, xm, numregs); 413 } 414 return new Unknown64(machInst); 415 } 416 417 StaticInstPtr 418 decodeSveStructStoreSSInsts(uint8_t esize, ExtMachInst machInst, 419 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 420 IntRegIndex xm, int numregs) 421 { 422 switch (esize) { 423 case 0: 424 return decodeSveStructStoreSSInstsByNReg<uint8_t>(esize, 425 machInst, zt, pg, xn, xm, numregs); 426 case 1: 427 return decodeSveStructStoreSSInstsByNReg<uint16_t>(esize, 428 machInst, zt, pg, xn, xm, numregs); 429 case 2: 430 return decodeSveStructStoreSSInstsByNReg<uint32_t>(esize, 431 machInst, zt, pg, xn, xm, numregs); 432 case 3: 433 return decodeSveStructStoreSSInstsByNReg<uint64_t>(esize, 434 machInst, zt, pg, xn, xm, numregs); 435 } 436 return new Unknown64(machInst); 437 } 438 439 StaticInstPtr 440 decodeSveGatherLoadVIInsts(uint8_t dtype, ExtMachInst machInst, 441 IntRegIndex zt, IntRegIndex pg, IntRegIndex zn, 442 uint64_t imm, bool esizeIs32, 443 bool firstFault) 444 { 445 const char* mn = firstFault ? "ldff1" : "ld1"; 446 switch (dtype) { 447 case 0x0: 448 if (esizeIs32) { 449 return new SveIndexedMemVI<int32_t, int8_t, 450 SveGatherLoadVIMicroop, 451 SveFirstFaultWritebackMicroop>( 452 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 453 } else { 454 return new SveIndexedMemVI<int64_t, int8_t, 455 SveGatherLoadVIMicroop, 456 SveFirstFaultWritebackMicroop>( 457 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 458 } 459 case 0x1: 460 if (esizeIs32) { 461 return new SveIndexedMemVI<uint32_t, uint8_t, 462 SveGatherLoadVIMicroop, 463 SveFirstFaultWritebackMicroop>( 464 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 465 } else { 466 return new SveIndexedMemVI<uint64_t, uint8_t, 467 SveGatherLoadVIMicroop, 468 SveFirstFaultWritebackMicroop>( 469 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 470 } 471 case 0x2: 472 if (esizeIs32) { 473 return new SveIndexedMemVI<int32_t, int16_t, 474 SveGatherLoadVIMicroop, 475 SveFirstFaultWritebackMicroop>( 476 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 477 } else { 478 return new SveIndexedMemVI<int64_t, int16_t, 479 SveGatherLoadVIMicroop, 480 SveFirstFaultWritebackMicroop>( 481 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 482 } 483 case 0x3: 484 if (esizeIs32) { 485 return new SveIndexedMemVI<uint32_t, uint16_t, 486 SveGatherLoadVIMicroop, 487 SveFirstFaultWritebackMicroop>( 488 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 489 } else { 490 return new SveIndexedMemVI<uint64_t, uint16_t, 491 SveGatherLoadVIMicroop, 492 SveFirstFaultWritebackMicroop>( 493 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 494 } 495 case 0x4: 496 if (esizeIs32) { 497 break; 498 } else { 499 return new SveIndexedMemVI<int64_t, int32_t, 500 SveGatherLoadVIMicroop, 501 SveFirstFaultWritebackMicroop>( 502 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 503 } 504 case 0x5: 505 if (esizeIs32) { 506 return new SveIndexedMemVI<uint32_t, uint32_t, 507 SveGatherLoadVIMicroop, 508 SveFirstFaultWritebackMicroop>( 509 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 510 } else { 511 return new SveIndexedMemVI<uint64_t, uint32_t, 512 SveGatherLoadVIMicroop, 513 SveFirstFaultWritebackMicroop>( 514 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 515 } 516 case 0x7: 517 if (esizeIs32) { 518 break; 519 } else { 520 return new SveIndexedMemVI<uint64_t, uint64_t, 521 SveGatherLoadVIMicroop, 522 SveFirstFaultWritebackMicroop>( 523 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 524 } 525 } 526 return new Unknown64(machInst); 527 } 528 529 StaticInstPtr 530 decodeSveGatherLoadSVInsts(uint8_t dtype, ExtMachInst machInst, 531 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 532 IntRegIndex zm, bool esizeIs32, bool offsetIs32, 533 bool offsetIsSigned, bool offsetIsScaled, 534 bool firstFault) 535 { 536 const char* mn = firstFault ? "ldff1" : "ld1"; 537 switch (dtype) { 538 case 0x0: 539 if (esizeIs32) { 540 return new SveIndexedMemSV<int32_t, int8_t, 541 SveGatherLoadSVMicroop, 542 SveFirstFaultWritebackMicroop>( 543 mn, machInst, MemReadOp, zt, pg, rn, zm, 544 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 545 } else { 546 return new SveIndexedMemSV<int64_t, int8_t, 547 SveGatherLoadSVMicroop, 548 SveFirstFaultWritebackMicroop>( 549 mn, machInst, MemReadOp, zt, pg, rn, zm, 550 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 551 } 552 case 0x1: 553 if (esizeIs32) { 554 return new SveIndexedMemSV<uint32_t, uint8_t, 555 SveGatherLoadSVMicroop, 556 SveFirstFaultWritebackMicroop>( 557 mn, machInst, MemReadOp, zt, pg, rn, zm, 558 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 559 } else { 560 return new SveIndexedMemSV<uint64_t, uint8_t, 561 SveGatherLoadSVMicroop, 562 SveFirstFaultWritebackMicroop>( 563 mn, machInst, MemReadOp, zt, pg, rn, zm, 564 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 565 } 566 case 0x2: 567 if (esizeIs32) { 568 return new SveIndexedMemSV<int32_t, int16_t, 569 SveGatherLoadSVMicroop, 570 SveFirstFaultWritebackMicroop>( 571 mn, machInst, MemReadOp, zt, pg, rn, zm, 572 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 573 } else { 574 return new SveIndexedMemSV<int64_t, int16_t, 575 SveGatherLoadSVMicroop, 576 SveFirstFaultWritebackMicroop>( 577 mn, machInst, MemReadOp, zt, pg, rn, zm, 578 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 579 } 580 case 0x3: 581 if (esizeIs32) { 582 return new SveIndexedMemSV<uint32_t, uint16_t, 583 SveGatherLoadSVMicroop, 584 SveFirstFaultWritebackMicroop>( 585 mn, machInst, MemReadOp, zt, pg, rn, zm, 586 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 587 } else { 588 return new SveIndexedMemSV<uint64_t, uint16_t, 589 SveGatherLoadSVMicroop, 590 SveFirstFaultWritebackMicroop>( 591 mn, machInst, MemReadOp, zt, pg, rn, zm, 592 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 593 } 594 case 0x4: 595 if (esizeIs32) { 596 break; 597 } else { 598 return new SveIndexedMemSV<int64_t, int32_t, 599 SveGatherLoadSVMicroop, 600 SveFirstFaultWritebackMicroop>( 601 mn, machInst, MemReadOp, zt, pg, rn, zm, 602 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 603 } 604 case 0x5: 605 if (esizeIs32) { 606 return new SveIndexedMemSV<uint32_t, uint32_t, 607 SveGatherLoadSVMicroop, 608 SveFirstFaultWritebackMicroop>( 609 mn, machInst, MemReadOp, zt, pg, rn, zm, 610 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 611 } else { 612 return new SveIndexedMemSV<uint64_t, uint32_t, 613 SveGatherLoadSVMicroop, 614 SveFirstFaultWritebackMicroop>( 615 mn, machInst, MemReadOp, zt, pg, rn, zm, 616 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 617 } 618 case 0x7: 619 if (esizeIs32) { 620 break; 621 } else { 622 return new SveIndexedMemSV<uint64_t, uint64_t, 623 SveGatherLoadSVMicroop, 624 SveFirstFaultWritebackMicroop>( 625 mn, machInst, MemReadOp, zt, pg, rn, zm, 626 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 627 } 628 } 629 return new Unknown64(machInst); 630 } 631 632 StaticInstPtr 633 decodeSveScatterStoreVIInsts(uint8_t msz, ExtMachInst machInst, 634 IntRegIndex zt, IntRegIndex pg, 635 IntRegIndex zn, uint64_t imm, 636 bool esizeIs32) 637 { 638 const char* mn = "st1"; 639 switch (msz) { 640 case 0x0: 641 if (esizeIs32) { 642 return new SveIndexedMemVI<uint32_t, uint8_t, 643 SveScatterStoreVIMicroop, 644 SveFirstFaultWritebackMicroop>( 645 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 646 } else { 647 return new SveIndexedMemVI<uint64_t, uint8_t, 648 SveScatterStoreVIMicroop, 649 SveFirstFaultWritebackMicroop>( 650 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 651 } 652 case 0x1: 653 if (esizeIs32) { 654 return new SveIndexedMemVI<uint32_t, uint16_t, 655 SveScatterStoreVIMicroop, 656 SveFirstFaultWritebackMicroop>( 657 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 658 } else { 659 return new SveIndexedMemVI<uint64_t, uint16_t, 660 SveScatterStoreVIMicroop, 661 SveFirstFaultWritebackMicroop>( 662 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 663 } 664 case 0x2: 665 if (esizeIs32) { 666 return new SveIndexedMemVI<uint32_t, uint32_t, 667 SveScatterStoreVIMicroop, 668 SveFirstFaultWritebackMicroop>( 669 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 670 } else { 671 return new SveIndexedMemVI<uint64_t, uint32_t, 672 SveScatterStoreVIMicroop, 673 SveFirstFaultWritebackMicroop>( 674 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 675 } 676 case 0x3: 677 if (esizeIs32) { 678 break; 679 } else { 680 return new SveIndexedMemVI<uint64_t, uint64_t, 681 SveScatterStoreVIMicroop, 682 SveFirstFaultWritebackMicroop>( 683 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 684 } 685 } 686 return new Unknown64(machInst); 687 } 688 689 StaticInstPtr 690 decodeSveScatterStoreSVInsts(uint8_t msz, ExtMachInst machInst, 691 IntRegIndex zt, IntRegIndex pg, 692 IntRegIndex rn, IntRegIndex zm, 693 bool esizeIs32, bool offsetIs32, 694 bool offsetIsSigned, bool offsetIsScaled) 695 { 696 const char* mn = "st1"; 697 switch (msz) { 698 case 0x0: 699 if (esizeIs32) { 700 return new SveIndexedMemSV<uint32_t, uint8_t, 701 SveScatterStoreSVMicroop, 702 SveFirstFaultWritebackMicroop>( 703 mn, machInst, MemWriteOp, zt, pg, rn, zm, 704 offsetIs32, offsetIsSigned, offsetIsScaled, false); 705 } else { 706 return new SveIndexedMemSV<uint64_t, uint8_t, 707 SveScatterStoreSVMicroop, 708 SveFirstFaultWritebackMicroop>( 709 mn, machInst, MemWriteOp, zt, pg, rn, zm, 710 offsetIs32, offsetIsSigned, offsetIsScaled, false); 711 } 712 case 0x1: 713 if (esizeIs32) { 714 return new SveIndexedMemSV<uint32_t, uint16_t, 715 SveScatterStoreSVMicroop, 716 SveFirstFaultWritebackMicroop>( 717 mn, machInst, MemWriteOp, zt, pg, rn, zm, 718 offsetIs32, offsetIsSigned, offsetIsScaled, false); 719 } else { 720 return new SveIndexedMemSV<uint64_t, uint16_t, 721 SveScatterStoreSVMicroop, 722 SveFirstFaultWritebackMicroop>( 723 mn, machInst, MemWriteOp, zt, pg, rn, zm, 724 offsetIs32, offsetIsSigned, offsetIsScaled, false); 725 } 726 case 0x2: 727 if (esizeIs32) { 728 return new SveIndexedMemSV<uint32_t, uint32_t, 729 SveScatterStoreSVMicroop, 730 SveFirstFaultWritebackMicroop>( 731 mn, machInst, MemWriteOp, zt, pg, rn, zm, 732 offsetIs32, offsetIsSigned, offsetIsScaled, false); 733 } else { 734 return new SveIndexedMemSV<uint64_t, uint32_t, 735 SveScatterStoreSVMicroop, 736 SveFirstFaultWritebackMicroop>( 737 mn, machInst, MemWriteOp, zt, pg, rn, zm, 738 offsetIs32, offsetIsSigned, offsetIsScaled, false); 739 } 740 case 0x3: 741 if (esizeIs32) { 742 break; 743 } else { 744 return new SveIndexedMemSV<uint64_t, uint64_t, 745 SveScatterStoreSVMicroop, 746 SveFirstFaultWritebackMicroop>( 747 mn, machInst, MemWriteOp, zt, pg, rn, zm, 748 offsetIs32, offsetIsSigned, offsetIsScaled, false); 749 } 750 } 751 return new Unknown64(machInst); 752 } 753 754}}; 755 756 757let {{ 758 759 header_output = '' 760 exec_output = '' 761 decoders = { 'Generic': {} } 762 763 SPAlignmentCheckCode = ''' 764 if (this->baseIsSP && bits(XBase, 3, 0) && 765 SPAlignmentCheckEnabled(xc->tcBase())) { 766 return std::make_shared<SPAlignmentFault>(); 767 } 768 ''' 769 770 def emitSveMemFillSpill(isPred): 771 global header_output, exec_output, decoders 772 eaCode = SPAlignmentCheckCode + ''' 773 int memAccessSize = %(memacc_size)s; 774 EA = XBase + ((int64_t) imm * %(memacc_size)s)''' % { 775 'memacc_size': 'eCount / 8' if isPred else 'eCount'} 776 loadRdEnableCode = ''' 777 auto rdEn = std::vector<bool>(); 778 ''' 779 if isPred: 780 loadMemAccCode = ''' 781 int index = 0; 782 uint8_t byte; 783 for (int i = 0; i < eCount / 8; i++) { 784 byte = memDataView[i]; 785 for (int j = 0; j < 8; j++, index++) { 786 PDest_x[index] = (byte >> j) & 1; 787 } 788 } 789 ''' 790 storeMemAccCode = ''' 791 int index = 0; 792 uint8_t byte; 793 for (int i = 0; i < eCount / 8; i++) { 794 byte = 0; 795 for (int j = 0; j < 8; j++, index++) { 796 byte |= PDest_x[index] << j; 797 } 798 memDataView[i] = byte; 799 } 800 ''' 801 storeWrEnableCode = ''' 802 auto wrEn = std::vector<bool>(eCount / 8, true); 803 ''' 804 else: 805 loadMemAccCode = ''' 806 for (int i = 0; i < eCount; i++) { 807 AA64FpDest_x[i] = memDataView[i]; 808 } 809 ''' 810 storeMemAccCode = ''' 811 for (int i = 0; i < eCount; i++) { 812 memDataView[i] = AA64FpDest_x[i]; 813 } 814 ''' 815 storeWrEnableCode = ''' 816 auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true); 817 ''' 818 loadIop = InstObjParams('ldr', 819 'SveLdrPred' if isPred else 'SveLdrVec', 820 'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill', 821 {'tpl_header': '', 822 'tpl_args': '', 823 'memacc_code': loadMemAccCode, 824 'ea_code' : sveEnabledCheckCode + eaCode, 825 'rden_code' : loadRdEnableCode, 826 'fault_code' : '', 827 'fa_code' : ''}, 828 ['IsMemRef', 'IsLoad']) 829 storeIop = InstObjParams('str', 830 'SveStrPred' if isPred else 'SveStrVec', 831 'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill', 832 {'tpl_header': '', 833 'tpl_args': '', 834 'wren_code': storeWrEnableCode, 835 'memacc_code': storeMemAccCode, 836 'ea_code' : sveEnabledCheckCode + eaCode, 837 'fa_code' : ''}, 838 ['IsMemRef', 'IsStore']) 839 header_output += SveMemFillSpillOpDeclare.subst(loadIop) 840 header_output += SveMemFillSpillOpDeclare.subst(storeIop) 841 exec_output += ( 842 SveContigLoadExecute.subst(loadIop) + 843 SveContigLoadInitiateAcc.subst(loadIop) + 844 SveContigLoadCompleteAcc.subst(loadIop) + 845 SveContigStoreExecute.subst(storeIop) + 846 SveContigStoreInitiateAcc.subst(storeIop) + 847 SveContigStoreCompleteAcc.subst(storeIop)) 848 849 loadTplArgs = ( 850 ('uint8_t', 'uint8_t'), 851 ('uint16_t', 'uint8_t'), 852 ('uint32_t', 'uint8_t'), 853 ('uint64_t', 'uint8_t'), 854 ('int64_t', 'int32_t'), 855 ('uint16_t', 'uint16_t'), 856 ('uint32_t', 'uint16_t'), 857 ('uint64_t', 'uint16_t'), 858 ('int64_t', 'int16_t'), 859 ('int32_t', 'int16_t'), 860 ('uint32_t', 'uint32_t'), 861 ('uint64_t', 'uint32_t'), 862 ('int64_t', 'int8_t'), 863 ('int32_t', 'int8_t'), 864 ('int16_t', 'int8_t'), 865 ('uint64_t', 'uint64_t'), 866 ) 867 868 storeTplArgs = ( 869 ('uint8_t', 'uint8_t'), 870 ('uint16_t', 'uint8_t'), 871 ('uint32_t', 'uint8_t'), 872 ('uint64_t', 'uint8_t'), 873 ('uint16_t', 'uint16_t'), 874 ('uint32_t', 'uint16_t'), 875 ('uint64_t', 'uint16_t'), 876 ('uint32_t', 'uint32_t'), 877 ('uint64_t', 'uint32_t'), 878 ('uint64_t', 'uint64_t'), 879 ) 880 881 gatherLoadTplArgs = ( 882 ('int32_t', 'int8_t'), 883 ('int64_t', 'int8_t'), 884 ('uint32_t', 'uint8_t'), 885 ('uint64_t', 'uint8_t'), 886 ('int32_t', 'int16_t'), 887 ('int64_t', 'int16_t'), 888 ('uint32_t', 'uint16_t'), 889 ('uint64_t', 'uint16_t'), 890 ('int64_t', 'int32_t'), 891 ('uint32_t', 'uint32_t'), 892 ('uint64_t', 'uint32_t'), 893 ('uint64_t', 'uint64_t'), 894 ) 895 896 scatterStoreTplArgs = ( 897 ('uint32_t', 'uint8_t'), 898 ('uint64_t', 'uint8_t'), 899 ('uint32_t', 'uint16_t'), 900 ('uint64_t', 'uint16_t'), 901 ('uint32_t', 'uint32_t'), 902 ('uint64_t', 'uint32_t'), 903 ('uint64_t', 'uint64_t'), 904 ) 905 906 # Generates definitions for SVE contiguous loads 907 def emitSveContigMemInsts(offsetIsImm): 908 global header_output, exec_output, decoders 909 # First-faulting instructions only have a scalar plus scalar form, 910 # while non-faulting instructions only a scalar plus immediate form, so 911 # `offsetIsImm` is used to determine which class of instructions is 912 # generated 913 firstFaulting = not offsetIsImm 914 tplHeader = 'template <class RegElemType, class MemElemType>' 915 tplArgs = '<RegElemType, MemElemType>' 916 eaCode = SPAlignmentCheckCode + ''' 917 int memAccessSize = eCount * sizeof(MemElemType); 918 EA = XBase + ''' 919 if offsetIsImm: 920 eaCode += '((int64_t) this->imm * eCount * sizeof(MemElemType))' 921 else: 922 eaCode += '(XOffset * sizeof(MemElemType));' 923 loadRdEnableCode = ''' 924 auto rdEn = std::vector<bool>(sizeof(MemElemType) * eCount, true); 925 for (int i = 0; i < eCount; i++) { 926 if (!GpOp_x[i]) { 927 for (int j = 0; j < sizeof(MemElemType); j++) { 928 rdEn[sizeof(MemElemType) * i + j] = false; 929 } 930 } 931 } 932 ''' 933 loadMemAccCode = ''' 934 for (int i = 0; i < eCount; i++) { 935 if (GpOp_x[i]) { 936 AA64FpDest_x[i] = memDataView[i]; 937 } else { 938 AA64FpDest_x[i] = 0; 939 } 940 } 941 ''' 942 storeMemAccCode = ''' 943 for (int i = 0; i < eCount; i++) { 944 if (GpOp_x[i]) { 945 memDataView[i] = AA64FpDest_x[i]; 946 } else { 947 memDataView[i] = 0; 948 for (int j = 0; j < sizeof(MemElemType); j++) { 949 wrEn[sizeof(MemElemType) * i + j] = false; 950 } 951 } 952 } 953 ''' 954 storeWrEnableCode = ''' 955 auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true); 956 ''' 957 ffrReadBackCode = ''' 958 auto& firstFaultReg = Ffr;''' 959 fautlingLoadmemAccCode = ''' 960 for (int i = 0; i < eCount; i++) { 961 if (GpOp_x[i] && firstFaultReg[i * sizeof(RegElemType)]) { 962 AA64FpDest_x[i] = memDataView[i]; 963 } else { 964 AA64FpDest_x[i] = 0; 965 } 966 } 967 ''' 968 nonFaultingCode = 'true ||' 969 faultCode = ''' 970 Addr fault_addr; 971 if (fault == NoFault || getFaultVAddr(fault, fault_addr)) { 972 unsigned fault_elem_index; 973 if (fault != NoFault) { 974 assert(fault_addr >= EA); 975 fault_elem_index = (fault_addr - EA) / sizeof(MemElemType); 976 } else { 977 fault_elem_index = eCount + 1; 978 } 979 int first_active_index; 980 for (first_active_index = 0; 981 first_active_index < eCount && !(GpOp_x[first_active_index]); 982 first_active_index++); 983 if (%s first_active_index < fault_elem_index) { 984 for (int i = 0; i < eCount; i++) { 985 for (int j = 0; j < sizeof(RegElemType); j++) { 986 if (i < fault_elem_index) { 987 Ffr_ub[i * sizeof(RegElemType) + j] = FfrAux_x[i]; 988 } else { 989 Ffr_ub[i * sizeof(RegElemType) + j] = 0; 990 } 991 } 992 } 993 fault = NoFault; 994 if (first_active_index >= fault_elem_index) { 995 // non-faulting load needs this 996 xc->setMemAccPredicate(false); 997 } 998 } 999 } 1000 ''' % ('' if firstFaulting else nonFaultingCode) 1001 1002 loadIop = InstObjParams('ld1', 1003 'SveContigLoadSI' if offsetIsImm else 'SveContigLoadSS', 1004 'SveContigMemSI' if offsetIsImm else 'SveContigMemSS', 1005 {'tpl_header': tplHeader, 1006 'tpl_args': tplArgs, 1007 'rden_code' : loadRdEnableCode, 1008 'memacc_code': loadMemAccCode, 1009 'ea_code' : sveEnabledCheckCode + eaCode, 1010 'fault_code' : '', 1011 'fa_code' : ''}, 1012 ['IsMemRef', 'IsLoad']) 1013 storeIop = InstObjParams('st1', 1014 'SveContigStoreSI' if offsetIsImm else 'SveContigStoreSS', 1015 'SveContigMemSI' if offsetIsImm else 'SveContigMemSS', 1016 {'tpl_header': tplHeader, 1017 'tpl_args': tplArgs, 1018 'wren_code': storeWrEnableCode, 1019 'memacc_code': storeMemAccCode, 1020 'ea_code' : sveEnabledCheckCode + eaCode, 1021 'fa_code' : ''}, 1022 ['IsMemRef', 'IsStore']) 1023 faultIop = InstObjParams('ldff1' if firstFaulting else 'ldnf1', 1024 'SveContigFFLoadSS' if firstFaulting else 'SveContigNFLoadSI', 1025 'SveContigMemSS' if firstFaulting else 'SveContigMemSI', 1026 {'tpl_header': tplHeader, 1027 'tpl_args': tplArgs, 1028 'rden_code' : loadRdEnableCode, 1029 'memacc_code': fautlingLoadmemAccCode, 1030 'ea_code' : sveEnabledCheckCode + eaCode, 1031 'fault_code' : faultCode, 1032 'fa_code' : ''}, 1033 ['IsMemRef', 'IsLoad']) 1034 faultIop.snippets['memacc_code'] = (ffrReadBackCode + 1035 faultIop.snippets['memacc_code']) 1036 if offsetIsImm: 1037 header_output += SveContigMemSIOpDeclare.subst(loadIop) 1038 header_output += SveContigMemSIOpDeclare.subst(storeIop) 1039 header_output += SveContigMemSIOpDeclare.subst(faultIop) 1040 else: 1041 header_output += SveContigMemSSOpDeclare.subst(loadIop) 1042 header_output += SveContigMemSSOpDeclare.subst(storeIop) 1043 header_output += SveContigMemSSOpDeclare.subst(faultIop) 1044 exec_output += ( 1045 SveContigLoadExecute.subst(loadIop) + 1046 SveContigLoadInitiateAcc.subst(loadIop) + 1047 SveContigLoadCompleteAcc.subst(loadIop) + 1048 SveContigStoreExecute.subst(storeIop) + 1049 SveContigStoreInitiateAcc.subst(storeIop) + 1050 SveContigStoreCompleteAcc.subst(storeIop) + 1051 SveContigLoadExecute.subst(faultIop) + 1052 SveContigLoadInitiateAcc.subst(faultIop) + 1053 SveContigLoadCompleteAcc.subst(faultIop)) 1054 1055 for args in loadTplArgs: 1056 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1057 'class_name': 'SveContigLoadSI' if offsetIsImm 1058 else 'SveContigLoadSS'} 1059 exec_output += SveContigMemExecDeclare.subst(substDict) 1060 for args in storeTplArgs: 1061 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1062 'class_name': 'SveContigStoreSI' if offsetIsImm 1063 else 'SveContigStoreSS'} 1064 exec_output += SveContigMemExecDeclare.subst(substDict) 1065 for args in loadTplArgs: 1066 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1067 'class_name': 'SveContigFFLoadSS' if firstFaulting 1068 else 'SveContigNFLoadSI'} 1069 exec_output += SveContigMemExecDeclare.subst(substDict) 1070 1071 1072 # Generates definitions for SVE load-and-replicate instructions 1073 def emitSveLoadAndRepl(): 1074 global header_output, exec_output, decoders 1075 tplHeader = 'template <class RegElemType, class MemElemType>' 1076 tplArgs = '<RegElemType, MemElemType>' 1077 eaCode = SPAlignmentCheckCode + ''' 1078 EA = XBase + imm * sizeof(MemElemType);''' 1079 memAccCode = ''' 1080 for (int i = 0; i < eCount; i++) { 1081 if (GpOp_x[i]) { 1082 AA64FpDest_x[i] = memData; 1083 } else { 1084 AA64FpDest_x[i] = 0; 1085 } 1086 } 1087 ''' 1088 iop = InstObjParams('ld1r', 1089 'SveLoadAndRepl', 1090 'SveContigMemSI', 1091 {'tpl_header': tplHeader, 1092 'tpl_args': tplArgs, 1093 'memacc_code': memAccCode, 1094 'ea_code' : sveEnabledCheckCode + eaCode, 1095 'fa_code' : ''}, 1096 ['IsMemRef', 'IsLoad']) 1097 header_output += SveContigMemSIOpDeclare.subst(iop) 1098 exec_output += ( 1099 SveLoadAndReplExecute.subst(iop) + 1100 SveLoadAndReplInitiateAcc.subst(iop) + 1101 SveLoadAndReplCompleteAcc.subst(iop)) 1102 for args in loadTplArgs: 1103 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1104 'class_name': 'SveLoadAndRepl'} 1105 exec_output += SveContigMemExecDeclare.subst(substDict) 1106 1107 class IndexedAddrForm: 1108 VEC_PLUS_IMM = 0 1109 SCA_PLUS_VEC = 1 1110 1111 # Generates definitions for the transfer microops of SVE indexed memory 1112 # operations (gather loads, scatter stores) 1113 def emitSveIndexedMemMicroops(indexed_addr_form): 1114 assert indexed_addr_form in (IndexedAddrForm.VEC_PLUS_IMM, 1115 IndexedAddrForm.SCA_PLUS_VEC) 1116 global header_output, exec_output, decoders 1117 tplHeader = 'template <class RegElemType, class MemElemType>' 1118 tplArgs = '<RegElemType, MemElemType>' 1119 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM:
| 2// All rights reserved 3// 4// The license below extends only to copyright in the software and shall 5// not be construed as granting a license to any other intellectual 6// property including but not limited to intellectual property relating 7// to a hardware implementation of the functionality of the software 8// licensed hereunder. You may use the software subject to the license 9// terms below provided that you ensure that this notice is replicated 10// unmodified and in its entirety in all distributions of the software, 11// modified or unmodified, in source code or in binary form. 12// 13// Redistribution and use in source and binary forms, with or without 14// modification, are permitted provided that the following conditions are 15// met: redistributions of source code must retain the above copyright 16// notice, this list of conditions and the following disclaimer; 17// redistributions in binary form must reproduce the above copyright 18// notice, this list of conditions and the following disclaimer in the 19// documentation and/or other materials provided with the distribution; 20// neither the name of the copyright holders nor the names of its 21// contributors may be used to endorse or promote products derived from 22// this software without specific prior written permission. 23// 24// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 27// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 28// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 30// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 34// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35// 36// Authors: Giacomo Gabrielli 37 38// @file Definition of SVE memory access instructions. 39 40output header {{ 41 42 // Decodes SVE contiguous load instructions, scalar plus scalar form. 43 template <template <typename T1, typename T2> class Base> 44 StaticInstPtr 45 decodeSveContigLoadSSInsts(uint8_t dtype, ExtMachInst machInst, 46 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 47 IntRegIndex rm, bool firstFaulting) 48 { 49 const char* mn = firstFaulting ? "ldff1" : "ld1"; 50 switch (dtype) { 51 case 0x0: 52 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 53 case 0x1: 54 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 55 case 0x2: 56 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 57 case 0x3: 58 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 59 case 0x4: 60 return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, rm); 61 case 0x5: 62 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 63 case 0x6: 64 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 65 case 0x7: 66 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 67 case 0x8: 68 return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, rm); 69 case 0x9: 70 return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, rm); 71 case 0xa: 72 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm); 73 case 0xb: 74 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm); 75 case 0xc: 76 return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, rm); 77 case 0xd: 78 return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, rm); 79 case 0xe: 80 return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, rm); 81 case 0xf: 82 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm); 83 } 84 return new Unknown64(machInst); 85 } 86 87 // Decodes SVE contiguous load instructions, scalar plus immediate form. 88 template <template <typename T1, typename T2> class Base> 89 StaticInstPtr 90 decodeSveContigLoadSIInsts(uint8_t dtype, ExtMachInst machInst, 91 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 92 uint64_t imm, bool nonFaulting, 93 bool replicate = false) 94 { 95 assert(!(nonFaulting && replicate)); 96 const char* mn = replicate ? "ld1r" : (nonFaulting ? "ldnf1" : "ld1"); 97 switch (dtype) { 98 case 0x0: 99 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 100 case 0x1: 101 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 102 case 0x2: 103 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 104 case 0x3: 105 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 106 case 0x4: 107 return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, imm); 108 case 0x5: 109 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 110 case 0x6: 111 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 112 case 0x7: 113 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 114 case 0x8: 115 return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, imm); 116 case 0x9: 117 return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, imm); 118 case 0xa: 119 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm); 120 case 0xb: 121 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm); 122 case 0xc: 123 return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, imm); 124 case 0xd: 125 return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, imm); 126 case 0xe: 127 return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, imm); 128 case 0xf: 129 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm); 130 } 131 return new Unknown64(machInst); 132 } 133 134 // Decodes SVE contiguous store instructions, scalar plus scalar form. 135 template <template <typename T1, typename T2> class Base> 136 StaticInstPtr 137 decodeSveContigStoreSSInsts(uint8_t dtype, ExtMachInst machInst, 138 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 139 IntRegIndex rm) 140 { 141 const char* mn = "st1"; 142 switch (dtype) { 143 case 0x0: 144 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 145 case 0x1: 146 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 147 case 0x2: 148 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 149 case 0x3: 150 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm); 151 case 0x5: 152 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 153 case 0x6: 154 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 155 case 0x7: 156 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm); 157 case 0xa: 158 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm); 159 case 0xb: 160 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm); 161 case 0xf: 162 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm); 163 } 164 return new Unknown64(machInst); 165 } 166 167 // Decodes SVE contiguous store instructions, scalar plus immediate form. 168 template <template <typename T1, typename T2> class Base> 169 StaticInstPtr 170 decodeSveContigStoreSIInsts(uint8_t dtype, ExtMachInst machInst, 171 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 172 int8_t imm) 173 { 174 const char* mn = "st1"; 175 switch (dtype) { 176 case 0x0: 177 return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 178 case 0x1: 179 return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 180 case 0x2: 181 return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 182 case 0x3: 183 return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm); 184 case 0x5: 185 return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 186 case 0x6: 187 return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 188 case 0x7: 189 return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm); 190 case 0xa: 191 return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm); 192 case 0xb: 193 return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm); 194 case 0xf: 195 return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm); 196 } 197 return new Unknown64(machInst); 198 } 199 200 // NOTE: SVE load-and-replicate instructions are decoded with 201 // decodeSveContigLoadSIInsts(...). 202 203}}; 204 205output decoder {{ 206 207 template <class etype> 208 StaticInstPtr 209 decodeSveStructLoadSIInstsByNReg(uint8_t esize, ExtMachInst machInst, 210 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 211 int64_t imm, int numregs) 212 { 213 static const char* nm[5][4] = { 214 { nullptr, nullptr, nullptr, nullptr}, 215 { nullptr, nullptr, nullptr, nullptr}, 216 { "ld2b", "ld2h", "ld2w", "ld2d" }, 217 { "ld3b", "ld3h", "ld3w", "ld3d" }, 218 { "ld4b", "ld4h", "ld4w", "ld4d" } }; 219 220 switch (numregs) { 221 case 2: 222 return new SveLdStructSI<etype, 223 SveLoadRegImmMicroop, 224 SveDeIntrlv2Microop>( 225 nm[numregs][esize], machInst, MemReadOp, 226 zt, pg, xn, imm, numregs); 227 case 3: 228 return new SveLdStructSI<etype, 229 SveLoadRegImmMicroop, 230 SveDeIntrlv3Microop>( 231 nm[numregs][esize], machInst, MemReadOp, 232 zt, pg, xn, imm, numregs); 233 case 4: 234 return new SveLdStructSI<etype, 235 SveLoadRegImmMicroop, 236 SveDeIntrlv4Microop>( 237 nm[numregs][esize], machInst, MemReadOp, 238 zt, pg, xn, imm, numregs); 239 } 240 return new Unknown64(machInst); 241 } 242 243 StaticInstPtr 244 decodeSveStructLoadSIInsts(uint8_t esize, ExtMachInst machInst, 245 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 246 int64_t imm, int numregs) 247 { 248 switch (esize) { 249 case 0: 250 return decodeSveStructLoadSIInstsByNReg<uint8_t>(esize, 251 machInst, zt, pg, xn, imm, numregs); 252 case 1: 253 return decodeSveStructLoadSIInstsByNReg<uint16_t>(esize, 254 machInst, zt, pg, xn, imm, numregs); 255 case 2: 256 return decodeSveStructLoadSIInstsByNReg<uint32_t>(esize, 257 machInst, zt, pg, xn, imm, numregs); 258 case 3: 259 return decodeSveStructLoadSIInstsByNReg<uint64_t>(esize, 260 machInst, zt, pg, xn, imm, numregs); 261 } 262 return new Unknown64(machInst); 263 } 264 265 template <class etype> 266 StaticInstPtr 267 decodeSveStructStoreSIInstsByNReg(uint8_t esize, ExtMachInst machInst, 268 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 269 int64_t imm, int numregs) 270 { 271 static const char* nm[5][4] = { 272 { nullptr, nullptr, nullptr, nullptr}, 273 { nullptr, nullptr, nullptr, nullptr}, 274 { "st2b", "st2h", "st2w", "st2d" }, 275 { "st3b", "st3h", "st3w", "st3d" }, 276 { "st4b", "st4h", "st4w", "st4d" } }; 277 278 switch (numregs) { 279 case 2: 280 return new SveStStructSI<etype, 281 SveStoreRegImmMicroop, 282 SveIntrlv2Microop>( 283 nm[numregs][esize], machInst, MemWriteOp, 284 zt, pg, xn, imm, numregs); 285 case 3: 286 return new SveStStructSI<etype, 287 SveStoreRegImmMicroop, 288 SveIntrlv3Microop>( 289 nm[numregs][esize], machInst, MemWriteOp, 290 zt, pg, xn, imm, numregs); 291 case 4: 292 return new SveStStructSI<etype, 293 SveStoreRegImmMicroop, 294 SveIntrlv4Microop>( 295 nm[numregs][esize], machInst, MemWriteOp, 296 zt, pg, xn, imm, numregs); 297 } 298 return new Unknown64(machInst); 299 } 300 301 StaticInstPtr 302 decodeSveStructStoreSIInsts(uint8_t esize, ExtMachInst machInst, 303 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 304 int64_t imm, int numregs) 305 { 306 switch (esize) { 307 case 0: 308 return decodeSveStructStoreSIInstsByNReg<uint8_t>(esize, 309 machInst, zt, pg, xn, imm, numregs); 310 case 1: 311 return decodeSveStructStoreSIInstsByNReg<uint16_t>(esize, 312 machInst, zt, pg, xn, imm, numregs); 313 case 2: 314 return decodeSveStructStoreSIInstsByNReg<uint32_t>(esize, 315 machInst, zt, pg, xn, imm, numregs); 316 case 3: 317 return decodeSveStructStoreSIInstsByNReg<uint64_t>(esize, 318 machInst, zt, pg, xn, imm, numregs); 319 } 320 return new Unknown64(machInst); 321 } 322 323 template <class etype> 324 StaticInstPtr 325 decodeSveStructLoadSSInstsByNReg(uint8_t esize, ExtMachInst machInst, 326 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 327 IntRegIndex xm, int numregs) 328 { 329 static const char* nm[5][4] = { 330 { nullptr, nullptr, nullptr, nullptr}, 331 { nullptr, nullptr, nullptr, nullptr}, 332 { "ld2b", "ld2h", "ld2w", "ld2d" }, 333 { "ld3b", "ld3h", "ld3w", "ld3d" }, 334 { "ld4b", "ld4h", "ld4w", "ld4d" } }; 335 336 switch (numregs) { 337 case 2: 338 return new SveLdStructSS<etype, 339 SveLoadRegRegMicroop, 340 SveDeIntrlv2Microop>( 341 nm[numregs][esize], machInst, MemReadOp, 342 zt, pg, xn, xm, numregs); 343 case 3: 344 return new SveLdStructSS<etype, 345 SveLoadRegRegMicroop, 346 SveDeIntrlv3Microop>( 347 nm[numregs][esize], machInst, MemReadOp, 348 zt, pg, xn, xm, numregs); 349 case 4: 350 return new SveLdStructSS<etype, 351 SveLoadRegRegMicroop, 352 SveDeIntrlv4Microop>( 353 nm[numregs][esize], machInst, MemReadOp, 354 zt, pg, xn, xm, numregs); 355 } 356 return new Unknown64(machInst); 357 } 358 359 StaticInstPtr 360 decodeSveStructLoadSSInsts(uint8_t esize, ExtMachInst machInst, 361 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 362 IntRegIndex xm, int numregs) 363 { 364 switch (esize) { 365 case 0: 366 return decodeSveStructLoadSSInstsByNReg<uint8_t>(esize, 367 machInst, zt, pg, xn, xm, numregs); 368 case 1: 369 return decodeSveStructLoadSSInstsByNReg<uint16_t>(esize, 370 machInst, zt, pg, xn, xm, numregs); 371 case 2: 372 return decodeSveStructLoadSSInstsByNReg<uint32_t>(esize, 373 machInst, zt, pg, xn, xm, numregs); 374 case 3: 375 return decodeSveStructLoadSSInstsByNReg<uint64_t>(esize, 376 machInst, zt, pg, xn, xm, numregs); 377 } 378 return new Unknown64(machInst); 379 } 380 381 template <class etype> 382 StaticInstPtr 383 decodeSveStructStoreSSInstsByNReg(uint8_t esize, ExtMachInst machInst, 384 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 385 IntRegIndex xm, int numregs) 386 { 387 static const char* nm[5][4] = { 388 { nullptr, nullptr, nullptr, nullptr}, 389 { nullptr, nullptr, nullptr, nullptr}, 390 { "st2b", "st2h", "st2w", "st2d" }, 391 { "st3b", "st3h", "st3w", "st3d" }, 392 { "st4b", "st4h", "st4w", "st4d" } }; 393 394 switch (numregs) { 395 case 2: 396 return new SveStStructSS<etype, 397 SveStoreRegRegMicroop, 398 SveIntrlv2Microop>( 399 nm[numregs][esize], machInst, MemWriteOp, 400 zt, pg, xn, xm, numregs); 401 case 3: 402 return new SveStStructSS<etype, 403 SveStoreRegRegMicroop, 404 SveIntrlv3Microop>( 405 nm[numregs][esize], machInst, MemWriteOp, 406 zt, pg, xn, xm, numregs); 407 case 4: 408 return new SveStStructSS<etype, 409 SveStoreRegRegMicroop, 410 SveIntrlv4Microop>( 411 nm[numregs][esize], machInst, MemWriteOp, 412 zt, pg, xn, xm, numregs); 413 } 414 return new Unknown64(machInst); 415 } 416 417 StaticInstPtr 418 decodeSveStructStoreSSInsts(uint8_t esize, ExtMachInst machInst, 419 IntRegIndex zt, IntRegIndex pg, IntRegIndex xn, 420 IntRegIndex xm, int numregs) 421 { 422 switch (esize) { 423 case 0: 424 return decodeSveStructStoreSSInstsByNReg<uint8_t>(esize, 425 machInst, zt, pg, xn, xm, numregs); 426 case 1: 427 return decodeSveStructStoreSSInstsByNReg<uint16_t>(esize, 428 machInst, zt, pg, xn, xm, numregs); 429 case 2: 430 return decodeSveStructStoreSSInstsByNReg<uint32_t>(esize, 431 machInst, zt, pg, xn, xm, numregs); 432 case 3: 433 return decodeSveStructStoreSSInstsByNReg<uint64_t>(esize, 434 machInst, zt, pg, xn, xm, numregs); 435 } 436 return new Unknown64(machInst); 437 } 438 439 StaticInstPtr 440 decodeSveGatherLoadVIInsts(uint8_t dtype, ExtMachInst machInst, 441 IntRegIndex zt, IntRegIndex pg, IntRegIndex zn, 442 uint64_t imm, bool esizeIs32, 443 bool firstFault) 444 { 445 const char* mn = firstFault ? "ldff1" : "ld1"; 446 switch (dtype) { 447 case 0x0: 448 if (esizeIs32) { 449 return new SveIndexedMemVI<int32_t, int8_t, 450 SveGatherLoadVIMicroop, 451 SveFirstFaultWritebackMicroop>( 452 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 453 } else { 454 return new SveIndexedMemVI<int64_t, int8_t, 455 SveGatherLoadVIMicroop, 456 SveFirstFaultWritebackMicroop>( 457 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 458 } 459 case 0x1: 460 if (esizeIs32) { 461 return new SveIndexedMemVI<uint32_t, uint8_t, 462 SveGatherLoadVIMicroop, 463 SveFirstFaultWritebackMicroop>( 464 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 465 } else { 466 return new SveIndexedMemVI<uint64_t, uint8_t, 467 SveGatherLoadVIMicroop, 468 SveFirstFaultWritebackMicroop>( 469 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 470 } 471 case 0x2: 472 if (esizeIs32) { 473 return new SveIndexedMemVI<int32_t, int16_t, 474 SveGatherLoadVIMicroop, 475 SveFirstFaultWritebackMicroop>( 476 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 477 } else { 478 return new SveIndexedMemVI<int64_t, int16_t, 479 SveGatherLoadVIMicroop, 480 SveFirstFaultWritebackMicroop>( 481 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 482 } 483 case 0x3: 484 if (esizeIs32) { 485 return new SveIndexedMemVI<uint32_t, uint16_t, 486 SveGatherLoadVIMicroop, 487 SveFirstFaultWritebackMicroop>( 488 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 489 } else { 490 return new SveIndexedMemVI<uint64_t, uint16_t, 491 SveGatherLoadVIMicroop, 492 SveFirstFaultWritebackMicroop>( 493 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 494 } 495 case 0x4: 496 if (esizeIs32) { 497 break; 498 } else { 499 return new SveIndexedMemVI<int64_t, int32_t, 500 SveGatherLoadVIMicroop, 501 SveFirstFaultWritebackMicroop>( 502 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 503 } 504 case 0x5: 505 if (esizeIs32) { 506 return new SveIndexedMemVI<uint32_t, uint32_t, 507 SveGatherLoadVIMicroop, 508 SveFirstFaultWritebackMicroop>( 509 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 510 } else { 511 return new SveIndexedMemVI<uint64_t, uint32_t, 512 SveGatherLoadVIMicroop, 513 SveFirstFaultWritebackMicroop>( 514 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 515 } 516 case 0x7: 517 if (esizeIs32) { 518 break; 519 } else { 520 return new SveIndexedMemVI<uint64_t, uint64_t, 521 SveGatherLoadVIMicroop, 522 SveFirstFaultWritebackMicroop>( 523 mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault); 524 } 525 } 526 return new Unknown64(machInst); 527 } 528 529 StaticInstPtr 530 decodeSveGatherLoadSVInsts(uint8_t dtype, ExtMachInst machInst, 531 IntRegIndex zt, IntRegIndex pg, IntRegIndex rn, 532 IntRegIndex zm, bool esizeIs32, bool offsetIs32, 533 bool offsetIsSigned, bool offsetIsScaled, 534 bool firstFault) 535 { 536 const char* mn = firstFault ? "ldff1" : "ld1"; 537 switch (dtype) { 538 case 0x0: 539 if (esizeIs32) { 540 return new SveIndexedMemSV<int32_t, int8_t, 541 SveGatherLoadSVMicroop, 542 SveFirstFaultWritebackMicroop>( 543 mn, machInst, MemReadOp, zt, pg, rn, zm, 544 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 545 } else { 546 return new SveIndexedMemSV<int64_t, int8_t, 547 SveGatherLoadSVMicroop, 548 SveFirstFaultWritebackMicroop>( 549 mn, machInst, MemReadOp, zt, pg, rn, zm, 550 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 551 } 552 case 0x1: 553 if (esizeIs32) { 554 return new SveIndexedMemSV<uint32_t, uint8_t, 555 SveGatherLoadSVMicroop, 556 SveFirstFaultWritebackMicroop>( 557 mn, machInst, MemReadOp, zt, pg, rn, zm, 558 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 559 } else { 560 return new SveIndexedMemSV<uint64_t, uint8_t, 561 SveGatherLoadSVMicroop, 562 SveFirstFaultWritebackMicroop>( 563 mn, machInst, MemReadOp, zt, pg, rn, zm, 564 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 565 } 566 case 0x2: 567 if (esizeIs32) { 568 return new SveIndexedMemSV<int32_t, int16_t, 569 SveGatherLoadSVMicroop, 570 SveFirstFaultWritebackMicroop>( 571 mn, machInst, MemReadOp, zt, pg, rn, zm, 572 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 573 } else { 574 return new SveIndexedMemSV<int64_t, int16_t, 575 SveGatherLoadSVMicroop, 576 SveFirstFaultWritebackMicroop>( 577 mn, machInst, MemReadOp, zt, pg, rn, zm, 578 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 579 } 580 case 0x3: 581 if (esizeIs32) { 582 return new SveIndexedMemSV<uint32_t, uint16_t, 583 SveGatherLoadSVMicroop, 584 SveFirstFaultWritebackMicroop>( 585 mn, machInst, MemReadOp, zt, pg, rn, zm, 586 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 587 } else { 588 return new SveIndexedMemSV<uint64_t, uint16_t, 589 SveGatherLoadSVMicroop, 590 SveFirstFaultWritebackMicroop>( 591 mn, machInst, MemReadOp, zt, pg, rn, zm, 592 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 593 } 594 case 0x4: 595 if (esizeIs32) { 596 break; 597 } else { 598 return new SveIndexedMemSV<int64_t, int32_t, 599 SveGatherLoadSVMicroop, 600 SveFirstFaultWritebackMicroop>( 601 mn, machInst, MemReadOp, zt, pg, rn, zm, 602 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 603 } 604 case 0x5: 605 if (esizeIs32) { 606 return new SveIndexedMemSV<uint32_t, uint32_t, 607 SveGatherLoadSVMicroop, 608 SveFirstFaultWritebackMicroop>( 609 mn, machInst, MemReadOp, zt, pg, rn, zm, 610 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 611 } else { 612 return new SveIndexedMemSV<uint64_t, uint32_t, 613 SveGatherLoadSVMicroop, 614 SveFirstFaultWritebackMicroop>( 615 mn, machInst, MemReadOp, zt, pg, rn, zm, 616 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 617 } 618 case 0x7: 619 if (esizeIs32) { 620 break; 621 } else { 622 return new SveIndexedMemSV<uint64_t, uint64_t, 623 SveGatherLoadSVMicroop, 624 SveFirstFaultWritebackMicroop>( 625 mn, machInst, MemReadOp, zt, pg, rn, zm, 626 offsetIs32, offsetIsSigned, offsetIsScaled, firstFault); 627 } 628 } 629 return new Unknown64(machInst); 630 } 631 632 StaticInstPtr 633 decodeSveScatterStoreVIInsts(uint8_t msz, ExtMachInst machInst, 634 IntRegIndex zt, IntRegIndex pg, 635 IntRegIndex zn, uint64_t imm, 636 bool esizeIs32) 637 { 638 const char* mn = "st1"; 639 switch (msz) { 640 case 0x0: 641 if (esizeIs32) { 642 return new SveIndexedMemVI<uint32_t, uint8_t, 643 SveScatterStoreVIMicroop, 644 SveFirstFaultWritebackMicroop>( 645 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 646 } else { 647 return new SveIndexedMemVI<uint64_t, uint8_t, 648 SveScatterStoreVIMicroop, 649 SveFirstFaultWritebackMicroop>( 650 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 651 } 652 case 0x1: 653 if (esizeIs32) { 654 return new SveIndexedMemVI<uint32_t, uint16_t, 655 SveScatterStoreVIMicroop, 656 SveFirstFaultWritebackMicroop>( 657 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 658 } else { 659 return new SveIndexedMemVI<uint64_t, uint16_t, 660 SveScatterStoreVIMicroop, 661 SveFirstFaultWritebackMicroop>( 662 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 663 } 664 case 0x2: 665 if (esizeIs32) { 666 return new SveIndexedMemVI<uint32_t, uint32_t, 667 SveScatterStoreVIMicroop, 668 SveFirstFaultWritebackMicroop>( 669 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 670 } else { 671 return new SveIndexedMemVI<uint64_t, uint32_t, 672 SveScatterStoreVIMicroop, 673 SveFirstFaultWritebackMicroop>( 674 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 675 } 676 case 0x3: 677 if (esizeIs32) { 678 break; 679 } else { 680 return new SveIndexedMemVI<uint64_t, uint64_t, 681 SveScatterStoreVIMicroop, 682 SveFirstFaultWritebackMicroop>( 683 mn, machInst, MemWriteOp, zt, pg, zn, imm, false); 684 } 685 } 686 return new Unknown64(machInst); 687 } 688 689 StaticInstPtr 690 decodeSveScatterStoreSVInsts(uint8_t msz, ExtMachInst machInst, 691 IntRegIndex zt, IntRegIndex pg, 692 IntRegIndex rn, IntRegIndex zm, 693 bool esizeIs32, bool offsetIs32, 694 bool offsetIsSigned, bool offsetIsScaled) 695 { 696 const char* mn = "st1"; 697 switch (msz) { 698 case 0x0: 699 if (esizeIs32) { 700 return new SveIndexedMemSV<uint32_t, uint8_t, 701 SveScatterStoreSVMicroop, 702 SveFirstFaultWritebackMicroop>( 703 mn, machInst, MemWriteOp, zt, pg, rn, zm, 704 offsetIs32, offsetIsSigned, offsetIsScaled, false); 705 } else { 706 return new SveIndexedMemSV<uint64_t, uint8_t, 707 SveScatterStoreSVMicroop, 708 SveFirstFaultWritebackMicroop>( 709 mn, machInst, MemWriteOp, zt, pg, rn, zm, 710 offsetIs32, offsetIsSigned, offsetIsScaled, false); 711 } 712 case 0x1: 713 if (esizeIs32) { 714 return new SveIndexedMemSV<uint32_t, uint16_t, 715 SveScatterStoreSVMicroop, 716 SveFirstFaultWritebackMicroop>( 717 mn, machInst, MemWriteOp, zt, pg, rn, zm, 718 offsetIs32, offsetIsSigned, offsetIsScaled, false); 719 } else { 720 return new SveIndexedMemSV<uint64_t, uint16_t, 721 SveScatterStoreSVMicroop, 722 SveFirstFaultWritebackMicroop>( 723 mn, machInst, MemWriteOp, zt, pg, rn, zm, 724 offsetIs32, offsetIsSigned, offsetIsScaled, false); 725 } 726 case 0x2: 727 if (esizeIs32) { 728 return new SveIndexedMemSV<uint32_t, uint32_t, 729 SveScatterStoreSVMicroop, 730 SveFirstFaultWritebackMicroop>( 731 mn, machInst, MemWriteOp, zt, pg, rn, zm, 732 offsetIs32, offsetIsSigned, offsetIsScaled, false); 733 } else { 734 return new SveIndexedMemSV<uint64_t, uint32_t, 735 SveScatterStoreSVMicroop, 736 SveFirstFaultWritebackMicroop>( 737 mn, machInst, MemWriteOp, zt, pg, rn, zm, 738 offsetIs32, offsetIsSigned, offsetIsScaled, false); 739 } 740 case 0x3: 741 if (esizeIs32) { 742 break; 743 } else { 744 return new SveIndexedMemSV<uint64_t, uint64_t, 745 SveScatterStoreSVMicroop, 746 SveFirstFaultWritebackMicroop>( 747 mn, machInst, MemWriteOp, zt, pg, rn, zm, 748 offsetIs32, offsetIsSigned, offsetIsScaled, false); 749 } 750 } 751 return new Unknown64(machInst); 752 } 753 754}}; 755 756 757let {{ 758 759 header_output = '' 760 exec_output = '' 761 decoders = { 'Generic': {} } 762 763 SPAlignmentCheckCode = ''' 764 if (this->baseIsSP && bits(XBase, 3, 0) && 765 SPAlignmentCheckEnabled(xc->tcBase())) { 766 return std::make_shared<SPAlignmentFault>(); 767 } 768 ''' 769 770 def emitSveMemFillSpill(isPred): 771 global header_output, exec_output, decoders 772 eaCode = SPAlignmentCheckCode + ''' 773 int memAccessSize = %(memacc_size)s; 774 EA = XBase + ((int64_t) imm * %(memacc_size)s)''' % { 775 'memacc_size': 'eCount / 8' if isPred else 'eCount'} 776 loadRdEnableCode = ''' 777 auto rdEn = std::vector<bool>(); 778 ''' 779 if isPred: 780 loadMemAccCode = ''' 781 int index = 0; 782 uint8_t byte; 783 for (int i = 0; i < eCount / 8; i++) { 784 byte = memDataView[i]; 785 for (int j = 0; j < 8; j++, index++) { 786 PDest_x[index] = (byte >> j) & 1; 787 } 788 } 789 ''' 790 storeMemAccCode = ''' 791 int index = 0; 792 uint8_t byte; 793 for (int i = 0; i < eCount / 8; i++) { 794 byte = 0; 795 for (int j = 0; j < 8; j++, index++) { 796 byte |= PDest_x[index] << j; 797 } 798 memDataView[i] = byte; 799 } 800 ''' 801 storeWrEnableCode = ''' 802 auto wrEn = std::vector<bool>(eCount / 8, true); 803 ''' 804 else: 805 loadMemAccCode = ''' 806 for (int i = 0; i < eCount; i++) { 807 AA64FpDest_x[i] = memDataView[i]; 808 } 809 ''' 810 storeMemAccCode = ''' 811 for (int i = 0; i < eCount; i++) { 812 memDataView[i] = AA64FpDest_x[i]; 813 } 814 ''' 815 storeWrEnableCode = ''' 816 auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true); 817 ''' 818 loadIop = InstObjParams('ldr', 819 'SveLdrPred' if isPred else 'SveLdrVec', 820 'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill', 821 {'tpl_header': '', 822 'tpl_args': '', 823 'memacc_code': loadMemAccCode, 824 'ea_code' : sveEnabledCheckCode + eaCode, 825 'rden_code' : loadRdEnableCode, 826 'fault_code' : '', 827 'fa_code' : ''}, 828 ['IsMemRef', 'IsLoad']) 829 storeIop = InstObjParams('str', 830 'SveStrPred' if isPred else 'SveStrVec', 831 'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill', 832 {'tpl_header': '', 833 'tpl_args': '', 834 'wren_code': storeWrEnableCode, 835 'memacc_code': storeMemAccCode, 836 'ea_code' : sveEnabledCheckCode + eaCode, 837 'fa_code' : ''}, 838 ['IsMemRef', 'IsStore']) 839 header_output += SveMemFillSpillOpDeclare.subst(loadIop) 840 header_output += SveMemFillSpillOpDeclare.subst(storeIop) 841 exec_output += ( 842 SveContigLoadExecute.subst(loadIop) + 843 SveContigLoadInitiateAcc.subst(loadIop) + 844 SveContigLoadCompleteAcc.subst(loadIop) + 845 SveContigStoreExecute.subst(storeIop) + 846 SveContigStoreInitiateAcc.subst(storeIop) + 847 SveContigStoreCompleteAcc.subst(storeIop)) 848 849 loadTplArgs = ( 850 ('uint8_t', 'uint8_t'), 851 ('uint16_t', 'uint8_t'), 852 ('uint32_t', 'uint8_t'), 853 ('uint64_t', 'uint8_t'), 854 ('int64_t', 'int32_t'), 855 ('uint16_t', 'uint16_t'), 856 ('uint32_t', 'uint16_t'), 857 ('uint64_t', 'uint16_t'), 858 ('int64_t', 'int16_t'), 859 ('int32_t', 'int16_t'), 860 ('uint32_t', 'uint32_t'), 861 ('uint64_t', 'uint32_t'), 862 ('int64_t', 'int8_t'), 863 ('int32_t', 'int8_t'), 864 ('int16_t', 'int8_t'), 865 ('uint64_t', 'uint64_t'), 866 ) 867 868 storeTplArgs = ( 869 ('uint8_t', 'uint8_t'), 870 ('uint16_t', 'uint8_t'), 871 ('uint32_t', 'uint8_t'), 872 ('uint64_t', 'uint8_t'), 873 ('uint16_t', 'uint16_t'), 874 ('uint32_t', 'uint16_t'), 875 ('uint64_t', 'uint16_t'), 876 ('uint32_t', 'uint32_t'), 877 ('uint64_t', 'uint32_t'), 878 ('uint64_t', 'uint64_t'), 879 ) 880 881 gatherLoadTplArgs = ( 882 ('int32_t', 'int8_t'), 883 ('int64_t', 'int8_t'), 884 ('uint32_t', 'uint8_t'), 885 ('uint64_t', 'uint8_t'), 886 ('int32_t', 'int16_t'), 887 ('int64_t', 'int16_t'), 888 ('uint32_t', 'uint16_t'), 889 ('uint64_t', 'uint16_t'), 890 ('int64_t', 'int32_t'), 891 ('uint32_t', 'uint32_t'), 892 ('uint64_t', 'uint32_t'), 893 ('uint64_t', 'uint64_t'), 894 ) 895 896 scatterStoreTplArgs = ( 897 ('uint32_t', 'uint8_t'), 898 ('uint64_t', 'uint8_t'), 899 ('uint32_t', 'uint16_t'), 900 ('uint64_t', 'uint16_t'), 901 ('uint32_t', 'uint32_t'), 902 ('uint64_t', 'uint32_t'), 903 ('uint64_t', 'uint64_t'), 904 ) 905 906 # Generates definitions for SVE contiguous loads 907 def emitSveContigMemInsts(offsetIsImm): 908 global header_output, exec_output, decoders 909 # First-faulting instructions only have a scalar plus scalar form, 910 # while non-faulting instructions only a scalar plus immediate form, so 911 # `offsetIsImm` is used to determine which class of instructions is 912 # generated 913 firstFaulting = not offsetIsImm 914 tplHeader = 'template <class RegElemType, class MemElemType>' 915 tplArgs = '<RegElemType, MemElemType>' 916 eaCode = SPAlignmentCheckCode + ''' 917 int memAccessSize = eCount * sizeof(MemElemType); 918 EA = XBase + ''' 919 if offsetIsImm: 920 eaCode += '((int64_t) this->imm * eCount * sizeof(MemElemType))' 921 else: 922 eaCode += '(XOffset * sizeof(MemElemType));' 923 loadRdEnableCode = ''' 924 auto rdEn = std::vector<bool>(sizeof(MemElemType) * eCount, true); 925 for (int i = 0; i < eCount; i++) { 926 if (!GpOp_x[i]) { 927 for (int j = 0; j < sizeof(MemElemType); j++) { 928 rdEn[sizeof(MemElemType) * i + j] = false; 929 } 930 } 931 } 932 ''' 933 loadMemAccCode = ''' 934 for (int i = 0; i < eCount; i++) { 935 if (GpOp_x[i]) { 936 AA64FpDest_x[i] = memDataView[i]; 937 } else { 938 AA64FpDest_x[i] = 0; 939 } 940 } 941 ''' 942 storeMemAccCode = ''' 943 for (int i = 0; i < eCount; i++) { 944 if (GpOp_x[i]) { 945 memDataView[i] = AA64FpDest_x[i]; 946 } else { 947 memDataView[i] = 0; 948 for (int j = 0; j < sizeof(MemElemType); j++) { 949 wrEn[sizeof(MemElemType) * i + j] = false; 950 } 951 } 952 } 953 ''' 954 storeWrEnableCode = ''' 955 auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true); 956 ''' 957 ffrReadBackCode = ''' 958 auto& firstFaultReg = Ffr;''' 959 fautlingLoadmemAccCode = ''' 960 for (int i = 0; i < eCount; i++) { 961 if (GpOp_x[i] && firstFaultReg[i * sizeof(RegElemType)]) { 962 AA64FpDest_x[i] = memDataView[i]; 963 } else { 964 AA64FpDest_x[i] = 0; 965 } 966 } 967 ''' 968 nonFaultingCode = 'true ||' 969 faultCode = ''' 970 Addr fault_addr; 971 if (fault == NoFault || getFaultVAddr(fault, fault_addr)) { 972 unsigned fault_elem_index; 973 if (fault != NoFault) { 974 assert(fault_addr >= EA); 975 fault_elem_index = (fault_addr - EA) / sizeof(MemElemType); 976 } else { 977 fault_elem_index = eCount + 1; 978 } 979 int first_active_index; 980 for (first_active_index = 0; 981 first_active_index < eCount && !(GpOp_x[first_active_index]); 982 first_active_index++); 983 if (%s first_active_index < fault_elem_index) { 984 for (int i = 0; i < eCount; i++) { 985 for (int j = 0; j < sizeof(RegElemType); j++) { 986 if (i < fault_elem_index) { 987 Ffr_ub[i * sizeof(RegElemType) + j] = FfrAux_x[i]; 988 } else { 989 Ffr_ub[i * sizeof(RegElemType) + j] = 0; 990 } 991 } 992 } 993 fault = NoFault; 994 if (first_active_index >= fault_elem_index) { 995 // non-faulting load needs this 996 xc->setMemAccPredicate(false); 997 } 998 } 999 } 1000 ''' % ('' if firstFaulting else nonFaultingCode) 1001 1002 loadIop = InstObjParams('ld1', 1003 'SveContigLoadSI' if offsetIsImm else 'SveContigLoadSS', 1004 'SveContigMemSI' if offsetIsImm else 'SveContigMemSS', 1005 {'tpl_header': tplHeader, 1006 'tpl_args': tplArgs, 1007 'rden_code' : loadRdEnableCode, 1008 'memacc_code': loadMemAccCode, 1009 'ea_code' : sveEnabledCheckCode + eaCode, 1010 'fault_code' : '', 1011 'fa_code' : ''}, 1012 ['IsMemRef', 'IsLoad']) 1013 storeIop = InstObjParams('st1', 1014 'SveContigStoreSI' if offsetIsImm else 'SveContigStoreSS', 1015 'SveContigMemSI' if offsetIsImm else 'SveContigMemSS', 1016 {'tpl_header': tplHeader, 1017 'tpl_args': tplArgs, 1018 'wren_code': storeWrEnableCode, 1019 'memacc_code': storeMemAccCode, 1020 'ea_code' : sveEnabledCheckCode + eaCode, 1021 'fa_code' : ''}, 1022 ['IsMemRef', 'IsStore']) 1023 faultIop = InstObjParams('ldff1' if firstFaulting else 'ldnf1', 1024 'SveContigFFLoadSS' if firstFaulting else 'SveContigNFLoadSI', 1025 'SveContigMemSS' if firstFaulting else 'SveContigMemSI', 1026 {'tpl_header': tplHeader, 1027 'tpl_args': tplArgs, 1028 'rden_code' : loadRdEnableCode, 1029 'memacc_code': fautlingLoadmemAccCode, 1030 'ea_code' : sveEnabledCheckCode + eaCode, 1031 'fault_code' : faultCode, 1032 'fa_code' : ''}, 1033 ['IsMemRef', 'IsLoad']) 1034 faultIop.snippets['memacc_code'] = (ffrReadBackCode + 1035 faultIop.snippets['memacc_code']) 1036 if offsetIsImm: 1037 header_output += SveContigMemSIOpDeclare.subst(loadIop) 1038 header_output += SveContigMemSIOpDeclare.subst(storeIop) 1039 header_output += SveContigMemSIOpDeclare.subst(faultIop) 1040 else: 1041 header_output += SveContigMemSSOpDeclare.subst(loadIop) 1042 header_output += SveContigMemSSOpDeclare.subst(storeIop) 1043 header_output += SveContigMemSSOpDeclare.subst(faultIop) 1044 exec_output += ( 1045 SveContigLoadExecute.subst(loadIop) + 1046 SveContigLoadInitiateAcc.subst(loadIop) + 1047 SveContigLoadCompleteAcc.subst(loadIop) + 1048 SveContigStoreExecute.subst(storeIop) + 1049 SveContigStoreInitiateAcc.subst(storeIop) + 1050 SveContigStoreCompleteAcc.subst(storeIop) + 1051 SveContigLoadExecute.subst(faultIop) + 1052 SveContigLoadInitiateAcc.subst(faultIop) + 1053 SveContigLoadCompleteAcc.subst(faultIop)) 1054 1055 for args in loadTplArgs: 1056 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1057 'class_name': 'SveContigLoadSI' if offsetIsImm 1058 else 'SveContigLoadSS'} 1059 exec_output += SveContigMemExecDeclare.subst(substDict) 1060 for args in storeTplArgs: 1061 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1062 'class_name': 'SveContigStoreSI' if offsetIsImm 1063 else 'SveContigStoreSS'} 1064 exec_output += SveContigMemExecDeclare.subst(substDict) 1065 for args in loadTplArgs: 1066 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1067 'class_name': 'SveContigFFLoadSS' if firstFaulting 1068 else 'SveContigNFLoadSI'} 1069 exec_output += SveContigMemExecDeclare.subst(substDict) 1070 1071 1072 # Generates definitions for SVE load-and-replicate instructions 1073 def emitSveLoadAndRepl(): 1074 global header_output, exec_output, decoders 1075 tplHeader = 'template <class RegElemType, class MemElemType>' 1076 tplArgs = '<RegElemType, MemElemType>' 1077 eaCode = SPAlignmentCheckCode + ''' 1078 EA = XBase + imm * sizeof(MemElemType);''' 1079 memAccCode = ''' 1080 for (int i = 0; i < eCount; i++) { 1081 if (GpOp_x[i]) { 1082 AA64FpDest_x[i] = memData; 1083 } else { 1084 AA64FpDest_x[i] = 0; 1085 } 1086 } 1087 ''' 1088 iop = InstObjParams('ld1r', 1089 'SveLoadAndRepl', 1090 'SveContigMemSI', 1091 {'tpl_header': tplHeader, 1092 'tpl_args': tplArgs, 1093 'memacc_code': memAccCode, 1094 'ea_code' : sveEnabledCheckCode + eaCode, 1095 'fa_code' : ''}, 1096 ['IsMemRef', 'IsLoad']) 1097 header_output += SveContigMemSIOpDeclare.subst(iop) 1098 exec_output += ( 1099 SveLoadAndReplExecute.subst(iop) + 1100 SveLoadAndReplInitiateAcc.subst(iop) + 1101 SveLoadAndReplCompleteAcc.subst(iop)) 1102 for args in loadTplArgs: 1103 substDict = {'tpl_args': '<%s>' % ', '.join(args), 1104 'class_name': 'SveLoadAndRepl'} 1105 exec_output += SveContigMemExecDeclare.subst(substDict) 1106 1107 class IndexedAddrForm: 1108 VEC_PLUS_IMM = 0 1109 SCA_PLUS_VEC = 1 1110 1111 # Generates definitions for the transfer microops of SVE indexed memory 1112 # operations (gather loads, scatter stores) 1113 def emitSveIndexedMemMicroops(indexed_addr_form): 1114 assert indexed_addr_form in (IndexedAddrForm.VEC_PLUS_IMM, 1115 IndexedAddrForm.SCA_PLUS_VEC) 1116 global header_output, exec_output, decoders 1117 tplHeader = 'template <class RegElemType, class MemElemType>' 1118 tplArgs = '<RegElemType, MemElemType>' 1119 if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM:
|