sve_mem.isa revision 14108:881e7d85baf7 
112137Sar4jc@virginia.edu// Copyright (c) 2017-2018 ARM Limited
212137Sar4jc@virginia.edu// All rights reserved
312137Sar4jc@virginia.edu//
412137Sar4jc@virginia.edu// The license below extends only to copyright in the software and shall
512137Sar4jc@virginia.edu// not be construed as granting a license to any other intellectual
612137Sar4jc@virginia.edu// property including but not limited to intellectual property relating
712137Sar4jc@virginia.edu// to a hardware implementation of the functionality of the software
812137Sar4jc@virginia.edu// licensed hereunder.  You may use the software subject to the license
912137Sar4jc@virginia.edu// terms below provided that you ensure that this notice is replicated
1012137Sar4jc@virginia.edu// unmodified and in its entirety in all distributions of the software,
1112137Sar4jc@virginia.edu// modified or unmodified, in source code or in binary form.
1212137Sar4jc@virginia.edu//
1312137Sar4jc@virginia.edu// Redistribution and use in source and binary forms, with or without
1412137Sar4jc@virginia.edu// modification, are permitted provided that the following conditions are
1512137Sar4jc@virginia.edu// met: redistributions of source code must retain the above copyright
1612137Sar4jc@virginia.edu// notice, this list of conditions and the following disclaimer;
1712137Sar4jc@virginia.edu// redistributions in binary form must reproduce the above copyright
1812137Sar4jc@virginia.edu// notice, this list of conditions and the following disclaimer in the
1912137Sar4jc@virginia.edu// documentation and/or other materials provided with the distribution;
2012137Sar4jc@virginia.edu// neither the name of the copyright holders nor the names of its
2112137Sar4jc@virginia.edu// contributors may be used to endorse or promote products derived from
2212137Sar4jc@virginia.edu// this software without specific prior written permission.
2312137Sar4jc@virginia.edu//
2412137Sar4jc@virginia.edu// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
2512137Sar4jc@virginia.edu// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
2612137Sar4jc@virginia.edu// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
2712137Sar4jc@virginia.edu// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
2812137Sar4jc@virginia.edu// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
2912137Sar4jc@virginia.edu// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
3012137Sar4jc@virginia.edu// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
3112137Sar4jc@virginia.edu// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
3212137Sar4jc@virginia.edu// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
3312137Sar4jc@virginia.edu// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
3412137Sar4jc@virginia.edu// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3512137Sar4jc@virginia.edu//
3612137Sar4jc@virginia.edu// Authors: Giacomo Gabrielli
3712137Sar4jc@virginia.edu
3812137Sar4jc@virginia.edu// @file Definition of SVE memory access instructions.
3912137Sar4jc@virginia.edu
4012137Sar4jc@virginia.eduoutput header {{
4112137Sar4jc@virginia.edu
4212137Sar4jc@virginia.edu    // Decodes SVE contiguous load instructions, scalar plus scalar form.
4312137Sar4jc@virginia.edu    template <template <typename T1, typename T2> class Base>
4412137Sar4jc@virginia.edu    StaticInstPtr
4512137Sar4jc@virginia.edu    decodeSveContigLoadSSInsts(uint8_t dtype, ExtMachInst machInst,
4612137Sar4jc@virginia.edu                               IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
4712137Sar4jc@virginia.edu                               IntRegIndex rm, bool firstFaulting)
4812137Sar4jc@virginia.edu    {
4912137Sar4jc@virginia.edu        const char* mn = firstFaulting ? "ldff1" : "ld1";
5012137Sar4jc@virginia.edu        switch (dtype) {
5112137Sar4jc@virginia.edu          case 0x0:
5212137Sar4jc@virginia.edu            return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
5312137Sar4jc@virginia.edu          case 0x1:
5412137Sar4jc@virginia.edu            return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
5512137Sar4jc@virginia.edu          case 0x2:
5612137Sar4jc@virginia.edu            return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
5712137Sar4jc@virginia.edu          case 0x3:
5812137Sar4jc@virginia.edu            return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
5912137Sar4jc@virginia.edu          case 0x4:
6012137Sar4jc@virginia.edu            return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, rm);
6112137Sar4jc@virginia.edu          case 0x5:
6212137Sar4jc@virginia.edu            return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
6312137Sar4jc@virginia.edu          case 0x6:
6412137Sar4jc@virginia.edu            return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
6512137Sar4jc@virginia.edu          case 0x7:
6612137Sar4jc@virginia.edu            return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
6712137Sar4jc@virginia.edu          case 0x8:
6812137Sar4jc@virginia.edu            return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, rm);
6912137Sar4jc@virginia.edu          case 0x9:
7012137Sar4jc@virginia.edu            return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, rm);
7112137Sar4jc@virginia.edu          case 0xa:
7212137Sar4jc@virginia.edu            return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
7312137Sar4jc@virginia.edu          case 0xb:
7412137Sar4jc@virginia.edu            return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
7512137Sar4jc@virginia.edu          case 0xc:
7612137Sar4jc@virginia.edu            return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, rm);
7712137Sar4jc@virginia.edu          case 0xd:
7812137Sar4jc@virginia.edu            return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, rm);
7912137Sar4jc@virginia.edu          case 0xe:
8012137Sar4jc@virginia.edu            return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, rm);
8112137Sar4jc@virginia.edu          case 0xf:
8212137Sar4jc@virginia.edu            return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm);
8312137Sar4jc@virginia.edu        }
8412137Sar4jc@virginia.edu        return new Unknown64(machInst);
8512137Sar4jc@virginia.edu    }
8612137Sar4jc@virginia.edu
8712137Sar4jc@virginia.edu    // Decodes SVE contiguous load instructions, scalar plus immediate form.
8812137Sar4jc@virginia.edu    template <template <typename T1, typename T2> class Base>
8912137Sar4jc@virginia.edu    StaticInstPtr
9012137Sar4jc@virginia.edu    decodeSveContigLoadSIInsts(uint8_t dtype, ExtMachInst machInst,
9112137Sar4jc@virginia.edu                               IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
9212137Sar4jc@virginia.edu                               uint64_t imm, bool nonFaulting,
9312137Sar4jc@virginia.edu                               bool replicate = false)
9412137Sar4jc@virginia.edu    {
9512137Sar4jc@virginia.edu        assert(!(nonFaulting && replicate));
9612137Sar4jc@virginia.edu        const char* mn = replicate ? "ld1r" : (nonFaulting ? "ldnf1" : "ld1");
9712137Sar4jc@virginia.edu        switch (dtype) {
9812137Sar4jc@virginia.edu          case 0x0:
9912137Sar4jc@virginia.edu            return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
10012137Sar4jc@virginia.edu          case 0x1:
10112137Sar4jc@virginia.edu            return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
10212137Sar4jc@virginia.edu          case 0x2:
10312137Sar4jc@virginia.edu            return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
10412137Sar4jc@virginia.edu          case 0x3:
10512137Sar4jc@virginia.edu            return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
10612137Sar4jc@virginia.edu          case 0x4:
10712137Sar4jc@virginia.edu            return new Base<int64_t, int32_t>(mn, machInst, zt, pg, rn, imm);
10812137Sar4jc@virginia.edu          case 0x5:
10912137Sar4jc@virginia.edu            return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
11012137Sar4jc@virginia.edu          case 0x6:
11112137Sar4jc@virginia.edu            return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
11212137Sar4jc@virginia.edu          case 0x7:
11312137Sar4jc@virginia.edu            return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
11412137Sar4jc@virginia.edu          case 0x8:
11512137Sar4jc@virginia.edu            return new Base<int64_t, int16_t>(mn, machInst, zt, pg, rn, imm);
11612137Sar4jc@virginia.edu          case 0x9:
11712137Sar4jc@virginia.edu            return new Base<int32_t, int16_t>(mn, machInst, zt, pg, rn, imm);
11812137Sar4jc@virginia.edu          case 0xa:
11912137Sar4jc@virginia.edu            return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
12012137Sar4jc@virginia.edu          case 0xb:
12112137Sar4jc@virginia.edu            return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
12212137Sar4jc@virginia.edu          case 0xc:
12312137Sar4jc@virginia.edu            return new Base<int64_t, int8_t>(mn, machInst, zt, pg, rn, imm);
12412137Sar4jc@virginia.edu          case 0xd:
12512137Sar4jc@virginia.edu            return new Base<int32_t, int8_t>(mn, machInst, zt, pg, rn, imm);
12612137Sar4jc@virginia.edu          case 0xe:
12712137Sar4jc@virginia.edu            return new Base<int16_t, int8_t>(mn, machInst, zt, pg, rn, imm);
12812137Sar4jc@virginia.edu          case 0xf:
12912137Sar4jc@virginia.edu            return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm);
13012137Sar4jc@virginia.edu        }
13112137Sar4jc@virginia.edu        return new Unknown64(machInst);
13212137Sar4jc@virginia.edu    }
13312137Sar4jc@virginia.edu
13412137Sar4jc@virginia.edu    // Decodes SVE contiguous store instructions, scalar plus scalar form.
13512137Sar4jc@virginia.edu    template <template <typename T1, typename T2> class Base>
13612137Sar4jc@virginia.edu    StaticInstPtr
13712137Sar4jc@virginia.edu    decodeSveContigStoreSSInsts(uint8_t dtype, ExtMachInst machInst,
13812137Sar4jc@virginia.edu                                IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
13912137Sar4jc@virginia.edu                                IntRegIndex rm)
14012137Sar4jc@virginia.edu    {
14112137Sar4jc@virginia.edu        const char* mn = "st1";
14212137Sar4jc@virginia.edu        switch (dtype) {
14312137Sar4jc@virginia.edu          case 0x0:
14412137Sar4jc@virginia.edu            return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
14512137Sar4jc@virginia.edu          case 0x1:
14612137Sar4jc@virginia.edu            return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
14712137Sar4jc@virginia.edu          case 0x2:
14812137Sar4jc@virginia.edu            return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
14912137Sar4jc@virginia.edu          case 0x3:
15012137Sar4jc@virginia.edu            return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, rm);
15112137Sar4jc@virginia.edu          case 0x5:
15212137Sar4jc@virginia.edu            return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
15312137Sar4jc@virginia.edu          case 0x6:
15412137Sar4jc@virginia.edu            return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
15512137Sar4jc@virginia.edu          case 0x7:
15612137Sar4jc@virginia.edu            return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, rm);
15712137Sar4jc@virginia.edu          case 0xa:
15812137Sar4jc@virginia.edu            return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
15912137Sar4jc@virginia.edu          case 0xb:
16012137Sar4jc@virginia.edu            return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, rm);
16112137Sar4jc@virginia.edu          case 0xf:
16212137Sar4jc@virginia.edu            return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, rm);
16312137Sar4jc@virginia.edu        }
16412137Sar4jc@virginia.edu        return new Unknown64(machInst);
16512137Sar4jc@virginia.edu    }
16612137Sar4jc@virginia.edu
16712137Sar4jc@virginia.edu    // Decodes SVE contiguous store instructions, scalar plus immediate form.
16812137Sar4jc@virginia.edu    template <template <typename T1, typename T2> class Base>
16912137Sar4jc@virginia.edu    StaticInstPtr
17012137Sar4jc@virginia.edu    decodeSveContigStoreSIInsts(uint8_t dtype, ExtMachInst machInst,
17112137Sar4jc@virginia.edu                                IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
17212137Sar4jc@virginia.edu                                int8_t imm)
17312137Sar4jc@virginia.edu    {
17412137Sar4jc@virginia.edu        const char* mn = "st1";
17512137Sar4jc@virginia.edu        switch (dtype) {
17612137Sar4jc@virginia.edu          case 0x0:
17712137Sar4jc@virginia.edu            return new Base<uint8_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
17812137Sar4jc@virginia.edu          case 0x1:
17912137Sar4jc@virginia.edu            return new Base<uint16_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
18012137Sar4jc@virginia.edu          case 0x2:
18112137Sar4jc@virginia.edu            return new Base<uint32_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
18212137Sar4jc@virginia.edu          case 0x3:
18312137Sar4jc@virginia.edu            return new Base<uint64_t, uint8_t>(mn, machInst, zt, pg, rn, imm);
18412137Sar4jc@virginia.edu          case 0x5:
18512137Sar4jc@virginia.edu            return new Base<uint16_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
18612137Sar4jc@virginia.edu          case 0x6:
18712137Sar4jc@virginia.edu            return new Base<uint32_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
18812137Sar4jc@virginia.edu          case 0x7:
18912137Sar4jc@virginia.edu            return new Base<uint64_t, uint16_t>(mn, machInst, zt, pg, rn, imm);
19012137Sar4jc@virginia.edu          case 0xa:
19112137Sar4jc@virginia.edu            return new Base<uint32_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
19212137Sar4jc@virginia.edu          case 0xb:
19312137Sar4jc@virginia.edu            return new Base<uint64_t, uint32_t>(mn, machInst, zt, pg, rn, imm);
19412137Sar4jc@virginia.edu          case 0xf:
19512137Sar4jc@virginia.edu            return new Base<uint64_t, uint64_t>(mn, machInst, zt, pg, rn, imm);
19612137Sar4jc@virginia.edu        }
19712137Sar4jc@virginia.edu        return new Unknown64(machInst);
19812137Sar4jc@virginia.edu    }
19912137Sar4jc@virginia.edu
20012137Sar4jc@virginia.edu    // NOTE: SVE load-and-replicate instructions are decoded with
20112137Sar4jc@virginia.edu    // decodeSveContigLoadSIInsts(...).
20212137Sar4jc@virginia.edu
20312137Sar4jc@virginia.edu}};
20412137Sar4jc@virginia.edu
20512137Sar4jc@virginia.eduoutput decoder {{
20612137Sar4jc@virginia.edu
20712137Sar4jc@virginia.edu    template <class etype>
20812137Sar4jc@virginia.edu    StaticInstPtr
20912137Sar4jc@virginia.edu    decodeSveStructLoadSIInstsByNReg(uint8_t esize, ExtMachInst machInst,
21012137Sar4jc@virginia.edu            IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
21112137Sar4jc@virginia.edu            int64_t imm, int numregs)
21212137Sar4jc@virginia.edu    {
21312137Sar4jc@virginia.edu        static const char* nm[5][4] = {
21412137Sar4jc@virginia.edu            { nullptr, nullptr, nullptr, nullptr},
215            { nullptr, nullptr, nullptr, nullptr},
216            { "ld2b", "ld2h", "ld2w", "ld2d" },
217            { "ld3b", "ld3h", "ld3w", "ld3d" },
218            { "ld4b", "ld4h", "ld4w", "ld4d" } };
219
220        switch (numregs) {
221            case 2:
222                return new SveLdStructSI<etype,
223                        SveLoadRegImmMicroop,
224                        SveDeIntrlv2Microop>(
225                               nm[numregs][esize], machInst, MemReadOp,
226                               zt, pg, xn, imm, numregs);
227            case 3:
228                return new SveLdStructSI<etype,
229                        SveLoadRegImmMicroop,
230                        SveDeIntrlv3Microop>(
231                               nm[numregs][esize], machInst, MemReadOp,
232                               zt, pg, xn, imm, numregs);
233            case 4:
234                return new SveLdStructSI<etype,
235                        SveLoadRegImmMicroop,
236                        SveDeIntrlv4Microop>(
237                               nm[numregs][esize], machInst, MemReadOp,
238                               zt, pg, xn, imm, numregs);
239        }
240        return new Unknown64(machInst);
241    }
242
243    StaticInstPtr
244    decodeSveStructLoadSIInsts(uint8_t esize, ExtMachInst machInst,
245            IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
246            int64_t imm, int numregs)
247    {
248        switch (esize) {
249            case 0:
250                return decodeSveStructLoadSIInstsByNReg<uint8_t>(esize,
251                        machInst, zt, pg, xn, imm, numregs);
252            case 1:
253                return decodeSveStructLoadSIInstsByNReg<uint16_t>(esize,
254                        machInst, zt, pg, xn, imm, numregs);
255            case 2:
256                return decodeSveStructLoadSIInstsByNReg<uint32_t>(esize,
257                        machInst, zt, pg, xn, imm, numregs);
258            case 3:
259                return decodeSveStructLoadSIInstsByNReg<uint64_t>(esize,
260                        machInst, zt, pg, xn, imm, numregs);
261        }
262        return new Unknown64(machInst);
263    }
264
265    template <class etype>
266    StaticInstPtr
267    decodeSveStructStoreSIInstsByNReg(uint8_t esize, ExtMachInst machInst,
268            IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
269            int64_t imm, int numregs)
270    {
271        static const char* nm[5][4] = {
272            { nullptr, nullptr, nullptr, nullptr},
273            { nullptr, nullptr, nullptr, nullptr},
274            { "st2b", "st2h", "st2w", "st2d" },
275            { "st3b", "st3h", "st3w", "st3d" },
276            { "st4b", "st4h", "st4w", "st4d" } };
277
278        switch (numregs) {
279            case 2:
280                return new SveStStructSI<etype,
281                        SveStoreRegImmMicroop,
282                        SveIntrlv2Microop>(
283                            nm[numregs][esize], machInst, MemWriteOp,
284                            zt, pg, xn, imm, numregs);
285            case 3:
286                return new SveStStructSI<etype,
287                        SveStoreRegImmMicroop,
288                        SveIntrlv3Microop>(
289                            nm[numregs][esize], machInst, MemWriteOp,
290                            zt, pg, xn, imm, numregs);
291           case 4:
292                return new SveStStructSI<etype,
293                        SveStoreRegImmMicroop,
294                        SveIntrlv4Microop>(
295                            nm[numregs][esize], machInst, MemWriteOp,
296                            zt, pg, xn, imm, numregs);
297        }
298        return new Unknown64(machInst);
299    }
300
301    StaticInstPtr
302    decodeSveStructStoreSIInsts(uint8_t esize, ExtMachInst machInst,
303            IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
304            int64_t imm, int numregs)
305    {
306        switch (esize) {
307            case 0:
308                return decodeSveStructStoreSIInstsByNReg<uint8_t>(esize,
309                    machInst, zt, pg, xn, imm, numregs);
310            case 1:
311                return decodeSveStructStoreSIInstsByNReg<uint16_t>(esize,
312                    machInst, zt, pg, xn, imm, numregs);
313            case 2:
314                return decodeSveStructStoreSIInstsByNReg<uint32_t>(esize,
315                    machInst, zt, pg, xn, imm, numregs);
316            case 3:
317                return decodeSveStructStoreSIInstsByNReg<uint64_t>(esize,
318                    machInst, zt, pg, xn, imm, numregs);
319        }
320        return new Unknown64(machInst);
321    }
322
323    template <class etype>
324    StaticInstPtr
325    decodeSveStructLoadSSInstsByNReg(uint8_t esize, ExtMachInst machInst,
326            IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
327            IntRegIndex xm, int numregs)
328    {
329        static const char* nm[5][4] = {
330            { nullptr, nullptr, nullptr, nullptr},
331            { nullptr, nullptr, nullptr, nullptr},
332            { "ld2b", "ld2h", "ld2w", "ld2d" },
333            { "ld3b", "ld3h", "ld3w", "ld3d" },
334            { "ld4b", "ld4h", "ld4w", "ld4d" } };
335
336        switch (numregs) {
337            case 2:
338                return new SveLdStructSS<etype,
339                        SveLoadRegRegMicroop,
340                        SveDeIntrlv2Microop>(
341                               nm[numregs][esize], machInst, MemReadOp,
342                               zt, pg, xn, xm, numregs);
343            case 3:
344                return new SveLdStructSS<etype,
345                        SveLoadRegRegMicroop,
346                        SveDeIntrlv3Microop>(
347                               nm[numregs][esize], machInst, MemReadOp,
348                               zt, pg, xn, xm, numregs);
349            case 4:
350                return new SveLdStructSS<etype,
351                        SveLoadRegRegMicroop,
352                        SveDeIntrlv4Microop>(
353                               nm[numregs][esize], machInst, MemReadOp,
354                               zt, pg, xn, xm, numregs);
355        }
356        return new Unknown64(machInst);
357    }
358
359    StaticInstPtr
360    decodeSveStructLoadSSInsts(uint8_t esize, ExtMachInst machInst,
361            IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
362            IntRegIndex xm, int numregs)
363    {
364        switch (esize) {
365            case 0:
366                return decodeSveStructLoadSSInstsByNReg<uint8_t>(esize,
367                            machInst, zt, pg, xn, xm, numregs);
368            case 1:
369                return decodeSveStructLoadSSInstsByNReg<uint16_t>(esize,
370                            machInst, zt, pg, xn, xm, numregs);
371            case 2:
372                return decodeSveStructLoadSSInstsByNReg<uint32_t>(esize,
373                            machInst, zt, pg, xn, xm, numregs);
374            case 3:
375                return decodeSveStructLoadSSInstsByNReg<uint64_t>(esize,
376                            machInst, zt, pg, xn, xm, numregs);
377        }
378        return new Unknown64(machInst);
379    }
380
381    template <class etype>
382    StaticInstPtr
383    decodeSveStructStoreSSInstsByNReg(uint8_t esize, ExtMachInst machInst,
384            IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
385            IntRegIndex xm, int numregs)
386    {
387        static const char* nm[5][4] = {
388            { nullptr, nullptr, nullptr, nullptr},
389            { nullptr, nullptr, nullptr, nullptr},
390            { "st2b", "st2h", "st2w", "st2d" },
391            { "st3b", "st3h", "st3w", "st3d" },
392            { "st4b", "st4h", "st4w", "st4d" } };
393
394        switch (numregs) {
395            case 2:
396                return new SveStStructSS<etype,
397                        SveStoreRegRegMicroop,
398                        SveIntrlv2Microop>(
399                               nm[numregs][esize], machInst, MemWriteOp,
400                               zt, pg, xn, xm, numregs);
401            case 3:
402                return new SveStStructSS<etype,
403                        SveStoreRegRegMicroop,
404                        SveIntrlv3Microop>(
405                               nm[numregs][esize], machInst, MemWriteOp,
406                               zt, pg, xn, xm, numregs);
407            case 4:
408                return new SveStStructSS<etype,
409                        SveStoreRegRegMicroop,
410                        SveIntrlv4Microop>(
411                               nm[numregs][esize], machInst, MemWriteOp,
412                               zt, pg, xn, xm, numregs);
413        }
414        return new Unknown64(machInst);
415    }
416
417    StaticInstPtr
418    decodeSveStructStoreSSInsts(uint8_t esize, ExtMachInst machInst,
419            IntRegIndex zt, IntRegIndex pg, IntRegIndex xn,
420            IntRegIndex xm, int numregs)
421    {
422        switch (esize) {
423            case 0:
424                return decodeSveStructStoreSSInstsByNReg<uint8_t>(esize,
425                            machInst, zt, pg, xn, xm, numregs);
426            case 1:
427                return decodeSveStructStoreSSInstsByNReg<uint16_t>(esize,
428                            machInst, zt, pg, xn, xm, numregs);
429            case 2:
430                return decodeSveStructStoreSSInstsByNReg<uint32_t>(esize,
431                            machInst, zt, pg, xn, xm, numregs);
432            case 3:
433                return decodeSveStructStoreSSInstsByNReg<uint64_t>(esize,
434                            machInst, zt, pg, xn, xm, numregs);
435        }
436        return new Unknown64(machInst);
437    }
438
439    StaticInstPtr
440    decodeSveGatherLoadVIInsts(uint8_t dtype, ExtMachInst machInst,
441                               IntRegIndex zt, IntRegIndex pg, IntRegIndex zn,
442                               uint64_t imm, bool esizeIs32,
443                               bool firstFault)
444    {
445        const char* mn = firstFault ? "ldff1" : "ld1";
446        switch (dtype) {
447          case 0x0:
448            if (esizeIs32) {
449                return new SveIndexedMemVI<int32_t, int8_t,
450                                           SveGatherLoadVIMicroop,
451                                           SveFirstFaultWritebackMicroop>(
452                    mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
453            } else {
454                return new SveIndexedMemVI<int64_t, int8_t,
455                                           SveGatherLoadVIMicroop,
456                                           SveFirstFaultWritebackMicroop>(
457                    mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
458            }
459          case 0x1:
460            if (esizeIs32) {
461                return new SveIndexedMemVI<uint32_t, uint8_t,
462                                           SveGatherLoadVIMicroop,
463                                           SveFirstFaultWritebackMicroop>(
464                    mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
465            } else {
466                return new SveIndexedMemVI<uint64_t, uint8_t,
467                                           SveGatherLoadVIMicroop,
468                                           SveFirstFaultWritebackMicroop>(
469                    mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
470            }
471          case 0x2:
472            if (esizeIs32) {
473                return new SveIndexedMemVI<int32_t, int16_t,
474                                           SveGatherLoadVIMicroop,
475                                           SveFirstFaultWritebackMicroop>(
476                    mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
477            } else {
478                return new SveIndexedMemVI<int64_t, int16_t,
479                                           SveGatherLoadVIMicroop,
480                                           SveFirstFaultWritebackMicroop>(
481                    mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
482            }
483          case 0x3:
484            if (esizeIs32) {
485                return new SveIndexedMemVI<uint32_t, uint16_t,
486                                           SveGatherLoadVIMicroop,
487                                           SveFirstFaultWritebackMicroop>(
488                    mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
489            } else {
490                return new SveIndexedMemVI<uint64_t, uint16_t,
491                                           SveGatherLoadVIMicroop,
492                                           SveFirstFaultWritebackMicroop>(
493                    mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
494            }
495          case 0x4:
496            if (esizeIs32) {
497                break;
498            } else {
499                return new SveIndexedMemVI<int64_t, int32_t,
500                                           SveGatherLoadVIMicroop,
501                                           SveFirstFaultWritebackMicroop>(
502                    mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
503            }
504          case 0x5:
505            if (esizeIs32) {
506                return new SveIndexedMemVI<uint32_t, uint32_t,
507                                           SveGatherLoadVIMicroop,
508                                           SveFirstFaultWritebackMicroop>(
509                    mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
510            } else {
511                return new SveIndexedMemVI<uint64_t, uint32_t,
512                                           SveGatherLoadVIMicroop,
513                                           SveFirstFaultWritebackMicroop>(
514                    mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
515            }
516          case 0x7:
517            if (esizeIs32) {
518                break;
519            } else {
520                return new SveIndexedMemVI<uint64_t, uint64_t,
521                                           SveGatherLoadVIMicroop,
522                                           SveFirstFaultWritebackMicroop>(
523                    mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
524            }
525        }
526        return new Unknown64(machInst);
527    }
528
529    StaticInstPtr
530    decodeSveGatherLoadSVInsts(uint8_t dtype, ExtMachInst machInst,
531                               IntRegIndex zt, IntRegIndex pg, IntRegIndex rn,
532                               IntRegIndex zm, bool esizeIs32, bool offsetIs32,
533                               bool offsetIsSigned, bool offsetIsScaled,
534                               bool firstFault)
535    {
536        const char* mn = firstFault ? "ldff1" : "ld1";
537        switch (dtype) {
538          case 0x0:
539            if (esizeIs32) {
540                return new SveIndexedMemSV<int32_t, int8_t,
541                                           SveGatherLoadSVMicroop,
542                                           SveFirstFaultWritebackMicroop>(
543                    mn, machInst, MemReadOp, zt, pg, rn, zm,
544                    offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
545            } else {
546                return new SveIndexedMemSV<int64_t, int8_t,
547                                           SveGatherLoadSVMicroop,
548                                           SveFirstFaultWritebackMicroop>(
549                    mn, machInst, MemReadOp, zt, pg, rn, zm,
550                    offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
551            }
552          case 0x1:
553            if (esizeIs32) {
554                return new SveIndexedMemSV<uint32_t, uint8_t,
555                                           SveGatherLoadSVMicroop,
556                                           SveFirstFaultWritebackMicroop>(
557                    mn, machInst, MemReadOp, zt, pg, rn, zm,
558                    offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
559            } else {
560                return new SveIndexedMemSV<uint64_t, uint8_t,
561                                           SveGatherLoadSVMicroop,
562                                           SveFirstFaultWritebackMicroop>(
563                    mn, machInst, MemReadOp, zt, pg, rn, zm,
564                    offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
565            }
566          case 0x2:
567            if (esizeIs32) {
568                return new SveIndexedMemSV<int32_t, int16_t,
569                                           SveGatherLoadSVMicroop,
570                                           SveFirstFaultWritebackMicroop>(
571                    mn, machInst, MemReadOp, zt, pg, rn, zm,
572                    offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
573            } else {
574                return new SveIndexedMemSV<int64_t, int16_t,
575                                           SveGatherLoadSVMicroop,
576                                           SveFirstFaultWritebackMicroop>(
577                    mn, machInst, MemReadOp, zt, pg, rn, zm,
578                    offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
579            }
580          case 0x3:
581            if (esizeIs32) {
582                return new SveIndexedMemSV<uint32_t, uint16_t,
583                                           SveGatherLoadSVMicroop,
584                                           SveFirstFaultWritebackMicroop>(
585                    mn, machInst, MemReadOp, zt, pg, rn, zm,
586                    offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
587            } else {
588                return new SveIndexedMemSV<uint64_t, uint16_t,
589                                           SveGatherLoadSVMicroop,
590                                           SveFirstFaultWritebackMicroop>(
591                    mn, machInst, MemReadOp, zt, pg, rn, zm,
592                    offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
593            }
594          case 0x4:
595            if (esizeIs32) {
596                break;
597            } else {
598                return new SveIndexedMemSV<int64_t, int32_t,
599                                           SveGatherLoadSVMicroop,
600                                           SveFirstFaultWritebackMicroop>(
601                    mn, machInst, MemReadOp, zt, pg, rn, zm,
602                    offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
603            }
604          case 0x5:
605            if (esizeIs32) {
606                return new SveIndexedMemSV<uint32_t, uint32_t,
607                                           SveGatherLoadSVMicroop,
608                                           SveFirstFaultWritebackMicroop>(
609                    mn, machInst, MemReadOp, zt, pg, rn, zm,
610                    offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
611            } else {
612                return new SveIndexedMemSV<uint64_t, uint32_t,
613                                           SveGatherLoadSVMicroop,
614                                           SveFirstFaultWritebackMicroop>(
615                    mn, machInst, MemReadOp, zt, pg, rn, zm,
616                    offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
617            }
618          case 0x7:
619            if (esizeIs32) {
620                break;
621            } else {
622                return new SveIndexedMemSV<uint64_t, uint64_t,
623                                           SveGatherLoadSVMicroop,
624                                           SveFirstFaultWritebackMicroop>(
625                    mn, machInst, MemReadOp, zt, pg, rn, zm,
626                    offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
627            }
628        }
629        return new Unknown64(machInst);
630    }
631
632    StaticInstPtr
633    decodeSveScatterStoreVIInsts(uint8_t msz, ExtMachInst machInst,
634                                 IntRegIndex zt, IntRegIndex pg,
635                                 IntRegIndex zn, uint64_t imm,
636                                 bool esizeIs32)
637    {
638        const char* mn = "st1";
639        switch (msz) {
640          case 0x0:
641            if (esizeIs32) {
642                return new SveIndexedMemVI<uint32_t, uint8_t,
643                                           SveScatterStoreVIMicroop,
644                                           SveFirstFaultWritebackMicroop>(
645                    mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
646            } else {
647                return new SveIndexedMemVI<uint64_t, uint8_t,
648                                           SveScatterStoreVIMicroop,
649                                           SveFirstFaultWritebackMicroop>(
650                    mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
651            }
652          case 0x1:
653            if (esizeIs32) {
654                return new SveIndexedMemVI<uint32_t, uint16_t,
655                                           SveScatterStoreVIMicroop,
656                                           SveFirstFaultWritebackMicroop>(
657                    mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
658            } else {
659                return new SveIndexedMemVI<uint64_t, uint16_t,
660                                           SveScatterStoreVIMicroop,
661                                           SveFirstFaultWritebackMicroop>(
662                    mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
663            }
664          case 0x2:
665            if (esizeIs32) {
666                return new SveIndexedMemVI<uint32_t, uint32_t,
667                                           SveScatterStoreVIMicroop,
668                                           SveFirstFaultWritebackMicroop>(
669                    mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
670            } else {
671                return new SveIndexedMemVI<uint64_t, uint32_t,
672                                           SveScatterStoreVIMicroop,
673                                           SveFirstFaultWritebackMicroop>(
674                    mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
675            }
676          case 0x3:
677            if (esizeIs32) {
678                break;
679            } else {
680                return new SveIndexedMemVI<uint64_t, uint64_t,
681                                           SveScatterStoreVIMicroop,
682                                           SveFirstFaultWritebackMicroop>(
683                    mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
684            }
685        }
686        return new Unknown64(machInst);
687    }
688
689    StaticInstPtr
690    decodeSveScatterStoreSVInsts(uint8_t msz, ExtMachInst machInst,
691                                 IntRegIndex zt, IntRegIndex pg,
692                                 IntRegIndex rn, IntRegIndex zm,
693                                 bool esizeIs32, bool offsetIs32,
694                                 bool offsetIsSigned, bool offsetIsScaled)
695    {
696        const char* mn = "st1";
697        switch (msz) {
698          case 0x0:
699            if (esizeIs32) {
700                return new SveIndexedMemSV<uint32_t, uint8_t,
701                                           SveScatterStoreSVMicroop,
702                                           SveFirstFaultWritebackMicroop>(
703                    mn, machInst, MemWriteOp, zt, pg, rn, zm,
704                    offsetIs32, offsetIsSigned, offsetIsScaled, false);
705            } else {
706                return new SveIndexedMemSV<uint64_t, uint8_t,
707                                           SveScatterStoreSVMicroop,
708                                           SveFirstFaultWritebackMicroop>(
709                    mn, machInst, MemWriteOp, zt, pg, rn, zm,
710                    offsetIs32, offsetIsSigned, offsetIsScaled, false);
711            }
712          case 0x1:
713            if (esizeIs32) {
714                return new SveIndexedMemSV<uint32_t, uint16_t,
715                                           SveScatterStoreSVMicroop,
716                                           SveFirstFaultWritebackMicroop>(
717                    mn, machInst, MemWriteOp, zt, pg, rn, zm,
718                    offsetIs32, offsetIsSigned, offsetIsScaled, false);
719            } else {
720                return new SveIndexedMemSV<uint64_t, uint16_t,
721                                           SveScatterStoreSVMicroop,
722                                           SveFirstFaultWritebackMicroop>(
723                    mn, machInst, MemWriteOp, zt, pg, rn, zm,
724                    offsetIs32, offsetIsSigned, offsetIsScaled, false);
725            }
726          case 0x2:
727            if (esizeIs32) {
728                return new SveIndexedMemSV<uint32_t, uint32_t,
729                                           SveScatterStoreSVMicroop,
730                                           SveFirstFaultWritebackMicroop>(
731                    mn, machInst, MemWriteOp, zt, pg, rn, zm,
732                    offsetIs32, offsetIsSigned, offsetIsScaled, false);
733            } else {
734                return new SveIndexedMemSV<uint64_t, uint32_t,
735                                           SveScatterStoreSVMicroop,
736                                           SveFirstFaultWritebackMicroop>(
737                    mn, machInst, MemWriteOp, zt, pg, rn, zm,
738                    offsetIs32, offsetIsSigned, offsetIsScaled, false);
739            }
740          case 0x3:
741            if (esizeIs32) {
742                break;
743            } else {
744                return new SveIndexedMemSV<uint64_t, uint64_t,
745                                           SveScatterStoreSVMicroop,
746                                           SveFirstFaultWritebackMicroop>(
747                    mn, machInst, MemWriteOp, zt, pg, rn, zm,
748                    offsetIs32, offsetIsSigned, offsetIsScaled, false);
749            }
750        }
751        return new Unknown64(machInst);
752    }
753
754}};
755
756
757let {{
758
759    header_output = ''
760    exec_output = ''
761    decoders = { 'Generic': {} }
762
763    SPAlignmentCheckCode = '''
764        if (this->baseIsSP && bits(XBase, 3, 0) &&
765            SPAlignmentCheckEnabled(xc->tcBase())) {
766            return std::make_shared<SPAlignmentFault>();
767        }
768    '''
769
770    def emitSveMemFillSpill(isPred):
771        global header_output, exec_output, decoders
772        eaCode = SPAlignmentCheckCode + '''
773        int memAccessSize = %(memacc_size)s;
774        EA = XBase + ((int64_t) imm * %(memacc_size)s)''' % {
775            'memacc_size': 'eCount / 8' if isPred else 'eCount'}
776        loadRdEnableCode = '''
777        auto rdEn = std::vector<bool>();
778        '''
779        if isPred:
780            loadMemAccCode = '''
781            int index = 0;
782            uint8_t byte;
783            for (int i = 0; i < eCount / 8; i++) {
784                byte = memDataView[i];
785                for (int j = 0; j < 8; j++, index++) {
786                    PDest_x[index] = (byte >> j) & 1;
787                }
788            }
789            '''
790            storeMemAccCode = '''
791            int index = 0;
792            uint8_t byte;
793            for (int i = 0; i < eCount / 8; i++) {
794                byte = 0;
795                for (int j = 0; j < 8; j++, index++) {
796                    byte |= PDest_x[index] << j;
797                }
798                memDataView[i] = byte;
799            }
800            '''
801            storeWrEnableCode = '''
802            auto wrEn = std::vector<bool>(eCount / 8, true);
803            '''
804        else:
805            loadMemAccCode = '''
806            for (int i = 0; i < eCount; i++) {
807                AA64FpDest_x[i] = memDataView[i];
808            }
809            '''
810            storeMemAccCode = '''
811            for (int i = 0; i < eCount; i++) {
812                memDataView[i] = AA64FpDest_x[i];
813            }
814            '''
815            storeWrEnableCode = '''
816            auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
817            '''
818        loadIop = InstObjParams('ldr',
819            'SveLdrPred' if isPred else 'SveLdrVec',
820            'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill',
821            {'tpl_header': '',
822             'tpl_args': '',
823             'memacc_code': loadMemAccCode,
824             'ea_code' : sveEnabledCheckCode + eaCode,
825             'rden_code' : loadRdEnableCode,
826             'fault_code' : '',
827             'fa_code' : ''},
828            ['IsMemRef', 'IsLoad'])
829        storeIop = InstObjParams('str',
830            'SveStrPred' if isPred else 'SveStrVec',
831            'SveMemPredFillSpill' if isPred else 'SveMemVecFillSpill',
832            {'tpl_header': '',
833             'tpl_args': '',
834             'wren_code': storeWrEnableCode,
835             'memacc_code': storeMemAccCode,
836             'ea_code' : sveEnabledCheckCode + eaCode,
837             'fa_code' : ''},
838            ['IsMemRef', 'IsStore'])
839        header_output += SveMemFillSpillOpDeclare.subst(loadIop)
840        header_output += SveMemFillSpillOpDeclare.subst(storeIop)
841        exec_output += (
842            SveContigLoadExecute.subst(loadIop) +
843            SveContigLoadInitiateAcc.subst(loadIop) +
844            SveContigLoadCompleteAcc.subst(loadIop) +
845            SveContigStoreExecute.subst(storeIop) +
846            SveContigStoreInitiateAcc.subst(storeIop) +
847            SveContigStoreCompleteAcc.subst(storeIop))
848
849    loadTplArgs = (
850        ('uint8_t', 'uint8_t'),
851        ('uint16_t', 'uint8_t'),
852        ('uint32_t', 'uint8_t'),
853        ('uint64_t', 'uint8_t'),
854        ('int64_t', 'int32_t'),
855        ('uint16_t', 'uint16_t'),
856        ('uint32_t', 'uint16_t'),
857        ('uint64_t', 'uint16_t'),
858        ('int64_t', 'int16_t'),
859        ('int32_t', 'int16_t'),
860        ('uint32_t', 'uint32_t'),
861        ('uint64_t', 'uint32_t'),
862        ('int64_t', 'int8_t'),
863        ('int32_t', 'int8_t'),
864        ('int16_t', 'int8_t'),
865        ('uint64_t', 'uint64_t'),
866    )
867
868    storeTplArgs = (
869        ('uint8_t', 'uint8_t'),
870        ('uint16_t', 'uint8_t'),
871        ('uint32_t', 'uint8_t'),
872        ('uint64_t', 'uint8_t'),
873        ('uint16_t', 'uint16_t'),
874        ('uint32_t', 'uint16_t'),
875        ('uint64_t', 'uint16_t'),
876        ('uint32_t', 'uint32_t'),
877        ('uint64_t', 'uint32_t'),
878        ('uint64_t', 'uint64_t'),
879    )
880
881    gatherLoadTplArgs = (
882        ('int32_t', 'int8_t'),
883        ('int64_t', 'int8_t'),
884        ('uint32_t', 'uint8_t'),
885        ('uint64_t', 'uint8_t'),
886        ('int32_t', 'int16_t'),
887        ('int64_t', 'int16_t'),
888        ('uint32_t', 'uint16_t'),
889        ('uint64_t', 'uint16_t'),
890        ('int64_t', 'int32_t'),
891        ('uint32_t', 'uint32_t'),
892        ('uint64_t', 'uint32_t'),
893        ('uint64_t', 'uint64_t'),
894    )
895
896    scatterStoreTplArgs = (
897        ('uint32_t', 'uint8_t'),
898        ('uint64_t', 'uint8_t'),
899        ('uint32_t', 'uint16_t'),
900        ('uint64_t', 'uint16_t'),
901        ('uint32_t', 'uint32_t'),
902        ('uint64_t', 'uint32_t'),
903        ('uint64_t', 'uint64_t'),
904    )
905
906    # Generates definitions for SVE contiguous loads
907    def emitSveContigMemInsts(offsetIsImm):
908        global header_output, exec_output, decoders
909        # First-faulting instructions only have a scalar plus scalar form,
910        # while non-faulting instructions only a scalar plus immediate form, so
911        # `offsetIsImm` is used to determine which class of instructions is
912        # generated
913        firstFaulting = not offsetIsImm
914        tplHeader = 'template <class RegElemType, class MemElemType>'
915        tplArgs = '<RegElemType, MemElemType>'
916        eaCode = SPAlignmentCheckCode + '''
917        int memAccessSize = eCount * sizeof(MemElemType);
918        EA = XBase + '''
919        if offsetIsImm:
920            eaCode += '((int64_t) this->imm * eCount * sizeof(MemElemType))'
921        else:
922            eaCode += '(XOffset * sizeof(MemElemType));'
923        loadRdEnableCode = '''
924        auto rdEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
925        for (int i = 0; i < eCount; i++) {
926            if (!GpOp_x[i]) {
927                for (int j = 0; j < sizeof(MemElemType); j++) {
928                    rdEn[sizeof(MemElemType) * i + j] = false;
929                }
930            }
931        }
932        '''
933        loadMemAccCode = '''
934        for (int i = 0; i < eCount; i++) {
935            if (GpOp_x[i]) {
936                AA64FpDest_x[i] = memDataView[i];
937            } else {
938                AA64FpDest_x[i] = 0;
939            }
940        }
941        '''
942        storeMemAccCode = '''
943        for (int i = 0; i < eCount; i++) {
944            if (GpOp_x[i]) {
945                memDataView[i] = AA64FpDest_x[i];
946            } else {
947                memDataView[i] = 0;
948                for (int j = 0; j < sizeof(MemElemType); j++) {
949                    wrEn[sizeof(MemElemType) * i + j] = false;
950                }
951            }
952        }
953        '''
954        storeWrEnableCode = '''
955        auto wrEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
956        '''
957        ffrReadBackCode = '''
958        auto& firstFaultReg = Ffr;'''
959        fautlingLoadmemAccCode = '''
960        for (int i = 0; i < eCount; i++) {
961            if (GpOp_x[i] && firstFaultReg[i * sizeof(RegElemType)]) {
962                AA64FpDest_x[i] = memDataView[i];
963            } else {
964                AA64FpDest_x[i] = 0;
965            }
966        }
967        '''
968        nonFaultingCode = 'true ||'
969        faultCode = '''
970        Addr fault_addr;
971        if (fault == NoFault || getFaultVAddr(fault, fault_addr)) {
972            unsigned fault_elem_index;
973            if (fault != NoFault) {
974                assert(fault_addr >= EA);
975                fault_elem_index = (fault_addr - EA) / sizeof(MemElemType);
976            } else {
977                fault_elem_index = eCount + 1;
978            }
979            int first_active_index;
980            for (first_active_index = 0;
981                 first_active_index < eCount && !(GpOp_x[first_active_index]);
982                 first_active_index++);
983            if (%s first_active_index < fault_elem_index) {
984                for (int i = 0; i < eCount; i++) {
985                    for (int j = 0; j < sizeof(RegElemType); j++) {
986                        if (i < fault_elem_index) {
987                            Ffr_ub[i * sizeof(RegElemType) + j] = FfrAux_x[i];
988                        } else {
989                            Ffr_ub[i * sizeof(RegElemType) + j] = 0;
990                        }
991                    }
992                }
993                fault = NoFault;
994                if (first_active_index >= fault_elem_index) {
995                    // non-faulting load needs this
996                    xc->setMemAccPredicate(false);
997                }
998            }
999        }
1000        ''' % ('' if firstFaulting else nonFaultingCode)
1001
1002        loadIop = InstObjParams('ld1',
1003            'SveContigLoadSI' if offsetIsImm else 'SveContigLoadSS',
1004            'SveContigMemSI' if offsetIsImm else 'SveContigMemSS',
1005            {'tpl_header': tplHeader,
1006             'tpl_args': tplArgs,
1007             'rden_code' : loadRdEnableCode,
1008             'memacc_code': loadMemAccCode,
1009             'ea_code' : sveEnabledCheckCode + eaCode,
1010             'fault_code' : '',
1011             'fa_code' : ''},
1012            ['IsMemRef', 'IsLoad'])
1013        storeIop = InstObjParams('st1',
1014            'SveContigStoreSI' if offsetIsImm else 'SveContigStoreSS',
1015            'SveContigMemSI' if offsetIsImm else 'SveContigMemSS',
1016            {'tpl_header': tplHeader,
1017             'tpl_args': tplArgs,
1018             'wren_code': storeWrEnableCode,
1019             'memacc_code': storeMemAccCode,
1020             'ea_code' : sveEnabledCheckCode + eaCode,
1021             'fa_code' : ''},
1022            ['IsMemRef', 'IsStore'])
1023        faultIop = InstObjParams('ldff1' if firstFaulting else 'ldnf1',
1024            'SveContigFFLoadSS' if firstFaulting else 'SveContigNFLoadSI',
1025            'SveContigMemSS' if firstFaulting else 'SveContigMemSI',
1026            {'tpl_header': tplHeader,
1027             'tpl_args': tplArgs,
1028             'rden_code' : loadRdEnableCode,
1029             'memacc_code': fautlingLoadmemAccCode,
1030             'ea_code' : sveEnabledCheckCode + eaCode,
1031             'fault_code' : faultCode,
1032             'fa_code' : ''},
1033            ['IsMemRef', 'IsLoad'])
1034        faultIop.snippets['memacc_code'] = (ffrReadBackCode +
1035                                           faultIop.snippets['memacc_code'])
1036        if offsetIsImm:
1037            header_output += SveContigMemSIOpDeclare.subst(loadIop)
1038            header_output += SveContigMemSIOpDeclare.subst(storeIop)
1039            header_output += SveContigMemSIOpDeclare.subst(faultIop)
1040        else:
1041            header_output += SveContigMemSSOpDeclare.subst(loadIop)
1042            header_output += SveContigMemSSOpDeclare.subst(storeIop)
1043            header_output += SveContigMemSSOpDeclare.subst(faultIop)
1044        exec_output += (
1045            SveContigLoadExecute.subst(loadIop) +
1046            SveContigLoadInitiateAcc.subst(loadIop) +
1047            SveContigLoadCompleteAcc.subst(loadIop) +
1048            SveContigStoreExecute.subst(storeIop) +
1049            SveContigStoreInitiateAcc.subst(storeIop) +
1050            SveContigStoreCompleteAcc.subst(storeIop) +
1051            SveContigLoadExecute.subst(faultIop) +
1052            SveContigLoadInitiateAcc.subst(faultIop) +
1053            SveContigLoadCompleteAcc.subst(faultIop))
1054
1055        for args in loadTplArgs:
1056            substDict = {'tpl_args': '<%s>' % ', '.join(args),
1057                         'class_name': 'SveContigLoadSI' if offsetIsImm
1058                                       else 'SveContigLoadSS'}
1059            exec_output += SveContigMemExecDeclare.subst(substDict)
1060        for args in storeTplArgs:
1061            substDict = {'tpl_args': '<%s>' % ', '.join(args),
1062                         'class_name': 'SveContigStoreSI' if offsetIsImm
1063                                       else 'SveContigStoreSS'}
1064            exec_output += SveContigMemExecDeclare.subst(substDict)
1065        for args in loadTplArgs:
1066            substDict = {'tpl_args': '<%s>' % ', '.join(args),
1067                         'class_name': 'SveContigFFLoadSS' if firstFaulting
1068                                       else 'SveContigNFLoadSI'}
1069            exec_output += SveContigMemExecDeclare.subst(substDict)
1070
1071
1072    # Generates definitions for SVE load-and-replicate instructions
1073    def emitSveLoadAndRepl():
1074        global header_output, exec_output, decoders
1075        tplHeader = 'template <class RegElemType, class MemElemType>'
1076        tplArgs = '<RegElemType, MemElemType>'
1077        eaCode = SPAlignmentCheckCode + '''
1078        EA = XBase + imm * sizeof(MemElemType);'''
1079        memAccCode = '''
1080        for (int i = 0; i < eCount; i++) {
1081            if (GpOp_x[i]) {
1082                AA64FpDest_x[i] = memData;
1083            } else {
1084                AA64FpDest_x[i] = 0;
1085            }
1086        }
1087        '''
1088        iop = InstObjParams('ld1r',
1089            'SveLoadAndRepl',
1090            'SveContigMemSI',
1091            {'tpl_header': tplHeader,
1092             'tpl_args': tplArgs,
1093             'memacc_code': memAccCode,
1094             'ea_code' : sveEnabledCheckCode + eaCode,
1095             'fa_code' : ''},
1096            ['IsMemRef', 'IsLoad'])
1097        header_output += SveContigMemSIOpDeclare.subst(iop)
1098        exec_output += (
1099            SveLoadAndReplExecute.subst(iop) +
1100            SveLoadAndReplInitiateAcc.subst(iop) +
1101            SveLoadAndReplCompleteAcc.subst(iop))
1102        for args in loadTplArgs:
1103            substDict = {'tpl_args': '<%s>' % ', '.join(args),
1104                         'class_name': 'SveLoadAndRepl'}
1105            exec_output += SveContigMemExecDeclare.subst(substDict)
1106
1107    class IndexedAddrForm:
1108        VEC_PLUS_IMM = 0
1109        SCA_PLUS_VEC = 1
1110
1111    # Generates definitions for the transfer microops of SVE indexed memory
1112    # operations (gather loads, scatter stores)
1113    def emitSveIndexedMemMicroops(indexed_addr_form):
1114        assert indexed_addr_form in (IndexedAddrForm.VEC_PLUS_IMM,
1115                                     IndexedAddrForm.SCA_PLUS_VEC)
1116        global header_output, exec_output, decoders
1117        tplHeader = 'template <class RegElemType, class MemElemType>'
1118        tplArgs = '<RegElemType, MemElemType>'
1119        if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM:
1120            eaCode = '''
1121        EA = AA64FpBase_x[elemIndex] + imm * sizeof(MemElemType)'''
1122        else:
1123            eaCode = '''
1124        uint64_t offset = AA64FpOffset_x[elemIndex];
1125        if (offsetIs32) {
1126            offset &= (1ULL << 32) - 1;
1127        }
1128        if (offsetIsSigned) {
1129            offset = sext<32>(offset);
1130        }
1131        if (offsetIsScaled) {
1132            offset *= sizeof(MemElemType);
1133        }
1134        EA = XBase + offset'''
1135        loadMemAccCode = '''
1136            AA64FpDest_x[elemIndex] = memData;
1137        '''
1138        storeMemAccCode = '''
1139            memData = AA64FpDest_x[elemIndex];
1140        '''
1141        predCheckCode = 'GpOp_x[index]'
1142        faultStatusSetCode = 'PUreg0_x[elemIndex] = 1;'
1143        faultStatusResetCode = 'PUreg0_x[elemIndex] = 0;'
1144        loadIop = InstObjParams('ld1',
1145            ('SveGatherLoadVIMicroop'
1146             if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM
1147             else 'SveGatherLoadSVMicroop'),
1148            'MicroOp',
1149            {'tpl_header': tplHeader,
1150             'tpl_args': tplArgs,
1151             'memacc_code': loadMemAccCode,
1152             'ea_code' : sveEnabledCheckCode + eaCode,
1153             'fault_status_set_code' : faultStatusSetCode,
1154             'fault_status_reset_code' : faultStatusResetCode,
1155             'pred_check_code' : predCheckCode,
1156             'fa_code' : ''},
1157            ['IsMicroop', 'IsMemRef', 'IsLoad'])
1158        storeIop = InstObjParams('st1',
1159            ('SveScatterStoreVIMicroop'
1160             if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM
1161             else 'SveScatterStoreSVMicroop'),
1162            'MicroOp',
1163            {'tpl_header': tplHeader,
1164             'tpl_args': tplArgs,
1165             'memacc_code': storeMemAccCode,
1166             'ea_code' : sveEnabledCheckCode + eaCode,
1167             'pred_check_code' : predCheckCode,
1168             'fa_code' : ''},
1169            ['IsMicroop', 'IsMemRef', 'IsStore'])
1170        if indexed_addr_form == IndexedAddrForm.VEC_PLUS_IMM:
1171            header_output += SveIndexedMemVIMicroopDeclare.subst(loadIop)
1172            header_output += SveIndexedMemVIMicroopDeclare.subst(storeIop)
1173        else:
1174            header_output += SveIndexedMemSVMicroopDeclare.subst(loadIop)
1175            header_output += SveIndexedMemSVMicroopDeclare.subst(storeIop)
1176        exec_output += (
1177            SveGatherLoadMicroopExecute.subst(loadIop) +
1178            SveGatherLoadMicroopInitiateAcc.subst(loadIop) +
1179            SveGatherLoadMicroopCompleteAcc.subst(loadIop) +
1180            SveScatterStoreMicroopExecute.subst(storeIop) +
1181            SveScatterStoreMicroopInitiateAcc.subst(storeIop) +
1182            SveScatterStoreMicroopCompleteAcc.subst(storeIop))
1183        for args in gatherLoadTplArgs:
1184            substDict = {'tpl_args': '<%s>' % ', '.join(args),
1185                         'class_name': (
1186                             'SveGatherLoadVIMicroop'
1187                             if indexed_addr_form == \
1188                                 IndexedAddrForm.VEC_PLUS_IMM
1189                             else 'SveGatherLoadSVMicroop')}
1190            # TODO: this should become SveMemExecDeclare
1191            exec_output += SveContigMemExecDeclare.subst(substDict)
1192        for args in scatterStoreTplArgs:
1193            substDict = {'tpl_args': '<%s>' % ', '.join(args),
1194                         'class_name': (
1195                             'SveScatterStoreVIMicroop'
1196                             if indexed_addr_form == \
1197                                 IndexedAddrForm.VEC_PLUS_IMM
1198                             else 'SveScatterStoreSVMicroop')}
1199            # TODO: this should become SveMemExecDeclare
1200            exec_output += SveContigMemExecDeclare.subst(substDict)
1201
1202    firstFaultTplArgs = ('int32_t', 'int64_t', 'uint32_t', 'uint64_t')
1203
1204    def emitSveFirstFaultWritebackMicroop():
1205        global header_output, exec_output, decoders
1206        tplHeader = 'template <class RegElemType>'
1207        tplArgs = '<RegElemType>'
1208        faultStatusCheckCode = 'PUreg0_x[index]'
1209        firstFaultResetCode = '''
1210        for(int j = 0; j < sizeof(RegElemType); j++) {
1211            Ffr_ub[index * sizeof(RegElemType) + j] = 0;
1212        }
1213        '''
1214        firstFaultForwardCode = '''
1215        for(int j = 0; j < sizeof(RegElemType); j++) {
1216            Ffr_ub[index * sizeof(RegElemType) + j] = FfrAux_x[index];
1217        }
1218        '''
1219        iop = InstObjParams('ldff1',
1220            'SveFirstFaultWritebackMicroop',
1221            'MicroOp',
1222            {'tpl_header': tplHeader,
1223             'tpl_args': tplArgs,
1224             'fault_status_check_code' : faultStatusCheckCode,
1225             'first_fault_reset_code' : firstFaultResetCode,
1226             'first_fault_forward_code' : firstFaultForwardCode},
1227             ['IsMicroop'])
1228        header_output += SveFirstFaultWritebackMicroopDeclare.subst(iop)
1229        exec_output += SveFirstFaultWritebackMicroopExecute.subst(iop)
1230        for args in firstFaultTplArgs:
1231            substDict = {'targs': args,
1232                         'class_name' : 'SveFirstFaultWritebackMicroop' }
1233            exec_output += SveOpExecDeclare.subst(substDict)
1234
1235    # Generates definitions for the first microop of SVE gather loads, required
1236    # to propagate the source vector register to the transfer microops
1237    def emitSveGatherLoadCpySrcVecMicroop():
1238        global header_output, exec_output, decoders
1239        code = sveEnabledCheckCode + '''
1240        unsigned eCount = ArmStaticInst::getCurSveVecLen<uint8_t>(
1241                xc->tcBase());
1242        for (unsigned i = 0; i < eCount; i++) {
1243            AA64FpUreg0_ub[i] = AA64FpOp1_ub[i];
1244        }'''
1245        iop = InstObjParams('ld1',
1246            'SveGatherLoadCpySrcVecMicroop',
1247            'MicroOp',
1248            {'code': code},
1249            ['IsMicroop'])
1250        header_output += SveGatherLoadCpySrcVecMicroopDeclare.subst(iop)
1251        exec_output += SveGatherLoadCpySrcVecMicroopExecute.subst(iop)
1252
1253    def emitSveInterleaveMicroop():
1254        global header_output, exec_output, decoders
1255        code2 = sveEnabledCheckCode + '''
1256        unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1257                xc->tcBase());
1258        for (unsigned int i = 0; i < eCount; ++i) {
1259            unsigned int absIdx = regIndex * eCount + i;
1260            unsigned int srcIdx = absIdx / numRegs;
1261            unsigned int srcVec = absIdx % numRegs;
1262            if (srcVec == 0)
1263                AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx];
1264            else if (srcVec == 1)
1265                AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx];
1266        }'''
1267
1268        code3 = sveEnabledCheckCode + '''
1269        unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1270                xc->tcBase());
1271        for (unsigned int i = 0; i < eCount; ++i) {
1272            unsigned int absIdx = regIndex * eCount + i;
1273            unsigned int srcIdx = absIdx / numRegs;
1274            unsigned int srcVec = absIdx % numRegs;
1275            if (srcVec == 0)
1276                AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx];
1277            else if (srcVec == 1)
1278                AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx];
1279            else if (srcVec == 2)
1280                AA64FpDest_x[i] = AA64FpOp1V2S_x[srcIdx];
1281        }'''
1282
1283        code4 = sveEnabledCheckCode + '''
1284        unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1285                xc->tcBase());
1286        for (unsigned int i = 0; i < eCount; ++i) {
1287            unsigned int absIdx = regIndex * eCount + i;
1288            unsigned int srcIdx = absIdx / numRegs;
1289            unsigned int srcVec = absIdx % numRegs;
1290            if (srcVec == 0)
1291                AA64FpDest_x[i] = AA64FpOp1V0S_x[srcIdx];
1292            else if (srcVec == 1)
1293                AA64FpDest_x[i] = AA64FpOp1V1S_x[srcIdx];
1294            else if (srcVec == 2)
1295                AA64FpDest_x[i] = AA64FpOp1V2S_x[srcIdx];
1296            else if (srcVec == 3)
1297                AA64FpDest_x[i] = AA64FpOp1V3S_x[srcIdx];
1298        }'''
1299
1300        iop2 = InstObjParams('intrlv',
1301                'SveIntrlv2Microop',
1302                'MicroOp',
1303                {'code': code2},
1304                ['IsMicroop'])
1305        iop3 = InstObjParams('intrlv',
1306                'SveIntrlv3Microop',
1307                'MicroOp',
1308                {'code': code3},
1309                ['IsMicroop'])
1310        iop4 = InstObjParams('intrlv',
1311                'SveIntrlv4Microop',
1312                'MicroOp',
1313                {'code': code4},
1314                ['IsMicroop'])
1315        header_output += SveIntrlvMicroopDeclare.subst(iop2);
1316        header_output += SveIntrlvMicroopDeclare.subst(iop3);
1317        header_output += SveIntrlvMicroopDeclare.subst(iop4);
1318        exec_output += SveIntrlvMicroopExecute.subst(iop2);
1319        exec_output += SveIntrlvMicroopExecute.subst(iop3);
1320        exec_output += SveIntrlvMicroopExecute.subst(iop4);
1321        for type in ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'):
1322            for nreg in range(2,5):
1323                substDict = {'targs' : type,
1324                        'class_name' : 'SveIntrlv' + str(nreg) + 'Microop'}
1325                exec_output += SveIntrlvMicroopExecDeclare.subst(substDict)
1326
1327    def emitSveDeInterleaveMicroop():
1328        global header_output, exec_output, decoders
1329        code2 = sveEnabledCheckCode + '''
1330        unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1331                xc->tcBase());
1332        for (unsigned int i = 0; i < eCount; ++i) {
1333            unsigned int absIdx = (regIndex + numRegs * i);
1334            unsigned int srcIdx = absIdx % eCount;
1335            unsigned int srcVec = absIdx / eCount;
1336            if (srcVec == 0)
1337                AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx];
1338            else if(srcVec == 1)
1339                AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx];
1340        }'''
1341
1342        code3 = sveEnabledCheckCode + '''
1343        unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1344                xc->tcBase());
1345        for (unsigned int i = 0; i < eCount; ++i) {
1346            unsigned int absIdx = (regIndex + numRegs * i);
1347            unsigned int srcIdx = absIdx % eCount;
1348            unsigned int srcVec = absIdx / eCount;
1349            if (srcVec == 0)
1350                AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx];
1351            else if(srcVec == 1)
1352                AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx];
1353            else if(srcVec == 2)
1354                AA64FpDest_x[i] = AA64IntrlvReg2_x[srcIdx];
1355        }'''
1356
1357        code4 = sveEnabledCheckCode + '''
1358        unsigned eCount = ArmStaticInst::getCurSveVecLen<Element>(
1359                xc->tcBase());
1360        for (unsigned int i = 0; i < eCount; ++i) {
1361            unsigned int absIdx = (regIndex + numRegs * i);
1362            unsigned int srcIdx = absIdx % eCount;
1363            unsigned int srcVec = absIdx / eCount;
1364            if (srcVec == 0)
1365                AA64FpDest_x[i] = AA64IntrlvReg0_x[srcIdx];
1366            else if(srcVec == 1)
1367                AA64FpDest_x[i] = AA64IntrlvReg1_x[srcIdx];
1368            else if(srcVec == 2)
1369                AA64FpDest_x[i] = AA64IntrlvReg2_x[srcIdx];
1370            else if(srcVec == 3)
1371                AA64FpDest_x[i] = AA64IntrlvReg3_x[srcIdx];
1372        }'''
1373
1374        iop2 = InstObjParams('deintrlv',
1375                'SveDeIntrlv2Microop',
1376                'MicroOp',
1377                {'code': code2},
1378                ['IsMicroop'])
1379        iop3 = InstObjParams('deintrlv',
1380                'SveDeIntrlv3Microop',
1381                'MicroOp',
1382                {'code': code3},
1383                ['IsMicroop'])
1384        iop4 = InstObjParams('deintrlv',
1385                'SveDeIntrlv4Microop',
1386                'MicroOp',
1387                {'code': code4},
1388                ['IsMicroop'])
1389        header_output += SveDeIntrlvMicroopDeclare.subst(iop2);
1390        header_output += SveDeIntrlvMicroopDeclare.subst(iop3);
1391        header_output += SveDeIntrlvMicroopDeclare.subst(iop4);
1392        exec_output += SveIntrlvMicroopExecute.subst(iop2);
1393        exec_output += SveIntrlvMicroopExecute.subst(iop3);
1394        exec_output += SveIntrlvMicroopExecute.subst(iop4);
1395        for type in ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'):
1396            for nreg in range(2,5):
1397                substDict = {'targs' : type,
1398                        'class_name' : 'SveDeIntrlv' + str(nreg) + 'Microop'}
1399                exec_output += SveIntrlvMicroopExecDeclare.subst(substDict)
1400
1401    # Generates definitions for SVE struct load/store microops
1402    def emitSveStructMemInsts(offsetIsImm):
1403        global header_output, exec_output, decoders
1404        eaCode = SPAlignmentCheckCode + '''
1405        int memAccessSize = eCount * sizeof(Element);
1406        EA = memAccessSize * regIndex + XBase + '''
1407        if offsetIsImm:
1408            eaCode += '((int64_t) this->imm * eCount * sizeof(Element))'
1409        else:
1410            eaCode += '(XOffset * sizeof(Element));'
1411        loadMemAccCode = '''
1412        for (int i = 0; i < eCount; i++) {
1413            int gpIdx = (regIndex * eCount + i) / numRegs;
1414            if (GpOp_x[gpIdx]) {
1415                AA64FpDest_x[i] = memDataView[i];
1416            } else {
1417                AA64FpDest_x[i] = 0;
1418            }
1419        }
1420        '''
1421        storeMemAccCode = '''
1422        for (int i = 0; i < eCount; i++) {
1423            int gpIdx = (regIndex * eCount + i) / numRegs;
1424            if (GpOp_x[gpIdx]) {
1425                memDataView[i] = AA64FpDest_x[i];
1426            } else {
1427                memDataView[i] = 0;
1428                for (int j = 0; j < sizeof(Element); j++) {
1429                    wrEn[sizeof(Element) * i + j] = false;
1430                }
1431            }
1432        }
1433        '''
1434        storeWrEnableCode = '''
1435        auto wrEn = std::vector<bool>(sizeof(Element) * eCount, true);
1436        '''
1437        loadIop = InstObjParams('ldxx',
1438            'SveLoadRegImmMicroop' if offsetIsImm else 'SveLoadRegRegMicroop',
1439            'MicroOp',
1440            {'targs': 'Element',
1441             'memacc_code': loadMemAccCode,
1442             'ea_code' : sveEnabledCheckCode + eaCode,
1443             'fa_code' : ''},
1444            ['IsMemRef', 'IsLoad', 'IsMicroop'])
1445        storeIop = InstObjParams('stxx',
1446            'SveStoreRegImmMicroop' if offsetIsImm
1447                                    else 'SveStoreRegRegMicroop',
1448            'MicroOp',
1449            {'targs': 'Element',
1450             'wren_code': storeWrEnableCode,
1451             'memacc_code': storeMemAccCode,
1452             'ea_code' : sveEnabledCheckCode + eaCode,
1453             'fa_code' : ''},
1454            ['IsMemRef', 'IsStore', 'IsMicroop'])
1455        if offsetIsImm:
1456            header_output += SveStructMemSIMicroopDeclare.subst(loadIop)
1457            header_output += SveStructMemSIMicroopDeclare.subst(storeIop)
1458        else:
1459            header_output += SveStructMemSSMicroopDeclare.subst(loadIop)
1460            header_output += SveStructMemSSMicroopDeclare.subst(storeIop)
1461        exec_output += (
1462            SveStructLoadExecute.subst(loadIop) +
1463            SveStructLoadInitiateAcc.subst(loadIop) +
1464            SveStructLoadCompleteAcc.subst(loadIop) +
1465            SveStructStoreExecute.subst(storeIop) +
1466            SveStructStoreInitiateAcc.subst(storeIop) +
1467            SveStructStoreCompleteAcc.subst(storeIop))
1468        tplArgs = ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t')
1469        for type in tplArgs:
1470            substDict = {'targs': type,
1471                         'class_name': 'SveLoadRegImmMicroop' if offsetIsImm
1472                                       else 'SveLoadRegRegMicroop'}
1473            exec_output += SveStructMemExecDeclare.subst(substDict)
1474            substDict['class_name'] = ('SveStoreRegImmMicroop' if offsetIsImm
1475                                       else 'SveStoreRegRegMicroop')
1476            exec_output += SveStructMemExecDeclare.subst(substDict)
1477
1478    # Generates definitions for SVE load-and-replicate quadword instructions
1479    def emitSveLoadAndReplQuad(offsetIsImm):
1480        global header_output, exec_output, decoders
1481        tplHeader = 'template <class RegElemType, class MemElemType>'
1482        tplArgs = '<RegElemType, MemElemType>'
1483        eaCode = SPAlignmentCheckCode + '''
1484        int memAccessSize = 16;
1485        EA = XBase + '''
1486        if offsetIsImm:
1487            eaCode += '(((int64_t) this->imm) * 16);'
1488        else:
1489            eaCode += '(XOffset * sizeof(MemElemType));'
1490        loadRdEnableCode = '''
1491        eCount = 16/sizeof(RegElemType);
1492        auto rdEn = std::vector<bool>(16, true);
1493        for (int i = 0; i < eCount; ++i) {
1494            if (!GpOp_x[i]) {
1495                for (int j = 0; j < sizeof(RegElemType); ++j) {
1496                    rdEn[sizeof(RegElemType) * i + j] = false;
1497                }
1498            }
1499        }
1500        '''
1501        memAccCode = '''
1502        __uint128_t qword;
1503        RegElemType* qp = reinterpret_cast<RegElemType*>(&qword);
1504        for (int i = 0; i < 16/sizeof(RegElemType); ++i) {
1505            if (GpOp_x[i]) {
1506                qp[i] = memDataView[i];
1507            } else {
1508                qp[i] = 0;
1509            }
1510        }
1511        eCount = ArmStaticInst::getCurSveVecLen<__uint128_t>(
1512                xc->tcBase());
1513        for (int i = 0; i < eCount; ++i) {
1514            AA64FpDest_uq[i] = qword;
1515        }
1516        '''
1517        iop = InstObjParams('ld1rq',
1518                'SveLd1RqSI' if offsetIsImm else 'SveLd1RqSS',
1519                'SveContigMemSI' if offsetIsImm else 'SveContigMemSS',
1520                {'tpl_header': tplHeader,
1521                 'tpl_args': tplArgs,
1522                 'rden_code': loadRdEnableCode,
1523                 'memacc_code': memAccCode,
1524                 'ea_code': sveEnabledCheckCode + eaCode,
1525                 'fault_code': '',
1526                 'fa_code': ''},
1527                ['IsMemRef', 'IsLoad'])
1528        if offsetIsImm:
1529            header_output += SveContigMemSIOpDeclare.subst(iop)
1530        else:
1531            header_output += SveContigMemSSOpDeclare.subst(iop)
1532        exec_output += (
1533                SveContigLoadExecute.subst(iop) +
1534                SveContigLoadInitiateAcc.subst(iop) +
1535                SveContigLoadCompleteAcc.subst(iop))
1536        for ttype in ('uint8_t', 'uint16_t', 'uint32_t', 'uint64_t'):
1537            substDict = {'tpl_args': '<%s, %s>' % (ttype, ttype),
1538                    'class_name': 'SveLd1RqSI' if offsetIsImm
1539                                  else 'SveLd1RqSS'}
1540            exec_output += SveContigMemExecDeclare.subst(substDict)
1541
1542    # LD1[S]{B,H,W,D} (scalar plus immediate)
1543    # ST1[S]{B,H,W,D} (scalar plus immediate)
1544    # LDNF1[S]{B,H,W,D} (scalar plus immediate)
1545    emitSveContigMemInsts(True)
1546    # LD1[S]{B,H,W,D} (scalar plus scalar)
1547    # ST1[S]{B,H,W,D} (scalar plus scalar)
1548    # LDFF1[S]{B,H,W,D} (scalar plus vector)
1549    emitSveContigMemInsts(False)
1550
1551    # LD1R[S]{B,H,W,D}
1552    emitSveLoadAndRepl()
1553
1554    # LD1RQ{B,H,W,D} (scalar plus immediate)
1555    emitSveLoadAndReplQuad(offsetIsImm = True)
1556    # LD1RQ{B,H,W,D} (scalar plus scalar)
1557    emitSveLoadAndReplQuad(offsetIsImm = False)
1558
1559    # LD{2,3,4}{B,H,W,D} (scalar plus immediate)
1560    # ST{2,3,4}{B,H,W,D} (scalar plus immediate)
1561    emitSveStructMemInsts(offsetIsImm = True)
1562    # LD{2,3,4}{B,H,W,D} (scalar plus scalar)
1563    # ST{2,3,4}{B,H,W,D} (scalar plus scalar)
1564    emitSveStructMemInsts(offsetIsImm = False)
1565
1566    # LDR (predicate), STR (predicate)
1567    emitSveMemFillSpill(True)
1568    # LDR (vector), STR (vector)
1569    emitSveMemFillSpill(False)
1570
1571    # LD1[S]{B,H,W,D} (vector plus immediate)
1572    # ST1[S]{B,H,W,D} (vector plus immediate)
1573    # LDFF1[S]{B,H,W,D} (scalar plus immediate)
1574    emitSveIndexedMemMicroops(IndexedAddrForm.VEC_PLUS_IMM)
1575    # LD1[S]{B,H,W,D} (scalar plus vector)
1576    # ST1[S]{B,H,W,D} (scalar plus vector)
1577    # LDFF1[S]{B,H,W,D} (scalar plus vector)
1578    emitSveIndexedMemMicroops(IndexedAddrForm.SCA_PLUS_VEC)
1579
1580    # FFR writeback microop for gather loads
1581    emitSveFirstFaultWritebackMicroop()
1582
1583    # Source vector copy microop for gather loads
1584    emitSveGatherLoadCpySrcVecMicroop()
1585
1586    # ST/LD struct de/interleave microops
1587    emitSveInterleaveMicroop()
1588    emitSveDeInterleaveMicroop()
1589}};
1590