92c92
< uint64_t imm, bool firstFaulting,
---
> uint64_t imm, bool nonFaulting,
95,98c95,96
< assert(!(replicate && firstFaulting));
<
< const char* mn = replicate ? "ld1r" :
< (firstFaulting ? "ldff1" : "ld1");
---
> assert(!(nonFaulting && replicate));
> const char* mn = replicate ? "ld1r" : (nonFaulting ? "ldnf1" : "ld1");
213c211
< bool firstFaulting)
---
> bool firstFault)
215c213
< const char* mn = firstFaulting ? "ldff1" : "ld1";
---
> const char* mn = firstFault ? "ldff1" : "ld1";
220,221c218,220
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
224,225c223,225
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
230,231c230,232
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
234,235c235,237
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
240,241c242,244
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
244,245c247,249
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
250,251c254,256
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
254,255c259,261
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
262,263c268,270
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
268,269c275,277
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
272,273c280,282
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
280,281c289,291
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
292c302
< bool firstFaulting)
---
> bool firstFault)
294c304
< const char* mn = firstFaulting ? "ldff1" : "ld1";
---
> const char* mn = firstFault ? "ldff1" : "ld1";
299c309,310
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
301c312
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
304c315,316
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
306c318
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
311c323,324
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
313c326
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
316c329,330
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
318c332
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
323c337,338
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
325c340
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
328c343,344
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
330c346
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
335c351,352
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
337c354
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
340c357,358
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
342c360
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
349c367,368
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
351c370
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
356c375,376
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
358c378
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
361c381,382
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
363c384
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
370c391,392
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
372c394
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
389,390c411,413
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
393,394c416,418
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
399,400c423,425
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
403,404c428,430
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
409,410c435,437
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
413,414c440,442
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
421,422c449,451
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
440c469,470
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
442c472
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
445c475,476
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
447c478
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
452c483,484
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
454c486
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
457c489,490
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
459c492
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
464c497,498
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
466c500
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
469c503,504
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
471c506
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
478c513,514
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
480c516
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
507a544,546
> loadRdEnableCode = '''
> auto rdEn = std::vector<bool>();
> '''
553a593,594
> 'rden_code' : loadRdEnableCode,
> 'fault_code' : '',
635a677,681
> # First-faulting instructions only have a scalar plus scalar form,
> # while non-faulting instructions only a scalar plus immediate form, so
> # `offsetIsImm` is used to determine which class of instructions is
> # generated
> firstFaulting = not offsetIsImm
644a691,700
> loadRdEnableCode = '''
> auto rdEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
> for (int i = 0; i < eCount; i++) {
> if (!GpOp_x[i]) {
> for (int j = 0; j < sizeof(MemElemType); j++) {
> rdEn[sizeof(MemElemType) * i + j] = false;
> }
> }
> }
> '''
668a725,769
> ffrReadBackCode = '''
> auto& firstFaultReg = Ffr;'''
> fautlingLoadmemAccCode = '''
> for (int i = 0; i < eCount; i++) {
> if (GpOp_x[i] && firstFaultReg[i * sizeof(RegElemType)]) {
> AA64FpDest_x[i] = memDataView[i];
> } else {
> AA64FpDest_x[i] = 0;
> }
> }
> '''
> nonFaultingCode = 'true ||'
> faultCode = '''
> Addr fault_addr;
> if (fault == NoFault || getFaultVAddr(fault, fault_addr)) {
> unsigned fault_elem_index;
> if (fault != NoFault) {
> assert(fault_addr >= EA);
> fault_elem_index = (fault_addr - EA) / sizeof(MemElemType);
> } else {
> fault_elem_index = eCount + 1;
> }
> int first_active_index;
> for (first_active_index = 0;
> first_active_index < eCount && !(GpOp_x[first_active_index]);
> first_active_index++);
> if (%s first_active_index < fault_elem_index) {
> for (int i = 0; i < eCount; i++) {
> for (int j = 0; j < sizeof(RegElemType); j++) {
> if (i < fault_elem_index) {
> Ffr_ub[i * sizeof(RegElemType) + j] = FfrAux_x[i];
> } else {
> Ffr_ub[i * sizeof(RegElemType) + j] = 0;
> }
> }
> }
> fault = NoFault;
> if (first_active_index >= fault_elem_index) {
> // non-faulting load needs this
> xc->setMemAccPredicate(false);
> }
> }
> }
> ''' % ('' if firstFaulting else nonFaultingCode)
>
673a775
> 'rden_code' : loadRdEnableCode,
675a778
> 'fault_code' : '',
687a791,803
> faultIop = InstObjParams('ldff1' if firstFaulting else 'ldnf1',
> 'SveContigFFLoadSS' if firstFaulting else 'SveContigNFLoadSI',
> 'SveContigMemSS' if firstFaulting else 'SveContigMemSI',
> {'tpl_header': tplHeader,
> 'tpl_args': tplArgs,
> 'rden_code' : loadRdEnableCode,
> 'memacc_code': fautlingLoadmemAccCode,
> 'ea_code' : sveEnabledCheckCode + eaCode,
> 'fault_code' : faultCode,
> 'fa_code' : ''},
> ['IsMemRef', 'IsLoad'])
> faultIop.snippets['memacc_code'] = (ffrReadBackCode +
> faultIop.snippets['memacc_code'])
690a807
> header_output += SveContigMemSIOpDeclare.subst(faultIop)
693a811
> header_output += SveContigMemSSOpDeclare.subst(faultIop)
700c818,822
< SveContigStoreCompleteAcc.subst(storeIop))
---
> SveContigStoreCompleteAcc.subst(storeIop) +
> SveContigLoadExecute.subst(faultIop) +
> SveContigLoadInitiateAcc.subst(faultIop) +
> SveContigLoadCompleteAcc.subst(faultIop))
>
710a833,837
> for args in loadTplArgs:
> substDict = {'tpl_args': '<%s>' % ', '.join(args),
> 'class_name': 'SveContigFFLoadSS' if firstFaulting
> else 'SveContigNFLoadSI'}
> exec_output += SveContigMemExecDeclare.subst(substDict)
711a839
>
776,780c904
< if (GpOp_x[elemIndex]) {
< AA64FpDest_x[elemIndex] = memData;
< } else {
< AA64FpDest_x[elemIndex] = 0;
< }
---
> AA64FpDest_x[elemIndex] = memData;
785c909,911
< predCheckCode = 'GpOp_x[elemIndex]'
---
> predCheckCode = 'GpOp_x[index]'
> faultStatusSetCode = 'PUreg0_x[elemIndex] = 1;'
> faultStatusResetCode = 'PUreg0_x[elemIndex] = 0;'
794a921,922
> 'fault_status_set_code' : faultStatusSetCode,
> 'fault_status_reset_code' : faultStatusResetCode,
841a970,1002
> firstFaultTplArgs = ('int32_t', 'int64_t', 'uint32_t', 'uint64_t')
>
> def emitSveFirstFaultWritebackMicroop():
> global header_output, exec_output, decoders
> tplHeader = 'template <class RegElemType>'
> tplArgs = '<RegElemType>'
> faultStatusCheckCode = 'PUreg0_x[index]'
> firstFaultResetCode = '''
> for(int j = 0; j < sizeof(RegElemType); j++) {
> Ffr_ub[index * sizeof(RegElemType) + j] = 0;
> }
> '''
> firstFaultForwardCode = '''
> for(int j = 0; j < sizeof(RegElemType); j++) {
> Ffr_ub[index * sizeof(RegElemType) + j] = FfrAux_x[index];
> }
> '''
> iop = InstObjParams('ldff1',
> 'SveFirstFaultWritebackMicroop',
> 'MicroOp',
> {'tpl_header': tplHeader,
> 'tpl_args': tplArgs,
> 'fault_status_check_code' : faultStatusCheckCode,
> 'first_fault_reset_code' : firstFaultResetCode,
> 'first_fault_forward_code' : firstFaultForwardCode},
> ['IsMicroop'])
> header_output += SveFirstFaultWritebackMicroopDeclare.subst(iop)
> exec_output += SveFirstFaultWritebackMicroopExecute.subst(iop)
> for args in firstFaultTplArgs:
> substDict = {'targs': args,
> 'class_name' : 'SveFirstFaultWritebackMicroop' }
> exec_output += SveOpExecDeclare.subst(substDict)
>
861a1023
> # LDNF1[S]{B,H,W,D} (scalar plus immediate)
864a1027
> # LDFF1[S]{B,H,W,D} (scalar plus vector)
876a1040
> # LDFF1[S]{B,H,W,D} (scalar plus immediate)
879a1044
> # LDFF1[S]{B,H,W,D} (scalar plus vector)
881a1047,1049
> # FFR writeback microop for gather loads
> emitSveFirstFaultWritebackMicroop()
>
884d1051
<
< uint64_t imm, bool firstFaulting,
---
> uint64_t imm, bool nonFaulting,
95,98c95,96
< assert(!(replicate && firstFaulting));
<
< const char* mn = replicate ? "ld1r" :
< (firstFaulting ? "ldff1" : "ld1");
---
> assert(!(nonFaulting && replicate));
> const char* mn = replicate ? "ld1r" : (nonFaulting ? "ldnf1" : "ld1");
213c211
< bool firstFaulting)
---
> bool firstFault)
215c213
< const char* mn = firstFaulting ? "ldff1" : "ld1";
---
> const char* mn = firstFault ? "ldff1" : "ld1";
220,221c218,220
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
224,225c223,225
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
230,231c230,232
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
234,235c235,237
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
240,241c242,244
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
244,245c247,249
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
250,251c254,256
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
254,255c259,261
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
262,263c268,270
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
268,269c275,277
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
272,273c280,282
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
280,281c289,291
< SveGatherLoadVIMicroop>(
< mn, machInst, MemReadOp, zt, pg, zn, imm);
---
> SveGatherLoadVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemReadOp, zt, pg, zn, imm, firstFault);
292c302
< bool firstFaulting)
---
> bool firstFault)
294c304
< const char* mn = firstFaulting ? "ldff1" : "ld1";
---
> const char* mn = firstFault ? "ldff1" : "ld1";
299c309,310
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
301c312
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
304c315,316
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
306c318
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
311c323,324
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
313c326
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
316c329,330
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
318c332
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
323c337,338
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
325c340
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
328c343,344
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
330c346
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
335c351,352
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
337c354
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
340c357,358
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
342c360
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
349c367,368
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
351c370
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
356c375,376
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
358c378
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
361c381,382
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
363c384
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
370c391,392
< SveGatherLoadSVMicroop>(
---
> SveGatherLoadSVMicroop,
> SveFirstFaultWritebackMicroop>(
372c394
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, firstFault);
389,390c411,413
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
393,394c416,418
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
399,400c423,425
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
403,404c428,430
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
409,410c435,437
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
413,414c440,442
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
421,422c449,451
< SveScatterStoreVIMicroop>(
< mn, machInst, MemWriteOp, zt, pg, zn, imm);
---
> SveScatterStoreVIMicroop,
> SveFirstFaultWritebackMicroop>(
> mn, machInst, MemWriteOp, zt, pg, zn, imm, false);
440c469,470
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
442c472
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
445c475,476
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
447c478
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
452c483,484
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
454c486
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
457c489,490
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
459c492
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
464c497,498
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
466c500
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
469c503,504
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
471c506
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
478c513,514
< SveScatterStoreSVMicroop>(
---
> SveScatterStoreSVMicroop,
> SveFirstFaultWritebackMicroop>(
480c516
< offsetIs32, offsetIsSigned, offsetIsScaled);
---
> offsetIs32, offsetIsSigned, offsetIsScaled, false);
507a544,546
> loadRdEnableCode = '''
> auto rdEn = std::vector<bool>();
> '''
553a593,594
> 'rden_code' : loadRdEnableCode,
> 'fault_code' : '',
635a677,681
> # First-faulting instructions only have a scalar plus scalar form,
> # while non-faulting instructions only a scalar plus immediate form, so
> # `offsetIsImm` is used to determine which class of instructions is
> # generated
> firstFaulting = not offsetIsImm
644a691,700
> loadRdEnableCode = '''
> auto rdEn = std::vector<bool>(sizeof(MemElemType) * eCount, true);
> for (int i = 0; i < eCount; i++) {
> if (!GpOp_x[i]) {
> for (int j = 0; j < sizeof(MemElemType); j++) {
> rdEn[sizeof(MemElemType) * i + j] = false;
> }
> }
> }
> '''
668a725,769
> ffrReadBackCode = '''
> auto& firstFaultReg = Ffr;'''
> fautlingLoadmemAccCode = '''
> for (int i = 0; i < eCount; i++) {
> if (GpOp_x[i] && firstFaultReg[i * sizeof(RegElemType)]) {
> AA64FpDest_x[i] = memDataView[i];
> } else {
> AA64FpDest_x[i] = 0;
> }
> }
> '''
> nonFaultingCode = 'true ||'
> faultCode = '''
> Addr fault_addr;
> if (fault == NoFault || getFaultVAddr(fault, fault_addr)) {
> unsigned fault_elem_index;
> if (fault != NoFault) {
> assert(fault_addr >= EA);
> fault_elem_index = (fault_addr - EA) / sizeof(MemElemType);
> } else {
> fault_elem_index = eCount + 1;
> }
> int first_active_index;
> for (first_active_index = 0;
> first_active_index < eCount && !(GpOp_x[first_active_index]);
> first_active_index++);
> if (%s first_active_index < fault_elem_index) {
> for (int i = 0; i < eCount; i++) {
> for (int j = 0; j < sizeof(RegElemType); j++) {
> if (i < fault_elem_index) {
> Ffr_ub[i * sizeof(RegElemType) + j] = FfrAux_x[i];
> } else {
> Ffr_ub[i * sizeof(RegElemType) + j] = 0;
> }
> }
> }
> fault = NoFault;
> if (first_active_index >= fault_elem_index) {
> // non-faulting load needs this
> xc->setMemAccPredicate(false);
> }
> }
> }
> ''' % ('' if firstFaulting else nonFaultingCode)
>
673a775
> 'rden_code' : loadRdEnableCode,
675a778
> 'fault_code' : '',
687a791,803
> faultIop = InstObjParams('ldff1' if firstFaulting else 'ldnf1',
> 'SveContigFFLoadSS' if firstFaulting else 'SveContigNFLoadSI',
> 'SveContigMemSS' if firstFaulting else 'SveContigMemSI',
> {'tpl_header': tplHeader,
> 'tpl_args': tplArgs,
> 'rden_code' : loadRdEnableCode,
> 'memacc_code': fautlingLoadmemAccCode,
> 'ea_code' : sveEnabledCheckCode + eaCode,
> 'fault_code' : faultCode,
> 'fa_code' : ''},
> ['IsMemRef', 'IsLoad'])
> faultIop.snippets['memacc_code'] = (ffrReadBackCode +
> faultIop.snippets['memacc_code'])
690a807
> header_output += SveContigMemSIOpDeclare.subst(faultIop)
693a811
> header_output += SveContigMemSSOpDeclare.subst(faultIop)
700c818,822
< SveContigStoreCompleteAcc.subst(storeIop))
---
> SveContigStoreCompleteAcc.subst(storeIop) +
> SveContigLoadExecute.subst(faultIop) +
> SveContigLoadInitiateAcc.subst(faultIop) +
> SveContigLoadCompleteAcc.subst(faultIop))
>
710a833,837
> for args in loadTplArgs:
> substDict = {'tpl_args': '<%s>' % ', '.join(args),
> 'class_name': 'SveContigFFLoadSS' if firstFaulting
> else 'SveContigNFLoadSI'}
> exec_output += SveContigMemExecDeclare.subst(substDict)
711a839
>
776,780c904
< if (GpOp_x[elemIndex]) {
< AA64FpDest_x[elemIndex] = memData;
< } else {
< AA64FpDest_x[elemIndex] = 0;
< }
---
> AA64FpDest_x[elemIndex] = memData;
785c909,911
< predCheckCode = 'GpOp_x[elemIndex]'
---
> predCheckCode = 'GpOp_x[index]'
> faultStatusSetCode = 'PUreg0_x[elemIndex] = 1;'
> faultStatusResetCode = 'PUreg0_x[elemIndex] = 0;'
794a921,922
> 'fault_status_set_code' : faultStatusSetCode,
> 'fault_status_reset_code' : faultStatusResetCode,
841a970,1002
> firstFaultTplArgs = ('int32_t', 'int64_t', 'uint32_t', 'uint64_t')
>
> def emitSveFirstFaultWritebackMicroop():
> global header_output, exec_output, decoders
> tplHeader = 'template <class RegElemType>'
> tplArgs = '<RegElemType>'
> faultStatusCheckCode = 'PUreg0_x[index]'
> firstFaultResetCode = '''
> for(int j = 0; j < sizeof(RegElemType); j++) {
> Ffr_ub[index * sizeof(RegElemType) + j] = 0;
> }
> '''
> firstFaultForwardCode = '''
> for(int j = 0; j < sizeof(RegElemType); j++) {
> Ffr_ub[index * sizeof(RegElemType) + j] = FfrAux_x[index];
> }
> '''
> iop = InstObjParams('ldff1',
> 'SveFirstFaultWritebackMicroop',
> 'MicroOp',
> {'tpl_header': tplHeader,
> 'tpl_args': tplArgs,
> 'fault_status_check_code' : faultStatusCheckCode,
> 'first_fault_reset_code' : firstFaultResetCode,
> 'first_fault_forward_code' : firstFaultForwardCode},
> ['IsMicroop'])
> header_output += SveFirstFaultWritebackMicroopDeclare.subst(iop)
> exec_output += SveFirstFaultWritebackMicroopExecute.subst(iop)
> for args in firstFaultTplArgs:
> substDict = {'targs': args,
> 'class_name' : 'SveFirstFaultWritebackMicroop' }
> exec_output += SveOpExecDeclare.subst(substDict)
>
861a1023
> # LDNF1[S]{B,H,W,D} (scalar plus immediate)
864a1027
> # LDFF1[S]{B,H,W,D} (scalar plus vector)
876a1040
> # LDFF1[S]{B,H,W,D} (scalar plus immediate)
879a1044
> # LDFF1[S]{B,H,W,D} (scalar plus vector)
881a1047,1049
> # FFR writeback microop for gather loads
> emitSveFirstFaultWritebackMicroop()
>
884d1051
<