1/*
2 * Copyright (c) 2012-2015 Advanced Micro Devices, Inc.
3 * All rights reserved.
4 *
5 * For use for simulation and test purposes only
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 * this list of conditions and the following disclaimer in the documentation
15 * and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the copyright holder nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 *
33 * Author: Steve Reinhardt
34 */
35
36#ifndef __ARCH_HSAIL_INSTS_MEM_HH__
37#define __ARCH_HSAIL_INSTS_MEM_HH__
38
39#include <type_traits>
40
41#include "arch/hsail/insts/decl.hh"
42#include "arch/hsail/insts/gpu_static_inst.hh"
43#include "arch/hsail/operand.hh"
44#include "gpu-compute/compute_unit.hh"
45
46namespace HsailISA
47{
48 class MemInst
49 {
50 public:
51 MemInst() : size(0), addr_operand(nullptr) { }
52
53 MemInst(Enums::MemType m_type)
54 {
55 if (m_type == Enums::M_U64 ||
56 m_type == Enums::M_S64 ||
57 m_type == Enums::M_F64) {
58 size = 8;
59 } else if (m_type == Enums::M_U32 ||
60 m_type == Enums::M_S32 ||
61 m_type == Enums::M_F32) {
62 size = 4;
63 } else if (m_type == Enums::M_U16 ||
64 m_type == Enums::M_S16 ||
65 m_type == Enums::M_F16) {
66 size = 2;
67 } else {
68 size = 1;
69 }
70
71 addr_operand = nullptr;
72 }
73
74 void
75 init_addr(AddrOperandBase *_addr_operand)
76 {
77 addr_operand = _addr_operand;
78 }
79
80 private:
81 int size;
82 AddrOperandBase *addr_operand;
83
84 public:
85 int getMemOperandSize() { return size; }
86 AddrOperandBase *getAddressOperand() { return addr_operand; }
87 };
88
89 template<typename DestOperandType, typename AddrOperandType>
90 class LdaInstBase : public HsailGPUStaticInst
91 {
92 public:
93 typename DestOperandType::DestOperand dest;
94 AddrOperandType addr;
95
96 LdaInstBase(const Brig::BrigInstBase *ib, const BrigObject *obj,
97 const char *_opcode)
98 : HsailGPUStaticInst(obj, _opcode)
99 {
100 using namespace Brig;
101
102 setFlag(ALU);
103
104 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
105 dest.init(op_offs, obj);
106 op_offs = obj->getOperandPtr(ib->operands, 1);
107 addr.init(op_offs, obj);
108 }
109
110 int numSrcRegOperands() override
111 { return(this->addr.isVectorRegister()); }
112 int numDstRegOperands() override
113 { return dest.isVectorRegister(); }
114 bool isVectorRegister(int operandIndex) override
115 {
116 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
117 return((operandIndex == 0) ? dest.isVectorRegister() :
118 this->addr.isVectorRegister());
119 }
120 bool isCondRegister(int operandIndex) override
121 {
122 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
123 return((operandIndex == 0) ? dest.isCondRegister() :
124 this->addr.isCondRegister());
125 }
126 bool isScalarRegister(int operandIndex) override
127 {
128 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
129 return((operandIndex == 0) ? dest.isScalarRegister() :
130 this->addr.isScalarRegister());
131 }
132 bool isSrcOperand(int operandIndex) override
133 {
134 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
135 if (operandIndex > 0)
136 return(this->addr.isVectorRegister());
137 return false;
138 }
139 bool isDstOperand(int operandIndex) override {
140 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
141 return(operandIndex == 0);
142 }
143 int getOperandSize(int operandIndex) override
144 {
145 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
146 return((operandIndex == 0) ? dest.opSize() :
147 this->addr.opSize());
148 }
149 int getRegisterIndex(int operandIndex) override
150 {
151 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
152 return((operandIndex == 0) ? dest.regIndex() :
153 this->addr.regIndex());
154 }
155 int getNumOperands() override
156 {
157 if (this->addr.isVectorRegister())
158 return 2;
159 return 1;
160 }
161 };
162
163 template<typename DestDataType, typename AddrOperandType>
164 class LdaInst :
165 public LdaInstBase<typename DestDataType::OperandType, AddrOperandType>,
166 public MemInst
167 {
168 public:
169 void generateDisassembly();
170
171 LdaInst(const Brig::BrigInstBase *ib, const BrigObject *obj,
172 const char *_opcode)
173 : LdaInstBase<typename DestDataType::OperandType,
174 AddrOperandType>(ib, obj, _opcode)
175 {
176 init_addr(&this->addr);
177 }
178
179 void execute(GPUDynInstPtr gpuDynInst);
180 };
181
182 template<typename DataType>
183 GPUStaticInst*
184 decodeLda(const Brig::BrigInstBase *ib, const BrigObject *obj)
185 {
186 unsigned op_offs = obj->getOperandPtr(ib->operands, 1);
187 BrigRegOperandInfo regDataType = findRegDataType(op_offs, obj);
188
189 if (regDataType.kind == Brig::BRIG_KIND_OPERAND_ADDRESS) {
190 return new LdaInst<DataType, NoRegAddrOperand>(ib, obj, "ldas");
191 } else if (regDataType.kind == Brig::BRIG_KIND_OPERAND_REGISTER) {
192 // V2/V4 not allowed
193 switch (regDataType.regKind) {
194 case Brig::BRIG_REGISTER_KIND_SINGLE:
195 return new LdaInst<DataType, SRegAddrOperand>(ib, obj, "ldas");
196 case Brig::BRIG_REGISTER_KIND_DOUBLE:
197 return new LdaInst<DataType, DRegAddrOperand>(ib, obj, "ldas");
198 default:
199 fatal("Bad ldas register operand type %d\n", regDataType.type);
200 }
201 } else {
202 fatal("Bad ldas register operand kind %d\n", regDataType.kind);
203 }
204 }
205
206 template<typename MemOperandType, typename DestOperandType,
207 typename AddrOperandType>
208 class LdInstBase : public HsailGPUStaticInst
209 {
210 public:
211 Brig::BrigWidth8_t width;
212 typename DestOperandType::DestOperand dest;
213 AddrOperandType addr;
214
215 Brig::BrigSegment segment;
216 Brig::BrigMemoryOrder memoryOrder;
217 Brig::BrigMemoryScope memoryScope;
218 unsigned int equivClass;
219
220 LdInstBase(const Brig::BrigInstBase *ib, const BrigObject *obj,
221 const char *_opcode)
222 : HsailGPUStaticInst(obj, _opcode)
223 {
224 using namespace Brig;
225
226 setFlag(MemoryRef);
227 setFlag(Load);
228
229 if (ib->opcode == BRIG_OPCODE_LD) {
230 const BrigInstMem *ldst = (const BrigInstMem*)ib;
231
232 segment = (BrigSegment)ldst->segment;
233 memoryOrder = BRIG_MEMORY_ORDER_NONE;
234 memoryScope = BRIG_MEMORY_SCOPE_NONE;
235 equivClass = ldst->equivClass;
236
237 width = ldst->width;
238 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
239 const Brig::BrigOperand *brigOp = obj->getOperand(op_offs);
240 if (brigOp->kind == BRIG_KIND_OPERAND_REGISTER)
241 dest.init(op_offs, obj);
242
243 op_offs = obj->getOperandPtr(ib->operands, 1);
244 addr.init(op_offs, obj);
245 } else {
246 const BrigInstAtomic *at = (const BrigInstAtomic*)ib;
247
248 segment = (BrigSegment)at->segment;
249 memoryOrder = (BrigMemoryOrder)at->memoryOrder;
250 memoryScope = (BrigMemoryScope)at->memoryScope;
251 equivClass = 0;
252
253 width = BRIG_WIDTH_1;
254 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
255 const Brig::BrigOperand *brigOp = obj->getOperand(op_offs);
256
257 if (brigOp->kind == BRIG_KIND_OPERAND_REGISTER)
258 dest.init(op_offs, obj);
259
260 op_offs = obj->getOperandPtr(ib->operands,1);
261 addr.init(op_offs, obj);
262 }
263
264 switch (memoryOrder) {
265 case BRIG_MEMORY_ORDER_NONE:
266 setFlag(NoOrder);
267 break;
268 case BRIG_MEMORY_ORDER_RELAXED:
269 setFlag(RelaxedOrder);
270 break;
271 case BRIG_MEMORY_ORDER_SC_ACQUIRE:
272 setFlag(Acquire);
273 break;
274 case BRIG_MEMORY_ORDER_SC_RELEASE:
275 setFlag(Release);
276 break;
277 case BRIG_MEMORY_ORDER_SC_ACQUIRE_RELEASE:
278 setFlag(AcquireRelease);
279 break;
280 default:
281 fatal("LdInst has bad memory order type\n");
282 }
283
284 switch (memoryScope) {
285 case BRIG_MEMORY_SCOPE_NONE:
286 setFlag(NoScope);
287 break;
288 case BRIG_MEMORY_SCOPE_WORKITEM:
289 setFlag(WorkitemScope);
290 break;
291 case BRIG_MEMORY_SCOPE_WORKGROUP:
292 setFlag(WorkgroupScope);
293 break;
294 case BRIG_MEMORY_SCOPE_AGENT:
295 setFlag(DeviceScope);
296 break;
297 case BRIG_MEMORY_SCOPE_SYSTEM:
298 setFlag(SystemScope);
299 break;
300 default:
301 fatal("LdInst has bad memory scope type\n");
302 }
303
304 switch (segment) {
305 case BRIG_SEGMENT_GLOBAL:
306 setFlag(GlobalSegment);
307 break;
308 case BRIG_SEGMENT_GROUP:
309 setFlag(GroupSegment);
310 break;
311 case BRIG_SEGMENT_PRIVATE:
312 setFlag(PrivateSegment);
313 break;
314 case BRIG_SEGMENT_READONLY:
315 setFlag(ReadOnlySegment);
316 break;
317 case BRIG_SEGMENT_SPILL:
318 setFlag(SpillSegment);
319 break;
320 case BRIG_SEGMENT_FLAT:
321 setFlag(Flat);
322 break;
323 case BRIG_SEGMENT_KERNARG:
324 setFlag(KernArgSegment);
325 break;
326 case BRIG_SEGMENT_ARG:
327 setFlag(ArgSegment);
328 break;
329 default:
330 panic("Ld: segment %d not supported\n", segment);
331 }
332 }
333
334 int numSrcRegOperands() override
335 { return(this->addr.isVectorRegister()); }
336 int numDstRegOperands() override { return dest.isVectorRegister(); }
337 int getNumOperands() override
338 {
339 if (this->addr.isVectorRegister())
340 return 2;
341 else
342 return 1;
343 }
344 bool isVectorRegister(int operandIndex) override
345 {
346 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
347 return((operandIndex == 0) ? dest.isVectorRegister() :
348 this->addr.isVectorRegister());
349 }
350 bool isCondRegister(int operandIndex) override
351 {
352 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
353 return((operandIndex == 0) ? dest.isCondRegister() :
354 this->addr.isCondRegister());
355 }
356 bool isScalarRegister(int operandIndex) override
357 {
358 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
359 return((operandIndex == 0) ? dest.isScalarRegister() :
360 this->addr.isScalarRegister());
361 }
362 bool isSrcOperand(int operandIndex) override
363 {
364 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
365 if (operandIndex > 0)
366 return(this->addr.isVectorRegister());
367 return false;
368 }
369 bool isDstOperand(int operandIndex) override
370 {
371 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
372 return(operandIndex == 0);
373 }
374 int getOperandSize(int operandIndex) override
375 {
376 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
377 return((operandIndex == 0) ? dest.opSize() :
378 this->addr.opSize());
379 }
380 int getRegisterIndex(int operandIndex) override
381 {
382 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
383 return((operandIndex == 0) ? dest.regIndex() :
384 this->addr.regIndex());
385 }
386 };
387
388 template<typename MemDataType, typename DestDataType,
389 typename AddrOperandType>
390 class LdInst :
391 public LdInstBase<typename MemDataType::CType,
392 typename DestDataType::OperandType, AddrOperandType>,
393 public MemInst
394 {
395 typename DestDataType::OperandType::DestOperand dest_vect[4];
396 uint16_t num_dest_operands;
397 void generateDisassembly() override;
398
399 public:
400 LdInst(const Brig::BrigInstBase *ib, const BrigObject *obj,
401 const char *_opcode)
402 : LdInstBase<typename MemDataType::CType,
403 typename DestDataType::OperandType,
404 AddrOperandType>(ib, obj, _opcode),
405 MemInst(MemDataType::memType)
406 {
407 init_addr(&this->addr);
408
409 unsigned op_offs = obj->getOperandPtr(ib->operands,0);
410 const Brig::BrigOperand *brigOp = obj->getOperand(op_offs);
411
412 if (brigOp->kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST) {
413 const Brig::BrigOperandOperandList *brigRegVecOp =
414 (const Brig::BrigOperandOperandList*)brigOp;
415
416 num_dest_operands =
417 *((unsigned*)obj->getData(brigRegVecOp->elements)) / 4;
418
419 assert(num_dest_operands <= 4);
420 } else {
421 num_dest_operands = 1;
422 }
423
424 if (num_dest_operands > 1) {
425 assert(brigOp->kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST);
426
427 for (int i = 0; i < num_dest_operands; ++i) {
428 dest_vect[i].init_from_vect(op_offs, obj, i);
429 }
430 }
431 }
432
433 void
434 initiateAcc(GPUDynInstPtr gpuDynInst) override
435 {
436 typedef typename MemDataType::CType c0;
437
438 gpuDynInst->statusBitVector = gpuDynInst->exec_mask;
439
440 if (num_dest_operands > 1) {
441 for (int i = 0; i < gpuDynInst->computeUnit()->wfSize(); ++i)
442 if (gpuDynInst->exec_mask[i])
443 gpuDynInst->statusVector.push_back(num_dest_operands);
444 else
445 gpuDynInst->statusVector.push_back(0);
446 }
447
448 for (int k = 0; k < num_dest_operands; ++k) {
449
450 c0 *d = &((c0*)gpuDynInst->d_data)
451 [k * gpuDynInst->computeUnit()->wfSize()];
452
453 for (int i = 0; i < gpuDynInst->computeUnit()->wfSize(); ++i) {
454 if (gpuDynInst->exec_mask[i]) {
455 Addr vaddr = gpuDynInst->addr[i] + k * sizeof(c0);
456
457 if (this->isLocalMem()) {
458 // load from shared memory
459 *d = gpuDynInst->wavefront()->ldsChunk->
460 read<c0>(vaddr);
461 } else {
462 Request *req = new Request(0, vaddr, sizeof(c0), 0,
463 gpuDynInst->computeUnit()->masterId(),
464 0, gpuDynInst->wfDynId);
465
466 gpuDynInst->setRequestFlags(req);
467 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
468 pkt->dataStatic(d);
469
470 if (gpuDynInst->computeUnit()->shader->
471 separate_acquire_release &&
472 gpuDynInst->isAcquire()) {
473 // if this load has acquire semantics,
474 // set the response continuation function
475 // to perform an Acquire request
476 gpuDynInst->execContinuation =
477 &GPUStaticInst::execLdAcq;
478
479 gpuDynInst->useContinuation = true;
480 } else {
481 // the request will be finished when
482 // the load completes
483 gpuDynInst->useContinuation = false;
484 }
485 // translation is performed in sendRequest()
486 gpuDynInst->computeUnit()->sendRequest(gpuDynInst,
487 i, pkt);
488 }
489 }
490 ++d;
491 }
492 }
493
494 gpuDynInst->updateStats();
495 }
496
497 void
498 completeAcc(GPUDynInstPtr gpuDynInst) override
499 {
500 typedef typename MemDataType::CType c1;
501
502 constexpr bool is_vt_32 = DestDataType::vgprType == VT_32;
503
504 /**
505 * this code essentially replaces the long if-else chain
506 * that was in used GlobalMemPipeline::exec() to infer the
507 * size (single/double) and type (floating point/integer) of
508 * the destination register. this is needed for load
509 * instructions because the loaded value and the
510 * destination type can be of different sizes, and we also
511 * need to know if the value we're writing back is floating
512 * point and signed/unsigned, so we can properly cast the
513 * writeback value
514 */
515 typedef typename std::conditional<is_vt_32,
516 typename std::conditional<std::is_floating_point<c1>::value,
517 float, typename std::conditional<std::is_signed<c1>::value,
518 int32_t, uint32_t>::type>::type,
519 typename std::conditional<std::is_floating_point<c1>::value,
520 double, typename std::conditional<std::is_signed<c1>::value,
521 int64_t, uint64_t>::type>::type>::type c0;
522
523
524 Wavefront *w = gpuDynInst->wavefront();
525
526 std::vector<uint32_t> regVec;
527 // iterate over number of destination register operands since
528 // this is a load
529 for (int k = 0; k < num_dest_operands; ++k) {
530 assert((sizeof(c1) * num_dest_operands)
531 <= MAX_WIDTH_FOR_MEM_INST);
532
533 int dst = this->dest.regIndex() + k;
534 if (num_dest_operands > MAX_REGS_FOR_NON_VEC_MEM_INST)
535 dst = dest_vect[k].regIndex();
536 // virtual->physical VGPR mapping
537 int physVgpr = w->remap(dst, sizeof(c0), 1);
538 // save the physical VGPR index
539 regVec.push_back(physVgpr);
540
541 c1 *p1 =
542 &((c1*)gpuDynInst->d_data)[k * w->computeUnit->wfSize()];
543
544 for (int i = 0; i < w->computeUnit->wfSize(); ++i) {
545 if (gpuDynInst->exec_mask[i]) {
546 DPRINTF(GPUReg, "CU%d, WF[%d][%d], lane %d: "
547 "$%s%d <- %d global ld done (src = wavefront "
548 "ld inst)\n", w->computeUnit->cu_id, w->simdId,
549 w->wfSlotId, i, sizeof(c0) == 4 ? "s" : "d",
550 dst, *p1);
551 // write the value into the physical VGPR. This is a
552 // purely functional operation. No timing is modeled.
553 w->computeUnit->vrf[w->simdId]->write<c0>(physVgpr,
554 *p1, i);
555 }
556 ++p1;
557 }
558 }
559
560 // Schedule the write operation of the load data on the VRF.
561 // This simply models the timing aspect of the VRF write operation.
562 // It does not modify the physical VGPR.
563 int loadVrfBankConflictCycles = gpuDynInst->computeUnit()->
564 vrf[w->simdId]->exec(gpuDynInst->seqNum(), w, regVec,
565 sizeof(c0), gpuDynInst->time);
566
567 if (this->isGlobalMem()) {
568 gpuDynInst->computeUnit()->globalMemoryPipe
569 .incLoadVRFBankConflictCycles(loadVrfBankConflictCycles);
570 } else {
571 assert(this->isLocalMem());
572 gpuDynInst->computeUnit()->localMemoryPipe
573 .incLoadVRFBankConflictCycles(loadVrfBankConflictCycles);
574 }
575 }
576
577 private:
578 void
579 execLdAcq(GPUDynInstPtr gpuDynInst) override
580 {
581 // after the load has complete and if the load has acquire
582 // semantics, issue an acquire request.
583 if (!this->isLocalMem()) {
584 if (gpuDynInst->computeUnit()->shader->separate_acquire_release
585 && gpuDynInst->isAcquire()) {
586 gpuDynInst->statusBitVector = VectorMask(1);
587 gpuDynInst->useContinuation = false;
588 // create request
589 Request *req = new Request(0, 0, 0, 0,
590 gpuDynInst->computeUnit()->masterId(),
591 0, gpuDynInst->wfDynId);
592 req->setFlags(Request::ACQUIRE);
593 gpuDynInst->computeUnit()->injectGlobalMemFence(gpuDynInst, false, req);
594 }
595 }
596 }
597
598 public:
599 bool isVectorRegister(int operandIndex) override
600 {
601 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
602 if ((num_dest_operands != getNumOperands()) &&
603 (operandIndex == (getNumOperands()-1)))
604 return(this->addr.isVectorRegister());
605 if (num_dest_operands > 1) {
606 return dest_vect[operandIndex].isVectorRegister();
607 }
608 else if (num_dest_operands == 1) {
609 return LdInstBase<typename MemDataType::CType,
610 typename DestDataType::OperandType,
611 AddrOperandType>::dest.isVectorRegister();
612 }
613 return false;
614 }
615 bool isCondRegister(int operandIndex) override
616 {
617 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
618 if ((num_dest_operands != getNumOperands()) &&
619 (operandIndex == (getNumOperands()-1)))
620 return(this->addr.isCondRegister());
621 if (num_dest_operands > 1)
622 return dest_vect[operandIndex].isCondRegister();
623 else if (num_dest_operands == 1)
624 return LdInstBase<typename MemDataType::CType,
625 typename DestDataType::OperandType,
626 AddrOperandType>::dest.isCondRegister();
627 return false;
628 }
629 bool isScalarRegister(int operandIndex) override
630 {
631 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
632 if ((num_dest_operands != getNumOperands()) &&
633 (operandIndex == (getNumOperands()-1)))
634 return(this->addr.isScalarRegister());
635 if (num_dest_operands > 1)
636 return dest_vect[operandIndex].isScalarRegister();
637 else if (num_dest_operands == 1)
638 return LdInstBase<typename MemDataType::CType,
639 typename DestDataType::OperandType,
640 AddrOperandType>::dest.isScalarRegister();
641 return false;
642 }
643 bool isSrcOperand(int operandIndex) override
644 {
645 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
646 if ((num_dest_operands != getNumOperands()) &&
647 (operandIndex == (getNumOperands()-1)))
648 return(this->addr.isVectorRegister());
649 return false;
650 }
651 bool isDstOperand(int operandIndex) override
652 {
653 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
654 if ((num_dest_operands != getNumOperands()) &&
655 (operandIndex == (getNumOperands()-1)))
656 return false;
657 return true;
658 }
659 int getOperandSize(int operandIndex) override
660 {
661 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
662 if ((num_dest_operands != getNumOperands()) &&
663 (operandIndex == (getNumOperands()-1)))
664 return(this->addr.opSize());
665 if (num_dest_operands > 1)
666 return(dest_vect[operandIndex].opSize());
667 else if (num_dest_operands == 1)
668 return(LdInstBase<typename MemDataType::CType,
669 typename DestDataType::OperandType,
670 AddrOperandType>::dest.opSize());
671 return 0;
672 }
673 int getRegisterIndex(int operandIndex) override
674 {
675 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
676 if ((num_dest_operands != getNumOperands()) &&
677 (operandIndex == (getNumOperands()-1)))
678 return(this->addr.regIndex());
679 if (num_dest_operands > 1)
680 return(dest_vect[operandIndex].regIndex());
681 else if (num_dest_operands == 1)
682 return(LdInstBase<typename MemDataType::CType,
683 typename DestDataType::OperandType,
684 AddrOperandType>::dest.regIndex());
685 return -1;
686 }
687 int getNumOperands() override
688 {
689 if (this->addr.isVectorRegister() || this->addr.isScalarRegister())
690 return(num_dest_operands+1);
691 else
692 return(num_dest_operands);
693 }
694 void execute(GPUDynInstPtr gpuDynInst) override;
695 };
696
697 template<typename MemDT, typename DestDT>
698 GPUStaticInst*
699 decodeLd2(const Brig::BrigInstBase *ib, const BrigObject *obj)
700 {
701 unsigned op_offs = obj->getOperandPtr(ib->operands,1);
702 BrigRegOperandInfo tmp = findRegDataType(op_offs, obj);
703
704 if (tmp.kind == Brig::BRIG_KIND_OPERAND_ADDRESS) {
705 return new LdInst<MemDT, DestDT, NoRegAddrOperand>(ib, obj, "ld");
706 } else if (tmp.kind == Brig::BRIG_KIND_OPERAND_REGISTER ||
707 tmp.kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST) {
708 switch (tmp.regKind) {
709 case Brig::BRIG_REGISTER_KIND_SINGLE:
710 return new LdInst<MemDT, DestDT,
711 SRegAddrOperand>(ib, obj, "ld");
712 case Brig::BRIG_REGISTER_KIND_DOUBLE:
713 return new LdInst<MemDT, DestDT,
714 DRegAddrOperand>(ib, obj, "ld");
715 default:
716 fatal("Bad ld register operand type %d\n", tmp.regKind);
717 }
718 } else {
719 fatal("Bad ld register operand kind %d\n", tmp.kind);
720 }
721 }
722
723 template<typename MemDT>
724 GPUStaticInst*
725 decodeLd(const Brig::BrigInstBase *ib, const BrigObject *obj)
726 {
727 unsigned op_offs = obj->getOperandPtr(ib->operands,0);
728 BrigRegOperandInfo dest = findRegDataType(op_offs, obj);
729
730 assert(dest.kind == Brig::BRIG_KIND_OPERAND_REGISTER ||
731 dest.kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST);
732 switch(dest.regKind) {
733 case Brig::BRIG_REGISTER_KIND_SINGLE:
734 switch (ib->type) {
735 case Brig::BRIG_TYPE_B8:
736 case Brig::BRIG_TYPE_B16:
737 case Brig::BRIG_TYPE_B32:
738 return decodeLd2<MemDT, B32>(ib, obj);
739 case Brig::BRIG_TYPE_U8:
740 case Brig::BRIG_TYPE_U16:
741 case Brig::BRIG_TYPE_U32:
742 return decodeLd2<MemDT, U32>(ib, obj);
743 case Brig::BRIG_TYPE_S8:
744 case Brig::BRIG_TYPE_S16:
745 case Brig::BRIG_TYPE_S32:
746 return decodeLd2<MemDT, S32>(ib, obj);
747 case Brig::BRIG_TYPE_F16:
748 case Brig::BRIG_TYPE_F32:
749 return decodeLd2<MemDT, U32>(ib, obj);
750 default:
751 fatal("Bad ld register operand type %d, %d\n",
752 dest.regKind, ib->type);
753 };
754 case Brig::BRIG_REGISTER_KIND_DOUBLE:
755 switch (ib->type) {
756 case Brig::BRIG_TYPE_B64:
757 return decodeLd2<MemDT, B64>(ib, obj);
758 case Brig::BRIG_TYPE_U64:
759 return decodeLd2<MemDT, U64>(ib, obj);
760 case Brig::BRIG_TYPE_S64:
761 return decodeLd2<MemDT, S64>(ib, obj);
762 case Brig::BRIG_TYPE_F64:
763 return decodeLd2<MemDT, U64>(ib, obj);
764 default:
765 fatal("Bad ld register operand type %d, %d\n",
766 dest.regKind, ib->type);
767 };
768 default:
769 fatal("Bad ld register operand type %d, %d\n", dest.regKind,
770 ib->type);
771 }
772 }
773
774 template<typename MemDataType, typename SrcOperandType,
775 typename AddrOperandType>
776 class StInstBase : public HsailGPUStaticInst
777 {
778 public:
779 typename SrcOperandType::SrcOperand src;
780 AddrOperandType addr;
781
782 Brig::BrigSegment segment;
783 Brig::BrigMemoryScope memoryScope;
784 Brig::BrigMemoryOrder memoryOrder;
785 unsigned int equivClass;
786
787 StInstBase(const Brig::BrigInstBase *ib, const BrigObject *obj,
788 const char *_opcode)
789 : HsailGPUStaticInst(obj, _opcode)
790 {
791 using namespace Brig;
792
793 setFlag(MemoryRef);
794 setFlag(Store);
795
796 if (ib->opcode == BRIG_OPCODE_ST) {
797 const BrigInstMem *ldst = (const BrigInstMem*)ib;
798
799 segment = (BrigSegment)ldst->segment;
800 memoryOrder = BRIG_MEMORY_ORDER_NONE;
801 memoryScope = BRIG_MEMORY_SCOPE_NONE;
802 equivClass = ldst->equivClass;
803
804 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
805 const BrigOperand *baseOp = obj->getOperand(op_offs);
806
807 if ((baseOp->kind == BRIG_KIND_OPERAND_CONSTANT_BYTES) ||
808 (baseOp->kind == BRIG_KIND_OPERAND_REGISTER)) {
809 src.init(op_offs, obj);
810 }
811
812 op_offs = obj->getOperandPtr(ib->operands, 1);
813 addr.init(op_offs, obj);
814 } else {
815 const BrigInstAtomic *at = (const BrigInstAtomic*)ib;
816
817 segment = (BrigSegment)at->segment;
818 memoryScope = (BrigMemoryScope)at->memoryScope;
819 memoryOrder = (BrigMemoryOrder)at->memoryOrder;
820 equivClass = 0;
821
822 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
823 addr.init(op_offs, obj);
824
825 op_offs = obj->getOperandPtr(ib->operands, 1);
826 src.init(op_offs, obj);
827 }
828
829 switch (memoryOrder) {
830 case BRIG_MEMORY_ORDER_NONE:
831 setFlag(NoOrder);
832 break;
833 case BRIG_MEMORY_ORDER_RELAXED:
834 setFlag(RelaxedOrder);
835 break;
836 case BRIG_MEMORY_ORDER_SC_ACQUIRE:
837 setFlag(Acquire);
838 break;
839 case BRIG_MEMORY_ORDER_SC_RELEASE:
840 setFlag(Release);
841 break;
842 case BRIG_MEMORY_ORDER_SC_ACQUIRE_RELEASE:
843 setFlag(AcquireRelease);
844 break;
845 default:
846 fatal("StInst has bad memory order type\n");
847 }
848
849 switch (memoryScope) {
850 case BRIG_MEMORY_SCOPE_NONE:
851 setFlag(NoScope);
852 break;
853 case BRIG_MEMORY_SCOPE_WORKITEM:
854 setFlag(WorkitemScope);
855 break;
856 case BRIG_MEMORY_SCOPE_WORKGROUP:
857 setFlag(WorkgroupScope);
858 break;
859 case BRIG_MEMORY_SCOPE_AGENT:
860 setFlag(DeviceScope);
861 break;
862 case BRIG_MEMORY_SCOPE_SYSTEM:
863 setFlag(SystemScope);
864 break;
865 default:
866 fatal("StInst has bad memory scope type\n");
867 }
868
869 switch (segment) {
870 case BRIG_SEGMENT_GLOBAL:
871 setFlag(GlobalSegment);
872 break;
873 case BRIG_SEGMENT_GROUP:
874 setFlag(GroupSegment);
875 break;
876 case BRIG_SEGMENT_PRIVATE:
877 setFlag(PrivateSegment);
878 break;
879 case BRIG_SEGMENT_READONLY:
880 setFlag(ReadOnlySegment);
881 break;
882 case BRIG_SEGMENT_SPILL:
883 setFlag(SpillSegment);
884 break;
885 case BRIG_SEGMENT_FLAT:
886 setFlag(Flat);
887 break;
888 case BRIG_SEGMENT_ARG:
889 setFlag(ArgSegment);
890 break;
891 default:
892 panic("St: segment %d not supported\n", segment);
893 }
894 }
895
896 int numDstRegOperands() override { return 0; }
897 int numSrcRegOperands() override
898 {
899 return src.isVectorRegister() + this->addr.isVectorRegister();
900 }
901 int getNumOperands() override
902 {
903 if (this->addr.isVectorRegister() || this->addr.isScalarRegister())
904 return 2;
905 else
906 return 1;
907 }
908 bool isVectorRegister(int operandIndex) override
909 {
910 assert(operandIndex >= 0 && operandIndex < getNumOperands());
911 return !operandIndex ? src.isVectorRegister() :
912 this->addr.isVectorRegister();
913 }
914 bool isCondRegister(int operandIndex) override
915 {
916 assert(operandIndex >= 0 && operandIndex < getNumOperands());
917 return !operandIndex ? src.isCondRegister() :
918 this->addr.isCondRegister();
919 }
920 bool isScalarRegister(int operandIndex) override
921 {
922 assert(operandIndex >= 0 && operandIndex < getNumOperands());
923 return !operandIndex ? src.isScalarRegister() :
924 this->addr.isScalarRegister();
925 }
926 bool isSrcOperand(int operandIndex) override
927 {
928 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
929 return true;
930 }
931 bool isDstOperand(int operandIndex) override { return false; }
932 int getOperandSize(int operandIndex) override
933 {
934 assert(operandIndex >= 0 && operandIndex < getNumOperands());
935 return !operandIndex ? src.opSize() : this->addr.opSize();
936 }
937 int getRegisterIndex(int operandIndex) override
938 {
939 assert(operandIndex >= 0 && operandIndex < getNumOperands());
940 return !operandIndex ? src.regIndex() : this->addr.regIndex();
941 }
942 };
943
944
945 template<typename MemDataType, typename SrcDataType,
946 typename AddrOperandType>
947 class StInst :
948 public StInstBase<MemDataType, typename SrcDataType::OperandType,
949 AddrOperandType>,
950 public MemInst
951 {
952 public:
953 typename SrcDataType::OperandType::SrcOperand src_vect[4];
954 uint16_t num_src_operands;
955 void generateDisassembly() override;
956
957 StInst(const Brig::BrigInstBase *ib, const BrigObject *obj,
958 const char *_opcode, int srcIdx)
959 : StInstBase<MemDataType, typename SrcDataType::OperandType,
960 AddrOperandType>(ib, obj, _opcode),
961 MemInst(SrcDataType::memType)
962 {
963 init_addr(&this->addr);
964
965 BrigRegOperandInfo rinfo;
966 unsigned op_offs = obj->getOperandPtr(ib->operands,srcIdx);
967 const Brig::BrigOperand *baseOp = obj->getOperand(op_offs);
968
969 if (baseOp->kind == Brig::BRIG_KIND_OPERAND_CONSTANT_BYTES) {
970 const Brig::BrigOperandConstantBytes *op =
971 (Brig::BrigOperandConstantBytes*)baseOp;
972
973 rinfo = BrigRegOperandInfo((Brig::BrigKind16_t)op->base.kind,
974 Brig::BRIG_TYPE_NONE);
975 } else {
976 rinfo = findRegDataType(op_offs, obj);
977 }
978
979 if (baseOp->kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST) {
980 const Brig::BrigOperandOperandList *brigRegVecOp =
981 (const Brig::BrigOperandOperandList*)baseOp;
982
983 num_src_operands =
984 *((unsigned*)obj->getData(brigRegVecOp->elements)) / 4;
985
986 assert(num_src_operands <= 4);
987 } else {
988 num_src_operands = 1;
989 }
990
991 if (num_src_operands > 1) {
992 assert(baseOp->kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST);
993
994 for (int i = 0; i < num_src_operands; ++i) {
995 src_vect[i].init_from_vect(op_offs, obj, i);
996 }
997 }
998 }
999
1000 void
1001 initiateAcc(GPUDynInstPtr gpuDynInst) override
1002 {
1003 // before performing a store, check if this store has
1004 // release semantics, and if so issue a release first
1005 if (!this->isLocalMem()) {
1006 if (gpuDynInst->computeUnit()->shader->separate_acquire_release
1007 && gpuDynInst->isRelease()) {
1008
1009 gpuDynInst->statusBitVector = VectorMask(1);
1010 gpuDynInst->execContinuation = &GPUStaticInst::execSt;
1011 gpuDynInst->useContinuation = true;
1012 // create request
1013 Request *req = new Request(0, 0, 0, 0,
1014 gpuDynInst->computeUnit()->masterId(),
1015 0, gpuDynInst->wfDynId);
1016 req->setFlags(Request::RELEASE);
1017 gpuDynInst->computeUnit()->injectGlobalMemFence(gpuDynInst, false, req);
1018
1019 return;
1020 }
1021 }
1022
1023 // if there is no release semantic, perform stores immediately
1024 execSt(gpuDynInst);
1025 }
1026
1027 // stores don't write anything back, so there is nothing
1028 // to do here. we only override this method to avoid the
1029 // fatal in the base class implementation
1030 void completeAcc(GPUDynInstPtr gpuDynInst) override { }
1031
1032 private:
1033 // execSt may be called through a continuation
1034 // if the store had release semantics. see comment for
1035 // execSt in gpu_static_inst.hh
1036 void
1037 execSt(GPUDynInstPtr gpuDynInst) override
1038 {
1039 typedef typename MemDataType::CType c0;
1040
1041 gpuDynInst->statusBitVector = gpuDynInst->exec_mask;
1042
1043 if (num_src_operands > 1) {
1044 for (int i = 0; i < gpuDynInst->computeUnit()->wfSize(); ++i)
1045 if (gpuDynInst->exec_mask[i])
1046 gpuDynInst->statusVector.push_back(num_src_operands);
1047 else
1048 gpuDynInst->statusVector.push_back(0);
1049 }
1050
1051 for (int k = 0; k < num_src_operands; ++k) {
1052 c0 *d = &((c0*)gpuDynInst->d_data)
1053 [k * gpuDynInst->computeUnit()->wfSize()];
1054
1055 for (int i = 0; i < gpuDynInst->computeUnit()->wfSize(); ++i) {
1056 if (gpuDynInst->exec_mask[i]) {
1057 Addr vaddr = gpuDynInst->addr[i] + k * sizeof(c0);
1058
1059 if (this->isLocalMem()) {
1060 //store to shared memory
1061 gpuDynInst->wavefront()->ldsChunk->write<c0>(vaddr,
1062 *d);
1063 } else {
1064 Request *req =
1065 new Request(0, vaddr, sizeof(c0), 0,
1066 gpuDynInst->computeUnit()->masterId(),
1067 0, gpuDynInst->wfDynId);
1068
1069 gpuDynInst->setRequestFlags(req);
1070 PacketPtr pkt = new Packet(req, MemCmd::WriteReq);
1071 pkt->dataStatic<c0>(d);
1072
1073 // translation is performed in sendRequest()
1074 // the request will be finished when the store completes
1075 gpuDynInst->useContinuation = false;
1076 gpuDynInst->computeUnit()->sendRequest(gpuDynInst,
1077 i, pkt);
1078
1079 }
1080 }
1081 ++d;
1082 }
1083 }
1084
1085 gpuDynInst->updateStats();
1086 }
1087
1088 public:
1089 bool isVectorRegister(int operandIndex) override
1090 {
1091 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1092 if (operandIndex == num_src_operands)
1093 return this->addr.isVectorRegister();
1094 if (num_src_operands > 1)
1095 return src_vect[operandIndex].isVectorRegister();
1096 else if (num_src_operands == 1)
1097 return StInstBase<MemDataType,
1098 typename SrcDataType::OperandType,
1099 AddrOperandType>::src.isVectorRegister();
1100 return false;
1101 }
1102 bool isCondRegister(int operandIndex) override
1103 {
1104 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1105 if (operandIndex == num_src_operands)
1106 return this->addr.isCondRegister();
1107 if (num_src_operands > 1)
1108 return src_vect[operandIndex].isCondRegister();
1109 else if (num_src_operands == 1)
1110 return StInstBase<MemDataType,
1111 typename SrcDataType::OperandType,
1112 AddrOperandType>::src.isCondRegister();
1113 return false;
1114 }
1115 bool isScalarRegister(int operandIndex) override
1116 {
1117 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1118 if (operandIndex == num_src_operands)
1119 return this->addr.isScalarRegister();
1120 if (num_src_operands > 1)
1121 return src_vect[operandIndex].isScalarRegister();
1122 else if (num_src_operands == 1)
1123 return StInstBase<MemDataType,
1124 typename SrcDataType::OperandType,
1125 AddrOperandType>::src.isScalarRegister();
1126 return false;
1127 }
1128 bool isSrcOperand(int operandIndex) override
1129 {
1130 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1131 return true;
1132 }
1133 bool isDstOperand(int operandIndex) override { return false; }
1134 int getOperandSize(int operandIndex) override
1135 {
1136 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1137 if (operandIndex == num_src_operands)
1138 return this->addr.opSize();
1139 if (num_src_operands > 1)
1140 return src_vect[operandIndex].opSize();
1141 else if (num_src_operands == 1)
1142 return StInstBase<MemDataType,
1143 typename SrcDataType::OperandType,
1144 AddrOperandType>::src.opSize();
1145 return 0;
1146 }
1147 int getRegisterIndex(int operandIndex) override
1148 {
1149 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1150 if (operandIndex == num_src_operands)
1151 return this->addr.regIndex();
1152 if (num_src_operands > 1)
1153 return src_vect[operandIndex].regIndex();
1154 else if (num_src_operands == 1)
1155 return StInstBase<MemDataType,
1156 typename SrcDataType::OperandType,
1157 AddrOperandType>::src.regIndex();
1158 return -1;
1159 }
1160 int getNumOperands() override
1161 {
1162 if (this->addr.isVectorRegister() || this->addr.isScalarRegister())
1163 return num_src_operands + 1;
1164 else
1165 return num_src_operands;
1166 }
1167 void execute(GPUDynInstPtr gpuDynInst) override;
1168 };
1169
1170 template<typename DataType, typename SrcDataType>
1171 GPUStaticInst*
1172 decodeSt(const Brig::BrigInstBase *ib, const BrigObject *obj)
1173 {
1174 int srcIdx = 0;
1175 int destIdx = 1;
1176 if (ib->opcode == Brig::BRIG_OPCODE_ATOMIC ||
1177 ib->opcode == Brig::BRIG_OPCODE_ATOMICNORET) {
1178 srcIdx = 1;
1179 destIdx = 0;
1180 }
1181 unsigned op_offs = obj->getOperandPtr(ib->operands,destIdx);
1182
1183 BrigRegOperandInfo tmp = findRegDataType(op_offs, obj);
1184
1185 if (tmp.kind == Brig::BRIG_KIND_OPERAND_ADDRESS) {
1186 return new StInst<DataType, SrcDataType,
1187 NoRegAddrOperand>(ib, obj, "st", srcIdx);
1188 } else if (tmp.kind == Brig::BRIG_KIND_OPERAND_REGISTER) {
1189 // V2/V4 not allowed
1190 switch (tmp.regKind) {
1191 case Brig::BRIG_REGISTER_KIND_SINGLE:
1192 return new StInst<DataType, SrcDataType,
1193 SRegAddrOperand>(ib, obj, "st", srcIdx);
1194 case Brig::BRIG_REGISTER_KIND_DOUBLE:
1195 return new StInst<DataType, SrcDataType,
1196 DRegAddrOperand>(ib, obj, "st", srcIdx);
1197 default:
1198 fatal("Bad st register operand type %d\n", tmp.type);
1199 }
1200 } else {
1201 fatal("Bad st register operand kind %d\n", tmp.kind);
1202 }
1203 }
1204
1205 template<typename OperandType, typename AddrOperandType, int NumSrcOperands,
1206 bool HasDst>
1207 class AtomicInstBase : public HsailGPUStaticInst
1208 {
1209 public:
1210 typename OperandType::DestOperand dest;
1211 typename OperandType::SrcOperand src[NumSrcOperands];
1212 AddrOperandType addr;
1213
1214 Brig::BrigSegment segment;
1215 Brig::BrigMemoryOrder memoryOrder;
1216 Brig::BrigAtomicOperation atomicOperation;
1217 Brig::BrigMemoryScope memoryScope;
1218 Brig::BrigOpcode opcode;
1219
1220 AtomicInstBase(const Brig::BrigInstBase *ib, const BrigObject *obj,
1221 const char *_opcode)
1222 : HsailGPUStaticInst(obj, _opcode)
1223 {
1224 using namespace Brig;
1225
1226 const BrigInstAtomic *at = (const BrigInstAtomic*)ib;
1227
1228 segment = (BrigSegment)at->segment;
1229 memoryScope = (BrigMemoryScope)at->memoryScope;
1230 memoryOrder = (BrigMemoryOrder)at->memoryOrder;
1231 atomicOperation = (BrigAtomicOperation)at->atomicOperation;
1232 opcode = (BrigOpcode)ib->opcode;
1233
1234 assert(opcode == Brig::BRIG_OPCODE_ATOMICNORET ||
1235 opcode == Brig::BRIG_OPCODE_ATOMIC);
1236
1237 setFlag(MemoryRef);
1238
1239 if (opcode == Brig::BRIG_OPCODE_ATOMIC) {
1240 setFlag(AtomicReturn);
1241 } else {
1242 setFlag(AtomicNoReturn);
1243 }
1244
1245 switch (memoryOrder) {
1246 case BRIG_MEMORY_ORDER_NONE:
1247 setFlag(NoOrder);
1248 break;
1249 case BRIG_MEMORY_ORDER_RELAXED:
1250 setFlag(RelaxedOrder);
1251 break;
1252 case BRIG_MEMORY_ORDER_SC_ACQUIRE:
1253 setFlag(Acquire);
1254 break;
1255 case BRIG_MEMORY_ORDER_SC_RELEASE:
1256 setFlag(Release);
1257 break;
1258 case BRIG_MEMORY_ORDER_SC_ACQUIRE_RELEASE:
1259 setFlag(AcquireRelease);
1260 break;
1261 default:
1262 fatal("AtomicInst has bad memory order type\n");
1263 }
1264
1265 switch (memoryScope) {
1266 case BRIG_MEMORY_SCOPE_NONE:
1267 setFlag(NoScope);
1268 break;
1269 case BRIG_MEMORY_SCOPE_WORKITEM:
1270 setFlag(WorkitemScope);
1271 break;
1272 case BRIG_MEMORY_SCOPE_WORKGROUP:
1273 setFlag(WorkgroupScope);
1274 break;
1275 case BRIG_MEMORY_SCOPE_AGENT:
1276 setFlag(DeviceScope);
1277 break;
1278 case BRIG_MEMORY_SCOPE_SYSTEM:
1279 setFlag(SystemScope);
1280 break;
1281 default:
1282 fatal("AtomicInst has bad memory scope type\n");
1283 }
1284
1285 switch (atomicOperation) {
1286 case Brig::BRIG_ATOMIC_AND:
1287 setFlag(AtomicAnd);
1288 break;
1289 case Brig::BRIG_ATOMIC_OR:
1290 setFlag(AtomicOr);
1291 break;
1292 case Brig::BRIG_ATOMIC_XOR:
1293 setFlag(AtomicXor);
1294 break;
1295 case Brig::BRIG_ATOMIC_CAS:
1296 setFlag(AtomicCAS);
1297 break;
1298 case Brig::BRIG_ATOMIC_EXCH:
1299 setFlag(AtomicExch);
1300 break;
1301 case Brig::BRIG_ATOMIC_ADD:
1302 setFlag(AtomicAdd);
1303 break;
1304 case Brig::BRIG_ATOMIC_WRAPINC:
1305 setFlag(AtomicInc);
1306 break;
1307 case Brig::BRIG_ATOMIC_WRAPDEC:
1308 setFlag(AtomicDec);
1309 break;
1310 case Brig::BRIG_ATOMIC_MIN:
1311 setFlag(AtomicMin);
1312 break;
1313 case Brig::BRIG_ATOMIC_MAX:
1314 setFlag(AtomicMax);
1315 break;
1316 case Brig::BRIG_ATOMIC_SUB:
1317 setFlag(AtomicSub);
1318 break;
1319 default:
1320 fatal("Bad BrigAtomicOperation code %d\n", atomicOperation);
1321 }
1322
1323 switch (segment) {
1324 case BRIG_SEGMENT_GLOBAL:
1325 setFlag(GlobalSegment);
1326 break;
1327 case BRIG_SEGMENT_GROUP:
1328 setFlag(GroupSegment);
1329 break;
1330 case BRIG_SEGMENT_FLAT:
1331 setFlag(Flat);
1332 break;
1333 default:
1334 panic("Atomic: segment %d not supported\n", segment);
1335 }
1336
1337 if (HasDst) {
1338 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
1339 dest.init(op_offs, obj);
1340
1341 op_offs = obj->getOperandPtr(ib->operands, 1);
1342 addr.init(op_offs, obj);
1343
1344 for (int i = 0; i < NumSrcOperands; ++i) {
1345 op_offs = obj->getOperandPtr(ib->operands, i + 2);
1346 src[i].init(op_offs, obj);
1347 }
1348 } else {
1349
1350 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
1351 addr.init(op_offs, obj);
1352
1353 for (int i = 0; i < NumSrcOperands; ++i) {
1354 op_offs = obj->getOperandPtr(ib->operands, i + 1);
1355 src[i].init(op_offs, obj);
1356 }
1357 }
1358 }
1359
1360 int numSrcRegOperands()
1361 {
1362 int operands = 0;
1363 for (int i = 0; i < NumSrcOperands; i++) {
1364 if (src[i].isVectorRegister()) {
1365 operands++;
1366 }
1367 }
1368 if (addr.isVectorRegister())
1369 operands++;
1370 return operands;
1371 }
1372 int numDstRegOperands() { return dest.isVectorRegister(); }
1373 int getNumOperands()
1374 {
1375 if (addr.isVectorRegister())
1376 return(NumSrcOperands + 2);
1377 return(NumSrcOperands + 1);
1378 }
1379 bool isVectorRegister(int operandIndex)
1380 {
1381 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1382 if (operandIndex < NumSrcOperands)
1383 return src[operandIndex].isVectorRegister();
1384 else if (operandIndex == NumSrcOperands)
1385 return(addr.isVectorRegister());
1386 else
1387 return dest.isVectorRegister();
1388 }
1389 bool isCondRegister(int operandIndex)
1390 {
1391 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1392 if (operandIndex < NumSrcOperands)
1393 return src[operandIndex].isCondRegister();
1394 else if (operandIndex == NumSrcOperands)
1395 return(addr.isCondRegister());
1396 else
1397 return dest.isCondRegister();
1398 }
1399 bool isScalarRegister(int operandIndex)
1400 {
1401 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1402 if (operandIndex < NumSrcOperands)
1403 return src[operandIndex].isScalarRegister();
1404 else if (operandIndex == NumSrcOperands)
1405 return(addr.isScalarRegister());
1406 else
1407 return dest.isScalarRegister();
1408 }
1409 bool isSrcOperand(int operandIndex)
1410 {
1411 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1412 if (operandIndex < NumSrcOperands)
1413 return true;
1414 else if (operandIndex == NumSrcOperands)
1415 return(addr.isVectorRegister());
1416 else
1417 return false;
1418 }
1419 bool isDstOperand(int operandIndex)
1420 {
1421 if (operandIndex <= NumSrcOperands)
1422 return false;
1423 else
1424 return true;
1425 }
1426 int getOperandSize(int operandIndex)
1427 {
1428 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1429 if (operandIndex < NumSrcOperands)
1430 return(src[operandIndex].opSize());
1431 else if (operandIndex == NumSrcOperands)
1432 return(addr.opSize());
1433 else
1434 return(dest.opSize());
1435 }
1436 int getRegisterIndex(int operandIndex)
1437 {
1438 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1439 if (operandIndex < NumSrcOperands)
1440 return(src[operandIndex].regIndex());
1441 else if (operandIndex == NumSrcOperands)
1442 return(addr.regIndex());
1443 else
1444 return(dest.regIndex());
1445 return -1;
1446 }
1447 };
1448
1449 template<typename MemDataType, typename AddrOperandType, int NumSrcOperands,
1450 bool HasDst>
1451 class AtomicInst :
1452 public AtomicInstBase<typename MemDataType::OperandType,
1453 AddrOperandType, NumSrcOperands, HasDst>,
1454 public MemInst
1455 {
1456 public:
1457 void generateDisassembly() override;
1458
1459 AtomicInst(const Brig::BrigInstBase *ib, const BrigObject *obj,
1460 const char *_opcode)
1461 : AtomicInstBase<typename MemDataType::OperandType, AddrOperandType,
1462 NumSrcOperands, HasDst>
1463 (ib, obj, _opcode),
1464 MemInst(MemDataType::memType)
1465 {
1466 init_addr(&this->addr);
1467 }
1468
1469 void
1470 initiateAcc(GPUDynInstPtr gpuDynInst) override
1471 {
1472 // before doing the RMW, check if this atomic has
1473 // release semantics, and if so issue a release first
1474 if (!this->isLocalMem()) {
1475 if (gpuDynInst->computeUnit()->shader->separate_acquire_release
1476 && (gpuDynInst->isRelease()
1477 || gpuDynInst->isAcquireRelease())) {
1478
1479 gpuDynInst->statusBitVector = VectorMask(1);
1480
1481 gpuDynInst->execContinuation = &GPUStaticInst::execAtomic;
1482 gpuDynInst->useContinuation = true;
1483
1484 // create request
1485 Request *req = new Request(0, 0, 0, 0,
1486 gpuDynInst->computeUnit()->masterId(),
1487 0, gpuDynInst->wfDynId);
1488 req->setFlags(Request::RELEASE);
1489 gpuDynInst->computeUnit()->injectGlobalMemFence(gpuDynInst, false, req);
1490
1491 return;
1492 }
1493 }
1494
1495 // if there is no release semantic, execute the RMW immediately
1496 execAtomic(gpuDynInst);
1497
1498 }
1499
1500 void
1501 completeAcc(GPUDynInstPtr gpuDynInst) override
1502 {
1503 // if this is not an atomic return op, then we
1504 // have nothing more to do.
1505 if (this->isAtomicRet()) {
1506 // the size of the src operands and the
1507 // memory being operated on must match
1508 // for HSAIL atomics - this assumption may
1509 // not apply to all ISAs
1510 typedef typename MemDataType::CType CType;
1511
1512 Wavefront *w = gpuDynInst->wavefront();
1513 int dst = this->dest.regIndex();
1514 std::vector<uint32_t> regVec;
1515 // virtual->physical VGPR mapping
1516 int physVgpr = w->remap(dst, sizeof(CType), 1);
1517 regVec.push_back(physVgpr);
1518 CType *p1 = &((CType*)gpuDynInst->d_data)[0];
1519
1520 for (int i = 0; i < w->computeUnit->wfSize(); ++i) {
1521 if (gpuDynInst->exec_mask[i]) {
1522 DPRINTF(GPUReg, "CU%d, WF[%d][%d], lane %d: "
1523 "$%s%d <- %d global ld done (src = wavefront "
1524 "ld inst)\n", w->computeUnit->cu_id, w->simdId,
1525 w->wfSlotId, i, sizeof(CType) == 4 ? "s" : "d",
1526 dst, *p1);
1527 // write the value into the physical VGPR. This is a
1528 // purely functional operation. No timing is modeled.
1529 w->computeUnit->vrf[w->simdId]->write<CType>(physVgpr, *p1, i);
1530 }
1531 ++p1;
1532 }
1533
1534 // Schedule the write operation of the load data on the VRF.
1535 // This simply models the timing aspect of the VRF write operation.
1536 // It does not modify the physical VGPR.
1537 int loadVrfBankConflictCycles = gpuDynInst->computeUnit()->
1538 vrf[w->simdId]->exec(gpuDynInst->seqNum(), w, regVec,
1539 sizeof(CType), gpuDynInst->time);
1540
1541 if (this->isGlobalMem()) {
1542 gpuDynInst->computeUnit()->globalMemoryPipe
1543 .incLoadVRFBankConflictCycles(loadVrfBankConflictCycles);
1544 } else {
1545 assert(this->isLocalMem());
1546 gpuDynInst->computeUnit()->localMemoryPipe
1547 .incLoadVRFBankConflictCycles(loadVrfBankConflictCycles);
1548 }
1549 }
1550 }
1551
1552 void execute(GPUDynInstPtr gpuDynInst) override;
1553
1554 private:
1555 // execAtomic may be called through a continuation
1556 // if the RMW had release semantics. see comment for
1557 // execContinuation in gpu_dyn_inst.hh
1558 void
1559 execAtomic(GPUDynInstPtr gpuDynInst) override
1560 {
1561 gpuDynInst->statusBitVector = gpuDynInst->exec_mask;
1562
1563 typedef typename MemDataType::CType c0;
1564
1565 c0 *d = &((c0*) gpuDynInst->d_data)[0];
1566 c0 *e = &((c0*) gpuDynInst->a_data)[0];
1567 c0 *f = &((c0*) gpuDynInst->x_data)[0];
1568
1569 for (int i = 0; i < gpuDynInst->computeUnit()->wfSize(); ++i) {
1570 if (gpuDynInst->exec_mask[i]) {
1571 Addr vaddr = gpuDynInst->addr[i];
1572
1573 if (this->isLocalMem()) {
1574 Wavefront *wavefront = gpuDynInst->wavefront();
1575 *d = wavefront->ldsChunk->read<c0>(vaddr);
1576
1577 if (this->isAtomicAdd()) {
1578 wavefront->ldsChunk->write<c0>(vaddr,
1579 wavefront->ldsChunk->read<c0>(vaddr) + (*e));
1580 } else if (this->isAtomicSub()) {
1581 wavefront->ldsChunk->write<c0>(vaddr,
1582 wavefront->ldsChunk->read<c0>(vaddr) - (*e));
1583 } else if (this->isAtomicMax()) {
1584 wavefront->ldsChunk->write<c0>(vaddr,
1585 std::max(wavefront->ldsChunk->read<c0>(vaddr),
1586 (*e)));
1587 } else if (this->isAtomicMin()) {
1588 wavefront->ldsChunk->write<c0>(vaddr,
1589 std::min(wavefront->ldsChunk->read<c0>(vaddr),
1590 (*e)));
1591 } else if (this->isAtomicAnd()) {
1592 wavefront->ldsChunk->write<c0>(vaddr,
1593 wavefront->ldsChunk->read<c0>(vaddr) & (*e));
1594 } else if (this->isAtomicOr()) {
1595 wavefront->ldsChunk->write<c0>(vaddr,
1596 wavefront->ldsChunk->read<c0>(vaddr) | (*e));
1597 } else if (this->isAtomicXor()) {
1598 wavefront->ldsChunk->write<c0>(vaddr,
1599 wavefront->ldsChunk->read<c0>(vaddr) ^ (*e));
1600 } else if (this->isAtomicInc()) {
1601 wavefront->ldsChunk->write<c0>(vaddr,
1602 wavefront->ldsChunk->read<c0>(vaddr) + 1);
1603 } else if (this->isAtomicDec()) {
1604 wavefront->ldsChunk->write<c0>(vaddr,
1605 wavefront->ldsChunk->read<c0>(vaddr) - 1);
1606 } else if (this->isAtomicExch()) {
1607 wavefront->ldsChunk->write<c0>(vaddr, (*e));
1608 } else if (this->isAtomicCAS()) {
1609 wavefront->ldsChunk->write<c0>(vaddr,
1610 (wavefront->ldsChunk->read<c0>(vaddr) == (*e)) ?
1611 (*f) : wavefront->ldsChunk->read<c0>(vaddr));
1612 } else {
1613 fatal("Unrecognized or invalid HSAIL atomic op "
1614 "type.\n");
1615 }
1616 } else {
1617 Request *req =
1618 new Request(0, vaddr, sizeof(c0), 0,
1619 gpuDynInst->computeUnit()->masterId(),
1620 0, gpuDynInst->wfDynId,
1621 gpuDynInst->makeAtomicOpFunctor<c0>(e,
1622 f));
1623
1624 gpuDynInst->setRequestFlags(req);
1625 PacketPtr pkt = new Packet(req, MemCmd::SwapReq);
1626 pkt->dataStatic(d);
1627
1628 if (gpuDynInst->computeUnit()->shader->
1629 separate_acquire_release &&
1630 (gpuDynInst->isAcquire())) {
1631 // if this atomic has acquire semantics,
1632 // schedule the continuation to perform an
1633 // acquire after the RMW completes
1634 gpuDynInst->execContinuation =
1635 &GPUStaticInst::execAtomicAcq;
1636
1637 gpuDynInst->useContinuation = true;
1638 } else {
1639 // the request will be finished when the RMW completes
1640 gpuDynInst->useContinuation = false;
1641 }
1642 // translation is performed in sendRequest()
1643 gpuDynInst->computeUnit()->sendRequest(gpuDynInst, i,
1644 pkt);
1645 }
1646 }
1647
1648 ++d;
1649 ++e;
1650 ++f;
1651 }
1652
1653 gpuDynInst->updateStats();
1654 }
1655
1656 // execAtomicACq will always be called through a continuation.
1657 // see comment for execContinuation in gpu_dyn_inst.hh
1658 void
1659 execAtomicAcq(GPUDynInstPtr gpuDynInst) override
1660 {
1661 // after performing the RMW, check to see if this instruction
1662 // has acquire semantics, and if so, issue an acquire
1663 if (!this->isLocalMem()) {
1664 if (gpuDynInst->computeUnit()->shader->separate_acquire_release
1665 && gpuDynInst->isAcquire()) {
1666 gpuDynInst->statusBitVector = VectorMask(1);
1667
1668 // the request will be finished when
1669 // the acquire completes
1670 gpuDynInst->useContinuation = false;
1671 // create request
1672 Request *req = new Request(0, 0, 0, 0,
1673 gpuDynInst->computeUnit()->masterId(),
1674 0, gpuDynInst->wfDynId);
1675 req->setFlags(Request::ACQUIRE);
1676 gpuDynInst->computeUnit()->injectGlobalMemFence(gpuDynInst, false, req);
1677 }
1678 }
1679 }
1680 };
1681
1682 template<typename DataType, typename AddrOperandType, int NumSrcOperands>
1683 GPUStaticInst*
1684 constructAtomic(const Brig::BrigInstBase *ib, const BrigObject *obj)
1685 {
1686 const Brig::BrigInstAtomic *at = (const Brig::BrigInstAtomic*)ib;
1687
1688 if (at->atomicOperation == Brig::BRIG_ATOMIC_LD) {
1689 return decodeLd<DataType>(ib, obj);
1690 } else if (at->atomicOperation == Brig::BRIG_ATOMIC_ST) {
1691 switch (ib->type) {
1692 case Brig::BRIG_TYPE_B8:
1693 return decodeSt<S8,S8>(ib, obj);
1694 case Brig::BRIG_TYPE_B16:
1695 return decodeSt<S16,S16>(ib, obj);
1696 case Brig::BRIG_TYPE_B32:
1697 return decodeSt<S32,S32>(ib, obj);
1698 case Brig::BRIG_TYPE_B64:
1699 return decodeSt<S64,S64>(ib, obj);
1700 default: fatal("AtomicSt: Operand type mismatch %d\n", ib->type);
1701 }
1702 } else {
1703 if ((Brig::BrigOpcode)ib->opcode == Brig::BRIG_OPCODE_ATOMICNORET)
1704 return new AtomicInst<DataType, AddrOperandType,
1705 NumSrcOperands, false>(ib, obj, "atomicnoret");
1706 else
1707 return new AtomicInst<DataType, AddrOperandType,
1708 NumSrcOperands, true>(ib, obj, "atomic");
1709 }
1710 }
1711
1712 template<typename DataType, int NumSrcOperands>
1713 GPUStaticInst*
1714 decodeAtomicHelper(const Brig::BrigInstBase *ib, const BrigObject *obj)
1715 {
1716 unsigned addrIndex = (Brig::BrigOpcode)ib->opcode ==
1717 Brig::BRIG_OPCODE_ATOMICNORET ? 0 : 1;
1718
1719 unsigned op_offs = obj->getOperandPtr(ib->operands,addrIndex);
1720
1721 BrigRegOperandInfo tmp = findRegDataType(op_offs, obj);
1722
1723 if (tmp.kind == Brig::BRIG_KIND_OPERAND_ADDRESS) {
1724 return constructAtomic<DataType, NoRegAddrOperand,
1725 NumSrcOperands>(ib, obj);
1726 } else if (tmp.kind == Brig::BRIG_KIND_OPERAND_REGISTER) {
1727 // V2/V4 not allowed
1728 switch (tmp.regKind) {
1729 case Brig::BRIG_REGISTER_KIND_SINGLE:
1730 return constructAtomic<DataType, SRegAddrOperand,
1731 NumSrcOperands>(ib, obj);
1732 case Brig::BRIG_REGISTER_KIND_DOUBLE:
1733 return constructAtomic<DataType, DRegAddrOperand,
1734 NumSrcOperands>(ib, obj);
1735 default:
1736 fatal("Bad atomic register operand type %d\n", tmp.type);
1737 }
1738 } else {
1739 fatal("Bad atomic register operand kind %d\n", tmp.kind);
1740 }
1741 }
1742
1743
1744 template<typename DataType>
1745 GPUStaticInst*
1746 decodeAtomic(const Brig::BrigInstBase *ib, const BrigObject *obj)
1747 {
1748 const Brig::BrigInstAtomic *at = (const Brig::BrigInstAtomic*)ib;
1749
1750 if (at->atomicOperation == Brig::BRIG_ATOMIC_CAS) {
1751 return decodeAtomicHelper<DataType, 2>(ib, obj);
1752 } else {
1753 return decodeAtomicHelper<DataType, 1>(ib, obj);
1754 }
1755 }
1756
1757 template<typename DataType>
1758 GPUStaticInst*
1759 decodeAtomicNoRet(const Brig::BrigInstBase *ib, const BrigObject *obj)
1760 {
1761 const Brig::BrigInstAtomic *at = (const Brig::BrigInstAtomic*)ib;
1762 if (at->atomicOperation == Brig::BRIG_ATOMIC_CAS) {
1763 return decodeAtomicHelper<DataType, 2>(ib, obj);
1764 } else {
1765 return decodeAtomicHelper<DataType, 1>(ib, obj);
1766 }
1767 }
1768} // namespace HsailISA
1769
1770#endif // __ARCH_HSAIL_INSTS_MEM_HH__
2 * Copyright (c) 2012-2015 Advanced Micro Devices, Inc.
3 * All rights reserved.
4 *
5 * For use for simulation and test purposes only
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright notice,
14 * this list of conditions and the following disclaimer in the documentation
15 * and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the copyright holder nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 *
33 * Author: Steve Reinhardt
34 */
35
36#ifndef __ARCH_HSAIL_INSTS_MEM_HH__
37#define __ARCH_HSAIL_INSTS_MEM_HH__
38
39#include <type_traits>
40
41#include "arch/hsail/insts/decl.hh"
42#include "arch/hsail/insts/gpu_static_inst.hh"
43#include "arch/hsail/operand.hh"
44#include "gpu-compute/compute_unit.hh"
45
46namespace HsailISA
47{
48 class MemInst
49 {
50 public:
51 MemInst() : size(0), addr_operand(nullptr) { }
52
53 MemInst(Enums::MemType m_type)
54 {
55 if (m_type == Enums::M_U64 ||
56 m_type == Enums::M_S64 ||
57 m_type == Enums::M_F64) {
58 size = 8;
59 } else if (m_type == Enums::M_U32 ||
60 m_type == Enums::M_S32 ||
61 m_type == Enums::M_F32) {
62 size = 4;
63 } else if (m_type == Enums::M_U16 ||
64 m_type == Enums::M_S16 ||
65 m_type == Enums::M_F16) {
66 size = 2;
67 } else {
68 size = 1;
69 }
70
71 addr_operand = nullptr;
72 }
73
74 void
75 init_addr(AddrOperandBase *_addr_operand)
76 {
77 addr_operand = _addr_operand;
78 }
79
80 private:
81 int size;
82 AddrOperandBase *addr_operand;
83
84 public:
85 int getMemOperandSize() { return size; }
86 AddrOperandBase *getAddressOperand() { return addr_operand; }
87 };
88
89 template<typename DestOperandType, typename AddrOperandType>
90 class LdaInstBase : public HsailGPUStaticInst
91 {
92 public:
93 typename DestOperandType::DestOperand dest;
94 AddrOperandType addr;
95
96 LdaInstBase(const Brig::BrigInstBase *ib, const BrigObject *obj,
97 const char *_opcode)
98 : HsailGPUStaticInst(obj, _opcode)
99 {
100 using namespace Brig;
101
102 setFlag(ALU);
103
104 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
105 dest.init(op_offs, obj);
106 op_offs = obj->getOperandPtr(ib->operands, 1);
107 addr.init(op_offs, obj);
108 }
109
110 int numSrcRegOperands() override
111 { return(this->addr.isVectorRegister()); }
112 int numDstRegOperands() override
113 { return dest.isVectorRegister(); }
114 bool isVectorRegister(int operandIndex) override
115 {
116 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
117 return((operandIndex == 0) ? dest.isVectorRegister() :
118 this->addr.isVectorRegister());
119 }
120 bool isCondRegister(int operandIndex) override
121 {
122 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
123 return((operandIndex == 0) ? dest.isCondRegister() :
124 this->addr.isCondRegister());
125 }
126 bool isScalarRegister(int operandIndex) override
127 {
128 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
129 return((operandIndex == 0) ? dest.isScalarRegister() :
130 this->addr.isScalarRegister());
131 }
132 bool isSrcOperand(int operandIndex) override
133 {
134 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
135 if (operandIndex > 0)
136 return(this->addr.isVectorRegister());
137 return false;
138 }
139 bool isDstOperand(int operandIndex) override {
140 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
141 return(operandIndex == 0);
142 }
143 int getOperandSize(int operandIndex) override
144 {
145 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
146 return((operandIndex == 0) ? dest.opSize() :
147 this->addr.opSize());
148 }
149 int getRegisterIndex(int operandIndex) override
150 {
151 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
152 return((operandIndex == 0) ? dest.regIndex() :
153 this->addr.regIndex());
154 }
155 int getNumOperands() override
156 {
157 if (this->addr.isVectorRegister())
158 return 2;
159 return 1;
160 }
161 };
162
163 template<typename DestDataType, typename AddrOperandType>
164 class LdaInst :
165 public LdaInstBase<typename DestDataType::OperandType, AddrOperandType>,
166 public MemInst
167 {
168 public:
169 void generateDisassembly();
170
171 LdaInst(const Brig::BrigInstBase *ib, const BrigObject *obj,
172 const char *_opcode)
173 : LdaInstBase<typename DestDataType::OperandType,
174 AddrOperandType>(ib, obj, _opcode)
175 {
176 init_addr(&this->addr);
177 }
178
179 void execute(GPUDynInstPtr gpuDynInst);
180 };
181
182 template<typename DataType>
183 GPUStaticInst*
184 decodeLda(const Brig::BrigInstBase *ib, const BrigObject *obj)
185 {
186 unsigned op_offs = obj->getOperandPtr(ib->operands, 1);
187 BrigRegOperandInfo regDataType = findRegDataType(op_offs, obj);
188
189 if (regDataType.kind == Brig::BRIG_KIND_OPERAND_ADDRESS) {
190 return new LdaInst<DataType, NoRegAddrOperand>(ib, obj, "ldas");
191 } else if (regDataType.kind == Brig::BRIG_KIND_OPERAND_REGISTER) {
192 // V2/V4 not allowed
193 switch (regDataType.regKind) {
194 case Brig::BRIG_REGISTER_KIND_SINGLE:
195 return new LdaInst<DataType, SRegAddrOperand>(ib, obj, "ldas");
196 case Brig::BRIG_REGISTER_KIND_DOUBLE:
197 return new LdaInst<DataType, DRegAddrOperand>(ib, obj, "ldas");
198 default:
199 fatal("Bad ldas register operand type %d\n", regDataType.type);
200 }
201 } else {
202 fatal("Bad ldas register operand kind %d\n", regDataType.kind);
203 }
204 }
205
206 template<typename MemOperandType, typename DestOperandType,
207 typename AddrOperandType>
208 class LdInstBase : public HsailGPUStaticInst
209 {
210 public:
211 Brig::BrigWidth8_t width;
212 typename DestOperandType::DestOperand dest;
213 AddrOperandType addr;
214
215 Brig::BrigSegment segment;
216 Brig::BrigMemoryOrder memoryOrder;
217 Brig::BrigMemoryScope memoryScope;
218 unsigned int equivClass;
219
220 LdInstBase(const Brig::BrigInstBase *ib, const BrigObject *obj,
221 const char *_opcode)
222 : HsailGPUStaticInst(obj, _opcode)
223 {
224 using namespace Brig;
225
226 setFlag(MemoryRef);
227 setFlag(Load);
228
229 if (ib->opcode == BRIG_OPCODE_LD) {
230 const BrigInstMem *ldst = (const BrigInstMem*)ib;
231
232 segment = (BrigSegment)ldst->segment;
233 memoryOrder = BRIG_MEMORY_ORDER_NONE;
234 memoryScope = BRIG_MEMORY_SCOPE_NONE;
235 equivClass = ldst->equivClass;
236
237 width = ldst->width;
238 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
239 const Brig::BrigOperand *brigOp = obj->getOperand(op_offs);
240 if (brigOp->kind == BRIG_KIND_OPERAND_REGISTER)
241 dest.init(op_offs, obj);
242
243 op_offs = obj->getOperandPtr(ib->operands, 1);
244 addr.init(op_offs, obj);
245 } else {
246 const BrigInstAtomic *at = (const BrigInstAtomic*)ib;
247
248 segment = (BrigSegment)at->segment;
249 memoryOrder = (BrigMemoryOrder)at->memoryOrder;
250 memoryScope = (BrigMemoryScope)at->memoryScope;
251 equivClass = 0;
252
253 width = BRIG_WIDTH_1;
254 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
255 const Brig::BrigOperand *brigOp = obj->getOperand(op_offs);
256
257 if (brigOp->kind == BRIG_KIND_OPERAND_REGISTER)
258 dest.init(op_offs, obj);
259
260 op_offs = obj->getOperandPtr(ib->operands,1);
261 addr.init(op_offs, obj);
262 }
263
264 switch (memoryOrder) {
265 case BRIG_MEMORY_ORDER_NONE:
266 setFlag(NoOrder);
267 break;
268 case BRIG_MEMORY_ORDER_RELAXED:
269 setFlag(RelaxedOrder);
270 break;
271 case BRIG_MEMORY_ORDER_SC_ACQUIRE:
272 setFlag(Acquire);
273 break;
274 case BRIG_MEMORY_ORDER_SC_RELEASE:
275 setFlag(Release);
276 break;
277 case BRIG_MEMORY_ORDER_SC_ACQUIRE_RELEASE:
278 setFlag(AcquireRelease);
279 break;
280 default:
281 fatal("LdInst has bad memory order type\n");
282 }
283
284 switch (memoryScope) {
285 case BRIG_MEMORY_SCOPE_NONE:
286 setFlag(NoScope);
287 break;
288 case BRIG_MEMORY_SCOPE_WORKITEM:
289 setFlag(WorkitemScope);
290 break;
291 case BRIG_MEMORY_SCOPE_WORKGROUP:
292 setFlag(WorkgroupScope);
293 break;
294 case BRIG_MEMORY_SCOPE_AGENT:
295 setFlag(DeviceScope);
296 break;
297 case BRIG_MEMORY_SCOPE_SYSTEM:
298 setFlag(SystemScope);
299 break;
300 default:
301 fatal("LdInst has bad memory scope type\n");
302 }
303
304 switch (segment) {
305 case BRIG_SEGMENT_GLOBAL:
306 setFlag(GlobalSegment);
307 break;
308 case BRIG_SEGMENT_GROUP:
309 setFlag(GroupSegment);
310 break;
311 case BRIG_SEGMENT_PRIVATE:
312 setFlag(PrivateSegment);
313 break;
314 case BRIG_SEGMENT_READONLY:
315 setFlag(ReadOnlySegment);
316 break;
317 case BRIG_SEGMENT_SPILL:
318 setFlag(SpillSegment);
319 break;
320 case BRIG_SEGMENT_FLAT:
321 setFlag(Flat);
322 break;
323 case BRIG_SEGMENT_KERNARG:
324 setFlag(KernArgSegment);
325 break;
326 case BRIG_SEGMENT_ARG:
327 setFlag(ArgSegment);
328 break;
329 default:
330 panic("Ld: segment %d not supported\n", segment);
331 }
332 }
333
334 int numSrcRegOperands() override
335 { return(this->addr.isVectorRegister()); }
336 int numDstRegOperands() override { return dest.isVectorRegister(); }
337 int getNumOperands() override
338 {
339 if (this->addr.isVectorRegister())
340 return 2;
341 else
342 return 1;
343 }
344 bool isVectorRegister(int operandIndex) override
345 {
346 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
347 return((operandIndex == 0) ? dest.isVectorRegister() :
348 this->addr.isVectorRegister());
349 }
350 bool isCondRegister(int operandIndex) override
351 {
352 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
353 return((operandIndex == 0) ? dest.isCondRegister() :
354 this->addr.isCondRegister());
355 }
356 bool isScalarRegister(int operandIndex) override
357 {
358 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
359 return((operandIndex == 0) ? dest.isScalarRegister() :
360 this->addr.isScalarRegister());
361 }
362 bool isSrcOperand(int operandIndex) override
363 {
364 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
365 if (operandIndex > 0)
366 return(this->addr.isVectorRegister());
367 return false;
368 }
369 bool isDstOperand(int operandIndex) override
370 {
371 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
372 return(operandIndex == 0);
373 }
374 int getOperandSize(int operandIndex) override
375 {
376 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
377 return((operandIndex == 0) ? dest.opSize() :
378 this->addr.opSize());
379 }
380 int getRegisterIndex(int operandIndex) override
381 {
382 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
383 return((operandIndex == 0) ? dest.regIndex() :
384 this->addr.regIndex());
385 }
386 };
387
388 template<typename MemDataType, typename DestDataType,
389 typename AddrOperandType>
390 class LdInst :
391 public LdInstBase<typename MemDataType::CType,
392 typename DestDataType::OperandType, AddrOperandType>,
393 public MemInst
394 {
395 typename DestDataType::OperandType::DestOperand dest_vect[4];
396 uint16_t num_dest_operands;
397 void generateDisassembly() override;
398
399 public:
400 LdInst(const Brig::BrigInstBase *ib, const BrigObject *obj,
401 const char *_opcode)
402 : LdInstBase<typename MemDataType::CType,
403 typename DestDataType::OperandType,
404 AddrOperandType>(ib, obj, _opcode),
405 MemInst(MemDataType::memType)
406 {
407 init_addr(&this->addr);
408
409 unsigned op_offs = obj->getOperandPtr(ib->operands,0);
410 const Brig::BrigOperand *brigOp = obj->getOperand(op_offs);
411
412 if (brigOp->kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST) {
413 const Brig::BrigOperandOperandList *brigRegVecOp =
414 (const Brig::BrigOperandOperandList*)brigOp;
415
416 num_dest_operands =
417 *((unsigned*)obj->getData(brigRegVecOp->elements)) / 4;
418
419 assert(num_dest_operands <= 4);
420 } else {
421 num_dest_operands = 1;
422 }
423
424 if (num_dest_operands > 1) {
425 assert(brigOp->kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST);
426
427 for (int i = 0; i < num_dest_operands; ++i) {
428 dest_vect[i].init_from_vect(op_offs, obj, i);
429 }
430 }
431 }
432
433 void
434 initiateAcc(GPUDynInstPtr gpuDynInst) override
435 {
436 typedef typename MemDataType::CType c0;
437
438 gpuDynInst->statusBitVector = gpuDynInst->exec_mask;
439
440 if (num_dest_operands > 1) {
441 for (int i = 0; i < gpuDynInst->computeUnit()->wfSize(); ++i)
442 if (gpuDynInst->exec_mask[i])
443 gpuDynInst->statusVector.push_back(num_dest_operands);
444 else
445 gpuDynInst->statusVector.push_back(0);
446 }
447
448 for (int k = 0; k < num_dest_operands; ++k) {
449
450 c0 *d = &((c0*)gpuDynInst->d_data)
451 [k * gpuDynInst->computeUnit()->wfSize()];
452
453 for (int i = 0; i < gpuDynInst->computeUnit()->wfSize(); ++i) {
454 if (gpuDynInst->exec_mask[i]) {
455 Addr vaddr = gpuDynInst->addr[i] + k * sizeof(c0);
456
457 if (this->isLocalMem()) {
458 // load from shared memory
459 *d = gpuDynInst->wavefront()->ldsChunk->
460 read<c0>(vaddr);
461 } else {
462 Request *req = new Request(0, vaddr, sizeof(c0), 0,
463 gpuDynInst->computeUnit()->masterId(),
464 0, gpuDynInst->wfDynId);
465
466 gpuDynInst->setRequestFlags(req);
467 PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
468 pkt->dataStatic(d);
469
470 if (gpuDynInst->computeUnit()->shader->
471 separate_acquire_release &&
472 gpuDynInst->isAcquire()) {
473 // if this load has acquire semantics,
474 // set the response continuation function
475 // to perform an Acquire request
476 gpuDynInst->execContinuation =
477 &GPUStaticInst::execLdAcq;
478
479 gpuDynInst->useContinuation = true;
480 } else {
481 // the request will be finished when
482 // the load completes
483 gpuDynInst->useContinuation = false;
484 }
485 // translation is performed in sendRequest()
486 gpuDynInst->computeUnit()->sendRequest(gpuDynInst,
487 i, pkt);
488 }
489 }
490 ++d;
491 }
492 }
493
494 gpuDynInst->updateStats();
495 }
496
497 void
498 completeAcc(GPUDynInstPtr gpuDynInst) override
499 {
500 typedef typename MemDataType::CType c1;
501
502 constexpr bool is_vt_32 = DestDataType::vgprType == VT_32;
503
504 /**
505 * this code essentially replaces the long if-else chain
506 * that was in used GlobalMemPipeline::exec() to infer the
507 * size (single/double) and type (floating point/integer) of
508 * the destination register. this is needed for load
509 * instructions because the loaded value and the
510 * destination type can be of different sizes, and we also
511 * need to know if the value we're writing back is floating
512 * point and signed/unsigned, so we can properly cast the
513 * writeback value
514 */
515 typedef typename std::conditional<is_vt_32,
516 typename std::conditional<std::is_floating_point<c1>::value,
517 float, typename std::conditional<std::is_signed<c1>::value,
518 int32_t, uint32_t>::type>::type,
519 typename std::conditional<std::is_floating_point<c1>::value,
520 double, typename std::conditional<std::is_signed<c1>::value,
521 int64_t, uint64_t>::type>::type>::type c0;
522
523
524 Wavefront *w = gpuDynInst->wavefront();
525
526 std::vector<uint32_t> regVec;
527 // iterate over number of destination register operands since
528 // this is a load
529 for (int k = 0; k < num_dest_operands; ++k) {
530 assert((sizeof(c1) * num_dest_operands)
531 <= MAX_WIDTH_FOR_MEM_INST);
532
533 int dst = this->dest.regIndex() + k;
534 if (num_dest_operands > MAX_REGS_FOR_NON_VEC_MEM_INST)
535 dst = dest_vect[k].regIndex();
536 // virtual->physical VGPR mapping
537 int physVgpr = w->remap(dst, sizeof(c0), 1);
538 // save the physical VGPR index
539 regVec.push_back(physVgpr);
540
541 c1 *p1 =
542 &((c1*)gpuDynInst->d_data)[k * w->computeUnit->wfSize()];
543
544 for (int i = 0; i < w->computeUnit->wfSize(); ++i) {
545 if (gpuDynInst->exec_mask[i]) {
546 DPRINTF(GPUReg, "CU%d, WF[%d][%d], lane %d: "
547 "$%s%d <- %d global ld done (src = wavefront "
548 "ld inst)\n", w->computeUnit->cu_id, w->simdId,
549 w->wfSlotId, i, sizeof(c0) == 4 ? "s" : "d",
550 dst, *p1);
551 // write the value into the physical VGPR. This is a
552 // purely functional operation. No timing is modeled.
553 w->computeUnit->vrf[w->simdId]->write<c0>(physVgpr,
554 *p1, i);
555 }
556 ++p1;
557 }
558 }
559
560 // Schedule the write operation of the load data on the VRF.
561 // This simply models the timing aspect of the VRF write operation.
562 // It does not modify the physical VGPR.
563 int loadVrfBankConflictCycles = gpuDynInst->computeUnit()->
564 vrf[w->simdId]->exec(gpuDynInst->seqNum(), w, regVec,
565 sizeof(c0), gpuDynInst->time);
566
567 if (this->isGlobalMem()) {
568 gpuDynInst->computeUnit()->globalMemoryPipe
569 .incLoadVRFBankConflictCycles(loadVrfBankConflictCycles);
570 } else {
571 assert(this->isLocalMem());
572 gpuDynInst->computeUnit()->localMemoryPipe
573 .incLoadVRFBankConflictCycles(loadVrfBankConflictCycles);
574 }
575 }
576
577 private:
578 void
579 execLdAcq(GPUDynInstPtr gpuDynInst) override
580 {
581 // after the load has complete and if the load has acquire
582 // semantics, issue an acquire request.
583 if (!this->isLocalMem()) {
584 if (gpuDynInst->computeUnit()->shader->separate_acquire_release
585 && gpuDynInst->isAcquire()) {
586 gpuDynInst->statusBitVector = VectorMask(1);
587 gpuDynInst->useContinuation = false;
588 // create request
589 Request *req = new Request(0, 0, 0, 0,
590 gpuDynInst->computeUnit()->masterId(),
591 0, gpuDynInst->wfDynId);
592 req->setFlags(Request::ACQUIRE);
593 gpuDynInst->computeUnit()->injectGlobalMemFence(gpuDynInst, false, req);
594 }
595 }
596 }
597
598 public:
599 bool isVectorRegister(int operandIndex) override
600 {
601 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
602 if ((num_dest_operands != getNumOperands()) &&
603 (operandIndex == (getNumOperands()-1)))
604 return(this->addr.isVectorRegister());
605 if (num_dest_operands > 1) {
606 return dest_vect[operandIndex].isVectorRegister();
607 }
608 else if (num_dest_operands == 1) {
609 return LdInstBase<typename MemDataType::CType,
610 typename DestDataType::OperandType,
611 AddrOperandType>::dest.isVectorRegister();
612 }
613 return false;
614 }
615 bool isCondRegister(int operandIndex) override
616 {
617 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
618 if ((num_dest_operands != getNumOperands()) &&
619 (operandIndex == (getNumOperands()-1)))
620 return(this->addr.isCondRegister());
621 if (num_dest_operands > 1)
622 return dest_vect[operandIndex].isCondRegister();
623 else if (num_dest_operands == 1)
624 return LdInstBase<typename MemDataType::CType,
625 typename DestDataType::OperandType,
626 AddrOperandType>::dest.isCondRegister();
627 return false;
628 }
629 bool isScalarRegister(int operandIndex) override
630 {
631 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
632 if ((num_dest_operands != getNumOperands()) &&
633 (operandIndex == (getNumOperands()-1)))
634 return(this->addr.isScalarRegister());
635 if (num_dest_operands > 1)
636 return dest_vect[operandIndex].isScalarRegister();
637 else if (num_dest_operands == 1)
638 return LdInstBase<typename MemDataType::CType,
639 typename DestDataType::OperandType,
640 AddrOperandType>::dest.isScalarRegister();
641 return false;
642 }
643 bool isSrcOperand(int operandIndex) override
644 {
645 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
646 if ((num_dest_operands != getNumOperands()) &&
647 (operandIndex == (getNumOperands()-1)))
648 return(this->addr.isVectorRegister());
649 return false;
650 }
651 bool isDstOperand(int operandIndex) override
652 {
653 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
654 if ((num_dest_operands != getNumOperands()) &&
655 (operandIndex == (getNumOperands()-1)))
656 return false;
657 return true;
658 }
659 int getOperandSize(int operandIndex) override
660 {
661 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
662 if ((num_dest_operands != getNumOperands()) &&
663 (operandIndex == (getNumOperands()-1)))
664 return(this->addr.opSize());
665 if (num_dest_operands > 1)
666 return(dest_vect[operandIndex].opSize());
667 else if (num_dest_operands == 1)
668 return(LdInstBase<typename MemDataType::CType,
669 typename DestDataType::OperandType,
670 AddrOperandType>::dest.opSize());
671 return 0;
672 }
673 int getRegisterIndex(int operandIndex) override
674 {
675 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
676 if ((num_dest_operands != getNumOperands()) &&
677 (operandIndex == (getNumOperands()-1)))
678 return(this->addr.regIndex());
679 if (num_dest_operands > 1)
680 return(dest_vect[operandIndex].regIndex());
681 else if (num_dest_operands == 1)
682 return(LdInstBase<typename MemDataType::CType,
683 typename DestDataType::OperandType,
684 AddrOperandType>::dest.regIndex());
685 return -1;
686 }
687 int getNumOperands() override
688 {
689 if (this->addr.isVectorRegister() || this->addr.isScalarRegister())
690 return(num_dest_operands+1);
691 else
692 return(num_dest_operands);
693 }
694 void execute(GPUDynInstPtr gpuDynInst) override;
695 };
696
697 template<typename MemDT, typename DestDT>
698 GPUStaticInst*
699 decodeLd2(const Brig::BrigInstBase *ib, const BrigObject *obj)
700 {
701 unsigned op_offs = obj->getOperandPtr(ib->operands,1);
702 BrigRegOperandInfo tmp = findRegDataType(op_offs, obj);
703
704 if (tmp.kind == Brig::BRIG_KIND_OPERAND_ADDRESS) {
705 return new LdInst<MemDT, DestDT, NoRegAddrOperand>(ib, obj, "ld");
706 } else if (tmp.kind == Brig::BRIG_KIND_OPERAND_REGISTER ||
707 tmp.kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST) {
708 switch (tmp.regKind) {
709 case Brig::BRIG_REGISTER_KIND_SINGLE:
710 return new LdInst<MemDT, DestDT,
711 SRegAddrOperand>(ib, obj, "ld");
712 case Brig::BRIG_REGISTER_KIND_DOUBLE:
713 return new LdInst<MemDT, DestDT,
714 DRegAddrOperand>(ib, obj, "ld");
715 default:
716 fatal("Bad ld register operand type %d\n", tmp.regKind);
717 }
718 } else {
719 fatal("Bad ld register operand kind %d\n", tmp.kind);
720 }
721 }
722
723 template<typename MemDT>
724 GPUStaticInst*
725 decodeLd(const Brig::BrigInstBase *ib, const BrigObject *obj)
726 {
727 unsigned op_offs = obj->getOperandPtr(ib->operands,0);
728 BrigRegOperandInfo dest = findRegDataType(op_offs, obj);
729
730 assert(dest.kind == Brig::BRIG_KIND_OPERAND_REGISTER ||
731 dest.kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST);
732 switch(dest.regKind) {
733 case Brig::BRIG_REGISTER_KIND_SINGLE:
734 switch (ib->type) {
735 case Brig::BRIG_TYPE_B8:
736 case Brig::BRIG_TYPE_B16:
737 case Brig::BRIG_TYPE_B32:
738 return decodeLd2<MemDT, B32>(ib, obj);
739 case Brig::BRIG_TYPE_U8:
740 case Brig::BRIG_TYPE_U16:
741 case Brig::BRIG_TYPE_U32:
742 return decodeLd2<MemDT, U32>(ib, obj);
743 case Brig::BRIG_TYPE_S8:
744 case Brig::BRIG_TYPE_S16:
745 case Brig::BRIG_TYPE_S32:
746 return decodeLd2<MemDT, S32>(ib, obj);
747 case Brig::BRIG_TYPE_F16:
748 case Brig::BRIG_TYPE_F32:
749 return decodeLd2<MemDT, U32>(ib, obj);
750 default:
751 fatal("Bad ld register operand type %d, %d\n",
752 dest.regKind, ib->type);
753 };
754 case Brig::BRIG_REGISTER_KIND_DOUBLE:
755 switch (ib->type) {
756 case Brig::BRIG_TYPE_B64:
757 return decodeLd2<MemDT, B64>(ib, obj);
758 case Brig::BRIG_TYPE_U64:
759 return decodeLd2<MemDT, U64>(ib, obj);
760 case Brig::BRIG_TYPE_S64:
761 return decodeLd2<MemDT, S64>(ib, obj);
762 case Brig::BRIG_TYPE_F64:
763 return decodeLd2<MemDT, U64>(ib, obj);
764 default:
765 fatal("Bad ld register operand type %d, %d\n",
766 dest.regKind, ib->type);
767 };
768 default:
769 fatal("Bad ld register operand type %d, %d\n", dest.regKind,
770 ib->type);
771 }
772 }
773
774 template<typename MemDataType, typename SrcOperandType,
775 typename AddrOperandType>
776 class StInstBase : public HsailGPUStaticInst
777 {
778 public:
779 typename SrcOperandType::SrcOperand src;
780 AddrOperandType addr;
781
782 Brig::BrigSegment segment;
783 Brig::BrigMemoryScope memoryScope;
784 Brig::BrigMemoryOrder memoryOrder;
785 unsigned int equivClass;
786
787 StInstBase(const Brig::BrigInstBase *ib, const BrigObject *obj,
788 const char *_opcode)
789 : HsailGPUStaticInst(obj, _opcode)
790 {
791 using namespace Brig;
792
793 setFlag(MemoryRef);
794 setFlag(Store);
795
796 if (ib->opcode == BRIG_OPCODE_ST) {
797 const BrigInstMem *ldst = (const BrigInstMem*)ib;
798
799 segment = (BrigSegment)ldst->segment;
800 memoryOrder = BRIG_MEMORY_ORDER_NONE;
801 memoryScope = BRIG_MEMORY_SCOPE_NONE;
802 equivClass = ldst->equivClass;
803
804 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
805 const BrigOperand *baseOp = obj->getOperand(op_offs);
806
807 if ((baseOp->kind == BRIG_KIND_OPERAND_CONSTANT_BYTES) ||
808 (baseOp->kind == BRIG_KIND_OPERAND_REGISTER)) {
809 src.init(op_offs, obj);
810 }
811
812 op_offs = obj->getOperandPtr(ib->operands, 1);
813 addr.init(op_offs, obj);
814 } else {
815 const BrigInstAtomic *at = (const BrigInstAtomic*)ib;
816
817 segment = (BrigSegment)at->segment;
818 memoryScope = (BrigMemoryScope)at->memoryScope;
819 memoryOrder = (BrigMemoryOrder)at->memoryOrder;
820 equivClass = 0;
821
822 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
823 addr.init(op_offs, obj);
824
825 op_offs = obj->getOperandPtr(ib->operands, 1);
826 src.init(op_offs, obj);
827 }
828
829 switch (memoryOrder) {
830 case BRIG_MEMORY_ORDER_NONE:
831 setFlag(NoOrder);
832 break;
833 case BRIG_MEMORY_ORDER_RELAXED:
834 setFlag(RelaxedOrder);
835 break;
836 case BRIG_MEMORY_ORDER_SC_ACQUIRE:
837 setFlag(Acquire);
838 break;
839 case BRIG_MEMORY_ORDER_SC_RELEASE:
840 setFlag(Release);
841 break;
842 case BRIG_MEMORY_ORDER_SC_ACQUIRE_RELEASE:
843 setFlag(AcquireRelease);
844 break;
845 default:
846 fatal("StInst has bad memory order type\n");
847 }
848
849 switch (memoryScope) {
850 case BRIG_MEMORY_SCOPE_NONE:
851 setFlag(NoScope);
852 break;
853 case BRIG_MEMORY_SCOPE_WORKITEM:
854 setFlag(WorkitemScope);
855 break;
856 case BRIG_MEMORY_SCOPE_WORKGROUP:
857 setFlag(WorkgroupScope);
858 break;
859 case BRIG_MEMORY_SCOPE_AGENT:
860 setFlag(DeviceScope);
861 break;
862 case BRIG_MEMORY_SCOPE_SYSTEM:
863 setFlag(SystemScope);
864 break;
865 default:
866 fatal("StInst has bad memory scope type\n");
867 }
868
869 switch (segment) {
870 case BRIG_SEGMENT_GLOBAL:
871 setFlag(GlobalSegment);
872 break;
873 case BRIG_SEGMENT_GROUP:
874 setFlag(GroupSegment);
875 break;
876 case BRIG_SEGMENT_PRIVATE:
877 setFlag(PrivateSegment);
878 break;
879 case BRIG_SEGMENT_READONLY:
880 setFlag(ReadOnlySegment);
881 break;
882 case BRIG_SEGMENT_SPILL:
883 setFlag(SpillSegment);
884 break;
885 case BRIG_SEGMENT_FLAT:
886 setFlag(Flat);
887 break;
888 case BRIG_SEGMENT_ARG:
889 setFlag(ArgSegment);
890 break;
891 default:
892 panic("St: segment %d not supported\n", segment);
893 }
894 }
895
896 int numDstRegOperands() override { return 0; }
897 int numSrcRegOperands() override
898 {
899 return src.isVectorRegister() + this->addr.isVectorRegister();
900 }
901 int getNumOperands() override
902 {
903 if (this->addr.isVectorRegister() || this->addr.isScalarRegister())
904 return 2;
905 else
906 return 1;
907 }
908 bool isVectorRegister(int operandIndex) override
909 {
910 assert(operandIndex >= 0 && operandIndex < getNumOperands());
911 return !operandIndex ? src.isVectorRegister() :
912 this->addr.isVectorRegister();
913 }
914 bool isCondRegister(int operandIndex) override
915 {
916 assert(operandIndex >= 0 && operandIndex < getNumOperands());
917 return !operandIndex ? src.isCondRegister() :
918 this->addr.isCondRegister();
919 }
920 bool isScalarRegister(int operandIndex) override
921 {
922 assert(operandIndex >= 0 && operandIndex < getNumOperands());
923 return !operandIndex ? src.isScalarRegister() :
924 this->addr.isScalarRegister();
925 }
926 bool isSrcOperand(int operandIndex) override
927 {
928 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
929 return true;
930 }
931 bool isDstOperand(int operandIndex) override { return false; }
932 int getOperandSize(int operandIndex) override
933 {
934 assert(operandIndex >= 0 && operandIndex < getNumOperands());
935 return !operandIndex ? src.opSize() : this->addr.opSize();
936 }
937 int getRegisterIndex(int operandIndex) override
938 {
939 assert(operandIndex >= 0 && operandIndex < getNumOperands());
940 return !operandIndex ? src.regIndex() : this->addr.regIndex();
941 }
942 };
943
944
945 template<typename MemDataType, typename SrcDataType,
946 typename AddrOperandType>
947 class StInst :
948 public StInstBase<MemDataType, typename SrcDataType::OperandType,
949 AddrOperandType>,
950 public MemInst
951 {
952 public:
953 typename SrcDataType::OperandType::SrcOperand src_vect[4];
954 uint16_t num_src_operands;
955 void generateDisassembly() override;
956
957 StInst(const Brig::BrigInstBase *ib, const BrigObject *obj,
958 const char *_opcode, int srcIdx)
959 : StInstBase<MemDataType, typename SrcDataType::OperandType,
960 AddrOperandType>(ib, obj, _opcode),
961 MemInst(SrcDataType::memType)
962 {
963 init_addr(&this->addr);
964
965 BrigRegOperandInfo rinfo;
966 unsigned op_offs = obj->getOperandPtr(ib->operands,srcIdx);
967 const Brig::BrigOperand *baseOp = obj->getOperand(op_offs);
968
969 if (baseOp->kind == Brig::BRIG_KIND_OPERAND_CONSTANT_BYTES) {
970 const Brig::BrigOperandConstantBytes *op =
971 (Brig::BrigOperandConstantBytes*)baseOp;
972
973 rinfo = BrigRegOperandInfo((Brig::BrigKind16_t)op->base.kind,
974 Brig::BRIG_TYPE_NONE);
975 } else {
976 rinfo = findRegDataType(op_offs, obj);
977 }
978
979 if (baseOp->kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST) {
980 const Brig::BrigOperandOperandList *brigRegVecOp =
981 (const Brig::BrigOperandOperandList*)baseOp;
982
983 num_src_operands =
984 *((unsigned*)obj->getData(brigRegVecOp->elements)) / 4;
985
986 assert(num_src_operands <= 4);
987 } else {
988 num_src_operands = 1;
989 }
990
991 if (num_src_operands > 1) {
992 assert(baseOp->kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST);
993
994 for (int i = 0; i < num_src_operands; ++i) {
995 src_vect[i].init_from_vect(op_offs, obj, i);
996 }
997 }
998 }
999
1000 void
1001 initiateAcc(GPUDynInstPtr gpuDynInst) override
1002 {
1003 // before performing a store, check if this store has
1004 // release semantics, and if so issue a release first
1005 if (!this->isLocalMem()) {
1006 if (gpuDynInst->computeUnit()->shader->separate_acquire_release
1007 && gpuDynInst->isRelease()) {
1008
1009 gpuDynInst->statusBitVector = VectorMask(1);
1010 gpuDynInst->execContinuation = &GPUStaticInst::execSt;
1011 gpuDynInst->useContinuation = true;
1012 // create request
1013 Request *req = new Request(0, 0, 0, 0,
1014 gpuDynInst->computeUnit()->masterId(),
1015 0, gpuDynInst->wfDynId);
1016 req->setFlags(Request::RELEASE);
1017 gpuDynInst->computeUnit()->injectGlobalMemFence(gpuDynInst, false, req);
1018
1019 return;
1020 }
1021 }
1022
1023 // if there is no release semantic, perform stores immediately
1024 execSt(gpuDynInst);
1025 }
1026
1027 // stores don't write anything back, so there is nothing
1028 // to do here. we only override this method to avoid the
1029 // fatal in the base class implementation
1030 void completeAcc(GPUDynInstPtr gpuDynInst) override { }
1031
1032 private:
1033 // execSt may be called through a continuation
1034 // if the store had release semantics. see comment for
1035 // execSt in gpu_static_inst.hh
1036 void
1037 execSt(GPUDynInstPtr gpuDynInst) override
1038 {
1039 typedef typename MemDataType::CType c0;
1040
1041 gpuDynInst->statusBitVector = gpuDynInst->exec_mask;
1042
1043 if (num_src_operands > 1) {
1044 for (int i = 0; i < gpuDynInst->computeUnit()->wfSize(); ++i)
1045 if (gpuDynInst->exec_mask[i])
1046 gpuDynInst->statusVector.push_back(num_src_operands);
1047 else
1048 gpuDynInst->statusVector.push_back(0);
1049 }
1050
1051 for (int k = 0; k < num_src_operands; ++k) {
1052 c0 *d = &((c0*)gpuDynInst->d_data)
1053 [k * gpuDynInst->computeUnit()->wfSize()];
1054
1055 for (int i = 0; i < gpuDynInst->computeUnit()->wfSize(); ++i) {
1056 if (gpuDynInst->exec_mask[i]) {
1057 Addr vaddr = gpuDynInst->addr[i] + k * sizeof(c0);
1058
1059 if (this->isLocalMem()) {
1060 //store to shared memory
1061 gpuDynInst->wavefront()->ldsChunk->write<c0>(vaddr,
1062 *d);
1063 } else {
1064 Request *req =
1065 new Request(0, vaddr, sizeof(c0), 0,
1066 gpuDynInst->computeUnit()->masterId(),
1067 0, gpuDynInst->wfDynId);
1068
1069 gpuDynInst->setRequestFlags(req);
1070 PacketPtr pkt = new Packet(req, MemCmd::WriteReq);
1071 pkt->dataStatic<c0>(d);
1072
1073 // translation is performed in sendRequest()
1074 // the request will be finished when the store completes
1075 gpuDynInst->useContinuation = false;
1076 gpuDynInst->computeUnit()->sendRequest(gpuDynInst,
1077 i, pkt);
1078
1079 }
1080 }
1081 ++d;
1082 }
1083 }
1084
1085 gpuDynInst->updateStats();
1086 }
1087
1088 public:
1089 bool isVectorRegister(int operandIndex) override
1090 {
1091 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1092 if (operandIndex == num_src_operands)
1093 return this->addr.isVectorRegister();
1094 if (num_src_operands > 1)
1095 return src_vect[operandIndex].isVectorRegister();
1096 else if (num_src_operands == 1)
1097 return StInstBase<MemDataType,
1098 typename SrcDataType::OperandType,
1099 AddrOperandType>::src.isVectorRegister();
1100 return false;
1101 }
1102 bool isCondRegister(int operandIndex) override
1103 {
1104 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1105 if (operandIndex == num_src_operands)
1106 return this->addr.isCondRegister();
1107 if (num_src_operands > 1)
1108 return src_vect[operandIndex].isCondRegister();
1109 else if (num_src_operands == 1)
1110 return StInstBase<MemDataType,
1111 typename SrcDataType::OperandType,
1112 AddrOperandType>::src.isCondRegister();
1113 return false;
1114 }
1115 bool isScalarRegister(int operandIndex) override
1116 {
1117 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1118 if (operandIndex == num_src_operands)
1119 return this->addr.isScalarRegister();
1120 if (num_src_operands > 1)
1121 return src_vect[operandIndex].isScalarRegister();
1122 else if (num_src_operands == 1)
1123 return StInstBase<MemDataType,
1124 typename SrcDataType::OperandType,
1125 AddrOperandType>::src.isScalarRegister();
1126 return false;
1127 }
1128 bool isSrcOperand(int operandIndex) override
1129 {
1130 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1131 return true;
1132 }
1133 bool isDstOperand(int operandIndex) override { return false; }
1134 int getOperandSize(int operandIndex) override
1135 {
1136 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1137 if (operandIndex == num_src_operands)
1138 return this->addr.opSize();
1139 if (num_src_operands > 1)
1140 return src_vect[operandIndex].opSize();
1141 else if (num_src_operands == 1)
1142 return StInstBase<MemDataType,
1143 typename SrcDataType::OperandType,
1144 AddrOperandType>::src.opSize();
1145 return 0;
1146 }
1147 int getRegisterIndex(int operandIndex) override
1148 {
1149 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1150 if (operandIndex == num_src_operands)
1151 return this->addr.regIndex();
1152 if (num_src_operands > 1)
1153 return src_vect[operandIndex].regIndex();
1154 else if (num_src_operands == 1)
1155 return StInstBase<MemDataType,
1156 typename SrcDataType::OperandType,
1157 AddrOperandType>::src.regIndex();
1158 return -1;
1159 }
1160 int getNumOperands() override
1161 {
1162 if (this->addr.isVectorRegister() || this->addr.isScalarRegister())
1163 return num_src_operands + 1;
1164 else
1165 return num_src_operands;
1166 }
1167 void execute(GPUDynInstPtr gpuDynInst) override;
1168 };
1169
1170 template<typename DataType, typename SrcDataType>
1171 GPUStaticInst*
1172 decodeSt(const Brig::BrigInstBase *ib, const BrigObject *obj)
1173 {
1174 int srcIdx = 0;
1175 int destIdx = 1;
1176 if (ib->opcode == Brig::BRIG_OPCODE_ATOMIC ||
1177 ib->opcode == Brig::BRIG_OPCODE_ATOMICNORET) {
1178 srcIdx = 1;
1179 destIdx = 0;
1180 }
1181 unsigned op_offs = obj->getOperandPtr(ib->operands,destIdx);
1182
1183 BrigRegOperandInfo tmp = findRegDataType(op_offs, obj);
1184
1185 if (tmp.kind == Brig::BRIG_KIND_OPERAND_ADDRESS) {
1186 return new StInst<DataType, SrcDataType,
1187 NoRegAddrOperand>(ib, obj, "st", srcIdx);
1188 } else if (tmp.kind == Brig::BRIG_KIND_OPERAND_REGISTER) {
1189 // V2/V4 not allowed
1190 switch (tmp.regKind) {
1191 case Brig::BRIG_REGISTER_KIND_SINGLE:
1192 return new StInst<DataType, SrcDataType,
1193 SRegAddrOperand>(ib, obj, "st", srcIdx);
1194 case Brig::BRIG_REGISTER_KIND_DOUBLE:
1195 return new StInst<DataType, SrcDataType,
1196 DRegAddrOperand>(ib, obj, "st", srcIdx);
1197 default:
1198 fatal("Bad st register operand type %d\n", tmp.type);
1199 }
1200 } else {
1201 fatal("Bad st register operand kind %d\n", tmp.kind);
1202 }
1203 }
1204
1205 template<typename OperandType, typename AddrOperandType, int NumSrcOperands,
1206 bool HasDst>
1207 class AtomicInstBase : public HsailGPUStaticInst
1208 {
1209 public:
1210 typename OperandType::DestOperand dest;
1211 typename OperandType::SrcOperand src[NumSrcOperands];
1212 AddrOperandType addr;
1213
1214 Brig::BrigSegment segment;
1215 Brig::BrigMemoryOrder memoryOrder;
1216 Brig::BrigAtomicOperation atomicOperation;
1217 Brig::BrigMemoryScope memoryScope;
1218 Brig::BrigOpcode opcode;
1219
1220 AtomicInstBase(const Brig::BrigInstBase *ib, const BrigObject *obj,
1221 const char *_opcode)
1222 : HsailGPUStaticInst(obj, _opcode)
1223 {
1224 using namespace Brig;
1225
1226 const BrigInstAtomic *at = (const BrigInstAtomic*)ib;
1227
1228 segment = (BrigSegment)at->segment;
1229 memoryScope = (BrigMemoryScope)at->memoryScope;
1230 memoryOrder = (BrigMemoryOrder)at->memoryOrder;
1231 atomicOperation = (BrigAtomicOperation)at->atomicOperation;
1232 opcode = (BrigOpcode)ib->opcode;
1233
1234 assert(opcode == Brig::BRIG_OPCODE_ATOMICNORET ||
1235 opcode == Brig::BRIG_OPCODE_ATOMIC);
1236
1237 setFlag(MemoryRef);
1238
1239 if (opcode == Brig::BRIG_OPCODE_ATOMIC) {
1240 setFlag(AtomicReturn);
1241 } else {
1242 setFlag(AtomicNoReturn);
1243 }
1244
1245 switch (memoryOrder) {
1246 case BRIG_MEMORY_ORDER_NONE:
1247 setFlag(NoOrder);
1248 break;
1249 case BRIG_MEMORY_ORDER_RELAXED:
1250 setFlag(RelaxedOrder);
1251 break;
1252 case BRIG_MEMORY_ORDER_SC_ACQUIRE:
1253 setFlag(Acquire);
1254 break;
1255 case BRIG_MEMORY_ORDER_SC_RELEASE:
1256 setFlag(Release);
1257 break;
1258 case BRIG_MEMORY_ORDER_SC_ACQUIRE_RELEASE:
1259 setFlag(AcquireRelease);
1260 break;
1261 default:
1262 fatal("AtomicInst has bad memory order type\n");
1263 }
1264
1265 switch (memoryScope) {
1266 case BRIG_MEMORY_SCOPE_NONE:
1267 setFlag(NoScope);
1268 break;
1269 case BRIG_MEMORY_SCOPE_WORKITEM:
1270 setFlag(WorkitemScope);
1271 break;
1272 case BRIG_MEMORY_SCOPE_WORKGROUP:
1273 setFlag(WorkgroupScope);
1274 break;
1275 case BRIG_MEMORY_SCOPE_AGENT:
1276 setFlag(DeviceScope);
1277 break;
1278 case BRIG_MEMORY_SCOPE_SYSTEM:
1279 setFlag(SystemScope);
1280 break;
1281 default:
1282 fatal("AtomicInst has bad memory scope type\n");
1283 }
1284
1285 switch (atomicOperation) {
1286 case Brig::BRIG_ATOMIC_AND:
1287 setFlag(AtomicAnd);
1288 break;
1289 case Brig::BRIG_ATOMIC_OR:
1290 setFlag(AtomicOr);
1291 break;
1292 case Brig::BRIG_ATOMIC_XOR:
1293 setFlag(AtomicXor);
1294 break;
1295 case Brig::BRIG_ATOMIC_CAS:
1296 setFlag(AtomicCAS);
1297 break;
1298 case Brig::BRIG_ATOMIC_EXCH:
1299 setFlag(AtomicExch);
1300 break;
1301 case Brig::BRIG_ATOMIC_ADD:
1302 setFlag(AtomicAdd);
1303 break;
1304 case Brig::BRIG_ATOMIC_WRAPINC:
1305 setFlag(AtomicInc);
1306 break;
1307 case Brig::BRIG_ATOMIC_WRAPDEC:
1308 setFlag(AtomicDec);
1309 break;
1310 case Brig::BRIG_ATOMIC_MIN:
1311 setFlag(AtomicMin);
1312 break;
1313 case Brig::BRIG_ATOMIC_MAX:
1314 setFlag(AtomicMax);
1315 break;
1316 case Brig::BRIG_ATOMIC_SUB:
1317 setFlag(AtomicSub);
1318 break;
1319 default:
1320 fatal("Bad BrigAtomicOperation code %d\n", atomicOperation);
1321 }
1322
1323 switch (segment) {
1324 case BRIG_SEGMENT_GLOBAL:
1325 setFlag(GlobalSegment);
1326 break;
1327 case BRIG_SEGMENT_GROUP:
1328 setFlag(GroupSegment);
1329 break;
1330 case BRIG_SEGMENT_FLAT:
1331 setFlag(Flat);
1332 break;
1333 default:
1334 panic("Atomic: segment %d not supported\n", segment);
1335 }
1336
1337 if (HasDst) {
1338 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
1339 dest.init(op_offs, obj);
1340
1341 op_offs = obj->getOperandPtr(ib->operands, 1);
1342 addr.init(op_offs, obj);
1343
1344 for (int i = 0; i < NumSrcOperands; ++i) {
1345 op_offs = obj->getOperandPtr(ib->operands, i + 2);
1346 src[i].init(op_offs, obj);
1347 }
1348 } else {
1349
1350 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
1351 addr.init(op_offs, obj);
1352
1353 for (int i = 0; i < NumSrcOperands; ++i) {
1354 op_offs = obj->getOperandPtr(ib->operands, i + 1);
1355 src[i].init(op_offs, obj);
1356 }
1357 }
1358 }
1359
1360 int numSrcRegOperands()
1361 {
1362 int operands = 0;
1363 for (int i = 0; i < NumSrcOperands; i++) {
1364 if (src[i].isVectorRegister()) {
1365 operands++;
1366 }
1367 }
1368 if (addr.isVectorRegister())
1369 operands++;
1370 return operands;
1371 }
1372 int numDstRegOperands() { return dest.isVectorRegister(); }
1373 int getNumOperands()
1374 {
1375 if (addr.isVectorRegister())
1376 return(NumSrcOperands + 2);
1377 return(NumSrcOperands + 1);
1378 }
1379 bool isVectorRegister(int operandIndex)
1380 {
1381 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1382 if (operandIndex < NumSrcOperands)
1383 return src[operandIndex].isVectorRegister();
1384 else if (operandIndex == NumSrcOperands)
1385 return(addr.isVectorRegister());
1386 else
1387 return dest.isVectorRegister();
1388 }
1389 bool isCondRegister(int operandIndex)
1390 {
1391 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1392 if (operandIndex < NumSrcOperands)
1393 return src[operandIndex].isCondRegister();
1394 else if (operandIndex == NumSrcOperands)
1395 return(addr.isCondRegister());
1396 else
1397 return dest.isCondRegister();
1398 }
1399 bool isScalarRegister(int operandIndex)
1400 {
1401 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1402 if (operandIndex < NumSrcOperands)
1403 return src[operandIndex].isScalarRegister();
1404 else if (operandIndex == NumSrcOperands)
1405 return(addr.isScalarRegister());
1406 else
1407 return dest.isScalarRegister();
1408 }
1409 bool isSrcOperand(int operandIndex)
1410 {
1411 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1412 if (operandIndex < NumSrcOperands)
1413 return true;
1414 else if (operandIndex == NumSrcOperands)
1415 return(addr.isVectorRegister());
1416 else
1417 return false;
1418 }
1419 bool isDstOperand(int operandIndex)
1420 {
1421 if (operandIndex <= NumSrcOperands)
1422 return false;
1423 else
1424 return true;
1425 }
1426 int getOperandSize(int operandIndex)
1427 {
1428 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1429 if (operandIndex < NumSrcOperands)
1430 return(src[operandIndex].opSize());
1431 else if (operandIndex == NumSrcOperands)
1432 return(addr.opSize());
1433 else
1434 return(dest.opSize());
1435 }
1436 int getRegisterIndex(int operandIndex)
1437 {
1438 assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1439 if (operandIndex < NumSrcOperands)
1440 return(src[operandIndex].regIndex());
1441 else if (operandIndex == NumSrcOperands)
1442 return(addr.regIndex());
1443 else
1444 return(dest.regIndex());
1445 return -1;
1446 }
1447 };
1448
1449 template<typename MemDataType, typename AddrOperandType, int NumSrcOperands,
1450 bool HasDst>
1451 class AtomicInst :
1452 public AtomicInstBase<typename MemDataType::OperandType,
1453 AddrOperandType, NumSrcOperands, HasDst>,
1454 public MemInst
1455 {
1456 public:
1457 void generateDisassembly() override;
1458
1459 AtomicInst(const Brig::BrigInstBase *ib, const BrigObject *obj,
1460 const char *_opcode)
1461 : AtomicInstBase<typename MemDataType::OperandType, AddrOperandType,
1462 NumSrcOperands, HasDst>
1463 (ib, obj, _opcode),
1464 MemInst(MemDataType::memType)
1465 {
1466 init_addr(&this->addr);
1467 }
1468
1469 void
1470 initiateAcc(GPUDynInstPtr gpuDynInst) override
1471 {
1472 // before doing the RMW, check if this atomic has
1473 // release semantics, and if so issue a release first
1474 if (!this->isLocalMem()) {
1475 if (gpuDynInst->computeUnit()->shader->separate_acquire_release
1476 && (gpuDynInst->isRelease()
1477 || gpuDynInst->isAcquireRelease())) {
1478
1479 gpuDynInst->statusBitVector = VectorMask(1);
1480
1481 gpuDynInst->execContinuation = &GPUStaticInst::execAtomic;
1482 gpuDynInst->useContinuation = true;
1483
1484 // create request
1485 Request *req = new Request(0, 0, 0, 0,
1486 gpuDynInst->computeUnit()->masterId(),
1487 0, gpuDynInst->wfDynId);
1488 req->setFlags(Request::RELEASE);
1489 gpuDynInst->computeUnit()->injectGlobalMemFence(gpuDynInst, false, req);
1490
1491 return;
1492 }
1493 }
1494
1495 // if there is no release semantic, execute the RMW immediately
1496 execAtomic(gpuDynInst);
1497
1498 }
1499
1500 void
1501 completeAcc(GPUDynInstPtr gpuDynInst) override
1502 {
1503 // if this is not an atomic return op, then we
1504 // have nothing more to do.
1505 if (this->isAtomicRet()) {
1506 // the size of the src operands and the
1507 // memory being operated on must match
1508 // for HSAIL atomics - this assumption may
1509 // not apply to all ISAs
1510 typedef typename MemDataType::CType CType;
1511
1512 Wavefront *w = gpuDynInst->wavefront();
1513 int dst = this->dest.regIndex();
1514 std::vector<uint32_t> regVec;
1515 // virtual->physical VGPR mapping
1516 int physVgpr = w->remap(dst, sizeof(CType), 1);
1517 regVec.push_back(physVgpr);
1518 CType *p1 = &((CType*)gpuDynInst->d_data)[0];
1519
1520 for (int i = 0; i < w->computeUnit->wfSize(); ++i) {
1521 if (gpuDynInst->exec_mask[i]) {
1522 DPRINTF(GPUReg, "CU%d, WF[%d][%d], lane %d: "
1523 "$%s%d <- %d global ld done (src = wavefront "
1524 "ld inst)\n", w->computeUnit->cu_id, w->simdId,
1525 w->wfSlotId, i, sizeof(CType) == 4 ? "s" : "d",
1526 dst, *p1);
1527 // write the value into the physical VGPR. This is a
1528 // purely functional operation. No timing is modeled.
1529 w->computeUnit->vrf[w->simdId]->write<CType>(physVgpr, *p1, i);
1530 }
1531 ++p1;
1532 }
1533
1534 // Schedule the write operation of the load data on the VRF.
1535 // This simply models the timing aspect of the VRF write operation.
1536 // It does not modify the physical VGPR.
1537 int loadVrfBankConflictCycles = gpuDynInst->computeUnit()->
1538 vrf[w->simdId]->exec(gpuDynInst->seqNum(), w, regVec,
1539 sizeof(CType), gpuDynInst->time);
1540
1541 if (this->isGlobalMem()) {
1542 gpuDynInst->computeUnit()->globalMemoryPipe
1543 .incLoadVRFBankConflictCycles(loadVrfBankConflictCycles);
1544 } else {
1545 assert(this->isLocalMem());
1546 gpuDynInst->computeUnit()->localMemoryPipe
1547 .incLoadVRFBankConflictCycles(loadVrfBankConflictCycles);
1548 }
1549 }
1550 }
1551
1552 void execute(GPUDynInstPtr gpuDynInst) override;
1553
1554 private:
1555 // execAtomic may be called through a continuation
1556 // if the RMW had release semantics. see comment for
1557 // execContinuation in gpu_dyn_inst.hh
1558 void
1559 execAtomic(GPUDynInstPtr gpuDynInst) override
1560 {
1561 gpuDynInst->statusBitVector = gpuDynInst->exec_mask;
1562
1563 typedef typename MemDataType::CType c0;
1564
1565 c0 *d = &((c0*) gpuDynInst->d_data)[0];
1566 c0 *e = &((c0*) gpuDynInst->a_data)[0];
1567 c0 *f = &((c0*) gpuDynInst->x_data)[0];
1568
1569 for (int i = 0; i < gpuDynInst->computeUnit()->wfSize(); ++i) {
1570 if (gpuDynInst->exec_mask[i]) {
1571 Addr vaddr = gpuDynInst->addr[i];
1572
1573 if (this->isLocalMem()) {
1574 Wavefront *wavefront = gpuDynInst->wavefront();
1575 *d = wavefront->ldsChunk->read<c0>(vaddr);
1576
1577 if (this->isAtomicAdd()) {
1578 wavefront->ldsChunk->write<c0>(vaddr,
1579 wavefront->ldsChunk->read<c0>(vaddr) + (*e));
1580 } else if (this->isAtomicSub()) {
1581 wavefront->ldsChunk->write<c0>(vaddr,
1582 wavefront->ldsChunk->read<c0>(vaddr) - (*e));
1583 } else if (this->isAtomicMax()) {
1584 wavefront->ldsChunk->write<c0>(vaddr,
1585 std::max(wavefront->ldsChunk->read<c0>(vaddr),
1586 (*e)));
1587 } else if (this->isAtomicMin()) {
1588 wavefront->ldsChunk->write<c0>(vaddr,
1589 std::min(wavefront->ldsChunk->read<c0>(vaddr),
1590 (*e)));
1591 } else if (this->isAtomicAnd()) {
1592 wavefront->ldsChunk->write<c0>(vaddr,
1593 wavefront->ldsChunk->read<c0>(vaddr) & (*e));
1594 } else if (this->isAtomicOr()) {
1595 wavefront->ldsChunk->write<c0>(vaddr,
1596 wavefront->ldsChunk->read<c0>(vaddr) | (*e));
1597 } else if (this->isAtomicXor()) {
1598 wavefront->ldsChunk->write<c0>(vaddr,
1599 wavefront->ldsChunk->read<c0>(vaddr) ^ (*e));
1600 } else if (this->isAtomicInc()) {
1601 wavefront->ldsChunk->write<c0>(vaddr,
1602 wavefront->ldsChunk->read<c0>(vaddr) + 1);
1603 } else if (this->isAtomicDec()) {
1604 wavefront->ldsChunk->write<c0>(vaddr,
1605 wavefront->ldsChunk->read<c0>(vaddr) - 1);
1606 } else if (this->isAtomicExch()) {
1607 wavefront->ldsChunk->write<c0>(vaddr, (*e));
1608 } else if (this->isAtomicCAS()) {
1609 wavefront->ldsChunk->write<c0>(vaddr,
1610 (wavefront->ldsChunk->read<c0>(vaddr) == (*e)) ?
1611 (*f) : wavefront->ldsChunk->read<c0>(vaddr));
1612 } else {
1613 fatal("Unrecognized or invalid HSAIL atomic op "
1614 "type.\n");
1615 }
1616 } else {
1617 Request *req =
1618 new Request(0, vaddr, sizeof(c0), 0,
1619 gpuDynInst->computeUnit()->masterId(),
1620 0, gpuDynInst->wfDynId,
1621 gpuDynInst->makeAtomicOpFunctor<c0>(e,
1622 f));
1623
1624 gpuDynInst->setRequestFlags(req);
1625 PacketPtr pkt = new Packet(req, MemCmd::SwapReq);
1626 pkt->dataStatic(d);
1627
1628 if (gpuDynInst->computeUnit()->shader->
1629 separate_acquire_release &&
1630 (gpuDynInst->isAcquire())) {
1631 // if this atomic has acquire semantics,
1632 // schedule the continuation to perform an
1633 // acquire after the RMW completes
1634 gpuDynInst->execContinuation =
1635 &GPUStaticInst::execAtomicAcq;
1636
1637 gpuDynInst->useContinuation = true;
1638 } else {
1639 // the request will be finished when the RMW completes
1640 gpuDynInst->useContinuation = false;
1641 }
1642 // translation is performed in sendRequest()
1643 gpuDynInst->computeUnit()->sendRequest(gpuDynInst, i,
1644 pkt);
1645 }
1646 }
1647
1648 ++d;
1649 ++e;
1650 ++f;
1651 }
1652
1653 gpuDynInst->updateStats();
1654 }
1655
1656 // execAtomicACq will always be called through a continuation.
1657 // see comment for execContinuation in gpu_dyn_inst.hh
1658 void
1659 execAtomicAcq(GPUDynInstPtr gpuDynInst) override
1660 {
1661 // after performing the RMW, check to see if this instruction
1662 // has acquire semantics, and if so, issue an acquire
1663 if (!this->isLocalMem()) {
1664 if (gpuDynInst->computeUnit()->shader->separate_acquire_release
1665 && gpuDynInst->isAcquire()) {
1666 gpuDynInst->statusBitVector = VectorMask(1);
1667
1668 // the request will be finished when
1669 // the acquire completes
1670 gpuDynInst->useContinuation = false;
1671 // create request
1672 Request *req = new Request(0, 0, 0, 0,
1673 gpuDynInst->computeUnit()->masterId(),
1674 0, gpuDynInst->wfDynId);
1675 req->setFlags(Request::ACQUIRE);
1676 gpuDynInst->computeUnit()->injectGlobalMemFence(gpuDynInst, false, req);
1677 }
1678 }
1679 }
1680 };
1681
1682 template<typename DataType, typename AddrOperandType, int NumSrcOperands>
1683 GPUStaticInst*
1684 constructAtomic(const Brig::BrigInstBase *ib, const BrigObject *obj)
1685 {
1686 const Brig::BrigInstAtomic *at = (const Brig::BrigInstAtomic*)ib;
1687
1688 if (at->atomicOperation == Brig::BRIG_ATOMIC_LD) {
1689 return decodeLd<DataType>(ib, obj);
1690 } else if (at->atomicOperation == Brig::BRIG_ATOMIC_ST) {
1691 switch (ib->type) {
1692 case Brig::BRIG_TYPE_B8:
1693 return decodeSt<S8,S8>(ib, obj);
1694 case Brig::BRIG_TYPE_B16:
1695 return decodeSt<S16,S16>(ib, obj);
1696 case Brig::BRIG_TYPE_B32:
1697 return decodeSt<S32,S32>(ib, obj);
1698 case Brig::BRIG_TYPE_B64:
1699 return decodeSt<S64,S64>(ib, obj);
1700 default: fatal("AtomicSt: Operand type mismatch %d\n", ib->type);
1701 }
1702 } else {
1703 if ((Brig::BrigOpcode)ib->opcode == Brig::BRIG_OPCODE_ATOMICNORET)
1704 return new AtomicInst<DataType, AddrOperandType,
1705 NumSrcOperands, false>(ib, obj, "atomicnoret");
1706 else
1707 return new AtomicInst<DataType, AddrOperandType,
1708 NumSrcOperands, true>(ib, obj, "atomic");
1709 }
1710 }
1711
1712 template<typename DataType, int NumSrcOperands>
1713 GPUStaticInst*
1714 decodeAtomicHelper(const Brig::BrigInstBase *ib, const BrigObject *obj)
1715 {
1716 unsigned addrIndex = (Brig::BrigOpcode)ib->opcode ==
1717 Brig::BRIG_OPCODE_ATOMICNORET ? 0 : 1;
1718
1719 unsigned op_offs = obj->getOperandPtr(ib->operands,addrIndex);
1720
1721 BrigRegOperandInfo tmp = findRegDataType(op_offs, obj);
1722
1723 if (tmp.kind == Brig::BRIG_KIND_OPERAND_ADDRESS) {
1724 return constructAtomic<DataType, NoRegAddrOperand,
1725 NumSrcOperands>(ib, obj);
1726 } else if (tmp.kind == Brig::BRIG_KIND_OPERAND_REGISTER) {
1727 // V2/V4 not allowed
1728 switch (tmp.regKind) {
1729 case Brig::BRIG_REGISTER_KIND_SINGLE:
1730 return constructAtomic<DataType, SRegAddrOperand,
1731 NumSrcOperands>(ib, obj);
1732 case Brig::BRIG_REGISTER_KIND_DOUBLE:
1733 return constructAtomic<DataType, DRegAddrOperand,
1734 NumSrcOperands>(ib, obj);
1735 default:
1736 fatal("Bad atomic register operand type %d\n", tmp.type);
1737 }
1738 } else {
1739 fatal("Bad atomic register operand kind %d\n", tmp.kind);
1740 }
1741 }
1742
1743
1744 template<typename DataType>
1745 GPUStaticInst*
1746 decodeAtomic(const Brig::BrigInstBase *ib, const BrigObject *obj)
1747 {
1748 const Brig::BrigInstAtomic *at = (const Brig::BrigInstAtomic*)ib;
1749
1750 if (at->atomicOperation == Brig::BRIG_ATOMIC_CAS) {
1751 return decodeAtomicHelper<DataType, 2>(ib, obj);
1752 } else {
1753 return decodeAtomicHelper<DataType, 1>(ib, obj);
1754 }
1755 }
1756
1757 template<typename DataType>
1758 GPUStaticInst*
1759 decodeAtomicNoRet(const Brig::BrigInstBase *ib, const BrigObject *obj)
1760 {
1761 const Brig::BrigInstAtomic *at = (const Brig::BrigInstAtomic*)ib;
1762 if (at->atomicOperation == Brig::BRIG_ATOMIC_CAS) {
1763 return decodeAtomicHelper<DataType, 2>(ib, obj);
1764 } else {
1765 return decodeAtomicHelper<DataType, 1>(ib, obj);
1766 }
1767 }
1768} // namespace HsailISA
1769
1770#endif // __ARCH_HSAIL_INSTS_MEM_HH__