Debian Codegraph @

visa/iga/IGALibrary/Backend/GED/Decoder.cpp

raw
1 /*========================== begin_copyright_notice ============================
2
3 Copyright (C) 2017-2021 Intel Corporation
4
5 SPDX-License-Identifier: MIT
6
7 ============================= end_copyright_notice ===========================*/
8
9 #include "Decoder.hpp"
10 #include "../../Frontend/Formatter.hpp"
11 #include "../../Frontend/IRToString.hpp"
12 #include "../../IR/Checker/IRChecker.hpp"
13 #include "../../IR/Messages.hpp"
14 #include "../../IR/SWSBSetter.hpp"
15 #include "../../MemManager/MemManager.hpp"
16 #include "../../asserts.hpp"
17 #include "../../strings.hpp"
18 #include "GEDToIGATranslation.hpp"
19 #include "IGAToGEDTranslation.hpp"
20
21 #include <cstring>
22 #include <sstream>
23
24
25 // Used to label expressions that need to be removed once GED is fixed
26 #define GED_WORKAROUND(X) (X)
27
28 using namespace ::iga;
29
30 DEFINE_GED_SOURCE_ACCESSORS_01(GED_ADDR_MODE, AddrMode)
31 DEFINE_GED_SOURCE_ACCESSORS_01(int32_t, AddrImm)
32 DEFINE_GED_SOURCE_ACCESSORS_01(uint32_t, AddrSubRegNum)
33 DEFINE_GED_SOURCE_ACCESSORS_01(uint32_t, Width)
34 DEFINE_GED_SOURCE_ACCESSORS_012(GED_DATA_TYPE, DataType)
35 DEFINE_GED_SOURCE_ACCESSORS_012(GED_MATH_MACRO_EXT, MathMacroExt)
36 DEFINE_GED_SOURCE_ACCESSORS_012(uint32_t, VertStride)
37 DEFINE_GED_SOURCE_ACCESSORS_012(uint32_t, ChanSel)
38 DEFINE_GED_SOURCE_ACCESSORS_012(GED_REP_CTRL, RepCtrl)
39 DEFINE_GED_SOURCE_ACCESSORS_012(GED_SRC_MOD, SrcMod)
40 DEFINE_GED_SOURCE_ACCESSORS_012(uint32_t, HorzStride)
41 DEFINE_GED_SOURCE_ACCESSORS_01234(uint32_t, RegNum)
42 DEFINE_GED_SOURCE_ACCESSORS_01234(uint32_t, SubRegNum)
43 DEFINE_GED_SOURCE_ACCESSORS_01234(GED_REG_FILE, RegFile)
44 static void setImmValKind(Type t, ImmVal &val) {
45 switch (t) {
46 case Type::B:
47 val.kind = ImmVal::Kind::S8;
48 break;
49 case Type::UB:
50 val.kind = ImmVal::Kind::U8;
51 break;
52 case Type::W:
53 val.kind = ImmVal::Kind::S16;
54 break;
55 case Type::UW:
56 val.kind = ImmVal::Kind::U16;
57 break;
58 case Type::D:
59 val.kind = ImmVal::Kind::S32;
60 break;
61 case Type::UD:
62 val.kind = ImmVal::Kind::U32;
63 break;
64 case Type::Q:
65 val.kind = ImmVal::Kind::S64;
66 break;
67 case Type::UQ:
68 val.kind = ImmVal::Kind::U64;
69 break;
70 case Type::HF:
71 val.kind = ImmVal::Kind::F16;
72 break;
73 case Type::F:
74 val.kind = ImmVal::Kind::F32;
75 break;
76 case Type::DF:
77 val.kind = ImmVal::Kind::F64;
78 break;
79 case Type::V: // fallthrough for the packed vector types
80 case Type::UV:
81 case Type::VF:
82 val.kind = ImmVal::Kind::U32;
83 break;
84 default:
85 break;
86 }
87 }
88
89 static Region macroDefaultSourceRegion(int srcOpIx, const OpSpec &os,
90 Platform p, ExecSize execSize) {
91 if (os.hasImplicitSrcRegion(srcOpIx, execSize, true)) {
92 return os.implicitSrcRegion(srcOpIx, execSize, true);
93 } else if (srcOpIx == 2) {
94 return Region::SRCXX1;
95 } else {
96 if (p >= Platform::XE) {
97 return os.isTernary() ? Region::SRC1X0 : // XE ternary packed region
98 Region::SRC110;
99 }
100 return os.isTernary() ? Region::SRC2X1 : Region::SRC441;
101 }
102 }
103
104 Decoder::Decoder(const Model &model, ErrorHandler &errHandler)
105 : GEDBitProcessor(model, errHandler) {
106 m_gedModel = lowerPlatform(model.platform);
107 m_SWSBEncodeMode = model.getSWSBEncodeMode();
108 IGA_ASSERT(m_gedModel != GED_MODEL_INVALID, "invalid GED model");
109 }
110
111 void Decoder::decodeSWSB(Instruction *inst) {
112 if (platform() >= Platform::XE) {
113 uint32_t swsbBits = 0;
114 if (inst->getOp() != Op::INVALID && inst->getOp() != Op::ILLEGAL) {
115 GED_DECODE_RAW_TO(SWSB, swsbBits);
116 }
117 // must convert the raw encoding bits to our SWSB IR
118 SWSB::InstType instType = inst->getSWSBInstType(m_SWSBEncodeMode);
119 SWSB sw;
120 SWSB_STATUS status = sw.decode(swsbBits, m_SWSBEncodeMode, instType);
121
122 switch (status) {
123 case SWSB_STATUS::SUCCESS:
124 break;
125 case SWSB_STATUS::ERROR_SET_ON_VARIABLE_LENGTH_ONLY:
126 errorT("SBID set is only allowed on variable latency ops");
127 break;
128 case SWSB_STATUS::ERROR_INVALID_SBID_VALUE:
129 errorT("invalid SBID value 0x%x", swsbBits);
130 break;
131 case SWSB_STATUS::ERROR_ENCODE_MODE:
132 errorT("invalid encoding mode for platform");
133 break;
134 default:
135 errorT("unknown error decoding SBID value 0x%x", swsbBits);
136 break;
137 }
138 inst->setSWSB(sw);
139 }
140 }
141
142 Kernel *Decoder::decodeKernelBlocks(const void *binary, size_t binarySize) {
143 return decodeKernel(binary, binarySize, false);
144 }
145
146 Kernel *Decoder::decodeKernelNumeric(const void *binary, size_t binarySize) {
147 return decodeKernel(binary, binarySize, true);
148 }
149
150 bool Decoder::isMacro() const {
151 return m_opSpec->is(Op::MADM) ||
152 (m_opSpec->is(Op::MATH) && IsMacro(m_subfunc.math));
153 }
154
155 Kernel *Decoder::decodeKernel(const void *binary, size_t binarySize,
156 bool numericLabels) {
157 m_binary = binary;
158 if (binarySize == 0) {
159 // edge case: empty kernel is okay
160 return new Kernel(m_model);
161 }
162 if (binarySize < 8) {
163 // bail if we don't have at least a compact instruction
164 errorT("binary size is too small");
165 return nullptr;
166 }
167 Kernel *kernel = new Kernel(m_model);
168
169 InstList insts;
170 // NOTE: we could pre-allocate instruction list here
171 // (this would block allocate everything)
172 // insts.reserve(binarySize / 8 + 1);
173
174 // Pass 1. decode them all into Instruction objects
175 decodeInstructions(*kernel, binary, binarySize, insts);
176
177 if (numericLabels) {
178 Block *block = kernel->createBlock();
179 block->setPC(0);
180 block->setID(1);
181 for (Instruction *inst : insts) {
182 block->appendInstruction(inst);
183 }
184 kernel->appendBlock(block);
185 } else {
186 auto blockStarts =
187 Block::inferBlocks(errorHandler(), kernel->getMemManager(), insts);
188 int id = 1;
189 for (const auto &bitr : blockStarts) {
190 bitr.second->setID(id++);
191 kernel->appendBlock(bitr.second);
192 }
193 }
194 return kernel;
195 }
196
197 Subfunction Decoder::decodeSubfunction(bool &valid) {
198 Subfunction sf = InvalidFC::INVALID;
199 const auto errsBefore = getErrorCount();
200 switch (m_opSpec->op) {
201 case Op::IF:
202 case Op::ELSE:
203 case Op::GOTO: {
204 BranchCntrl bc = BranchCntrl::OFF;
205 if (m_opSpec->supportsBranchCtrl()) {
206 // HSW doesn't support this
207 GED_DECODE_TO(BranchCtrl, translate, bc);
208 }
209 sf = bc;
210 break;
211 }
212 case Op::MATH: {
213 GED_DECODE(MathFC, GED_MATH_FC, mathFc, MathFC);
214 sf = mathFc;
215 if (mathFc == MathFC::INVALID) {
216 errorT("invalid MathFC");
217 }
218 break;
219 }
220 case Op::SHFL: {
221 GED_DECODE(ShuffleFC, GED_SHUFFLE_FC, shuffleFc, ShuffleFC);
222 sf = shuffleFc;
223 if (shuffleFc == ShuffleFC::INVALID) {
224 errorT("invalid ShuffleFC");
225 }
226 break;
227 }
228 case Op::LFSR: {
229 GED_DECODE(LfsrFC, GED_LFSR_FC, lfsrFc, LfsrFC);
230 sf = lfsrFc;
231 if (lfsrFc == LfsrFC::INVALID) {
232 errorT("invalid ShuffleFC");
233 }
234 break;
235 }
236 case Op::DNSCL: {
237 GED_DECODE(ConvSrcDataType, GED_CONV_SRC_DATATYPE, srcTy, ConvSrcDataType);
238 GED_DECODE(ConvDstDataType, GED_CONV_DST_DATATYPE, dstTy, ConvDstDataType);
239 GED_DECODE(DnsclMode, GED_DNSCL_MODE, dmode, DnsclMode);
240 GED_DECODE(RoundingMode, GED_ROUNDING_MODE, rmode, RoundingMode);
241 sf = DnsclFC(srcTy, dstTy, dmode, rmode);
242 if (srcTy == ConvSrcDataType::INVALID ||
243 dstTy == ConvDstDataType::INVALID ||
244 dmode == DnsclMode::INVALID ||
245 rmode == RoundingMode::INVALID) {
246 // FIXME: vague error message to avoid IP issue
247 errorT("invalid FC");
248 }
249 break;
250 }
251 case Op::BFN: {
252 GED_DECODE_RAW(uint32_t, lut8, BfnFC);
253 sf = BfnFC((uint8_t)lut8);
254 break;
255 }
256 case Op::DPAS:
257 case Op::DPASW:
258 {
259 // GED splits this field up; we fuse it as a single subfunction
260 uint32_t systolicDepth;
261 GED_DECODE_RAW_TO_SRC(systolicDepth, uint32_t, SystolicDepth);
262 uint32_t repeatCount;
263 GED_DECODE_RAW_TO_SRC(repeatCount, uint32_t, RepeatCount);
264 sf = GetDpasFC(systolicDepth, repeatCount);
265 break;
266 }
267 case Op::BDPAS:
268 {
269 // in bdpas, repeat count is set to 8; no repeat count field
270 // GED splits this field up; we fuse it as a single subfunction
271 uint32_t systolicDepth;
272 // Note that bdpas's enum class for systolic depth is different from dpas'.
273 // Hence GED has different API name (GED_SetSystolicDepthBlkScl).
274 GED_DECODE_RAW_TO_SRC(systolicDepth, uint32_t, SystolicDepthBlkScl);
275 const uint32_t repeatCount = 8;
276 sf = GetDpasFC(systolicDepth, repeatCount);
277 break;
278 }
279 case Op::SYNC: {
280 GED_DECODE(SyncFC, GED_SYNC_FC, syncFC, SyncFC);
281 sf = syncFC;
282 break;
283 }
284 case Op::SENDSC:
285 case Op::SENDS:
286 // handled in descriptor decoding
287 break;
288 case Op::SENDC:
289 case Op::SEND:
290 if (platform() >= Platform::XE) {
291 GED_DECODE(SFID, GED_SFID, sfid, SFID);
292 sf = sfid;
293 } // else handled in descriptor decoding
294 break;
295 case Op::SENDG:
296 case Op::SENDGC:
297 case Op::SENDGX:
298 case Op::SENDGXC: {
299 GED_DECODE(SFID, GED_SFID, sfid, SFID);
300 sf = sfid;
301 break;
302 }
303 default:
304 IGA_ASSERT(!m_opSpec->supportsSubfunction(),
305 "we need to decode a subfunction here");
306 sf = InvalidFC::INVALID;
307 break;
308 }
309
310 // GED_DECODE didn't fail anywhere above
311 valid = getErrorCount() == errsBefore;
312
313 return sf;
314 }
315
316 const OpSpec *Decoder::decodeOpSpec(Op op) {
317 auto os = &m_model.lookupOpSpec(op);
318 return os;
319 }
320
321 // Pass 1. decode all instructions in Instruction*
322 void Decoder::decodeInstructions(Kernel &kernel, const void *binaryStart,
323 size_t binarySize, InstList &insts) {
324 restart();
325 uint32_t nextId = 1;
326 const unsigned char *binary = (const unsigned char *)binaryStart;
327
328 int32_t bytesLeft = (int32_t)binarySize;
329 while (bytesLeft > 0) {
330 // need at least 4 bytes to check compaction control
331 if (bytesLeft < 4) {
332 warningT("unexpected padding at end of kernel");
333 break;
334 }
335 // ensure there's enough buffer left
336 int32_t iLen = getBitField(COMPACTION_CONTROL, 1) != 0 ? COMPACTED_SIZE
337 : UNCOMPACTED_SIZE;
338 if (bytesLeft < iLen) {
339 warningT("unexpected padding at end of kernel");
340 break;
341 }
342 memset(&m_currGedInst, 0, sizeof(m_currGedInst));
343 GED_RETURN_VALUE status =
344 GED_DecodeIns(m_gedModel, binary, (uint32_t)binarySize, &m_currGedInst);
345 Instruction *inst = nullptr;
346 if (status == GED_RETURN_VALUE_NO_COMPACT_FORM) {
347 errorT("error decoding instruction (no compacted form)");
348 inst =
349 createErrorInstruction(kernel, "unable to decompact", binary, iLen);
350 // fall through: GED can sort of decode some things here
351 } else if (status != GED_RETURN_VALUE_SUCCESS) {
352 errorT("error decoding instruction");
353 inst = createErrorInstruction(kernel, "GED error decoding instruction",
354 binary, iLen);
355 } else {
356 const GED_OPCODE gedOp = GED_GetOpcode(&m_currGedInst);
357 const Op op = translate(gedOp);
358 m_opSpec = decodeOpSpec(op);
359 if (!m_opSpec->isValid()) {
360 // figure out if we failed to resolve the primary op
361 // or if it's an unmapped subfunction (e.g. math function)
362 auto os = m_model.lookupOpSpec(op);
363 std::stringstream ss;
364 if (os.isValid()) {
365 ss << " invalid subfuction under " << os.mnemonic.str()
366 << ": unsupported opcode on this platform";
367 } else {
368 ss << " ISA opcode 0x" << iga::hex((unsigned)*(const uint8_t *)binary, 2) << ""
369 << ": unsupported opcode on this platform";
370 }
371 std::string str = ss.str();
372 errorT(str);
373 inst = createErrorInstruction(kernel, str.c_str(), binary, iLen);
374 } else {
375 bool validSf = false;
376 m_subfunc = decodeSubfunction(validSf);
377 if (validSf) {
378 try {
379 inst = decodeNextInstruction(kernel);
380 } catch (const FatalError &) {
381 // error is already logged
382 inst = createErrorInstruction(
383 kernel, errorHandler().getErrors().back().message.c_str(),
384 binary, iLen);
385 }
386 } else {
387 // error is already logged
388 inst = createErrorInstruction(kernel, "invalid subfunction", binary,
389 iLen);
390 }
391 }
392 }
393 inst->setPC(currentPc());
394 inst->setID(nextId++);
395 inst->setLoc(currentPc());
396 insts.emplace_back(inst);
397 #if _DEBUG
398 if (!errorHandler().hasErrors()) {
399 // only validate if there weren't errors
400 inst->validate();
401 }
402 #endif
403 advancePc(iLen);
404 binary += iLen;
405 bytesLeft -= iLen;
406 }
407 }
408
409 void Decoder::decodeNextInstructionEpilog(Instruction *inst) {
410 decodeSWSB(inst);
411 }
412
413 // Decodes a GED instruction to IGA IR and appends it to a given block
414 Instruction *Decoder::decodeNextInstruction(Kernel &kernel) {
415 Instruction *inst = nullptr;
416
417 switch (m_opSpec->format) {
418 case OpSpec::NULLARY:
419 if (m_opSpec->op == Op::ILLEGAL) {
420 inst = kernel.createIllegalInstruction();
421 } else if (m_opSpec->op == Op::NOP) {
422 inst = kernel.createNopInstruction();
423 } else if (m_opSpec->op == Op::THRYLD) {
424 inst = kernel.createThryldInstruction();
425 } else {
426 std::stringstream ss;
427 ss << "at pc " << currentPc() << ": invalid operation format";
428 IGA_ASSERT_FALSE(ss.str().c_str());
429 return kernel.createIllegalInstruction();
430 }
431 break;
432 case OpSpec::BASIC_UNARY_REG:
433 case OpSpec::BASIC_UNARY_REGIMM:
434 case OpSpec::BASIC_BINARY_REG_IMM:
435 case OpSpec::BASIC_BINARY_REG_REG:
436 case OpSpec::BASIC_BINARY_REG_REGIMM:
437 case OpSpec::MATH_BINARY_REG_REGIMM:
438 inst = decodeBasicInstruction(kernel);
439 break;
440 case OpSpec::JUMP_UNARY_REG:
441 case OpSpec::JUMP_UNARY_IMM:
442 case OpSpec::JUMP_UNARY_REGIMM:
443 case OpSpec::JUMP_UNARY_CALL_REGIMM:
444 case OpSpec::JUMP_BINARY_BRC:
445 case OpSpec::JUMP_BINARY_IMM_IMM:
446 if (m_model.supportsSimplifiedBranches()) {
447 inst = decodeBranchSimplifiedInstruction(kernel);
448 } else {
449 inst = decodeBranchInstruction(kernel);
450 }
451 break;
452 case OpSpec::TERNARY_REGIMM_REG_REGIMM:
453 inst = decodeTernaryInstruction(kernel);
454 break;
455 case OpSpec::QUINARY_SRC:
456 inst = decodeQuinaryInstruction(kernel);
457 break;
458 case OpSpec::SEND_UNARY:
459 case OpSpec::SEND_BINARY:
460 inst = decodeSendInstruction(kernel);
461 break;
462 case OpSpec::SYNC_UNARY:
463 if (platform() < Platform::XE) {
464 inst = decodeWaitInstruction(kernel);
465 } else {
466 inst = decodeSyncInstruction(kernel);
467 }
468 break;
469 default: {
470 std::stringstream ss;
471 ss << "at pc " << currentPc() << ": invalid operation format\n";
472 ss << FormatOpBits(m_model, (const uint8_t *)m_binary + currentPc());
473
474 IGA_ASSERT_FALSE(ss.str().c_str());
475 return kernel.createIllegalInstruction();
476 } // default:
477 } // switch
478
479 decodeOptions(inst);
480 decodeNextInstructionEpilog(inst);
481
482 return inst;
483 }
484
485 bool Decoder::hasImm64Src0Overlap() {
486 if (platform() < Platform::XE)
487 return false;
488
489 // SWSB overlaps the flag modifier with src0
490 // check if it's 64 bit imm
491 GED_REG_FILE regFile = decodeSrcRegFile<SourceIndex::SRC0>();
492 Type t = decodeSrcType<SourceIndex::SRC0>();
493 return (TypeIs64b(t) && (regFile == GED_REG_FILE_IMM));
494 }
495
496 ///////////////////////////////////////////////////////////////////////
497 // BASIC INSTRUCTIONS
498 ///////////////////////////////////////////////////////////////////////
499
500 Instruction *Decoder::decodeBasicInstruction(Kernel &kernel) {
501 FlagRegInfo fri = decodeFlagRegInfo(hasImm64Src0Overlap());
502 Instruction *inst = kernel.createBasicInstruction(
503 *m_opSpec, fri.pred, fri.reg, decodeExecSize(), decodeChannelOffset(),
504 decodeMaskCtrl(), fri.modifier, m_subfunc);
505
506 GED_ACCESS_MODE accessMode = decodeAccessMode();
507 if (m_opSpec->supportsDestination()) {
508 decodeBasicDestination(inst, accessMode);
509 }
510 switch (m_opSpec->format) {
511 case OpSpec::BASIC_UNARY_REG:
512 case OpSpec::BASIC_UNARY_REGIMM:
513 decodeBasicUnaryInstruction(inst, accessMode);
514 break;
515 case OpSpec::BASIC_BINARY_REG_IMM:
516 case OpSpec::BASIC_BINARY_REG_REG:
517 case OpSpec::BASIC_BINARY_REG_REGIMM:
518 case OpSpec::MATH_BINARY_REG_REGIMM:
519 decodeSourceBasic<SourceIndex::SRC0>(inst, accessMode);
520 if (inst->getSourceCount() > 1) {
521 // math can have one or two ops
522 decodeSourceBasic<SourceIndex::SRC1>(inst, accessMode);
523 }
524 break;
525 default:
526 std::stringstream ss;
527 ss << "IGA INTERNAL ERROR: ";
528 ss << FormatOpBits(m_model, (const char *)m_binary + currentPc());
529 ss << ": unexpected format for basic instruction";
530 IGA_ASSERT_FALSE(ss.str().c_str());
531 errorT(ss.str());
532 inst = kernel.createIllegalInstruction();
533 }
534
535 return inst;
536 }
537
538 void Decoder::decodeBasicUnaryInstruction(Instruction *inst,
539 GED_ACCESS_MODE accessMode) {
540 decodeSourceBasic<SourceIndex::SRC0>(inst, accessMode);
541 if (m_opSpec->op == Op::MOVI && platform() >= Platform::GEN10) {
542 // movi can takes two parameters on on this platform
543 // movi (..) reg reg (imm|null)
544 decodeSourceBasic<SourceIndex::SRC1>(inst, accessMode);
545 }
546 }
547
548 void Decoder::decodeBasicDestinationAlign16(Instruction *inst) {
549 GED_DECODE_RAW(GED_ADDR_MODE, addrMode, DstAddrMode);
550
551 DstModifier dstMod = DstModifier::NONE;
552 if (inst->getOpSpec().supportsSaturation()) {
553 GED_DECODE_RAW(GED_SATURATE, mod, Saturate);
554 dstMod = translate(mod);
555 }
556
557 GED_DECODE(Type, GED_DATA_TYPE, type, DstDataType);
558
559 switch (addrMode) {
560 case GED_ADDR_MODE_Direct: {
561 DirRegOpInfo dri = decodeDstDirRegInfo();
562 if (inst->isMacro()) {
563 MathMacroExt MathMacroReg = decodeDestinationMathMacroRegFromChEn();
564 inst->setMacroDestination(dstMod, dri.regName, dri.regRef, MathMacroReg,
565 Region::Horz::HZ_1, type);
566 } else {
567 // normal Align16 destination
568 uint32_t hStride = 1;
569 GED_DECODE_RAW(GED_DST_CHAN_EN, chEn, DstChanEn);
570 if (dri.regName == RegName::ARF_MME &&
571 isAlign16MathMacroRegisterCsrPlatform()) {
572 // special access to acc2-acc9 via ChEn encoding
573 // (for context save and restore)
574 dri.regRef.regNum = (uint16_t)decodeDestinationRegNumAccBitsFromChEn();
575 } else if (chEn == GED_DST_CHAN_EN_xyzw) {
576 hStride = 1;
577 } else {
578 fatalT("dst: unsupported Align16 ChEn; only <1> (.xyzw) supported");
579 }
580
581 GED_DECODE_RAW(uint32_t, subRegNum, DstSubRegNum);
582 inst->setDirectDestination(dstMod, dri.regName, dri.regRef,
583 translateRgnH(hStride), type);
584 }
585 break;
586 }
587 case GED_ADDR_MODE_Indirect: {
588 GED_DECODE_RAW(GED_DST_CHAN_EN, chEn, DstChanEn);
589 if (chEn == GED_DST_CHAN_EN_xyzw) {
590 warningT("converting unary/binary Align16 dst to equivalent Align1");
591 } else {
592 fatalT("unsupported Align16 Dst.ChEn (only .xyzw supported)");
593 }
594
595 GED_DECODE_RAW(int32_t, addrImm, DstAddrImm);
596 GED_DECODE_RAW(uint32_t, subRegNum, DstAddrSubRegNum);
597 RegRef a0(0u, subRegNum);
598 inst->setIndirectDestination(dstMod, a0, (uint16_t)addrImm,
599 Region::Horz::HZ_1, type);
600 break;
601 }
602 default:
603 fatalT("invalid addressing mode on dst");
604 break;
605 } // switch
606 }
607
608 void Decoder::decodeBasicDestinationAlign1(Instruction *inst) {
609 GED_ADDR_MODE addrMode = GED_ADDR_MODE_Direct;
610 GED_DECODE_RAW(GED_ADDR_MODE, taddrMode, DstAddrMode);
611 addrMode = taddrMode;
612
613 DstModifier dstMod = DstModifier::NONE;
614 if (inst->getOpSpec().supportsSaturation()) {
615 GED_DECODE_RAW(GED_SATURATE, mod, Saturate);
616 dstMod = translate(mod);
617 }
618
619 GED_DECODE_RAW(uint32_t, hStride, DstHorzStride);
620 Region::Horz rgnHzDec = translateRgnH(hStride);
621 if (inst->getOpSpec().hasImplicitDstRegion(isMacro())) {
622 Region::Horz rgnHzImpl =
623 inst->getOpSpec().implicitDstRegion(isMacro()).getHz();
624 if (rgnHzImpl != rgnHzDec) {
625 warningT("dst has wrong region for binary normal form");
626 }
627 }
628
629 GED_DECODE(Type, GED_DATA_TYPE, type, DstDataType);
630
631 switch (addrMode) {
632 case GED_ADDR_MODE_Direct: {
633 GED_DECODE_RAW(GED_REG_FILE, regFile, DstRegFile);
634 if (regFile != GED_REG_FILE_ARF && regFile != GED_REG_FILE_GRF) {
635 errorT("invalid reg file on dst");
636 }
637
638 DirRegOpInfo dri = decodeDstDirRegInfo();
639 if (inst->isMacro()) {
640 GED_DECODE(MathMacroExt, GED_MATH_MACRO_EXT, mme, DstMathMacroExt);
641 inst->setMacroDestination(dstMod, dri.regName, dri.regRef, mme, rgnHzDec,
642 type);
643 } else {
644 // normal Align1 destination
645 // it's a normal Align1 destination
646 GED_DECODE_RAW(uint32_t, subRegNum, DstSubRegNum);
647 inst->setDirectDestination(dstMod, dri.regName, dri.regRef, rgnHzDec,
648 type);
649 }
650 break;
651 }
652 case GED_ADDR_MODE_Indirect: {
653 GED_DECODE_RAW(int32_t, addrImm, DstAddrImm);
654 GED_DECODE_RAW(uint32_t, subRegNum, DstAddrSubRegNum);
655 RegRef a0(0u, subRegNum);
656 inst->setIndirectDestination(dstMod, a0, (uint16_t)addrImm, rgnHzDec, type);
657 break;
658 }
659 default:
660 fatalT("invalid addressing mode on dst");
661 break;
662 } // switch
663 }
664
665 ///////////////////////////////////////////////////////////////////////
666 // QUINARY INSTRUCTIONS
667 ///////////////////////////////////////////////////////////////////////
668 Instruction *Decoder::decodeQuinaryInstruction(Kernel &kernel) {
669 FlagRegInfo fri = decodeFlagRegInfo();
670 Instruction *inst = kernel.createBasicInstruction(
671 *m_opSpec, fri.pred, fri.reg, decodeExecSize(), decodeChannelOffset(),
672 decodeMaskCtrl(), fri.modifier, m_subfunc);
673
674 decodeQuinaryInstructionOperands(inst);
675 return inst;
676 }
677
678 void Decoder::decodeQuinaryInstructionOperands(Instruction *inst) {
679 if (m_opSpec->is(Op::BDPAS)) {
680 decodeInstructionOperandsForBDPAS(inst);
681 } else {
682 // at the moment, bdpas is the only instruction with 5 operands
683 errorT("incorrect instruction");
684 }
685 }
686
687 void Decoder::decodeInstructionOperandsForBDPAS(Instruction *inst) {
688 decodeBDPASDestination(inst);
689 // breaking up into function that deals with sources 0, 1, 2 and function
690 // that deals with sources 3 and 4 (scaling sources)
691 // Certain GED functions are specific to source operands
692 decodeBDPASSource<SourceIndex::SRC0>(inst);
693 decodeBDPASSource<SourceIndex::SRC1>(inst);
694 decodeBDPASSource<SourceIndex::SRC2>(inst);
695 decodeBDPASScalingSource<SourceIndex::SRC3>(inst);
696 decodeBDPASScalingSource<SourceIndex::SRC4>(inst);
697 }
698
699 void Decoder::decodeBDPASDestination(Instruction *inst) {
700 DirRegOpInfo dri = decodeDstDirRegInfo();
701
702 GED_DECODE_RAW(uint32_t, hStride, DstHorzStride);
703 inst->setDirectDestination(DstModifier::NONE, dri.regName, dri.regRef,
704 translateRgnH(hStride), dri.type);
705 }
706
707 template<SourceIndex S>
708 void Decoder::decodeBDPASScalingSource(Instruction *inst) {
709 const OpSpec &os = inst->getOpSpec();
710
711 uint32_t regNumBits = decodeSrcRegNum<S>();
712 RegName regName = RegName::INVALID;
713 GED_REG_FILE regFile = decodeSrcRegFile<S>();
714 RegRef regRef;
715 decodeReg((int)S, regFile, regNumBits, regName, regRef);
716 // src 3 and 4 have sub reg num
717 regRef.subRegNum = (uint16_t)decodeSrcSubRegNum<S>();
718
719 Region dftRgn =
720 os.implicitSrcRegion((int)S, inst->getExecSize(), isMacro());
721 // src3 and src4 are restricted to UB data types
722 inst->setDirectSource(S, SrcModifier::NONE, regName, regRef, dftRgn,
723 Type::UB);
724 }
725
726 template <SourceIndex S>
727 void Decoder::decodeBDPASSource(Instruction *inst) {
728 const OpSpec &os = inst->getOpSpec();
729 Type ty = decodeSrcType<S>();
730
731 uint32_t regNumBits = decodeSrcRegNum<S>();
732 RegName regName = RegName::INVALID;
733 GED_REG_FILE regFile = decodeSrcRegFile<S>();
734 RegRef regRef;
735 decodeReg((int)S, regFile, regNumBits, regName, regRef);
736 // src 1, 2 do not have sub reg number
737 regRef.subRegNum = 0;
738
739 if (S == SourceIndex::SRC0) {
740 // src0 has sub reg number
741 auto binoff = (uint16_t)decodeSrcSubRegNum<S>();
742 regRef.subRegNum = (uint16_t)BinaryOffsetToSubReg(binoff, regName, ty,
743 m_model.platform);
744 } if (S == SourceIndex::SRC1) {
745 GED_DECODE_TO(Src1Precision, translate, ty);
746 } else if (S == SourceIndex::SRC2) {
747 GED_DECODE_TO(Src2Precision, translate, ty);
748 }
749
750 Region dftRgn =
751 os.implicitSrcRegion((int)S, inst->getExecSize(), isMacro());
752 inst->setDirectSource(S, SrcModifier::NONE, regName, regRef, dftRgn,
753 ty);
754 }
755 ///////////////////////////////////////////////////////////////////////
756 // TERNARY INSTRUCTIONS
757 ///////////////////////////////////////////////////////////////////////
758 Instruction *Decoder::decodeTernaryInstruction(Kernel &kernel) {
759 FlagRegInfo fri = decodeFlagRegInfo();
760 Instruction *inst = kernel.createBasicInstruction(
761 *m_opSpec, fri.pred, fri.reg, decodeExecSize(), decodeChannelOffset(),
762 decodeMaskCtrl(), fri.modifier, m_subfunc);
763
764 GED_ACCESS_MODE accessMode = decodeAccessMode();
765 decodeTernaryInstructionOperands(kernel, inst, accessMode);
766
767 return inst;
768 }
769
770 void Decoder::decodeTernaryInstructionOperands(Kernel &kernel,
771 Instruction *inst,
772 GED_ACCESS_MODE accessMode) {
773 if (accessMode == GED_ACCESS_MODE_Align16) {
774 if (m_opSpec->supportsDestination()) {
775 decodeTernaryDestinationAlign16(inst);
776 }
777 decodeTernarySourceAlign16<SourceIndex::SRC0>(inst);
778 decodeTernarySourceAlign16<SourceIndex::SRC1>(inst);
779 decodeTernarySourceAlign16<SourceIndex::SRC2>(inst);
780 } else {
781 if (platform() >= Platform::GEN10) {
782 if (m_opSpec->supportsDestination()) {
783 decodeTernaryDestinationAlign1(inst);
784 }
785 decodeTernarySourceAlign1<SourceIndex::SRC0>(inst);
786 decodeTernarySourceAlign1<SourceIndex::SRC1>(inst);
787 decodeTernarySourceAlign1<SourceIndex::SRC2>(inst);
788 } else {
789 errorT("unexpected Align1 Ternary in current platform");
790 inst = kernel.createIllegalInstruction();
791 }
792 }
793 }
794
795 void Decoder::decodeTernaryDestinationAlign16(Instruction *inst) {
796 GED_DECODE_RAW(uint32_t, regNumBits, DstRegNum);
797 DstModifier dstMod = DstModifier::NONE;
798 if (m_opSpec->supportsSaturation()) {
799 GED_DECODE_RAW(GED_SATURATE, mod, Saturate);
800 dstMod = translate(mod);
801 }
802 GED_DECODE(Type, GED_DATA_TYPE, type, DstDataType);
803 GED_DECODE_RAW(GED_REG_FILE, regFile, DstRegFile);
804 GED_DECODE_RAW(GED_DST_CHAN_EN, chEn, DstChanEn);
805
806 RegName regName = RegName::INVALID;
807 RegRef regRef;
808 decodeReg(-1, regFile, regNumBits, regName, regRef);
809
810 if (inst->isMacro()) {
811 MathMacroExt MathMacroReg = decodeDestinationMathMacroRegFromChEn();
812 inst->setMacroDestination(dstMod, regName, regRef, MathMacroReg,
813 Region::Horz::HZ_1, type);
814 } else {
815 // We have to translate Align16 ternary instructions to equivalent
816 // Align1 where posssible. The goal of these translations is to
817 // capture everything the IGC compiler generates. There will be
818 // valid Align16 sequences that we choose not to represent in Align1.
819 //
820 // CASES:
821 // SIMD1: illegal (hardware disallows this, we use a SIMD4 with ChEn to
822 // emulate) SIMD2: We accept .xy, .zw only if the type is :df. E.g.
823 // mad (2) r5.0.xy:df ... // means r5.0<1>
824 // mad (2) r5.0.zw:df ... // means r5.1<1>
825 // *** we convert the exec size to SIMD1 ***
826 // SIMD4: This can be a true SIMD4 or an emulation of a SIMD1 scalar.
827 // We decide based on the ChEn mask.
828 // .xyzw is a SIMD4, but .{x,y,z,w} means scalar SIMD1
829 // (since other lanes are masked out)
830 // *** we convert the exec size to SIMD1 for the scalar case ***
831 // I.e.
832 // mad (4) r5.0.xyzw:f ... // gets converted cleanly
833 // to
834 // mad (4) r5.0<1>:f
835 // but
836 // mad (4) r5.0.x:f ... {NoMask) // gets treated as scalar and
837 // translates to (W) mad (1) r5.0.x
838 //
839 // SIMD8, SIMD16: we only accept .xyzw and .r as packed and scalar
840 // this seems to capture everything used in practice
841 //
842 // NOTE: this is an appalling hack, but creates the least technical
843 // debt for the project. These problems all go away in GEN10 when
844 // we get Align1 ternary and Align16 fades into the sunset.
845 uint8_t subregOffAlign16Elems =
846 0; // in elements not bytes (add after conversion)
847 switch (inst->getExecSize()) {
848 case ExecSize::SIMD2:
849 if (chEn != GED_DST_CHAN_EN_xyzw) {
850 // this is a special case of below for DF and Q
851 inst->setExecSize(ExecSize::SIMD1);
852 if (chEn == GED_DST_CHAN_EN_xy && type == Type::DF) {
853 subregOffAlign16Elems = 0; // dst.k.xy:df => dst.(k+0)<1>:df
854 } else if (chEn == GED_DST_CHAN_EN_zw && type == Type::DF) {
855 subregOffAlign16Elems = 1; // dst.k.xy:df => dst.(k+1)<1>:df
856 } else {
857 errorT("unsupported Align16 ternary destination for SIMD2"
858 " (must be .xywz or .{xy,zw} for :df)");
859 }
860 }
861 break;
862 case ExecSize::SIMD4:
863 if (chEn != GED_DST_CHAN_EN_xyzw) {
864 // with Align16, we emulate a scalar (SIMD1) by masking out
865 // all, but one of the channels.
866 // we must translate a SIMD4
867 // mad (4) r5.0.x:f ... {NoMask}
868 // to the following
869 // (W) mad (1) r5.0<1>:f ...
870 // we have to twiddle the execution size here too
871 inst->setExecSize(ExecSize::SIMD1);
872 switch (chEn) {
873 case GED_DST_CHAN_EN_x:
874 subregOffAlign16Elems = 0; // subregister is already aligned
875 break;
876 case GED_DST_CHAN_EN_y:
877 subregOffAlign16Elems =
878 1; // dst.k.y => dst.(k+1)<1> (e.g. dst.4.y => dst.5<1>)
879 break;
880 case GED_DST_CHAN_EN_z:
881 subregOffAlign16Elems = 2; // dst.k.z => dst.(k+2)<1>
882 break;
883 case GED_DST_CHAN_EN_w:
884 subregOffAlign16Elems = 3; // dst.k.w => dst.(k+3)<1>
885 break;
886 default:
887 errorT("unsupported Align16 ternary destination for SIMD4"
888 " (must be .xywz or .{x,y,z,w})");
889 break;
890 }
891 } // else { it's an .xyzw ChEn: we can leave the subregister alone }
892 break;
893 case ExecSize::SIMD8:
894 case ExecSize::SIMD16:
895 case ExecSize::SIMD32: // can appear for things like :hf, :w, or :b
896 if (chEn != GED_DST_CHAN_EN_xyzw) {
897 errorT("unsupported Align16 ternary destination for SIMD{8,16}"
898 " (must be .xywz)");
899 }
900 // the access must already be aligned for .xyzw
901 break;
902 default:
903 // SIMD1 is illegal
904 errorT("unsupported Align16 ternary destination (unsupported SIMD)");
905 break;
906 }
907
908 GED_DECODE_RAW(uint32_t, subRegNumBytes, DstSubRegNum);
909 uint16_t subRegNumber =
910 type == Type::INVALID
911 ? 0
912 : (uint16_t)BinaryOffsetToSubReg(subRegNumBytes, regName, type,
913 m_model.platform);
914 regRef.subRegNum = (uint16_t)(subRegNumber + subregOffAlign16Elems);
915 inst->setDirectDestination(dstMod, regName, regRef, Region::Horz::HZ_1,
916 type);
917 }
918 }
919
920 template <SourceIndex S>
921 void Decoder::decodeTernarySourceAlign16(Instruction *inst) {
922 bool isMacro = inst->isMacro(); // madm or math.invm or math.rsqrt
923
924 if (!isMacro && m_model.supportsAlign16MacroOnly()) {
925 warningT("src", (int)S,
926 ": converting Align16 to Align1 "
927 "(bits will re-assemble to Align1)");
928 }
929
930 SrcModifier srcMod = decodeSrcModifier<S>();
931
932 ///////////////////////////////////////////////////////////////////////////
933 // register name, number and and register file
934 // (will be GRF)
935 uint32_t regNum = decodeSrcRegNum<S>();
936
937 ///////////////////////////////////////////////////////////////////////////
938 // swizzling / region
939 GED_DATA_TYPE gedType;
940 GED_DECODE_RAW_TO(SrcDataType, gedType);
941 if (platform() >= Platform::GEN8LP &&
942 (gedType == GED_DATA_TYPE_f || gedType == GED_DATA_TYPE_hf) &&
943 S > SourceIndex::SRC0) {
944 // CHV+ mixed mode
945 gedType = decodeSrcDataType<S>();
946 }
947 Type type = translate(gedType);
948
949 if (isMacro) {
950 MathMacroExt MathMacroReg = decodeSrcMathMacroReg<S>();
951 RegRef rr(regNum, 0u);
952 Region macroDftSrcRgn = macroDefaultSourceRegion(
953 (int)S, inst->getOpSpec(), platform(), inst->getExecSize());
954 inst->setMacroSource(S, srcMod, RegName::GRF_R, rr, MathMacroReg,
955 macroDftSrcRgn, type);
956 } else {
957 int subReg =
958 type == Type::INVALID
959 ? 0
960 : BinaryOffsetToSubReg(decodeSrcSubRegNum<S>(), RegName::GRF_R,
961 type, m_model.platform);
962 RegRef reg = RegRef(regNum, (uint32_t)subReg);
963 Region rgn;
964 if (decodeSrcRepCtrl<S>() == GED_REP_CTRL_NoRep) {
965 GED_SWIZZLE swizzle[4];
966 decodeChSelToSwizzle(decodeSrcChanSel<S>(), swizzle);
967 bool isFullSwizzle =
968 (swizzle[0] == GED_SWIZZLE_x && swizzle[1] == GED_SWIZZLE_y &&
969 swizzle[2] == GED_SWIZZLE_z && swizzle[3] == GED_SWIZZLE_w);
970
971 bool isXYSwizzle =
972 (swizzle[0] == GED_SWIZZLE_x && swizzle[1] == GED_SWIZZLE_y &&
973 swizzle[2] == GED_SWIZZLE_x && swizzle[3] == GED_SWIZZLE_y);
974
975 bool isYZSwizzle =
976 (swizzle[0] == GED_SWIZZLE_z && swizzle[1] == GED_SWIZZLE_w &&
977 swizzle[2] == GED_SWIZZLE_z && swizzle[3] == GED_SWIZZLE_w);
978
979 bool invalidSwizzle = false;
980 if (TypeIs64b(type)) {
981 invalidSwizzle = !isFullSwizzle && !isXYSwizzle && !isYZSwizzle;
982 } else {
983 invalidSwizzle = !isFullSwizzle;
984 }
985
986 if (invalidSwizzle) {
987 fatalT("unconvertible ternary align16 operand");
988 }
989
990 // mad (8) r46.0.xyzw:df r46.0.xyzw:df r50.0.xyzw:df r48.0.xyzw:df
991 // {Align16, Q1} mad (2) r5.0.xy:df r5.0.xyxy:df r92.2.xyxy:df
992 // r93.0.xyxy:df {Align16, Q1, NoMask} a HW hack for scalar operation on
993 // DF
994 if (type == Type::DF && (isXYSwizzle || isYZSwizzle)) {
995 if (S == SourceIndex::SRC2) {
996 rgn = Region::SRCXX0;
997 } else {
998 rgn = Region::SRC0X0;
999 }
1000 if (isYZSwizzle) {
1001 reg.subRegNum += 1;
1002 }
1003 } else {
1004 // we accept r#.xyzw as r#<2;1>
1005 if (S == SourceIndex::SRC2) {
1006 rgn = Region::SRCXX1;
1007 } else {
1008 rgn = Region::SRC2X1;
1009 }
1010 }
1011 } else {
1012 // r#.r is the same as r#<0;0>
1013 if (S == SourceIndex::SRC2) {
1014 rgn = Region::SRCXX0;
1015 } else {
1016 rgn = Region::SRC0X0;
1017 }
1018 }
1019 inst->setDirectSource(S, srcMod, RegName::GRF_R, reg, rgn, type);
1020 }
1021 }
1022
1023 static bool ternaryDstOmitsHzStride(const OpSpec &os) {
1024 return os.isDpasFormat();
1025 }
1026
1027 void Decoder::decodeTernaryDestinationAlign1(Instruction *inst) {
1028 const OpSpec &os = inst->getOpSpec();
1029
1030 DstModifier dstMod = DstModifier::NONE;
1031 if (os.supportsSaturation()) {
1032 GED_DECODE_RAW(GED_SATURATE, mod, Saturate);
1033 dstMod = translate(mod);
1034 }
1035
1036 DirRegOpInfo dri = decodeDstDirRegInfo();
1037
1038 if (inst->isMacro()) {
1039 GED_DECODE(MathMacroExt, GED_MATH_MACRO_EXT, mme, DstMathMacroExt);
1040 inst->setMacroDestination(dstMod, dri.regName, dri.regRef, mme,
1041 Region::Horz::HZ_1, dri.type);
1042 } else {
1043 if (ternaryDstOmitsHzStride(inst->getOpSpec())) {
1044 Region::Horz dftRgnHz = os.hasImplicitDstRegion(isMacro())
1045 ? os.implicitDstRegion(isMacro()).getHz()
1046 : Region::Horz::HZ_1;
1047 inst->setDirectDestination(dstMod, dri.regName, dri.regRef, dftRgnHz,
1048 dri.type);
1049 } else {
1050 GED_DECODE_RAW(uint32_t, hStride, DstHorzStride);
1051
1052 inst->setDirectDestination(dstMod, dri.regName, dri.regRef,
1053 translateRgnH(hStride), dri.type);
1054 }
1055 }
1056 }
1057
1058 template <SourceIndex S>
1059 Region Decoder::decodeSrcRegionTernaryAlign1(const OpSpec &os) {
1060 uint32_t rgnVt = static_cast<uint32_t>(Region::Vert::VT_INVALID);
1061 bool hasRgnVt = S != SourceIndex::SRC2;
1062 if (hasRgnVt) {
1063 rgnVt = decodeSrcVertStride<S>();
1064 }
1065 //
1066 uint32_t rgnHz = decodeSrcHorzStride<S>();
1067 //
1068 return transateGEDtoIGARegion(
1069 rgnVt, static_cast<uint32_t>(Region::Width::WI_INVALID), rgnHz);
1070 }
1071
1072 // get full region from dst and src type. This function must be called
1073 // after src and dst type are decoded.
1074 static Region retrieveReducedRegionTernary(Type srcType, Type dstType,
1075 Region dstRegion) {
1076 IGA_ASSERT(dstRegion.getHz() != Region::Horz::HZ_INVALID &&
1077 srcType != Type::INVALID && dstType != Type::INVALID,
1078 "Decoder: cannot derive src1 region");
1079 auto sVertStride =
1080 dstRegion.h * TypeSizeInBits(dstType) / TypeSizeInBits(srcType);
1081 switch (sVertStride) {
1082 case 1: return Region::SRC1X0;
1083 case 2: return Region::SRC4X2;
1084 case 4: return Region::SRC8X4;
1085 default:
1086 break;
1087 }
1088 IGA_ASSERT_FALSE("Decoder: cannot derive src1 region");
1089 return Region::INVALID;
1090 }
1091
1092 template <SourceIndex S>
1093 void Decoder::decodeTernarySourceAlign1(Instruction *inst) {
1094 if (platform() < Platform::GEN10) {
1095 fatalT("Align1 not available on this platform");
1096 }
1097
1098 GED_REG_FILE regFile = decodeSrcRegFile<S>();
1099 const OpSpec &os = inst->getOpSpec();
1100 if (os.isDpasFormat()) {
1101 // DPAS specific
1102 // DPAS allowed src0 as null:
1103 // When Src0 is specified as null, it is treated as an immediate value of +0
1104 if (!(S == SourceIndex::SRC0 && regFile == GED_REG_FILE_ARF)) {
1105 if (regFile != GED_REG_FILE_GRF) {
1106 fatalT("invalid register file in src", (int)S);
1107 }
1108 }
1109
1110 RegRef regRef;
1111 RegName regName = decodeSourceReg<S>(regRef);
1112
1113 // only ARF_NULL is allowed at src0 if it's not GRF
1114 if (S == SourceIndex::SRC0 && regFile == GED_REG_FILE_ARF)
1115 if (regName != RegName::ARF_NULL)
1116 fatalT("non grf src0 register file must be null for this op");
1117
1118 Type ty = decodeSrcType<S>();
1119 if (S == SourceIndex::SRC1) {
1120 GED_DECODE_TO(Src1Precision, translate, ty);
1121 } else if (S == SourceIndex::SRC2) {
1122 GED_DECODE_TO(Src2Precision, translate, ty);
1123 } // else ty is valid already for dst/src0
1124 regRef.subRegNum = (uint16_t)BinaryOffsetToSubReg(regRef.subRegNum, regName,
1125 ty, m_model.platform);
1126 Region dftRgn =
1127 os.implicitSrcRegion((int)S, inst->getExecSize(), isMacro());
1128 inst->setDirectSource(S, decodeSrcModifier<S>(), regName, regRef, dftRgn,
1129 ty);
1130 return;
1131 } // DPAS
1132
1133 // regular ternary align1 source operand
1134 if (regFile == GED_REG_FILE_IMM) {
1135 Type type = decodeSrcType<S>();
1136
1137 ImmVal val;
1138 if (platform() < Platform::GEN10) {
1139 val = decodeSrcImmVal(type);
1140 } else {
1141 val = decodeTernarySrcImmVal<S>(type);
1142 }
1143
1144 inst->setImmediateSource(S, val, type);
1145 } else if (regFile == GED_REG_FILE_GRF || regFile == GED_REG_FILE_ARF) {
1146 // addressing mode is always direct in Align1 ternary
1147 if (inst->isMacro()) {
1148 if (m_model.supportsAlign16ImplicitAcc()) {
1149 fatalT("src", (int)S,
1150 ": macro instructions must be Align16 "
1151 "for this platform");
1152 }
1153 RegRef regRef;
1154 RegName regName = decodeSourceReg<S>(regRef);
1155 Region macroDftSrcRgn = macroDefaultSourceRegion(
1156 (int)S, inst->getOpSpec(), platform(), inst->getExecSize());
1157 inst->setMacroSource(S, decodeSrcModifier<S>(), regName, regRef,
1158 decodeSrcMathMacroReg<S>(), macroDftSrcRgn,
1159 decodeSrcType<S>());
1160 } else {
1161 // normal access
1162 std::optional<Region> rgn;
1163 bool hasReduceRegion = m_model.srcHasReducedRegion(static_cast<uint32_t>(S));
1164 if (hasReduceRegion)
1165 rgn = decodeSrcReducedRegionTernary<S>();
1166 else
1167 rgn = decodeSrcRegionTernaryAlign1<S>(inst->getOpSpec());
1168
1169 DirRegOpInfo opInfo = decodeSrcDirRegOpInfo<S>();
1170 if (hasReduceRegion && !rgn) {
1171 rgn = retrieveReducedRegionTernary(opInfo.type,
1172 inst->getDestination().getType(),
1173 inst->getDestination().getRegion());
1174 }
1175
1176 inst->setDirectSource(S, decodeSrcModifier<S>(), opInfo.regName,
1177 opInfo.regRef, *rgn, opInfo.type);
1178 }
1179 } else { // GED_REG_FILE_INVALID
1180 fatalT("invalid register file in src", (int)S);
1181 }
1182 }
1183
1184
1185 ///////////////////////////////////////////////////////////////////////
1186 // SEND INSTRUCTIONS
1187 ///////////////////////////////////////////////////////////////////////
1188 SendDesc Decoder::decodeSendExDesc() {
1189 // ex_desc
1190 GED_REG_FILE exDescRegFile = GED_REG_FILE_IMM;
1191 if (m_opSpec->format & OpSpec::Format::SEND_BINARY) {
1192 // only sends/sendsc has ExDescRegFile
1193 GED_DECODE_RAW_TO(ExDescRegFile, exDescRegFile);
1194 }
1195
1196 SendDesc exDesc;
1197 if (exDescRegFile == GED_REG_FILE_IMM) {
1198 exDesc.type = SendDesc::Kind::IMM;
1199 GED_DECODE_RAW_TO(ExMsgDescImm, exDesc.imm);
1200 } else {
1201 // For sends GED interprets SelReg32ExDesc and returns default values
1202 GED_DECODE_RAW(uint32_t, subRegNum, ExDescAddrSubRegNum);
1203 exDesc.type = SendDesc::Kind::REG32A;
1204 exDesc.reg.regNum = 0; // a0 is implied
1205 exDesc.reg.subRegNum = (uint16_t)(subRegNum / 2);
1206 }
1207 return exDesc;
1208 }
1209
1210 SendDesc Decoder::decodeSendDesc() {
1211 GED_REG_FILE descRegFile = GED_REG_FILE_IMM;
1212 GED_DECODE_RAW_TO(DescRegFile, descRegFile);
1213 SendDesc desc;
1214 if (descRegFile == GED_REG_FILE_IMM) {
1215 desc.type = SendDesc::Kind::IMM;
1216 GED_DECODE_RAW_TO(MsgDesc, desc.imm);
1217 } else {
1218 // desc register is hardwired to a0.0 (ex-desc can vary)
1219 desc.type = SendDesc::Kind::REG32A;
1220 desc.reg.regNum = 0;
1221 desc.reg.subRegNum = 0;
1222 }
1223 return desc;
1224 }
1225
1226 static void decodeMLenRlenFromDesc(const SendDesc &desc, int &src0Len,
1227 int &dstLen) {
1228 if (desc.isImm()) {
1229 src0Len = (int)(desc.imm >> 25) & 0xF;
1230 dstLen = (int)(desc.imm >> 20) & 0x1F;
1231 }
1232 }
1233
1234 void Decoder::decodeSendInfoPreXe(SendDescodeInfo &sdi) {
1235 if (sdi.exDesc.isImm()) {
1236 // in <=GEN11, it's ExDesc[3:0]
1237 // if the extended descriptor is immediate, we can extract it
1238 // from that
1239 sdi.sfid = sfidFromEncoding(platform(), sdi.exDesc.imm);
1240 } else if (sdi.exDesc.isReg()) {
1241 // given <=GEN11 and reg exdesc
1242 sdi.sfid = SFID::A0REG;
1243 }
1244 if (sdi.exDesc.isImm()) {
1245 sdi.src1Len = (int)(sdi.exDesc.imm >> 6) & 0x1F;
1246 }
1247 decodeMLenRlenFromDesc(sdi.desc, sdi.src0Len, sdi.dstLen);
1248 }
1249 void Decoder::decodeSendInfoXe(SendDescodeInfo &sdi) {
1250 sdi.sfid = m_subfunc.send;
1251 if (sdi.exDesc.isImm()) {
1252 sdi.src1Len = (int)(sdi.exDesc.imm >> 6) & 0x1F;
1253 }
1254 decodeMLenRlenFromDesc(sdi.desc, sdi.src0Len, sdi.dstLen);
1255 }
1256
1257 void Decoder::decodeSendInfoXeHP(SendDescodeInfo &sdi) {
1258 sdi.sfid = m_subfunc.send;
1259 if (sdi.exDesc.isImm()) {
1260 sdi.src1Len = (int)(sdi.exDesc.imm >> 6) & 0x1F;
1261 }
1262 decodeMLenRlenFromDesc(sdi.desc, sdi.src0Len, sdi.dstLen);
1263
1264 if (sdi.exDesc.isReg()) {
1265 // if ExBSO is set, decode Src1Length and CPS
1266 GED_DECODE_RAW(uint32_t, exBSO, ExBSO);
1267 sdi.hasExBSO = exBSO != 0;
1268 if (sdi.hasExBSO) {
1269 GED_DECODE_RAW(uint32_t, cps, CPS);
1270 sdi.hasCps = cps != 0;
1271 GED_DECODE_RAW_TO(Src1Length, sdi.src1Len);
1272 }
1273 }
1274 }
1275
1276 void Decoder::decodeSendInfoXeHPG(SendDescodeInfo &sdi) {
1277 // This is exactly the same as XeHP except that:
1278 // - all immediate descriptors encode Src1Len in the EU bits
1279 decodeSendInfoXeHP(sdi);
1280 if (sdi.exDesc.isImm()) {
1281 // >=XeHPG all immediate descriptors also have Src1Length
1282 // clobber the value XeHP decoding set
1283 GED_DECODE_RAW_TO(Src1Length, sdi.src1Len);
1284 }
1285 }
1286
1287 void Decoder::decodeSendInfoXe2(SendDescodeInfo &sdi) {
1288 // In Xe2 is similar to XeHP/XeHPG/XeHPC
1289 // - Given an Imm ExDesc Src1Len is an EU field
1290 // - When ExDesc is a reg we get the extra field
1291 // ExDescImm to add more bits to the extended descriptor
1292 // - CPS goes away unconditionally (this is now ExDescImm[11])
1293 // - if SFID is ugm then ExBSO holds an ExDescImm bit as well
1294 // - if ExDesc is a reg and SFID is not ugm, ExBSO bit denotes if
1295 // Src1Len in EU field or in ExDesc
1296 sdi.sfid = m_subfunc.send;
1297 if (sdi.exDesc.isReg()) {
1298 GED_DECODE_RAW_TO(ExMsgDescImm, sdi.exImmOffDesc);
1299 sdi.hasExBSO = false;
1300 // GED gives us the overlaps too (ExBSO and ExDescImm[15]), so
1301 // we must mask them out for non-UGM
1302 if (m_subfunc.send != SFID::UGM) {
1303 // ExDescImm overlaps ExBSO for non-UGM; clear it
1304 sdi.exImmOffDesc &= ~(1 << 15); // ExBSO overlaps ExDescImm[15]
1305
1306 // if ExBSO is set, decode Src1Length from EU field
1307 GED_DECODE_RAW(uint32_t, exBSO, ExBSO);
1308 sdi.hasExBSO = exBSO != 0;
1309 if (sdi.hasExBSO)
1310 GED_DECODE_RAW_TO(Src1Length, sdi.src1Len);
1311 } else {
1312 // for UGM, src1Length must be in EU field
1313 GED_DECODE_RAW_TO(Src1Length, sdi.src1Len);
1314 }
1315 sdi.exImmOffDesc &= ~(0x7 << 16); // ExDesc.AddrSubReg
1316 } else {
1317 // exDes is Imm, src1.len must be in EU field
1318 GED_DECODE_RAW_TO(Src1Length, sdi.src1Len);
1319 }
1320 decodeMLenRlenFromDesc(sdi.desc, sdi.src0Len, sdi.dstLen);
1321 }
1322
1323 Instruction *Decoder::decodeSendInstruction(Kernel &kernel) {
1324 if (m_opSpec->isSendgFormat()) {
1325 if (m_opSpec->is(Op::SENDGX) || m_opSpec->is(Op::SENDGXC)) {
1326 return decodeSendgxInstructionXe3p(kernel);
1327 }
1328 return decodeSendgInstructionXe3(kernel);
1329 }
1330 SendDescodeInfo sdi;
1331 sdi.desc = decodeSendDesc();
1332 sdi.exDesc = decodeSendExDesc();
1333 if (platform() < Platform::XE) {
1334 decodeSendInfoPreXe(sdi);
1335 } else if (platform() == Platform::XE) {
1336 decodeSendInfoXe(sdi);
1337 } else if (platform() == Platform::XE_HP) {
1338 decodeSendInfoXeHP(sdi);
1339 } else if (platform() >= Platform::XE_HPG &&
1340 platform() < Platform::XE2) {
1341 decodeSendInfoXeHPG(sdi);
1342 } else if (platform() >= Platform::XE2) {
1343 decodeSendInfoXe2(sdi);
1344 } else {
1345 IGA_ASSERT_FALSE("unsupported platform");
1346 }
1347
1348 FlagRegInfo fri = decodeFlagRegInfo();
1349 Instruction *inst = kernel.createSendInstruction(
1350 *m_opSpec, sdi.sfid, fri.pred, fri.reg, decodeExecSize(),
1351 decodeChannelOffset(), decodeMaskCtrl(),
1352 sdi.exImmOffDesc,
1353 sdi.exDesc, sdi.desc);
1354
1355 if ((m_opSpec->format & OpSpec::Format::SEND_BINARY) ==
1356 OpSpec::Format::SEND_BINARY) { // send is binary
1357 decodeSendDestination(inst);
1358 decodeSendSource0(inst);
1359 decodeSendSource1(inst);
1360 if (sdi.src1Len < 0 && inst->getSource(SourceIndex::SRC1).isNull()) {
1361 // if src1Len comes from a0.#[24:20], but src1 is null, then
1362 // we can still assume it's 0.
1363 sdi.src1Len = 0;
1364 }
1365 } else { // if (m_opSpec->isSendFormat()) {
1366 decodeSendDestination(inst);
1367 decodeSendSource0(inst);
1368 }
1369
1370 // No fusion in XeHPC+
1371 bool hasFusionCtrl =
1372 platform() >= Platform::XE && platform() < Platform::XE_HPC;
1373 if (hasFusionCtrl) {
1374 GED_FUSION_CTRL fusionCtrl = GED_FUSION_CTRL_Normal;
1375 GED_DECODE_RAW_TO(FusionCtrl, fusionCtrl);
1376 if (fusionCtrl == GED_FUSION_CTRL_Serialized) {
1377 inst->addInstOpt(InstOpt::SERIALIZE);
1378 }
1379 }
1380
1381 if (sdi.hasExBSO)
1382 inst->addInstOpt(InstOpt::EXBSO);
1383 if (sdi.hasCps)
1384 inst->addInstOpt(InstOpt::CPS);
1385
1386 // in case the operand lengths come from a seprate source
1387 if (inst->getSrc0Length() < 0)
1388 inst->setSrc0Length(sdi.src0Len);
1389 if (inst->getSrc1Length() < 0)
1390 inst->setSrc1Length(sdi.src1Len);
1391
1392 return inst;
1393 }
1394
1395 Instruction *Decoder::decodeSendgxInstructionXe3p(Kernel &kernel) {
1396 // Descriptor stuff
1397 RegRef id0 = REGREF_INVALID, id1 = REGREF_INVALID;
1398 GED_DECODE_RAW(uint32_t, id0Present, IndMsgDesc0IsPresent);
1399 if (id0Present) {
1400 GED_DECODE_RAW(uint32_t, id0Subreg, IndMsgDesc0Addr);
1401 id0 = RegRef((uint16_t)0, (uint16_t)(8 * id0Subreg));
1402 }
1403 GED_DECODE_RAW(uint32_t, id1Present, IndMsgDesc1IsPresent);
1404 if (id1Present) {
1405 GED_DECODE_RAW(uint32_t, id1Subreg, IndMsgDesc1Addr);
1406 id1 = RegRef((uint16_t)0, (uint16_t)(8 * id1Subreg));
1407 }
1408
1409 GED_DECODE_RAW(uint64_t, desc, MsgDesc64);
1410
1411 // Possibilties for sources
1412 // 1. Canonical send with both GRF
1413 // send... ... r10:4 r20:2 ...
1414 // Src0.RegFile = Src1.RegFile = GRF
1415 // Src0.Reg.Len = valid
1416 // Src1.Reg.Len = valid
1417 //
1418 // 2. Null as a zero register for src0. (e.g. scratch or SLM loads)
1419 // send... ... null src1 ....descs
1420 GED_DECODE_RAW(uint32_t, src0LenU, Src0Length);
1421 int src0Len = (int)src0LenU;
1422
1423 FlagRegInfo fri = decodeFlagRegInfo();
1424 Instruction *inst = kernel.createSendgInstruction(
1425 *m_opSpec, m_subfunc.send, fri.pred, fri.reg, decodeExecSize(),
1426 decodeChannelOffset(), decodeMaskCtrl(), src0Len,
1427 0, /* src1Len will be decoded later */
1428 id0, id1,
1429 desc);
1430
1431 decodeSendgxDestinationXe3p(inst);
1432 decodeSendgxSource0Xe3p(inst);
1433 decodeSendgxSource1Xe3p(inst);
1434
1435 // decode src1Len after decoding src0 reg number
1436 GED_DECODE_RAW(uint32_t, src1LenU, Src1Length);
1437 inst->setSrc1Length(src1LenU);
1438
1439 return inst;
1440 }
1441
1442 Instruction *Decoder::decodeSendgInstructionXe3(Kernel &kernel) {
1443 // Descriptor stuff
1444 RegRef id0 = REGREF_INVALID, id1 = REGREF_INVALID;
1445 GED_DECODE_RAW(uint32_t, id0Present, IndMsgDesc0IsPresent);
1446 if (id0Present) {
1447 GED_DECODE_RAW(uint32_t, id0Subreg, IndMsgDesc0Addr);
1448 id0 = RegRef((uint16_t)0, (uint16_t)(8 * id0Subreg));
1449 }
1450 GED_DECODE_RAW(uint32_t, id1Present, IndMsgDesc1IsPresent);
1451 if (id1Present) {
1452 GED_DECODE_RAW(uint32_t, id1Subreg, IndMsgDesc1Addr);
1453 id1 = RegRef((uint16_t)0, (uint16_t)(8 * id1Subreg));
1454 }
1455
1456 GED_DECODE_RAW(uint64_t, desc, MsgDesc64);
1457
1458 // Possibilties for sources
1459 // 1. Canonical send with both GRF
1460 // send... ... r10:4 r20:2 ...
1461 // Src0.RegFile = Src1.RegFile = GRF
1462 // Src0.Reg.Len = valid
1463 // Src1.Reg.Len = valid
1464 //
1465 // 2. Null as a zero register for src0. (e.g. scratch or SLM loads)
1466 // send... ... null src1 ....descs
1467 //
1468 // 3. Gathering send (no explicit src1, src0.subreg takes src1.length)
1469 // send... ... r[s0.#]:s0len ....descs
1470 //
1471 // Src0.RF Src0.REG Src0.SUBREG Src0.LEN Src1.RF Src1.REG
1472 // Src1.LEN
1473 // Canonical: GRF vaild N/A valid GRF/ARF vaild valid
1474 // (null would mean 0) Null src0: ARF null N/A valid ARF
1475 // null don't care (implied 0) Gathering: ARF s0.X .X valid
1476 // ARF null invalid (bits overlap Src0.SUBREG)
1477 // (today gathering is determined by Src1.RegFile==ARF)
1478 //
1479 // Bits[103:99] holds either Src1.LEN or Src0.SUBREG (for gathering)
1480 GED_DECODE_RAW(uint32_t, src0LenU, Src0Length);
1481 int src0Len = (int)src0LenU;
1482 GED_DECODE_RAW(GED_REG_FILE, src1RegFile, Src1RegFile);
1483 int src1Len = 0;
1484 if (src1RegFile == GED_REG_FILE_GRF) {
1485 // register file type for sendgx can only be GRF
1486 GED_DECODE_RAW(uint32_t, src1LenU, Src1Length);
1487 src1Len = (int)src1LenU;
1488 }
1489
1490 FlagRegInfo fri = decodeFlagRegInfo();
1491 Instruction *inst = kernel.createSendgInstruction(
1492 *m_opSpec, m_subfunc.send, fri.pred, fri.reg, decodeExecSize(),
1493 decodeChannelOffset(), decodeMaskCtrl(), src0Len, src1Len, id0, id1,
1494 desc);
1495
1496 decodeSendgDestinationXe3(inst);
1497 decodeSendgSource0Xe3(inst);
1498 decodeSendgSource1Xe3(inst);
1499
1500 return inst;
1501 }
1502
1503 void Decoder::decodeSendDestination(Instruction *inst) {
1504 GED_ACCESS_MODE accessMode = decodeAccessMode();
1505 GED_DECODE_RAW(GED_REG_FILE, regFile, DstRegFile);
1506 GED_ADDR_MODE addrMode = GED_ADDR_MODE_Direct;
1507
1508 if (platform() <= Platform::GEN11) {
1509 GED_DECODE_RAW_TO(DstAddrMode, addrMode);
1510 }
1511
1512 if (addrMode == GED_ADDR_MODE_Indirect) {
1513 if (regFile == GED_REG_FILE_GRF) {
1514 decodeBasicDestination(inst, accessMode);
1515 } else {
1516 errorT("error decoding instruction: SEND dst ARF");
1517 }
1518 } else {
1519 DirRegOpInfo dri = decodeDstDirRegInfo();
1520
1521 Region::Horz rgnHz = Region::Horz::HZ_1;
1522 if (m_opSpec->hasImplicitDstRegion(isMacro())) {
1523 rgnHz = m_opSpec->implicitDstRegion(isMacro()).getHz();
1524 }
1525
1526 inst->setDirectDestination(DstModifier::NONE, dri.regName, dri.regRef,
1527 rgnHz, dri.type);
1528 }
1529 }
1530
1531 GED_ADDR_MODE Decoder::decodeSendSource0AddressMode() {
1532 GED_ADDR_MODE addrMode = GED_ADDR_MODE_Direct;
1533 if (platform() <= Platform::GEN11) {
1534 addrMode = decodeSrcAddrMode<SourceIndex::SRC0>();
1535 }
1536 return addrMode;
1537 }
1538
1539 void Decoder::decodeSendSource0(Instruction *inst) {
1540 GED_ACCESS_MODE accessMode = decodeAccessMode();
1541 GED_REG_FILE regFile = decodeSrcRegFile<SourceIndex::SRC0>();
1542
1543 GED_ADDR_MODE addrMode = decodeSendSource0AddressMode();
1544
1545 if (regFile == GED_REG_FILE_GRF && addrMode == GED_ADDR_MODE_Indirect) {
1546 decodeSourceBasic<SourceIndex::SRC0>(inst, accessMode);
1547 } else {
1548 DirRegOpInfo dri = decodeSrcDirRegOpInfo<SourceIndex::SRC0>();
1549
1550 Region rgn =
1551 inst->getOpSpec().implicitSrcRegion(0, inst->getExecSize(), isMacro());
1552 bool hasSrcRgnEncoding =
1553 inst->getOpSpec().isSendFormat() && platform() < Platform::GEN9;
1554
1555 hasSrcRgnEncoding &= platform() <= Platform::GEN11;
1556
1557 if (hasSrcRgnEncoding) {
1558 // these bits are implicitly set by GED on SKL, and they disallow access
1559 rgn = *decodeSrcRegionVWH<SourceIndex::SRC0>();
1560 }
1561
1562 if (dri.regName == RegName::ARF_S) {
1563 // Scalar src0 implies a gathering send.
1564 //
1565 // This is the only case that send can have subRegNum.
1566 // Decode subRegNum separately here.
1567 // (Scaling is always in bytes for all s0.# in sends)
1568 dri.regRef.subRegNum = (uint16_t)decodeSrcSubRegNum<SourceIndex::SRC0>();
1569 inst->setIndirectSource(SourceIndex::SRC0, SrcModifier::NONE,
1570 dri.regName, dri.regRef, 0, rgn, dri.type);
1571 } else
1572 inst->setDirectSource(SourceIndex::SRC0, SrcModifier::NONE, dri.regName,
1573 dri.regRef, rgn, dri.type);
1574 }
1575 }
1576
1577 void Decoder::decodeSendSource1(Instruction *inst) {
1578 RegRef regRef;
1579 RegName regName = decodeSourceReg<SourceIndex::SRC1>(regRef);
1580 const OpSpec &os = inst->getOpSpec();
1581 Region rgn = os.implicitSrcRegion(1, inst->getExecSize(), isMacro());
1582 Type implSrcType = os.implicitSrcType(1, false);
1583 inst->setDirectSource(SourceIndex::SRC1, SrcModifier::NONE, regName, regRef,
1584 rgn, implSrcType);
1585 }
1586
1587 void Decoder::decodeSendgDestinationXe3(Instruction *inst) {
1588 GED_DECODE_RAW(GED_REG_FILE, regFile, DstRegFile);
1589 RegName dstRegName = RegName::ARF_NULL;
1590 RegRef dstReg = REGREF_ZERO_ZERO;
1591 if (regFile == GED_REG_FILE_GRF) {
1592 GED_DECODE_RAW(uint32_t, regNumBits, DstRegNum);
1593 dstReg.regNum = (uint16_t)regNumBits;
1594 dstRegName = RegName::GRF_R;
1595 } // else it's null
1596 inst->setDirectDestination(DstModifier::NONE, dstRegName, dstReg,
1597 Region::Horz::HZ_INVALID,
1598 m_opSpec->implicitDstType());
1599 }
1600
1601 bool Decoder::decodeSendgSource0Xe3(Instruction *inst) {
1602 RegRef regRef = REGREF_ZERO_ZERO;
1603 RegName regName = RegName::ARF_NULL;
1604 GED_REG_FILE regFile = decodeSrcRegFile<SourceIndex::SRC0>();
1605
1606 if (regFile == GED_REG_FILE_GRF) {
1607 (void)decodeSourceReg<SourceIndex::SRC0>(regRef);
1608 regName = RegName::GRF_R;
1609 } else {
1610 regName = decodeSourceReg<SourceIndex::SRC0>(regRef);
1611 }
1612 Type implSrcType = inst->getOpSpec().implicitSrcType(1, false);
1613
1614 Region rgn =
1615 inst->getOpSpec().implicitSrcRegion(0, inst->getExecSize(), false);
1616
1617 if (regName == RegName::ARF_S) {
1618 // ARF_S src0 must be sendg (gather send)
1619 // sendg src0 is a indirect register: r[s0.0]. This is the only case that
1620 // send can have subRegNum. Decode subRegNum separatly here.
1621 regRef.subRegNum = (uint16_t)decodeSrcSubRegNum<SourceIndex::SRC0>();
1622 inst->setIndirectSource(SourceIndex::SRC0, SrcModifier::NONE, regName,
1623 regRef, 0, rgn, implSrcType);
1624 } else {
1625 inst->setDirectSource(SourceIndex::SRC0, SrcModifier::NONE, regName, regRef,
1626 rgn, implSrcType);
1627 }
1628 return regName == RegName::ARF_S;
1629 }
1630
1631 void Decoder::decodeSendgSource1Xe3(Instruction *inst) {
1632 // This only exists if Src1.RegFile = GRF, otherwise we put a null parameter
1633 // there
1634 RegRef regRef = REGREF_ZERO_ZERO;
1635 RegName regName = RegName::ARF_NULL;
1636 GED_REG_FILE regFile = decodeSrcRegFile<SourceIndex::SRC1>();
1637
1638 if (regFile == GED_REG_FILE_GRF) {
1639 (void)decodeSourceReg<SourceIndex::SRC1>(regRef);
1640 regName = RegName::GRF_R;
1641 } // else: don't touch Src1.Reg or GED will blow up
1642
1643 const OpSpec &os = inst->getOpSpec();
1644 Region rgn = os.implicitSrcRegion(1, inst->getExecSize(), isMacro());
1645 Type implSrcType = os.implicitSrcType(1, false);
1646 inst->setDirectSource(SourceIndex::SRC1, SrcModifier::NONE, regName, regRef,
1647 rgn, implSrcType);
1648 }
1649
1650 void Decoder::decodeSendgxDestinationXe3p(Instruction *inst) {
1651 RegName dstRegName = RegName::GRF_R;
1652 RegRef dstReg = REGREF_ZERO_ZERO;
1653
1654 GED_DECODE_RAW(uint32_t, regNumBits, DstRegNum);
1655 dstReg.regNum = (uint16_t)regNumBits;
1656
1657 if (dstReg.regNum == 511) {
1658 // r511 is decoded as null
1659 dstRegName = RegName::ARF_NULL;
1660 dstReg.regNum = 0;
1661 }
1662 inst->setDirectDestination(DstModifier::NONE, dstRegName, dstReg,
1663 Region::Horz::HZ_INVALID,
1664 m_opSpec->implicitDstType());
1665 }
1666
1667 void Decoder::decodeSendgxSource0Xe3p(Instruction *inst) {
1668 RegRef regRef = REGREF_ZERO_ZERO;
1669 RegName regName = RegName::GRF_R;
1670
1671 decodeGrfSourceReg<SourceIndex::SRC0>(regRef);
1672
1673 if (regRef.regNum == 511) {
1674 regName = RegName::ARF_NULL;
1675 regRef.regNum = 0;
1676 }
1677
1678 Type implSrcType = inst->getOpSpec().implicitSrcType(1, false);
1679
1680 Region rgn =
1681 inst->getOpSpec().implicitSrcRegion(0, inst->getExecSize(), false);
1682
1683
1684 inst->setDirectSource(SourceIndex::SRC0, SrcModifier::NONE, regName, regRef,
1685 rgn, implSrcType);
1686 }
1687
1688 void Decoder::decodeSendgxSource1Xe3p(Instruction *inst) {
1689 RegRef regRef = REGREF_ZERO_ZERO;
1690 RegName regName = RegName::GRF_R;
1691
1692 decodeGrfSourceReg<SourceIndex::SRC1>(regRef);
1693
1694 if (regRef.regNum == 511) {
1695 // r511 is decoded as null
1696 regName = RegName::ARF_NULL;
1697 regRef.regNum = 0;
1698 }
1699
1700 const OpSpec &os = inst->getOpSpec();
1701 Region rgn = os.implicitSrcRegion(1, inst->getExecSize(), isMacro());
1702 Type implSrcType = os.implicitSrcType(1, false);
1703 inst->setDirectSource(SourceIndex::SRC1, SrcModifier::NONE, regName, regRef,
1704 rgn, implSrcType);
1705 }
1706
1707 ///////////////////////////////////////////////////////////////////////
1708 // BRANCH INSTRUCTIONS
1709 ///////////////////////////////////////////////////////////////////////
1710 Instruction *Decoder::decodeBranchInstruction(Kernel &kernel) {
1711 if (decodeAccessMode() == GED_ACCESS_MODE_Align16) {
1712 errorT("Align16 branches not supported");
1713 return kernel.createIllegalInstruction();
1714 }
1715
1716 FlagRegInfo fri = decodeFlagRegInfo();
1717 Instruction *inst = kernel.createBranchInstruction(
1718 *m_opSpec, fri.pred, fri.reg, decodeExecSize(), decodeChannelOffset(),
1719 decodeMaskCtrl(), m_subfunc);
1720
1721 if (m_opSpec->op == Op::JMPI) {
1722 // jmpi (1) JIP
1723 // is encoded as:
1724 // jmpi (1) ip ip JIP
1725 GED_REG_FILE regFile = GED_REG_FILE_INVALID;
1726
1727 GED_DECODE_RAW(GED_REG_FILE, regTFile, Src1RegFile);
1728 regFile = regTFile;
1729
1730 // TODO: make and use m_opSpec->hasImplicit{Source,Destination}()
1731 if (regFile != GED_REG_FILE_IMM) {
1732 if (m_model.supportsSrc1CtrlFlow()) {
1733 decodeSourceBasic<SourceIndex::SRC1>(inst, SourceIndex::SRC0,
1734 GED_ACCESS_MODE_Align1);
1735 } else {
1736 Region rgn = *decodeSrcRegionVWH<SourceIndex::SRC0>();
1737 DirRegOpInfo opInfo = decodeSrcDirRegOpInfo<SourceIndex::SRC0>();
1738 inst->setDirectSource(SourceIndex::SRC0, SrcModifier::NONE,
1739 opInfo.regName, opInfo.regRef, rgn, opInfo.type);
1740 }
1741 } else {
1742 GED_DECODE_RAW(int32_t, jip, JIP);
1743 // jmpi is stored post-increment; normalize it to pre-increment
1744 jip += GED_InsSize(&m_currGedInst);
1745 Type dataType = Type::INVALID;
1746 if (m_model.supportsSrc1CtrlFlow()) {
1747 dataType = decodeSrcType<SourceIndex::SRC1>();
1748 } else {
1749 dataType = decodeSrcType<SourceIndex::SRC0>();
1750 }
1751 inst->setLabelSource(SourceIndex::SRC0, jip, dataType);
1752 }
1753 } else if (m_opSpec->op == Op::RET) {
1754 // ret encodes as:
1755 // ret (..) null src0
1756 // we leave then null implicit
1757 decodeSourceBasicAlign1<SourceIndex::SRC0>(inst);
1758 } else if (m_opSpec->op == Op::CALL || m_opSpec->op == Op::CALLA) {
1759 // calla (..) reg imm32
1760 // call (..) reg imm32
1761 // call (..) reg reg32
1762 //
1763 // call can take register or immediate (jip)
1764 // call stores register info in src1
1765 decodeBasicDestinationAlign1(inst);
1766 GED_REG_FILE regFile = GED_REG_FILE_INVALID;
1767 Type srcType = Type::INVALID;
1768
1769 GED_DECODE_RAW(GED_REG_FILE, regTFile, Src1RegFile);
1770 regFile = regTFile;
1771 srcType = decodeSrcType<SourceIndex::SRC1>();
1772
1773 if (regFile == GED_REG_FILE_IMM) {
1774 // calla (..) reg imm32
1775 // call (..) reg imm32
1776 decodeJipToSrc(inst, SourceIndex::SRC0, srcType);
1777 } else {
1778 // call (..) reg reg32
1779 decodeSourceBasicAlign1<SourceIndex::SRC1>(inst, SourceIndex::SRC0);
1780 }
1781 } else if (m_opSpec->op == Op::BRC || m_opSpec->op == Op::BRD) {
1782 // brc (..) lbl16 lbl16 [PreBDW]
1783 // brc (..) lbl32 lbl32 [BDW+]
1784 // brc (..) reg32 [PreHSW]
1785 // brc (..) reg64 [HSW]
1786 // brc (..) reg64 [BDW+]
1787 //
1788 // brd (..) imm16 [IVB,HSW]
1789 // brd (..) reg32 [IVB,HSW]
1790 // brd (..) lbl32 [BDW+]
1791 // brd (..) reg32 [BDW+]
1792 GED_DECODE_RAW(GED_REG_FILE, regFile, Src0RegFile);
1793 if (regFile == GED_REG_FILE_IMM) {
1794 Type type = decodeSrcType<SourceIndex::SRC0>();
1795 decodeJipToSrc(inst, SourceIndex::SRC0, type);
1796 if (m_opSpec->op == Op::BRC) {
1797 decodeUipToSrc1(inst, type);
1798 }
1799 } else {
1800 // register argument
1801 decodeSourceBasicAlign1<SourceIndex::SRC0>(inst);
1802 if (m_opSpec->op == Op::BRC) {
1803 // add an implicit null parameter
1804 inst->setSource(SourceIndex::SRC1, Operand::SRC_REG_NULL_UD);
1805 }
1806 }
1807 } else {
1808 // e.g. if, else, endif, while, cont, break, ...
1809 decodeJipToSrc(inst);
1810 if (m_opSpec->format != OpSpec::Format::JUMP_UNARY_IMM) {
1811 decodeUipToSrc1(inst, Type::INVALID);
1812 }
1813 }
1814
1815 return inst;
1816 }
1817
1818 Instruction *Decoder::decodeBranchSimplifiedInstruction(Kernel &kernel) {
1819 BranchCntrl branchCtrl = BranchCntrl::OFF;
1820 if (m_opSpec->supportsBranchCtrl()) {
1821 GED_DECODE_TO(BranchCtrl, translate, branchCtrl);
1822 }
1823 FlagRegInfo fri = decodeFlagRegInfo();
1824 Instruction *inst = kernel.createBranchInstruction(
1825 *m_opSpec, fri.pred, fri.reg, decodeExecSize(), decodeChannelOffset(),
1826 decodeMaskCtrl(), branchCtrl);
1827
1828 if (inst->getOpSpec().supportsDestination()) {
1829 decodeBranchDestination(inst);
1830 }
1831
1832 GED_DECODE_RAW(GED_REG_FILE, regFile, Src0RegFile);
1833 if (regFile != GED_REG_FILE_IMM) {
1834 Region rgn = m_opSpec->implicitSrcRegion(0, inst->getExecSize(), isMacro());
1835 DirRegOpInfo opInfo = decodeSrcDirRegOpInfo<SourceIndex::SRC0>();
1836 inst->setDirectSource(SourceIndex::SRC0, SrcModifier::NONE, opInfo.regName,
1837 opInfo.regRef, rgn, opInfo.type);
1838 } else {
1839 decodeJipToSrc(
1840 inst, SourceIndex::SRC0,
1841 m_opSpec->implicitSrcType(static_cast<int>(SourceIndex::SRC0), false));
1842 }
1843 // brc/brd read both UIP and JIP from one register (64-bits)
1844 bool isReg64 = ((m_opSpec->op == Op::BRC || m_opSpec->op == Op::BRD) &&
1845 regFile == GED_REG_FILE_GRF);
1846 bool isUnary = (m_opSpec->format & OpSpec::Format::UNARY) != 0;
1847 if (!isReg64 && !isUnary) {
1848 decodeUipToSrc1(inst, Type::INVALID);
1849 }
1850 return inst;
1851 }
1852
1853 void Decoder::decodeBranchDestination(Instruction *inst) {
1854 DirRegOpInfo dri = decodeDstDirRegInfo();
1855 Type dty = Type::UD;
1856 if (inst->getOpSpec().hasImplicitDstType()) {
1857 dty = m_opSpec->implicitDstType();
1858 }
1859 inst->setDirectDestination(DstModifier::NONE, dri.regName, dri.regRef,
1860 Region::Horz::HZ_1, dty);
1861 }
1862
1863 ///////////////////////////////////////////////////////////////////////
1864 // OTHER INSTRUCTIONS
1865 ///////////////////////////////////////////////////////////////////////
1866 Instruction *Decoder::decodeWaitInstruction(Kernel &kernel) {
1867 // wait encodes as
1868 // wait (..) nreg nreg null
1869 GED_ACCESS_MODE accessMode = decodeAccessMode();
1870 FlagRegInfo fri = decodeFlagRegInfo();
1871 Instruction *inst = kernel.createBasicInstruction(
1872 *m_opSpec, fri.pred, fri.reg, decodeExecSize(), decodeChannelOffset(),
1873 decodeMaskCtrl(), fri.modifier, m_subfunc);
1874 decodeSourceBasic<SourceIndex::SRC0>(inst, accessMode);
1875 return inst;
1876 }
1877
1878 Instruction *Decoder::decodeSyncInstruction(Kernel &kernel) {
1879 FlagRegInfo fri = decodeFlagRegInfo();
1880 Instruction *inst = kernel.createBasicInstruction(
1881 *m_opSpec, fri.pred, fri.reg, decodeExecSize(), decodeChannelOffset(),
1882 decodeMaskCtrl(), fri.modifier, m_subfunc);
1883 GED_REG_FILE regFile = decodeSrcRegFile<SourceIndex::SRC0>();
1884
1885 if (regFile == GED_REG_FILE_ARF) {
1886 // e.g.
1887 // sync.nop null
1888 // sync.allrd null
1889 // ...
1890 // Since XeHPC, sync.bar supports flag src0
1891 if (platform() >= Platform::XE_HPC) {
1892 decodeSourceBasic<SourceIndex::SRC0>(inst, GED_ACCESS_MODE_Align1);
1893 } else {
1894 inst->setSource(SourceIndex::SRC0, Operand::SRC_REG_NULL_UB);
1895 }
1896 } else {
1897 // e.g.
1898 // sync.allrd 0x15
1899 // ...
1900 decodeSourceBasic<SourceIndex::SRC0>(inst, GED_ACCESS_MODE_Align1);
1901 }
1902 return inst;
1903 }
1904
1905 Predication Decoder::decodePredication() {
1906 Predication pred = {PredCtrl::NONE, false};
1907 GED_DECODE_RAW(GED_PRED_CTRL, pc, PredCtrl);
1908 pred.function = translate(pc);
1909 return pred;
1910 }
1911
1912 void Decoder::decodePredInv(Predication &pred) {
1913 GED_DECODE_RAW(GED_PRED_INV, pi, PredInv);
1914 pred.inverse = (pi == GED_PRED_INV_Invert);
1915 }
1916
1917 MaskCtrl Decoder::decodeMaskCtrl() {
1918 GED_DECODE(MaskCtrl, GED_MASK_CTRL, ctrl, MaskCtrl);
1919 return ctrl;
1920 }
1921
1922 FlagRegInfo Decoder::decodeFlagRegInfo(bool imm64Src0Overlaps) {
1923
1924 FlagRegInfo fri = {
1925 {PredCtrl::NONE, false}, FlagModifier::NONE, REGREF_ZERO_ZERO};
1926 if (m_opSpec->supportsPredication()) {
1927 fri.pred = decodePredication();
1928 }
1929 if (m_opSpec->supportsFlagModifier() && !imm64Src0Overlaps) {
1930 // XE SWSB overlaps CondModifier and Imm64 values
1931 GED_DECODE_RAW(GED_COND_MODIFIER, condMod, CondModifier);
1932 fri.modifier = translate(condMod);
1933 } else if (m_opSpec->is(Op::MATH) && isMacro()) {
1934 // math.inv and math.rsqrtm both implicitly support EO
1935 // currently math is the only case, and its flagModifier must be EO
1936 fri.modifier = FlagModifier::EO;
1937 }
1938
1939 // For XeHPC PredIvn field only exists when
1940 // PredCtrl or CondCtrl (flag modifier) exits
1941 if (platform() >= Platform::XE_HPC) {
1942 if (fri.pred.function != PredCtrl::NONE ||
1943 fri.modifier != FlagModifier::NONE)
1944 decodePredInv(fri.pred);
1945 } else if (m_opSpec->supportsPredication()) {
1946 decodePredInv(fri.pred);
1947 }
1948
1949 if (fri.pred.function != PredCtrl::NONE ||
1950 fri.modifier != FlagModifier::NONE) {
1951 GED_DECODE_RAW(uint32_t, flagRegNum, FlagRegNum);
1952 fri.reg.regNum = (uint16_t)flagRegNum;
1953 GED_DECODE_RAW(uint32_t, flagSubRegNum, FlagSubRegNum);
1954 fri.reg.subRegNum = (uint16_t)flagSubRegNum;
1955 }
1956
1957 return fri;
1958 }
1959
1960 ExecSize Decoder::decodeExecSize() {
1961 GED_DECODE_RAW(uint32_t, execSize, ExecSize);
1962 return translateExecSize(execSize);
1963 }
1964
1965 ChannelOffset Decoder::decodeChannelOffset() {
1966 if (m_opSpec->supportsQtrCtrl()) {
1967 GED_DECODE(ChannelOffset, GED_CHANNEL_OFFSET, em, ChannelOffset);
1968 return em;
1969 } else {
1970 return ChannelOffset::M0;
1971 }
1972 }
1973
1974 GED_ACCESS_MODE Decoder::decodeAccessMode() {
1975 if (m_model.supportsAccessMode()) {
1976 GED_DECODE_RAW(GED_ACCESS_MODE, accessMode, AccessMode);
1977 return accessMode;
1978 }
1979 return GED_ACCESS_MODE_Align1;
1980 }
1981
1982 void Decoder::decodeJipToSrc(Instruction *inst, SourceIndex s, Type type) {
1983 inst->setLabelSource(s, decodeJip(), type);
1984 }
1985 void Decoder::decodeUipToSrc1(Instruction *inst, Type type) {
1986 inst->setLabelSource(SourceIndex::SRC1, decodeUip(), type);
1987 }
1988
1989 // PreBDW JIP and UIP are in QWORDS in <GEN8 except for a few
1990 // exceptions for the above instructions
1991 #define PC_SCALE \
1992 ((platform() < Platform::GEN8 && m_opSpec->op != Op::CALL && \
1993 m_opSpec->op != Op::CALLA && m_opSpec->op != Op::JMPI) \
1994 ? 8 \
1995 : 1)
1996 int32_t Decoder::decodeJip() {
1997 GED_DECODE_RAW(int32_t, jip, JIP);
1998 return jip * PC_SCALE;
1999 }
2000 int32_t Decoder::decodeUip() {
2001 GED_DECODE_RAW(int32_t, uip, UIP);
2002 return uip * PC_SCALE;
2003 }
2004
2005 int Decoder::decodeDestinationRegNumAccBitsFromChEn() {
2006 // this is used by the math macro register (implicit accumulator access)
2007 // and for context save and restore access to those registers
2008 GED_DECODE_RAW(GED_DST_CHAN_EN, chEn, DstChanEn);
2009 switch (chEn) {
2010 case GED_DST_CHAN_EN_None:
2011 return 0; // 0000b => mme0 (acc2)
2012 case GED_DST_CHAN_EN_x:
2013 return 1; // 0001b => mme1 (acc3)
2014 case GED_DST_CHAN_EN_y:
2015 return 2; // 0010b => mme2 (acc4)
2016 case GED_DST_CHAN_EN_xy:
2017 return 3; // 0011b => mme3 (acc5)
2018 case GED_DST_CHAN_EN_z:
2019 return 4; // 0100b => mme4 (acc6)
2020 case GED_DST_CHAN_EN_xz:
2021 return 5; // 0101b => mme5 (acc7)
2022 case GED_DST_CHAN_EN_yz:
2023 return 6; // 0110b => mme6 (acc8)
2024 case GED_DST_CHAN_EN_xyz:
2025 return 7; // 0111b => mme7 (acc9)
2026 //
2027 // every thing else unreachable because this is an explicit operand
2028 // not an implicit math macro acc reference
2029 //
2030 case GED_DST_CHAN_EN_w:
2031 return 0; // 1000b => noacc
2032 //
2033 // HACK: for context save and restore, acc9 encodes as .xyzw
2034 // I think this is because of
2035 // mov(8) acc2:ud r103:ud {NoMask, Align16} //acc9
2036 // ^.xyzw implied
2037 // Seems like it should just be .xyz
2038 case GED_DST_CHAN_EN_xyzw:
2039 return 7; // 1111b => mme7 (acc9)
2040 default:
2041 errorT("dst: invalid math macro register (from ChEn)");
2042 return -1;
2043 }
2044 }
2045
2046 MathMacroExt Decoder::decodeDestinationMathMacroRegFromChEn() {
2047 // this is used by the math macro register (implicit accumulator) access
2048 // and for context save and restore access to those registers
2049 GED_DECODE_RAW(GED_DST_CHAN_EN, chEn, DstChanEn);
2050 switch (chEn) {
2051 case GED_DST_CHAN_EN_None:
2052 return MathMacroExt::MME0; // 0000b => mme0 (acc2)
2053 case GED_DST_CHAN_EN_x:
2054 return MathMacroExt::MME1; // 0001b => mme1 (acc3)
2055 case GED_DST_CHAN_EN_y:
2056 return MathMacroExt::MME2; // 0010b => mme2 (acc4)
2057 case GED_DST_CHAN_EN_xy:
2058 return MathMacroExt::MME3; // 0011b => mme3 (acc5)
2059 case GED_DST_CHAN_EN_z:
2060 return MathMacroExt::MME4; // 0100b => mme4 (acc6)
2061 case GED_DST_CHAN_EN_xz:
2062 return MathMacroExt::MME5; // 0101b => mme5 (acc7)
2063 case GED_DST_CHAN_EN_yz:
2064 return MathMacroExt::MME6; // 0110b => mme6 (acc8)
2065 case GED_DST_CHAN_EN_xyz:
2066 return MathMacroExt::MME7; // 0111b => mme7 (acc9)
2067 case GED_DST_CHAN_EN_w:
2068 return MathMacroExt::NOMME; // 1000b => nomme (noacc)
2069 default:
2070 errorT("invalid dst implicit accumulator reference (in ChEn)");
2071 return MathMacroExt::INVALID;
2072 }
2073 }
2074
2075 void Decoder::decodeDstDirSubRegNum(DirRegOpInfo &dri) {
2076 if (isMacro() || m_opSpec->isAnySendFormat()) {
2077 dri.regRef.subRegNum = 0;
2078 } else {
2079 Type scalingType = dri.type;
2080 if (scalingType == Type::INVALID)
2081 scalingType = m_opSpec->isBranching() ? Type::D : Type::UB;
2082
2083 GED_DECODE_RAW(uint32_t, subRegNum, DstSubRegNum);
2084 dri.regRef.subRegNum = (uint16_t)BinaryOffsetToSubReg(
2085 subRegNum, dri.regName, scalingType, m_model.platform);
2086 }
2087 }
2088
2089 void Decoder::decodeReg(int opIx, GED_REG_FILE regFile, uint32_t regNumBits,
2090 RegName &regName,
2091 RegRef &regRef) // works for src or dst
2092 {
2093 const char *opName = opIx == 0 ? "src0"
2094 : opIx == 1 ? "src1"
2095 : opIx == 2 ? "src2"
2096 : opIx == 3 ? "src meta"
2097 : "dst";
2098 if (regFile == GED_REG_FILE_GRF) {
2099 regName = RegName::GRF_R;
2100 regRef.regNum = (uint16_t)regNumBits;
2101 } else if (regFile == GED_REG_FILE_ARF) { // ARF
2102 regName = RegName::INVALID;
2103 int arfRegNum = 0;
2104 const RegInfo *ri = m_model.lookupArfRegInfoByRegNum((uint8_t)regNumBits);
2105 if (ri == nullptr) {
2106 errorT(opName, ": ", iga::fmtHex(regNumBits, 2),
2107 ": invalid arf register");
2108 } else {
2109 regName = ri->regName;
2110 if (!ri->decode((uint8_t)regNumBits, arfRegNum)) {
2111 errorT(opName, ": ", ri->syntax, arfRegNum,
2112 ": invalid register number ");
2113 }
2114 }
2115 regRef.regNum = (uint16_t)arfRegNum;
2116 } else { // e.g. 10b
2117 errorT(opName, ": invalid register file");
2118 }
2119 }
2120
2121 DirRegOpInfo Decoder::decodeDstDirRegInfo() {
2122 DirRegOpInfo dri;
2123 dri.type = m_opSpec->implicitDstType();
2124 bool hasDstType = true;
2125 if (platform() >= Platform::XE) {
2126 hasDstType &= !m_opSpec->isAnySendFormat();
2127 hasDstType &= !m_opSpec->isBranching();
2128 }
2129 if (hasDstType) {
2130 dri.type = decodeDstType();
2131 }
2132
2133 GED_DECODE_RAW(GED_REG_FILE, gedRegFile, DstRegFile);
2134 GED_DECODE_RAW(uint32_t, regNumBits, DstRegNum);
2135 decodeReg(-1, gedRegFile, regNumBits, dri.regName, dri.regRef);
2136 decodeDstDirSubRegNum(dri);
2137
2138 return dri;
2139 }
2140
2141 Type Decoder::decodeDstType() {
2142 GED_DECODE(Type, GED_DATA_TYPE, t, DstDataType);
2143 return t;
2144 }
2145
2146 bool Decoder::hasImplicitScalingType(Type &type, DirRegOpInfo &dri) {
2147 // FIXME: when entering this function, assuming it MUST NOT be imm or label
2148 // src
2149 if (platform() >= Platform::XE &&
2150 (m_opSpec->isSendFormat() || m_opSpec->isBranching())) {
2151 dri.type =
2152 m_opSpec->implicitSrcType(static_cast<int>(SourceIndex::SRC0), false);
2153 type = Type::D;
2154 return true;
2155 }
2156 return false;
2157 }
2158
2159 ImmVal Decoder::decodeSrcImmVal(Type t) {
2160 ImmVal val;
2161 val.kind = ImmVal::Kind::UNDEF;
2162 val.reset();
2163
2164 GED_DECODE_RAW_TO(Imm, val.u64);
2165 setImmValKind(t, val);
2166 return val;
2167 }
2168
2169 // get full region from dst and src type. This function must be called
2170 // after src and dst type are decoded.
2171 static Region retrieveReducedRegionVWH(Type srcType, Type dstType,
2172 Region dstRegion) {
2173 IGA_ASSERT(dstRegion.getHz() != Region::Horz::HZ_INVALID &&
2174 srcType != Type::INVALID && dstType != Type::INVALID,
2175 "Decoder: cannot derive src1 region");
2176 auto sVertStride =
2177 dstRegion.h * TypeSizeInBits(dstType) / TypeSizeInBits(srcType);
2178 switch (sVertStride) {
2179 case 1: return Region::SRC110;
2180 case 2: return Region::SRC210;
2181 case 4: return Region::SRC410;
2182 case 8: return Region::SRC810;
2183 case 16: return Region::SRC1610;
2184 default:
2185 break;
2186 }
2187 IGA_ASSERT_FALSE("Decoder: cannot derive src1 region");
2188 return Region::INVALID;
2189 }
2190
2191 template <SourceIndex S>
2192 void Decoder::decodeSourceBasicAlign1(Instruction *inst, SourceIndex toSrcIxE) {
2193 const int toSrcIx = static_cast<int>(toSrcIxE);
2194 GED_REG_FILE regFile = decodeSrcRegFile<S>();
2195 if (regFile == GED_REG_FILE_IMM) {
2196 // immediate operand
2197 Type type = decodeSrcType<S>();
2198 inst->setImmediateSource(toSrcIxE, decodeSrcImmVal(type), type);
2199 } else if (regFile == GED_REG_FILE_ARF || regFile == GED_REG_FILE_GRF) {
2200 // register operand
2201 GED_ADDR_MODE addrMode = GED_ADDR_MODE_Direct;
2202 addrMode = decodeSrcAddrMode<S>();
2203
2204 SrcModifier srcMod = decodeSrcModifier<S>();
2205 // region (implicit accumulator if Align16 and <GEN11)
2206 Region implRgn = Region::INVALID;
2207 if (inst->getOpSpec().hasImplicitSrcRegion(toSrcIx, inst->getExecSize(),
2208 isMacro())) {
2209 implRgn = inst->getOpSpec().implicitSrcRegion(
2210 toSrcIx, inst->getExecSize(), isMacro());
2211 }
2212 std::optional<Region> decRgn;
2213 if (m_opSpec->isAnySendFormat()) {
2214 decRgn = implRgn;
2215 } else {
2216 decRgn = decodeSrcRegionVWH<S>();
2217 if (!decRgn) {
2218 IGA_ASSERT(m_model.srcHasReducedRegion(static_cast<uint32_t>(S)), "Invalid src region");
2219 decRgn = retrieveReducedRegionVWH(decodeSrcType<S>(),
2220 inst->getDestination().getType(), inst->getDestination().getRegion());
2221 }
2222 }
2223 // ensure the region matches any implicit region rules
2224 if (!m_opSpec->isAnySendFormat() &&
2225 inst->getOpSpec().hasImplicitSrcRegion(toSrcIx, inst->getExecSize(),
2226 isMacro())) {
2227 if (implRgn != decRgn) {
2228 warningT("src", (int)S, ".Rgn should have ", ToSyntax(implRgn),
2229 " for binary normal form");
2230 }
2231 }
2232
2233 if (addrMode == GED_ADDR_MODE_Direct) {
2234 if (inst->isMacro()) {
2235 // GEN11 macros are stored in the subregister
2236 if (m_model.supportsAlign16()) {
2237 fatalT("src", (int)S,
2238 ": macro instructions must be "
2239 "Align16 for this platform");
2240 }
2241 MathMacroExt mme = decodeSrcMathMacroReg<S>();
2242 RegRef regRef{0, 0};
2243 RegName regName = decodeSourceReg<S>(regRef);
2244 inst->setMacroSource(toSrcIxE, srcMod, regName, regRef, mme, *decRgn,
2245 decodeSrcType<S>());
2246 } else {
2247 // normal access
2248 DirRegOpInfo opInfo = decodeSrcDirRegOpInfo<S>();
2249 inst->setDirectSource(toSrcIxE, srcMod, opInfo.regName, opInfo.regRef,
2250 *decRgn, opInfo.type);
2251 }
2252 } else if (addrMode == GED_ADDR_MODE_Indirect) {
2253 RegRef a0(0u, decodeSrcAddrSubRegNum<S>());
2254 int16_t addrImm = (uint16_t)decodeSrcAddrImm<S>();
2255 inst->setIndirectSource(
2256 toSrcIxE, srcMod,
2257 RegName::GRF_R, // set to GRF for indirect register access
2258 a0, addrImm, *decRgn, decodeSrcType<S>());
2259 } else { // == GED_ADDR_MODE_INVALID
2260 fatalT("invalid addressing mode in src", (int)S);
2261 }
2262 } else { // GED_REG_FILE_INVALID
2263 fatalT("invalid register file in src", (int)S);
2264 }
2265 }
2266
2267 template <SourceIndex S>
2268 void Decoder::decodeSourceBasicAlign16(Instruction *inst, SourceIndex toSrcIx) {
2269 GED_REG_FILE regFile = decodeSrcRegFile<S>();
2270 if (regFile == GED_REG_FILE_IMM) {
2271 // immediate operand
2272 Type type = decodeSrcType<S>();
2273 inst->setImmediateSource(toSrcIx, decodeSrcImmVal(type), type);
2274 } else if (regFile == GED_REG_FILE_ARF || regFile == GED_REG_FILE_GRF) {
2275 // register operand (direct or indirect)
2276 SrcModifier srcMod = decodeSrcModifier<S>();
2277
2278 // reg and subreg (if direct)
2279 GED_ADDR_MODE addrMode = decodeSrcAddrMode<S>();
2280
2281 // special context save/restore access to acc2-acc9
2282 uint32_t vs = decodeSrcVertStride<S>();
2283
2284 if (addrMode == GED_ADDR_MODE_Direct) {
2285 DirRegOpInfo opInfo = decodeSrcDirRegOpInfo<S>();
2286 if (inst->isMacro()) { // math macro operand (macro inst)
2287 if (!((vs == 2 && opInfo.type == Type::DF) ||
2288 (vs == 4 && opInfo.type != Type::DF))) {
2289 fatalT("src", (int)S, ": inconvertible align16 operand");
2290 }
2291 // <GEN11 macros are stored in swizzle bits
2292 MathMacroExt MathMacroReg = decodeSrcMathMacroReg<S>();
2293 Region macroDftSrcRgn = macroDefaultSourceRegion(
2294 (int)S, inst->getOpSpec(), platform(), inst->getExecSize());
2295 inst->setMacroSource(toSrcIx, srcMod, opInfo.regName, opInfo.regRef,
2296 MathMacroReg, macroDftSrcRgn, opInfo.type);
2297 } else {
2298 if (vs != 4) {
2299 fatalT("src", (int)S, ": inconvertible align16 operand");
2300 }
2301 Region rgn = Region::SRC110;
2302 if (opInfo.regName == RegName::ARF_MME &&
2303 isAlign16MathMacroRegisterCsrPlatform()) {
2304 // GEN8-9: context save and restore of acc3-9 hack
2305 // (remember acc2 is Align1 and misses this path)
2306 // So if we are here, it'll look like we just
2307 // decoded "mme0" (acc2), but really we need to consult
2308 // ChSel for the real mme# (acc#+2).
2309 uint32_t chanSel = decodeSrcChanSel<S>() & 0xF;
2310 // We do have to strip off the top bits of ChSel since
2311 // it's really only ChSel[3:0].
2312 //
2313 // the ChSel[3:0] value is taken as interpreted as the
2314 // mme register (e.g. we used to map 0 to 2 for acc2)
2315 // mme's start at 0 (mme0 is acc2).
2316 opInfo.regRef.regNum = (uint8_t)chanSel;
2317 opInfo.regRef.subRegNum = 0;
2318 // In GEN10, this all gets cleaned up and acc3 really is
2319 // RegName[7:4] = 0101b ("acc")
2320 // RegName[3:0] = 0011b ("3")
2321 // (Which we map back to mme1.) Moreover, that'll be
2322 // Align1 code and this path won't be hit.
2323 } else {
2324 // conversion of some other Align16 (e.g math macros)
2325 if (isChanSelPacked<S>()) {
2326 fatalT("src", (int)S, ": inconvertible align16 operand");
2327 }
2328 }
2329 inst->setDirectSource(toSrcIx, srcMod, opInfo.regName, opInfo.regRef,
2330 rgn, opInfo.type);
2331 }
2332 } else if (addrMode == GED_ADDR_MODE_Indirect) {
2333 if (!isChanSelPacked<S>() && vs == 4) {
2334 fatalT("src", (int)S, ": inconvertible align16 operand");
2335 }
2336 int32_t subRegNum = decodeSrcAddrSubRegNum<S>();
2337 int32_t addrImm = decodeSrcAddrImm<S>();
2338 RegRef indReg = {0, (uint8_t)subRegNum};
2339 inst->setIndirectSource(toSrcIx, srcMod, RegName::GRF_R, indReg,
2340 (int16_t)addrImm, Region::SRC110,
2341 decodeSrcType<S>());
2342 } else {
2343 // == GED_ADDR_MODE_INVALID
2344 fatalT("src", (int)S, ": invalid addressing mode");
2345 }
2346 } else { // GED_REG_FILE_INVALID
2347 fatalT("invalid register file in src", (int)S);
2348 }
2349 }
2350
2351 void Decoder::decodeChSelToSwizzle(uint32_t chanSel, GED_SWIZZLE swizzle[4]) {
2352 GED_RETURN_VALUE status = GED_RETURN_VALUE_INVALID_FIELD;
2353
2354 swizzle[0] = GED_GetSwizzleX(chanSel, m_gedModel, &status);
2355 if (status != GED_RETURN_VALUE_SUCCESS) {
2356 fatalT("swizzle X could not be retrieved");
2357 }
2358 swizzle[1] = GED_GetSwizzleY(chanSel, m_gedModel, &status);
2359 if (status != GED_RETURN_VALUE_SUCCESS) {
2360 fatalT("swizzle Y could not be retrieved");
2361 }
2362 swizzle[2] = GED_GetSwizzleZ(chanSel, m_gedModel, &status);
2363 if (status != GED_RETURN_VALUE_SUCCESS) {
2364 fatalT("swizzle Z could not be retrieved");
2365 }
2366 swizzle[3] = GED_GetSwizzleW(chanSel, m_gedModel, &status);
2367 if (status != GED_RETURN_VALUE_SUCCESS) {
2368 fatalT("swizzle W could not be retrieved");
2369 }
2370 }
2371
2372 template <SourceIndex S> bool Decoder::isChanSelPacked() {
2373 uint32_t chanSel = decodeSrcChanSel<S>();
2374 GED_SWIZZLE swizzle[4];
2375 decodeChSelToSwizzle(chanSel, swizzle);
2376 return swizzle[0] != GED_SWIZZLE_x && swizzle[1] != GED_SWIZZLE_y &&
2377 swizzle[2] != GED_SWIZZLE_z && swizzle[3] != GED_SWIZZLE_w;
2378 }
2379
2380 void Decoder::decodeThreadOptions(Instruction *inst, GED_THREAD_CTRL trdCntrl) {
2381 switch (trdCntrl) {
2382 case GED_THREAD_CTRL_Atomic:
2383 inst->addInstOpt(InstOpt::ATOMIC);
2384 break;
2385 case GED_THREAD_CTRL_Switch:
2386 inst->addInstOpt(InstOpt::SWITCH);
2387 break;
2388 case GED_THREAD_CTRL_NoPreempt:
2389 inst->addInstOpt(InstOpt::NOPREEMPT);
2390 break;
2391 case GED_THREAD_CTRL_INVALID:
2392 default:
2393 break;
2394 }
2395 }
2396
2397 template <SourceIndex S> ImmVal Decoder::decodeTernarySrcImmVal(Type t) {
2398 ImmVal val;
2399 val.kind = ImmVal::Kind::UNDEF;
2400 val.reset();
2401
2402 if (S == SourceIndex::SRC0) {
2403 GED_DECODE_RAW_TO(Src0TernaryImm, val.u64);
2404 } else if (S == SourceIndex::SRC2) {
2405 GED_DECODE_RAW_TO(Src2TernaryImm, val.u64);
2406 } else {
2407 errorT("src1: no immediate supported here on ternary instruction");
2408 }
2409
2410 setImmValKind(t, val);
2411
2412 return val;
2413 }
2414
2415 void Decoder::decodeOptions(Instruction *inst) {
2416 const OpSpec &os = inst->getOpSpec();
2417 if (os.supportsAccWrEn()) {
2418 // * GED doesn't allow AccWrEn on send's
2419 // * BrnchCtrl overlaps AccWrEn, so anything using that is out
2420 GED_ACC_WR_CTRL accWrEn = GED_ACC_WR_CTRL_Normal;
2421 GED_DECODE_RAW_TO(AccWrCtrl, accWrEn);
2422 if (accWrEn == GED_ACC_WR_CTRL_AccWrEn) {
2423 inst->addInstOpt(InstOpt::ACCWREN);
2424 }
2425 }
2426
2427 if (os.supportsDebugCtrl()) {
2428 GED_DEBUG_CTRL debugCtrl = GED_DEBUG_CTRL_Normal;
2429 GED_DECODE_RAW_TO(DebugCtrl, debugCtrl);
2430 if (debugCtrl == GED_DEBUG_CTRL_Breakpoint) {
2431 inst->addInstOpt(InstOpt::BREAKPOINT);
2432 }
2433 }
2434
2435 if (os.isAnySendFormat()) {
2436 GED_EOT eot = GED_EOT_None;
2437 GED_DECODE_RAW_TO(EOT, eot);
2438 if (eot == GED_EOT_EOT) {
2439 inst->addInstOpt(InstOpt::EOT);
2440 }
2441 }
2442
2443 if (os.supportsFwdCtrl()) {
2444 uint32_t fwdCtrl = 0;
2445 GED_DECODE_RAW_TO(FwdCtrl, fwdCtrl);
2446 if (fwdCtrl)
2447 inst->addInstOpt(InstOpt::FWD);
2448 }
2449
2450 if (os.supportsDepCtrl()) {
2451 GED_DEP_CTRL dpCtrl = GED_DEP_CTRL_Normal;
2452 GED_DECODE_RAW_TO(DepCtrl, dpCtrl);
2453 if (dpCtrl == GED_DEP_CTRL_NoDDClr) {
2454 inst->addInstOpt(InstOpt::NODDCLR);
2455 } else if (dpCtrl == GED_DEP_CTRL_NoDDChk) {
2456 inst->addInstOpt(InstOpt::NODDCHK);
2457 } else if (dpCtrl == GED_DEP_CTRL_NoDDClr_NoDDChk) {
2458 inst->addInstOpt(InstOpt::NODDCLR);
2459 inst->addInstOpt(InstOpt::NODDCHK);
2460 }
2461 }
2462
2463 if (os.supportsThreadCtrl()) {
2464 GED_THREAD_CTRL trdCntrl = GED_THREAD_CTRL_Normal;
2465 GED_DECODE_RAW_TO(ThreadCtrl, trdCntrl);
2466 decodeThreadOptions(inst, trdCntrl);
2467 }
2468
2469 if (m_model.supportNoSrcDepSet() && os.isAnySendFormat()) {
2470 GED_NO_SRC_DEP_SET srcDep;
2471 GED_DECODE_RAW_TO(NoSrcDepSet, srcDep);
2472 if (srcDep == GED_NO_SRC_DEP_SET_NoSrcDepSet) {
2473 inst->addInstOpt(InstOpt::NOSRCDEPSET);
2474 }
2475 }
2476
2477 if (GED_IsCompact(&m_currGedInst)) {
2478 inst->addInstOpt(InstOpt::COMPACTED);
2479 }
2480 }
2481
2482 Instruction *Decoder::createErrorInstruction(Kernel &kernel,
2483 const char *message,
2484 const void *binary, int32_t iLen) {
2485 Instruction *inst = kernel.createIllegalInstruction();
2486
2487 std::stringstream ss;
2488 ss << FormatOpBits(m_model, binary);
2489 if (*message) {
2490 ss << ": " << message;
2491 }
2492 size_t bufLen = (size_t)ss.tellp() + 1;
2493 char *buf = (char *)kernel.getMemManager().alloc(bufLen);
2494 ss.read(buf, bufLen - 1);
2495 buf[bufLen - 1] = 0;
2496
2497 if (iLen == 8) {
2498 inst->addInstOpt(InstOpt::COMPACTED);
2499 }
2500 inst->setComment(buf);
2501 return inst;
2502 }
2503
2504 uint32_t Decoder::getBitField(int ix, int len) const {
2505 const uint32_t *ws = (const uint32_t *)((const char *)m_binary + currentPc());
2506 // shift is only well-defined for values <32, use 0xFFFFFFFF
2507 uint32_t mask = len >= 32 ? 0xFFFFFFFF : (1 << (uint32_t)len) - 1;
2508 IGA_ASSERT(len <= 32 && ((ix + len - 1) / 32 == ix / 32),
2509 "getBitField: bitfield spans DWord");
2510 return (ws[ix / 32] >> (ix % 32)) & mask;
2511 }
2512
2513 void Decoder::handleGedDecoderError(int line, const char *field,
2514 GED_RETURN_VALUE status) {
2515 std::stringstream ss;
2516 ss << "GED reports ";
2517 if (status == GED_RETURN_VALUE_INVALID_VALUE) {
2518 ss << "invalid value";
2519 } else if (status == GED_RETURN_VALUE_INVALID_FIELD) {
2520 ss << "invalid field";
2521 } else if (status != GED_RETURN_VALUE_SUCCESS) {
2522 ss << "error (" << (int)status << ")";
2523 }
2524 ss << " for field " << field << " (line " << line << ")\n";
2525 ss << FormatOpBits(m_model, (const char *)m_binary + currentPc());
2526 if (status == GED_RETURN_VALUE_INVALID_VALUE) {
2527 // indicates something wrong with the bits given, but we can
2528 // continue trying to decode things
2529 errorT(ss.str());
2530 } else {
2531 // indicates IGA is totally wrong and we should probably bail out
2532 fatalT(ss.str());
2533 }
2534 }
2535
2536 // These template class member functions are not defined in header (only
2537 // declared in header), thus their definitions are not available to other
2538 // .cpp. We need to explicitly instantiate those template functions so
2539 // the other .cpp can reference them.
2540 template void
2541 Decoder::decodeSourceBasicAlign16<SourceIndex::SRC0>(Instruction *inst,
2542 SourceIndex toSrcIx);
2543 template void
2544 Decoder::decodeSourceBasicAlign16<SourceIndex::SRC1>(Instruction *inst,
2545 SourceIndex toSrcIx);
2546 template void
2547 Decoder::decodeSourceBasicAlign1<SourceIndex::SRC0>(Instruction *inst,
2548 SourceIndex toSrcIx);
2549 template void
2550 Decoder::decodeSourceBasicAlign1<SourceIndex::SRC1>(Instruction *inst,
2551 SourceIndex toSrcIx);