Debian Codegraph @

visa/iga/IGALibrary/Backend/GED/Encoder.cpp

raw
1 /*========================== begin_copyright_notice ============================
2
3 Copyright (C) 2017-2021 Intel Corporation
4
5 SPDX-License-Identifier: MIT
6
7 ============================= end_copyright_notice ===========================*/
8
9 #include "Encoder.hpp"
10 #include "../../Frontend/IRToString.hpp"
11 #include "../../IR/Kernel.hpp"
12 #include "../../IR/SWSBSetter.hpp"
13 #include "../../Models/Models.hpp"
14 #include "../../Timer/Timer.hpp"
15 #include "../../bits.hpp"
16 #include "../../strings.hpp"
17 #include "IGAToGEDTranslation.hpp"
18
19 #include <cstring>
20
21 using namespace iga;
22
23
24 static const char *gedReturnValueToString(GED_RETURN_VALUE rv) {
25 switch (rv) {
26 case GED_RETURN_VALUE_SUCCESS:
27 return "success";
28 case GED_RETURN_VALUE_CYCLIC_DEPENDENCY:
29 return "cyclic dependency";
30 case GED_RETURN_VALUE_NULL_POINTER:
31 return "null pointer";
32 case GED_RETURN_VALUE_OPCODE_NOT_SUPPORTED:
33 return "unsupported opcode";
34 case GED_RETURN_VALUE_NO_COMPACT_FORM:
35 return "no compact form";
36 case GED_RETURN_VALUE_INVALID_FIELD:
37 return "invalid field";
38 case GED_RETURN_VALUE_INVALID_VALUE:
39 return "invalid value";
40 case GED_RETURN_VALUE_INVALID_INTERPRETER:
41 return "invalid interpreter";
42 default:
43 return "other error";
44 }
45 }
46 void Encoder::handleGedError(int line, const char *setter,
47 GED_RETURN_VALUE status) {
48 errorT("IGALibrary/GED/Encoder.cpp:", line, ": GED_Set", setter, ": ",
49 gedReturnValueToString(status));
50 }
51
52 Encoder::Encoder(const Model &model, ErrorHandler &errHandler,
53 const EncoderOpts &opts)
54 : GEDBitProcessor(model, errHandler), m_opts(opts),
55 m_numberInstructionsEncoded(0), m_mem(nullptr) {
56 // derive the swsb encoding mode from platform if not set
57 if (opts.swsbEncodeMode == SWSB_ENCODE_MODE::SWSBInvalidMode) {
58 m_opts.swsbEncodeMode = model.getSWSBEncodeMode();
59 }
60 }
61
62 void Encoder::encodeKernelPreProcess(Kernel &k) { doEncodeKernelPreProcess(k); }
63
64 void Encoder::doEncodeKernelPreProcess(Kernel &k) {
65 if (m_opts.autoDepSet && platform() >= Platform::XE) {
66 SWSBAnalyzer swsbAnalyzer(k, errorHandler(), m_opts.swsbEncodeMode,
67 m_opts.sbidCount, m_opts.autoSBIDCounter);
68 swsbAnalyzer.run();
69 }
70 }
71
72 double Encoder::getElapsedTimeMicros(unsigned int idx) {
73 return getIGATimerUS(idx);
74 }
75
76 int64_t Encoder::getElapsedTimeTicks(unsigned int idx) {
77 return getIGATimerTicks(idx);
78 }
79
80 std::string Encoder::getTimerName(unsigned int idx) {
81 return getIGATimerNames(idx);
82 }
83
84 size_t Encoder::getNumInstructionsEncoded() const {
85 return m_numberInstructionsEncoded;
86 }
87
88 void Encoder::encodeKernel(Kernel &k, MemManager &mem, void *&bits,
89 uint32_t &bitsLen) {
90 #ifndef IGA_DISABLE_ENCODER_EXCEPTIONS
91 try {
92 #endif
93 initIGATimer();
94 setIGAKernelName("test");
95 IGA_ASSERT(k.getModel().platform == platform(),
96 "kernel/encoder model mismatch");
97
98 encodeKernelPreProcess(k);
99 m_needToPatch.clear();
100 m_mem = &mem;
101 m_numberInstructionsEncoded = k.getInstructionCount();
102 size_t allocLen = m_numberInstructionsEncoded * UNCOMPACTED_SIZE;
103 if (allocLen == 0) // for empty kernel case
104 allocLen = 4;
105 m_instBuf = (uint8_t *)mem.alloc(allocLen);
106 if (!m_instBuf) {
107 fatalAtT(0, "failed to allocate memory for kernel binary");
108 return;
109 }
110
111 for (auto blk : k.getBlockList()) {
112 START_ENCODER_TIMER();
113 encodeBlock(k, blk);
114 STOP_ENCODER_TIMER();
115 if (hasFatalError()) {
116 return;
117 }
118 }
119 START_ENCODER_TIMER();
120 patchJumpOffsets();
121 STOP_ENCODER_TIMER();
122
123 // setting actual size
124 bitsLen = currentPc();
125 bits = m_instBuf;
126
127 applyGedWorkarounds(k, currentPc());
128
129 // clear any padding
130 memset(m_instBuf + bitsLen, 0, allocLen - bitsLen);
131 #ifndef IGA_DISABLE_ENCODER_EXCEPTIONS
132 } catch (const iga::FatalError &) {
133 // error is already reported
134 }
135 #endif
136 }
137
138 void Encoder::encodeInlineBinaryInst(Instruction &inst) {
139 setEncodedPC(&inst, currentPc());
140 for (auto i = 0, j = 3; i < 4; ++i, --j) {
141 uint8_t *src = (uint8_t *)&inst.getInlineBinary().at(j);
142 uint8_t *dst = m_instBuf + currentPc() + i * 4;
143 memcpy_s(dst, 4, src, 4);
144 }
145
146 // Inline-binary-instruction must have size 16
147 advancePc(16);
148 }
149
150 void Encoder::encodeBlock(Kernel &k, Block *blk) {
151 m_blockToOffsetMap[blk] = currentPc();
152
153 InstList &instList = blk->getInstList();
154 const auto instListEnd = instList.end();
155 for (auto instIter = instList.begin(); instIter != instListEnd; ++instIter) {
156 Instruction *inst = *instIter;
157
158 if (inst->hasInstOpt(InstOpt::CACHELINEALIGN)) {
159 while (currentPc() / 64 != (currentPc() + 31) / 64) {
160 SWSB swsb(SWSB::DistType::NO_DIST, SWSB::TokenType::NOTOKEN, 0, 0);
161 Instruction *syncInst = k.createSyncNopInstruction(swsb);
162 setCurrInst(syncInst);
163 encodeInstruction(*syncInst);
164 if (hasFatalError()) {
165 return;
166 }
167 setEncodedPC(syncInst, currentPc());
168 GED_RETURN_VALUE status = GED_RETURN_VALUE_SIZE;
169 status = GED_EncodeIns(&m_gedInst, GED_INS_TYPE_NATIVE,
170 m_instBuf + currentPc());
171 if (status != GED_RETURN_VALUE_SUCCESS) {
172 errorAtT(inst->getLoc(), "GED unable to encode instruction: ",
173 gedReturnValueToString(status));
174 }
175 blk->insertInstBefore(instIter, syncInst);
176 advancePc(16);
177 }
178 }
179
180 setCurrInst(inst);
181
182 if (inst->isInlineBinaryInstruction()) {
183 encodeInlineBinaryInst(*inst);
184 continue;
185 }
186
187 encodeInstruction(*inst);
188 if (hasFatalError()) {
189 return;
190 }
191 setEncodedPC(inst, currentPc());
192
193 GED_RETURN_VALUE status = GED_RETURN_VALUE_SIZE;
194
195 // If -Xforce-no-compact is set, do not compact any insruction
196 // Otherwise, if {NoCompact} is set, do not compact the instruction
197 // Otherwise, if {Compacted} is set on the instruction, try to compact it
198 // and report error on fail Otherwise, if no compaction setting on the
199 // instruction, try to compact the instruction if -Xauto-compact Otherwise,
200 // do not compact the instruction
201 bool mustCompact = inst->hasInstOpt(InstOpt::COMPACTED);
202 bool mustNotCompact = inst->hasInstOpt(InstOpt::NOCOMPACT);
203 if (m_opts.forceNoCompact) {
204 mustCompact = false;
205 mustNotCompact = true;
206 }
207
208 int32_t iLen = 16;
209 if (mustCompact || (!mustNotCompact && m_opts.autoCompact)) {
210 // try compact first
211 status = GED_EncodeIns(&m_gedInst, GED_INS_TYPE_COMPACT,
212 m_instBuf + currentPc());
213 if (status == GED_RETURN_VALUE_SUCCESS) {
214 // If auto compation is turned on, in case we need to patch later.
215 inst->addInstOpt(InstOpt::COMPACTED);
216 iLen = 8;
217 } else if (status == GED_RETURN_VALUE_NO_COMPACT_FORM) {
218 if (mustCompact) {
219 if (m_opts.explicitCompactMissIsWarning) {
220 warningAtT(inst->getLoc(), "GED unable to compact instruction");
221 } else {
222 errorAtT(inst->getLoc(), "GED unable to compact instruction");
223 }
224 }
225 } // else: some other error (unreachable?)
226 }
227
228 // try native encoding if compaction failed
229 if (status != GED_RETURN_VALUE_SUCCESS) {
230 inst->removeInstOpt(InstOpt::COMPACTED);
231 status = GED_EncodeIns(&m_gedInst, GED_INS_TYPE_NATIVE,
232 m_instBuf + currentPc());
233 if (status != GED_RETURN_VALUE_SUCCESS) {
234 errorAtT(inst->getLoc(), "GED unable to encode instruction: ",
235 gedReturnValueToString(status));
236 }
237 }
238 advancePc(iLen);
239 }
240 }
241
242 bool Encoder::getBlockOffset(const Block *b, uint32_t &pc) {
243 auto iter = m_blockToOffsetMap.find(b);
244 if (iter != m_blockToOffsetMap.end()) {
245 pc = iter->second;
246 return true;
247 }
248 return false;
249 }
250
251 void Encoder::setEncodedPC(Instruction *inst, int32_t encodedPC) {
252 #if 0
253 auto iter = m_instPcs.find(inst);
254 IGA_ASSERT(iter == m_instPcs.end(), "resetting encode PC");
255 ((void)iter); // dummy use where ASSERT_USER compiles out
256 m_instPcs[inst] = encodedPC;
257 #else
258 inst->setPC(encodedPC);
259 #endif
260 }
261
262 int32_t Encoder::getEncodedPC(const Instruction *inst) const {
263 #if 0
264 auto iter = m_instPcs.find(inst);
265 if (iter == m_instPcs.end()) {
266 IGA_ASSERT_FALSE("inst PC not found");
267 return 0;
268 }
269 return iter->second;
270 #else
271 return inst->getPC();
272 #endif
273 }
274
275 void Encoder::encodeFC(const Instruction &i) {
276 const OpSpec &os = i.getOpSpec();
277
278 if (os.is(Op::MATH)) {
279 GED_MATH_FC mfc = lowerMathFC(i.getMathFc());
280 GED_ENCODE(MathFC, mfc);
281 } else if (os.is(Op::BFN)) {
282 GED_ENCODE(BfnFC, i.getBfnFc().value);
283 } else if (os.isDpasFormat()) {
284 auto sf = i.getDpasFc();
285 auto sdepth = GetDpasSystolicDepth(sf);
286 auto rc = GetDpasRepeatCount(sf);
287 if (os.is(Op::BDPAS)) {
288 // in bdpas, repeat count is fixed to 8
289 // there is no instruction encoding bits for repeat count
290 // Note that bdpas's enum class for systolic depth is different from
291 // dpas'. Hence GED has different API name (GED_GetSystolicDepthBlkScl).
292 GED_ENCODE(SystolicDepthBlkScl, sdepth);
293 } else {
294 GED_ENCODE(SystolicDepth, sdepth);
295 GED_ENCODE(RepeatCount, rc);
296 }
297 } else if (os.isAnySendFormat()) {
298 if (platform() >= Platform::XE) {
299 // on earlier platforms this is stowed in ExDesc
300 auto sfid = lowerSFID(i.getSendFc());
301 GED_ENCODE(SFID, sfid);
302 }
303 } else if (os.is(Op::SYNC)) {
304 GED_SYNC_FC wfc = lowerSyncFC(i.getSyncFc());
305 GED_ENCODE(SyncFC, wfc);
306 } else if (os.is(Op::SHFL)) {
307 GED_SHUFFLE_FC sfc = lowerShuffleFC(i.getShuffleFc());
308 GED_ENCODE(ShuffleFC, sfc);
309 } else if (os.is(Op::LFSR)) {
310 GED_LFSR_FC fc = lowerLfsrFC(i.getLfsrFc());
311 GED_ENCODE(LfsrFC, fc);
312 } else if (os.is(Op::DNSCL)) {
313 auto fc = i.getDnsclFc();
314 GED_ENCODE(ConvSrcDataType, lowerConvSrcDataType(fc.convSrcTy));
315 GED_ENCODE(ConvDstDataType, lowerConvDstDataType(fc.convDstTy));
316 GED_ENCODE(DnsclMode, lowerDnsclMode(fc.dnsclMode));
317 GED_ENCODE(RoundingMode, lowerRoundingMode(fc.roundingMode));
318 } else if (os.supportsBranchCtrl()) {
319 GED_ENCODE(BranchCtrl, lowerBranchCntrl(i.getBranchCtrl()));
320 } else if (os.supportsSubfunction()) {
321 IGA_ASSERT_FALSE("encoder needs to encode subfunction for this op");
322 }
323 }
324
325 void Encoder::encodeInstruction(Instruction &inst) {
326 m_opcode = inst.getOp();
327 const auto gedPlat = lowerPlatform(platform());
328 const auto gedOp = lowerOpcode(m_opcode);
329 if (gedOp == GED_OPCODE_INVALID) {
330 fatalAtT(inst.getLoc(), "OpSpec failed to map to a valid GED opcode");
331 return;
332 }
333 GED_RETURN_VALUE status = GED_InitEmptyIns(gedPlat, &m_gedInst, gedOp);
334 if (status != GED_RETURN_VALUE_SUCCESS) {
335 fatalAtT(inst.getLoc(), "GED failed to create instruction template");
336 return;
337 }
338
339 if (m_opcode == Op::ILLEGAL || m_opcode == Op::THRYLD) {
340 // GED does all the work for this instruction
341 return;
342 } else if (m_opcode == Op::NOP) {
343 // nop supports {Breakpoint}
344 encodeOptions(inst);
345 return;
346 }
347 const OpSpec &os = inst.getOpSpec();
348
349 // Dwindling cases where we must use Align16
350 // Pre-GEN10 ternary ops are all align16
351 bool isTernary = platform() < Platform::GEN10 && os.isTernary();
352 bool contextSaveRestoreNeedsAlign16 =
353 isAlign16MathMacroRegisterCsrOperand(inst.getDestination()) ||
354 isAlign16MathMacroRegisterCsrOperand(inst.getSource(0));
355 // IEEE macro instructions (math.invm and math.rsqrtm)
356 bool align16MacroInst = m_model.supportsAlign16MacroInst() && inst.isMacro();
357 m_encodeAlign16 =
358 isTernary || contextSaveRestoreNeedsAlign16 || align16MacroInst;
359 GED_ACCESS_MODE accessMode =
360 m_encodeAlign16 ? GED_ACCESS_MODE_Align16 : GED_ACCESS_MODE_Align1;
361 if (m_model.supportsAccessMode()) {
362 GED_ENCODE(AccessMode, accessMode);
363 } // else GED will crash given this call (even given Align1)
364
365 ExecSize execSize = inst.getExecSize();
366 if (os.isTernary() && m_model.supportsAlign16Ternary() &&
367 inst.getExecSize() == ExecSize::SIMD1) {
368 // scalar ternary workaround for Align16
369 // (c.f. Encoder::encodeTernaryDestinationAlign16)
370 execSize = inst.getDestination().getType() == Type::DF ? ExecSize::SIMD2
371 : ExecSize::SIMD4;
372 }
373 GED_ENCODE(ExecSize, lowerExecSize(execSize));
374
375 encodeFC(inst);
376
377 if (os.supportsQtrCtrl()) {
378 // use ExecSize from above since it may have been modified
379 GED_CHANNEL_OFFSET qtrCtrl = lowerQtrCtrl(inst.getChannelOffset());
380 GED_ENCODE(ChannelOffset, qtrCtrl);
381 }
382
383 GED_ENCODE(MaskCtrl, lowerEmask(inst.getMaskCtrl()));
384
385 // Predicate
386 const Predication &pred = inst.getPredication();
387 if (os.supportsPredication()) {
388 GED_ENCODE(PredCtrl, lowerPredCtrl(pred.function));
389 } else {
390 GED_ENCODE(PredCtrl, GED_PRED_CTRL_Normal);
391 }
392
393 bool isImm64Src0Overlap =
394 platform() >= Platform::XE &&
395 inst.getSource(0).getKind() == Operand::Kind::IMMEDIATE &&
396 TypeIs64b(inst.getSource(0).getType());
397
398 if (!isImm64Src0Overlap && inst.getOpSpec().supportsFlagModifier()) {
399 if (os.op == Op::BFN) {
400 switch (inst.getFlagModifier()) {
401 case FlagModifier::NONE:
402 case FlagModifier::EQ:
403 case FlagModifier::GT:
404 case FlagModifier::LT:
405 // GED does the special mapping to CondMod2
406 // only a subset of cond modifiers are supported on this op
407 GED_ENCODE(CondModifier, lowerCondModifier(inst.getFlagModifier()));
408 break;
409 default:
410 errorT("this instruction format only supports "
411 "(eq), (gt), and (lt) conditional modifiers");
412 }
413 } else {
414 GED_ENCODE(CondModifier, lowerCondModifier(inst.getFlagModifier()));
415 }
416 }
417
418 bool hasFlagRegField = true;
419 // For >= XE_HPC, Some fields only exist when having CondCtrl or PredCtrl:
420 // PredInv, FlagRegNum, FlagSubRegNum
421 // In GED, either CondCtrl or PredCtrl have to be set to non-zero before
422 // these fields can be set
423 if (platform() >= Platform::XE_HPC) {
424 hasFlagRegField = (inst.getFlagModifier() != FlagModifier::NONE) ||
425 (pred.function != PredCtrl::NONE) || inst.isBranching();
426 }
427
428 if (os.supportsPredication() && hasFlagRegField)
429 GED_ENCODE(PredInv,
430 pred.inverse ? GED_PRED_INV_Invert : GED_PRED_INV_Normal);
431
432 // GED_ExecutionDataType
433 RegRef flagReg = inst.getFlagReg();
434 if (hasFlagRegField && (flagReg != REGREF_INVALID)) {
435 GED_ENCODE(FlagRegNum, static_cast<uint32_t>(inst.getFlagReg().regNum));
436 GED_ENCODE(FlagSubRegNum, inst.getFlagReg().subRegNum);
437 }
438
439 // set AccWrEn where supported
440 if (inst.hasInstOpt(InstOpt::ACCWREN)) {
441 GED_ENCODE(AccWrCtrl, GED_ACC_WR_CTRL_AccWrEn);
442 }
443
444 if (os.isBranching()) {
445 if (m_model.supportsSimplifiedBranches()) {
446 encodeBranchingInstructionSimplified(inst);
447 } else {
448 encodeBranchingInstruction(inst);
449 }
450 // options encoded internally
451 } else if (os.isTernary()) {
452 encodeTernaryInstruction(inst, accessMode);
453 } else if (os.isQuinary()) {
454 encodeQuinaryInstruction(inst);
455 } else if (os.isAnySendFormat()) {
456 encodeSendInstruction(inst);
457 } else if (os.is(Op::SYNC)) {
458 encodeSyncInstruction(inst);
459 } else {
460 encodeBasicInstruction(inst, accessMode);
461 }
462
463 if (!hasFatalError()) {
464 encodeOptions(inst);
465
466 // setup for back patching on branching ops
467 if (os.isBranching() || inst.isMovWithLabel()) {
468 bool src0IsLabel = inst.getSource(0).isImm();
469 bool src1IsLabel = inst.getSourceCount() > 1 && inst.getSource(1).isImm();
470 if (src0IsLabel || src1IsLabel) {
471 m_needToPatch.emplace_back(&inst, m_gedInst, m_instBuf + currentPc());
472 // Force not to compact label instructions to avoid the compaction error
473 // when auto-compaction is enabled. We could set this inst to
474 // compactable during Encoder::encodeBlock when the value is unknown
475 // (and assume to be 0). But we can only compact imm values use up to 12
476 // bits.
477 inst.addInstOpt(InstOpt::NOCOMPACT);
478 }
479 }
480 }
481 }
482
483 void Encoder::encodeBasicInstruction(const Instruction &inst,
484 GED_ACCESS_MODE accessMode) {
485 const OpSpec &os = inst.getOpSpec();
486
487 if (os.isBinaryWithExecDataType())
488 GED_ENCODE(ExecutionDataType,
489 lowerExecDataType(inst.getDestination().getType()));
490
491 switch (inst.getSourceCount()) {
492 case 2:
493 encodeBasicSource<SourceIndex::SRC1>(inst, inst.getSource(1), accessMode);
494 // vvvv fall through vvvv
495 case 1:
496 encodeBasicSource<SourceIndex::SRC0>(inst, inst.getSource(0), accessMode);
497 }
498
499 if (os.supportsDestination()) {
500 encodeBasicDestination(inst, inst.getDestination(), accessMode);
501 if (os.op == Op::BDPAS) {
502 GED_ENCODE(ExecutionDataType,
503 lowerExecDataType(inst.getDestination().getType()));
504 }
505 } else if (os.op == Op::WAIT) {
506 // wait has an implicit destination (same as first source)
507 // but with dst region of <1>
508 Operand copy(inst.getSource(0));
509 copy.setRegion(Region::DST1);
510 encodeBasicDestination(inst, copy);
511 }
512 }
513
514 void Encoder::encodeTernaryDestinationAlign1(const Instruction &inst) {
515 const Operand &dst = inst.getDestination();
516
517 if (inst.getOpSpec().supportsSaturation()) {
518 GED_ENCODE(Saturate, lowerSaturate(dst.getDstModifier()));
519 }
520 GED_ENCODE(DstDataType, lowerDataType(dst.getType()));
521 GED_ENCODE(DstRegFile, lowerRegFile(dst.getDirRegName()));
522 encodeDstReg(dst.getDirRegName(), dst.getDirRegRef().regNum);
523
524 if (inst.isMacro()) {
525 GED_ENCODE(DstMathMacroExt, lowerMathMacroReg(dst.getMathMacroExt()));
526 // GED_ENCODE(DstHorzStride, 1);
527 } else {
528 GED_ENCODE(DstSubRegNum,
529 SubRegToBinaryOffset(dst.getDirRegRef().subRegNum,
530 dst.getDirRegName(), dst.getType(),
531 m_model.platform));
532 if (inst.getOpSpec().hasDstHorzStride())
533 GED_ENCODE(DstHorzStride, static_cast<int>(dst.getRegion().getHz()));
534 }
535 }
536 template <SourceIndex S>
537 void Encoder::encodeQuinarySource(const Instruction &inst) {
538 assert(platform() > Platform::XE3 &&
539 "Quinary sources not supported on platform");
540 assert(inst.getOpSpec().isBDPAS() &&
541 "Quinary source not supported for instruction; only supported for bdpas");
542
543 const Operand &src = inst.getSource(S);
544 Type srcType = src.getType();
545
546 auto getSubRegOff = [&]() {
547 return SubRegToBinaryOffset(src.getDirRegRef().subRegNum,
548 src.getDirRegName(), srcType,
549 m_model.platform);
550 };
551 if (S == SourceIndex::SRC0) {
552 GED_ENCODE(Src0DataType, lowerDataType(srcType));
553 encodeSrcSubRegNum<S>(getSubRegOff());
554 } else if (S == SourceIndex::SRC1) {
555 GED_ENCODE(Src1Precision, lowerSubBytePrecision(srcType));
556 } else if (S == SourceIndex::SRC2) {
557 GED_ENCODE(Src2Precision, lowerSubBytePrecision(srcType));
558 } else if (S == SourceIndex::SRC3) {
559 encodeSrcSubRegNum<S>(getSubRegOff());
560 } else if (S == SourceIndex::SRC4) {
561 encodeSrcSubRegNum<S>(getSubRegOff());
562 }
563 encodeSrcRegFile<S>(lowerRegFile(src.getDirRegName()));
564 encodeSrcReg<S>(src.getDirRegName(), src.getDirRegRef().regNum);
565 }
566
567 template <SourceIndex S>
568 void Encoder::encodeTernarySourceAlign1(const Instruction &inst) {
569 // CNL+ align1 ternary
570 if (platform() < Platform::GEN10) {
571 fatalT("src", (int)S,
572 ": align1 ternary is not supported on this "
573 "platform");
574 return;
575 }
576
577 const Operand &src = inst.getSource(S);
578 Type srcType = src.getType();
579 // DPAS
580 if (inst.getOpSpec().isDpasFormat()) {
581 // src0's type is the type for all sources
582
583 if (S == SourceIndex::SRC0) {
584 GED_ENCODE(Src0DataType, lowerDataType(srcType));
585 // GED: src0 HS = 0, VS=3
586 } else if (S == SourceIndex::SRC1) {
587 GED_ENCODE(Src1Precision, lowerSubBytePrecision(srcType));
588 // GED sets both the type and the precision at the same time for us
589 // GED: src1 HS = 1, VS=3
590 // via this higher-level API
591 } else if (S == SourceIndex::SRC2) {
592 GED_ENCODE(Src2Precision, lowerSubBytePrecision(srcType));
593 // GED: src2 HS = 3
594 // GED sets both the type and the precision at the same time for us
595 // via this higher-level API
596 }
597 encodeSrcRegFile<S>(lowerRegFile(src.getDirRegName()));
598 encodeSrcReg<S>(src.getDirRegName(), src.getDirRegRef().regNum);
599 encodeSrcSubRegNum<S>(SubRegToBinaryOffset(src.getDirRegRef().subRegNum,
600 src.getDirRegName(), srcType,
601 m_model.platform));
602
603 return;
604 }
605
606 // GED will catch any mismatch between float and int (illegal mixed mode)
607 encodeSrcType<S>(srcType); // GED dependency requires type before reg file
608
609 switch (src.getKind()) {
610 case Operand::Kind::DIRECT:
611 case Operand::Kind::MACRO: {
612 encodeSrcRegFile<S>(lowerRegFile(src.getDirRegName()));
613
614 if (platform() <= Platform::GEN11) {
615 encodeSrcAddrMode<S>(GED_ADDR_MODE_Direct);
616 }
617
618 // source modifiers
619 if (inst.getOpSpec().supportsSourceModifiers()) {
620 encodeSrcModifier<S>(src.getSrcModifier());
621 }
622
623 // regioning
624 //
625 // ternary align1 puts SpcAcc into subreg, so regions may be set
626 // in all cases
627 // * madm doesn't have a region in GEN9 ...
628 // it does in GEN10+, but we haven't supported it in syntax yet
629 // and leave it to GED to set it
630 // * src0 and src1 only has <w;h>, src2 only has <h>
631 bool hasSrcRgnHz = !inst.isMacro();
632 bool hasSrcRgnVt = !inst.isMacro() && S < SourceIndex::SRC2;
633 bool hasReducedRegion =
634 m_model.srcHasReducedRegion(static_cast<uint32_t>(S));
635 auto rgn = src.getRegion();
636 if (hasReducedRegion) {
637 encodeSrcReducedRegion<S>(rgn);
638 } else {
639 if (hasSrcRgnHz)
640 encodeSrcRegionHorz<S>(rgn.getHz());
641 if (hasSrcRgnVt)
642 encodeTernarySrcRegionVert(S, rgn.getVt());
643 }
644 // register and subregister
645 encodeSrcReg<S>(src.getDirRegName(), src.getDirRegRef().regNum);
646 if (inst.isMacro()) {
647 if (platform() < Platform::GEN11) {
648 fatalT("src", (int)S, ": math macro operands require Align16");
649 return;
650 }
651 encodeSrcMathMacroReg<S>(src.getMathMacroExt());
652 if (!hasReducedRegion) {
653 if (S != SourceIndex::SRC2) {
654 encodeTernarySrcRegionVert(S, Region::Vert::VT_4);
655 }
656 encodeSrcRegionHorz<S>(Region::Horz::HZ_1);
657 }
658 } else {
659 auto subReg = SubRegToBinaryOffset(src.getDirRegRef().subRegNum,
660 src.getDirRegName(), src.getType(),
661 m_model.platform);
662 encodeSrcSubRegNum<S>(subReg);
663 }
664 break;
665 }
666 case Operand::Kind::IMMEDIATE:
667 if (S == SourceIndex::SRC1) {
668 fatalT("src1: immediate operand in ternary align1 must be "
669 "src0 or src2");
670 return;
671 }
672 encodeSrcRegFile<S>(GED_REG_FILE_IMM);
673 if (platform() < Platform::GEN10) {
674 encodeImmVal(src.getImmediateValue(), src.getType());
675 } else {
676 encodeTernaryImmVal<S>(src.getImmediateValue(), src.getType());
677 }
678 break;
679 default:
680 fatalT("src", (int)S, ": invalid operand kind");
681 return;
682 }
683 }
684 void Encoder::encodeQuinaryInstruction(const Instruction &inst) {
685 // execution data type set based on dst
686 Type dstType = inst.getDestination().getType();
687 GED_EXECUTION_DATA_TYPE execDataType;
688
689 // use ternary instruction function support for dst, src0, src1 and src2
690 if (isTernaryAlign1Floating(dstType)) {
691 execDataType = GED_EXECUTION_DATA_TYPE_Float;
692 } else if (isTernaryAlign1Integral(dstType)) {
693 execDataType = GED_EXECUTION_DATA_TYPE_Integer;
694 } else {
695 fatalT("unsupported execution data type");
696 return;
697 }
698 GED_ENCODE(ExecutionDataType, execDataType);
699
700 if (inst.getOpSpec().supportsDestination()) {
701 encodeTernaryDestinationAlign1(inst);
702 }
703 encodeQuinarySource<SourceIndex::SRC0>(inst);
704 encodeQuinarySource<SourceIndex::SRC1>(inst);
705 encodeQuinarySource<SourceIndex::SRC2>(inst);
706 encodeQuinarySource<SourceIndex::SRC3>(inst);
707 encodeQuinarySource<SourceIndex::SRC4>(inst);
708 }
709
710 void Encoder::encodeTernaryInstruction(const Instruction &inst,
711 GED_ACCESS_MODE accessMode) {
712 if (accessMode == GED_ACCESS_MODE_Align1) {
713 encodeTernaryAlign1Instruction(inst);
714 } else {
715 encodeTernaryAlign16Instruction(inst);
716 }
717 }
718 void Encoder::encodeTernaryAlign16Instruction(const Instruction &inst) {
719 if (inst.getOpSpec().supportsDestination()) {
720 encodeTernaryDestinationAlign16(inst);
721 }
722 encodeTernarySourceAlign16<SourceIndex::SRC0>(inst);
723 encodeTernarySourceAlign16<SourceIndex::SRC1>(inst);
724 encodeTernarySourceAlign16<SourceIndex::SRC2>(inst);
725 }
726 void Encoder::encodeTernaryAlign1Instruction(const Instruction &inst) {
727 // set ExecutionDataType (integral or floating)
728 // the operands must be part of the same type set
729 Type src0Type = inst.getSource(0).getType();
730 GED_EXECUTION_DATA_TYPE execDataType;
731 if (isTernaryAlign1Floating(src0Type)) {
732 execDataType = GED_EXECUTION_DATA_TYPE_Float;
733 } else if (isTernaryAlign1Integral(src0Type)) {
734 execDataType = GED_EXECUTION_DATA_TYPE_Integer;
735 } else {
736 fatalT("src0: unsupported type for ternary align1 encoding");
737 return;
738 }
739 GED_ENCODE(ExecutionDataType, execDataType);
740
741 if (inst.getOpSpec().supportsDestination()) {
742 encodeTernaryDestinationAlign1(inst);
743 }
744 encodeTernarySourceAlign1<SourceIndex::SRC0>(inst);
745 encodeTernarySourceAlign1<SourceIndex::SRC1>(inst);
746 encodeTernarySourceAlign1<SourceIndex::SRC2>(inst);
747 }
748
749 void Encoder::encodeBranchingInstruction(const Instruction &inst) {
750 // the destination stride is always 1 for all control flow
751 GED_ENCODE(DstHorzStride, 1);
752
753 // control flow instructions require patching later if any operand is a label
754 bool src0IsLabel = inst.getSource(0).getKind() == Operand::Kind::LABEL;
755
756 // break up instructions into various classes
757 // - stuff with implicit operands: jmpi
758 // - stuff that can take register operands: call, calla, return
759 // - everything else: if, else, while, ..., goto, join, ...
760 if (m_opcode == Op::JMPI) {
761 // jmpi encodes the syntax
762 // jmpi (1) LABEL
763 // jmpi (1) reg32
764 // as
765 // jmpi (1) ip ip LABEL
766 // jmpi (1) ip ip reg32
767 //
768 // "Restriction: The index data type must be D (Signed DWord Integer)."
769 //
770 // implicit IP ...
771 encodeBasicDestination(inst, Operand::DST_REG_IP_UD);
772 encodeBasicSource<SourceIndex::SRC0>(inst, Operand::SRC_REG_IP_UD);
773 GED_ENCODE(Src1DataType, GED_DATA_TYPE_d);
774 if (src0IsLabel) {
775 // jmpi (1) LABEL (encodes into Src1)
776 GED_ENCODE(Src1RegFile, GED_REG_FILE_IMM);
777 } else {
778 // jmpi (1) reg32 (encodes into Src1)
779 encodeBasicSource<SourceIndex::SRC1>(inst, inst.getSource(0));
780 }
781 } else if (m_opcode == Op::CALL || m_opcode == Op::CALLA ||
782 m_opcode == Op::RET) {
783 // e.g. call, calla, ret
784 // call (..) imm32
785 // call (..) reg32
786 // calla (..) imm32
787 // ret (...) reg32 => encodes as ret (...) null reg
788 //
789 if (m_opcode == Op::CALL || m_opcode == Op::CALLA) {
790 encodeBasicDestination(inst, inst.getDestination());
791 } else if (m_opcode == Op::RET) {
792 encodeBasicDestination(inst, Operand::DST_REG_NULL_UD);
793 encodeBasicSource<SourceIndex::SRC0>(inst, inst.getSource(0));
794 }
795
796 if (m_opcode == Op::CALL || m_opcode == Op::CALLA) {
797 if (src0IsLabel) {
798 // op == CALL (since it's a label), hence we have
799 // call (..) imm32 => which uses src1
800 GED_ENCODE(Src1RegFile, GED_REG_FILE_IMM);
801 GED_ENCODE(Src1DataType, GED_DATA_TYPE_d);
802 } else {
803 // call (..) reg32
804 // calla (..) imm32
805 encodeBasicSource<SourceIndex::SRC1>(inst, inst.getSource(0));
806 }
807
808 // <2;2,1> restriction for CALL and CALLA restriction is only for
809 // IVB+HSW, but simulator has it until CNL. So we have to support it
810 // until we get CNL HW validation moves to it
811 if (callNeedsSrc0Region221(inst)) {
812 GED_ENCODE(Src0VertStride, 2);
813 GED_ENCODE(Src0Width, 2);
814 GED_ENCODE(Src0HorzStride, 1);
815 }
816 // though it's not stated in the spec, ICL requires src0 region be set to
817 // <2;4,1>
818 else if (callNeedsSrc0Region241(inst)) {
819 GED_ENCODE(Src0VertStride, 2);
820 GED_ENCODE(Src0Width, 4);
821 GED_ENCODE(Src0HorzStride, 1);
822 }
823 }
824 } else if (m_opcode == Op::BRD || m_opcode == Op::BRC) {
825 // [brd/brc]: The ip register must be used (for example, by the assembler)
826 // as dst.
827 encodeBasicDestination(inst, Operand::DST_REG_IP_D);
828 if (!src0IsLabel) {
829 encodeBasicSource<SourceIndex::SRC0>(inst, inst.getSource(0));
830 }
831 GED_DATA_TYPE ty =
832 platform() < Platform::GEN8 ? GED_DATA_TYPE_w : GED_DATA_TYPE_d;
833 GED_ENCODE(Src0RegFile, src0IsLabel ? GED_REG_FILE_IMM : GED_REG_FILE_GRF);
834 GED_ENCODE(Src0DataType, ty);
835 // GED automatically sets?
836 // if (m_opcode == Op::BRC && src0IsLabel) {
837 // GED_ENCODE(Src1DataType, ty);
838 // }
839 // if (m_opcode == Op::BRD && m_inst->getOpSpec().hasImplicitSrcRegion(0)) {
840 // encodeSrcRegion(SourceIndex::SRC0,
841 // m_inst->getOpSpec().implicitSrcRegion(0));
842 // }
843 } else {
844 // regular control flow that only accepts immediate values
845 // e.g. if, else, endif, while, cont, break, goto, join, halt
846
847 // Apparently, the implicit destination on these instructions
848 // is null instead of ip (unlike jmpi etc)
849 // destination is ip<1>:ud
850 //
851 // encodeDestination(&Operand::DST_IP);
852 encodeBasicDestination(inst, Operand::DST_REG_NULL_UD);
853
854 // UIP
855 if (m_opcode != Op::ENDIF && m_opcode != Op::WHILE &&
856 m_opcode != Op::JOIN) {
857 // if/else/halt/brk/cont.... all require :d on operands
858 GED_DATA_TYPE ty =
859 platform() < Platform::GEN8 ? GED_DATA_TYPE_w : GED_DATA_TYPE_d;
860 GED_ENCODE(Src0RegFile, GED_REG_FILE_IMM);
861 GED_ENCODE(Src0DataType, ty);
862 }
863 // before XE don't need to set JIP for control flow instructions that have
864 // UIP JIP
865 if (m_opcode == Op::WHILE || m_opcode == Op::ENDIF ||
866 m_opcode == Op::JOIN) {
867 GED_ENCODE(Src1RegFile, GED_REG_FILE_IMM);
868 GED_ENCODE(Src1DataType, GED_DATA_TYPE_d);
869 }
870 }
871 }
872
873 void Encoder::encodeBranchingInstructionSimplified(const Instruction &inst) {
874 const OpSpec &os = inst.getOpSpec();
875
876 // set branch control
877 if (os.supportsBranchCtrl()) {
878 GED_ENCODE(BranchCtrl, lowerBranchCntrl(inst.getBranchCtrl()));
879 }
880
881 // control flow instructions require patching later if any operand is a label
882 const Operand &src0 = inst.getSource(0);
883 bool src0IsLabel = src0.getKind() == Operand::Kind::LABEL;
884
885 // for jmpi HW will take care of IP so don't need to encode it for dst/src0
886 if (inst.getOpSpec().supportsDestination()) {
887 encodeBranchDestination(inst.getDestination());
888 } else {
889 encodeBranchDestination(Operand::DST_REG_NULL_UD);
890 }
891 // regualar control flow that only accepts immediate values
892 // e.g. if, else, endif, while, cont, break, goto, join, halt
893
894 // encoding JIP
895 if (src0IsLabel) {
896 GED_ENCODE(Src0RegFile, GED_REG_FILE_IMM);
897 // if (src0.getTargetBlock() == nullptr) {
898 // // the input value is immediate; use m_immVal as the value
899 // encodeBranchSource(src0);
900 // }
901 } else {
902 // jmpi, call, brc, ...
903 if (src0.getKind() == Operand::Kind::INDIRECT)
904 errorT("branch instructions forbid indirect register mode");
905 encodeBranchSource(src0);
906 }
907
908 if (inst.getSourceCount() == 2) {
909 // encoding UIP always IMM except for brc with a register argument
910 if (inst.getOp() != Op::BRC || src0.isImm()) {
911 GED_ENCODE(Src1RegFile, GED_REG_FILE_IMM);
912 }
913 }
914 }
915
916 void Encoder::encodeSendInstruction(const Instruction &i) {
917 ////////////////////////////////////////////
918 // send operands
919 const OpSpec &os = i.getOpSpec();
920 encodeSendDestination(i.getDestination());
921 if (os.isSendsFormat()) {
922 // old sends sources get special treatment
923 encodeSendsSource0(i.getSource(0));
924 encodeSendsSource1(i.getSource(1));
925 } else {
926 encodeSendSource0(i.getSource(0));
927 if (m_model.supportsXeSend()) {
928 encodeSendsSource1(i.getSource(1));
929 }
930 }
931
932 if (os.isSendgFormat()) {
933 GED_ENCODE(Src0Length, (uint32_t)i.getSrc0Length());
934 }
935
936 ////////////////////////////////////////////
937 // send descriptors and other gunk
938 encodeSendDescs(i);
939
940 ////////////////////////////////////////////
941 // send options
942
943 // FusionCtrl is removed from XeHPC+
944 bool hasFusion = platform() >= Platform::XE && platform() < Platform::XE_HPC;
945 if (hasFusion) {
946 GED_ENCODE(FusionCtrl, i.hasInstOpt(InstOpt::SERIALIZE)
947 ? GED_FUSION_CTRL_Serialized
948 : GED_FUSION_CTRL_Normal);
949 }
950
951 if (i.hasInstOpt(InstOpt::EOT)) {
952 GED_ENCODE(EOT, GED_EOT_EOT);
953 }
954 } // end: encodeSendInstruction
955
956 void Encoder::encodeSendDescs(const Instruction &i) {
957 if (platform() < Platform::XE) {
958 encodeSendDescsPreXe(i);
959 } else if (platform() == Platform::XE) {
960 encodeSendDescsXe(i);
961 } else if (platform() == Platform::XE_HP) {
962 encodeSendDescsXeHP(i);
963 } else if (platform() == Platform::XE_HPG || platform() == Platform::XE_HPC
964 ) {
965 encodeSendDescsXeHPG(i);
966 } else if (i.getOpSpec().isSendgFormat()) {
967 encodeSendgDescsXe3(i);
968 } else if (platform() >= Platform::XE2) {
969 encodeSendDescsXe2(i);
970 } else {
971 errorT("unsupported platform");
972 }
973
974 bool noEOTinExDesc = m_model.supportsXeSend();
975 noEOTinExDesc &= platform() <= Platform::XE3; // efficient 64b send desc
976 if (noEOTinExDesc && i.getExtMsgDescriptor().isImm() &&
977 (i.getExtMsgDescriptor().imm & 1 << 5))
978 errorT("Encoder: Send exDesc[5] must not be set (the legacy EOT bit)");
979 }
980
981 void Encoder::encodeSendDescsPreXe(const Instruction &i) {
982 SendDesc exDesc = i.getExtMsgDescriptor();
983 const OpSpec &os = i.getOpSpec();
984 if (exDesc.isReg()) {
985 if (os.isSendFormat()) {
986 errorT("unary send forbids register ExDesc");
987 }
988 GED_ENCODE(ExDescRegFile, GED_REG_FILE_ARF);
989 GED_ENCODE(ExDescAddrSubRegNum, 2 * exDesc.reg.subRegNum);
990 } else {
991 GED_ENCODE(ExDescRegFile, GED_REG_FILE_IMM);
992 GED_ENCODE(ExMsgDescImm, exDesc.imm);
993 }
994
995 SendDesc desc = i.getMsgDescriptor();
996 if (desc.isReg()) {
997 if (platform() == Platform::GEN9) {
998 uint32_t msgDescriptor = 0;
999 // There is a HW bug on SKL where HW will only copy bits 0-28 from
1000 // the address register (descriptor register) and will miss bit 30
1001 // of the descriptor. Hence, even in the case of an register
1002 // descriptor we must program bit 30 as immediate (it will be
1003 // taken from the encoding and OR'd in correctly)
1004 //
1005 // E.g. (old syntax)
1006 // sends (8) r74:hf r16 r73 0x42:ud a0.0 {Align1, Q1, NoMask}
1007 // // sampler, resLen=3, msgLen=1, extMsgLen=1
1008 // On SKL, HW will copy bits 29-31 from the actual immediate
1009 // descriptor bits. Hence, we must set immediate descriptor
1010 // bit 30 even in the case of a register descriptor. (For SKL).
1011 //
1012 // For 3D sampler bit 30 indicates HF/F return format.
1013 // For render target write bit 30 indicates HF/F input...
1014 // Thankfully for SKL the 3D sampler doesn't support HF input.
1015 // For CNL it does, and that will be bit 29.
1016 // But this bug should be fixed in CNL.
1017 if (platform() == Platform::GEN9 && desc.isReg()) {
1018 if (i.getDestination().getType() == Type::HF ||
1019 i.getSource(0).getType() == Type::HF) {
1020 msgDescriptor |= (1 << 30);
1021 }
1022 }
1023 GED_ENCODE(DescRegFile, GED_REG_FILE_IMM);
1024 GED_ENCODE(MsgDesc, msgDescriptor);
1025 }
1026 GED_ENCODE(DescRegFile, GED_REG_FILE_ARF);
1027 const RegInfo *ri = m_model.lookupRegInfoByRegName(RegName::ARF_A);
1028 IGA_ASSERT(ri, "failed to find a0 register");
1029 GED_ENCODE(DescRegNum, ri->encodeARFRegNum((int)desc.reg.regNum));
1030 } else if (desc.isImm()) {
1031 GED_ENCODE(DescRegFile, GED_REG_FILE_IMM);
1032 GED_ENCODE(MsgDesc, desc.imm);
1033 }
1034 }
1035 void Encoder::encodeSendDescsXe(const Instruction &i) {
1036 SendDesc exDesc = i.getExtMsgDescriptor();
1037 if (exDesc.isReg()) {
1038 GED_ENCODE(ExDescRegFile, GED_REG_FILE_ARF);
1039 GED_ENCODE(ExDescAddrSubRegNum, 2 * exDesc.reg.subRegNum);
1040 } else {
1041 GED_ENCODE(ExDescRegFile, GED_REG_FILE_IMM);
1042 GED_ENCODE(ExMsgDescImm, exDesc.imm);
1043 }
1044
1045 SendDesc desc = i.getMsgDescriptor();
1046 if (desc.isReg()) {
1047 GED_ENCODE(DescRegFile, GED_REG_FILE_ARF);
1048 // a0.0 is implied (there's no field)
1049 if (desc.reg.subRegNum != 0) {
1050 errorT("send with reg desc must be a0.0");
1051 }
1052 } else {
1053 GED_ENCODE(DescRegFile, GED_REG_FILE_IMM);
1054 GED_ENCODE(MsgDesc, desc.imm);
1055 }
1056 }
1057
1058 // A bit harder than Xe
1059 // * If ExBSO is set then Src1Length holds xlen
1060 // * CPS has it's own field (ExDesc[11]) only if ExDesc.IsReg
1061 void Encoder::encodeSendDescsXeHP(const Instruction &i) {
1062 SendDesc exDesc = i.getExtMsgDescriptor();
1063 if (exDesc.isReg()) {
1064 GED_ENCODE(ExDescRegFile, GED_REG_FILE_ARF);
1065 GED_ENCODE(ExDescAddrSubRegNum, 2 * exDesc.reg.subRegNum);
1066 GED_ENCODE(ExBSO, i.hasInstOpt(InstOpt::EXBSO) ? 1 : 0);
1067 if (i.hasInstOpt(InstOpt::EXBSO)) {
1068 GED_ENCODE(CPS, i.hasInstOpt(InstOpt::CPS) ? 1 : 0);
1069 GED_ENCODE(Src1Length, (uint32_t)i.getSrc1Length());
1070 } else if (i.hasInstOpt(InstOpt::CPS)) {
1071 errorT("{CPS} requires {ExBSO}");
1072 }
1073 } else {
1074 if (i.hasInstOpt(InstOpt::CPS)) {
1075 warningT("when ExDesc is immediate use ExDesc[11] rather than {CPS}");
1076 exDesc.imm |= 1 << 11;
1077 }
1078 GED_ENCODE(ExDescRegFile, GED_REG_FILE_IMM);
1079 GED_ENCODE(ExMsgDescImm, exDesc.imm);
1080 }
1081
1082 SendDesc desc = i.getMsgDescriptor();
1083 if (desc.isReg()) {
1084 GED_ENCODE(DescRegFile, GED_REG_FILE_ARF);
1085 if (desc.reg.subRegNum != 0) { // a0.0 is implied (there's no field)
1086 errorT("send with reg desc must be a0.0");
1087 }
1088 } else {
1089 GED_ENCODE(DescRegFile, GED_REG_FILE_IMM);
1090 GED_ENCODE(MsgDesc, desc.imm);
1091 }
1092 }
1093
1094 // Similar to XeHP, except
1095 // * ExDesc.IsImm implies use of Src1Length (Src.Length is in EU bits)
1096 void Encoder::encodeSendDescsXeHPG(const Instruction &i) {
1097 SendDesc exDesc = i.getExtMsgDescriptor();
1098 if (exDesc.isReg()) {
1099 GED_ENCODE(ExDescRegFile, GED_REG_FILE_ARF);
1100 GED_ENCODE(ExDescAddrSubRegNum, 2 * exDesc.reg.subRegNum);
1101 GED_ENCODE(ExBSO, i.hasInstOpt(InstOpt::EXBSO) ? 1 : 0);
1102 if (i.hasInstOpt(InstOpt::EXBSO)) {
1103 GED_ENCODE(CPS, i.hasInstOpt(InstOpt::CPS) ? 1 : 0);
1104 GED_ENCODE(Src1Length, (uint32_t)i.getSrc1Length());
1105 } else if (i.hasInstOpt(InstOpt::CPS)) {
1106 errorT("{CPS} requires {ExBSO}");
1107 }
1108 } else {
1109 if (i.hasInstOpt(InstOpt::CPS)) {
1110 warningT("when ExDesc is immediate use ExDesc[11] rather than {CPS}");
1111 exDesc.imm |= 1 << 11;
1112 }
1113 GED_ENCODE(ExDescRegFile, GED_REG_FILE_IMM);
1114 GED_ENCODE(ExMsgDescImm, exDesc.imm);
1115 GED_ENCODE(Src1Length, (uint32_t)i.getSrc1Length());
1116 }
1117
1118 SendDesc desc = i.getMsgDescriptor();
1119 if (desc.isReg()) {
1120 GED_ENCODE(DescRegFile, GED_REG_FILE_ARF);
1121 if (desc.reg.subRegNum != 0) { // a0.0 is implied (there's no field)
1122 errorT("send with reg desc must be a0.0");
1123 }
1124 } else {
1125 GED_ENCODE(DescRegFile, GED_REG_FILE_IMM);
1126 GED_ENCODE(MsgDesc, desc.imm);
1127 }
1128 }
1129
1130 // Similar to XeHPG/XeHPC except:
1131 // * ExBSO doesn't exist for UGM (high bit of ExImmOffDesc)
1132 // * if ExBSO field exists, it denotes if Src1Len in EU field or in ExDesc
1133 // * CPS is gone (use ExImmOff[11])
1134 // * If ExDesc.IsReg, then we also set ExImmOffDesc. And if it's UGM,
1135 // ExDescImm[15] is the same bit as ExBSO (ExBSO not)
1136 void Encoder::encodeSendDescsXe2(const Instruction &i) {
1137 SendDesc exDesc = i.getExtMsgDescriptor();
1138 bool hasSrc1Length = true;
1139 // Gather Send lacks Src1.Len
1140 // (the same field [103:99] encoding field becomes Src0.SubRegNum)
1141 if (i.isGatherSend())
1142 hasSrc1Length = false;
1143 if (exDesc.isReg()) {
1144 GED_ENCODE(ExDescRegFile, GED_REG_FILE_ARF);
1145 // set ExBSO for non-UGM
1146 // (UGM uses the ExBSO bit for ExDescImm[15])
1147 uint32_t exImmOffDesc = i.getExtImmOffDescriptor();
1148 if (i.getSendFc() != SFID::UGM) {
1149 // ExDescImm[15] must not be set for non-UGM which the bit is ExBSO
1150 if (exImmOffDesc & (1 << 15)) {
1151 exImmOffDesc &= ~(1 << 15);
1152 errorT("ExDescImm[15] overlaps ExBSO for this SFID and must be 0");
1153 }
1154 }
1155 if (exImmOffDesc & (0x7 << 16)) {
1156 exImmOffDesc &= ~(0x7 << 16);
1157 errorT("ExDescImm[18:16] overlaps ExDesc.Reg and must be 0");
1158 }
1159 if (exImmOffDesc & 0x7FF) {
1160 exImmOffDesc &= ~0x7FF;
1161 errorT("ExDescImm[10:0] are unmapped and must be 0");
1162 }
1163 GED_ENCODE(ExMsgDescImm, exImmOffDesc);
1164
1165 if (i.getSendFc() != SFID::UGM) {
1166 // must be done *after* ExMsgDescImm since this will stomp
1167 // ExDescImm[15] (we could flip it around and just set ExDescImm[15])
1168 GED_ENCODE(ExBSO, i.hasInstOpt(InstOpt::EXBSO) ? 1 : 0);
1169 if (i.hasInstOpt(InstOpt::EXBSO)) {
1170 GED_ENCODE(Src1Length, (uint32_t)i.getSrc1Length());
1171 }
1172 } else {
1173 if (i.hasInstOpt(InstOpt::EXBSO))
1174 warningAtT(i.getLoc(),
1175 "{ExBSO} does not exist for send.ugm on this platform");
1176 // for UGM, src1.len must be in EU field
1177 if (hasSrc1Length)
1178 GED_ENCODE(Src1Length, (uint32_t)i.getSrc1Length());
1179 }
1180
1181 if (i.hasInstOpt(InstOpt::CPS)) {
1182 // this is ExDesc[11]
1183 errorT("{CPS} does not exist on this platform");
1184 }
1185
1186 // has to be set after ExMsgDescImm due to overlap
1187 GED_ENCODE(ExDescAddrSubRegNum, 2 * exDesc.reg.subRegNum);
1188 } else {
1189 GED_ENCODE(ExDescRegFile, GED_REG_FILE_IMM);
1190 GED_ENCODE(ExMsgDescImm, exDesc.imm);
1191 if (hasSrc1Length)
1192 GED_ENCODE(Src1Length, (uint32_t)i.getSrc1Length());
1193 }
1194
1195 SendDesc desc = i.getMsgDescriptor();
1196 if (desc.isReg()) {
1197 GED_ENCODE(DescRegFile, GED_REG_FILE_ARF);
1198 if (desc.reg.subRegNum != 0) { // a0.0 is implied (there's no field)
1199 errorT("send with reg desc must be a0.0");
1200 }
1201 } else {
1202 GED_ENCODE(DescRegFile, GED_REG_FILE_IMM);
1203 GED_ENCODE(MsgDesc, desc.imm);
1204 }
1205 }
1206
1207 // Completely new encoding.
1208 //
1209 void Encoder::encodeSendgDescsXe3(const Instruction &i) {
1210 int src1Len = i.getSrc1Length();
1211 if (src1Len > 0) {
1212 bool encoded = false;
1213 if (!encoded)
1214 GED_ENCODE(Src1Length, (uint32_t)src1Len);
1215 }
1216 auto id0 = i.getSendgIndDesc0Reg();
1217 if (id0 != REGREF_INVALID) {
1218 GED_ENCODE(IndMsgDesc0IsPresent, 1);
1219 if (id0.subRegNum % 8) {
1220 errorT("ID0 must be a 8-byte-aligned offset");
1221 }
1222 GED_ENCODE(IndMsgDesc0Addr, id0.subRegNum / 8);
1223 }
1224 auto id1 = i.getSendgIndDesc1Reg();
1225 if (id1 != REGREF_INVALID) {
1226 if (id1.subRegNum % 8) {
1227 errorT("ID1 must be a 8-byte-aligned offset");
1228 }
1229 GED_ENCODE(IndMsgDesc1IsPresent, 1);
1230 GED_ENCODE(IndMsgDesc1Addr, id1.subRegNum / 8);
1231 }
1232 auto desc = i.getSendgDesc();
1233 if (id1 != REGREF_INVALID && (desc & ~((1ull << 42) - 1))) {
1234 errorT("Desc[46:42] overlaps ID1 (those bits must be zero in desc)");
1235 }
1236 GED_ENCODE(MsgDesc64, desc);
1237 }
1238
1239 void Encoder::encodeSyncInstruction(const Instruction &inst) {
1240 // Set the Dst.HorStride to 1 so that "sync.bar null" can be compacted
1241 GED_ENCODE(DstHorzStride, 1);
1242
1243 const Operand &src = inst.getSource(0);
1244 if (src.getKind() == Operand::Kind::IMMEDIATE) {
1245 encodeSrcRegFile<SourceIndex::SRC0>(GED_REG_FILE_IMM);
1246 encodeSrcType<SourceIndex::SRC0>(src.getType());
1247 encodeImmVal(src.getImmediateValue(), src.getType());
1248 } else {
1249 if (platform() <= Platform::XE_HPG) {
1250 encodeSrcRegFile<SourceIndex::SRC0>(GED_REG_FILE_ARF);
1251 } else {
1252 // XeHPC+ supports sync with reg32. For earlier platforms encode it to the
1253 // null reg anyway. If not doing so we'll encounter some weird behavior on
1254 // validation. Suspect it's becuase on some previous platforms' testcase
1255 // there are reg32 those are not valid, but IGA workaround (set it to
1256 // NULL) them
1257 if (src.isNull()) {
1258 encodeSrcRegFile<SourceIndex::SRC0>(GED_REG_FILE_ARF);
1259 } else {
1260 // currently only flag register is supported in sync.bar
1261 encodeSrcRegFile<SourceIndex::SRC0>(lowerRegFile(src.getDirRegName()));
1262 encodeSrcReg<SourceIndex::SRC0>(src.getDirRegName(),
1263 src.getDirRegRef().regNum);
1264 encodeSrcType<SourceIndex::SRC0>(src.getType());
1265 // must be flag register (otherwise GED will return error), encode the
1266 // subreg directly.
1267 GED_ENCODE(Src0SubRegNum,
1268 SubRegToBinaryOffset(src.getDirRegRef().subRegNum,
1269 src.getDirRegName(), src.getType(),
1270 m_model.platform));
1271 }
1272 }
1273 }
1274 }
1275
1276 void Encoder::encodeBranchDestination(const Operand &dst) {
1277 GED_ENCODE(DstRegFile, lowerRegFile(dst.getDirRegName()));
1278 encodeDstReg(dst.getDirRegName(), dst.getDirRegRef().regNum);
1279 GED_ENCODE(DstSubRegNum,
1280 SubRegToBinaryOffset(dst.getDirRegRef().subRegNum,
1281 dst.getDirRegName(), dst.getType(),
1282 m_model.platform));
1283 }
1284
1285 void Encoder::encodeBasicDestination(const Instruction &inst,
1286 const Operand &dst,
1287 GED_ACCESS_MODE accessMode) {
1288 IGA_ASSERT(accessMode != GED_ACCESS_MODE_Align16 || m_model.supportsAlign16(),
1289 "Align16 not supported on this platform.");
1290
1291 GED_ENCODE(DstRegFile, lowerRegFile(dst.getDirRegName()));
1292 switch (dst.getKind()) {
1293 case Operand::Kind::DIRECT:
1294 case Operand::Kind::MACRO:
1295 GED_ENCODE(DstAddrMode, GED_ADDR_MODE_Direct);
1296 GED_ENCODE(DstDataType, lowerDataType(dst.getType()));
1297 if (inst.getOpSpec().supportsSaturation()) {
1298 GED_ENCODE(Saturate, lowerSaturate(dst.getDstModifier()));
1299 }
1300 // VVVVV fallthrough VVVVV
1301 default:
1302 break;
1303 }
1304
1305 switch (dst.getKind()) {
1306 case Operand::Kind::DIRECT:
1307 if (accessMode == GED_ACCESS_MODE_Align16) {
1308 if (dst.getRegion() != Region::DST1) {
1309 fatalT("dst has inconvertible region for Align16 encoding");
1310 return;
1311 }
1312 if (isAlign16MathMacroRegisterCsrOperand(dst)) {
1313 // acc2.XXXX on BDW .. SKL is context save and restore
1314 // This is really mme0
1315 encodeDstReg(RegName::ARF_MME, 0);
1316 // on GEN8 and GEN9 all encode as acc2, but the mux varies
1317 // to distinguish which acc it really is.
1318 GED_DST_CHAN_EN chEn;
1319 switch (dst.getDirRegRef().regNum) {
1320 /// case 0: ... acc2 actually uses Align1!
1321 // old-style for acc2 would be:
1322 // mov(8) r113:ud acc2:ud {NoMask} // acc2
1323 //
1324 // acc3-9 are Align16
1325 case 1:
1326 chEn = GED_DST_CHAN_EN_x;
1327 break; // mme0/acc3 -> acc2.x (0001b)
1328 case 2:
1329 chEn = GED_DST_CHAN_EN_y;
1330 break; // mme1/acc4 -> acc2.y (0010b)
1331 case 3:
1332 chEn = GED_DST_CHAN_EN_xy;
1333 break;
1334 case 4:
1335 chEn = GED_DST_CHAN_EN_z;
1336 break;
1337 case 5:
1338 chEn = GED_DST_CHAN_EN_xz;
1339 break;
1340 case 6:
1341 chEn = GED_DST_CHAN_EN_yz;
1342 break;
1343 case 7:
1344 chEn = GED_DST_CHAN_EN_xyzw;
1345 break; // mme7/acc9 -> acc2.xyzw (0111b)
1346 default:
1347 IGA_ASSERT_FALSE("unreachable");
1348 chEn = GED_DST_CHAN_EN_x;
1349 }
1350 GED_ENCODE(DstChanEn, chEn);
1351 } else {
1352 // normal align16 destination (this still might be a
1353 // CSR work around op if the src is "acc2")
1354 encodeDstReg(dst.getDirRegName(), dst.getDirRegRef().regNum);
1355 GED_ENCODE(DstChanEn, GED_DST_CHAN_EN_xyzw);
1356 }
1357 GED_ENCODE(DstSubRegNum,
1358 SubRegToBinaryOffset(dst.getDirRegRef().subRegNum,
1359 dst.getDirRegName(), dst.getType(),
1360 m_model.platform));
1361 } else { // Align1
1362 encodeDstReg(dst.getDirRegName(), dst.getDirRegRef().regNum);
1363 GED_ENCODE(DstSubRegNum,
1364 SubRegToBinaryOffset(dst.getDirRegRef().subRegNum,
1365 dst.getDirRegName(), dst.getType(),
1366 m_model.platform));
1367 }
1368 break;
1369 case Operand::Kind::MACRO:
1370 encodeDstReg(dst.getDirRegName(), dst.getDirRegRef().regNum);
1371 GED_ENCODE(DstMathMacroExt, lowerSpecialAcc(dst.getMathMacroExt()));
1372 if (accessMode == GED_ACCESS_MODE_Align1 &&
1373 m_model.supportsAlign16ImplicitAcc()) {
1374 fatalT("Align1 dst math macro unsupported on this platform.");
1375 return;
1376 }
1377 break;
1378 case Operand::Kind::INDIRECT:
1379 GED_ENCODE(DstAddrMode, GED_ADDR_MODE_Indirect);
1380 GED_ENCODE(DstDataType, lowerDataType(dst.getType()));
1381 if (inst.getOpSpec().supportsSaturation()) {
1382 GED_ENCODE(Saturate, lowerSaturate(dst.getDstModifier()));
1383 }
1384
1385 GED_ENCODE(DstAddrImm, dst.getIndImmAddr());
1386 GED_ENCODE(DstAddrSubRegNum, dst.getIndAddrReg().subRegNum);
1387 break;
1388 default:
1389 IGA_ASSERT_FALSE("unsupported operand kind");
1390 break;
1391 }
1392
1393 if (accessMode == GED_ACCESS_MODE_Align1) {
1394 auto dstRgn = dst.getRegion();
1395 if (inst.getOpSpec().hasImplicitDstRegion(inst.isMacro())) {
1396 auto dstRgnImpl = inst.getOpSpec().implicitDstRegion(inst.isMacro());
1397 if (dstRgn != dstRgnImpl) {
1398 warningT("dst region should be ", ToSyntax(dstRgnImpl));
1399 }
1400 }
1401 GED_ENCODE(DstHorzStride, lowerRegionHorz(dstRgn.getHz()));
1402 }
1403 }
1404
1405 static void createChSelForCtxSavRst(GED_SWIZZLE *chSel, GED_SWIZZLE x,
1406 GED_SWIZZLE y) {
1407 // following IsaAsm rules here
1408 // reg.ab expands to reg.abbb
1409 chSel[0] = x;
1410 chSel[1] = chSel[2] = chSel[3] = y;
1411 }
1412
1413 void Encoder::encodeBranchSource(const Operand &src) {
1414 encodeSrcRegFile<SourceIndex::SRC0>(lowerRegFile(src.getDirRegName()));
1415 encodeSrcReg<SourceIndex::SRC0>(src.getDirRegName(),
1416 src.getDirRegRef().regNum);
1417 auto subReg =
1418 SubRegToBinaryOffset(src.getDirRegRef().subRegNum, src.getDirRegName(),
1419 Type::D, m_model.platform);
1420 encodeSrcSubRegNum<SourceIndex::SRC0>(subReg);
1421 }
1422
1423 template <SourceIndex S>
1424 void Encoder::encodeBasicSource(const Instruction &inst,
1425 const Operand &src,
1426 GED_ACCESS_MODE accessMode) {
1427 // setting the reg file must precede must precede setting the type in GED
1428 switch (src.getKind()) {
1429 case Operand::Kind::DIRECT:
1430 case Operand::Kind::MACRO:
1431 case Operand::Kind::INDIRECT:
1432 encodeSrcRegFile<S>(lowerRegFile(src.getDirRegName()));
1433 if (inst.getOpSpec().supportsSourceModifiers()) {
1434 encodeSrcModifier<S>(src.getSrcModifier());
1435 } else if (src.getSrcModifier() != SrcModifier::NONE) {
1436 // better be invalid in the IR if unsupported
1437 errorT("src", (int)S, " source modifier not supported (invalid IR)");
1438 }
1439 break;
1440 case Operand::Kind::IMMEDIATE:
1441 encodeSrcRegFile<S>(GED_REG_FILE_IMM);
1442 break;
1443 default:
1444 break;
1445 }
1446
1447 encodeSrcType<S>(src.getType());
1448
1449 switch (src.getKind()) {
1450 case Operand::Kind::DIRECT:
1451 case Operand::Kind::MACRO: {
1452 encodeSrcAddrMode<S>(GED_ADDR_MODE_Direct);
1453 if (src.getKind() == Operand::Kind::DIRECT) {
1454 if (isAlign16MathMacroRegisterCsrOperand(src)) {
1455 // BDW..SKL context save and restore of acc3...acc9
1456 // encode as acc2.####, ChSel will be changed in regioning code
1457 // recall acc2 is remapped to mme0
1458 encodeSrcReg<S>(RegName::ARF_MME, 0);
1459 } else {
1460 encodeSrcReg<S>(src.getDirRegName(), src.getDirRegRef().regNum);
1461 auto subReg = SubRegToBinaryOffset(src.getDirRegRef().subRegNum,
1462 src.getDirRegName(), src.getType(),
1463 m_model.platform);
1464 encodeSrcSubRegNum<S>(subReg);
1465 }
1466 } else { // (src.getKind() == Operand::Kind::MACRO)
1467 encodeSrcReg<S>(src.getDirRegName(), src.getDirRegRef().regNum);
1468 encodeSrcMathMacroReg<S>(src.getMathMacroExt());
1469 if (accessMode == GED_ACCESS_MODE_Align16) {
1470 // vertical stride has to be halved for 8B types
1471 if (src.getType() == Type::DF) {
1472 encodeSrcRegionVert<S>(Region::Vert::VT_2);
1473 } else {
1474 encodeSrcRegionVert<S>(Region::Vert::VT_4);
1475 }
1476 }
1477 }
1478 break;
1479 }
1480 case Operand::Kind::INDIRECT:
1481 encodeSrcAddrMode<S>(GED_ADDR_MODE_Indirect);
1482 encodeSrcAddrImm<S>(src.getIndImmAddr());
1483 encodeSrcAddrSubRegNum<S>(src.getIndAddrReg().subRegNum);
1484 break;
1485 case Operand::Kind::IMMEDIATE:
1486 encodeImmVal(src.getImmediateValue(), src.getType());
1487 break;
1488 default:
1489 // support mov label
1490 if (static_cast<int>(S) == 0 && inst.isMovWithLabel()) {
1491 GED_ENCODE(Src0RegFile, GED_REG_FILE_IMM);
1492 } else {
1493 fatalT("src", (int)S,
1494 ": unsupported source operand kind "
1495 "(malformed IR)");
1496 return;
1497 }
1498 break;
1499 }
1500
1501 // sets stuff found in all register accesses (not macros)
1502 // - region
1503 switch (src.getKind()) {
1504 case Operand::Kind::DIRECT:
1505 case Operand::Kind::INDIRECT:
1506 if (accessMode == GED_ACCESS_MODE_Align16) {
1507 // r13.0<4>.xyzw is the only supported ChEn
1508 // ^^^
1509 encodeSrcRegionVert<S>(Region::Vert::VT_4);
1510 GED_SWIZZLE chSel[4] = {GED_SWIZZLE_x, GED_SWIZZLE_y, GED_SWIZZLE_z,
1511 GED_SWIZZLE_w};
1512 if (isAlign16MathMacroRegisterCsrOperand(src)) {
1513 // context save and restore workaround on GEN8 and GEN9
1514 switch (src.getDirRegRef().regNum) {
1515 case 1: // acc2.yx = mme1 (acc3)
1516 createChSelForCtxSavRst(chSel, GED_SWIZZLE_y, GED_SWIZZLE_x);
1517 break;
1518 case 2: // acc2.zx = mme2 (acc4)
1519 createChSelForCtxSavRst(chSel, GED_SWIZZLE_z, GED_SWIZZLE_x);
1520 break;
1521 case 3: // acc2.wx = mme3 (acc5)
1522 createChSelForCtxSavRst(chSel, GED_SWIZZLE_w, GED_SWIZZLE_x);
1523 break;
1524 case 4: // acc2.xy = mme4 (acc6)
1525 createChSelForCtxSavRst(chSel, GED_SWIZZLE_x, GED_SWIZZLE_y);
1526 break;
1527 case 5: // acc2.yy = mme5 (acc7)
1528 createChSelForCtxSavRst(chSel, GED_SWIZZLE_y, GED_SWIZZLE_y);
1529 break;
1530 case 6: // acc2.zy = mme6 (acc8)
1531 createChSelForCtxSavRst(chSel, GED_SWIZZLE_z, GED_SWIZZLE_y);
1532 break;
1533 case 7: // acc2.wy = mme7 (acc9)
1534 createChSelForCtxSavRst(chSel, GED_SWIZZLE_w, GED_SWIZZLE_y);
1535 break;
1536 }
1537 } else {
1538 // normal Align16 that we are converting to Align1
1539 if (src.getRegion() != Region::SRC110 &&
1540 // supports legacy bits that may use <K;K,1> for "block"
1541 // access; this allows us to assemble/reassemble similar bits
1542 src.getRegion() != Region::SRC221 &&
1543 src.getRegion() != Region::SRC441 &&
1544 src.getRegion() != Region::SRC881 &&
1545 src.getRegion() != Region::SRCFF1) {
1546 fatalT("src", (int)S,
1547 ": unsupported region for "
1548 "translation to align16 encoding");
1549 return;
1550 }
1551 // TODO: we could permit SIMD4 with .x to mean broadcast read
1552 // of subreg 0, but I don't think any System Routine code uses
1553 // this.
1554 //
1555 // NOTE: technically we could convert
1556 // r13.0<0>.xxxx to r13.0<0;1,0>
1557 // r13.0<0>.yyyy to r13.1<0;1,0>
1558 // r13.0<0>.zzzz to r13.2<0;1,0>
1559 // r13.0<0>.wwww to r13.3<0;1,0>
1560 // Also be sure to handle stuff like:
1561 // r13.4<0>.zzzz (would be r13.7<0;1,0>)
1562 //
1563 // Let's wait until we need this though.
1564 }
1565 encodeSrcChanSel<S>(chSel[0], chSel[1], chSel[2], chSel[3]);
1566 } else { // Align1
1567 if (m_model.srcHasReducedRegion(static_cast<uint32_t>(S)))
1568 encodeSrcReducedRegion<S>(src.getRegion());
1569 else
1570 encodeSrcRegion<S>(src.getRegion(), true);
1571 }
1572 break;
1573 case Operand::Kind::MACRO:
1574 if (accessMode == GED_ACCESS_MODE_Align1) {
1575 if (m_model.srcHasReducedRegion(static_cast<uint32_t>(S)))
1576 encodeSrcReducedRegion<S>(src.getRegion());
1577 else
1578 encodeSrcRegion<S>(src.getRegion());
1579 } // else {align16 macros use the regioning bits, don't clobber them}
1580 break;
1581 default:
1582 break;
1583 }
1584 }
1585
1586 void Encoder::encodeSendDirectDestination(const Operand &dst) {
1587 if (platform() >= Platform::XE) {
1588 // auto t = dst.getType() == Type::INVALID ? Type::UD : dst.getType();
1589 // GED_ENCODE(DstDataType, lowerDataType(t));
1590 GED_ENCODE(DstRegNum, dst.getDirRegRef().regNum);
1591 } else {
1592 auto t = dst.getType() == Type::INVALID ? Type::UD : dst.getType();
1593 GED_ENCODE(DstDataType, lowerDataType(t));
1594
1595 // GED_ENCODE(Saturate, lowerSaturate(dst->getDstModifier()));
1596 if (m_opcode != Op::SENDS && m_opcode != Op::SENDSC) {
1597 GED_ENCODE(
1598 DstHorzStride,
1599 static_cast<uint32_t>(dst.getRegion().getHz())); // not used for sends
1600 }
1601
1602 GED_ENCODE(DstRegNum, dst.getDirRegRef().regNum);
1603 // GED_ENCODE(DstSubRegNum,
1604 // SubRegToBinaryOffset(dst.getDirRegRef().subRegNum, RegName::GRF_R,
1605 // dst.getType(), m_model.platform));
1606 }
1607 }
1608
1609 void Encoder::encodeSendDestinationDataType(const Operand &dst) {
1610 if (platform() >= Platform::XE)
1611 return;
1612
1613 auto t = dst.getType() == Type::INVALID ? Type::UD : dst.getType();
1614 GED_ENCODE(DstDataType, lowerDataType(t));
1615 }
1616
1617 void Encoder::encodeSendDestination(const Operand &dst) {
1618 if (platform() > Platform::XE3) {
1619 if (m_opcode == Op::SENDGX || m_opcode == Op::SENDGXC) {
1620 encodeSendgxDestinationXe3p(dst);
1621 return;
1622 }
1623 encodeSendgDestinationXe3p(dst);
1624 return;
1625 }
1626 if (m_model.supportsUnarySend()) {
1627 switch (dst.getKind()) {
1628 case Operand::Kind::DIRECT:
1629 GED_ENCODE(DstAddrMode, GED_ADDR_MODE_Direct);
1630 break;
1631 case Operand::Kind::INDIRECT:
1632 GED_ENCODE(DstAddrMode, GED_ADDR_MODE_Indirect);
1633 break;
1634 default:
1635 fatalT("dst: unsupported destination operand kind/addrMode "
1636 "(malformed IR)");
1637 return;
1638 }
1639 }
1640
1641 GED_ENCODE(DstRegFile, lowerRegFile(dst.getDirRegName()));
1642
1643 if (dst.getKind() == Operand::Kind::DIRECT) {
1644 encodeSendDirectDestination(dst);
1645 } else if (dst.getKind() == Operand::Kind::INDIRECT) {
1646 encodeSendDestinationDataType(dst);
1647 if (m_opcode != Op::SENDS && m_opcode != Op::SENDSC) {
1648 GED_ENCODE(
1649 DstHorzStride,
1650 static_cast<uint32_t>(dst.getRegion().getHz())); // not used for sends
1651 }
1652 GED_ENCODE(DstAddrImm, dst.getIndImmAddr());
1653 GED_ENCODE(DstAddrSubRegNum, dst.getIndAddrReg().subRegNum);
1654 }
1655 }
1656 void Encoder::encodeSendgxDestinationXe3p(const Operand& dst) {
1657 if (dst.getKind() != Operand::Kind::DIRECT) {
1658 errorT("invalid dst operand kind");
1659 }
1660 if (dst.getDirRegName() == RegName::ARF_NULL) {
1661 GED_ENCODE(DstRegNum, (uint16_t)511);
1662 } else {
1663 GED_ENCODE(DstRegNum, dst.getDirRegRef().regNum);
1664 }
1665 }
1666 void Encoder::encodeSendgDestinationXe3p(const Operand &dst) {
1667 if (dst.getKind() != Operand::Kind::DIRECT) {
1668 errorT("invalid dst operand kind");
1669 }
1670 if (dst.isNull()) {
1671 GED_ENCODE(DstRegFile, GED_REG_FILE_ARF);
1672 } else {
1673 GED_ENCODE(DstRegFile, GED_REG_FILE_GRF);
1674 GED_ENCODE(DstRegNum, dst.getDirRegRef().regNum);
1675 }
1676 }
1677 void Encoder::encodeSendgxSource1Xe3p(const Operand& src) {
1678 if (src.getDirRegName() == RegName::ARF_NULL) {
1679 GED_ENCODE(Src1RegNum, (uint16_t)511);
1680 }
1681 else {
1682 GED_ENCODE(Src1RegNum, src.getDirRegRef().regNum);
1683 }
1684 }
1685
1686 void Encoder::encodeSendgxSource0Xe3p(const Operand& src) {
1687 if (src.getDirRegName() == RegName::ARF_NULL) {
1688 GED_ENCODE(Src0RegNum, (uint16_t)511);
1689 }
1690 else {
1691 GED_ENCODE(Src0RegNum, src.getDirRegRef().regNum);
1692 }
1693 }
1694
1695 void Encoder::encodeSendSource0(const Operand &src) {
1696 if (m_model.supportsUnarySend()) {
1697 switch (src.getKind()) {
1698 case Operand::Kind::DIRECT:
1699 GED_ENCODE(Src0AddrMode, GED_ADDR_MODE_Direct);
1700 break;
1701 case Operand::Kind::INDIRECT:
1702 GED_ENCODE(Src0AddrMode, GED_ADDR_MODE_Indirect);
1703 break;
1704 default:
1705 fatalT("src0: unsupported source operand kind/addrMode "
1706 "(malformed IR)");
1707 return;
1708 }
1709 }
1710
1711 GED_REG_FILE gedRegFile = lowerRegFile(src.getDirRegName());
1712 // for sendgx, src0 reg file is repurposed for storing high bit of reg num
1713 if (m_opcode == Op::SENDGX || m_opcode == Op::SENDGXC) {
1714 return encodeSendgxSource0Xe3p(src);
1715 }
1716 GED_ENCODE(Src0RegFile, gedRegFile);
1717
1718 auto t = src.getType() == Type::INVALID ? Type::UD : src.getType();
1719
1720 if (src.getKind() == Operand::Kind::DIRECT) {
1721 if (m_model.supportsXeSend()) {
1722 GED_ENCODE(Src0RegNum, src.getDirRegRef().regNum);
1723 } else {
1724 GED_ENCODE(Src0DataType, lowerDataType(t));
1725 GED_ENCODE(Src0RegNum, src.getDirRegRef().regNum);
1726 GED_ENCODE(Src0SubRegNum, src.getDirRegRef().subRegNum);
1727 }
1728 } else if (src.getKind() == Operand::Kind::INDIRECT) {
1729 // Gather Send src0 operand
1730 if (src.getDirRegName() == RegName::ARF_S) {
1731 // Though src0 is indirect register access, its reg number and subreg
1732 // number can be get from "DirRegRef" since it is not a0 access
1733 encodeSrcReg<SourceIndex::SRC0>(src.getDirRegName(),
1734 src.getDirRegRef().regNum);
1735 auto subReg = src.getDirRegRef().subRegNum;
1736 if (subReg % 2)
1737 errorT("src0 subreg must be 16b aligned");
1738 GED_ENCODE(Src0SubRegNum, subReg);
1739 } else {
1740 // legacy send indirect src operand
1741 GED_ENCODE(Src0DataType, lowerDataType(t));
1742 GED_ENCODE(Src0AddrSubRegNum, src.getIndAddrReg().subRegNum);
1743 // For platform >= XeHPC, the ImmAddr is represented in Word Offset in
1744 // binary,
1745 // platform < XeHPC, the ImmAddr is represented in Byte Offset in
1746 // binary
1747 // And for all platforms, the ImmAddr is represented in Byte Offset in
1748 // assembly syntax
1749 if (platform() >= Platform::XE_HPC) {
1750 GED_ENCODE(Src0AddrImm, src.getIndImmAddr() / 2);
1751 } else {
1752 GED_ENCODE(Src0AddrImm, src.getIndImmAddr());
1753 }
1754 }
1755 }
1756 }
1757
1758 // The sends opCode exists on gen9+. There is no sends opcode on pre-gen9.
1759 // Starting from XE, send opcode can have two sources, so the sends opcode
1760 // is not needed.
1761
1762 void Encoder::encodeSendsSource0(const Operand &src) {
1763 // "...for sends/sendsc instructions Src0.SrcMod, ... and Src0.SrcType are not
1764 // used." "Src0.RegFile[1], Src1.RegFile[1] are implicitly set to 0,
1765 // and Src0.RegFile[0] is implicitly set as 1 for sends/sendsc instructions."
1766 switch (src.getKind()) {
1767 case Operand::Kind::DIRECT:
1768 GED_ENCODE(Src0AddrMode, GED_ADDR_MODE_Direct);
1769 break;
1770 case Operand::Kind::INDIRECT:
1771 GED_ENCODE(Src0AddrMode, GED_ADDR_MODE_Indirect);
1772 break;
1773 default:
1774 fatalT("src0: unsupported source operand kind/addrMode (malformed IR)");
1775 return;
1776 break;
1777 }
1778
1779 if (src.getKind() == Operand::Kind::DIRECT) {
1780 GED_ENCODE(Src0RegNum, src.getDirRegRef().regNum);
1781 GED_ENCODE(Src0SubRegNum, src.getDirRegRef().subRegNum);
1782 } else if (src.getKind() == Operand::Kind::INDIRECT) {
1783 auto immAddr = src.getIndImmAddr();
1784 // For platforms >= XeHPC, ImmAddr is encoded as words,
1785 // platforms < XeHPC, ImmAddr is encoded as bytes
1786 // For all platforms, ImmAddr is represented in Byte Offset in syntax
1787 if (platform() >= Platform::XE_HPC) {
1788 immAddr /= 2;
1789 }
1790 GED_ENCODE(Src0AddrImm, immAddr);
1791 GED_ENCODE(Src0AddrSubRegNum, src.getIndAddrReg().subRegNum);
1792 }
1793 }
1794
1795 void Encoder::encodeSendsSource1(const Operand &src) {
1796 // GED_ENCODE(Src1AddrMode, GED_ADDR_MODE_Direct);
1797 GED_REG_FILE gedRegFile = lowerRegFile(src.getDirRegName());
1798 if (m_opcode == Op::SENDGX || m_opcode == Op::SENDGXC) {
1799 // for sendgx, src1 reg file is repurposed for storing high bit of reg num
1800 return encodeSendgxSource1Xe3p(src);
1801 }
1802 GED_ENCODE(Src1RegFile, gedRegFile);
1803 if (gedRegFile == GED_REG_FILE_GRF)
1804 GED_ENCODE(Src1RegNum, src.getDirRegRef().regNum);
1805 }
1806
1807 void Encoder::encodeSendsDestination(const Operand &dst) {
1808 GED_ENCODE(DstAddrMode, GED_ADDR_MODE_Direct);
1809 GED_ENCODE(DstRegFile, lowerRegFile(dst.getDirRegName()));
1810 // send types use :ud where possible
1811 auto t = dst.getType() == Type::INVALID ? Type::UD : dst.getType();
1812 GED_ENCODE(DstDataType, lowerDataType(t));
1813
1814 // GED_ENCODE(Saturate, lowerSaturate(dst->getDstModifier()));
1815 // GED_ENCODE(DstHorzStride, static_cast<uint32_t>(dst->getHz()));
1816
1817 GED_ENCODE(DstRegNum, dst.getDirRegRef().regNum);
1818 // TODO: set correct regType
1819 GED_ENCODE(DstSubRegNum,
1820 SubRegToBinaryOffset(dst.getDirRegRef().subRegNum, RegName::GRF_R,
1821 dst.getType(), m_model.platform));
1822 }
1823
1824 template <SourceIndex S>
1825 void Encoder::encodeTernarySourceAlign16(const Instruction &inst) {
1826 // PreCNL Align16
1827 // GRF-only
1828 encodeSrcAddrMode<S>(GED_ADDR_MODE_Direct);
1829
1830 const Operand &src = inst.getSource(S);
1831
1832 if (inst.getOpSpec().supportsSourceModifiers()) {
1833 encodeSrcModifier<S>(src.getSrcModifier());
1834 }
1835
1836 // set the data type
1837 GED_DATA_TYPE gedType = lowerDataType(src.getType());
1838 if (S == SourceIndex::SRC0) {
1839 GED_ENCODE(SrcDataType, gedType);
1840 } else {
1841 const Operand &src0 = inst.getSource(SourceIndex::SRC0);
1842 bool src0IsFloating =
1843 src0.getType() == Type::F || src0.getType() == Type::HF;
1844 if (platform() >= Platform::GEN8LP && src0IsFloating) {
1845 bool srcNIsFloating =
1846 src.getType() == Type::F || src.getType() == Type::HF;
1847 if (src0IsFloating && srcNIsFloating) {
1848 encodeSrcType<S>(src.getType());
1849 } else {
1850 fatalT("src", (int)S,
1851 ": mixed types require :f and :hf "
1852 "(or vice versa)");
1853 return;
1854 }
1855 }
1856 }
1857
1858 if (!inst.isMacro()) {
1859 const Region &rgn = src.getRegion();
1860 const RegRef &reg = src.getDirRegRef();
1861 // Adjusting sub register when going from align1 to align16 representation.
1862 // in align 16 subregister is always 16 byte alligned, but we can play
1863 // with swizzle to access none aligned sub register
1864 uint16_t subRegNumber = reg.subRegNum;
1865 // mad (8) r46.0.xyzw:df r46.0.xyzw:df r50.0.xyzw:df r48.0.xyzw:df {Align16,
1866 // Q1} mad (2) r5.0.xy:df r5.0.xyxy:df r92.2.xyxy:df r93.0.xyxy:df {Align16,
1867 // Q1, NoMask} // BDW,SKL
1868 if (S != SourceIndex::SRC2) {
1869 if (rgn == Region::SRC8X1 || rgn == Region::SRC4X1 ||
1870 rgn == Region::SRC2X1) {
1871 encodeSrcRepCtrl<S>(GED_REP_CTRL_NoRep);
1872 encodeSrcChanSel<S>(GED_SWIZZLE_x, GED_SWIZZLE_y, GED_SWIZZLE_z,
1873 GED_SWIZZLE_w);
1874 } else if (rgn == Region::SRC0X0) {
1875 if (src.getType() == Type::DF) {
1876 if (reg.subRegNum % 2 == 0) {
1877 encodeSrcChanSel<S>(GED_SWIZZLE_x, GED_SWIZZLE_y, GED_SWIZZLE_x,
1878 GED_SWIZZLE_y);
1879 } else {
1880 encodeSrcChanSel<S>(GED_SWIZZLE_z, GED_SWIZZLE_w, GED_SWIZZLE_z,
1881 GED_SWIZZLE_w);
1882 subRegNumber -= 1;
1883 }
1884 } else {
1885 encodeSrcRepCtrl<S>(GED_REP_CTRL_Rep);
1886 }
1887 } else {
1888 fatalT("src", (int)S, ": unsupported region for Align16 encoding");
1889 return;
1890 }
1891 } else {
1892 if (rgn == Region::SRCXX1) {
1893 encodeSrcRepCtrl<S>(GED_REP_CTRL_NoRep);
1894 encodeSrcChanSel<S>(GED_SWIZZLE_x, GED_SWIZZLE_y, GED_SWIZZLE_z,
1895 GED_SWIZZLE_w);
1896 } else if (rgn == Region::SRCXX0) {
1897 if (src.getType() == Type::DF) {
1898 if (src.getDirRegRef().subRegNum % 2 == 0) {
1899 encodeSrcChanSel<S>(GED_SWIZZLE_x, GED_SWIZZLE_y, GED_SWIZZLE_x,
1900 GED_SWIZZLE_y);
1901 } else {
1902 encodeSrcChanSel<S>(GED_SWIZZLE_z, GED_SWIZZLE_w, GED_SWIZZLE_z,
1903 GED_SWIZZLE_w);
1904 subRegNumber -= 1;
1905 }
1906 } else {
1907 encodeSrcRepCtrl<S>(GED_REP_CTRL_Rep);
1908 }
1909 } else if (rgn == Region::SRC0X0 && src.getType() == Type::DF) {
1910 encodeSrcChanSel<S>(GED_SWIZZLE_x, GED_SWIZZLE_y, GED_SWIZZLE_x,
1911 GED_SWIZZLE_y);
1912 } else {
1913 fatalT("src", (int)S, ": unsupported region for Align16 encoding");
1914 return;
1915 }
1916 }
1917 uint32_t regNum = reg.regNum;
1918 encodeSrcReg<S>(RegName::GRF_R, (uint16_t)regNum);
1919 auto subReg = SubRegToBinaryOffset(subRegNumber, src.getDirRegName(),
1920 src.getType(), m_model.platform);
1921 encodeSrcSubRegNum<S>(subReg);
1922 } else {
1923 // implicit operand accumulator
1924 // e.g. madm (4) ... -r14.acc3
1925 encodeSrcReg<S>(RegName::GRF_R, src.getDirRegRef().regNum);
1926 encodeSrcMathMacroReg<S>(src.getMathMacroExt());
1927 }
1928 }
1929
1930 void Encoder::encodeTernaryDestinationAlign16(const Instruction &inst) {
1931 const Operand &dst = inst.getDestination();
1932 if (inst.getOpSpec().supportsSaturation()) {
1933 GED_ENCODE(Saturate, lowerSaturate(dst.getDstModifier()));
1934 }
1935 GED_ENCODE(DstDataType, lowerDataType(dst.getType()));
1936 if (dst.getDirRegName() != RegName::GRF_R) {
1937 fatalT("align16 ternary dst must be to GRF");
1938 return;
1939 }
1940
1941 // register / info (must be GRF)
1942 GED_ENCODE(DstRegFile, lowerRegFile(dst.getDirRegName()));
1943 uint32_t regNum = dst.getDirRegRef().regNum;
1944 GED_ENCODE(DstRegNum, regNum);
1945 if (inst.isMacro()) {
1946 // macro only
1947 GED_DST_CHAN_EN chanEn = mathMacroRegToChEn(dst.getMathMacroExt());
1948 GED_ENCODE(DstChanEn, chanEn);
1949 } else {
1950 // Align16 instruction (we must convert from Align1)
1951 //
1952 // As long as the Align1 sequences are packed (.xyzw), this is
1953 // straightforward. However, "scalar" (braoadcast) sequences are
1954 // a bit harder as we must carefully choose the ChEn based on the
1955 // subregister that would be used in Align1
1956 // (See also Decoder::decodeDestinationTernaryAlign16)
1957 GED_DST_CHAN_EN chanEn = GED_DST_CHAN_EN_xyzw;
1958 auto reg = dst.getDirRegRef();
1959 if (inst.getExecSize() == ExecSize::SIMD1) {
1960 // SIMD1 MAD is not allowed, so MDF (and IGC) are generating use
1961 // SIMD4 and SIMD2 with specific channel masks to selectively
1962 // enable just the bottom channel.
1963 if (dst.getType() == Type::DF) {
1964 // For 64-bit types we use a mad (2) ...
1965 // Note, only :df is needed since :q and :uq are not supported
1966 //
1967 // e.g. mad (2) r5.0.xy:df ... {Align16, Q1, NoMask} //
1968 if (dst.getDirRegRef().subRegNum % 2 == 0) {
1969 chanEn = GED_DST_CHAN_EN_xy;
1970 } else {
1971 // e.g. mad (1) r5.1<1>:df
1972 // encodes as
1973 // mad (2) r5.0.zw:df
1974 // ^ SIMD2 and .zw (~= .1)
1975 chanEn = GED_DST_CHAN_EN_zw;
1976 reg.subRegNum -= 1;
1977 }
1978 } else {
1979 // 32-bit or 16-bit type (:hf). We use a SIMD4
1980 //
1981 // one channel enabled. E.g. we'll parse
1982 // mad (1|M0) r53.6<1>:f ...
1983 // and encode it as
1984 // mad (4) r53.4.z:f
1985 // ^ SIMD4 ^^^ aligned subreg .4.z == subreg .6:f Align1
1986 switch (reg.subRegNum % 4) {
1987 case 0:
1988 chanEn = GED_DST_CHAN_EN_x;
1989 break;
1990 case 1:
1991 chanEn = GED_DST_CHAN_EN_y;
1992 break;
1993 case 2:
1994 chanEn = GED_DST_CHAN_EN_z;
1995 break;
1996 case 3:
1997 chanEn = GED_DST_CHAN_EN_w;
1998 break;
1999 }
2000 // align the subregister
2001 reg.subRegNum -= (reg.subRegNum % 4);
2002 }
2003 }
2004 GED_ENCODE(DstChanEn, chanEn);
2005 GED_ENCODE(DstSubRegNum,
2006 SubRegToBinaryOffset(reg.subRegNum, dst.getDirRegName(),
2007 dst.getType(), m_model.platform));
2008 }
2009 }
2010
2011 void Encoder::encodeDstReg(RegName regName, uint16_t regNum) {
2012 // encodes ARF or GRF
2013 uint32_t gedBits = translateRegNum(-1, regName, regNum);
2014 GED_ENCODE(DstRegNum, gedBits);
2015 }
2016
2017 void Encoder::encodeImmVal(const ImmVal &val, Type type) {
2018 GED_ENCODE(Imm, typeConvesionHelper(val, type));
2019 }
2020
2021 template <SourceIndex S> void Encoder::encodeSrcRepCtrl(GED_REP_CTRL rep) {
2022 if (S == SourceIndex::SRC0) {
2023 GED_ENCODE(Src0RepCtrl, rep);
2024 } else if (S == SourceIndex::SRC1) {
2025 GED_ENCODE(Src1RepCtrl, rep);
2026 } else {
2027 GED_ENCODE(Src2RepCtrl, rep);
2028 }
2029 }
2030
2031 template <SourceIndex S>
2032 void Encoder::encodeSrcChanSel(GED_SWIZZLE chSelX, GED_SWIZZLE chSelY,
2033 GED_SWIZZLE chSelZ, GED_SWIZZLE chSelW) {
2034 uint32_t chSelBits = createChanSel(chSelX, chSelY, chSelZ, chSelW);
2035 if (S == SourceIndex::SRC0) {
2036 GED_ENCODE(Src0ChanSel, chSelBits);
2037 } else if (S == SourceIndex::SRC1) {
2038 GED_ENCODE(Src1ChanSel, chSelBits);
2039 } else {
2040 GED_ENCODE(Src2ChanSel, chSelBits);
2041 }
2042 }
2043
2044 uint32_t Encoder::translateRegNum(int opIx, RegName regName, uint16_t regNum) {
2045 const char *whichOp = opIx == 0 ? "src0"
2046 : opIx == 1 ? "src1"
2047 : opIx == 2 ? "src2"
2048 : "dst";
2049
2050 const RegInfo *ri = m_model.lookupRegInfoByRegName(regName);
2051 if (ri == nullptr) {
2052 errorT(whichOp, ": invalid register name for this platform");
2053 } else if (!ri->isRegNumberValid((int)regNum)) {
2054 errorT(whichOp, ": ", ri->syntax, regNum, " number out of range");
2055 } else if (regName != RegName::GRF_R) {
2056 return ri->encodeARFRegNum((int)regNum);
2057 }
2058 return regNum; // widen for GED
2059 }
2060
2061 uint32_t Encoder::mathMacroRegToBits(int src, MathMacroExt implAcc) {
2062 uint32_t bits = 8; // NOACC
2063 switch (implAcc) {
2064 /// or 00000000b (GEN11)
2065 case MathMacroExt::MME0:
2066 bits = 0;
2067 break; // 0000b
2068 case MathMacroExt::MME1:
2069 bits = 1;
2070 break;
2071 case MathMacroExt::MME2:
2072 bits = 2;
2073 break;
2074 case MathMacroExt::MME3:
2075 bits = 3;
2076 break;
2077 case MathMacroExt::MME4:
2078 bits = 4;
2079 break;
2080 case MathMacroExt::MME5:
2081 bits = 5;
2082 break;
2083 case MathMacroExt::MME6:
2084 bits = 6;
2085 break;
2086 case MathMacroExt::MME7:
2087 bits = 7;
2088 break;
2089 /// or 00008000b (GEN11)
2090 case MathMacroExt::NOMME:
2091 bits = 8;
2092 break; // 1000b
2093 default:
2094 if (src < 0) {
2095 fatalT("dst operand has invalid math macro register");
2096 } else {
2097 fatalT("src", src, " operand has invalid math macro register");
2098 }
2099 return bits;
2100 }
2101 return bits;
2102 }
2103 GED_DST_CHAN_EN Encoder::mathMacroRegToChEn(MathMacroExt implAcc) {
2104 GED_DST_CHAN_EN bits = GED_DST_CHAN_EN_w; // NOACC
2105 switch (implAcc) {
2106 case MathMacroExt::MME0:
2107 bits = GED_DST_CHAN_EN_None;
2108 break; // 0000b
2109 case MathMacroExt::MME1:
2110 bits = GED_DST_CHAN_EN_x;
2111 break;
2112 case MathMacroExt::MME2:
2113 bits = GED_DST_CHAN_EN_y;
2114 break;
2115 case MathMacroExt::MME3:
2116 bits = GED_DST_CHAN_EN_xy;
2117 break;
2118 case MathMacroExt::MME4:
2119 bits = GED_DST_CHAN_EN_z;
2120 break; // 0100b
2121 case MathMacroExt::MME5:
2122 bits = GED_DST_CHAN_EN_xz;
2123 break;
2124 case MathMacroExt::MME6:
2125 bits = GED_DST_CHAN_EN_yz;
2126 break;
2127 case MathMacroExt::MME7:
2128 bits = GED_DST_CHAN_EN_xyz;
2129 break;
2130 case MathMacroExt::NOMME:
2131 bits = GED_DST_CHAN_EN_w;
2132 break; // 1000b
2133 default:
2134 fatalT("operand has invalid math macro register");
2135 }
2136 return bits;
2137 }
2138
2139 void Encoder::encodeOptionsThreadControl(const Instruction &inst) {
2140 if (inst.hasInstOpt(InstOpt::NOPREEMPT)) {
2141 if (m_model.supportsNoPreempt()) {
2142 GED_ENCODE(ThreadCtrl, GED_THREAD_CTRL_NoPreempt);
2143 } else {
2144 warningT("NoPreempt not supported on this platform (dropping)");
2145 }
2146 }
2147 }
2148
2149 void Encoder::encodeOptions(const Instruction &inst) {
2150 GED_ENCODE(DebugCtrl, inst.hasInstOpt(InstOpt::BREAKPOINT)
2151 ? GED_DEBUG_CTRL_Breakpoint
2152 : GED_DEBUG_CTRL_Normal);
2153
2154 auto &os = inst.getOpSpec();
2155 if (os.supportsDepCtrl()) {
2156 if (inst.hasInstOpt(InstOpt::NODDCHK) &&
2157 !inst.hasInstOpt(InstOpt::NODDCLR)) {
2158 GED_ENCODE(DepCtrl, GED_DEP_CTRL_NoDDChk);
2159 } else if (!inst.hasInstOpt(InstOpt::NODDCHK) &&
2160 inst.hasInstOpt(InstOpt::NODDCLR)) {
2161 GED_ENCODE(DepCtrl, GED_DEP_CTRL_NoDDClr);
2162 } else if (inst.hasInstOpt(InstOpt::NODDCHK) &&
2163 inst.hasInstOpt(InstOpt::NODDCLR)) {
2164 GED_ENCODE(DepCtrl, GED_DEP_CTRL_NoDDClr_NoDDChk);
2165 } else if (!inst.getOpSpec().isAnySendFormat() && inst.getOp() != Op::NOP) {
2166 GED_ENCODE(DepCtrl, GED_DEP_CTRL_Normal);
2167 }
2168 }
2169
2170 if (inst.hasInstOpt(InstOpt::ATOMIC)) {
2171 // this bit overlaps
2172 if (os.isAnySendFormat() && platform() >= Platform::XE3) {
2173 errorT("{Atomic} not available on sends for this platform");
2174 }
2175 GED_ENCODE(ThreadCtrl, GED_THREAD_CTRL_Atomic);
2176 }
2177
2178 if (inst.hasInstOpt(InstOpt::FWD)) {
2179 if (!os.supportsFwdCtrl())
2180 errorT("Invalid option {Fwd}");
2181 GED_ENCODE(FwdCtrl, 1);
2182 }
2183
2184 if (inst.hasInstOpt(InstOpt::SWITCH) && m_model.supportsHwDeps()) {
2185 if (inst.getOp() == Op::NOP) {
2186 warningT("nop doesn't support Switch option (dropping)");
2187 } else {
2188 GED_ENCODE(ThreadCtrl, GED_THREAD_CTRL_Switch);
2189 }
2190 }
2191 encodeOptionsThreadControl(inst);
2192
2193 if (!inst.hasInstOpt(InstOpt::ATOMIC) && !inst.hasInstOpt(InstOpt::SWITCH) &&
2194 !inst.hasInstOpt(InstOpt::NOPREEMPT) &&
2195 !inst.getOpSpec().isAnySendFormat() && inst.getOp() != Op::NOP) {
2196 GED_ENCODE(ThreadCtrl, GED_THREAD_CTRL_Normal);
2197 }
2198
2199 if (inst.hasInstOpt(InstOpt::NOSRCDEPSET)) {
2200 GED_ENCODE(NoSrcDepSet, GED_NO_SRC_DEP_SET_NoSrcDepSet);
2201 } else if (inst.getOpSpec().isAnySendFormat() &&
2202 m_model.supportNoSrcDepSet()) {
2203 GED_ENCODE(NoSrcDepSet, GED_NO_SRC_DEP_SET_Normal);
2204 }
2205
2206 if (platform() >= Platform::XE && m_opcode != Op::ILLEGAL &&
2207 m_opcode != Op::THRYLD) {
2208 SWSB::InstType inst_type = inst.getSWSBInstType(m_opts.swsbEncodeMode);
2209 uint32_t swsbBinary =
2210 inst.getSWSB().encode(m_opts.swsbEncodeMode, inst_type);
2211 IGA_ASSERT(inst.getSWSB().verify(m_opts.swsbEncodeMode, inst_type),
2212 "INTERNAL ERROR: invalid SWSB (parser/IR-creator should have "
2213 "prevented this)");
2214
2215 GED_ENCODE(SWSB, swsbBinary);
2216 }
2217 }
2218
2219 void Encoder::patchJumpOffsets() {
2220 for (JumpPatch &jp : m_needToPatch) {
2221 const Instruction *inst = jp.inst;
2222 IGA_ASSERT(inst->getOpSpec().isBranching() || inst->isMovWithLabel(),
2223 "patching non-control-flow/non-mov instruction");
2224
2225 // on some platforms jmpi os post-increment
2226 uint32_t jmpiExtraOffset = 0;
2227 bool isPostIncrementJmpi =
2228 inst->getOp() == Op::JMPI && !m_model.supportsSimplifiedBranches();
2229 if (isPostIncrementJmpi) {
2230 // jmpi is relative to the incremented PC, hence we must add
2231 // the size of the instruction here. jmpi probably will never
2232 // compact, but we'll be careful here
2233 jmpiExtraOffset = inst->hasInstOpt(InstOpt::COMPACTED) ? 8 : 16;
2234 IGA_ASSERT(inst->getSource(0).getKind() == Operand::Kind::LABEL,
2235 "patching non label op");
2236 // skip registers
2237 }
2238
2239 // calla and mov is an absolute offset
2240 uint32_t encodePC =
2241 (inst->getOpSpec().isJipAbsolute()) || (inst->getOp() == Op::MOV)
2242 ? 0
2243 : getEncodedPC(inst);
2244
2245 uint32_t jumpPC = 0;
2246 const Block *jipBlk = inst->getJIP();
2247 if (jipBlk == nullptr) {
2248 // immediate offset: we have to treat this as a relative offset
2249 jumpPC = inst->getSource(0).getImmediateValue().s32 + encodePC;
2250 } else if (!getBlockOffset(jipBlk, jumpPC)) {
2251 // For call, its target symbol may not be resolvable until in the
2252 // link stage when other kernels are available.
2253 if (inst->getOp() != Op::CALL && inst->getOp() != Op::CALLA) {
2254 fatalAtT(inst->getLoc(), "jip label invalid");
2255 }
2256 }
2257
2258 int32_t jip = jumpPC - encodePC - jmpiExtraOffset;
2259 // JIP and UIP are in QWORDS for most ops on PreBDW
2260 int32_t pcUnscale = arePcsInQWords(inst->getOpSpec()) ? 8 : 1;
2261
2262 if (inst->isMovWithLabel()) {
2263 // encode mov label
2264 GED_DATA_TYPE src0_ty = lowerDataType(inst->getSource(0).getType());
2265 GED_ENCODE_TO(Src0DataType, src0_ty, &jp.gedInst);
2266 GED_ENCODE_TO(Imm, jip, &jp.gedInst);
2267 } else {
2268 // encode other branch instructions
2269 GED_ENCODE_TO(JIP, jip / pcUnscale, &jp.gedInst);
2270 }
2271
2272 if (inst->getSourceCount() == 2 &&
2273 (inst->getOp() != Op::BRC || inst->getSource(1).isImm())) {
2274 // No need to set src1 regFile and type,
2275 // it will be over written by UIP
2276 const Block *uipBlk = inst->getUIP();
2277 if (uipBlk == nullptr) {
2278 jumpPC = inst->getSource(1).getImmediateValue().s32 + encodePC;
2279 } else if (!getBlockOffset(uipBlk, jumpPC)) {
2280 fatalAtT(inst->getLoc(), "uip label invalid");
2281 }
2282 encodePC = getEncodedPC(inst);
2283 int32_t uip = jumpPC - encodePC;
2284 GED_ENCODE_TO(UIP, uip / pcUnscale, &jp.gedInst);
2285 }
2286
2287 // re-encode branch
2288 START_GED_TIMER();
2289 GED_RETURN_VALUE status = GED_EncodeIns(&jp.gedInst,
2290 inst->hasInstOpt(InstOpt::COMPACTED)
2291 ? GED_INS_TYPE_COMPACT
2292 : GED_INS_TYPE_NATIVE,
2293 jp.bits);
2294 STOP_GED_TIMER();
2295 if (status != GED_RETURN_VALUE_SUCCESS) {
2296 fatalAtT(inst->getLoc(),
2297 "GED_EncodeIns failed: ", gedReturnValueToString(status));
2298 }
2299 }
2300 }
2301
2302 bool Encoder::arePcsInQWords(const OpSpec &os) const {
2303 // everything is in bytes except:
2304 // HSW calla, call, and jmpi
2305 return platform() < Platform::GEN8 && os.op != Op::JMPI &&
2306 os.op != Op::CALL && os.op != Op::CALLA;
2307 }
2308
2309 bool Encoder::callNeedsSrc0Region221(const Instruction &inst) const {
2310 // [call]: "Restriction: The src0 regioning control must be <2;2,1>"
2311 // [calla]: "Restriction: The src0 regioning control must be <2;2,1>"
2312 return (inst.getOp() == Op::CALL && platform() < Platform::GEN8) ||
2313 (inst.getOp() == Op::CALL && platform() == Platform::GEN9) ||
2314 (inst.getOp() == Op::CALLA && platform() <= Platform::GEN10);
2315 }
2316
2317 bool Encoder::callNeedsSrc0Region241(const Instruction &inst) const {
2318 return (inst.getOp() == Op::CALL && platform() == Platform::GEN11);
2319 }
2320
2321 void Encoder::encodeTernarySrcRegionVert(SourceIndex S, Region::Vert v) {
2322 if (S == SourceIndex::SRC0) {
2323 GED_ENCODE(Src0VertStride, lowerRegionVert(v));
2324 } else { // (S == SourceIndex::SRC1)
2325 GED_ENCODE(Src1VertStride, lowerRegionVert(v));
2326 } // S != SRC2 since ternary Align1 doesn't have bits for that
2327 }
2328
2329 // fixes stuff where GED just ignores or where it refuses to allow us to
2330 // set bits. This should be empty unless GED fixes are in flight.
2331 void Encoder::applyGedWorkarounds(const Kernel &, size_t) {
2332 // Also consider
2333 // GED_RETURN_VALUE GED_SetRawBits(ged_ins_t* ins, uint8_t low, uint8_t high,
2334 // const uint64_t value)
2335 }