Debian Codegraph @

visa/iga/IGALibrary/Backend/GED/Encoder.hpp

raw
1 /*========================== begin_copyright_notice ============================
2
3 Copyright (C) 2017-2021 Intel Corporation
4
5 SPDX-License-Identifier: MIT
6
7 ============================= end_copyright_notice ===========================*/
8
9 #ifndef _IGA_BACKEND_GED_ENCODER_H_
10 #define _IGA_BACKEND_GED_ENCODER_H_
11
12 #include "../../ErrorHandler.hpp"
13 #include "../../IR/Block.hpp"
14 #include "../../IR/Instruction.hpp"
15 #include "../../IR/Kernel.hpp"
16 #include "../../Timer/Timer.hpp"
17 #include "../EncoderOpts.hpp"
18 #include "GEDBitProcessor.hpp"
19 #include "IGAToGEDTranslation.hpp"
20
21 #include <list>
22 #include <map>
23 #include <vector>
24
25 namespace iga {
26 #define GED_ENCODE(FUNC, ARG) GED_ENCODE_TO(FUNC, ARG, &m_gedInst)
27 #if defined(GED_TIMER) || defined(_DEBUG)
28 #define START_GED_TIMER() startIGATimer(TIMER_GED)
29 #define STOP_GED_TIMER() stopIGATimer(TIMER_GED)
30 #else
31 #define START_GED_TIMER()
32 #define STOP_GED_TIMER()
33 #endif
34
35 #if defined(TOTAL_ENCODE_TIMER) || defined(_DEBUG)
36 #define START_ENCODER_TIMER() startIGATimer(TIMER_TOTAL)
37 #define STOP_ENCODER_TIMER() stopIGATimer(TIMER_TOTAL)
38 #else
39 #define START_ENCODER_TIMER()
40 #define STOP_ENCODER_TIMER()
41 #endif
42
43 // uncomment to emit all the GED calls
44 // #define TRACE_GED_CALLS
45
46 #ifdef TRACE_GED_CALLS
47 #define TRACE_GED_SETTER(FIELD, ARG, STATUS) \
48 std::cerr << "Encoder.cpp:" << __LINE__ << ": GED_Set" << #FIELD << "(..," \
49 << (ARG) << ") results in " << (STATUS) << "\n"
50 #else
51 #define TRACE_GED_SETTER(FIELD, ARG, STATUS)
52 #endif
53
54 #define GED_ENCODE_TO(FIELD, ARG, GED) \
55 do { \
56 GED_RETURN_VALUE _status; \
57 START_GED_TIMER(); \
58 _status = GED_Set##FIELD(GED, ARG); \
59 STOP_GED_TIMER(); \
60 TRACE_GED_SETTER(FIELD, ARG, _status); \
61 if (_status != GED_RETURN_VALUE_SUCCESS) { \
62 handleGedError(__LINE__, #FIELD, _status); \
63 } \
64 } while (0)
65
66 using BlockList = std::list<Block *, std_arena_based_allocator<Block *>>;
67
68 class Encoder : protected GEDBitProcessor {
69 public:
70 Encoder(const Model &model, ErrorHandler &eh, const EncoderOpts &eos);
71
72 void encodeKernel(Kernel &k, MemManager &m, void *&bits, uint32_t &bitsLen);
73
74 size_t getNumInstructionsEncoded() const;
75
76 ///////////////////////////////////////////////////////////////////////
77 // PROFILING API FOR TESTING (must be compiled into)
78 //
79 // compile with MEASURE_COMPILATION_TIME
80 double getElapsedTimeMicros(unsigned int idx);
81 int64_t getElapsedTimeTicks(unsigned int idx);
82 std::string getTimerName(unsigned int idx);
83
84 protected:
85 // TODO: phase these out
86 void encodeKernelPreProcess(Kernel &k);
87 void doEncodeKernelPreProcess(Kernel &k);
88 void encodeFC(const Instruction &i);
89 void encodeTernaryInstruction(const Instruction &inst,
90 GED_ACCESS_MODE accessMode);
91 void encodeTernaryAlign1Instruction(const Instruction &inst);
92 void encodeTernaryAlign16Instruction(const Instruction &inst);
93 void encodeQuinaryInstruction(const Instruction &inst);
94 void encodeSendDirectDestination(const Operand &dst);
95 void encodeSendDestinationDataType(const Operand &dst);
96 void encodeOptionsThreadControl(const Instruction &inst);
97
98 protected:
99 void encodeDstReg(RegName regName, uint16_t regNum); // just the regnum
100
101 void encodeImmVal(const ImmVal &val, Type type);
102
103 template <SourceIndex S> void encodeSrcRegFile(GED_REG_FILE rf);
104 template <SourceIndex S> void encodeSrcRegionVert(Region::Vert v);
105 template <SourceIndex S> void encodeSrcType(Type t);
106 template <SourceIndex S> void encodeSrcAddrMode(GED_ADDR_MODE x);
107 template <SourceIndex S> void encodeSrcModifier(SrcModifier x);
108 template <SourceIndex S> void encodeSrcSubRegNum(uint32_t subRegInByte);
109
110 template <SourceIndex S> void encodeSrcMathMacroReg(MathMacroExt a);
111
112 template <SourceIndex S> void encodeSrcReg(RegName regName, uint16_t regNum);
113 template <SourceIndex S> void encodeSrcAddrImm(int32_t addrImm);
114 template <SourceIndex S> void encodeSrcAddrSubRegNum(uint32_t addrSubReg);
115 template <SourceIndex S>
116 void encodeSrcRegion(const Region &r, bool hasRgnWi = true);
117 template <SourceIndex S> void encodeSrcRegionWidth(Region::Width w);
118 template <SourceIndex S> void encodeSrcReducedRegion(const Region& rgn);
119 template <SourceIndex S> void encodeSrcRepCtrl(GED_REP_CTRL rep);
120 template <SourceIndex S>
121 void encodeSrcChanSel(GED_SWIZZLE chSelX, GED_SWIZZLE chSelY,
122 GED_SWIZZLE chSelZ, GED_SWIZZLE chSelW);
123
124 template <SourceIndex S>
125 void encodeTernarySourceAlign16(const Instruction &inst);
126 void encodeTernaryDestinationAlign16(const Instruction &inst);
127
128 template <SourceIndex S>
129 void encodeTernarySourceAlign1(const Instruction &inst);
130 template <SourceIndex S>
131 void encodeQuinarySource(const Instruction &inst);
132 void encodeTernaryDestinationAlign1(const Instruction &inst);
133
134 void encodeTernarySrcRegionVert(SourceIndex S, Region::Vert v);
135
136 template <SourceIndex S>
137 void encodeTernaryImmVal(const ImmVal &val, Type type);
138
139 template <SourceIndex S> void encodeSrcRegionHorz(Region::Horz s);
140
141 void handleGedError(int line, const char *setter, GED_RETURN_VALUE status);
142
143 // state that is valid over instance life
144 EncoderOpts m_opts;
145
146 // state that is valid over encodeInst()
147 ged_ins_t m_gedInst;
148 bool m_encodeAlign16 = false;
149 Op m_opcode = Op::INVALID;
150 size_t m_numberInstructionsEncoded;
151
152 private:
153 void operator delete(void *, MemManager *){};
154 void *operator new(size_t sz, MemManager *m) { return m->alloc(sz); };
155
156 void encodeBlock(Kernel &k, Block *blk);
157 void encodeInstruction(Instruction &inst);
158 void patchJumpOffsets();
159
160 ///////////////////////////////////////////////////////////////////////
161 // BASIC INSTRUCTIONS
162 ///////////////////////////////////////////////////////////////////////
163 void encodeBasicInstruction(const Instruction &inst,
164 GED_ACCESS_MODE accessMode);
165 void
166 encodeBasicDestination(const Instruction &inst, const Operand &dst,
167 GED_ACCESS_MODE accessMode = GED_ACCESS_MODE_Align1);
168 template <SourceIndex S>
169 void encodeBasicSource(const Instruction &inst, const Operand &src,
170 GED_ACCESS_MODE accessMode = GED_ACCESS_MODE_Align1);
171 ///////////////////////////////////////////////////////////////////////
172 // BRANCH INSTRUCTIONS
173 ///////////////////////////////////////////////////////////////////////
174 void encodeBranchingInstruction(const Instruction &inst);
175 void encodeBranchingInstructionSimplified(const Instruction &inst);
176 void encodeBranchDestination(const Operand &dst);
177 void encodeBranchSource(const Operand &src);
178 ///////////////////////////////////////////////////////////////////////
179 // SEND INSTRUCTIONS
180 ///////////////////////////////////////////////////////////////////////
181 void encodeSendInstruction(const Instruction &inst);
182 void encodeSendSource0(const Operand &src);
183 void encodeSendDestination(const Operand &dst);
184 void encodeSendgDestinationXe3p(const Operand &dst);
185 void encodeSendgxDestinationXe3p(const Operand &dst);
186 void encodeSendgxSource0Xe3p(const Operand &src);
187 void encodeSendgxSource1Xe3p(const Operand &src);
188 void encodeSendsSource0(const Operand &src);
189 void encodeSendsSource1(const Operand &src);
190 void encodeSendsDestination(const Operand &dst);
191
192 void encodeSendDescs(const Instruction &inst);
193 void encodeSendDescsPreXe(const Instruction &inst);
194 void encodeSendDescsXe(const Instruction &inst);
195 void encodeSendDescsXeHP(const Instruction &inst);
196 void encodeSendDescsXeHPG(const Instruction &inst);
197 void encodeSendDescsXe2(const Instruction &inst);
198 void encodeSendgDescsXe3(const Instruction &inst);
199
200 void encodeInlineBinaryInst(Instruction &inst);
201 ///////////////////////////////////////////////////////////////////////
202 // SYNC INSTRUCTIONS
203 ///////////////////////////////////////////////////////////////////////
204 void encodeSyncInstruction(const Instruction &inst);
205
206 ///////////////////////////////////////////////////////////////////////
207 // OTHER HELPER FUNCTIONS
208 ///////////////////////////////////////////////////////////////////////
209 void setEncodedPC(Instruction *inst, int32_t encodePC);
210 int32_t getEncodedPC(const Instruction *inst) const;
211
212 bool getBlockOffset(const Block *b, uint32_t &pc);
213
214 // handles encoding ARF registers as well as the easy GRF case
215 // caller actually pulls the trigger and encodes the bits, but this
216 // call can raise the encoding error
217 uint32_t translateRegNum(int opIx, RegName reg, uint16_t regNum);
218 uint32_t mathMacroRegToBits(int src, MathMacroExt mme); // ChSel and SubReg
219 GED_DST_CHAN_EN mathMacroRegToChEn(MathMacroExt mme);
220
221 void applyGedWorkarounds(const Kernel &k, size_t bitsLen);
222 void encodeOptions(const Instruction &inst);
223
224
225 //////////////////////////////////////////////////////////////////////
226 // platform specific queries *but sometimes need the instruction too)
227 //
228 // GEN7p5 implicitly scales PC offsets by QW except for a few instructions
229 bool arePcsInQWords(const OpSpec &os) const;
230
231 // Call need to have src0 region be set to:
232 // SKL and before: <2;2,1>
233 // ICL: <2;4,1>
234 // Later GENs ignore the region completely
235 bool callNeedsSrc0Region221(const Instruction &inst) const;
236 bool callNeedsSrc0Region241(const Instruction &inst) const;
237
238 /////////////////////////////////////////////////////////////
239 // state valid over encodeKernel()
240 MemManager *m_mem;
241 uint8_t *m_instBuf = nullptr; // the output bits
242 struct JumpPatch { // JIP and UIP label patching
243 Instruction *inst; // the instruction
244 ged_ins_t gedInst; // the partially constructed GED instruction
245 uint8_t *bits; // where to encode it in the heap
246 JumpPatch(Instruction *i, const ged_ins_t &gi, uint8_t *bs)
247 : inst(i), gedInst(gi), bits(bs) {}
248 };
249 std::vector<JumpPatch> m_needToPatch;
250 std::map<const Block *, int32_t> m_blockToOffsetMap;
251
252 public:
253 ////////////////////////////////////////////////////////////////
254 static uint64_t typeConvesionHelper(const ImmVal &val, Type type) {
255 uint64_t value = 0;
256 switch (type) {
257 case Type::UD:
258 case Type::F:
259 case Type::V:
260 case Type::UV:
261 case Type::VF:
262 value = (uint64_t)val.u32;
263 break;
264 case Type::D:
265 value = (uint64_t)val.s32;
266 break;
267 case Type::W:
268 value = (uint64_t)val.s16;
269 break;
270 case Type::UW:
271 case Type::HF:
272 case Type::BF:
273 value = (uint64_t)val.u16;
274 break;
275 case Type::DF:
276 case Type::UQ:
277 case Type::Q:
278 value = val.u64;
279 break;
280 case Type::B:
281 case Type::UB:
282 // technically not reachable since we don't permit byte moves
283 // from immediates
284 value = val.u64;
285 break;
286 case Type::BF8:
287 case Type::HF8:
288 value = (uint64_t)val.u8;
289 break;
290 case Type::TF32:
291 value = (uint64_t)val.u32;
292 break;
293 default:
294 break;
295 }
296
297 return value;
298 }
299
300 protected:
301 ////////////////////////////////////////////////////////////////
302 // allowable types for ternary Align1
303 static bool isTernaryAlign1Floating(Type t) {
304 return lowerExecDataType(t) ==
305 GED_EXECUTION_DATA_TYPE::GED_EXECUTION_DATA_TYPE_Float;
306 }
307
308 // allowable types for ternary Align1
309 static bool isTernaryAlign1Integral(Type t) {
310 return lowerExecDataType(t) ==
311 GED_EXECUTION_DATA_TYPE::GED_EXECUTION_DATA_TYPE_Integer;
312 }
313 }; // end: class definition Encoder
314
315 template <SourceIndex S> void Encoder::encodeSrcRegFile(GED_REG_FILE rf) {
316 if (S == SourceIndex::SRC0) {
317 GED_ENCODE(Src0RegFile, rf);
318 } else if (S == SourceIndex::SRC1) {
319 GED_ENCODE(Src1RegFile, rf);
320 } else if (S == SourceIndex::SRC2) {
321 GED_ENCODE(Src2RegFile, rf);
322 } else if (S == SourceIndex::SRC3) {
323 GED_ENCODE(Src3RegFile, rf);
324 } else if (S == SourceIndex::SRC4) {
325 GED_ENCODE(Src4RegFile, rf);
326 }
327 }
328
329 template <SourceIndex S> void Encoder::encodeSrcRegionVert(Region::Vert v) {
330 if (S == SourceIndex::SRC0) {
331 GED_ENCODE(Src0VertStride, lowerRegionVert(v));
332 } else { // (S == SourceIndex::SRC1)
333 GED_ENCODE(Src1VertStride, lowerRegionVert(v));
334 } // S != SRC2 since ternary Align1 doesn't have bits for that
335 }
336
337 template <SourceIndex S> void Encoder::encodeSrcType(Type t) {
338 if (S == SourceIndex::SRC0) {
339 GED_ENCODE(Src0DataType, lowerDataType(t));
340 } else if (S == SourceIndex::SRC1) {
341 GED_ENCODE(Src1DataType, lowerDataType(t));
342 } else {
343 GED_ENCODE(Src2DataType, lowerDataType(t));
344 }
345 }
346
347 template <SourceIndex S> void Encoder::encodeSrcAddrMode(GED_ADDR_MODE x) {
348 if (S == SourceIndex::SRC0) {
349 GED_ENCODE(Src0AddrMode, x);
350 } else {
351 GED_ENCODE(Src1AddrMode, x);
352 }
353 }
354
355 template <SourceIndex S> void Encoder::encodeSrcModifier(SrcModifier x) {
356 if (S == SourceIndex::SRC0) {
357 GED_ENCODE(Src0SrcMod, lowerSrcMod(x));
358 } else if (S == SourceIndex::SRC1) {
359 GED_ENCODE(Src1SrcMod, lowerSrcMod(x));
360 } else {
361 GED_ENCODE(Src2SrcMod, lowerSrcMod(x));
362 }
363 }
364
365 template <SourceIndex S>
366 void Encoder::encodeSrcSubRegNum(uint32_t subRegInByte) {
367 if (S == SourceIndex::SRC0) {
368 GED_ENCODE(Src0SubRegNum, subRegInByte);
369 } else if (S == SourceIndex::SRC1) {
370 GED_ENCODE(Src1SubRegNum, subRegInByte);
371 } else if (S == SourceIndex::SRC2) {
372 GED_ENCODE(Src2SubRegNum, subRegInByte);
373 } else if (S == SourceIndex::SRC3) {
374 GED_ENCODE(Src3SubRegNum, subRegInByte);
375 } else if (S == SourceIndex::SRC4) {
376 GED_ENCODE(Src4SubRegNum, subRegInByte);
377 }
378 }
379
380 template <SourceIndex S> void Encoder::encodeSrcMathMacroReg(MathMacroExt a) {
381 if (S == SourceIndex::SRC0) {
382 GED_ENCODE(Src0MathMacroExt, lowerSpecialAcc(a));
383 } else if (S == SourceIndex::SRC1) {
384 GED_ENCODE(Src1MathMacroExt, lowerSpecialAcc(a));
385 } else {
386 GED_ENCODE(Src2MathMacroExt, lowerSpecialAcc(a));
387 }
388 }
389 template <SourceIndex S>
390 void Encoder::encodeSrcReg(RegName regName, uint16_t regNum) {
391 uint32_t regBits = 0;
392 if (regName == RegName::GRF_R) {
393 regBits = regNum; // GRF fast path
394 } else { // ARF slower path
395 const RegInfo *ri = m_model.lookupRegInfoByRegName(regName);
396 if (!ri) {
397 errorT("src", (int)S, ": unexpected register on this platform");
398 } else {
399 uint16_t reg;
400 ri->encode((int)regNum, reg);
401 regBits = reg; // widen for GED
402 }
403 }
404 if (S == SourceIndex::SRC0) {
405 GED_ENCODE(Src0RegNum, regBits);
406 } else if (S == SourceIndex::SRC1) {
407 GED_ENCODE(Src1RegNum, regBits);
408 } else if (S == SourceIndex::SRC2) {
409 GED_ENCODE(Src2RegNum, regBits);
410 } else if (S == SourceIndex::SRC3) {
411 GED_ENCODE(Src3RegNum, regBits);
412 } else if (S == SourceIndex::SRC4) {
413 GED_ENCODE(Src4RegNum, regBits);
414 }
415 }
416
417 template <SourceIndex S> void Encoder::encodeSrcAddrImm(int32_t addrImm) {
418 if (S == SourceIndex::SRC0) {
419 GED_ENCODE(Src0AddrImm, addrImm);
420 } else {
421 GED_ENCODE(Src1AddrImm, addrImm);
422 }
423 }
424 template <SourceIndex S>
425 void Encoder::encodeSrcAddrSubRegNum(uint32_t addrSubReg) {
426 if (S == SourceIndex::SRC0) {
427 GED_ENCODE(Src0AddrSubRegNum, addrSubReg);
428 } else {
429 GED_ENCODE(Src1AddrSubRegNum, addrSubReg);
430 }
431 }
432 template <SourceIndex S>
433 void Encoder::encodeSrcRegion(const Region &rgn, bool hasRgnWi) {
434 uint32_t v = 0;
435 if (rgn.getVt() == Region::Vert::VT_VxH) {
436 v = 0x3;
437 } else if (rgn.getVt() != Region::Vert::VT_INVALID) {
438 v = static_cast<uint32_t>(rgn.v);
439 } else {
440 errorT(S == SourceIndex::SRC0 ? "invalid region vertical stride on src0"
441 : "invalid region vertical stride on src1");
442 }
443
444 uint32_t w = static_cast<uint32_t>(rgn.getWi());
445 if (rgn.getWi() == Region::Width::WI_INVALID) {
446 errorT(S == SourceIndex::SRC0 ? "invalid region width on src0"
447 : "invalid region width on src1");
448 }
449
450 uint32_t h = static_cast<uint32_t>(rgn.getHz());
451 if (rgn.getHz() == Region::Horz::HZ_INVALID) {
452 h = 1;
453 errorT(S == SourceIndex::SRC0 ? "invalid region horizontal stride on src0"
454 : "invalid region horizontal stride on src1");
455 }
456
457 if (S == SourceIndex::SRC0) {
458 GED_ENCODE(Src0VertStride, v);
459 if (hasRgnWi) {
460 GED_ENCODE(Src0Width, w);
461 } else {
462 // some ops have an implicit width region
463 // (e.g. some specialized instructions poaches Src0.RgnWi)
464 //
465 // Within the IR we use 1 so logic that depends on regioning
466 // gets the correct behavior (hardware assumes w=1).
467 w = 1;
468 }
469 GED_ENCODE(Src0HorzStride, h);
470 } else if (S == SourceIndex::SRC1) {
471 GED_ENCODE(Src1VertStride, v);
472 GED_ENCODE(Src1Width, w);
473 GED_ENCODE(Src1HorzStride, h);
474 } else {
475 IGA_ASSERT_FALSE("Encoder::encodeSrcRegion: only works on src0 and src1");
476 }
477 }
478
479 template <SourceIndex S>
480 void Encoder::encodeSrcReducedRegion(const Region& rgn) {
481 IGA_ASSERT(S == SourceIndex::SRC1, "Invalid SourceIndex");
482 if (rgn.isScalar())
483 GED_ENCODE(Src1ScalarReg, 1);
484 else
485 GED_ENCODE(Src1ScalarReg, 0);
486 return;
487 IGA_ASSERT_FALSE("Unreachable: Unsupported region");
488 }
489
490 template <SourceIndex S> void Encoder::encodeSrcRegionWidth(Region::Width w) {
491 if (S == SourceIndex::SRC0) {
492 GED_ENCODE(Src0Width, lowerRegionWidth(w));
493 } else { // (S == SourceIndex::SRC1)
494 GED_ENCODE(Src1Width, lowerRegionWidth(w));
495 } // S != SRC2 since ternary Align1 doesn't have bits for that
496 }
497
498 template <SourceIndex S>
499 void Encoder::encodeTernaryImmVal(const ImmVal &val, Type type) {
500 if (S == SourceIndex::SRC0) {
501 GED_ENCODE(Src0TernaryImm, typeConvesionHelper(val, type));
502 } else if (S == SourceIndex::SRC2) {
503 GED_ENCODE(Src2TernaryImm, typeConvesionHelper(val, type));
504 } else {
505 errorT("immediate operands not supported in src1 of ternary formats");
506 }
507 }
508
509 template <SourceIndex S> void Encoder::encodeSrcRegionHorz(Region::Horz s) {
510 if (S == SourceIndex::SRC0) {
511 GED_ENCODE(Src0HorzStride, lowerRegionHorz(s));
512 } else if (S == SourceIndex::SRC1) {
513 GED_ENCODE(Src1HorzStride, lowerRegionHorz(s));
514 } else {
515 GED_ENCODE(Src2HorzStride, lowerRegionHorz(s));
516 }
517 }
518 } // namespace iga
519
520 namespace iga {
521 typedef Encoder Encoder;
522 }
523 #endif