Debian Codegraph @

visa/iga/IGALibrary/IR/RegDeps.cpp

raw
1 /*========================== begin_copyright_notice ============================
2
3 Copyright (C) 2017-2021 Intel Corporation
4
5 SPDX-License-Identifier: MIT
6
7 ============================= end_copyright_notice ===========================*/
8
9 #include "RegDeps.hpp"
10 #include "../asserts.hpp"
11 #include "../bits.hpp"
12
13 #include <algorithm>
14 #include <cstring>
15 #include <limits>
16 #include <sstream>
17
18 using namespace iga;
19
20 static DEP_CLASS getClassFromPipeType(DEP_PIPE type, const OpSpec &opspec) {
21 if (opspec.is(Op::SYNC) || opspec.is(Op::ILLEGAL) || opspec.is(Op::THRYLD))
22 return DEP_CLASS::OTHER;
23
24 switch (type) {
25 case DEP_PIPE::NONE:
26 case DEP_PIPE::SHORT:
27 case DEP_PIPE::LONG:
28 case DEP_PIPE::CONTROL_FLOW:
29 return DEP_CLASS::IN_ORDER;
30
31 case DEP_PIPE::SEND:
32 case DEP_PIPE::MATH:
33 return DEP_CLASS::OUT_OF_ORDER;
34
35 case DEP_PIPE::FLOAT:
36 case DEP_PIPE::INTEGER:
37 case DEP_PIPE::LONG64:
38 case DEP_PIPE::MATH_INORDER:
39 case DEP_PIPE::SCALAR:
40 return DEP_CLASS::IN_ORDER;
41
42 case DEP_PIPE::DPAS:
43 case DEP_PIPE::SEND_SLM:
44 case DEP_PIPE::SEND_UNKNOWN:
45 return DEP_CLASS::OUT_OF_ORDER;
46 }
47 return DEP_CLASS::NONE;
48 }
49
50 static void setDEPPipeClass_SingleDistPipe(DepSet &dep,
51 const Instruction &inst) {
52 auto opsec = inst.getOpSpec();
53 dep.setDepPipe(DEP_PIPE::SHORT);
54 if (opsec.is(Op::MATH)) {
55 dep.setDepPipe(DEP_PIPE::MATH);
56 } else if (opsec.isAnySendFormat()) {
57 dep.setDepPipe(DEP_PIPE::SEND);
58 } else if (opsec.isBranching()) {
59 dep.setDepPipe(DEP_PIPE::CONTROL_FLOW);
60 } else {
61 for (uint32_t i = 0; i < inst.getSourceCount(); ++i) {
62 const auto &src = inst.getSource(i);
63 if (src.getType() == Type::DF || src.getType() == Type::Q ||
64 src.getType() == Type::UQ) {
65 dep.setDepPipe(DEP_PIPE::LONG);
66 break;
67 }
68 }
69 if (opsec.supportsDestination()) {
70 const auto &dst = inst.getDestination();
71 if (dst.getType() == Type::DF || dst.getType() == Type::Q ||
72 dst.getType() == Type::UQ) {
73 dep.setDepPipe(DEP_PIPE::LONG);
74 }
75 }
76 }
77
78 dep.setDepClass(getClassFromPipeType(dep.getDepPipe(), opsec));
79 }
80
81 // XeHP+
82 static void setSendPipeType(DEP_PIPE &pipe_type, const Instruction &inst,
83 const Model &model) {
84 assert(inst.getOpSpec().isAnySendFormat());
85 pipe_type = DEP_PIPE::SEND;
86 // XeHPG+: slm send should be considered in different pipes
87 // Check if it's SLM
88 if (inst.getOpSpec().isSendgFormat()) {
89 if (inst.getSendFc() == SFID::SLM)
90 pipe_type = DEP_PIPE::SEND_SLM;
91 return;
92 }
93
94 if (model.platform >= Platform::XE_HPG) {
95 if (inst.getMsgDescriptor().isReg())
96 pipe_type = DEP_PIPE::SEND_UNKNOWN;
97 else {
98 SFID sfid = inst.getSendFc();
99 if (sfid == SFID::SLM) {
100 pipe_type = DEP_PIPE::SEND_SLM;
101 } else {
102 uint32_t desc = inst.getMsgDescriptor().imm;
103 bool btiSlm = (0xFE == getBits<uint32_t>(desc, 0, 8));
104 bool scratchBlockMsg = (1 == getBits<uint32_t>(desc, 18, 1));
105 bool headerPresent = (1 == getBits<uint32_t>(desc, 19, 1));
106 bool sidebandOffsetEn = (1 == getBits<uint32_t>(desc, 7, 1));
107 if ((sfid == SFID::DC0 && btiSlm && !scratchBlockMsg) ||
108 (sfid == SFID::DC1 && btiSlm) ||
109 (sfid == SFID::DC2 && !headerPresent && sidebandOffsetEn))
110 pipe_type = DEP_PIPE::SEND_SLM;
111 }
112 }
113 }
114 }
115
116 // MTL
117 static void setDEPPipeClass_ThreeDistPipeDPMath(DepSet &dep,
118 const Instruction &inst,
119 const Model &model) {
120 // The same as ThreeDistPipe, only that instructions with DP (fp64) types
121 // will be in Math pipe
122 auto opsec = inst.getOpSpec();
123
124 DEP_PIPE pipe_type = DEP_PIPE::NONE;
125 if (opsec.is(Op::MATH)) {
126 pipe_type = DEP_PIPE::MATH;
127 } else if (opsec.isAnySendFormat()) {
128 setSendPipeType(pipe_type, inst, model);
129 } else if (opsec.isDpasFormat()) {
130 pipe_type = DEP_PIPE::DPAS;
131 } else if (opsec.isBranching()) {
132 pipe_type = DEP_PIPE::INTEGER;
133 } else {
134 // In order instruction:
135 // if destination type is FP32/FP16/BF16 then it goes to float pipe,
136 // if destination type is int32 / 16 / 8 it goes to integer pipe and
137 // if destination or source type is int64 then it goes to long pipe
138 // if destination or source type is FP64 then it goes to out-of-order Math
139 // pipe for conversion instructions float2int goes to integer pipe and
140 // int2float goes to float pipe, If destination type is null then source0
141 // data type will determine the pipe
142 Type inst_type = Type::INVALID;
143 if (opsec.supportsDestination())
144 inst_type = inst.getDestination().getType();
145 else if (inst.getSourceCount())
146 inst_type = inst.getSource(0).getType();
147
148 if (inst_type != Type::INVALID) {
149 if (TypeIs64b(inst_type)) {
150 if (TypeIsFloating(inst_type))
151 pipe_type = DEP_PIPE::MATH;
152 else
153 pipe_type = DEP_PIPE::LONG64;
154 } else if (TypeIsFloating(inst_type))
155 pipe_type = DEP_PIPE::FLOAT;
156 else
157 pipe_type = DEP_PIPE::INTEGER;
158 }
159
160 // any of src has 64-bit type --> math or long pipe
161 for (uint32_t i = 0; i < inst.getSourceCount(); ++i) {
162 const auto &src = inst.getSource(i);
163 if (TypeIs64b(src.getType())) {
164 if (TypeIsFloating(src.getType()))
165 pipe_type = DEP_PIPE::MATH;
166 else
167 pipe_type = DEP_PIPE::LONG64;
168 break;
169 }
170 }
171 }
172
173 // default set to Integer pipe (e.g. NOP)
174 if (pipe_type == DEP_PIPE::NONE)
175 pipe_type = DEP_PIPE::INTEGER;
176
177 dep.setDepClass(getClassFromPipeType(pipe_type, opsec));
178 dep.setDepPipe(pipe_type);
179 }
180
181 // XeHP
182 static void setDEPPipeClass_ThreeDistPipe(DepSet &dep, const Instruction &inst,
183 const Model &model) {
184 auto opsec = inst.getOpSpec();
185
186 DEP_PIPE pipe_type = DEP_PIPE::NONE;
187 if (opsec.is(Op::MATH)) {
188 pipe_type = DEP_PIPE::MATH;
189 } else if (opsec.isAnySendFormat()) {
190 setSendPipeType(pipe_type, inst, model);
191 } else if (opsec.isDpasFormat()) {
192 pipe_type = DEP_PIPE::DPAS;
193 } else if (opsec.isBranching()) {
194 pipe_type = DEP_PIPE::INTEGER;
195 } else {
196 // In order instruction:
197 // if destination type is FP32/FP16/BF16 then it goes to float pipe,
198 // if destination type is int32 / 16 / 8 it goes to integer pipe and
199 // if destination or source type is int64/FP64 then it goes to long pipe
200
201 // for conversion instructions float2int goes to integer pipe and int2float
202 // goes to float pipe, anything2doublefloat goes to long pipe and
203 // doublefloat2anything goes to long pipe
204
205 // If destination type is null then source0 data type will determine the
206 // pipe
207 Type inst_type = Type::INVALID;
208 if (opsec.supportsDestination())
209 inst_type = inst.getDestination().getType();
210 else if (inst.getSourceCount())
211 inst_type = inst.getSource(0).getType();
212
213 if (inst_type != Type::INVALID) {
214 if (TypeIs64b(inst_type))
215 pipe_type = DEP_PIPE::LONG64;
216 else if (TypeIsFloating(inst_type))
217 pipe_type = DEP_PIPE::FLOAT;
218 else
219 pipe_type = DEP_PIPE::INTEGER;
220 }
221
222 for (uint32_t i = 0; i < inst.getSourceCount(); ++i) {
223 const auto &src = inst.getSource(i);
224 if (TypeIs64b(src.getType())) {
225 pipe_type = DEP_PIPE::LONG64;
226 break;
227 }
228 }
229 }
230
231 // default set to Integer pipe (e.g. NOP)
232 if (pipe_type == DEP_PIPE::NONE)
233 pipe_type = DEP_PIPE::INTEGER;
234
235 dep.setDepClass(getClassFromPipeType(pipe_type, opsec));
236 dep.setDepPipe(pipe_type);
237 }
238
239 // XeHPC+
240 static void setDEPPipeClass_FourDistPipe(DepSet &dep, const Instruction &inst,
241 const Model &model) {
242 setDEPPipeClass_ThreeDistPipe(dep, inst, model);
243 if (dep.getDepPipe() == DEP_PIPE::MATH) {
244 dep.setDepClass(DEP_CLASS::IN_ORDER);
245 dep.setDepPipe(DEP_PIPE::MATH_INORDER);
246 }
247 }
248
249 static void setDEPPipeClass_FourDistPipeReduction(DepSet &dep,
250 const Instruction &inst,
251 const Model &model) {
252 // The difference between FourDistPipe and FourDistPipeReduction
253 // is that for FourDistPipeReduction: only fp64-dst-instructions goto Long
254 // pipe (LONG64), and for FourDistPipe: instructions having 64b types on src
255 // or dst goto Long pipe.
256 auto opsec = inst.getOpSpec();
257
258 DEP_PIPE pipe_type = DEP_PIPE::NONE;
259 if (opsec.is(Op::MATH)) {
260 dep.setDepClass(DEP_CLASS::IN_ORDER);
261 dep.setDepPipe(DEP_PIPE::MATH_INORDER);
262 return;
263 } else if (opsec.isAnySendFormat()) {
264 setSendPipeType(pipe_type, inst, model);
265 } else if (opsec.isDpasFormat()) {
266 pipe_type = DEP_PIPE::DPAS;
267 } else if (opsec.isBranching()) {
268 pipe_type = DEP_PIPE::INTEGER;
269 } else {
270 // In order instruction:
271 // RegDistFloat: Specify distance dependancy in float32 pipe
272 // (i.e. all instructions with float32/16/8 dst)
273 // RegDistLong: Specify distance dependancy in float64 pipe
274 // (i.e. all instructions with float64 dst)
275 // RegDistInt: Specify distance dependancy in Int32/16/8 and Int64 pipe
276 // (i.e. all instructions with integer dst)
277 // RegDistMath: Specify distance dependancy in Math pipe
278 // (i.e. all math instructions)
279
280 Type inst_type = Type::INVALID;
281 // only instructions with f64 dst go into Long, others with f32/16/8
282 // go into FLOAT
283 if (opsec.supportsDestination()) {
284 inst_type = inst.getDestination().getType();
285 if (TypeIsFloating(inst_type)) {
286 if (TypeIs64b(inst_type))
287 pipe_type = DEP_PIPE::LONG64;
288 else
289 pipe_type = DEP_PIPE::FLOAT;
290 } else {
291 pipe_type = DEP_PIPE::INTEGER;
292 }
293 }
294 }
295
296 // default set to Integer pipe (e.g. NOP)
297 if (pipe_type == DEP_PIPE::NONE)
298 pipe_type = DEP_PIPE::INTEGER;
299
300 dep.setDepClass(getClassFromPipeType(pipe_type, opsec));
301 dep.setDepPipe(pipe_type);
302 }
303
304 static bool isScalarPipeInst(const Instruction &inst) {
305 return inst.getOp() == Op::MOV &&
306 inst.getDestination().getDirRegName() == RegName::ARF_S &&
307 inst.getSource(0).isImm();
308 }
309
310 static void setDEPPipeClass_FiveDistPipe(DepSet &dep, const Instruction &inst,
311 const Model &model) {
312 // TODO: Implement FiveDistPipe
313 setDEPPipeClass_FourDistPipeReduction(dep, inst, model);
314
315 // mov instructions with scalar dst and imm src goto SCALAR pipe
316 if (isScalarPipeInst(inst)) {
317 dep.setDepPipe(DEP_PIPE::SCALAR);
318 IGA_ASSERT(dep.getDepClass() == DEP_CLASS::IN_ORDER,
319 "dep class for mov should be set already");
320 }
321 }
322
323 static void setDEPPipeClass_FiveDistPipeReduction(DepSet &dep,
324 const Instruction &inst,
325 const Model &model) {
326 // The only difference between FiveDistPipeReduction and FiveDistPipe
327 // is that Q to F coversion instructions are now in INT pipe
328 setDEPPipeClass_FiveDistPipe(dep, inst, model);
329
330 if (dep.getDepClass() == DEP_CLASS::IN_ORDER) {
331 auto opsec = inst.getOpSpec();
332 if (opsec.supportsDestination() && inst.getSourceCount() > 1) {
333 Type dstType = inst.getDestination().getType();
334 Type src0Type = inst.getSource(0).getType();
335 if (dstType == Type::F && (src0Type == Type::Q || src0Type == Type::UQ)) {
336 dep.setDepPipe(DEP_PIPE::INTEGER);
337 }
338 }
339 }
340 }
341
342 static void setDEPPipeClass_FiveDistPipeCvtToInt(DepSet &dep,
343 const Instruction &inst,
344 const Model &model) {
345 // Support feature "Mov Cvt from FPU0 (FLOAT) to FPU1 (INT) pipe".
346 // The difference to FiveDistPipe is:
347 // - srnd in INT pipe always
348 // - mov:
349 // df destination always use LONG pipe
350 // f to f, hf to hf, bf to bf use FLOAT pipe
351 // any other datatype to f/hf/tf/bf/bf8/hf8 use INT pipe
352 // integer destination always use INT pipe (except for scalar pipe mov)
353
354 if (inst.getOp() == Op::SRND) {
355 dep.setDepPipe(DEP_PIPE::INTEGER);
356 dep.setDepClass(DEP_CLASS::IN_ORDER);
357 return;
358 }
359
360 // mov instructions with scalar dst and imm src goto SCALAR pipe
361 if (isScalarPipeInst(inst)) {
362 dep.setDepPipe(DEP_PIPE::SCALAR);
363 dep.setDepClass(DEP_CLASS::IN_ORDER);
364 return;
365 }
366
367 if (inst.getOp() == Op::MOV) {
368 auto dstType = inst.getDestination().getType();
369 auto srcType = inst.getSource(0).getType();
370 if (dstType == Type::DF)
371 dep.setDepPipe(DEP_PIPE::LONG64);
372 else if (dstType == srcType &&
373 (dstType == Type::F || dstType == Type::HF || dstType == Type::BF))
374 dep.setDepPipe(DEP_PIPE::FLOAT);
375 else
376 dep.setDepPipe(DEP_PIPE::INTEGER);
377 dep.setDepClass(DEP_CLASS::IN_ORDER);
378 return;
379 }
380
381 // The rest is the same as FiveDistPipe
382 setDEPPipeClass_FiveDistPipe(dep, inst, model);
383 }
384
385 static void setDEPPipeClass(SWSB_ENCODE_MODE enc_mode, DepSet &dep,
386 const Instruction &inst, const Model &model) {
387 switch (enc_mode) {
388 case SWSB_ENCODE_MODE::SingleDistPipe:
389 setDEPPipeClass_SingleDistPipe(dep, inst);
390 break;
391 case SWSB_ENCODE_MODE::ThreeDistPipeDPMath:
392 setDEPPipeClass_ThreeDistPipeDPMath(dep, inst, model);
393 break;
394 case SWSB_ENCODE_MODE::ThreeDistPipe:
395 setDEPPipeClass_ThreeDistPipe(dep, inst, model);
396 break;
397 case SWSB_ENCODE_MODE::FourDistPipe:
398 setDEPPipeClass_FourDistPipe(dep, inst, model);
399 break;
400 case SWSB_ENCODE_MODE::FourDistPipeReduction:
401 setDEPPipeClass_FourDistPipeReduction(dep, inst, model);
402 break;
403 case SWSB_ENCODE_MODE::FiveDistPipe:
404 setDEPPipeClass_FiveDistPipe(dep, inst, model);
405 break;
406 case SWSB_ENCODE_MODE::FiveDistPipeReduction:
407 setDEPPipeClass_FiveDistPipeReduction(dep, inst, model);
408 break;
409 case SWSB_ENCODE_MODE::FiveDistPipeCvtToInt:
410 case SWSB_ENCODE_MODE::FiveDistPipeCvtToIntNoFwd:
411 setDEPPipeClass_FiveDistPipeCvtToInt(dep, inst, model);
412 break;
413 case SWSB_ENCODE_MODE::FiveDistPipeSWSBCntr:
414 setDEPPipeClass_FiveDistPipe(dep, inst, model);
415 break;
416 default:
417 break;
418 }
419 }
420
421 DepSet::DepSet(const InstIDs &instIdCntr, const DepSetBuilder &dsb)
422 : m_InstIDs(instIdCntr.global, instIdCntr.inOrder, instIdCntr.floatPipe,
423 instIdCntr.intPipe, instIdCntr.longPipe, instIdCntr.mathPipe,
424 instIdCntr.scalarPipe), m_DB(dsb) {
425 m_bucketList.reserve(4);
426 bits = new BitSet<>(dsb.getTOTAL_BITS());
427 }
428
429 uint32_t DepSet::getDPASOpsPerChan(Type src1_ty, Type src2_ty, bool isDF) {
430 // get OPS_PER_CHAN, the number of dot product operations per dword channel,
431 // depending on element type
432 if (isDF)
433 return 1;
434
435 if (src1_ty == Type::HF || src1_ty == Type::BF) {
436 IGA_ASSERT(src1_ty == src2_ty, "dpas: invalid src1/src2 types combination");
437 return 2;
438 } else if (src1_ty == Type::BF8 || src1_ty == Type::HF8) {
439 IGA_ASSERT(src2_ty == Type::BF8 || src2_ty == Type::HF8,
440 "dpas: invalid src1/src2 types combination");
441 return 4;
442 } else if (src1_ty == Type::TF32) {
443 IGA_ASSERT(src1_ty == src2_ty, "dpas: invalid src1/src2 types combination");
444 return 1;
445 } else {
446 // if both src1 and src2 are int2 or int4, than ops_per_chan will be 8
447 int src1_size = TypeSizeInBits(src1_ty);
448 int src2_size = TypeSizeInBits(src2_ty);
449 // Type: ub, b, u4, s4, u2, s2
450 IGA_ASSERT((src1_size <= 8), "OPS_PER_CHAN: unsupported src1 type");
451 IGA_ASSERT((src2_size <= 8), "OPS_PER_CHAN: unsupported src2 type");
452 if ((src1_size == 2 || src1_size == 4) &&
453 (src2_size == 2 || src2_size == 4))
454 return 8;
455 return 4;
456 }
457 }
458
459 // lowBound - start register address offset in byte
460 // UpBound - upper register address offset in byte
461 uint32_t DepSet::getDPASSrcDepUpBound(unsigned idx, Type srcType,
462 uint32_t execSize, uint32_t lowBound,
463 uint32_t systolicDepth,
464 uint32_t repeatCount,
465 uint32_t opsPerChan) {
466 auto typeSizeInBits = TypeSizeInBitsWithDefault(srcType, 32);
467 // elements_size is the size of total elements to be calculated in one
468 // operation
469 uint32_t elements_size = execSize * typeSizeInBits / 8;
470
471 uint32_t upBound = lowBound;
472 if (idx == 0)
473 upBound += elements_size * repeatCount;
474 else if (idx == 1)
475 upBound += elements_size * opsPerChan * systolicDepth;
476 else if (idx == 2)
477 upBound += (repeatCount - 1) * opsPerChan * 8 * typeSizeInBits /
478 8 + /* start offset of the last repeated block */
479 opsPerChan * systolicDepth * typeSizeInBits /
480 8; /* size of used register in last repeated block */
481 else {
482 IGA_ASSERT(idx <= 4, "Invalid dpas src index");
483 // bdpas may have src3/src4. The register access pattern of them must not
484 // cross a GRF. Assume its register footprint is till the end of the start
485 // register. We consider dpas src register in register granularity during
486 // swsb setting so this is good enough.
487 auto startReg = lowBound / m_DB.getGRF_BYTES_PER_REG();
488 RegRef tmpRR = RegRef((uint16_t)startReg + 1, 0);
489 upBound = addressOf(RegName::GRF_R, tmpRR, typeSizeInBits) - 1;
490 }
491 return upBound;
492 }
493
494 void DepSet::getDpasSrcDependency(const Instruction &inst,
495 RegRangeListType &reg_range,
496 RegRangeListType &extra_regs,
497 const Model &model) {
498 uint32_t execSize = static_cast<uint32_t>(inst.getExecSize());
499
500 IGA_ASSERT((!inst.isDF() && execSize == (m_DB.getGRF_BYTES_PER_REG() / 4)) ||
501 (inst.isDF() && execSize == 8),
502 "Invalid ExecSize for this op");
503
504 // check src operand and add the dependency
505 uint32_t repeatCount = GetDpasRepeatCount(inst.getDpasFc());
506 uint32_t systolicDepth = GetDpasSystolicDepth(inst.getDpasFc());
507 uint32_t ops_per_chan = getDPASOpsPerChan(
508 inst.getSource(1).getType(), inst.getSource(2).getType(), inst.isDF());
509
510 for (unsigned srcIx = 0; srcIx < inst.getSourceCount(); ++srcIx) {
511 const Operand &op = inst.getSource(srcIx);
512 // the src0 could be null, in that case no need to set the dependency
513 if (op.getDirRegName() == RegName::ARF_NULL) {
514 // if src is null, set the reg range to max() to specify its actually
515 // empty
516 reg_range.push_back(std::make_pair(std::numeric_limits<uint32_t>::max(),
517 std::numeric_limits<uint32_t>::max()));
518 continue;
519 }
520 IGA_ASSERT(op.getDirRegName() == RegName::GRF_R,
521 "GRF or null required on this op");
522
523 // calculate register region
524 auto tType = op.getType();
525 auto typeSizeInBits = TypeSizeInBitsWithDefault(tType, 32);
526 uint32_t lowBound =
527 addressOf(op.getDirRegName(), op.getDirRegRef(), typeSizeInBits);
528 uint32_t upBound =
529 getDPASSrcDepUpBound(srcIx, tType, execSize, lowBound, systolicDepth,
530 repeatCount, ops_per_chan);
531 IGA_ASSERT(upBound >= lowBound, "source region footprint computation got "
532 "it wrong: upBound is less than lowBound");
533
534 uint32_t startRegNum = lowBound / m_DB.getGRF_BYTES_PER_REG();
535 uint32_t upperRegNum = (upBound - 1) / m_DB.getGRF_BYTES_PER_REG();
536 reg_range.push_back(std::make_pair(startRegNum, upperRegNum));
537 // calculate extra_regs for HW workaround: treat Src2 as dpas.8x8 when
538 // calculating register footpring (4 registers)
539 if (m_DB.needDPASSrc2WA()) {
540 if (srcIx == 2 && (repeatCount != 8 || systolicDepth != 8)) {
541 uint32_t extraUpBound = getDPASSrcDepUpBound(
542 srcIx, tType, execSize, lowBound, 8, 8, ops_per_chan);
543 uint32_t extraUpRegNum =
544 (extraUpBound - 1) / m_DB.getGRF_BYTES_PER_REG();
545 if (extraUpRegNum >= m_DB.getGRF_REGS())
546 IGA_FATAL("IGA RegDeps: DPAS src2 out of bounds due to HW WA");
547 extra_regs.push_back(std::make_pair(startRegNum, extraUpRegNum));
548 }
549 }
550 // calculate extra_regs for HW workaround: src1 always have 8 register
551 // footpring
552 if (m_DB.needDPASSrc1WA()) {
553 if (srcIx == 1) {
554 uint32_t extraUpBound = lowBound + m_DB.getGRF_BYTES_PER_REG() * 8;
555 uint32_t extraUpRegNum =
556 (extraUpBound - 1) / m_DB.getGRF_BYTES_PER_REG();
557 if (extraUpRegNum >= m_DB.getGRF_REGS())
558 IGA_FATAL("IGA RegDeps: DPAS src1 out of bounds due to HW WA");
559 extra_regs.push_back(std::make_pair(startRegNum, extraUpRegNum));
560 }
561 }
562 }
563 }
564
565 void DepSet::addDependency(const RegRangeType &reg_range) {
566 for (uint32_t regNum = reg_range.first; regNum <= reg_range.second;
567 regNum++) {
568 addGrf(regNum);
569 addToBucket(regNum);
570 }
571
572 // Using one of the special registers to add write dependency in to special
573 // bucket This way it will always check that implicit dependency
574 m_bucketList.push_back(m_DB.getBucketStart(RegName::ARF_CR));
575 }
576
577 void DepSet::addDependency(const RegRangeListType &reg_range) {
578 for (RegRangeType pair : reg_range) {
579 // when range is max(), which means it's null, skip it
580 if (pair.first == std::numeric_limits<uint32_t>::max())
581 continue;
582 // u64 to avoid overflow if pair.second == ~0U
583 for (size_t regNum = pair.first; regNum <= pair.second; regNum++) {
584 addGrf(regNum);
585 addToBucket(regNum);
586 }
587 }
588
589 // Using one of the special registers to add read dependency into special
590 // bucket This way it will always check that implicit dependency
591 m_bucketList.push_back(m_DB.getBucketStart(RegName::ARF_CR));
592 }
593
594 bool DepSetBuilder::needDstReadSuppressionWA(const Instruction &inst) const {
595 if (mPlatformModel.platform != Platform::XE_HPG)
596 return false;
597
598 if (inst.getOpSpec().is(Op::MATH))
599 return true;
600
601 if (inst.isDF())
602 return true;
603
604 return false;
605 }
606
607 uint32_t DepSetBuilder::getBucketStart(RegName regname) const {
608 uint32_t bucket = 0;
609 switch (regname) {
610 case iga::RegName::GRF_R:
611 bucket = getGRF_START() / getBYTES_PER_BUCKET();
612 break;
613 case iga::RegName::ARF_A:
614 bucket = getARF_A_START() / getBYTES_PER_BUCKET();
615 break;
616 case iga::RegName::ARF_ACC:
617 bucket = getARF_ACC_START() / getBYTES_PER_BUCKET();
618 break;
619 case iga::RegName::ARF_F:
620 bucket = getARF_F_START() / getBYTES_PER_BUCKET();
621 break;
622 case RegName::ARF_CR:
623 case RegName::ARF_SR:
624 bucket = getARF_SPECIAL_START() / getBYTES_PER_BUCKET();
625 break;
626 case RegName::ARF_S:
627 bucket = getARF_SCALAR_START() / getBYTES_PER_BUCKET();
628 break;
629 default:
630 // putting rest of archtecture registers in to same bucket
631 bucket = getARF_F_START() / 32;
632 break;
633 }
634 return bucket;
635 }
636
637 size_t DepSetBuilder::DpasMacroBuilder::getNumberOfSuppresionGroups(
638 uint32_t srcIdx) const {
639
640 if (srcIdx == 1) {
641 if (m_model.platform > Platform::XE3)
642 return 4;
643 return 1;
644 }
645 if (srcIdx == 2) {
646 if (m_model.platform > Platform::XE3)
647 return 16;
648 if (m_model.platform >= Platform::XE_HPC)
649 return 4;
650 }
651 return 0;
652 }
653
654 size_t DepSetBuilder::DpasMacroBuilder::formSrcSuppressionBlock(
655 InstListIterator startIt) {
656 // get the candidate block
657 BitSet<> allDstBits(m_dsBuilder.getGRF_LEN());
658 BitSet<> allSrcBits(m_dsBuilder.getGRF_LEN());
659 BitSet<> allDstNoLastBits(m_dsBuilder.getGRF_LEN());
660 BitSet<> allSrcNoLastBits(m_dsBuilder.getGRF_LEN());
661
662 SuppressBlock bptr;
663
664 size_t numSuppressed = 0;
665 InstListIterator it = startIt;
666
667 // keep track of previous range in case we need to check if canFwd when
668 // hasProducerConsumerDep
669 Instruction *prev_inst = nullptr;
670 SrcRegRangeType prev_src_range, prev_extra_src_range;
671 DstRegRangeType prev_dst_range;
672
673 // find until the last instruction that can be suppressed
674 while (it != m_instList.end()) {
675 SrcRegRangeType src_range, src_extra_range;
676 DstRegRangeType dst_range;
677 m_inps.getDpasSrcDependency(**it, src_range, src_extra_range, m_model);
678 m_inps.getDpasDstDependency(**it, dst_range);
679 if (hasInternalDep(**it, dst_range, src_range,
680 GetDpasSystolicDepth((*it)->getDpasFc()) == 8))
681 break;
682
683 bool skipSetLastBits = false;
684 if (hasProducerConsumerDep(dst_range, src_range, allDstBits)) {
685 // if the only reason this instruction can't be in this block is because
686 // of having producer-consume dependency, and that dependency can be
687 // resolved by Fwd, then this instruction can be in the block
688 if (prev_inst &&
689 (prev_inst->hasInstOpt(InstOpt::FWD) ||
690 canFwd(*prev_inst, **it, prev_dst_range, prev_src_range, dst_range,
691 src_range, allDstNoLastBits, allSrcNoLastBits))) {
692 // set the forward block. It's sure that it can be forwarded, even if
693 // later we found out this suppression block can't suppress any
694 // instruction and we decide not to form macro for the suppression, we
695 // can still keep the fwd block
696 prev_inst->addInstOpt(InstOpt::ATOMIC);
697 prev_inst->addInstOpt(InstOpt::FWD);
698 // when fwd bits is formed, skip setting
699 // allDstNoLastBits/allSrcNoLastBits so that a continuous fwd
700 // instructions could be formed. For example: dpas.8x8 (16|M0) r110:f
701 // r10:f r2:bf r68.0:bf {Fwd, Atomic} dpas.8x8 (16|M0) r110:f r110:f
702 // r2:bf r76.0:bf {Fwd, Atomic} dpas.8x8 (16|M0) r110:f r110:f r2:bf
703 // r84.0:b
704 skipSetLastBits = true;
705 } else
706 break;
707 }
708
709 // at this point, we can add this DPAS into the macro
710 ++numSuppressed;
711 bptr.addRegRanges(src_range, src_extra_range, dst_range);
712 if (!skipSetLastBits) {
713 allSrcNoLastBits = allSrcBits;
714 allDstNoLastBits = allDstBits;
715 }
716 setDstSrcBits(src_range, dst_range, allSrcBits, allDstBits);
717 InstListIterator nextIt = std::next(it, 1);
718 if (nextIt == m_instList.end())
719 break;
720 if (nextIsNotMacroCandidate(**it, **nextIt))
721 break;
722 prev_inst = *it;
723 prev_src_range = src_range;
724 prev_dst_range = dst_range;
725 it = nextIt;
726 }
727
728 if (numSuppressed) {
729 // at least one instruction can be suppressed, the candidate block can be in
730 // the macro udpate register footprint into DepSet
731 updateRegFootprintsToDepSets(bptr.allSrcRange, bptr.allExtraSrcRange,
732 bptr.allDstRange);
733
734 // return the total instructions found can be in the macro
735 return numSuppressed;
736 }
737 return 0;
738 }
739
740 bool DepSetBuilder::DpasMacroBuilder::hasProducerConsumerDep(
741 const DstRegRangeType &dst_range, const SrcRegRangeType &src_range,
742 const BitSet<> &target_dst_bits) const {
743
744 BitSet<> new_srcbits(m_dsBuilder.getGRF_LEN());
745 BitSet<> new_dstbits(m_dsBuilder.getGRF_LEN());
746 setDstSrcBits(src_range, dst_range, new_srcbits, new_dstbits);
747
748 // check if there is RAW dependency
749 if (target_dst_bits.intersects(new_srcbits))
750 return true;
751 return false;
752 }
753
754 // add given src and dst register ranges into DepSet
755 void DepSetBuilder::DpasMacroBuilder::updateRegFootprintsToDepSets(
756 SrcRegRangeType &src_range, SrcRegRangeType &extra_src_range,
757 DstRegRangeType &dst_range) {
758 m_inps.addDependency(src_range);
759 m_inps.addDependency(extra_src_range);
760 m_oups.addDependency(dst_range);
761 };
762
763 void DepSetBuilder::DpasMacroBuilder::updateRegFootprintsToDepSets(
764 RegRangeListType &src_range, RegRangeListType &extra_src_range,
765 RegRangeListType &dst_range) {
766 m_inps.addDependency(src_range);
767 m_inps.addDependency(extra_src_range);
768 m_oups.addDependency(dst_range);
769 };
770
771 Instruction &DepSetBuilder::DpasMacroBuilder::formMacro(size_t &dpasCnt) {
772 dpasCnt = 1;
773 InstListIterator cur = m_firstDpasIt;
774
775 SrcRegRangeType src_range, src_extra_range;
776 DstRegRangeType dst_range;
777 m_inps.getDpasSrcDependency(**cur, src_range, src_extra_range, m_model);
778 m_inps.getDpasDstDependency(**cur, dst_range);
779 InstListIterator next = cur;
780 next++;
781 // early exit if there is no following instructions or dpas depth is not 8
782 if (next == m_instList.end() ||
783 GetDpasSystolicDepth((*cur)->getDpasFc()) != 8) {
784 updateRegFootprintsToDepSets(src_range, src_extra_range, dst_range);
785 return **cur;
786 }
787
788 // Firstly check if there is FWD block from the first, so that it won't break
789 // the suppression block due to having producer-consumer dependency at the
790 // first instruction (formSrcSuppressionBlock checks FWD block start from the
791 // 2nd instruction)
792 dpasCnt = std::max(dpasCnt, formFwdBlock(m_firstDpasIt));
793
794 dpasCnt = std::max(dpasCnt, formSrcSuppressionBlock(m_firstDpasIt));
795
796 if (dpasCnt == 1) {
797 updateRegFootprintsToDepSets(src_range, src_extra_range, dst_range);
798 return **cur;
799 }
800
801 // Set Atomic to all dpas in the macro except the last one
802 // Also clean-up their SWSB if set from the input.
803 InstListIterator it = m_firstDpasIt;
804 for (size_t i = 0; i < dpasCnt - 1; ++it, ++i) {
805 (*it)->addInstOpt(InstOpt::ATOMIC);
806 (*it)->setSWSB(SWSB());
807 }
808
809 InstListIterator last = m_firstDpasIt;
810 std::advance(last, dpasCnt - 1);
811 assert(last != m_instList.end());
812 // if the last dpas happens to have Fwd set, remove it since the next
813 // instruction it can forward to is not in this macro The case happens at
814 // getSuppressionBlockCandidate when FWD is set but nothing found can be
815 // suppressed later on
816 if ((*last)->hasInstOpt(InstOpt::FWD)) {
817 (*last)->removeInstOpt(InstOpt::ATOMIC);
818 (*last)->removeInstOpt(InstOpt::FWD);
819 }
820 return **last;
821 }
822
823 bool DepSetBuilder::DpasMacroBuilder::nextIsNotMacroCandidate(
824 const Instruction &dpas, const Instruction &next_inst) const {
825 if (!next_inst.getOpSpec().isDpasFormat())
826 return true;
827
828 // DPAS and DPASW should not be in the same macro
829 if (next_inst.getOp() != dpas.getOp())
830 return true;
831
832 // DPAS with different CtrlMask (NoMask and no NoMask) cannot be in the same
833 // macro
834 if (next_inst.getMaskCtrl() != dpas.getMaskCtrl())
835 return true;
836
837 // DPAS with different execution mask cannot be in the same macro
838 if (next_inst.getChannelOffset() != dpas.getChannelOffset())
839 return true;
840
841 // dpas with different depth should not be in the same macro
842 // Note that different repeat count is allowed in the same macro
843 uint32_t dpasSystolicDepth = GetDpasSystolicDepth(dpas.getDpasFc());
844 uint32_t nextSystolicDepth = GetDpasSystolicDepth(next_inst.getDpasFc());
845 if (dpasSystolicDepth != nextSystolicDepth)
846 return true;
847
848 // Dpas in the same macro must have the same datatypes in all src and dst
849 // Except for src0 and dst which can accept having a mix of bf16 and fp32
850 // datatypes
851 assert(dpas.getSourceCount() == next_inst.getSourceCount());
852 for (size_t i = 0; i < dpas.getSourceCount(); ++i) {
853 Type curType = dpas.getSource(i).getType();
854 Type nextType = next_inst.getSource(i).getType();
855 if (curType != nextType &&
856 (i != 0 || !isValidMixedTypes(curType, nextType)))
857 return true;
858 }
859 Type curDstType = dpas.getDestination().getType();
860 Type nextDstType = next_inst.getDestination().getType();
861 if (curDstType != nextDstType &&
862 !isValidMixedTypes(curDstType, nextDstType))
863 return true;
864 return false;
865 }
866
867 bool DepSetBuilder::DpasMacroBuilder::isValidMixedTypes(Type curType,
868 Type nextType) const {
869 // bf16 and fp32 can be mixed
870 return m_model.platform > Platform::XE3 &&
871 (curType == Type::BF || curType == Type::F) &&
872 (nextType == Type::BF || nextType == Type::F);
873 }
874
875 // set register range from start_reg to upper_reg into bit_set
876 void DepSetBuilder::DpasMacroBuilder::setBits(BitSet<> &bit_set,
877 uint32_t start_reg,
878 uint32_t upper_reg) const {
879 // if the given register is max(), which means there's no register,
880 // then no need to do anything
881 if (start_reg == std::numeric_limits<uint32_t>::max() ||
882 upper_reg == std::numeric_limits<uint32_t>::max())
883 return;
884 for (uint32_t i = start_reg; i <= upper_reg; ++i) {
885 uint32_t grf_addr = i * m_dsBuilder.getGRF_BYTES_PER_REG();
886 bit_set.set(grf_addr, m_dsBuilder.getGRF_BYTES_PER_REG());
887 }
888 }
889
890 // set dst_range to dst_bits and src_range to src_bits
891 void DepSetBuilder::DpasMacroBuilder::setDstSrcBits(
892 const SrcRegRangeType &src_range, const DstRegRangeType &dst_range,
893 BitSet<> &src_bits, BitSet<> &dst_bits) const {
894 for (const auto &regs : src_range) {
895 setBits(src_bits, regs.first, regs.second);
896 }
897 setBits(dst_bits, dst_range.first, dst_range.second);
898 }
899
900 // check if the given register ranges having intersection
901 bool DepSetBuilder::DpasMacroBuilder::hasIntersect(
902 const DepSet::RegRangeType &rr1, const DepSet::RegRangeType &rr2) const {
903 BitSet<> rr1bits(m_dsBuilder.getGRF_LEN());
904 BitSet<> rr2bits(m_dsBuilder.getGRF_LEN());
905 setBits(rr1bits, rr1.first, rr1.second);
906 setBits(rr2bits, rr2.first, rr2.second);
907
908 return rr1bits.intersects(rr2bits);
909 }
910
911 // If rr1 and rr2 footprint are all the same, return true.
912 bool DepSetBuilder::DpasMacroBuilder::hasEntireOverlap(
913 const DepSet::RegRangeType &rr1, const DepSet::RegRangeType &rr2) const {
914
915 // overlap, check if it's completely overlap
916 BitSet<> rr1bits(m_dsBuilder.getGRF_LEN());
917 BitSet<> rr2bits(m_dsBuilder.getGRF_LEN());
918 setBits(rr1bits, rr1.first, rr1.second);
919 setBits(rr2bits, rr2.first, rr2.second);
920 return rr1bits.equal(rr2bits);
921 }
922
923 // check if the instruction having internal dependency
924 // Instruction having internal dependency on dst to src is not allowed to be in
925 // a macro. Only for depth 8 dpas, internal dep on dst and src0 is allowed, but
926 // only when src0 and dst memory footprint is entirely the same
927 bool DepSetBuilder::DpasMacroBuilder::hasInternalDep(
928 const Instruction &dpas, const DstRegRangeType &dst_range,
929 const SrcRegRangeType &src_range, bool isDepth8) const {
930
931 const auto &dstOp = dpas.getDestination();
932 const auto &src0Op = dpas.getSource(0);
933
934 if (hasIntersect(dst_range, src_range[1]))
935 return true;
936
937 if (hasIntersect(dst_range, src_range[2]))
938 return true;
939
940 // if src0 is null, skip it
941 if (src_range[0].first != std::numeric_limits<uint32_t>::max()) {
942 if (!isDepth8 && hasIntersect(dst_range, src_range[0]))
943 return true;
944
945 // for depth 8 dpas, sr0 and dst having the same footprint is
946 // treated as no internal dependency for other rep_count, having
947 // intersect is internal dependency
948 if (isDepth8) {
949 // if the ranges entirely overlap but the subregisters are different, we
950 // have an internal dependency and hence, cannot form a macro
951 if (hasEntireOverlap(dst_range, src_range[0])) {
952 if (dstOp.getDirRegRef().subRegNum != src0Op.getDirRegRef().subRegNum)
953 return true;
954 // if there is partial overlap in registers, we cannot form a macro
955 } else if (hasIntersect(dst_range, src_range[0]))
956 return true;
957 }
958 }
959 return false;
960 }
961
962 size_t DepSetBuilder::DpasMacroBuilder::formFwdBlock(InstListIterator first) {
963 size_t count = 0;
964 InstListIterator cur = first;
965 SrcRegRangeType src_range, src_extra_range;
966 DstRegRangeType dst_range;
967 m_inps.getDpasSrcDependency(**cur, src_range, src_extra_range, m_model);
968 m_inps.getDpasDstDependency(**cur, dst_range);
969 if (hasInternalDep(**cur, dst_range, src_range,
970 GetDpasSystolicDepth((*cur)->getDpasFc()) == 8))
971 return count;
972
973 InstListIterator next = cur;
974 next++;
975 while (next != m_instList.end()) {
976
977 if (nextIsNotMacroCandidate(**cur, **next))
978 break;
979 SrcRegRangeType next_src_range, next_src_extra_range;
980 DstRegRangeType next_dst_range;
981 m_inps.getDpasSrcDependency(**next, next_src_range, next_src_extra_range,
982 m_model);
983 m_inps.getDpasDstDependency(**next, next_dst_range);
984 // no need to check the producer-consumer dependency (leave
985 // allDstBits/allSrcBits as 0) since if FWD block can be formed, add the dst
986 // must be the same
987 BitSet<> allDstBits(m_dsBuilder.getGRF_LEN());
988 BitSet<> allSrcBits(m_dsBuilder.getGRF_LEN());
989 if (canFwd(**cur, **next, dst_range, src_range, next_dst_range,
990 next_src_range, allDstBits, allSrcBits)) {
991 (*cur)->addInstOpt(InstOpt::ATOMIC);
992 (*cur)->addInstOpt(InstOpt::FWD);
993 updateRegFootprintsToDepSets(next_src_range, next_src_extra_range,
994 next_dst_range);
995 if (!count) {
996 updateRegFootprintsToDepSets(src_range, src_extra_range, dst_range);
997 count++;
998 }
999 count++;
1000 } else {
1001 break;
1002 }
1003 src_range.assign(next_src_range.begin(), next_src_range.end());
1004 src_extra_range.assign(next_src_extra_range.begin(),
1005 next_src_extra_range.end());
1006 dst_range = next_dst_range;
1007 cur = next;
1008 next++;
1009 }
1010 return count;
1011 }
1012
1013 // Check if this dpas can have {Fwd} set with the next dpas within this macro
1014 bool DepSetBuilder::DpasMacroBuilder::canFwd(
1015 const Instruction &cur, const Instruction &next,
1016 const DstRegRangeType &cur_dst_range, const SrcRegRangeType &cur_src_range,
1017 const DstRegRangeType &next_dst_range,
1018 const SrcRegRangeType &next_src_range, const BitSet<> &prev_dst_bits,
1019 const BitSet<> &prev_src_bits) const {
1020
1021 // this function must be called after next is verified can be a candidate
1022 assert(!nextIsNotMacroCandidate(cur, next));
1023
1024 // 0. check if can use Fwd
1025 if (!cur.getOpSpec().supportsFwdCtrl() ||
1026 m_encMode == SWSB_ENCODE_MODE::FiveDistPipeCvtToIntNoFwd)
1027 return false;
1028
1029 // 1. this and next DPAS are dpas8x8
1030 if (cur.getDpasFc() != DpasFC::F_8X8 || next.getDpasFc() != DpasFC::F_8X8)
1031 return false;
1032
1033 // 2. the very next DPAS's src0 and dst are both identical to current DPAS's
1034 // dst Execution size of both instructions must be the same at this point
1035 // (checked in nextIsNotMacroCandidate), so only needs to check if they have
1036 // the same register number
1037 if (next.getSource(0).getDirRegName() == RegName::ARF_NULL)
1038 return false;
1039 RegRef regref = cur.getDestination().getDirRegRef();
1040 if (regref != next.getDestination().getDirRegRef() ||
1041 regref != next.getSource(0).getDirRegRef())
1042 return false;
1043
1044 // 3. If current's dst type is int32, current dst and next src0/dst types must
1045 // all be the same. If current's and next's dst type is fp32 or bf16,
1046 // Fwd is allowd as long as current's dst type is the same as next's src0's
1047 Type dstType = cur.getDestination().getType();
1048 // The valid types are fp32, int32, bf16
1049 if (dstType != Type::BF && TypeSizeInBits(dstType) != 32)
1050 return false;
1051 if (dstType != next.getSource(0).getType())
1052 return false;
1053 Type nextDstType = next.getDestination().getType();
1054 if (dstType != nextDstType &&
1055 !isValidMixedTypes(dstType, nextDstType))
1056 return false;
1057
1058 // 4. next DPAS's src1 and src2 can't have dependency to current DPAS's dst
1059 BitSet<> cur_dst_bits(m_dsBuilder.getGRF_LEN());
1060 BitSet<> next_src12_bits(m_dsBuilder.getGRF_LEN());
1061 setBits(cur_dst_bits, cur_dst_range.first, cur_dst_range.second);
1062 auto i = next_src_range.begin();
1063 ++i;
1064 for (; i != next_src_range.end(); ++i) {
1065 setBits(next_src12_bits, (*i).first, (*i).second);
1066 }
1067 if (cur_dst_bits.intersects(next_src12_bits))
1068 return false;
1069
1070 // 5. both instructions can't have producer-consumer dependency to prior
1071 // instructions in the macro
1072 if (hasProducerConsumerDep(cur_dst_range, cur_src_range, prev_dst_bits) ||
1073 hasProducerConsumerDep(next_dst_range, next_src_range, prev_dst_bits))
1074 return false;
1075
1076 return true;
1077 }
1078
1079 std::pair<DepSet *, DepSet *> DepSetBuilder::createDPASSrcDstDepSet(
1080 const InstList &insList, InstListIterator instIt,
1081 const InstIDs &inst_id_counter, size_t &dpasCnt,
1082 SWSB_ENCODE_MODE enc_mode) {
1083
1084 // create DepSet for input
1085 DepSet *inps = new DepSet(inst_id_counter, *this);
1086 mAllDepSet.push_back(inps);
1087 inps->setDepType(DEP_TYPE::READ);
1088 setDEPPipeClass(enc_mode, *inps, **instIt, mPlatformModel);
1089
1090 // create DepSet for output
1091 DepSet *oups = new DepSet(inst_id_counter, *this);
1092 mAllDepSet.push_back(oups);
1093 oups->setDepType(DEP_TYPE::WRITE);
1094 setDEPPipeClass(enc_mode, *oups, **instIt, mPlatformModel);
1095
1096 // identify dpas macro
1097 DpasMacroBuilder dmb(
1098 *this, mPlatformModel, insList, instIt, *inps, *oups, enc_mode);
1099 Instruction &lastDpas = dmb.formMacro(dpasCnt);
1100
1101 // let the last instruciton in the macro represent this DepSet
1102 inps->m_instruction = &lastDpas;
1103 oups->m_instruction = &lastDpas;
1104
1105 return std::make_pair(inps, oups);
1106 }
1107
1108 DepSet *DepSetBuilder::createSrcDepSet(Instruction &i,
1109 const InstIDs &inst_id_counter,
1110 SWSB_ENCODE_MODE enc_mode) {
1111 DepSet *inps = new DepSet(inst_id_counter, *this);
1112 mAllDepSet.push_back(inps);
1113
1114 inps->m_instruction = &i;
1115 inps->setDepType(DEP_TYPE::READ);
1116
1117 setDEPPipeClass(enc_mode, *inps, i, mPlatformModel);
1118
1119 inps->setInputsFlagDep();
1120 if (i.getOpSpec().isAnySendFormat())
1121 inps->setInputsSendDescDep();
1122 inps->setInputsSrcDep();
1123 return inps;
1124 }
1125
1126 void DepSet::addGrf(size_t reg) {
1127 addGrfBytes(reg, 0, m_DB.getGRF_BYTES_PER_REG());
1128 }
1129
1130 void DepSet::setInputsFlagDep() {
1131 // does it read the flag register
1132 // predication does this
1133 // conditional modifier on 'sel' does this
1134 const Predication &pred = m_instruction->getPredication();
1135 const FlagModifier fm = m_instruction->getFlagModifier();
1136 bool readsFlagRegister =
1137 pred.function != PredCtrl::NONE ||
1138 (m_instruction->getOp() == Op::SEL && fm != FlagModifier::NONE);
1139 if (readsFlagRegister) {
1140 // add the ARF offset from ExecMaskOffset
1141 // E.g.
1142 // (f1.0) op (16|M16) ...
1143 // is touching f1.1
1144 const RegRef &fr = m_instruction->getFlagReg();
1145 size_t fByteOff =
1146 (size_t)fr.regNum * m_DB.getARF_F_BYTES_PER_REG() +
1147 (size_t)fr.subRegNum * 2; // FIXME: magic number (needs some thought
1148 // should be bytes per subreg)
1149 size_t execOff =
1150 4 * (static_cast<size_t>(m_instruction->getChannelOffset()));
1151 fByteOff += execOff / 8; // move over by ARF offset
1152 size_t execSize = static_cast<size_t>(m_instruction->getExecSize());
1153 size_t addr = (size_t)m_DB.getARF_F_START() + fByteOff;
1154 addFBytes(addr, execSize);
1155 m_bucketList.push_back(addr / m_DB.getBYTES_PER_BUCKET());
1156 }
1157 }
1158
1159 void DepSet::setInputsSendDescDep() {
1160 IGA_ASSERT(m_instruction->getOpSpec().isAnySendFormat(),
1161 "DepSet::setInputsSendDescDep: must be send format");
1162 // set up reg footprint for ID0, ID1
1163 if (m_instruction->getOpSpec().isSendgFormat()) {
1164 // ID0 and ID1 are qword descriptors
1165 auto id0 = m_instruction->getSendgIndDesc0Reg();
1166 if (id0 != REGREF_INVALID)
1167 setSendSrcScalarRegRegion(id0, 8);
1168
1169 auto id1 = m_instruction->getSendgIndDesc1Reg();
1170 if (id1 != REGREF_INVALID)
1171 setSendSrcScalarRegRegion(id1, 8);
1172 return;
1173 }
1174 // set up reg footprint for desc and exdesc if they are registers
1175 auto desc = m_instruction->getMsgDescriptor();
1176 if (desc.isReg()) {
1177 addABytesAndBukets(desc.reg.regNum);
1178 }
1179 auto exDesc = m_instruction->getExtMsgDescriptor();
1180 if (exDesc.isReg()) {
1181 addABytesAndBukets(exDesc.reg.regNum);
1182 }
1183 }
1184
1185 void DepSet::setInputsSrcDep() {
1186 uint32_t execSize = static_cast<uint32_t>(m_instruction->getExecSize());
1187
1188 // mac/mach has implicitly read to acc0
1189 if (m_instruction->getOp() == Op::MAC || m_instruction->getOp() == Op::MACH ||
1190 m_instruction->getOp() == Op::MACL) {
1191 setSrcRegion(RegName::ARF_ACC, RegRef(0, 0), Region::SRC110, execSize,
1192 16); // assume it's :w, though for acc access it actually does
1193 // not matter, because its footprint will always count as
1194 // acc0/acc1 pair
1195 }
1196
1197 // For the instruction having no srcs, we still need to add ARF_CR to mark the
1198 // dependency This is for the case that:
1199 // and (1|M0) cr0.0<1>:ud cr0.0<0;1,0>:ud 0xFFFFFFCF:ud
1200 // {A@1} nop {A@1}
1201 // It's requested that the instruction following the one having architecture
1202 // register (CR/CE/SR) access, It must mark to sync with all pipes, even if
1203 // it's a nop. nop has no src and dst so we mark it here to force setting swsb
1204 // if required
1205 //
1206 // For sync instructions we do not need to consider its dependency. Though it
1207 // may still apply the above case so still set ARF_CR to it.
1208 if (m_instruction->getSourceCount() == 0 ||
1209 m_instruction->getOpSpec().is(Op::SYNC)) {
1210 m_bucketList.push_back(m_DB.getBucketStart(RegName::ARF_CR));
1211 return;
1212 }
1213
1214 // Check if inst has intra-read-suppresion: If a multi-src inst has two or
1215 // more src read to the same register, regardless if they are overlapping,
1216 // it is considered having intra-read-suppression. Will need to force set
1217 // the src's footprint to the full register when needIntraReadSuppressionWA
1218 // Keep track of the touch register number in trackUseGrfNum and check
1219 // during setting each src's footprint.
1220 std::vector<unsigned> trackUseGrfNum;
1221
1222 // check all the source operands
1223 for (unsigned srcIx = 0, numSrcs = m_instruction->getSourceCount();
1224 srcIx < numSrcs; srcIx++) {
1225 const Operand &op = m_instruction->getSource(srcIx);
1226 auto tType = op.getType();
1227 uint32_t typeSizeInBits = TypeSizeInBitsWithDefault(tType, 32);
1228
1229 // possibly need to fixup the region
1230 Region rgn = op.getRegion();
1231 if (numSrcs == 3) {
1232 if (m_instruction->isMacro()) {
1233 // macros and sends are packed access only
1234 // madm (4) r3.acc1 r7.acc2 r9.acc4 r6.noacc
1235 rgn = Region::SRC110;
1236 } else {
1237 // ternary align 1 has some implicit regions that need filling in
1238 //
1239 // c.f. https://gfxspecs.intel.com/Predator/Home/Index/3017
1240 // "GEN10 Regioning Rules for Align1 Ternary Operations"
1241 // 1. Width is 1 when Vertical and Horizontal Strides are both zero
1242 // (broadcast access).
1243 // 2. Width is equal to Vertical Stride when Horizontal Stride is
1244 // zero.
1245 // 3. Width is equal to Vertical Stride/Horizontal Stride when both
1246 // Strides are non-zero.
1247 // 4. Vertical Stride must not be zero if Horizontal Stride is
1248 // non-zero.
1249 // This implies Vertical Stride is always greater than Horizontal
1250 // Stride.
1251 // 5. For Source 2, if Horizontal Stride is non-zero, then Width is
1252 // the
1253 // a register's width of elements (e.g. 8 for a 32-bit data type).
1254 // Otherwise, if Horizontal Stride is 0, then so is the Vertical
1255 // (and rule 1 applies). This means Vertical Stride is always
1256 // 'Width' * 'Horizontal Stride'.
1257 if (srcIx < 2) {
1258 // <V;H>
1259 if (rgn.getVt() == Region::Vert::VT_0 &&
1260 rgn.getHz() == Region::Horz::HZ_0) {
1261 rgn.set(Region::Width::WI_1); // by #1
1262 } else if (rgn.getHz() == Region::Horz::HZ_0) {
1263 rgn.w = rgn.v; // by #2
1264 } else if (rgn.getVt() != Region::Vert::VT_0 &&
1265 rgn.getHz() != Region::Horz::HZ_0) {
1266 rgn.w = rgn.v / rgn.h; // by #3
1267 } else {
1268 // error condition #4, just use vertical stride
1269 // SPECIFY: should this be an assertion?
1270 rgn.w = rgn.v;
1271 }
1272 } else { // (srcIx == 2)
1273 // <H>
1274 // <H> -> <H;1,0>
1275 // if (rgn.h == Region::HZ_0) {
1276 // rgn.w = Region::WI_1; // by #5 and #1
1277 //} else {
1278 // rgn.w = static_cast<Region::Width>(GRF_BYTES_PER_REG/typeSize);
1279 // // by #5
1280 //}
1281 // rgn.v = static_cast<Region::Vert>(
1282 // static_cast<int>(rgn.w)*static_cast<int>(rgn.h)/typeSize); // by
1283 // #5
1284 rgn.set(static_cast<Region::Vert>(static_cast<size_t>(rgn.h)),
1285 Region::Width::WI_1, Region::Horz::HZ_0);
1286 }
1287 }
1288 } else if (numSrcs == 2 && m_instruction->isMacro()) {
1289 // math macro
1290 rgn = Region::SRC110;
1291 } // not ternary, not macro
1292
1293 switch (op.getKind()) {
1294 case Operand::Kind::DIRECT:
1295 if (m_instruction->getOpSpec().isAnySendFormat()) {
1296 if (op.getDirRegName() == RegName::GRF_R) {
1297 // send source GRF (not null reg)
1298 int nregs = (srcIx == 0) ? m_instruction->getSrc0Length()
1299 : m_instruction->getSrc1Length();
1300 // if we can't tell the number of registers
1301 // (e.g. the descriptor is in a register),
1302 // then we must conservatively assume the worst (32)
1303 if (nregs < 0)
1304 nregs = 32;
1305 uint32_t regNum = op.getDirRegRef().regNum;
1306 for (uint32_t i = 0; i < (uint32_t)nregs; i++) {
1307 if ((regNum + i) >= m_DB.getGRF_REGS()) {
1308 break;
1309 }
1310 addGrf((size_t)regNum + i);
1311 addToBucket(regNum + i);
1312 }
1313 addToBucket(m_DB.getBucketStart(RegName::ARF_CR));
1314 }
1315 } else {
1316 if (m_instruction->getOp() == Op::BRC) {
1317 rgn = Region::SRC221;
1318 }
1319
1320 auto useBuckets = setSrcRegion(op.getDirRegName(), op.getDirRegRef(),
1321 rgn, execSize, typeSizeInBits);
1322
1323 // check and add extra footprint for intra read suppression instructions.
1324 // The WA only applies to GRF. In the case, bucket number is equal to
1325 // GRF number.
1326 if (m_DB.needIntraReadSuppressionWA() &&
1327 op.getDirRegName() == RegName::GRF_R) {
1328 IGA_ASSERT(useBuckets.first != m_DB.getTOTAL_BUCKETS(),
1329 "SWSB: GRF src must have valid bucket number");
1330 uint32_t grfNum = useBuckets.first;
1331 for (uint32_t i = 0; i < useBuckets.second; ++i, ++grfNum) {
1332 if (std::find(trackUseGrfNum.begin(), trackUseGrfNum.end(),
1333 useBuckets.first) != trackUseGrfNum.end()) {
1334 // found the same grf use by different srcs. Set this DepSet to
1335 // have full register footprint
1336 addGrf(grfNum);
1337 } else {
1338 // no intra read to previous src
1339 trackUseGrfNum.push_back(grfNum);
1340 }
1341 }
1342 }
1343 }
1344 break;
1345 case Operand::Kind::MACRO: {
1346 setSrcRegion(op.getDirRegName(), op.getDirRegRef(), rgn, execSize,
1347 typeSizeInBits);
1348 auto mme = op.getMathMacroExt();
1349 if (mme != MathMacroExt::NOMME && mme != MathMacroExt::INVALID) {
1350 int mmeNum = static_cast<int>(op.getMathMacroExt()) -
1351 static_cast<int>(MathMacroExt::MME0);
1352 setSrcRegion(RegName::ARF_ACC, RegRef(mmeNum, 0), Region::SRC110,
1353 execSize, typeSizeInBits);
1354 }
1355 break;
1356 }
1357 case Operand::Kind::INDIRECT: {
1358 setHasIndirect();
1359 setDepType(DEP_TYPE::READ_ALWAYS_INTERFERE);
1360
1361 if (op.getDirRegName() == RegName::ARF_S) {
1362 IGA_ASSERT(m_instruction->isGatherSend(),
1363 "Non-send instruction has indirect scalar register src");
1364 // Mark src access to 1 byte that we will insert force sync before and
1365 // after gather send so the actual length doesn't matter.
1366 setSendSrcScalarRegRegion(op.getIndAddrReg(), 1);
1367 break;
1368 }
1369
1370 auto rgn = op.getRegion();
1371 if (rgn.getVt() == Region::Vert::VT_VxH) {
1372 // VxH or Vx1 mode
1373 // op (K) dst r[a0.0]<W,1>:w (reads K/W elements)
1374 // op (K) dst r[a0.0]<1,0>:w (reads K/W elements)
1375 setSrcRegion(RegName::ARF_A, op.getIndAddrReg(), Region::SRC110,
1376 execSize / rgn.w, //
1377 16); // :w is 16-bits a piece
1378 } else {
1379 // uniform: consumes one value in a0
1380 // op (..) dst r[a0.0]<16,8,1>:w
1381 setSrcRegion(RegName::ARF_A, op.getIndAddrReg(), Region::SRC110,
1382 1, // 1 element only
1383 16); // :w is 16-bits a piece
1384 }
1385 // we can't do anything else for this
1386 break;
1387 }
1388 default:
1389 break;
1390 }
1391 }
1392 }
1393
1394 void DepSet::setOutputsFlagDep() {
1395 const FlagModifier fm = m_instruction->getFlagModifier();
1396 bool writesFlagRegister =
1397 fm != FlagModifier::NONE &&
1398 m_instruction->getOp() != Op::SEL; // sel uses flag modifier as input
1399 if (writesFlagRegister) {
1400 const RegRef &fr = m_instruction->getFlagReg();
1401 int fByteOff = fr.regNum * m_DB.getARF_F_BYTES_PER_REG() +
1402 fr.subRegNum * 2; // 2 bytes per subreg
1403 int execOff = 4 * (static_cast<int>(m_instruction->getChannelOffset()));
1404 fByteOff += execOff / 8; // move over by ARF offset
1405 int execSize = static_cast<int>(m_instruction->getExecSize()); // 1 <<
1406 // (static_cast<int>(m_instruction->getExecSize())
1407 // - 1);
1408 size_t addr = (size_t)m_DB.getARF_F_START() + fByteOff;
1409 addFBytes(addr, execSize);
1410 m_bucketList.push_back(addr / m_DB.getBYTES_PER_BUCKET());
1411 }
1412 }
1413
1414 void DepSet::setOutputsDstDep() {
1415 int execOff = 4 * (static_cast<int>(m_instruction->getChannelOffset()));
1416 int execSize = static_cast<int>(m_instruction->getExecSize()); // 1 <<
1417 // (static_cast<int>(m_instruction->getExecSize())
1418 // - 1);
1419
1420 if (!m_instruction->getOpSpec().supportsDestination()) {
1421 // For the instruction having no srcs, we still need to add ARF_CR to mark
1422 // the dependency This is for the case that:
1423 // and (1|M0) cr0.0<1>:ud cr0.0<0;1,0>:ud 0xFFFFFFCF:ud
1424 // {A@1} nop {A@1}
1425 // It's requested that the instruction following the one having architecture
1426 // register (CR/CE/SR) access, It must mark to sync with all pipes, even if
1427 // it's a nop. nop has no src and dst so we mark it here to force setting
1428 // swsb if required
1429 m_bucketList.push_back(m_DB.getBucketStart(RegName::ARF_CR));
1430 return;
1431 }
1432
1433 const auto &op = m_instruction->getDestination();
1434 auto tType = op.getType();
1435 auto typeSizeBits = TypeSizeInBitsWithDefault(tType, 32);
1436
1437 // A trick to correct the call dst footprint that its dst type is implicit dw
1438 // but it actually writes to continuous two dw. This causes the issue
1439 // for SIMD1 case. Double the exec_size to workaround the case. For other simd
1440 // width, it's ok since we already conservative assume the dst footprint by
1441 // times of exec_size, which is not neccessary for call/calla that their dst
1442 // is scalar 2 dw.
1443 if (m_instruction->getOp() == Op::CALL ||
1444 m_instruction->getOp() == Op::CALLA) {
1445 assert(tType == Type::D && op.getKind() == Operand::Kind::DIRECT &&
1446 op.getDirRegName() == RegName::GRF_R);
1447 if (execSize == 1)
1448 execSize = 2;
1449 // for 64b IP platform, the dst register footprint size of call/calla is
1450 // 3 dws
1451 if (m_DB.has64bitIPAddress() && execSize < 3) {
1452 execSize = 3;
1453 }
1454 }
1455
1456 // Instructions having implicit write to acc
1457 if (m_instruction->hasInstOpt(InstOpt::ACCWREN) ||
1458 m_instruction->getOp() == Op::SUBB ||
1459 m_instruction->getOp() == Op::ADDC ||
1460 m_instruction->getOp() == Op::MACL ||
1461 m_instruction->getOp() == Op::MACH) {
1462 auto elemsPerAccReg = 8 * m_DB.getARF_ACC_BYTES_PER_REG() /
1463 typeSizeBits; // e.g. 8 subreg elems for :f
1464 RegRef ar;
1465 ar.regNum = (uint16_t)(execOff / elemsPerAccReg);
1466 ar.subRegNum = (uint16_t)(execOff % elemsPerAccReg);
1467 setDstRegion(RegName::ARF_ACC, ar, Region::DST1, execSize, typeSizeBits);
1468 }
1469
1470 Region rgn = op.getRegion();
1471 switch (op.getKind()) {
1472 case Operand::Kind::DIRECT:
1473 // send target (a GRF, not null reg)
1474 if (m_instruction->getOpSpec().isAnySendFormat() &&
1475 op.getDirRegName() == RegName::GRF_R) {
1476 int nregs = m_instruction->getDstLength();
1477 // getDstLength return -1 when it's not able to deduce the length
1478 // we have to be conservative and use the max
1479 if (nregs < 0)
1480 nregs = 32;
1481 for (uint32_t i = 0; i < (uint32_t)nregs; i++) {
1482 uint32_t regNum = op.getDirRegRef().regNum;
1483 if ((regNum + i) >= m_DB.getGRF_REGS()) {
1484 break;
1485 }
1486
1487 addGrf((size_t)regNum + i);
1488 addToBucket(regNum + i);
1489 }
1490 rgn = Region::DST1;
1491 addToBucket(m_DB.getBucketStart(RegName::ARF_CR));
1492
1493 } else if (m_instruction->getOpSpec().is(Op::MATH) &&
1494 op.getDirRegName() == RegName::GRF_R &&
1495 m_instruction->getMathFc() == MathFC::IDIV) {
1496 uint16_t regNum = op.getDirRegRef().regNum;
1497 addGrf(regNum);
1498 addToBucket(regNum);
1499 addGrf((size_t)regNum + 1);
1500 addToBucket((size_t)regNum + 1);
1501 addToBucket(m_DB.getBucketStart(RegName::ARF_CR));
1502 } else {
1503 // normal GRF target
1504 setDstRegion(op.getDirRegName(), op.getDirRegRef(), op.getRegion(),
1505 execSize, typeSizeBits);
1506 }
1507 break;
1508 case Operand::Kind::MACRO: {
1509 // math macro
1510 // GRF + ACC
1511 setDstRegion(op.getDirRegName(), op.getDirRegRef(), Region::DST1, execSize,
1512 typeSizeBits);
1513 auto mme = op.getMathMacroExt();
1514 if (mme != MathMacroExt::NOMME && mme != MathMacroExt::INVALID) {
1515 // and the accumultator
1516 int mmeNum = static_cast<int>(mme) - static_cast<int>(MathMacroExt::MME0);
1517 RegRef mmeReg{static_cast<uint8_t>(mmeNum), 0};
1518 setDstRegion(RegName::ARF_ACC, mmeReg, Region::DST1, execSize,
1519 typeSizeBits);
1520 }
1521 break;
1522 }
1523 case Operand::Kind::INDIRECT:
1524 setHasIndirect();
1525
1526 // indirect destinations use a0
1527 //
1528 // writes use one a0 value
1529 setDstRegion(RegName::ARF_A, op.getIndAddrReg(), Region::DST1,
1530 1, // one element only
1531 16); // :w is 16-bits
1532 setDepType(DEP_TYPE::WRITE_ALWAYS_INTERFERE);
1533
1534 break;
1535 default:
1536 break;
1537 }
1538
1539 if (m_instruction->getOpSpec().isSendgFormat() &&
1540 m_DB.needSyncAfterFence() &&
1541 m_DB.isSendgFence(*m_instruction))
1542 setDepType(DEP_TYPE::WRITE_ALWAYS_INTERFERE);
1543 }
1544
1545 bool DepSetBuilder::isSendgFence(const Instruction& sendg) const {
1546 auto desc = sendg.getSendgDesc();
1547 // decode the message op code from desc bit[5:0]
1548 // fence has opcode value 31
1549 if ((desc & 0x7F) == 31)
1550 return true;
1551 return false;
1552 }
1553
1554 DepSet *DepSetBuilder::createDstDepSet(Instruction &i,
1555 const InstIDs &inst_id_counter,
1556 SWSB_ENCODE_MODE enc_mode) {
1557 DepSet *oups = new DepSet(inst_id_counter, *this);
1558 mAllDepSet.push_back(oups);
1559
1560 oups->m_instruction = &i;
1561 setDEPPipeClass(enc_mode, *oups, i, mPlatformModel);
1562
1563 oups->setDepType(DEP_TYPE::WRITE);
1564
1565 oups->setOutputsFlagDep();
1566 oups->setOutputsDstDep();
1567 return oups;
1568 }
1569
1570 DepSet *DepSetBuilder::createDstDepSetFullGrf(Instruction &i,
1571 const InstIDs &inst_id_counter,
1572 SWSB_ENCODE_MODE enc_mode,
1573 bool setFlagDep) {
1574 DepSet *oups = new DepSet(inst_id_counter, *this);
1575 mAllDepSet.push_back(oups);
1576
1577 oups->m_instruction = &i;
1578 setDEPPipeClass(enc_mode, *oups, i, mPlatformModel);
1579
1580 oups->setDepType(DEP_TYPE::WRITE);
1581 if (setFlagDep)
1582 oups->setOutputsFlagDep();
1583 oups->setOutputsDstDepFullGrf();
1584 return oups;
1585 }
1586
1587 void DepSet::setOutputsDstDepFullGrf() {
1588 size_t execSize = static_cast<size_t>(m_instruction->getExecSize()); // 1 <<
1589 // (static_cast<int>(m_instruction->getExecSize())
1590 // - 1);
1591
1592 const auto &op = m_instruction->getDestination();
1593 auto tType = op.getType();
1594 auto typeSizeBits = TypeSizeInBitsWithDefault(tType, 32);
1595
1596 Region rgn = op.getRegion();
1597 switch (op.getKind()) {
1598 case Operand::Kind::DIRECT: {
1599 RegName rn = op.getDirRegName();
1600 if (rn != RegName::GRF_R)
1601 return;
1602 RegRef rr = op.getDirRegRef();
1603 m_dType = DEP_TYPE::WRITE;
1604
1605 size_t hz = static_cast<size_t>(rgn.getHz());
1606 // sets a region for a basic operand
1607 size_t grfAddr = addressOf(rn, rr, typeSizeBits);
1608 if (grfAddr >= m_DB.getTOTAL_BITS())
1609 return;
1610
1611 size_t lowBound = m_DB.getTOTAL_BITS();
1612 size_t upperBound = 0;
1613
1614 // caculate the access registers range from region
1615 for (size_t ch = 0; ch < execSize; ch++) {
1616 size_t offset = ch * hz * typeSizeBits / 8;
1617 size_t start = grfAddr + offset;
1618 // bits.set(start, typeSizeBits / 8);
1619
1620 if (start < lowBound) {
1621 lowBound = start;
1622 }
1623 if (start + typeSizeBits / 8 > upperBound) {
1624 upperBound = start + typeSizeBits / 8;
1625 }
1626 }
1627 size_t startRegNum = lowBound / m_DB.getBYTES_PER_BUCKET();
1628 size_t upperRegNum = (upperBound - 1) / m_DB.getBYTES_PER_BUCKET();
1629
1630 for (size_t i = startRegNum; i <= upperRegNum; i++) {
1631 // set the entire grf
1632 addGrf(i);
1633 m_bucketList.push_back(i);
1634 }
1635 addToBucket(m_DB.getBucketStart(RegName::ARF_CR));
1636 break;
1637 }
1638 default:
1639 // if the dst is not a GRF, set dep as regular cases
1640 setOutputsDstDep();
1641 break;
1642 }
1643 }
1644
1645 void DepSet::getDpasDstDependency(const Instruction &inst,
1646 RegRangeType &reg_range) {
1647 uint32_t execSize = static_cast<uint32_t>(inst.getExecSize());
1648
1649 const auto &op = inst.getDestination();
1650 auto tType = op.getType();
1651 uint32_t typeSizeBits = TypeSizeInBitsWithDefault(tType, 32);
1652 IGA_ASSERT(op.getDirRegName() == RegName::GRF_R, "DPAS with non-GRF");
1653
1654 // calculated used register region low and upper bound
1655 uint32_t lowBound =
1656 addressOf(op.getDirRegName(), op.getDirRegRef(), typeSizeBits);
1657 // elements_size is the size of total elements to be calculated in one
1658 // operation
1659 uint32_t elements_size = execSize * typeSizeBits / 8;
1660
1661 // For dpas, the destination region depends on Reapeat count
1662 uint32_t repeatCount = GetDpasRepeatCount(inst.getDpasFc());
1663 uint32_t upperBound = lowBound + elements_size * repeatCount;
1664
1665 uint32_t startRegNum = lowBound / m_DB.getGRF_BYTES_PER_REG();
1666 uint32_t upperRegNum = (upperBound - 1) / m_DB.getGRF_BYTES_PER_REG();
1667
1668 reg_range.first = startRegNum;
1669 reg_range.second = upperRegNum;
1670 }
1671
1672 bool static const isSpecial(RegName rn) {
1673 // The special registers (architecture registers) are CR/SR/CE
1674 // All the other ARF should have explicitly access so no need
1675 // the special handling on it
1676 // Others are SP, IP, IM, DBG
1677 if (rn == RegName::ARF_CR || rn == RegName::ARF_SR || rn == RegName::ARF_CE ||
1678 rn == RegName::ARF_FC) {
1679 return true;
1680 }
1681 return false;
1682 }
1683
1684 void DepSet::setSendSrcScalarRegRegion(const RegRef &rr, uint32_t numBytes) {
1685 uint32_t lowBound = addressOf(RegName::ARF_S, rr, 8);
1686 uint32_t upperBound = lowBound + numBytes;
1687 uint32_t startBucketNum = lowBound / m_DB.getBYTES_PER_BUCKET();
1688 uint32_t upperBucketNum = (upperBound - 1) / m_DB.getBYTES_PER_BUCKET();
1689
1690 bits->set(lowBound, numBytes);
1691 for (uint32_t i = startBucketNum; i <= upperBucketNum; i++) {
1692 m_bucketList.push_back(i);
1693 }
1694
1695 // Using one of the special registers to add read dependency in to special
1696 // bucket This way it will always check that implicit dependency
1697 m_bucketList.push_back(m_DB.getBucketStart(RegName::ARF_CR));
1698 }
1699
1700 std::pair<uint32_t, uint32_t>
1701 DepSet::setSrcRegion(RegName rn, RegRef rr, Region rgn, uint32_t execSize,
1702 uint32_t typeSizeBits) {
1703 // Previously we also skip acc register with subRegNum < 2 ==> WHY?
1704 // The condition is (rn == RegName::ARF_ACC && rr.subRegNum < 2)
1705 if (!isRegTracked(rn)) {
1706 // In the case that an instruction has only non-tracked register access,
1707 // we still want it to mark swsb if its previous instruction having
1708 // architecture register access, so add the special bucket ARF_CR anyway
1709 // in case of swsb is required
1710 // e.g.
1711 // (W) mov(1|M0) sr0.2<1>:ud 0xFFFFFFFF:ud {A@1}
1712 // (W) mov(1|M0) f0.0<1>:ud 0x0 : ud {A@1}
1713 // the second mov required A@1 be set
1714 m_bucketList.push_back(m_DB.getBucketStart(RegName::ARF_CR));
1715 return std::make_pair(m_DB.getTOTAL_BUCKETS(), 0);
1716 }
1717
1718 uint32_t v = 1, w = 1, h = 0; // e.g. old-style default regions
1719 if (rgn != Region::INVALID) {
1720 v = static_cast<uint32_t>(rgn.getVt());
1721 w = static_cast<uint32_t>(rgn.getWi());
1722 w = w == 0 ? 1 : w;
1723 h = static_cast<uint32_t>(rgn.getHz());
1724 }
1725
1726 uint32_t lowBound = m_DB.getTOTAL_BITS();
1727 uint32_t upperBound = 0;
1728
1729 // sets a region for a basic operand
1730 uint32_t rowBase = addressOf(rn, rr, typeSizeBits);
1731 if (rowBase >= m_DB.getTOTAL_BITS()) // tm0 or something
1732 return std::make_pair(m_DB.getTOTAL_BUCKETS(), 0);
1733
1734 // the acc0/acc1 could have overlapping. (same as acc2/acc3, acc4/acc5, ...)
1735 // To be conservative we always mark them together when
1736 // one of it is used
1737 // This is to resolve the acc arrangement difference between int and float
1738 // E.g. acc3.3:f and acc2.7:w have an overlap
1739 if (rn == RegName::ARF_ACC) {
1740 RegRef tmp_rr;
1741 tmp_rr.regNum = (rr.regNum % 2) == 0 ? rr.regNum : rr.regNum - 1;
1742 tmp_rr.subRegNum = 0;
1743 lowBound = addressOf(rn, tmp_rr, typeSizeBits);
1744 // reg access cross two acc
1745 upperBound = lowBound + 2 * m_DB.getARF_A_BYTES_PER_REG();
1746 bits->set(lowBound, 2 * (size_t)m_DB.getARF_A_BYTES_PER_REG());
1747 } else {
1748 uint32_t rows = execSize / w;
1749 rows = (rows != 0) ? rows : 1;
1750 // size_t bytesTouched = 0;
1751 for (uint32_t y = 0; y < rows; y++) {
1752 uint32_t offset = rowBase;
1753 for (uint32_t x = 0; x < w; x++) {
1754 bits->set(offset, typeSizeBits / 8);
1755 if (offset < lowBound) {
1756 lowBound = offset;
1757 }
1758 if (offset + typeSizeBits / 8 > upperBound) {
1759 upperBound = offset + typeSizeBits / 8;
1760 }
1761 offset += h * typeSizeBits / 8;
1762 // bytesTouched += typeSize;
1763 }
1764 rowBase += v * typeSizeBits / 8;
1765 }
1766 }
1767
1768 // size_t extentTouched = upperBound - lowBound;
1769 IGA_ASSERT(upperBound >= lowBound,
1770 "source region footprint computation got it wrong: upperBound "
1771 "is less than lowBound");
1772 uint32_t startRegNum = lowBound / m_DB.getBYTES_PER_BUCKET();
1773 uint32_t upperRegNum = (upperBound - 1) / m_DB.getBYTES_PER_BUCKET();
1774
1775 for (uint32_t i = startRegNum; i <= upperRegNum; i++) {
1776 m_bucketList.push_back(i);
1777 }
1778
1779 // Special registers has side effects so even if there is no direct
1780 // interference subsequent instrucion might depend on it. Also the prior
1781 // instrucions may depends on it so force to sync all pipes to this
1782 // instruction and the following instruction
1783 if (isSpecial(rn)) {
1784 setHasSR();
1785 m_dType = DEP_TYPE::READ_ALWAYS_INTERFERE;
1786 } else {
1787 // Using one of the special registers to add write dependency in to special
1788 // bucket This way it will always check that implicit dependency
1789 m_bucketList.push_back(m_DB.getBucketStart(RegName::ARF_CR));
1790 }
1791 return std::make_pair(startRegNum, upperRegNum - startRegNum + 1);
1792 }
1793
1794 void DepSet::setDstRegion(RegName rn, RegRef rr, Region rgn, uint32_t execSize,
1795 uint32_t typeSizeBits) {
1796 // Previously we also skip acc register with subRegNum < 2 ==> WHY?
1797 // The condition is (rn == RegName::ARF_ACC && rr.subRegNum < 2)
1798 if (!isRegTracked(rn)) {
1799 // In the case that an instruction has only non-tracked register access,
1800 // we still want it to mark swsb if its previous instruction having
1801 // architecture register access, so add the special bucket ARF_CR anyway
1802 // in case of swsb is required
1803 // e.g.
1804 // (W) mov(1|M0) sr0.2<1>:ud 0xFFFFFFFF:ud {A@1}
1805 // (W) mov(1|M0) f0.0<1>:ud 0x0 : ud {A@1}
1806 // the second mov required A@1 be set
1807 m_bucketList.push_back(m_DB.getBucketStart(RegName::ARF_CR));
1808 return;
1809 }
1810
1811 m_dType = DEP_TYPE::WRITE;
1812
1813 auto setBitsFromRegRef = [&](RegRef regRef) {
1814 // sets a region for a basic operand
1815 uint32_t grfAddr = addressOf(rn, regRef, typeSizeBits);
1816 if (grfAddr >= m_DB.getTOTAL_BITS())
1817 return;
1818 uint32_t lowBound = m_DB.getTOTAL_BITS();
1819 uint32_t upperBound = 0;
1820
1821 // the acc0/acc1 could have overlapping. (same as acc2/acc3, acc4/acc5, ...)
1822 // To be conservative we always mark them together when one of it is used
1823 // This is to resolve the acc arrangement difference between int and float
1824 // E.g. acc3.3:f and acc2.7:w have an overlap
1825 if (rn == RegName::ARF_ACC) {
1826 RegRef tmp_rr;
1827 tmp_rr.regNum =
1828 (regRef.regNum % 2) == 0 ? regRef.regNum : regRef.regNum - 1;
1829 tmp_rr.subRegNum = 0;
1830 lowBound = addressOf(rn, tmp_rr, typeSizeBits);
1831
1832 // reg access cross two acc
1833 upperBound = lowBound + 2 * m_DB.getARF_A_BYTES_PER_REG();
1834 bits->set(lowBound, 2 * (size_t)m_DB.getARF_A_BYTES_PER_REG());
1835 } else {
1836 // otherwise caculate the access registers range from region
1837 uint32_t hz = static_cast<uint32_t>(rgn.getHz());
1838 for (uint32_t ch = 0; ch < execSize; ch++) {
1839 uint32_t offset = ch * hz * typeSizeBits / 8;
1840 uint32_t start = grfAddr + offset;
1841 bits->set(start, typeSizeBits / 8);
1842
1843 if (start < lowBound) {
1844 lowBound = start;
1845 }
1846 if (start + typeSizeBits / 8 > upperBound) {
1847 upperBound = start + typeSizeBits / 8;
1848 }
1849 }
1850 }
1851
1852 if (isRegTracked(rn)) {
1853 uint32_t startRegNum = lowBound / m_DB.getBYTES_PER_BUCKET();
1854 uint32_t upperRegNum = (upperBound - 1) / m_DB.getBYTES_PER_BUCKET();
1855
1856 for (uint32_t i = startRegNum; i <= upperRegNum; i++) {
1857 // note: bucket start is already included in 'i' calculation
1858 // used to be: m_bucketList.push_back(i + DepSet::getBucketStart(rn));
1859 m_bucketList.push_back(i);
1860 }
1861 }
1862 };
1863 setBitsFromRegRef(rr);
1864
1865 if (m_instruction->getOp() == Op::MULLH) {
1866 // Execution sizes 1,2,4,8 and 16: write to dst, dst+1
1867 RegRef tmpRR((uint16_t)(rr.regNum + 1), rr.subRegNum);
1868 setBitsFromRegRef(tmpRR);
1869 // Execution size 32: write to dst, dst+1, dst+2, dst+3
1870 if (m_instruction->getExecSize() == ExecSize::SIMD32) {
1871 tmpRR.regNum++;
1872 setBitsFromRegRef(tmpRR);
1873 tmpRR.regNum++;
1874 setBitsFromRegRef(tmpRR);
1875 }
1876 }
1877
1878 // Special registers has side effects so even if there is no direct
1879 // interference subsequent instrucion might depend on it. Also the prior
1880 // instrucions may depends on it so force to sync all pipes to this
1881 // instruction and the following instruction
1882 if (isSpecial(rn)) {
1883 setHasSR();
1884 m_dType = DEP_TYPE::WRITE_ALWAYS_INTERFERE;
1885 } else {
1886 // Using one of the special registers to add write dependency in to special
1887 // bucket This way it will always check that implicit dependency
1888 m_bucketList.push_back(m_DB.getBucketStart(RegName::ARF_CR));
1889 }
1890 }
1891
1892 uint32_t DepSet::addressOf(RegName rnm, const RegRef &rr,
1893 uint32_t typeSizeBits) const {
1894 auto getAddressOf = [&rr, &typeSizeBits](uint32_t regStart,
1895 uint32_t bytePerReg) {
1896 return regStart + rr.regNum * bytePerReg + rr.subRegNum * typeSizeBits / 8;
1897 };
1898
1899 switch (rnm) {
1900 case RegName::GRF_R:
1901 return getAddressOf(m_DB.getGRF_START(), m_DB.getGRF_BYTES_PER_REG());
1902 case RegName::ARF_A:
1903 return getAddressOf(m_DB.getARF_A_START(), m_DB.getARF_A_BYTES_PER_REG());
1904 case RegName::ARF_ACC:
1905 return getAddressOf(m_DB.getARF_ACC_START(),
1906 m_DB.getARF_ACC_BYTES_PER_REG());
1907 case RegName::ARF_F:
1908 return getAddressOf(m_DB.getARF_F_START(), m_DB.getARF_F_BYTES_PER_REG());
1909 case RegName::ARF_CR:
1910 case RegName::ARF_SR:
1911 case RegName::ARF_CE:
1912 case RegName::ARF_FC:
1913 return getAddressOf(m_DB.getARF_SPECIAL_START(),
1914 m_DB.getARF_SPECIAL_BYTES_PER_REG());
1915 case RegName::ARF_S:
1916 return getAddressOf(m_DB.getARF_SCALAR_START(),
1917 m_DB.getARF_SCALAR_BYTES_PER_REG());
1918 default:
1919 return m_DB.getTOTAL_BITS();
1920 }
1921 }
1922
1923 bool DepSet::isRegTracked(RegName rnm) const {
1924 switch (rnm) {
1925 case RegName::GRF_R:
1926 case RegName::ARF_A:
1927 case RegName::ARF_ACC:
1928 case RegName::ARF_F:
1929 case RegName::ARF_S:
1930 return true;
1931 default:
1932 return false || isSpecial(rnm);
1933 }
1934 }
1935
1936 void DepSet::addGrfBytes(size_t reg, size_t subRegBytes, size_t num_bytes) {
1937 size_t grf_addr =
1938 m_DB.getGRF_START() + reg * m_DB.getGRF_BYTES_PER_REG() + subRegBytes;
1939 if (grf_addr < m_DB.getGRF_START() ||
1940 (grf_addr + num_bytes >
1941 (size_t)m_DB.getGRF_START() + m_DB.getGRF_LEN())) {
1942 IGA_FATAL("RegDeps: GRF index is out of bounds");
1943 }
1944 bits->set(grf_addr, num_bytes);
1945 }
1946
1947 void DepSet::addABytesAndBukets(size_t reg) {
1948 IGA_ASSERT(reg < m_DB.getARF_A_REGS(),
1949 "a# register out of bounds");
1950 size_t addr = m_DB.getARF_A_START() + m_DB.getARF_A_BYTES_PER_REG() * reg;
1951 IGA_ASSERT(addr < (size_t)m_DB.getARF_A_START() + m_DB.getARF_A_LEN(),
1952 "a# byte address out of bounds");
1953
1954 // ARF_A's swsb is tracked at per register base. sub-register number
1955 // doesn't matter.
1956 auto num_bytes = m_DB.getARF_A_BYTES_PER_REG();
1957 bits->set(addr, num_bytes);
1958 // set the corresponding buckets
1959 for (auto i = addr / m_DB.getBYTES_PER_BUCKET();
1960 i <= (addr + num_bytes - 1) / m_DB.getBYTES_PER_BUCKET(); i++) {
1961 addToBucket((uint32_t)i);
1962 }
1963 }
1964 void DepSet::addFBytes(size_t fByteOff, size_t num_bytes) {
1965 bits->set(fByteOff, num_bytes);
1966 }
1967
1968 bool DepSet::destructiveSubtract(const DepSet &rhs) {
1969 return bits->andNot(*rhs.bits);
1970 }
1971
1972 static void emitComma(std::ostream &os, bool &first) {
1973 if (first) {
1974 first = false;
1975 } else {
1976 os << ", ";
1977 }
1978 }
1979 void DepSet::formatShortReg(std::ostream &os, bool &first, const char *reg_name,
1980 size_t reg_num, // ((size_t)-1) means we ignore
1981 size_t reg_start, size_t reg_len) const {
1982 if (bits->testAll(reg_start, reg_len)) {
1983 // full register is set
1984 emitComma(os, first);
1985 os << reg_name;
1986 if (reg_num != (size_t)-1)
1987 os << reg_num;
1988 } else if (bits->testAny(reg_start, reg_len)) {
1989 // partial register
1990 for (size_t ri = 0; ri < reg_len; ri++) {
1991 // find starting bit
1992 if (!bits->test(reg_start + ri)) {
1993 continue;
1994 }
1995 // ri is first set
1996 emitComma(os, first);
1997 os << reg_name;
1998 if (reg_num != (size_t)-1)
1999 os << reg_num;
2000 os << "." << ri;
2001
2002 // find the ending position
2003 size_t end_ri = ri;
2004 while (end_ri < reg_start + reg_len && bits->test(reg_start + end_ri))
2005 end_ri++;
2006 // e.g. r23.0-3 (a full DWORD)
2007 if (end_ri > ri + 1) // case of 1 byte omits range r0.0 (for :b)
2008 os << '-' << (end_ri - 1);
2009 // else: e.g. r23.0 (just one byte)
2010 ri = end_ri; // will move to the next element
2011 } // for
2012 } // else: nothing set in this register
2013 }
2014 // emits a shorter description... something like
2015 // {r0,r1.0-3,r7}
2016 void DepSet::str(std::ostream &os) const {
2017 bool first = true;
2018 os << "{";
2019
2020 for (size_t ri = 0; ri < m_DB.getGRF_REGS(); ri++) {
2021 formatShortReg(os, first, "r", ri,
2022 m_DB.getGRF_START() + ri * m_DB.getGRF_BYTES_PER_REG(),
2023 m_DB.getGRF_BYTES_PER_REG());
2024 }
2025
2026 for (uint32_t ai = 0; ai < m_DB.getARF_A_REGS(); ai++) {
2027 formatShortReg(os, first, "a", ai,
2028 m_DB.getARF_A_START() +
2029 (size_t)ai * m_DB.getARF_A_BYTES_PER_REG(),
2030 m_DB.getARF_A_BYTES_PER_REG());
2031 }
2032
2033 for (uint32_t acci = 0; acci < m_DB.getARF_ACC_REGS(); acci++) {
2034 formatShortReg(os, first, "acc", acci,
2035 m_DB.getARF_ACC_START() +
2036 (size_t)acci * m_DB.getARF_ACC_BYTES_PER_REG(),
2037 m_DB.getARF_ACC_BYTES_PER_REG());
2038 }
2039
2040 for (uint32_t fi = 0; fi < m_DB.getARF_F_REGS(); fi++) {
2041 formatShortReg(os, first, "f", fi,
2042 m_DB.getARF_F_START() +
2043 (size_t)fi * m_DB.getARF_F_BYTES_PER_REG(),
2044 m_DB.getARF_F_BYTES_PER_REG());
2045 }
2046
2047 for (uint32_t fi = 0; fi < m_DB.getARF_SCALAR_REGS(); fi++) {
2048 formatShortReg(os, first, "s", fi,
2049 m_DB.getARF_SCALAR_START() +
2050 (size_t)fi * m_DB.getARF_SCALAR_BYTES_PER_REG(),
2051 m_DB.getARF_SCALAR_BYTES_PER_REG());
2052 }
2053
2054 os << "}";
2055 }
2056 std::string DepSet::str() const {
2057 std::stringstream ss;
2058 str(ss);
2059 return ss.str();
2060 }