Debian Codegraph @

visa/iga/IGALibrary/IR/SWSBSetter.cpp

raw
1 /*========================== begin_copyright_notice ============================
2
3 Copyright (C) 2017-2021 Intel Corporation
4
5 SPDX-License-Identifier: MIT
6
7 ============================= end_copyright_notice ===========================*/
8
9 #include "BitSet.hpp"
10 #include "RegDeps.hpp"
11 #include "Traversals.hpp"
12
13 #include <iterator>
14 #include <limits>
15 #include <optional>
16
17 using namespace iga;
18
19 /**
20 * RAW: R kill W R-->live explict dependence
21 * WAW: different pipelines W2 kill W1 W2-->live explict dependence
22 * WAR: different pipelines W kill R W-->live explict dependence
23 * WAR: same pipeline W kill R W-->live implict dependence
24 * AR: sample pipeline R2 kill R1 R2-->live no dependence
25 * RAR: different pipelines R1,R2-->live no dependence
26 *
27 * Different pipeline
28 * send, math, control flow, long/short (type df)
29 *
30 * add (8) r10 r20 r30
31 * add (8) r11 r21 r22
32 * if (..)
33 * // if instruction doesn't count in if calculations, but it takes about 6
34 * // cycles to resolve for fall through can treat it as continue BB.
35 * // Only when this BB has one predecessor
36 * add r40 r10 r50 {@2}
37 * else
38 * add r60 r70 r80
39 * endif
40 * //Both control flows converge on this. Conservative analysis start with 1.
41 * //By the time jmp happens counter should be at 0 anyway.
42 * add r90 r100 r110 {@1}
43 * add r91 r101 r111 {@2}
44 *
45 *
46 * Types of Dependencies
47 * dst src0 src1
48 * grf ind grf // set distance to 1. If SBID list not empty insert test
49 * instruction. Optimization if SBID == 1 AND grf depends on it, set SBID, clear
50 * SBIDList
51 */
52
53 /**
54 Bucket - represents a GRF line
55 ID - sequential numbering of instructions. Resets at 0 with new BB
56 Implicit assumption. Various data structures have pointers to Dependencies.
57
58 For each BB scan instructions down
59 if new BB
60 reset buckets //can use bit mask
61 reset distanceTracker
62 reset ID
63 For each instruction
64 Calculate dependcy on srcs and destination
65 if currDistance < MAX_DIST
66 record distance = currDistance // for a case where we at new BB
67
68 For each dependency and bucket it touches look in to a bucket
69 if bucket not empty
70 find potential dependencies //bit mask intersects
71 for each dependency found
72 if appopriate (WAW, RAR, RAW) Dependence exists
73 Clear dependency bits from bucket dependency
74 if dep empty remove from bucket
75 if swsb //no out of order
76 if instDistance > (currDistance - depID)
77 //We found dependence closer
78 record distance = currDistance - depID
79 //CurrDistance > depID AND min(currDist - depID, 1) else //sbid record SBID
80 ID
81
82 if dependencyRecord NOT empty
83 Generate appropriate SWSB/test instruction
84 IF SBID
85 if all dependencies are clear
86 add SBID to free list
87 remove entry SBID -> dependencies
88 Remove MAX_DIST DEP from buckets
89 Add current instruction Dependencies to buckets
90 if instruction isVariableExecTime //send, math.idiv
91 if freeSBITLIst IS empty
92 pick one SBID
93 generate test instruction
94 move SBID to free list
95 clear dependency from bucket/sbidList
96
97 assign SBID from free list
98 if end of block AND SBID list NOT empty
99 generate test instructions
100 */
101
102 #include "SWSBSetter.hpp"
103 /*
104 WAW
105 explicit dependence
106 math.fc (except idiv) r10 ...
107 add r10 ....
108
109 add r10 ... //long: type DF/Q
110 add r10 ... //short:
111
112 WAW
113 no dependence
114 add r10 ...
115 add r10 ...
116
117
118 Math.sin r10 r20 r30
119 Math.cos r20 r40 r50
120 Not required - same pipe
121
122 Math.sin r20 r10 r30
123 Math.cos r20 r40 r50
124 Not required - same pipe
125
126 FPU_long r20 r10 r30
127 Math.sin r20 r40 r50
128 Explicit dep required as math can overtake FPU_long - since they are in
129 different pipes.
130
131 RAW
132 add r10 ...
133 add r20 ...
134 add ... r20 ... {@1}
135 add ... r10 {@3} <--- technically speaking this depending is not necesary
136 since they are in same pipe and previous instruction will
137 stall so last instruction dependence is cleared. But in terms of runtime there
138 is no impact so not worth special handling
139
140 assuming two grfs are written/read
141 send r10
142 send r11
143
144 add (16) ... r10 ...
145 second send has dependency on first send
146 add has dependency on second send
147 if sends written 1 grf, and add still read two grfs it will have dependence on
148 both sends
149
150 send r10 //set$1 writes r10/r11
151 add(8) r10 {$1.dst}
152 add(8) r11 {}
153
154
155 */
156 void SWSBAnalyzer::clearDepBuckets(DepSet &depMatch) {
157 for (auto bucketID : depMatch.getBuckets()) {
158 auto bucket = &m_buckets[bucketID];
159 auto numDepSets = bucket->getNumDependencies();
160 for (uint32_t i = 0; i < numDepSets; ++i) {
161 DepSet *dep = bucket->getDepSet(i);
162 // See if anything matches for this GRF bucket.
163 // originally was checking for intersect but was removing extra dependence
164 // in case like this
165 /*
166 (W) and (1|M0) r0.0<1>:ud r0.0<0;1,0>:ud
167 0xFFFFBFFF:ud {} (W) mov (16|M0) r25.0<1>:f 0:d
168 (W) mov (1|M0) r0.2<1>:ud 0x0:ud
169 (W) mov (1|M0) r0.2<1>:ud 0x0:ud
170 (W) and (1|M0) r0.0<1>:ud r0.0<0;1,0>:ud
171 0xFFFFF7FF:ud {} mov (16|M0) r120.0<1>:ud
172 r17.0<8;8,1>:ud mov (16|M0) r122.0<1>:ud r19.0<8;8,1>:ud
173 mov (16|M0) r124.0<1>:ud r21.0<8;8,1>:ud
174 mov (16|M0) r126.0<1>:ud r23.0<8;8,1>:ud
175 (W) mov (16|M0) r118.0<1>:ud r0.0<8;8,1>:ud {}
176 */
177 // the r0 dependece was already cleared by second r0
178 // but when clearing from buckets it would find the second r0 and clear it
179 // by mistake
180 if (dep && depMatch.getInstGlobalID() == dep->getInstGlobalID() &&
181 (dep->getDepType() == depMatch.getDepType())) {
182 bucket->clearDepSet(i);
183 }
184 }
185 }
186 depMatch.reset();
187 }
188
189 void SWSBAnalyzer::checkAccFlagRAW(bool &isRAW, const DepSet &currDep,
190 const DepSet &targetDep)
191 {
192 // The function must be called when there is RAW dependency detected
193 assert(isRAW);
194
195 auto check_dep_reg = [&](const DepSet &in_dep, uint32_t reg_start,
196 uint32_t reg_len) {
197 return in_dep.getBitSet().intersects(currDep.getBitSet(), reg_start,
198 reg_len);
199 };
200 auto has_grf_dep = [&](const DepSet &in_dep) {
201 return check_dep_reg(in_dep, m_DB->getGRF_START(), m_DB->getGRF_LEN());
202 };
203 auto has_arf_a_dep = [&](const DepSet &in_dep) {
204 return check_dep_reg(in_dep, m_DB->getARF_A_START(), m_DB->getARF_A_LEN());
205 };
206 auto has_acc_dep = [&](const DepSet &in_dep) {
207 return check_dep_reg(in_dep, m_DB->getARF_ACC_START(),
208 m_DB->getARF_ACC_LEN());
209 };
210 auto has_flag_dep = [&](const DepSet &in_dep) {
211 return check_dep_reg(in_dep, m_DB->getARF_F_START(), m_DB->getARF_F_LEN());
212 };
213 auto has_sp_dep = [&](const DepSet &in_dep) {
214 return check_dep_reg(in_dep, m_DB->getARF_SPECIAL_START(),
215 m_DB->getARF_SPECIAL_LEN());
216 };
217 auto has_scalar_dep = [&](const DepSet &in_dep) {
218 return check_dep_reg(in_dep, m_DB->getARF_SCALAR_START(),
219 m_DB->getARF_SCALAR_LEN());
220 };
221
222 // is acc dependecy
223 if (has_acc_dep(targetDep)) {
224 // and no dependency on other registers
225 if (!(has_grf_dep(targetDep) || has_arf_a_dep(targetDep) ||
226 has_flag_dep(targetDep) || has_sp_dep(targetDep)))
227 if (!has_scalar_dep(targetDep))
228 isRAW = false;
229 }
230
231 // is flag dependency
232 if (has_flag_dep(targetDep)) {
233 // and no dependency on other registers
234 if (!(has_grf_dep(targetDep) || has_arf_a_dep(targetDep) ||
235 has_acc_dep(targetDep) || has_sp_dep(targetDep)))
236 if (!has_scalar_dep(targetDep))
237 isRAW = false;
238 // flag and acc only
239 if (has_acc_dep(targetDep))
240 if (!(has_grf_dep(targetDep) || has_arf_a_dep(targetDep) ||
241 has_sp_dep(targetDep)))
242 if (!has_scalar_dep(targetDep))
243 isRAW = false;
244 }
245 }
246
247 /**
248 * This function takes in a current instruction dependency.
249 * Either SRC or DST
250 * It then checks against previous dependencies.
251 * It sets mininum valid distance
252 * and creates an active list of SBIDs this instruction depends on
253 * It clears and removes previous dependencies.
254 * The approach is bucket based.
255 * Each bucket is one GRF.
256 * So if instruction writes in to more then one GRF then multiple buckets will
257 * have the dependency
258 */
259 void SWSBAnalyzer::calculateDependence(DepSet &currDep,
260 SWSB &swsb,
261 const Instruction &currInst,
262 activeSBIDsTy &activeSBIDs,
263 bool &needSyncForShootDownInst) {
264 needSyncForShootDownInst = false;
265 auto currDepType = currDep.getDepType();
266 auto currDepPipe = currDep.getDepPipe();
267
268 for (auto bucketID : currDep.getBuckets()) {
269 // iterates over Dependencies in a GRF bucket
270 // Assumption there shouldn't be more then 1-2
271 Bucket *bucket = &m_buckets[bucketID];
272 size_t numDepSets = bucket->getNumDependencies();
273 for (uint32_t i = 0; i < numDepSets; ++i) {
274 uint32_t index = static_cast<uint32_t>(numDepSets - 1 - i);
275 auto dep = bucket->getDepSet(index);
276
277 if (dep && (dep->getDepType() == DEP_TYPE::WRITE_ALWAYS_INTERFERE ||
278 dep->getDepType() == DEP_TYPE::READ_ALWAYS_INTERFERE)) {
279 // force to sync with dep
280 if (dep->getDepClass() == DEP_CLASS::OUT_OF_ORDER) {
281 setSbidDependency(*dep, currInst, needSyncForShootDownInst,
282 activeSBIDs);
283 } else {
284 // Set to sync with all in-order-pipes. WRITE/READ_ALWAYS_INTERFERE
285 // could be used to mark arf dependency, which is required to be all
286 // pipes instead of dep's pipe only
287 swsb.minDist = 1;
288 if (getNumOfDistPipe() == 1)
289 swsb.distType = SWSB::DistType::REG_DIST;
290 else
291 swsb.distType = SWSB::DistType::REG_DIST_ALL;
292 bucket->clearDepSet(index);
293 }
294 }
295
296 // See if anything matches for this GRF bucket.
297 if (dep && dep->getBitSet().intersects(currDep.getBitSet())) {
298 /*
299 * RAW: R kill W R-->live explict
300 * dependence WAW: different pipelines W2 kill W1 W2-->live explict
301 * dependence WAR: different pipelines W kill R W-->live explict
302 * dependence WAR: same pipeline W kill R W-->live implict
303 * dependence AR: sample pipeline R2 kill R1 R2-->live no
304 * dependence RAR: different pipelines R1,R2-->live no
305 * dependence
306 */
307 // RAW: R kill W R-->live explict
308 // dependence
309 DEP_TYPE prevDepType = dep->getDepType();
310 DEP_PIPE prevDepPipe = dep->getDepPipe();
311 DEP_CLASS prevDepClass = dep->getDepClass();
312
313 // Send with different SFID could write to different pipes
314 bool sendInDiffPipe = false;
315 if (dep->getInstruction()->getOpSpec().isAnySendFormat() &&
316 currDep.getInstruction()->getOpSpec().isAnySendFormat()) {
317 sendInDiffPipe = (dep->getInstruction()->getSendFc() !=
318 currDep.getInstruction()->getSendFc());
319 // for send in unknown pipe, always treated as different pipe
320 if (!sendInDiffPipe) {
321 sendInDiffPipe = dep->getDepPipe() == DEP_PIPE::SEND_UNKNOWN ||
322 currDep.getDepPipe() == DEP_PIPE::SEND_UNKNOWN;
323 }
324 }
325
326 auto isRead = [](DEP_TYPE ty) {
327 return ty == DEP_TYPE::READ ||
328 ty == DEP_TYPE::READ_ALWAYS_INTERFERE;
329 };
330 auto isWrite = [](DEP_TYPE ty) {
331 return ty == DEP_TYPE::WRITE ||
332 ty == DEP_TYPE::WRITE_ALWAYS_INTERFERE;
333 };
334
335 bool isRAW = isRead(currDepType) && isWrite(prevDepType);
336 // WAW: different pipelines W2 kill W1 W2-->live explict
337 // dependence
338 bool isWAW =
339 (isWrite(currDepType) && isWrite(prevDepType) &&
340 (currDepPipe != prevDepPipe || sendInDiffPipe));
341 // WAR: different pipelines W kill R W-->live explict
342 // dependence
343 bool isWAR = isWrite(currDepType) &&
344 isRead(prevDepType) &&
345 (currDepPipe != prevDepPipe || sendInDiffPipe);
346 bool isWAW_out_of_order =
347 (isWrite(currDepType) && isWrite(prevDepType) &&
348 prevDepClass == DEP_CLASS::OUT_OF_ORDER);
349
350 // Special case handling for acc/flag dependency:
351 // if the RAW dependency on acc/flag and acc/flag only, and it's whithin
352 // the same pipe, HW can handle it that we don't need to set swsb
353 if (isRAW && currDepPipe == prevDepPipe)
354 checkAccFlagRAW(isRAW, currDep, *dep);
355
356 if (isWAR || isWAW || isRAW || isWAW_out_of_order) {
357 if (dep->getBitSet().empty()) {
358 m_errorHandler.reportWarning(
359 currInst.getPC(), "Dependency in bucket with no bits set");
360 }
361 // removing from bucket if there is nothing
362 if (!dep->getBitSet().testAny(bucketID * 32,
363 m_DB->getGRF_BYTES_PER_REG())) {
364 bucket->clearDepSet(index);
365 }
366
367 if (prevDepClass == DEP_CLASS::IN_ORDER) {
368 setDistanceDependency(dep, swsb, isWAW, prevDepPipe, currDepPipe,
369 currInst);
370 } else if (prevDepClass == DEP_CLASS::OUT_OF_ORDER) {
371 setSbidDependency(*dep, currInst, needSyncForShootDownInst,
372 activeSBIDs);
373 }
374 // for the instruction in "OTHER" DEP_CLASS, such as sync, we don't
375 // need to consider their dependency that is implied by hardware
376 }
377 }
378 }
379 }
380 }
381
382 void SWSBAnalyzer::setDistanceDependency(DepSet *dep, SWSB &swsb, bool isWAW,
383 DEP_PIPE prevDepPipe, DEP_PIPE currDepPipe, const Instruction &currInst) {
384
385 if (getNumOfDistPipe() == 1) {
386 // FOR WAW if PREV is SHORT and curr is LONG then write will
387 // finish before current write, no need to set swsb
388 bool isWAWHazard =
389 ((prevDepPipe == DEP_PIPE::SHORT && currDepPipe == DEP_PIPE::LONG) ||
390 (prevDepPipe == DEP_PIPE::SHORT && currDepPipe == DEP_PIPE::SHORT)) &&
391 isWAW;
392 // require swsb for all the other kinds of dependency
393 if (!isWAWHazard) {
394 // setting minimum distance
395 uint32_t newDistance =
396 m_InstIdCounter.inOrder - dep->getInstIDs().inOrder;
397 swsb.minDist =
398 swsb.minDist == 0 ? newDistance : std::min(swsb.minDist, newDistance);
399 // clamp the distance to max distance
400 swsb.minDist = std::min(swsb.minDist, (uint32_t)MAX_VALID_DISTANCE);
401 swsb.distType = SWSB::DistType::REG_DIST;
402 }
403 } else {
404 // For multiple in-order pipeline architecuture, all cases should
405 // be considered The distance is depended on the previous
406 // instruction's pipeline
407 uint32_t newDistance = 0;
408 SWSB::DistType newDepPipe = SWSB::DistType::NO_DIST;
409 switch (prevDepPipe) {
410 case DEP_PIPE::FLOAT:
411 newDistance = m_InstIdCounter.floatPipe - dep->getInstIDs().floatPipe;
412 newDepPipe = SWSB::DistType::REG_DIST_FLOAT;
413 break;
414 case DEP_PIPE::INTEGER:
415 newDistance = m_InstIdCounter.intPipe - dep->getInstIDs().intPipe;
416 newDepPipe = SWSB::DistType::REG_DIST_INT;
417 break;
418 case DEP_PIPE::LONG64:
419 newDistance = m_InstIdCounter.longPipe - dep->getInstIDs().longPipe;
420 newDepPipe = SWSB::DistType::REG_DIST_LONG;
421 break;
422 case DEP_PIPE::MATH_INORDER:
423 newDistance = m_InstIdCounter.mathPipe - dep->getInstIDs().mathPipe;
424 newDepPipe = SWSB::DistType::REG_DIST_MATH;
425 break;
426 case DEP_PIPE::SCALAR:
427 newDistance = m_InstIdCounter.scalarPipe - dep->getInstIDs().scalarPipe;
428 newDepPipe = SWSB::DistType::REG_DIST_SCALAR;
429 break;
430 default:
431 IGA_ASSERT(0, "Unsupported DEP_PIPE for in-order instructions");
432 break;
433 }
434
435 if (swsb.minDist) {
436 // the instruction already has dependency to others. Take the
437 // min distance and set distType to ALL if they are not the same
438 newDistance = std::min(swsb.minDist, newDistance);
439 if (swsb.distType != newDepPipe)
440 swsb.distType = SWSB::DistType::REG_DIST_ALL;
441 } else {
442 swsb.distType = newDepPipe;
443 }
444 assert(swsb.distType != SWSB::DistType::NO_DIST);
445 // clamp the distance to max distance
446 swsb.minDist = std::min(newDistance, (uint32_t)MAX_VALID_DISTANCE);
447 } // end of if (m_enableMultiDistPipe)
448 // clear this instruction's dependency since it is satisfied
449 clearDepBuckets(*dep);
450
451 // clear its companion because when an in-order instruction is
452 // synced, both its input and output dependency are satisfied. The
453 // only case is that if it has read/write_always_interfere
454 // dependency, it should be reserved. The restriction is that: When
455 // certain Arch Registers (sr, cr, ce) are used, the very next
456 // instruction requires dependency to be set on all pipes {A@1} e.g.
457 // mov (1|M0) r104.0<1>:ud sr0.1<0;1,0>:ud
458 // cmp(16 | M0) (ne)f0.0 null:ud r104.0<0; 1, 0> : ub
459 // r62.4<0; 1, 0> : uw
460 // A@1 is required for cmp instead of I@1
461 if (dep->getCompanion() != nullptr) {
462 // In the case that this DepSet is generated from math_wa_info, it
463 // won't have companion
464 if (dep->getCompanion()->getDepType() != DEP_TYPE::WRITE_ALWAYS_INTERFERE &&
465 dep->getCompanion()->getDepType() != DEP_TYPE::READ_ALWAYS_INTERFERE) {
466 clearDepBuckets(*dep->getCompanion());
467 }
468 }
469 }
470
471 void SWSBAnalyzer::setSbidDependency(DepSet &dep, const Instruction &currInst,
472 bool &needSyncForShootDownInst,
473 activeSBIDsTy &activeSBIDs) {
474 /* For out of order we don't know how long it will finish
475 * so need to test for SBID.
476 * Instruction can depend on more then one SBID
477 * send r10
478 * send r20
479 * send r30
480 * ....
481 * add r10 r20 r30
482 * between different buckets and srcs/dst dependencies instruction can rely on
483 * multiple SBID
484 */
485 SBID depSBID = dep.getSBID();
486 if (depSBID.isFree) {
487 m_errorHandler.reportError((int)dep.getInstGlobalID(),
488 "SBID SHOULDN'T BE FREE!");
489 }
490 // clears all the buckets
491 clearDepBuckets(dep);
492
493 // In case of shooting down of this instruction, we need to add sync to
494 // preserve the swsb id sync, so that it's safe to clear the dep
495 if (currInst.hasPredication() ||
496 (currInst.getExecSize() != dep.getInstruction()->getExecSize()) ||
497 (currInst.getChannelOffset() != dep.getInstruction()->getChannelOffset()))
498 needSyncForShootDownInst = true;
499
500 // used to set read or write dependency
501 depSBID.dType = dep.getDepType();
502
503 // activeSBIDs stores all sbid that this inst has dependency on
504 // and it'll be processed in processactiveSBIDs
505 bool insert_new_sbid = true;
506 // making sure there are no duplicates
507 for (auto &aSBID : activeSBIDs) {
508 if (aSBID.first.sbid == depSBID.sbid) {
509 // write takes longer then read
510 // force the SBID to WRITE type (e.g. $10.dst) if the newly added
511 // dependency is a write dependency.
512 if (depSBID.dType == DEP_TYPE::WRITE ||
513 depSBID.dType == DEP_TYPE::WRITE_ALWAYS_INTERFERE) {
514 aSBID.first.dType = depSBID.dType;
515 }
516 // add DepSet into this activeSBIDs's associated list
517 aSBID.second.push_back(&dep);
518 // the sbid is already in the list, we don't need to add it to
519 // activeSBIDs
520 insert_new_sbid = false;
521 break;
522 }
523 }
524
525 // adding to active SBID
526 // in Run function we will see how many this instruction relies on
527 // and generate approriate SWSB and if needed test instruction
528 // in that level also will add them back to free list
529 if (insert_new_sbid) {
530 activeSBIDs.push_back(std::make_pair(depSBID, std::vector<DepSet*>()));
531 activeSBIDs.back().second.push_back(&dep);
532 }
533 }
534
535 void SWSBAnalyzer::insertSyncAllRdWr(InstList::iterator insertPoint,
536 Block *bb) {
537 SWSB swsb;
538 auto clearRD = m_kernel.createSyncAllRdInstruction(swsb);
539 auto clearWR = m_kernel.createSyncAllWrInstruction(swsb);
540
541 if (insertPoint == bb->getInstList().end()) {
542 bb->getInstList().push_back(clearRD);
543 bb->getInstList().push_back(clearWR);
544 } else {
545 bb->insertInstBefore(insertPoint, clearRD);
546 bb->insertInstBefore(insertPoint, clearWR);
547 }
548 }
549
550 // TODO this should also clear up grf dependency to handle this case:
551 /*
552 call (16|M0) r8.0:ud 32
553 sendc.rc (16|M0) null r118 null 0x0 0x140B1000 {} //
554 wr:10h, rd:0, Render Target Write msc:16, to #0 (W) mov (1|M0) a0.0<1>:ud
555 r7.0<0;1,0>:ud sendc.rc (16|M0) null r100 null 0x0 0x140B1000 {}
556 // wr:10h, rd:0, Render Target Write msc:16, to #0 sendc.rc (16|M0) null r118
557 null 0x0 0x140B1000 {} // wr:10h, rd:0, Render Target Write msc:16,
558 to #0 (W) mov (16|M0) r118.0<1>:ud r6.0<8;8,1>:ud (W)
559 send.dc0 (16|M0) r38 r118 null 0x0 a0.0 ret (16|M0)
560
561 Right now mov will have false dependense on the first send.
562 */
563 bool SWSBAnalyzer::clearSBIDDependence(InstList::iterator insertPoint,
564 Instruction *lastInst, Block *bb) {
565
566 auto clearSBID = [&](const SBID& in) {
567 assert(m_IdToDepSetMap.find(in.sbid) != m_IdToDepSetMap.end());
568 assert(m_IdToDepSetMap[in.sbid].first->getDepClass() ==
569 DEP_CLASS::OUT_OF_ORDER);
570 clearDepBuckets(*m_IdToDepSetMap[in.sbid].first);
571 clearDepBuckets(*m_IdToDepSetMap[in.sbid].second);
572 m_freeSBIDList[in.sbid].reset();
573 };
574
575 bool sbidInUse = false;
576 for (uint32_t i = 0; i < m_SBIDCount; ++i) {
577 // there are still dependencies that might be used outside of this basic
578 // block
579 if (!m_freeSBIDList[i].isFree) {
580 clearSBID(m_freeSBIDList[i]);
581 sbidInUse = true;
582 }
583 }
584
585 // if last instruction in basic block is EOT no need to generate flushes
586 // hardware will take care of it
587 if (lastInst && lastInst->getOpSpec().isAnySendFormat() &&
588 lastInst->hasInstOpt(InstOpt::EOT)) {
589 sbidInUse = false;
590 }
591
592 // platform check is mainly for testing purposes
593 if (sbidInUse) {
594 insertSyncAllRdWr(insertPoint, bb);
595 }
596
597 return sbidInUse;
598 }
599
600 // Keeping track of dependencies that need to be cleared because they are no
601 // longer relevant right now each BB ends with control flow instruction, and we
602 // reset at each BB
603 void SWSBAnalyzer::clearBuckets(DepSet *input, DepSet *output) {
604 if (input->getDepClass() != DEP_CLASS::IN_ORDER)
605 return;
606
607 if (m_initPoint) {
608 m_distanceTracker.emplace_back(input, output);
609 m_initPoint = false;
610
611 } else {
612 // add DepSet to m_distanceTracker
613 m_distanceTracker.emplace_back(input, output);
614
615 auto get_depset_id = [&](DEP_PIPE pipe_type, DepSet &dep_set) {
616 if (getNumOfDistPipe() == 1)
617 return dep_set.getInstIDs().inOrder;
618 switch (pipe_type) {
619 case DEP_PIPE::FLOAT:
620 return dep_set.getInstIDs().floatPipe;
621 case DEP_PIPE::INTEGER:
622 return dep_set.getInstIDs().intPipe;
623 case DEP_PIPE::LONG64:
624 return dep_set.getInstIDs().longPipe;
625 case DEP_PIPE::MATH_INORDER:
626 return dep_set.getInstIDs().mathPipe;
627 case DEP_PIPE::SCALAR:
628 return dep_set.getInstIDs().scalarPipe;
629 default:
630 IGA_ASSERT(0, "SWSB: unhandled in-order DEP_PIPE for XeHP+ encoding");
631 break;
632 }
633 return (uint32_t)0;
634 };
635
636 auto get_latency = [&](DEP_PIPE pipe_type) {
637 if (pipe_type == DEP_PIPE::LONG64)
638 return m_LatencyLong64Pipe;
639 else if (pipe_type == DEP_PIPE::MATH_INORDER)
640 return m_LatencyInOrderMath;
641 return m_LatencyInOrderPipe;
642 };
643
644 DEP_PIPE new_pipe = input->getDepPipe();
645 // max B2B latency of thie pipe
646 size_t max_dis = get_latency(new_pipe);
647 // Remove nodes from the Tracker if the latency is already satified
648 m_distanceTracker.remove_if([=](const distanceTrackerNode &node) {
649 // bypass nodes those are not belong to the same pipe
650 if (node.input->getDepPipe() != new_pipe)
651 return false;
652
653 // if the distance >= max_latency, clear buckets for corresponding
654 // input and output Dependency
655 size_t new_id = get_depset_id(new_pipe, *input);
656 if ((new_id - get_depset_id(new_pipe, *node.input)) >= max_dis) {
657 clearDepBuckets(*node.input);
658 clearDepBuckets(*node.output);
659 return true;
660 }
661 return false;
662 });
663 }
664 }
665
666 bool SWSBAnalyzer::trySBIDCounter(activeSBIDsTy &activeSBIDs,
667 const DepSet &curDepSet,
668 SBID &retSBID) {
669 // There is only one active sbid. It doesn't worth using sbid counter
670 if (activeSBIDs.size() <= 1)
671 return false;
672
673 // curDepSet is a force-sync-all, all dependency will be sync-ed so there
674 // is no benefit to do sbid-counter
675 if (curDepSet.getDepType() == DEP_TYPE::WRITE_ALWAYS_INTERFERE ||
676 curDepSet.getDepType() == DEP_TYPE::READ_ALWAYS_INTERFERE)
677 return false;
678
679 // To simplify the design, we only handle the case that current instruction
680 // (which has dependency to activeSBIDs) is an in-order-pipe instruction
681 if (curDepSet.getDepClass() != DEP_CLASS::IN_ORDER)
682 return false;
683
684 // If there is non-send instruction the activeSBIDs associated with, sbid
685 // counter cannot be used
686 // keep track of if there is any sbid counter used in activeSBIDs already
687 // If there is one, we will replace to it if possible
688 std::optional<SBID> hasCounterID = std::nullopt;
689 for (const auto& aSBID : activeSBIDs) {
690 for (auto depSet : aSBID.second) {
691 auto inst = depSet->getInstruction();
692 // Also skip counter replacement if the send has NOACCSBSET set or its
693 // sbid is already a counter
694 if (!inst->getOpSpec().isAnySendFormat() ||
695 inst->getSWSB().hasSpecialToken()) {
696 return false;
697 }
698 if (inst->getSWSB().tokenType == SWSB::TokenType::INC)
699 hasCounterID = aSBID.first;
700 }
701 }
702
703 // Replace the associated instructions' sbid to sbid counter. For example,
704 // From:
705 // send.ugm (1) r50 r31 null:0 0xFF000000 0x6219C500 {$0}
706 // send.ugm (1) r60 r31 null:0 0xFF000000 0x6219C500 {$1}
707 // // add's activeSBIDs are $0 and $1
708 // add (1|M0) r90:ud r50:ud r60:ud
709 // To:
710 // send.ugm (1) r50 r31 null:0 0xFF000000 0x6219C500 {$0.inc}
711 // send.ugm (1) r60 r31 null:0 0xFF000000 0x6219C500 {$0.inc}
712 // add (1|M0) r90:ud r50:ud r60:ud {$0.dst}
713 //
714 // These send instructions must have no .dst dependency to each other, otherwise
715 // the dependency should be resovled and they won't both be activeSBID here.
716 // Hence we don't need to check the dependency among these instructions.
717 // Note, having .src dependency to each other is fine. Sbid counter resolves
718 // the .src dependency.
719
720 // pick the existing counter (if any), or the first activeSBID as the SBID
721 // counter.
722 retSBID = hasCounterID ? *hasCounterID : activeSBIDs.front().first;
723 // Keep track of the DepSet's DepType (Read or Write)
724 retSBID.dType = activeSBIDs.front().first.dType;
725
726 auto isActiveSBID = [&activeSBIDs](uint32_t sbid) {
727 for(const auto& aSBID : activeSBIDs) {
728 if (sbid == aSBID.first.sbid)
729 return true;
730 }
731 return false;
732 };
733
734 for (const auto& aSBID : activeSBIDs) {
735 for (auto depSet : aSBID.second) {
736 // Updated the set SWSB on send instructions
737 auto inst = depSet->getInstruction();
738 auto instSWSB = inst->getSWSB();
739 instSWSB.sbid = retSBID.sbid;
740 assert(instSWSB.tokenType == SWSB::TokenType::SET ||
741 instSWSB.tokenType == SWSB::TokenType::INC);
742 instSWSB.tokenType = SWSB::TokenType::INC;
743 inst->setSWSB(instSWSB);
744 // Updated the DepSet and its companion DepSet's sbid
745 // So that later instructions having dependency to these DepSet can have
746 // the correct sbid ref
747 SBID newSBID = SBID(depSet->getSBID());
748 newSBID.sbid = retSBID.sbid;
749 depSet->setSBID(newSBID);
750 assert(depSet->getCompanion());
751 depSet->getCompanion()->setSBID(newSBID);
752 // Updated instructions those set to have dependency to these SBIDs
753 for (auto i : depSet->getSrcDepInsts()) {
754 auto swsb = i->getSWSB();
755 // update only when the instruction has sbid that's the same as
756 // activeSBIDs. There is case that sbid.src was set to an instruction
757 // (and hence be added to SrcDepInsts). But that sbid.src was moved to
758 // a sync.nop and the instruction was assigned an irrelevant sbid.
759 // We cannot avoid adding that instruction into SrcDepInsts, so we
760 // need to do the additional check here to avoid updating its sbid.
761 if (!isActiveSBID(swsb.sbid))
762 continue;
763 swsb.sbid = retSBID.sbid;
764 i->setSWSB(swsb);
765 }
766 }
767 // Set depType to the strongest one (WRITE or WRITE_ALWAYS_INTERFERE)
768 // if any. Note, it makes no difference for it is READ or
769 // READ_ALWAYS_INTERFERE here
770 if (retSBID.dType == DEP_TYPE::WRITE_ALWAYS_INTERFERE)
771 continue;
772 if (aSBID.first.dType == DEP_TYPE::WRITE ||
773 aSBID.first.dType == DEP_TYPE::WRITE_ALWAYS_INTERFERE)
774 retSBID.dType = aSBID.first.dType;
775 }
776
777 return true;
778 }
779
780 void SWSBAnalyzer::processactiveSBIDs(SWSB &swsb,
781 const DepSet *input, Block *bb,
782 InstList::iterator instIter,
783 activeSBIDsTy &activeSBIDs) {
784 // free SBID in activeSBIDs and clear the dependecies of associated
785 // DepSet
786 auto clearSBID = [&](const SBID& in) {
787 m_freeSBIDList[in.sbid].reset();
788 assert(m_IdToDepSetMap.find(in.sbid) != m_IdToDepSetMap.end());
789 assert(m_IdToDepSetMap[in.sbid].first->getDepClass() ==
790 DEP_CLASS::OUT_OF_ORDER);
791 clearDepBuckets(*m_IdToDepSetMap[in.sbid].first);
792 clearDepBuckets(*m_IdToDepSetMap[in.sbid].second);
793 };
794
795 // If the instruction depends on more than one sbid, and those sbids all
796 // comes from send instructions, we can replace the sbid by sbid counter
797 if (enableSBIDCounterSetting()) {
798 SBID chosenSBID;
799 if (trySBIDCounter(activeSBIDs, *input, chosenSBID)) {
800 // build swsb for current instruction according to the chosen SBID
801 assert(swsb.tokenType == SWSB::TokenType::NOTOKEN);
802 swsb.sbid = chosenSBID.sbid;
803 if (chosenSBID.dType == DEP_TYPE::READ ||
804 chosenSBID.dType == DEP_TYPE::READ_ALWAYS_INTERFERE) {
805 swsb.tokenType = SWSB::TokenType::SRC;
806 } else {
807 swsb.tokenType = SWSB::TokenType::DST;
808 // All associated DepSet are cleared with .dst dependency here.
809 // free the SBIDs and clear dependency
810 for (const auto& aSBID : activeSBIDs)
811 clearSBID(aSBID.first);
812 }
813 // verify if the combination of token and dist is valid, if not, move the
814 // token dependency out and add a sync for it
815 adjustSWSB(*bb, instIter, swsb, true);
816 return;
817 }
818 }
819
820 // If instruction depends on one or more SBIDS, first one goes in to SWSB
821 // field for rest we generate wait instructions.
822 for (const auto &aSBID : activeSBIDs) {
823 // Could be we had operation depending on the write
824 /*
825 * This case also gets triggered when we have send in BB and dependence in
826 * another BB L0: call (16|M0) r8.0 L64 L16: sendc.rc
827 * (16|M0) null r118 null 0x0 0x140B1000 {$0} //
828 * wr:10h, rd:0, Render Target Write msc:16, to #0 L64: (W) mov (16|M0)
829 * r118.0<1>:ud r6.0<8;8,1>:ud (W) send.dc0 (16|M0) r38 r118
830 * null 0x0 a0.0 {@1, $0} ret (16|M0) r8.0 {@3} After first
831 * BB in which sendc.rc ends we clear all SBID and generate sync
832 * instructions On mov it detects dependense, but all SBID are freed.
833 */
834 if (m_freeSBIDList[aSBID.first.sbid].isFree) {
835 continue;
836 }
837
838 SWSB::TokenType tType = SWSB::TokenType::NOTOKEN;
839 if (aSBID.first.dType == DEP_TYPE::READ ||
840 aSBID.first.dType == DEP_TYPE::READ_ALWAYS_INTERFERE) {
841 tType = SWSB::TokenType::SRC;
842 } else {
843 tType = SWSB::TokenType::DST;
844 // if SBID is cleared add it back to free pool
845 // write is last thing. So if instruction depends on it we know read is
846 // done but not vice versa
847 clearSBID(aSBID.first);
848 }
849
850 // Setting first SBID as part of instruction
851 // If this instruction depends on more SBID, generate sync for the extra ids
852 // TODO: Is it safe to clear SBID here?
853 if (swsb.tokenType == SWSB::TokenType::NOTOKEN) {
854 swsb.tokenType = tType;
855 swsb.sbid = aSBID.first.sbid;
856 } else {
857 // add sync for the id
858 SWSB sync_swsb(SWSB::DistType::NO_DIST, tType, 0, aSBID.first.sbid);
859 auto nopInst = m_kernel.createSyncNopInstruction(sync_swsb);
860 bb->insertInstBefore(instIter, nopInst);
861 m_IdToDepSetMap[aSBID.first.sbid].first->addSrcDepInsts(nopInst);
862 m_IdToDepSetMap[aSBID.first.sbid].first->getCompanion()->addSrcDepInsts(nopInst);
863 m_IdToDepSetMap[aSBID.first.sbid].second->addSrcDepInsts(nopInst);
864 }
865 }
866
867 // verify if the combination of token and dist is valid, if not, move the
868 // token dependency out and add a sync for it
869 auto syncInsts = adjustSWSB(*bb, instIter, swsb, true);
870
871 // update the DepSet's SrcDepInsts if current instruction or any newly added
872 // sync is associated with it
873 auto checkAndUpdateSrcDepInsts =
874 [&activeSBIDs](Instruction* inst, SWSB swsb) {
875 if (!swsb.hasToken())
876 return;
877 if (swsb.tokenType != SWSB::TokenType::SET &&
878 swsb.tokenType != SWSB::TokenType::SRC)
879 return;
880 for (const auto &aSBID : activeSBIDs) {
881 if (swsb.sbid == aSBID.first.sbid) {
882 for (auto& ds : aSBID.second) {
883 ds->addSrcDepInsts(inst);
884 ds->getCompanion()->addSrcDepInsts(inst);
885 }
886 }
887 }
888 };
889
890 for (auto sync : syncInsts) {
891 checkAndUpdateSrcDepInsts(sync, sync->getSWSB());
892 }
893 checkAndUpdateSrcDepInsts(*instIter, swsb);
894 }
895
896 uint32_t SWSBAnalyzer::getNumOfDistPipe() {
897 return getNumOfDistPipe(m_swsbMode);
898 }
899
900 uint32_t SWSBAnalyzer::getNumOfDistPipe(SWSB_ENCODE_MODE mode) {
901 switch (mode) {
902 case SWSB_ENCODE_MODE::SingleDistPipe:
903 return 1;
904 case SWSB_ENCODE_MODE::ThreeDistPipe:
905 return 3;
906 case SWSB_ENCODE_MODE::ThreeDistPipeDPMath:
907 return 3;
908 case SWSB_ENCODE_MODE::FourDistPipe:
909 case SWSB_ENCODE_MODE::FourDistPipeReduction:
910 return 4;
911 case SWSB_ENCODE_MODE::FiveDistPipe:
912 case SWSB_ENCODE_MODE::FiveDistPipeReduction:
913 case SWSB_ENCODE_MODE::FiveDistPipeCvtToInt:
914 case SWSB_ENCODE_MODE::FiveDistPipeSWSBCntr:
915 case SWSB_ENCODE_MODE::FiveDistPipeCvtToIntNoFwd:
916 return 5;
917 default:
918 break;
919 }
920 return 0;
921 }
922
923 void SWSBAnalyzer::advanceInorderInstCounter(DEP_PIPE dep_pipe) {
924 ++m_InstIdCounter.inOrder;
925 if (getNumOfDistPipe() == 1)
926 return;
927
928 switch (dep_pipe) {
929 case DEP_PIPE::FLOAT:
930 ++m_InstIdCounter.floatPipe;
931 break;
932 case DEP_PIPE::INTEGER:
933 ++m_InstIdCounter.intPipe;
934 break;
935 case DEP_PIPE::LONG64:
936 ++m_InstIdCounter.longPipe;
937 break;
938 case DEP_PIPE::MATH_INORDER:
939 ++m_InstIdCounter.mathPipe;
940 break;
941 case DEP_PIPE::SCALAR:
942 ++m_InstIdCounter.scalarPipe;
943 break;
944 default:
945 IGA_ASSERT(0, "unhandled in-order DEP_PIPE for XE_HP encoding");
946 break;
947 }
948 }
949
950 void SWSBAnalyzer::addRMWDependencyIfReqruied(DepSet &input, DepSet &output) {
951 const Instruction *inst = input.getInstruction();
952 // return if the instruction has no dst, or the dst is not GRF or not byte
953 // type
954 const Operand &dst = inst->getDestination();
955 if (dst.getKind() != Operand::Kind::DIRECT)
956 return;
957
958 if (dst.getDirRegName() != RegName::GRF_R)
959 return;
960
961 if (TypeSizeInBitsWithDefault(dst.getType(), 32) != 8)
962 return;
963
964 // When there is RMW behavior, the instruction will read the Word first,
965 // modify the byte value in it and then write back the entire Word.
966 // we assume the instruction will read/write the entire register to simplify
967 // the logic
968
969 // add the entire grf of the dst register into input and output DepSet
970 // All registers being touched are added into Bucket. We can get the touched
971 // grf number from added bucket index
972 const std::vector<size_t> &out_buk = output.getBuckets();
973 for (auto i : out_buk) {
974 // we only need grf bucket
975 if (i >= m_DB->getBucketStart(RegName::ARF_A))
976 continue;
977 input.addGrf(i);
978 input.addToBucket((uint32_t)i);
979 output.addGrf(i);
980 }
981 }
982
983 void SWSBAnalyzer::addSWSBToInst(InstListIterator instIt, const SWSB &swsb,
984 Block &block) {
985 assert(instIt != block.getInstList().end());
986 Instruction &inst = **instIt;
987 SWSB new_swsb(inst.getSWSB());
988 // handling distance
989 if (swsb.hasDist()) {
990 if (!inst.getSWSB().hasDist()) {
991 new_swsb.distType = swsb.distType;
992 new_swsb.minDist = swsb.minDist;
993 } else {
994 // for single dist pipe platform, distType must be REG_DIST, so won't
995 // be set to REG_DIST_ALL
996 new_swsb.distType = (inst.getSWSB().distType == swsb.distType)
997 ? swsb.distType
998 : SWSB::DistType::REG_DIST_ALL;
999 new_swsb.minDist = std::min(inst.getSWSB().minDist, swsb.minDist);
1000 }
1001 }
1002
1003 // handling token
1004 if (swsb.hasToken()) {
1005 if (!inst.getSWSB().hasToken()) {
1006 new_swsb.tokenType = swsb.tokenType;
1007 new_swsb.sbid = swsb.sbid;
1008 } else {
1009 // if both has id, and are different, then insert a sync to carry
1010 // the new one, otherwise do nothing
1011 if ((inst.getSWSB().tokenType != swsb.tokenType) ||
1012 (inst.getSWSB().sbid != swsb.sbid)) {
1013 SWSB tmp_swsb(SWSB::DistType::NO_DIST, swsb.tokenType, 0, swsb.sbid);
1014 Instruction *sync_inst = m_kernel.createSyncNopInstruction(tmp_swsb);
1015 block.insertInstBefore(instIt, sync_inst);
1016 }
1017 }
1018 }
1019
1020 // check if the new swsb combination is valid, if not, move the dist out to a
1021 // sync
1022 // FIXME: move the dist out here to let the sbid set on the instruction could
1023 // have better readability, but a potential issue is that A@1 is required to
1024 // be set on the instruction having architecture read/write. This case A@1
1025 // will be moved out from the instruction
1026 adjustSWSB(block, instIt, new_swsb, false);
1027 inst.setSWSB(new_swsb);
1028 }
1029
1030 std::vector<Instruction*>
1031 SWSBAnalyzer::adjustSWSB(Block &block, const InstListIterator instIt,
1032 SWSB &swsb, bool preferMoveOutSBID) {
1033 std::vector<Instruction*> syncInsts;
1034 const Instruction* inst = *instIt;
1035 if (swsb.verify(m_swsbMode, inst->getSWSBInstType(m_swsbMode)))
1036 return syncInsts;
1037
1038 auto movDistToSync = [&]() {
1039 assert(swsb.hasDist());
1040 SWSB tmp_swsb(swsb.distType, SWSB::TokenType::NOTOKEN, swsb.minDist, 0);
1041 Instruction *sync_inst = m_kernel.createSyncNopInstruction(tmp_swsb);
1042 syncInsts.push_back(sync_inst);
1043 block.insertInstBefore(instIt, sync_inst);
1044 swsb.distType = SWSB::DistType::NO_DIST;
1045 swsb.minDist = 0;
1046 };
1047 auto movTokenToSync = [&]() {
1048 assert(swsb.hasToken());
1049 SWSB tmp_swsb(SWSB::DistType::NO_DIST, swsb.tokenType , 0, swsb.sbid);
1050 Instruction *sync_inst = m_kernel.createSyncNopInstruction(tmp_swsb);
1051 syncInsts.push_back(sync_inst);
1052 block.insertInstBefore(instIt, sync_inst);
1053 swsb.tokenType = SWSB::TokenType::NOTOKEN;
1054 swsb.sbid = 0;
1055 };
1056
1057 // Send can only have SBID.set and its combination on the instruction
1058 if (inst->getOpSpec().isAnySendFormat()) {
1059 if (swsb.tokenType == SWSB::TokenType::INC) {
1060 // swsb has SBID.inc. It cannot be combined with other swsb.
1061 if (swsb.hasDist())
1062 movDistToSync();
1063 assert(swsb.verify(m_swsbMode, inst->getSWSBInstType(m_swsbMode)));
1064 return syncInsts;
1065 }
1066 if (swsb.tokenType == SWSB::TokenType::SET) {
1067 // swsb has SBID.set but the combination with distance is invalid.
1068 // Move distance to sync.
1069 movDistToSync();
1070 assert(swsb.verify(m_swsbMode, inst->getSWSBInstType(m_swsbMode)));
1071 } else {
1072 // swsb has no SBID.set at the moment. Send must have SBID.set
1073 // later (SWSBAnalyzer::assignSBID) so allow distance swsb and once
1074 // SBID.set is set, the distance may be able to combined with it.
1075 // Move only Token out of the instruction.
1076 if (swsb.hasToken())
1077 movTokenToSync();
1078 }
1079 return syncInsts;
1080 }
1081
1082 // For other instructions, solely SBID or DIST are valid SWSB.
1083 // Move one to sync according to the preference.
1084 if (preferMoveOutSBID)
1085 movTokenToSync();
1086 else
1087 movDistToSync();
1088 assert(swsb.verify(m_swsbMode, inst->getSWSBInstType(m_swsbMode)));
1089 return syncInsts;
1090 }
1091
1092 static bool isSyncNop(const Instruction &i) {
1093 return i.is(Op::SYNC) && i.getSyncFc() == SyncFC::NOP;
1094 };
1095
1096 void SWSBAnalyzer::postProcessReadModifiedWriteOnByteDst() {
1097 // move all swsb set within the Atomic block out for the "instruction write
1098 // combined" cases Atomic are provided in the input so assume they are correct
1099 // and have no internal dependency within the macro. Move all swsb to the
1100 // first instruction in the Atomic block and also add the distance swsb to the
1101 // intruction following the Atomic block in case of it has dependency to the
1102 // block but was resolved within the instruction in the block E.g.
1103 // (W) mov (32|M0) r13.0<2>:ub r50.0<1;1,0>:uw {Atomic, I@1}
1104 // (W) mov (32|M0) r13.1<2>:ub r52.0<1;1,0>:uw {Atomic}
1105 // (W) mov (32|M0) r13.2<2>:ub r54.0<1;1,0>:uw {Atomic}
1106 // (W) mov (32|M0) r13.3<2>:ub r56.0<1;1,0>:uw
1107 // add (1) r13.0<1>:df r100.0<0;1,0>:df {I@1}
1108 for (Block *bb : m_kernel.getBlockList()) {
1109 InstList &instList = bb->getInstList();
1110 for (auto inst_it = instList.begin(); inst_it != instList.end();
1111 ++inst_it) {
1112 auto isWriteCombinedCandidate = [&](Instruction &inst) {
1113 return (inst.is(Op::MOV) || inst.is(Op::SRND)) &&
1114 inst.getDestination().getKind() == Operand::Kind::DIRECT &&
1115 inst.getDestination().getDirRegName() == RegName::GRF_R &&
1116 TypeSizeInBitsWithDefault(inst.getDestination().getType(),
1117 32) == 8;
1118 };
1119 // add distance swsb in "from" into "to"
1120 auto updateDistanceSWSB = [](const SWSB &from, SWSB &to) {
1121 if (!from.hasDist())
1122 return;
1123
1124 if (!to.hasDist()) {
1125 to.distType = from.distType;
1126 to.minDist = from.minDist;
1127 } else {
1128 to.distType = (to.distType == from.distType)
1129 ? to.distType
1130 : SWSB::DistType::REG_DIST_ALL;
1131 to.minDist = std::min(to.minDist, from.minDist);
1132 }
1133 };
1134
1135 if ((*inst_it)->hasInstOpt(InstOpt::ATOMIC) &&
1136 isWriteCombinedCandidate(**inst_it)) {
1137 // found the marcro start
1138 InstListIterator firstit = inst_it;
1139 SWSB allDistSWSB = SWSB();
1140 ++inst_it;
1141 InstList sync_insts;
1142 // iterate to the end of the macro and move all swsb to the first
1143 for (; inst_it != instList.end(); ++inst_it) {
1144 Instruction &cur_inst = **inst_it;
1145 // found sync, prepare to move to before the firstinst
1146 if (cur_inst.is(Op::SYNC)) {
1147 sync_insts.push_back(
1148 m_kernel.createSyncNopInstruction(cur_inst.getSWSB()));
1149 // remove swsb in current sync so that this sync will be removed
1150 // in the following pass
1151 cur_inst.setSWSB(SWSB());
1152 continue;
1153 }
1154 // All instructions within the write-combined atomic block must be
1155 // write-combined candidate
1156 if (isWriteCombinedCandidate(cur_inst)) {
1157 // move the swsb within the atomic block to the firstinst and keep
1158 // track of distance swsb
1159 if (cur_inst.getSWSB().hasSWSB()) {
1160 updateDistanceSWSB(cur_inst.getSWSB(), allDistSWSB);
1161 addSWSBToInst(firstit, cur_inst.getSWSB(), *bb);
1162 cur_inst.setSWSB(SWSB());
1163 }
1164 // found the last instruction of the Atomic block
1165 if (!cur_inst.hasInstOpt(InstOpt::ATOMIC))
1166 break;
1167 } else {
1168 m_errorHandler.reportError(
1169 cur_inst.getPC(),
1170 "Instruction found in the write-combined atomic block is not "
1171 "a write-combined candidate");
1172 break;
1173 }
1174 }
1175 // insert sync to before firstinst
1176 if (!sync_insts.empty())
1177 instList.insert(firstit, sync_insts.begin(), sync_insts.end());
1178
1179 if (inst_it == instList.end()) {
1180 m_errorHandler.reportError(
1181 (*firstit)->getPC(),
1182 "The last instruction in this write-combined atomic block has "
1183 "{Atomic} set");
1184 break;
1185 }
1186
1187 // insert distance dependency to the instruction following the block
1188 InstListIterator next = inst_it;
1189 next++;
1190 // if the last instruction in the atomic block is the last instruction
1191 // in the BB, insert a sync.nop to carry the required SWSB info
1192 // For example:
1193 // BB0:
1194 // (W) mov (32|M0) r13.0<2>:ub r50.0<1;1,0>:uw {Atomic}
1195 // (W) mov (32|M0) r13.1<2>:ub r52.0<1;1,0>:uw {Atomic}
1196 // (W) mov (32|M0) r13.2<2>:ub r54.0<1;1,0>:uw {Atomic}
1197 // (W) mov (32|M0) r13.3<2>:ub r56.0<1;1,0>:uw
1198 // (W) sync.nop {I@1} // insert nop
1199 // BB1:
1200 // add (1) r13.0<1>:df r100.0<0;1,0>:df
1201 if (next == instList.end()) {
1202 instList.push_back(m_kernel.createSyncNopInstruction(allDistSWSB));
1203 break;
1204 }
1205 addSWSBToInst(next, allDistSWSB, *bb);
1206 }
1207 }
1208 }
1209 }
1210
1211 void SWSBAnalyzer::postProcessSBIDCounterInst() {
1212 for (Block *bb : m_kernel.getBlockList()) {
1213 InstList &instList = bb->getInstList();
1214 for (auto inst_it = instList.begin(); inst_it != instList.end();
1215 ++inst_it) {
1216 Instruction *inst = *inst_it;
1217 if(!inst->getOpSpec().isAnySendFormat())
1218 continue;
1219 SWSB swsb = inst->getSWSB();
1220 adjustSWSB(*bb, inst_it, swsb, false);
1221 inst->setSWSB(swsb);
1222 }
1223 }
1224 }
1225
1226 void SWSBAnalyzer::postProcessRemoveRedundantSync() {
1227 // Case 1:
1228 // sync.nop carry the sbid the same as the sbid set on the following
1229 // instruction can be removed since it'll automatically be sync-ed when sbid
1230 // is reused. For example:
1231 // sync.nop null {$0.dst} // can be removed
1232 // math.exp(8|M0) r12.0<1>:f r10.0<8;8,1>:f {$0}
1233 // Case 2:
1234 // This case may occur after sbid counter insertion. We only check the
1235 // redundant token (but no dist) since that's the only possible cases.
1236 // Continuous sync.nop with the same swsb. One of it can be removed:
1237 // sync.nop null {$0.src}
1238 // sync.nop null {$0.src}
1239 for (Block *bb : m_kernel.getBlockList()) {
1240 InstList &instList = bb->getInstList();
1241 if (instList.empty())
1242 continue;
1243 auto inst_it = instList.begin();
1244 // skip the first instruction, which must not be sync
1245 ++inst_it;
1246 for (; inst_it != instList.end(); ++inst_it) {
1247 Instruction *inst = *inst_it;
1248 if (isSyncNop(*inst))
1249 continue;
1250
1251 SWSB cur_swsb = inst->getSWSB();
1252 // iterate through the previous sync if any
1253 auto sync_it = inst_it;
1254 --sync_it;
1255 // keep track of the seen SWSB's TokenType and sbid number
1256 std::set<std::pair<SWSB::TokenType, uint32_t>> seenSBID;
1257 while (sync_it != instList.begin()) {
1258 Instruction *sync_inst = *sync_it;
1259 if (!isSyncNop(*sync_inst))
1260 break;
1261 SWSB sync_swsb = sync_inst->getSWSB();
1262 // Case 1
1263 // if the sync has sbid set, it could be the reserved sbid for shoot
1264 // down instructions, we should keep it. Otherwise if the its sbid
1265 // is the same as current's instrcutions' sbid.set, we can remove it.
1266 if(cur_swsb.hasToken() && cur_swsb.tokenType == SWSB::TokenType::SET) {
1267 if (sync_swsb.hasToken() &&
1268 sync_swsb.tokenType != SWSB::TokenType::SET &&
1269 sync_swsb.sbid == cur_swsb.sbid) {
1270 // clean the swsb so that we can remove this instruction later
1271 sync_inst->setSWSB(SWSB());
1272 }
1273 }
1274 // Case 2
1275 // keep track of visited SWSB on sync and remove the duplicated one
1276 sync_swsb = sync_inst->getSWSB();
1277 if (sync_swsb.hasToken()) {
1278 auto sbidPair = std::make_pair(sync_swsb.tokenType, sync_swsb.sbid);
1279 if (seenSBID.find(sbidPair) != seenSBID.end()) {
1280 sync_inst->setSWSB(SWSB());
1281 } else {
1282 seenSBID.insert(sbidPair);
1283 }
1284 }
1285 --sync_it;
1286 }
1287 }
1288 // remove the redundant sync.nop (sync.nop with no swsb)
1289 instList.remove_if([](const Instruction *inst) {
1290 return isSyncNop(*inst) && !inst->getSWSB().hasSWSB();
1291 });
1292 }
1293 }
1294
1295 void SWSBAnalyzer::postProcess() {
1296 // revisit all instructions to handle write-combined Atomic block
1297 if (m_kernel.getModel().hasReadModifiedWriteOnByteDst()) {
1298 postProcessReadModifiedWriteOnByteDst();
1299 }
1300
1301 // we may update sbid token to .inc after a send's swsb is adjusted.
1302 // adjustSWSB again in case the swsb doesn't fit on the inst
1303 if (enableSBIDCounterSetting()) {
1304 postProcessSBIDCounterInst();
1305 }
1306
1307 // revisit all instructions to remove redundant sync.nop
1308 postProcessRemoveRedundantSync();
1309 }
1310
1311 SBID &SWSBAnalyzer::assignSBID(DepSet *input, DepSet *output, Instruction &inst,
1312 SWSB &swsb, InstList::iterator insertPoint,
1313 Block *curBB, bool needSyncForShootDown) {
1314 bool foundFree = false;
1315 SBID *sbidFree = nullptr;
1316 for (uint32_t i = 0; i < m_SBIDCount; ++i) {
1317 if (m_freeSBIDList[i].isFree) {
1318 foundFree = true;
1319 sbidFree = &m_freeSBIDList[i];
1320 m_freeSBIDList[i].sbid = i;
1321 break;
1322 }
1323 }
1324 // no free SBID.
1325 if (!foundFree) {
1326 unsigned int index = (m_SBIDRRCounter++) % m_SBIDCount;
1327
1328 // While swsb id being reuse, the dependency will automatically resolved by
1329 // hardware, so cleanup the dependency bucket for instruction that
1330 // previously used this id
1331 assert(m_IdToDepSetMap.find(index) != m_IdToDepSetMap.end());
1332 assert(m_IdToDepSetMap[index].first->getDepClass() ==
1333 DEP_CLASS::OUT_OF_ORDER);
1334 clearDepBuckets(*m_IdToDepSetMap[index].first);
1335 clearDepBuckets(*m_IdToDepSetMap[index].second);
1336
1337 m_freeSBIDList[index].reset();
1338 sbidFree = &m_freeSBIDList[index];
1339 sbidFree->sbid = index;
1340 }
1341 sbidFree->isFree = false;
1342 input->setSBID(*sbidFree);
1343 output->setSBID(*sbidFree);
1344 if (m_IdToDepSetMap.find(sbidFree->sbid) != m_IdToDepSetMap.end())
1345 m_IdToDepSetMap.erase(sbidFree->sbid);
1346 m_IdToDepSetMap.emplace(sbidFree->sbid, std::make_pair(input, output));
1347
1348 if (needSyncForShootDown) {
1349 // check if the instruction has A@1 on it. If it does, create another sync
1350 // to move A@1 to before the sync.nop in case of violating the rule "When
1351 // certain Arch Registers (sr, cr, ce) are used, the very next instruction
1352 // requires dependency to be set on all pipes {A@1}".
1353 // At this point we do not know if the A@1 is inserted for this rule or
1354 // other dependency. Conservatively move A@1 above to the very first sync
1355 // after the previous non-sync instruction
1356 if (swsb.hasDist() &&
1357 (swsb.distType == SWSB::DistType::REG_DIST_ALL ||
1358 swsb.distType == SWSB::DistType::REG_DIST) &&
1359 swsb.minDist == 1) {
1360 SWSB ss(swsb.distType, SWSB::TokenType::NOTOKEN, 1, 0);
1361 curBB->insertInstBefore(insertPoint,
1362 m_kernel.createSyncNopInstruction(ss));
1363 swsb.distType = SWSB::DistType::NO_DIST;
1364 swsb.minDist = 0;
1365 }
1366 // add a sync to preserve the token for possibly shooting down instruction
1367 SWSB tDep(SWSB::DistType::NO_DIST, SWSB::TokenType::SET, 0, sbidFree->sbid);
1368 auto syncInst = m_kernel.createSyncNopInstruction(tDep);
1369 curBB->insertInstBefore(insertPoint, syncInst);
1370 input->addSrcDepInsts(syncInst);
1371 output->addSrcDepInsts(syncInst);
1372 }
1373
1374 // adding the set for this SBID
1375 // if the swsb has the token set already, move it out to a sync
1376 if (swsb.tokenType != SWSB::TokenType::NOTOKEN) {
1377 SWSB tDep(SWSB::DistType::NO_DIST, swsb.tokenType, 0, swsb.sbid);
1378 Instruction *tInst = m_kernel.createSyncNopInstruction(tDep);
1379 curBB->insertInstBefore(insertPoint, tInst);
1380 // the depset of send corresponding to the sbid must be updated with this
1381 // sync
1382 m_IdToDepSetMap[swsb.sbid].first->addSrcDepInsts(tInst);
1383 m_IdToDepSetMap[swsb.sbid].first->getCompanion()->addSrcDepInsts(tInst);
1384 m_IdToDepSetMap[swsb.sbid].second->addSrcDepInsts(tInst);
1385 }
1386 // set the sbid
1387 swsb.tokenType = SWSB::TokenType::SET;
1388 swsb.sbid = sbidFree->sbid;
1389
1390 // verify if the token and dist combination is valid, if not, move the dist
1391 // out to a sync move the dist out here to let the sbid set on the instruction
1392 // so that other instructions having dependency to this SBID is sync to the
1393 // correct instruction (but not a sync.nop)
1394 // FIXME: a potential issue is that A@1 is required to be set on the
1395 // instruction having architecture read/write. This case A@1 will be moved out
1396 // from the instruction
1397 adjustSWSB(*curBB, insertPoint, swsb, false);
1398
1399 assert(sbidFree != nullptr);
1400 return *sbidFree;
1401 }
1402
1403 void SWSBAnalyzer::run() {
1404 m_initPoint = true;
1405 m_distanceTracker.clear();
1406
1407 for (uint32_t i = 0; i < MAX_GRF_BUCKETS; ++i) {
1408 m_buckets[i].clearDependency();
1409 }
1410
1411 // init in order pipe id counters
1412 m_InstIdCounter.init();
1413 math_wa_info.reset();
1414
1415 Instruction *inst = nullptr;
1416 Block *lastBB = nullptr;
1417 for (auto bb : m_kernel.getBlockList()) {
1418 bool blockEndsWithNonBranchInst = false;
1419 // resetting things for each bb
1420 lastBB = bb;
1421 InstList &instList = bb->getInstList(); // Don't use auto for over loaded
1422 // return which has const...
1423 const auto instListEnd = instList.end();
1424 for (auto instIter = instList.begin(); instIter != instListEnd;
1425 ++instIter) {
1426 m_InstIdCounter.global++;
1427 inst = *instIter;
1428 // skip illeagl instruction. It cannot have dependency to others, nor does
1429 // it affect the in-pipe instruction count. Skipping it can avoid we
1430 // accidentally set SWSB on it (e.g. when forcing B2B dependency). Illeagl
1431 // instructions cannot carry SWSB info.
1432 if (inst->getOp() == Op::ILLEGAL || inst->getOp() == Op::THRYLD)
1433 continue;
1434 DepSet *input = nullptr;
1435 DepSet *output = nullptr;
1436 size_t dpas_cnt_in_macro = 0;
1437
1438 if (math_wa_info.math_inst != nullptr)
1439 math_wa_info.previous_is_math = true;
1440 if (inst->getOpSpec().is(Op::MATH)) {
1441 math_wa_info.math_inst = inst;
1442
1443 // if the math following a math, we only care about the last math
1444 math_wa_info.previous_is_math = false;
1445 }
1446
1447 // recored the first instruction of a dpas macro, in case that inserting
1448 // instructions (e.g. sync) before the macro, those instructions have to
1449 // be insert before first_inst_in_dpas_macro
1450 InstListIterator first_inst_in_dpas_macro = instList.end();
1451 if (inst->getOpSpec().isDpasFormat()) {
1452 std::pair<DepSet *, DepSet *> dep_set_pair =
1453 m_DB->createDPASSrcDstDepSet(instList, instIter, m_InstIdCounter,
1454 dpas_cnt_in_macro, m_swsbMode);
1455 input = dep_set_pair.first;
1456 output = dep_set_pair.second;
1457
1458 first_inst_in_dpas_macro = instIter;
1459 // bypass dpas insturctions in the macro, the last dpas represents the
1460 // macro
1461 for (size_t i = 0; i < dpas_cnt_in_macro - 1; ++i)
1462 ++instIter;
1463 inst = *instIter;
1464 } else {
1465 input = m_DB->createSrcDepSet(*inst, m_InstIdCounter, m_swsbMode);
1466 if (m_DB->needDstReadSuppressionWA(*inst))
1467 output = m_DB->createDstDepSetFullGrf(*inst, m_InstIdCounter,
1468 m_swsbMode, true);
1469 else
1470 output = m_DB->createDstDepSet(*inst, m_InstIdCounter, m_swsbMode);
1471 }
1472 input->setCompanion(output);
1473 output->setCompanion(input);
1474
1475 // XeHPC+ features
1476 if (m_kernel.getModel().hasReadModifiedWriteOnByteDst())
1477 addRMWDependencyIfReqruied(*input, *output);
1478
1479 SWSB swsb;
1480
1481 // Either source or destination are indirect, or there are SR access,
1482 // We don't know what registers are being accessed
1483 // Need to flush all the sbids and set distance to 1
1484 if (input->hasIndirect() || output->hasIndirect() || input->hasSR() ||
1485 output->hasSR()) {
1486 // clear out-of-order dependency, insert sync.allrd and sync.allwr
1487 // if there are un-resolved sbid dependecny
1488 // if this instruction itself is an out-of-order instruction, insert
1489 // sync.all anyway.
1490 InstListIterator insert_point = instIter;
1491 if (first_inst_in_dpas_macro != instList.end())
1492 insert_point = first_inst_in_dpas_macro;
1493 bool forceSyncAll = clearSBIDDependence(insert_point, inst, bb);
1494 if (!forceSyncAll && input->getDepClass() == DEP_CLASS::OUT_OF_ORDER)
1495 insertSyncAllRdWr(insert_point, bb);
1496
1497 // clear in-order dependency
1498 clearBuckets(input, output);
1499
1500 // will add direct accesses to buckets
1501 // adding dependencies to buckets
1502 for (auto bucketID : input->getBuckets()) {
1503 m_buckets[bucketID].addDepSet(input);
1504 }
1505 for (auto bucketID : output->getBuckets()) {
1506 m_buckets[bucketID].addDepSet(output);
1507 }
1508
1509 // set to check all dist pipes
1510 if (getNumOfDistPipe() == 1)
1511 swsb.distType = SWSB::DistType::REG_DIST;
1512 else
1513 swsb.distType = SWSB::DistType::REG_DIST_ALL;
1514 swsb.minDist = 1;
1515
1516 // input and output must have the same dep class and in the same pipe
1517 // so check the input only to add the instCounter
1518 // FIXME: is it possilbe that a instruction has output and no input?
1519 if (input->getDepClass() == DEP_CLASS::IN_ORDER)
1520 advanceInorderInstCounter(input->getDepPipe());
1521
1522 // if this is an out-of-order instruction, we still need to assign an
1523 // sbid for it
1524 if (output->getDepClass() == DEP_CLASS::OUT_OF_ORDER)
1525 assignSBID(input, output, *inst, swsb, insert_point, bb,
1526 false);
1527
1528 inst->setSWSB(swsb);
1529 // clean up math_wa_info, this instruction force to sync all, no need to
1530 // consider math wa
1531 if (math_wa_info.previous_is_math) {
1532 math_wa_info.reset();
1533 }
1534 // early out, no need to calculateDependence that all dependencies are
1535 // resolved.
1536 continue;
1537 } // end indirect access handling
1538
1539 if (math_wa_info.previous_is_math) {
1540 // math WA affect the instruction right after the math, and with
1541 // different predication Add the WA math dst region to Buckets
1542 if (math_wa_info.math_inst->getPredication().function !=
1543 inst->getPredication().function) {
1544 math_wa_info.dep_set = m_DB->createDstDepSetFullGrf(
1545 *math_wa_info.math_inst, math_wa_info.math_id, m_swsbMode, false);
1546 math_wa_info.dep_set->setSBID(math_wa_info.math_sbid);
1547 for (auto bucketID : math_wa_info.dep_set->getBuckets()) {
1548 IGA_ASSERT(bucketID < m_DB->getTOTAL_BUCKETS(),
1549 "buckedID out of range");
1550 m_buckets[bucketID].addDepSet(math_wa_info.dep_set);
1551 }
1552 }
1553 }
1554
1555 // keep track of the depended SBID current inst depends on,
1556 // and the list of DepSet where this SBID comes from
1557 activeSBIDsTy activeSBIDs;
1558 bool needSyncForShootDown = false;
1559 // Calculates dependence between this instruction dependencies and
1560 // previous ones.
1561 calculateDependence(*input, swsb, *inst, activeSBIDs,
1562 needSyncForShootDown);
1563 calculateDependence(*output, swsb, *inst, activeSBIDs,
1564 needSyncForShootDown);
1565
1566 // clean up math_wa_info
1567 if (math_wa_info.previous_is_math) {
1568 if (math_wa_info.dep_set != nullptr)
1569 clearDepBuckets(*math_wa_info.dep_set);
1570 math_wa_info.reset();
1571 }
1572
1573 if (first_inst_in_dpas_macro != instList.end())
1574 processactiveSBIDs(swsb, input, bb,
1575 first_inst_in_dpas_macro, activeSBIDs);
1576 else
1577 processactiveSBIDs(swsb, input, bb, instIter, activeSBIDs);
1578
1579 // Need to set SBID
1580 if (output->getDepClass() == DEP_CLASS::OUT_OF_ORDER) {
1581 InstList::iterator insertPoint = instIter;
1582 if (first_inst_in_dpas_macro != instList.end())
1583 insertPoint = first_inst_in_dpas_macro;
1584 SBID &assigned_id = assignSBID(input, output, *inst, swsb,
1585 insertPoint, bb, needSyncForShootDown);
1586
1587 // record the sbid if it's math, for use of math wa
1588 if (inst->getOpSpec().is(Op::MATH)) {
1589 math_wa_info.math_sbid = assigned_id;
1590 }
1591 }
1592
1593 clearBuckets(input, output);
1594
1595 /*
1596 * Handling the case where everything is in one bb, and send with EOT is
1597 * in the middle of instruction stream call (16|M0) r8.0:ud 32
1598 * sendc.rc (16|M0) null r118 null 0x0 0x140B1000
1599 * {EOT} // wr:10h, rd:0, Render Target Write msc:16, to #0
1600 * ...
1601 * ret (16|M0) r8.0
1602 */
1603 if (!(inst->getOpSpec().isAnySendFormat() &&
1604 inst->hasInstOpt(InstOpt::EOT))) {
1605 // adding dependencies to buckets
1606 for (auto bucketID : input->getBuckets()) {
1607 // We want to check dependncy of regular instructions against
1608 // WRITE_ALWAYS_INTERFERE without adding them themselves
1609 if (bucketID == m_DB->getBucketStart(RegName::ARF_CR) &&
1610 input->getDepType() != DEP_TYPE::WRITE_ALWAYS_INTERFERE &&
1611 input->getDepType() != DEP_TYPE::READ_ALWAYS_INTERFERE) {
1612 continue;
1613 }
1614 m_buckets[bucketID].addDepSet(input);
1615 }
1616 for (auto bucketID : output->getBuckets()) {
1617 IGA_ASSERT(bucketID < m_DB->getTOTAL_BUCKETS(),
1618 "buckedID out of range");
1619 // We want to check dependncy of regular instructions against
1620 // WRITE_ALWAYS_INTERFERE without adding them themselves
1621 if (bucketID == m_DB->getBucketStart(RegName::ARF_CR) &&
1622 output->getDepType() != DEP_TYPE::WRITE_ALWAYS_INTERFERE &&
1623 output->getDepType() != DEP_TYPE::READ_ALWAYS_INTERFERE) {
1624 continue;
1625 }
1626 m_buckets[bucketID].addDepSet(output);
1627 }
1628 }
1629
1630 if (input->getDepClass() == DEP_CLASS::IN_ORDER) {
1631 advanceInorderInstCounter(input->getDepPipe());
1632 }
1633
1634 // for dpas block, set the distance at the first inst in the block, and
1635 // set the swsb id at the last inst in the block.
1636 if ((first_inst_in_dpas_macro != instList.end()) &&
1637 (*first_inst_in_dpas_macro != inst)) {
1638 (*first_inst_in_dpas_macro)->setSWSB(SWSB(swsb.distType,
1639 SWSB::TokenType::NOTOKEN, swsb.minDist, 0));
1640 inst->setSWSB(SWSB(SWSB::DistType::NO_DIST, swsb.tokenType, 0,
1641 swsb.sbid));
1642 } else {
1643 // if the input SWSB is a special token, preserve it and insert a sync
1644 // before to carry the dependency info Note that dpas must not have
1645 // special token so we only do this check for non-dpas here
1646 if (inst->getSWSB().hasSpecialToken()) {
1647 if (swsb.hasSWSB()) {
1648 Instruction *syncInst =
1649 m_kernel.createSyncNopInstruction(swsb);
1650 bb->insertInstBefore(instIter, syncInst);
1651 }
1652 } else {
1653 inst->setSWSB(swsb);
1654 }
1655 }
1656 assert(swsb.verify(m_swsbMode, inst->getSWSBInstType(m_swsbMode)));
1657
1658 if (inst->isBranching()) {
1659 // TODO: konrad : this is somewhat conservative, some
1660 // branch instructions might not need sync (join)
1661 blockEndsWithNonBranchInst = false;
1662 clearSBIDDependence(instIter, inst, bb);
1663 continue;
1664 } else {
1665 blockEndsWithNonBranchInst = true;
1666 }
1667 } // iterate on instr
1668 // clear read
1669 // clear write
1670 if (blockEndsWithNonBranchInst) {
1671 clearSBIDDependence(instList.end(), inst, bb);
1672 }
1673 } // iterate on basic block
1674
1675 if (inst)
1676 forceSyncLastInst(*inst, *lastBB);
1677
1678 postProcess();
1679 return;
1680 }
1681
1682 void SWSBAnalyzer::forceSyncLastInst(const Instruction &lastInst,
1683 Block &lastBB) {
1684 // this code is for FC composite
1685 // if last instruction is not EOT we will insert flush instructions
1686 // and stall the pipeline since we do not do global analysis
1687 if ((lastInst.getOpSpec().isAnySendFormat() &&
1688 !lastInst.getInstOpts().contains(InstOpt::EOT)) ||
1689 !lastInst.getOpSpec().isAnySendFormat()) {
1690 SWSB swsb;
1691 if (getNumOfDistPipe() == 1)
1692 swsb.distType = SWSB::DistType::REG_DIST;
1693 else
1694 swsb.distType = SWSB::DistType::REG_DIST_ALL;
1695 swsb.minDist = 1;
1696 Instruction *syncInst = m_kernel.createSyncNopInstruction(swsb);
1697 lastBB.getInstList().push_back(syncInst);
1698 }
1699 }