| 1 |
/*========================== begin_copyright_notice ============================ |
| 2 |
|
| 3 |
Copyright (C) 2017-2021 Intel Corporation |
| 4 |
|
| 5 |
SPDX-License-Identifier: MIT |
| 6 |
|
| 7 |
============================= end_copyright_notice ===========================*/ |
| 8 |
|
| 9 |
#include "BitSet.hpp" |
| 10 |
#include "RegDeps.hpp" |
| 11 |
#include "Traversals.hpp" |
| 12 |
|
| 13 |
#include <iterator> |
| 14 |
#include <limits> |
| 15 |
#include <optional> |
| 16 |
|
| 17 |
using namespace iga; |
| 18 |
|
| 19 |
/** |
| 20 |
* RAW: R kill W R-->live explict dependence |
| 21 |
* WAW: different pipelines W2 kill W1 W2-->live explict dependence |
| 22 |
* WAR: different pipelines W kill R W-->live explict dependence |
| 23 |
* WAR: same pipeline W kill R W-->live implict dependence |
| 24 |
* AR: sample pipeline R2 kill R1 R2-->live no dependence |
| 25 |
* RAR: different pipelines R1,R2-->live no dependence |
| 26 |
* |
| 27 |
* Different pipeline |
| 28 |
* send, math, control flow, long/short (type df) |
| 29 |
* |
| 30 |
* add (8) r10 r20 r30 |
| 31 |
* add (8) r11 r21 r22 |
| 32 |
* if (..) |
| 33 |
* // if instruction doesn't count in if calculations, but it takes about 6 |
| 34 |
* // cycles to resolve for fall through can treat it as continue BB. |
| 35 |
* // Only when this BB has one predecessor |
| 36 |
* add r40 r10 r50 {@2} |
| 37 |
* else |
| 38 |
* add r60 r70 r80 |
| 39 |
* endif |
| 40 |
* //Both control flows converge on this. Conservative analysis start with 1. |
| 41 |
* //By the time jmp happens counter should be at 0 anyway. |
| 42 |
* add r90 r100 r110 {@1} |
| 43 |
* add r91 r101 r111 {@2} |
| 44 |
* |
| 45 |
* |
| 46 |
* Types of Dependencies |
| 47 |
* dst src0 src1 |
| 48 |
* grf ind grf // set distance to 1. If SBID list not empty insert test |
| 49 |
* instruction. Optimization if SBID == 1 AND grf depends on it, set SBID, clear |
| 50 |
* SBIDList |
| 51 |
*/ |
| 52 |
|
| 53 |
/** |
| 54 |
Bucket - represents a GRF line |
| 55 |
ID - sequential numbering of instructions. Resets at 0 with new BB |
| 56 |
Implicit assumption. Various data structures have pointers to Dependencies. |
| 57 |
|
| 58 |
For each BB scan instructions down |
| 59 |
if new BB |
| 60 |
reset buckets //can use bit mask |
| 61 |
reset distanceTracker |
| 62 |
reset ID |
| 63 |
For each instruction |
| 64 |
Calculate dependcy on srcs and destination |
| 65 |
if currDistance < MAX_DIST |
| 66 |
record distance = currDistance // for a case where we at new BB |
| 67 |
|
| 68 |
For each dependency and bucket it touches look in to a bucket |
| 69 |
if bucket not empty |
| 70 |
find potential dependencies //bit mask intersects |
| 71 |
for each dependency found |
| 72 |
if appopriate (WAW, RAR, RAW) Dependence exists |
| 73 |
Clear dependency bits from bucket dependency |
| 74 |
if dep empty remove from bucket |
| 75 |
if swsb //no out of order |
| 76 |
if instDistance > (currDistance - depID) |
| 77 |
//We found dependence closer |
| 78 |
record distance = currDistance - depID |
| 79 |
//CurrDistance > depID AND min(currDist - depID, 1) else //sbid record SBID |
| 80 |
ID |
| 81 |
|
| 82 |
if dependencyRecord NOT empty |
| 83 |
Generate appropriate SWSB/test instruction |
| 84 |
IF SBID |
| 85 |
if all dependencies are clear |
| 86 |
add SBID to free list |
| 87 |
remove entry SBID -> dependencies |
| 88 |
Remove MAX_DIST DEP from buckets |
| 89 |
Add current instruction Dependencies to buckets |
| 90 |
if instruction isVariableExecTime //send, math.idiv |
| 91 |
if freeSBITLIst IS empty |
| 92 |
pick one SBID |
| 93 |
generate test instruction |
| 94 |
move SBID to free list |
| 95 |
clear dependency from bucket/sbidList |
| 96 |
|
| 97 |
assign SBID from free list |
| 98 |
if end of block AND SBID list NOT empty |
| 99 |
generate test instructions |
| 100 |
*/ |
| 101 |
|
| 102 |
#include "SWSBSetter.hpp" |
| 103 |
/* |
| 104 |
WAW |
| 105 |
explicit dependence |
| 106 |
math.fc (except idiv) r10 ... |
| 107 |
add r10 .... |
| 108 |
|
| 109 |
add r10 ... //long: type DF/Q |
| 110 |
add r10 ... //short: |
| 111 |
|
| 112 |
WAW |
| 113 |
no dependence |
| 114 |
add r10 ... |
| 115 |
add r10 ... |
| 116 |
|
| 117 |
|
| 118 |
Math.sin r10 r20 r30 |
| 119 |
Math.cos r20 r40 r50 |
| 120 |
Not required - same pipe |
| 121 |
|
| 122 |
Math.sin r20 r10 r30 |
| 123 |
Math.cos r20 r40 r50 |
| 124 |
Not required - same pipe |
| 125 |
|
| 126 |
FPU_long r20 r10 r30 |
| 127 |
Math.sin r20 r40 r50 |
| 128 |
Explicit dep required as math can overtake FPU_long - since they are in |
| 129 |
different pipes. |
| 130 |
|
| 131 |
RAW |
| 132 |
add r10 ... |
| 133 |
add r20 ... |
| 134 |
add ... r20 ... {@1} |
| 135 |
add ... r10 {@3} <--- technically speaking this depending is not necesary |
| 136 |
since they are in same pipe and previous instruction will |
| 137 |
stall so last instruction dependence is cleared. But in terms of runtime there |
| 138 |
is no impact so not worth special handling |
| 139 |
|
| 140 |
assuming two grfs are written/read |
| 141 |
send r10 |
| 142 |
send r11 |
| 143 |
|
| 144 |
add (16) ... r10 ... |
| 145 |
second send has dependency on first send |
| 146 |
add has dependency on second send |
| 147 |
if sends written 1 grf, and add still read two grfs it will have dependence on |
| 148 |
both sends |
| 149 |
|
| 150 |
send r10 //set$1 writes r10/r11 |
| 151 |
add(8) r10 {$1.dst} |
| 152 |
add(8) r11 {} |
| 153 |
|
| 154 |
|
| 155 |
*/ |
| 156 |
void SWSBAnalyzer::clearDepBuckets(DepSet &depMatch) { |
| 157 |
for (auto bucketID : depMatch.getBuckets()) { |
| 158 |
auto bucket = &m_buckets[bucketID]; |
| 159 |
auto numDepSets = bucket->getNumDependencies(); |
| 160 |
for (uint32_t i = 0; i < numDepSets; ++i) { |
| 161 |
DepSet *dep = bucket->getDepSet(i); |
| 162 |
// See if anything matches for this GRF bucket. |
| 163 |
// originally was checking for intersect but was removing extra dependence |
| 164 |
// in case like this |
| 165 |
/* |
| 166 |
(W) and (1|M0) r0.0<1>:ud r0.0<0;1,0>:ud |
| 167 |
0xFFFFBFFF:ud {} (W) mov (16|M0) r25.0<1>:f 0:d |
| 168 |
(W) mov (1|M0) r0.2<1>:ud 0x0:ud |
| 169 |
(W) mov (1|M0) r0.2<1>:ud 0x0:ud |
| 170 |
(W) and (1|M0) r0.0<1>:ud r0.0<0;1,0>:ud |
| 171 |
0xFFFFF7FF:ud {} mov (16|M0) r120.0<1>:ud |
| 172 |
r17.0<8;8,1>:ud mov (16|M0) r122.0<1>:ud r19.0<8;8,1>:ud |
| 173 |
mov (16|M0) r124.0<1>:ud r21.0<8;8,1>:ud |
| 174 |
mov (16|M0) r126.0<1>:ud r23.0<8;8,1>:ud |
| 175 |
(W) mov (16|M0) r118.0<1>:ud r0.0<8;8,1>:ud {} |
| 176 |
*/ |
| 177 |
// the r0 dependece was already cleared by second r0 |
| 178 |
// but when clearing from buckets it would find the second r0 and clear it |
| 179 |
// by mistake |
| 180 |
if (dep && depMatch.getInstGlobalID() == dep->getInstGlobalID() && |
| 181 |
(dep->getDepType() == depMatch.getDepType())) { |
| 182 |
bucket->clearDepSet(i); |
| 183 |
} |
| 184 |
} |
| 185 |
} |
| 186 |
depMatch.reset(); |
| 187 |
} |
| 188 |
|
| 189 |
void SWSBAnalyzer::checkAccFlagRAW(bool &isRAW, const DepSet &currDep, |
| 190 |
const DepSet &targetDep) |
| 191 |
{ |
| 192 |
// The function must be called when there is RAW dependency detected |
| 193 |
assert(isRAW); |
| 194 |
|
| 195 |
auto check_dep_reg = [&](const DepSet &in_dep, uint32_t reg_start, |
| 196 |
uint32_t reg_len) { |
| 197 |
return in_dep.getBitSet().intersects(currDep.getBitSet(), reg_start, |
| 198 |
reg_len); |
| 199 |
}; |
| 200 |
auto has_grf_dep = [&](const DepSet &in_dep) { |
| 201 |
return check_dep_reg(in_dep, m_DB->getGRF_START(), m_DB->getGRF_LEN()); |
| 202 |
}; |
| 203 |
auto has_arf_a_dep = [&](const DepSet &in_dep) { |
| 204 |
return check_dep_reg(in_dep, m_DB->getARF_A_START(), m_DB->getARF_A_LEN()); |
| 205 |
}; |
| 206 |
auto has_acc_dep = [&](const DepSet &in_dep) { |
| 207 |
return check_dep_reg(in_dep, m_DB->getARF_ACC_START(), |
| 208 |
m_DB->getARF_ACC_LEN()); |
| 209 |
}; |
| 210 |
auto has_flag_dep = [&](const DepSet &in_dep) { |
| 211 |
return check_dep_reg(in_dep, m_DB->getARF_F_START(), m_DB->getARF_F_LEN()); |
| 212 |
}; |
| 213 |
auto has_sp_dep = [&](const DepSet &in_dep) { |
| 214 |
return check_dep_reg(in_dep, m_DB->getARF_SPECIAL_START(), |
| 215 |
m_DB->getARF_SPECIAL_LEN()); |
| 216 |
}; |
| 217 |
auto has_scalar_dep = [&](const DepSet &in_dep) { |
| 218 |
return check_dep_reg(in_dep, m_DB->getARF_SCALAR_START(), |
| 219 |
m_DB->getARF_SCALAR_LEN()); |
| 220 |
}; |
| 221 |
|
| 222 |
// is acc dependecy |
| 223 |
if (has_acc_dep(targetDep)) { |
| 224 |
// and no dependency on other registers |
| 225 |
if (!(has_grf_dep(targetDep) || has_arf_a_dep(targetDep) || |
| 226 |
has_flag_dep(targetDep) || has_sp_dep(targetDep))) |
| 227 |
if (!has_scalar_dep(targetDep)) |
| 228 |
isRAW = false; |
| 229 |
} |
| 230 |
|
| 231 |
// is flag dependency |
| 232 |
if (has_flag_dep(targetDep)) { |
| 233 |
// and no dependency on other registers |
| 234 |
if (!(has_grf_dep(targetDep) || has_arf_a_dep(targetDep) || |
| 235 |
has_acc_dep(targetDep) || has_sp_dep(targetDep))) |
| 236 |
if (!has_scalar_dep(targetDep)) |
| 237 |
isRAW = false; |
| 238 |
// flag and acc only |
| 239 |
if (has_acc_dep(targetDep)) |
| 240 |
if (!(has_grf_dep(targetDep) || has_arf_a_dep(targetDep) || |
| 241 |
has_sp_dep(targetDep))) |
| 242 |
if (!has_scalar_dep(targetDep)) |
| 243 |
isRAW = false; |
| 244 |
} |
| 245 |
} |
| 246 |
|
| 247 |
/** |
| 248 |
* This function takes in a current instruction dependency. |
| 249 |
* Either SRC or DST |
| 250 |
* It then checks against previous dependencies. |
| 251 |
* It sets mininum valid distance |
| 252 |
* and creates an active list of SBIDs this instruction depends on |
| 253 |
* It clears and removes previous dependencies. |
| 254 |
* The approach is bucket based. |
| 255 |
* Each bucket is one GRF. |
| 256 |
* So if instruction writes in to more then one GRF then multiple buckets will |
| 257 |
* have the dependency |
| 258 |
*/ |
| 259 |
void SWSBAnalyzer::calculateDependence(DepSet &currDep, |
| 260 |
SWSB &swsb, |
| 261 |
const Instruction &currInst, |
| 262 |
activeSBIDsTy &activeSBIDs, |
| 263 |
bool &needSyncForShootDownInst) { |
| 264 |
needSyncForShootDownInst = false; |
| 265 |
auto currDepType = currDep.getDepType(); |
| 266 |
auto currDepPipe = currDep.getDepPipe(); |
| 267 |
|
| 268 |
for (auto bucketID : currDep.getBuckets()) { |
| 269 |
// iterates over Dependencies in a GRF bucket |
| 270 |
// Assumption there shouldn't be more then 1-2 |
| 271 |
Bucket *bucket = &m_buckets[bucketID]; |
| 272 |
size_t numDepSets = bucket->getNumDependencies(); |
| 273 |
for (uint32_t i = 0; i < numDepSets; ++i) { |
| 274 |
uint32_t index = static_cast<uint32_t>(numDepSets - 1 - i); |
| 275 |
auto dep = bucket->getDepSet(index); |
| 276 |
|
| 277 |
if (dep && (dep->getDepType() == DEP_TYPE::WRITE_ALWAYS_INTERFERE || |
| 278 |
dep->getDepType() == DEP_TYPE::READ_ALWAYS_INTERFERE)) { |
| 279 |
// force to sync with dep |
| 280 |
if (dep->getDepClass() == DEP_CLASS::OUT_OF_ORDER) { |
| 281 |
setSbidDependency(*dep, currInst, needSyncForShootDownInst, |
| 282 |
activeSBIDs); |
| 283 |
} else { |
| 284 |
// Set to sync with all in-order-pipes. WRITE/READ_ALWAYS_INTERFERE |
| 285 |
// could be used to mark arf dependency, which is required to be all |
| 286 |
// pipes instead of dep's pipe only |
| 287 |
swsb.minDist = 1; |
| 288 |
if (getNumOfDistPipe() == 1) |
| 289 |
swsb.distType = SWSB::DistType::REG_DIST; |
| 290 |
else |
| 291 |
swsb.distType = SWSB::DistType::REG_DIST_ALL; |
| 292 |
bucket->clearDepSet(index); |
| 293 |
} |
| 294 |
} |
| 295 |
|
| 296 |
// See if anything matches for this GRF bucket. |
| 297 |
if (dep && dep->getBitSet().intersects(currDep.getBitSet())) { |
| 298 |
/* |
| 299 |
* RAW: R kill W R-->live explict |
| 300 |
* dependence WAW: different pipelines W2 kill W1 W2-->live explict |
| 301 |
* dependence WAR: different pipelines W kill R W-->live explict |
| 302 |
* dependence WAR: same pipeline W kill R W-->live implict |
| 303 |
* dependence AR: sample pipeline R2 kill R1 R2-->live no |
| 304 |
* dependence RAR: different pipelines R1,R2-->live no |
| 305 |
* dependence |
| 306 |
*/ |
| 307 |
// RAW: R kill W R-->live explict |
| 308 |
// dependence |
| 309 |
DEP_TYPE prevDepType = dep->getDepType(); |
| 310 |
DEP_PIPE prevDepPipe = dep->getDepPipe(); |
| 311 |
DEP_CLASS prevDepClass = dep->getDepClass(); |
| 312 |
|
| 313 |
// Send with different SFID could write to different pipes |
| 314 |
bool sendInDiffPipe = false; |
| 315 |
if (dep->getInstruction()->getOpSpec().isAnySendFormat() && |
| 316 |
currDep.getInstruction()->getOpSpec().isAnySendFormat()) { |
| 317 |
sendInDiffPipe = (dep->getInstruction()->getSendFc() != |
| 318 |
currDep.getInstruction()->getSendFc()); |
| 319 |
// for send in unknown pipe, always treated as different pipe |
| 320 |
if (!sendInDiffPipe) { |
| 321 |
sendInDiffPipe = dep->getDepPipe() == DEP_PIPE::SEND_UNKNOWN || |
| 322 |
currDep.getDepPipe() == DEP_PIPE::SEND_UNKNOWN; |
| 323 |
} |
| 324 |
} |
| 325 |
|
| 326 |
auto isRead = [](DEP_TYPE ty) { |
| 327 |
return ty == DEP_TYPE::READ || |
| 328 |
ty == DEP_TYPE::READ_ALWAYS_INTERFERE; |
| 329 |
}; |
| 330 |
auto isWrite = [](DEP_TYPE ty) { |
| 331 |
return ty == DEP_TYPE::WRITE || |
| 332 |
ty == DEP_TYPE::WRITE_ALWAYS_INTERFERE; |
| 333 |
}; |
| 334 |
|
| 335 |
bool isRAW = isRead(currDepType) && isWrite(prevDepType); |
| 336 |
// WAW: different pipelines W2 kill W1 W2-->live explict |
| 337 |
// dependence |
| 338 |
bool isWAW = |
| 339 |
(isWrite(currDepType) && isWrite(prevDepType) && |
| 340 |
(currDepPipe != prevDepPipe || sendInDiffPipe)); |
| 341 |
// WAR: different pipelines W kill R W-->live explict |
| 342 |
// dependence |
| 343 |
bool isWAR = isWrite(currDepType) && |
| 344 |
isRead(prevDepType) && |
| 345 |
(currDepPipe != prevDepPipe || sendInDiffPipe); |
| 346 |
bool isWAW_out_of_order = |
| 347 |
(isWrite(currDepType) && isWrite(prevDepType) && |
| 348 |
prevDepClass == DEP_CLASS::OUT_OF_ORDER); |
| 349 |
|
| 350 |
// Special case handling for acc/flag dependency: |
| 351 |
// if the RAW dependency on acc/flag and acc/flag only, and it's whithin |
| 352 |
// the same pipe, HW can handle it that we don't need to set swsb |
| 353 |
if (isRAW && currDepPipe == prevDepPipe) |
| 354 |
checkAccFlagRAW(isRAW, currDep, *dep); |
| 355 |
|
| 356 |
if (isWAR || isWAW || isRAW || isWAW_out_of_order) { |
| 357 |
if (dep->getBitSet().empty()) { |
| 358 |
m_errorHandler.reportWarning( |
| 359 |
currInst.getPC(), "Dependency in bucket with no bits set"); |
| 360 |
} |
| 361 |
// removing from bucket if there is nothing |
| 362 |
if (!dep->getBitSet().testAny(bucketID * 32, |
| 363 |
m_DB->getGRF_BYTES_PER_REG())) { |
| 364 |
bucket->clearDepSet(index); |
| 365 |
} |
| 366 |
|
| 367 |
if (prevDepClass == DEP_CLASS::IN_ORDER) { |
| 368 |
setDistanceDependency(dep, swsb, isWAW, prevDepPipe, currDepPipe, |
| 369 |
currInst); |
| 370 |
} else if (prevDepClass == DEP_CLASS::OUT_OF_ORDER) { |
| 371 |
setSbidDependency(*dep, currInst, needSyncForShootDownInst, |
| 372 |
activeSBIDs); |
| 373 |
} |
| 374 |
// for the instruction in "OTHER" DEP_CLASS, such as sync, we don't |
| 375 |
// need to consider their dependency that is implied by hardware |
| 376 |
} |
| 377 |
} |
| 378 |
} |
| 379 |
} |
| 380 |
} |
| 381 |
|
| 382 |
void SWSBAnalyzer::setDistanceDependency(DepSet *dep, SWSB &swsb, bool isWAW, |
| 383 |
DEP_PIPE prevDepPipe, DEP_PIPE currDepPipe, const Instruction &currInst) { |
| 384 |
|
| 385 |
if (getNumOfDistPipe() == 1) { |
| 386 |
// FOR WAW if PREV is SHORT and curr is LONG then write will |
| 387 |
// finish before current write, no need to set swsb |
| 388 |
bool isWAWHazard = |
| 389 |
((prevDepPipe == DEP_PIPE::SHORT && currDepPipe == DEP_PIPE::LONG) || |
| 390 |
(prevDepPipe == DEP_PIPE::SHORT && currDepPipe == DEP_PIPE::SHORT)) && |
| 391 |
isWAW; |
| 392 |
// require swsb for all the other kinds of dependency |
| 393 |
if (!isWAWHazard) { |
| 394 |
// setting minimum distance |
| 395 |
uint32_t newDistance = |
| 396 |
m_InstIdCounter.inOrder - dep->getInstIDs().inOrder; |
| 397 |
swsb.minDist = |
| 398 |
swsb.minDist == 0 ? newDistance : std::min(swsb.minDist, newDistance); |
| 399 |
// clamp the distance to max distance |
| 400 |
swsb.minDist = std::min(swsb.minDist, (uint32_t)MAX_VALID_DISTANCE); |
| 401 |
swsb.distType = SWSB::DistType::REG_DIST; |
| 402 |
} |
| 403 |
} else { |
| 404 |
// For multiple in-order pipeline architecuture, all cases should |
| 405 |
// be considered The distance is depended on the previous |
| 406 |
// instruction's pipeline |
| 407 |
uint32_t newDistance = 0; |
| 408 |
SWSB::DistType newDepPipe = SWSB::DistType::NO_DIST; |
| 409 |
switch (prevDepPipe) { |
| 410 |
case DEP_PIPE::FLOAT: |
| 411 |
newDistance = m_InstIdCounter.floatPipe - dep->getInstIDs().floatPipe; |
| 412 |
newDepPipe = SWSB::DistType::REG_DIST_FLOAT; |
| 413 |
break; |
| 414 |
case DEP_PIPE::INTEGER: |
| 415 |
newDistance = m_InstIdCounter.intPipe - dep->getInstIDs().intPipe; |
| 416 |
newDepPipe = SWSB::DistType::REG_DIST_INT; |
| 417 |
break; |
| 418 |
case DEP_PIPE::LONG64: |
| 419 |
newDistance = m_InstIdCounter.longPipe - dep->getInstIDs().longPipe; |
| 420 |
newDepPipe = SWSB::DistType::REG_DIST_LONG; |
| 421 |
break; |
| 422 |
case DEP_PIPE::MATH_INORDER: |
| 423 |
newDistance = m_InstIdCounter.mathPipe - dep->getInstIDs().mathPipe; |
| 424 |
newDepPipe = SWSB::DistType::REG_DIST_MATH; |
| 425 |
break; |
| 426 |
case DEP_PIPE::SCALAR: |
| 427 |
newDistance = m_InstIdCounter.scalarPipe - dep->getInstIDs().scalarPipe; |
| 428 |
newDepPipe = SWSB::DistType::REG_DIST_SCALAR; |
| 429 |
break; |
| 430 |
default: |
| 431 |
IGA_ASSERT(0, "Unsupported DEP_PIPE for in-order instructions"); |
| 432 |
break; |
| 433 |
} |
| 434 |
|
| 435 |
if (swsb.minDist) { |
| 436 |
// the instruction already has dependency to others. Take the |
| 437 |
// min distance and set distType to ALL if they are not the same |
| 438 |
newDistance = std::min(swsb.minDist, newDistance); |
| 439 |
if (swsb.distType != newDepPipe) |
| 440 |
swsb.distType = SWSB::DistType::REG_DIST_ALL; |
| 441 |
} else { |
| 442 |
swsb.distType = newDepPipe; |
| 443 |
} |
| 444 |
assert(swsb.distType != SWSB::DistType::NO_DIST); |
| 445 |
// clamp the distance to max distance |
| 446 |
swsb.minDist = std::min(newDistance, (uint32_t)MAX_VALID_DISTANCE); |
| 447 |
} // end of if (m_enableMultiDistPipe) |
| 448 |
// clear this instruction's dependency since it is satisfied |
| 449 |
clearDepBuckets(*dep); |
| 450 |
|
| 451 |
// clear its companion because when an in-order instruction is |
| 452 |
// synced, both its input and output dependency are satisfied. The |
| 453 |
// only case is that if it has read/write_always_interfere |
| 454 |
// dependency, it should be reserved. The restriction is that: When |
| 455 |
// certain Arch Registers (sr, cr, ce) are used, the very next |
| 456 |
// instruction requires dependency to be set on all pipes {A@1} e.g. |
| 457 |
// mov (1|M0) r104.0<1>:ud sr0.1<0;1,0>:ud |
| 458 |
// cmp(16 | M0) (ne)f0.0 null:ud r104.0<0; 1, 0> : ub |
| 459 |
// r62.4<0; 1, 0> : uw |
| 460 |
// A@1 is required for cmp instead of I@1 |
| 461 |
if (dep->getCompanion() != nullptr) { |
| 462 |
// In the case that this DepSet is generated from math_wa_info, it |
| 463 |
// won't have companion |
| 464 |
if (dep->getCompanion()->getDepType() != DEP_TYPE::WRITE_ALWAYS_INTERFERE && |
| 465 |
dep->getCompanion()->getDepType() != DEP_TYPE::READ_ALWAYS_INTERFERE) { |
| 466 |
clearDepBuckets(*dep->getCompanion()); |
| 467 |
} |
| 468 |
} |
| 469 |
} |
| 470 |
|
| 471 |
void SWSBAnalyzer::setSbidDependency(DepSet &dep, const Instruction &currInst, |
| 472 |
bool &needSyncForShootDownInst, |
| 473 |
activeSBIDsTy &activeSBIDs) { |
| 474 |
/* For out of order we don't know how long it will finish |
| 475 |
* so need to test for SBID. |
| 476 |
* Instruction can depend on more then one SBID |
| 477 |
* send r10 |
| 478 |
* send r20 |
| 479 |
* send r30 |
| 480 |
* .... |
| 481 |
* add r10 r20 r30 |
| 482 |
* between different buckets and srcs/dst dependencies instruction can rely on |
| 483 |
* multiple SBID |
| 484 |
*/ |
| 485 |
SBID depSBID = dep.getSBID(); |
| 486 |
if (depSBID.isFree) { |
| 487 |
m_errorHandler.reportError((int)dep.getInstGlobalID(), |
| 488 |
"SBID SHOULDN'T BE FREE!"); |
| 489 |
} |
| 490 |
// clears all the buckets |
| 491 |
clearDepBuckets(dep); |
| 492 |
|
| 493 |
// In case of shooting down of this instruction, we need to add sync to |
| 494 |
// preserve the swsb id sync, so that it's safe to clear the dep |
| 495 |
if (currInst.hasPredication() || |
| 496 |
(currInst.getExecSize() != dep.getInstruction()->getExecSize()) || |
| 497 |
(currInst.getChannelOffset() != dep.getInstruction()->getChannelOffset())) |
| 498 |
needSyncForShootDownInst = true; |
| 499 |
|
| 500 |
// used to set read or write dependency |
| 501 |
depSBID.dType = dep.getDepType(); |
| 502 |
|
| 503 |
// activeSBIDs stores all sbid that this inst has dependency on |
| 504 |
// and it'll be processed in processactiveSBIDs |
| 505 |
bool insert_new_sbid = true; |
| 506 |
// making sure there are no duplicates |
| 507 |
for (auto &aSBID : activeSBIDs) { |
| 508 |
if (aSBID.first.sbid == depSBID.sbid) { |
| 509 |
// write takes longer then read |
| 510 |
// force the SBID to WRITE type (e.g. $10.dst) if the newly added |
| 511 |
// dependency is a write dependency. |
| 512 |
if (depSBID.dType == DEP_TYPE::WRITE || |
| 513 |
depSBID.dType == DEP_TYPE::WRITE_ALWAYS_INTERFERE) { |
| 514 |
aSBID.first.dType = depSBID.dType; |
| 515 |
} |
| 516 |
// add DepSet into this activeSBIDs's associated list |
| 517 |
aSBID.second.push_back(&dep); |
| 518 |
// the sbid is already in the list, we don't need to add it to |
| 519 |
// activeSBIDs |
| 520 |
insert_new_sbid = false; |
| 521 |
break; |
| 522 |
} |
| 523 |
} |
| 524 |
|
| 525 |
// adding to active SBID |
| 526 |
// in Run function we will see how many this instruction relies on |
| 527 |
// and generate approriate SWSB and if needed test instruction |
| 528 |
// in that level also will add them back to free list |
| 529 |
if (insert_new_sbid) { |
| 530 |
activeSBIDs.push_back(std::make_pair(depSBID, std::vector<DepSet*>())); |
| 531 |
activeSBIDs.back().second.push_back(&dep); |
| 532 |
} |
| 533 |
} |
| 534 |
|
| 535 |
void SWSBAnalyzer::insertSyncAllRdWr(InstList::iterator insertPoint, |
| 536 |
Block *bb) { |
| 537 |
SWSB swsb; |
| 538 |
auto clearRD = m_kernel.createSyncAllRdInstruction(swsb); |
| 539 |
auto clearWR = m_kernel.createSyncAllWrInstruction(swsb); |
| 540 |
|
| 541 |
if (insertPoint == bb->getInstList().end()) { |
| 542 |
bb->getInstList().push_back(clearRD); |
| 543 |
bb->getInstList().push_back(clearWR); |
| 544 |
} else { |
| 545 |
bb->insertInstBefore(insertPoint, clearRD); |
| 546 |
bb->insertInstBefore(insertPoint, clearWR); |
| 547 |
} |
| 548 |
} |
| 549 |
|
| 550 |
// TODO this should also clear up grf dependency to handle this case: |
| 551 |
/* |
| 552 |
call (16|M0) r8.0:ud 32 |
| 553 |
sendc.rc (16|M0) null r118 null 0x0 0x140B1000 {} // |
| 554 |
wr:10h, rd:0, Render Target Write msc:16, to #0 (W) mov (1|M0) a0.0<1>:ud |
| 555 |
r7.0<0;1,0>:ud sendc.rc (16|M0) null r100 null 0x0 0x140B1000 {} |
| 556 |
// wr:10h, rd:0, Render Target Write msc:16, to #0 sendc.rc (16|M0) null r118 |
| 557 |
null 0x0 0x140B1000 {} // wr:10h, rd:0, Render Target Write msc:16, |
| 558 |
to #0 (W) mov (16|M0) r118.0<1>:ud r6.0<8;8,1>:ud (W) |
| 559 |
send.dc0 (16|M0) r38 r118 null 0x0 a0.0 ret (16|M0) |
| 560 |
|
| 561 |
Right now mov will have false dependense on the first send. |
| 562 |
*/ |
| 563 |
bool SWSBAnalyzer::clearSBIDDependence(InstList::iterator insertPoint, |
| 564 |
Instruction *lastInst, Block *bb) { |
| 565 |
|
| 566 |
auto clearSBID = [&](const SBID& in) { |
| 567 |
assert(m_IdToDepSetMap.find(in.sbid) != m_IdToDepSetMap.end()); |
| 568 |
assert(m_IdToDepSetMap[in.sbid].first->getDepClass() == |
| 569 |
DEP_CLASS::OUT_OF_ORDER); |
| 570 |
clearDepBuckets(*m_IdToDepSetMap[in.sbid].first); |
| 571 |
clearDepBuckets(*m_IdToDepSetMap[in.sbid].second); |
| 572 |
m_freeSBIDList[in.sbid].reset(); |
| 573 |
}; |
| 574 |
|
| 575 |
bool sbidInUse = false; |
| 576 |
for (uint32_t i = 0; i < m_SBIDCount; ++i) { |
| 577 |
// there are still dependencies that might be used outside of this basic |
| 578 |
// block |
| 579 |
if (!m_freeSBIDList[i].isFree) { |
| 580 |
clearSBID(m_freeSBIDList[i]); |
| 581 |
sbidInUse = true; |
| 582 |
} |
| 583 |
} |
| 584 |
|
| 585 |
// if last instruction in basic block is EOT no need to generate flushes |
| 586 |
// hardware will take care of it |
| 587 |
if (lastInst && lastInst->getOpSpec().isAnySendFormat() && |
| 588 |
lastInst->hasInstOpt(InstOpt::EOT)) { |
| 589 |
sbidInUse = false; |
| 590 |
} |
| 591 |
|
| 592 |
// platform check is mainly for testing purposes |
| 593 |
if (sbidInUse) { |
| 594 |
insertSyncAllRdWr(insertPoint, bb); |
| 595 |
} |
| 596 |
|
| 597 |
return sbidInUse; |
| 598 |
} |
| 599 |
|
| 600 |
// Keeping track of dependencies that need to be cleared because they are no |
| 601 |
// longer relevant right now each BB ends with control flow instruction, and we |
| 602 |
// reset at each BB |
| 603 |
void SWSBAnalyzer::clearBuckets(DepSet *input, DepSet *output) { |
| 604 |
if (input->getDepClass() != DEP_CLASS::IN_ORDER) |
| 605 |
return; |
| 606 |
|
| 607 |
if (m_initPoint) { |
| 608 |
m_distanceTracker.emplace_back(input, output); |
| 609 |
m_initPoint = false; |
| 610 |
|
| 611 |
} else { |
| 612 |
// add DepSet to m_distanceTracker |
| 613 |
m_distanceTracker.emplace_back(input, output); |
| 614 |
|
| 615 |
auto get_depset_id = [&](DEP_PIPE pipe_type, DepSet &dep_set) { |
| 616 |
if (getNumOfDistPipe() == 1) |
| 617 |
return dep_set.getInstIDs().inOrder; |
| 618 |
switch (pipe_type) { |
| 619 |
case DEP_PIPE::FLOAT: |
| 620 |
return dep_set.getInstIDs().floatPipe; |
| 621 |
case DEP_PIPE::INTEGER: |
| 622 |
return dep_set.getInstIDs().intPipe; |
| 623 |
case DEP_PIPE::LONG64: |
| 624 |
return dep_set.getInstIDs().longPipe; |
| 625 |
case DEP_PIPE::MATH_INORDER: |
| 626 |
return dep_set.getInstIDs().mathPipe; |
| 627 |
case DEP_PIPE::SCALAR: |
| 628 |
return dep_set.getInstIDs().scalarPipe; |
| 629 |
default: |
| 630 |
IGA_ASSERT(0, "SWSB: unhandled in-order DEP_PIPE for XeHP+ encoding"); |
| 631 |
break; |
| 632 |
} |
| 633 |
return (uint32_t)0; |
| 634 |
}; |
| 635 |
|
| 636 |
auto get_latency = [&](DEP_PIPE pipe_type) { |
| 637 |
if (pipe_type == DEP_PIPE::LONG64) |
| 638 |
return m_LatencyLong64Pipe; |
| 639 |
else if (pipe_type == DEP_PIPE::MATH_INORDER) |
| 640 |
return m_LatencyInOrderMath; |
| 641 |
return m_LatencyInOrderPipe; |
| 642 |
}; |
| 643 |
|
| 644 |
DEP_PIPE new_pipe = input->getDepPipe(); |
| 645 |
// max B2B latency of thie pipe |
| 646 |
size_t max_dis = get_latency(new_pipe); |
| 647 |
// Remove nodes from the Tracker if the latency is already satified |
| 648 |
m_distanceTracker.remove_if([=](const distanceTrackerNode &node) { |
| 649 |
// bypass nodes those are not belong to the same pipe |
| 650 |
if (node.input->getDepPipe() != new_pipe) |
| 651 |
return false; |
| 652 |
|
| 653 |
// if the distance >= max_latency, clear buckets for corresponding |
| 654 |
// input and output Dependency |
| 655 |
size_t new_id = get_depset_id(new_pipe, *input); |
| 656 |
if ((new_id - get_depset_id(new_pipe, *node.input)) >= max_dis) { |
| 657 |
clearDepBuckets(*node.input); |
| 658 |
clearDepBuckets(*node.output); |
| 659 |
return true; |
| 660 |
} |
| 661 |
return false; |
| 662 |
}); |
| 663 |
} |
| 664 |
} |
| 665 |
|
| 666 |
bool SWSBAnalyzer::trySBIDCounter(activeSBIDsTy &activeSBIDs, |
| 667 |
const DepSet &curDepSet, |
| 668 |
SBID &retSBID) { |
| 669 |
// There is only one active sbid. It doesn't worth using sbid counter |
| 670 |
if (activeSBIDs.size() <= 1) |
| 671 |
return false; |
| 672 |
|
| 673 |
// curDepSet is a force-sync-all, all dependency will be sync-ed so there |
| 674 |
// is no benefit to do sbid-counter |
| 675 |
if (curDepSet.getDepType() == DEP_TYPE::WRITE_ALWAYS_INTERFERE || |
| 676 |
curDepSet.getDepType() == DEP_TYPE::READ_ALWAYS_INTERFERE) |
| 677 |
return false; |
| 678 |
|
| 679 |
// To simplify the design, we only handle the case that current instruction |
| 680 |
// (which has dependency to activeSBIDs) is an in-order-pipe instruction |
| 681 |
if (curDepSet.getDepClass() != DEP_CLASS::IN_ORDER) |
| 682 |
return false; |
| 683 |
|
| 684 |
// If there is non-send instruction the activeSBIDs associated with, sbid |
| 685 |
// counter cannot be used |
| 686 |
// keep track of if there is any sbid counter used in activeSBIDs already |
| 687 |
// If there is one, we will replace to it if possible |
| 688 |
std::optional<SBID> hasCounterID = std::nullopt; |
| 689 |
for (const auto& aSBID : activeSBIDs) { |
| 690 |
for (auto depSet : aSBID.second) { |
| 691 |
auto inst = depSet->getInstruction(); |
| 692 |
// Also skip counter replacement if the send has NOACCSBSET set or its |
| 693 |
// sbid is already a counter |
| 694 |
if (!inst->getOpSpec().isAnySendFormat() || |
| 695 |
inst->getSWSB().hasSpecialToken()) { |
| 696 |
return false; |
| 697 |
} |
| 698 |
if (inst->getSWSB().tokenType == SWSB::TokenType::INC) |
| 699 |
hasCounterID = aSBID.first; |
| 700 |
} |
| 701 |
} |
| 702 |
|
| 703 |
// Replace the associated instructions' sbid to sbid counter. For example, |
| 704 |
// From: |
| 705 |
// send.ugm (1) r50 r31 null:0 0xFF000000 0x6219C500 {$0} |
| 706 |
// send.ugm (1) r60 r31 null:0 0xFF000000 0x6219C500 {$1} |
| 707 |
// // add's activeSBIDs are $0 and $1 |
| 708 |
// add (1|M0) r90:ud r50:ud r60:ud |
| 709 |
// To: |
| 710 |
// send.ugm (1) r50 r31 null:0 0xFF000000 0x6219C500 {$0.inc} |
| 711 |
// send.ugm (1) r60 r31 null:0 0xFF000000 0x6219C500 {$0.inc} |
| 712 |
// add (1|M0) r90:ud r50:ud r60:ud {$0.dst} |
| 713 |
// |
| 714 |
// These send instructions must have no .dst dependency to each other, otherwise |
| 715 |
// the dependency should be resovled and they won't both be activeSBID here. |
| 716 |
// Hence we don't need to check the dependency among these instructions. |
| 717 |
// Note, having .src dependency to each other is fine. Sbid counter resolves |
| 718 |
// the .src dependency. |
| 719 |
|
| 720 |
// pick the existing counter (if any), or the first activeSBID as the SBID |
| 721 |
// counter. |
| 722 |
retSBID = hasCounterID ? *hasCounterID : activeSBIDs.front().first; |
| 723 |
// Keep track of the DepSet's DepType (Read or Write) |
| 724 |
retSBID.dType = activeSBIDs.front().first.dType; |
| 725 |
|
| 726 |
auto isActiveSBID = [&activeSBIDs](uint32_t sbid) { |
| 727 |
for(const auto& aSBID : activeSBIDs) { |
| 728 |
if (sbid == aSBID.first.sbid) |
| 729 |
return true; |
| 730 |
} |
| 731 |
return false; |
| 732 |
}; |
| 733 |
|
| 734 |
for (const auto& aSBID : activeSBIDs) { |
| 735 |
for (auto depSet : aSBID.second) { |
| 736 |
// Updated the set SWSB on send instructions |
| 737 |
auto inst = depSet->getInstruction(); |
| 738 |
auto instSWSB = inst->getSWSB(); |
| 739 |
instSWSB.sbid = retSBID.sbid; |
| 740 |
assert(instSWSB.tokenType == SWSB::TokenType::SET || |
| 741 |
instSWSB.tokenType == SWSB::TokenType::INC); |
| 742 |
instSWSB.tokenType = SWSB::TokenType::INC; |
| 743 |
inst->setSWSB(instSWSB); |
| 744 |
// Updated the DepSet and its companion DepSet's sbid |
| 745 |
// So that later instructions having dependency to these DepSet can have |
| 746 |
// the correct sbid ref |
| 747 |
SBID newSBID = SBID(depSet->getSBID()); |
| 748 |
newSBID.sbid = retSBID.sbid; |
| 749 |
depSet->setSBID(newSBID); |
| 750 |
assert(depSet->getCompanion()); |
| 751 |
depSet->getCompanion()->setSBID(newSBID); |
| 752 |
// Updated instructions those set to have dependency to these SBIDs |
| 753 |
for (auto i : depSet->getSrcDepInsts()) { |
| 754 |
auto swsb = i->getSWSB(); |
| 755 |
// update only when the instruction has sbid that's the same as |
| 756 |
// activeSBIDs. There is case that sbid.src was set to an instruction |
| 757 |
// (and hence be added to SrcDepInsts). But that sbid.src was moved to |
| 758 |
// a sync.nop and the instruction was assigned an irrelevant sbid. |
| 759 |
// We cannot avoid adding that instruction into SrcDepInsts, so we |
| 760 |
// need to do the additional check here to avoid updating its sbid. |
| 761 |
if (!isActiveSBID(swsb.sbid)) |
| 762 |
continue; |
| 763 |
swsb.sbid = retSBID.sbid; |
| 764 |
i->setSWSB(swsb); |
| 765 |
} |
| 766 |
} |
| 767 |
// Set depType to the strongest one (WRITE or WRITE_ALWAYS_INTERFERE) |
| 768 |
// if any. Note, it makes no difference for it is READ or |
| 769 |
// READ_ALWAYS_INTERFERE here |
| 770 |
if (retSBID.dType == DEP_TYPE::WRITE_ALWAYS_INTERFERE) |
| 771 |
continue; |
| 772 |
if (aSBID.first.dType == DEP_TYPE::WRITE || |
| 773 |
aSBID.first.dType == DEP_TYPE::WRITE_ALWAYS_INTERFERE) |
| 774 |
retSBID.dType = aSBID.first.dType; |
| 775 |
} |
| 776 |
|
| 777 |
return true; |
| 778 |
} |
| 779 |
|
| 780 |
void SWSBAnalyzer::processactiveSBIDs(SWSB &swsb, |
| 781 |
const DepSet *input, Block *bb, |
| 782 |
InstList::iterator instIter, |
| 783 |
activeSBIDsTy &activeSBIDs) { |
| 784 |
// free SBID in activeSBIDs and clear the dependecies of associated |
| 785 |
// DepSet |
| 786 |
auto clearSBID = [&](const SBID& in) { |
| 787 |
m_freeSBIDList[in.sbid].reset(); |
| 788 |
assert(m_IdToDepSetMap.find(in.sbid) != m_IdToDepSetMap.end()); |
| 789 |
assert(m_IdToDepSetMap[in.sbid].first->getDepClass() == |
| 790 |
DEP_CLASS::OUT_OF_ORDER); |
| 791 |
clearDepBuckets(*m_IdToDepSetMap[in.sbid].first); |
| 792 |
clearDepBuckets(*m_IdToDepSetMap[in.sbid].second); |
| 793 |
}; |
| 794 |
|
| 795 |
// If the instruction depends on more than one sbid, and those sbids all |
| 796 |
// comes from send instructions, we can replace the sbid by sbid counter |
| 797 |
if (enableSBIDCounterSetting()) { |
| 798 |
SBID chosenSBID; |
| 799 |
if (trySBIDCounter(activeSBIDs, *input, chosenSBID)) { |
| 800 |
// build swsb for current instruction according to the chosen SBID |
| 801 |
assert(swsb.tokenType == SWSB::TokenType::NOTOKEN); |
| 802 |
swsb.sbid = chosenSBID.sbid; |
| 803 |
if (chosenSBID.dType == DEP_TYPE::READ || |
| 804 |
chosenSBID.dType == DEP_TYPE::READ_ALWAYS_INTERFERE) { |
| 805 |
swsb.tokenType = SWSB::TokenType::SRC; |
| 806 |
} else { |
| 807 |
swsb.tokenType = SWSB::TokenType::DST; |
| 808 |
// All associated DepSet are cleared with .dst dependency here. |
| 809 |
// free the SBIDs and clear dependency |
| 810 |
for (const auto& aSBID : activeSBIDs) |
| 811 |
clearSBID(aSBID.first); |
| 812 |
} |
| 813 |
// verify if the combination of token and dist is valid, if not, move the |
| 814 |
// token dependency out and add a sync for it |
| 815 |
adjustSWSB(*bb, instIter, swsb, true); |
| 816 |
return; |
| 817 |
} |
| 818 |
} |
| 819 |
|
| 820 |
// If instruction depends on one or more SBIDS, first one goes in to SWSB |
| 821 |
// field for rest we generate wait instructions. |
| 822 |
for (const auto &aSBID : activeSBIDs) { |
| 823 |
// Could be we had operation depending on the write |
| 824 |
/* |
| 825 |
* This case also gets triggered when we have send in BB and dependence in |
| 826 |
* another BB L0: call (16|M0) r8.0 L64 L16: sendc.rc |
| 827 |
* (16|M0) null r118 null 0x0 0x140B1000 {$0} // |
| 828 |
* wr:10h, rd:0, Render Target Write msc:16, to #0 L64: (W) mov (16|M0) |
| 829 |
* r118.0<1>:ud r6.0<8;8,1>:ud (W) send.dc0 (16|M0) r38 r118 |
| 830 |
* null 0x0 a0.0 {@1, $0} ret (16|M0) r8.0 {@3} After first |
| 831 |
* BB in which sendc.rc ends we clear all SBID and generate sync |
| 832 |
* instructions On mov it detects dependense, but all SBID are freed. |
| 833 |
*/ |
| 834 |
if (m_freeSBIDList[aSBID.first.sbid].isFree) { |
| 835 |
continue; |
| 836 |
} |
| 837 |
|
| 838 |
SWSB::TokenType tType = SWSB::TokenType::NOTOKEN; |
| 839 |
if (aSBID.first.dType == DEP_TYPE::READ || |
| 840 |
aSBID.first.dType == DEP_TYPE::READ_ALWAYS_INTERFERE) { |
| 841 |
tType = SWSB::TokenType::SRC; |
| 842 |
} else { |
| 843 |
tType = SWSB::TokenType::DST; |
| 844 |
// if SBID is cleared add it back to free pool |
| 845 |
// write is last thing. So if instruction depends on it we know read is |
| 846 |
// done but not vice versa |
| 847 |
clearSBID(aSBID.first); |
| 848 |
} |
| 849 |
|
| 850 |
// Setting first SBID as part of instruction |
| 851 |
// If this instruction depends on more SBID, generate sync for the extra ids |
| 852 |
// TODO: Is it safe to clear SBID here? |
| 853 |
if (swsb.tokenType == SWSB::TokenType::NOTOKEN) { |
| 854 |
swsb.tokenType = tType; |
| 855 |
swsb.sbid = aSBID.first.sbid; |
| 856 |
} else { |
| 857 |
// add sync for the id |
| 858 |
SWSB sync_swsb(SWSB::DistType::NO_DIST, tType, 0, aSBID.first.sbid); |
| 859 |
auto nopInst = m_kernel.createSyncNopInstruction(sync_swsb); |
| 860 |
bb->insertInstBefore(instIter, nopInst); |
| 861 |
m_IdToDepSetMap[aSBID.first.sbid].first->addSrcDepInsts(nopInst); |
| 862 |
m_IdToDepSetMap[aSBID.first.sbid].first->getCompanion()->addSrcDepInsts(nopInst); |
| 863 |
m_IdToDepSetMap[aSBID.first.sbid].second->addSrcDepInsts(nopInst); |
| 864 |
} |
| 865 |
} |
| 866 |
|
| 867 |
// verify if the combination of token and dist is valid, if not, move the |
| 868 |
// token dependency out and add a sync for it |
| 869 |
auto syncInsts = adjustSWSB(*bb, instIter, swsb, true); |
| 870 |
|
| 871 |
// update the DepSet's SrcDepInsts if current instruction or any newly added |
| 872 |
// sync is associated with it |
| 873 |
auto checkAndUpdateSrcDepInsts = |
| 874 |
[&activeSBIDs](Instruction* inst, SWSB swsb) { |
| 875 |
if (!swsb.hasToken()) |
| 876 |
return; |
| 877 |
if (swsb.tokenType != SWSB::TokenType::SET && |
| 878 |
swsb.tokenType != SWSB::TokenType::SRC) |
| 879 |
return; |
| 880 |
for (const auto &aSBID : activeSBIDs) { |
| 881 |
if (swsb.sbid == aSBID.first.sbid) { |
| 882 |
for (auto& ds : aSBID.second) { |
| 883 |
ds->addSrcDepInsts(inst); |
| 884 |
ds->getCompanion()->addSrcDepInsts(inst); |
| 885 |
} |
| 886 |
} |
| 887 |
} |
| 888 |
}; |
| 889 |
|
| 890 |
for (auto sync : syncInsts) { |
| 891 |
checkAndUpdateSrcDepInsts(sync, sync->getSWSB()); |
| 892 |
} |
| 893 |
checkAndUpdateSrcDepInsts(*instIter, swsb); |
| 894 |
} |
| 895 |
|
| 896 |
uint32_t SWSBAnalyzer::getNumOfDistPipe() { |
| 897 |
return getNumOfDistPipe(m_swsbMode); |
| 898 |
} |
| 899 |
|
| 900 |
uint32_t SWSBAnalyzer::getNumOfDistPipe(SWSB_ENCODE_MODE mode) { |
| 901 |
switch (mode) { |
| 902 |
case SWSB_ENCODE_MODE::SingleDistPipe: |
| 903 |
return 1; |
| 904 |
case SWSB_ENCODE_MODE::ThreeDistPipe: |
| 905 |
return 3; |
| 906 |
case SWSB_ENCODE_MODE::ThreeDistPipeDPMath: |
| 907 |
return 3; |
| 908 |
case SWSB_ENCODE_MODE::FourDistPipe: |
| 909 |
case SWSB_ENCODE_MODE::FourDistPipeReduction: |
| 910 |
return 4; |
| 911 |
case SWSB_ENCODE_MODE::FiveDistPipe: |
| 912 |
case SWSB_ENCODE_MODE::FiveDistPipeReduction: |
| 913 |
case SWSB_ENCODE_MODE::FiveDistPipeCvtToInt: |
| 914 |
case SWSB_ENCODE_MODE::FiveDistPipeSWSBCntr: |
| 915 |
case SWSB_ENCODE_MODE::FiveDistPipeCvtToIntNoFwd: |
| 916 |
return 5; |
| 917 |
default: |
| 918 |
break; |
| 919 |
} |
| 920 |
return 0; |
| 921 |
} |
| 922 |
|
| 923 |
void SWSBAnalyzer::advanceInorderInstCounter(DEP_PIPE dep_pipe) { |
| 924 |
++m_InstIdCounter.inOrder; |
| 925 |
if (getNumOfDistPipe() == 1) |
| 926 |
return; |
| 927 |
|
| 928 |
switch (dep_pipe) { |
| 929 |
case DEP_PIPE::FLOAT: |
| 930 |
++m_InstIdCounter.floatPipe; |
| 931 |
break; |
| 932 |
case DEP_PIPE::INTEGER: |
| 933 |
++m_InstIdCounter.intPipe; |
| 934 |
break; |
| 935 |
case DEP_PIPE::LONG64: |
| 936 |
++m_InstIdCounter.longPipe; |
| 937 |
break; |
| 938 |
case DEP_PIPE::MATH_INORDER: |
| 939 |
++m_InstIdCounter.mathPipe; |
| 940 |
break; |
| 941 |
case DEP_PIPE::SCALAR: |
| 942 |
++m_InstIdCounter.scalarPipe; |
| 943 |
break; |
| 944 |
default: |
| 945 |
IGA_ASSERT(0, "unhandled in-order DEP_PIPE for XE_HP encoding"); |
| 946 |
break; |
| 947 |
} |
| 948 |
} |
| 949 |
|
| 950 |
void SWSBAnalyzer::addRMWDependencyIfReqruied(DepSet &input, DepSet &output) { |
| 951 |
const Instruction *inst = input.getInstruction(); |
| 952 |
// return if the instruction has no dst, or the dst is not GRF or not byte |
| 953 |
// type |
| 954 |
const Operand &dst = inst->getDestination(); |
| 955 |
if (dst.getKind() != Operand::Kind::DIRECT) |
| 956 |
return; |
| 957 |
|
| 958 |
if (dst.getDirRegName() != RegName::GRF_R) |
| 959 |
return; |
| 960 |
|
| 961 |
if (TypeSizeInBitsWithDefault(dst.getType(), 32) != 8) |
| 962 |
return; |
| 963 |
|
| 964 |
// When there is RMW behavior, the instruction will read the Word first, |
| 965 |
// modify the byte value in it and then write back the entire Word. |
| 966 |
// we assume the instruction will read/write the entire register to simplify |
| 967 |
// the logic |
| 968 |
|
| 969 |
// add the entire grf of the dst register into input and output DepSet |
| 970 |
// All registers being touched are added into Bucket. We can get the touched |
| 971 |
// grf number from added bucket index |
| 972 |
const std::vector<size_t> &out_buk = output.getBuckets(); |
| 973 |
for (auto i : out_buk) { |
| 974 |
// we only need grf bucket |
| 975 |
if (i >= m_DB->getBucketStart(RegName::ARF_A)) |
| 976 |
continue; |
| 977 |
input.addGrf(i); |
| 978 |
input.addToBucket((uint32_t)i); |
| 979 |
output.addGrf(i); |
| 980 |
} |
| 981 |
} |
| 982 |
|
| 983 |
void SWSBAnalyzer::addSWSBToInst(InstListIterator instIt, const SWSB &swsb, |
| 984 |
Block &block) { |
| 985 |
assert(instIt != block.getInstList().end()); |
| 986 |
Instruction &inst = **instIt; |
| 987 |
SWSB new_swsb(inst.getSWSB()); |
| 988 |
// handling distance |
| 989 |
if (swsb.hasDist()) { |
| 990 |
if (!inst.getSWSB().hasDist()) { |
| 991 |
new_swsb.distType = swsb.distType; |
| 992 |
new_swsb.minDist = swsb.minDist; |
| 993 |
} else { |
| 994 |
// for single dist pipe platform, distType must be REG_DIST, so won't |
| 995 |
// be set to REG_DIST_ALL |
| 996 |
new_swsb.distType = (inst.getSWSB().distType == swsb.distType) |
| 997 |
? swsb.distType |
| 998 |
: SWSB::DistType::REG_DIST_ALL; |
| 999 |
new_swsb.minDist = std::min(inst.getSWSB().minDist, swsb.minDist); |
| 1000 |
} |
| 1001 |
} |
| 1002 |
|
| 1003 |
// handling token |
| 1004 |
if (swsb.hasToken()) { |
| 1005 |
if (!inst.getSWSB().hasToken()) { |
| 1006 |
new_swsb.tokenType = swsb.tokenType; |
| 1007 |
new_swsb.sbid = swsb.sbid; |
| 1008 |
} else { |
| 1009 |
// if both has id, and are different, then insert a sync to carry |
| 1010 |
// the new one, otherwise do nothing |
| 1011 |
if ((inst.getSWSB().tokenType != swsb.tokenType) || |
| 1012 |
(inst.getSWSB().sbid != swsb.sbid)) { |
| 1013 |
SWSB tmp_swsb(SWSB::DistType::NO_DIST, swsb.tokenType, 0, swsb.sbid); |
| 1014 |
Instruction *sync_inst = m_kernel.createSyncNopInstruction(tmp_swsb); |
| 1015 |
block.insertInstBefore(instIt, sync_inst); |
| 1016 |
} |
| 1017 |
} |
| 1018 |
} |
| 1019 |
|
| 1020 |
// check if the new swsb combination is valid, if not, move the dist out to a |
| 1021 |
// sync |
| 1022 |
// FIXME: move the dist out here to let the sbid set on the instruction could |
| 1023 |
// have better readability, but a potential issue is that A@1 is required to |
| 1024 |
// be set on the instruction having architecture read/write. This case A@1 |
| 1025 |
// will be moved out from the instruction |
| 1026 |
adjustSWSB(block, instIt, new_swsb, false); |
| 1027 |
inst.setSWSB(new_swsb); |
| 1028 |
} |
| 1029 |
|
| 1030 |
std::vector<Instruction*> |
| 1031 |
SWSBAnalyzer::adjustSWSB(Block &block, const InstListIterator instIt, |
| 1032 |
SWSB &swsb, bool preferMoveOutSBID) { |
| 1033 |
std::vector<Instruction*> syncInsts; |
| 1034 |
const Instruction* inst = *instIt; |
| 1035 |
if (swsb.verify(m_swsbMode, inst->getSWSBInstType(m_swsbMode))) |
| 1036 |
return syncInsts; |
| 1037 |
|
| 1038 |
auto movDistToSync = [&]() { |
| 1039 |
assert(swsb.hasDist()); |
| 1040 |
SWSB tmp_swsb(swsb.distType, SWSB::TokenType::NOTOKEN, swsb.minDist, 0); |
| 1041 |
Instruction *sync_inst = m_kernel.createSyncNopInstruction(tmp_swsb); |
| 1042 |
syncInsts.push_back(sync_inst); |
| 1043 |
block.insertInstBefore(instIt, sync_inst); |
| 1044 |
swsb.distType = SWSB::DistType::NO_DIST; |
| 1045 |
swsb.minDist = 0; |
| 1046 |
}; |
| 1047 |
auto movTokenToSync = [&]() { |
| 1048 |
assert(swsb.hasToken()); |
| 1049 |
SWSB tmp_swsb(SWSB::DistType::NO_DIST, swsb.tokenType , 0, swsb.sbid); |
| 1050 |
Instruction *sync_inst = m_kernel.createSyncNopInstruction(tmp_swsb); |
| 1051 |
syncInsts.push_back(sync_inst); |
| 1052 |
block.insertInstBefore(instIt, sync_inst); |
| 1053 |
swsb.tokenType = SWSB::TokenType::NOTOKEN; |
| 1054 |
swsb.sbid = 0; |
| 1055 |
}; |
| 1056 |
|
| 1057 |
// Send can only have SBID.set and its combination on the instruction |
| 1058 |
if (inst->getOpSpec().isAnySendFormat()) { |
| 1059 |
if (swsb.tokenType == SWSB::TokenType::INC) { |
| 1060 |
// swsb has SBID.inc. It cannot be combined with other swsb. |
| 1061 |
if (swsb.hasDist()) |
| 1062 |
movDistToSync(); |
| 1063 |
assert(swsb.verify(m_swsbMode, inst->getSWSBInstType(m_swsbMode))); |
| 1064 |
return syncInsts; |
| 1065 |
} |
| 1066 |
if (swsb.tokenType == SWSB::TokenType::SET) { |
| 1067 |
// swsb has SBID.set but the combination with distance is invalid. |
| 1068 |
// Move distance to sync. |
| 1069 |
movDistToSync(); |
| 1070 |
assert(swsb.verify(m_swsbMode, inst->getSWSBInstType(m_swsbMode))); |
| 1071 |
} else { |
| 1072 |
// swsb has no SBID.set at the moment. Send must have SBID.set |
| 1073 |
// later (SWSBAnalyzer::assignSBID) so allow distance swsb and once |
| 1074 |
// SBID.set is set, the distance may be able to combined with it. |
| 1075 |
// Move only Token out of the instruction. |
| 1076 |
if (swsb.hasToken()) |
| 1077 |
movTokenToSync(); |
| 1078 |
} |
| 1079 |
return syncInsts; |
| 1080 |
} |
| 1081 |
|
| 1082 |
// For other instructions, solely SBID or DIST are valid SWSB. |
| 1083 |
// Move one to sync according to the preference. |
| 1084 |
if (preferMoveOutSBID) |
| 1085 |
movTokenToSync(); |
| 1086 |
else |
| 1087 |
movDistToSync(); |
| 1088 |
assert(swsb.verify(m_swsbMode, inst->getSWSBInstType(m_swsbMode))); |
| 1089 |
return syncInsts; |
| 1090 |
} |
| 1091 |
|
| 1092 |
static bool isSyncNop(const Instruction &i) { |
| 1093 |
return i.is(Op::SYNC) && i.getSyncFc() == SyncFC::NOP; |
| 1094 |
}; |
| 1095 |
|
| 1096 |
void SWSBAnalyzer::postProcessReadModifiedWriteOnByteDst() { |
| 1097 |
// move all swsb set within the Atomic block out for the "instruction write |
| 1098 |
// combined" cases Atomic are provided in the input so assume they are correct |
| 1099 |
// and have no internal dependency within the macro. Move all swsb to the |
| 1100 |
// first instruction in the Atomic block and also add the distance swsb to the |
| 1101 |
// intruction following the Atomic block in case of it has dependency to the |
| 1102 |
// block but was resolved within the instruction in the block E.g. |
| 1103 |
// (W) mov (32|M0) r13.0<2>:ub r50.0<1;1,0>:uw {Atomic, I@1} |
| 1104 |
// (W) mov (32|M0) r13.1<2>:ub r52.0<1;1,0>:uw {Atomic} |
| 1105 |
// (W) mov (32|M0) r13.2<2>:ub r54.0<1;1,0>:uw {Atomic} |
| 1106 |
// (W) mov (32|M0) r13.3<2>:ub r56.0<1;1,0>:uw |
| 1107 |
// add (1) r13.0<1>:df r100.0<0;1,0>:df {I@1} |
| 1108 |
for (Block *bb : m_kernel.getBlockList()) { |
| 1109 |
InstList &instList = bb->getInstList(); |
| 1110 |
for (auto inst_it = instList.begin(); inst_it != instList.end(); |
| 1111 |
++inst_it) { |
| 1112 |
auto isWriteCombinedCandidate = [&](Instruction &inst) { |
| 1113 |
return (inst.is(Op::MOV) || inst.is(Op::SRND)) && |
| 1114 |
inst.getDestination().getKind() == Operand::Kind::DIRECT && |
| 1115 |
inst.getDestination().getDirRegName() == RegName::GRF_R && |
| 1116 |
TypeSizeInBitsWithDefault(inst.getDestination().getType(), |
| 1117 |
32) == 8; |
| 1118 |
}; |
| 1119 |
// add distance swsb in "from" into "to" |
| 1120 |
auto updateDistanceSWSB = [](const SWSB &from, SWSB &to) { |
| 1121 |
if (!from.hasDist()) |
| 1122 |
return; |
| 1123 |
|
| 1124 |
if (!to.hasDist()) { |
| 1125 |
to.distType = from.distType; |
| 1126 |
to.minDist = from.minDist; |
| 1127 |
} else { |
| 1128 |
to.distType = (to.distType == from.distType) |
| 1129 |
? to.distType |
| 1130 |
: SWSB::DistType::REG_DIST_ALL; |
| 1131 |
to.minDist = std::min(to.minDist, from.minDist); |
| 1132 |
} |
| 1133 |
}; |
| 1134 |
|
| 1135 |
if ((*inst_it)->hasInstOpt(InstOpt::ATOMIC) && |
| 1136 |
isWriteCombinedCandidate(**inst_it)) { |
| 1137 |
// found the marcro start |
| 1138 |
InstListIterator firstit = inst_it; |
| 1139 |
SWSB allDistSWSB = SWSB(); |
| 1140 |
++inst_it; |
| 1141 |
InstList sync_insts; |
| 1142 |
// iterate to the end of the macro and move all swsb to the first |
| 1143 |
for (; inst_it != instList.end(); ++inst_it) { |
| 1144 |
Instruction &cur_inst = **inst_it; |
| 1145 |
// found sync, prepare to move to before the firstinst |
| 1146 |
if (cur_inst.is(Op::SYNC)) { |
| 1147 |
sync_insts.push_back( |
| 1148 |
m_kernel.createSyncNopInstruction(cur_inst.getSWSB())); |
| 1149 |
// remove swsb in current sync so that this sync will be removed |
| 1150 |
// in the following pass |
| 1151 |
cur_inst.setSWSB(SWSB()); |
| 1152 |
continue; |
| 1153 |
} |
| 1154 |
// All instructions within the write-combined atomic block must be |
| 1155 |
// write-combined candidate |
| 1156 |
if (isWriteCombinedCandidate(cur_inst)) { |
| 1157 |
// move the swsb within the atomic block to the firstinst and keep |
| 1158 |
// track of distance swsb |
| 1159 |
if (cur_inst.getSWSB().hasSWSB()) { |
| 1160 |
updateDistanceSWSB(cur_inst.getSWSB(), allDistSWSB); |
| 1161 |
addSWSBToInst(firstit, cur_inst.getSWSB(), *bb); |
| 1162 |
cur_inst.setSWSB(SWSB()); |
| 1163 |
} |
| 1164 |
// found the last instruction of the Atomic block |
| 1165 |
if (!cur_inst.hasInstOpt(InstOpt::ATOMIC)) |
| 1166 |
break; |
| 1167 |
} else { |
| 1168 |
m_errorHandler.reportError( |
| 1169 |
cur_inst.getPC(), |
| 1170 |
"Instruction found in the write-combined atomic block is not " |
| 1171 |
"a write-combined candidate"); |
| 1172 |
break; |
| 1173 |
} |
| 1174 |
} |
| 1175 |
// insert sync to before firstinst |
| 1176 |
if (!sync_insts.empty()) |
| 1177 |
instList.insert(firstit, sync_insts.begin(), sync_insts.end()); |
| 1178 |
|
| 1179 |
if (inst_it == instList.end()) { |
| 1180 |
m_errorHandler.reportError( |
| 1181 |
(*firstit)->getPC(), |
| 1182 |
"The last instruction in this write-combined atomic block has " |
| 1183 |
"{Atomic} set"); |
| 1184 |
break; |
| 1185 |
} |
| 1186 |
|
| 1187 |
// insert distance dependency to the instruction following the block |
| 1188 |
InstListIterator next = inst_it; |
| 1189 |
next++; |
| 1190 |
// if the last instruction in the atomic block is the last instruction |
| 1191 |
// in the BB, insert a sync.nop to carry the required SWSB info |
| 1192 |
// For example: |
| 1193 |
// BB0: |
| 1194 |
// (W) mov (32|M0) r13.0<2>:ub r50.0<1;1,0>:uw {Atomic} |
| 1195 |
// (W) mov (32|M0) r13.1<2>:ub r52.0<1;1,0>:uw {Atomic} |
| 1196 |
// (W) mov (32|M0) r13.2<2>:ub r54.0<1;1,0>:uw {Atomic} |
| 1197 |
// (W) mov (32|M0) r13.3<2>:ub r56.0<1;1,0>:uw |
| 1198 |
// (W) sync.nop {I@1} // insert nop |
| 1199 |
// BB1: |
| 1200 |
// add (1) r13.0<1>:df r100.0<0;1,0>:df |
| 1201 |
if (next == instList.end()) { |
| 1202 |
instList.push_back(m_kernel.createSyncNopInstruction(allDistSWSB)); |
| 1203 |
break; |
| 1204 |
} |
| 1205 |
addSWSBToInst(next, allDistSWSB, *bb); |
| 1206 |
} |
| 1207 |
} |
| 1208 |
} |
| 1209 |
} |
| 1210 |
|
| 1211 |
void SWSBAnalyzer::postProcessSBIDCounterInst() { |
| 1212 |
for (Block *bb : m_kernel.getBlockList()) { |
| 1213 |
InstList &instList = bb->getInstList(); |
| 1214 |
for (auto inst_it = instList.begin(); inst_it != instList.end(); |
| 1215 |
++inst_it) { |
| 1216 |
Instruction *inst = *inst_it; |
| 1217 |
if(!inst->getOpSpec().isAnySendFormat()) |
| 1218 |
continue; |
| 1219 |
SWSB swsb = inst->getSWSB(); |
| 1220 |
adjustSWSB(*bb, inst_it, swsb, false); |
| 1221 |
inst->setSWSB(swsb); |
| 1222 |
} |
| 1223 |
} |
| 1224 |
} |
| 1225 |
|
| 1226 |
void SWSBAnalyzer::postProcessRemoveRedundantSync() { |
| 1227 |
// Case 1: |
| 1228 |
// sync.nop carry the sbid the same as the sbid set on the following |
| 1229 |
// instruction can be removed since it'll automatically be sync-ed when sbid |
| 1230 |
// is reused. For example: |
| 1231 |
// sync.nop null {$0.dst} // can be removed |
| 1232 |
// math.exp(8|M0) r12.0<1>:f r10.0<8;8,1>:f {$0} |
| 1233 |
// Case 2: |
| 1234 |
// This case may occur after sbid counter insertion. We only check the |
| 1235 |
// redundant token (but no dist) since that's the only possible cases. |
| 1236 |
// Continuous sync.nop with the same swsb. One of it can be removed: |
| 1237 |
// sync.nop null {$0.src} |
| 1238 |
// sync.nop null {$0.src} |
| 1239 |
for (Block *bb : m_kernel.getBlockList()) { |
| 1240 |
InstList &instList = bb->getInstList(); |
| 1241 |
if (instList.empty()) |
| 1242 |
continue; |
| 1243 |
auto inst_it = instList.begin(); |
| 1244 |
// skip the first instruction, which must not be sync |
| 1245 |
++inst_it; |
| 1246 |
for (; inst_it != instList.end(); ++inst_it) { |
| 1247 |
Instruction *inst = *inst_it; |
| 1248 |
if (isSyncNop(*inst)) |
| 1249 |
continue; |
| 1250 |
|
| 1251 |
SWSB cur_swsb = inst->getSWSB(); |
| 1252 |
// iterate through the previous sync if any |
| 1253 |
auto sync_it = inst_it; |
| 1254 |
--sync_it; |
| 1255 |
// keep track of the seen SWSB's TokenType and sbid number |
| 1256 |
std::set<std::pair<SWSB::TokenType, uint32_t>> seenSBID; |
| 1257 |
while (sync_it != instList.begin()) { |
| 1258 |
Instruction *sync_inst = *sync_it; |
| 1259 |
if (!isSyncNop(*sync_inst)) |
| 1260 |
break; |
| 1261 |
SWSB sync_swsb = sync_inst->getSWSB(); |
| 1262 |
// Case 1 |
| 1263 |
// if the sync has sbid set, it could be the reserved sbid for shoot |
| 1264 |
// down instructions, we should keep it. Otherwise if the its sbid |
| 1265 |
// is the same as current's instrcutions' sbid.set, we can remove it. |
| 1266 |
if(cur_swsb.hasToken() && cur_swsb.tokenType == SWSB::TokenType::SET) { |
| 1267 |
if (sync_swsb.hasToken() && |
| 1268 |
sync_swsb.tokenType != SWSB::TokenType::SET && |
| 1269 |
sync_swsb.sbid == cur_swsb.sbid) { |
| 1270 |
// clean the swsb so that we can remove this instruction later |
| 1271 |
sync_inst->setSWSB(SWSB()); |
| 1272 |
} |
| 1273 |
} |
| 1274 |
// Case 2 |
| 1275 |
// keep track of visited SWSB on sync and remove the duplicated one |
| 1276 |
sync_swsb = sync_inst->getSWSB(); |
| 1277 |
if (sync_swsb.hasToken()) { |
| 1278 |
auto sbidPair = std::make_pair(sync_swsb.tokenType, sync_swsb.sbid); |
| 1279 |
if (seenSBID.find(sbidPair) != seenSBID.end()) { |
| 1280 |
sync_inst->setSWSB(SWSB()); |
| 1281 |
} else { |
| 1282 |
seenSBID.insert(sbidPair); |
| 1283 |
} |
| 1284 |
} |
| 1285 |
--sync_it; |
| 1286 |
} |
| 1287 |
} |
| 1288 |
// remove the redundant sync.nop (sync.nop with no swsb) |
| 1289 |
instList.remove_if([](const Instruction *inst) { |
| 1290 |
return isSyncNop(*inst) && !inst->getSWSB().hasSWSB(); |
| 1291 |
}); |
| 1292 |
} |
| 1293 |
} |
| 1294 |
|
| 1295 |
void SWSBAnalyzer::postProcess() { |
| 1296 |
// revisit all instructions to handle write-combined Atomic block |
| 1297 |
if (m_kernel.getModel().hasReadModifiedWriteOnByteDst()) { |
| 1298 |
postProcessReadModifiedWriteOnByteDst(); |
| 1299 |
} |
| 1300 |
|
| 1301 |
// we may update sbid token to .inc after a send's swsb is adjusted. |
| 1302 |
// adjustSWSB again in case the swsb doesn't fit on the inst |
| 1303 |
if (enableSBIDCounterSetting()) { |
| 1304 |
postProcessSBIDCounterInst(); |
| 1305 |
} |
| 1306 |
|
| 1307 |
// revisit all instructions to remove redundant sync.nop |
| 1308 |
postProcessRemoveRedundantSync(); |
| 1309 |
} |
| 1310 |
|
| 1311 |
SBID &SWSBAnalyzer::assignSBID(DepSet *input, DepSet *output, Instruction &inst, |
| 1312 |
SWSB &swsb, InstList::iterator insertPoint, |
| 1313 |
Block *curBB, bool needSyncForShootDown) { |
| 1314 |
bool foundFree = false; |
| 1315 |
SBID *sbidFree = nullptr; |
| 1316 |
for (uint32_t i = 0; i < m_SBIDCount; ++i) { |
| 1317 |
if (m_freeSBIDList[i].isFree) { |
| 1318 |
foundFree = true; |
| 1319 |
sbidFree = &m_freeSBIDList[i]; |
| 1320 |
m_freeSBIDList[i].sbid = i; |
| 1321 |
break; |
| 1322 |
} |
| 1323 |
} |
| 1324 |
// no free SBID. |
| 1325 |
if (!foundFree) { |
| 1326 |
unsigned int index = (m_SBIDRRCounter++) % m_SBIDCount; |
| 1327 |
|
| 1328 |
// While swsb id being reuse, the dependency will automatically resolved by |
| 1329 |
// hardware, so cleanup the dependency bucket for instruction that |
| 1330 |
// previously used this id |
| 1331 |
assert(m_IdToDepSetMap.find(index) != m_IdToDepSetMap.end()); |
| 1332 |
assert(m_IdToDepSetMap[index].first->getDepClass() == |
| 1333 |
DEP_CLASS::OUT_OF_ORDER); |
| 1334 |
clearDepBuckets(*m_IdToDepSetMap[index].first); |
| 1335 |
clearDepBuckets(*m_IdToDepSetMap[index].second); |
| 1336 |
|
| 1337 |
m_freeSBIDList[index].reset(); |
| 1338 |
sbidFree = &m_freeSBIDList[index]; |
| 1339 |
sbidFree->sbid = index; |
| 1340 |
} |
| 1341 |
sbidFree->isFree = false; |
| 1342 |
input->setSBID(*sbidFree); |
| 1343 |
output->setSBID(*sbidFree); |
| 1344 |
if (m_IdToDepSetMap.find(sbidFree->sbid) != m_IdToDepSetMap.end()) |
| 1345 |
m_IdToDepSetMap.erase(sbidFree->sbid); |
| 1346 |
m_IdToDepSetMap.emplace(sbidFree->sbid, std::make_pair(input, output)); |
| 1347 |
|
| 1348 |
if (needSyncForShootDown) { |
| 1349 |
// check if the instruction has A@1 on it. If it does, create another sync |
| 1350 |
// to move A@1 to before the sync.nop in case of violating the rule "When |
| 1351 |
// certain Arch Registers (sr, cr, ce) are used, the very next instruction |
| 1352 |
// requires dependency to be set on all pipes {A@1}". |
| 1353 |
// At this point we do not know if the A@1 is inserted for this rule or |
| 1354 |
// other dependency. Conservatively move A@1 above to the very first sync |
| 1355 |
// after the previous non-sync instruction |
| 1356 |
if (swsb.hasDist() && |
| 1357 |
(swsb.distType == SWSB::DistType::REG_DIST_ALL || |
| 1358 |
swsb.distType == SWSB::DistType::REG_DIST) && |
| 1359 |
swsb.minDist == 1) { |
| 1360 |
SWSB ss(swsb.distType, SWSB::TokenType::NOTOKEN, 1, 0); |
| 1361 |
curBB->insertInstBefore(insertPoint, |
| 1362 |
m_kernel.createSyncNopInstruction(ss)); |
| 1363 |
swsb.distType = SWSB::DistType::NO_DIST; |
| 1364 |
swsb.minDist = 0; |
| 1365 |
} |
| 1366 |
// add a sync to preserve the token for possibly shooting down instruction |
| 1367 |
SWSB tDep(SWSB::DistType::NO_DIST, SWSB::TokenType::SET, 0, sbidFree->sbid); |
| 1368 |
auto syncInst = m_kernel.createSyncNopInstruction(tDep); |
| 1369 |
curBB->insertInstBefore(insertPoint, syncInst); |
| 1370 |
input->addSrcDepInsts(syncInst); |
| 1371 |
output->addSrcDepInsts(syncInst); |
| 1372 |
} |
| 1373 |
|
| 1374 |
// adding the set for this SBID |
| 1375 |
// if the swsb has the token set already, move it out to a sync |
| 1376 |
if (swsb.tokenType != SWSB::TokenType::NOTOKEN) { |
| 1377 |
SWSB tDep(SWSB::DistType::NO_DIST, swsb.tokenType, 0, swsb.sbid); |
| 1378 |
Instruction *tInst = m_kernel.createSyncNopInstruction(tDep); |
| 1379 |
curBB->insertInstBefore(insertPoint, tInst); |
| 1380 |
// the depset of send corresponding to the sbid must be updated with this |
| 1381 |
// sync |
| 1382 |
m_IdToDepSetMap[swsb.sbid].first->addSrcDepInsts(tInst); |
| 1383 |
m_IdToDepSetMap[swsb.sbid].first->getCompanion()->addSrcDepInsts(tInst); |
| 1384 |
m_IdToDepSetMap[swsb.sbid].second->addSrcDepInsts(tInst); |
| 1385 |
} |
| 1386 |
// set the sbid |
| 1387 |
swsb.tokenType = SWSB::TokenType::SET; |
| 1388 |
swsb.sbid = sbidFree->sbid; |
| 1389 |
|
| 1390 |
// verify if the token and dist combination is valid, if not, move the dist |
| 1391 |
// out to a sync move the dist out here to let the sbid set on the instruction |
| 1392 |
// so that other instructions having dependency to this SBID is sync to the |
| 1393 |
// correct instruction (but not a sync.nop) |
| 1394 |
// FIXME: a potential issue is that A@1 is required to be set on the |
| 1395 |
// instruction having architecture read/write. This case A@1 will be moved out |
| 1396 |
// from the instruction |
| 1397 |
adjustSWSB(*curBB, insertPoint, swsb, false); |
| 1398 |
|
| 1399 |
assert(sbidFree != nullptr); |
| 1400 |
return *sbidFree; |
| 1401 |
} |
| 1402 |
|
| 1403 |
void SWSBAnalyzer::run() { |
| 1404 |
m_initPoint = true; |
| 1405 |
m_distanceTracker.clear(); |
| 1406 |
|
| 1407 |
for (uint32_t i = 0; i < MAX_GRF_BUCKETS; ++i) { |
| 1408 |
m_buckets[i].clearDependency(); |
| 1409 |
} |
| 1410 |
|
| 1411 |
// init in order pipe id counters |
| 1412 |
m_InstIdCounter.init(); |
| 1413 |
math_wa_info.reset(); |
| 1414 |
|
| 1415 |
Instruction *inst = nullptr; |
| 1416 |
Block *lastBB = nullptr; |
| 1417 |
for (auto bb : m_kernel.getBlockList()) { |
| 1418 |
bool blockEndsWithNonBranchInst = false; |
| 1419 |
// resetting things for each bb |
| 1420 |
lastBB = bb; |
| 1421 |
InstList &instList = bb->getInstList(); // Don't use auto for over loaded |
| 1422 |
// return which has const... |
| 1423 |
const auto instListEnd = instList.end(); |
| 1424 |
for (auto instIter = instList.begin(); instIter != instListEnd; |
| 1425 |
++instIter) { |
| 1426 |
m_InstIdCounter.global++; |
| 1427 |
inst = *instIter; |
| 1428 |
// skip illeagl instruction. It cannot have dependency to others, nor does |
| 1429 |
// it affect the in-pipe instruction count. Skipping it can avoid we |
| 1430 |
// accidentally set SWSB on it (e.g. when forcing B2B dependency). Illeagl |
| 1431 |
// instructions cannot carry SWSB info. |
| 1432 |
if (inst->getOp() == Op::ILLEGAL || inst->getOp() == Op::THRYLD) |
| 1433 |
continue; |
| 1434 |
DepSet *input = nullptr; |
| 1435 |
DepSet *output = nullptr; |
| 1436 |
size_t dpas_cnt_in_macro = 0; |
| 1437 |
|
| 1438 |
if (math_wa_info.math_inst != nullptr) |
| 1439 |
math_wa_info.previous_is_math = true; |
| 1440 |
if (inst->getOpSpec().is(Op::MATH)) { |
| 1441 |
math_wa_info.math_inst = inst; |
| 1442 |
|
| 1443 |
// if the math following a math, we only care about the last math |
| 1444 |
math_wa_info.previous_is_math = false; |
| 1445 |
} |
| 1446 |
|
| 1447 |
// recored the first instruction of a dpas macro, in case that inserting |
| 1448 |
// instructions (e.g. sync) before the macro, those instructions have to |
| 1449 |
// be insert before first_inst_in_dpas_macro |
| 1450 |
InstListIterator first_inst_in_dpas_macro = instList.end(); |
| 1451 |
if (inst->getOpSpec().isDpasFormat()) { |
| 1452 |
std::pair<DepSet *, DepSet *> dep_set_pair = |
| 1453 |
m_DB->createDPASSrcDstDepSet(instList, instIter, m_InstIdCounter, |
| 1454 |
dpas_cnt_in_macro, m_swsbMode); |
| 1455 |
input = dep_set_pair.first; |
| 1456 |
output = dep_set_pair.second; |
| 1457 |
|
| 1458 |
first_inst_in_dpas_macro = instIter; |
| 1459 |
// bypass dpas insturctions in the macro, the last dpas represents the |
| 1460 |
// macro |
| 1461 |
for (size_t i = 0; i < dpas_cnt_in_macro - 1; ++i) |
| 1462 |
++instIter; |
| 1463 |
inst = *instIter; |
| 1464 |
} else { |
| 1465 |
input = m_DB->createSrcDepSet(*inst, m_InstIdCounter, m_swsbMode); |
| 1466 |
if (m_DB->needDstReadSuppressionWA(*inst)) |
| 1467 |
output = m_DB->createDstDepSetFullGrf(*inst, m_InstIdCounter, |
| 1468 |
m_swsbMode, true); |
| 1469 |
else |
| 1470 |
output = m_DB->createDstDepSet(*inst, m_InstIdCounter, m_swsbMode); |
| 1471 |
} |
| 1472 |
input->setCompanion(output); |
| 1473 |
output->setCompanion(input); |
| 1474 |
|
| 1475 |
// XeHPC+ features |
| 1476 |
if (m_kernel.getModel().hasReadModifiedWriteOnByteDst()) |
| 1477 |
addRMWDependencyIfReqruied(*input, *output); |
| 1478 |
|
| 1479 |
SWSB swsb; |
| 1480 |
|
| 1481 |
// Either source or destination are indirect, or there are SR access, |
| 1482 |
// We don't know what registers are being accessed |
| 1483 |
// Need to flush all the sbids and set distance to 1 |
| 1484 |
if (input->hasIndirect() || output->hasIndirect() || input->hasSR() || |
| 1485 |
output->hasSR()) { |
| 1486 |
// clear out-of-order dependency, insert sync.allrd and sync.allwr |
| 1487 |
// if there are un-resolved sbid dependecny |
| 1488 |
// if this instruction itself is an out-of-order instruction, insert |
| 1489 |
// sync.all anyway. |
| 1490 |
InstListIterator insert_point = instIter; |
| 1491 |
if (first_inst_in_dpas_macro != instList.end()) |
| 1492 |
insert_point = first_inst_in_dpas_macro; |
| 1493 |
bool forceSyncAll = clearSBIDDependence(insert_point, inst, bb); |
| 1494 |
if (!forceSyncAll && input->getDepClass() == DEP_CLASS::OUT_OF_ORDER) |
| 1495 |
insertSyncAllRdWr(insert_point, bb); |
| 1496 |
|
| 1497 |
// clear in-order dependency |
| 1498 |
clearBuckets(input, output); |
| 1499 |
|
| 1500 |
// will add direct accesses to buckets |
| 1501 |
// adding dependencies to buckets |
| 1502 |
for (auto bucketID : input->getBuckets()) { |
| 1503 |
m_buckets[bucketID].addDepSet(input); |
| 1504 |
} |
| 1505 |
for (auto bucketID : output->getBuckets()) { |
| 1506 |
m_buckets[bucketID].addDepSet(output); |
| 1507 |
} |
| 1508 |
|
| 1509 |
// set to check all dist pipes |
| 1510 |
if (getNumOfDistPipe() == 1) |
| 1511 |
swsb.distType = SWSB::DistType::REG_DIST; |
| 1512 |
else |
| 1513 |
swsb.distType = SWSB::DistType::REG_DIST_ALL; |
| 1514 |
swsb.minDist = 1; |
| 1515 |
|
| 1516 |
// input and output must have the same dep class and in the same pipe |
| 1517 |
// so check the input only to add the instCounter |
| 1518 |
// FIXME: is it possilbe that a instruction has output and no input? |
| 1519 |
if (input->getDepClass() == DEP_CLASS::IN_ORDER) |
| 1520 |
advanceInorderInstCounter(input->getDepPipe()); |
| 1521 |
|
| 1522 |
// if this is an out-of-order instruction, we still need to assign an |
| 1523 |
// sbid for it |
| 1524 |
if (output->getDepClass() == DEP_CLASS::OUT_OF_ORDER) |
| 1525 |
assignSBID(input, output, *inst, swsb, insert_point, bb, |
| 1526 |
false); |
| 1527 |
|
| 1528 |
inst->setSWSB(swsb); |
| 1529 |
// clean up math_wa_info, this instruction force to sync all, no need to |
| 1530 |
// consider math wa |
| 1531 |
if (math_wa_info.previous_is_math) { |
| 1532 |
math_wa_info.reset(); |
| 1533 |
} |
| 1534 |
// early out, no need to calculateDependence that all dependencies are |
| 1535 |
// resolved. |
| 1536 |
continue; |
| 1537 |
} // end indirect access handling |
| 1538 |
|
| 1539 |
if (math_wa_info.previous_is_math) { |
| 1540 |
// math WA affect the instruction right after the math, and with |
| 1541 |
// different predication Add the WA math dst region to Buckets |
| 1542 |
if (math_wa_info.math_inst->getPredication().function != |
| 1543 |
inst->getPredication().function) { |
| 1544 |
math_wa_info.dep_set = m_DB->createDstDepSetFullGrf( |
| 1545 |
*math_wa_info.math_inst, math_wa_info.math_id, m_swsbMode, false); |
| 1546 |
math_wa_info.dep_set->setSBID(math_wa_info.math_sbid); |
| 1547 |
for (auto bucketID : math_wa_info.dep_set->getBuckets()) { |
| 1548 |
IGA_ASSERT(bucketID < m_DB->getTOTAL_BUCKETS(), |
| 1549 |
"buckedID out of range"); |
| 1550 |
m_buckets[bucketID].addDepSet(math_wa_info.dep_set); |
| 1551 |
} |
| 1552 |
} |
| 1553 |
} |
| 1554 |
|
| 1555 |
// keep track of the depended SBID current inst depends on, |
| 1556 |
// and the list of DepSet where this SBID comes from |
| 1557 |
activeSBIDsTy activeSBIDs; |
| 1558 |
bool needSyncForShootDown = false; |
| 1559 |
// Calculates dependence between this instruction dependencies and |
| 1560 |
// previous ones. |
| 1561 |
calculateDependence(*input, swsb, *inst, activeSBIDs, |
| 1562 |
needSyncForShootDown); |
| 1563 |
calculateDependence(*output, swsb, *inst, activeSBIDs, |
| 1564 |
needSyncForShootDown); |
| 1565 |
|
| 1566 |
// clean up math_wa_info |
| 1567 |
if (math_wa_info.previous_is_math) { |
| 1568 |
if (math_wa_info.dep_set != nullptr) |
| 1569 |
clearDepBuckets(*math_wa_info.dep_set); |
| 1570 |
math_wa_info.reset(); |
| 1571 |
} |
| 1572 |
|
| 1573 |
if (first_inst_in_dpas_macro != instList.end()) |
| 1574 |
processactiveSBIDs(swsb, input, bb, |
| 1575 |
first_inst_in_dpas_macro, activeSBIDs); |
| 1576 |
else |
| 1577 |
processactiveSBIDs(swsb, input, bb, instIter, activeSBIDs); |
| 1578 |
|
| 1579 |
// Need to set SBID |
| 1580 |
if (output->getDepClass() == DEP_CLASS::OUT_OF_ORDER) { |
| 1581 |
InstList::iterator insertPoint = instIter; |
| 1582 |
if (first_inst_in_dpas_macro != instList.end()) |
| 1583 |
insertPoint = first_inst_in_dpas_macro; |
| 1584 |
SBID &assigned_id = assignSBID(input, output, *inst, swsb, |
| 1585 |
insertPoint, bb, needSyncForShootDown); |
| 1586 |
|
| 1587 |
// record the sbid if it's math, for use of math wa |
| 1588 |
if (inst->getOpSpec().is(Op::MATH)) { |
| 1589 |
math_wa_info.math_sbid = assigned_id; |
| 1590 |
} |
| 1591 |
} |
| 1592 |
|
| 1593 |
clearBuckets(input, output); |
| 1594 |
|
| 1595 |
/* |
| 1596 |
* Handling the case where everything is in one bb, and send with EOT is |
| 1597 |
* in the middle of instruction stream call (16|M0) r8.0:ud 32 |
| 1598 |
* sendc.rc (16|M0) null r118 null 0x0 0x140B1000 |
| 1599 |
* {EOT} // wr:10h, rd:0, Render Target Write msc:16, to #0 |
| 1600 |
* ... |
| 1601 |
* ret (16|M0) r8.0 |
| 1602 |
*/ |
| 1603 |
if (!(inst->getOpSpec().isAnySendFormat() && |
| 1604 |
inst->hasInstOpt(InstOpt::EOT))) { |
| 1605 |
// adding dependencies to buckets |
| 1606 |
for (auto bucketID : input->getBuckets()) { |
| 1607 |
// We want to check dependncy of regular instructions against |
| 1608 |
// WRITE_ALWAYS_INTERFERE without adding them themselves |
| 1609 |
if (bucketID == m_DB->getBucketStart(RegName::ARF_CR) && |
| 1610 |
input->getDepType() != DEP_TYPE::WRITE_ALWAYS_INTERFERE && |
| 1611 |
input->getDepType() != DEP_TYPE::READ_ALWAYS_INTERFERE) { |
| 1612 |
continue; |
| 1613 |
} |
| 1614 |
m_buckets[bucketID].addDepSet(input); |
| 1615 |
} |
| 1616 |
for (auto bucketID : output->getBuckets()) { |
| 1617 |
IGA_ASSERT(bucketID < m_DB->getTOTAL_BUCKETS(), |
| 1618 |
"buckedID out of range"); |
| 1619 |
// We want to check dependncy of regular instructions against |
| 1620 |
// WRITE_ALWAYS_INTERFERE without adding them themselves |
| 1621 |
if (bucketID == m_DB->getBucketStart(RegName::ARF_CR) && |
| 1622 |
output->getDepType() != DEP_TYPE::WRITE_ALWAYS_INTERFERE && |
| 1623 |
output->getDepType() != DEP_TYPE::READ_ALWAYS_INTERFERE) { |
| 1624 |
continue; |
| 1625 |
} |
| 1626 |
m_buckets[bucketID].addDepSet(output); |
| 1627 |
} |
| 1628 |
} |
| 1629 |
|
| 1630 |
if (input->getDepClass() == DEP_CLASS::IN_ORDER) { |
| 1631 |
advanceInorderInstCounter(input->getDepPipe()); |
| 1632 |
} |
| 1633 |
|
| 1634 |
// for dpas block, set the distance at the first inst in the block, and |
| 1635 |
// set the swsb id at the last inst in the block. |
| 1636 |
if ((first_inst_in_dpas_macro != instList.end()) && |
| 1637 |
(*first_inst_in_dpas_macro != inst)) { |
| 1638 |
(*first_inst_in_dpas_macro)->setSWSB(SWSB(swsb.distType, |
| 1639 |
SWSB::TokenType::NOTOKEN, swsb.minDist, 0)); |
| 1640 |
inst->setSWSB(SWSB(SWSB::DistType::NO_DIST, swsb.tokenType, 0, |
| 1641 |
swsb.sbid)); |
| 1642 |
} else { |
| 1643 |
// if the input SWSB is a special token, preserve it and insert a sync |
| 1644 |
// before to carry the dependency info Note that dpas must not have |
| 1645 |
// special token so we only do this check for non-dpas here |
| 1646 |
if (inst->getSWSB().hasSpecialToken()) { |
| 1647 |
if (swsb.hasSWSB()) { |
| 1648 |
Instruction *syncInst = |
| 1649 |
m_kernel.createSyncNopInstruction(swsb); |
| 1650 |
bb->insertInstBefore(instIter, syncInst); |
| 1651 |
} |
| 1652 |
} else { |
| 1653 |
inst->setSWSB(swsb); |
| 1654 |
} |
| 1655 |
} |
| 1656 |
assert(swsb.verify(m_swsbMode, inst->getSWSBInstType(m_swsbMode))); |
| 1657 |
|
| 1658 |
if (inst->isBranching()) { |
| 1659 |
// TODO: konrad : this is somewhat conservative, some |
| 1660 |
// branch instructions might not need sync (join) |
| 1661 |
blockEndsWithNonBranchInst = false; |
| 1662 |
clearSBIDDependence(instIter, inst, bb); |
| 1663 |
continue; |
| 1664 |
} else { |
| 1665 |
blockEndsWithNonBranchInst = true; |
| 1666 |
} |
| 1667 |
} // iterate on instr |
| 1668 |
// clear read |
| 1669 |
// clear write |
| 1670 |
if (blockEndsWithNonBranchInst) { |
| 1671 |
clearSBIDDependence(instList.end(), inst, bb); |
| 1672 |
} |
| 1673 |
} // iterate on basic block |
| 1674 |
|
| 1675 |
if (inst) |
| 1676 |
forceSyncLastInst(*inst, *lastBB); |
| 1677 |
|
| 1678 |
postProcess(); |
| 1679 |
return; |
| 1680 |
} |
| 1681 |
|
| 1682 |
void SWSBAnalyzer::forceSyncLastInst(const Instruction &lastInst, |
| 1683 |
Block &lastBB) { |
| 1684 |
// this code is for FC composite |
| 1685 |
// if last instruction is not EOT we will insert flush instructions |
| 1686 |
// and stall the pipeline since we do not do global analysis |
| 1687 |
if ((lastInst.getOpSpec().isAnySendFormat() && |
| 1688 |
!lastInst.getInstOpts().contains(InstOpt::EOT)) || |
| 1689 |
!lastInst.getOpSpec().isAnySendFormat()) { |
| 1690 |
SWSB swsb; |
| 1691 |
if (getNumOfDistPipe() == 1) |
| 1692 |
swsb.distType = SWSB::DistType::REG_DIST; |
| 1693 |
else |
| 1694 |
swsb.distType = SWSB::DistType::REG_DIST_ALL; |
| 1695 |
swsb.minDist = 1; |
| 1696 |
Instruction *syncInst = m_kernel.createSyncNopInstruction(swsb); |
| 1697 |
lastBB.getInstList().push_back(syncInst); |
| 1698 |
} |
| 1699 |
} |