| 1 |
/*========================== begin_copyright_notice ============================ |
| 2 |
|
| 3 |
Copyright (C) 2017-2021 Intel Corporation |
| 4 |
|
| 5 |
SPDX-License-Identifier: MIT |
| 6 |
|
| 7 |
============================= end_copyright_notice ===========================*/ |
| 8 |
|
| 9 |
#ifndef _IGA_SWSBSETTER_H_ |
| 10 |
#define _IGA_SWSBSETTER_H_ |
| 11 |
|
| 12 |
#include "../ErrorHandler.hpp" |
| 13 |
#include "Block.hpp" |
| 14 |
#include "Instruction.hpp" |
| 15 |
#include "Kernel.hpp" |
| 16 |
#include "Operand.hpp" |
| 17 |
#include "RegDeps.hpp" |
| 18 |
#include <limits> |
| 19 |
|
| 20 |
namespace iga { |
| 21 |
// Bucket represents a GRF and maps to all instructions that access it |
| 22 |
class Bucket { |
| 23 |
public: |
| 24 |
Bucket() { dependencies.reserve(5); } |
| 25 |
void clearDependency() { dependencies.clear(); } |
| 26 |
bool isEmpty() const { return dependencies.empty(); } |
| 27 |
size_t getNumDependencies() const { return dependencies.size(); } |
| 28 |
DepSet *getDepSet(uint32_t index) { return dependencies[index]; } |
| 29 |
void clearDepSet(uint32_t index) { dependencies[index] = nullptr; } |
| 30 |
// Most of the time dependecy vector will have 1 or two entries |
| 31 |
void addDepSet(DepSet *dep) { |
| 32 |
bool depSet = false; |
| 33 |
for (size_t i = 0; i < dependencies.size(); ++i) { |
| 34 |
if (!dependencies[i]) { |
| 35 |
dependencies[i] = dep; |
| 36 |
depSet = true; |
| 37 |
break; |
| 38 |
} |
| 39 |
} |
| 40 |
if (!depSet) { |
| 41 |
dependencies.push_back(dep); |
| 42 |
} |
| 43 |
} |
| 44 |
|
| 45 |
private: |
| 46 |
std::vector<DepSet *> dependencies; |
| 47 |
}; |
| 48 |
|
| 49 |
class SWSBAnalyzer { |
| 50 |
public: |
| 51 |
typedef DepSet::InstIDs InstIDs; |
| 52 |
|
| 53 |
public: |
| 54 |
// Blocks have already been created |
| 55 |
SWSBAnalyzer(Kernel &k, ErrorHandler &errHandler, |
| 56 |
SWSB_ENCODE_MODE encode_mode, int sbid_count, |
| 57 |
bool enableSBIDCounterAutoSet = false) |
| 58 |
: m_kernel(k), m_errorHandler(errHandler), m_SBIDRRCounter(0), |
| 59 |
m_initPoint(false), |
| 60 |
m_enableSBIDCounter(enableSBIDCounterAutoSet), |
| 61 |
MAX_VALID_DISTANCE(k.getModel().getSWSBMaxValidDistance()) { |
| 62 |
// Set SWSB_ENCODE_MODE |
| 63 |
if (encode_mode != SWSB_ENCODE_MODE::SWSBInvalidMode) |
| 64 |
m_swsbMode = encode_mode; |
| 65 |
else |
| 66 |
m_swsbMode = k.getModel().getSWSBEncodeMode(); |
| 67 |
|
| 68 |
if (m_swsbMode == SWSB_ENCODE_MODE::FourDistPipeReduction || |
| 69 |
m_swsbMode >= SWSB_ENCODE_MODE::FiveDistPipe) { |
| 70 |
m_LatencyLong64Pipe = 12; |
| 71 |
} |
| 72 |
|
| 73 |
m_DB = new DepSetBuilder(k.getModel()); |
| 74 |
m_buckets = new Bucket[m_DB->getTOTAL_BUCKETS()]; |
| 75 |
|
| 76 |
// set sbid count to 16 if not given |
| 77 |
if (sbid_count) |
| 78 |
m_SBIDCount = sbid_count; |
| 79 |
else |
| 80 |
m_SBIDCount = 16; |
| 81 |
m_freeSBIDList.resize(m_SBIDCount); |
| 82 |
} |
| 83 |
|
| 84 |
~SWSBAnalyzer() { |
| 85 |
delete m_DB; |
| 86 |
delete[] m_buckets; |
| 87 |
} |
| 88 |
SWSBAnalyzer(const SWSBAnalyzer&) = delete; |
| 89 |
SWSBAnalyzer& operator=(const SWSBAnalyzer&) = delete; |
| 90 |
|
| 91 |
void run(); |
| 92 |
|
| 93 |
// getNumOfDistPipe - get number of in-order distance pipes of the given mode |
| 94 |
static uint32_t getNumOfDistPipe(SWSB_ENCODE_MODE mode); |
| 95 |
|
| 96 |
private: |
| 97 |
// postProcess - last step of "run" |
| 98 |
void postProcess(); |
| 99 |
|
| 100 |
void postProcessReadModifiedWriteOnByteDst(); |
| 101 |
void postProcessRemoveRedundantSync(); |
| 102 |
void postProcessSBIDCounterInst(); |
| 103 |
|
| 104 |
private: |
| 105 |
// activeSBIDsTy - keep track of the depended SBID an inst depends on, |
| 106 |
// and the list of DepSet where this SBID comes from |
| 107 |
typedef std::vector<std::pair<SBID, std::vector<DepSet*>>> activeSBIDsTy; |
| 108 |
|
| 109 |
// activeSBIDs: input list that the sbid this dep has dependency on will be |
| 110 |
// added into. This list will later on be pass to processactiveSBIDs to set the |
| 111 |
// swsb id dependency to inst accordingly needSyncForShootDownInst: if the |
| 112 |
// sync to the sbid on the instruction is required. If the instruction is |
| 113 |
// possiblely being shoot down, we have to add a sync to the id is synced with |
| 114 |
// because we will clear the dependency |
| 115 |
void calculateDependence(DepSet &dep, SWSB &swsb, |
| 116 |
const Instruction &currInst, |
| 117 |
activeSBIDsTy &activeSBIDs, |
| 118 |
bool &needSyncForShootDownInst); |
| 119 |
void processactiveSBIDs(SWSB &distanceDependency, const DepSet *input, |
| 120 |
Block *bb, InstList::iterator iter, |
| 121 |
activeSBIDsTy &activeSBIDs); |
| 122 |
// Check and replace the associated instructions in activeSBIDs to use sbid |
| 123 |
// counter if possible. Return true on success. |
| 124 |
// DepSet->Instruction tracked by activeSBIDs is also updated to replaced |
| 125 |
// its sbid to ".inc" one, |
| 126 |
// If cannot replace to sbid counter, return false and activeSBIDs remains |
| 127 |
// unchanged. |
| 128 |
// retSBID: the chosen SBID that is make to be counter |
| 129 |
bool trySBIDCounter(activeSBIDsTy &activeSBIDs, const DepSet &curDepSet, |
| 130 |
SBID &retSBID); |
| 131 |
|
| 132 |
// helper function to pick a free SBID and set it to distanceDependency. |
| 133 |
// This function only set SBID to distanceDependency, will not assign |
| 134 |
// distanceDependency to the inst |
| 135 |
SBID &assignSBID(DepSet *input, DepSet *output, Instruction &inst, |
| 136 |
SWSB &distanceDependency, InstList::iterator insertPoint, |
| 137 |
Block *curBB, bool needSyncForShootDown); |
| 138 |
|
| 139 |
// helper fuction to set out-of-order dependecy. Called by |
| 140 |
// calculateDependence. This function create SBID with depedency to given dep, |
| 141 |
// and add it to activeSBIDs it also set needSyncForShootDownInst if required. |
| 142 |
void setSbidDependency(DepSet &dep, const Instruction &currInst, |
| 143 |
bool &needSyncForShootDownInst, |
| 144 |
activeSBIDsTy &activeSBIDs); |
| 145 |
|
| 146 |
// helper fuction to set swsb for dependency to in-order instructions. |
| 147 |
void setDistanceDependency(DepSet *dep, SWSB &swsb, bool isWAW, |
| 148 |
DEP_PIPE prevDepPipe, DEP_PIPE currDepPipe, |
| 149 |
const Instruction &currInst); |
| 150 |
|
| 151 |
// clear dependency of the given dep |
| 152 |
void clearDepBuckets(DepSet &dep); |
| 153 |
// clear all sbid, set ids to all free and insert sync to sync with all pipes |
| 154 |
// return if there are active sbid in-use before and sync.allrd/sync.allwr |
| 155 |
// instructions are inserted to sync them |
| 156 |
bool clearSBIDDependence(InstList::iterator insertPoint, |
| 157 |
Instruction *lastInst, Block *bb); |
| 158 |
// clear given input and output dependency in the buckets (for in-order pipes |
| 159 |
// only) |
| 160 |
void clearBuckets(DepSet *input, DepSet *output); |
| 161 |
|
| 162 |
// helper function to insert sync.allrd and sync.allwr before the given |
| 163 |
// inserPoint of given bb |
| 164 |
void insertSyncAllRdWr(InstList::iterator insertPoint, Block *bb); |
| 165 |
|
| 166 |
// a helper function to increase inst id counter based on the current encoding |
| 167 |
// mode |
| 168 |
void advanceInorderInstCounter(DEP_PIPE dep_pipe); |
| 169 |
|
| 170 |
// get number of dist pipe according to SWSB_ENCODE_MODE |
| 171 |
uint32_t getNumOfDistPipe(); |
| 172 |
|
| 173 |
// addRMWDependencyIfReqruied - add read dependnecy to input DepSet if the |
| 174 |
// instruction has RMW behavior, which is, has byte type dst XeHPC+ feature |
| 175 |
void addRMWDependencyIfReqruied(DepSet &input, DepSet &output); |
| 176 |
|
| 177 |
// add swsb into instruction, insert sync if the added swsb is not compatible |
| 178 |
// with the existed swsb in the inst |
| 179 |
void addSWSBToInst(InstListIterator instIt, const SWSB &swsb, Block &block); |
| 180 |
|
| 181 |
// a helper function to check if the RAW depedency to acc/flag can be |
| 182 |
// handled by HW, and hence swsb is not required. isRAW will be set to false for the |
| 183 |
// case. This function is used by calculateDependence. |
| 184 |
// The function must be called when there is RAW dependency detected. |
| 185 |
void checkAccFlagRAW(bool& isRAW, const DepSet &currDep, const DepSet &targetDep); |
| 186 |
|
| 187 |
// insert sync and A@1 for the last instruction in a BB |
| 188 |
void forceSyncLastInst(const Instruction &lastInst, Block &lastBB); |
| 189 |
|
| 190 |
// verify if the swsb is valid on the instruction instIt. If not, insert sync.nop |
| 191 |
// for swsb information until it is valid, but do not set swsb to the instruction. |
| 192 |
// preferMoveOutSBID - perfer move out sbid to sync.nop if possible. If set to |
| 193 |
// false, try to move distance out first. |
| 194 |
// Return the newly inserted sync.nop instructions if any |
| 195 |
std::vector<Instruction*> |
| 196 |
adjustSWSB(Block &block, const InstListIterator instIt, |
| 197 |
SWSB &swsb, bool preferMoveOutSBID); |
| 198 |
|
| 199 |
bool enableSBIDCounterSetting() { |
| 200 |
// enable sbid counter setting only when the encoding mode supports it |
| 201 |
if(m_enableSBIDCounter && |
| 202 |
m_swsbMode >= SWSB_ENCODE_MODE::FiveDistPipeCvtToInt) |
| 203 |
return true; |
| 204 |
return false; |
| 205 |
} |
| 206 |
|
| 207 |
/// ------------ HW Workaround Information ------------ /// |
| 208 |
// MathWAInfo: For a math instruction, when the following instruction has |
| 209 |
// different predication to the math, should assume the math taking the entire |
| 210 |
// GRF in it's dst no matter the access region and channels are. |
| 211 |
struct MathWAInfo { |
| 212 |
bool previous_is_math = false; |
| 213 |
DepSet *dep_set = nullptr; |
| 214 |
// a special id to identify this DepSet when trying to clean it from buckets |
| 215 |
const InstIDs math_id = {std::numeric_limits<uint32_t>::max(), 0}; |
| 216 |
Instruction *math_inst = nullptr; |
| 217 |
SBID math_sbid = {0, true, DEP_TYPE::NONE}; |
| 218 |
|
| 219 |
void reset() { |
| 220 |
previous_is_math = false; |
| 221 |
dep_set = nullptr; |
| 222 |
math_inst = nullptr; |
| 223 |
math_sbid = {0, true, DEP_TYPE::NONE}; |
| 224 |
} |
| 225 |
} math_wa_info; |
| 226 |
|
| 227 |
private: |
| 228 |
const uint32_t MAX_GRF_BUCKETS = 128; |
| 229 |
// Instruction having 64-bit type destination having 14 latency |
| 230 |
// All the other instructions having 10 latency |
| 231 |
int m_LatencyLong64Pipe = 14; |
| 232 |
int m_LatencyInOrderMath = 18; |
| 233 |
int m_LatencyInOrderPipe = 10; |
| 234 |
|
| 235 |
private: |
| 236 |
// m_SBIDCound - number of SBID can be used |
| 237 |
uint32_t m_SBIDCount; |
| 238 |
|
| 239 |
// m_InstIdCounter - record the current instruction state |
| 240 |
InstIDs m_InstIdCounter; |
| 241 |
|
| 242 |
Kernel &m_kernel; |
| 243 |
ErrorHandler &m_errorHandler; |
| 244 |
|
| 245 |
Bucket *m_buckets = nullptr; |
| 246 |
DepSetBuilder *m_DB = nullptr; |
| 247 |
|
| 248 |
// This is the list to recored all sbid, if it's free or not |
| 249 |
std::vector<SBID> m_freeSBIDList; |
| 250 |
|
| 251 |
// m_SBIDRRCounter - the round robin counter for SB id reuse |
| 252 |
unsigned int m_SBIDRRCounter; |
| 253 |
|
| 254 |
// id to dep set mapping, this tracks for which instructions' dependency that |
| 255 |
// this id is currently on. While we're re-using id, we clean up the |
| 256 |
// dependency |
| 257 |
std::map<uint32_t, std::pair<DepSet *, DepSet *>> m_IdToDepSetMap; |
| 258 |
|
| 259 |
// m_distanceTracker - Track the DepSet of in-order instructions to see if |
| 260 |
// their latency is satisfied. If the distance to current instruction is |
| 261 |
// larger then the latency, then we no need to track the dependency anymore, |
| 262 |
// remove the node from m_distanceTracker |
| 263 |
struct distanceTrackerNode { |
| 264 |
distanceTrackerNode(DepSet *in, DepSet *out) : input(in), output(out) {} |
| 265 |
DepSet *input; |
| 266 |
DepSet *output; |
| 267 |
}; |
| 268 |
std::list<distanceTrackerNode> m_distanceTracker; |
| 269 |
|
| 270 |
bool m_initPoint; |
| 271 |
|
| 272 |
SWSB_ENCODE_MODE m_swsbMode; |
| 273 |
|
| 274 |
const int MAX_VALID_DISTANCE; |
| 275 |
|
| 276 |
bool m_enableSBIDCounter = false; |
| 277 |
}; |
| 278 |
} // namespace iga |
| 279 |
#endif |