| 1 |
/* |
| 2 |
* Copyright 2011 Christoph Bumiller |
| 3 |
* |
| 4 |
* Permission is hereby granted, free of charge, to any person obtaining a |
| 5 |
* copy of this software and associated documentation files (the "Software"), |
| 6 |
* to deal in the Software without restriction, including without limitation |
| 7 |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 |
* and/or sell copies of the Software, and to permit persons to whom the |
| 9 |
* Software is furnished to do so, subject to the following conditions: |
| 10 |
* |
| 11 |
* The above copyright notice and this permission notice shall be included in |
| 12 |
* all copies or substantial portions of the Software. |
| 13 |
* |
| 14 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 15 |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 16 |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 17 |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR |
| 18 |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
| 19 |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
| 20 |
* OTHER DEALINGS IN THE SOFTWARE. |
| 21 |
*/ |
| 22 |
|
| 23 |
#include "nv50_ir.h" |
| 24 |
#include "nv50_ir_build_util.h" |
| 25 |
|
| 26 |
#include "nv50_ir_target_nvc0.h" |
| 27 |
#include "nv50_ir_lowering_nvc0.h" |
| 28 |
|
| 29 |
#include <limits> |
| 30 |
|
| 31 |
namespace nv50_ir { |
| 32 |
|
| 33 |
#define QOP_ADD 0 |
| 34 |
#define QOP_SUBR 1 |
| 35 |
#define QOP_SUB 2 |
| 36 |
#define QOP_MOV2 3 |
| 37 |
|
| 38 |
// UL UR LL LR |
| 39 |
#define QUADOP(q, r, s, t) \ |
| 40 |
((QOP_##q << 6) | (QOP_##r << 4) | \ |
| 41 |
(QOP_##s << 2) | (QOP_##t << 0)) |
| 42 |
|
| 43 |
void |
| 44 |
NVC0LegalizeSSA::handleDIV(Instruction *i) |
| 45 |
{ |
| 46 |
FlowInstruction *call; |
| 47 |
int builtin; |
| 48 |
|
| 49 |
bld.setPosition(i, false); |
| 50 |
|
| 51 |
// Generate movs to the input regs for the call we want to generate |
| 52 |
for (int s = 0; i->srcExists(s); ++s) { |
| 53 |
Instruction *ld = i->getSrc(s)->getInsn(); |
| 54 |
// check if we are moving an immediate, propagate it in that case |
| 55 |
if (!ld || ld->fixed || (ld->op != OP_LOAD && ld->op != OP_MOV) || |
| 56 |
!(ld->src(0).getFile() == FILE_IMMEDIATE)) |
| 57 |
bld.mkMovToReg(s, i->getSrc(s)); |
| 58 |
else { |
| 59 |
assert(ld->getSrc(0) != NULL); |
| 60 |
bld.mkMovToReg(s, ld->getSrc(0)); |
| 61 |
// Clear the src, to make code elimination possible here before we |
| 62 |
// delete the instruction i later |
| 63 |
i->setSrc(s, NULL); |
| 64 |
if (ld->isDead()) |
| 65 |
delete_Instruction(prog, ld); |
| 66 |
} |
| 67 |
} |
| 68 |
|
| 69 |
switch (i->dType) { |
| 70 |
case TYPE_U32: builtin = NVC0_BUILTIN_DIV_U32; break; |
| 71 |
case TYPE_S32: builtin = NVC0_BUILTIN_DIV_S32; break; |
| 72 |
default: |
| 73 |
return; |
| 74 |
} |
| 75 |
call = bld.mkFlow(OP_CALL, NULL, CC_ALWAYS, NULL); |
| 76 |
bld.mkMovFromReg(i->getDef(0), i->op == OP_DIV ? 0 : 1); |
| 77 |
bld.mkClobber(FILE_GPR, (i->op == OP_DIV) ? 0xe : 0xd, 2); |
| 78 |
bld.mkClobber(FILE_PREDICATE, (i->dType == TYPE_S32) ? 0xf : 0x3, 0); |
| 79 |
|
| 80 |
call->fixed = 1; |
| 81 |
call->absolute = call->builtin = 1; |
| 82 |
call->target.builtin = builtin; |
| 83 |
delete_Instruction(prog, i); |
| 84 |
} |
| 85 |
|
| 86 |
void |
| 87 |
NVC0LegalizeSSA::handleRCPRSQLib(Instruction *i, Value *src[]) |
| 88 |
{ |
| 89 |
FlowInstruction *call; |
| 90 |
Value *def[2]; |
| 91 |
int builtin; |
| 92 |
|
| 93 |
def[0] = bld.mkMovToReg(0, src[0])->getDef(0); |
| 94 |
def[1] = bld.mkMovToReg(1, src[1])->getDef(0); |
| 95 |
|
| 96 |
if (i->op == OP_RCP) |
| 97 |
builtin = NVC0_BUILTIN_RCP_F64; |
| 98 |
else |
| 99 |
builtin = NVC0_BUILTIN_RSQ_F64; |
| 100 |
|
| 101 |
call = bld.mkFlow(OP_CALL, NULL, CC_ALWAYS, NULL); |
| 102 |
def[0] = bld.getSSA(); |
| 103 |
def[1] = bld.getSSA(); |
| 104 |
bld.mkMovFromReg(def[0], 0); |
| 105 |
bld.mkMovFromReg(def[1], 1); |
| 106 |
bld.mkClobber(FILE_GPR, 0x3fc, 2); |
| 107 |
bld.mkClobber(FILE_PREDICATE, i->op == OP_RSQ ? 0x3 : 0x1, 0); |
| 108 |
bld.mkOp2(OP_MERGE, TYPE_U64, i->getDef(0), def[0], def[1]); |
| 109 |
|
| 110 |
call->fixed = 1; |
| 111 |
call->absolute = call->builtin = 1; |
| 112 |
call->target.builtin = builtin; |
| 113 |
delete_Instruction(prog, i); |
| 114 |
|
| 115 |
prog->fp64 = true; |
| 116 |
} |
| 117 |
|
| 118 |
void |
| 119 |
NVC0LegalizeSSA::handleRCPRSQ(Instruction *i) |
| 120 |
{ |
| 121 |
assert(i->dType == TYPE_F64); |
| 122 |
// There are instructions that will compute the high 32 bits of the 64-bit |
| 123 |
// float. We will just stick 0 in the bottom 32 bits. |
| 124 |
|
| 125 |
bld.setPosition(i, false); |
| 126 |
|
| 127 |
// 1. Take the source and it up. |
| 128 |
Value *src[2], *dst[2], *def = i->getDef(0); |
| 129 |
bld.mkSplit(src, 4, i->getSrc(0)); |
| 130 |
|
| 131 |
int chip = prog->getTarget()->getChipset(); |
| 132 |
if (chip >= NVISA_GK104_CHIPSET) { |
| 133 |
handleRCPRSQLib(i, src); |
| 134 |
return; |
| 135 |
} |
| 136 |
|
| 137 |
// 2. We don't care about the low 32 bits of the destination. Stick a 0 in. |
| 138 |
dst[0] = bld.loadImm(NULL, 0); |
| 139 |
dst[1] = bld.getSSA(); |
| 140 |
|
| 141 |
// 3. The new version of the instruction takes the high 32 bits of the |
| 142 |
// source and outputs the high 32 bits of the destination. |
| 143 |
i->setSrc(0, src[1]); |
| 144 |
i->setDef(0, dst[1]); |
| 145 |
i->setType(TYPE_F32); |
| 146 |
i->subOp = NV50_IR_SUBOP_RCPRSQ_64H; |
| 147 |
|
| 148 |
// 4. Recombine the two dst pieces back into the original destination. |
| 149 |
bld.setPosition(i, true); |
| 150 |
bld.mkOp2(OP_MERGE, TYPE_U64, def, dst[0], dst[1]); |
| 151 |
} |
| 152 |
|
| 153 |
void |
| 154 |
NVC0LegalizeSSA::handleFTZ(Instruction *i) |
| 155 |
{ |
| 156 |
// Only want to flush float inputs |
| 157 |
assert(i->sType == TYPE_F32); |
| 158 |
|
| 159 |
// If we're already flushing denorms (and NaN's) to zero, no need for this. |
| 160 |
if (i->dnz) |
| 161 |
return; |
| 162 |
|
| 163 |
// Only certain classes of operations can flush |
| 164 |
OpClass cls = prog->getTarget()->getOpClass(i->op); |
| 165 |
if (cls != OPCLASS_ARITH && cls != OPCLASS_COMPARE && |
| 166 |
cls != OPCLASS_CONVERT) |
| 167 |
return; |
| 168 |
|
| 169 |
i->ftz = true; |
| 170 |
} |
| 171 |
|
| 172 |
void |
| 173 |
NVC0LegalizeSSA::handleTEXLOD(TexInstruction *i) |
| 174 |
{ |
| 175 |
if (i->tex.levelZero) |
| 176 |
return; |
| 177 |
|
| 178 |
ImmediateValue lod; |
| 179 |
|
| 180 |
// The LOD argument comes right after the coordinates (before depth bias, |
| 181 |
// offsets, etc). |
| 182 |
int arg = i->tex.target.getArgCount(); |
| 183 |
|
| 184 |
// SM30+ stores the indirect handle as a separate arg, which comes before |
| 185 |
// the LOD. |
| 186 |
if (prog->getTarget()->getChipset() >= NVISA_GK104_CHIPSET && |
| 187 |
i->tex.rIndirectSrc >= 0) |
| 188 |
arg++; |
| 189 |
// SM20 stores indirect handle combined with array coordinate |
| 190 |
if (prog->getTarget()->getChipset() < NVISA_GK104_CHIPSET && |
| 191 |
!i->tex.target.isArray() && |
| 192 |
i->tex.rIndirectSrc >= 0) |
| 193 |
arg++; |
| 194 |
|
| 195 |
if (!i->src(arg).getImmediate(lod) || !lod.isInteger(0)) |
| 196 |
return; |
| 197 |
|
| 198 |
if (i->op == OP_TXL) |
| 199 |
i->op = OP_TEX; |
| 200 |
i->tex.levelZero = true; |
| 201 |
i->moveSources(arg + 1, -1); |
| 202 |
} |
| 203 |
|
| 204 |
void |
| 205 |
NVC0LegalizeSSA::handleShift(Instruction *lo) |
| 206 |
{ |
| 207 |
Value *shift = lo->getSrc(1); |
| 208 |
Value *dst64 = lo->getDef(0); |
| 209 |
Value *src[2], *dst[2]; |
| 210 |
operation op = lo->op; |
| 211 |
|
| 212 |
bld.setPosition(lo, false); |
| 213 |
|
| 214 |
bld.mkSplit(src, 4, lo->getSrc(0)); |
| 215 |
|
| 216 |
// SM30 and prior don't have the fancy new SHF.L/R ops. So the logic has to |
| 217 |
// be completely emulated. For SM35+, we can use the more directed SHF |
| 218 |
// operations. |
| 219 |
if (prog->getTarget()->getChipset() < NVISA_GK20A_CHIPSET) { |
| 220 |
// The strategy here is to handle shifts >= 32 and less than 32 as |
| 221 |
// separate parts. |
| 222 |
// |
| 223 |
// For SHL: |
| 224 |
// If the shift is <= 32, then |
| 225 |
// (HI,LO) << x = (HI << x | (LO >> (32 - x)), LO << x) |
| 226 |
// If the shift is > 32, then |
| 227 |
// (HI,LO) << x = (LO << (x - 32), 0) |
| 228 |
// |
| 229 |
// For SHR: |
| 230 |
// If the shift is <= 32, then |
| 231 |
// (HI,LO) >> x = (HI >> x, (HI << (32 - x)) | LO >> x) |
| 232 |
// If the shift is > 32, then |
| 233 |
// (HI,LO) >> x = (0, HI >> (x - 32)) |
| 234 |
// |
| 235 |
// Note that on NVIDIA hardware, a shift > 32 yields a 0 value, which we |
| 236 |
// can use to our advantage. Also note the structural similarities |
| 237 |
// between the right/left cases. The main difference is swapping hi/lo |
| 238 |
// on input and output. |
| 239 |
|
| 240 |
Value *x32_minus_shift, *pred, *hi1, *hi2; |
| 241 |
DataType type = isSignedIntType(lo->dType) ? TYPE_S32 : TYPE_U32; |
| 242 |
operation antiop = op == OP_SHR ? OP_SHL : OP_SHR; |
| 243 |
if (op == OP_SHR) |
| 244 |
std::swap(src[0], src[1]); |
| 245 |
bld.mkOp2(OP_ADD, TYPE_U32, (x32_minus_shift = bld.getSSA()), shift, bld.mkImm(0x20)) |
| 246 |
->src(0).mod = Modifier(NV50_IR_MOD_NEG); |
| 247 |
bld.mkCmp(OP_SET, CC_LE, TYPE_U8, (pred = bld.getSSA(1, FILE_PREDICATE)), |
| 248 |
TYPE_U32, shift, bld.mkImm(32)); |
| 249 |
// Compute HI (shift <= 32) |
| 250 |
bld.mkOp2(OP_OR, TYPE_U32, (hi1 = bld.getSSA()), |
| 251 |
bld.mkOp2v(op, TYPE_U32, bld.getSSA(), src[1], shift), |
| 252 |
bld.mkOp2v(antiop, TYPE_U32, bld.getSSA(), src[0], x32_minus_shift)) |
| 253 |
->setPredicate(CC_P, pred); |
| 254 |
// Compute LO (all shift values) |
| 255 |
bld.mkOp2(op, type, (dst[0] = bld.getSSA()), src[0], shift); |
| 256 |
// Compute HI (shift > 32) |
| 257 |
bld.mkOp2(op, type, (hi2 = bld.getSSA()), src[0], |
| 258 |
bld.mkOp1v(OP_NEG, TYPE_S32, bld.getSSA(), x32_minus_shift)) |
| 259 |
->setPredicate(CC_NOT_P, pred); |
| 260 |
bld.mkOp2(OP_UNION, TYPE_U32, (dst[1] = bld.getSSA()), hi1, hi2); |
| 261 |
if (op == OP_SHR) |
| 262 |
std::swap(dst[0], dst[1]); |
| 263 |
bld.mkOp2(OP_MERGE, TYPE_U64, dst64, dst[0], dst[1]); |
| 264 |
delete_Instruction(prog, lo); |
| 265 |
return; |
| 266 |
} |
| 267 |
|
| 268 |
Instruction *hi = new_Instruction(func, op, TYPE_U32); |
| 269 |
lo->bb->insertAfter(lo, hi); |
| 270 |
|
| 271 |
hi->sType = lo->sType; |
| 272 |
lo->dType = TYPE_U32; |
| 273 |
|
| 274 |
hi->setDef(0, (dst[1] = bld.getSSA())); |
| 275 |
if (lo->op == OP_SHR) |
| 276 |
hi->subOp |= NV50_IR_SUBOP_SHIFT_HIGH; |
| 277 |
lo->setDef(0, (dst[0] = bld.getSSA())); |
| 278 |
|
| 279 |
bld.setPosition(hi, true); |
| 280 |
|
| 281 |
if (lo->op == OP_SHL) |
| 282 |
std::swap(hi, lo); |
| 283 |
|
| 284 |
hi->setSrc(0, new_ImmediateValue(prog, 0u)); |
| 285 |
hi->setSrc(1, shift); |
| 286 |
hi->setSrc(2, lo->op == OP_SHL ? src[0] : src[1]); |
| 287 |
|
| 288 |
lo->setSrc(0, src[0]); |
| 289 |
lo->setSrc(1, shift); |
| 290 |
lo->setSrc(2, src[1]); |
| 291 |
|
| 292 |
bld.mkOp2(OP_MERGE, TYPE_U64, dst64, dst[0], dst[1]); |
| 293 |
} |
| 294 |
|
| 295 |
void |
| 296 |
NVC0LegalizeSSA::handleSET(CmpInstruction *cmp) |
| 297 |
{ |
| 298 |
DataType hTy = cmp->sType == TYPE_S64 ? TYPE_S32 : TYPE_U32; |
| 299 |
Value *carry; |
| 300 |
Value *src0[2], *src1[2]; |
| 301 |
bld.setPosition(cmp, false); |
| 302 |
|
| 303 |
bld.mkSplit(src0, 4, cmp->getSrc(0)); |
| 304 |
bld.mkSplit(src1, 4, cmp->getSrc(1)); |
| 305 |
bld.mkOp2(OP_SUB, hTy, NULL, src0[0], src1[0]) |
| 306 |
->setFlagsDef(0, (carry = bld.getSSA(1, FILE_FLAGS))); |
| 307 |
cmp->setFlagsSrc(cmp->srcCount(), carry); |
| 308 |
cmp->setSrc(0, src0[1]); |
| 309 |
cmp->setSrc(1, src1[1]); |
| 310 |
cmp->sType = hTy; |
| 311 |
} |
| 312 |
|
| 313 |
void |
| 314 |
NVC0LegalizeSSA::handleBREV(Instruction *i) |
| 315 |
{ |
| 316 |
i->op = OP_EXTBF; |
| 317 |
i->subOp = NV50_IR_SUBOP_EXTBF_REV; |
| 318 |
i->setSrc(1, bld.mkImm(0x2000)); |
| 319 |
} |
| 320 |
|
| 321 |
bool |
| 322 |
NVC0LegalizeSSA::visit(Function *fn) |
| 323 |
{ |
| 324 |
bld.setProgram(fn->getProgram()); |
| 325 |
return true; |
| 326 |
} |
| 327 |
|
| 328 |
bool |
| 329 |
NVC0LegalizeSSA::visit(BasicBlock *bb) |
| 330 |
{ |
| 331 |
Instruction *next; |
| 332 |
for (Instruction *i = bb->getEntry(); i; i = next) { |
| 333 |
next = i->next; |
| 334 |
|
| 335 |
if (i->sType == TYPE_F32 && prog->getType() != Program::TYPE_COMPUTE) |
| 336 |
handleFTZ(i); |
| 337 |
|
| 338 |
switch (i->op) { |
| 339 |
case OP_DIV: |
| 340 |
case OP_MOD: |
| 341 |
if (i->sType != TYPE_F32) |
| 342 |
handleDIV(i); |
| 343 |
break; |
| 344 |
case OP_RCP: |
| 345 |
case OP_RSQ: |
| 346 |
if (i->dType == TYPE_F64) |
| 347 |
handleRCPRSQ(i); |
| 348 |
break; |
| 349 |
case OP_TXL: |
| 350 |
case OP_TXF: |
| 351 |
handleTEXLOD(i->asTex()); |
| 352 |
break; |
| 353 |
case OP_SHR: |
| 354 |
case OP_SHL: |
| 355 |
if (typeSizeof(i->sType) == 8) |
| 356 |
handleShift(i); |
| 357 |
break; |
| 358 |
case OP_SET: |
| 359 |
case OP_SET_AND: |
| 360 |
case OP_SET_OR: |
| 361 |
case OP_SET_XOR: |
| 362 |
if (typeSizeof(i->sType) == 8 && i->sType != TYPE_F64) |
| 363 |
handleSET(i->asCmp()); |
| 364 |
break; |
| 365 |
case OP_BREV: |
| 366 |
handleBREV(i); |
| 367 |
break; |
| 368 |
default: |
| 369 |
break; |
| 370 |
} |
| 371 |
} |
| 372 |
return true; |
| 373 |
} |
| 374 |
|
| 375 |
NVC0LegalizePostRA::NVC0LegalizePostRA(const Program *prog) |
| 376 |
: rZero(NULL), |
| 377 |
carry(NULL), |
| 378 |
pOne(NULL), |
| 379 |
needTexBar(prog->getTarget()->getChipset() >= 0xe0 && |
| 380 |
prog->getTarget()->getChipset() < 0x110) |
| 381 |
{ |
| 382 |
} |
| 383 |
|
| 384 |
bool |
| 385 |
NVC0LegalizePostRA::insnDominatedBy(const Instruction *later, |
| 386 |
const Instruction *early) const |
| 387 |
{ |
| 388 |
if (early->bb == later->bb) |
| 389 |
return early->serial < later->serial; |
| 390 |
return later->bb->dominatedBy(early->bb); |
| 391 |
} |
| 392 |
|
| 393 |
void |
| 394 |
NVC0LegalizePostRA::addTexUse(std::list<TexUse> &uses, |
| 395 |
Instruction *usei, const Instruction *texi) |
| 396 |
{ |
| 397 |
bool add = true; |
| 398 |
bool dominated = insnDominatedBy(usei, texi); |
| 399 |
// Uses before the tex have to all be included. Just because an earlier |
| 400 |
// instruction dominates another instruction doesn't mean that there's no |
| 401 |
// way to get from the tex to the later instruction. For example you could |
| 402 |
// have nested loops, with the tex in the inner loop, and uses before it in |
| 403 |
// both loops - even though the outer loop's instruction would dominate the |
| 404 |
// inner's, we still want a texbar before the inner loop's instruction. |
| 405 |
// |
| 406 |
// However we can still use the eliding logic between uses dominated by the |
| 407 |
// tex instruction, as that is unambiguously correct. |
| 408 |
if (dominated) { |
| 409 |
for (std::list<TexUse>::iterator it = uses.begin(); it != uses.end();) { |
| 410 |
if (it->after) { |
| 411 |
if (insnDominatedBy(usei, it->insn)) { |
| 412 |
add = false; |
| 413 |
break; |
| 414 |
} |
| 415 |
if (insnDominatedBy(it->insn, usei)) { |
| 416 |
it = uses.erase(it); |
| 417 |
continue; |
| 418 |
} |
| 419 |
} |
| 420 |
++it; |
| 421 |
} |
| 422 |
} |
| 423 |
if (add) |
| 424 |
uses.push_back(TexUse(usei, texi, dominated)); |
| 425 |
} |
| 426 |
|
| 427 |
// While it might be tempting to use the an algorithm that just looks at tex |
| 428 |
// uses, not all texture results are guaranteed to be used on all paths. In |
| 429 |
// the case where along some control flow path a texture result is never used, |
| 430 |
// we might reuse that register for something else, creating a |
| 431 |
// write-after-write hazard. So we have to manually look through all |
| 432 |
// instructions looking for ones that reference the registers in question. |
| 433 |
void |
| 434 |
NVC0LegalizePostRA::findFirstUses( |
| 435 |
Instruction *texi, std::list<TexUse> &uses) |
| 436 |
{ |
| 437 |
int minGPR = texi->def(0).rep()->reg.data.id; |
| 438 |
int maxGPR = minGPR + texi->def(0).rep()->reg.size / 4 - 1; |
| 439 |
|
| 440 |
std::unordered_set<const BasicBlock *> visited; |
| 441 |
findFirstUsesBB(minGPR, maxGPR, texi->next, texi, uses, visited); |
| 442 |
} |
| 443 |
|
| 444 |
void |
| 445 |
NVC0LegalizePostRA::findFirstUsesBB( |
| 446 |
int minGPR, int maxGPR, Instruction *start, |
| 447 |
const Instruction *texi, std::list<TexUse> &uses, |
| 448 |
std::unordered_set<const BasicBlock *> &visited) |
| 449 |
{ |
| 450 |
const BasicBlock *bb = start->bb; |
| 451 |
|
| 452 |
// We don't process the whole bb the first time around. This is correct, |
| 453 |
// however we might be in a loop and hit this BB again, and need to process |
| 454 |
// the full thing. So only mark a bb as visited if we processed it from the |
| 455 |
// beginning. |
| 456 |
if (start == bb->getEntry()) { |
| 457 |
if (visited.find(bb) != visited.end()) |
| 458 |
return; |
| 459 |
visited.insert(bb); |
| 460 |
} |
| 461 |
|
| 462 |
for (Instruction *insn = start; insn != bb->getExit(); insn = insn->next) { |
| 463 |
if (insn->isNop()) |
| 464 |
continue; |
| 465 |
|
| 466 |
for (int d = 0; insn->defExists(d); ++d) { |
| 467 |
const Value *def = insn->def(d).rep(); |
| 468 |
if (insn->def(d).getFile() != FILE_GPR || |
| 469 |
def->reg.data.id + def->reg.size / 4 - 1 < minGPR || |
| 470 |
def->reg.data.id > maxGPR) |
| 471 |
continue; |
| 472 |
addTexUse(uses, insn, texi); |
| 473 |
return; |
| 474 |
} |
| 475 |
|
| 476 |
for (int s = 0; insn->srcExists(s); ++s) { |
| 477 |
const Value *src = insn->src(s).rep(); |
| 478 |
if (insn->src(s).getFile() != FILE_GPR || |
| 479 |
src->reg.data.id + src->reg.size / 4 - 1 < minGPR || |
| 480 |
src->reg.data.id > maxGPR) |
| 481 |
continue; |
| 482 |
addTexUse(uses, insn, texi); |
| 483 |
return; |
| 484 |
} |
| 485 |
} |
| 486 |
|
| 487 |
for (Graph::EdgeIterator ei = bb->cfg.outgoing(); !ei.end(); ei.next()) { |
| 488 |
findFirstUsesBB(minGPR, maxGPR, BasicBlock::get(ei.getNode())->getEntry(), |
| 489 |
texi, uses, visited); |
| 490 |
} |
| 491 |
} |
| 492 |
|
| 493 |
// Texture barriers: |
| 494 |
// This pass is a bit long and ugly and can probably be optimized. |
| 495 |
// |
| 496 |
// 1. obtain a list of TEXes and their outputs' first use(s) |
| 497 |
// 2. calculate the barrier level of each first use (minimal number of TEXes, |
| 498 |
// over all paths, between the TEX and the use in question) |
| 499 |
// 3. for each barrier, if all paths from the source TEX to that barrier |
| 500 |
// contain a barrier of lesser level, it can be culled |
| 501 |
bool |
| 502 |
NVC0LegalizePostRA::insertTextureBarriers(Function *fn) |
| 503 |
{ |
| 504 |
std::list<TexUse> *uses; |
| 505 |
std::vector<Instruction *> texes; |
| 506 |
std::vector<int> bbFirstTex; |
| 507 |
std::vector<int> bbFirstUse; |
| 508 |
std::vector<int> texCounts; |
| 509 |
std::vector<TexUse> useVec; |
| 510 |
ArrayList insns; |
| 511 |
|
| 512 |
fn->orderInstructions(insns); |
| 513 |
|
| 514 |
texCounts.resize(fn->allBBlocks.getSize(), 0); |
| 515 |
bbFirstTex.resize(fn->allBBlocks.getSize(), insns.getSize()); |
| 516 |
bbFirstUse.resize(fn->allBBlocks.getSize(), insns.getSize()); |
| 517 |
|
| 518 |
// tag BB CFG nodes by their id for later |
| 519 |
for (ArrayList::Iterator i = fn->allBBlocks.iterator(); !i.end(); i.next()) { |
| 520 |
BasicBlock *bb = reinterpret_cast<BasicBlock *>(i.get()); |
| 521 |
if (bb) |
| 522 |
bb->cfg.tag = bb->getId(); |
| 523 |
} |
| 524 |
|
| 525 |
// gather the first uses for each TEX |
| 526 |
for (unsigned int i = 0; i < insns.getSize(); ++i) { |
| 527 |
Instruction *tex = reinterpret_cast<Instruction *>(insns.get(i)); |
| 528 |
if (isTextureOp(tex->op)) { |
| 529 |
texes.push_back(tex); |
| 530 |
if (!texCounts.at(tex->bb->getId())) |
| 531 |
bbFirstTex[tex->bb->getId()] = texes.size() - 1; |
| 532 |
texCounts[tex->bb->getId()]++; |
| 533 |
} |
| 534 |
} |
| 535 |
insns.clear(); |
| 536 |
if (texes.empty()) |
| 537 |
return false; |
| 538 |
uses = new std::list<TexUse>[texes.size()]; |
| 539 |
if (!uses) |
| 540 |
return false; |
| 541 |
for (size_t i = 0; i < texes.size(); ++i) { |
| 542 |
findFirstUses(texes[i], uses[i]); |
| 543 |
} |
| 544 |
|
| 545 |
// determine the barrier level at each use |
| 546 |
for (size_t i = 0; i < texes.size(); ++i) { |
| 547 |
for (std::list<TexUse>::iterator u = uses[i].begin(); u != uses[i].end(); |
| 548 |
++u) { |
| 549 |
BasicBlock *tb = texes[i]->bb; |
| 550 |
BasicBlock *ub = u->insn->bb; |
| 551 |
if (tb == ub) { |
| 552 |
u->level = 0; |
| 553 |
for (size_t j = i + 1; j < texes.size() && |
| 554 |
texes[j]->bb == tb && texes[j]->serial < u->insn->serial; |
| 555 |
++j) |
| 556 |
u->level++; |
| 557 |
} else { |
| 558 |
u->level = fn->cfg.findLightestPathWeight(&tb->cfg, |
| 559 |
&ub->cfg, texCounts); |
| 560 |
if (u->level < 0) { |
| 561 |
WARN("Failed to find path TEX -> TEXBAR\n"); |
| 562 |
u->level = 0; |
| 563 |
continue; |
| 564 |
} |
| 565 |
// this counted all TEXes in the origin block, correct that |
| 566 |
u->level -= i - bbFirstTex.at(tb->getId()) + 1 /* this TEX */; |
| 567 |
// and did not count the TEXes in the destination block, add those |
| 568 |
for (size_t j = bbFirstTex.at(ub->getId()); j < texes.size() && |
| 569 |
texes[j]->bb == ub && texes[j]->serial < u->insn->serial; |
| 570 |
++j) |
| 571 |
u->level++; |
| 572 |
} |
| 573 |
assert(u->level >= 0); |
| 574 |
useVec.push_back(*u); |
| 575 |
} |
| 576 |
} |
| 577 |
delete[] uses; |
| 578 |
|
| 579 |
// insert the barriers |
| 580 |
for (size_t i = 0; i < useVec.size(); ++i) { |
| 581 |
Instruction *prev = useVec[i].insn->prev; |
| 582 |
if (useVec[i].level < 0) |
| 583 |
continue; |
| 584 |
if (prev && prev->op == OP_TEXBAR) { |
| 585 |
if (prev->subOp > useVec[i].level) |
| 586 |
prev->subOp = useVec[i].level; |
| 587 |
prev->setSrc(prev->srcCount(), useVec[i].tex->getDef(0)); |
| 588 |
} else { |
| 589 |
Instruction *bar = new_Instruction(func, OP_TEXBAR, TYPE_NONE); |
| 590 |
bar->fixed = 1; |
| 591 |
bar->subOp = useVec[i].level; |
| 592 |
// make use explicit to ease latency calculation |
| 593 |
bar->setSrc(bar->srcCount(), useVec[i].tex->getDef(0)); |
| 594 |
useVec[i].insn->bb->insertBefore(useVec[i].insn, bar); |
| 595 |
} |
| 596 |
} |
| 597 |
|
| 598 |
if (fn->getProgram()->optLevel < 3) |
| 599 |
return true; |
| 600 |
|
| 601 |
std::vector<Limits> limitT, limitB, limitS; // entry, exit, single |
| 602 |
|
| 603 |
limitT.resize(fn->allBBlocks.getSize(), Limits(0, 0)); |
| 604 |
limitB.resize(fn->allBBlocks.getSize(), Limits(0, 0)); |
| 605 |
limitS.resize(fn->allBBlocks.getSize()); |
| 606 |
|
| 607 |
// cull unneeded barriers (should do that earlier, but for simplicity) |
| 608 |
IteratorRef bi = fn->cfg.iteratorCFG(); |
| 609 |
// first calculate min/max outstanding TEXes for each BB |
| 610 |
for (bi->reset(); !bi->end(); bi->next()) { |
| 611 |
Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get()); |
| 612 |
BasicBlock *bb = BasicBlock::get(n); |
| 613 |
int min = 0; |
| 614 |
int max = std::numeric_limits<int>::max(); |
| 615 |
for (Instruction *i = bb->getFirst(); i; i = i->next) { |
| 616 |
if (isTextureOp(i->op)) { |
| 617 |
min++; |
| 618 |
if (max < std::numeric_limits<int>::max()) |
| 619 |
max++; |
| 620 |
} else |
| 621 |
if (i->op == OP_TEXBAR) { |
| 622 |
min = MIN2(min, i->subOp); |
| 623 |
max = MIN2(max, i->subOp); |
| 624 |
} |
| 625 |
} |
| 626 |
// limits when looking at an isolated block |
| 627 |
limitS[bb->getId()].min = min; |
| 628 |
limitS[bb->getId()].max = max; |
| 629 |
} |
| 630 |
// propagate the min/max values |
| 631 |
for (unsigned int l = 0; l <= fn->loopNestingBound; ++l) { |
| 632 |
for (bi->reset(); !bi->end(); bi->next()) { |
| 633 |
Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get()); |
| 634 |
BasicBlock *bb = BasicBlock::get(n); |
| 635 |
const int bbId = bb->getId(); |
| 636 |
for (Graph::EdgeIterator ei = n->incident(); !ei.end(); ei.next()) { |
| 637 |
BasicBlock *in = BasicBlock::get(ei.getNode()); |
| 638 |
const int inId = in->getId(); |
| 639 |
limitT[bbId].min = MAX2(limitT[bbId].min, limitB[inId].min); |
| 640 |
limitT[bbId].max = MAX2(limitT[bbId].max, limitB[inId].max); |
| 641 |
} |
| 642 |
// I just hope this is correct ... |
| 643 |
if (limitS[bbId].max == std::numeric_limits<int>::max()) { |
| 644 |
// no barrier |
| 645 |
limitB[bbId].min = limitT[bbId].min + limitS[bbId].min; |
| 646 |
limitB[bbId].max = limitT[bbId].max + limitS[bbId].min; |
| 647 |
} else { |
| 648 |
// block contained a barrier |
| 649 |
limitB[bbId].min = MIN2(limitS[bbId].max, |
| 650 |
limitT[bbId].min + limitS[bbId].min); |
| 651 |
limitB[bbId].max = MIN2(limitS[bbId].max, |
| 652 |
limitT[bbId].max + limitS[bbId].min); |
| 653 |
} |
| 654 |
} |
| 655 |
} |
| 656 |
// finally delete unnecessary barriers |
| 657 |
for (bi->reset(); !bi->end(); bi->next()) { |
| 658 |
Graph::Node *n = reinterpret_cast<Graph::Node *>(bi->get()); |
| 659 |
BasicBlock *bb = BasicBlock::get(n); |
| 660 |
Instruction *prev = NULL; |
| 661 |
Instruction *next; |
| 662 |
int max = limitT[bb->getId()].max; |
| 663 |
for (Instruction *i = bb->getFirst(); i; i = next) { |
| 664 |
next = i->next; |
| 665 |
if (i->op == OP_TEXBAR) { |
| 666 |
if (i->subOp >= max) { |
| 667 |
delete_Instruction(prog, i); |
| 668 |
i = NULL; |
| 669 |
} else { |
| 670 |
max = i->subOp; |
| 671 |
if (prev && prev->op == OP_TEXBAR && prev->subOp >= max) { |
| 672 |
delete_Instruction(prog, prev); |
| 673 |
prev = NULL; |
| 674 |
} |
| 675 |
} |
| 676 |
} else |
| 677 |
if (isTextureOp(i->op)) { |
| 678 |
max++; |
| 679 |
} |
| 680 |
if (i && !i->isNop()) |
| 681 |
prev = i; |
| 682 |
} |
| 683 |
} |
| 684 |
return true; |
| 685 |
} |
| 686 |
|
| 687 |
bool |
| 688 |
NVC0LegalizePostRA::visit(Function *fn) |
| 689 |
{ |
| 690 |
if (needTexBar) |
| 691 |
insertTextureBarriers(fn); |
| 692 |
|
| 693 |
rZero = new_LValue(fn, FILE_GPR); |
| 694 |
pOne = new_LValue(fn, FILE_PREDICATE); |
| 695 |
carry = new_LValue(fn, FILE_FLAGS); |
| 696 |
|
| 697 |
rZero->reg.data.id = (prog->getTarget()->getChipset() >= NVISA_GK20A_CHIPSET) ? 255 : 63; |
| 698 |
carry->reg.data.id = 0; |
| 699 |
pOne->reg.data.id = 7; |
| 700 |
|
| 701 |
return true; |
| 702 |
} |
| 703 |
|
| 704 |
void |
| 705 |
NVC0LegalizePostRA::replaceZero(Instruction *i) |
| 706 |
{ |
| 707 |
for (int s = 0; i->srcExists(s); ++s) { |
| 708 |
if (s == 2 && i->op == OP_SUCLAMP) |
| 709 |
continue; |
| 710 |
if (s == 1 && i->op == OP_SHLADD) |
| 711 |
continue; |
| 712 |
ImmediateValue *imm = i->getSrc(s)->asImm(); |
| 713 |
if (imm) { |
| 714 |
if (i->op == OP_SELP && s == 2) { |
| 715 |
i->setSrc(s, pOne); |
| 716 |
if (imm->reg.data.u64 == 0) |
| 717 |
i->src(s).mod = i->src(s).mod ^ Modifier(NV50_IR_MOD_NOT); |
| 718 |
} else if (imm->reg.data.u64 == 0) { |
| 719 |
i->setSrc(s, rZero); |
| 720 |
} |
| 721 |
} |
| 722 |
} |
| 723 |
} |
| 724 |
|
| 725 |
// replace CONT with BRA for single unconditional continue |
| 726 |
bool |
| 727 |
NVC0LegalizePostRA::tryReplaceContWithBra(BasicBlock *bb) |
| 728 |
{ |
| 729 |
if (bb->cfg.incidentCount() != 2 || bb->getEntry()->op != OP_PRECONT) |
| 730 |
return false; |
| 731 |
Graph::EdgeIterator ei = bb->cfg.incident(); |
| 732 |
if (ei.getType() != Graph::Edge::BACK) |
| 733 |
ei.next(); |
| 734 |
if (ei.getType() != Graph::Edge::BACK) |
| 735 |
return false; |
| 736 |
BasicBlock *contBB = BasicBlock::get(ei.getNode()); |
| 737 |
|
| 738 |
if (!contBB->getExit() || contBB->getExit()->op != OP_CONT || |
| 739 |
contBB->getExit()->getPredicate()) |
| 740 |
return false; |
| 741 |
contBB->getExit()->op = OP_BRA; |
| 742 |
bb->remove(bb->getEntry()); // delete PRECONT |
| 743 |
|
| 744 |
ei.next(); |
| 745 |
assert(ei.end() || ei.getType() != Graph::Edge::BACK); |
| 746 |
return true; |
| 747 |
} |
| 748 |
|
| 749 |
// replace branches to join blocks with join ops |
| 750 |
void |
| 751 |
NVC0LegalizePostRA::propagateJoin(BasicBlock *bb) |
| 752 |
{ |
| 753 |
if (bb->getEntry()->op != OP_JOIN || bb->getEntry()->asFlow()->limit) |
| 754 |
return; |
| 755 |
for (Graph::EdgeIterator ei = bb->cfg.incident(); !ei.end(); ei.next()) { |
| 756 |
BasicBlock *in = BasicBlock::get(ei.getNode()); |
| 757 |
Instruction *exit = in->getExit(); |
| 758 |
if (!exit) { |
| 759 |
in->insertTail(new FlowInstruction(func, OP_JOIN, bb)); |
| 760 |
// there should always be a terminator instruction |
| 761 |
WARN("inserted missing terminator in BB:%i\n", in->getId()); |
| 762 |
} else |
| 763 |
if (exit->op == OP_BRA) { |
| 764 |
exit->op = OP_JOIN; |
| 765 |
exit->asFlow()->limit = 1; // must-not-propagate marker |
| 766 |
} |
| 767 |
} |
| 768 |
bb->remove(bb->getEntry()); |
| 769 |
} |
| 770 |
|
| 771 |
// replaces instructions which would end up as f2f or i2i with faster |
| 772 |
// alternatives: |
| 773 |
// - fabs(a) -> fadd(0, abs a) |
| 774 |
// - fneg(a) -> fadd(neg 0, neg a) |
| 775 |
// - ineg(a) -> iadd(0, neg a) |
| 776 |
// - fneg(abs a) -> fadd(neg 0, neg abs a) |
| 777 |
// - sat(a) -> sat add(0, a) |
| 778 |
void |
| 779 |
NVC0LegalizePostRA::replaceCvt(Instruction *cvt) |
| 780 |
{ |
| 781 |
if (!isFloatType(cvt->sType) && typeSizeof(cvt->sType) != 4) |
| 782 |
return; |
| 783 |
if (cvt->sType != cvt->dType) |
| 784 |
return; |
| 785 |
// we could make it work, but in this case we have optimizations disabled |
| 786 |
// and we don't really care either way. |
| 787 |
if (cvt->src(0).getFile() != FILE_GPR && |
| 788 |
cvt->src(0).getFile() != FILE_MEMORY_CONST) |
| 789 |
return; |
| 790 |
|
| 791 |
Modifier mod0, mod1; |
| 792 |
|
| 793 |
switch (cvt->op) { |
| 794 |
case OP_ABS: |
| 795 |
if (cvt->src(0).mod) |
| 796 |
return; |
| 797 |
if (!isFloatType(cvt->sType)) |
| 798 |
return; |
| 799 |
mod0 = 0; |
| 800 |
mod1 = NV50_IR_MOD_ABS; |
| 801 |
break; |
| 802 |
case OP_NEG: |
| 803 |
if (!isFloatType(cvt->sType) && cvt->src(0).mod) |
| 804 |
return; |
| 805 |
if (isFloatType(cvt->sType) && |
| 806 |
(cvt->src(0).mod && cvt->src(0).mod != Modifier(NV50_IR_MOD_ABS))) |
| 807 |
return; |
| 808 |
|
| 809 |
mod0 = isFloatType(cvt->sType) ? NV50_IR_MOD_NEG : 0; |
| 810 |
mod1 = cvt->src(0).mod == Modifier(NV50_IR_MOD_ABS) ? |
| 811 |
NV50_IR_MOD_NEG_ABS : NV50_IR_MOD_NEG; |
| 812 |
break; |
| 813 |
case OP_SAT: |
| 814 |
if (!isFloatType(cvt->sType) && cvt->src(0).mod.abs()) |
| 815 |
return; |
| 816 |
mod0 = 0; |
| 817 |
mod1 = cvt->src(0).mod; |
| 818 |
cvt->saturate = true; |
| 819 |
break; |
| 820 |
default: |
| 821 |
return; |
| 822 |
} |
| 823 |
|
| 824 |
cvt->op = OP_ADD; |
| 825 |
cvt->moveSources(0, 1); |
| 826 |
cvt->setSrc(0, rZero); |
| 827 |
cvt->src(0).mod = mod0; |
| 828 |
cvt->src(1).mod = mod1; |
| 829 |
} |
| 830 |
|
| 831 |
bool |
| 832 |
NVC0LegalizePostRA::visit(BasicBlock *bb) |
| 833 |
{ |
| 834 |
Instruction *i, *next; |
| 835 |
|
| 836 |
// remove pseudo operations and non-fixed no-ops, split 64 bit operations |
| 837 |
for (i = bb->getFirst(); i; i = next) { |
| 838 |
next = i->next; |
| 839 |
if (i->op == OP_EMIT || i->op == OP_RESTART) { |
| 840 |
if (!i->getDef(0)->refCount()) |
| 841 |
i->setDef(0, NULL); |
| 842 |
if (i->src(0).getFile() == FILE_IMMEDIATE) |
| 843 |
i->setSrc(0, rZero); // initial value must be 0 |
| 844 |
replaceZero(i); |
| 845 |
} else |
| 846 |
if (i->isNop()) { |
| 847 |
bb->remove(i); |
| 848 |
} else |
| 849 |
if (i->op == OP_BAR && i->subOp == NV50_IR_SUBOP_BAR_SYNC && |
| 850 |
prog->getType() != Program::TYPE_COMPUTE) { |
| 851 |
// It seems like barriers are never required for tessellation since |
| 852 |
// the warp size is 32, and there are always at most 32 tcs threads. |
| 853 |
bb->remove(i); |
| 854 |
} else |
| 855 |
if (i->op == OP_LOAD && i->subOp == NV50_IR_SUBOP_LDC_IS) { |
| 856 |
int offset = i->src(0).get()->reg.data.offset; |
| 857 |
if (abs(offset) >= 0x10000) |
| 858 |
i->src(0).get()->reg.fileIndex += offset >> 16; |
| 859 |
i->src(0).get()->reg.data.offset = (int)(short)offset; |
| 860 |
} else { |
| 861 |
// TODO: Move this to before register allocation for operations that |
| 862 |
// need the $c register ! |
| 863 |
if (typeSizeof(i->sType) == 8 || typeSizeof(i->dType) == 8) { |
| 864 |
Instruction *hi; |
| 865 |
hi = BuildUtil::split64BitOpPostRA(func, i, rZero, carry); |
| 866 |
if (hi) |
| 867 |
next = hi; |
| 868 |
} |
| 869 |
|
| 870 |
if (i->op != OP_MOV && i->op != OP_PFETCH) |
| 871 |
replaceZero(i); |
| 872 |
|
| 873 |
if (i->op == OP_SAT || i->op == OP_NEG || i->op == OP_ABS) |
| 874 |
replaceCvt(i); |
| 875 |
} |
| 876 |
} |
| 877 |
if (!bb->getEntry()) |
| 878 |
return true; |
| 879 |
|
| 880 |
if (!tryReplaceContWithBra(bb)) |
| 881 |
propagateJoin(bb); |
| 882 |
|
| 883 |
return true; |
| 884 |
} |
| 885 |
|
| 886 |
NVC0LoweringPass::NVC0LoweringPass(Program *prog) : targ(prog->getTarget()), |
| 887 |
gpEmitAddress(NULL) |
| 888 |
{ |
| 889 |
bld.setProgram(prog); |
| 890 |
} |
| 891 |
|
| 892 |
bool |
| 893 |
NVC0LoweringPass::visit(Function *fn) |
| 894 |
{ |
| 895 |
if (prog->getType() == Program::TYPE_GEOMETRY) { |
| 896 |
assert(!strncmp(fn->getName(), "MAIN", 4)); |
| 897 |
// TODO: when we generate actual functions pass this value along somehow |
| 898 |
bld.setPosition(BasicBlock::get(fn->cfg.getRoot()), false); |
| 899 |
gpEmitAddress = bld.loadImm(NULL, 0)->asLValue(); |
| 900 |
if (fn->cfgExit) { |
| 901 |
bld.setPosition(BasicBlock::get(fn->cfgExit)->getExit(), false); |
| 902 |
if (prog->getTarget()->getChipset() >= NVISA_GV100_CHIPSET) |
| 903 |
bld.mkOp1(OP_FINAL, TYPE_NONE, NULL, gpEmitAddress)->fixed = 1; |
| 904 |
bld.mkMovToReg(0, gpEmitAddress); |
| 905 |
} |
| 906 |
} |
| 907 |
return true; |
| 908 |
} |
| 909 |
|
| 910 |
bool |
| 911 |
NVC0LoweringPass::visit(BasicBlock *bb) |
| 912 |
{ |
| 913 |
return true; |
| 914 |
} |
| 915 |
|
| 916 |
inline Value * |
| 917 |
NVC0LoweringPass::loadTexHandle(Value *ptr, unsigned int slot) |
| 918 |
{ |
| 919 |
uint8_t b = prog->driver->io.auxCBSlot; |
| 920 |
uint32_t off = prog->driver->io.texBindBase + slot * 4; |
| 921 |
|
| 922 |
if (ptr) |
| 923 |
ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(2)); |
| 924 |
|
| 925 |
return bld. |
| 926 |
mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr); |
| 927 |
} |
| 928 |
|
| 929 |
// move array source to first slot, convert to u16, add indirections |
| 930 |
bool |
| 931 |
NVC0LoweringPass::handleTEX(TexInstruction *i) |
| 932 |
{ |
| 933 |
const int dim = i->tex.target.getDim() + i->tex.target.isCube(); |
| 934 |
const int arg = i->tex.target.getArgCount() - i->tex.target.isMS(); |
| 935 |
const int lyr = arg - 1; |
| 936 |
const int chipset = prog->getTarget()->getChipset(); |
| 937 |
|
| 938 |
/* Only normalize in the non-explicit derivatives case. For explicit |
| 939 |
* derivatives, this is handled in handleManualTXD. |
| 940 |
*/ |
| 941 |
if (i->tex.target.isCube() && i->dPdx[0].get() == NULL) { |
| 942 |
Value *src[3], *val; |
| 943 |
int c; |
| 944 |
for (c = 0; c < 3; ++c) |
| 945 |
src[c] = bld.mkOp1v(OP_ABS, TYPE_F32, bld.getSSA(), i->getSrc(c)); |
| 946 |
val = bld.getScratch(); |
| 947 |
bld.mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]); |
| 948 |
bld.mkOp2(OP_MAX, TYPE_F32, val, src[2], val); |
| 949 |
bld.mkOp1(OP_RCP, TYPE_F32, val, val); |
| 950 |
for (c = 0; c < 3; ++c) { |
| 951 |
i->setSrc(c, bld.mkOp2v(OP_MUL, TYPE_F32, bld.getSSA(), |
| 952 |
i->getSrc(c), val)); |
| 953 |
} |
| 954 |
} |
| 955 |
|
| 956 |
// Arguments to the TEX instruction are a little insane. Even though the |
| 957 |
// encoding is identical between SM20 and SM30, the arguments mean |
| 958 |
// different things between Fermi and Kepler+. A lot of arguments are |
| 959 |
// optional based on flags passed to the instruction. This summarizes the |
| 960 |
// order of things. |
| 961 |
// |
| 962 |
// Fermi: |
| 963 |
// array/indirect |
| 964 |
// coords |
| 965 |
// sample |
| 966 |
// lod bias |
| 967 |
// depth compare |
| 968 |
// offsets: |
| 969 |
// - tg4: 8 bits each, either 2 (1 offset reg) or 8 (2 offset reg) |
| 970 |
// - other: 4 bits each, single reg |
| 971 |
// |
| 972 |
// Kepler+: |
| 973 |
// indirect handle |
| 974 |
// array (+ offsets for txd in upper 16 bits) |
| 975 |
// coords |
| 976 |
// sample |
| 977 |
// lod bias |
| 978 |
// depth compare |
| 979 |
// offsets (same as fermi, except txd which takes it with array) |
| 980 |
// |
| 981 |
// Maxwell (tex): |
| 982 |
// array |
| 983 |
// coords |
| 984 |
// indirect handle |
| 985 |
// sample |
| 986 |
// lod bias |
| 987 |
// depth compare |
| 988 |
// offsets |
| 989 |
// |
| 990 |
// Maxwell (txd): |
| 991 |
// indirect handle |
| 992 |
// coords |
| 993 |
// array + offsets |
| 994 |
// derivatives |
| 995 |
|
| 996 |
if (chipset >= NVISA_GK104_CHIPSET) { |
| 997 |
if (i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) { |
| 998 |
// XXX this ignores tsc, and assumes a 1:1 mapping |
| 999 |
assert(i->tex.rIndirectSrc >= 0); |
| 1000 |
if (!i->tex.bindless) { |
| 1001 |
Value *hnd = loadTexHandle(i->getIndirectR(), i->tex.r); |
| 1002 |
i->tex.r = 0xff; |
| 1003 |
i->tex.s = 0x1f; |
| 1004 |
i->setIndirectR(hnd); |
| 1005 |
} |
| 1006 |
i->setIndirectS(NULL); |
| 1007 |
} else if (i->tex.r == i->tex.s || i->op == OP_TXF) { |
| 1008 |
if (i->tex.r == 0xffff) |
| 1009 |
i->tex.r = prog->driver->io.fbtexBindBase / 4; |
| 1010 |
else |
| 1011 |
i->tex.r += prog->driver->io.texBindBase / 4; |
| 1012 |
i->tex.s = 0; // only a single cX[] value possible here |
| 1013 |
} else { |
| 1014 |
Value *hnd = bld.getScratch(); |
| 1015 |
Value *rHnd = loadTexHandle(NULL, i->tex.r); |
| 1016 |
Value *sHnd = loadTexHandle(NULL, i->tex.s); |
| 1017 |
|
| 1018 |
bld.mkOp3(OP_INSBF, TYPE_U32, hnd, rHnd, bld.mkImm(0x1400), sHnd); |
| 1019 |
|
| 1020 |
i->tex.r = 0; // not used for indirect tex |
| 1021 |
i->tex.s = 0; |
| 1022 |
i->setIndirectR(hnd); |
| 1023 |
} |
| 1024 |
if (i->tex.target.isArray()) { |
| 1025 |
LValue *layer = new_LValue(func, FILE_GPR); |
| 1026 |
Value *src = i->getSrc(lyr); |
| 1027 |
/* Vulkan requires that a negative index on a texelFetch() count as |
| 1028 |
* out-of-bounds but a negative index on any other texture operation |
| 1029 |
* gets clamped to 0. (See the spec section entitled "(u,v,w,a) to |
| 1030 |
* (i,j,k,l,n) Transformation And Array Layer Selection"). |
| 1031 |
* |
| 1032 |
* For TXF, we take a U32 MAX with 0xffff, ensuring that negative |
| 1033 |
* array indices clamp to 0xffff and will be considered out-of-bounds |
| 1034 |
* by the hardware (there are a maximum of 2048 array indices in an |
| 1035 |
* image descriptor). For everything else, we use a saturating F32 |
| 1036 |
* to U16 conversion which will clamp negative array indices to 0 and |
| 1037 |
* large positive indices to 0xffff. The hardware will further clamp |
| 1038 |
* positive array indices to the maximum in the image descriptor. |
| 1039 |
*/ |
| 1040 |
if (i->op == OP_TXF) { |
| 1041 |
bld.mkOp2(OP_MIN, TYPE_U32, layer, src, bld.loadImm(NULL, 0xffff)); |
| 1042 |
} else { |
| 1043 |
bld.mkCvt(OP_CVT, TYPE_U16, layer, TYPE_F32, src)->saturate = true; |
| 1044 |
} |
| 1045 |
if (i->op != OP_TXD || chipset < NVISA_GM107_CHIPSET) { |
| 1046 |
for (int s = dim; s >= 1; --s) |
| 1047 |
i->setSrc(s, i->getSrc(s - 1)); |
| 1048 |
i->setSrc(0, layer); |
| 1049 |
} else { |
| 1050 |
i->setSrc(dim, layer); |
| 1051 |
} |
| 1052 |
} |
| 1053 |
// Move the indirect reference to the first place |
| 1054 |
if (i->tex.rIndirectSrc >= 0 && ( |
| 1055 |
i->op == OP_TXD || chipset < NVISA_GM107_CHIPSET)) { |
| 1056 |
Value *hnd = i->getIndirectR(); |
| 1057 |
|
| 1058 |
i->setIndirectR(NULL); |
| 1059 |
i->moveSources(0, 1); |
| 1060 |
i->setSrc(0, hnd); |
| 1061 |
i->tex.rIndirectSrc = 0; |
| 1062 |
i->tex.sIndirectSrc = -1; |
| 1063 |
} |
| 1064 |
// Move the indirect reference to right after the coords |
| 1065 |
else if (i->tex.rIndirectSrc >= 0 && chipset >= NVISA_GM107_CHIPSET) { |
| 1066 |
Value *hnd = i->getIndirectR(); |
| 1067 |
|
| 1068 |
i->setIndirectR(NULL); |
| 1069 |
i->moveSources(arg, 1); |
| 1070 |
i->setSrc(arg, hnd); |
| 1071 |
i->tex.rIndirectSrc = 0; |
| 1072 |
i->tex.sIndirectSrc = -1; |
| 1073 |
} |
| 1074 |
} else |
| 1075 |
// (nvc0) generate and move the tsc/tic/array source to the front |
| 1076 |
if (i->tex.target.isArray() || i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) { |
| 1077 |
LValue *src = new_LValue(func, FILE_GPR); // 0xttxsaaaa |
| 1078 |
|
| 1079 |
Value *ticRel = i->getIndirectR(); |
| 1080 |
Value *tscRel = i->getIndirectS(); |
| 1081 |
|
| 1082 |
if (i->tex.r == 0xffff) { |
| 1083 |
i->tex.r = 0x20; |
| 1084 |
i->tex.s = 0x10; |
| 1085 |
} |
| 1086 |
|
| 1087 |
if (ticRel) { |
| 1088 |
i->setSrc(i->tex.rIndirectSrc, NULL); |
| 1089 |
if (i->tex.r) |
| 1090 |
ticRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(), |
| 1091 |
ticRel, bld.mkImm(i->tex.r)); |
| 1092 |
} |
| 1093 |
if (tscRel) { |
| 1094 |
i->setSrc(i->tex.sIndirectSrc, NULL); |
| 1095 |
if (i->tex.s) |
| 1096 |
tscRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(), |
| 1097 |
tscRel, bld.mkImm(i->tex.s)); |
| 1098 |
} |
| 1099 |
|
| 1100 |
Value *arrayIndex = i->tex.target.isArray() ? i->getSrc(lyr) : NULL; |
| 1101 |
if (arrayIndex) { |
| 1102 |
for (int s = dim; s >= 1; --s) |
| 1103 |
i->setSrc(s, i->getSrc(s - 1)); |
| 1104 |
i->setSrc(0, arrayIndex); |
| 1105 |
} else { |
| 1106 |
i->moveSources(0, 1); |
| 1107 |
} |
| 1108 |
|
| 1109 |
if (arrayIndex) { |
| 1110 |
if (i->op == OP_TXF) { |
| 1111 |
bld.mkOp2(OP_MIN, TYPE_U32, src, arrayIndex, bld.loadImm(NULL, 0xffff)); |
| 1112 |
} else { |
| 1113 |
bld.mkCvt(OP_CVT, TYPE_U16, src, TYPE_F32, arrayIndex)->saturate = true; |
| 1114 |
} |
| 1115 |
} else { |
| 1116 |
bld.loadImm(src, 0); |
| 1117 |
} |
| 1118 |
|
| 1119 |
if (ticRel) |
| 1120 |
bld.mkOp3(OP_INSBF, TYPE_U32, src, ticRel, bld.mkImm(0x0917), src); |
| 1121 |
if (tscRel) |
| 1122 |
bld.mkOp3(OP_INSBF, TYPE_U32, src, tscRel, bld.mkImm(0x0710), src); |
| 1123 |
|
| 1124 |
i->setSrc(0, src); |
| 1125 |
} |
| 1126 |
|
| 1127 |
// For nvc0, the sample id has to be in the second operand, as the offset |
| 1128 |
// does. Right now we don't know how to pass both in, and this case can't |
| 1129 |
// happen with OpenGL. On nve0, the sample id is part of the texture |
| 1130 |
// coordinate argument. |
| 1131 |
assert(chipset >= NVISA_GK104_CHIPSET || |
| 1132 |
!i->tex.useOffsets || !i->tex.target.isMS()); |
| 1133 |
|
| 1134 |
// offset is between lod and dc |
| 1135 |
if (i->tex.useOffsets) { |
| 1136 |
int n, c; |
| 1137 |
int s = i->srcCount(0xff, true); |
| 1138 |
if (i->op != OP_TXD || chipset < NVISA_GK104_CHIPSET) { |
| 1139 |
if (i->tex.target.isShadow()) |
| 1140 |
s--; |
| 1141 |
if (i->srcExists(s)) // move potential predicate out of the way |
| 1142 |
i->moveSources(s, 1); |
| 1143 |
if (i->tex.useOffsets == 4 && i->srcExists(s + 1)) |
| 1144 |
i->moveSources(s + 1, 1); |
| 1145 |
} |
| 1146 |
if (i->op == OP_TXG) { |
| 1147 |
// Either there is 1 offset, which goes into the 2 low bytes of the |
| 1148 |
// first source, or there are 4 offsets, which go into 2 sources (8 |
| 1149 |
// values, 1 byte each). |
| 1150 |
Value *offs[2] = {NULL, NULL}; |
| 1151 |
for (n = 0; n < i->tex.useOffsets; n++) { |
| 1152 |
for (c = 0; c < 2; ++c) { |
| 1153 |
if ((n % 2) == 0 && c == 0) |
| 1154 |
bld.mkMov(offs[n / 2] = bld.getScratch(), i->offset[n][c].get()); |
| 1155 |
else |
| 1156 |
bld.mkOp3(OP_INSBF, TYPE_U32, |
| 1157 |
offs[n / 2], |
| 1158 |
i->offset[n][c].get(), |
| 1159 |
bld.mkImm(0x800 | ((n * 16 + c * 8) % 32)), |
| 1160 |
offs[n / 2]); |
| 1161 |
} |
| 1162 |
} |
| 1163 |
i->setSrc(s, offs[0]); |
| 1164 |
if (offs[1]) |
| 1165 |
i->setSrc(s + 1, offs[1]); |
| 1166 |
} else { |
| 1167 |
unsigned imm = 0; |
| 1168 |
assert(i->tex.useOffsets == 1); |
| 1169 |
for (c = 0; c < 3; ++c) { |
| 1170 |
ImmediateValue val; |
| 1171 |
if (!i->offset[0][c].getImmediate(val)) |
| 1172 |
assert(!"non-immediate offset passed to non-TXG"); |
| 1173 |
imm |= (val.reg.data.u32 & 0xf) << (c * 4); |
| 1174 |
} |
| 1175 |
if (i->op == OP_TXD && chipset >= NVISA_GK104_CHIPSET) { |
| 1176 |
// The offset goes into the upper 16 bits of the array index. So |
| 1177 |
// create it if it's not already there, and INSBF it if it already |
| 1178 |
// is. |
| 1179 |
s = (i->tex.rIndirectSrc >= 0) ? 1 : 0; |
| 1180 |
if (chipset >= NVISA_GM107_CHIPSET) |
| 1181 |
s += dim; |
| 1182 |
if (i->tex.target.isArray()) { |
| 1183 |
Value *offset = bld.getScratch(); |
| 1184 |
bld.mkOp3(OP_INSBF, TYPE_U32, offset, |
| 1185 |
bld.loadImm(NULL, imm), bld.mkImm(0xc10), |
| 1186 |
i->getSrc(s)); |
| 1187 |
i->setSrc(s, offset); |
| 1188 |
} else { |
| 1189 |
i->moveSources(s, 1); |
| 1190 |
i->setSrc(s, bld.loadImm(NULL, imm << 16)); |
| 1191 |
} |
| 1192 |
} else { |
| 1193 |
i->setSrc(s, bld.loadImm(NULL, imm)); |
| 1194 |
} |
| 1195 |
} |
| 1196 |
} |
| 1197 |
|
| 1198 |
return true; |
| 1199 |
} |
| 1200 |
|
| 1201 |
bool |
| 1202 |
NVC0LoweringPass::handleManualTXD(TexInstruction *i) |
| 1203 |
{ |
| 1204 |
// Always done from the l0 perspective. This is the way that NVIDIA's |
| 1205 |
// driver does it, and doing it from the "current" lane's perspective |
| 1206 |
// doesn't seem to always work for reasons that aren't altogether clear, |
| 1207 |
// even in frag shaders. |
| 1208 |
// |
| 1209 |
// Note that we must move not only the coordinates into lane0, but also all |
| 1210 |
// ancillary arguments, like array indices and depth compare as they may |
| 1211 |
// differ between lanes. Offsets for TXD are supposed to be uniform, so we |
| 1212 |
// leave them alone. |
| 1213 |
static const uint8_t qOps[2] = |
| 1214 |
{ QUADOP(MOV2, ADD, MOV2, ADD), QUADOP(MOV2, MOV2, ADD, ADD) }; |
| 1215 |
|
| 1216 |
Value *def[4][4]; |
| 1217 |
Value *crd[3], *arr[2], *shadow; |
| 1218 |
Instruction *tex; |
| 1219 |
Value *zero = bld.loadImm(bld.getSSA(), 0); |
| 1220 |
int l, c; |
| 1221 |
const int dim = i->tex.target.getDim() + i->tex.target.isCube(); |
| 1222 |
|
| 1223 |
// This function is invoked after handleTEX lowering, so we have to expect |
| 1224 |
// the arguments in the order that the hw wants them. For Fermi, array and |
| 1225 |
// indirect are both in the leading arg, while for Kepler, array and |
| 1226 |
// indirect are separate (and both precede the coordinates). Maxwell is |
| 1227 |
// handled in a separate function. |
| 1228 |
int array; |
| 1229 |
if (targ->getChipset() < NVISA_GK104_CHIPSET) |
| 1230 |
array = i->tex.target.isArray() || i->tex.rIndirectSrc >= 0; |
| 1231 |
else |
| 1232 |
array = i->tex.target.isArray() + (i->tex.rIndirectSrc >= 0); |
| 1233 |
|
| 1234 |
i->op = OP_TEX; // no need to clone dPdx/dPdy later |
| 1235 |
|
| 1236 |
for (c = 0; c < dim; ++c) |
| 1237 |
crd[c] = bld.getScratch(); |
| 1238 |
for (c = 0; c < array; ++c) |
| 1239 |
arr[c] = bld.getScratch(); |
| 1240 |
shadow = bld.getScratch(); |
| 1241 |
|
| 1242 |
for (l = 0; l < 4; ++l) { |
| 1243 |
Value *src[3], *val; |
| 1244 |
|
| 1245 |
bld.mkOp(OP_QUADON, TYPE_NONE, NULL); |
| 1246 |
// we're using the texture result from lane 0 in all cases, so make sure |
| 1247 |
// that lane 0 is pointing at the proper array index, indirect value, |
| 1248 |
// and depth compare. |
| 1249 |
if (l != 0) { |
| 1250 |
for (c = 0; c < array; ++c) |
| 1251 |
bld.mkQuadop(0x00, arr[c], l, i->getSrc(c), zero); |
| 1252 |
if (i->tex.target.isShadow()) { |
| 1253 |
// The next argument after coords is the depth compare |
| 1254 |
bld.mkQuadop(0x00, shadow, l, i->getSrc(array + dim), zero); |
| 1255 |
} |
| 1256 |
} |
| 1257 |
// mov position coordinates from lane l to all lanes |
| 1258 |
for (c = 0; c < dim; ++c) |
| 1259 |
bld.mkQuadop(0x00, crd[c], l, i->getSrc(c + array), zero); |
| 1260 |
// add dPdx from lane l to lanes dx |
| 1261 |
for (c = 0; c < dim; ++c) |
| 1262 |
bld.mkQuadop(qOps[0], crd[c], l, i->dPdx[c].get(), crd[c]); |
| 1263 |
// add dPdy from lane l to lanes dy |
| 1264 |
for (c = 0; c < dim; ++c) |
| 1265 |
bld.mkQuadop(qOps[1], crd[c], l, i->dPdy[c].get(), crd[c]); |
| 1266 |
// normalize cube coordinates |
| 1267 |
if (i->tex.target.isCube()) { |
| 1268 |
for (c = 0; c < 3; ++c) |
| 1269 |
src[c] = bld.mkOp1v(OP_ABS, TYPE_F32, bld.getSSA(), crd[c]); |
| 1270 |
val = bld.getScratch(); |
| 1271 |
bld.mkOp2(OP_MAX, TYPE_F32, val, src[0], src[1]); |
| 1272 |
bld.mkOp2(OP_MAX, TYPE_F32, val, src[2], val); |
| 1273 |
bld.mkOp1(OP_RCP, TYPE_F32, val, val); |
| 1274 |
for (c = 0; c < 3; ++c) |
| 1275 |
src[c] = bld.mkOp2v(OP_MUL, TYPE_F32, bld.getSSA(), crd[c], val); |
| 1276 |
} else { |
| 1277 |
for (c = 0; c < dim; ++c) |
| 1278 |
src[c] = crd[c]; |
| 1279 |
} |
| 1280 |
// texture |
| 1281 |
bld.insert(tex = cloneForward(func, i)); |
| 1282 |
if (l != 0) { |
| 1283 |
for (c = 0; c < array; ++c) |
| 1284 |
tex->setSrc(c, arr[c]); |
| 1285 |
if (i->tex.target.isShadow()) |
| 1286 |
tex->setSrc(array + dim, shadow); |
| 1287 |
} |
| 1288 |
for (c = 0; c < dim; ++c) |
| 1289 |
tex->setSrc(c + array, src[c]); |
| 1290 |
// broadcast results from lane 0 to all lanes so that the moves *into* |
| 1291 |
// the target lane pick up the proper value. |
| 1292 |
if (l != 0) |
| 1293 |
for (c = 0; i->defExists(c); ++c) |
| 1294 |
bld.mkQuadop(0x00, tex->getDef(c), 0, tex->getDef(c), zero); |
| 1295 |
bld.mkOp(OP_QUADPOP, TYPE_NONE, NULL); |
| 1296 |
|
| 1297 |
// save results |
| 1298 |
for (c = 0; i->defExists(c); ++c) { |
| 1299 |
Instruction *mov; |
| 1300 |
def[c][l] = bld.getSSA(); |
| 1301 |
mov = bld.mkMov(def[c][l], tex->getDef(c)); |
| 1302 |
mov->fixed = 1; |
| 1303 |
mov->lanes = 1 << l; |
| 1304 |
} |
| 1305 |
} |
| 1306 |
|
| 1307 |
for (c = 0; i->defExists(c); ++c) { |
| 1308 |
Instruction *u = bld.mkOp(OP_UNION, TYPE_U32, i->getDef(c)); |
| 1309 |
for (l = 0; l < 4; ++l) |
| 1310 |
u->setSrc(l, def[c][l]); |
| 1311 |
} |
| 1312 |
|
| 1313 |
i->bb->remove(i); |
| 1314 |
return true; |
| 1315 |
} |
| 1316 |
|
| 1317 |
bool |
| 1318 |
NVC0LoweringPass::handleTXD(TexInstruction *txd) |
| 1319 |
{ |
| 1320 |
int dim = txd->tex.target.getDim() + txd->tex.target.isCube(); |
| 1321 |
unsigned arg = txd->tex.target.getArgCount(); |
| 1322 |
unsigned expected_args = arg; |
| 1323 |
const int chipset = prog->getTarget()->getChipset(); |
| 1324 |
|
| 1325 |
if (chipset >= NVISA_GK104_CHIPSET) { |
| 1326 |
if (!txd->tex.target.isArray() && txd->tex.useOffsets) |
| 1327 |
expected_args++; |
| 1328 |
if (txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0) |
| 1329 |
expected_args++; |
| 1330 |
} else { |
| 1331 |
if (txd->tex.useOffsets) |
| 1332 |
expected_args++; |
| 1333 |
if (!txd->tex.target.isArray() && ( |
| 1334 |
txd->tex.rIndirectSrc >= 0 || txd->tex.sIndirectSrc >= 0)) |
| 1335 |
expected_args++; |
| 1336 |
} |
| 1337 |
|
| 1338 |
if (expected_args > 4 || |
| 1339 |
dim > 2 || |
| 1340 |
txd->tex.target.isShadow()) |
| 1341 |
txd->op = OP_TEX; |
| 1342 |
|
| 1343 |
handleTEX(txd); |
| 1344 |
while (txd->srcExists(arg)) |
| 1345 |
++arg; |
| 1346 |
|
| 1347 |
txd->tex.derivAll = true; |
| 1348 |
if (txd->op == OP_TEX) |
| 1349 |
return handleManualTXD(txd); |
| 1350 |
|
| 1351 |
assert(arg == expected_args); |
| 1352 |
for (int c = 0; c < dim; ++c) { |
| 1353 |
txd->setSrc(arg + c * 2 + 0, txd->dPdx[c]); |
| 1354 |
txd->setSrc(arg + c * 2 + 1, txd->dPdy[c]); |
| 1355 |
txd->dPdx[c].set(NULL); |
| 1356 |
txd->dPdy[c].set(NULL); |
| 1357 |
} |
| 1358 |
|
| 1359 |
// In this case we have fewer than 4 "real" arguments, which means that |
| 1360 |
// handleTEX didn't apply any padding. However we have to make sure that |
| 1361 |
// the second "group" of arguments still gets padded up to 4. |
| 1362 |
if (chipset >= NVISA_GK104_CHIPSET) { |
| 1363 |
int s = arg + 2 * dim; |
| 1364 |
if (s >= 4 && s < 7) { |
| 1365 |
if (txd->srcExists(s)) // move potential predicate out of the way |
| 1366 |
txd->moveSources(s, 7 - s); |
| 1367 |
while (s < 7) |
| 1368 |
txd->setSrc(s++, bld.loadImm(NULL, 0)); |
| 1369 |
} |
| 1370 |
} |
| 1371 |
|
| 1372 |
return true; |
| 1373 |
} |
| 1374 |
|
| 1375 |
bool |
| 1376 |
NVC0LoweringPass::handleTXQ(TexInstruction *txq) |
| 1377 |
{ |
| 1378 |
const int chipset = prog->getTarget()->getChipset(); |
| 1379 |
if (chipset >= NVISA_GK104_CHIPSET && txq->tex.rIndirectSrc < 0) |
| 1380 |
txq->tex.r += prog->driver->io.texBindBase / 4; |
| 1381 |
|
| 1382 |
if (txq->tex.rIndirectSrc < 0) |
| 1383 |
return true; |
| 1384 |
|
| 1385 |
Value *ticRel = txq->getIndirectR(); |
| 1386 |
|
| 1387 |
txq->setIndirectS(NULL); |
| 1388 |
txq->tex.sIndirectSrc = -1; |
| 1389 |
|
| 1390 |
assert(ticRel); |
| 1391 |
|
| 1392 |
if (chipset < NVISA_GK104_CHIPSET) { |
| 1393 |
LValue *src = new_LValue(func, FILE_GPR); // 0xttxsaaaa |
| 1394 |
|
| 1395 |
txq->setSrc(txq->tex.rIndirectSrc, NULL); |
| 1396 |
if (txq->tex.r) |
| 1397 |
ticRel = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(), |
| 1398 |
ticRel, bld.mkImm(txq->tex.r)); |
| 1399 |
|
| 1400 |
bld.mkOp2(OP_SHL, TYPE_U32, src, ticRel, bld.mkImm(0x17)); |
| 1401 |
|
| 1402 |
txq->moveSources(0, 1); |
| 1403 |
txq->setSrc(0, src); |
| 1404 |
} else { |
| 1405 |
Value *hnd; |
| 1406 |
if (txq->tex.bindless) { |
| 1407 |
hnd = txq->getIndirectR(); |
| 1408 |
} else { |
| 1409 |
hnd = loadTexHandle(txq->getIndirectR(), txq->tex.r); |
| 1410 |
txq->tex.r = 0xff; |
| 1411 |
txq->tex.s = 0x1f; |
| 1412 |
} |
| 1413 |
|
| 1414 |
txq->setIndirectR(NULL); |
| 1415 |
txq->moveSources(0, 1); |
| 1416 |
txq->setSrc(0, hnd); |
| 1417 |
txq->tex.rIndirectSrc = 0; |
| 1418 |
} |
| 1419 |
|
| 1420 |
return true; |
| 1421 |
} |
| 1422 |
|
| 1423 |
bool |
| 1424 |
NVC0LoweringPass::handleTXLQ(TexInstruction *i) |
| 1425 |
{ |
| 1426 |
/* The outputs are inverted compared to what the TGSI instruction |
| 1427 |
* expects. Take that into account in the mask. |
| 1428 |
*/ |
| 1429 |
assert((i->tex.mask & ~3) == 0); |
| 1430 |
if (i->tex.mask == 1) |
| 1431 |
i->tex.mask = 2; |
| 1432 |
else if (i->tex.mask == 2) |
| 1433 |
i->tex.mask = 1; |
| 1434 |
handleTEX(i); |
| 1435 |
bld.setPosition(i, true); |
| 1436 |
|
| 1437 |
/* The returned values are not quite what we want: |
| 1438 |
* (a) convert from s16/u16 to f32 |
| 1439 |
* (b) multiply by 1/256 |
| 1440 |
*/ |
| 1441 |
for (int def = 0; def < 2; ++def) { |
| 1442 |
if (!i->defExists(def)) |
| 1443 |
continue; |
| 1444 |
enum DataType type = TYPE_S16; |
| 1445 |
if (i->tex.mask == 2 || def > 0) |
| 1446 |
type = TYPE_U16; |
| 1447 |
bld.mkCvt(OP_CVT, TYPE_F32, i->getDef(def), type, i->getDef(def)); |
| 1448 |
bld.mkOp2(OP_MUL, TYPE_F32, i->getDef(def), |
| 1449 |
i->getDef(def), bld.loadImm(NULL, 1.0f / 256)); |
| 1450 |
} |
| 1451 |
if (i->tex.mask == 3) { |
| 1452 |
LValue *t = new_LValue(func, FILE_GPR); |
| 1453 |
bld.mkMov(t, i->getDef(0)); |
| 1454 |
bld.mkMov(i->getDef(0), i->getDef(1)); |
| 1455 |
bld.mkMov(i->getDef(1), t); |
| 1456 |
} |
| 1457 |
return true; |
| 1458 |
} |
| 1459 |
|
| 1460 |
bool |
| 1461 |
NVC0LoweringPass::handleBUFQ(Instruction *bufq) |
| 1462 |
{ |
| 1463 |
bufq->op = OP_MOV; |
| 1464 |
bufq->setSrc(0, loadBufLength32(bufq->getIndirect(0, 1), |
| 1465 |
bufq->getSrc(0)->reg.fileIndex * 16)); |
| 1466 |
bufq->setIndirect(0, 0, NULL); |
| 1467 |
bufq->setIndirect(0, 1, NULL); |
| 1468 |
return true; |
| 1469 |
} |
| 1470 |
|
| 1471 |
void |
| 1472 |
NVC0LoweringPass::handleSharedATOMNVE4(Instruction *atom) |
| 1473 |
{ |
| 1474 |
assert(atom->src(0).getFile() == FILE_MEMORY_SHARED); |
| 1475 |
|
| 1476 |
BasicBlock *currBB = atom->bb; |
| 1477 |
BasicBlock *tryLockBB = atom->bb->splitBefore(atom, false); |
| 1478 |
BasicBlock *joinBB = atom->bb->splitAfter(atom); |
| 1479 |
BasicBlock *setAndUnlockBB = new BasicBlock(func); |
| 1480 |
BasicBlock *failLockBB = new BasicBlock(func); |
| 1481 |
|
| 1482 |
bld.setPosition(currBB, true); |
| 1483 |
assert(!currBB->joinAt); |
| 1484 |
currBB->joinAt = bld.mkFlow(OP_JOINAT, joinBB, CC_ALWAYS, NULL); |
| 1485 |
|
| 1486 |
CmpInstruction *pred = |
| 1487 |
bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE), |
| 1488 |
TYPE_U32, bld.mkImm(0), bld.mkImm(1)); |
| 1489 |
|
| 1490 |
bld.mkFlow(OP_BRA, tryLockBB, CC_ALWAYS, NULL); |
| 1491 |
currBB->cfg.attach(&tryLockBB->cfg, Graph::Edge::TREE); |
| 1492 |
|
| 1493 |
bld.setPosition(tryLockBB, true); |
| 1494 |
|
| 1495 |
Instruction *ld = |
| 1496 |
bld.mkLoad(TYPE_U32, atom->getDef(0), atom->getSrc(0)->asSym(), |
| 1497 |
atom->getIndirect(0, 0)); |
| 1498 |
ld->setDef(1, bld.getSSA(1, FILE_PREDICATE)); |
| 1499 |
ld->subOp = NV50_IR_SUBOP_LOAD_LOCKED; |
| 1500 |
|
| 1501 |
bld.mkFlow(OP_BRA, setAndUnlockBB, CC_P, ld->getDef(1)); |
| 1502 |
bld.mkFlow(OP_BRA, failLockBB, CC_ALWAYS, NULL); |
| 1503 |
tryLockBB->cfg.attach(&failLockBB->cfg, Graph::Edge::CROSS); |
| 1504 |
tryLockBB->cfg.attach(&setAndUnlockBB->cfg, Graph::Edge::TREE); |
| 1505 |
|
| 1506 |
tryLockBB->cfg.detach(&joinBB->cfg); |
| 1507 |
bld.remove(atom); |
| 1508 |
|
| 1509 |
bld.setPosition(setAndUnlockBB, true); |
| 1510 |
Value *stVal; |
| 1511 |
if (atom->subOp == NV50_IR_SUBOP_ATOM_EXCH) { |
| 1512 |
// Read the old value, and write the new one. |
| 1513 |
stVal = atom->getSrc(1); |
| 1514 |
} else if (atom->subOp == NV50_IR_SUBOP_ATOM_CAS) { |
| 1515 |
CmpInstruction *set = |
| 1516 |
bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(), |
| 1517 |
TYPE_U32, ld->getDef(0), atom->getSrc(1)); |
| 1518 |
|
| 1519 |
bld.mkCmp(OP_SLCT, CC_NE, TYPE_U32, (stVal = bld.getSSA()), |
| 1520 |
TYPE_U32, atom->getSrc(2), ld->getDef(0), set->getDef(0)); |
| 1521 |
} else { |
| 1522 |
operation op; |
| 1523 |
|
| 1524 |
switch (atom->subOp) { |
| 1525 |
case NV50_IR_SUBOP_ATOM_ADD: |
| 1526 |
op = OP_ADD; |
| 1527 |
break; |
| 1528 |
case NV50_IR_SUBOP_ATOM_AND: |
| 1529 |
op = OP_AND; |
| 1530 |
break; |
| 1531 |
case NV50_IR_SUBOP_ATOM_OR: |
| 1532 |
op = OP_OR; |
| 1533 |
break; |
| 1534 |
case NV50_IR_SUBOP_ATOM_XOR: |
| 1535 |
op = OP_XOR; |
| 1536 |
break; |
| 1537 |
case NV50_IR_SUBOP_ATOM_MIN: |
| 1538 |
op = OP_MIN; |
| 1539 |
break; |
| 1540 |
case NV50_IR_SUBOP_ATOM_MAX: |
| 1541 |
op = OP_MAX; |
| 1542 |
break; |
| 1543 |
default: |
| 1544 |
assert(0); |
| 1545 |
return; |
| 1546 |
} |
| 1547 |
|
| 1548 |
stVal = bld.mkOp2v(op, atom->dType, bld.getSSA(), ld->getDef(0), |
| 1549 |
atom->getSrc(1)); |
| 1550 |
} |
| 1551 |
|
| 1552 |
Instruction *st = |
| 1553 |
bld.mkStore(OP_STORE, TYPE_U32, atom->getSrc(0)->asSym(), |
| 1554 |
atom->getIndirect(0, 0), stVal); |
| 1555 |
st->setDef(0, pred->getDef(0)); |
| 1556 |
st->subOp = NV50_IR_SUBOP_STORE_UNLOCKED; |
| 1557 |
|
| 1558 |
bld.mkFlow(OP_BRA, failLockBB, CC_ALWAYS, NULL); |
| 1559 |
setAndUnlockBB->cfg.attach(&failLockBB->cfg, Graph::Edge::TREE); |
| 1560 |
|
| 1561 |
// Lock until the store has not been performed. |
| 1562 |
bld.setPosition(failLockBB, true); |
| 1563 |
bld.mkFlow(OP_BRA, tryLockBB, CC_NOT_P, pred->getDef(0)); |
| 1564 |
bld.mkFlow(OP_BRA, joinBB, CC_ALWAYS, NULL); |
| 1565 |
failLockBB->cfg.attach(&tryLockBB->cfg, Graph::Edge::BACK); |
| 1566 |
failLockBB->cfg.attach(&joinBB->cfg, Graph::Edge::TREE); |
| 1567 |
|
| 1568 |
bld.setPosition(joinBB, false); |
| 1569 |
bld.mkFlow(OP_JOIN, NULL, CC_ALWAYS, NULL)->fixed = 1; |
| 1570 |
} |
| 1571 |
|
| 1572 |
void |
| 1573 |
NVC0LoweringPass::handleSharedATOM(Instruction *atom) |
| 1574 |
{ |
| 1575 |
assert(atom->src(0).getFile() == FILE_MEMORY_SHARED); |
| 1576 |
|
| 1577 |
BasicBlock *currBB = atom->bb; |
| 1578 |
BasicBlock *tryLockAndSetBB = atom->bb->splitBefore(atom, false); |
| 1579 |
BasicBlock *joinBB = atom->bb->splitAfter(atom); |
| 1580 |
|
| 1581 |
bld.setPosition(currBB, true); |
| 1582 |
assert(!currBB->joinAt); |
| 1583 |
currBB->joinAt = bld.mkFlow(OP_JOINAT, joinBB, CC_ALWAYS, NULL); |
| 1584 |
|
| 1585 |
bld.mkFlow(OP_BRA, tryLockAndSetBB, CC_ALWAYS, NULL); |
| 1586 |
currBB->cfg.attach(&tryLockAndSetBB->cfg, Graph::Edge::TREE); |
| 1587 |
|
| 1588 |
bld.setPosition(tryLockAndSetBB, true); |
| 1589 |
|
| 1590 |
Instruction *ld = |
| 1591 |
bld.mkLoad(TYPE_U32, atom->getDef(0), atom->getSrc(0)->asSym(), |
| 1592 |
atom->getIndirect(0, 0)); |
| 1593 |
ld->setDef(1, bld.getSSA(1, FILE_PREDICATE)); |
| 1594 |
ld->subOp = NV50_IR_SUBOP_LOAD_LOCKED; |
| 1595 |
|
| 1596 |
Value *stVal; |
| 1597 |
if (atom->subOp == NV50_IR_SUBOP_ATOM_EXCH) { |
| 1598 |
// Read the old value, and write the new one. |
| 1599 |
stVal = atom->getSrc(1); |
| 1600 |
} else if (atom->subOp == NV50_IR_SUBOP_ATOM_CAS) { |
| 1601 |
CmpInstruction *set = |
| 1602 |
bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE), |
| 1603 |
TYPE_U32, ld->getDef(0), atom->getSrc(1)); |
| 1604 |
set->setPredicate(CC_P, ld->getDef(1)); |
| 1605 |
|
| 1606 |
Instruction *selp = |
| 1607 |
bld.mkOp3(OP_SELP, TYPE_U32, bld.getSSA(), ld->getDef(0), |
| 1608 |
atom->getSrc(2), set->getDef(0)); |
| 1609 |
selp->src(2).mod = Modifier(NV50_IR_MOD_NOT); |
| 1610 |
selp->setPredicate(CC_P, ld->getDef(1)); |
| 1611 |
|
| 1612 |
stVal = selp->getDef(0); |
| 1613 |
} else { |
| 1614 |
operation op; |
| 1615 |
|
| 1616 |
switch (atom->subOp) { |
| 1617 |
case NV50_IR_SUBOP_ATOM_ADD: |
| 1618 |
op = OP_ADD; |
| 1619 |
break; |
| 1620 |
case NV50_IR_SUBOP_ATOM_AND: |
| 1621 |
op = OP_AND; |
| 1622 |
break; |
| 1623 |
case NV50_IR_SUBOP_ATOM_OR: |
| 1624 |
op = OP_OR; |
| 1625 |
break; |
| 1626 |
case NV50_IR_SUBOP_ATOM_XOR: |
| 1627 |
op = OP_XOR; |
| 1628 |
break; |
| 1629 |
case NV50_IR_SUBOP_ATOM_MIN: |
| 1630 |
op = OP_MIN; |
| 1631 |
break; |
| 1632 |
case NV50_IR_SUBOP_ATOM_MAX: |
| 1633 |
op = OP_MAX; |
| 1634 |
break; |
| 1635 |
default: |
| 1636 |
assert(0); |
| 1637 |
return; |
| 1638 |
} |
| 1639 |
|
| 1640 |
Instruction *i = |
| 1641 |
bld.mkOp2(op, atom->dType, bld.getSSA(), ld->getDef(0), |
| 1642 |
atom->getSrc(1)); |
| 1643 |
i->setPredicate(CC_P, ld->getDef(1)); |
| 1644 |
|
| 1645 |
stVal = i->getDef(0); |
| 1646 |
} |
| 1647 |
|
| 1648 |
Instruction *st = |
| 1649 |
bld.mkStore(OP_STORE, TYPE_U32, atom->getSrc(0)->asSym(), |
| 1650 |
atom->getIndirect(0, 0), stVal); |
| 1651 |
st->setPredicate(CC_P, ld->getDef(1)); |
| 1652 |
st->subOp = NV50_IR_SUBOP_STORE_UNLOCKED; |
| 1653 |
|
| 1654 |
// Loop until the lock is acquired. |
| 1655 |
bld.mkFlow(OP_BRA, tryLockAndSetBB, CC_NOT_P, ld->getDef(1)); |
| 1656 |
tryLockAndSetBB->cfg.attach(&tryLockAndSetBB->cfg, Graph::Edge::BACK); |
| 1657 |
tryLockAndSetBB->cfg.attach(&joinBB->cfg, Graph::Edge::CROSS); |
| 1658 |
bld.mkFlow(OP_BRA, joinBB, CC_ALWAYS, NULL); |
| 1659 |
|
| 1660 |
bld.remove(atom); |
| 1661 |
|
| 1662 |
bld.setPosition(joinBB, false); |
| 1663 |
bld.mkFlow(OP_JOIN, NULL, CC_ALWAYS, NULL)->fixed = 1; |
| 1664 |
} |
| 1665 |
|
| 1666 |
bool |
| 1667 |
NVC0LoweringPass::handleATOM(Instruction *atom) |
| 1668 |
{ |
| 1669 |
SVSemantic sv; |
| 1670 |
Value *ptr = atom->getIndirect(0, 0), *ind = atom->getIndirect(0, 1), *base; |
| 1671 |
|
| 1672 |
switch (atom->src(0).getFile()) { |
| 1673 |
case FILE_MEMORY_LOCAL: |
| 1674 |
sv = SV_LBASE; |
| 1675 |
break; |
| 1676 |
case FILE_MEMORY_SHARED: |
| 1677 |
// For Fermi/Kepler, we have to use ld lock/st unlock to perform atomic |
| 1678 |
// operations on shared memory. For Maxwell, ATOMS is enough. |
| 1679 |
if (targ->getChipset() < NVISA_GK104_CHIPSET) |
| 1680 |
handleSharedATOM(atom); |
| 1681 |
else if (targ->getChipset() < NVISA_GM107_CHIPSET) |
| 1682 |
handleSharedATOMNVE4(atom); |
| 1683 |
return true; |
| 1684 |
case FILE_MEMORY_GLOBAL: |
| 1685 |
return true; |
| 1686 |
default: |
| 1687 |
assert(atom->src(0).getFile() == FILE_MEMORY_BUFFER); |
| 1688 |
base = loadBufInfo64(ind, atom->getSrc(0)->reg.fileIndex * 16); |
| 1689 |
assert(base->reg.size == 8); |
| 1690 |
if (ptr) |
| 1691 |
base = bld.mkOp2v(OP_ADD, TYPE_U64, base, base, ptr); |
| 1692 |
assert(base->reg.size == 8); |
| 1693 |
atom->setIndirect(0, 0, base); |
| 1694 |
atom->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL; |
| 1695 |
|
| 1696 |
// Harden against out-of-bounds accesses |
| 1697 |
Value *offset = bld.loadImm(NULL, atom->getSrc(0)->reg.data.offset + typeSizeof(atom->sType)); |
| 1698 |
Value *length = loadBufLength32(ind, atom->getSrc(0)->reg.fileIndex * 16); |
| 1699 |
Value *pred = new_LValue(func, FILE_PREDICATE); |
| 1700 |
if (ptr) |
| 1701 |
bld.mkOp2(OP_ADD, TYPE_U32, offset, offset, ptr); |
| 1702 |
bld.mkCmp(OP_SET, CC_GT, TYPE_U32, pred, TYPE_U32, offset, length); |
| 1703 |
atom->setPredicate(CC_NOT_P, pred); |
| 1704 |
if (atom->defExists(0)) { |
| 1705 |
Value *zero, *dst = atom->getDef(0); |
| 1706 |
atom->setDef(0, bld.getSSA()); |
| 1707 |
|
| 1708 |
bld.setPosition(atom, true); |
| 1709 |
bld.mkMov((zero = bld.getSSA()), bld.mkImm(0)) |
| 1710 |
->setPredicate(CC_P, pred); |
| 1711 |
bld.mkOp2(OP_UNION, TYPE_U32, dst, atom->getDef(0), zero); |
| 1712 |
} |
| 1713 |
|
| 1714 |
return true; |
| 1715 |
} |
| 1716 |
base = |
| 1717 |
bld.mkOp1v(OP_RDSV, TYPE_U32, bld.getScratch(), bld.mkSysVal(sv, 0)); |
| 1718 |
|
| 1719 |
atom->setSrc(0, cloneShallow(func, atom->getSrc(0))); |
| 1720 |
atom->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL; |
| 1721 |
if (ptr) |
| 1722 |
base = bld.mkOp2v(OP_ADD, TYPE_U32, base, base, ptr); |
| 1723 |
atom->setIndirect(0, 1, NULL); |
| 1724 |
atom->setIndirect(0, 0, base); |
| 1725 |
|
| 1726 |
return true; |
| 1727 |
} |
| 1728 |
|
| 1729 |
bool |
| 1730 |
NVC0LoweringPass::handleATOMCctl(Instruction *atom) { |
| 1731 |
// Flush L1 cache manually since atomics go directly to L2. This ensures |
| 1732 |
// that any later CA reads retrieve the updated data. |
| 1733 |
|
| 1734 |
if (atom->cache != nv50_ir::CACHE_CA) |
| 1735 |
return false; |
| 1736 |
|
| 1737 |
bld.setPosition(atom, true); |
| 1738 |
|
| 1739 |
Instruction *cctl = bld.mkOp1(OP_CCTL, TYPE_NONE, NULL, atom->getSrc(0)); |
| 1740 |
cctl->setIndirect(0, 0, atom->getIndirect(0, 0)); |
| 1741 |
cctl->fixed = 1; |
| 1742 |
cctl->subOp = NV50_IR_SUBOP_CCTL_IV; |
| 1743 |
if (atom->isPredicated()) |
| 1744 |
cctl->setPredicate(atom->cc, atom->getPredicate()); |
| 1745 |
|
| 1746 |
return true; |
| 1747 |
} |
| 1748 |
|
| 1749 |
bool |
| 1750 |
NVC0LoweringPass::handleCasExch(Instruction *cas) |
| 1751 |
{ |
| 1752 |
if (targ->getChipset() < NVISA_GM107_CHIPSET) { |
| 1753 |
if (cas->src(0).getFile() == FILE_MEMORY_SHARED) { |
| 1754 |
// ATOM_CAS and ATOM_EXCH are handled in handleSharedATOM(). |
| 1755 |
return false; |
| 1756 |
} |
| 1757 |
} |
| 1758 |
|
| 1759 |
if (cas->subOp != NV50_IR_SUBOP_ATOM_CAS && |
| 1760 |
cas->subOp != NV50_IR_SUBOP_ATOM_EXCH) |
| 1761 |
return false; |
| 1762 |
|
| 1763 |
if (cas->subOp == NV50_IR_SUBOP_ATOM_CAS && |
| 1764 |
targ->getChipset() < NVISA_GV100_CHIPSET) { |
| 1765 |
// CAS is crazy. It's 2nd source is a double reg, and the 3rd source |
| 1766 |
// should be set to the high part of the double reg or bad things will |
| 1767 |
// happen elsewhere in the universe. |
| 1768 |
// Also, it sometimes returns the new value instead of the old one |
| 1769 |
// under mysterious circumstances. |
| 1770 |
DataType ty = typeOfSize(typeSizeof(cas->dType) * 2); |
| 1771 |
Value *dreg = bld.getSSA(typeSizeof(ty)); |
| 1772 |
bld.setPosition(cas, false); |
| 1773 |
bld.mkOp2(OP_MERGE, ty, dreg, cas->getSrc(1), cas->getSrc(2)); |
| 1774 |
cas->setSrc(1, dreg); |
| 1775 |
cas->setSrc(2, dreg); |
| 1776 |
} |
| 1777 |
|
| 1778 |
return true; |
| 1779 |
} |
| 1780 |
|
| 1781 |
inline Value * |
| 1782 |
NVC0LoweringPass::loadResInfo32(Value *ptr, uint32_t off, uint16_t base) |
| 1783 |
{ |
| 1784 |
uint8_t b = prog->driver->io.auxCBSlot; |
| 1785 |
off += base; |
| 1786 |
|
| 1787 |
return bld. |
| 1788 |
mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr); |
| 1789 |
} |
| 1790 |
|
| 1791 |
inline Value * |
| 1792 |
NVC0LoweringPass::loadResInfo64(Value *ptr, uint32_t off, uint16_t base) |
| 1793 |
{ |
| 1794 |
uint8_t b = prog->driver->io.auxCBSlot; |
| 1795 |
off += base; |
| 1796 |
|
| 1797 |
if (ptr) |
| 1798 |
ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getScratch(), ptr, bld.mkImm(4)); |
| 1799 |
|
| 1800 |
return bld. |
| 1801 |
mkLoadv(TYPE_U64, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U64, off), ptr); |
| 1802 |
} |
| 1803 |
|
| 1804 |
inline Value * |
| 1805 |
NVC0LoweringPass::loadResLength32(Value *ptr, uint32_t off, uint16_t base) |
| 1806 |
{ |
| 1807 |
uint8_t b = prog->driver->io.auxCBSlot; |
| 1808 |
off += base; |
| 1809 |
|
| 1810 |
if (ptr) |
| 1811 |
ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getScratch(), ptr, bld.mkImm(4)); |
| 1812 |
|
| 1813 |
return bld. |
| 1814 |
mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U64, off + 8), ptr); |
| 1815 |
} |
| 1816 |
|
| 1817 |
inline Value * |
| 1818 |
NVC0LoweringPass::loadBufInfo64(Value *ptr, uint32_t off) |
| 1819 |
{ |
| 1820 |
return loadResInfo64(ptr, off, prog->driver->io.bufInfoBase); |
| 1821 |
} |
| 1822 |
|
| 1823 |
inline Value * |
| 1824 |
NVC0LoweringPass::loadBufLength32(Value *ptr, uint32_t off) |
| 1825 |
{ |
| 1826 |
return loadResLength32(ptr, off, prog->driver->io.bufInfoBase); |
| 1827 |
} |
| 1828 |
|
| 1829 |
inline Value * |
| 1830 |
NVC0LoweringPass::loadUboInfo64(Value *ptr, uint32_t off) |
| 1831 |
{ |
| 1832 |
return loadResInfo64(ptr, off, prog->driver->io.uboInfoBase); |
| 1833 |
} |
| 1834 |
|
| 1835 |
inline Value * |
| 1836 |
NVC0LoweringPass::loadUboLength32(Value *ptr, uint32_t off) |
| 1837 |
{ |
| 1838 |
return loadResLength32(ptr, off, prog->driver->io.uboInfoBase); |
| 1839 |
} |
| 1840 |
|
| 1841 |
inline Value * |
| 1842 |
NVC0LoweringPass::loadMsInfo32(Value *ptr, uint32_t off) |
| 1843 |
{ |
| 1844 |
uint8_t b = prog->driver->io.msInfoCBSlot; |
| 1845 |
off += prog->driver->io.msInfoBase; |
| 1846 |
return bld. |
| 1847 |
mkLoadv(TYPE_U32, bld.mkSymbol(FILE_MEMORY_CONST, b, TYPE_U32, off), ptr); |
| 1848 |
} |
| 1849 |
|
| 1850 |
inline Value * |
| 1851 |
NVC0LoweringPass::loadSuInfo32(Value *ptr, int slot, uint32_t off, bool bindless) |
| 1852 |
{ |
| 1853 |
uint32_t base = slot * NVC0_SU_INFO__STRIDE; |
| 1854 |
|
| 1855 |
// We don't upload surface info for bindless for GM107+ |
| 1856 |
assert(!bindless || targ->getChipset() < NVISA_GM107_CHIPSET); |
| 1857 |
|
| 1858 |
if (ptr) { |
| 1859 |
ptr = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(slot)); |
| 1860 |
if (bindless) |
| 1861 |
ptr = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(511)); |
| 1862 |
else |
| 1863 |
ptr = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(7)); |
| 1864 |
ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(6)); |
| 1865 |
base = 0; |
| 1866 |
} |
| 1867 |
off += base; |
| 1868 |
|
| 1869 |
return loadResInfo32(ptr, off, bindless ? prog->driver->io.bindlessBase : |
| 1870 |
prog->driver->io.suInfoBase); |
| 1871 |
} |
| 1872 |
|
| 1873 |
Value * |
| 1874 |
NVC0LoweringPass::loadMsAdjInfo32(TexInstruction::Target target, uint32_t index, int slot, Value *ind, bool bindless) |
| 1875 |
{ |
| 1876 |
if (!bindless || targ->getChipset() < NVISA_GM107_CHIPSET) |
| 1877 |
return loadSuInfo32(ind, slot, NVC0_SU_INFO_MS(index), bindless); |
| 1878 |
|
| 1879 |
assert(bindless); |
| 1880 |
|
| 1881 |
Value *samples = bld.getSSA(); |
| 1882 |
// this shouldn't be lowered because it's being inserted before the current instruction |
| 1883 |
TexInstruction *tex = new_TexInstruction(func, OP_TXQ); |
| 1884 |
tex->tex.target = target; |
| 1885 |
tex->tex.query = TXQ_TYPE; |
| 1886 |
tex->tex.mask = 0x4; |
| 1887 |
tex->tex.r = 0xff; |
| 1888 |
tex->tex.s = 0x1f; |
| 1889 |
tex->tex.rIndirectSrc = 0; |
| 1890 |
tex->setDef(0, samples); |
| 1891 |
tex->setSrc(0, ind); |
| 1892 |
tex->setSrc(1, bld.loadImm(NULL, 0)); |
| 1893 |
bld.insert(tex); |
| 1894 |
|
| 1895 |
// doesn't work with sample counts other than 1/2/4/8 but they aren't supported |
| 1896 |
switch (index) { |
| 1897 |
case 0: { |
| 1898 |
Value *tmp = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), samples, bld.mkImm(2)); |
| 1899 |
return bld.mkOp2v(OP_SHR, TYPE_U32, bld.getSSA(), tmp, bld.mkImm(2)); |
| 1900 |
} |
| 1901 |
case 1: { |
| 1902 |
Value *tmp = bld.mkCmp(OP_SET, CC_GT, TYPE_U32, bld.getSSA(), TYPE_U32, samples, bld.mkImm(2))->getDef(0); |
| 1903 |
return bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), tmp, bld.mkImm(1)); |
| 1904 |
} |
| 1905 |
default: { |
| 1906 |
assert(false); |
| 1907 |
return NULL; |
| 1908 |
} |
| 1909 |
} |
| 1910 |
} |
| 1911 |
|
| 1912 |
static inline uint16_t getSuClampSubOp(const TexInstruction *su, int c) |
| 1913 |
{ |
| 1914 |
switch (su->tex.target.getEnum()) { |
| 1915 |
case TEX_TARGET_BUFFER: return NV50_IR_SUBOP_SUCLAMP_PL(0, 1); |
| 1916 |
case TEX_TARGET_RECT: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2); |
| 1917 |
case TEX_TARGET_1D: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2); |
| 1918 |
case TEX_TARGET_1D_ARRAY: return (c == 1) ? |
| 1919 |
NV50_IR_SUBOP_SUCLAMP_PL(0, 2) : |
| 1920 |
NV50_IR_SUBOP_SUCLAMP_SD(0, 2); |
| 1921 |
case TEX_TARGET_2D: return NV50_IR_SUBOP_SUCLAMP_BL(0, 2); |
| 1922 |
case TEX_TARGET_2D_MS: return NV50_IR_SUBOP_SUCLAMP_BL(0, 2); |
| 1923 |
case TEX_TARGET_2D_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2); |
| 1924 |
case TEX_TARGET_2D_MS_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2); |
| 1925 |
case TEX_TARGET_3D: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2); |
| 1926 |
case TEX_TARGET_CUBE: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2); |
| 1927 |
case TEX_TARGET_CUBE_ARRAY: return NV50_IR_SUBOP_SUCLAMP_SD(0, 2); |
| 1928 |
default: |
| 1929 |
assert(0); |
| 1930 |
return 0; |
| 1931 |
} |
| 1932 |
} |
| 1933 |
|
| 1934 |
bool |
| 1935 |
NVC0LoweringPass::handleSUQ(TexInstruction *suq) |
| 1936 |
{ |
| 1937 |
int mask = suq->tex.mask; |
| 1938 |
int dim = suq->tex.target.getDim(); |
| 1939 |
int arg = dim + (suq->tex.target.isArray() || suq->tex.target.isCube()); |
| 1940 |
Value *ind = suq->getIndirectR(); |
| 1941 |
int slot = suq->tex.r; |
| 1942 |
int c, d; |
| 1943 |
|
| 1944 |
for (c = 0, d = 0; c < 3; ++c, mask >>= 1) { |
| 1945 |
if (c >= arg || !(mask & 1)) |
| 1946 |
continue; |
| 1947 |
|
| 1948 |
int offset; |
| 1949 |
|
| 1950 |
if (c == 1 && suq->tex.target == TEX_TARGET_1D_ARRAY) { |
| 1951 |
offset = NVC0_SU_INFO_SIZE(2); |
| 1952 |
} else { |
| 1953 |
offset = NVC0_SU_INFO_SIZE(c); |
| 1954 |
} |
| 1955 |
bld.mkMov(suq->getDef(d++), loadSuInfo32(ind, slot, offset, suq->tex.bindless)); |
| 1956 |
if (c == 2 && suq->tex.target.isCube()) |
| 1957 |
bld.mkOp2(OP_DIV, TYPE_U32, suq->getDef(d - 1), suq->getDef(d - 1), |
| 1958 |
bld.loadImm(NULL, 6)); |
| 1959 |
} |
| 1960 |
|
| 1961 |
if (mask & 1) { |
| 1962 |
if (suq->tex.target.isMS()) { |
| 1963 |
Value *ms_x = loadSuInfo32(ind, slot, NVC0_SU_INFO_MS(0), suq->tex.bindless); |
| 1964 |
Value *ms_y = loadSuInfo32(ind, slot, NVC0_SU_INFO_MS(1), suq->tex.bindless); |
| 1965 |
Value *ms = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getScratch(), ms_x, ms_y); |
| 1966 |
bld.mkOp2(OP_SHL, TYPE_U32, suq->getDef(d++), bld.loadImm(NULL, 1), ms); |
| 1967 |
} else { |
| 1968 |
bld.mkMov(suq->getDef(d++), bld.loadImm(NULL, 1)); |
| 1969 |
} |
| 1970 |
} |
| 1971 |
|
| 1972 |
bld.remove(suq); |
| 1973 |
return true; |
| 1974 |
} |
| 1975 |
|
| 1976 |
void |
| 1977 |
NVC0LoweringPass::adjustCoordinatesMS(TexInstruction *tex) |
| 1978 |
{ |
| 1979 |
const int arg = tex->tex.target.getArgCount(); |
| 1980 |
int slot = tex->tex.r; |
| 1981 |
|
| 1982 |
if (tex->tex.target == TEX_TARGET_2D_MS) |
| 1983 |
tex->tex.target = TEX_TARGET_2D; |
| 1984 |
else |
| 1985 |
if (tex->tex.target == TEX_TARGET_2D_MS_ARRAY) |
| 1986 |
tex->tex.target = TEX_TARGET_2D_ARRAY; |
| 1987 |
else |
| 1988 |
return; |
| 1989 |
|
| 1990 |
Value *x = tex->getSrc(0); |
| 1991 |
Value *y = tex->getSrc(1); |
| 1992 |
Value *s = tex->getSrc(arg - 1); |
| 1993 |
|
| 1994 |
Value *tx = bld.getSSA(), *ty = bld.getSSA(), *ts = bld.getSSA(); |
| 1995 |
Value *ind = tex->getIndirectR(); |
| 1996 |
|
| 1997 |
Value *ms_x = loadMsAdjInfo32(tex->tex.target, 0, slot, ind, tex->tex.bindless); |
| 1998 |
Value *ms_y = loadMsAdjInfo32(tex->tex.target, 1, slot, ind, tex->tex.bindless); |
| 1999 |
|
| 2000 |
bld.mkOp2(OP_SHL, TYPE_U32, tx, x, ms_x); |
| 2001 |
bld.mkOp2(OP_SHL, TYPE_U32, ty, y, ms_y); |
| 2002 |
|
| 2003 |
s = bld.mkOp2v(OP_AND, TYPE_U32, ts, s, bld.loadImm(NULL, 0x7)); |
| 2004 |
s = bld.mkOp2v(OP_SHL, TYPE_U32, ts, ts, bld.mkImm(3)); |
| 2005 |
|
| 2006 |
Value *dx = loadMsInfo32(ts, 0x0); |
| 2007 |
Value *dy = loadMsInfo32(ts, 0x4); |
| 2008 |
|
| 2009 |
bld.mkOp2(OP_ADD, TYPE_U32, tx, tx, dx); |
| 2010 |
bld.mkOp2(OP_ADD, TYPE_U32, ty, ty, dy); |
| 2011 |
|
| 2012 |
tex->setSrc(0, tx); |
| 2013 |
tex->setSrc(1, ty); |
| 2014 |
tex->moveSources(arg, -1); |
| 2015 |
} |
| 2016 |
|
| 2017 |
// Sets 64-bit "generic address", predicate and format sources for SULD/SUST. |
| 2018 |
// They're computed from the coordinates using the surface info in c[] space. |
| 2019 |
void |
| 2020 |
NVC0LoweringPass::processSurfaceCoordsNVE4(TexInstruction *su) |
| 2021 |
{ |
| 2022 |
Instruction *insn; |
| 2023 |
const bool atom = su->op == OP_SUREDB || su->op == OP_SUREDP; |
| 2024 |
const bool raw = |
| 2025 |
su->op == OP_SULDB || su->op == OP_SUSTB || su->op == OP_SUREDB; |
| 2026 |
const int slot = su->tex.r; |
| 2027 |
const int dim = su->tex.target.getDim(); |
| 2028 |
const bool array = su->tex.target.isArray() || su->tex.target.isCube(); |
| 2029 |
const int arg = dim + array; |
| 2030 |
int c; |
| 2031 |
Value *zero = bld.mkImm(0); |
| 2032 |
Value *p1 = NULL; |
| 2033 |
Value *v; |
| 2034 |
Value *src[3]; |
| 2035 |
Value *bf, *eau, *off; |
| 2036 |
Value *addr, *pred; |
| 2037 |
Value *ind = su->getIndirectR(); |
| 2038 |
Value *y, *z; |
| 2039 |
|
| 2040 |
off = bld.getScratch(4); |
| 2041 |
bf = bld.getScratch(4); |
| 2042 |
addr = bld.getSSA(8); |
| 2043 |
pred = bld.getScratch(1, FILE_PREDICATE); |
| 2044 |
|
| 2045 |
bld.setPosition(su, false); |
| 2046 |
|
| 2047 |
adjustCoordinatesMS(su); |
| 2048 |
|
| 2049 |
// calculate clamped coordinates |
| 2050 |
for (c = 0; c < arg; ++c) { |
| 2051 |
int dimc = c; |
| 2052 |
|
| 2053 |
if (c == 1 && su->tex.target == TEX_TARGET_1D_ARRAY) { |
| 2054 |
// The array index is stored in the Z component for 1D arrays. |
| 2055 |
dimc = 2; |
| 2056 |
} |
| 2057 |
|
| 2058 |
src[c] = bld.getScratch(); |
| 2059 |
if (c == 0 && raw) |
| 2060 |
v = loadSuInfo32(ind, slot, NVC0_SU_INFO_RAW_X, su->tex.bindless); |
| 2061 |
else |
| 2062 |
v = loadSuInfo32(ind, slot, NVC0_SU_INFO_DIM(dimc), su->tex.bindless); |
| 2063 |
bld.mkOp3(OP_SUCLAMP, TYPE_S32, src[c], su->getSrc(c), v, zero) |
| 2064 |
->subOp = getSuClampSubOp(su, dimc); |
| 2065 |
} |
| 2066 |
for (; c < 3; ++c) |
| 2067 |
src[c] = zero; |
| 2068 |
|
| 2069 |
if (dim == 2 && !array) { |
| 2070 |
v = loadSuInfo32(ind, slot, NVC0_SU_INFO_UNK1C, su->tex.bindless); |
| 2071 |
src[2] = bld.mkOp2v(OP_SHR, TYPE_U32, bld.getSSA(), |
| 2072 |
v, bld.loadImm(NULL, 16)); |
| 2073 |
|
| 2074 |
v = loadSuInfo32(ind, slot, NVC0_SU_INFO_DIM(2), su->tex.bindless); |
| 2075 |
bld.mkOp3(OP_SUCLAMP, TYPE_S32, src[2], src[2], v, zero) |
| 2076 |
->subOp = NV50_IR_SUBOP_SUCLAMP_SD(0, 2); |
| 2077 |
} |
| 2078 |
|
| 2079 |
// set predicate output |
| 2080 |
if (su->tex.target == TEX_TARGET_BUFFER) { |
| 2081 |
src[0]->getInsn()->setFlagsDef(1, pred); |
| 2082 |
} else |
| 2083 |
if (array) { |
| 2084 |
p1 = bld.getSSA(1, FILE_PREDICATE); |
| 2085 |
src[dim]->getInsn()->setFlagsDef(1, p1); |
| 2086 |
} |
| 2087 |
|
| 2088 |
// calculate pixel offset |
| 2089 |
if (dim == 1) { |
| 2090 |
y = z = zero; |
| 2091 |
if (su->tex.target != TEX_TARGET_BUFFER) |
| 2092 |
bld.mkOp2(OP_AND, TYPE_U32, off, src[0], bld.loadImm(NULL, 0xffff)); |
| 2093 |
} else { |
| 2094 |
y = src[1]; |
| 2095 |
z = src[2]; |
| 2096 |
|
| 2097 |
v = loadSuInfo32(ind, slot, NVC0_SU_INFO_UNK1C, su->tex.bindless); |
| 2098 |
bld.mkOp3(OP_MADSP, TYPE_U32, off, src[2], v, src[1]) |
| 2099 |
->subOp = NV50_IR_SUBOP_MADSP(4,4,8); // u16l u16l u16l |
| 2100 |
|
| 2101 |
v = loadSuInfo32(ind, slot, NVC0_SU_INFO_PITCH, su->tex.bindless); |
| 2102 |
bld.mkOp3(OP_MADSP, TYPE_U32, off, off, v, src[0]) |
| 2103 |
->subOp = array ? |
| 2104 |
NV50_IR_SUBOP_MADSP_SD : NV50_IR_SUBOP_MADSP(0,2,8); // u32 u16l u16l |
| 2105 |
} |
| 2106 |
|
| 2107 |
// calculate effective address part 1 |
| 2108 |
if (su->tex.target == TEX_TARGET_BUFFER) { |
| 2109 |
if (raw) { |
| 2110 |
bf = src[0]; |
| 2111 |
} else { |
| 2112 |
v = loadSuInfo32(ind, slot, NVC0_SU_INFO_FMT, su->tex.bindless); |
| 2113 |
bld.mkOp3(OP_VSHL, TYPE_U32, bf, src[0], v, zero) |
| 2114 |
->subOp = NV50_IR_SUBOP_V1(7,6,8|2); |
| 2115 |
} |
| 2116 |
} else { |
| 2117 |
uint16_t subOp = 0; |
| 2118 |
|
| 2119 |
switch (dim) { |
| 2120 |
case 1: |
| 2121 |
break; |
| 2122 |
case 2: |
| 2123 |
if (array) { |
| 2124 |
z = off; |
| 2125 |
} else { |
| 2126 |
subOp = NV50_IR_SUBOP_SUBFM_3D; |
| 2127 |
} |
| 2128 |
break; |
| 2129 |
default: |
| 2130 |
subOp = NV50_IR_SUBOP_SUBFM_3D; |
| 2131 |
assert(dim == 3); |
| 2132 |
break; |
| 2133 |
} |
| 2134 |
insn = bld.mkOp3(OP_SUBFM, TYPE_U32, bf, src[0], y, z); |
| 2135 |
insn->subOp = subOp; |
| 2136 |
insn->setFlagsDef(1, pred); |
| 2137 |
} |
| 2138 |
|
| 2139 |
// part 2 |
| 2140 |
v = loadSuInfo32(ind, slot, NVC0_SU_INFO_ADDR, su->tex.bindless); |
| 2141 |
|
| 2142 |
if (su->tex.target == TEX_TARGET_BUFFER) { |
| 2143 |
eau = v; |
| 2144 |
} else { |
| 2145 |
eau = bld.mkOp3v(OP_SUEAU, TYPE_U32, bld.getScratch(4), off, bf, v); |
| 2146 |
} |
| 2147 |
// add array layer offset |
| 2148 |
if (array) { |
| 2149 |
v = loadSuInfo32(ind, slot, NVC0_SU_INFO_ARRAY, su->tex.bindless); |
| 2150 |
if (dim == 1) |
| 2151 |
bld.mkOp3(OP_MADSP, TYPE_U32, eau, src[1], v, eau) |
| 2152 |
->subOp = NV50_IR_SUBOP_MADSP(4,0,0); // u16 u24 u32 |
| 2153 |
else |
| 2154 |
bld.mkOp3(OP_MADSP, TYPE_U32, eau, v, src[2], eau) |
| 2155 |
->subOp = NV50_IR_SUBOP_MADSP(0,0,0); // u32 u24 u32 |
| 2156 |
// combine predicates |
| 2157 |
assert(p1); |
| 2158 |
bld.mkOp2(OP_OR, TYPE_U8, pred, pred, p1); |
| 2159 |
} |
| 2160 |
|
| 2161 |
if (atom) { |
| 2162 |
Value *lo = bf; |
| 2163 |
if (su->tex.target == TEX_TARGET_BUFFER) { |
| 2164 |
lo = zero; |
| 2165 |
bld.mkMov(off, bf); |
| 2166 |
} |
| 2167 |
// bf == g[] address & 0xff |
| 2168 |
// eau == g[] address >> 8 |
| 2169 |
bld.mkOp3(OP_PERMT, TYPE_U32, bf, lo, bld.loadImm(NULL, 0x6540), eau); |
| 2170 |
bld.mkOp3(OP_PERMT, TYPE_U32, eau, zero, bld.loadImm(NULL, 0x0007), eau); |
| 2171 |
} else |
| 2172 |
if (su->op == OP_SULDP && su->tex.target == TEX_TARGET_BUFFER) { |
| 2173 |
// Convert from u32 to u8 address format, which is what the library code |
| 2174 |
// doing SULDP currently uses. |
| 2175 |
// XXX: can SUEAU do this ? |
| 2176 |
// XXX: does it matter that we don't mask high bytes in bf ? |
| 2177 |
// Grrr. |
| 2178 |
bld.mkOp2(OP_SHR, TYPE_U32, off, bf, bld.mkImm(8)); |
| 2179 |
bld.mkOp2(OP_ADD, TYPE_U32, eau, eau, off); |
| 2180 |
} |
| 2181 |
|
| 2182 |
bld.mkOp2(OP_MERGE, TYPE_U64, addr, bf, eau); |
| 2183 |
|
| 2184 |
if (atom && su->tex.target == TEX_TARGET_BUFFER) |
| 2185 |
bld.mkOp2(OP_ADD, TYPE_U64, addr, addr, off); |
| 2186 |
|
| 2187 |
// let's just set it 0 for raw access and hope it works |
| 2188 |
v = raw ? |
| 2189 |
bld.mkImm(0) : loadSuInfo32(ind, slot, NVC0_SU_INFO_FMT, su->tex.bindless); |
| 2190 |
|
| 2191 |
// get rid of old coordinate sources, make space for fmt info and predicate |
| 2192 |
su->moveSources(arg, 3 - arg); |
| 2193 |
// set 64 bit address and 32-bit format sources |
| 2194 |
su->setSrc(0, addr); |
| 2195 |
su->setSrc(1, v); |
| 2196 |
su->setSrc(2, pred); |
| 2197 |
su->setIndirectR(NULL); |
| 2198 |
|
| 2199 |
// prevent read fault when the image is not actually bound |
| 2200 |
CmpInstruction *pred1 = |
| 2201 |
bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE), |
| 2202 |
TYPE_U32, bld.mkImm(0), |
| 2203 |
loadSuInfo32(ind, slot, NVC0_SU_INFO_ADDR, su->tex.bindless)); |
| 2204 |
|
| 2205 |
if (su->op != OP_SUSTP && su->tex.format) { |
| 2206 |
const TexInstruction::ImgFormatDesc *format = su->tex.format; |
| 2207 |
int blockwidth = format->bits[0] + format->bits[1] + |
| 2208 |
format->bits[2] + format->bits[3]; |
| 2209 |
|
| 2210 |
// make sure that the format doesn't mismatch |
| 2211 |
assert(format->components != 0); |
| 2212 |
bld.mkCmp(OP_SET_OR, CC_NE, TYPE_U32, pred1->getDef(0), |
| 2213 |
TYPE_U32, bld.loadImm(NULL, blockwidth / 8), |
| 2214 |
loadSuInfo32(ind, slot, NVC0_SU_INFO_BSIZE, su->tex.bindless), |
| 2215 |
pred1->getDef(0)); |
| 2216 |
} |
| 2217 |
su->setPredicate(CC_NOT_P, pred1->getDef(0)); |
| 2218 |
|
| 2219 |
// TODO: initialize def values to 0 when the surface operation is not |
| 2220 |
// performed (not needed for stores). Also, fix the "address bounds test" |
| 2221 |
// subtests from arb_shader_image_load_store-invalid for buffers, because it |
| 2222 |
// seems like that the predicate is not correctly set by suclamp. |
| 2223 |
} |
| 2224 |
|
| 2225 |
static DataType |
| 2226 |
getSrcType(const TexInstruction::ImgFormatDesc *t, int c) |
| 2227 |
{ |
| 2228 |
switch (t->type) { |
| 2229 |
case FLOAT: return t->bits[c] == 16 ? TYPE_F16 : TYPE_F32; |
| 2230 |
case UNORM: return t->bits[c] == 8 ? TYPE_U8 : TYPE_U16; |
| 2231 |
case SNORM: return t->bits[c] == 8 ? TYPE_S8 : TYPE_S16; |
| 2232 |
case UINT: |
| 2233 |
return (t->bits[c] == 8 ? TYPE_U8 : |
| 2234 |
(t->bits[c] == 16 ? TYPE_U16 : TYPE_U32)); |
| 2235 |
case SINT: |
| 2236 |
return (t->bits[c] == 8 ? TYPE_S8 : |
| 2237 |
(t->bits[c] == 16 ? TYPE_S16 : TYPE_S32)); |
| 2238 |
} |
| 2239 |
return TYPE_NONE; |
| 2240 |
} |
| 2241 |
|
| 2242 |
static DataType |
| 2243 |
getDestType(const ImgType type) { |
| 2244 |
switch (type) { |
| 2245 |
case FLOAT: |
| 2246 |
case UNORM: |
| 2247 |
case SNORM: |
| 2248 |
return TYPE_F32; |
| 2249 |
case UINT: |
| 2250 |
return TYPE_U32; |
| 2251 |
case SINT: |
| 2252 |
return TYPE_S32; |
| 2253 |
default: |
| 2254 |
assert(!"Impossible type"); |
| 2255 |
return TYPE_NONE; |
| 2256 |
} |
| 2257 |
} |
| 2258 |
|
| 2259 |
void |
| 2260 |
NVC0LoweringPass::convertSurfaceFormat(TexInstruction *su, Instruction **loaded) |
| 2261 |
{ |
| 2262 |
const TexInstruction::ImgFormatDesc *format = su->tex.format; |
| 2263 |
int width = format->bits[0] + format->bits[1] + |
| 2264 |
format->bits[2] + format->bits[3]; |
| 2265 |
Value *untypedDst[4] = {}; |
| 2266 |
Value *typedDst[4] = {}; |
| 2267 |
|
| 2268 |
// We must convert this to a generic load. |
| 2269 |
su->op = OP_SULDB; |
| 2270 |
|
| 2271 |
su->dType = typeOfSize(width / 8); |
| 2272 |
su->sType = TYPE_U8; |
| 2273 |
|
| 2274 |
for (int i = 0; i < width / 32; i++) |
| 2275 |
untypedDst[i] = bld.getSSA(); |
| 2276 |
if (width < 32) |
| 2277 |
untypedDst[0] = bld.getSSA(); |
| 2278 |
|
| 2279 |
if (loaded && loaded[0]) { |
| 2280 |
for (int i = 0; i < 4; i++) { |
| 2281 |
if (loaded[i]) |
| 2282 |
typedDst[i] = loaded[i]->getDef(0); |
| 2283 |
} |
| 2284 |
} else { |
| 2285 |
for (int i = 0; i < 4; i++) { |
| 2286 |
typedDst[i] = su->getDef(i); |
| 2287 |
} |
| 2288 |
} |
| 2289 |
|
| 2290 |
// Set the untyped dsts as the su's destinations |
| 2291 |
if (loaded && loaded[0]) { |
| 2292 |
for (int i = 0; i < 4; i++) |
| 2293 |
if (loaded[i]) |
| 2294 |
loaded[i]->setDef(0, untypedDst[i]); |
| 2295 |
} else { |
| 2296 |
for (int i = 0; i < 4; i++) |
| 2297 |
su->setDef(i, untypedDst[i]); |
| 2298 |
|
| 2299 |
bld.setPosition(su, true); |
| 2300 |
} |
| 2301 |
|
| 2302 |
// Unpack each component into the typed dsts |
| 2303 |
int bits = 0; |
| 2304 |
for (int i = 0; i < 4; bits += format->bits[i], i++) { |
| 2305 |
if (!typedDst[i]) |
| 2306 |
continue; |
| 2307 |
|
| 2308 |
if (loaded && loaded[0]) |
| 2309 |
bld.setPosition(loaded[i], true); |
| 2310 |
|
| 2311 |
if (i >= format->components) { |
| 2312 |
if (format->type == FLOAT || |
| 2313 |
format->type == UNORM || |
| 2314 |
format->type == SNORM) |
| 2315 |
bld.loadImm(typedDst[i], i == 3 ? 1.0f : 0.0f); |
| 2316 |
else |
| 2317 |
bld.loadImm(typedDst[i], i == 3 ? 1 : 0); |
| 2318 |
continue; |
| 2319 |
} |
| 2320 |
|
| 2321 |
// Get just that component's data into the relevant place |
| 2322 |
if (format->bits[i] == 32) |
| 2323 |
bld.mkMov(typedDst[i], untypedDst[i]); |
| 2324 |
else if (format->bits[i] == 16) |
| 2325 |
bld.mkCvt(OP_CVT, getDestType(format->type), typedDst[i], |
| 2326 |
getSrcType(format, i), untypedDst[i / 2]) |
| 2327 |
->subOp = (i & 1) << (format->type == FLOAT ? 0 : 1); |
| 2328 |
else if (format->bits[i] == 8) |
| 2329 |
bld.mkCvt(OP_CVT, getDestType(format->type), typedDst[i], |
| 2330 |
getSrcType(format, i), untypedDst[0])->subOp = i; |
| 2331 |
else { |
| 2332 |
bld.mkOp2(OP_EXTBF, TYPE_U32, typedDst[i], untypedDst[bits / 32], |
| 2333 |
bld.mkImm((bits % 32) | (format->bits[i] << 8))); |
| 2334 |
if (format->type == UNORM || format->type == SNORM) |
| 2335 |
bld.mkCvt(OP_CVT, TYPE_F32, typedDst[i], getSrcType(format, i), typedDst[i]); |
| 2336 |
} |
| 2337 |
|
| 2338 |
// Normalize / convert as necessary |
| 2339 |
if (format->type == UNORM) |
| 2340 |
bld.mkOp2(OP_MUL, TYPE_F32, typedDst[i], typedDst[i], bld.loadImm(NULL, 1.0f / ((1 << format->bits[i]) - 1))); |
| 2341 |
else if (format->type == SNORM) |
| 2342 |
bld.mkOp2(OP_MUL, TYPE_F32, typedDst[i], typedDst[i], bld.loadImm(NULL, 1.0f / ((1 << (format->bits[i] - 1)) - 1))); |
| 2343 |
else if (format->type == FLOAT && format->bits[i] < 16) { |
| 2344 |
bld.mkOp2(OP_SHL, TYPE_U32, typedDst[i], typedDst[i], bld.loadImm(NULL, 15 - format->bits[i])); |
| 2345 |
bld.mkCvt(OP_CVT, TYPE_F32, typedDst[i], TYPE_F16, typedDst[i]); |
| 2346 |
} |
| 2347 |
} |
| 2348 |
|
| 2349 |
if (format->bgra) { |
| 2350 |
std::swap(typedDst[0], typedDst[2]); |
| 2351 |
} |
| 2352 |
} |
| 2353 |
|
| 2354 |
void |
| 2355 |
NVC0LoweringPass::insertOOBSurfaceOpResult(TexInstruction *su) |
| 2356 |
{ |
| 2357 |
if (!su->getPredicate()) |
| 2358 |
return; |
| 2359 |
|
| 2360 |
bld.setPosition(su, true); |
| 2361 |
|
| 2362 |
for (unsigned i = 0; su->defExists(i); ++i) { |
| 2363 |
Value *def = su->getDef(i); |
| 2364 |
Value *newDef = bld.getSSA(); |
| 2365 |
su->setDef(i, newDef); |
| 2366 |
|
| 2367 |
Instruction *mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0)); |
| 2368 |
assert(su->cc == CC_NOT_P); |
| 2369 |
mov->setPredicate(CC_P, su->getPredicate()); |
| 2370 |
Instruction *uni = bld.mkOp2(OP_UNION, TYPE_U32, bld.getSSA(), newDef, mov->getDef(0)); |
| 2371 |
bld.mkMov(def, uni->getDef(0)); |
| 2372 |
} |
| 2373 |
} |
| 2374 |
|
| 2375 |
void |
| 2376 |
NVC0LoweringPass::handleSurfaceOpNVE4(TexInstruction *su) |
| 2377 |
{ |
| 2378 |
processSurfaceCoordsNVE4(su); |
| 2379 |
|
| 2380 |
if (su->op == OP_SULDP && su->tex.format) { |
| 2381 |
convertSurfaceFormat(su, NULL); |
| 2382 |
insertOOBSurfaceOpResult(su); |
| 2383 |
} |
| 2384 |
|
| 2385 |
if (su->op == OP_SUREDB || su->op == OP_SUREDP) { |
| 2386 |
assert(su->getPredicate()); |
| 2387 |
Value *pred = |
| 2388 |
bld.mkOp2v(OP_OR, TYPE_U8, bld.getScratch(1, FILE_PREDICATE), |
| 2389 |
su->getPredicate(), su->getSrc(2)); |
| 2390 |
|
| 2391 |
Instruction *red = bld.mkOp(OP_ATOM, su->dType, bld.getSSA()); |
| 2392 |
red->subOp = su->subOp; |
| 2393 |
red->setSrc(0, bld.mkSymbol(FILE_MEMORY_GLOBAL, 0, TYPE_U32, 0)); |
| 2394 |
red->setSrc(1, su->getSrc(3)); |
| 2395 |
if (su->subOp == NV50_IR_SUBOP_ATOM_CAS) |
| 2396 |
red->setSrc(2, su->getSrc(4)); |
| 2397 |
red->setIndirect(0, 0, su->getSrc(0)); |
| 2398 |
|
| 2399 |
// make sure to initialize dst value when the atomic operation is not |
| 2400 |
// performed |
| 2401 |
Instruction *mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0)); |
| 2402 |
|
| 2403 |
assert(su->cc == CC_NOT_P); |
| 2404 |
red->setPredicate(su->cc, pred); |
| 2405 |
mov->setPredicate(CC_P, pred); |
| 2406 |
|
| 2407 |
bld.mkOp2(OP_UNION, TYPE_U32, su->getDef(0), |
| 2408 |
red->getDef(0), mov->getDef(0)); |
| 2409 |
|
| 2410 |
delete_Instruction(bld.getProgram(), su); |
| 2411 |
|
| 2412 |
handleATOMCctl(red); |
| 2413 |
handleCasExch(red); |
| 2414 |
} |
| 2415 |
|
| 2416 |
if (su->op == OP_SUSTB || su->op == OP_SUSTP) |
| 2417 |
su->sType = (su->tex.target == TEX_TARGET_BUFFER) ? TYPE_U32 : TYPE_U8; |
| 2418 |
} |
| 2419 |
|
| 2420 |
void |
| 2421 |
NVC0LoweringPass::processSurfaceCoordsNVC0(TexInstruction *su) |
| 2422 |
{ |
| 2423 |
const int slot = su->tex.r; |
| 2424 |
const int dim = su->tex.target.getDim(); |
| 2425 |
const int arg = dim + (su->tex.target.isArray() || su->tex.target.isCube()); |
| 2426 |
int c; |
| 2427 |
Value *zero = bld.mkImm(0); |
| 2428 |
Value *src[3]; |
| 2429 |
Value *v; |
| 2430 |
Value *ind = su->getIndirectR(); |
| 2431 |
|
| 2432 |
bld.setPosition(su, false); |
| 2433 |
|
| 2434 |
adjustCoordinatesMS(su); |
| 2435 |
|
| 2436 |
if (ind) { |
| 2437 |
Value *ptr; |
| 2438 |
ptr = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), ind, bld.mkImm(su->tex.r)); |
| 2439 |
ptr = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ptr, bld.mkImm(7)); |
| 2440 |
su->setIndirectR(ptr); |
| 2441 |
} |
| 2442 |
|
| 2443 |
// get surface coordinates |
| 2444 |
for (c = 0; c < arg; ++c) |
| 2445 |
src[c] = su->getSrc(c); |
| 2446 |
for (; c < 3; ++c) |
| 2447 |
src[c] = zero; |
| 2448 |
|
| 2449 |
// calculate pixel offset |
| 2450 |
if (su->op == OP_SULDP || su->op == OP_SUREDP) { |
| 2451 |
v = loadSuInfo32(ind, slot, NVC0_SU_INFO_BSIZE, su->tex.bindless); |
| 2452 |
su->setSrc(0, (src[0] = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), src[0], v))); |
| 2453 |
} |
| 2454 |
|
| 2455 |
// add array layer offset |
| 2456 |
if (su->tex.target.isArray() || su->tex.target.isCube()) { |
| 2457 |
v = loadSuInfo32(ind, slot, NVC0_SU_INFO_ARRAY, su->tex.bindless); |
| 2458 |
assert(dim > 1); |
| 2459 |
su->setSrc(2, (src[2] = bld.mkOp2v(OP_MUL, TYPE_U32, bld.getSSA(), src[2], v))); |
| 2460 |
} |
| 2461 |
|
| 2462 |
// 3d is special-cased. Note that a single "slice" of a 3d image may |
| 2463 |
// also be attached as 2d, so we have to do the same 3d processing for |
| 2464 |
// 2d as well, just in case. In order to remap a 3d image onto a 2d |
| 2465 |
// image, we have to retile it "by hand". |
| 2466 |
if (su->tex.target == TEX_TARGET_3D || su->tex.target == TEX_TARGET_2D) { |
| 2467 |
Value *z = loadSuInfo32(ind, slot, NVC0_SU_INFO_UNK1C, su->tex.bindless); |
| 2468 |
Value *y_size_aligned = |
| 2469 |
bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), |
| 2470 |
loadSuInfo32(ind, slot, NVC0_SU_INFO_DIM_Y, su->tex.bindless), |
| 2471 |
bld.loadImm(NULL, 0x0000ffff)); |
| 2472 |
// Add the z coordinate for actual 3d-images |
| 2473 |
if (dim > 2) |
| 2474 |
src[2] = bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), z, src[2]); |
| 2475 |
else |
| 2476 |
src[2] = z; |
| 2477 |
|
| 2478 |
// Compute the surface parameters from tile shifts |
| 2479 |
Value *tile_shift[3]; |
| 2480 |
Value *tile_extbf[3]; |
| 2481 |
// Fetch the "real" tiling parameters of the underlying surface |
| 2482 |
for (int i = 0; i < 3; i++) { |
| 2483 |
tile_extbf[i] = |
| 2484 |
bld.mkOp2v(OP_SHR, TYPE_U32, bld.getSSA(), |
| 2485 |
loadSuInfo32(ind, slot, NVC0_SU_INFO_DIM(i), su->tex.bindless), |
| 2486 |
bld.loadImm(NULL, 16)); |
| 2487 |
tile_shift[i] = |
| 2488 |
bld.mkOp2v(OP_SHR, TYPE_U32, bld.getSSA(), |
| 2489 |
loadSuInfo32(ind, slot, NVC0_SU_INFO_DIM(i), su->tex.bindless), |
| 2490 |
bld.loadImm(NULL, 24)); |
| 2491 |
} |
| 2492 |
|
| 2493 |
// However for load/atomics, we use byte-indexing. And for byte |
| 2494 |
// indexing, the X tile size is always the same. This leads to slightly |
| 2495 |
// better code. |
| 2496 |
if (su->op == OP_SULDP || su->op == OP_SUREDP) { |
| 2497 |
tile_extbf[0] = bld.loadImm(NULL, 0x600); |
| 2498 |
tile_shift[0] = bld.loadImm(NULL, 6); |
| 2499 |
} |
| 2500 |
|
| 2501 |
// Compute the location of given coordinate, both inside the tile as |
| 2502 |
// well as which (linearly-laid out) tile it's in. |
| 2503 |
Value *coord_in_tile[3]; |
| 2504 |
Value *tile[3]; |
| 2505 |
for (int i = 0; i < 3; i++) { |
| 2506 |
coord_in_tile[i] = bld.mkOp2v(OP_EXTBF, TYPE_U32, bld.getSSA(), src[i], tile_extbf[i]); |
| 2507 |
tile[i] = bld.mkOp2v(OP_SHR, TYPE_U32, bld.getSSA(), src[i], tile_shift[i]); |
| 2508 |
} |
| 2509 |
|
| 2510 |
// Based on the "real" tiling parameters, compute x/y coordinates in the |
| 2511 |
// larger surface with 2d tiling that was supplied to the hardware. This |
| 2512 |
// was determined and verified with the help of the tiling pseudocode in |
| 2513 |
// the envytools docs. |
| 2514 |
// |
| 2515 |
// adj_x = x_coord_in_tile + x_tile * x_tile_size * z_tile_size + |
| 2516 |
// z_coord_in_tile * x_tile_size |
| 2517 |
// adj_y = y_coord_in_tile + y_tile * y_tile_size + |
| 2518 |
// z_tile * y_tile_size * y_tiles |
| 2519 |
// |
| 2520 |
// Note: STRIDE_Y = y_tile_size * y_tiles |
| 2521 |
|
| 2522 |
su->setSrc(0, bld.mkOp2v( |
| 2523 |
OP_ADD, TYPE_U32, bld.getSSA(), |
| 2524 |
bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), |
| 2525 |
coord_in_tile[0], |
| 2526 |
bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), |
| 2527 |
tile[0], |
| 2528 |
bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), |
| 2529 |
tile_shift[2], tile_shift[0]))), |
| 2530 |
bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), |
| 2531 |
coord_in_tile[2], tile_shift[0]))); |
| 2532 |
|
| 2533 |
su->setSrc(1, bld.mkOp2v( |
| 2534 |
OP_ADD, TYPE_U32, bld.getSSA(), |
| 2535 |
bld.mkOp2v(OP_MUL, TYPE_U32, bld.getSSA(), |
| 2536 |
tile[2], y_size_aligned), |
| 2537 |
bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), |
| 2538 |
coord_in_tile[1], |
| 2539 |
bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), |
| 2540 |
tile[1], tile_shift[1])))); |
| 2541 |
|
| 2542 |
if (su->tex.target == TEX_TARGET_3D) { |
| 2543 |
su->moveSources(3, -1); |
| 2544 |
su->tex.target = TEX_TARGET_2D; |
| 2545 |
} |
| 2546 |
} |
| 2547 |
|
| 2548 |
// prevent read fault when the image is not actually bound |
| 2549 |
CmpInstruction *pred = |
| 2550 |
bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE), |
| 2551 |
TYPE_U32, bld.mkImm(0), |
| 2552 |
loadSuInfo32(ind, slot, NVC0_SU_INFO_ADDR, su->tex.bindless)); |
| 2553 |
if (su->op != OP_SUSTP && su->tex.format) { |
| 2554 |
const TexInstruction::ImgFormatDesc *format = su->tex.format; |
| 2555 |
int blockwidth = format->bits[0] + format->bits[1] + |
| 2556 |
format->bits[2] + format->bits[3]; |
| 2557 |
|
| 2558 |
assert(format->components != 0); |
| 2559 |
// make sure that the format doesn't mismatch when it's not FMT_NONE |
| 2560 |
bld.mkCmp(OP_SET_OR, CC_NE, TYPE_U32, pred->getDef(0), |
| 2561 |
TYPE_U32, bld.loadImm(NULL, ffs(blockwidth / 8) - 1), |
| 2562 |
loadSuInfo32(ind, slot, NVC0_SU_INFO_BSIZE, su->tex.bindless), |
| 2563 |
pred->getDef(0)); |
| 2564 |
} |
| 2565 |
su->setPredicate(CC_NOT_P, pred->getDef(0)); |
| 2566 |
} |
| 2567 |
|
| 2568 |
void |
| 2569 |
NVC0LoweringPass::handleSurfaceOpNVC0(TexInstruction *su) |
| 2570 |
{ |
| 2571 |
if (su->tex.target == TEX_TARGET_1D_ARRAY) { |
| 2572 |
/* As 1d arrays also need 3 coordinates, switching to TEX_TARGET_2D_ARRAY |
| 2573 |
* will simplify the lowering pass and the texture constraints. */ |
| 2574 |
su->moveSources(1, 1); |
| 2575 |
su->setSrc(1, bld.loadImm(NULL, 0)); |
| 2576 |
su->tex.target = TEX_TARGET_2D_ARRAY; |
| 2577 |
} |
| 2578 |
|
| 2579 |
processSurfaceCoordsNVC0(su); |
| 2580 |
|
| 2581 |
if (su->op == OP_SULDP && su->tex.format) { |
| 2582 |
convertSurfaceFormat(su, NULL); |
| 2583 |
insertOOBSurfaceOpResult(su); |
| 2584 |
} |
| 2585 |
|
| 2586 |
if (su->op == OP_SUREDB || su->op == OP_SUREDP) { |
| 2587 |
const int dim = su->tex.target.getDim(); |
| 2588 |
const int arg = dim + (su->tex.target.isArray() || su->tex.target.isCube()); |
| 2589 |
LValue *addr = bld.getSSA(8); |
| 2590 |
Value *def = su->getDef(0); |
| 2591 |
|
| 2592 |
su->op = OP_SULEA; |
| 2593 |
|
| 2594 |
// Set the destination to the address |
| 2595 |
su->dType = TYPE_U64; |
| 2596 |
su->setDef(0, addr); |
| 2597 |
su->setDef(1, su->getPredicate()); |
| 2598 |
|
| 2599 |
bld.setPosition(su, true); |
| 2600 |
|
| 2601 |
// Perform the atomic op |
| 2602 |
Instruction *red = bld.mkOp(OP_ATOM, su->sType, bld.getSSA()); |
| 2603 |
red->subOp = su->subOp; |
| 2604 |
red->setSrc(0, bld.mkSymbol(FILE_MEMORY_GLOBAL, 0, su->sType, 0)); |
| 2605 |
red->setSrc(1, su->getSrc(arg)); |
| 2606 |
if (red->subOp == NV50_IR_SUBOP_ATOM_CAS) |
| 2607 |
red->setSrc(2, su->getSrc(arg + 1)); |
| 2608 |
red->setIndirect(0, 0, addr); |
| 2609 |
|
| 2610 |
// make sure to initialize dst value when the atomic operation is not |
| 2611 |
// performed |
| 2612 |
Instruction *mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0)); |
| 2613 |
|
| 2614 |
assert(su->cc == CC_NOT_P); |
| 2615 |
red->setPredicate(su->cc, su->getPredicate()); |
| 2616 |
mov->setPredicate(CC_P, su->getPredicate()); |
| 2617 |
|
| 2618 |
bld.mkOp2(OP_UNION, TYPE_U32, def, red->getDef(0), mov->getDef(0)); |
| 2619 |
|
| 2620 |
handleCasExch(red); |
| 2621 |
} |
| 2622 |
} |
| 2623 |
|
| 2624 |
TexInstruction * |
| 2625 |
NVC0LoweringPass::processSurfaceCoordsGM107(TexInstruction *su, Instruction *ret[4]) |
| 2626 |
{ |
| 2627 |
const int slot = su->tex.r; |
| 2628 |
const int dim = su->tex.target.getDim(); |
| 2629 |
const bool array = su->tex.target.isArray() || su->tex.target.isCube(); |
| 2630 |
const int arg = dim + array; |
| 2631 |
Value *ind = su->getIndirectR(); |
| 2632 |
Value *handle; |
| 2633 |
Instruction *pred = NULL, *pred2d = NULL; |
| 2634 |
int pos = 0; |
| 2635 |
|
| 2636 |
bld.setPosition(su, false); |
| 2637 |
|
| 2638 |
adjustCoordinatesMS(su); |
| 2639 |
|
| 2640 |
// add texture handle |
| 2641 |
switch (su->op) { |
| 2642 |
case OP_SUSTP: |
| 2643 |
pos = 4; |
| 2644 |
break; |
| 2645 |
case OP_SUREDP: |
| 2646 |
pos = (su->subOp == NV50_IR_SUBOP_ATOM_CAS) ? 2 : 1; |
| 2647 |
break; |
| 2648 |
default: |
| 2649 |
assert(pos == 0); |
| 2650 |
break; |
| 2651 |
} |
| 2652 |
|
| 2653 |
if (dim == 2 && !array) { |
| 2654 |
// This might be a 2d slice of a 3d texture, try to load the z |
| 2655 |
// coordinate in. |
| 2656 |
Value *v; |
| 2657 |
if (!su->tex.bindless) |
| 2658 |
v = loadSuInfo32(ind, slot, NVC0_SU_INFO_UNK1C, su->tex.bindless); |
| 2659 |
else |
| 2660 |
v = bld.mkOp2v(OP_SHR, TYPE_U32, bld.getSSA(), ind, bld.mkImm(11)); |
| 2661 |
Value *is_3d = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), v, bld.mkImm(1)); |
| 2662 |
pred2d = bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE), |
| 2663 |
TYPE_U32, bld.mkImm(0), is_3d); |
| 2664 |
|
| 2665 |
bld.mkOp2(OP_SHR, TYPE_U32, v, v, bld.loadImm(NULL, 16)); |
| 2666 |
su->moveSources(dim, 1); |
| 2667 |
su->setSrc(dim, v); |
| 2668 |
su->tex.target = nv50_ir::TEX_TARGET_3D; |
| 2669 |
pos++; |
| 2670 |
} |
| 2671 |
|
| 2672 |
if (su->tex.bindless) |
| 2673 |
handle = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ind, bld.mkImm(2047)); |
| 2674 |
else |
| 2675 |
handle = loadTexHandle(ind, slot + 32); |
| 2676 |
|
| 2677 |
su->setSrc(arg + pos, handle); |
| 2678 |
|
| 2679 |
// The address check doesn't make sense here. The format check could make |
| 2680 |
// sense but it's a bit of a pain. |
| 2681 |
if (!su->tex.bindless) { |
| 2682 |
// prevent read fault when the image is not actually bound |
| 2683 |
pred = |
| 2684 |
bld.mkCmp(OP_SET, CC_EQ, TYPE_U32, bld.getSSA(1, FILE_PREDICATE), |
| 2685 |
TYPE_U32, bld.mkImm(0), |
| 2686 |
loadSuInfo32(ind, slot, NVC0_SU_INFO_ADDR, su->tex.bindless)); |
| 2687 |
if (su->op != OP_SUSTP && su->tex.format) { |
| 2688 |
const TexInstruction::ImgFormatDesc *format = su->tex.format; |
| 2689 |
int blockwidth = format->bits[0] + format->bits[1] + |
| 2690 |
format->bits[2] + format->bits[3]; |
| 2691 |
|
| 2692 |
assert(format->components != 0); |
| 2693 |
// make sure that the format doesn't mismatch when it's not FMT_NONE |
| 2694 |
bld.mkCmp(OP_SET_OR, CC_NE, TYPE_U32, pred->getDef(0), |
| 2695 |
TYPE_U32, bld.loadImm(NULL, blockwidth / 8), |
| 2696 |
loadSuInfo32(ind, slot, NVC0_SU_INFO_BSIZE, su->tex.bindless), |
| 2697 |
pred->getDef(0)); |
| 2698 |
} |
| 2699 |
} |
| 2700 |
|
| 2701 |
// Now we have "pred" which (optionally) contains whether to do the surface |
| 2702 |
// op at all, and a "pred2d" which indicates that, in case of doing the |
| 2703 |
// surface op, we have to create a 2d and 3d version, conditioned on pred2d. |
| 2704 |
TexInstruction *su2d = NULL; |
| 2705 |
if (pred2d) { |
| 2706 |
su2d = cloneForward(func, su)->asTex(); |
| 2707 |
for (unsigned i = 0; su->defExists(i); ++i) |
| 2708 |
su2d->setDef(i, bld.getSSA()); |
| 2709 |
su2d->moveSources(dim + 1, -1); |
| 2710 |
su2d->tex.target = nv50_ir::TEX_TARGET_2D; |
| 2711 |
} |
| 2712 |
if (pred2d && pred) { |
| 2713 |
Instruction *pred3d = bld.mkOp2(OP_AND, TYPE_U8, |
| 2714 |
bld.getSSA(1, FILE_PREDICATE), |
| 2715 |
pred->getDef(0), pred2d->getDef(0)); |
| 2716 |
pred3d->src(0).mod = Modifier(NV50_IR_MOD_NOT); |
| 2717 |
pred3d->src(1).mod = Modifier(NV50_IR_MOD_NOT); |
| 2718 |
su->setPredicate(CC_P, pred3d->getDef(0)); |
| 2719 |
pred2d = bld.mkOp2(OP_AND, TYPE_U8, bld.getSSA(1, FILE_PREDICATE), |
| 2720 |
pred->getDef(0), pred2d->getDef(0)); |
| 2721 |
pred2d->src(0).mod = Modifier(NV50_IR_MOD_NOT); |
| 2722 |
} else if (pred) { |
| 2723 |
su->setPredicate(CC_NOT_P, pred->getDef(0)); |
| 2724 |
} else if (pred2d) { |
| 2725 |
su->setPredicate(CC_NOT_P, pred2d->getDef(0)); |
| 2726 |
} |
| 2727 |
if (su2d) { |
| 2728 |
su2d->setPredicate(CC_P, pred2d->getDef(0)); |
| 2729 |
bld.insert(su2d); |
| 2730 |
|
| 2731 |
// Create a UNION so that RA assigns the same registers |
| 2732 |
bld.setPosition(su, true); |
| 2733 |
for (unsigned i = 0; su->defExists(i); ++i) { |
| 2734 |
assert(i < 4); |
| 2735 |
|
| 2736 |
Value *def = su->getDef(i); |
| 2737 |
Value *newDef = bld.getSSA(); |
| 2738 |
ValueDef &def2 = su2d->def(i); |
| 2739 |
Instruction *mov = NULL; |
| 2740 |
|
| 2741 |
su->setDef(i, newDef); |
| 2742 |
if (pred) { |
| 2743 |
mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0)); |
| 2744 |
mov->setPredicate(CC_P, pred->getDef(0)); |
| 2745 |
} |
| 2746 |
|
| 2747 |
Instruction *uni = ret[i] = bld.mkOp2(OP_UNION, TYPE_U32, |
| 2748 |
bld.getSSA(), |
| 2749 |
newDef, def2.get()); |
| 2750 |
if (mov) |
| 2751 |
uni->setSrc(2, mov->getDef(0)); |
| 2752 |
bld.mkMov(def, uni->getDef(0)); |
| 2753 |
} |
| 2754 |
} else if (pred) { |
| 2755 |
// Create a UNION so that RA assigns the same registers |
| 2756 |
bld.setPosition(su, true); |
| 2757 |
for (unsigned i = 0; su->defExists(i); ++i) { |
| 2758 |
assert(i < 4); |
| 2759 |
|
| 2760 |
Value *def = su->getDef(i); |
| 2761 |
Value *newDef = bld.getSSA(); |
| 2762 |
su->setDef(i, newDef); |
| 2763 |
|
| 2764 |
Instruction *mov = bld.mkMov(bld.getSSA(), bld.loadImm(NULL, 0)); |
| 2765 |
mov->setPredicate(CC_P, pred->getDef(0)); |
| 2766 |
|
| 2767 |
Instruction *uni = ret[i] = bld.mkOp2(OP_UNION, TYPE_U32, |
| 2768 |
bld.getSSA(), |
| 2769 |
newDef, mov->getDef(0)); |
| 2770 |
bld.mkMov(def, uni->getDef(0)); |
| 2771 |
} |
| 2772 |
} |
| 2773 |
|
| 2774 |
return su2d; |
| 2775 |
} |
| 2776 |
|
| 2777 |
void |
| 2778 |
NVC0LoweringPass::handleSurfaceOpGM107(TexInstruction *su) |
| 2779 |
{ |
| 2780 |
// processSurfaceCoords also takes care of fixing up the outputs and |
| 2781 |
// union'ing them with 0 as necessary. Additionally it may create a second |
| 2782 |
// surface which needs some of the similar fixups. |
| 2783 |
|
| 2784 |
Instruction *loaded[4] = {}; |
| 2785 |
TexInstruction *su2 = processSurfaceCoordsGM107(su, loaded); |
| 2786 |
|
| 2787 |
if (su->op == OP_SULDP && su->tex.format) { |
| 2788 |
convertSurfaceFormat(su, loaded); |
| 2789 |
} |
| 2790 |
|
| 2791 |
if (su->op == OP_SUREDP) { |
| 2792 |
su->op = OP_SUREDB; |
| 2793 |
} |
| 2794 |
|
| 2795 |
// If we fixed up the type of the regular surface load instruction, we also |
| 2796 |
// have to fix up the copy. |
| 2797 |
if (su2) { |
| 2798 |
su2->op = su->op; |
| 2799 |
su2->dType = su->dType; |
| 2800 |
su2->sType = su->sType; |
| 2801 |
} |
| 2802 |
} |
| 2803 |
|
| 2804 |
void |
| 2805 |
NVC0LoweringPass::handleLDST(Instruction *i) |
| 2806 |
{ |
| 2807 |
if (i->src(0).getFile() == FILE_SHADER_INPUT) { |
| 2808 |
if (prog->getType() == Program::TYPE_COMPUTE) { |
| 2809 |
i->getSrc(0)->reg.file = FILE_MEMORY_CONST; |
| 2810 |
i->getSrc(0)->reg.fileIndex = 0; |
| 2811 |
} else |
| 2812 |
if (prog->getType() == Program::TYPE_GEOMETRY && |
| 2813 |
i->src(0).isIndirect(0)) { |
| 2814 |
// XXX: this assumes vec4 units |
| 2815 |
Value *ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), |
| 2816 |
i->getIndirect(0, 0), bld.mkImm(4)); |
| 2817 |
i->setIndirect(0, 0, ptr); |
| 2818 |
i->op = OP_VFETCH; |
| 2819 |
} else { |
| 2820 |
i->op = OP_VFETCH; |
| 2821 |
assert(prog->getType() != Program::TYPE_FRAGMENT); // INTERP |
| 2822 |
} |
| 2823 |
} else if (i->src(0).getFile() == FILE_MEMORY_CONST) { |
| 2824 |
int8_t fileIndex = i->getSrc(0)->reg.fileIndex - 1; |
| 2825 |
Value *ind = i->getIndirect(0, 1); |
| 2826 |
|
| 2827 |
if (targ->getChipset() >= NVISA_GK104_CHIPSET && |
| 2828 |
prog->getType() == Program::TYPE_COMPUTE && |
| 2829 |
(fileIndex >= 6 || ind)) { |
| 2830 |
// The launch descriptor only allows to set up 8 CBs, but OpenGL |
| 2831 |
// requires at least 12 UBOs. To bypass this limitation, for constant |
| 2832 |
// buffers 7+, we store the addrs into the driver constbuf and we |
| 2833 |
// directly load from the global memory. |
| 2834 |
if (ind) { |
| 2835 |
// Clamp the UBO index when an indirect access is used to avoid |
| 2836 |
// loading information from the wrong place in the driver cb. |
| 2837 |
// TODO - synchronize the max with the driver. |
| 2838 |
ind = bld.mkOp2v(OP_MIN, TYPE_U32, bld.getSSA(), |
| 2839 |
bld.mkOp2v(OP_ADD, TYPE_U32, bld.getSSA(), |
| 2840 |
ind, bld.loadImm(NULL, fileIndex)), |
| 2841 |
bld.loadImm(NULL, 13)); |
| 2842 |
fileIndex = 0; |
| 2843 |
} |
| 2844 |
|
| 2845 |
Value *offset = bld.loadImm(NULL, i->getSrc(0)->reg.data.offset + typeSizeof(i->sType)); |
| 2846 |
Value *ptr = loadUboInfo64(ind, fileIndex * 16); |
| 2847 |
Value *length = loadUboLength32(ind, fileIndex * 16); |
| 2848 |
Value *pred = new_LValue(func, FILE_PREDICATE); |
| 2849 |
if (i->src(0).isIndirect(0)) { |
| 2850 |
bld.mkOp2(OP_ADD, TYPE_U64, ptr, ptr, i->getIndirect(0, 0)); |
| 2851 |
bld.mkOp2(OP_ADD, TYPE_U32, offset, offset, i->getIndirect(0, 0)); |
| 2852 |
} |
| 2853 |
i->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL; |
| 2854 |
i->setIndirect(0, 1, NULL); |
| 2855 |
i->setIndirect(0, 0, ptr); |
| 2856 |
bld.mkCmp(OP_SET, CC_GT, TYPE_U32, pred, TYPE_U32, offset, length); |
| 2857 |
i->setPredicate(CC_NOT_P, pred); |
| 2858 |
Value *zero, *dst = i->getDef(0); |
| 2859 |
i->setDef(0, bld.getSSA()); |
| 2860 |
|
| 2861 |
bld.setPosition(i, true); |
| 2862 |
bld.mkMov((zero = bld.getSSA()), bld.mkImm(0)) |
| 2863 |
->setPredicate(CC_P, pred); |
| 2864 |
bld.mkOp2(OP_UNION, TYPE_U32, dst, i->getDef(0), zero); |
| 2865 |
} else if (i->src(0).isIndirect(1)) { |
| 2866 |
Value *ptr; |
| 2867 |
if (i->src(0).isIndirect(0)) |
| 2868 |
ptr = bld.mkOp3v(OP_INSBF, TYPE_U32, bld.getSSA(), |
| 2869 |
i->getIndirect(0, 1), bld.mkImm(0x1010), |
| 2870 |
i->getIndirect(0, 0)); |
| 2871 |
else |
| 2872 |
ptr = bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), |
| 2873 |
i->getIndirect(0, 1), bld.mkImm(16)); |
| 2874 |
i->setIndirect(0, 1, NULL); |
| 2875 |
i->setIndirect(0, 0, ptr); |
| 2876 |
i->subOp = NV50_IR_SUBOP_LDC_IS; |
| 2877 |
} |
| 2878 |
} else if (i->src(0).getFile() == FILE_SHADER_OUTPUT) { |
| 2879 |
assert(prog->getType() == Program::TYPE_TESSELLATION_CONTROL); |
| 2880 |
i->op = OP_VFETCH; |
| 2881 |
} else if (i->src(0).getFile() == FILE_MEMORY_BUFFER) { |
| 2882 |
Value *ind = i->getIndirect(0, 1); |
| 2883 |
Value *ptr = loadBufInfo64(ind, i->getSrc(0)->reg.fileIndex * 16); |
| 2884 |
// XXX come up with a way not to do this for EVERY little access but |
| 2885 |
// rather to batch these up somehow. Unfortunately we've lost the |
| 2886 |
// information about the field width by the time we get here. |
| 2887 |
Value *offset = bld.loadImm(NULL, i->getSrc(0)->reg.data.offset + typeSizeof(i->sType)); |
| 2888 |
Value *length = loadBufLength32(ind, i->getSrc(0)->reg.fileIndex * 16); |
| 2889 |
Value *pred = new_LValue(func, FILE_PREDICATE); |
| 2890 |
if (i->src(0).isIndirect(0)) { |
| 2891 |
bld.mkOp2(OP_ADD, TYPE_U64, ptr, ptr, i->getIndirect(0, 0)); |
| 2892 |
bld.mkOp2(OP_ADD, TYPE_U32, offset, offset, i->getIndirect(0, 0)); |
| 2893 |
} |
| 2894 |
i->setIndirect(0, 1, NULL); |
| 2895 |
i->setIndirect(0, 0, ptr); |
| 2896 |
i->getSrc(0)->reg.file = FILE_MEMORY_GLOBAL; |
| 2897 |
bld.mkCmp(OP_SET, CC_GT, TYPE_U32, pred, TYPE_U32, offset, length); |
| 2898 |
i->setPredicate(CC_NOT_P, pred); |
| 2899 |
if (i->defExists(0)) { |
| 2900 |
Value *zero, *dst = i->getDef(0); |
| 2901 |
uint8_t size = dst->reg.size; |
| 2902 |
i->setDef(0, bld.getSSA(size)); |
| 2903 |
|
| 2904 |
bld.setPosition(i, true); |
| 2905 |
bld.mkMov((zero = bld.getSSA(size)), bld.mkImm(0), i->dType) |
| 2906 |
->setPredicate(CC_P, pred); |
| 2907 |
bld.mkOp2(OP_UNION, i->dType, dst, i->getDef(0), zero); |
| 2908 |
} |
| 2909 |
} |
| 2910 |
} |
| 2911 |
|
| 2912 |
void |
| 2913 |
NVC0LoweringPass::readTessCoord(LValue *dst, int c) |
| 2914 |
{ |
| 2915 |
// In case of SPIRV the domain can be specified in the tesc shader, |
| 2916 |
// but this should be passed to tese shader by merge_tess_info. |
| 2917 |
const uint8_t domain = prog->driver_out->prop.tp.domain; |
| 2918 |
assert( |
| 2919 |
domain == MESA_PRIM_LINES || |
| 2920 |
domain == MESA_PRIM_TRIANGLES || |
| 2921 |
domain == MESA_PRIM_QUADS); |
| 2922 |
|
| 2923 |
Value *laneid = bld.getSSA(); |
| 2924 |
Value *x, *y; |
| 2925 |
|
| 2926 |
bld.mkOp1(OP_RDSV, TYPE_U32, laneid, bld.mkSysVal(SV_LANEID, 0)); |
| 2927 |
|
| 2928 |
if (c == 0) { |
| 2929 |
x = dst; |
| 2930 |
y = NULL; |
| 2931 |
} else |
| 2932 |
if (c == 1) { |
| 2933 |
x = NULL; |
| 2934 |
y = dst; |
| 2935 |
} else { |
| 2936 |
assert(c == 2); |
| 2937 |
if (domain != MESA_PRIM_TRIANGLES) { |
| 2938 |
// optimize out tesscoord.z |
| 2939 |
bld.mkMov(dst, bld.loadImm(NULL, 0)); |
| 2940 |
return; |
| 2941 |
} |
| 2942 |
x = bld.getSSA(); |
| 2943 |
y = bld.getSSA(); |
| 2944 |
} |
| 2945 |
if (x) |
| 2946 |
bld.mkFetch(x, TYPE_F32, FILE_SHADER_OUTPUT, 0x2f0, NULL, laneid); |
| 2947 |
if (y) |
| 2948 |
bld.mkFetch(y, TYPE_F32, FILE_SHADER_OUTPUT, 0x2f4, NULL, laneid); |
| 2949 |
|
| 2950 |
if (c == 2) { |
| 2951 |
// compute tesscoord.z from x, y |
| 2952 |
bld.mkOp2(OP_ADD, TYPE_F32, dst, x, y); |
| 2953 |
bld.mkOp2(OP_SUB, TYPE_F32, dst, bld.loadImm(NULL, 1.0f), dst); |
| 2954 |
} |
| 2955 |
} |
| 2956 |
|
| 2957 |
bool |
| 2958 |
NVC0LoweringPass::handleRDSV(Instruction *i) |
| 2959 |
{ |
| 2960 |
Symbol *sym = i->getSrc(0)->asSym(); |
| 2961 |
const SVSemantic sv = sym->reg.data.sv.sv; |
| 2962 |
Value *vtx = NULL; |
| 2963 |
Instruction *ld; |
| 2964 |
uint32_t addr = targ->getSVAddress(FILE_SHADER_INPUT, sym); |
| 2965 |
|
| 2966 |
if (addr >= 0x400) { |
| 2967 |
// mov $sreg |
| 2968 |
if (sym->reg.data.sv.index == 3) { |
| 2969 |
// TGSI backend may use 4th component of TID,NTID,CTAID,NCTAID |
| 2970 |
i->op = OP_MOV; |
| 2971 |
i->setSrc(0, bld.mkImm((sv == SV_NTID || sv == SV_NCTAID) ? 1 : 0)); |
| 2972 |
} else |
| 2973 |
if (sv == SV_TID) { |
| 2974 |
// Help CSE combine TID fetches |
| 2975 |
Value *tid = bld.mkOp1v(OP_RDSV, TYPE_U32, bld.getScratch(), |
| 2976 |
bld.mkSysVal(SV_COMBINED_TID, 0)); |
| 2977 |
i->op = OP_EXTBF; |
| 2978 |
i->setSrc(0, tid); |
| 2979 |
switch (sym->reg.data.sv.index) { |
| 2980 |
case 0: i->setSrc(1, bld.mkImm(0x1000)); break; |
| 2981 |
case 1: i->setSrc(1, bld.mkImm(0x0a10)); break; |
| 2982 |
case 2: i->setSrc(1, bld.mkImm(0x061a)); break; |
| 2983 |
} |
| 2984 |
} |
| 2985 |
if (sv == SV_VERTEX_COUNT) { |
| 2986 |
bld.setPosition(i, true); |
| 2987 |
bld.mkOp2(OP_EXTBF, TYPE_U32, i->getDef(0), i->getDef(0), bld.mkImm(0x808)); |
| 2988 |
} |
| 2989 |
return true; |
| 2990 |
} |
| 2991 |
|
| 2992 |
switch (sv) { |
| 2993 |
case SV_POSITION: |
| 2994 |
assert(prog->getType() == Program::TYPE_FRAGMENT); |
| 2995 |
if (i->srcExists(1)) { |
| 2996 |
// Pass offset through to the interpolation logic |
| 2997 |
ld = bld.mkInterp(NV50_IR_INTERP_LINEAR | NV50_IR_INTERP_OFFSET, |
| 2998 |
i->getDef(0), addr, NULL); |
| 2999 |
ld->setSrc(1, i->getSrc(1)); |
| 3000 |
} else { |
| 3001 |
bld.mkInterp(NV50_IR_INTERP_LINEAR, i->getDef(0), addr, NULL); |
| 3002 |
} |
| 3003 |
break; |
| 3004 |
case SV_FACE: |
| 3005 |
{ |
| 3006 |
Value *face = i->getDef(0); |
| 3007 |
bld.mkInterp(NV50_IR_INTERP_FLAT, face, addr, NULL); |
| 3008 |
if (i->dType == TYPE_F32) { |
| 3009 |
bld.mkOp2(OP_OR, TYPE_U32, face, face, bld.mkImm(0x00000001)); |
| 3010 |
bld.mkOp1(OP_NEG, TYPE_S32, face, face); |
| 3011 |
bld.mkCvt(OP_CVT, TYPE_F32, face, TYPE_S32, face); |
| 3012 |
} |
| 3013 |
} |
| 3014 |
break; |
| 3015 |
case SV_TESS_COORD: |
| 3016 |
assert(prog->getType() == Program::TYPE_TESSELLATION_EVAL); |
| 3017 |
readTessCoord(i->getDef(0)->asLValue(), i->getSrc(0)->reg.data.sv.index); |
| 3018 |
break; |
| 3019 |
case SV_NTID: |
| 3020 |
case SV_NCTAID: |
| 3021 |
case SV_GRIDID: |
| 3022 |
assert(targ->getChipset() >= NVISA_GK104_CHIPSET); // mov $sreg otherwise |
| 3023 |
if (sym->reg.data.sv.index == 3) { |
| 3024 |
i->op = OP_MOV; |
| 3025 |
i->setSrc(0, bld.mkImm(sv == SV_GRIDID ? 0 : 1)); |
| 3026 |
return true; |
| 3027 |
} |
| 3028 |
FALLTHROUGH; |
| 3029 |
case SV_WORK_DIM: |
| 3030 |
addr += prog->driver->prop.cp.gridInfoBase; |
| 3031 |
bld.mkLoad(TYPE_U32, i->getDef(0), |
| 3032 |
bld.mkSymbol(FILE_MEMORY_CONST, prog->driver->io.auxCBSlot, |
| 3033 |
TYPE_U32, addr), NULL); |
| 3034 |
break; |
| 3035 |
case SV_SAMPLE_INDEX: |
| 3036 |
// TODO: Properly pass source as an address in the PIX address space |
| 3037 |
// (which can be of the form [r0+offset]). But this is currently |
| 3038 |
// unnecessary. |
| 3039 |
ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0)); |
| 3040 |
ld->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID; |
| 3041 |
break; |
| 3042 |
case SV_SAMPLE_POS: { |
| 3043 |
Value *sampleID = bld.getScratch(); |
| 3044 |
ld = bld.mkOp1(OP_PIXLD, TYPE_U32, sampleID, bld.mkImm(0)); |
| 3045 |
ld->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID; |
| 3046 |
Value *offset = calculateSampleOffset(sampleID); |
| 3047 |
|
| 3048 |
assert(prog->driver_out->prop.fp.readsSampleLocations); |
| 3049 |
|
| 3050 |
if (targ->getChipset() >= NVISA_GM200_CHIPSET) { |
| 3051 |
bld.mkLoad(TYPE_F32, |
| 3052 |
i->getDef(0), |
| 3053 |
bld.mkSymbol( |
| 3054 |
FILE_MEMORY_CONST, prog->driver->io.auxCBSlot, |
| 3055 |
TYPE_U32, prog->driver->io.sampleInfoBase), |
| 3056 |
offset); |
| 3057 |
bld.mkOp2(OP_EXTBF, TYPE_U32, i->getDef(0), i->getDef(0), |
| 3058 |
bld.mkImm(0x040c + sym->reg.data.sv.index * 16)); |
| 3059 |
bld.mkCvt(OP_CVT, TYPE_F32, i->getDef(0), TYPE_U32, i->getDef(0)); |
| 3060 |
bld.mkOp2(OP_MUL, TYPE_F32, i->getDef(0), i->getDef(0), bld.mkImm(1.0f / 16.0f)); |
| 3061 |
} else { |
| 3062 |
bld.mkLoad(TYPE_F32, |
| 3063 |
i->getDef(0), |
| 3064 |
bld.mkSymbol( |
| 3065 |
FILE_MEMORY_CONST, prog->driver->io.auxCBSlot, |
| 3066 |
TYPE_U32, prog->driver->io.sampleInfoBase + |
| 3067 |
4 * sym->reg.data.sv.index), |
| 3068 |
offset); |
| 3069 |
} |
| 3070 |
break; |
| 3071 |
} |
| 3072 |
case SV_SAMPLE_MASK: { |
| 3073 |
ld = bld.mkOp1(OP_PIXLD, TYPE_U32, i->getDef(0), bld.mkImm(0)); |
| 3074 |
ld->subOp = NV50_IR_SUBOP_PIXLD_COVMASK; |
| 3075 |
Instruction *sampleid = |
| 3076 |
bld.mkOp1(OP_PIXLD, TYPE_U32, bld.getSSA(), bld.mkImm(0)); |
| 3077 |
sampleid->subOp = NV50_IR_SUBOP_PIXLD_SAMPLEID; |
| 3078 |
Value *masked = |
| 3079 |
bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), ld->getDef(0), |
| 3080 |
bld.mkOp2v(OP_SHL, TYPE_U32, bld.getSSA(), |
| 3081 |
bld.loadImm(NULL, 1), sampleid->getDef(0))); |
| 3082 |
if (prog->persampleInvocation) { |
| 3083 |
bld.mkMov(i->getDef(0), masked); |
| 3084 |
} else { |
| 3085 |
bld.mkOp3(OP_SELP, TYPE_U32, i->getDef(0), ld->getDef(0), masked, |
| 3086 |
bld.mkImm(0)) |
| 3087 |
->subOp = 1; |
| 3088 |
} |
| 3089 |
break; |
| 3090 |
} |
| 3091 |
case SV_BASEVERTEX: |
| 3092 |
case SV_BASEINSTANCE: |
| 3093 |
case SV_DRAWID: |
| 3094 |
ld = bld.mkLoad(TYPE_U32, i->getDef(0), |
| 3095 |
bld.mkSymbol(FILE_MEMORY_CONST, |
| 3096 |
prog->driver->io.auxCBSlot, |
| 3097 |
TYPE_U32, |
| 3098 |
prog->driver->io.drawInfoBase + |
| 3099 |
4 * (sv - SV_BASEVERTEX)), |
| 3100 |
NULL); |
| 3101 |
break; |
| 3102 |
default: |
| 3103 |
if (prog->getType() == Program::TYPE_TESSELLATION_EVAL && !i->perPatch) |
| 3104 |
vtx = bld.mkOp1v(OP_PFETCH, TYPE_U32, bld.getSSA(), bld.mkImm(0)); |
| 3105 |
if (prog->getType() == Program::TYPE_FRAGMENT) { |
| 3106 |
bld.mkInterp(NV50_IR_INTERP_FLAT, i->getDef(0), addr, NULL); |
| 3107 |
} else { |
| 3108 |
ld = bld.mkFetch(i->getDef(0), i->dType, |
| 3109 |
FILE_SHADER_INPUT, addr, i->getIndirect(0, 0), vtx); |
| 3110 |
ld->perPatch = i->perPatch; |
| 3111 |
} |
| 3112 |
break; |
| 3113 |
} |
| 3114 |
bld.getBB()->remove(i); |
| 3115 |
return true; |
| 3116 |
} |
| 3117 |
|
| 3118 |
bool |
| 3119 |
NVC0LoweringPass::handleDIV(Instruction *i) |
| 3120 |
{ |
| 3121 |
if (!isFloatType(i->dType)) |
| 3122 |
return true; |
| 3123 |
bld.setPosition(i, false); |
| 3124 |
Instruction *rcp = bld.mkOp1(OP_RCP, i->dType, bld.getSSA(typeSizeof(i->dType)), i->getSrc(1)); |
| 3125 |
i->op = OP_MUL; |
| 3126 |
i->setSrc(1, rcp->getDef(0)); |
| 3127 |
return true; |
| 3128 |
} |
| 3129 |
|
| 3130 |
bool |
| 3131 |
NVC0LoweringPass::handleMOD(Instruction *i) |
| 3132 |
{ |
| 3133 |
if (!isFloatType(i->dType)) |
| 3134 |
return true; |
| 3135 |
LValue *value = bld.getScratch(typeSizeof(i->dType)); |
| 3136 |
bld.mkOp1(OP_RCP, i->dType, value, i->getSrc(1)); |
| 3137 |
bld.mkOp2(OP_MUL, i->dType, value, i->getSrc(0), value); |
| 3138 |
bld.mkOp1(OP_TRUNC, i->dType, value, value); |
| 3139 |
bld.mkOp2(OP_MUL, i->dType, value, i->getSrc(1), value); |
| 3140 |
i->op = OP_SUB; |
| 3141 |
i->setSrc(1, value); |
| 3142 |
return true; |
| 3143 |
} |
| 3144 |
|
| 3145 |
bool |
| 3146 |
NVC0LoweringPass::handleSQRT(Instruction *i) |
| 3147 |
{ |
| 3148 |
if (targ->isOpSupported(OP_SQRT, i->dType)) |
| 3149 |
return true; |
| 3150 |
|
| 3151 |
if (i->dType == TYPE_F64) { |
| 3152 |
Value *pred = bld.getSSA(1, FILE_PREDICATE); |
| 3153 |
Value *zero = bld.loadImm(NULL, 0.0); |
| 3154 |
Value *dst = bld.getSSA(8); |
| 3155 |
bld.mkOp1(OP_RSQ, i->dType, dst, i->getSrc(0)); |
| 3156 |
bld.mkCmp(OP_SET, CC_LE, i->dType, pred, i->dType, i->getSrc(0), zero); |
| 3157 |
bld.mkOp3(OP_SELP, TYPE_U64, dst, zero, dst, pred); |
| 3158 |
i->op = OP_MUL; |
| 3159 |
i->setSrc(1, dst); |
| 3160 |
// TODO: Handle this properly with a library function |
| 3161 |
} else { |
| 3162 |
bld.setPosition(i, true); |
| 3163 |
i->op = OP_RSQ; |
| 3164 |
bld.mkOp1(OP_RCP, i->dType, i->getDef(0), i->getDef(0)); |
| 3165 |
} |
| 3166 |
|
| 3167 |
return true; |
| 3168 |
} |
| 3169 |
|
| 3170 |
bool |
| 3171 |
NVC0LoweringPass::handleEXPORT(Instruction *i) |
| 3172 |
{ |
| 3173 |
if (prog->getType() == Program::TYPE_FRAGMENT) { |
| 3174 |
int id = i->getSrc(0)->reg.data.offset / 4; |
| 3175 |
|
| 3176 |
if (i->src(0).isIndirect(0)) // TODO, ugly |
| 3177 |
return false; |
| 3178 |
i->op = OP_MOV; |
| 3179 |
i->subOp = NV50_IR_SUBOP_MOV_FINAL; |
| 3180 |
i->src(0).set(i->src(1)); |
| 3181 |
i->setSrc(1, NULL); |
| 3182 |
i->setDef(0, new_LValue(func, FILE_GPR)); |
| 3183 |
i->getDef(0)->reg.data.id = id; |
| 3184 |
|
| 3185 |
prog->maxGPR = MAX2(prog->maxGPR, id); |
| 3186 |
} else |
| 3187 |
if (prog->getType() == Program::TYPE_GEOMETRY) { |
| 3188 |
i->setIndirect(0, 1, gpEmitAddress); |
| 3189 |
} |
| 3190 |
return true; |
| 3191 |
} |
| 3192 |
|
| 3193 |
bool |
| 3194 |
NVC0LoweringPass::handleOUT(Instruction *i) |
| 3195 |
{ |
| 3196 |
Instruction *prev = i->prev; |
| 3197 |
ImmediateValue stream, prevStream; |
| 3198 |
|
| 3199 |
// Only merge if the stream ids match. Also, note that the previous |
| 3200 |
// instruction would have already been lowered, so we take arg1 from it. |
| 3201 |
if (i->op == OP_RESTART && prev && prev->op == OP_EMIT && |
| 3202 |
i->src(0).getImmediate(stream) && |
| 3203 |
prev->src(1).getImmediate(prevStream) && |
| 3204 |
stream.reg.data.u32 == prevStream.reg.data.u32) { |
| 3205 |
i->prev->subOp = NV50_IR_SUBOP_EMIT_RESTART; |
| 3206 |
delete_Instruction(prog, i); |
| 3207 |
} else { |
| 3208 |
assert(gpEmitAddress); |
| 3209 |
i->setDef(0, gpEmitAddress); |
| 3210 |
i->setSrc(1, i->getSrc(0)); |
| 3211 |
i->setSrc(0, gpEmitAddress); |
| 3212 |
} |
| 3213 |
return true; |
| 3214 |
} |
| 3215 |
|
| 3216 |
Value * |
| 3217 |
NVC0LoweringPass::calculateSampleOffset(Value *sampleID) |
| 3218 |
{ |
| 3219 |
Value *offset = bld.getScratch(); |
| 3220 |
if (targ->getChipset() >= NVISA_GM200_CHIPSET) { |
| 3221 |
// Sample location offsets (in bytes) are calculated like so: |
| 3222 |
// offset = (SV_POSITION.y % 4 * 2) + (SV_POSITION.x % 2) |
| 3223 |
// offset = offset * 32 + sampleID % 8 * 4; |
| 3224 |
// which is equivalent to: |
| 3225 |
// offset = (SV_POSITION.y & 0x3) << 6 + (SV_POSITION.x & 0x1) << 5; |
| 3226 |
// offset += sampleID << 2 |
| 3227 |
|
| 3228 |
// The second operand (src1) of the INSBF instructions are like so: |
| 3229 |
// 0xssll where ss is the size and ll is the offset. |
| 3230 |
// so: dest = src2 | (src0 & (1 << ss - 1)) << ll |
| 3231 |
|
| 3232 |
// Add sample ID (offset = (sampleID & 0x7) << 2) |
| 3233 |
bld.mkOp3(OP_INSBF, TYPE_U32, offset, sampleID, bld.mkImm(0x0302), bld.mkImm(0x0)); |
| 3234 |
|
| 3235 |
Symbol *xSym = bld.mkSysVal(SV_POSITION, 0); |
| 3236 |
Symbol *ySym = bld.mkSysVal(SV_POSITION, 1); |
| 3237 |
Value *coord = bld.getScratch(); |
| 3238 |
|
| 3239 |
// Add X coordinate (offset |= (SV_POSITION.x & 0x1) << 5) |
| 3240 |
bld.mkInterp(NV50_IR_INTERP_LINEAR, coord, |
| 3241 |
targ->getSVAddress(FILE_SHADER_INPUT, xSym), NULL); |
| 3242 |
bld.mkCvt(OP_CVT, TYPE_U32, coord, TYPE_F32, coord) |
| 3243 |
->rnd = ROUND_ZI; |
| 3244 |
bld.mkOp3(OP_INSBF, TYPE_U32, offset, coord, bld.mkImm(0x0105), offset); |
| 3245 |
|
| 3246 |
// Add Y coordinate (offset |= (SV_POSITION.y & 0x3) << 6) |
| 3247 |
bld.mkInterp(NV50_IR_INTERP_LINEAR, coord, |
| 3248 |
targ->getSVAddress(FILE_SHADER_INPUT, ySym), NULL); |
| 3249 |
bld.mkCvt(OP_CVT, TYPE_U32, coord, TYPE_F32, coord) |
| 3250 |
->rnd = ROUND_ZI; |
| 3251 |
bld.mkOp3(OP_INSBF, TYPE_U32, offset, coord, bld.mkImm(0x0206), offset); |
| 3252 |
} else { |
| 3253 |
bld.mkOp2(OP_SHL, TYPE_U32, offset, sampleID, bld.mkImm(3)); |
| 3254 |
} |
| 3255 |
return offset; |
| 3256 |
} |
| 3257 |
|
| 3258 |
// Handle programmable sample locations for GM20x+ |
| 3259 |
void |
| 3260 |
NVC0LoweringPass::handlePIXLD(Instruction *i) |
| 3261 |
{ |
| 3262 |
if (i->subOp != NV50_IR_SUBOP_PIXLD_OFFSET) |
| 3263 |
return; |
| 3264 |
if (targ->getChipset() < NVISA_GM200_CHIPSET) |
| 3265 |
return; |
| 3266 |
|
| 3267 |
assert(prog->driver_out->prop.fp.readsSampleLocations); |
| 3268 |
|
| 3269 |
bld.mkLoad(TYPE_F32, |
| 3270 |
i->getDef(0), |
| 3271 |
bld.mkSymbol( |
| 3272 |
FILE_MEMORY_CONST, prog->driver->io.auxCBSlot, |
| 3273 |
TYPE_U32, prog->driver->io.sampleInfoBase), |
| 3274 |
calculateSampleOffset(i->getSrc(0))); |
| 3275 |
|
| 3276 |
bld.getBB()->remove(i); |
| 3277 |
} |
| 3278 |
|
| 3279 |
// Generate a binary predicate if an instruction is predicated by |
| 3280 |
// e.g. an f32 value. |
| 3281 |
void |
| 3282 |
NVC0LoweringPass::checkPredicate(Instruction *insn) |
| 3283 |
{ |
| 3284 |
Value *pred = insn->getPredicate(); |
| 3285 |
Value *pdst; |
| 3286 |
|
| 3287 |
if (!pred || pred->reg.file == FILE_PREDICATE) |
| 3288 |
return; |
| 3289 |
pdst = new_LValue(func, FILE_PREDICATE); |
| 3290 |
|
| 3291 |
// CAUTION: don't use pdst->getInsn, the definition might not be unique, |
| 3292 |
// delay turning PSET(FSET(x,y),0) into PSET(x,y) to a later pass |
| 3293 |
|
| 3294 |
bld.mkCmp(OP_SET, CC_NEU, insn->dType, pdst, insn->dType, bld.mkImm(0), pred); |
| 3295 |
|
| 3296 |
insn->setPredicate(insn->cc, pdst); |
| 3297 |
} |
| 3298 |
|
| 3299 |
// |
| 3300 |
// - add quadop dance for texturing |
| 3301 |
// - put FP outputs in GPRs |
| 3302 |
// - convert instruction sequences |
| 3303 |
// |
| 3304 |
bool |
| 3305 |
NVC0LoweringPass::visit(Instruction *i) |
| 3306 |
{ |
| 3307 |
bool ret = true; |
| 3308 |
bld.setPosition(i, false); |
| 3309 |
|
| 3310 |
if (i->cc != CC_ALWAYS) |
| 3311 |
checkPredicate(i); |
| 3312 |
|
| 3313 |
switch (i->op) { |
| 3314 |
case OP_TEX: |
| 3315 |
case OP_TXB: |
| 3316 |
case OP_TXL: |
| 3317 |
case OP_TXF: |
| 3318 |
case OP_TXG: |
| 3319 |
return handleTEX(i->asTex()); |
| 3320 |
case OP_TXD: |
| 3321 |
return handleTXD(i->asTex()); |
| 3322 |
case OP_TXLQ: |
| 3323 |
return handleTXLQ(i->asTex()); |
| 3324 |
case OP_TXQ: |
| 3325 |
return handleTXQ(i->asTex()); |
| 3326 |
case OP_EX2: |
| 3327 |
bld.mkOp1(OP_PREEX2, TYPE_F32, i->getDef(0), i->getSrc(0)); |
| 3328 |
i->setSrc(0, i->getDef(0)); |
| 3329 |
break; |
| 3330 |
case OP_DIV: |
| 3331 |
return handleDIV(i); |
| 3332 |
case OP_MOD: |
| 3333 |
return handleMOD(i); |
| 3334 |
case OP_SQRT: |
| 3335 |
return handleSQRT(i); |
| 3336 |
case OP_EXPORT: |
| 3337 |
ret = handleEXPORT(i); |
| 3338 |
break; |
| 3339 |
case OP_EMIT: |
| 3340 |
case OP_RESTART: |
| 3341 |
return handleOUT(i); |
| 3342 |
case OP_RDSV: |
| 3343 |
return handleRDSV(i); |
| 3344 |
case OP_STORE: |
| 3345 |
case OP_LOAD: |
| 3346 |
handleLDST(i); |
| 3347 |
break; |
| 3348 |
case OP_ATOM: |
| 3349 |
{ |
| 3350 |
const bool cctl = i->src(0).getFile() == FILE_MEMORY_BUFFER; |
| 3351 |
handleATOM(i); |
| 3352 |
if (cctl) |
| 3353 |
handleATOMCctl(i); |
| 3354 |
handleCasExch(i); |
| 3355 |
} |
| 3356 |
break; |
| 3357 |
case OP_SULDB: |
| 3358 |
case OP_SULDP: |
| 3359 |
case OP_SUSTB: |
| 3360 |
case OP_SUSTP: |
| 3361 |
case OP_SUREDB: |
| 3362 |
case OP_SUREDP: |
| 3363 |
if (targ->getChipset() >= NVISA_GM107_CHIPSET) |
| 3364 |
handleSurfaceOpGM107(i->asTex()); |
| 3365 |
else if (targ->getChipset() >= NVISA_GK104_CHIPSET) |
| 3366 |
handleSurfaceOpNVE4(i->asTex()); |
| 3367 |
else |
| 3368 |
handleSurfaceOpNVC0(i->asTex()); |
| 3369 |
break; |
| 3370 |
case OP_SUQ: |
| 3371 |
handleSUQ(i->asTex()); |
| 3372 |
break; |
| 3373 |
case OP_BUFQ: |
| 3374 |
handleBUFQ(i); |
| 3375 |
break; |
| 3376 |
case OP_PIXLD: |
| 3377 |
handlePIXLD(i); |
| 3378 |
break; |
| 3379 |
default: |
| 3380 |
break; |
| 3381 |
} |
| 3382 |
|
| 3383 |
/* Kepler+ has a special opcode to compute a new base address to be used |
| 3384 |
* for indirect loads. |
| 3385 |
* |
| 3386 |
* Maxwell+ has an additional similar requirement for indirect |
| 3387 |
* interpolation ops in frag shaders. |
| 3388 |
*/ |
| 3389 |
bool doAfetch = false; |
| 3390 |
if (targ->getChipset() >= NVISA_GK104_CHIPSET && |
| 3391 |
!i->perPatch && |
| 3392 |
(i->op == OP_VFETCH || i->op == OP_EXPORT) && |
| 3393 |
i->src(0).isIndirect(0)) { |
| 3394 |
doAfetch = true; |
| 3395 |
} |
| 3396 |
if (targ->getChipset() >= NVISA_GM107_CHIPSET && |
| 3397 |
(i->op == OP_LINTERP || i->op == OP_PINTERP) && |
| 3398 |
i->src(0).isIndirect(0)) { |
| 3399 |
doAfetch = true; |
| 3400 |
} |
| 3401 |
|
| 3402 |
if (doAfetch) { |
| 3403 |
Value *addr = cloneShallow(func, i->getSrc(0)); |
| 3404 |
Instruction *afetch = bld.mkOp1(OP_AFETCH, TYPE_U32, bld.getSSA(), |
| 3405 |
i->getSrc(0)); |
| 3406 |
afetch->setIndirect(0, 0, i->getIndirect(0, 0)); |
| 3407 |
addr->reg.data.offset = 0; |
| 3408 |
i->setSrc(0, addr); |
| 3409 |
i->setIndirect(0, 0, afetch->getDef(0)); |
| 3410 |
i->subOp = NV50_IR_SUBOP_VFETCH_PHYS; |
| 3411 |
} |
| 3412 |
|
| 3413 |
return ret; |
| 3414 |
} |
| 3415 |
|
| 3416 |
bool |
| 3417 |
TargetNVC0::runLegalizePass(Program *prog, CGStage stage) const |
| 3418 |
{ |
| 3419 |
if (stage == CG_STAGE_PRE_SSA) { |
| 3420 |
NVC0LoweringPass pass(prog); |
| 3421 |
return pass.run(prog, false, true); |
| 3422 |
} else |
| 3423 |
if (stage == CG_STAGE_POST_RA) { |
| 3424 |
NVC0LegalizePostRA pass(prog); |
| 3425 |
return pass.run(prog, false, true); |
| 3426 |
} else |
| 3427 |
if (stage == CG_STAGE_SSA) { |
| 3428 |
NVC0LegalizeSSA pass; |
| 3429 |
return pass.run(prog, false, true); |
| 3430 |
} |
| 3431 |
return false; |
| 3432 |
} |
| 3433 |
|
| 3434 |
} // namespace nv50_ir |