| 1 |
// Copyright (c) 2018 Google LLC |
| 2 |
// |
| 3 |
// Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 |
// you may not use this file except in compliance with the License. |
| 5 |
// You may obtain a copy of the License at |
| 6 |
// |
| 7 |
// http://www.apache.org/licenses/LICENSE-2.0 |
| 8 |
// |
| 9 |
// Unless required by applicable law or agreed to in writing, software |
| 10 |
// distributed under the License is distributed on an "AS IS" BASIS, |
| 11 |
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 |
// See the License for the specific language governing permissions and |
| 13 |
// limitations under the License. |
| 14 |
|
| 15 |
#include "upgrade_memory_model.h" |
| 16 |
|
| 17 |
#include <utility> |
| 18 |
|
| 19 |
#include "source/opt/ir_builder.h" |
| 20 |
#include "source/opt/ir_context.h" |
| 21 |
#include "source/spirv_constant.h" |
| 22 |
#include "source/util/make_unique.h" |
| 23 |
|
| 24 |
namespace spvtools { |
| 25 |
namespace opt { |
| 26 |
|
| 27 |
Pass::Status UpgradeMemoryModel::Process() { |
| 28 |
// TODO: This pass needs changes to support cooperative matrices. |
| 29 |
if (context()->get_feature_mgr()->HasCapability( |
| 30 |
SpvCapabilityCooperativeMatrixNV)) { |
| 31 |
return Pass::Status::SuccessWithoutChange; |
| 32 |
} |
| 33 |
|
| 34 |
// Only update Logical GLSL450 to Logical VulkanKHR. |
| 35 |
Instruction* memory_model = get_module()->GetMemoryModel(); |
| 36 |
if (memory_model->GetSingleWordInOperand(0u) != SpvAddressingModelLogical || |
| 37 |
memory_model->GetSingleWordInOperand(1u) != SpvMemoryModelGLSL450) { |
| 38 |
return Pass::Status::SuccessWithoutChange; |
| 39 |
} |
| 40 |
|
| 41 |
UpgradeMemoryModelInstruction(); |
| 42 |
UpgradeInstructions(); |
| 43 |
CleanupDecorations(); |
| 44 |
UpgradeBarriers(); |
| 45 |
UpgradeMemoryScope(); |
| 46 |
|
| 47 |
return Pass::Status::SuccessWithChange; |
| 48 |
} |
| 49 |
|
| 50 |
void UpgradeMemoryModel::UpgradeMemoryModelInstruction() { |
| 51 |
// Overall changes necessary: |
| 52 |
// 1. Add the OpExtension. |
| 53 |
// 2. Add the OpCapability. |
| 54 |
// 3. Modify the memory model. |
| 55 |
Instruction* memory_model = get_module()->GetMemoryModel(); |
| 56 |
context()->AddCapability(MakeUnique<Instruction>( |
| 57 |
context(), SpvOpCapability, 0, 0, |
| 58 |
std::initializer_list<Operand>{ |
| 59 |
{SPV_OPERAND_TYPE_CAPABILITY, {SpvCapabilityVulkanMemoryModelKHR}}})); |
| 60 |
const std::string extension = "SPV_KHR_vulkan_memory_model"; |
| 61 |
std::vector<uint32_t> words(extension.size() / 4 + 1, 0); |
| 62 |
char* dst = reinterpret_cast<char*>(words.data()); |
| 63 |
strncpy(dst, extension.c_str(), extension.size()); |
| 64 |
context()->AddExtension( |
| 65 |
MakeUnique<Instruction>(context(), SpvOpExtension, 0, 0, |
| 66 |
std::initializer_list<Operand>{ |
| 67 |
{SPV_OPERAND_TYPE_LITERAL_STRING, words}})); |
| 68 |
memory_model->SetInOperand(1u, {SpvMemoryModelVulkanKHR}); |
| 69 |
} |
| 70 |
|
| 71 |
void UpgradeMemoryModel::UpgradeInstructions() { |
| 72 |
// Coherent and Volatile decorations are deprecated. Remove them and replace |
| 73 |
// with flags on the memory/image operations. The decorations can occur on |
| 74 |
// OpVariable, OpFunctionParameter (of pointer type) and OpStructType (member |
| 75 |
// decoration). Trace from the decoration target(s) to the final memory/image |
| 76 |
// instructions. Additionally, Workgroup storage class variables and function |
| 77 |
// parameters are implicitly coherent in GLSL450. |
| 78 |
|
| 79 |
// Upgrade modf and frexp first since they generate new stores. |
| 80 |
// In SPIR-V 1.4 or later, normalize OpCopyMemory* access operands. |
| 81 |
for (auto& func : *get_module()) { |
| 82 |
func.ForEachInst([this](Instruction* inst) { |
| 83 |
if (inst->opcode() == SpvOpExtInst) { |
| 84 |
auto ext_inst = inst->GetSingleWordInOperand(1u); |
| 85 |
if (ext_inst == GLSLstd450Modf || ext_inst == GLSLstd450Frexp) { |
| 86 |
auto import = |
| 87 |
get_def_use_mgr()->GetDef(inst->GetSingleWordInOperand(0u)); |
| 88 |
if (reinterpret_cast<char*>(import->GetInOperand(0u).words.data()) == |
| 89 |
std::string("GLSL.std.450")) { |
| 90 |
UpgradeExtInst(inst); |
| 91 |
} |
| 92 |
} |
| 93 |
} else if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) { |
| 94 |
if (inst->opcode() == SpvOpCopyMemory || |
| 95 |
inst->opcode() == SpvOpCopyMemorySized) { |
| 96 |
uint32_t start_operand = inst->opcode() == SpvOpCopyMemory ? 2u : 3u; |
| 97 |
if (inst->NumInOperands() > start_operand) { |
| 98 |
auto num_access_words = MemoryAccessNumWords( |
| 99 |
inst->GetSingleWordInOperand(start_operand)); |
| 100 |
if ((num_access_words + start_operand) == inst->NumInOperands()) { |
| 101 |
// There is a single memory access operand. Duplicate it to have a |
| 102 |
// separate operand for both source and target. |
| 103 |
for (uint32_t i = 0; i < num_access_words; ++i) { |
| 104 |
auto operand = inst->GetInOperand(start_operand + i); |
| 105 |
inst->AddOperand(std::move(operand)); |
| 106 |
} |
| 107 |
} |
| 108 |
} else { |
| 109 |
// Add two memory access operands. |
| 110 |
inst->AddOperand( |
| 111 |
{SPV_OPERAND_TYPE_MEMORY_ACCESS, {SpvMemoryAccessMaskNone}}); |
| 112 |
inst->AddOperand( |
| 113 |
{SPV_OPERAND_TYPE_MEMORY_ACCESS, {SpvMemoryAccessMaskNone}}); |
| 114 |
} |
| 115 |
} |
| 116 |
} |
| 117 |
}); |
| 118 |
} |
| 119 |
|
| 120 |
UpgradeMemoryAndImages(); |
| 121 |
UpgradeAtomics(); |
| 122 |
} |
| 123 |
|
| 124 |
void UpgradeMemoryModel::UpgradeMemoryAndImages() { |
| 125 |
for (auto& func : *get_module()) { |
| 126 |
func.ForEachInst([this](Instruction* inst) { |
| 127 |
bool is_coherent = false; |
| 128 |
bool is_volatile = false; |
| 129 |
bool src_coherent = false; |
| 130 |
bool src_volatile = false; |
| 131 |
bool dst_coherent = false; |
| 132 |
bool dst_volatile = false; |
| 133 |
uint32_t start_operand = 0u; |
| 134 |
SpvScope scope = SpvScopeQueueFamilyKHR; |
| 135 |
SpvScope src_scope = SpvScopeQueueFamilyKHR; |
| 136 |
SpvScope dst_scope = SpvScopeQueueFamilyKHR; |
| 137 |
switch (inst->opcode()) { |
| 138 |
case SpvOpLoad: |
| 139 |
case SpvOpStore: |
| 140 |
std::tie(is_coherent, is_volatile, scope) = |
| 141 |
GetInstructionAttributes(inst->GetSingleWordInOperand(0u)); |
| 142 |
break; |
| 143 |
case SpvOpImageRead: |
| 144 |
case SpvOpImageSparseRead: |
| 145 |
case SpvOpImageWrite: |
| 146 |
std::tie(is_coherent, is_volatile, scope) = |
| 147 |
GetInstructionAttributes(inst->GetSingleWordInOperand(0u)); |
| 148 |
break; |
| 149 |
case SpvOpCopyMemory: |
| 150 |
case SpvOpCopyMemorySized: |
| 151 |
std::tie(dst_coherent, dst_volatile, dst_scope) = |
| 152 |
GetInstructionAttributes(inst->GetSingleWordInOperand(0u)); |
| 153 |
std::tie(src_coherent, src_volatile, src_scope) = |
| 154 |
GetInstructionAttributes(inst->GetSingleWordInOperand(1u)); |
| 155 |
break; |
| 156 |
default: |
| 157 |
break; |
| 158 |
} |
| 159 |
|
| 160 |
switch (inst->opcode()) { |
| 161 |
case SpvOpLoad: |
| 162 |
UpgradeFlags(inst, 1u, is_coherent, is_volatile, kVisibility, |
| 163 |
kMemory); |
| 164 |
break; |
| 165 |
case SpvOpStore: |
| 166 |
UpgradeFlags(inst, 2u, is_coherent, is_volatile, kAvailability, |
| 167 |
kMemory); |
| 168 |
break; |
| 169 |
case SpvOpCopyMemory: |
| 170 |
case SpvOpCopyMemorySized: |
| 171 |
start_operand = inst->opcode() == SpvOpCopyMemory ? 2u : 3u; |
| 172 |
if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) { |
| 173 |
// There are guaranteed to be two memory access operands at this |
| 174 |
// point so treat source and target separately. |
| 175 |
uint32_t num_access_words = MemoryAccessNumWords( |
| 176 |
inst->GetSingleWordInOperand(start_operand)); |
| 177 |
UpgradeFlags(inst, start_operand, dst_coherent, dst_volatile, |
| 178 |
kAvailability, kMemory); |
| 179 |
UpgradeFlags(inst, start_operand + num_access_words, src_coherent, |
| 180 |
src_volatile, kVisibility, kMemory); |
| 181 |
} else { |
| 182 |
UpgradeFlags(inst, start_operand, dst_coherent, dst_volatile, |
| 183 |
kAvailability, kMemory); |
| 184 |
UpgradeFlags(inst, start_operand, src_coherent, src_volatile, |
| 185 |
kVisibility, kMemory); |
| 186 |
} |
| 187 |
break; |
| 188 |
case SpvOpImageRead: |
| 189 |
case SpvOpImageSparseRead: |
| 190 |
UpgradeFlags(inst, 2u, is_coherent, is_volatile, kVisibility, kImage); |
| 191 |
break; |
| 192 |
case SpvOpImageWrite: |
| 193 |
UpgradeFlags(inst, 3u, is_coherent, is_volatile, kAvailability, |
| 194 |
kImage); |
| 195 |
break; |
| 196 |
default: |
| 197 |
break; |
| 198 |
} |
| 199 |
|
| 200 |
// |is_coherent| is never used for the same instructions as |
| 201 |
// |src_coherent| and |dst_coherent|. |
| 202 |
if (is_coherent) { |
| 203 |
inst->AddOperand( |
| 204 |
{SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(scope)}}); |
| 205 |
} |
| 206 |
if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) { |
| 207 |
// There are two memory access operands. The first is for the target and |
| 208 |
// the second is for the source. |
| 209 |
if (dst_coherent || src_coherent) { |
| 210 |
start_operand = inst->opcode() == SpvOpCopyMemory ? 2u : 3u; |
| 211 |
std::vector<Operand> new_operands; |
| 212 |
uint32_t num_access_words = |
| 213 |
MemoryAccessNumWords(inst->GetSingleWordInOperand(start_operand)); |
| 214 |
// The flags were already updated so subtract if we're adding a |
| 215 |
// scope. |
| 216 |
if (dst_coherent) --num_access_words; |
| 217 |
for (uint32_t i = 0; i < start_operand + num_access_words; ++i) { |
| 218 |
new_operands.push_back(inst->GetInOperand(i)); |
| 219 |
} |
| 220 |
// Add the target scope if necessary. |
| 221 |
if (dst_coherent) { |
| 222 |
new_operands.push_back( |
| 223 |
{SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(dst_scope)}}); |
| 224 |
} |
| 225 |
// Copy the remaining current operands. |
| 226 |
for (uint32_t i = start_operand + num_access_words; |
| 227 |
i < inst->NumInOperands(); ++i) { |
| 228 |
new_operands.push_back(inst->GetInOperand(i)); |
| 229 |
} |
| 230 |
// Add the source scope if necessary. |
| 231 |
if (src_coherent) { |
| 232 |
new_operands.push_back( |
| 233 |
{SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(src_scope)}}); |
| 234 |
} |
| 235 |
inst->SetInOperands(std::move(new_operands)); |
| 236 |
} |
| 237 |
} else { |
| 238 |
// According to SPV_KHR_vulkan_memory_model, if both available and |
| 239 |
// visible flags are used the first scope operand is for availability |
| 240 |
// (writes) and the second is for visibility (reads). |
| 241 |
if (dst_coherent) { |
| 242 |
inst->AddOperand( |
| 243 |
{SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(dst_scope)}}); |
| 244 |
} |
| 245 |
if (src_coherent) { |
| 246 |
inst->AddOperand( |
| 247 |
{SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(src_scope)}}); |
| 248 |
} |
| 249 |
} |
| 250 |
}); |
| 251 |
} |
| 252 |
} |
| 253 |
|
| 254 |
void UpgradeMemoryModel::UpgradeAtomics() { |
| 255 |
for (auto& func : *get_module()) { |
| 256 |
func.ForEachInst([this](Instruction* inst) { |
| 257 |
if (spvOpcodeIsAtomicOp(inst->opcode())) { |
| 258 |
bool unused_coherent = false; |
| 259 |
bool is_volatile = false; |
| 260 |
SpvScope unused_scope = SpvScopeQueueFamilyKHR; |
| 261 |
std::tie(unused_coherent, is_volatile, unused_scope) = |
| 262 |
GetInstructionAttributes(inst->GetSingleWordInOperand(0)); |
| 263 |
|
| 264 |
UpgradeSemantics(inst, 2u, is_volatile); |
| 265 |
if (inst->opcode() == SpvOpAtomicCompareExchange || |
| 266 |
inst->opcode() == SpvOpAtomicCompareExchangeWeak) { |
| 267 |
UpgradeSemantics(inst, 3u, is_volatile); |
| 268 |
} |
| 269 |
} |
| 270 |
}); |
| 271 |
} |
| 272 |
} |
| 273 |
|
| 274 |
void UpgradeMemoryModel::UpgradeSemantics(Instruction* inst, |
| 275 |
uint32_t in_operand, |
| 276 |
bool is_volatile) { |
| 277 |
if (!is_volatile) return; |
| 278 |
|
| 279 |
uint32_t semantics_id = inst->GetSingleWordInOperand(in_operand); |
| 280 |
const analysis::Constant* constant = |
| 281 |
context()->get_constant_mgr()->FindDeclaredConstant(semantics_id); |
| 282 |
const analysis::Integer* type = constant->type()->AsInteger(); |
| 283 |
assert(type && type->width() == 32); |
| 284 |
uint32_t value = 0; |
| 285 |
if (type->IsSigned()) { |
| 286 |
value = static_cast<uint32_t>(constant->GetS32()); |
| 287 |
} else { |
| 288 |
value = constant->GetU32(); |
| 289 |
} |
| 290 |
|
| 291 |
value |= SpvMemorySemanticsVolatileMask; |
| 292 |
auto new_constant = context()->get_constant_mgr()->GetConstant(type, {value}); |
| 293 |
auto new_semantics = |
| 294 |
context()->get_constant_mgr()->GetDefiningInstruction(new_constant); |
| 295 |
inst->SetInOperand(in_operand, {new_semantics->result_id()}); |
| 296 |
} |
| 297 |
|
| 298 |
std::tuple<bool, bool, SpvScope> UpgradeMemoryModel::GetInstructionAttributes( |
| 299 |
uint32_t id) { |
| 300 |
// |id| is a pointer used in a memory/image instruction. Need to determine if |
| 301 |
// that pointer points to volatile or coherent memory. Workgroup storage |
| 302 |
// class is implicitly coherent and cannot be decorated with volatile, so |
| 303 |
// short circuit that case. |
| 304 |
Instruction* inst = context()->get_def_use_mgr()->GetDef(id); |
| 305 |
analysis::Type* type = context()->get_type_mgr()->GetType(inst->type_id()); |
| 306 |
if (type->AsPointer() && |
| 307 |
type->AsPointer()->storage_class() == SpvStorageClassWorkgroup) { |
| 308 |
return std::make_tuple(true, false, SpvScopeWorkgroup); |
| 309 |
} |
| 310 |
|
| 311 |
bool is_coherent = false; |
| 312 |
bool is_volatile = false; |
| 313 |
std::unordered_set<uint32_t> visited; |
| 314 |
std::tie(is_coherent, is_volatile) = |
| 315 |
TraceInstruction(context()->get_def_use_mgr()->GetDef(id), |
| 316 |
std::vector<uint32_t>(), &visited); |
| 317 |
|
| 318 |
return std::make_tuple(is_coherent, is_volatile, SpvScopeQueueFamilyKHR); |
| 319 |
} |
| 320 |
|
| 321 |
std::pair<bool, bool> UpgradeMemoryModel::TraceInstruction( |
| 322 |
Instruction* inst, std::vector<uint32_t> indices, |
| 323 |
std::unordered_set<uint32_t>* visited) { |
| 324 |
auto iter = cache_.find(std::make_pair(inst->result_id(), indices)); |
| 325 |
if (iter != cache_.end()) { |
| 326 |
return iter->second; |
| 327 |
} |
| 328 |
|
| 329 |
if (!visited->insert(inst->result_id()).second) { |
| 330 |
return std::make_pair(false, false); |
| 331 |
} |
| 332 |
|
| 333 |
// Initialize the cache before |indices| is (potentially) modified. |
| 334 |
auto& cached_result = cache_[std::make_pair(inst->result_id(), indices)]; |
| 335 |
cached_result.first = false; |
| 336 |
cached_result.second = false; |
| 337 |
|
| 338 |
bool is_coherent = false; |
| 339 |
bool is_volatile = false; |
| 340 |
switch (inst->opcode()) { |
| 341 |
case SpvOpVariable: |
| 342 |
case SpvOpFunctionParameter: |
| 343 |
is_coherent |= HasDecoration(inst, 0, SpvDecorationCoherent); |
| 344 |
is_volatile |= HasDecoration(inst, 0, SpvDecorationVolatile); |
| 345 |
if (!is_coherent || !is_volatile) { |
| 346 |
bool type_coherent = false; |
| 347 |
bool type_volatile = false; |
| 348 |
std::tie(type_coherent, type_volatile) = |
| 349 |
CheckType(inst->type_id(), indices); |
| 350 |
is_coherent |= type_coherent; |
| 351 |
is_volatile |= type_volatile; |
| 352 |
} |
| 353 |
break; |
| 354 |
case SpvOpAccessChain: |
| 355 |
case SpvOpInBoundsAccessChain: |
| 356 |
// Store indices in reverse order. |
| 357 |
for (uint32_t i = inst->NumInOperands() - 1; i > 0; --i) { |
| 358 |
indices.push_back(inst->GetSingleWordInOperand(i)); |
| 359 |
} |
| 360 |
break; |
| 361 |
case SpvOpPtrAccessChain: |
| 362 |
// Store indices in reverse order. Skip the |Element| operand. |
| 363 |
for (uint32_t i = inst->NumInOperands() - 1; i > 1; --i) { |
| 364 |
indices.push_back(inst->GetSingleWordInOperand(i)); |
| 365 |
} |
| 366 |
break; |
| 367 |
default: |
| 368 |
break; |
| 369 |
} |
| 370 |
|
| 371 |
// No point searching further. |
| 372 |
if (is_coherent && is_volatile) { |
| 373 |
cached_result.first = true; |
| 374 |
cached_result.second = true; |
| 375 |
return std::make_pair(true, true); |
| 376 |
} |
| 377 |
|
| 378 |
// Variables and function parameters are sources. Continue searching until we |
| 379 |
// reach them. |
| 380 |
if (inst->opcode() != SpvOpVariable && |
| 381 |
inst->opcode() != SpvOpFunctionParameter) { |
| 382 |
inst->ForEachInId([this, &is_coherent, &is_volatile, &indices, |
| 383 |
&visited](const uint32_t* id_ptr) { |
| 384 |
Instruction* op_inst = context()->get_def_use_mgr()->GetDef(*id_ptr); |
| 385 |
const analysis::Type* type = |
| 386 |
context()->get_type_mgr()->GetType(op_inst->type_id()); |
| 387 |
if (type && |
| 388 |
(type->AsPointer() || type->AsImage() || type->AsSampledImage())) { |
| 389 |
bool operand_coherent = false; |
| 390 |
bool operand_volatile = false; |
| 391 |
std::tie(operand_coherent, operand_volatile) = |
| 392 |
TraceInstruction(op_inst, indices, visited); |
| 393 |
is_coherent |= operand_coherent; |
| 394 |
is_volatile |= operand_volatile; |
| 395 |
} |
| 396 |
}); |
| 397 |
} |
| 398 |
|
| 399 |
cached_result.first = is_coherent; |
| 400 |
cached_result.second = is_volatile; |
| 401 |
return std::make_pair(is_coherent, is_volatile); |
| 402 |
} |
| 403 |
|
| 404 |
std::pair<bool, bool> UpgradeMemoryModel::CheckType( |
| 405 |
uint32_t type_id, const std::vector<uint32_t>& indices) { |
| 406 |
bool is_coherent = false; |
| 407 |
bool is_volatile = false; |
| 408 |
Instruction* type_inst = context()->get_def_use_mgr()->GetDef(type_id); |
| 409 |
assert(type_inst->opcode() == SpvOpTypePointer); |
| 410 |
Instruction* element_inst = context()->get_def_use_mgr()->GetDef( |
| 411 |
type_inst->GetSingleWordInOperand(1u)); |
| 412 |
for (int i = (int)indices.size() - 1; i >= 0; --i) { |
| 413 |
if (is_coherent && is_volatile) break; |
| 414 |
|
| 415 |
if (element_inst->opcode() == SpvOpTypePointer) { |
| 416 |
element_inst = context()->get_def_use_mgr()->GetDef( |
| 417 |
element_inst->GetSingleWordInOperand(1u)); |
| 418 |
} else if (element_inst->opcode() == SpvOpTypeStruct) { |
| 419 |
uint32_t index = indices.at(i); |
| 420 |
Instruction* index_inst = context()->get_def_use_mgr()->GetDef(index); |
| 421 |
assert(index_inst->opcode() == SpvOpConstant); |
| 422 |
uint64_t value = GetIndexValue(index_inst); |
| 423 |
is_coherent |= HasDecoration(element_inst, static_cast<uint32_t>(value), |
| 424 |
SpvDecorationCoherent); |
| 425 |
is_volatile |= HasDecoration(element_inst, static_cast<uint32_t>(value), |
| 426 |
SpvDecorationVolatile); |
| 427 |
element_inst = context()->get_def_use_mgr()->GetDef( |
| 428 |
element_inst->GetSingleWordInOperand(static_cast<uint32_t>(value))); |
| 429 |
} else { |
| 430 |
assert(spvOpcodeIsComposite(element_inst->opcode())); |
| 431 |
element_inst = context()->get_def_use_mgr()->GetDef( |
| 432 |
element_inst->GetSingleWordInOperand(0u)); |
| 433 |
} |
| 434 |
} |
| 435 |
|
| 436 |
if (!is_coherent || !is_volatile) { |
| 437 |
bool remaining_coherent = false; |
| 438 |
bool remaining_volatile = false; |
| 439 |
std::tie(remaining_coherent, remaining_volatile) = |
| 440 |
CheckAllTypes(element_inst); |
| 441 |
is_coherent |= remaining_coherent; |
| 442 |
is_volatile |= remaining_volatile; |
| 443 |
} |
| 444 |
|
| 445 |
return std::make_pair(is_coherent, is_volatile); |
| 446 |
} |
| 447 |
|
| 448 |
std::pair<bool, bool> UpgradeMemoryModel::CheckAllTypes( |
| 449 |
const Instruction* inst) { |
| 450 |
std::unordered_set<const Instruction*> visited; |
| 451 |
std::vector<const Instruction*> stack; |
| 452 |
stack.push_back(inst); |
| 453 |
|
| 454 |
bool is_coherent = false; |
| 455 |
bool is_volatile = false; |
| 456 |
while (!stack.empty()) { |
| 457 |
const Instruction* def = stack.back(); |
| 458 |
stack.pop_back(); |
| 459 |
|
| 460 |
if (!visited.insert(def).second) continue; |
| 461 |
|
| 462 |
if (def->opcode() == SpvOpTypeStruct) { |
| 463 |
// Any member decorated with coherent and/or volatile is enough to have |
| 464 |
// the related operation be flagged as coherent and/or volatile. |
| 465 |
is_coherent |= HasDecoration(def, std::numeric_limits<uint32_t>::max(), |
| 466 |
SpvDecorationCoherent); |
| 467 |
is_volatile |= HasDecoration(def, std::numeric_limits<uint32_t>::max(), |
| 468 |
SpvDecorationVolatile); |
| 469 |
if (is_coherent && is_volatile) |
| 470 |
return std::make_pair(is_coherent, is_volatile); |
| 471 |
|
| 472 |
// Check the subtypes. |
| 473 |
for (uint32_t i = 0; i < def->NumInOperands(); ++i) { |
| 474 |
stack.push_back(context()->get_def_use_mgr()->GetDef( |
| 475 |
def->GetSingleWordInOperand(i))); |
| 476 |
} |
| 477 |
} else if (spvOpcodeIsComposite(def->opcode())) { |
| 478 |
stack.push_back(context()->get_def_use_mgr()->GetDef( |
| 479 |
def->GetSingleWordInOperand(0u))); |
| 480 |
} else if (def->opcode() == SpvOpTypePointer) { |
| 481 |
stack.push_back(context()->get_def_use_mgr()->GetDef( |
| 482 |
def->GetSingleWordInOperand(1u))); |
| 483 |
} |
| 484 |
} |
| 485 |
|
| 486 |
return std::make_pair(is_coherent, is_volatile); |
| 487 |
} |
| 488 |
|
| 489 |
uint64_t UpgradeMemoryModel::GetIndexValue(Instruction* index_inst) { |
| 490 |
const analysis::Constant* index_constant = |
| 491 |
context()->get_constant_mgr()->GetConstantFromInst(index_inst); |
| 492 |
assert(index_constant->AsIntConstant()); |
| 493 |
if (index_constant->type()->AsInteger()->IsSigned()) { |
| 494 |
if (index_constant->type()->AsInteger()->width() == 32) { |
| 495 |
return index_constant->GetS32(); |
| 496 |
} else { |
| 497 |
return index_constant->GetS64(); |
| 498 |
} |
| 499 |
} else { |
| 500 |
if (index_constant->type()->AsInteger()->width() == 32) { |
| 501 |
return index_constant->GetU32(); |
| 502 |
} else { |
| 503 |
return index_constant->GetU64(); |
| 504 |
} |
| 505 |
} |
| 506 |
} |
| 507 |
|
| 508 |
bool UpgradeMemoryModel::HasDecoration(const Instruction* inst, uint32_t value, |
| 509 |
SpvDecoration decoration) { |
| 510 |
// If the iteration was terminated early then an appropriate decoration was |
| 511 |
// found. |
| 512 |
return !context()->get_decoration_mgr()->WhileEachDecoration( |
| 513 |
inst->result_id(), decoration, [value](const Instruction& i) { |
| 514 |
if (i.opcode() == SpvOpDecorate || i.opcode() == SpvOpDecorateId) { |
| 515 |
return false; |
| 516 |
} else if (i.opcode() == SpvOpMemberDecorate) { |
| 517 |
if (value == i.GetSingleWordInOperand(1u) || |
| 518 |
value == std::numeric_limits<uint32_t>::max()) |
| 519 |
return false; |
| 520 |
} |
| 521 |
|
| 522 |
return true; |
| 523 |
}); |
| 524 |
} |
| 525 |
|
| 526 |
void UpgradeMemoryModel::UpgradeFlags(Instruction* inst, uint32_t in_operand, |
| 527 |
bool is_coherent, bool is_volatile, |
| 528 |
OperationType operation_type, |
| 529 |
InstructionType inst_type) { |
| 530 |
if (!is_coherent && !is_volatile) return; |
| 531 |
|
| 532 |
uint32_t flags = 0; |
| 533 |
if (inst->NumInOperands() > in_operand) { |
| 534 |
flags |= inst->GetSingleWordInOperand(in_operand); |
| 535 |
} |
| 536 |
if (is_coherent) { |
| 537 |
if (inst_type == kMemory) { |
| 538 |
flags |= SpvMemoryAccessNonPrivatePointerKHRMask; |
| 539 |
if (operation_type == kVisibility) { |
| 540 |
flags |= SpvMemoryAccessMakePointerVisibleKHRMask; |
| 541 |
} else { |
| 542 |
flags |= SpvMemoryAccessMakePointerAvailableKHRMask; |
| 543 |
} |
| 544 |
} else { |
| 545 |
flags |= SpvImageOperandsNonPrivateTexelKHRMask; |
| 546 |
if (operation_type == kVisibility) { |
| 547 |
flags |= SpvImageOperandsMakeTexelVisibleKHRMask; |
| 548 |
} else { |
| 549 |
flags |= SpvImageOperandsMakeTexelAvailableKHRMask; |
| 550 |
} |
| 551 |
} |
| 552 |
} |
| 553 |
|
| 554 |
if (is_volatile) { |
| 555 |
if (inst_type == kMemory) { |
| 556 |
flags |= SpvMemoryAccessVolatileMask; |
| 557 |
} else { |
| 558 |
flags |= SpvImageOperandsVolatileTexelKHRMask; |
| 559 |
} |
| 560 |
} |
| 561 |
|
| 562 |
if (inst->NumInOperands() > in_operand) { |
| 563 |
inst->SetInOperand(in_operand, {flags}); |
| 564 |
} else if (inst_type == kMemory) { |
| 565 |
inst->AddOperand({SPV_OPERAND_TYPE_OPTIONAL_MEMORY_ACCESS, {flags}}); |
| 566 |
} else { |
| 567 |
inst->AddOperand({SPV_OPERAND_TYPE_OPTIONAL_IMAGE, {flags}}); |
| 568 |
} |
| 569 |
} |
| 570 |
|
| 571 |
uint32_t UpgradeMemoryModel::GetScopeConstant(SpvScope scope) { |
| 572 |
analysis::Integer int_ty(32, false); |
| 573 |
uint32_t int_id = context()->get_type_mgr()->GetTypeInstruction(&int_ty); |
| 574 |
const analysis::Constant* constant = |
| 575 |
context()->get_constant_mgr()->GetConstant( |
| 576 |
context()->get_type_mgr()->GetType(int_id), |
| 577 |
{static_cast<uint32_t>(scope)}); |
| 578 |
return context() |
| 579 |
->get_constant_mgr() |
| 580 |
->GetDefiningInstruction(constant) |
| 581 |
->result_id(); |
| 582 |
} |
| 583 |
|
| 584 |
void UpgradeMemoryModel::CleanupDecorations() { |
| 585 |
// All of the volatile and coherent decorations have been dealt with, so now |
| 586 |
// we can just remove them. |
| 587 |
get_module()->ForEachInst([this](Instruction* inst) { |
| 588 |
if (inst->result_id() != 0) { |
| 589 |
context()->get_decoration_mgr()->RemoveDecorationsFrom( |
| 590 |
inst->result_id(), [](const Instruction& dec) { |
| 591 |
switch (dec.opcode()) { |
| 592 |
case SpvOpDecorate: |
| 593 |
case SpvOpDecorateId: |
| 594 |
if (dec.GetSingleWordInOperand(1u) == SpvDecorationCoherent || |
| 595 |
dec.GetSingleWordInOperand(1u) == SpvDecorationVolatile) |
| 596 |
return true; |
| 597 |
break; |
| 598 |
case SpvOpMemberDecorate: |
| 599 |
if (dec.GetSingleWordInOperand(2u) == SpvDecorationCoherent || |
| 600 |
dec.GetSingleWordInOperand(2u) == SpvDecorationVolatile) |
| 601 |
return true; |
| 602 |
break; |
| 603 |
default: |
| 604 |
break; |
| 605 |
} |
| 606 |
return false; |
| 607 |
}); |
| 608 |
} |
| 609 |
}); |
| 610 |
} |
| 611 |
|
| 612 |
void UpgradeMemoryModel::UpgradeBarriers() { |
| 613 |
std::vector<Instruction*> barriers; |
| 614 |
// Collects all the control barriers in |function|. Returns true if the |
| 615 |
// function operates on the Output storage class. |
| 616 |
ProcessFunction CollectBarriers = [this, &barriers](Function* function) { |
| 617 |
bool operates_on_output = false; |
| 618 |
for (auto& block : *function) { |
| 619 |
block.ForEachInst([this, &barriers, |
| 620 |
&operates_on_output](Instruction* inst) { |
| 621 |
if (inst->opcode() == SpvOpControlBarrier) { |
| 622 |
barriers.push_back(inst); |
| 623 |
} else if (!operates_on_output) { |
| 624 |
// This instruction operates on output storage class if it is a |
| 625 |
// pointer to output type or any input operand is a pointer to output |
| 626 |
// type. |
| 627 |
analysis::Type* type = |
| 628 |
context()->get_type_mgr()->GetType(inst->type_id()); |
| 629 |
if (type && type->AsPointer() && |
| 630 |
type->AsPointer()->storage_class() == SpvStorageClassOutput) { |
| 631 |
operates_on_output = true; |
| 632 |
return; |
| 633 |
} |
| 634 |
inst->ForEachInId([this, &operates_on_output](uint32_t* id_ptr) { |
| 635 |
Instruction* op_inst = |
| 636 |
context()->get_def_use_mgr()->GetDef(*id_ptr); |
| 637 |
analysis::Type* op_type = |
| 638 |
context()->get_type_mgr()->GetType(op_inst->type_id()); |
| 639 |
if (op_type && op_type->AsPointer() && |
| 640 |
op_type->AsPointer()->storage_class() == SpvStorageClassOutput) |
| 641 |
operates_on_output = true; |
| 642 |
}); |
| 643 |
} |
| 644 |
}); |
| 645 |
} |
| 646 |
return operates_on_output; |
| 647 |
}; |
| 648 |
|
| 649 |
std::queue<uint32_t> roots; |
| 650 |
for (auto& e : get_module()->entry_points()) |
| 651 |
if (e.GetSingleWordInOperand(0u) == SpvExecutionModelTessellationControl) { |
| 652 |
roots.push(e.GetSingleWordInOperand(1u)); |
| 653 |
if (context()->ProcessCallTreeFromRoots(CollectBarriers, &roots)) { |
| 654 |
for (auto barrier : barriers) { |
| 655 |
// Add OutputMemoryKHR to the semantics of the barriers. |
| 656 |
uint32_t semantics_id = barrier->GetSingleWordInOperand(2u); |
| 657 |
Instruction* semantics_inst = |
| 658 |
context()->get_def_use_mgr()->GetDef(semantics_id); |
| 659 |
analysis::Type* semantics_type = |
| 660 |
context()->get_type_mgr()->GetType(semantics_inst->type_id()); |
| 661 |
uint64_t semantics_value = GetIndexValue(semantics_inst); |
| 662 |
const analysis::Constant* constant = |
| 663 |
context()->get_constant_mgr()->GetConstant( |
| 664 |
semantics_type, {static_cast<uint32_t>(semantics_value) | |
| 665 |
SpvMemorySemanticsOutputMemoryKHRMask}); |
| 666 |
barrier->SetInOperand(2u, {context() |
| 667 |
->get_constant_mgr() |
| 668 |
->GetDefiningInstruction(constant) |
| 669 |
->result_id()}); |
| 670 |
} |
| 671 |
} |
| 672 |
barriers.clear(); |
| 673 |
} |
| 674 |
} |
| 675 |
|
| 676 |
void UpgradeMemoryModel::UpgradeMemoryScope() { |
| 677 |
get_module()->ForEachInst([this](Instruction* inst) { |
| 678 |
// Don't need to handle all the operations that take a scope. |
| 679 |
// * Group operations can only be subgroup |
| 680 |
// * Non-uniform can only be workgroup or subgroup |
| 681 |
// * Named barriers are not supported by Vulkan |
| 682 |
// * Workgroup ops (e.g. async_copy) have at most workgroup scope. |
| 683 |
if (spvOpcodeIsAtomicOp(inst->opcode())) { |
| 684 |
if (IsDeviceScope(inst->GetSingleWordInOperand(1))) { |
| 685 |
inst->SetInOperand(1, {GetScopeConstant(SpvScopeQueueFamilyKHR)}); |
| 686 |
} |
| 687 |
} else if (inst->opcode() == SpvOpControlBarrier) { |
| 688 |
if (IsDeviceScope(inst->GetSingleWordInOperand(1))) { |
| 689 |
inst->SetInOperand(1, {GetScopeConstant(SpvScopeQueueFamilyKHR)}); |
| 690 |
} |
| 691 |
} else if (inst->opcode() == SpvOpMemoryBarrier) { |
| 692 |
if (IsDeviceScope(inst->GetSingleWordInOperand(0))) { |
| 693 |
inst->SetInOperand(0, {GetScopeConstant(SpvScopeQueueFamilyKHR)}); |
| 694 |
} |
| 695 |
} |
| 696 |
}); |
| 697 |
} |
| 698 |
|
| 699 |
bool UpgradeMemoryModel::IsDeviceScope(uint32_t scope_id) { |
| 700 |
const analysis::Constant* constant = |
| 701 |
context()->get_constant_mgr()->FindDeclaredConstant(scope_id); |
| 702 |
assert(constant && "Memory scope must be a constant"); |
| 703 |
|
| 704 |
const analysis::Integer* type = constant->type()->AsInteger(); |
| 705 |
assert(type); |
| 706 |
assert(type->width() == 32 || type->width() == 64); |
| 707 |
if (type->width() == 32) { |
| 708 |
if (type->IsSigned()) |
| 709 |
return static_cast<uint32_t>(constant->GetS32()) == SpvScopeDevice; |
| 710 |
else |
| 711 |
return static_cast<uint32_t>(constant->GetU32()) == SpvScopeDevice; |
| 712 |
} else { |
| 713 |
if (type->IsSigned()) |
| 714 |
return static_cast<uint32_t>(constant->GetS64()) == SpvScopeDevice; |
| 715 |
else |
| 716 |
return static_cast<uint32_t>(constant->GetU64()) == SpvScopeDevice; |
| 717 |
} |
| 718 |
|
| 719 |
assert(false); |
| 720 |
return false; |
| 721 |
} |
| 722 |
|
| 723 |
void UpgradeMemoryModel::UpgradeExtInst(Instruction* ext_inst) { |
| 724 |
const bool is_modf = ext_inst->GetSingleWordInOperand(1u) == GLSLstd450Modf; |
| 725 |
auto ptr_id = ext_inst->GetSingleWordInOperand(3u); |
| 726 |
auto ptr_type_id = get_def_use_mgr()->GetDef(ptr_id)->type_id(); |
| 727 |
auto pointee_type_id = |
| 728 |
get_def_use_mgr()->GetDef(ptr_type_id)->GetSingleWordInOperand(1u); |
| 729 |
auto element_type_id = ext_inst->type_id(); |
| 730 |
std::vector<const analysis::Type*> element_types(2); |
| 731 |
element_types[0] = context()->get_type_mgr()->GetType(element_type_id); |
| 732 |
element_types[1] = context()->get_type_mgr()->GetType(pointee_type_id); |
| 733 |
analysis::Struct struct_type(element_types); |
| 734 |
uint32_t struct_id = |
| 735 |
context()->get_type_mgr()->GetTypeInstruction(&struct_type); |
| 736 |
// Change the operation |
| 737 |
GLSLstd450 new_op = is_modf ? GLSLstd450ModfStruct : GLSLstd450FrexpStruct; |
| 738 |
ext_inst->SetOperand(3u, {static_cast<uint32_t>(new_op)}); |
| 739 |
// Remove the pointer argument |
| 740 |
ext_inst->RemoveOperand(5u); |
| 741 |
// Set the type id to the new struct. |
| 742 |
ext_inst->SetResultType(struct_id); |
| 743 |
|
| 744 |
// The result is now a struct of the original result. The zero'th element is |
| 745 |
// old result and should replace the old result. The one'th element needs to |
| 746 |
// be stored via a new instruction. |
| 747 |
auto where = ext_inst->NextNode(); |
| 748 |
InstructionBuilder builder( |
| 749 |
context(), where, |
| 750 |
IRContext::kAnalysisDefUse | IRContext::kAnalysisInstrToBlockMapping); |
| 751 |
auto extract_0 = |
| 752 |
builder.AddCompositeExtract(element_type_id, ext_inst->result_id(), {0}); |
| 753 |
context()->ReplaceAllUsesWith(ext_inst->result_id(), extract_0->result_id()); |
| 754 |
// The extract's input was just changed to itself, so fix that. |
| 755 |
extract_0->SetInOperand(0u, {ext_inst->result_id()}); |
| 756 |
auto extract_1 = |
| 757 |
builder.AddCompositeExtract(pointee_type_id, ext_inst->result_id(), {1}); |
| 758 |
builder.AddStore(ptr_id, extract_1->result_id()); |
| 759 |
} |
| 760 |
|
| 761 |
uint32_t UpgradeMemoryModel::MemoryAccessNumWords(uint32_t mask) { |
| 762 |
uint32_t result = 1; |
| 763 |
if (mask & SpvMemoryAccessAlignedMask) ++result; |
| 764 |
if (mask & SpvMemoryAccessMakePointerAvailableKHRMask) ++result; |
| 765 |
if (mask & SpvMemoryAccessMakePointerVisibleKHRMask) ++result; |
| 766 |
return result; |
| 767 |
} |
| 768 |
|
| 769 |
} // namespace opt |
| 770 |
} // namespace spvtools |