Mem model spv 1.4 (#2565)
* Update memory model support for SPIR-V 1.4
Fixes #2552
* Upgrade memory model now supports two memory access operands for
OpCopyMemory*
* in all cases the pass will first generate two operands by either
adding them or copying
* updates accounts for multiple operands
* tests
diff --git a/source/opt/upgrade_memory_model.cpp b/source/opt/upgrade_memory_model.cpp
index d8836a4..31c0abd 100644
--- a/source/opt/upgrade_memory_model.cpp
+++ b/source/opt/upgrade_memory_model.cpp
@@ -18,6 +18,7 @@
#include "source/opt/ir_builder.h"
#include "source/opt/ir_context.h"
+#include "source/spirv_constant.h"
#include "source/util/make_unique.h"
namespace spvtools {
@@ -70,6 +71,7 @@
// parameters are implicitly coherent in GLSL450.
// Upgrade modf and frexp first since they generate new stores.
+ // In SPIR-V 1.4 or later, normalize OpCopyMemory* access operands.
for (auto& func : *get_module()) {
func.ForEachInst([this](Instruction* inst) {
if (inst->opcode() == SpvOpExtInst) {
@@ -82,6 +84,29 @@
UpgradeExtInst(inst);
}
}
+ } else if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) {
+ if (inst->opcode() == SpvOpCopyMemory ||
+ inst->opcode() == SpvOpCopyMemorySized) {
+ uint32_t start_operand = inst->opcode() == SpvOpCopyMemory ? 2u : 3u;
+ if (inst->NumInOperands() > start_operand) {
+ auto num_access_words = MemoryAccessNumWords(
+ inst->GetSingleWordInOperand(start_operand));
+ if ((num_access_words + start_operand) == inst->NumInOperands()) {
+ // There is a single memory access operand. Duplicate it to have a
+ // separate operand for both source and target.
+ for (uint32_t i = 0; i < num_access_words; ++i) {
+ auto operand = inst->GetInOperand(start_operand + i);
+ inst->AddOperand(std::move(operand));
+ }
+ }
+ } else {
+ // Add two memory access operands.
+ inst->AddOperand(
+ {SPV_OPERAND_TYPE_MEMORY_ACCESS, {SpvMemoryAccessMaskNone}});
+ inst->AddOperand(
+ {SPV_OPERAND_TYPE_MEMORY_ACCESS, {SpvMemoryAccessMaskNone}});
+ }
+ }
}
});
}
@@ -93,6 +118,7 @@
bool src_volatile = false;
bool dst_coherent = false;
bool dst_volatile = false;
+ uint32_t start_operand = 0u;
SpvScope scope = SpvScopeQueueFamilyKHR;
SpvScope src_scope = SpvScopeQueueFamilyKHR;
SpvScope dst_scope = SpvScopeQueueFamilyKHR;
@@ -129,16 +155,23 @@
kMemory);
break;
case SpvOpCopyMemory:
- UpgradeFlags(inst, 2u, dst_coherent, dst_volatile, kAvailability,
- kMemory);
- UpgradeFlags(inst, 2u, src_coherent, src_volatile, kVisibility,
- kMemory);
- break;
case SpvOpCopyMemorySized:
- UpgradeFlags(inst, 3u, dst_coherent, dst_volatile, kAvailability,
- kMemory);
- UpgradeFlags(inst, 3u, src_coherent, src_volatile, kVisibility,
- kMemory);
+ start_operand = inst->opcode() == SpvOpCopyMemory ? 2u : 3u;
+ if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) {
+ // There are guaranteed to be two memory access operands at this
+ // point so treat source and target separately.
+ uint32_t num_access_words = MemoryAccessNumWords(
+ inst->GetSingleWordInOperand(start_operand));
+ UpgradeFlags(inst, start_operand, dst_coherent, dst_volatile,
+ kAvailability, kMemory);
+ UpgradeFlags(inst, start_operand + num_access_words, src_coherent,
+ src_volatile, kVisibility, kMemory);
+ } else {
+ UpgradeFlags(inst, start_operand, dst_coherent, dst_volatile,
+ kAvailability, kMemory);
+ UpgradeFlags(inst, start_operand, src_coherent, src_volatile,
+ kVisibility, kMemory);
+ }
break;
case SpvOpImageRead:
case SpvOpImageSparseRead:
@@ -158,16 +191,49 @@
inst->AddOperand(
{SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(scope)}});
}
- // According to SPV_KHR_vulkan_memory_model, if both available and
- // visible flags are used the first scope operand is for availability
- // (writes) and the second is for visibility (reads).
- if (dst_coherent) {
- inst->AddOperand(
- {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(dst_scope)}});
- }
- if (src_coherent) {
- inst->AddOperand(
- {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(src_scope)}});
+ if (get_module()->version() >= SPV_SPIRV_VERSION_WORD(1, 4)) {
+ // There are two memory access operands. The first is for the target and
+ // the second is for the source.
+ if (dst_coherent || src_coherent) {
+ start_operand = inst->opcode() == SpvOpCopyMemory ? 2u : 3u;
+ std::vector<Operand> new_operands;
+ uint32_t num_access_words =
+ MemoryAccessNumWords(inst->GetSingleWordInOperand(start_operand));
+ // The flags were already updated so subtract if we're adding a
+ // scope.
+ if (dst_coherent) --num_access_words;
+ for (uint32_t i = 0; i < start_operand + num_access_words; ++i) {
+ new_operands.push_back(inst->GetInOperand(i));
+ }
+ // Add the target scope if necessary.
+ if (dst_coherent) {
+ new_operands.push_back(
+ {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(dst_scope)}});
+ }
+ // Copy the remaining current operands.
+ for (uint32_t i = start_operand + num_access_words;
+ i < inst->NumInOperands(); ++i) {
+ new_operands.push_back(inst->GetInOperand(i));
+ }
+ // Add the source scope if necessary.
+ if (src_coherent) {
+ new_operands.push_back(
+ {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(src_scope)}});
+ }
+ inst->SetInOperands(std::move(new_operands));
+ }
+ } else {
+ // According to SPV_KHR_vulkan_memory_model, if both available and
+ // visible flags are used the first scope operand is for availability
+ // (writes) and the second is for visibility (reads).
+ if (dst_coherent) {
+ inst->AddOperand(
+ {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(dst_scope)}});
+ }
+ if (src_coherent) {
+ inst->AddOperand(
+ {SPV_OPERAND_TYPE_SCOPE_ID, {GetScopeConstant(src_scope)}});
+ }
}
});
}
@@ -636,5 +702,13 @@
builder.AddStore(ptr_id, extract_1->result_id());
}
+uint32_t UpgradeMemoryModel::MemoryAccessNumWords(uint32_t mask) {
+ uint32_t result = 1;
+ if (mask & SpvMemoryAccessAlignedMask) ++result;
+ if (mask & SpvMemoryAccessMakePointerAvailableKHRMask) ++result;
+ if (mask & SpvMemoryAccessMakePointerVisibleKHRMask) ++result;
+ return result;
+}
+
} // namespace opt
} // namespace spvtools
diff --git a/source/opt/upgrade_memory_model.h b/source/opt/upgrade_memory_model.h
index 9adc33b..0dcd4fd 100644
--- a/source/opt/upgrade_memory_model.h
+++ b/source/opt/upgrade_memory_model.h
@@ -15,11 +15,11 @@
#ifndef LIBSPIRV_OPT_UPGRADE_MEMORY_MODEL_H_
#define LIBSPIRV_OPT_UPGRADE_MEMORY_MODEL_H_
-#include "pass.h"
-
#include <functional>
#include <tuple>
+#include "pass.h"
+
namespace spvtools {
namespace opt {
@@ -123,6 +123,10 @@
// facilitate adding memory model flags.
void UpgradeExtInst(Instruction* modf);
+ // Returns the number of words taken up by a memory access argument and its
+ // implied operands.
+ uint32_t MemoryAccessNumWords(uint32_t mask);
+
// Caches the result of TraceInstruction. For a given result id and set of
// indices, stores whether that combination is coherent and/or volatile.
std::unordered_map<std::pair<uint32_t, std::vector<uint32_t>>,
diff --git a/test/opt/upgrade_memory_model_test.cpp b/test/opt/upgrade_memory_model_test.cpp
index 9d2d762..8de7e02 100644
--- a/test/opt/upgrade_memory_model_test.cpp
+++ b/test/opt/upgrade_memory_model_test.cpp
@@ -1713,4 +1713,328 @@
SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
}
+TEST_F(UpgradeMemoryModelTest, SPV14NormalizeCopyMemoryAddOperands) {
+ const std::string text = R"(
+; CHECK: OpCopyMemory {{%\w+}} {{%\w+}} None None
+OpCapability Shader
+OpMemoryModel Logical GLSL450
+OpEntryPoint GLCompute %func "func" %src %dst
+%void = OpTypeVoid
+%int = OpTypeInt 32 0
+%ptr_ssbo_int = OpTypePointer StorageBuffer %int
+%src = OpVariable %ptr_ssbo_int StorageBuffer
+%dst = OpVariable %ptr_ssbo_int StorageBuffer
+%void_fn = OpTypeFunction %void
+%func = OpFunction %void None %void_fn
+%entry = OpLabel
+OpCopyMemory %dst %src
+OpReturn
+OpFunctionEnd
+)";
+
+ SetTargetEnv(SPV_ENV_UNIVERSAL_1_4);
+ SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
+}
+
+TEST_F(UpgradeMemoryModelTest, SPV14NormalizeCopyMemoryDuplicateOperand) {
+ const std::string text = R"(
+; CHECK: OpCopyMemory {{%\w+}} {{%\w+}} Nontemporal Nontemporal
+OpCapability Shader
+OpMemoryModel Logical GLSL450
+OpEntryPoint GLCompute %func "func" %src %dst
+%void = OpTypeVoid
+%int = OpTypeInt 32 0
+%ptr_ssbo_int = OpTypePointer StorageBuffer %int
+%src = OpVariable %ptr_ssbo_int StorageBuffer
+%dst = OpVariable %ptr_ssbo_int StorageBuffer
+%void_fn = OpTypeFunction %void
+%func = OpFunction %void None %void_fn
+%entry = OpLabel
+OpCopyMemory %dst %src Nontemporal
+OpReturn
+OpFunctionEnd
+)";
+
+ SetTargetEnv(SPV_ENV_UNIVERSAL_1_4);
+ SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
+}
+
+TEST_F(UpgradeMemoryModelTest, SPV14NormalizeCopyMemoryDuplicateOperands) {
+ const std::string text = R"(
+; CHECK: OpCopyMemory {{%\w+}} {{%\w+}} Aligned 4 Aligned 4
+OpCapability Shader
+OpMemoryModel Logical GLSL450
+OpEntryPoint GLCompute %func "func" %src %dst
+%void = OpTypeVoid
+%int = OpTypeInt 32 0
+%ptr_ssbo_int = OpTypePointer StorageBuffer %int
+%src = OpVariable %ptr_ssbo_int StorageBuffer
+%dst = OpVariable %ptr_ssbo_int StorageBuffer
+%void_fn = OpTypeFunction %void
+%func = OpFunction %void None %void_fn
+%entry = OpLabel
+OpCopyMemory %dst %src Aligned 4
+OpReturn
+OpFunctionEnd
+)";
+
+ SetTargetEnv(SPV_ENV_UNIVERSAL_1_4);
+ SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
+}
+
+TEST_F(UpgradeMemoryModelTest, SPV14CopyMemoryDstCoherent) {
+ const std::string text = R"(
+; CHECK: [[scope:%\w+]] = OpConstant {{%\w+}} 5
+; CHECK: OpCopyMemory {{%\w+}} {{%\w+}} MakePointerAvailableKHR|NonPrivatePointerKHR [[scope]] None
+OpCapability Shader
+OpMemoryModel Logical GLSL450
+OpEntryPoint GLCompute %func "func" %src %dst
+OpDecorate %dst Coherent
+%void = OpTypeVoid
+%int = OpTypeInt 32 0
+%ptr_ssbo_int = OpTypePointer StorageBuffer %int
+%src = OpVariable %ptr_ssbo_int StorageBuffer
+%dst = OpVariable %ptr_ssbo_int StorageBuffer
+%void_fn = OpTypeFunction %void
+%func = OpFunction %void None %void_fn
+%entry = OpLabel
+OpCopyMemory %dst %src
+OpReturn
+OpFunctionEnd
+)";
+
+ SetTargetEnv(SPV_ENV_UNIVERSAL_1_4);
+ SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
+}
+
+TEST_F(UpgradeMemoryModelTest, SPV14CopyMemoryDstCoherentPreviousArgs) {
+ const std::string text = R"(
+; CHECK: [[scope:%\w+]] = OpConstant {{%\w+}} 5
+; CHECK: OpCopyMemory {{%\w+}} {{%\w+}} Aligned|MakePointerAvailableKHR|NonPrivatePointerKHR 4 [[scope]] Aligned 4
+OpCapability Shader
+OpMemoryModel Logical GLSL450
+OpEntryPoint GLCompute %func "func" %src %dst
+OpDecorate %dst Coherent
+%void = OpTypeVoid
+%int = OpTypeInt 32 0
+%ptr_ssbo_int = OpTypePointer StorageBuffer %int
+%src = OpVariable %ptr_ssbo_int StorageBuffer
+%dst = OpVariable %ptr_ssbo_int StorageBuffer
+%void_fn = OpTypeFunction %void
+%func = OpFunction %void None %void_fn
+%entry = OpLabel
+OpCopyMemory %dst %src Aligned 4
+OpReturn
+OpFunctionEnd
+)";
+
+ SetTargetEnv(SPV_ENV_UNIVERSAL_1_4);
+ SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
+}
+
+TEST_F(UpgradeMemoryModelTest, SPV14CopyMemorySrcCoherent) {
+ const std::string text = R"(
+; CHECK: [[scope:%\w+]] = OpConstant {{%\w+}} 5
+; CHECK: OpCopyMemory {{%\w+}} {{%\w+}} None MakePointerVisibleKHR|NonPrivatePointerKHR [[scope]]
+OpCapability Shader
+OpMemoryModel Logical GLSL450
+OpEntryPoint GLCompute %func "func" %src %dst
+OpDecorate %src Coherent
+%void = OpTypeVoid
+%int = OpTypeInt 32 0
+%ptr_ssbo_int = OpTypePointer StorageBuffer %int
+%src = OpVariable %ptr_ssbo_int StorageBuffer
+%dst = OpVariable %ptr_ssbo_int StorageBuffer
+%void_fn = OpTypeFunction %void
+%func = OpFunction %void None %void_fn
+%entry = OpLabel
+OpCopyMemory %dst %src
+OpReturn
+OpFunctionEnd
+)";
+
+ SetTargetEnv(SPV_ENV_UNIVERSAL_1_4);
+ SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
+}
+
+TEST_F(UpgradeMemoryModelTest, SPV14CopyMemorySrcCoherentPreviousArgs) {
+ const std::string text = R"(
+; CHECK: [[scope:%\w+]] = OpConstant {{%\w+}} 5
+; CHECK: OpCopyMemory {{%\w+}} {{%\w+}} Aligned 4 Aligned|MakePointerVisibleKHR|NonPrivatePointerKHR 4 [[scope]]
+OpCapability Shader
+OpMemoryModel Logical GLSL450
+OpEntryPoint GLCompute %func "func" %src %dst
+OpDecorate %src Coherent
+%void = OpTypeVoid
+%int = OpTypeInt 32 0
+%ptr_ssbo_int = OpTypePointer StorageBuffer %int
+%src = OpVariable %ptr_ssbo_int StorageBuffer
+%dst = OpVariable %ptr_ssbo_int StorageBuffer
+%void_fn = OpTypeFunction %void
+%func = OpFunction %void None %void_fn
+%entry = OpLabel
+OpCopyMemory %dst %src Aligned 4
+OpReturn
+OpFunctionEnd
+)";
+
+ SetTargetEnv(SPV_ENV_UNIVERSAL_1_4);
+ SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
+}
+
+TEST_F(UpgradeMemoryModelTest, SPV14CopyMemoryBothCoherent) {
+ const std::string text = R"(
+; CHECK-DAG: [[queue:%\w+]] = OpConstant {{%\w+}} 5
+; CHECK-DAG: [[wg:%\w+]] = OpConstant {{%\w+}} 2
+; CHECK: OpCopyMemory {{%\w+}} {{%\w+}} MakePointerAvailableKHR|NonPrivatePointerKHR [[wg]] MakePointerVisibleKHR|NonPrivatePointerKHR [[queue]]
+OpCapability Shader
+OpMemoryModel Logical GLSL450
+OpEntryPoint GLCompute %func "func" %src %dst
+OpDecorate %src Coherent
+%void = OpTypeVoid
+%int = OpTypeInt 32 0
+%ptr_ssbo_int = OpTypePointer StorageBuffer %int
+%ptr_wg_int = OpTypePointer Workgroup %int
+%src = OpVariable %ptr_ssbo_int StorageBuffer
+%dst = OpVariable %ptr_wg_int Workgroup
+%void_fn = OpTypeFunction %void
+%func = OpFunction %void None %void_fn
+%entry = OpLabel
+OpCopyMemory %dst %src
+OpReturn
+OpFunctionEnd
+)";
+
+ SetTargetEnv(SPV_ENV_UNIVERSAL_1_4);
+ SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
+}
+
+TEST_F(UpgradeMemoryModelTest, SPV14CopyMemoryBothCoherentPreviousArgs) {
+ const std::string text = R"(
+; CHECK-DAG: [[queue:%\w+]] = OpConstant {{%\w+}} 5
+; CHECK-DAG: [[wg:%\w+]] = OpConstant {{%\w+}} 2
+; CHECK: OpCopyMemory {{%\w+}} {{%\w+}} Aligned|MakePointerAvailableKHR|NonPrivatePointerKHR 4 [[queue]] Aligned|MakePointerVisibleKHR|NonPrivatePointerKHR 4 [[wg]]
+OpCapability Shader
+OpMemoryModel Logical GLSL450
+OpEntryPoint GLCompute %func "func" %src %dst
+OpDecorate %dst Coherent
+%void = OpTypeVoid
+%int = OpTypeInt 32 0
+%ptr_ssbo_int = OpTypePointer StorageBuffer %int
+%ptr_wg_int = OpTypePointer Workgroup %int
+%src = OpVariable %ptr_wg_int Workgroup
+%dst = OpVariable %ptr_ssbo_int StorageBuffer
+%void_fn = OpTypeFunction %void
+%func = OpFunction %void None %void_fn
+%entry = OpLabel
+OpCopyMemory %dst %src Aligned 4
+OpReturn
+OpFunctionEnd
+)";
+
+ SetTargetEnv(SPV_ENV_UNIVERSAL_1_4);
+ SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
+}
+
+TEST_F(UpgradeMemoryModelTest, SPV14CopyMemoryBothVolatile) {
+ const std::string text = R"(
+; CHECK: OpCopyMemory {{%\w+}} {{%\w+}} Volatile Volatile
+OpCapability Shader
+OpMemoryModel Logical GLSL450
+OpEntryPoint GLCompute %func "func" %src %dst
+OpDecorate %src Volatile
+OpDecorate %dst Volatile
+%void = OpTypeVoid
+%int = OpTypeInt 32 0
+%ptr_ssbo_int = OpTypePointer StorageBuffer %int
+%src = OpVariable %ptr_ssbo_int StorageBuffer
+%dst = OpVariable %ptr_ssbo_int StorageBuffer
+%void_fn = OpTypeFunction %void
+%func = OpFunction %void None %void_fn
+%entry = OpLabel
+OpCopyMemory %dst %src
+OpReturn
+OpFunctionEnd
+)";
+
+ SetTargetEnv(SPV_ENV_UNIVERSAL_1_4);
+ SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
+}
+
+TEST_F(UpgradeMemoryModelTest, SPV14CopyMemoryBothVolatilePreviousArgs) {
+ const std::string text = R"(
+; CHECK: OpCopyMemory {{%\w+}} {{%\w+}} Volatile|Aligned 4 Volatile|Aligned 4
+OpCapability Shader
+OpMemoryModel Logical GLSL450
+OpEntryPoint GLCompute %func "func" %src %dst
+OpDecorate %src Volatile
+OpDecorate %dst Volatile
+%void = OpTypeVoid
+%int = OpTypeInt 32 0
+%ptr_ssbo_int = OpTypePointer StorageBuffer %int
+%src = OpVariable %ptr_ssbo_int StorageBuffer
+%dst = OpVariable %ptr_ssbo_int StorageBuffer
+%void_fn = OpTypeFunction %void
+%func = OpFunction %void None %void_fn
+%entry = OpLabel
+OpCopyMemory %dst %src Aligned 4
+OpReturn
+OpFunctionEnd
+)";
+
+ SetTargetEnv(SPV_ENV_UNIVERSAL_1_4);
+ SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
+}
+
+TEST_F(UpgradeMemoryModelTest, SPV14CopyMemoryDstCoherentTwoOperands) {
+ const std::string text = R"(
+; CHECK: [[scope:%\w+]] = OpConstant {{%\w+}} 5
+; CHECK: OpCopyMemory {{%\w+}} {{%\w+}} MakePointerAvailableKHR|NonPrivatePointerKHR [[scope]] None
+OpCapability Shader
+OpMemoryModel Logical GLSL450
+OpEntryPoint GLCompute %func "func" %src %dst
+OpDecorate %dst Coherent
+%void = OpTypeVoid
+%int = OpTypeInt 32 0
+%ptr_ssbo_int = OpTypePointer StorageBuffer %int
+%src = OpVariable %ptr_ssbo_int StorageBuffer
+%dst = OpVariable %ptr_ssbo_int StorageBuffer
+%void_fn = OpTypeFunction %void
+%func = OpFunction %void None %void_fn
+%entry = OpLabel
+OpCopyMemory %dst %src None None
+OpReturn
+OpFunctionEnd
+)";
+
+ SetTargetEnv(SPV_ENV_UNIVERSAL_1_4);
+ SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
+}
+
+TEST_F(UpgradeMemoryModelTest,
+ SPV14CopyMemoryDstCoherentPreviousArgsTwoOperands) {
+ const std::string text = R"(
+; CHECK: [[scope:%\w+]] = OpConstant {{%\w+}} 5
+; CHECK: OpCopyMemory {{%\w+}} {{%\w+}} Aligned|MakePointerAvailableKHR|NonPrivatePointerKHR 4 [[scope]] Aligned 8
+OpCapability Shader
+OpMemoryModel Logical GLSL450
+OpEntryPoint GLCompute %func "func" %src %dst
+OpDecorate %dst Coherent
+%void = OpTypeVoid
+%int = OpTypeInt 32 0
+%ptr_ssbo_int = OpTypePointer StorageBuffer %int
+%src = OpVariable %ptr_ssbo_int StorageBuffer
+%dst = OpVariable %ptr_ssbo_int StorageBuffer
+%void_fn = OpTypeFunction %void
+%func = OpFunction %void None %void_fn
+%entry = OpLabel
+OpCopyMemory %dst %src Aligned 4 Aligned 8
+OpReturn
+OpFunctionEnd
+)";
+
+ SetTargetEnv(SPV_ENV_UNIVERSAL_1_4);
+ SinglePassRunAndMatch<opt::UpgradeMemoryModel>(text, true);
+}
+
} // namespace
