Merge pull request #1707 from billhollings/SingleQueueSemaphore
Vulkan semaphore functional improvements.
diff --git a/Docs/Whats_New.md b/Docs/Whats_New.md
index 09569a5..1edf4a1 100644
--- a/Docs/Whats_New.md
+++ b/Docs/Whats_New.md
@@ -20,6 +20,10 @@
- Add support for extensions:
- `VK_KHR_shader_float_controls`
+- Vulkan semaphore functional improvements:
+ - Replace use of `MTLFence` with an option to limit to a single Vulkan queue and use Metal's implicit submisison order guarantees.
+ - Support option to force use of `MTLEvents` for Vulkan semaphores on NVIDIA and Rosetta2.
+ - `MVKConfiguration` replace booleans `semaphoreUseMTLEvent` and `semaphoreUseMTLFence` with enumerated `semaphoreSupportStyle`.
- Support config option to automatically use Metal argument buffers when `VK_EXT_descriptor_indexing`
extension is enabled. `MVKConfiguration::useMetalArgumentBuffers` (`MVK_CONFIG_USE_METAL_ARGUMENT_BUFFERS`)
is now an enum field. The use of Metal argument buffers is still disabled by default (`MVK_CONFIG_USE_METAL_ARGUMENT_BUFFERS_NEVER`).
diff --git a/MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h b/MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h
index 11aed6b..683a435 100644
--- a/MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h
+++ b/MoltenVK/MoltenVK/API/vk_mvk_moltenvk.h
@@ -104,6 +104,15 @@
MVK_CONFIG_USE_METAL_ARGUMENT_BUFFERS_MAX_ENUM = 0x7FFFFFFF
} MVKUseMetalArgumentBuffers;
+/** Identifies the Metal functionality used to support Vulkan semaphore functionality (VkSemaphore). */
+typedef enum MVKVkSemaphoreSupportStyle {
+ MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_CALLBACK = 0, /**< Use CPU callbacks upon GPU submission completion. This is the slowest technique. */
+ MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_METAL_EVENTS_WHERE_SAFE = 1, /**< Use Metal events (MTLEvent) when available on the platform, and where safe. This will revert to same as MVK_CONFIG_VK_SEMAPHORE_USE_SINGLE_QUEUE on some NVIDIA GPUs and Rosetta2, due to potential challenges with MTLEvents on those platforms. */
+ MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_METAL_EVENTS = 2, /**< Always use Metal events (MTLEvent) when available on the platform. */
+ MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_SINGLE_QUEUE = 3, /**< Limit Vulkan to a single queue, with no explicit semaphore synchronization, and use Metal's implicit guarantees that all operations submitted to a queue will give the same result as if they had been run in submission order. */
+ MVK_CONFIG_VK_SEMAPHORE_MAX_ENUM = 0x7FFFFFFF
+} MVKVkSemaphoreSupportStyle;
+
/**
* MoltenVK configuration settings.
*
@@ -565,50 +574,37 @@
*/
VkBool32 forceLowPowerGPU;
- /**
- * Use Metal's implicit guarantees that all operations submitted to a queue will give the same result as
- * if they had been run in submission order to implement VkSemaphore synchronization as no-ops.
- *
- * This requires all submissions be made to the same queue, and to guarantee that, MoltenVK will expose
- * only one queue to the application.
- *
- * In the special case of VK_SEMAPHORE_TYPE_TIMELINE semaphores, MoltenVK will always
- * use MTLSharedEvent if it is available on the platform, regardless of the values of
- * semaphoreUseMTLEvent or semaphoreUseMTLFence.
- *
- * The value of this parameter must be changed before creating a VkDevice for the change to take effect.
- *
- * The initial value or this parameter is set by the
- * MVK_ALLOW_SINGLE_QUEUE_SEMAPHORE
- * runtime environment variable or MoltenVK compile-time build setting.
- * If neither is set, this setting is enabled by default, and VkSemaphore will force a single queue
- * on NVIDIA GPUs and whenever MVK_ALLOW_METAL_EVENTS is not also set.
- */
- VkBool32 semaphoreUseSingleQueue;
+ /** Deprecated. Use semaphoreSupportStyle instead. */
+ VkBool32 semaphoreUseMTLFence;
/**
- * Use MTLEvent, if it is available on the device, for VkSemaphore synchronization behaviour.
+ * Determines the style used to implement Vulkan semaphore (VkSemaphore) functionality in Metal.
+ * See the documentation of the MVKVkSemaphoreSupportStyle for the options.
*
- * This parameter interacts with semaphoreUseSingleQueue. If both are enabled, on GPUs other than
- * NVIDIA, semaphoreUseMTLEvent takes priority and MTLEvent will be used if it is available,
- * otherwise MTLFence will be used if it is available. On NVIDIA GPUs, MTLEvent is disabled
- * for VkSemaphores, so CPU-based synchronization will be used unless semaphoreUseSingleQueue
- * is enabled.
+ * In the special case of VK_SEMAPHORE_TYPE_TIMELINE semaphores, MoltenVK will always use
+ * MTLSharedEvent if it is available on the platform, regardless of the value of this parameter.
*
- * In the special case of VK_SEMAPHORE_TYPE_TIMELINE semaphores, MoltenVK will always
- * use MTLSharedEvent if it is available on the platform, regardless of the values of
- * semaphoreUseMTLEvent or semaphoreUseMTLFence.
- *
- * The value of this parameter must be changed before creating a VkDevice,
+ * The value of this parameter must be changed before creating a VkInstance,
* for the change to take effect.
*
* The initial value or this parameter is set by the
- * MVK_ALLOW_METAL_EVENTS
+ * MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE
* runtime environment variable or MoltenVK compile-time build setting.
- * If neither is set, this setting is enabled by default, and VkSemaphore will use MTLEvent,
- * if it is available, except on NVIDIA GPUs.
+ * If neither is set, this setting is set to
+ * MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_METAL_EVENTS_WHERE_SAFE by default,
+ * and MoltenVK will use MTLEvent, except on NVIDIA GPU, and Rosetta2 environments,
+ * where it will use a single queue with implicit synchronization
+ * (as if this parameter was set to MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_SINGLE_QUEUE).
+ *
+ * This parameter interacts with the deprecated legacy parameters semaphoreUseMTLEvent
+ * and semaphoreUseMTLFence. If semaphoreUseMTLEvent is enabled, this parameter
+ * will be set to MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_METAL_EVENTS_WHERE_SAFE.
+ * If semaphoreUseMTLEvent is disabled, and semaphoreUseMTLFence is enabled,
+ * this parameter will be set to MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_SINGLE_QUEUE.
+ * Structurally, this parameter replaces, and is aliased by, semaphoreUseMTLEvent.
*/
- VkBool32 semaphoreUseMTLEvent;
+ MVKVkSemaphoreSupportStyle semaphoreSupportStyle;
+#define semaphoreUseMTLEvent semaphoreSupportStyle
/**
* Controls whether Metal should run an automatic GPU capture without the user having to
diff --git a/MoltenVK/MoltenVK/GPUObjects/MVKDevice.h b/MoltenVK/MoltenVK/GPUObjects/MVKDevice.h
index 3ce2caa..d378fd9 100644
--- a/MoltenVK/MoltenVK/GPUObjects/MVKDevice.h
+++ b/MoltenVK/MoltenVK/GPUObjects/MVKDevice.h
@@ -403,6 +403,7 @@
void initLimits();
void initGPUInfoProperties();
void initMemoryProperties();
+ void initVkSemaphoreStyle();
void setMemoryHeap(uint32_t heapIndex, VkDeviceSize heapSize, VkMemoryHeapFlags heapFlags);
void setMemoryType(uint32_t typeIndex, uint32_t heapIndex, VkMemoryPropertyFlags propertyFlags);
uint64_t getVRAMSize();
@@ -413,7 +414,6 @@
void initExtensions();
void initCounterSets();
bool needsCounterSetRetained();
- MVKSemaphoreStyle getSemaphoreStyle();
MVKArrayRef<MVKQueueFamily*> getQueueFamilies();
void initPipelineCacheUUID();
uint32_t getHighestGPUCapability();
@@ -434,6 +434,7 @@
MVKSmallVector<MVKQueueFamily*, kMVKQueueFamilyCount> _queueFamilies;
MVKPixelFormats _pixelFormats;
id<MTLCounterSet> _timestampMTLCounterSet;
+ MVKSemaphoreStyle _vkSemaphoreStyle;
uint32_t _allMemoryTypes;
uint32_t _hostVisibleMemoryTypes;
uint32_t _hostCoherentMemoryTypes;
@@ -885,7 +886,6 @@
id<MTLBuffer> _globalVisibilityResultMTLBuffer = nil;
id<MTLSamplerState> _defaultMTLSamplerState = nil;
id<MTLBuffer> _dummyBlitMTLBuffer = nil;
- MVKSemaphoreStyle _vkSemaphoreStyle = MVKSemaphoreStyleUseEmulation;
uint32_t _globalVisibilityQueryCount = 0;
bool _logActivityPerformanceInline = false;
bool _isPerformanceTracking = false;
diff --git a/MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm b/MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm
index dc13832..f43f942 100644
--- a/MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm
+++ b/MoltenVK/MoltenVK/GPUObjects/MVKDevice.mm
@@ -1311,8 +1311,7 @@
// Single queue semaphore requires using a single queue for everything
// So don't allow anyone to have more than one
- if (getSemaphoreStyle() != MVKSemaphoreStyleSingleQueue)
- {
+ if (_vkSemaphoreStyle != MVKSemaphoreStyleSingleQueue) {
// Dedicated graphics queue family...or another general-purpose queue family.
if (specialize) { qfProps.queueFlags = (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_TRANSFER_BIT); }
_queueFamilies.push_back(new MVKQueueFamily(this, qfIdx++, &qfProps));
@@ -1445,6 +1444,7 @@
initMemoryProperties();
initExternalMemoryProperties();
initCounterSets();
+ initVkSemaphoreStyle();
logGPUInfo();
}
@@ -3115,21 +3115,33 @@
}
}
-MVKSemaphoreStyle MVKPhysicalDevice::getSemaphoreStyle() {
- // Decide whether Vulkan semaphores should use a MTLEvent or forcing a single queue if they are available.
- // Prefer MTLEvent, because MTLEvent handles sync across MTLCommandBuffers and MTLCommandQueues.
- // However, do not allow use of MTLEvents on Rosetta2 (x86 build on M1 runtime) or NVIDIA GPUs,
- // which have demonstrated trouble with MTLEvents. In that case, since a single queue will be used
- // unless the option for it it has been disabled, in which case CPU emulation will be used
- bool isNVIDIA = _properties.vendorID == kNVVendorId;
- bool isRosetta2 = _properties.vendorID == kAppleVendorId && !MVK_APPLE_SILICON;
- if (_metalFeatures.events && mvkConfig().semaphoreUseMTLEvent && !(isRosetta2 || isNVIDIA)) {
- return MVKSemaphoreStyleUseMTLEvent;
+// Determine whether Vulkan semaphores should use a MTLEvent, CPU callbacks, or should limit
+// Vulkan to a single queue and use Metal's implicit guarantees that all operations submitted
+// to a queue will give the same result as if they had been run in submission order.
+// MTLEvents for semaphores can sometimes prove troublesome on some platforms,
+// and so may optionally be disabled on those platforms.
+void MVKPhysicalDevice::initVkSemaphoreStyle() {
+
+ // Default to CPU callback if other options unavailable.
+ _vkSemaphoreStyle = MVKSemaphoreStyleUseEmulation;
+
+ switch (mvkConfig().semaphoreSupportStyle) {
+ case MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_METAL_EVENTS_WHERE_SAFE: {
+ bool isNVIDIA = _properties.vendorID == kNVVendorId;
+ bool isRosetta2 = _properties.vendorID == kAppleVendorId && !MVK_APPLE_SILICON;
+ if (_metalFeatures.events && !(isRosetta2 || isNVIDIA)) { _vkSemaphoreStyle = MVKSemaphoreStyleUseMTLEvent; }
+ break;
+ }
+ case MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_METAL_EVENTS:
+ if (_metalFeatures.events) { _vkSemaphoreStyle = MVKSemaphoreStyleUseMTLEvent; }
+ break;
+ case MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_SINGLE_QUEUE:
+ _vkSemaphoreStyle = MVKSemaphoreStyleSingleQueue;
+ break;
+ case MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_CALLBACK:
+ default:
+ break;
}
- if (mvkConfig().semaphoreUseSingleQueue) {
- return MVKSemaphoreStyleSingleQueue;
- }
- return MVKSemaphoreStyleUseEmulation;
}
// Workaround for a bug in Intel Iris Plus Graphics driver where the counterSets array is
@@ -3664,7 +3676,7 @@
return new MVKTimelineSemaphoreEmulated(this, pCreateInfo, pTypeCreateInfo, pExportInfo, pImportInfo);
}
} else {
- switch (_vkSemaphoreStyle) {
+ switch (_physicalDevice->_vkSemaphoreStyle) {
case MVKSemaphoreStyleUseMTLEvent: return new MVKSemaphoreMTLEvent(this, pCreateInfo, pExportInfo, pImportInfo);
case MVKSemaphoreStyleUseEmulation: return new MVKSemaphoreEmulated(this, pCreateInfo, pExportInfo, pImportInfo);
case MVKSemaphoreStyleSingleQueue: return new MVKSemaphoreSingleQueue(this, pCreateInfo, pExportInfo, pImportInfo);
@@ -4460,16 +4472,15 @@
_pProperties = &_physicalDevice->_properties;
_pMemoryProperties = &_physicalDevice->_memoryProperties;
- _vkSemaphoreStyle = _physicalDevice->getSemaphoreStyle();
- switch (_vkSemaphoreStyle) {
+ switch (_physicalDevice->_vkSemaphoreStyle) {
case MVKSemaphoreStyleUseMTLEvent:
- MVKLogInfo("Using MTLEvent for Vulkan semaphores.");
+ MVKLogInfo("Vulkan semaphores using MTLEvent.");
break;
case MVKSemaphoreStyleUseEmulation:
- MVKLogInfo("Using emulation for Vulkan semaphores.");
+ MVKLogInfo("Vulkan semaphores using CPU callbacks upon GPU submission completion.");
break;
case MVKSemaphoreStyleSingleQueue:
- MVKLogInfo("Using Metal implicit guarantees within a single queue for Vulkan semaphores.");
+ MVKLogInfo("Vulkan semaphores using Metal implicit guarantees within a single queue.");
break;
}
}
diff --git a/MoltenVK/MoltenVK/Utility/MVKEnvironment.cpp b/MoltenVK/MoltenVK/Utility/MVKEnvironment.cpp
index 56876d3..f28af09 100644
--- a/MoltenVK/MoltenVK/Utility/MVKEnvironment.cpp
+++ b/MoltenVK/MoltenVK/Utility/MVKEnvironment.cpp
@@ -46,12 +46,11 @@
MVK_SET_FROM_ENV_OR_BUILD_BOOL (evCfg.defaultGPUCaptureScopeQueueFamilyIndex, MVK_CONFIG_DEFAULT_GPU_CAPTURE_SCOPE_QUEUE_FAMILY_INDEX);
MVK_SET_FROM_ENV_OR_BUILD_BOOL (evCfg.defaultGPUCaptureScopeQueueIndex, MVK_CONFIG_DEFAULT_GPU_CAPTURE_SCOPE_QUEUE_INDEX);
MVK_SET_FROM_ENV_OR_BUILD_BOOL (evCfg.fastMathEnabled, MVK_CONFIG_FAST_MATH_ENABLED);
-
MVK_SET_FROM_ENV_OR_BUILD_INT32 (evCfg.logLevel, MVK_CONFIG_LOG_LEVEL);
MVK_SET_FROM_ENV_OR_BUILD_INT32 (evCfg.traceVulkanCalls, MVK_CONFIG_TRACE_VULKAN_CALLS);
MVK_SET_FROM_ENV_OR_BUILD_BOOL (evCfg.forceLowPowerGPU, MVK_CONFIG_FORCE_LOW_POWER_GPU);
- MVK_SET_FROM_ENV_OR_BUILD_BOOL (evCfg.semaphoreUseSingleQueue, MVK_ALLOW_SINGLE_QUEUE_SEMAPHORE);
- MVK_SET_FROM_ENV_OR_BUILD_BOOL (evCfg.semaphoreUseMTLEvent, MVK_ALLOW_METAL_EVENTS);
+ MVK_SET_FROM_ENV_OR_BUILD_BOOL (evCfg.semaphoreUseMTLFence, MVK_ALLOW_METAL_FENCES);
+ MVK_SET_FROM_ENV_OR_BUILD_INT32 (evCfg.semaphoreSupportStyle, MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE);
MVK_SET_FROM_ENV_OR_BUILD_INT32 (evCfg.autoGPUCaptureScope, MVK_CONFIG_AUTO_GPU_CAPTURE_SCOPE);
MVK_SET_FROM_ENV_OR_BUILD_STRING(evCfg.autoGPUCaptureOutputFilepath, MVK_CONFIG_AUTO_GPU_CAPTURE_OUTPUT_FILE, evGPUCapFileStrObj);
MVK_SET_FROM_ENV_OR_BUILD_BOOL (evCfg.texture1DAs2D, MVK_CONFIG_TEXTURE_1D_AS_2D);
@@ -63,6 +62,18 @@
MVK_SET_FROM_ENV_OR_BUILD_BOOL (evCfg.resumeLostDevice, MVK_CONFIG_RESUME_LOST_DEVICE);
MVK_SET_FROM_ENV_OR_BUILD_INT32 (evCfg.useMetalArgumentBuffers, MVK_CONFIG_USE_METAL_ARGUMENT_BUFFERS);
+ // Deprected legacy VkSemaphore MVK_ALLOW_METAL_FENCES and MVK_ALLOW_METAL_EVENTS config.
+ // Legacy MVK_ALLOW_METAL_EVENTS is covered by MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE,
+ // but for backwards compatibility, if legacy MVK_ALLOW_METAL_EVENTS is explicitly
+ // disabled, disable semaphoreUseMTLEvent (aliased as semaphoreSupportStyle value
+ // MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_CALLBACK), and let mvkSetConfig() further
+ // process legacy behavior based on the value of legacy semaphoreUseMTLFence).
+ bool sem4UseMTLEvent;
+ MVK_SET_FROM_ENV_OR_BUILD_BOOL(sem4UseMTLEvent, MVK_ALLOW_METAL_EVENTS);
+ if ( !sem4UseMTLEvent ) {
+ evCfg.semaphoreUseMTLEvent = (MVKVkSemaphoreSupportStyle)false; // Disabled. Also semaphoreSupportStyle MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_CALLBACK.
+ }
+
mvkSetConfig(evCfg);
}
@@ -90,7 +101,14 @@
_mvkConfig.apiVersionToAdvertise = VK_MAKE_VERSION(VK_VERSION_MAJOR(_mvkConfig.apiVersionToAdvertise),
VK_VERSION_MINOR(_mvkConfig.apiVersionToAdvertise),
VK_HEADER_VERSION);
-
+
+ // Deprecated legacy support for specific case where semaphoreUseMTLFence is enabled and legacy
+ // semaphoreUseMTLEvent (now aliased to semaphoreSupportStyle) is disabled. In this case the user
+ // had been using the legacy MTLFence, so use MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_SINGLE_QUEUE now.
+ if (_mvkConfig.semaphoreUseMTLFence && !_mvkConfig.semaphoreUseMTLEvent) {
+ _mvkConfig.semaphoreSupportStyle = MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_SINGLE_QUEUE;
+ }
+
// Set capture file path string
if (_mvkConfig.autoGPUCaptureOutputFilepath) {
_autoGPUCaptureOutputFile = _mvkConfig.autoGPUCaptureOutputFilepath;
diff --git a/MoltenVK/MoltenVK/Utility/MVKEnvironment.h b/MoltenVK/MoltenVK/Utility/MVKEnvironment.h
index 840381d..307038d 100644
--- a/MoltenVK/MoltenVK/Utility/MVKEnvironment.h
+++ b/MoltenVK/MoltenVK/Utility/MVKEnvironment.h
@@ -231,16 +231,17 @@
#endif
/**
- * Allow the use of a single queue or MTLEvent for VkSemaphore synchronization behaviour.
- * By default:
- * - MVK_ALLOW_METAL_EVENTS is enabled
- * - MVK_ALLOW_SINGLE_QUEUE_SEMAPHORE is enabled
- * */
-#ifndef MVK_ALLOW_METAL_EVENTS
+ * Determines the style used to implement Vulkan semaphore (VkSemaphore) functionality in Metal.
+ * By default, use Metal events, if availalble, on most platforms.
+ */
+#ifndef MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE
+# define MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE_METAL_EVENTS_WHERE_SAFE
+#endif
+#ifndef MVK_ALLOW_METAL_EVENTS // Deprecated
# define MVK_ALLOW_METAL_EVENTS 1
#endif
-#ifndef MVK_ALLOW_SINGLE_QUEUE_SEMAPHORE
-# define MVK_ALLOW_SINGLE_QUEUE_SEMAPHORE 1
+#ifndef MVK_ALLOW_METAL_FENCES // Deprecated
+# define MVK_ALLOW_METAL_FENCES 0
#endif
/** Substitute Metal 2D textures for Vulkan 1D images. Enabled by default. */
diff --git a/Scripts/runcts b/Scripts/runcts
index 540723f..6a8761d 100755
--- a/Scripts/runcts
+++ b/Scripts/runcts
@@ -108,6 +108,7 @@
export MVK_CONFIG_FAST_MATH_ENABLED=1
export MVK_CONFIG_USE_METAL_ARGUMENT_BUFFERS=0 #(2 = VK_EXT_descriptor_indexing enabled)
export MVK_CONFIG_FORCE_LOW_POWER_GPU=0
+export MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE=2 #(2 = MTLEvents always)
# -------------- Operation --------------------
