device/vulkan.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | /// @file vulkan.cpp | ||
| 3 | /// @brief Vulkan results as device errors, probing physical devices against Ernie's requirements | ||
| 4 | /// and listing them. | ||
| 5 | #include "ernie/device/vulkan.h" | ||
| 6 | |||
| 7 | #include "ernie/device/external.h" | ||
| 8 | #include "ernie/device/instance.h" | ||
| 9 | #include "ernie/util/algorithm.h" | ||
| 10 | #include "ernie/util/text.h" | ||
| 11 | |||
| 12 | #include <algorithm> | ||
| 13 | #include <cstdio> | ||
| 14 | |||
| 15 | namespace ernie { | ||
| 16 | namespace { | ||
| 17 | |||
| 18 | /// @brief An optional extension and the capability it is part of. | ||
| 19 | struct Extension { | ||
| 20 | const char* name; | ||
| 21 | bool DeviceCapabilities::* capability; | ||
| 22 | }; | ||
| 23 | |||
| 24 | /// Every optional extension Ernie enables; Vulkan 1.3 core covers all it requires. | ||
| 25 | const std::array kExtensions{ | ||
| 26 | Extension{VK_KHR_ACCELERATION_STRUCTURE_EXTENSION_NAME, &DeviceCapabilities::rayQuery}, | ||
| 27 | Extension{VK_KHR_RAY_QUERY_EXTENSION_NAME, &DeviceCapabilities::rayQuery}, | ||
| 28 | Extension{VK_KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME, &DeviceCapabilities::rayQuery}, | ||
| 29 | Extension{kExternalMemoryExtension, &DeviceCapabilities::externalMemory}, | ||
| 30 | Extension{kExternalSemaphoreExtension, &DeviceCapabilities::externalTimeline}, | ||
| 31 | Extension{VK_EXT_MEMORY_BUDGET_EXTENSION_NAME, &DeviceCapabilities::memoryBudget}, | ||
| 32 | Extension{VK_EXT_DEVICE_FAULT_EXTENSION_NAME, &DeviceCapabilities::deviceFault}, | ||
| 33 | Extension{VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME, | ||
| 34 | &DeviceCapabilities::calibratedTimestamps}, | ||
| 35 | Extension{VK_KHR_SWAPCHAIN_EXTENSION_NAME, &DeviceCapabilities::present}, | ||
| 36 | }; | ||
| 37 | |||
| 38 | /// @brief Drops capabilities drivers report but get wrong, keyed by driver, all in one place. | ||
| 39 | /// Shader-level workarounds live beside the shaders they change; notes on them are here too: | ||
| 40 | /// - NVIDIA's compiler crashes on ray-query `CandidateObjectRay*`: procedural intersections take | ||
| 41 | /// the candidate's world-to-object matrix instead. | ||
| 42 | /// - Vulkan does not promise `fma` rounds once (lavapipe does not): double-float products split | ||
| 43 | /// their operands instead. | ||
| 44 | void WorkAroundDrivers(VkDriverId driver, DeviceCapabilities& capabilities) { | ||
| 45 | #if defined(__aarch64__) || defined(_M_ARM64) | ||
| 46 | // lavapipe on arm64 crashes or hangs building and tracing acceleration structures: Homebrew's | ||
| 47 | // on macOS, and Mesa 24's on Ubuntu 24.04. | ||
| 48 | if (driver == VK_DRIVER_ID_MESA_LLVMPIPE) { | ||
| 49 | capabilities.rayQuery = false; | ||
| 50 | } | ||
| 51 | #else | ||
| 52 | static_cast<void>(driver); | ||
| 53 | static_cast<void>(capabilities); | ||
| 54 | #endif | ||
| 55 | } | ||
| 56 | |||
| 57 | /// @brief A feature bit Ernie requires: its name, for the error naming it, and where it lives. | ||
| 58 | struct RequiredBit { | ||
| 59 | const char* name; | ||
| 60 | VkBool32* (*field)(FeatureChain& chain); | ||
| 61 | }; | ||
| 62 | |||
| 63 | #define ERNIE_REQUIRE(part, bit) \ | ||
| 64 | RequiredBit { \ | ||
| 65 | #bit, [](FeatureChain& f) { return &f.part.bit; } \ | ||
| 66 | } | ||
| 67 | |||
| 68 | const std::array kRequirements{ | ||
| 69 | ERNIE_REQUIRE(v12, bufferDeviceAddress), | ||
| 70 | ERNIE_REQUIRE(v12, timelineSemaphore), | ||
| 71 | ERNIE_REQUIRE(v12, scalarBlockLayout), | ||
| 72 | ERNIE_REQUIRE(v12, descriptorIndexing), | ||
| 73 | ERNIE_REQUIRE(v12, runtimeDescriptorArray), | ||
| 74 | ERNIE_REQUIRE(v12, descriptorBindingPartiallyBound), | ||
| 75 | ERNIE_REQUIRE(v12, descriptorBindingSampledImageUpdateAfterBind), | ||
| 76 | ERNIE_REQUIRE(v12, descriptorBindingStorageImageUpdateAfterBind), | ||
| 77 | ERNIE_REQUIRE(v12, descriptorBindingUpdateUnusedWhilePending), | ||
| 78 | ERNIE_REQUIRE(v12, shaderSampledImageArrayNonUniformIndexing), | ||
| 79 | ERNIE_REQUIRE(v12, shaderStorageImageArrayNonUniformIndexing), | ||
| 80 | ERNIE_REQUIRE(v13, synchronization2), | ||
| 81 | ERNIE_REQUIRE(v13, maintenance4), | ||
| 82 | ERNIE_REQUIRE(v13, dynamicRendering), | ||
| 83 | }; | ||
| 84 | |||
| 85 | #undef ERNIE_REQUIRE | ||
| 86 | |||
| 87 | 82 | VulkanDeviceType ToDeviceType(VkPhysicalDeviceType type) { | |
| 88 | 82 | switch (type) { | |
| 89 | case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: | ||
| 90 | return VulkanDeviceType::IntegratedGpu; | ||
| 91 | ✗ | case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: | |
| 92 | ✗ | return VulkanDeviceType::DiscreteGpu; | |
| 93 | ✗ | case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: | |
| 94 | ✗ | return VulkanDeviceType::VirtualGpu; | |
| 95 | 82 | case VK_PHYSICAL_DEVICE_TYPE_CPU: | |
| 96 | 82 | return VulkanDeviceType::Cpu; | |
| 97 | ✗ | default: | |
| 98 | ✗ | return VulkanDeviceType::Other; | |
| 99 | } | ||
| 100 | } | ||
| 101 | |||
| 102 | } // namespace | ||
| 103 | |||
| 104 | 1839 | void CheckVk(VkResult result, std::string_view what) { | |
| 105 | 1839 | if (result >= VK_SUCCESS) { | |
| 106 | 1839 | return; | |
| 107 | } | ||
| 108 | ✗ | const DeviceErrorKind kind = | |
| 109 | ✗ | result == VK_ERROR_OUT_OF_DEVICE_MEMORY || result == VK_ERROR_OUT_OF_HOST_MEMORY | |
| 110 | ✗ | ? DeviceErrorKind::OutOfMemory | |
| 111 | ✗ | : result == VK_ERROR_DEVICE_LOST ? DeviceErrorKind::DeviceLost | |
| 112 | ✗ | : result == VK_ERROR_FEATURE_NOT_PRESENT || result == VK_ERROR_EXTENSION_NOT_PRESENT || | |
| 113 | result == VK_ERROR_INCOMPATIBLE_DRIVER | ||
| 114 | ✗ | ? DeviceErrorKind::Unsupported | |
| 115 | : DeviceErrorKind::Failed; | ||
| 116 | ✗ | throw DeviceError(kind, std::string(what) + " failed (VkResult " + | |
| 117 | ✗ | std::to_string(static_cast<int>(result)) + ")"); | |
| 118 | } | ||
| 119 | |||
| 120 | 591 | void Barrier(VkCommandBuffer commands, VkPipelineStageFlags2 srcStage, VkAccessFlags2 srcAccess, | |
| 121 | VkPipelineStageFlags2 dstStage, VkAccessFlags2 dstAccess) { | ||
| 122 | 591 | const VkMemoryBarrier2 barrier{.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, | |
| 123 | .srcStageMask = srcStage, | ||
| 124 | .srcAccessMask = srcAccess, | ||
| 125 | .dstStageMask = dstStage, | ||
| 126 | 591 | .dstAccessMask = dstAccess}; | |
| 127 | 591 | const VkDependencyInfo dependency{.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, | |
| 128 | .memoryBarrierCount = 1, | ||
| 129 | 591 | .pMemoryBarriers = &barrier}; | |
| 130 | 591 | vkCmdPipelineBarrier2(commands, &dependency); | |
| 131 | 591 | } | |
| 132 | |||
| 133 | 29 | void CopyBuffers(VkCommandBuffer commands, std::span<const BufferCopies> copies) { | |
| 134 | 58 | if (std::ranges::all_of(copies, [](const BufferCopies& c) { return c.regions.empty(); })) { | |
| 135 | return; | ||
| 136 | } | ||
| 137 | 29 | Barrier(commands, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, kAnyAccess, | |
| 138 | VK_PIPELINE_STAGE_2_COPY_BIT, kAnyAccess); | ||
| 139 | 58 | for (const BufferCopies& copy : copies) { | |
| 140 | 29 | if (!copy.regions.empty()) { | |
| 141 | 29 | vkCmdCopyBuffer(commands, copy.source, copy.destination, | |
| 142 | 29 | static_cast<std::uint32_t>(copy.regions.size()), copy.regions.data()); | |
| 143 | } | ||
| 144 | } | ||
| 145 | 29 | Barrier(commands, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT, | |
| 146 | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, kAnyAccess); | ||
| 147 | } | ||
| 148 | |||
| 149 | 26 | void CopyBuffer(VkCommandBuffer commands, VkBuffer source, VkBuffer destination, | |
| 150 | std::span<const VkBufferCopy> regions) { | ||
| 151 | 26 | const BufferCopies copy{source, destination, regions}; | |
| 152 | 26 | CopyBuffers(commands, std::span(©, 1)); | |
| 153 | 26 | } | |
| 154 | |||
| 155 | 232 | FeatureChain& FeatureChain::operator=(const FeatureChain& other) { | |
| 156 | 232 | core = other.core; | |
| 157 | 232 | v11 = other.v11; | |
| 158 | 232 | v12 = other.v12; | |
| 159 | 232 | v13 = other.v13; | |
| 160 | 232 | accelerationStructure = other.accelerationStructure; | |
| 161 | 232 | rayQuery = other.rayQuery; | |
| 162 | 232 | fault = other.fault; | |
| 163 | 232 | optional_ = other.optional_; | |
| 164 | 232 | Relink(); | |
| 165 | 232 | return *this; | |
| 166 | } | ||
| 167 | |||
| 168 | 159 | void FeatureChain::Link(bool accelerationStructures, bool rayQueries, bool faults) { | |
| 169 | 159 | optional_ = {accelerationStructures, rayQueries, faults}; | |
| 170 | 159 | Relink(); | |
| 171 | 159 | } | |
| 172 | |||
| 173 | 782 | void FeatureChain::Relink() { | |
| 174 | 782 | core.pNext = &v11; | |
| 175 | 782 | v11.pNext = &v12; | |
| 176 | 782 | v12.pNext = &v13; | |
| 177 | 782 | void** next = &v13.pNext; | |
| 178 | 2346 | const auto add = [&](auto& features, bool included) { | |
| 179 | 2346 | if (included) { | |
| 180 | 782 | *next = &features; | |
| 181 | 782 | next = &features.pNext; | |
| 182 | } | ||
| 183 | 782 | }; | |
| 184 | 782 | add(accelerationStructure, optional_[0]); | |
| 185 | 782 | add(rayQuery, optional_[1]); | |
| 186 | 782 | add(fault, optional_[2]); | |
| 187 | 782 | *next = nullptr; | |
| 188 | 782 | } | |
| 189 | |||
| 190 | 82 | PhysicalDevice Probe(VkPhysicalDevice handle) { | |
| 191 | 82 | PhysicalDevice device; | |
| 192 | 82 | device.handle = handle; | |
| 193 | 82 | std::uint32_t count = 0; | |
| 194 | 82 | vkEnumerateDeviceExtensionProperties(handle, nullptr, &count, nullptr); | |
| 195 | 82 | std::vector<VkExtensionProperties> extensions(count); | |
| 196 | 82 | vkEnumerateDeviceExtensionProperties(handle, nullptr, &count, extensions.data()); | |
| 197 | 82 | device.extensions = Map( | |
| 198 | 13284 | extensions, [](const VkExtensionProperties& e) { return std::string(e.extensionName); }); | |
| 199 | 1066 | const auto has = [&](std::string_view name) { | |
| 200 | 984 | return std::ranges::find(device.extensions, name) != device.extensions.end(); | |
| 201 | 82 | }; | |
| 202 | |||
| 203 | 82 | VkPhysicalDeviceDriverProperties driver{ | |
| 204 | .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; | ||
| 205 | 82 | VkPhysicalDeviceIDProperties ids{.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES, | |
| 206 | 82 | .pNext = &driver}; | |
| 207 | 82 | VkPhysicalDeviceProperties2 properties{.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, | |
| 208 | 82 | .pNext = &ids}; | |
| 209 | 82 | vkGetPhysicalDeviceProperties2(handle, &properties); | |
| 210 | // Optional structs are read only when their extension is there. | ||
| 211 | 82 | FeatureChain& features = device.features; | |
| 212 | 82 | features.Link(has(VK_KHR_ACCELERATION_STRUCTURE_EXTENSION_NAME), | |
| 213 | 246 | has(VK_KHR_RAY_QUERY_EXTENSION_NAME), has(VK_EXT_DEVICE_FAULT_EXTENSION_NAME)); | |
| 214 | 82 | vkGetPhysicalDeviceFeatures2(handle, &features.core); | |
| 215 | |||
| 216 | 82 | const VkPhysicalDeviceProperties& p = properties.properties; | |
| 217 | 82 | device.info = {.name = p.deviceName, | |
| 218 | 82 | .type = ToDeviceType(p.deviceType), | |
| 219 | 82 | .apiMajor = VK_API_VERSION_MAJOR(p.apiVersion), | |
| 220 | 82 | .apiMinor = VK_API_VERSION_MINOR(p.apiVersion), | |
| 221 | 82 | .uuid = Hex(ids.deviceUUID, VK_UUID_SIZE)}; | |
| 222 | 82 | VkPhysicalDeviceMemoryProperties memory{}; | |
| 223 | 82 | vkGetPhysicalDeviceMemoryProperties(handle, &memory); | |
| 224 | 164 | for (std::uint32_t i = 0; i < memory.memoryHeapCount; ++i) { | |
| 225 | 82 | if ((memory.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0) { | |
| 226 | 82 | device.deviceLocalBytes += memory.memoryHeaps[i].size; | |
| 227 | } | ||
| 228 | } | ||
| 229 | |||
| 230 | 82 | if (p.apiVersion < VK_API_VERSION_1_3) { | |
| 231 | ✗ | device.missing = "Vulkan 1.3"; | |
| 232 | } | ||
| 233 | 1230 | for (const RequiredBit& requirement : kRequirements) { | |
| 234 | 1148 | if (device.missing.empty() && *requirement.field(features) != VK_TRUE) { | |
| 235 | 1148 | device.missing = requirement.name; | |
| 236 | } | ||
| 237 | } | ||
| 238 | // A capability needs every one of its extensions and, for ray queries, the feature bits. | ||
| 239 | 82 | DeviceCapabilities& capabilities = device.info.capabilities; | |
| 240 | 82 | capabilities = {.rayQuery = true, | |
| 241 | .externalMemory = true, | ||
| 242 | .externalTimeline = true, | ||
| 243 | .memoryBudget = true, | ||
| 244 | .deviceFault = true, | ||
| 245 | .calibratedTimestamps = true, | ||
| 246 | .present = true}; | ||
| 247 | 820 | for (const Extension& extension : kExtensions) { | |
| 248 | 738 | capabilities.*extension.capability &= has(extension.name); | |
| 249 | } | ||
| 250 | 82 | capabilities.rayQuery &= features.accelerationStructure.accelerationStructure == VK_TRUE && | |
| 251 | 82 | features.rayQuery.rayQuery == VK_TRUE; | |
| 252 | 82 | capabilities.deviceFault &= features.fault.deviceFault == VK_TRUE; | |
| 253 | // Timestamp queries reset from the host, so GPU zones need no command buffers of their own. | ||
| 254 | 164 | capabilities.calibratedTimestamps &= | |
| 255 | 82 | features.v12.hostQueryReset == VK_TRUE && p.limits.timestampComputeAndGraphics == VK_TRUE; | |
| 256 | 82 | capabilities.externalMemory = capabilities.externalMemory && SharesMemory(handle); | |
| 257 | 82 | capabilities.externalTimeline = capabilities.externalTimeline && SharesTimelines(handle); | |
| 258 | 82 | capabilities.shaderInt64 = features.core.features.shaderInt64 == VK_TRUE; | |
| 259 | 82 | capabilities.shaderFloat16 = features.v12.shaderFloat16 == VK_TRUE; | |
| 260 | 82 | capabilities.sparseResidency = features.core.features.sparseResidencyImage2D == VK_TRUE; | |
| 261 | 82 | WorkAroundDrivers(driver.driverID, capabilities); | |
| 262 | 82 | device.info.supported = device.missing.empty(); | |
| 263 | 82 | return device; | |
| 264 | 328 | } | |
| 265 | |||
| 266 | 77 | std::vector<const char*> PhysicalDevice::EnabledExtensions() const { | |
| 267 | 77 | std::vector<const char*> names; | |
| 268 | 770 | for (const Extension& extension : kExtensions) { | |
| 269 | 693 | if (info.capabilities.*extension.capability) { | |
| 270 | 462 | names.push_back(extension.name); | |
| 271 | } | ||
| 272 | } | ||
| 273 | // Drivers layered over Metal require their portability subset enabled whenever they offer it. | ||
| 274 | 77 | if (std::ranges::find(extensions, "VK_KHR_portability_subset") != extensions.end()) { | |
| 275 | ✗ | names.push_back("VK_KHR_portability_subset"); | |
| 276 | } | ||
| 277 | 77 | return names; | |
| 278 | ✗ | } | |
| 279 | |||
| 280 | 77 | FeatureChain PhysicalDevice::EnabledFeatures() const { | |
| 281 | 77 | const DeviceCapabilities& capabilities = info.capabilities; | |
| 282 | 77 | FeatureChain supported = features; | |
| 283 | 77 | FeatureChain enabled; | |
| 284 | 1155 | for (const RequiredBit& requirement : kRequirements) { | |
| 285 | 1078 | *requirement.field(enabled) = *requirement.field(supported); | |
| 286 | } | ||
| 287 | 462 | const auto bit = [](bool on) { return on ? VK_TRUE : VK_FALSE; }; | |
| 288 | 77 | enabled.core.features.shaderInt64 = bit(capabilities.shaderInt64); | |
| 289 | 77 | enabled.v12.shaderFloat16 = bit(capabilities.shaderFloat16); | |
| 290 | 77 | enabled.accelerationStructure.accelerationStructure = bit(capabilities.rayQuery); | |
| 291 | 77 | enabled.rayQuery.rayQuery = bit(capabilities.rayQuery); | |
| 292 | 77 | enabled.fault.deviceFault = bit(capabilities.deviceFault); | |
| 293 | 77 | enabled.v12.hostQueryReset = bit(capabilities.calibratedTimestamps); | |
| 294 | 77 | enabled.Link(capabilities.rayQuery, capabilities.rayQuery, capabilities.deviceFault); | |
| 295 | 77 | return enabled; | |
| 296 | } | ||
| 297 | |||
| 298 | ✗ | std::uint64_t PhysicalDevice::Score() const { | |
| 299 | ✗ | if (!missing.empty()) { | |
| 300 | return 0; | ||
| 301 | } | ||
| 302 | ✗ | std::uint64_t kind = 1; | |
| 303 | ✗ | switch (info.type) { | |
| 304 | ✗ | case VulkanDeviceType::DiscreteGpu: | |
| 305 | ✗ | kind = 4; | |
| 306 | ✗ | break; | |
| 307 | ✗ | case VulkanDeviceType::IntegratedGpu: | |
| 308 | ✗ | kind = 3; | |
| 309 | ✗ | break; | |
| 310 | ✗ | case VulkanDeviceType::VirtualGpu: | |
| 311 | ✗ | kind = 2; | |
| 312 | ✗ | break; | |
| 313 | default: | ||
| 314 | break; | ||
| 315 | } | ||
| 316 | // The kind decides; memory, in MiB, breaks ties. | ||
| 317 | ✗ | return (kind << 48) + (deviceLocalBytes >> 20); | |
| 318 | } | ||
| 319 | |||
| 320 | 4 | std::vector<VulkanDeviceInfo> ListVulkanDevices() { | |
| 321 | 4 | const Instance instance; | |
| 322 | 4 | return Map(instance.PhysicalDevices(), | |
| 323 | 12 | [](VkPhysicalDevice device) { return Probe(device).info; }); | |
| 324 | 4 | } | |
| 325 | |||
| 326 | 2 | const char* ToString(VulkanDeviceType type) { | |
| 327 | 2 | switch (type) { | |
| 328 | case VulkanDeviceType::IntegratedGpu: | ||
| 329 | return "integrated gpu"; | ||
| 330 | ✗ | case VulkanDeviceType::DiscreteGpu: | |
| 331 | ✗ | return "discrete gpu"; | |
| 332 | ✗ | case VulkanDeviceType::VirtualGpu: | |
| 333 | ✗ | return "virtual gpu"; | |
| 334 | 2 | case VulkanDeviceType::Cpu: | |
| 335 | 2 | return "cpu"; | |
| 336 | ✗ | default: | |
| 337 | ✗ | return "other"; | |
| 338 | } | ||
| 339 | } | ||
| 340 | |||
| 341 | } // namespace ernie | ||
| 342 |