device/context.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | /// @file context.cpp | ||
| 3 | /// @brief Choosing a device, creating it with its queues and allocator, and retiring objects. | ||
| 4 | #include "ernie/device/context.h" | ||
| 5 | |||
| 6 | #include "ernie/util/algorithm.h" | ||
| 7 | #include "ernie/util/file.h" | ||
| 8 | #include "ernie/util/on_failure.h" | ||
| 9 | #include "ernie/util/profiler.h" | ||
| 10 | #include "ernie/util/text.h" | ||
| 11 | |||
| 12 | #include <algorithm> | ||
| 13 | #include <array> | ||
| 14 | #include <cstring> | ||
| 15 | #include <optional> | ||
| 16 | #include <span> | ||
| 17 | #include <stdexcept> | ||
| 18 | |||
| 19 | namespace ernie { | ||
| 20 | namespace { | ||
| 21 | |||
| 22 | /// The releases this thread is running, which a `WaitIdle` inside one must not wait for. | ||
| 23 | thread_local std::size_t releasingHere = 0; | ||
| 24 | |||
| 25 | 78 | PhysicalDevice Choose(const Instance& instance, std::string_view wanted) { | |
| 26 | 78 | std::vector<PhysicalDevice> devices = Map(instance.PhysicalDevices(), Probe); | |
| 27 | 78 | std::erase_if(devices, [&](const PhysicalDevice& device) { | |
| 28 | 79 | return !wanted.empty() && device.info.uuid != wanted && | |
| 29 | 81 | ToLower(device.info.name).find(ToLower(wanted)) == std::string::npos; | |
| 30 | }); | ||
| 31 | 78 | const auto best = std::ranges::max_element(devices, {}, &PhysicalDevice::Score); | |
| 32 | 78 | if (best == devices.end()) { | |
| 33 | 1 | throw DeviceError(DeviceErrorKind::Unsupported, | |
| 34 | 1 | wanted.empty() | |
| 35 | 3 | ? "No Vulkan device" | |
| 36 | 4 | : "No Vulkan device matches '" + std::string(wanted) + "'"); | |
| 37 | } | ||
| 38 | 77 | if (!best->missing.empty()) { | |
| 39 | ✗ | throw DeviceError(DeviceErrorKind::Unsupported, | |
| 40 | ✗ | best->info.name + " lacks " + best->missing + ", which Ernie requires"); | |
| 41 | } | ||
| 42 | 154 | return *best; | |
| 43 | 78 | } | |
| 44 | |||
| 45 | /// @brief The families the queues come from: main (graphics and compute), compute and transfer. | ||
| 46 | struct Families { | ||
| 47 | std::uint32_t main = 0; | ||
| 48 | std::uint32_t compute = 0; | ||
| 49 | std::uint32_t transfer = 0; | ||
| 50 | }; | ||
| 51 | |||
| 52 | /// Prefers a compute-only family for compute and a transfer-only one for uploads, where present. | ||
| 53 | 77 | Families FindFamilies(VkPhysicalDevice device) { | |
| 54 | 77 | std::uint32_t count = 0; | |
| 55 | 77 | vkGetPhysicalDeviceQueueFamilyProperties(device, &count, nullptr); | |
| 56 | 77 | std::vector<VkQueueFamilyProperties> families(count); | |
| 57 | 77 | vkGetPhysicalDeviceQueueFamilyProperties(device, &count, families.data()); | |
| 58 | 308 | const auto find = [&](VkQueueFlags required, | |
| 59 | VkQueueFlags excluded) -> std::optional<std::uint32_t> { | ||
| 60 | 385 | for (std::uint32_t i = 0; i < count; ++i) { | |
| 61 | 231 | if ((families[i].queueFlags & required) == required && | |
| 62 | 231 | (families[i].queueFlags & excluded) == 0 && families[i].queueCount > 0) { | |
| 63 | 77 | return i; | |
| 64 | } | ||
| 65 | } | ||
| 66 | 154 | return std::nullopt; | |
| 67 | 77 | }; | |
| 68 | 77 | const std::optional<std::uint32_t> main = find(VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT, 0); | |
| 69 | 77 | if (!main) { | |
| 70 | ✗ | throw DeviceError(DeviceErrorKind::Unsupported, | |
| 71 | ✗ | "No queue family does graphics and compute"); | |
| 72 | } | ||
| 73 | 77 | Families result{*main, *main, *main}; | |
| 74 | 77 | result.compute = find(VK_QUEUE_COMPUTE_BIT, VK_QUEUE_GRAPHICS_BIT).value_or(*main); | |
| 75 | 77 | result.transfer = find(VK_QUEUE_TRANSFER_BIT, VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT) | |
| 76 | 77 | .value_or(result.compute); | |
| 77 | 77 | return result; | |
| 78 | 77 | } | |
| 79 | |||
| 80 | /// @return The cache saved at `path`, when it was saved by the same driver on the same device, | ||
| 81 | /// else nothing: a driver may crash on another's data rather than reject it. | ||
| 82 | 77 | std::vector<std::byte> LoadPipelineCache(const std::filesystem::path& path, | |
| 83 | const VkPhysicalDeviceProperties& properties) { | ||
| 84 | 77 | std::error_code missing; | |
| 85 | 77 | if (path.empty() || !std::filesystem::is_regular_file(path, missing)) { | |
| 86 | 75 | return {}; | |
| 87 | } | ||
| 88 | 2 | try { | |
| 89 | 2 | const MappedFile file(path); | |
| 90 | 2 | const std::span<const std::byte> bytes = file.Bytes(); | |
| 91 | VkPipelineCacheHeaderVersionOne header{}; | ||
| 92 | 2 | if (bytes.size() < sizeof(header)) { | |
| 93 | ✗ | return {}; | |
| 94 | } | ||
| 95 | 2 | std::memcpy(&header, bytes.data(), sizeof(header)); | |
| 96 | 2 | if (header.headerSize < sizeof(header) || | |
| 97 | 2 | header.headerVersion != VK_PIPELINE_CACHE_HEADER_VERSION_ONE || | |
| 98 | 2 | header.vendorID != properties.vendorID || header.deviceID != properties.deviceID || | |
| 99 | 2 | std::memcmp(header.pipelineCacheUUID, properties.pipelineCacheUUID, VK_UUID_SIZE) != | |
| 100 | 0) { | ||
| 101 | 1 | return {}; | |
| 102 | } | ||
| 103 | 1 | return {bytes.begin(), bytes.end()}; | |
| 104 | 2 | } catch (const std::runtime_error&) { | |
| 105 | ✗ | return {}; | |
| 106 | ✗ | } | |
| 107 | } | ||
| 108 | |||
| 109 | } // namespace | ||
| 110 | |||
| 111 | 78 | DeviceContext::DeviceContext(const DeviceContextOptions& options) | |
| 112 | 78 | : instance_({.validation = options.validation, .extensions = options.instanceExtensions}), | |
| 113 | 78 | physical_(Choose(instance_, options.device)) { | |
| 114 | 77 | ERNIE_ZONE("Create device"); | |
| 115 | // Swapchains need the surfaces only a window system's instance extensions bring. | ||
| 116 | 154 | physical_.info.capabilities.present &= | |
| 117 | 77 | std::ranges::find(options.instanceExtensions, VK_KHR_SURFACE_EXTENSION_NAME) != | |
| 118 | 77 | options.instanceExtensions.end(); | |
| 119 | 77 | VkPhysicalDeviceMaintenance4Properties maintenance4{ | |
| 120 | .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_4_PROPERTIES}; | ||
| 121 | 77 | VkPhysicalDeviceProperties2 properties{.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, | |
| 122 | 77 | .pNext = &maintenance4}; | |
| 123 | 77 | vkGetPhysicalDeviceProperties2(physical_.handle, &properties); | |
| 124 | 77 | limits_ = properties.properties.limits; | |
| 125 | 154 | maxBufferSize_ = | |
| 126 | 77 | std::min<std::uint64_t>(maintenance4.maxBufferSize, limits_.maxStorageBufferRange); | |
| 127 | |||
| 128 | 77 | const Families families = FindFamilies(physical_.handle); | |
| 129 | 77 | std::vector<std::uint32_t> distinct{families.main}; | |
| 130 | 231 | for (const std::uint32_t family : {families.compute, families.transfer}) { | |
| 131 | 154 | if (std::ranges::find(distinct, family) == distinct.end()) { | |
| 132 | ✗ | distinct.push_back(family); | |
| 133 | } | ||
| 134 | } | ||
| 135 | 77 | const float priority = 1.0f; | |
| 136 | 77 | const std::vector<VkDeviceQueueCreateInfo> queueInfos = | |
| 137 | ✗ | Map(distinct, [&](std::uint32_t family) { | |
| 138 | 77 | return VkDeviceQueueCreateInfo{.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, | |
| 139 | .queueFamilyIndex = family, | ||
| 140 | .queueCount = 1, | ||
| 141 | 77 | .pQueuePriorities = &priority}; | |
| 142 | 77 | }); | |
| 143 | 77 | const FeatureChain features = physical_.EnabledFeatures(); | |
| 144 | 77 | const std::vector<const char*> extensions = physical_.EnabledExtensions(); | |
| 145 | 77 | const VkDeviceCreateInfo info{ | |
| 146 | .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, | ||
| 147 | .pNext = &features.core, | ||
| 148 | 77 | .queueCreateInfoCount = static_cast<std::uint32_t>(queueInfos.size()), | |
| 149 | 77 | .pQueueCreateInfos = queueInfos.data(), | |
| 150 | 77 | .enabledExtensionCount = static_cast<std::uint32_t>(extensions.size()), | |
| 151 | 77 | .ppEnabledExtensionNames = extensions.data()}; | |
| 152 | 77 | CheckVk(vkCreateDevice(physical_.handle, &info, nullptr, &device_), | |
| 153 | 77 | "Creating the device on " + physical_.info.name); | |
| 154 | 77 | const OnFailure freeDevice([&] { vkDestroyDevice(device_, nullptr); }); | |
| 155 | |||
| 156 | 77 | const VmaVulkanFunctions functions{.vkGetInstanceProcAddr = vkGetInstanceProcAddr, | |
| 157 | 77 | .vkGetDeviceProcAddr = vkGetDeviceProcAddr}; | |
| 158 | 77 | VmaAllocatorCreateInfo allocator{.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT, | |
| 159 | 77 | .physicalDevice = physical_.handle, | |
| 160 | 77 | .device = device_, | |
| 161 | .pVulkanFunctions = &functions, | ||
| 162 | 77 | .instance = instance_.Handle(), | |
| 163 | 77 | .vulkanApiVersion = VK_API_VERSION_1_3}; | |
| 164 | 77 | if (physical_.info.capabilities.memoryBudget) { | |
| 165 | 77 | allocator.flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT; | |
| 166 | } | ||
| 167 | 77 | CheckVk(vmaCreateAllocator(&allocator, &allocator_), "Creating the memory allocator"); | |
| 168 | 77 | const OnFailure freeAllocator([&] { | |
| 169 | ✗ | queues_.clear(); | |
| 170 | ✗ | vmaDestroyAllocator(allocator_); | |
| 171 | 77 | }); | |
| 172 | |||
| 173 | 77 | const std::array names{"main", "compute", "transfer"}; | |
| 174 | 77 | const std::array wanted{families.main, families.compute, families.transfer}; | |
| 175 | 77 | std::array<Queue*, 3> chosen{}; | |
| 176 | 308 | for (std::size_t i = 0; i < wanted.size(); ++i) { | |
| 177 | 231 | const auto existing = std::ranges::find(queues_, wanted[i], &Queue::Family); | |
| 178 | 231 | if (existing != queues_.end()) { | |
| 179 | 154 | chosen[i] = existing->get(); | |
| 180 | 154 | continue; | |
| 181 | } | ||
| 182 | 77 | queues_.push_back(std::make_unique<Queue>(device_, wanted[i], names[i], | |
| 183 | 77 | physical_.info.capabilities.externalTimeline)); | |
| 184 | 77 | chosen[i] = queues_.back().get(); | |
| 185 | 77 | Name(VK_OBJECT_TYPE_QUEUE, reinterpret_cast<std::uint64_t>(chosen[i]->Handle()), names[i]); | |
| 186 | } | ||
| 187 | 77 | main_ = chosen[0]; | |
| 188 | 77 | compute_ = chosen[1]; | |
| 189 | 77 | transfer_ = chosen[2]; | |
| 190 | |||
| 191 | 77 | pipelineCachePath_ = options.pipelineCache; | |
| 192 | 77 | const std::vector<std::byte> cached = | |
| 193 | 77 | LoadPipelineCache(pipelineCachePath_, properties.properties); | |
| 194 | 77 | const VkPipelineCacheCreateInfo cache{.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, | |
| 195 | 77 | .initialDataSize = cached.size(), | |
| 196 | 77 | .pInitialData = cached.data()}; | |
| 197 | 77 | CheckVk(vkCreatePipelineCache(device_, &cache, nullptr, &pipelineCache_), | |
| 198 | "Creating the pipeline cache"); | ||
| 199 | 77 | const OnFailure freeCache([&] { vkDestroyPipelineCache(device_, pipelineCache_, nullptr); }); | |
| 200 | 77 | profiler_ = GpuProfiler::Create(*this); | |
| 201 | 156 | } | |
| 202 | |||
| 203 | 77 | DeviceContext::~DeviceContext() { | |
| 204 | 154 | Quietly([this] { WaitIdle(); }); | |
| 205 | 154 | Quietly([this] { SavePipelineCache(); }); | |
| 206 | 77 | vkDestroyPipelineCache(device_, pipelineCache_, nullptr); | |
| 207 | 77 | profiler_.reset(); | |
| 208 | 77 | queues_.clear(); | |
| 209 | 77 | vmaDestroyAllocator(allocator_); | |
| 210 | 77 | vkDestroyDevice(device_, nullptr); | |
| 211 | 154 | } | |
| 212 | |||
| 213 | 79 | std::vector<Queue*> DeviceContext::Queues() const { | |
| 214 | 158 | return Map(queues_, [](const std::unique_ptr<Queue>& queue) { return queue.get(); }); | |
| 215 | } | ||
| 216 | |||
| 217 | 214 | void DeviceContext::Retire(std::shared_ptr<const void> object) { | |
| 218 | 214 | if (object != nullptr) { | |
| 219 | 856 | Retire([held = std::move(object)]() mutable { held.reset(); }); | |
| 220 | } | ||
| 221 | 214 | } | |
| 222 | |||
| 223 | 1028 | void DeviceContext::Retire(std::function<void()> release) { | |
| 224 | 2056 | std::vector<Ticket> tickets = Map(queues_, [](const auto& queue) { return queue->Last(); }); | |
| 225 | 1028 | const std::lock_guard lock(retiredMutex_); | |
| 226 | 2056 | retired_.push_back({std::move(release), std::move(tickets)}); | |
| 227 | 2056 | } | |
| 228 | |||
| 229 | 535 | void DeviceContext::Collect() { | |
| 230 | 535 | if (profiler_) { | |
| 231 | 535 | profiler_->Collect(); | |
| 232 | } | ||
| 233 | 535 | std::vector<Retired> done; | |
| 234 | 535 | { | |
| 235 | 535 | const std::lock_guard lock(retiredMutex_); | |
| 236 | 535 | const auto finished = std::ranges::partition(retired_, [](const Retired& retired) { | |
| 237 | 1078 | return !std::ranges::all_of(retired.tickets, | |
| 238 | 1078 | [](const Ticket& ticket) { return ticket.Done(); }); | |
| 239 | }); | ||
| 240 | 535 | done.assign(std::make_move_iterator(finished.begin()), | |
| 241 | std::make_move_iterator(finished.end())); | ||
| 242 | 535 | retired_.erase(finished.begin(), finished.end()); | |
| 243 | 535 | releasing_ += done.size(); | |
| 244 | ✗ | } | |
| 245 | // Released without the lock: a release may retire more. | ||
| 246 | 1563 | for (Retired& retired : done) { | |
| 247 | 1028 | ++releasingHere; | |
| 248 | 1028 | retired.release(); | |
| 249 | 1028 | --releasingHere; | |
| 250 | 1028 | const std::lock_guard lock(retiredMutex_); | |
| 251 | 1028 | --releasing_; | |
| 252 | 1028 | } | |
| 253 | 535 | released_.notify_all(); | |
| 254 | 535 | } | |
| 255 | |||
| 256 | 232 | void DeviceContext::WaitIdle() { | |
| 257 | 464 | for (const auto& queue : queues_) { | |
| 258 | 232 | queue->Last().Wait(); | |
| 259 | } | ||
| 260 | 232 | Collect(); | |
| 261 | // Releases another thread took are done too, so their owners may go. | ||
| 262 | 232 | std::unique_lock lock(retiredMutex_); | |
| 263 | 232 | released_.wait(lock, [this] { return releasing_ <= releasingHere; }); | |
| 264 | 232 | } | |
| 265 | |||
| 266 | 77 | void DeviceContext::SavePipelineCache() const { | |
| 267 | 77 | if (pipelineCachePath_.empty()) { | |
| 268 | 72 | return; | |
| 269 | } | ||
| 270 | // A cache that grows between sizing and reading reads incomplete: size it again. | ||
| 271 | 5 | std::vector<std::byte> bytes; | |
| 272 | 5 | VkResult read = VK_INCOMPLETE; | |
| 273 | 10 | while (read == VK_INCOMPLETE) { | |
| 274 | 5 | std::size_t size = 0; | |
| 275 | 5 | CheckVk(vkGetPipelineCacheData(device_, pipelineCache_, &size, nullptr), | |
| 276 | "Sizing the pipeline cache"); | ||
| 277 | 5 | bytes.resize(size); | |
| 278 | 5 | read = vkGetPipelineCacheData(device_, pipelineCache_, &size, bytes.data()); | |
| 279 | 5 | CheckVk(read, "Reading the pipeline cache"); | |
| 280 | 5 | bytes.resize(size); | |
| 281 | } | ||
| 282 | 5 | WriteBinaryFile(pipelineCachePath_, bytes); | |
| 283 | 5 | } | |
| 284 | |||
| 285 | 689 | void DeviceContext::Name(VkObjectType type, std::uint64_t handle, std::string_view name) const { | |
| 286 | 689 | instance_.Name(device_, type, handle, name); | |
| 287 | 689 | } | |
| 288 | |||
| 289 | } // namespace ernie | ||
| 290 |