device/acceleration.h
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | /// @file acceleration.h | ||
| 3 | /// @brief Hardware acceleration structures for ray queries: bottom levels over triangles or | ||
| 4 | /// procedural boxes read in place, top levels over instances a kernel writes, kept in a pool, built | ||
| 5 | /// in batches and compacted without host waits. | ||
| 6 | /// | ||
| 7 | /// AccelerationPool pool(context, "scene"); | ||
| 8 | /// AccelerationStructure mesh(pool, Sizes(context, triangles, kStatic), "mesh"); | ||
| 9 | /// const BottomBuild build{.target = &mesh, .triangles = triangles}; | ||
| 10 | /// // In a pass, after the inputs' writers and before the trace: | ||
| 11 | /// RecordBuilds(commands, std::span(&build, 1), {}, scratch); | ||
| 12 | /// | ||
| 13 | /// Barriers between passes are the frame graph's: a pass that builds writes its targets, one that | ||
| 14 | /// builds a top level reads its bottom levels, and one that traces reads the top level. | ||
| 15 | /// | ||
| 16 | /// References: | ||
| 17 | /// - Khronos, "VK_KHR_acceleration_structure" and "VK_KHR_ray_query", the Vulkan specification. | ||
| 18 | /// - NVIDIA, nvpro_core2's nvvk::AccelerationStructureBuilder (batched, budgeted, compacted | ||
| 19 | /// builds) and vk_lod_clusters (sizes read back without waiting), Apache-2.0. | ||
| 20 | #pragma once | ||
| 21 | |||
| 22 | #include "ernie/device/arena.h" | ||
| 23 | |||
| 24 | #include <cstdint> | ||
| 25 | #include <optional> | ||
| 26 | #include <span> | ||
| 27 | #include <string> | ||
| 28 | #include <string_view> | ||
| 29 | #include <vector> | ||
| 30 | |||
| 31 | namespace ernie { | ||
| 32 | |||
| 33 | /// @brief A place in a buffer an acceleration structure reads its input from. | ||
| 34 | struct BufferView { | ||
| 35 | const Buffer* buffer = nullptr; | ||
| 36 | std::uint64_t offset = 0; | ||
| 37 | }; | ||
| 38 | |||
| 39 | /// @brief Triangles: float x, y, z positions `stride` bytes apart, three uint32 indices each. | ||
| 40 | struct TriangleGeometry { | ||
| 41 | BufferView vertices; | ||
| 42 | std::uint64_t stride = 3 * sizeof(float); | ||
| 43 | std::uint32_t vertexCount = 0; | ||
| 44 | BufferView indices; | ||
| 45 | std::uint32_t triangleCount = 0; | ||
| 46 | }; | ||
| 47 | |||
| 48 | /// @brief Procedural primitives as axis-aligned boxes, min then max corner in floats, which ray | ||
| 49 | /// queries report as candidates for the shader to intersect exactly. | ||
| 50 | struct BoxGeometry { | ||
| 51 | BufferView boxes; | ||
| 52 | std::uint64_t stride = 6 * sizeof(float); | ||
| 53 | std::uint32_t count = 0; | ||
| 54 | }; | ||
| 55 | |||
| 56 | /// @brief How a structure is built: tuned to trace or to build fast, and whether it may be | ||
| 57 | /// compacted once built or refitted in place. | ||
| 58 | struct BuildFlags { | ||
| 59 | bool fastTrace = true; | ||
| 60 | bool compactable = false; | ||
| 61 | bool updatable = false; | ||
| 62 | }; | ||
| 63 | |||
| 64 | /// Bottom levels that rarely change: traced fast and compacted. | ||
| 65 | inline constexpr BuildFlags kStatic{.fastTrace = true, .compactable = true}; | ||
| 66 | /// Bottom levels that deform: built fast and refitted. | ||
| 67 | inline constexpr BuildFlags kDeforming{.fastTrace = false, .updatable = true}; | ||
| 68 | /// Top levels rebuilt each step. | ||
| 69 | inline constexpr BuildFlags kRebuilt{.fastTrace = false}; | ||
| 70 | |||
| 71 | /// @brief One instance of a top level, as the device reads it (VkAccelerationStructureInstanceKHR): | ||
| 72 | /// a row-major 3x4 transform, a 24-bit index shaders see and an 8-bit mask, a 24-bit offset and | ||
| 73 | /// 8 bits of flags, and the bottom level's address. | ||
| 74 | struct AccelerationInstance { | ||
| 75 | float transform[3][4] = {{1, 0, 0, 0}, {0, 1, 0, 0}, {0, 0, 1, 0}}; | ||
| 76 | std::uint32_t indexAndMask = 0xff000000u; | ||
| 77 | std::uint32_t offsetAndFlags = 0; | ||
| 78 | std::uint64_t bottom = 0; | ||
| 79 | }; | ||
| 80 | static_assert(sizeof(AccelerationInstance) == 64); | ||
| 81 | |||
| 82 | enum class AccelerationKind : std::uint8_t { Bottom, Top }; | ||
| 83 | |||
| 84 | /// @brief What a structure needs: its storage, scratch for a build and for a refit, and the most | ||
| 85 | /// primitives (or instances) of each geometry its builds may hold. | ||
| 86 | 152 | struct AccelerationSizes { | |
| 87 | AccelerationKind kind = AccelerationKind::Bottom; | ||
| 88 | BuildFlags flags; | ||
| 89 | std::uint64_t storage = 0; | ||
| 90 | std::uint64_t buildScratch = 0; | ||
| 91 | std::uint64_t updateScratch = 0; | ||
| 92 | std::vector<std::uint32_t> primitives; | ||
| 93 | }; | ||
| 94 | |||
| 95 | /// @return What a bottom level over `triangles` or `boxes` (one kind only) needs. | ||
| 96 | /// @throws std::invalid_argument for no geometry, both kinds, or triangles without vertices. | ||
| 97 | /// @throws DeviceError on devices without ray queries. | ||
| 98 | AccelerationSizes Sizes(const DeviceContext& context, std::span<const TriangleGeometry> triangles, | ||
| 99 | BuildFlags flags); | ||
| 100 | AccelerationSizes Sizes(const DeviceContext& context, std::span<const BoxGeometry> boxes, | ||
| 101 | BuildFlags flags); | ||
| 102 | /// @return What a top level of up to `capacity` instances needs. | ||
| 103 | AccelerationSizes Sizes(const DeviceContext& context, std::uint32_t capacity, BuildFlags flags); | ||
| 104 | |||
| 105 | /// @brief Memory acceleration structures live in: large arenas handed out in 256-byte-aligned | ||
| 106 | /// ranges, so thousands of bottom levels share a few buffers. Ranges come back once the device is | ||
| 107 | /// done with them. | ||
| 108 | 8 | class AccelerationPool { | |
| 109 | public: | ||
| 110 | /// @param arenaSize Bytes per arena; a larger structure gets an arena of its own. | ||
| 111 | AccelerationPool(DeviceContext& context, std::string_view name, | ||
| 112 | std::uint64_t arenaSize = std::uint64_t{128} << 20); | ||
| 113 | AccelerationPool(const AccelerationPool&) = delete; | ||
| 114 | AccelerationPool& operator=(const AccelerationPool&) = delete; | ||
| 115 | |||
| 116 | /// @brief Bytes in use and structures alive, by kind, for memory reports. | ||
| 117 | 38 | struct Statistics { | |
| 118 | std::uint64_t bottomBytes = 0; | ||
| 119 | std::uint64_t topBytes = 0; | ||
| 120 | std::uint32_t bottoms = 0; | ||
| 121 | std::uint32_t tops = 0; | ||
| 122 | }; | ||
| 123 | const Statistics& Usage() const { | ||
| 124 | return usage_; | ||
| 125 | } | ||
| 126 | 152 | DeviceContext& Context() const { | |
| 127 | 152 | return context_; | |
| 128 | } | ||
| 129 | 28 | const std::string& Name() const { | |
| 130 | 28 | return name_; | |
| 131 | } | ||
| 132 | /// @return The alignment every build's scratch needs. | ||
| 133 | 101 | std::uint64_t ScratchAlignment() const { | |
| 134 | 101 | return scratchAlignment_; | |
| 135 | } | ||
| 136 | |||
| 137 | private: | ||
| 138 | friend class AccelerationStructure; | ||
| 139 | BufferRange Allocate(AccelerationKind kind, std::uint64_t bytes); | ||
| 140 | void Free(AccelerationKind kind, const BufferRange& range); | ||
| 141 | |||
| 142 | DeviceContext& context_; | ||
| 143 | std::string name_; | ||
| 144 | Arenas arenas_; | ||
| 145 | std::uint64_t scratchAlignment_; | ||
| 146 | Statistics usage_; | ||
| 147 | }; | ||
| 148 | |||
| 149 | /// @brief A bottom or top level in a pool. Destroying one retires it: the device may still trace | ||
| 150 | /// it, and its range and handle go once the work submitted so far is done. | ||
| 151 | class AccelerationStructure { | ||
| 152 | public: | ||
| 153 | /// @throws DeviceError when the pool cannot hold it. | ||
| 154 | AccelerationStructure(AccelerationPool& pool, const AccelerationSizes& sizes, | ||
| 155 | std::string_view name); | ||
| 156 | ~AccelerationStructure(); | ||
| 157 | AccelerationStructure(const AccelerationStructure&) = delete; | ||
| 158 | AccelerationStructure& operator=(const AccelerationStructure&) = delete; | ||
| 159 | |||
| 160 | 162 | AccelerationKind Kind() const { | |
| 161 | 162 | return sizes_.kind; | |
| 162 | } | ||
| 163 | 8 | const AccelerationSizes& Sizes() const { | |
| 164 | 8 | return sizes_; | |
| 165 | } | ||
| 166 | /// @return The address instances name a bottom level by, and kernels trace a top level by. | ||
| 167 | 86 | std::uint64_t Address() const { | |
| 168 | 86 | return address_; | |
| 169 | } | ||
| 170 | /// @return The buffer it lives in and its range there: what the frame graph tracks. | ||
| 171 | const Buffer& Storage() const; | ||
| 172 | const BufferRange& Range() const { | ||
| 173 | return range_; | ||
| 174 | } | ||
| 175 | 39 | const std::string& Name() const { | |
| 176 | 39 | return name_; | |
| 177 | } | ||
| 178 | 101 | const AccelerationPool& Pool() const { | |
| 179 | 101 | return pool_; | |
| 180 | } | ||
| 181 | 114 | VkAccelerationStructureKHR Handle() const { | |
| 182 | 113 | return structure_; | |
| 183 | } | ||
| 184 | |||
| 185 | private: | ||
| 186 | AccelerationPool& pool_; | ||
| 187 | AccelerationSizes sizes_; | ||
| 188 | std::string name_; | ||
| 189 | BufferRange range_; | ||
| 190 | VkAccelerationStructureKHR structure_ = VK_NULL_HANDLE; | ||
| 191 | std::uint64_t address_ = 0; | ||
| 192 | }; | ||
| 193 | |||
| 194 | /// @brief A bottom level's build: from scratch, or a refit in place of one built `updatable`. | ||
| 195 | struct BottomBuild { | ||
| 196 | const AccelerationStructure* target = nullptr; | ||
| 197 | std::span<const TriangleGeometry> triangles; | ||
| 198 | std::span<const BoxGeometry> boxes; | ||
| 199 | bool update = false; | ||
| 200 | }; | ||
| 201 | |||
| 202 | /// @brief A top level's build over `count` instances at `instances`, 16-byte aligned. | ||
| 203 | struct TopBuild { | ||
| 204 | const AccelerationStructure* target = nullptr; | ||
| 205 | BufferView instances; | ||
| 206 | std::uint32_t count = 0; | ||
| 207 | bool update = false; | ||
| 208 | }; | ||
| 209 | |||
| 210 | /// @return The scratch `RecordBuilds` needs for these builds, each at its own aligned offset. | ||
| 211 | std::uint64_t ScratchBytes(std::span<const BottomBuild> bottoms, std::span<const TopBuild> tops); | ||
| 212 | |||
| 213 | /// @brief Records every build in one call, so the device runs them together. The builds must not | ||
| 214 | /// depend on one another: bottom levels a top level reads are built in an earlier call, with a | ||
| 215 | /// barrier between. | ||
| 216 | /// @param scratch At least `ScratchBytes`, used only by these builds until they finish. | ||
| 217 | /// @throws std::invalid_argument for a build larger than its target was sized for, an update of | ||
| 218 | /// a structure not built updatable, or a target of the wrong kind. | ||
| 219 | void RecordBuilds(VkCommandBuffer commands, std::span<const BottomBuild> bottoms, | ||
| 220 | std::span<const TopBuild> tops, BufferView scratch); | ||
| 221 | |||
| 222 | /// @brief The sizes structures compact to, written after their builds and read back without | ||
| 223 | /// waiting once the device has written them. | ||
| 224 | class CompactionQuery { | ||
| 225 | public: | ||
| 226 | CompactionQuery(DeviceContext& context, std::uint32_t capacity); | ||
| 227 | ~CompactionQuery(); | ||
| 228 | CompactionQuery(const CompactionQuery&) = delete; | ||
| 229 | CompactionQuery& operator=(const CompactionQuery&) = delete; | ||
| 230 | |||
| 231 | /// @brief Records writing the compacted sizes of `built`, after a barrier from their builds. | ||
| 232 | /// @throws std::invalid_argument for more than the capacity, or a structure not compactable. | ||
| 233 | void Record(VkCommandBuffer commands, std::span<const AccelerationStructure* const> built); | ||
| 234 | /// @return The sizes in `Record`'s order once the device has written them; none before. | ||
| 235 | std::optional<std::vector<std::uint64_t>> Sizes() const; | ||
| 236 | |||
| 237 | private: | ||
| 238 | VkDevice device_; | ||
| 239 | VkQueryPool pool_ = VK_NULL_HANDLE; | ||
| 240 | std::uint32_t capacity_; | ||
| 241 | std::uint32_t count_ = 0; | ||
| 242 | }; | ||
| 243 | |||
| 244 | /// @brief Records copying `source` into `compacted`, sized from a `CompactionQuery`. | ||
| 245 | void RecordCompaction(VkCommandBuffer commands, const AccelerationStructure& source, | ||
| 246 | const AccelerationStructure& compacted); | ||
| 247 | |||
| 248 | } // namespace ernie | ||
| 249 |