GCC Code Coverage Report

ernie/
Coverage:
low: ≥ 0%
medium: ≥ 75.0%
high: ≥ 90.0%
Lines:
255 / 269
94.8%
Branches:
129 / 218
59.2%
Functions:
33 / 33
100.0%
device/acceleration.cpp
Line Branch Exec Source
1 // SPDX-License-Identifier: BSD-3-Clause
2 /// @file acceleration.cpp
3 /// @brief Sizing acceleration structures, keeping them in a pool, and recording their builds,
4 /// compacted-size queries and compaction.
5 #include "ernie/device/acceleration.h"
6
7 #include "ernie/device/context.h"
8
9 #include <stdexcept>
10 #include <utility>
11
12 namespace ernie {
13 namespace {
14
15 /// The alignment the specification requires of an acceleration structure's offset.
16 constexpr std::uint64_t kStructureAlignment = 256;
17 /// The alignment instance arrays need.
18 constexpr std::uint64_t kInstanceAlignment = 16;
19
20 289 VkDeviceAddress DeviceAddress(const BufferView& view) {
21
2/2
✓ Branch 0 taken 257 times.
✓ Branch 1 taken 32 times.
 289 return view.buffer == nullptr ? 0 : view.buffer->Address() + view.offset;
22 }
23
24 107 void RequireRayQueries(const DeviceContext& context) {
25
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 107 times.
 107 if (!context.Capabilities().rayQuery) {
26 ✗ throw DeviceError(DeviceErrorKind::Unsupported,
27
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ context.Info().name + " has no hardware acceleration structures");
28 }
29 107 }
30
31 148 VkBuildAccelerationStructureFlagsKHR ToVk(const BuildFlags& flags) {
32 296 VkBuildAccelerationStructureFlagsKHR vk =
33
2/2
✓ Branch 0 taken 59 times.
✓ Branch 1 taken 89 times.
 148 flags.fastTrace ? VK_BUILD_ACCELERATION_STRUCTURE_PREFER_FAST_TRACE_BIT_KHR
34 : VK_BUILD_ACCELERATION_STRUCTURE_PREFER_FAST_BUILD_BIT_KHR;
35
2/2
✓ Branch 0 taken 89 times.
✓ Branch 1 taken 59 times.
 148 if (flags.compactable) {
36 89 vk |= VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_COMPACTION_BIT_KHR;
37 }
38
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 145 times.
 148 if (flags.updatable) {
39 3 vk |= VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_UPDATE_BIT_KHR;
40 }
41 148 return vk;
42 }
43
44 224 VkAccelerationStructureTypeKHR ToVk(AccelerationKind kind) {
45
2/2
✓ Branch 0 taken 78 times.
✓ Branch 1 taken 146 times.
 224 return kind == AccelerationKind::Bottom ? VK_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL_KHR
46 224 : VK_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL_KHR;
47 }
48
49 49 VkAccelerationStructureGeometryKHR Geometry(const TriangleGeometry& mesh) {
50 49 VkAccelerationStructureGeometryTrianglesDataKHR triangles{
51 .sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_TRIANGLES_DATA_KHR};
52 49 triangles.vertexFormat = VK_FORMAT_R32G32B32_SFLOAT;
53 49 triangles.vertexData.deviceAddress = DeviceAddress(mesh.vertices);
54 49 triangles.vertexStride = mesh.stride;
55 49 triangles.maxVertex = mesh.vertexCount - 1;
56 49 triangles.indexType = VK_INDEX_TYPE_UINT32;
57 49 triangles.indexData.deviceAddress = DeviceAddress(mesh.indices);
58 49 VkAccelerationStructureGeometryKHR geometry{
59 .sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_KHR,
60 .geometryType = VK_GEOMETRY_TYPE_TRIANGLES_KHR,
61 .flags = VK_GEOMETRY_OPAQUE_BIT_KHR};
62 49 geometry.geometry.triangles = triangles;
63 49 return geometry;
64 }
65
66 44 VkAccelerationStructureGeometryKHR Geometry(const BoxGeometry& boxes) {
67 44 VkAccelerationStructureGeometryAabbsDataKHR aabbs{
68 .sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_AABBS_DATA_KHR};
69 44 aabbs.data.deviceAddress = DeviceAddress(boxes.boxes);
70 44 aabbs.stride = boxes.stride;
71 // Opaque: a candidate is committed by the shader that intersects it exactly.
72 44 VkAccelerationStructureGeometryKHR geometry{
73 .sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_KHR,
74 .geometryType = VK_GEOMETRY_TYPE_AABBS_KHR,
75 .flags = VK_GEOMETRY_OPAQUE_BIT_KHR};
76 44 geometry.geometry.aabbs = aabbs;
77 44 return geometry;
78 }
79
80 56 VkAccelerationStructureGeometryKHR Geometry(const BufferView& instances) {
81 56 VkAccelerationStructureGeometryInstancesDataKHR data{
82 .sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_INSTANCES_DATA_KHR};
83 56 data.data.deviceAddress = DeviceAddress(instances);
84 56 VkAccelerationStructureGeometryKHR geometry{
85 .sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_KHR,
86 .geometryType = VK_GEOMETRY_TYPE_INSTANCES_KHR};
87 56 geometry.geometry.instances = data;
88 56 return geometry;
89 }
90
91 /// A bottom level's geometries and their primitive counts.
92 140 struct BottomInput {
93 std::vector<VkAccelerationStructureGeometryKHR> geometry;
94 std::vector<std::uint32_t> primitives;
95 };
96
97 94 BottomInput Input(std::span<const TriangleGeometry> triangles, std::span<const BoxGeometry> boxes) {
98
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 93 times.
 94 if (triangles.empty() == boxes.empty()) {
99
1/2
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1 throw std::invalid_argument("A bottom level holds triangles or boxes, one kind only");
100 }
101 93 BottomInput input;
102
2/2
✓ Branch 0 taken 49 times.
✓ Branch 1 taken 93 times.
 142 for (const TriangleGeometry& mesh : triangles) {
103
2/4
✓ Branch 0 taken 49 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 49 times.
 49 if (mesh.vertexCount == 0 || mesh.triangleCount == 0) {
104
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ throw std::invalid_argument("A bottom level's triangles need vertices");
105 }
106
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 input.geometry.push_back(Geometry(mesh));
107
1/2
✓ Branch 1 taken 49 times.
✗ Branch 2 not taken.
 49 input.primitives.push_back(mesh.triangleCount);
108 }
109
2/2
✓ Branch 0 taken 44 times.
✓ Branch 1 taken 93 times.
 137 for (const BoxGeometry& box : boxes) {
110
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 input.geometry.push_back(Geometry(box));
111
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 44 input.primitives.push_back(box.count);
112 }
113 93 return input;
114 ✗ }
115
116 VkAccelerationStructureBuildGeometryInfoKHR
117 148 BuildInfo(AccelerationKind kind, const BuildFlags& flags,
118 std::span<const VkAccelerationStructureGeometryKHR> geometry) {
119 148 return {.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_GEOMETRY_INFO_KHR,
120 148 .type = ToVk(kind),
121 148 .flags = ToVk(flags),
122 .mode = VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR,
123 148 .geometryCount = static_cast<std::uint32_t>(geometry.size()),
124 148 .pGeometries = geometry.data()};
125 }
126
127 68 AccelerationSizes Query(const DeviceContext& context, AccelerationKind kind,
128 const BuildFlags& flags,
129 std::span<const VkAccelerationStructureGeometryKHR> geometry,
130 std::span<const std::uint32_t> primitives) {
131 68 const VkAccelerationStructureBuildGeometryInfoKHR build = BuildInfo(kind, flags, geometry);
132 68 VkAccelerationStructureBuildSizesInfoKHR sizes{
133 .sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_SIZES_INFO_KHR};
134 68 vkGetAccelerationStructureBuildSizesKHR(context.Device(),
135 VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR, &build,
136 primitives.data(), &sizes);
137 68 return {.kind = kind,
138 .flags = flags,
139 68 .storage = sizes.accelerationStructureSize,
140 68 .buildScratch = sizes.buildScratchSize,
141 1 .updateScratch = flags.updatable ? sizes.updateScratchSize : 0,
142 68 .primitives = {primitives.begin(), primitives.end()}};
143
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 67 times.
 69 }
144
145 /// Checks a build holds no more geometries or primitives than its target was sized for.
146 81 void CheckFits(const AccelerationStructure& target, std::span<const std::uint32_t> primitives) {
147 81 const std::vector<std::uint32_t>& sized = target.Sizes().primitives;
148 81 bool fits = primitives.size() <= sized.size();
149
4/4
✓ Branch 0 taken 161 times.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 81 times.
✓ Branch 3 taken 80 times.
 162 for (std::size_t i = 0; fits && i < primitives.size(); ++i) {
150 81 fits = primitives[i] <= sized[i];
151 }
152
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 80 times.
 81 if (!fits) {
153
1/2
✓ Branch 1 taken 1 time.
✗ Branch 2 not taken.
 2 throw std::invalid_argument("A build of " + target.Name() +
154
2/4
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 time.
✗ Branch 6 not taken.
 3 " holds more than it was sized for");
155 }
156 80 }
157
158 38 std::uint64_t QueryScratchAlignment(const DeviceContext& context) {
159 38 VkPhysicalDeviceAccelerationStructurePropertiesKHR properties{
160 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ACCELERATION_STRUCTURE_PROPERTIES_KHR};
161 38 VkPhysicalDeviceProperties2 all{.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2,
162 38 .pNext = &properties};
163 38 vkGetPhysicalDeviceProperties2(context.Physical(), &all);
164
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 38 times.
 38 return std::max<std::uint64_t>(properties.minAccelerationStructureScratchOffsetAlignment, 1);
165 }
166
167 /// @return The pool the builds' targets share, which sets their scratch alignment.
168 101 const AccelerationPool& BuildPool(std::span<const BottomBuild> bottoms,
169 std::span<const TopBuild> tops) {
170
2/2
✓ Branch 0 taken 62 times.
✓ Branch 1 taken 39 times.
 101 const AccelerationStructure* first = bottoms.empty() ? tops[0].target : bottoms[0].target;
171
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 101 times.
 101 if (first == nullptr) {
172
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ throw std::invalid_argument("A build has no target");
173 }
174 101 return first->Pool();
175 }
176
177 206 std::uint64_t AlignUp(std::uint64_t value, std::uint64_t alignment) {
178 206 return (value + alignment - 1) / alignment * alignment;
179 }
180
181 149 std::uint64_t Scratch(const AccelerationStructure& target, bool update) {
182
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 148 times.
 149 return update ? target.Sizes().updateScratch : target.Sizes().buildScratch;
183 }
184
185 82 void CheckTarget(const AccelerationStructure* target, AccelerationKind kind, bool update) {
186
2/4
✓ Branch 0 taken 82 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 82 times.
 82 if (target == nullptr || target->Kind() != kind) {
187
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ throw std::invalid_argument("A build's target is missing or of the other kind");
188 }
189
4/4
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 80 times.
✓ Branch 2 taken 1 time.
✓ Branch 3 taken 1 time.
 82 if (update && !target->Sizes().flags.updatable) {
190
2/4
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
✓ Branch 5 taken 1 time.
✗ Branch 6 not taken.
 2 throw std::invalid_argument(target->Name() + " was not built to be updated");
191 }
192 81 }
193
194 } // namespace
195
196 25 AccelerationSizes Sizes(const DeviceContext& context, std::span<const TriangleGeometry> triangles,
197 BuildFlags flags) {
198 25 RequireRayQueries(context);
199 25 const BottomInput input = Input(triangles, {});
200
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 return Query(context, AccelerationKind::Bottom, flags, input.geometry, input.primitives);
201 24 }
202
203 22 AccelerationSizes Sizes(const DeviceContext& context, std::span<const BoxGeometry> boxes,
204 BuildFlags flags) {
205 22 RequireRayQueries(context);
206 22 const BottomInput input = Input({}, boxes);
207
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 return Query(context, AccelerationKind::Bottom, flags, input.geometry, input.primitives);
208 22 }
209
210 22 AccelerationSizes Sizes(const DeviceContext& context, std::uint32_t capacity, BuildFlags flags) {
211 22 RequireRayQueries(context);
212 22 const VkAccelerationStructureGeometryKHR instances = Geometry(BufferView{});
213 22 return Query(context, AccelerationKind::Top, flags, std::span(&instances, 1),
214 22 std::span(&capacity, 1));
215 }
216
217 38 AccelerationPool::AccelerationPool(DeviceContext& context, std::string_view name,
218 38 std::uint64_t arenaSize)
219
1/2
✓ Branch 2 taken 38 times.
✗ Branch 3 not taken.
 38 : context_(context), name_(name), arenas_(context, MemoryUse::Device, name, arenaSize),
220
2/4
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 38 times.
✗ Branch 5 not taken.
 38 scratchAlignment_(QueryScratchAlignment(context)) {
221
1/2
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
 38 RequireRayQueries(context);
222 38 }
223
224 76 BufferRange AccelerationPool::Allocate(AccelerationKind kind, std::uint64_t bytes) {
225 76 const BufferRange range = arenas_.Allocate(bytes, kStructureAlignment);
226
2/2
✓ Branch 0 taken 54 times.
✓ Branch 1 taken 22 times.
 76 if (kind == AccelerationKind::Bottom) {
227 54 usage_.bottomBytes += bytes;
228 54 ++usage_.bottoms;
229 } else {
230 22 usage_.topBytes += bytes;
231 22 ++usage_.tops;
232 }
233 76 return range;
234 }
235
236 76 void AccelerationPool::Free(AccelerationKind kind, const BufferRange& range) {
237 76 arenas_.Free(range);
238
2/2
✓ Branch 0 taken 54 times.
✓ Branch 1 taken 22 times.
 76 if (kind == AccelerationKind::Bottom) {
239 54 usage_.bottomBytes -= range.size;
240 54 --usage_.bottoms;
241 } else {
242 22 usage_.topBytes -= range.size;
243 22 --usage_.tops;
244 }
245 76 }
246
247 76 AccelerationStructure::AccelerationStructure(AccelerationPool& pool, const AccelerationSizes& sizes,
248 76 std::string_view name)
249
2/4
✓ Branch 2 taken 76 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 76 times.
✗ Branch 6 not taken.
 76 : pool_(pool), sizes_(sizes), name_(name), range_(pool.Allocate(sizes.kind, sizes.storage)) {
250
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 DeviceContext& context = pool.Context();
251 76 const VkAccelerationStructureCreateInfoKHR info{
252 .sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_CREATE_INFO_KHR,
253
2/4
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 76 times.
✗ Branch 5 not taken.
 76 .buffer = Storage().Handle(),
254 76 .offset = range_.offset,
255 76 .size = sizes.storage,
256 76 .type = ToVk(sizes.kind)};
257 76 const VkResult created =
258
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 vkCreateAccelerationStructureKHR(context.Device(), &info, nullptr, &structure_);
259
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 76 times.
 76 if (created < VK_SUCCESS) {
260
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 ✗ pool.Free(sizes.kind, range_);
261 }
262
1/2
✓ Branch 2 taken 76 times.
✗ Branch 3 not taken.
 76 CheckVk(created, [&] { return "Creating acceleration structure " + name_; });
263 76 context.Name(VK_OBJECT_TYPE_ACCELERATION_STRUCTURE_KHR,
264
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 reinterpret_cast<std::uint64_t>(structure_), name);
265 76 const VkAccelerationStructureDeviceAddressInfoKHR address{
266 .sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_DEVICE_ADDRESS_INFO_KHR,
267 76 .accelerationStructure = structure_};
268
1/2
✓ Branch 1 taken 76 times.
✗ Branch 2 not taken.
 76 address_ = vkGetAccelerationStructureDeviceAddressKHR(context.Device(), &address);
269 76 }
270
271 76 AccelerationStructure::~AccelerationStructure() {
272 // Work submitted so far may still trace it, or build a top level that refers to it.
273 76 DeviceContext& context = pool_.Context();
274 76 context.Retire([device = context.Device(), structure = structure_] {
275 76 vkDestroyAccelerationStructureKHR(device, structure, nullptr);
276 });
277 76 pool_.Free(sizes_.kind, range_);
278 76 }
279
280 106 const Buffer& AccelerationStructure::Storage() const {
281 106 return pool_.arenas_.Storage(range_);
282 }
283
284 44 std::uint64_t ScratchBytes(std::span<const BottomBuild> bottoms, std::span<const TopBuild> tops) {
285
3/4
✓ Branch 0 taken 28 times.
✓ Branch 1 taken 16 times.
✓ Branch 2 taken 28 times.
✗ Branch 3 not taken.
 44 if (bottoms.empty() && tops.empty()) {
286 return 0;
287 }
288 44 const std::uint64_t alignment = BuildPool(bottoms, tops).ScratchAlignment();
289 // One alignment's slack places the first build's scratch, wherever `scratch` starts.
290 44 std::uint64_t bytes = alignment;
291
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 44 times.
 85 for (const BottomBuild& build : bottoms) {
292 41 bytes += AlignUp(Scratch(*build.target, build.update), alignment);
293 }
294
2/2
✓ Branch 0 taken 28 times.
✓ Branch 1 taken 44 times.
 72 for (const TopBuild& build : tops) {
295 28 bytes += AlignUp(Scratch(*build.target, build.update), alignment);
296 }
297 44 return bytes;
298 }
299
300 57 void RecordBuilds(VkCommandBuffer commands, std::span<const BottomBuild> bottoms,
301 std::span<const TopBuild> tops, BufferView scratch) {
302
3/4
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 23 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 34 times.
 57 if (bottoms.empty() && tops.empty()) {
303 ✗ return;
304 }
305 57 const AccelerationPool& pool = BuildPool(bottoms, tops);
306
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 const std::uint64_t alignment = pool.ScratchAlignment();
307 57 VkDeviceAddress next = AlignUp(DeviceAddress(scratch), alignment);
308 57 std::vector<BottomInput> inputs;
309 57 std::vector<VkAccelerationStructureGeometryKHR> instances;
310
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 inputs.reserve(bottoms.size());
311
1/2
✓ Branch 1 taken 57 times.
✗ Branch 2 not taken.
 57 instances.reserve(tops.size());
312 57 std::vector<VkAccelerationStructureBuildGeometryInfoKHR> infos;
313 57 std::vector<std::vector<VkAccelerationStructureBuildRangeInfoKHR>> ranges;
314 137 const auto add = [&](const AccelerationStructure& target, bool update,
315 std::span<const VkAccelerationStructureGeometryKHR> geometry,
316 std::span<const std::uint32_t> primitives) {
317 80 VkAccelerationStructureBuildGeometryInfoKHR info =
318
2/2
✓ Branch 0 taken 79 times.
✓ Branch 1 taken 1 time.
 80 BuildInfo(target.Kind(), target.Sizes().flags, geometry);
319
2/2
✓ Branch 0 taken 79 times.
✓ Branch 1 taken 1 time.
 80 info.mode = update ? VK_BUILD_ACCELERATION_STRUCTURE_MODE_UPDATE_KHR
320 : VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR;
321
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 79 times.
 80 info.srcAccelerationStructure = update ? target.Handle() : VK_NULL_HANDLE;
322 80 info.dstAccelerationStructure = target.Handle();
323 80 info.scratchData.deviceAddress = next;
324 80 next += AlignUp(Scratch(target, update), alignment);
325 80 infos.push_back(info);
326 80 std::vector<VkAccelerationStructureBuildRangeInfoKHR>& range = ranges.emplace_back();
327
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 80 times.
 160 for (const std::uint32_t count : primitives) {
328 80 range.push_back({.primitiveCount = count});
329 }
330 137 };
331
2/2
✓ Branch 0 taken 48 times.
✓ Branch 1 taken 55 times.
 103 for (const BottomBuild& build : bottoms) {
332
2/2
✓ Branch 1 taken 47 times.
✓ Branch 2 taken 1 time.
 48 CheckTarget(build.target, AccelerationKind::Bottom, build.update);
333
2/4
✓ Branch 1 taken 47 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 47 times.
✗ Branch 5 not taken.
 47 const BottomInput& input = inputs.emplace_back(Input(build.triangles, build.boxes));
334
2/2
✓ Branch 1 taken 46 times.
✓ Branch 2 taken 1 time.
 47 CheckFits(*build.target, input.primitives);
335
1/2
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
 46 add(*build.target, build.update, input.geometry, input.primitives);
336 }
337
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 55 times.
 89 for (const TopBuild& build : tops) {
338
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 CheckTarget(build.target, AccelerationKind::Top, build.update);
339
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 34 times.
 34 if (DeviceAddress(build.instances) % kInstanceAlignment != 0) {
340
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ throw std::invalid_argument("A top level's instances need 16-byte alignment");
341 }
342
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 CheckFits(*build.target, std::span(&build.count, 1));
343 34 const VkAccelerationStructureGeometryKHR& geometry =
344
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 instances.emplace_back(Geometry(build.instances));
345
1/2
✓ Branch 1 taken 34 times.
✗ Branch 2 not taken.
 34 add(*build.target, build.update, std::span(&geometry, 1), std::span(&build.count, 1));
346 }
347 55 std::vector<const VkAccelerationStructureBuildRangeInfoKHR*> pointers;
348
2/2
✓ Branch 0 taken 80 times.
✓ Branch 1 taken 55 times.
 135 for (const auto& range : ranges) {
349
1/2
✓ Branch 1 taken 80 times.
✗ Branch 2 not taken.
 80 pointers.push_back(range.data());
350 }
351
1/2
✓ Branch 1 taken 55 times.
✗ Branch 2 not taken.
 55 vkCmdBuildAccelerationStructuresKHR(commands, static_cast<std::uint32_t>(infos.size()),
352
1/2
✓ Branch 1 taken 55 times.
✗ Branch 2 not taken.
 55 infos.data(), pointers.data());
353 63 }
354
355 15 CompactionQuery::CompactionQuery(DeviceContext& context, std::uint32_t capacity)
356 15 : device_(context.Device()), capacity_(capacity) {
357 15 const VkQueryPoolCreateInfo info{.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO,
358 .queryType =
359 VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR,
360 15 .queryCount = capacity};
361 15 CheckVk(vkCreateQueryPool(device_, &info, nullptr, &pool_), "Creating a compaction query");
362 15 }
363
364 15 CompactionQuery::~CompactionQuery() {
365 15 vkDestroyQueryPool(device_, pool_, nullptr);
366 15 }
367
368 15 void CompactionQuery::Record(VkCommandBuffer commands,
369 std::span<const AccelerationStructure* const> built) {
370
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 15 times.
 15 if (built.size() > capacity_) {
371
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ throw std::invalid_argument("A compaction query holds " + std::to_string(capacity_) +
372
0/6
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
 ✗ " sizes, not " + std::to_string(built.size()));
373 }
374 15 std::vector<VkAccelerationStructureKHR> handles;
375
2/2
✓ Branch 0 taken 25 times.
✓ Branch 1 taken 15 times.
 40 for (const AccelerationStructure* structure : built) {
376
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 25 times.
 25 if (!structure->Sizes().flags.compactable) {
377
0/4
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
 ✗ throw std::invalid_argument(structure->Name() + " was not built to be compacted");
378 }
379
1/2
✓ Branch 1 taken 25 times.
✗ Branch 2 not taken.
 25 handles.push_back(structure->Handle());
380 }
381
1/2
✓ Branch 1 taken 15 times.
✗ Branch 2 not taken.
 15 count_ = static_cast<std::uint32_t>(handles.size());
382
1/2
✓ Branch 1 taken 15 times.
✗ Branch 2 not taken.
 15 vkCmdResetQueryPool(commands, pool_, 0, capacity_);
383
1/2
✓ Branch 0 taken 15 times.
✗ Branch 1 not taken.
 15 if (count_ > 0) {
384
1/2
✓ Branch 1 taken 15 times.
✗ Branch 2 not taken.
 15 vkCmdWriteAccelerationStructuresPropertiesKHR(
385
1/2
✓ Branch 1 taken 15 times.
✗ Branch 2 not taken.
 15 commands, count_, handles.data(),
386 VK_QUERY_TYPE_ACCELERATION_STRUCTURE_COMPACTED_SIZE_KHR, pool_, 0);
387 }
388 15 }
389
390 6 std::optional<std::vector<std::uint64_t>> CompactionQuery::Sizes() const {
391 // Each size followed by its availability.
392 6 std::vector<std::uint64_t> results(2 * std::size_t{count_});
393 6 const VkResult result =
394 6 count_ == 0
395
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 ? VK_SUCCESS
396
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
 6 : vkGetQueryPoolResults(device_, pool_, 0, count_,
397
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
 6 results.size() * sizeof(std::uint64_t), results.data(),
398 2 * sizeof(std::uint64_t),
399 12 VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
400
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
 6 if (result < VK_SUCCESS) {
401
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 ✗ CheckVk(result, "Reading compacted sizes");
402 }
403 6 std::vector<std::uint64_t> sizes;
404
2/2
✓ Branch 0 taken 8 times.
✓ Branch 1 taken 6 times.
 14 for (std::uint32_t i = 0; i < count_; ++i) {
405
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 8 times.
 8 if (results[2 * i + 1] == 0) {
406 ✗ return std::nullopt;
407 }
408
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 sizes.push_back(results[2 * i]);
409 }
410 6 return sizes;
411 6 }
412
413 8 void RecordCompaction(VkCommandBuffer commands, const AccelerationStructure& source,
414 const AccelerationStructure& compacted) {
415 8 const VkCopyAccelerationStructureInfoKHR copy{
416 .sType = VK_STRUCTURE_TYPE_COPY_ACCELERATION_STRUCTURE_INFO_KHR,
417 8 .src = source.Handle(),
418 8 .dst = compacted.Handle(),
419 8 .mode = VK_COPY_ACCELERATION_STRUCTURE_MODE_COMPACT_KHR};
420 8 vkCmdCopyAccelerationStructureKHR(commands, &copy);
421 8 }
422
423 } // namespace ernie
424