GCC Code Coverage Report

ernie/
Coverage:
low: ≥ 0%
medium: ≥ 75.0%
high: ≥ 90.0%
Lines:
161 / 178
90.4%
Branches:
111 / 202
55.0%
Functions:
15 / 15
100.0%
render/rt/bottom_levels.cpp
Line Branch Exec Source
1 // SPDX-License-Identifier: BSD-3-Clause
2 /// @file bottom_levels.cpp
3 /// @brief Building, refitting and compacting each geometry's bottom level as frame-graph passes.
4 #include "ernie/render/rt/bottom_levels.h"
5
6 #include "ernie/util/profiler.h"
7
8 #include <algorithm>
9
10 namespace ernie {
11
12 15 BottomLevels::BottomLevels(DeviceContext& context, Arenas& arenas, const GeometryPool& geometry,
13 15 std::uint64_t scratchBudget)
14 15 : context_(context), geometry_(geometry), scratchBudget_(scratchBudget),
15
1/2
✓ Branch 2 taken 15 times.
✗ Branch 3 not taken.
 15 pool_(context, "bottom levels"), uploader_(context, context.Main()),
16 // Addresses fit an int64 column bit for bit; kernels read them as uint64.
17
1/2
✓ Branch 1 taken 15 times.
✗ Branch 2 not taken.
 15 addresses_(std::make_unique<DeviceColumn>(arenas, "bottom-level addresses", DType::Int64,
18
1/2
✓ Branch 1 taken 15 times.
✗ Branch 2 not taken.
 30 std::vector<std::int64_t>{}, 0)) {}
19
20 57 BottomLevels::Level& BottomLevels::At(GeometryId geometry) {
21
2/2
✓ Branch 0 taken 23 times.
✓ Branch 1 taken 34 times.
 57 if (geometry >= levels_.size()) {
22 23 levels_.resize(geometry + 1);
23 23 hostAddresses_.resize(geometry + 1, 0);
24 }
25 57 return levels_[geometry];
26 }
27
28 32 void BottomLevels::SetAddress(GeometryId geometry, std::uint64_t address) {
29 32 At(geometry);
30
1/2
✓ Branch 0 taken 32 times.
✗ Branch 1 not taken.
 32 if (hostAddresses_[geometry] != address) {
31 32 hostAddresses_[geometry] = address;
32 32 dirtyAddresses_.push_back(geometry);
33 }
34 32 }
35
36 24 void BottomLevels::Queue(GeometryId geometry, bool refit) {
37 24 const auto queued = std::ranges::find(pending_, geometry, &Work::geometry);
38
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (queued == pending_.end()) {
39 24 pending_.push_back({geometry, refit});
40 } else {
41 // A rebuild covers a refit.
42
0/4
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ queued->refit = queued->refit && refit;
43 }
44 24 }
45
46 27 void BottomLevels::Apply(const GeometryPool::Changes& changes) {
47
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 27 times.
 28 for (const GeometryId geometry : changes.freed) {
48
0/16
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 1 std::erase_if(pending_, [&](const Work& work) { return work.geometry == geometry; });
49 // Frames in flight may trace it: destroying it retires it with their tickets.
50
2/4
✓ Branch 1 taken 1 time.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 1 time.
 1 At(geometry) = {};
51 1 SetAddress(geometry, 0);
52 }
53
2/2
✓ Branch 0 taken 24 times.
✓ Branch 1 taken 27 times.
 51 for (const GeometryId geometry : changes.rebuilt) {
54 24 At(geometry).refits = 0;
55 24 Queue(geometry, false);
56 }
57
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 27 times.
 27 for (const GeometryId geometry : changes.refitted) {
58 ✗ Level& level = At(geometry);
59 // A level built static is rebuilt refittable the first time its points move.
60 ✗ const bool refit =
61
0/6
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 ✗ level.structure && level.deforming && level.refits < kRefitsBeforeRebuild;
62 ✗ if (!refit) {
63 ✗ level.deforming = true;
64 ✗ level.refits = 0;
65 }
66 ✗ Queue(geometry, refit);
67 }
68 27 }
69
70 48 BottomBuild BottomLevels::Build(GeometryId id, bool refit, std::vector<TriangleGeometry>& triangles,
71 std::vector<BoxGeometry>& boxes) const {
72
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 42 times.
 48 const Geometry& geometry = geometry_.Get(id);
73
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 42 times.
 48 BottomBuild build{.target = levels_[id].structure.get(), .update = refit};
74
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 42 times.
 48 if (geometry.kind == GeometryKind::Triangles) {
75 24 triangles.push_back(
76 6 {.vertices = {&geometry_.Storage(geometry.vertices), geometry.vertices.offset},
77 6 .vertexCount = geometry.vertexCount,
78 6 .indices = {&geometry_.Storage(geometry.indices), geometry.indices.offset},
79 6 .triangleCount = geometry.triangleCount});
80 6 build.triangles = std::span(&triangles.back(), 1);
81 } else {
82 126 boxes.push_back(
83 42 {.boxes = {&geometry_.Storage(geometry.box), geometry.box.offset}, .count = 1});
84 42 build.boxes = std::span(&boxes.back(), 1);
85 }
86 48 return build;
87 }
88
89 27 void BottomLevels::AddBuilds(graph::Graph& graph) {
90 27 ERNIE_ZONE("Prepare the bottom levels");
91
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 Moves moves = TakeCompactedSizes();
92
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 FrameBuilds builds = PlanBuilds();
93
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 Compaction compaction = StartCompactionQuery(builds.built);
94
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 AddAddressUpload(graph);
95
4/4
✓ Branch 0 taken 13 times.
✓ Branch 1 taken 14 times.
✓ Branch 2 taken 5 times.
✓ Branch 3 taken 8 times.
 27 if (!builds.builds.empty() || !moves.empty()) {
96
1/2
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 AddBuildPass(graph, std::move(builds), std::move(moves), compaction);
97 }
98
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 13 times.
 27 if (!compaction.geometries.empty()) {
99
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 compactions_.push_back(std::move(compaction));
100 }
101 27 }
102
103 27 BottomLevels::Moves BottomLevels::TakeCompactedSizes() {
104 // Each level whose compacted size came back moves into a structure of that size.
105 27 Moves moves;
106
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 std::erase_if(compactions_, [&](Compaction& compaction) {
107 5 const std::optional<std::vector<std::uint64_t>> sizes = compaction.query->Sizes();
108
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (!sizes) {
109 return false;
110 }
111
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 5 times.
 12 for (std::size_t i = 0; i < compaction.geometries.size(); ++i) {
112
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 Level& level = levels_[compaction.geometries[i]];
113 7 level.compacting = false;
114 // Rebuilt, refitted or freed since: its size no longer applies.
115
2/4
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 if (level.structure != compaction.structures[i] || level.deforming) {
116 ✗ continue;
117 }
118
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 AccelerationSizes sizesCompacted = level.structure->Sizes();
119
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 sizesCompacted.storage = (*sizes)[i];
120 7 auto compacted = std::make_shared<AccelerationStructure>(
121
2/4
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 5 not taken.
 7 pool_, sizesCompacted, level.structure->Name() + " compacted");
122 7 ++totals_.compacted;
123
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 totals_.saved += level.structure->Sizes().storage - sizesCompacted.storage;
124
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 moves.emplace_back(level.structure, compacted);
125 7 level.structure = compacted;
126
1/2
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
 7 SetAddress(compaction.geometries[i], compacted->Address());
127 7 }
128 return true;
129 5 });
130 27 return moves;
131 ✗ }
132
133 27 BottomLevels::FrameBuilds BottomLevels::PlanBuilds() {
134 27 FrameBuilds frame;
135 // Builds point into these, so they hold every geometry without growing.
136
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 frame.triangles.reserve(pending_.size());
137
1/2
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
 27 frame.boxes.reserve(pending_.size());
138 27 std::vector<Work> later;
139
2/2
✓ Branch 0 taken 24 times.
✓ Branch 1 taken 27 times.
 51 for (const Work& work : pending_) {
140
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 21 times.
 24 const Geometry& geometry = geometry_.Get(work.geometry);
141
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 21 times.
 24 Level& level = levels_[work.geometry];
142
3/4
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 21 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3 times.
 24 if (geometry.kind == GeometryKind::Triangles && geometry.triangleCount == 0) {
143 ✗ level.structure.reset();
144
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 ✗ SetAddress(work.geometry, 0);
145 ✗ continue;
146 }
147
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (!work.refit) {
148 24 std::vector<TriangleGeometry> sizing;
149 24 std::vector<BoxGeometry> sizingBoxes;
150
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 const BuildFlags flags = level.deforming ? kDeforming : kStatic;
151
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 const BottomBuild input = Build(work.geometry, false, sizing, sizingBoxes);
152
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 21 times.
 24 const AccelerationSizes sizes = !input.triangles.empty()
153
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 21 times.
 24 ? Sizes(context_, input.triangles, flags)
154
2/4
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 21 times.
✗ Branch 5 not taken.
 24 : Sizes(context_, input.boxes, flags);
155 // Over the budget, the level keeps tracing as it was until a later frame.
156
3/4
✓ Branch 0 taken 10 times.
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 10 times.
 24 if (!frame.builds.empty() && frame.scratch + sizes.buildScratch > scratchBudget_) {
157
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 ✗ later.push_back(work);
158 ✗ continue;
159 }
160 // A new structure, so frames in flight keep tracing the old one until they finish.
161 24 level.structure = std::make_shared<AccelerationStructure>(
162
3/6
✓ Branch 2 taken 24 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 24 times.
✗ Branch 6 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 24 times.
 48 pool_, sizes, "geometry " + std::to_string(work.geometry));
163 24 level.refits = 0;
164 24 } else {
165 ✗ ++level.refits;
166 }
167
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 24 times.
 24 ++(work.refit ? totals_.refitted : totals_.built);
168
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 const BottomBuild build = Build(work.geometry, work.refit, frame.triangles, frame.boxes);
169 48 frame.scratch +=
170
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 24 times.
 24 work.refit ? build.target->Sizes().updateScratch : build.target->Sizes().buildScratch;
171
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 frame.builds.push_back(build);
172
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 frame.built.push_back(work.geometry);
173
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 SetAddress(work.geometry, level.structure->Address());
174 }
175 27 pending_ = std::move(later);
176 27 return frame;
177 27 }
178
179 27 BottomLevels::Compaction BottomLevels::StartCompactionQuery(const std::vector<GeometryId>& built) {
180 // Static levels have their compacted sizes read back for a later frame.
181 27 Compaction compaction;
182
2/2
✓ Branch 0 taken 24 times.
✓ Branch 1 taken 27 times.
 51 for (const GeometryId geometry : built) {
183
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 Level& level = levels_[geometry];
184
2/4
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 24 times.
✗ Branch 3 not taken.
 24 if (!level.deforming && !level.compacting) {
185 24 level.compacting = true;
186
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 compaction.geometries.push_back(geometry);
187
1/2
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
 24 compaction.structures.push_back(level.structure);
188 }
189 }
190
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 13 times.
 27 if (!compaction.geometries.empty()) {
191 42 compaction.query = std::make_unique<CompactionQuery>(
192
1/2
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
 14 context_, static_cast<std::uint32_t>(compaction.geometries.size()));
193 }
194 27 return compaction;
195 ✗ }
196
197 27 void BottomLevels::AddAddressUpload(graph::Graph& graph) {
198
2/2
✓ Branch 0 taken 8 times.
✓ Branch 1 taken 19 times.
 27 if (dirtyAddresses_.empty()) {
199 8 return;
200 }
201 19 addresses_->Resize(hostAddresses_.size(), context_.Main());
202 19 StageRows(uploader_, *addresses_, hostAddresses_, dirtyAddresses_);
203 19 dirtyAddresses_.clear();
204 19 graph::Buffer addresses = graph.Import(*addresses_);
205 19 graph::AddUploadPass(graph, "Upload bottom-level addresses", uploader_,
206 std::span(&addresses, 1));
207 }
208
209 19 void BottomLevels::AddBuildPass(graph::Graph& graph, FrameBuilds frame, Moves moves,
210 const Compaction& compaction) {
211 19 graph::Acceleration levels = graph.Import(pool_);
212 19 std::vector<graph::Buffer> inputs = geometry_.Import(graph);
213 19 graph::Buffer scratchBuffer =
214
4/6
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 5 times.
✓ Branch 3 taken 14 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 14 times.
✗ Branch 7 not taken.
 19 frame.scratch > 0 ? graph.Create(ScratchBytes(frame.builds, {}), "bottom-level scratch")
215 5 : graph::Buffer{};
216
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 CompactionQuery* query = compaction.query.get();
217
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 std::vector<const AccelerationStructure*> queried(compaction.structures.size());
218 19 std::ranges::transform(compaction.structures, queried.begin(),
219 24 [](const auto& structure) { return structure.get(); });
220
1/2
✓ Branch 1 taken 19 times.
✗ Branch 2 not taken.
 19 graph.Add(
221 "Build bottom levels", graph::Kind::Build,
222 57 [&](graph::PassBuilder& pass) {
223
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 19 times.
 38 for (const graph::Buffer& input : inputs) {
224 19 pass.Read(input, graph::Use::BuildInput);
225 }
226
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 5 times.
 19 if (scratchBuffer.Valid()) {
227 14 scratchBuffer = pass.Write(scratchBuffer, graph::Use::BuildScratch);
228 }
229 19 pass.Read(levels, graph::Use::BuildInput);
230 19 levels = pass.Build(levels);
231 19 },
232
2/6
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
✓ Branch 5 taken 19 times.
✗ Branch 6 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
 19 [frame = std::move(frame), moves = std::move(moves), queried, query,
233 scratchBuffer](graph::PassContext& pass) {
234
2/2
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 19 times.
 26 for (const auto& [source, compacted] : moves) {
235 7 RecordCompaction(pass.commands, *source, *compacted);
236 // The copy reads the source: it goes once this frame's work is done.
237
2/4
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 7 times.
✗ Branch 4 not taken.
 21 pass.Keep(source);
238 }
239
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 5 times.
 19 if (!frame.builds.empty()) {
240 14 const graph::BufferSlice space = pass.Get(scratchBuffer);
241 14 RecordBuilds(pass.commands, frame.builds, {}, {space.buffer, space.offset});
242 }
243
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 5 times.
 19 if (query != nullptr) {
244 // Sizes are written once the builds are done.
245 14 Barrier(pass.commands, VK_PIPELINE_STAGE_2_ACCELERATION_STRUCTURE_BUILD_BIT_KHR,
246 VK_ACCESS_2_ACCELERATION_STRUCTURE_WRITE_BIT_KHR,
247 VK_PIPELINE_STAGE_2_ACCELERATION_STRUCTURE_BUILD_BIT_KHR,
248 VK_ACCESS_2_ACCELERATION_STRUCTURE_READ_BIT_KHR);
249 14 query->Record(pass.commands, queried);
250 }
251 19 });
252 19 }
253
254 } // namespace ernie
255