GCC Code Coverage Report

ernie/
Coverage:
low: ≥ 0%
medium: ≥ 75.0%
high: ≥ 90.0%
Lines:
127 / 139
91.4%
Branches:
57 / 102
55.9%
Functions:
16 / 16
100.0%
device/profile.cpp
Line Branch Exec Source
1 // SPDX-License-Identifier: BSD-3-Clause
2 /// @file profile.cpp
3 /// @brief Ernie's GPU timer and Tracy's host-calibrated Vulkan context, and their zones around
4 /// commands.
5 #include "ernie/device/profile.h"
6
7 #include "ernie/device/context.h"
8
9 #include <algorithm>
10 #include <array>
11 #include <cstring>
12 #include <vector>
13
14 #if ERNIE_TRACY
15 #include <tracy/TracyVulkan.hpp>
16
17 #include <new>
18 #endif
19
20 namespace ernie {
21 namespace {
22
23 /// Zones in flight at once, two queries each.
24 constexpr std::uint32_t kZones = 4096;
25 /// Frames after which a zone still without timestamps is taken as never submitted.
26 constexpr std::uint64_t kStaleFrames = 64;
27 /// How often the device's clock is read again with the host's, against drift.
28 constexpr std::uint64_t kRecalibrateNs = 1'000'000'000;
29
30 #if defined(__linux__)
31 /// The host clock the steady clock reads, so timestamps convert to `profiler::Now`.
32 constexpr VkTimeDomainEXT kSteadyDomain = VK_TIME_DOMAIN_CLOCK_MONOTONIC_EXT;
33 #endif
34
35 /// @brief The zones this thread has open, for their nesting depths.
36 struct OpenZones {
37 std::array<VkCommandBuffer, 32> commands{};
38 std::uint32_t count = 0;
39 };
40 thread_local OpenZones openZones;
41
42 /// @return Whether the device can read its clock together with the steady clock's.
43 77 bool CalibratesSteadyClock(VkPhysicalDevice physical) {
44 #if defined(__linux__)
45 77 std::uint32_t count = 0;
46 77 vkGetPhysicalDeviceCalibrateableTimeDomainsEXT(physical, &count, nullptr);
47 77 std::vector<VkTimeDomainEXT> domains(count);
48
1/2
✓ Branch 1 taken 77 times.
✗ Branch 2 not taken.
 77 vkGetPhysicalDeviceCalibrateableTimeDomainsEXT(physical, &count, domains.data());
49
2/4
✓ Branch 1 taken 77 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 77 times.
 154 return std::ranges::find(domains, kSteadyDomain) != domains.end() &&
50
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 77 times.
 77 std::ranges::find(domains, VK_TIME_DOMAIN_DEVICE_EXT) != domains.end();
51 #else
52 static_cast<void>(physical);
53 return false;
54 #endif
55 77 }
56
57 /// @return The timestamp bits every queue's family keeps; 0 if one keeps none.
58 77 std::uint32_t TimestampBits(const DeviceContext& context) {
59 77 std::uint32_t count = 0;
60 77 vkGetPhysicalDeviceQueueFamilyProperties(context.Physical(), &count, nullptr);
61 77 std::vector<VkQueueFamilyProperties> families(count);
62
1/2
✓ Branch 1 taken 77 times.
✗ Branch 2 not taken.
 77 vkGetPhysicalDeviceQueueFamilyProperties(context.Physical(), &count, families.data());
63 77 std::uint32_t bits = 64;
64
3/4
✓ Branch 1 taken 77 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 77 times.
✓ Branch 4 taken 77 times.
 154 for (const Queue* queue : context.Queues()) {
65
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 77 times.
 77 bits = std::min(bits, families[queue->Family()].timestampValidBits);
66 77 }
67 77 return bits;
68 77 }
69
70 } // namespace
71
72 /// @brief Timestamp queries around zones, read back into `profiler::AddGpuZone` on the steady
73 /// clock: calibrated where the device reads its clock with the steady clock's, else aligned once
74 /// to when the first zone was recorded.
75 class GpuProfiler::Timer {
76 public:
77 77 Timer(const DeviceContext& context, std::uint32_t bits)
78
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 77 times.
 77 : device_(context.Device()), period_(context.Limits().timestampPeriod),
79
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 77 times.
 77 mask_(bits >= 64 ? ~std::uint64_t{0} : (std::uint64_t{1} << bits) - 1),
80
3/4
✗ Branch 0 not taken.
✓ Branch 1 taken 77 times.
✓ Branch 3 taken 315315 times.
✓ Branch 4 taken 77 times.
 315392 calibrates_(CalibratesSteadyClock(context.Physical())) {
81 77 const VkQueryPoolCreateInfo info{.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO,
82 .queryType = VK_QUERY_TYPE_TIMESTAMP,
83 .queryCount = 2 * kZones};
84 77 CheckVk(vkCreateQueryPool(device_, &info, nullptr, &pool_), "Creating the GPU timer");
85 77 context.Name(VK_OBJECT_TYPE_QUERY_POOL, reinterpret_cast<std::uint64_t>(pool_),
86 "GPU timer");
87
1/2
✓ Branch 0 taken 77 times.
✗ Branch 1 not taken.
 77 if (calibrates_) {
88 77 Calibrate();
89 }
90 77 }
91 77 ~Timer() {
92 77 vkDestroyQueryPool(device_, pool_, nullptr);
93 77 }
94 Timer(const Timer&) = delete;
95 Timer& operator=(const Timer&) = delete;
96
97 22 std::uint32_t Begin(VkCommandBuffer commands, const char* name) {
98 22 OpenZones& open = openZones;
99
1/2
✓ Branch 0 taken 22 times.
✗ Branch 1 not taken.
 22 if (open.count == open.commands.size()) {
100 return kUntimed;
101 }
102 22 const std::uint32_t depth = static_cast<std::uint32_t>(
103 22 std::count(open.commands.begin(), open.commands.begin() + open.count, commands));
104 22 std::uint32_t zone = 0;
105 22 {
106 22 const std::lock_guard lock(mutex_);
107
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
 22 if (tail_ - head_ == kZones) {
108 ✗ return kUntimed;
109 }
110 22 zone = static_cast<std::uint32_t>(tail_++ % kZones);
111
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 44 slots_[zone] = {.name = profiler::Intern(name),
112
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 .frame = profiler::CurrentFrame(),
113
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 .recorded = profiler::Now(),
114 .depth = depth,
115 .state = State::Open};
116 ✗ }
117 22 open.commands[open.count++] = commands;
118 22 vkResetQueryPool(device_, pool_, 2 * zone, 2);
119 22 vkCmdWriteTimestamp2(commands, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, pool_, 2 * zone);
120 22 return zone;
121 }
122
123 22 void End(VkCommandBuffer commands, std::uint32_t zone) {
124 22 vkCmdWriteTimestamp2(commands, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, pool_, 2 * zone + 1);
125 22 --openZones.count;
126 22 const std::lock_guard lock(mutex_);
127 22 slots_[zone].state = State::Closed;
128 22 }
129
130 535 void Collect() {
131 535 const std::lock_guard lock(mutex_);
132
4/6
✓ Branch 0 taken 535 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 535 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 8 times.
✓ Branch 6 taken 527 times.
 535 if (calibrates_ && profiler::Now() - calibratedAt_ > kRecalibrateNs) {
133
1/2
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
 8 Calibrate();
134 }
135
1/2
✓ Branch 1 taken 535 times.
✗ Branch 2 not taken.
 535 const std::uint64_t frame = profiler::CurrentFrame();
136
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 535 times.
 557 for (std::uint64_t i = head_; i < tail_; ++i) {
137 22 const auto zone = static_cast<std::uint32_t>(i % kZones);
138
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
 22 Slot& slot = slots_[zone];
139
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
 22 if (slot.state != State::Closed) {
140 ✗ continue;
141 }
142 // Each timestamp followed by its availability.
143 22 std::array<std::uint64_t, 4> results{};
144
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 const VkResult result = vkGetQueryPoolResults(
145
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 device_, pool_, 2 * zone, 2, sizeof(results), results.data(),
146 2 * sizeof(std::uint64_t),
147 VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
148
3/6
✓ Branch 0 taken 22 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 22 times.
 22 if (result >= 0 && results[1] != 0 && results[3] != 0) {
149
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
 22 if (!aligned_) {
150 ✗ deviceTicks_ = results[0] & mask_;
151 ✗ hostNs_ = slot.recorded;
152 ✗ aligned_ = true;
153 }
154 22 const std::uint64_t start = ToHost(results[0]);
155 66 profiler::AddGpuZone(slot.frame, {.name = slot.name,
156 .start = start,
157
1/2
✓ Branch 0 taken 22 times.
✗ Branch 1 not taken.
 22 .end = std::max(start, ToHost(results[2])),
158
1/2
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
 22 .depth = slot.depth});
159 22 slot.state = State::Read;
160
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ } else if (slot.frame + kStaleFrames < frame) {
161 ✗ slot.state = State::Read;
162 }
163 }
164
3/4
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 535 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 22 times.
 557 while (head_ < tail_ && slots_[head_ % kZones].state == State::Read) {
165 22 slots_[head_++ % kZones].state = State::Free;
166 }
167 535 }
168
169 private:
170 enum class State : std::uint8_t { Free, Open, Closed, Read };
171 struct Slot {
172 const char* name = "";
173 std::uint64_t frame = 0;
174 /// When it was recorded, on the steady clock.
175 std::uint64_t recorded = 0;
176 std::uint32_t depth = 0;
177 State state = State::Free;
178 };
179
180 /// Reads the device's clock with the steady clock's.
181 85 void Calibrate() {
182 #if defined(__linux__)
183 85 const std::array<VkCalibratedTimestampInfoEXT, 2> domains{
184 VkCalibratedTimestampInfoEXT{.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_EXT,
185 .timeDomain = VK_TIME_DOMAIN_DEVICE_EXT},
186 VkCalibratedTimestampInfoEXT{.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_EXT,
187 .timeDomain = kSteadyDomain}};
188 85 std::array<std::uint64_t, 2> timestamps{};
189 85 std::uint64_t deviation = 0;
190
1/2
✓ Branch 1 taken 85 times.
✗ Branch 2 not taken.
 85 if (vkGetCalibratedTimestampsEXT(device_, 2, domains.data(), timestamps.data(),
191 &deviation) == VK_SUCCESS) {
192 85 deviceTicks_ = timestamps[0] & mask_;
193 85 hostNs_ = timestamps[1];
194 85 aligned_ = true;
195 }
196 #endif
197 85 calibratedAt_ = profiler::Now();
198 85 }
199
200 /// @return Device ticks on the steady clock, in nanoseconds.
201 44 std::uint64_t ToHost(std::uint64_t ticks) const {
202 // Ticks wrap at the family's valid bits; those before the reference count back.
203 44 const std::uint64_t ahead = (ticks - deviceTicks_) & mask_;
204 44 const std::uint64_t behind = (deviceTicks_ - ticks) & mask_;
205
1/2
✓ Branch 0 taken 44 times.
✗ Branch 1 not taken.
 44 const double ns = ahead <= mask_ / 2 ? static_cast<double>(ahead) * period_
206 ✗ : -static_cast<double>(behind) * period_;
207 44 return static_cast<std::uint64_t>(static_cast<double>(hostNs_) + ns);
208 }
209
210 VkDevice device_;
211 VkQueryPool pool_ = VK_NULL_HANDLE;
212 double period_;
213 std::uint64_t mask_;
214 bool calibrates_;
215 std::mutex mutex_;
216 std::array<Slot, kZones> slots_{};
217 /// Monotonic slot counts: the oldest unread and the next to record.
218 std::uint64_t head_ = 0;
219 std::uint64_t tail_ = 0;
220 /// A device tick and the steady clock's nanosecond at the same moment.
221 bool aligned_ = false;
222 std::uint64_t deviceTicks_ = 0;
223 std::uint64_t hostNs_ = 0;
224 std::uint64_t calibratedAt_ = 0;
225 };
226
227 77 GpuProfiler::GpuProfiler(void* tracy, std::unique_ptr<Timer> timer)
228 77 : tracy_(tracy), timer_(std::move(timer)) {}
229
230 77 std::unique_ptr<GpuProfiler> GpuProfiler::Create(const DeviceContext& context) {
231
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 77 times.
 77 if (!context.Capabilities().calibratedTimestamps) {
232 ✗ return nullptr;
233 }
234 77 const std::uint32_t bits = TimestampBits(context);
235
1/2
✓ Branch 0 taken 77 times.
✗ Branch 1 not taken.
 77 std::unique_ptr<Timer> timer = bits > 0 ? std::make_unique<Timer>(context, bits) : nullptr;
236 77 void* tracy = nullptr;
237 #if ERNIE_TRACY
238 tracy::VkCtx* vk = tracy::CreateVkContext(
239 context.Physical(), context.Device(), vkResetQueryPool,
240 vkGetPhysicalDeviceCalibrateableTimeDomainsEXT, vkGetCalibratedTimestampsEXT);
241 const std::string& name = context.Info().name;
242 vk->Name(name.c_str(), static_cast<std::uint16_t>(name.size()));
243 tracy = vk;
244 #endif
245
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 77 times.
 77 if (tracy == nullptr && timer == nullptr) {
246 ✗ return nullptr;
247 }
248
1/2
✓ Branch 1 taken 77 times.
✗ Branch 2 not taken.
 77 return std::unique_ptr<GpuProfiler>(new GpuProfiler(tracy, std::move(timer)));
249 77 }
250
251 77 GpuProfiler::~GpuProfiler() {
252 #if ERNIE_TRACY
253 tracy::DestroyVkContext(static_cast<tracy::VkCtx*>(tracy_));
254 #endif
255 77 }
256
257 535 void GpuProfiler::Collect() {
258 // One thread collects at a time; a busy one is collecting already.
259 535 const std::unique_lock lock(collecting_, std::try_to_lock);
260
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 535 times.
 535 if (!lock.owns_lock()) {
261 ✗ return;
262 }
263
1/2
✓ Branch 0 taken 535 times.
✗ Branch 1 not taken.
 535 if (timer_) {
264
1/2
✓ Branch 1 taken 535 times.
✗ Branch 2 not taken.
 535 timer_->Collect();
265 }
266 #if ERNIE_TRACY
267 static_cast<tracy::VkCtx*>(tracy_)->Collect(VK_NULL_HANDLE);
268 #endif
269 535 }
270
271 22 std::uint32_t GpuProfiler::Begin(VkCommandBuffer commands, const char* name) {
272
1/2
✓ Branch 0 taken 22 times.
✗ Branch 1 not taken.
 22 return timer_ ? timer_->Begin(commands, name) : kUntimed;
273 }
274
275 22 void GpuProfiler::End(VkCommandBuffer commands, std::uint32_t zone) {
276 22 timer_->End(commands, zone);
277 22 }
278
279 450 GpuZone::GpuZone(const DeviceContext& context, VkCommandBuffer commands, const char* name)
280 450 : context_(context), commands_(commands) {
281 450 context.BeginLabel(commands, name);
282
1/2
✓ Branch 0 taken 450 times.
✗ Branch 1 not taken.
 450 GpuProfiler* profiler = context.Profiler();
283
1/2
✓ Branch 0 taken 450 times.
✗ Branch 1 not taken.
 450 if (profiler == nullptr) {
284 return;
285 }
286 #if ERNIE_TRACY
287 static_assert(sizeof(tracy::VkCtxScope) <= sizeof(zone_) &&
288 alignof(tracy::VkCtxScope) <= alignof(std::max_align_t));
289 // A transient zone: its name is copied, so it may be built at run time.
290 static constexpr char kFunction[] = "GpuZone";
291 new (zone_) tracy::VkCtxScope(static_cast<tracy::VkCtx*>(profiler->Tracy()), __LINE__, __FILE__,
292 std::strlen(__FILE__), kFunction, sizeof(kFunction) - 1, name,
293 std::strlen(name), commands, true);
294 tracy_ = true;
295 #endif
296
2/2
✓ Branch 1 taken 22 times.
✓ Branch 2 taken 428 times.
 450 if (profiler::GpuEnabled()) {
297 22 timed_ = profiler->Begin(commands, name);
298 }
299 }
300
301 450 GpuZone::~GpuZone() {
302
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 428 times.
 450 if (timed_ != GpuProfiler::kUntimed) {
303 22 context_.Profiler()->End(commands_, timed_);
304 }
305 #if ERNIE_TRACY
306 if (tracy_) {
307 std::launder(reinterpret_cast<tracy::VkCtxScope*>(zone_))->~VkCtxScope();
308 }
309 #endif
310 450 context_.EndLabel(commands_);
311 450 }
312
313 } // namespace ernie
314