GCC Code Coverage Report

ernie/
Coverage:
low: ≥ 0%
medium: ≥ 75.0%
high: ≥ 90.0%
Lines:
102 / 103
99.0%
Branches:
63 / 96
65.6%
Functions:
14 / 14
100.0%
device/staging.cpp
Line Branch Exec Source
1 // SPDX-License-Identifier: BSD-3-Clause
2 /// @file staging.cpp
3 /// @brief Staged uploads in batched copy regions, and readbacks.
4 #include "ernie/device/staging.h"
5
6 #include <algorithm>
7 #include <cstring>
8 #include <stdexcept>
9
10 namespace ernie {
11 namespace {
12
13 /// Orders copies after earlier work that reads or writes what they touch.
14 203 void BeforeCopies(VkCommandBuffer commands) {
15 203 Barrier(commands, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT,
16 VK_PIPELINE_STAGE_2_COPY_BIT,
17 VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT);
18 203 }
19
20 } // namespace
21
22 115 Uploader::Uploader(DeviceContext& context, Queue& queue, std::uint64_t chunk)
23 115 : context_(context), queue_(queue), chunk_(chunk) {}
24
25 115 Uploader::~Uploader() {
26 230 Quietly([this] { Flush(); });
27 115 }
28
29 481 void Uploader::Write(const Buffer& destination, std::uint64_t offset,
30 std::span<const std::byte> bytes) {
31
3/4
✓ Branch 0 taken 481 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✓ Branch 3 taken 480 times.
 481 if (offset > destination.Size() || bytes.size() > destination.Size() - offset) {
32
1/2
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1 throw std::out_of_range("Writing past the end of a buffer");
33 }
34
1/2
✓ Branch 0 taken 480 times.
✗ Branch 1 not taken.
 480 if (bytes.empty()) {
35 return;
36 }
37
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 479 times.
 480 context_.CountUpload(bytes.size());
38
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 479 times.
 480 if (!destination.Mapped().empty()) {
39 1 std::memcpy(destination.Mapped().data() + offset, bytes.data(), bytes.size());
40 1 destination.Flush(offset, bytes.size());
41 1 return;
42 }
43
4/4
✓ Branch 0 taken 360 times.
✓ Branch 1 taken 119 times.
✓ Branch 2 taken 1 time.
✓ Branch 3 taken 359 times.
 479 if (staging_.empty() || used_ + bytes.size() > staging_.back()->Size()) {
44 120 FlushStaging();
45
2/4
✓ Branch 2 taken 120 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 120 times.
 120 staging_.push_back(std::make_shared<Buffer>(
46
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 118 times.
 120 context_, std::max<std::uint64_t>(chunk_, bytes.size()), MemoryUse::Upload, "staging"));
47 120 used_ = 0;
48 }
49 479 const Buffer& staging = *staging_.back();
50 479 std::memcpy(staging.Mapped().data() + used_, bytes.data(), bytes.size());
51 479 copies_.push_back({staging_.size() - 1, destination.Handle(), {used_, offset, bytes.size()}});
52 479 used_ += bytes.size();
53 479 pending_ += bytes.size();
54 }
55
56 239 void Uploader::FlushStaging() {
57 // Once a staging buffer is full or about to be copied from, not once a write.
58
3/4
✓ Branch 0 taken 120 times.
✓ Branch 1 taken 119 times.
✓ Branch 2 taken 120 times.
✗ Branch 3 not taken.
 239 if (!staging_.empty() && used_ > 0) {
59 120 staging_.back()->Flush(0, used_);
60 }
61 239 }
62
63 451 void Uploader::WriteRows(const Buffer& destination, std::uint64_t offset, std::uint64_t rowBytes,
64 std::span<const std::byte> host, std::span<const std::uint32_t> rows) {
65
2/2
✓ Branch 0 taken 225 times.
✓ Branch 1 taken 451 times.
 676 for (std::size_t first = 0; first < rows.size();) {
66 225 std::size_t last = first + 1;
67
4/4
✓ Branch 0 taken 5029 times.
✓ Branch 1 taken 220 times.
✓ Branch 2 taken 5024 times.
✓ Branch 3 taken 5 times.
 5249 while (last < rows.size() && rows[last] == rows[last - 1] + 1) {
68 5024 ++last;
69 }
70
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 225 times.
 225 const std::uint64_t begin = rows[first];
71 225 const std::uint64_t count = rows[last - 1] + 1 - begin;
72
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 225 times.
 225 Write(destination, offset + begin * rowBytes,
73 host.subspan(begin * rowBytes, count * rowBytes));
74 225 first = last;
75 }
76 451 }
77
78 190 Ticket Uploader::Flush(std::span<const Ticket> waits) {
79 // Staging of earlier flushes goes once its copies are done.
80 190 context_.Collect();
81
2/2
✓ Branch 0 taken 146 times.
✓ Branch 1 taken 44 times.
 190 if (copies_.empty()) {
82 146 return {};
83 }
84 44 ERNIE_ZONE("Upload");
85 44 std::vector<std::shared_ptr<const Buffer>> staging;
86 44 const Ticket ticket =
87
1/2
✓ Branch 1 taken 44 times.
✗ Branch 2 not taken.
 88 queue_.Submit([&](VkCommandBuffer commands) { staging = Record(commands); }, waits);
88
2/2
✓ Branch 0 taken 45 times.
✓ Branch 1 taken 44 times.
 89 for (std::shared_ptr<const Buffer>& buffer : staging) {
89
1/2
✓ Branch 1 taken 45 times.
✗ Branch 2 not taken.
 90 context_.Retire(std::move(buffer));
90 }
91 44 return ticket;
92 44 }
93
94 120 std::vector<std::shared_ptr<const Buffer>> Uploader::Record(VkCommandBuffer commands) {
95
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 119 times.
 120 if (copies_.empty()) {
96 1 return {};
97 }
98 119 ERNIE_PLOT("Uploaded bytes", pending_);
99 119 FlushStaging();
100 119 BeforeCopies(commands);
101 // Regions run to the same buffers in one call each.
102 119 std::vector<VkBufferCopy> regions;
103 119 std::size_t first = 0;
104
2/2
✓ Branch 0 taken 131 times.
✓ Branch 1 taken 119 times.
 250 while (first < copies_.size()) {
105 131 std::size_t last = first + 1;
106
1/2
✓ Branch 1 taken 131 times.
✗ Branch 2 not taken.
 131 regions.assign(1, copies_[first].region);
107
4/4
✓ Branch 0 taken 360 times.
✓ Branch 1 taken 119 times.
✓ Branch 2 taken 359 times.
✓ Branch 3 taken 1 time.
 479 while (last < copies_.size() && copies_[last].staging == copies_[first].staging &&
108
2/2
✓ Branch 0 taken 348 times.
✓ Branch 1 taken 11 times.
 359 copies_[last].destination == copies_[first].destination) {
109
1/2
✓ Branch 1 taken 348 times.
✗ Branch 2 not taken.
 348 regions.push_back(copies_[last++].region);
110 }
111
1/2
✓ Branch 1 taken 131 times.
✗ Branch 2 not taken.
 131 vkCmdCopyBuffer(commands, staging_[copies_[first].staging]->Handle(),
112
1/2
✓ Branch 1 taken 131 times.
✗ Branch 2 not taken.
 131 copies_[first].destination, static_cast<std::uint32_t>(regions.size()),
113
1/2
✓ Branch 1 taken 131 times.
✗ Branch 2 not taken.
 131 regions.data());
114 first = last;
115 }
116
1/2
✓ Branch 1 taken 119 times.
✗ Branch 2 not taken.
 119 Barrier(commands, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT,
117 VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, kAnyAccess);
118
1/2
✓ Branch 1 taken 119 times.
✗ Branch 2 not taken.
 119 std::vector<std::shared_ptr<const Buffer>> staging(staging_.begin(), staging_.end());
119 119 staging_.clear();
120 119 copies_.clear();
121 119 used_ = 0;
122 119 pending_ = 0;
123 119 return staging;
124 119 }
125
126 84 Readback::Readback(DeviceContext& context, Queue& queue, const Buffer& source, std::uint64_t offset,
127 84 std::uint64_t size, std::span<const Ticket> waits)
128 84 : context_(context), host_(std::make_shared<Buffer>(context, std::max<std::uint64_t>(size, 1),
129
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 84 times.
✓ Branch 2 taken 84 times.
✗ Branch 3 not taken.
 168 MemoryUse::Readback, "readback")) {
130
2/4
✓ Branch 0 taken 84 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 84 times.
 84 if (offset > source.Size() || size > source.Size() - offset) {
131
0/2
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ throw std::out_of_range("Reading past the end of a buffer");
132 }
133
1/2
✓ Branch 1 taken 84 times.
✗ Branch 2 not taken.
 84 context.CountReadback(size);
134
1/2
✓ Branch 1 taken 84 times.
✗ Branch 2 not taken.
 84 ticket_ = queue.Submit(
135
1/2
✓ Branch 1 taken 84 times.
✗ Branch 2 not taken.
 84 [&](VkCommandBuffer commands) {
136 84 BeforeCopies(commands);
137 84 const VkBufferCopy region{offset, 0, size};
138
1/2
✓ Branch 0 taken 84 times.
✗ Branch 1 not taken.
 84 if (size > 0) {
139 84 vkCmdCopyBuffer(commands, source.Handle(), host_->Handle(), 1, &region);
140 }
141 // Visible to the host's reads once the ticket is reached.
142 84 Barrier(commands, VK_PIPELINE_STAGE_2_COPY_BIT, VK_ACCESS_2_TRANSFER_WRITE_BIT,
143 VK_PIPELINE_STAGE_2_HOST_BIT, VK_ACCESS_2_HOST_READ_BIT);
144 84 },
145 waits);
146
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 84 }
147
148 84 Readback::~Readback() {
149
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 84 times.
 84 context_.Retire(std::move(host_));
150
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 84 times.
 84 }
151
152 84 std::span<const std::byte> Readback::Bytes() {
153 84 ERNIE_ZONE("Read back");
154
1/2
✓ Branch 1 taken 84 times.
✗ Branch 2 not taken.
 84 ticket_.Wait();
155
1/2
✓ Branch 1 taken 84 times.
✗ Branch 2 not taken.
 84 host_->Invalidate(0, host_->Size());
156 84 return host_->Mapped();
157 84 }
158
159 } // namespace ernie
160