device/column.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | /// @file column.cpp | ||
| 3 | /// @brief Allocating device columns and growing them by device copies. | ||
| 4 | #include "ernie/device/column.h" | ||
| 5 | |||
| 6 | #include "ernie/util/on_failure.h" | ||
| 7 | |||
| 8 | #include <algorithm> | ||
| 9 | #include <functional> | ||
| 10 | #include <limits> | ||
| 11 | #include <numeric> | ||
| 12 | #include <stdexcept> | ||
| 13 | #include <utility> | ||
| 14 | |||
| 15 | namespace ernie { | ||
| 16 | namespace { | ||
| 17 | |||
| 18 | 329 | std::uint64_t ElementBytes(DType type, const std::vector<std::int64_t>& element) { | |
| 19 | 329 | return ByteSize(type) * std::accumulate(element.begin(), element.end(), std::uint64_t{1}, | |
| 20 | 287 | [](std::uint64_t product, std::int64_t extent) { | |
| 21 | 287 | return product * static_cast<std::uint64_t>(extent); | |
| 22 | 329 | }); | |
| 23 | } | ||
| 24 | |||
| 25 | } // namespace | ||
| 26 | |||
| 27 | 329 | DeviceColumn::DeviceColumn(Arenas& arenas, std::string_view name, DType type, | |
| 28 | 329 | std::vector<std::int64_t> element, std::uint64_t rows) | |
| 29 | 329 | : arenas_(&arenas), name_(name), type_(type), element_(std::move(element)), | |
| 30 | 329 | rowBytes_(ElementBytes(type, element_)), rows_(rows), capacity_(rows), | |
| 31 | 658 | range_(Allocate(rows)) {} | |
| 32 | |||
| 33 | ✗ | DeviceColumn::DeviceColumn(DeviceContext& context, std::shared_ptr<const Buffer> storage, | |
| 34 | std::uint64_t offset, std::string_view name, DType type, | ||
| 35 | std::vector<std::int64_t> element, std::uint64_t rows, | ||
| 36 | ✗ | std::shared_ptr<const void> owner) | |
| 37 | ✗ | : context_(&context), storage_(std::move(storage)), owner_(std::move(owner)), name_(name), | |
| 38 | ✗ | type_(type), element_(std::move(element)), rowBytes_(ElementBytes(type, element_)), | |
| 39 | ✗ | rows_(rows), capacity_(rows), range_{0, offset, rows * rowBytes_} { | |
| 40 | ✗ | if (offset > storage_->Size() || range_.size > storage_->Size() - offset) { | |
| 41 | ✗ | throw std::out_of_range(name_ + "'s rows run past the end of their memory"); | |
| 42 | } | ||
| 43 | ✗ | } | |
| 44 | |||
| 45 | 329 | DeviceColumn::~DeviceColumn() { | |
| 46 | 329 | if (arenas_ != nullptr) { | |
| 47 | 329 | arenas_->Free(range_); | |
| 48 | } else { | ||
| 49 | ✗ | context_->Retire(std::move(storage_)); | |
| 50 | ✗ | context_->Retire(std::move(owner_)); | |
| 51 | } | ||
| 52 | 329 | } | |
| 53 | |||
| 54 | 565 | BufferRange DeviceColumn::Allocate(std::uint64_t rows) { | |
| 55 | 565 | if (rowBytes_ != 0 && rows > std::numeric_limits<std::uint64_t>::max() / rowBytes_) { | |
| 56 | ✗ | throw DeviceError(DeviceErrorKind::Unsupported, | |
| 57 | ✗ | std::to_string(rows) + " rows of " + name_ + " overflow"); | |
| 58 | } | ||
| 59 | // Arenas hand out no empty ranges. | ||
| 60 | 848 | return arenas_->Allocate(std::max<std::uint64_t>(rowBytes_ * rows, 1)); | |
| 61 | } | ||
| 62 | |||
| 63 | 477 | Ticket DeviceColumn::Resize(std::uint64_t rows, Queue& queue, std::span<const Ticket> waits) { | |
| 64 | 477 | if (rows <= capacity_) { | |
| 65 | 241 | rows_ = rows; | |
| 66 | 241 | return {}; | |
| 67 | } | ||
| 68 | 236 | if (arenas_ == nullptr) { | |
| 69 | ✗ | throw std::logic_error(name_ + " is another API's memory, which Ernie cannot grow"); | |
| 70 | } | ||
| 71 | // Nothing changes until the new range holds the rows, so a failure leaves the column whole. | ||
| 72 | 236 | const std::uint64_t capacity = std::max(rows, 2 * capacity_); | |
| 73 | 236 | const BufferRange grown = Allocate(capacity); | |
| 74 | 236 | const OnFailure freeGrown([&] { arenas_->Free(grown); }); | |
| 75 | 236 | const std::uint64_t kept = rowBytes_ * rows_; | |
| 76 | 236 | Ticket ticket; | |
| 77 | 236 | if (kept > 0) { | |
| 78 | 26 | ticket = queue.Submit( | |
| 79 | 52 | [&](VkCommandBuffer commands) { | |
| 80 | 26 | const VkBufferCopy region{range_.offset, grown.offset, kept}; | |
| 81 | 26 | CopyBuffer(commands, arenas_->Storage(range_).Handle(), | |
| 82 | 26 | arenas_->Storage(grown).Handle(), std::span(®ion, 1)); | |
| 83 | 26 | }, | |
| 84 | waits); | ||
| 85 | } | ||
| 86 | 236 | arenas_->Free(std::exchange(range_, grown)); | |
| 87 | 236 | capacity_ = capacity; | |
| 88 | 236 | rows_ = rows; | |
| 89 | 236 | return ticket; | |
| 90 | 236 | } | |
| 91 | |||
| 92 | } // namespace ernie | ||
| 93 |