graph/graph.h
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | /// @file graph.h | ||
| 3 | /// @brief The frame graph: passes declare the images and buffers they read and write, and the | ||
| 4 | /// graph culls what nothing needs, backs transients with pooled memory and records the passes in | ||
| 5 | /// order with the barriers between them. | ||
| 6 | /// | ||
| 7 | /// graph::Graph graph(context, pool); | ||
| 8 | /// const graph::Buffer poses = graph.Import(poseColumn, "poses"); | ||
| 9 | /// graph::Image radiance = graph.Create(description, "radiance"); | ||
| 10 | /// graph.Add("Shade", graph::Kind::Compute, | ||
| 11 | /// [&](graph::PassBuilder& pass) { | ||
| 12 | /// pass.ReadAddress(poses); | ||
| 13 | /// radiance = pass.Write(radiance); | ||
| 14 | /// }, | ||
| 15 | /// [=](graph::PassContext& pass) { | ||
| 16 | /// shade.Dispatch(pass.commands, groups, Parameters{pass.Get(poses).Address(), ...}); | ||
| 17 | /// }); | ||
| 18 | /// const Ticket done = graph.Execute(context.Main()); | ||
| 19 | /// | ||
| 20 | /// Setup runs inside `Add`; run callbacks run in `Execute`. Passes run in the order they were | ||
| 21 | /// added. Imports enter and leave in the general layout. Bindless and device-address reads are | ||
| 22 | /// invisible to the graph, so passes declare them (`ReadBindless`, `ReadAddress`). | ||
| 23 | /// ERNIE_GRAPH_FULL_BARRIERS=1 adds a full barrier before every pass, to rule out a missing one. | ||
| 24 | /// | ||
| 25 | /// References: | ||
| 26 | /// - Y. O'Donnell, "FrameGraph: Extensible Rendering Architecture in Frostbite", GDC 2017. | ||
| 27 | /// - Epic Games, Unreal Engine's Render Dependency Graph (RenderCore/RenderGraphBuilder.cpp). | ||
| 28 | /// - Google, Filament's FrameGraph. https://github.com/google/filament | ||
| 29 | /// - H.-K. Arntzen, "Render graphs and Vulkan — a deep dive", 2017. | ||
| 30 | #pragma once | ||
| 31 | |||
| 32 | #include "ernie/device/acceleration.h" | ||
| 33 | #include "ernie/device/column.h" | ||
| 34 | #include "ernie/device/image.h" | ||
| 35 | #include "ernie/graph/blackboard.h" | ||
| 36 | #include "ernie/graph/dependencies.h" | ||
| 37 | #include "ernie/graph/pool.h" | ||
| 38 | |||
| 39 | #include <cstdint> | ||
| 40 | #include <functional> | ||
| 41 | #include <memory> | ||
| 42 | #include <span> | ||
| 43 | #include <stdexcept> | ||
| 44 | #include <string> | ||
| 45 | #include <utility> | ||
| 46 | #include <vector> | ||
| 47 | |||
| 48 | namespace ernie::graph { | ||
| 49 | |||
| 50 | /// @brief A graph that cannot run: a pass reads what nothing wrote, or uses a resource in a way | ||
| 51 | /// it cannot. | ||
| 52 | class GraphError : public std::runtime_error { | ||
| 53 | public: | ||
| 54 | 1 | using std::runtime_error::runtime_error; | |
| 55 | }; | ||
| 56 | |||
| 57 | /// @brief A graph's image; each write makes a new version. | ||
| 58 | struct Image { | ||
| 59 | std::uint32_t id = ~0u; | ||
| 60 | /// Writes declared before this handle was made. | ||
| 61 | std::uint32_t version = 0; | ||
| 62 | bool Valid() const { | ||
| 63 | return id != ~0u; | ||
| 64 | } | ||
| 65 | }; | ||
| 66 | |||
| 67 | /// @brief A graph's buffer; each write makes a new version. | ||
| 68 | struct Buffer { | ||
| 69 | std::uint32_t id = ~0u; | ||
| 70 | std::uint32_t version = 0; | ||
| 71 | 273 | bool Valid() const { | |
| 72 | 273 | return id != ~0u; | |
| 73 | } | ||
| 74 | }; | ||
| 75 | |||
| 76 | /// @brief A graph's acceleration structure; each build or refit makes a new version. | ||
| 77 | struct Acceleration { | ||
| 78 | std::uint32_t id = ~0u; | ||
| 79 | std::uint32_t version = 0; | ||
| 80 | bool Valid() const { | ||
| 81 | return id != ~0u; | ||
| 82 | } | ||
| 83 | }; | ||
| 84 | |||
| 85 | /// @brief A buffer resource as a pass records with it: a range of a device buffer. | ||
| 86 | struct BufferSlice { | ||
| 87 | const ernie::Buffer* buffer = nullptr; | ||
| 88 | std::uint64_t offset = 0; | ||
| 89 | std::uint64_t size = 0; | ||
| 90 | 216 | VkDeviceAddress Address() const { | |
| 91 | 215 | return buffer->Address() + offset; | |
| 92 | } | ||
| 93 | /// @return A shader's handle of the slice as `T`s. | ||
| 94 | 83 | template <typename T> shaders::BufferRef<T> BufferRef() const { | |
| 95 | 83 | return {Address()}; | |
| 96 | } | ||
| 97 | }; | ||
| 98 | |||
| 99 | class Graph; | ||
| 100 | |||
| 101 | /// @brief Declares one pass's accesses during setup. | ||
| 102 | class PassBuilder { | ||
| 103 | public: | ||
| 104 | /// @return The same handle. | ||
| 105 | /// @throws std::invalid_argument for a use that writes. | ||
| 106 | Image Read(Image image, Use use = Use::Sampled, Layers layers = {}); | ||
| 107 | /// @brief Declares a write, or a read and write. | ||
| 108 | /// @return The new version. | ||
| 109 | /// @throws std::invalid_argument for a use that only reads. | ||
| 110 | Image Write(Image image, Use use = Use::StorageWrite, Layers layers = {}); | ||
| 111 | Buffer Read(Buffer buffer, Use use = Use::StorageRead); | ||
| 112 | Buffer Write(Buffer buffer, Use use = Use::StorageWrite); | ||
| 113 | /// @brief Declares a read through the bindless heap. | ||
| 114 | Image ReadBindless(Image image, Use use = Use::Sampled, Layers layers = {}); | ||
| 115 | /// @brief Declares a read through a device address. | ||
| 116 | Buffer ReadAddress(Buffer buffer, Use use = Use::StorageRead); | ||
| 117 | /// @brief Declares a write through a device address. | ||
| 118 | Buffer WriteAddress(Buffer buffer, Use use = Use::StorageReadWrite); | ||
| 119 | /// @brief Declares building, refitting or compacting into a structure. | ||
| 120 | /// @return The new version. | ||
| 121 | Acceleration Build(Acceleration structure); | ||
| 122 | /// @brief Declares tracing a structure, or reading it to build a top level, copy or query it | ||
| 123 | /// (`Use::BuildInput`). | ||
| 124 | /// @throws std::invalid_argument for a use other than tracing or a build's input. | ||
| 125 | Acceleration Read(Acceleration structure, Use use = Use::Trace); | ||
| 126 | /// @brief Keeps the pass though nothing reads what it writes, as for one the host reads. | ||
| 127 | void NeverCull(); | ||
| 128 | |||
| 129 | private: | ||
| 130 | friend class Graph; | ||
| 131 | enum class Shape : std::uint8_t { Image, Buffer, Acceleration }; | ||
| 132 | 244 | PassBuilder(Graph& graph, std::uint32_t pass) : graph_(graph), pass_(pass) {} | |
| 133 | |||
| 134 | std::uint32_t Declare(std::uint32_t id, std::uint32_t version, Shape shape, Use use, | ||
| 135 | Layers layers, Reach reach, bool write); | ||
| 136 | |||
| 137 | Graph& graph_; | ||
| 138 | std::uint32_t pass_; | ||
| 139 | }; | ||
| 140 | |||
| 141 | /// @brief What a pass's run callback records with. | ||
| 142 | class PassContext { | ||
| 143 | public: | ||
| 144 | VkCommandBuffer commands = VK_NULL_HANDLE; | ||
| 145 | |||
| 146 | /// @throws std::logic_error unless the pass declared it, or for a pool's structure. | ||
| 147 | DeviceImage& Get(Image image) const; | ||
| 148 | BufferSlice Get(Buffer buffer) const; | ||
| 149 | const AccelerationStructure& Get(Acceleration structure) const; | ||
| 150 | /// @return The image's layout during the pass, for attachments. | ||
| 151 | VkImageLayout Layout(Image image) const; | ||
| 152 | DeviceContext& Context() const; | ||
| 153 | /// @brief Keeps an object, such as the staging a copy reads, until the graph's work is done. | ||
| 154 | void Keep(std::shared_ptr<const void> object) const; | ||
| 155 | |||
| 156 | private: | ||
| 157 | friend class Graph; | ||
| 158 | 238 | PassContext(const Graph& graph, std::uint32_t pass) : graph_(graph), pass_(pass) {} | |
| 159 | |||
| 160 | void Check(std::uint32_t id) const; | ||
| 161 | |||
| 162 | const Graph& graph_; | ||
| 163 | std::uint32_t pass_; | ||
| 164 | }; | ||
| 165 | |||
| 166 | /// @brief Passes in order and the resources they use, built for one frame: set up, compiled | ||
| 167 | /// and executed once. | ||
| 168 | class Graph { | ||
| 169 | public: | ||
| 170 | /// @param pool Where transients come from; outlives the graph. | ||
| 171 | Graph(DeviceContext& context, TransientPool& pool); | ||
| 172 | ~Graph(); | ||
| 173 | Graph(const Graph&) = delete; | ||
| 174 | Graph& operator=(const Graph&) = delete; | ||
| 175 | |||
| 176 | /// @brief Uses an image owned outside the graph; importing it again returns the same | ||
| 177 | /// resource. The image outlives `Execute`'s work. | ||
| 178 | /// @return The handle of its latest version. | ||
| 179 | Image Import(DeviceImage& image, std::string_view name); | ||
| 180 | /// @brief Uses a column's rows, as kernels, CUDA or Warp write them, under `name` or the | ||
| 181 | /// column's own. | ||
| 182 | Buffer Import(const DeviceColumn& column, std::string_view name = {}); | ||
| 183 | /// @brief Uses a range of a buffer owned outside the graph. | ||
| 184 | Buffer Import(const ernie::Buffer& buffer, std::uint64_t offset, std::uint64_t size, | ||
| 185 | std::string_view name); | ||
| 186 | /// @brief Uses an acceleration structure owned outside the graph, such as a pooled bottom or | ||
| 187 | /// top level; importing it again returns the same resource. | ||
| 188 | Acceleration Import(const AccelerationStructure& structure); | ||
| 189 | /// @brief Uses every structure in a pool as one resource, as thousands of bottom levels are | ||
| 190 | /// built by one pass and read by the top level's build: the barriers are the same, at the | ||
| 191 | /// cost of one resource rather than thousands. | ||
| 192 | Acceleration Import(const AccelerationPool& pool); | ||
| 193 | /// @brief Declares an image living for one execution; the usage its passes declare is | ||
| 194 | /// added to `description`'s. | ||
| 195 | Image Create(const ImageDescription& description, std::string_view name); | ||
| 196 | /// @brief Declares a device-local buffer living for one execution. | ||
| 197 | Buffer Create(std::uint64_t bytes, std::string_view name); | ||
| 198 | |||
| 199 | /// @brief Adds a pass after the others, running `setup` now. | ||
| 200 | /// @param setup Called with a `PassBuilder&` to declare accesses. | ||
| 201 | /// @param run Called with a `PassContext&` in `Execute`. | ||
| 202 | /// @throws std::logic_error after `Compile`. | ||
| 203 | template <typename Setup, typename Run> | ||
| 204 | 244 | void Add(std::string_view name, Kind kind, Setup&& setup, Run&& run) { | |
| 205 | 244 | PassBuilder builder(*this, NewPass(name, kind)); | |
| 206 | 242 | std::forward<Setup>(setup)(builder); | |
| 207 | 242 | passes_.back().run = std::forward<Run>(run); | |
| 208 | 242 | } | |
| 209 | |||
| 210 | /// @brief A named group of the passes added while it lives, labelled and timed as one region. | ||
| 211 | class [[nodiscard]] ScopeGuard { | ||
| 212 | public: | ||
| 213 | ~ScopeGuard(); | ||
| 214 | ScopeGuard(const ScopeGuard&) = delete; | ||
| 215 | ScopeGuard& operator=(const ScopeGuard&) = delete; | ||
| 216 | |||
| 217 | private: | ||
| 218 | friend class Graph; | ||
| 219 | 55 | explicit ScopeGuard(Graph& graph) : graph_(graph) {} | |
| 220 | Graph& graph_; | ||
| 221 | }; | ||
| 222 | /// @brief Opens a scope for the passes added until the guard is destroyed; scopes nest. | ||
| 223 | ScopeGuard Scope(std::string_view name); | ||
| 224 | |||
| 225 | /// @return Handles that passes share by type. | ||
| 226 | 52 | Blackboard& Board() { | |
| 227 | 52 | return board_; | |
| 228 | } | ||
| 229 | |||
| 230 | /// @brief Drops passes nothing needs, backs transients and computes the barriers. | ||
| 231 | /// `Execute` calls it. | ||
| 232 | /// @throws GraphError if a pass reads what nothing wrote, or uses a resource as it cannot. | ||
| 233 | void Compile(); | ||
| 234 | /// @brief Records every kept pass in order with its barriers, submits them on `queue` after | ||
| 235 | /// `waits`, and gives transients back to the pool. | ||
| 236 | /// @param binary Semaphores the graph's work waits on and signals, as a swapchain image's. | ||
| 237 | /// @return The ticket of the graph's work, which CUDA can wait on too. | ||
| 238 | /// @throws GraphError as `Compile`; std::logic_error if executed already. | ||
| 239 | Ticket Execute(Queue& queue, std::span<const Ticket> waits = {}, BinarySemaphores binary = {}); | ||
| 240 | |||
| 241 | /// @brief Adds a full barrier before every pass; on by default with | ||
| 242 | /// ERNIE_GRAPH_FULL_BARRIERS=1. | ||
| 243 | 2 | void SetFullBarriers(bool on) { | |
| 244 | 2 | fullBarriers_ = on; | |
| 245 | } | ||
| 246 | /// @brief Submits each pass on its own and waits for it, so a device fault names its pass. | ||
| 247 | 2 | void SetFlushEachPass(bool on) { | |
| 248 | 2 | flushEachPass_ = on; | |
| 249 | } | ||
| 250 | |||
| 251 | /// @return The dependencies `Compile` computed, for the passes it kept. | ||
| 252 | const PassDependencies& Dependencies() const { | ||
| 253 | return dependencies_; | ||
| 254 | } | ||
| 255 | /// @return The names of the passes `Compile` kept, in order. | ||
| 256 | std::vector<std::string> ExecutedPasses() const; | ||
| 257 | /// @return The graph in Graphviz's dot language. | ||
| 258 | std::string Dot() const; | ||
| 259 | |||
| 260 | private: | ||
| 261 | friend class PassBuilder; | ||
| 262 | friend class PassContext; | ||
| 263 | |||
| 264 | static constexpr std::uint32_t kNoScope = ~0u; | ||
| 265 | |||
| 266 | 1920 | struct Resource { | |
| 267 | std::string name; | ||
| 268 | bool image = false; | ||
| 269 | bool imported = false; | ||
| 270 | DeviceImage* deviceImage = nullptr; | ||
| 271 | /// A buffer's range, or the memory a structure lives in. | ||
| 272 | BufferSlice slice; | ||
| 273 | const AccelerationStructure* acceleration = nullptr; | ||
| 274 | /// Every structure of a pool, tracked as one. | ||
| 275 | const AccelerationPool* structures = nullptr; | ||
| 276 | 4361 | bool Structure() const { | |
| 277 | 4361 | return acceleration != nullptr || structures != nullptr; | |
| 278 | } | ||
| 279 | /// A transient's description, or size, and its leased range. | ||
| 280 | ImageDescription description; | ||
| 281 | std::uint64_t bytes = 0; | ||
| 282 | BufferRange range; | ||
| 283 | std::uint32_t version = 0; | ||
| 284 | Origin origin = Origin::Fresh; | ||
| 285 | }; | ||
| 286 | struct Pass { | ||
| 287 | std::string name; | ||
| 288 | Kind kind = Kind::Compute; | ||
| 289 | std::vector<DependencyAccess> accesses; | ||
| 290 | std::function<void(PassContext&)> run; | ||
| 291 | /// The innermost scope it was added in. | ||
| 292 | std::uint32_t scope = kNoScope; | ||
| 293 | bool neverCull = false; | ||
| 294 | }; | ||
| 295 | 190 | struct ScopeNode { | |
| 296 | std::string name; | ||
| 297 | std::uint32_t parent = kNoScope; | ||
| 298 | std::uint32_t depth = 0; | ||
| 299 | }; | ||
| 300 | |||
| 301 | std::uint32_t NewPass(std::string_view name, Kind kind); | ||
| 302 | std::uint32_t AddResource(Resource resource); | ||
| 303 | std::vector<std::uint32_t> LivePasses() const; | ||
| 304 | void Lease(const std::vector<std::uint32_t>& live); | ||
| 305 | void ReleaseLeases(); | ||
| 306 | void Record(VkCommandBuffer commands, const Dependency& dependency) const; | ||
| 307 | /// Records a pass with its barriers, inside a GPU zone for itself and each scope from | ||
| 308 | /// `depth` in. | ||
| 309 | void RecordPass(VkCommandBuffer commands, std::uint32_t order, std::uint32_t depth = 0) const; | ||
| 310 | |||
| 311 | DeviceContext& context_; | ||
| 312 | TransientPool& pool_; | ||
| 313 | std::vector<Resource> resources_; | ||
| 314 | std::vector<Pass> passes_; | ||
| 315 | std::vector<ScopeNode> scopes_; | ||
| 316 | std::uint32_t scope_ = kNoScope; | ||
| 317 | Blackboard board_; | ||
| 318 | /// Objects passes keep, retired once the graph's work is submitted. | ||
| 319 | mutable std::vector<std::shared_ptr<const void>> kept_; | ||
| 320 | /// The passes kept, in order, and their dependencies. | ||
| 321 | std::vector<std::uint32_t> live_; | ||
| 322 | PassDependencies dependencies_; | ||
| 323 | bool compiled_ = false; | ||
| 324 | bool executed_ = false; | ||
| 325 | bool leased_ = false; | ||
| 326 | bool fullBarriers_ = false; | ||
| 327 | bool flushEachPass_ = false; | ||
| 328 | }; | ||
| 329 | |||
| 330 | /// @brief Adds a copy pass recording what `uploader` staged into `targets`, which later passes then | ||
| 331 | /// see written; invalid handles among them are skipped. The staging buffers live until the graph's | ||
| 332 | /// work is done. Nothing is added when nothing is staged. | ||
| 333 | void AddUploadPass(Graph& graph, std::string_view name, Uploader& uploader, | ||
| 334 | std::span<Buffer> targets); | ||
| 335 | |||
| 336 | } // namespace ernie::graph | ||
| 337 |