graph/graph.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | /// @file graph.cpp | ||
| 3 | /// @brief Declaring, culling, backing and recording the frame graph's passes. | ||
| 4 | #include "ernie/graph/graph.h" | ||
| 5 | |||
| 6 | #include <algorithm> | ||
| 7 | #include <cstdlib> | ||
| 8 | #include <sstream> | ||
| 9 | #include <string_view> | ||
| 10 | |||
| 11 | #ifndef ERNIE_GRAPH_FULL_BARRIERS | ||
| 12 | #define ERNIE_GRAPH_FULL_BARRIERS 0 | ||
| 13 | #endif | ||
| 14 | |||
| 15 | namespace ernie::graph { | ||
| 16 | namespace { | ||
| 17 | |||
| 18 | 47 | bool FullBarriersByDefault() { | |
| 19 | #if ERNIE_GRAPH_FULL_BARRIERS | ||
| 20 | return true; | ||
| 21 | #else | ||
| 22 | 47 | const char* value = std::getenv("ERNIE_GRAPH_FULL_BARRIERS"); | |
| 23 | 47 | return value != nullptr && std::string_view(value) == "1"; | |
| 24 | #endif | ||
| 25 | } | ||
| 26 | |||
| 27 | 2473 | bool Writes(Use use) { | |
| 28 | 2473 | switch (use) { | |
| 29 | case Use::StorageWrite: | ||
| 30 | case Use::StorageReadWrite: | ||
| 31 | case Use::ColorWrite: | ||
| 32 | case Use::ColorReadWrite: | ||
| 33 | case Use::DepthWrite: | ||
| 34 | case Use::DepthReadWrite: | ||
| 35 | case Use::CopyDst: | ||
| 36 | case Use::BuildScratch: | ||
| 37 | case Use::BuildOutput: | ||
| 38 | return true; | ||
| 39 | 1314 | default: | |
| 40 | 1314 | return false; | |
| 41 | } | ||
| 42 | } | ||
| 43 | |||
| 44 | 867 | bool Reads(Use use) { | |
| 45 | 867 | return !Writes(use) || use == Use::StorageReadWrite || use == Use::ColorReadWrite || | |
| 46 | 867 | use == Use::DepthReadWrite; | |
| 47 | } | ||
| 48 | |||
| 49 | 867 | ImageUsage NeededUsage(Use use) { | |
| 50 | 867 | switch (use) { | |
| 51 | case Use::Sampled: | ||
| 52 | return ImageUsage::Sampled; | ||
| 53 | 458 | case Use::StorageRead: | |
| 54 | 458 | case Use::StorageWrite: | |
| 55 | 458 | case Use::StorageReadWrite: | |
| 56 | 458 | return ImageUsage::Storage; | |
| 57 | ✗ | case Use::ColorWrite: | |
| 58 | ✗ | case Use::ColorReadWrite: | |
| 59 | ✗ | case Use::DepthWrite: | |
| 60 | ✗ | case Use::DepthReadWrite: | |
| 61 | ✗ | case Use::DepthRead: | |
| 62 | ✗ | return ImageUsage::Attachment; | |
| 63 | 403 | default: | |
| 64 | // Every image takes transfer usage. | ||
| 65 | 403 | return ImageUsage{}; | |
| 66 | } | ||
| 67 | } | ||
| 68 | |||
| 69 | } // namespace | ||
| 70 | |||
| 71 | 871 | std::uint32_t PassBuilder::Declare(std::uint32_t id, std::uint32_t version, Shape shape, Use use, | |
| 72 | Layers layers, Reach reach, bool write) { | ||
| 73 | 1742 | const auto resourceShape = [](const Graph::Resource& resource) { | |
| 74 | 871 | return resource.image ? Shape::Image | |
| 75 | 67 | : resource.Structure() ? Shape::Acceleration | |
| 76 | : Shape::Buffer; | ||
| 77 | }; | ||
| 78 | 1716 | if (id >= graph_.resources_.size() || resourceShape(graph_.resources_[id]) != shape) { | |
| 79 | ✗ | static constexpr const char* kShapes[] = {"image", "buffer", "acceleration structure"}; | |
| 80 | ✗ | throw std::invalid_argument(std::string("An invalid ") + | |
| 81 | ✗ | kShapes[static_cast<std::size_t>(shape)] + " handle"); | |
| 82 | } | ||
| 83 | 871 | Graph::Resource& resource = graph_.resources_[id]; | |
| 84 | 871 | Graph::Pass& pass = graph_.passes_[pass_]; | |
| 85 | 871 | if (Writes(use) != write) { | |
| 86 | ✗ | throw std::invalid_argument("Pass '" + pass.name + "' declares " + ToString(use) + " on '" + | |
| 87 | ✗ | resource.name + "' as a " + (write ? "write" : "read")); | |
| 88 | } | ||
| 89 | // Structures are built and traced; the memory they live in is no plain buffer, and the other | ||
| 90 | // way round. | ||
| 91 | 871 | const bool structureUse = use == Use::BuildOutput || use == Use::Trace; | |
| 92 | 871 | if (structureUse != (shape == Shape::Acceleration) && use != Use::BuildInput) { | |
| 93 | 1 | throw std::invalid_argument( | |
| 94 | 6 | "Pass '" + pass.name + "' declares " + ToString(use) + " on '" + resource.name + | |
| 95 | 2 | "', which " + | |
| 96 | 3 | (structureUse ? "is no acceleration structure" : "is an acceleration structure")); | |
| 97 | } | ||
| 98 | 870 | if (version > resource.version) { | |
| 99 | ✗ | throw GraphError("Pass '" + pass.name + "' uses a version of '" + resource.name + | |
| 100 | ✗ | "' not written yet"); | |
| 101 | } | ||
| 102 | 870 | pass.accesses.push_back({id, use, layers, reach}); | |
| 103 | 870 | if (write) { | |
| 104 | 431 | ++resource.version; | |
| 105 | } | ||
| 106 | 870 | return resource.version; | |
| 107 | } | ||
| 108 | |||
| 109 | 10 | Image PassBuilder::Read(Image image, Use use, Layers layers) { | |
| 110 | 10 | Declare(image.id, image.version, Shape::Image, use, layers, Reach::Descriptor, false); | |
| 111 | 10 | return image; | |
| 112 | } | ||
| 113 | |||
| 114 | 16 | Image PassBuilder::Write(Image image, Use use, Layers layers) { | |
| 115 | 16 | return {image.id, | |
| 116 | 16 | Declare(image.id, image.version, Shape::Image, use, layers, Reach::Descriptor, true)}; | |
| 117 | } | ||
| 118 | |||
| 119 | 46 | Buffer PassBuilder::Read(Buffer buffer, Use use) { | |
| 120 | 46 | Declare(buffer.id, buffer.version, Shape::Buffer, use, {}, Reach::Descriptor, false); | |
| 121 | 45 | return buffer; | |
| 122 | } | ||
| 123 | |||
| 124 | 229 | Buffer PassBuilder::Write(Buffer buffer, Use use) { | |
| 125 | 229 | return {buffer.id, | |
| 126 | 229 | Declare(buffer.id, buffer.version, Shape::Buffer, use, {}, Reach::Descriptor, true)}; | |
| 127 | } | ||
| 128 | |||
| 129 | ✗ | Image PassBuilder::ReadBindless(Image image, Use use, Layers layers) { | |
| 130 | ✗ | Declare(image.id, image.version, Shape::Image, use, layers, Reach::Bindless, false); | |
| 131 | ✗ | return image; | |
| 132 | } | ||
| 133 | |||
| 134 | 310 | Buffer PassBuilder::ReadAddress(Buffer buffer, Use use) { | |
| 135 | 310 | Declare(buffer.id, buffer.version, Shape::Buffer, use, {}, Reach::Address, false); | |
| 136 | 310 | return buffer; | |
| 137 | } | ||
| 138 | |||
| 139 | 137 | Buffer PassBuilder::WriteAddress(Buffer buffer, Use use) { | |
| 140 | 137 | return {buffer.id, | |
| 141 | 137 | Declare(buffer.id, buffer.version, Shape::Buffer, use, {}, Reach::Address, true)}; | |
| 142 | } | ||
| 143 | |||
| 144 | 49 | Acceleration PassBuilder::Build(Acceleration structure) { | |
| 145 | 49 | return {structure.id, Declare(structure.id, structure.version, Shape::Acceleration, | |
| 146 | 49 | Use::BuildOutput, {}, Reach::Address, true)}; | |
| 147 | } | ||
| 148 | |||
| 149 | 75 | Acceleration PassBuilder::Read(Acceleration structure, Use use) { | |
| 150 | 75 | if (use != Use::Trace && use != Use::BuildInput) { | |
| 151 | 1 | throw std::invalid_argument(std::string("An acceleration structure is traced or read by " | |
| 152 | 2 | "builds, not used as ") + | |
| 153 | 3 | ToString(use)); | |
| 154 | } | ||
| 155 | 74 | Declare(structure.id, structure.version, Shape::Acceleration, use, {}, Reach::Address, false); | |
| 156 | 74 | return structure; | |
| 157 | } | ||
| 158 | |||
| 159 | 8 | void PassBuilder::NeverCull() { | |
| 160 | 8 | graph_.passes_[pass_].neverCull = true; | |
| 161 | 8 | } | |
| 162 | |||
| 163 | 304 | void PassContext::Check(std::uint32_t id) const { | |
| 164 | 304 | const Graph::Pass& pass = graph_.passes_[pass_]; | |
| 165 | 304 | if (std::ranges::none_of( | |
| 166 | 1387 | pass.accesses, [&](const DependencyAccess& access) { return access.resource == id; })) { | |
| 167 | ✗ | throw std::logic_error( | |
| 168 | ✗ | "Pass '" + pass.name + "' did not declare '" + | |
| 169 | ✗ | (id < graph_.resources_.size() ? graph_.resources_[id].name : std::string("?")) + "'"); | |
| 170 | } | ||
| 171 | 304 | } | |
| 172 | |||
| 173 | 8 | DeviceImage& PassContext::Get(Image image) const { | |
| 174 | 8 | Check(image.id); | |
| 175 | 8 | return *graph_.resources_[image.id].deviceImage; | |
| 176 | } | ||
| 177 | |||
| 178 | 287 | BufferSlice PassContext::Get(Buffer buffer) const { | |
| 179 | 287 | Check(buffer.id); | |
| 180 | 287 | return graph_.resources_[buffer.id].slice; | |
| 181 | } | ||
| 182 | |||
| 183 | 1 | const AccelerationStructure& PassContext::Get(Acceleration structure) const { | |
| 184 | 1 | Check(structure.id); | |
| 185 | 1 | const Graph::Resource& resource = graph_.resources_[structure.id]; | |
| 186 | 1 | if (resource.acceleration == nullptr) { | |
| 187 | ✗ | throw std::logic_error("'" + resource.name + "' is a pool of structures, not one"); | |
| 188 | } | ||
| 189 | 1 | return *resource.acceleration; | |
| 190 | } | ||
| 191 | |||
| 192 | 8 | VkImageLayout PassContext::Layout(Image image) const { | |
| 193 | 8 | Check(image.id); | |
| 194 | 8 | const Graph::Pass& pass = graph_.passes_[pass_]; | |
| 195 | 8 | const auto access = std::ranges::find(pass.accesses, image.id, &DependencyAccess::resource); | |
| 196 | 8 | return ToState(pass.kind, access->use, true).layout; | |
| 197 | } | ||
| 198 | |||
| 199 | ✗ | DeviceContext& PassContext::Context() const { | |
| 200 | ✗ | return graph_.context_; | |
| 201 | } | ||
| 202 | |||
| 203 | 81 | void PassContext::Keep(std::shared_ptr<const void> object) const { | |
| 204 | 81 | graph_.kept_.push_back(std::move(object)); | |
| 205 | 81 | } | |
| 206 | |||
| 207 | 47 | Graph::Graph(DeviceContext& context, TransientPool& pool) | |
| 208 | 47 | : context_(context), pool_(pool), fullBarriers_(FullBarriersByDefault()) {} | |
| 209 | |||
| 210 | 47 | Graph::~Graph() { | |
| 211 | 47 | ReleaseLeases(); | |
| 212 | 47 | } | |
| 213 | |||
| 214 | 548 | std::uint32_t Graph::AddResource(Resource resource) { | |
| 215 | 548 | resources_.push_back(std::move(resource)); | |
| 216 | 548 | return static_cast<std::uint32_t>(resources_.size() - 1); | |
| 217 | } | ||
| 218 | |||
| 219 | 12 | Image Graph::Import(DeviceImage& image, std::string_view name) { | |
| 220 | 12 | const auto found = std::ranges::find(resources_, &image, &Resource::deviceImage); | |
| 221 | 12 | if (found != resources_.end()) { | |
| 222 | ✗ | return {static_cast<std::uint32_t>(found - resources_.begin()), found->version}; | |
| 223 | } | ||
| 224 | 12 | return {AddResource({.name = std::string(name), | |
| 225 | .image = true, | ||
| 226 | .imported = true, | ||
| 227 | .deviceImage = &image, | ||
| 228 | .origin = Origin::Imported}), | ||
| 229 | 12 | 0}; | |
| 230 | 24 | } | |
| 231 | |||
| 232 | 493 | Buffer Graph::Import(const DeviceColumn& column, std::string_view name) { | |
| 233 | 968 | return Import(column.Storage(), column.Range().offset, column.Bytes(), | |
| 234 | 493 | name.empty() ? column.Name() : name); | |
| 235 | } | ||
| 236 | |||
| 237 | 31 | Acceleration Graph::Import(const AccelerationStructure& structure) { | |
| 238 | 31 | const auto found = std::ranges::find(resources_, &structure, &Resource::acceleration); | |
| 239 | 31 | if (found != resources_.end()) { | |
| 240 | 1 | return {static_cast<std::uint32_t>(found - resources_.begin()), found->version}; | |
| 241 | } | ||
| 242 | 60 | return {AddResource( | |
| 243 | 30 | {.name = structure.Name(), | |
| 244 | .imported = true, | ||
| 245 | 30 | .slice = {&structure.Storage(), structure.Range().offset, structure.Range().size}, | |
| 246 | .acceleration = &structure, | ||
| 247 | .origin = Origin::Imported}), | ||
| 248 | 30 | 0}; | |
| 249 | 60 | } | |
| 250 | |||
| 251 | 48 | Acceleration Graph::Import(const AccelerationPool& pool) { | |
| 252 | 48 | const auto found = std::ranges::find(resources_, &pool, &Resource::structures); | |
| 253 | 48 | if (found != resources_.end()) { | |
| 254 | 20 | return {static_cast<std::uint32_t>(found - resources_.begin()), found->version}; | |
| 255 | } | ||
| 256 | 56 | return {AddResource({.name = pool.Name(), | |
| 257 | .imported = true, | ||
| 258 | .structures = &pool, | ||
| 259 | .origin = Origin::Imported}), | ||
| 260 | 28 | 0}; | |
| 261 | 28 | } | |
| 262 | |||
| 263 | 534 | Buffer Graph::Import(const ernie::Buffer& buffer, std::uint64_t offset, std::uint64_t size, | |
| 264 | std::string_view name) { | ||
| 265 | 534 | const auto found = std::ranges::find_if(resources_, [&](const Resource& resource) { | |
| 266 | 3535 | return resource.imported && !resource.image && !resource.Structure() && | |
| 267 | 7088 | resource.slice.buffer == &buffer && resource.slice.offset == offset && | |
| 268 | 159 | resource.slice.size == size; | |
| 269 | }); | ||
| 270 | 534 | if (found != resources_.end()) { | |
| 271 | 152 | return {static_cast<std::uint32_t>(found - resources_.begin()), found->version}; | |
| 272 | } | ||
| 273 | 382 | return {AddResource({.name = std::string(name), | |
| 274 | .imported = true, | ||
| 275 | .slice = {&buffer, offset, size}, | ||
| 276 | .origin = Origin::Imported}), | ||
| 277 | 382 | 0}; | |
| 278 | 764 | } | |
| 279 | |||
| 280 | 4 | Image Graph::Create(const ImageDescription& description, std::string_view name) { | |
| 281 | 4 | return {AddResource({.name = std::string(name), .image = true, .description = description}), 0}; | |
| 282 | 8 | } | |
| 283 | |||
| 284 | 92 | Buffer Graph::Create(std::uint64_t bytes, std::string_view name) { | |
| 285 | 92 | return {AddResource({.name = std::string(name), .bytes = bytes}), 0}; | |
| 286 | 184 | } | |
| 287 | |||
| 288 | 244 | std::uint32_t Graph::NewPass(std::string_view name, Kind kind) { | |
| 289 | 244 | if (compiled_) { | |
| 290 | ✗ | throw std::logic_error("Passes cannot be added after Compile"); | |
| 291 | } | ||
| 292 | 244 | passes_.push_back({.name = std::string(name), .kind = kind, .scope = scope_}); | |
| 293 | 244 | return static_cast<std::uint32_t>(passes_.size() - 1); | |
| 294 | 488 | } | |
| 295 | |||
| 296 | 55 | Graph::ScopeGuard::~ScopeGuard() { | |
| 297 | 55 | graph_.scope_ = graph_.scopes_[graph_.scope_].parent; | |
| 298 | 55 | } | |
| 299 | |||
| 300 | 55 | Graph::ScopeGuard Graph::Scope(std::string_view name) { | |
| 301 | 55 | const std::uint32_t depth = scope_ == kNoScope ? 0 : scopes_[scope_].depth + 1; | |
| 302 | 110 | scopes_.push_back({.name = std::string(name), .parent = scope_, .depth = depth}); | |
| 303 | 55 | scope_ = static_cast<std::uint32_t>(scopes_.size() - 1); | |
| 304 | 55 | return ScopeGuard(*this); | |
| 305 | 110 | } | |
| 306 | |||
| 307 | 46 | std::vector<std::uint32_t> Graph::LivePasses() const { | |
| 308 | // Walk back from the passes that must run, keeping passes that write what a kept pass reads. | ||
| 309 | 46 | std::vector<bool> needed(resources_.size(), false); | |
| 310 | 46 | std::vector<bool> keep(passes_.size(), false); | |
| 311 | 286 | for (std::size_t p = passes_.size(); p-- > 0;) { | |
| 312 | 240 | const Pass& pass = passes_[p]; | |
| 313 | 240 | const bool live = | |
| 314 | 240 | pass.neverCull || std::ranges::any_of(pass.accesses, [&](const DependencyAccess& a) { | |
| 315 | 721 | return Writes(a.use) && (resources_[a.resource].imported || needed[a.resource]); | |
| 316 | 240 | }); | |
| 317 | 240 | if (!live) { | |
| 318 | 1 | continue; | |
| 319 | } | ||
| 320 | 239 | keep[p] = true; | |
| 321 | 1106 | for (const DependencyAccess& access : pass.accesses) { | |
| 322 | 867 | if (Reads(access.use)) { | |
| 323 | 481 | needed[access.resource] = true; | |
| 324 | } | ||
| 325 | } | ||
| 326 | } | ||
| 327 | 46 | std::vector<std::uint32_t> live; | |
| 328 | 286 | for (std::size_t p = 0; p < passes_.size(); ++p) { | |
| 329 | 240 | if (keep[p]) { | |
| 330 | 239 | live.push_back(static_cast<std::uint32_t>(p)); | |
| 331 | } | ||
| 332 | } | ||
| 333 | 46 | return live; | |
| 334 | 46 | } | |
| 335 | |||
| 336 | 46 | void Graph::Lease(const std::vector<std::uint32_t>& live) { | |
| 337 | 46 | std::vector<bool> used(resources_.size(), false); | |
| 338 | 46 | std::vector<ImageUsage> usage(resources_.size(), ImageUsage{}); | |
| 339 | 285 | for (const std::uint32_t p : live) { | |
| 340 | 1106 | for (const DependencyAccess& access : passes_[p].accesses) { | |
| 341 | 867 | used[access.resource] = true; | |
| 342 | 867 | usage[access.resource] = usage[access.resource] | NeededUsage(access.use); | |
| 343 | } | ||
| 344 | } | ||
| 345 | 591 | for (std::size_t i = 0; i < resources_.size(); ++i) { | |
| 346 | 545 | Resource& resource = resources_[i]; | |
| 347 | 545 | if (resource.imported || !used[i]) { | |
| 348 | 455 | continue; | |
| 349 | } | ||
| 350 | 90 | if (resource.image) { | |
| 351 | 4 | ImageDescription description = resource.description; | |
| 352 | 4 | description.usage = description.usage | usage[i]; | |
| 353 | 4 | const TransientPool::ImageLease lease = pool_.AcquireImage(description); | |
| 354 | 4 | resource.deviceImage = lease.image; | |
| 355 | 6 | resource.origin = lease.recycled ? Origin::Recycled : Origin::Fresh; | |
| 356 | } else { | ||
| 357 | // Freed ranges come back only once the device is done with them. | ||
| 358 | 86 | resource.range = pool_.AcquireBuffer(resource.bytes); | |
| 359 | 86 | resource.slice = {&pool_.Buffers().Storage(resource.range), resource.range.offset, | |
| 360 | 86 | resource.bytes}; | |
| 361 | 86 | resource.origin = Origin::Fresh; | |
| 362 | } | ||
| 363 | 90 | leased_ = true; | |
| 364 | } | ||
| 365 | 46 | } | |
| 366 | |||
| 367 | 93 | void Graph::ReleaseLeases() { | |
| 368 | 93 | if (!leased_) { | |
| 369 | return; | ||
| 370 | } | ||
| 371 | 559 | for (Resource& resource : resources_) { | |
| 372 | 522 | if (resource.imported) { | |
| 373 | 426 | continue; | |
| 374 | } | ||
| 375 | 96 | if (resource.deviceImage != nullptr) { | |
| 376 | 4 | pool_.Release(*resource.deviceImage); | |
| 377 | 4 | resource.deviceImage = nullptr; | |
| 378 | } | ||
| 379 | 96 | if (resource.slice.buffer != nullptr) { | |
| 380 | 86 | pool_.Release(resource.range); | |
| 381 | 86 | resource.slice = {}; | |
| 382 | } | ||
| 383 | } | ||
| 384 | 37 | leased_ = false; | |
| 385 | } | ||
| 386 | |||
| 387 | 46 | void Graph::Compile() { | |
| 388 | 46 | if (compiled_) { | |
| 389 | ✗ | return; | |
| 390 | } | ||
| 391 | 46 | ERNIE_ZONE("Compile the frame graph"); | |
| 392 | 46 | live_ = LivePasses(); | |
| 393 | 46 | Lease(live_); | |
| 394 | 46 | std::vector<DependencyResource> resources; | |
| 395 | 46 | resources.reserve(resources_.size()); | |
| 396 | 591 | for (const Resource& resource : resources_) { | |
| 397 | 545 | resources.push_back({resource.name, resource.image, resource.origin}); | |
| 398 | } | ||
| 399 | 46 | std::vector<DependencyPass> passes; | |
| 400 | 46 | passes.reserve(live_.size()); | |
| 401 | 285 | for (const std::uint32_t p : live_) { | |
| 402 | 717 | passes.push_back({passes_[p].name, passes_[p].kind, passes_[p].accesses}); | |
| 403 | } | ||
| 404 | 46 | dependencies_ = ComputeDependencies(resources, passes); | |
| 405 | 46 | if (!dependencies_.errors.empty()) { | |
| 406 | 1 | ReleaseLeases(); | |
| 407 | 1 | std::string message = "The frame graph cannot run:"; | |
| 408 | 2 | for (const std::string& error : dependencies_.errors) { | |
| 409 | 3 | message += "\n " + error; | |
| 410 | } | ||
| 411 | 1 | throw GraphError(message); | |
| 412 | 1 | } | |
| 413 | 45 | compiled_ = true; | |
| 414 | 832 | } | |
| 415 | |||
| 416 | 283 | void Graph::Record(VkCommandBuffer commands, const Dependency& dependency) const { | |
| 417 | 283 | if (dependency.Empty()) { | |
| 418 | 8 | return; | |
| 419 | } | ||
| 420 | 275 | std::vector<VkImageMemoryBarrier2> images; | |
| 421 | 275 | images.reserve(dependency.images.size()); | |
| 422 | 311 | for (const Barrier& b : dependency.images) { | |
| 423 | 36 | const Layers layers = b.layers; | |
| 424 | 36 | images.push_back({.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, | |
| 425 | 36 | .srcStageMask = b.srcStages, | |
| 426 | 36 | .srcAccessMask = b.srcAccess, | |
| 427 | 36 | .dstStageMask = b.dstStages, | |
| 428 | 36 | .dstAccessMask = b.dstAccess, | |
| 429 | 36 | .oldLayout = b.oldLayout, | |
| 430 | 36 | .newLayout = b.newLayout, | |
| 431 | .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, | ||
| 432 | .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, | ||
| 433 | 36 | .image = resources_[b.resource].deviceImage->Handle(), | |
| 434 | .subresourceRange = {.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, | ||
| 435 | .levelCount = VK_REMAINING_MIP_LEVELS, | ||
| 436 | 36 | .baseArrayLayer = layers.first, | |
| 437 | .layerCount = layers.count == Layers::kAll | ||
| 438 | ? VK_REMAINING_ARRAY_LAYERS | ||
| 439 | 36 | : layers.count}}); | |
| 440 | } | ||
| 441 | // Buffers are ranges reached by address: their dependencies are one memory barrier. | ||
| 442 | 275 | VkMemoryBarrier2 memory{.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2}; | |
| 443 | 1292 | for (const Barrier& b : dependency.buffers) { | |
| 444 | 1017 | memory.srcStageMask |= b.srcStages; | |
| 445 | 1017 | memory.srcAccessMask |= b.srcAccess; | |
| 446 | 1017 | memory.dstStageMask |= b.dstStages; | |
| 447 | 1017 | memory.dstAccessMask |= b.dstAccess; | |
| 448 | } | ||
| 449 | 275 | const VkDependencyInfo info{.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, | |
| 450 | 275 | .memoryBarrierCount = dependency.buffers.empty() ? 0u : 1u, | |
| 451 | .pMemoryBarriers = &memory, | ||
| 452 | .imageMemoryBarrierCount = | ||
| 453 | 275 | static_cast<std::uint32_t>(images.size()), | |
| 454 | 275 | .pImageMemoryBarriers = images.data()}; | |
| 455 | 275 | vkCmdPipelineBarrier2(commands, &info); | |
| 456 | 275 | } | |
| 457 | |||
| 458 | 391 | void Graph::RecordPass(VkCommandBuffer commands, std::uint32_t order, std::uint32_t depth) const { | |
| 459 | 391 | const Pass& pass = passes_[live_[order]]; | |
| 460 | // Scopes open outermost first, each zone living on the stack of the call nested in it. | ||
| 461 | 391 | std::uint32_t scope = pass.scope; | |
| 462 | 392 | while (scope != kNoScope && scopes_[scope].depth > depth) { | |
| 463 | 1 | scope = scopes_[scope].parent; | |
| 464 | } | ||
| 465 | 391 | if (scope != kNoScope && scopes_[scope].depth == depth) { | |
| 466 | 153 | ERNIE_GPU_ZONE(context_, commands, scopes_[scope].name.c_str()); | |
| 467 | 153 | RecordPass(commands, order, depth + 1); | |
| 468 | 153 | return; | |
| 469 | 153 | } | |
| 470 | 238 | ERNIE_ZONE("Record a pass"); | |
| 471 | 238 | ERNIE_ZONE_TEXT(pass.name); | |
| 472 | 238 | ERNIE_GPU_ZONE(context_, commands, pass.name.c_str()); | |
| 473 | 238 | Record(commands, dependencies_.passes[order]); | |
| 474 | 238 | if (fullBarriers_) { | |
| 475 | 4 | ernie::Barrier(commands, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_WRITE_BIT, | |
| 476 | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, kAnyAccess); | ||
| 477 | } | ||
| 478 | 238 | PassContext context(*this, live_[order]); | |
| 479 | 238 | context.commands = commands; | |
| 480 | 238 | if (pass.run) { | |
| 481 | 238 | pass.run(context); | |
| 482 | } | ||
| 483 | 238 | } | |
| 484 | |||
| 485 | 45 | Ticket Graph::Execute(Queue& queue, std::span<const Ticket> waits, BinarySemaphores binary) { | |
| 486 | 45 | if (executed_) { | |
| 487 | ✗ | throw std::logic_error("A frame graph executes once"); | |
| 488 | } | ||
| 489 | 45 | Compile(); | |
| 490 | 45 | executed_ = true; | |
| 491 | // Transient images new to the pool are made ready on the main queue. | ||
| 492 | 45 | std::vector<Ticket> after(waits.begin(), waits.end()); | |
| 493 | 588 | for (const Resource& resource : resources_) { | |
| 494 | 543 | if (resource.image && resource.deviceImage != nullptr) { | |
| 495 | 16 | after.push_back(resource.deviceImage->Ready()); | |
| 496 | } | ||
| 497 | } | ||
| 498 | 45 | Ticket ticket; | |
| 499 | 45 | if (flushEachPass_) { | |
| 500 | // The first submission waits on the binary semaphore; the last signals its own. | ||
| 501 | 3 | for (std::uint32_t order = 0; order < live_.size(); ++order) { | |
| 502 | 2 | ticket = queue.Submit([&](VkCommandBuffer commands) { RecordPass(commands, order); }, | |
| 503 | 2 | after, {.wait = std::exchange(binary.wait, VK_NULL_HANDLE)}); | |
| 504 | 2 | ticket.Wait(); | |
| 505 | } | ||
| 506 | } | ||
| 507 | 45 | ticket = queue.Submit( | |
| 508 | 45 | [&](VkCommandBuffer commands) { | |
| 509 | 45 | if (!flushEachPass_) { | |
| 510 | 280 | for (std::uint32_t order = 0; order < live_.size(); ++order) { | |
| 511 | 236 | RecordPass(commands, order); | |
| 512 | } | ||
| 513 | } | ||
| 514 | 45 | Record(commands, dependencies_.after); | |
| 515 | 45 | }, | |
| 516 | after, binary); | ||
| 517 | 45 | ReleaseLeases(); | |
| 518 | 126 | for (std::shared_ptr<const void>& object : kept_) { | |
| 519 | 162 | context_.Retire(std::move(object)); | |
| 520 | } | ||
| 521 | 45 | kept_.clear(); | |
| 522 | 45 | return ticket; | |
| 523 | 45 | } | |
| 524 | |||
| 525 | 3 | std::vector<std::string> Graph::ExecutedPasses() const { | |
| 526 | 3 | std::vector<std::string> names; | |
| 527 | 16 | for (const std::uint32_t p : live_) { | |
| 528 | 13 | names.push_back(passes_[p].name); | |
| 529 | } | ||
| 530 | 3 | return names; | |
| 531 | ✗ | } | |
| 532 | |||
| 533 | 2 | std::string Graph::Dot() const { | |
| 534 | 2 | std::vector<bool> live(passes_.size(), !compiled_); | |
| 535 | 4 | for (const std::uint32_t p : live_) { | |
| 536 | 2 | live[p] = true; | |
| 537 | } | ||
| 538 | 2 | std::ostringstream dot; | |
| 539 | 2 | dot << "digraph FrameGraph {\n rankdir=LR;\n"; | |
| 540 | 13 | for (std::size_t r = 0; r < resources_.size(); ++r) { | |
| 541 | 11 | const Resource& resource = resources_[r]; | |
| 542 | 11 | const char* shape = resource.image ? "ellipse" | |
| 543 | ✗ | : resource.Structure() ? "hexagon" | |
| 544 | 11 | : "cylinder"; | |
| 545 | 22 | dot << " r" << r << " [shape=" << shape << ", label=\"" << resource.name << "\"" | |
| 546 | 26 | << (resource.imported ? ", style=bold" : "") << "];\n"; | |
| 547 | } | ||
| 548 | 8 | for (std::size_t p = 0; p < passes_.size(); ++p) { | |
| 549 | 6 | const Pass& pass = passes_[p]; | |
| 550 | 12 | dot << " p" << p << " [shape=box, label=\"" << pass.name << "\"" | |
| 551 | 13 | << (live[p] ? "" : ", style=dashed") << "];\n"; | |
| 552 | 20 | for (const DependencyAccess& access : pass.accesses) { | |
| 553 | 14 | if (Writes(access.use)) { | |
| 554 | 8 | dot << " p" << p << " -> r" << access.resource; | |
| 555 | } else { | ||
| 556 | 6 | dot << " r" << access.resource << " -> p" << p; | |
| 557 | } | ||
| 558 | 28 | dot << " [label=\"" << ToString(access.use) << "\"" | |
| 559 | 24 | << (access.reach == Reach::Descriptor ? "" : ", style=dotted") << "];\n"; | |
| 560 | } | ||
| 561 | } | ||
| 562 | 2 | dot << "}\n"; | |
| 563 | 2 | return dot.str(); | |
| 564 | 2 | } | |
| 565 | |||
| 566 | 91 | void AddUploadPass(Graph& graph, std::string_view name, Uploader& uploader, | |
| 567 | std::span<Buffer> targets) { | ||
| 568 | 91 | if (!uploader.Staged()) { | |
| 569 | return; | ||
| 570 | } | ||
| 571 | 73 | graph.Add( | |
| 572 | name, Kind::Copy, | ||
| 573 | 219 | [&](PassBuilder& pass) { | |
| 574 | 301 | for (Buffer& target : targets) { | |
| 575 | 228 | if (target.Valid()) { | |
| 576 | 180 | target = pass.Write(target, Use::CopyDst); | |
| 577 | } | ||
| 578 | } | ||
| 579 | 73 | }, | |
| 580 | 73 | [&uploader](PassContext& pass) { | |
| 581 | 146 | for (std::shared_ptr<const ernie::Buffer>& staging : uploader.Record(pass.commands)) { | |
| 582 | 146 | pass.Keep(std::move(staging)); | |
| 583 | 73 | } | |
| 584 | 73 | }); | |
| 585 | } | ||
| 586 | |||
| 587 | } // namespace ernie::graph | ||
| 588 |