From dc49526f7d24f57744a0fdaf51a0d82da0dd952c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dar=C3=ADo=20Rodr=C3=ADguez=20Mart=C3=ADnez?= Date: Fri, 9 Oct 2026 18:15:59 +0200 Subject: [PATCH 1/7] game frame limiter --- editor/Engine/ToastEngine.cs | 30 +++++++++++++++++++ .../schemas/project_settings.schema.json | 7 +++++ engine/ffi/engine.h | 3 ++ engine/src/toast/engine.cpp | 4 +++ engine/src/toast/project_settings.cpp | 3 ++ engine/src/toast/project_settings.hpp | 5 ++++ tools/player/program.cs | 17 ++++++----- tools/player/toast_engine.cs | 5 ++++ 8 files changed, 67 insertions(+), 7 deletions(-) diff --git a/editor/Engine/ToastEngine.cs b/editor/Engine/ToastEngine.cs index 3a08f5cde..473398c2c 100644 --- a/editor/Engine/ToastEngine.cs +++ b/editor/Engine/ToastEngine.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Runtime.ExceptionServices; @@ -307,6 +308,8 @@ private static string EngineDllPath() { private void TickLoop(CancellationToken token) { m_tickThreadId = Environment.CurrentManagedThreadId; m_ticking = true; + var stopwatch = Stopwatch.StartNew(); + var nextTick = stopwatch.Elapsed; try { while (!token.IsCancellationRequested && toast_should_close() != 1) { // Wait until the gate is open @@ -322,6 +325,8 @@ private void TickLoop(CancellationToken token) { } finally { m_tickIdle.Set(); } + + PaceTick(stopwatch, ref nextTick, token); } } catch (OperationCanceledException) { // Dispose() cancels the token while the loop is on m_tickGate @@ -331,6 +336,28 @@ private void TickLoop(CancellationToken token) { } } + // Max tick rate comes from the project settings, which only change on the tick thread + private static void PaceTick(Stopwatch stopwatch, ref TimeSpan nextTick, CancellationToken token) { + var maxRate = toast_get_max_tick_rate(); + if (maxRate == 0) { + nextTick = stopwatch.Elapsed; + return; + } + + var interval = TimeSpan.FromSeconds(1.0 / maxRate); + nextTick += interval; + var remaining = nextTick - stopwatch.Elapsed; + if (remaining <= TimeSpan.Zero) { + // fell behind, resync instead of bursting to catch up + nextTick = stopwatch.Elapsed; + return; + } + + if (remaining > TimeSpan.FromMilliseconds(2)) + token.WaitHandle.WaitOne(remaining - TimeSpan.FromMilliseconds(1)); + while (stopwatch.Elapsed < nextTick && !token.IsCancellationRequested) Thread.SpinWait(50); + } + private void CreateMainWindow() { var desktop = (IClassicDesktopStyleApplicationLifetime)Application.Current!.ApplicationLifetime!; desktop.MainWindow = new MainWindowView(this) { @@ -487,6 +514,9 @@ public static void ReloadManifest() { [LibraryImport(EngineLib)] private static partial void toast_reload_project_settings(); + [LibraryImport(EngineLib)] + private static partial uint toast_get_max_tick_rate(); + public static void ReloadProjectSettings() { if (IsEngineReady) OnTickThread(toast_reload_project_settings); } diff --git a/engine/assets/schemas/project_settings.schema.json b/engine/assets/schemas/project_settings.schema.json index 4b9a4119b..79d8af4c4 100644 --- a/engine/assets/schemas/project_settings.schema.json +++ b/engine/assets/schemas/project_settings.schema.json @@ -21,6 +21,13 @@ "description": "Each entry X creates an X:// VFS scheme rooted at {ProjectPath}/X", "default": ["assets"] }, + "max_tick_rate": { + "type": "integer", + "x-toast-type": "int", + "minimum": 0, + "description": "Maximum game ticks per second in the editor and the player; 0 is uncapped", + "default": 500 + }, "gameplay": { "x-toast-type": "gameplay", "description": "Gameplay entry-point settings" diff --git a/engine/ffi/engine.h b/engine/ffi/engine.h index a26bcaafb..edf2d8233 100644 --- a/engine/ffi/engine.h +++ b/engine/ffi/engine.h @@ -112,6 +112,9 @@ TOAST_C_API int toast_viewport_get_frame(void* dst, uint32_t dst_capacity, toast TOAST_C_API void toast_reload_project_settings(void) NOEXCEPT; +/// @return Maximum ticks per second from the project settings, 0 is uncapped +TOAST_C_API uint32_t toast_get_max_tick_rate(void) NOEXCEPT; + /// @brief Tells the renderer whether the host application is focused and whether it is minimized /// @param focused 0 holds rendering at 30 fps (or the configured cap when lower), 1 restores the configured cap /// @param minimized 1 stops rendering entirely until called again with 0 diff --git a/engine/src/toast/engine.cpp b/engine/src/toast/engine.cpp index 431ee009d..d1e6262a5 100644 --- a/engine/src/toast/engine.cpp +++ b/engine/src/toast/engine.cpp @@ -1103,6 +1103,10 @@ void toast_reload_project_settings() noexcept { toast::Engine::get()->reloadSettings(); } +auto toast_get_max_tick_rate() noexcept -> uint32_t { + return toast::ProjectSettings::maxTickRate(); +} + void toast_set_window_state(int focused, int minimized) noexcept { auto* vk_renderer = renderer::VulkanRenderer::instance; if (vk_renderer == nullptr) { diff --git a/engine/src/toast/project_settings.cpp b/engine/src/toast/project_settings.cpp index d35bf5c35..47550cb42 100644 --- a/engine/src/toast/project_settings.cpp +++ b/engine/src/toast/project_settings.cpp @@ -38,6 +38,9 @@ ProjectSettings::ProjectSettings(const std::filesystem::path& path) { m_databases.emplace_back("assets"); } + m_max_tick_rate = + static_cast(std::max(0, table["max_tick_rate"].value_or(k_default_max_tick_rate))); + if (auto* gameplay = table["gameplay"].as_table()) { auto make_handle = [](std::string_view uri) -> assets::Handle { return {nullptr, toast::UID::make(), uri}; diff --git a/engine/src/toast/project_settings.hpp b/engine/src/toast/project_settings.hpp index a82f125bb..afdaad282 100644 --- a/engine/src/toast/project_settings.hpp +++ b/engine/src/toast/project_settings.hpp @@ -77,13 +77,18 @@ class TOAST_API ProjectSettings { static auto uiSettings() -> const UISettings& { return instance->m_ui_settings; } + /// 0 is uncapped + static auto maxTickRate() -> unsigned { return instance != nullptr ? instance->m_max_tick_rate : k_default_max_tick_rate; } + private: + static constexpr unsigned k_default_max_tick_rate = 500; static inline ProjectSettings* instance = nullptr; std::string m_name; std::array m_version {1, 0, 0}; std::vector m_databases; GameplaySettings m_gameplay_settings; UISettings m_ui_settings; + unsigned m_max_tick_rate = k_default_max_tick_rate; }; } diff --git a/tools/player/program.cs b/tools/player/program.cs index 4c4706abf..2a1a98604 100644 --- a/tools/player/program.cs +++ b/tools/player/program.cs @@ -4,9 +4,6 @@ namespace player; internal class Program { - // Caps the tick loop rate to 500 - private const double TargetTickHz = 500.0; - public static void Main() { var engine = new ToastEngine(); var game = new ApplicationLayer(); @@ -33,22 +30,28 @@ public static void Main() { engine.CreateSdlWindow("Toast Engine"); engine.StartGame(); - var targetInterval = TimeSpan.FromSeconds(1.0 / TargetTickHz); var stopwatch = Stopwatch.StartNew(); - var nextTick = stopwatch.Elapsed + targetInterval; + var nextTick = stopwatch.Elapsed; while (!engine.ShouldClose()) { engine.Tick(); + var maxRate = engine.MaxTickRate; + if (maxRate == 0) { + nextTick = stopwatch.Elapsed; + continue; + } + + var targetInterval = TimeSpan.FromSeconds(1.0 / maxRate); + nextTick += targetInterval; var remaining = nextTick - stopwatch.Elapsed; if (remaining > TimeSpan.Zero) { if (remaining > TimeSpan.FromMilliseconds(2)) Thread.Sleep(remaining - TimeSpan.FromMilliseconds(1)); while (stopwatch.Elapsed < nextTick) Thread.SpinWait(50); - nextTick += targetInterval; } else { // fell behind, resync instead of bursting to catch up - nextTick = stopwatch.Elapsed + targetInterval; + nextTick = stopwatch.Elapsed; } } diff --git a/tools/player/toast_engine.cs b/tools/player/toast_engine.cs index cc8216bf2..ddf631f7c 100644 --- a/tools/player/toast_engine.cs +++ b/tools/player/toast_engine.cs @@ -32,6 +32,8 @@ public void Tick() { toast_tick(m_handle); } + public uint MaxTickRate => toast_get_max_tick_rate(); + public bool ShouldClose() { return toast_should_close(m_handle) != 0; } @@ -73,6 +75,9 @@ public void StartGame() { [DllImport("__ENGINE_LIB__", CallingConvention = CallingConvention.Cdecl)] private static extern int toast_should_close(IntPtr engine); + [DllImport("__ENGINE_LIB__", CallingConvention = CallingConvention.Cdecl)] + private static extern uint toast_get_max_tick_rate(); + [DllImport("__ENGINE_LIB__", CallingConvention = CallingConvention.Cdecl)] private static extern void toast_destroy(IntPtr engine); From 3208aee0d16d891c40f81be4b3a58c649e5e20f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dar=C3=ADo=20Rodr=C3=ADguez=20Mart=C3=ADnez?= Date: Sat, 10 Oct 2026 15:02:15 +0200 Subject: [PATCH 2/7] compute queue apparently it works, but needs further testing --- engine/CMakeLists.txt | 4 + engine/assets/shaders/depth_convention.slang | 54 ++ .../shaders/depth_convention.slang.meta | 4 + engine/assets/shaders/environment.slang | 6 +- engine/assets/shaders/ssao.slang | 7 +- engine/assets/shaders/ssr.slang | 9 +- engine/assets/shaders/taa.slang | 5 +- engine/assets/shaders/traced_shadows.slang | 3 +- engine/assets/shaders/voxel.slang | 16 +- engine/src/toast/renderer/async_compute.cpp | 157 ++++ engine/src/toast/renderer/async_compute.hpp | 96 ++ .../src/toast/renderer/async_compute_task.cpp | 385 ++++++++ .../src/toast/renderer/async_compute_task.hpp | 210 +++++ engine/src/toast/renderer/cubemap_target.cpp | 46 +- engine/src/toast/renderer/cubemap_target.hpp | 12 +- .../src/toast/renderer/depth_convention.hpp | 33 + engine/src/toast/renderer/frustum.hpp | 9 +- engine/src/toast/renderer/gpu_sync.cpp | 331 +++++++ engine/src/toast/renderer/gpu_sync.hpp | 182 ++++ engine/src/toast/renderer/gpu_timer.cpp | 13 +- engine/src/toast/renderer/gpu_timer.hpp | 7 + .../src/toast/renderer/passes/bloom_pass.cpp | 136 +-- .../src/toast/renderer/passes/bloom_pass.hpp | 18 +- .../renderer/passes/cluster_lighting_pass.cpp | 48 +- .../src/toast/renderer/passes/debug_pass.cpp | 358 +------- .../src/toast/renderer/passes/debug_pass.hpp | 7 +- .../renderer/passes/environment_pass.cpp | 37 +- .../toast/renderer/passes/material_pass.cpp | 2 +- .../src/toast/renderer/passes/perf_window.cpp | 843 ++++++++++++++++++ .../src/toast/renderer/passes/perf_window.hpp | 148 +++ .../renderer/passes/reflection_probe_pass.cpp | 118 +-- .../src/toast/renderer/passes/shadow_pass.cpp | 59 +- .../src/toast/renderer/passes/shadow_pass.hpp | 5 +- .../src/toast/renderer/passes/voxel_pass.cpp | 6 + .../src/toast/renderer/passes/voxel_pass.hpp | 5 + .../toast/renderer/post_process_target.cpp | 46 +- .../toast/renderer/post_process_target.hpp | 2 +- .../src/toast/renderer/sdl_output_target.cpp | 19 +- engine/src/toast/renderer/shader_compiler.cpp | 13 +- .../renderer/shared_texture_output_target.cpp | 29 +- engine/src/toast/renderer/voxel_debug.cpp | 3 +- .../src/toast/renderer/voxel_gpu_storage.cpp | 60 +- .../src/toast/renderer/voxel_gpu_storage.hpp | 8 + engine/src/toast/renderer/vulkan_common.hpp | 44 +- engine/src/toast/renderer/vulkan_core.cpp | 221 ++++- engine/src/toast/renderer/vulkan_core.hpp | 41 + engine/src/toast/renderer/vulkan_mesh.cpp | 62 +- engine/src/toast/renderer/vulkan_mesh.hpp | 11 + engine/src/toast/renderer/vulkan_pipeline.cpp | 4 +- engine/src/toast/renderer/vulkan_pipeline.hpp | 5 +- engine/src/toast/renderer/vulkan_renderer.cpp | 630 ++++--------- engine/src/toast/renderer/vulkan_renderer.hpp | 34 +- .../toast/renderer/vulkan_resource_base.hpp | 9 + engine/src/toast/renderer/vulkan_texture.cpp | 87 +- engine/src/toast/renderer/vulkan_texture.hpp | 5 + .../src/toast/ui/render/rmlui_renderer_vk.cpp | 61 +- .../src/toast/ui/render/rmlui_renderer_vk.h | 93 +- .../ui/render/rmlui_renderer_vk_effects.cpp | 88 +- engine/src/toast/ui/ui_system.cpp | 8 +- engine/src/toast/world/camera.cpp | 7 +- 60 files changed, 3432 insertions(+), 1537 deletions(-) create mode 100644 engine/assets/shaders/depth_convention.slang create mode 100644 engine/assets/shaders/depth_convention.slang.meta create mode 100644 engine/src/toast/renderer/async_compute.cpp create mode 100644 engine/src/toast/renderer/async_compute.hpp create mode 100644 engine/src/toast/renderer/async_compute_task.cpp create mode 100644 engine/src/toast/renderer/async_compute_task.hpp create mode 100644 engine/src/toast/renderer/depth_convention.hpp create mode 100644 engine/src/toast/renderer/gpu_sync.cpp create mode 100644 engine/src/toast/renderer/gpu_sync.hpp create mode 100644 engine/src/toast/renderer/passes/perf_window.cpp create mode 100644 engine/src/toast/renderer/passes/perf_window.hpp diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index 83edf01e8..c11148989 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -140,8 +140,12 @@ if (TOAST_UNITY_BUILD) set_target_properties(toast_engine PROPERTIES UNITY_BUILD ON UNITY_BUILD_BATCH_SIZE 8) endif () +# Vulkan synchronization validation +option(TOAST_SYNC_VALIDATION "Enable Vulkan synchronization validation in Debug builds" ON) + target_compile_definitions(toast_engine PUBLIC $<$:DEBUG> + $<$,$>:TOAST_SYNC_VALIDATION> $<$,$>:NDEBUG> $<$,$>:TRACY_ENABLE> $<$:TOAST_LUA_OWNERSHIP_CHECKS> diff --git a/engine/assets/shaders/depth_convention.slang b/engine/assets/shaders/depth_convention.slang new file mode 100644 index 000000000..bc2d58bac --- /dev/null +++ b/engine/assets/shaders/depth_convention.slang @@ -0,0 +1,54 @@ +// Mirrors renderer/depth_convention.hpp through TOAST_REVERSED_Z +// Shadow maps keep standard depth and do not use these + +// The depth buffer clears to it +public static const float kFarDepth = TOAST_REVERSED_Z ? 0.0 : 1.0; + +public static const float kNearDepth = TOAST_REVERSED_Z ? 1.0 : 0.0; + +// Sky pixels +public bool isFarDepth(float depth) +{ +#if TOAST_REVERSED_Z + return depth <= 0.0; +#else + return depth >= 1.0; +#endif +} + +public bool isCloserDepth(float a, float b) +{ +#if TOAST_REVERSED_Z + return a > b; +#else + return a < b; +#endif +} + +public float closerDepth(float a, float b) +{ +#if TOAST_REVERSED_Z + return max(a, b); +#else + return min(a, b); +#endif +} + +public float fartherDepth(float a, float b) +{ +#if TOAST_REVERSED_Z + return min(a, b); +#else + return max(a, b); +#endif +} + +// Clip space distance from the near plane and >= 0 on or beyond it +public float clipNearDistance(float4 clip) +{ +#if TOAST_REVERSED_Z + return clip.w - clip.z; +#else + return clip.z; +#endif +} diff --git a/engine/assets/shaders/depth_convention.slang.meta b/engine/assets/shaders/depth_convention.slang.meta new file mode 100644 index 000000000..c5bbbd720 --- /dev/null +++ b/engine/assets/shaders/depth_convention.slang.meta @@ -0,0 +1,4 @@ +uid = "olYeZGySqTs" +type = "slang_module" +created_at = "2026-10-09T17:15:00.0000000Z" +modified_at = "2026-10-09T17:15:00.0000000Z" diff --git a/engine/assets/shaders/environment.slang b/engine/assets/shaders/environment.slang index a8549b656..f849becfe 100644 --- a/engine/assets/shaders/environment.slang +++ b/engine/assets/shaders/environment.slang @@ -1,3 +1,5 @@ +import depth_convention; + struct EnvironmentParams { float4 faceRight; @@ -80,13 +82,13 @@ float4 fragmentEquirect(VSOutput input) : SV_Target0 return float4(gEquirectangular.SampleLevel(uv, 0.0).rgb * gParams.intensity, 1.0); } -// Far plane so it needs a less or equal depth test +// Far plane so it needs a closer or equal depth test [shader("vertex")] VSOutput vertexSkybox(uint vertexId : SV_VertexID) { VSOutput output; output.uv = float2((vertexId << 1) & 2, vertexId & 2); - output.position = float4(output.uv * 2.0 - 1.0, 1.0, 1.0); + output.position = float4(output.uv * 2.0 - 1.0, kFarDepth, 1.0); return output; } diff --git a/engine/assets/shaders/ssao.slang b/engine/assets/shaders/ssao.slang index 13dcf3a9f..a54c837cf 100644 --- a/engine/assets/shaders/ssao.slang +++ b/engine/assets/shaders/ssao.slang @@ -1,4 +1,5 @@ import fullscreen; +import depth_convention; struct SsaoParams { @@ -75,7 +76,7 @@ float4 aoMain(VSOutput input) : SV_Target0 } float depth = gDepth.Sample(input.uv).r; - if (depth >= 1.0) + if (isFarDepth(depth)) { return float4(1.0, kNoSurfaceDistance, 0.0, 1.0); } @@ -119,7 +120,7 @@ float4 aoMain(VSOutput input) : SV_Target0 } float sceneDepth = gDepth.SampleLevel(sampleUv, 0.0).r; - if (sceneDepth >= 1.0) + if (isFarDepth(sceneDepth)) { validSamples += 1.0; continue; @@ -212,7 +213,7 @@ float4 compositeMain(VSOutput input) : SV_Target0 return float4(aoOnly ? float3(1.0, 1.0, 1.0) : sceneColor, 1.0); } - if (depth >= 1.0) + if (isFarDepth(depth)) { return float4(aoOnly ? float3(1.0, 1.0, 1.0) : sceneColor, 1.0); } diff --git a/engine/assets/shaders/ssr.slang b/engine/assets/shaders/ssr.slang index bf295f506..138e7439c 100644 --- a/engine/assets/shaders/ssr.slang +++ b/engine/assets/shaders/ssr.slang @@ -1,4 +1,5 @@ import fullscreen; +import depth_convention; struct SsrParams { @@ -84,7 +85,7 @@ float4 traceMain(VSOutput input) : SV_Target0 } float depth = gDepth.Sample(input.uv).r; - if (depth >= 1.0) + if (isFarDepth(depth)) { return float4(0.0, 0.0, 0.0, kNoSurfaceDistance); } @@ -128,7 +129,7 @@ float4 traceMain(VSOutput input) : SV_Target0 } float sceneDepth = gDepth.SampleLevel(sampleUv, 0.0).r; - if (sceneDepth >= 1.0) + if (isFarDepth(sceneDepth)) { previousPosition = rayPosition; continue; @@ -221,7 +222,7 @@ bool resolveHit(float2 uv, float3 worldPosition, float3 N, float3 R, float pixel } float sceneDepth = gDepth.SampleLevel(candidateUv, 0.0).r; - if (sceneDepth >= 1.0) + if (isFarDepth(sceneDepth)) { continue; } @@ -282,7 +283,7 @@ float4 compositeMain(VSOutput input) : SV_Target0 bool ssrOnly = (renderMode == 14); float3 passthrough = ssrOnly ? float3(0.0, 0.0, 0.0) : sceneColor; - if (dot(normalRoughness.xyz, normalRoughness.xyz) < 1e-4 || depth >= 1.0) + if (dot(normalRoughness.xyz, normalRoughness.xyz) < 1e-4 || isFarDepth(depth)) { return float4(passthrough, 1.0); } diff --git a/engine/assets/shaders/taa.slang b/engine/assets/shaders/taa.slang index a0b05aeb3..ae220b6ef 100644 --- a/engine/assets/shaders/taa.slang +++ b/engine/assets/shaders/taa.slang @@ -1,4 +1,5 @@ import fullscreen; +import depth_convention; struct TaaParams { @@ -116,7 +117,7 @@ float4 fragmentMain(VSOutput input) : SV_Target0 float3 center = float3(0.0, 0.0, 0.0); float3 moment1 = float3(0.0, 0.0, 0.0); float3 moment2 = float3(0.0, 0.0, 0.0); - float closestDepth = 1.0; + float closestDepth = kFarDepth; float2 closestUv = uv; for (int y = -1; y <= 1; ++y) @@ -134,7 +135,7 @@ float4 fragmentMain(VSOutput input) : SV_Target0 // Keeps a moving edge from trailing the background float depth = gDepth.SampleLevel(sampleUv, 0.0).r; - if (depth < closestDepth) + if (isCloserDepth(depth, closestDepth)) { closestDepth = depth; closestUv = sampleUv; diff --git a/engine/assets/shaders/traced_shadows.slang b/engine/assets/shaders/traced_shadows.slang index a37942c62..706c78ecf 100644 --- a/engine/assets/shaders/traced_shadows.slang +++ b/engine/assets/shaders/traced_shadows.slang @@ -1,4 +1,5 @@ import fullscreen; +import depth_convention; struct TracedShadowParams { @@ -48,7 +49,7 @@ float4 fragmentMain(VSOutput input) : SV_Target0 float4 normalRoughness = gNormalRoughness.Sample(input.uv); float depth = gDepth.Sample(input.uv).r; - if (dot(normalRoughness.xyz, normalRoughness.xyz) < 1e-4 || depth >= 1.0) + if (dot(normalRoughness.xyz, normalRoughness.xyz) < 1e-4 || isFarDepth(depth)) { return float4(applied ? sceneColor : float3(1.0, 1.0, 1.0), 1.0); } diff --git a/engine/assets/shaders/voxel.slang b/engine/assets/shaders/voxel.slang index f76311d9a..c28052f6a 100644 --- a/engine/assets/shaders/voxel.slang +++ b/engine/assets/shaders/voxel.slang @@ -1,6 +1,7 @@ import voxel_dda; import voxel_gbuffer; import lighting; +import depth_convention; struct PushConstants { @@ -261,10 +262,10 @@ bool shade(VSOutput input, out FSOutput output, out float depth) const float3 origin = mul(instance.worldToVoxel, float4(camera, 1.0)).xyz; const float3 direction = mul(instance.worldToVoxel, float4(worldDirection, 0.0)).xyz; - // Vulkan clips at z = 0 so never hit in front of the near plane - const float clipOriginZ = mul(gCamera.viewProjection, float4(camera, 1.0)).z; - const float clipStepZ = mul(gCamera.viewProjection, float4(worldDirection, 0.0)).z; - const float tMin = clipStepZ > 0.0 ? max(0.0, -clipOriginZ / clipStepZ) : 0.0; + // Never hit in front of the near plane whose clip distance is linear along the ray in either convention + const float nearOrigin = clipNearDistance(mul(gCamera.viewProjection, float4(camera, 1.0))); + const float nearStep = clipNearDistance(mul(gCamera.viewProjection, float4(worldDirection, 0.0))); + const float tMin = nearStep > 0.0 ? max(0.0, -nearOrigin / nearStep) : 0.0; const RayHit hit = marchRay(instance.recordIndex, origin, direction, tMin, kFloatMax); @@ -322,8 +323,13 @@ bool shade(VSOutput input, out FSOutput output, out float depth) return true; } +// The DDA hit is never closer than the rasterized face which keeps early depth testing on [shader("fragment")] +#if TOAST_REVERSED_Z +FSOutput fragmentOutside(VSOutput input, out float depth : SV_DepthLessEqual) +#else FSOutput fragmentOutside(VSOutput input, out float depth : SV_DepthGreaterEqual) +#endif { FSOutput output; if (!shade(input, output, depth)) @@ -332,7 +338,7 @@ FSOutput fragmentOutside(VSOutput input, out float depth : SV_DepthGreaterEqual) } // Slab entry can land a hair in front of the face - depth = max(depth, input.position.z); + depth = fartherDepth(depth, input.position.z); return output; } diff --git a/engine/src/toast/renderer/async_compute.cpp b/engine/src/toast/renderer/async_compute.cpp new file mode 100644 index 000000000..c6feebc51 --- /dev/null +++ b/engine/src/toast/renderer/async_compute.cpp @@ -0,0 +1,157 @@ +/// @file async_compute.cpp +/// @author dario +/// @date 09/10/2026 + +#include "async_compute.hpp" + +#include +#include +#include +#include +#include +#include + +namespace renderer { + +AsyncCompute::AsyncCompute(const vk::raii::Device& device, uint32_t queue_family_index, vk::Queue queue, std::mutex* queue_mutex) + : m_device(&device), + m_queue_family_index(queue_family_index), + m_queue(queue), + m_queue_mutex(queue_mutex) { + vk::SemaphoreTypeCreateInfo type_ci {}; + type_ci.semaphoreType = vk::SemaphoreType::eTimeline; + type_ci.initialValue = k_no_ticket; + + vk::SemaphoreCreateInfo semaphore_ci {}; + semaphore_ci.pNext = &type_ci; + m_timeline = vk::raii::Semaphore(device, semaphore_ci); + + TOAST_INFO("Render", "Async compute ready on queue family {}", queue_family_index); +} + +AsyncCompute::~AsyncCompute() { + // No queue lock since the core may already have destroyed it + const Ticket last = lastSubmittedTicket(); + if (last != k_no_ticket && !wait(last)) { + TOAST_WARN("Render", "Async compute shut down with ticket {} unfinished", last); + } +} + +void AsyncCompute::reclaimLocked() { + const Ticket completed = m_timeline.getCounterValue(); + for (auto it = m_in_flight.begin(); it != m_in_flight.end();) { + if (it->ticket <= completed) { + m_free.push_back(std::move(*it)); + it = m_in_flight.erase(it); + } else { + ++it; + } + } +} + +auto AsyncCompute::acquireSlot() -> Slot { + { + std::scoped_lock lock(m_mutex); + reclaimLocked(); + if (!m_free.empty()) { + Slot slot = std::move(m_free.back()); + m_free.pop_back(); + slot.pool.reset(); + return slot; + } + } + + // One pool per slot so threads record without sharing a pool + Slot slot; + slot.pool = vk::raii::CommandPool( + *m_device, vk::CommandPoolCreateInfo(vk::CommandPoolCreateFlagBits::eTransient, m_queue_family_index) + ); + auto buffers = m_device->allocateCommandBuffers(vk::CommandBufferAllocateInfo(*slot.pool, vk::CommandBufferLevel::ePrimary, 1)); + slot.cmd = std::move(buffers[0]); + return slot; +} + +auto AsyncCompute::submit(const std::function& record, std::string_view debug_name) -> Ticket { + ZoneScoped; + Slot slot = acquireSlot(); + const vk::CommandBuffer cmd = *slot.cmd; + + try { + cmd.begin(vk::CommandBufferBeginInfo(vk::CommandBufferUsageFlagBits::eOneTimeSubmit)); + record(cmd); + cmd.end(); + } catch (...) { + std::scoped_lock lock(m_mutex); + m_free.push_back(std::move(slot)); + throw; + } + + vk::CommandBufferSubmitInfo command_info {}; + command_info.commandBuffer = cmd; + + vk::SemaphoreSubmitInfo signal {}; + signal.semaphore = *m_timeline; + signal.stageMask = vk::PipelineStageFlagBits2::eAllCommands; + + vk::SubmitInfo2 info {}; + info.setCommandBufferInfos(command_info); + info.setSignalSemaphoreInfos(signal); + + std::scoped_lock lock(m_mutex); + // Same lock as the submit so values reach the queue in increasing order + const Ticket ticket = m_last_submitted + 1; + signal.value = ticket; + { + std::unique_lock queue_lock; + if (m_queue_mutex != nullptr) { + queue_lock = std::unique_lock(*m_queue_mutex); + } + m_queue.submit2(info, nullptr); + } + m_last_submitted = ticket; + + if (!debug_name.empty()) { + TOAST_TRACE("Render", "Async compute job '{}' submitted as ticket {}", debug_name, ticket); + } + + slot.ticket = ticket; + m_in_flight.push_back(std::move(slot)); + return ticket; +} + +auto AsyncCompute::isComplete(Ticket ticket) const -> bool { + return ticket == k_no_ticket || m_timeline.getCounterValue() >= ticket; +} + +auto AsyncCompute::wait(Ticket ticket, uint64_t timeout_ns) const -> bool { + if (isComplete(ticket)) { + return true; + } + const vk::Semaphore semaphore = *m_timeline; + const vk::SemaphoreWaitInfo wait_info({}, semaphore, ticket); + return m_device->waitSemaphores(wait_info, timeout_ns) == vk::Result::eSuccess; +} + +auto AsyncCompute::completedTicket() const -> Ticket { + return m_timeline.getCounterValue(); +} + +auto AsyncCompute::lastSubmittedTicket() const -> Ticket { + std::scoped_lock lock(m_mutex); + return m_last_submitted; +} + +auto AsyncCompute::sharedFamilies(uint32_t graphics_family) const noexcept -> SharedFamilies { + if (graphics_family == m_queue_family_index) { + return { + .indices = {graphics_family, graphics_family}, + .count = 1 + }; + } + return { + .indices = {graphics_family, m_queue_family_index}, + .count = 2 + }; +} + +} diff --git a/engine/src/toast/renderer/async_compute.hpp b/engine/src/toast/renderer/async_compute.hpp new file mode 100644 index 000000000..5a39a5ad0 --- /dev/null +++ b/engine/src/toast/renderer/async_compute.hpp @@ -0,0 +1,96 @@ +/// @file async_compute.hpp +/// @author dario +/// @date 09/10/2026 + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace renderer { + +class TOAST_API AsyncCompute { +public: + /// Timeline value so completing one completes every earlier one + using Ticket = uint64_t; + + static constexpr Ticket k_no_ticket = 0; + + /// @param queue_mutex non null when another subsystem submits to the same VkQueue + AsyncCompute(const vk::raii::Device& device, uint32_t queue_family_index, vk::Queue queue, std::mutex* queue_mutex); + + ~AsyncCompute(); + + AsyncCompute(const AsyncCompute&) = delete; + auto operator=(const AsyncCompute&) -> AsyncCompute& = delete; + AsyncCompute(AsyncCompute&&) = delete; + auto operator=(AsyncCompute&&) -> AsyncCompute& = delete; + + /// Thread safe and records outside the lock so threads record in parallel + /// An exception from @p record propagates and no ticket is handed out + auto submit(const std::function& record, std::string_view debug_name = {}) -> Ticket; + + [[nodiscard]] + auto isComplete(Ticket ticket) const -> bool; + + /// @return false on timeout + auto wait(Ticket ticket, uint64_t timeout_ns = std::numeric_limits::max()) const -> bool; + + [[nodiscard]] + auto completedTicket() const -> Ticket; + + [[nodiscard]] + auto lastSubmittedTicket() const -> Ticket; + + [[nodiscard]] + auto queueFamilyIndex() const noexcept -> uint32_t { + return m_queue_family_index; + } + + /// Never wait on it from a frame submit or the frame couples to async work again + [[nodiscard]] + auto timeline() const noexcept -> vk::Semaphore { + return *m_timeline; + } + + /// For eConcurrent create infos with count entries since one family repeats + struct SharedFamilies { + std::array indices {}; + uint32_t count = 0; + }; + + [[nodiscard]] + auto sharedFamilies(uint32_t graphics_family) const noexcept -> SharedFamilies; + +private: + struct Slot { + vk::raii::CommandPool pool = nullptr; + vk::raii::CommandBuffer cmd = nullptr; + Ticket ticket = k_no_ticket; + }; + + /// Caller holds m_mutex + void reclaimLocked(); + auto acquireSlot() -> Slot; + + const vk::raii::Device* m_device = nullptr; + uint32_t m_queue_family_index = 0; + vk::Queue m_queue = nullptr; + std::mutex* m_queue_mutex = nullptr; + + vk::raii::Semaphore m_timeline = nullptr; + + mutable std::mutex m_mutex; + Ticket m_last_submitted = k_no_ticket; + std::vector m_in_flight; + std::vector m_free; +}; + +} diff --git a/engine/src/toast/renderer/async_compute_task.cpp b/engine/src/toast/renderer/async_compute_task.cpp new file mode 100644 index 000000000..23486f88d --- /dev/null +++ b/engine/src/toast/renderer/async_compute_task.cpp @@ -0,0 +1,385 @@ +/// @file async_compute_task.cpp +/// @author dario +/// @date 10/10/2026 + +#include "async_compute_task.hpp" + +#include "shader_cache.hpp" +#include "vulkan_core.hpp" +#include "vulkan_debug.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace renderer { + +namespace { + +constexpr vk::BufferUsageFlags k_task_buffer_usage = + vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferSrc | vk::BufferUsageFlagBits::eTransferDst; + +} + +void AsyncComputeTask::StepContext::dispatchRaw( + uint32_t entry, std::array groups, const void* push, uint32_t push_size +) { + if (entry >= m_task.m_pipelines.size() || !m_task.m_pipelines[entry]->isReady()) { + return; + } + + // Dispatches in one command buffer do not order their storage writes + if (m_dispatched) { + sync::BarrierBatch barriers; + barriers.memory(sync::Usage::compute_storage_write, sync::Usage::compute_storage_write); + barriers.flush(m_cmd); + } + m_dispatched = true; + + const vk::PipelineLayout layout = *m_task.m_layout.getPipelineLayout(); + m_cmd.bindPipeline(vk::PipelineBindPoint::eCompute, m_task.m_pipelines[entry]->getPipeline()); + m_cmd.bindDescriptorSets(vk::PipelineBindPoint::eCompute, layout, 0, {*m_task.m_slots[m_slot].set}, {}); + if (push != nullptr && push_size > 0) { + m_cmd.pushConstants(layout, vk::ShaderStageFlagBits::eAll, 0, push_size, push); + } + m_cmd.dispatch(groups[0], groups[1], groups[2]); +} + +auto AsyncComputeTask::FrameView::buffer(BufferId id) const -> vk::Buffer { + if (m_task == nullptr || id >= m_task->m_buffers.size() || !m_task->m_buffers[id].ring) { + return nullptr; + } + return *m_task->m_buffers[id].copies[m_slot]; +} + +AsyncComputeTask::AsyncComputeTask(Config config) : m_config(std::move(config)), m_core(&getCore()) { + ZoneScoped; + m_frame_slot.fill(-1); + + const auto uid = assets::resolveURI(m_config.shader_uri); + const auto shader = uid.has_value() ? ShaderCache::get().acquire(*uid) : nullptr; + if (!shader) { + TOAST_ERROR("Render", "{} shader {} unavailable", m_config.debug_name, m_config.shader_uri); + return; + } + + m_layout.rebuild(*m_core, shader->reflection, m_config.debug_name); + for (const std::string& entry : m_config.entries) { + VulkanPipeline::Config pipeline_config; + pipeline_config.pipeline_type = VulkanPipeline::PipelineType::compute; + pipeline_config.debug_name = std::format("{} {}", m_config.debug_name, entry); + pipeline_config.shader_spirv = shader->spirv; + pipeline_config.pipeline_layout = *m_layout.getPipelineLayout(); + pipeline_config.compute_entry = entry; + auto& pipeline = m_pipelines.emplace_back(std::make_unique()); + pipeline->rebuild(*m_core, pipeline_config); + } + m_ready = !m_pipelines.empty() && std::ranges::all_of(m_pipelines, [](const auto& p) { return p->isReady(); }); +} + +AsyncComputeTask::~AsyncComputeTask() { + // Buffers and sets must outlive every job that uses them + const AsyncCompute& compute = m_core->asyncCompute(); + auto unused = compute.wait(compute.lastSubmittedTicket()); +} + +/// This is superturbobad do not overuse it +auto AsyncComputeTask::addRingBuffer(vk::DeviceSize size, bool cpu_readback) -> BufferId { + const auto shared = m_core->asyncCompute().sharedFamilies(m_core->getGraphicsQueueFamilyIndex()); + vk::BufferCreateInfo buffer_ci({}, size, k_task_buffer_usage); + // Concurrent so the renderer reads it with a visibility barrier and no ownership transfer + if (shared.count > 1) { + buffer_ci.sharingMode = vk::SharingMode::eConcurrent; + buffer_ci.queueFamilyIndexCount = shared.count; + buffer_ci.pQueueFamilyIndices = shared.indices.data(); + } + vma::AllocationCreateInfo device_alloc {}; + device_alloc.usage = vma::MemoryUsage::eAutoPreferDevice; + + vk::BufferCreateInfo readback_ci({}, size, vk::BufferUsageFlagBits::eTransferDst); + vma::AllocationCreateInfo readback_alloc {}; + readback_alloc.usage = vma::MemoryUsage::eAutoPreferHost; + readback_alloc.flags = vma::AllocationCreateFlagBits::eMapped | vma::AllocationCreateFlagBits::eHostAccessRandom; + + const auto id = static_cast(m_buffers.size()); + Buffer& buffer = m_buffers.emplace_back(); + buffer.ring = true; + buffer.cpu_readback = cpu_readback; + buffer.size = size; + for (uint32_t slot = 0; slot < k_slot_count; ++slot) { + buffer.copies.push_back(m_core->getAllocator().createBuffer(buffer_ci, device_alloc)); + setDebugName(*m_core, *buffer.copies.back(), std::format("{} Ring{}[{}]", m_config.debug_name, id, slot)); + if (cpu_readback) { + buffer.readbacks.push_back(m_core->getAllocator().createBuffer(readback_ci, readback_alloc)); + } + } + return id; +} + +auto AsyncComputeTask::addPersistentBuffer(vk::DeviceSize size) -> BufferId { + // Exclusive to the compute family since only this task touches it + const vk::BufferCreateInfo buffer_ci({}, size, k_task_buffer_usage); + vma::AllocationCreateInfo device_alloc {}; + device_alloc.usage = vma::MemoryUsage::eAutoPreferDevice; + + const auto id = static_cast(m_buffers.size()); + Buffer& buffer = m_buffers.emplace_back(); + buffer.size = size; + buffer.copies.push_back(m_core->getAllocator().createBuffer(buffer_ci, device_alloc)); + setDebugName(*m_core, *buffer.copies.back(), std::format("{} Persistent{}", m_config.debug_name, id)); + return id; +} + +void AsyncComputeTask::bind(uint32_t binding, BufferId id, Copy copy) { + m_bindings.push_back({.binding = binding, .id = id, .copy = copy}); +} + +void AsyncComputeTask::createDescriptors() { + if (*m_pool || !m_ready) { + return; + } + const auto& layouts = m_layout.getDescriptorSetLayouts(); + if (layouts.empty()) { + return; + } + + const auto& device = m_core->getDevice(); + // Own pool since the renderer pool belongs to the render thread + const std::array pool_sizes {vk::DescriptorPoolSize( + vk::DescriptorType::eStorageBuffer, std::max(1, static_cast(m_bindings.size())) * k_slot_count + )}; + m_pool = vk::raii::DescriptorPool( + device, vk::DescriptorPoolCreateInfo(vk::DescriptorPoolCreateFlagBits::eFreeDescriptorSet, k_slot_count, pool_sizes) + ); + + const vk::DescriptorSetLayout set_layout = *layouts[0]; + for (Slot& slot : m_slots) { + auto sets = device.allocateDescriptorSets(vk::DescriptorSetAllocateInfo(*m_pool, 1, &set_layout)); + slot.set = std::move(sets[0]); + } +} + +void AsyncComputeTask::writeDescriptors(uint32_t slot) { + // Before the first step every ring copy holds the upload so any other slot works as previous + const uint32_t previous = m_last_written >= 0 ? static_cast(m_last_written) : (slot + 1) % k_slot_count; + + std::vector infos; + infos.reserve(m_bindings.size()); + std::vector writes; + writes.reserve(m_bindings.size()); + for (const Binding& binding : m_bindings) { + const Buffer& buffer = m_buffers[binding.id]; + const uint32_t copy = !buffer.ring ? 0 : (binding.copy == Copy::current ? slot : previous); + infos.emplace_back(*buffer.copies[copy], 0, buffer.size); + writes.emplace_back(*m_slots[slot].set, binding.binding, 0, 1, vk::DescriptorType::eStorageBuffer, nullptr, &infos.back()); + } + // The set of a claimed slot is not bound by any pending job so rewriting it is safe + m_core->getDevice().updateDescriptorSets(writes, {}); +} + +void AsyncComputeTask::upload(BufferId id, std::span data) { + ZoneScoped; + if (id >= m_buffers.size()) { + return; + } + Buffer& buffer = m_buffers[id]; + if (buffer.ring && m_step_counter > 0) { + TOAST_ERROR("Render", "{} ring buffer upload after the first step would race frames reading it", m_config.debug_name); + return; + } + const vk::DeviceSize bytes = std::min(data.size(), buffer.size); + + vk::BufferCreateInfo staging_ci({}, bytes, vk::BufferUsageFlagBits::eTransferSrc); + vma::AllocationCreateInfo staging_alloc {}; + staging_alloc.usage = vma::MemoryUsage::eAutoPreferHost; + staging_alloc.flags = vma::AllocationCreateFlagBits::eMapped | vma::AllocationCreateFlagBits::eHostAccessSequentialWrite; + vma::raii::Buffer staging = m_core->getAllocator().createBuffer(staging_ci, staging_alloc); + std::memcpy(staging.getAllocation().getInfo().pMappedData, data.data(), bytes); + staging.getAllocation().flush(0, bytes); + + AsyncCompute& compute = m_core->asyncCompute(); + const AsyncCompute::Ticket ticket = compute.submit( + [&](vk::CommandBuffer cmd) { + // Earlier steps may still read or write these copies + sync::BarrierBatch barriers; + barriers.memory(sync::Usage::compute_storage_write, sync::Usage::transfer_dst); + barriers.flush(cmd); + for (const vma::raii::Buffer& copy : buffer.copies) { + cmd.copyBuffer(*staging, *copy, vk::BufferCopy(0, 0, bytes)); + } + }, + std::format("{} upload", m_config.debug_name) + ); + // Simplest correct lifetime for the staging buffer and uploads are rare + auto unused = compute.wait(ticket); +} + +void AsyncComputeTask::step(const std::function& record, std::string_view debug_name) { + ZoneScoped; + if (!m_ready) { + return; + } + createDescriptors(); + + const uint32_t slot = claimFreeSlot(); + writeDescriptors(slot); + + const AsyncCompute::Ticket ticket = m_core->asyncCompute().submit( + [&](vk::CommandBuffer cmd) { + // Submits on one queue may overlap so the previous step and any upload must land first + sync::BarrierBatch barriers; + barriers.memory(sync::Usage::compute_storage_write, sync::Usage::compute_storage_write); + barriers.memory(sync::Usage::transfer_dst, sync::Usage::compute_storage_write); + barriers.flush(cmd); + + StepContext context(*this, cmd, slot); + record(context); + + // CPU copies of this step for read() + bool any_readback = false; + for (const Buffer& buffer : m_buffers) { + if (buffer.cpu_readback) { + barriers.buffer(*buffer.copies[slot], sync::Usage::compute_storage_write, sync::Usage::transfer_src); + any_readback = true; + } + } + if (!any_readback) { + return; + } + barriers.flush(cmd); + for (const Buffer& buffer : m_buffers) { + if (buffer.cpu_readback) { + cmd.copyBuffer(*buffer.copies[slot], *buffer.readbacks[slot], vk::BufferCopy(0, 0, buffer.size)); + barriers.buffer(*buffer.readbacks[slot], sync::Usage::transfer_dst, sync::Usage::host_read); + } + } + barriers.flush(cmd); + }, + debug_name.empty() ? std::string_view(m_config.debug_name) : debug_name + ); + + publish(slot, ticket); +} + +auto AsyncComputeTask::read(BufferId id, std::span out) -> bool { + if (id >= m_buffers.size() || !m_buffers[id].cpu_readback) { + return false; + } + int32_t newest = -1; + { + std::scoped_lock lock(m_mutex); + newest = newestFinishedLocked(); + } + // Only the owner thread rewrites slots so it stays valid after the lock + if (newest < 0) { + return false; + } + const vma::raii::Buffer& readback = m_buffers[id].readbacks[static_cast(newest)]; + const vk::DeviceSize bytes = std::min(out.size(), m_buffers[id].size); + // Non coherent host memory needs an invalidate before the CPU sees device writes + readback.getAllocation().invalidate(0, bytes); + std::memcpy(out.data(), readback.getAllocation().getInfo().pMappedData, bytes); + return true; +} + +auto AsyncComputeTask::acquireForFrame(vk::CommandBuffer cmd, uint32_t frame_index, sync::Usage next) -> FrameView { + if (frame_index >= m_frame_slot.size()) { + return {}; + } + + int32_t newest = -1; + uint64_t step = 0; + { + std::scoped_lock lock(m_mutex); + if (const int32_t old = m_frame_slot[frame_index]; old >= 0) { + --m_slots[static_cast(old)].pins; + m_frame_slot[frame_index] = -1; + } + newest = newestFinishedLocked(); + if (newest < 0) { + return {}; + } + ++m_slots[static_cast(newest)].pins; + m_frame_slot[frame_index] = newest; + step = m_slots[static_cast(newest)].step; + } + const auto slot = static_cast(newest); + + if (m_acquired_step[slot] != step) { + const sync::QueueHandoff handoff { + .from = m_core->asyncCompute().queueFamilyIndex(), .to = m_core->getGraphicsQueueFamilyIndex(), .exclusive = false + }; + sync::BarrierBatch barriers; + for (const Buffer& buffer : m_buffers) { + if (buffer.ring) { + barriers.acquire(*buffer.copies[slot], next, handoff); + } + } + barriers.flush(cmd); + m_acquired_step[slot] = step; + } + + FrameView view; + view.m_task = this; + view.m_slot = slot; + view.m_step = step; + return view; +} + +auto AsyncComputeTask::finishedStep() const -> uint64_t { + std::scoped_lock lock(m_mutex); + const int32_t newest = newestFinishedLocked(); + return newest < 0 ? 0 : m_slots[static_cast(newest)].step; +} + +auto AsyncComputeTask::claimFreeSlot() -> uint32_t { + const AsyncCompute& compute = m_core->asyncCompute(); + while (true) { + { + std::scoped_lock lock(m_mutex); + // Never the next step input, the result the renderer shows next or a slot a frame still binds + const int32_t newest = newestFinishedLocked(); + for (uint32_t i = 0; i < k_slot_count; ++i) { + Slot& slot = m_slots[i]; + const auto index = static_cast(i); + if (index != m_last_written && index != newest && slot.pins == 0 && compute.isComplete(slot.ticket)) { + slot.step = 0; + return i; + } + } + } + // Every slot busy means the GPU is behind or frames still bind them, thats bad + if (compute.completedTicket() < compute.lastSubmittedTicket()) { + auto unused = compute.wait(compute.completedTicket() + 1); + } else { + std::this_thread::yield(); + } + } +} + +void AsyncComputeTask::publish(uint32_t slot, AsyncCompute::Ticket ticket) { + std::scoped_lock lock(m_mutex); + m_slots[slot].ticket = ticket; + m_slots[slot].step = ++m_step_counter; + m_last_written = static_cast(slot); +} + +auto AsyncComputeTask::newestFinishedLocked() const -> int32_t { + const AsyncCompute& compute = m_core->asyncCompute(); + int32_t best = -1; + uint64_t best_step = 0; + for (uint32_t i = 0; i < k_slot_count; ++i) { + const Slot& slot = m_slots[i]; + if (slot.step > best_step && compute.isComplete(slot.ticket)) { + best = static_cast(i); + best_step = slot.step; + } + } + return best; +} + +} diff --git a/engine/src/toast/renderer/async_compute_task.hpp b/engine/src/toast/renderer/async_compute_task.hpp new file mode 100644 index 000000000..609868a8b --- /dev/null +++ b/engine/src/toast/renderer/async_compute_task.hpp @@ -0,0 +1,210 @@ +/// @file async_compute_task.hpp +/// @author dario +/// @date 10/10/2026 + +#pragma once + +#include "async_compute.hpp" +#include "gpu_sync.hpp" +#include "shader_layout.hpp" +#include "vulkan_pipeline.hpp" +#include "vulkan_renderer.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace renderer { + +/// Base for a compute simulation that runs on the async compute queue and feeds the renderer +/// +/// A derived class declares its buffers and bindings in its constructor and then calls step() from its own thread +/// +/// Ring buffers are written once per step and read by the renderer and the CPU +/// Persistent buffers are state only the task touches, like velocities +class TOAST_API AsyncComputeTask { +public: + using BufferId = uint32_t; + + /// Which copy of a ring buffer a binding sees during a step + enum class Copy : uint8_t { + /// Written by this step + current, + /// Result of the step before, read only + previous, + }; + + struct Config { + std::string shader_uri; + std::vector entries {"computeMain"}; + std::string debug_name = "AsyncComputeTask"; + }; + + /// Handed to the step callback. Barriers between dispatches of one step are inserted for you + class TOAST_API StepContext { + public: + template + void dispatch(uint32_t groups_x, const Push& push, uint32_t entry = 0, uint32_t groups_y = 1, uint32_t groups_z = 1) { + dispatchRaw(entry, {groups_x, groups_y, groups_z}, &push, sizeof(Push)); + } + + void dispatch(uint32_t groups_x, uint32_t entry = 0, uint32_t groups_y = 1, uint32_t groups_z = 1) { + dispatchRaw(entry, {groups_x, groups_y, groups_z}, nullptr, 0); + } + + /// For anything the helpers do not cover + [[nodiscard]] + auto commandBuffer() const -> vk::CommandBuffer { + return m_cmd; + } + + private: + friend class AsyncComputeTask; + + StepContext(AsyncComputeTask& task, vk::CommandBuffer cmd, uint32_t slot) : m_task(task), m_cmd(cmd), m_slot(slot) { } + + void dispatchRaw(uint32_t entry, std::array groups, const void* push, uint32_t push_size); + + AsyncComputeTask& m_task; + vk::CommandBuffer m_cmd; + uint32_t m_slot; + bool m_dispatched = false; + }; + + /// Buffers the renderer may read this frame + class FrameView { + public: + [[nodiscard]] + explicit operator bool() const { + return m_task != nullptr; + } + + /// Null for a persistent buffer since only the task touches those + [[nodiscard]] + auto buffer(BufferId id) const -> vk::Buffer; + + /// Step counter of the data + [[nodiscard]] + auto step() const -> uint64_t { + return m_step; + } + + private: + friend class AsyncComputeTask; + const AsyncComputeTask* m_task = nullptr; + uint32_t m_slot = 0; + uint64_t m_step = 0; + }; + + virtual ~AsyncComputeTask(); + + AsyncComputeTask(const AsyncComputeTask&) = delete; + auto operator=(const AsyncComputeTask&) -> AsyncComputeTask& = delete; + AsyncComputeTask(AsyncComputeTask&&) = delete; + auto operator=(AsyncComputeTask&&) -> AsyncComputeTask& = delete; + + /// Render thread after the fence of frame_index was waited on, empty until a step finished + /// next is how the renderer reads the ring buffers + auto acquireForFrame(vk::CommandBuffer cmd, uint32_t frame_index, sync::Usage next) -> FrameView; + + /// Newest finished CPU copy of a ring buffer added with cpu_readback and false until one exists + auto read(BufferId id, std::span out) -> bool; + + template + auto read(BufferId id, std::span out) -> bool { + return read(id, std::as_writable_bytes(out)); + } + + /// Steps finished so far + [[nodiscard]] + auto finishedStep() const -> uint64_t; + + [[nodiscard]] + auto isReady() const -> bool { + return m_ready; + } + +protected: + explicit AsyncComputeTask(Config config); + + /// Constructor only + auto addRingBuffer(vk::DeviceSize size, bool cpu_readback = false) -> BufferId; + auto addPersistentBuffer(vk::DeviceSize size) -> BufferId; + void bind(uint32_t binding, BufferId id, Copy copy = Copy::current); + + /// Owner thread, blocks until the copy landed gpu side + void upload(BufferId id, std::span data); + + template + void upload(BufferId id, std::span data) { + upload(id, std::as_bytes(data)); + } + + /// Owner thread + void step(const std::function& record, std::string_view debug_name = {}); + +private: + /// Frames in flight plus the step input plus the newest result plus the one being written + static constexpr uint32_t k_slot_count = VulkanRenderer::k_frames_in_flight + 3; + + struct Buffer { + bool ring = false; + bool cpu_readback = false; + vk::DeviceSize size = 0; + /// One per slot for a ring buffer and a single entry otherwise + std::vector copies; + std::vector readbacks; + }; + + struct Binding { + uint32_t binding = 0; + BufferId id = 0; + Copy copy = Copy::current; + }; + + struct Slot { + vk::raii::DescriptorSet set = nullptr; + AsyncCompute::Ticket ticket = AsyncCompute::k_no_ticket; + /// Frames in flight still binding this slot + uint32_t pins = 0; + /// Step it holds and 0 while being written + uint64_t step = 0; + }; + + void createDescriptors(); + void writeDescriptors(uint32_t slot); + auto claimFreeSlot() -> uint32_t; + void publish(uint32_t slot, AsyncCompute::Ticket ticket); + /// Caller holds m_mutex + auto newestFinishedLocked() const -> int32_t; + + Config m_config; + const VulkanCore* m_core = nullptr; + bool m_ready = false; + + ShaderLayout m_layout; + std::vector> m_pipelines; + vk::raii::DescriptorPool m_pool = nullptr; + + std::vector m_buffers; + std::vector m_bindings; + + mutable std::mutex m_mutex; + std::array m_slots; + uint64_t m_step_counter = 0; + int32_t m_last_written = -1; + + /// Render thread only + std::array m_frame_slot {}; + std::array m_acquired_step {}; +}; + +} diff --git a/engine/src/toast/renderer/cubemap_target.cpp b/engine/src/toast/renderer/cubemap_target.cpp index bb1acff39..318bc0229 100644 --- a/engine/src/toast/renderer/cubemap_target.cpp +++ b/engine/src/toast/renderer/cubemap_target.cpp @@ -28,7 +28,7 @@ void CubemapTarget::create( m_size = size; m_mip_levels = std::max(mip_levels, 1u); - m_layout = vk::ImageLayout::eUndefined; + m_state = {}; vk::ImageCreateInfo image_ci {}; // imageCubeArray is not enabled on this device @@ -49,6 +49,7 @@ void CubemapTarget::create( m_image.emplace(core.getAllocator().createImage(image_ci, allocation_ci)); setDebugName(core, **m_image, std::string(debug_name)); + m_state.reset(**m_image, vk::ImageSubresourceRange(vk::ImageAspectFlagBits::eColor, 0, m_mip_levels, 0, k_faces)); vk::ImageViewCreateInfo cube_view_ci {}; cube_view_ci.image = **m_image; @@ -76,47 +77,8 @@ void CubemapTarget::create( } } -void CubemapTarget::transition( - vk::CommandBuffer cmd, vk::ImageLayout new_layout, vk::AccessFlags dst_access, vk::PipelineStageFlags dst_stage -) { - if (!m_image.has_value() || m_layout == new_layout) { - return; - } - - vk::PipelineStageFlags src_stage = vk::PipelineStageFlagBits::eTopOfPipe; - vk::AccessFlags src_access {}; - switch (m_layout) { - case vk::ImageLayout::eColorAttachmentOptimal: - src_stage = vk::PipelineStageFlagBits::eColorAttachmentOutput; - src_access = vk::AccessFlagBits::eColorAttachmentWrite; - break; - case vk::ImageLayout::eTransferDstOptimal: - src_stage = vk::PipelineStageFlagBits::eTransfer; - src_access = vk::AccessFlagBits::eTransferWrite; - break; - case vk::ImageLayout::eShaderReadOnlyOptimal: - src_stage = vk::PipelineStageFlagBits::eFragmentShader; - src_access = vk::AccessFlagBits::eShaderRead; - break; - default: break; - } - - const vk::ImageMemoryBarrier barrier( - src_access, - dst_access, - m_layout, - new_layout, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - **m_image, - vk::ImageSubresourceRange(vk::ImageAspectFlagBits::eColor, 0, m_mip_levels, 0, k_faces) - ); - cmd.pipelineBarrier(src_stage, dst_stage, {}, nullptr, nullptr, barrier); - m_layout = new_layout; -} - -void CubemapTarget::setLayout(vk::ImageLayout layout) noexcept { - m_layout = layout; +void CubemapTarget::transition(vk::CommandBuffer cmd, sync::Usage next) { + sync::transition(cmd, m_state, next); } auto CubemapTarget::faceView(uint32_t mip, uint32_t face) const -> vk::ImageView { diff --git a/engine/src/toast/renderer/cubemap_target.hpp b/engine/src/toast/renderer/cubemap_target.hpp index 662940e30..7c454c1d6 100644 --- a/engine/src/toast/renderer/cubemap_target.hpp +++ b/engine/src/toast/renderer/cubemap_target.hpp @@ -22,10 +22,10 @@ class CubemapTarget { vk::ImageUsageFlags extra_usage = {}, bool with_face_views = true ); - void - transition(vk::CommandBuffer cmd, vk::ImageLayout new_layout, vk::AccessFlags dst_access, vk::PipelineStageFlags dst_stage); + void transition(vk::CommandBuffer cmd, sync::Usage next); - void setLayout(vk::ImageLayout layout) noexcept; + /// For code that transitioned the whole image itself like the vulkan_common upload helpers + void setUsage(sync::Usage usage) noexcept { m_state.usage = usage; } [[nodiscard]] auto faceView(uint32_t mip, uint32_t face) const -> vk::ImageView; @@ -56,8 +56,8 @@ class CubemapTarget { } [[nodiscard]] - auto layout() const noexcept -> vk::ImageLayout { - return m_layout; + auto usage() const noexcept -> sync::Usage { + return m_state.usage; } [[nodiscard]] @@ -75,7 +75,7 @@ class CubemapTarget { uint32_t m_size = 0; uint32_t m_mip_levels = 1; - vk::ImageLayout m_layout = vk::ImageLayout::eUndefined; + sync::ImageState m_state; }; } diff --git a/engine/src/toast/renderer/depth_convention.hpp b/engine/src/toast/renderer/depth_convention.hpp new file mode 100644 index 000000000..95c6afcba --- /dev/null +++ b/engine/src/toast/renderer/depth_convention.hpp @@ -0,0 +1,33 @@ +/// @file depth_convention.hpp +/// @author dario +/// @date 09/10/2026 + +#pragma once + +#include +#include +#include + +namespace renderer::depth { + +/// Near 1 far 0 for even float precision over distance +inline constexpr bool k_reversed_z = true; + +inline constexpr float k_near = k_reversed_z ? 1.0f : 0.0f; +inline constexpr float k_far = k_reversed_z ? 0.0f : 1.0f; + +inline constexpr float k_clear = k_far; + +inline constexpr vk::CompareOp k_closer = k_reversed_z ? vk::CompareOp::eGreater : vk::CompareOp::eLess; +/// Drawing over a depth prepass and far plane draws such as the skybox +inline constexpr vk::CompareOp k_closer_or_equal = k_reversed_z ? vk::CompareOp::eGreaterOrEqual : vk::CompareOp::eLessOrEqual; + +/// Callers still apply their own Y flip +[[nodiscard]] +inline auto perspective(float fovy, float aspect, float near_plane, float far_plane) -> glm::mat4 { + // Swapping the planes maps near to 1 and far to 0 + return k_reversed_z ? glm::perspectiveRH_ZO(fovy, aspect, far_plane, near_plane) + : glm::perspectiveRH_ZO(fovy, aspect, near_plane, far_plane); +} + +} diff --git a/engine/src/toast/renderer/frustum.hpp b/engine/src/toast/renderer/frustum.hpp index 2d4ac1d7c..7eed009d3 100644 --- a/engine/src/toast/renderer/frustum.hpp +++ b/engine/src/toast/renderer/frustum.hpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace renderer { @@ -17,9 +18,10 @@ struct Box { glm::vec3 max {0.0f}; }; -/// Inward normalized planes and with 0 to 1 clip depth near is row 2 alone +/// Inward normalized planes ordered left right bottom top near far +/// @p reversed_depth is depth::k_reversed_z for a scene camera and false for shadow views [[nodiscard]] -inline auto extractFrustumPlanes(const glm::mat4& view_projection) -> FrustumPlanes { +inline auto extractFrustumPlanes(const glm::mat4& view_projection, bool reversed_depth = false) -> FrustumPlanes { const auto& m = view_projection; // glm is column major const glm::vec4 row0(m[0][0], m[1][0], m[2][0], m[3][0]); @@ -28,6 +30,9 @@ inline auto extractFrustumPlanes(const glm::mat4& view_projection) -> FrustumPla const glm::vec4 row3(m[0][3], m[1][3], m[2][3], m[3][3]); FrustumPlanes planes {row3 + row0, row3 - row0, row3 + row1, row3 - row1, row2, row3 - row2}; + if (reversed_depth) { + std::swap(planes[4], planes[5]); + } for (auto& plane : planes) { const float length = glm::length(glm::vec3(plane)); diff --git a/engine/src/toast/renderer/gpu_sync.cpp b/engine/src/toast/renderer/gpu_sync.cpp new file mode 100644 index 000000000..9574db1d4 --- /dev/null +++ b/engine/src/toast/renderer/gpu_sync.cpp @@ -0,0 +1,331 @@ +/// @file gpu_sync.cpp +/// @author dario +/// @date 09/10/2026 + +#include "gpu_sync.hpp" + +namespace renderer::sync { + +namespace { + +using Stage = vk::PipelineStageFlagBits2; +using Access = vk::AccessFlagBits2; +using Layout = vk::ImageLayout; + +/// Only writes need making available since a read only needs the execution dependency +constexpr vk::AccessFlags2 k_write_access = Access::eShaderWrite | Access::eShaderStorageWrite | Access::eColorAttachmentWrite | + Access::eDepthStencilAttachmentWrite | Access::eTransferWrite | Access::eHostWrite | + Access::eMemoryWrite | Access::eAccelerationStructureWriteKHR; + +auto makeImageBarrier(vk::Image image, const vk::ImageSubresourceRange& range, Usage previous, Usage next) + -> vk::ImageMemoryBarrier2 { + const UsageInfo src = info(previous); + const UsageInfo dst = info(next); + + vk::ImageMemoryBarrier2 barrier {}; + barrier.srcStageMask = src.stages; + barrier.srcAccessMask = src.access & k_write_access; + barrier.dstStageMask = dst.stages; + barrier.dstAccessMask = dst.access; + barrier.oldLayout = src.layout; + barrier.newLayout = dst.layout; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image; + barrier.subresourceRange = range; + return barrier; +} + +/// A read in a new stage still needs a barrier since the write was only made visible to the first read stages +auto readCoveredBy(Usage previous, Usage next) -> bool { + const UsageInfo src = info(previous); + const UsageInfo dst = info(next); + return (dst.stages & src.stages) == dst.stages && (dst.access & src.access) == dst.access; +} + +auto needsImageBarrier(Usage previous, Usage next) -> bool { + if (next == Usage::none) { + return false; + } + if (writes(previous) || writes(next) || info(previous).layout != info(next).layout) { + return true; + } + return !readCoveredBy(previous, next); +} + +auto needsBufferBarrier(Usage previous, Usage next) -> bool { + if (next == Usage::none || previous == Usage::none) { + return false; + } + return writes(previous) || writes(next) || !readCoveredBy(previous, next); +} + +} + +auto info(Usage usage) -> UsageInfo { + switch (usage) { + case Usage::none: return {Stage::eNone, Access::eNone, Layout::eUndefined}; + + case Usage::color_attachment: + return { + Stage::eColorAttachmentOutput, + Access::eColorAttachmentRead | Access::eColorAttachmentWrite, + Layout::eColorAttachmentOptimal, + }; + case Usage::depth_attachment: + return { + Stage::eEarlyFragmentTests | Stage::eLateFragmentTests, + Access::eDepthStencilAttachmentRead | Access::eDepthStencilAttachmentWrite, + Layout::eDepthAttachmentOptimal, + }; + case Usage::depth_sampled: return {Stage::eFragmentShader, Access::eShaderSampledRead, Layout::eDepthReadOnlyOptimal}; + + case Usage::fragment_sampled: return {Stage::eFragmentShader, Access::eShaderSampledRead, Layout::eShaderReadOnlyOptimal}; + case Usage::compute_sampled: return {Stage::eComputeShader, Access::eShaderSampledRead, Layout::eShaderReadOnlyOptimal}; + case Usage::shader_sampled: + return { + Stage::eVertexShader | Stage::eFragmentShader | Stage::eComputeShader, + Access::eShaderSampledRead, + Layout::eShaderReadOnlyOptimal, + }; + + case Usage::compute_storage_read: return {Stage::eComputeShader, Access::eShaderStorageRead, Layout::eGeneral}; + case Usage::compute_storage_write: + return {Stage::eComputeShader, Access::eShaderStorageRead | Access::eShaderStorageWrite, Layout::eGeneral}; + case Usage::vertex_storage_read: return {Stage::eVertexShader, Access::eShaderStorageRead, Layout::eGeneral}; + case Usage::fragment_storage_read: return {Stage::eFragmentShader, Access::eShaderStorageRead, Layout::eGeneral}; + case Usage::uniform_read: + return { + Stage::eVertexShader | Stage::eFragmentShader | Stage::eComputeShader, + Access::eUniformRead, + Layout::eUndefined, + }; + case Usage::vertex_input: + return { + Stage::eVertexAttributeInput | Stage::eIndexInput, + Access::eVertexAttributeRead | Access::eIndexRead, + Layout::eUndefined, + }; + case Usage::indirect_read: return {Stage::eDrawIndirect, Access::eIndirectCommandRead, Layout::eUndefined}; + + case Usage::transfer_src: return {Stage::eTransfer, Access::eTransferRead, Layout::eTransferSrcOptimal}; + case Usage::transfer_dst: return {Stage::eTransfer, Access::eTransferWrite, Layout::eTransferDstOptimal}; + case Usage::host_read: return {Stage::eHost, Access::eHostRead, Layout::eGeneral}; + case Usage::host_write: return {Stage::eHost, Access::eHostWrite, Layout::eGeneral}; + + case Usage::present: return {Stage::eColorAttachmentOutput, Access::eNone, Layout::ePresentSrcKHR}; + + case Usage::acceleration_build: + return { + Stage::eAccelerationStructureBuildKHR, + Access::eAccelerationStructureReadKHR | Access::eAccelerationStructureWriteKHR, + Layout::eUndefined, + }; + case Usage::acceleration_trace_fragment: + return {Stage::eFragmentShader, Access::eAccelerationStructureReadKHR, Layout::eUndefined}; + + case Usage::general: return {Stage::eAllCommands, Access::eMemoryRead | Access::eMemoryWrite, Layout::eGeneral}; + } + return {Stage::eAllCommands, Access::eMemoryRead | Access::eMemoryWrite, Layout::eGeneral}; +} + +auto writes(Usage usage) -> bool { + return static_cast(info(usage).access & k_write_access); +} + +void BarrierBatch::image(ImageState& state, Usage next) { + if (!state) { + return; + } + image(state.image, state.range, state.usage, next); + state.usage = next; +} + +void BarrierBatch::image(vk::Image image, const vk::ImageSubresourceRange& range, Usage previous, Usage next) { + if (image == VK_NULL_HANDLE || !needsImageBarrier(previous, next)) { + return; + } + m_images.push_back(makeImageBarrier(image, range, previous, next)); +} + +void BarrierBatch::buffer(vk::Buffer buffer, Usage previous, Usage next, vk::DeviceSize offset, vk::DeviceSize size) { + if (buffer == VK_NULL_HANDLE || size == 0 || !needsBufferBarrier(previous, next)) { + return; + } + const UsageInfo src = info(previous); + const UsageInfo dst = info(next); + + vk::BufferMemoryBarrier2 barrier {}; + barrier.srcStageMask = src.stages; + barrier.srcAccessMask = src.access & k_write_access; + barrier.dstStageMask = dst.stages; + barrier.dstAccessMask = dst.access; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer; + barrier.offset = offset; + barrier.size = size; + m_buffers.push_back(barrier); +} + +void BarrierBatch::memory(Usage previous, Usage next) { + if (!needsBufferBarrier(previous, next)) { + return; + } + const UsageInfo src = info(previous); + const UsageInfo dst = info(next); + + vk::MemoryBarrier2 barrier {}; + barrier.srcStageMask = src.stages; + barrier.srcAccessMask = src.access & k_write_access; + barrier.dstStageMask = dst.stages; + barrier.dstAccessMask = dst.access; + m_memory.push_back(barrier); +} + +void BarrierBatch::release( + vk::Image image, const vk::ImageSubresourceRange& range, Usage previous, Usage next, const QueueHandoff& handoff +) { + if (!handoff.crossesQueues()) { + this->image(image, range, previous, next); + return; + } + if (image == VK_NULL_HANDLE) { + return; + } + vk::ImageMemoryBarrier2 barrier = makeImageBarrier(image, range, previous, next); + // The consumer acquire carries the destination scope + barrier.dstStageMask = Stage::eNone; + barrier.dstAccessMask = Access::eNone; + if (handoff.exclusive) { + barrier.srcQueueFamilyIndex = handoff.from; + barrier.dstQueueFamilyIndex = handoff.to; + } + m_images.push_back(barrier); +} + +void BarrierBatch::acquire( + vk::Image image, const vk::ImageSubresourceRange& range, Usage previous, Usage next, const QueueHandoff& handoff +) { + if (!handoff.crossesQueues() || image == VK_NULL_HANDLE) { + return; + } + vk::ImageMemoryBarrier2 barrier = makeImageBarrier(image, range, previous, next); + // The release carried the source scope and the fence wait orders this after it + barrier.srcStageMask = Stage::eNone; + barrier.srcAccessMask = Access::eNone; + if (handoff.exclusive) { + barrier.srcQueueFamilyIndex = handoff.from; + barrier.dstQueueFamilyIndex = handoff.to; + } else { + // The release already transitioned a concurrent image + barrier.oldLayout = barrier.newLayout; + } + m_images.push_back(barrier); +} + +void BarrierBatch::release( + vk::Buffer buffer, Usage previous, Usage next, const QueueHandoff& handoff, vk::DeviceSize offset, vk::DeviceSize size +) { + if (!handoff.crossesQueues()) { + this->buffer(buffer, previous, next, offset, size); + return; + } + if (buffer == VK_NULL_HANDLE || size == 0) { + return; + } + const UsageInfo src = info(previous); + + vk::BufferMemoryBarrier2 barrier {}; + barrier.srcStageMask = src.stages; + barrier.srcAccessMask = src.access & k_write_access; + barrier.dstStageMask = Stage::eNone; + barrier.dstAccessMask = Access::eNone; + barrier.srcQueueFamilyIndex = handoff.exclusive ? handoff.from : VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = handoff.exclusive ? handoff.to : VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer; + barrier.offset = offset; + barrier.size = size; + m_buffers.push_back(barrier); +} + +void BarrierBatch::acquire( + vk::Buffer buffer, Usage next, const QueueHandoff& handoff, vk::DeviceSize offset, vk::DeviceSize size +) { + if (!handoff.crossesQueues() || buffer == VK_NULL_HANDLE || size == 0) { + return; + } + const UsageInfo dst = info(next); + + vk::BufferMemoryBarrier2 barrier {}; + barrier.srcStageMask = Stage::eNone; + barrier.srcAccessMask = Access::eNone; + barrier.dstStageMask = dst.stages; + barrier.dstAccessMask = dst.access; + barrier.srcQueueFamilyIndex = handoff.exclusive ? handoff.from : VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = handoff.exclusive ? handoff.to : VK_QUEUE_FAMILY_IGNORED; + barrier.buffer = buffer; + barrier.offset = offset; + barrier.size = size; + m_buffers.push_back(barrier); +} + +void BarrierBatch::flush(vk::CommandBuffer cmd) { + if (empty()) { + return; + } + + vk::DependencyInfo dependency {}; + dependency.memoryBarrierCount = static_cast(m_memory.size()); + dependency.pMemoryBarriers = m_memory.data(); + dependency.bufferMemoryBarrierCount = static_cast(m_buffers.size()); + dependency.pBufferMemoryBarriers = m_buffers.data(); + dependency.imageMemoryBarrierCount = static_cast(m_images.size()); + dependency.pImageMemoryBarriers = m_images.data(); + cmd.pipelineBarrier2(dependency); + + m_images.clear(); + m_buffers.clear(); + m_memory.clear(); +} + +void transition(vk::CommandBuffer cmd, vk::Image image, const vk::ImageSubresourceRange& range, Usage previous, Usage next) { + BarrierBatch batch; + batch.image(image, range, previous, next); + batch.flush(cmd); +} + +void transition(vk::CommandBuffer cmd, ImageState& state, Usage next) { + BarrierBatch batch; + batch.image(state, next); + batch.flush(cmd); +} + +void submit(vk::Queue queue, const SubmitDesc& desc) { + std::vector waits; + waits.reserve(desc.waits.size()); + for (const auto& wait : desc.waits) { + waits.push_back(vk::SemaphoreSubmitInfo {}.setSemaphore(wait.semaphore).setStageMask(wait.stages)); + } + + std::vector signals; + signals.reserve(desc.signals.size()); + for (const vk::Semaphore semaphore : desc.signals) { + signals.push_back(vk::SemaphoreSubmitInfo {}.setSemaphore(semaphore).setStageMask(vk::PipelineStageFlagBits2::eAllCommands)); + } + + std::vector commands; + commands.reserve(desc.commands.size()); + for (const vk::CommandBuffer cmd : desc.commands) { + commands.push_back(vk::CommandBufferSubmitInfo {}.setCommandBuffer(cmd)); + } + + vk::SubmitInfo2 info {}; + info.pNext = desc.next; + info.setWaitSemaphoreInfos(waits); + info.setCommandBufferInfos(commands); + info.setSignalSemaphoreInfos(signals); + queue.submit2(info, desc.fence); +} + +} diff --git a/engine/src/toast/renderer/gpu_sync.hpp b/engine/src/toast/renderer/gpu_sync.hpp new file mode 100644 index 000000000..fd522e941 --- /dev/null +++ b/engine/src/toast/renderer/gpu_sync.hpp @@ -0,0 +1,182 @@ +/// @file gpu_sync.hpp +/// @author dario +/// @date 09/10/2026 + +#pragma once + +#include +#include +#include +#include +#include + +namespace renderer::sync { + +enum class Usage : uint8_t { + /// Contents discarded and the layout undefined + none, + + /// Includes load and store ops + color_attachment, + /// eDepthAttachmentOptimal including load and store ops + depth_attachment, + /// eDepthReadOnlyOptimal sampled by fragment shaders + depth_sampled, + + fragment_sampled, + compute_sampled, + /// Vertex fragment and compute + shader_sampled, + + compute_storage_read, + compute_storage_write, + /// Vertex pulling + vertex_storage_read, + fragment_storage_read, + /// Any shader stage + uniform_read, + /// Vertex and index buffers + vertex_input, + indirect_read, + + transfer_src, + transfer_dst, + host_read, + host_write, + + /// Colour attachment output stage so the next use chains onto the acquire semaphore wait + present, + + /// Build input output and scratch + acceleration_build, + /// Fragment shader ray queries + acceleration_trace_fragment, + + /// Every stage and access in eGeneral so correct anywhere and slow everywhere + general, +}; + +struct UsageInfo { + vk::PipelineStageFlags2 stages; + vk::AccessFlags2 access; + vk::ImageLayout layout = vk::ImageLayout::eUndefined; +}; + +[[nodiscard]] +TOAST_API auto info(Usage usage) -> UsageInfo; + +[[nodiscard]] +TOAST_API auto writes(Usage usage) -> bool; + +/// Owned next to the image it describes +struct ImageState { + vk::Image image = VK_NULL_HANDLE; + vk::ImageSubresourceRange range {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}; + Usage usage = Usage::none; + + /// On every image creation since its contents start undefined + void reset(vk::Image new_image, const vk::ImageSubresourceRange& new_range) { + image = new_image; + range = new_range; + usage = Usage::none; + } + + [[nodiscard]] + explicit operator bool() const { + return image != VK_NULL_HANDLE; + } +}; + +struct QueueHandoff { + uint32_t from = VK_QUEUE_FAMILY_IGNORED; + uint32_t to = VK_QUEUE_FAMILY_IGNORED; + /// False for eConcurrent resources which need visibility on the consumer but no ownership transfer + bool exclusive = true; + + [[nodiscard]] + auto crossesQueues() const noexcept -> bool { + return from != to; + } +}; + +/// Skips a barrier only when neither side writes and the layout and next read stages are already covered +class TOAST_API BarrierBatch { +public: + /// No op for an image that does not exist. The widest usage until the next write avoids read to read barriers + void image(ImageState& state, Usage next); + + void image(vk::Image image, const vk::ImageSubresourceRange& range, Usage previous, Usage next); + + void buffer(vk::Buffer buffer, Usage previous, Usage next, vk::DeviceSize offset = 0, vk::DeviceSize size = VK_WHOLE_SIZE); + + void memory(Usage previous, Usage next); + + /// On the writing queue. A plain transition within one family. Across families @p next may name stages this queue lacks + /// since the acquire waits for them + void release(vk::Image image, const vk::ImageSubresourceRange& range, Usage previous, Usage next, const QueueHandoff& handoff); + + /// On the consuming queue after the release fence was waited on. @p previous and @p next must match the release + /// exactly and nothing is recorded within one family + void acquire(vk::Image image, const vk::ImageSubresourceRange& range, Usage previous, Usage next, const QueueHandoff& handoff); + + void release( + vk::Buffer buffer, Usage previous, Usage next, const QueueHandoff& handoff, vk::DeviceSize offset = 0, + vk::DeviceSize size = VK_WHOLE_SIZE + ); + + void acquire( + vk::Buffer buffer, Usage next, const QueueHandoff& handoff, vk::DeviceSize offset = 0, vk::DeviceSize size = VK_WHOLE_SIZE + ); + + void flush(vk::CommandBuffer cmd); + + [[nodiscard]] + auto empty() const -> bool { + return m_images.empty() && m_buffers.empty() && m_memory.empty(); + } + + [[nodiscard]] + auto pendingImages() const -> std::span { + return m_images; + } + + [[nodiscard]] + auto pendingBuffers() const -> std::span { + return m_buffers; + } + + [[nodiscard]] + auto pendingMemory() const -> std::span { + return m_memory; + } + +private: + std::vector m_images; + std::vector m_buffers; + std::vector m_memory; +}; + +TOAST_API void + transition(vk::CommandBuffer cmd, vk::Image image, const vk::ImageSubresourceRange& range, Usage previous, Usage next); + +TOAST_API void transition(vk::CommandBuffer cmd, ImageState& state, Usage next); + +struct SemaphoreWait { + vk::Semaphore semaphore; + vk::PipelineStageFlags2 stages; +}; + +/// Signals fire once every command in the submit completes +struct SubmitDesc { + std::span commands; + std::span waits; + std::span signals; + vk::Fence fence; + /// Chained to SubmitInfo2 such as a FrameBoundaryEXT + const void* next = nullptr; +}; + +/// The caller holds any lock the queue needs +TOAST_API void submit(vk::Queue queue, const SubmitDesc& desc); + +} diff --git a/engine/src/toast/renderer/gpu_timer.cpp b/engine/src/toast/renderer/gpu_timer.cpp index 84de7fed8..4d7e23241 100644 --- a/engine/src/toast/renderer/gpu_timer.cpp +++ b/engine/src/toast/renderer/gpu_timer.cpp @@ -79,14 +79,14 @@ void GpuTimer::beginGraphics(uint32_t slot, vk::CommandBuffer cmd) { // The compute pair is reset by the compute command buffer cmd.resetQueryPool(*m_pool, base(slot), k_queries_per_slot - 2); - cmd.writeTimestamp(vk::PipelineStageFlagBits::eTopOfPipe, *m_pool, base(slot)); + cmd.writeTimestamp2(vk::PipelineStageFlagBits2::eNone, *m_pool, base(slot)); } void GpuTimer::endGraphics(uint32_t slot, vk::CommandBuffer cmd) { if (!m_supported || slot >= m_slots.size()) { return; } - cmd.writeTimestamp(vk::PipelineStageFlagBits::eBottomOfPipe, *m_pool, base(slot) + 1); + cmd.writeTimestamp2(vk::PipelineStageFlagBits2::eAllCommands, *m_pool, base(slot) + 1); m_slots[slot].graphics_written = true; } @@ -98,14 +98,14 @@ void GpuTimer::beginCompute(uint32_t slot, vk::CommandBuffer cmd) { const uint32_t first = base(slot) + k_queries_per_slot - 2; cmd.resetQueryPool(*m_pool, first, 2); - cmd.writeTimestamp(vk::PipelineStageFlagBits::eTopOfPipe, *m_pool, first); + cmd.writeTimestamp2(vk::PipelineStageFlagBits2::eNone, *m_pool, first); } void GpuTimer::endCompute(uint32_t slot, vk::CommandBuffer cmd) { if (!m_compute_supported || slot >= m_slots.size()) { return; } - cmd.writeTimestamp(vk::PipelineStageFlagBits::eBottomOfPipe, *m_pool, base(slot) + k_queries_per_slot - 1); + cmd.writeTimestamp2(vk::PipelineStageFlagBits2::eAllCommands, *m_pool, base(slot) + k_queries_per_slot - 1); m_slots[slot].compute_written = true; } @@ -121,7 +121,7 @@ auto GpuTimer::beginScope(uint32_t slot, vk::CommandBuffer cmd, std::string_view const auto index = static_cast(state.scopes.size()); state.scopes.push_back({.name = ScopeName(name), .depth = state.depth, .cpu_start = std::chrono::steady_clock::now()}); ++state.depth; - cmd.writeTimestamp(vk::PipelineStageFlagBits::eTopOfPipe, *m_pool, base(slot) + 2 + (2 * index)); + cmd.writeTimestamp2(vk::PipelineStageFlagBits2::eNone, *m_pool, base(slot) + 2 + (2 * index)); return index; } @@ -137,7 +137,7 @@ void GpuTimer::endScope(uint32_t slot, vk::CommandBuffer cmd, uint32_t scope) { PendingScope& pending = state.scopes[scope]; pending.cpu_ms = std::chrono::duration(std::chrono::steady_clock::now() - pending.cpu_start).count(); state.depth = pending.depth; - cmd.writeTimestamp(vk::PipelineStageFlagBits::eBottomOfPipe, *m_pool, base(slot) + 3 + (2 * scope)); + cmd.writeTimestamp2(vk::PipelineStageFlagBits2::eAllCommands, *m_pool, base(slot) + 3 + (2 * scope)); } auto GpuTimer::ticksToMs(uint64_t begin, uint64_t end, uint64_t mask) const noexcept -> double { @@ -170,6 +170,7 @@ void GpuTimer::collect(uint32_t slot) { const std::vector& ticks = graphics.value; m_frame.valid = true; + ++m_serial; m_frame.graphics_ms = ticksToMs(ticks[0], ticks[1], m_graphics_mask); m_frame.compute_ms = 0.0; m_frame.scopes.clear(); diff --git a/engine/src/toast/renderer/gpu_timer.hpp b/engine/src/toast/renderer/gpu_timer.hpp index 54e485fcf..f363e1573 100644 --- a/engine/src/toast/renderer/gpu_timer.hpp +++ b/engine/src/toast/renderer/gpu_timer.hpp @@ -75,6 +75,12 @@ class GpuTimer { return m_frame; } + /// Tells a fresh last() from a repeat + [[nodiscard]] + auto frameSerial() const noexcept -> uint64_t { + return m_serial; + } + [[nodiscard]] auto smoothed() const noexcept -> const Timings& { return m_smoothed; @@ -114,6 +120,7 @@ class GpuTimer { bool m_supported = false; bool m_compute_supported = false; bool m_reported_unavailable = false; + uint64_t m_serial = 0; Timings m_frame; Timings m_smoothed; diff --git a/engine/src/toast/renderer/passes/bloom_pass.cpp b/engine/src/toast/renderer/passes/bloom_pass.cpp index e6ebbe465..82b48ba91 100644 --- a/engine/src/toast/renderer/passes/bloom_pass.cpp +++ b/engine/src/toast/renderer/passes/bloom_pass.cpp @@ -96,7 +96,6 @@ void BloomPass::createTargets(const VulkanCore& core, vk::Extent2D extent) { const auto make_target = [&](Mip& mip, vk::Extent2D size, std::string_view debug_name) { mip.extent = size; - mip.layout = vk::ImageLayout::eUndefined; const auto image_ci = colorTargetImageInfo(size, m_format); @@ -105,6 +104,7 @@ void BloomPass::createTargets(const VulkanCore& core, vk::Extent2D extent) { mip.image.emplace(core.getAllocator().createImage(image_ci, allocation_ci)); setDebugName(core, **mip.image, std::string(debug_name)); + mip.state.reset(**mip.image, colorSubresourceRange()); vk::ImageViewCreateInfo view_ci {}; view_ci.image = **mip.image; @@ -142,17 +142,21 @@ void BloomPass::createTargets(const VulkanCore& core, vk::Extent2D extent) { return std::move(allocated[0]); }; - m_downsample_sets.clear(); - m_upsample_sets.clear(); - for (size_t i = 0; i < m_mips.size(); ++i) { - m_downsample_sets.push_back(allocate()); - } - for (size_t i = 0; i + 1 < m_mips.size(); ++i) { - m_upsample_sets.push_back(allocate()); + m_frame_sets.clear(); + m_frame_sets.resize(VulkanRenderer::k_frames_in_flight); + for (size_t frame = 0; frame < m_frame_sets.size(); ++frame) { + FrameSets& sets = m_frame_sets[frame]; + for (size_t i = 0; i < m_mips.size(); ++i) { + sets.downsample.push_back(allocate()); + setDebugName(core, *sets.downsample.back(), std::format("BloomPass Downsample[{}][{}]", frame, i)); + } + for (size_t i = 0; i + 1 < m_mips.size(); ++i) { + sets.upsample.push_back(allocate()); + setDebugName(core, *sets.upsample.back(), std::format("BloomPass Upsample[{}][{}]", frame, i)); + } + sets.composite = allocate(); + setDebugName(core, *sets.composite, std::format("BloomPass Composite[{}]", frame)); } - m_composite_set = allocate(); - - m_bound_source = nullptr; TOAST_TRACE("Render", "BloomPass targets created: {} mips from {}x{}", m_mips.size(), extent.width, extent.height); } @@ -174,35 +178,6 @@ void BloomPass::writeDescriptor(vk::DescriptorSet set, vk::ImageView source, vk: m_core->getDevice().updateDescriptorSets(writes, {}); } -void BloomPass::transition( - vk::CommandBuffer cmd, Mip& mip, vk::ImageLayout new_layout, vk::AccessFlags dst_access, vk::PipelineStageFlags dst_stage -) { - if (!mip.image.has_value() || mip.layout == new_layout) { - return; - } - - const bool was_attachment = mip.layout == vk::ImageLayout::eColorAttachmentOptimal; - const vk::ImageMemoryBarrier barrier( - was_attachment ? vk::AccessFlagBits::eColorAttachmentWrite : vk::AccessFlags {}, - dst_access, - mip.layout, - new_layout, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - **mip.image, - vk::ImageSubresourceRange(vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1) - ); - cmd.pipelineBarrier( - was_attachment ? vk::PipelineStageFlagBits::eColorAttachmentOutput : vk::PipelineStageFlagBits::eTopOfPipe, - dst_stage, - {}, - nullptr, - nullptr, - barrier - ); - mip.layout = new_layout; -} - void BloomPass::drawInto( vk::CommandBuffer cmd, const VulkanPipeline& pipeline, vk::DescriptorSet set, const Mip& target, const Params& params, bool additive @@ -239,23 +214,24 @@ void BloomPass::drawInto( auto BloomPass::record(vk::CommandBuffer cmd, uint32_t frame_index, vk::ImageView source_view) -> vk::ImageView { ZoneScoped; - (void)frame_index; - - if (m_mips.empty() || !m_composite.view.has_value() || !m_composite_pipeline.isReady() || !source_view) { + if (m_mips.empty() || !m_composite.view.has_value() || !m_composite_pipeline.isReady() || !source_view || + frame_index >= m_frame_sets.size()) { return source_view; } - if (m_bound_source != source_view) { - writeDescriptor(*m_downsample_sets[0], source_view, source_view); + // Safe since the fence of this slot was waited on + FrameSets& sets = m_frame_sets[frame_index]; + if (sets.bound_source != source_view) { + writeDescriptor(*sets.downsample[0], source_view, source_view); for (size_t i = 1; i < m_mips.size(); ++i) { - writeDescriptor(*m_downsample_sets[i], **m_mips[i - 1].view, source_view); + writeDescriptor(*sets.downsample[i], **m_mips[i - 1].view, source_view); } for (size_t i = 0; i + 1 < m_mips.size(); ++i) { const size_t from = m_mips.size() - 1 - i; - writeDescriptor(*m_upsample_sets[i], **m_mips[from].view, source_view); + writeDescriptor(*sets.upsample[i], **m_mips[from].view, source_view); } - writeDescriptor(*m_composite_set, **m_mips[0].view, source_view); - m_bound_source = source_view; + writeDescriptor(*sets.composite, **m_mips[0].view, source_view); + sets.bound_source = source_view; } const auto* frame = VulkanRenderer::instance->renderingFrame(); @@ -273,23 +249,11 @@ auto BloomPass::record(vk::CommandBuffer cmd, uint32_t frame_index, vk::ImageVie params.inverse_source_size = glm::vec2(1.0f / static_cast(source_size.width), 1.0f / static_cast(source_size.height)); - transition( - cmd, - m_mips[i], - vk::ImageLayout::eColorAttachmentOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::PipelineStageFlagBits::eColorAttachmentOutput - ); - - drawInto(cmd, i == 0 ? m_downsample_first_pipeline : m_downsample_pipeline, *m_downsample_sets[i], m_mips[i], params, false); - - transition( - cmd, - m_mips[i], - vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits::eShaderRead, - vk::PipelineStageFlagBits::eFragmentShader - ); + sync::transition(cmd, m_mips[i].state, sync::Usage::color_attachment); + + drawInto(cmd, i == 0 ? m_downsample_first_pipeline : m_downsample_pipeline, *sets.downsample[i], m_mips[i], params, false); + + sync::transition(cmd, m_mips[i].state, sync::Usage::fragment_sampled); } for (size_t i = 0; i + 1 < m_mips.size(); ++i) { @@ -299,40 +263,16 @@ auto BloomPass::record(vk::CommandBuffer cmd, uint32_t frame_index, vk::ImageVie params.inverse_source_size = glm::vec2(1.0f / static_cast(m_mips[from].extent.width), 1.0f / static_cast(m_mips[from].extent.height)); - transition( - cmd, - m_mips[to], - vk::ImageLayout::eColorAttachmentOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::PipelineStageFlagBits::eColorAttachmentOutput - ); - - drawInto(cmd, m_upsample_pipeline, *m_upsample_sets[i], m_mips[to], params, true); - - transition( - cmd, - m_mips[to], - vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits::eShaderRead, - vk::PipelineStageFlagBits::eFragmentShader - ); + sync::transition(cmd, m_mips[to].state, sync::Usage::color_attachment); + + drawInto(cmd, m_upsample_pipeline, *sets.upsample[i], m_mips[to], params, true); + + sync::transition(cmd, m_mips[to].state, sync::Usage::fragment_sampled); } - transition( - cmd, - m_composite, - vk::ImageLayout::eColorAttachmentOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::PipelineStageFlagBits::eColorAttachmentOutput - ); - drawInto(cmd, m_composite_pipeline, *m_composite_set, m_composite, params, false); - transition( - cmd, - m_composite, - vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits::eShaderRead, - vk::PipelineStageFlagBits::eFragmentShader - ); + sync::transition(cmd, m_composite.state, sync::Usage::color_attachment); + drawInto(cmd, m_composite_pipeline, *sets.composite, m_composite, params, false); + sync::transition(cmd, m_composite.state, sync::Usage::shader_sampled); return **m_composite.view; } diff --git a/engine/src/toast/renderer/passes/bloom_pass.hpp b/engine/src/toast/renderer/passes/bloom_pass.hpp index 514edc409..dc43488f5 100644 --- a/engine/src/toast/renderer/passes/bloom_pass.hpp +++ b/engine/src/toast/renderer/passes/bloom_pass.hpp @@ -46,7 +46,7 @@ class BloomPass : public IPostProcessPass { std::optional image; std::optional view; vk::Extent2D extent; - vk::ImageLayout layout = vk::ImageLayout::eUndefined; + sync::ImageState state; }; void createTargets(const VulkanCore& core, vk::Extent2D extent); @@ -54,10 +54,6 @@ class BloomPass : public IPostProcessPass { void writeDescriptor(vk::DescriptorSet set, vk::ImageView source, vk::ImageView scene); - void transition( - vk::CommandBuffer cmd, Mip& mip, vk::ImageLayout new_layout, vk::AccessFlags dst_access, vk::PipelineStageFlags dst_stage - ); - void drawInto( vk::CommandBuffer cmd, const VulkanPipeline& pipeline, vk::DescriptorSet set, const Mip& target, const Params& params, bool additive @@ -78,11 +74,15 @@ class BloomPass : public IPostProcessPass { Mip m_composite; - std::vector m_downsample_sets; - std::vector m_upsample_sets; - vk::raii::DescriptorSet m_composite_set = nullptr; + /// Per frame in flight since the TAA ping pong source changes and a pending frame still binds the old sets + struct FrameSets { + std::vector downsample; + std::vector upsample; + vk::raii::DescriptorSet composite = nullptr; + vk::ImageView bound_source = nullptr; + }; - vk::ImageView m_bound_source = nullptr; + std::vector m_frame_sets; }; } diff --git a/engine/src/toast/renderer/passes/cluster_lighting_pass.cpp b/engine/src/toast/renderer/passes/cluster_lighting_pass.cpp index 3bb81000e..e3932f235 100644 --- a/engine/src/toast/renderer/passes/cluster_lighting_pass.cpp +++ b/engine/src/toast/renderer/passes/cluster_lighting_pass.cpp @@ -25,22 +25,11 @@ namespace renderer { auto ClusterLightingPass::createBuffer( const renderer::VulkanCore& core, vk::DeviceSize size, vk::BufferUsageFlags usage, HostAccess access ) -> vma::raii::Buffer { - const uint32_t graphics_family = core.getGraphicsQueueFamilyIndex(); - const uint32_t compute_family = core.getComputeQueueFamilyIndex(); - const bool use_concurrent_sharing = graphics_family != compute_family; - + // Graphics queue only since light culling runs in the frame command buffer vk::BufferCreateInfo buffer_ci {}; buffer_ci.size = size; buffer_ci.usage = usage; - - std::array family_indices {graphics_family, compute_family}; - if (use_concurrent_sharing) { - buffer_ci.sharingMode = vk::SharingMode::eConcurrent; - buffer_ci.queueFamilyIndexCount = 2; - buffer_ci.pQueueFamilyIndices = family_indices.data(); - } else { - buffer_ci.sharingMode = vk::SharingMode::eExclusive; - } + buffer_ci.sharingMode = vk::SharingMode::eExclusive; vma::AllocationCreateInfo alloc_ci {}; switch (access) { @@ -235,18 +224,9 @@ void ClusterLightingPass::dispatch(vk::CommandBuffer cmd, uint32_t frame_index) // Same command buffer does not order shader memory access so this needs its own barrier auto& fb = m_frame_buffers[frame_index]; - const vk::BufferMemoryBarrier barrier( - vk::AccessFlagBits::eShaderWrite, - vk::AccessFlagBits::eShaderRead, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - **fb.cluster_aabb.gpu_buffer, - 0, - VK_WHOLE_SIZE - ); - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eComputeShader, vk::PipelineStageFlagBits::eComputeShader, {}, nullptr, barrier, nullptr - ); + sync::BarrierBatch barriers; + barriers.buffer(**fb.cluster_aabb.gpu_buffer, sync::Usage::compute_storage_write, sync::Usage::compute_storage_read); + barriers.flush(cmd); cmd.bindPipeline(vk::PipelineBindPoint::eCompute, m_cull_lights_pipeline.getPipeline()); cmd.dispatch(group_count, 1, 1); @@ -256,22 +236,16 @@ void ClusterLightingPass::dispatch(vk::CommandBuffer cmd, uint32_t frame_index) if (fb.readback_valid) { m_readback_hold.fetch_sub(1, std::memory_order_relaxed); - const vk::BufferMemoryBarrier to_transfer( - vk::AccessFlagBits::eShaderWrite, - vk::AccessFlagBits::eTransferRead, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - **fb.cluster_light_grid.gpu_buffer, - 0, - VK_WHOLE_SIZE - ); - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eComputeShader, vk::PipelineStageFlagBits::eTransfer, {}, nullptr, to_transfer, nullptr - ); + barriers.buffer(**fb.cluster_light_grid.gpu_buffer, sync::Usage::compute_storage_write, sync::Usage::transfer_src); + barriers.flush(cmd); constexpr vk::DeviceSize k_grid_size = sizeof(uint32_t) * k_cluster_count; const vk::BufferCopy region(0, 0, k_grid_size); cmd.copyBuffer(**fb.cluster_light_grid.gpu_buffer, **fb.cluster_light_grid_readback.gpu_buffer, region); + + // A fence wait alone does not make the copy visible to the host + barriers.buffer(**fb.cluster_light_grid_readback.gpu_buffer, sync::Usage::transfer_dst, sync::Usage::host_read); + barriers.flush(cmd); } } diff --git a/engine/src/toast/renderer/passes/debug_pass.cpp b/engine/src/toast/renderer/passes/debug_pass.cpp index 218d020b0..c59e3f97b 100644 --- a/engine/src/toast/renderer/passes/debug_pass.cpp +++ b/engine/src/toast/renderer/passes/debug_pass.cpp @@ -16,6 +16,7 @@ #include "../vulkan_texture.hpp" #include "cluster_lighting_pass.hpp" #include "environment_pass.hpp" +#include "perf_window.hpp" #include "shadow_pass.hpp" #include "skinning_pass.hpp" @@ -39,41 +40,6 @@ namespace renderer { -struct DebugPass::PerfOverlay { - static constexpr size_t k_history = 240; - - std::chrono::steady_clock::time_point last_sample; - VulkanRenderer::PerfCounters last {}; - uint32_t last_skipped = 0; - bool has_sample = false; - - float render_fps = 0.0f; - float game_fps = 0.0f; - float tick_rate = 0.0f; - float skipped_per_second = 0.0f; - float repeated_ratio = 0.0f; - float wait_ratio = 0.0f; - - float draw_ms = 0.0f; - float upload_ms = 0.0f; - float gpu_wait_ms = 0.0f; - float acquire_ms = 0.0f; - float record_ms = 0.0f; - float submit_ms = 0.0f; - float present_ms = 0.0f; - float frame_wait_ms = 0.0f; - float pacing_ms = 0.0f; - - float build_ms = 0.0f; - float slot_wait_ms = 0.0f; - - std::array interval_history {}; - std::array gpu_history {}; - size_t history_cursor = 0; - size_t history_filled = 0; - float p99_interval_ms = 0.0f; -}; - namespace { constexpr std::array k_axis_handles { @@ -421,323 +387,6 @@ auto clusterHeatmapColorImGui(uint32_t light_count) -> ImVec4 { } // namespace -void DebugPass::drawPerformanceWindow() { - ZoneScoped; - if (!m_perf) { - m_perf = std::make_unique(); - } - PerfOverlay& perf = *m_perf; - - auto* renderer = VulkanRenderer::instance; - const VulkanRenderer::PerfCounters counters = renderer->perfCounters(); - const uint32_t skipped = renderer->getSkippedBuildCount(); - const GpuTimer* gpu_timer = renderer->gpuTimer(); - const GpuTimer::Timings* gpu = gpu_timer != nullptr && gpu_timer->smoothed().valid ? &gpu_timer->smoothed() : nullptr; - const GpuTimer::Timings* gpu_last = gpu_timer != nullptr && gpu_timer->last().valid ? &gpu_timer->last() : nullptr; - const auto now = std::chrono::steady_clock::now(); - - const float frame_interval_ms = ImGui::GetIO().DeltaTime * 1000.0f; - perf.interval_history[perf.history_cursor] = frame_interval_ms; - perf.gpu_history[perf.history_cursor] = - gpu_last != nullptr ? static_cast(gpu_last->graphics_ms + gpu_last->compute_ms) : 0.0f; - perf.history_cursor = (perf.history_cursor + 1) % PerfOverlay::k_history; - perf.history_filled = std::min(perf.history_filled + 1, PerfOverlay::k_history); - - constexpr auto k_sample_interval = std::chrono::milliseconds(500); - const bool first_sample = perf.last_sample.time_since_epoch().count() == 0; - const bool sample_due = !first_sample && now - perf.last_sample >= k_sample_interval; - - if (sample_due) { - const double seconds = std::chrono::duration(now - perf.last_sample).count(); - const VulkanRenderer::PerfCounters& last = perf.last; - const auto draws = static_cast(counters.draws - last.draws); - const auto built = static_cast(counters.frames_built - last.frames_built); - const auto ticks = static_cast(counters.ticks - last.ticks); - - const auto per = [](uint64_t current, uint64_t previous, double count) { - return count > 0.0 ? static_cast(static_cast(current - previous) / count / 1.0e6) : 0.0f; - }; - - perf.render_fps = static_cast(draws / seconds); - perf.game_fps = static_cast(built / seconds); - perf.tick_rate = static_cast(ticks / seconds); - perf.skipped_per_second = static_cast(static_cast(skipped - perf.last_skipped) / seconds); - perf.repeated_ratio = - draws > 0.0 ? static_cast(static_cast(counters.repeated_draws - last.repeated_draws) / draws) : 0.0f; - perf.wait_ratio = - std::min(1.0f, static_cast(static_cast(counters.slot_wait_ns - last.slot_wait_ns) / 1.0e9 / seconds)); - - perf.draw_ms = per(counters.draw_work_ns, last.draw_work_ns, draws); - perf.upload_ms = per(counters.upload_ns, last.upload_ns, draws); - perf.gpu_wait_ms = per(counters.gpu_wait_ns, last.gpu_wait_ns, draws); - perf.acquire_ms = per(counters.acquire_ns, last.acquire_ns, draws); - perf.record_ms = per(counters.record_ns, last.record_ns, draws); - perf.submit_ms = per(counters.submit_ns, last.submit_ns, draws); - perf.present_ms = per(counters.present_ns, last.present_ns, draws); - perf.frame_wait_ms = per(counters.frame_wait_ns, last.frame_wait_ns, draws); - perf.pacing_ms = per(counters.pacing_ns, last.pacing_ns, draws); - perf.build_ms = per(counters.build_ns, last.build_ns, built); - perf.slot_wait_ms = per(counters.slot_wait_ns, last.slot_wait_ns, ticks); - - std::array sorted = perf.interval_history; - const size_t filled = perf.history_filled; - std::sort(sorted.begin(), sorted.begin() + static_cast(filled)); - perf.p99_interval_ms = filled > 0 ? sorted[std::min(filled - 1, filled * 99 / 100)] : 0.0f; - - perf.has_sample = true; - } - - if (first_sample || sample_due) { - perf.last_sample = now; - perf.last = counters; - perf.last_skipped = skipped; - } - - const ImGuiIO& io = ImGui::GetIO(); - ImGui::SetNextWindowPos(ImVec2(io.DisplaySize.x - 12.0f, 12.0f), ImGuiCond_FirstUseEver, ImVec2(1.0f, 0.0f)); - ImGui::SetNextWindowBgAlpha(0.85f); - constexpr ImGuiWindowFlags k_flags = - ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoFocusOnAppearing | ImGuiWindowFlags_NoNav; - - const bool open = ImGui::Begin("Performance", nullptr, k_flags); - if (!open || !perf.has_sample) { - if (open) { - ImGui::TextDisabled("Sampling..."); - } - ImGui::End(); - return; - } - - const ImVec4 k_bad(1.0f, 0.35f, 0.35f, 1.0f); - const ImVec4 k_warn(1.0f, 0.7f, 0.2f, 1.0f); - const ImVec4 k_good(0.4f, 0.9f, 0.4f, 1.0f); - constexpr float k_trend = 0.25f; - - float avg_ms = 0.0f; - float min_ms = 0.0f; - float max_ms = 0.0f; - renderer->getFrameTimeStats(avg_ms, min_ms, max_ms); - const auto cap = static_cast(renderer->effectiveFrameRateLimit()); - const float interval_ms = perf.render_fps > 0.0f ? 1000.0f / perf.render_fps : 0.0f; - const float gpu_ms = gpu != nullptr ? static_cast(gpu->graphics_ms + gpu->compute_ms) : 0.0f; - const auto share = [interval_ms](float ms) { return interval_ms > 0.0f ? ms / interval_ms : 0.0f; }; - - const auto heaviest = [gpu](bool by_gpu) -> const GpuTimer::ScopeTiming* { - if (gpu == nullptr) { - return nullptr; - } - const GpuTimer::ScopeTiming* best = nullptr; - for (size_t i = 0; i < gpu->scopes.size(); ++i) { - const auto& scope = gpu->scopes[i]; - if (i + 1 < gpu->scopes.size() && gpu->scopes[i + 1].depth > scope.depth) { - continue; - } - const double value = by_gpu ? scope.gpu_ms : scope.cpu_ms; - if (best == nullptr || value > (by_gpu ? best->gpu_ms : best->cpu_ms)) { - best = &scope; - } - } - return best; - }; - - ImGui::PushTextWrapPos(ImGui::GetFontSize() * 28.0f); - - ImGui::Text("Renderer %6.1f fps %6.2f ms", perf.render_fps, interval_ms); - ImGui::TextDisabled(" avg %.2f min %.2f max %.2f p99 %.2f ms", avg_ms, min_ms, max_ms, perf.p99_interval_ms); - ImGui::Text("Game %6.1f fps %6.0f ticks/s", perf.game_fps, perf.tick_rate); - if (gpu != nullptr) { - ImGui::Text("GPU %6.2f ms %3.0f%%", gpu_ms, std::min(share(gpu_ms), 1.0f) * 100.0f); - } else { - ImGui::TextDisabled("GPU n/a"); - } - if (cap > 0.0f) { - ImGui::TextDisabled("Cap %6.0f fps%s", cap, renderer->applicationFocused() ? "" : " unfocused"); - } else { - ImGui::TextDisabled("Cap off"); - } - - const float graph_width = ImGui::GetFontSize() * 28.0f; - const float graph_max = std::max({perf.p99_interval_ms * 1.25f, interval_ms * 1.5f, gpu_ms * 1.5f, 1.0f}); - const auto history_count = static_cast(perf.history_filled); - const int history_offset = perf.history_filled < PerfOverlay::k_history ? 0 : static_cast(perf.history_cursor); - const std::string interval_label = std::format("frame ms (max {:.1f})", graph_max); - ImGui::PlotLines( - "##frame_interval", - perf.interval_history.data(), - history_count, - history_offset, - interval_label.c_str(), - 0.0f, - graph_max, - ImVec2(graph_width, 50.0f) - ); - if (gpu != nullptr) { - ImGui::PlotLines( - "##gpu_time", - perf.gpu_history.data(), - history_count, - history_offset, - "GPU ms", - 0.0f, - graph_max, - ImVec2(graph_width, 36.0f) - ); - } - - ImGui::Separator(); - - const bool at_cap = cap > 0.0f && perf.render_fps >= cap * 0.95f; - ImGui::TextDisabled("Game "); - ImGui::SameLine(); - if (perf.skipped_per_second > 0.0f || (!at_cap && perf.wait_ratio >= k_trend)) { - ImGui::TextColored(k_bad, "Renderer exhausted"); - ImGui::SameLine(); - ImGui::TextDisabled("wait %.0f%% skipped %.1f/s", perf.wait_ratio * 100.0f, perf.skipped_per_second); - } else if (perf.repeated_ratio >= k_trend) { - ImGui::TextColored(k_warn, "CPU starved"); - ImGui::SameLine(); - ImGui::TextDisabled("repeated %.0f%%", perf.repeated_ratio * 100.0f); - } else if (at_cap) { - ImGui::TextColored(k_good, "Capped"); - } else { - ImGui::TextColored(k_good, "Balanced"); - } - - const float cpu_work_ms = perf.upload_ms + perf.record_ms + perf.submit_ms; - const float display_ms = perf.acquire_ms + perf.present_ms; - const float idle_ms = perf.frame_wait_ms + perf.pacing_ms; - - ImGui::TextDisabled("Render"); - ImGui::SameLine(); - if (share(perf.gpu_wait_ms) >= k_trend || (gpu != nullptr && share(gpu_ms) >= 0.85f && share(idle_ms) < k_trend)) { - ImGui::TextColored(k_bad, "GPU bound"); - ImGui::SameLine(); - ImGui::TextDisabled("wait %.0f%% gpu %.2f ms", share(perf.gpu_wait_ms) * 100.0f, gpu_ms); - if (const auto* scope = heaviest(true); scope != nullptr) { - const std::string_view name = scope->name.view(); - ImGui::TextDisabled("top %.*s %.2f ms", static_cast(name.size()), name.data(), scope->gpu_ms); - } - } else if (share(display_ms) >= k_trend) { - ImGui::TextColored(k_warn, "Display paced"); - ImGui::SameLine(); - ImGui::TextDisabled("acquire+present %.0f%%", share(display_ms) * 100.0f); - } else if (share(cpu_work_ms) >= 0.6f) { - ImGui::TextColored(k_bad, "CPU bound"); - ImGui::SameLine(); - ImGui::TextDisabled("cpu %.2f ms gpu %.2f ms", cpu_work_ms, gpu_ms); - if (const auto* scope = heaviest(false); scope != nullptr) { - const std::string_view name = scope->name.view(); - ImGui::TextDisabled("top %.*s %.2f ms", static_cast(name.size()), name.data(), scope->cpu_ms); - } - } else if (share(idle_ms) >= k_trend) { - ImGui::TextColored(k_good, "Headroom"); - ImGui::SameLine(); - ImGui::TextDisabled("idle %.0f%%", share(idle_ms) * 100.0f); - } else { - ImGui::TextColored(k_good, "Balanced"); - } - - if (ImGui::CollapsingHeader("Render thread", ImGuiTreeNodeFlags_DefaultOpen)) { - struct Phase { - const char* label; - float ms; - }; - - const std::array phases { - Phase { "GPU wait", perf.gpu_wait_ms}, - Phase { "Record", perf.record_ms}, - Phase { "Submit", perf.submit_ms}, - Phase { "Uploads", perf.upload_ms}, - Phase { "Acquire", perf.acquire_ms}, - Phase { "Present", perf.present_ms}, - Phase {"Game wait", perf.frame_wait_ms}, - Phase {"Cap sleep", perf.pacing_ms}, - }; - - float accounted_ms = 0.0f; - if (ImGui::BeginTable("##render_phases", 3, ImGuiTableFlags_SizingFixedFit)) { - for (const Phase& phase : phases) { - accounted_ms += phase.ms; - ImGui::TableNextRow(); - ImGui::TableNextColumn(); - ImGui::TextUnformatted(phase.label); - ImGui::TableNextColumn(); - ImGui::Text("%6.2f ms", phase.ms); - ImGui::TableNextColumn(); - const float fraction = std::clamp(share(phase.ms), 0.0f, 1.0f); - ImGui::ProgressBar( - fraction, ImVec2(ImGui::GetFontSize() * 10.0f, 0.0f), std::format("{:.0f}%", fraction * 100.0f).c_str() - ); - } - ImGui::EndTable(); - } - ImGui::TextDisabled("Other %6.2f ms", std::max(0.0f, interval_ms - accounted_ms)); - } - - if (ImGui::CollapsingHeader("GPU passes", ImGuiTreeNodeFlags_DefaultOpen)) { - if (gpu_timer == nullptr || !gpu_timer->supported()) { - ImGui::TextDisabled("No timestamp queries"); - } else if (gpu == nullptr) { - ImGui::TextDisabled("No data yet"); - } else { - ImGui::Text("Graphics %.2f ms", gpu->graphics_ms); - if (gpu_timer->computeSupported()) { - ImGui::SameLine(); - ImGui::Text(" Compute %.2f ms", gpu->compute_ms); - } - - constexpr ImGuiTableFlags k_table_flags = - ImGuiTableFlags_SizingFixedFit | ImGuiTableFlags_RowBg | ImGuiTableFlags_BordersInnerV; - if (ImGui::BeginTable("##gpu_scopes", 4, k_table_flags)) { - ImGui::TableSetupColumn("Pass"); - ImGui::TableSetupColumn("CPU"); - ImGui::TableSetupColumn("GPU"); - ImGui::TableSetupColumn("Runs"); - ImGui::TableHeadersRow(); - for (const auto& scope : gpu->scopes) { - const std::string_view name = scope.name.view(); - const auto indent = static_cast(scope.depth * 2); - ImGui::TableNextRow(); - ImGui::TableNextColumn(); - if (scope.cpu_ms < 0.01 && scope.gpu_ms < 0.01) { - ImGui::TextDisabled("%*s%.*s", indent, "", static_cast(name.size()), name.data()); - } else { - ImGui::Text("%*s%.*s", indent, "", static_cast(name.size()), name.data()); - } - ImGui::TableNextColumn(); - ImGui::Text("%6.2f ms", scope.cpu_ms); - ImGui::TableNextColumn(); - ImGui::Text("%6.2f ms", scope.gpu_ms); - ImGui::TableNextColumn(); - ImGui::Text("%u", scope.count); - } - ImGui::EndTable(); - } - } - } - - if (ImGui::CollapsingHeader("Game thread")) { - ImGui::Text("Ticks %6.0f /s", perf.tick_rate); - ImGui::Text("Built %6.1f /s %6.2f ms", perf.game_fps, perf.build_ms); - ImGui::Text("Slot wait %6.2f ms %3.0f%%", perf.slot_wait_ms, perf.wait_ratio * 100.0f); - ImGui::Text("Repeated %6.0f%%", perf.repeated_ratio * 100.0f); - ImGui::Text("Skipped %6.1f /s", perf.skipped_per_second); - } - - if (ImGui::CollapsingHeader("Scene")) { - if (const auto* frame = renderer->renderingFrame(); frame != nullptr) { - const auto visible = std::ranges::count_if(frame->mesh_instances, [](const auto& proxy) { return proxy.visible; }); - ImGui::Text("Mesh instances %zu visible of %zu", static_cast(visible), frame->mesh_instances.size()); - ImGui::Text("Material ranges %zu", frame->material_ranges.size()); - ImGui::Text("Lights %zu of %u submitted", frame->lights.size(), frame->light_stats.submitted); - ImGui::Text("Voxel volumes %zu", frame->voxel_instances.size()); - } - } - - ImGui::PopTextWrapPos(); - ImGui::End(); -} - void DebugPass::update(uint32_t frame_index, float dt) { ZoneScoped; const auto* frame = VulkanRenderer::instance->renderingFrame(); @@ -774,7 +423,10 @@ void DebugPass::update(uint32_t frame_index, float dt) { ImGui_ImplVulkan_NewFrame(); ImGui::NewFrame(); - drawPerformanceWindow(); + if (!m_perf) { + m_perf = std::make_unique(); + } + m_perf->draw(); if (!m_voxels) { m_voxels = std::make_unique(); diff --git a/engine/src/toast/renderer/passes/debug_pass.hpp b/engine/src/toast/renderer/passes/debug_pass.hpp index 40a2a6f84..9f4a3e3d3 100644 --- a/engine/src/toast/renderer/passes/debug_pass.hpp +++ b/engine/src/toast/renderer/passes/debug_pass.hpp @@ -20,6 +20,8 @@ namespace renderer { class VulkanCore; class ClusterLightingPass; +class PerfWindow; + namespace voxel_debug { class Monitor; } @@ -131,13 +133,10 @@ class DebugPass : public IRenderPass { bool m_imgui_ready = false; bool m_editor_panels = true; - struct PerfOverlay; - std::unique_ptr m_perf; + std::unique_ptr m_perf; std::unique_ptr m_voxels; - void drawPerformanceWindow(); - const ClusterLightingPass* m_cluster_lighting_pass = nullptr; float m_sky_intensity_ui = -1.0f; diff --git a/engine/src/toast/renderer/passes/environment_pass.cpp b/engine/src/toast/renderer/passes/environment_pass.cpp index 05052d905..6c6cd0606 100644 --- a/engine/src/toast/renderer/passes/environment_pass.cpp +++ b/engine/src/toast/renderer/passes/environment_pass.cpp @@ -106,7 +106,7 @@ void EnvironmentPass::createPipelines(const VulkanCore& core) { config.depth_format = m_depth_format; config.depth_test = true; config.depth_write = false; - config.depth_compare = vk::CompareOp::eLessOrEqual; + config.depth_compare = depth::k_closer_or_equal; config.extra_color_formats = worldStageExtraColorFormats(); m_skybox_pipeline.rebuild(core, config); } @@ -204,7 +204,7 @@ auto EnvironmentPass::uploadEquirect(const assets::HdrImage& image) -> bool { cmd.end(); - submitAndWait(device, m_core->getGraphicsQueue(), *cmd); + submitAndWait(device, m_core->getGraphicsQueue(), m_core->graphicsSubmitMutex(), *cmd); vk::ImageViewCreateInfo view_ci {}; view_ci.image = **m_equirect_image; @@ -372,12 +372,7 @@ void EnvironmentPass::recordPre(vk::CommandBuffer cmd, uint32_t frame_index, uin return params; }; - m_sky.transition( - cmd, - vk::ImageLayout::eColorAttachmentOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::PipelineStageFlagBits::eColorAttachmentOutput - ); + m_sky.transition(cmd, sync::Usage::color_attachment); const bool use_equirect = m_has_equirect && m_equirect_pipeline.isReady() && *m_equirect_set != VK_NULL_HANDLE; for (uint32_t face = 0; face < 6; ++face) { if (use_equirect) { @@ -386,29 +381,15 @@ void EnvironmentPass::recordPre(vk::CommandBuffer cmd, uint32_t frame_index, uin renderFace(cmd, m_sky_pipeline, nullptr, m_sky, 0, face, make_params(face, 0.0f)); } } - m_sky.transition( - cmd, vk::ImageLayout::eShaderReadOnlyOptimal, vk::AccessFlagBits::eShaderRead, vk::PipelineStageFlagBits::eFragmentShader - ); + m_sky.transition(cmd, sync::Usage::fragment_sampled); - m_irradiance.transition( - cmd, - vk::ImageLayout::eColorAttachmentOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::PipelineStageFlagBits::eColorAttachmentOutput - ); + m_irradiance.transition(cmd, sync::Usage::color_attachment); for (uint32_t face = 0; face < 6; ++face) { renderFace(cmd, m_irradiance_pipeline, *m_sky_source_set, m_irradiance, 0, face, make_params(face, 0.0f)); } - m_irradiance.transition( - cmd, vk::ImageLayout::eShaderReadOnlyOptimal, vk::AccessFlagBits::eShaderRead, vk::PipelineStageFlagBits::eFragmentShader - ); + m_irradiance.transition(cmd, sync::Usage::fragment_sampled); - m_prefiltered.transition( - cmd, - vk::ImageLayout::eColorAttachmentOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::PipelineStageFlagBits::eColorAttachmentOutput - ); + m_prefiltered.transition(cmd, sync::Usage::color_attachment); for (uint32_t mip = 0; mip < m_prefiltered.mipLevels(); ++mip) { const float roughness = m_prefiltered.mipLevels() > 1 ? static_cast(mip) / static_cast(m_prefiltered.mipLevels() - 1) : 0.0f; @@ -416,9 +397,7 @@ void EnvironmentPass::recordPre(vk::CommandBuffer cmd, uint32_t frame_index, uin renderFace(cmd, m_prefilter_pipeline, *m_sky_source_set, m_prefiltered, mip, face, make_params(face, roughness)); } } - m_prefiltered.transition( - cmd, vk::ImageLayout::eShaderReadOnlyOptimal, vk::AccessFlagBits::eShaderRead, vk::PipelineStageFlagBits::eFragmentShader - ); + m_prefiltered.transition(cmd, sync::Usage::fragment_sampled); TOAST_INFO( "Render", diff --git a/engine/src/toast/renderer/passes/material_pass.cpp b/engine/src/toast/renderer/passes/material_pass.cpp index c3d5becf2..c4055c9f3 100644 --- a/engine/src/toast/renderer/passes/material_pass.cpp +++ b/engine/src/toast/renderer/passes/material_pass.cpp @@ -154,7 +154,7 @@ void MaterialPass::rebuildPipeline() { config.depth_write = settings.depth_write; // eLessOrEqual since the prepass writes the exact same depth - config.depth_compare = vk::CompareOp::eLessOrEqual; + config.depth_compare = depth::k_closer_or_equal; config.cull_mode = toCullMode(settings.cull_mode); config.blend_preset = toBlendPreset(settings.blend_mode); config.extra_color_formats = worldStageExtraColorFormats(); diff --git a/engine/src/toast/renderer/passes/perf_window.cpp b/engine/src/toast/renderer/passes/perf_window.cpp new file mode 100644 index 000000000..e641c713e --- /dev/null +++ b/engine/src/toast/renderer/passes/perf_window.cpp @@ -0,0 +1,843 @@ +/// @file perf_window.cpp +/// @author dario +/// @date 09/10/2026 + +#include "perf_window.hpp" + +#include "../async_compute.hpp" +#include "../vulkan_core.hpp" + +#include +#include +#include +#include +#include +#include + +namespace renderer { + +namespace { + +// Prefixed since unity builds merge this with the other passes + +constexpr ImU32 perfColor(int r, int g, int b, int a = 255) { + return IM_COL32(r, g, b, a); +} + +const ImVec4 k_perf_good(0.45f, 0.85f, 0.45f, 1.0f); +const ImVec4 k_perf_warn(1.0f, 0.72f, 0.25f, 1.0f); +const ImVec4 k_perf_bad(1.0f, 0.38f, 0.38f, 1.0f); + +constexpr ImU32 k_perf_bar_good = perfColor(92, 184, 92); +constexpr ImU32 k_perf_bar_warn = perfColor(230, 170, 60); +constexpr ImU32 k_perf_bar_bad = perfColor(220, 80, 80); +constexpr ImU32 k_perf_gpu_line = perfColor(110, 170, 255); +constexpr ImU32 k_perf_budget_line = perfColor(255, 255, 255, 110); +constexpr ImU32 k_perf_graph_bg = perfColor(0, 0, 0, 90); + +/// Frame share above which a cost counts as a trend rather than noise +constexpr float k_perf_trend = 0.25f; + +auto perfMiB(uint64_t bytes) -> double { + return static_cast(bytes) / (1024.0 * 1024.0); +} + +void perfSwatch(ImU32 color) { + const float size = ImGui::GetTextLineHeight() * 0.7f; + const ImVec2 cursor = ImGui::GetCursorScreenPos(); + const float pad = (ImGui::GetTextLineHeight() - size) * 0.5f; + ImGui::GetWindowDrawList()->AddRectFilled( + ImVec2(cursor.x, cursor.y + pad), ImVec2(cursor.x + size, cursor.y + pad + size), color, 2.0f + ); + ImGui::Dummy(ImVec2(size, ImGui::GetTextLineHeight())); + ImGui::SameLine(); +} + +void perfVerdictText(const char* heading, const ImVec4& color, const char* label, const std::string& detail) { + ImGui::TextDisabled("%s", heading); + ImGui::SameLine(); + ImGui::TextColored(color, "%s", label); + if (!detail.empty()) { + ImGui::SameLine(); + ImGui::TextDisabled("%s", detail.c_str()); + } +} + +struct PerfSegment { + std::string_view label; + float value; + ImU32 color; +}; + +void perfStackedBar(const char* id, float width, float height, float total, const std::vector& segments) { + const ImVec2 origin = ImGui::GetCursorScreenPos(); + ImGui::InvisibleButton(id, ImVec2(width, height)); + const bool hovered = ImGui::IsItemHovered(); + const float mouse_x = ImGui::GetIO().MousePos.x; + + ImDrawList* draw = ImGui::GetWindowDrawList(); + draw->AddRectFilled(origin, ImVec2(origin.x + width, origin.y + height), k_perf_graph_bg, 3.0f); + if (total <= 0.0f) { + return; + } + + float x = origin.x; + for (const PerfSegment& segment : segments) { + const float segment_width = std::clamp(segment.value / total, 0.0f, 1.0f) * width; + if (segment_width < 0.5f) { + continue; + } + const float end = std::min(x + segment_width, origin.x + width); + draw->AddRectFilled(ImVec2(x, origin.y), ImVec2(end, origin.y + height), segment.color); + if (hovered && mouse_x >= x && mouse_x < end) { + draw->AddRect(ImVec2(x, origin.y), ImVec2(end, origin.y + height), perfColor(255, 255, 255, 200)); + ImGui::SetTooltip( + "%.*s\n%.2f ms %.0f%%", + static_cast(segment.label.size()), + segment.label.data(), + segment.value, + segment.value / total * 100.0f + ); + } + x = end; + } +} + +} + +auto PerfWindow::historyOffset() const -> int { + return m_filled < k_history ? 0 : static_cast(m_cursor); +} + +void PerfWindow::record(const GpuTimer* timer) { + const GpuTimer::Timings* gpu_last = timer != nullptr && timer->last().valid ? &timer->last() : nullptr; + // The timer keeps its last frame through probe captures and stalls + const bool fresh_gpu = gpu_last != nullptr && timer->frameSerial() != m_last_gpu_serial; + + m_interval_history[m_cursor] = ImGui::GetIO().DeltaTime * 1000.0f; + m_gpu_history[m_cursor] = gpu_last != nullptr ? static_cast(gpu_last->graphics_ms) : 0.0f; + m_cursor = (m_cursor + 1) % k_history; + m_filled = std::min(m_filled + 1, k_history); + + if (fresh_gpu) { + m_last_gpu_serial = timer->frameSerial(); + accumulatePasses(*gpu_last); + } +} + +void PerfWindow::accumulatePasses(const GpuTimer::Timings& frame) { + const auto now = std::chrono::steady_clock::now(); + ++m_window_frames; + m_graphics_sum += frame.graphics_ms; + + // Unknown scopes go right after the previous scope of this frame so existing rows never move + size_t insert_at = 0; + for (const auto& scope : frame.scopes) { + const std::string_view name = scope.name.view(); + auto it = + std::ranges::find_if(m_passes, [&](const PassStats& pass) { return pass.depth == scope.depth && pass.name == name; }); + if (it == m_passes.end()) { + PassStats pass; + pass.name = std::string(name); + pass.depth = scope.depth; + it = m_passes.insert(m_passes.begin() + static_cast(std::min(insert_at, m_passes.size())), std::move(pass)); + } + insert_at = static_cast(it - m_passes.begin()) + 1; + + it->last_seen = now; + it->gpu_sum += scope.gpu_ms; + it->cpu_sum += scope.cpu_ms; + it->gpu_peak_acc = std::max(it->gpu_peak_acc, scope.gpu_ms); + it->runs_sum += scope.count; + ++it->frames_run; + } +} + +void PerfWindow::publishPasses() { + if (m_window_frames == 0) { + return; + } + const auto frames = static_cast(m_window_frames); + m_graphics_avg = static_cast(m_graphics_sum / frames); + m_graphics_sum = 0.0; + + for (PassStats& pass : m_passes) { + const double ran = std::max(pass.frames_run, 1u); + pass.gpu_avg = static_cast(pass.gpu_sum / ran); + pass.cpu_avg = static_cast(pass.cpu_sum / ran); + pass.gpu_peak = static_cast(pass.gpu_peak_acc); + pass.runs = static_cast(static_cast(pass.runs_sum) / ran); + pass.presence = static_cast(static_cast(pass.frames_run) / frames); + + pass.gpu_sum = 0.0; + pass.cpu_sum = 0.0; + pass.gpu_peak_acc = 0.0; + pass.runs_sum = 0; + pass.frames_run = 0; + } + m_window_frames = 0; + + // Passes gone for a while drop out instead of piling up + constexpr auto k_forget_after = std::chrono::seconds(10); + const auto now = std::chrono::steady_clock::now(); + std::erase_if(m_passes, [&](const PassStats& pass) { return now - pass.last_seen > k_forget_after; }); +} + +auto PerfWindow::heaviestPass(bool by_gpu) const -> const PassStats* { + // Leaf passes only since a parent time is the sum of its children + const PassStats* best = nullptr; + for (size_t i = 0; i < m_passes.size(); ++i) { + const PassStats& pass = m_passes[i]; + if (i + 1 < m_passes.size() && m_passes[i + 1].depth > pass.depth) { + continue; + } + const float value = by_gpu ? pass.perFrame() : pass.cpu_avg * pass.presence; + if (best == nullptr || value > (by_gpu ? best->perFrame() : best->cpu_avg * best->presence)) { + best = &pass; + } + } + return best; +} + +void PerfWindow::sample(const VulkanRenderer::PerfCounters& counters, uint32_t skipped) { + constexpr auto k_sample_interval = std::chrono::milliseconds(500); + const auto now = std::chrono::steady_clock::now(); + const bool first_sample = m_last_sample.time_since_epoch().count() == 0; + const bool sample_due = !first_sample && now - m_last_sample >= k_sample_interval; + + if (sample_due) { + const double seconds = std::chrono::duration(now - m_last_sample).count(); + const VulkanRenderer::PerfCounters& last = m_last_counters; + const auto draws = static_cast(counters.draws - last.draws); + const auto built = static_cast(counters.frames_built - last.frames_built); + const auto ticks = static_cast(counters.ticks - last.ticks); + + const auto per = [](uint64_t current, uint64_t previous, double count) { + return count > 0.0 ? static_cast(static_cast(current - previous) / count / 1.0e6) : 0.0f; + }; + + Sample& s = m_sample; + s.render_fps = static_cast(draws / seconds); + s.game_fps = static_cast(built / seconds); + s.tick_rate = static_cast(ticks / seconds); + s.skipped_per_second = static_cast(static_cast(skipped - m_last_skipped) / seconds); + s.repeated_ratio = + draws > 0.0 ? static_cast(static_cast(counters.repeated_draws - last.repeated_draws) / draws) : 0.0f; + s.wait_ratio = + std::min(1.0f, static_cast(static_cast(counters.slot_wait_ns - last.slot_wait_ns) / 1.0e9 / seconds)); + + s.draw_ms = per(counters.draw_work_ns, last.draw_work_ns, draws); + s.upload_ms = per(counters.upload_ns, last.upload_ns, draws); + s.gpu_wait_ms = per(counters.gpu_wait_ns, last.gpu_wait_ns, draws); + s.acquire_ms = per(counters.acquire_ns, last.acquire_ns, draws); + s.record_ms = per(counters.record_ns, last.record_ns, draws); + s.submit_ms = per(counters.submit_ns, last.submit_ns, draws); + s.present_ms = per(counters.present_ns, last.present_ns, draws); + s.frame_wait_ms = per(counters.frame_wait_ns, last.frame_wait_ns, draws); + s.pacing_ms = per(counters.pacing_ns, last.pacing_ns, draws); + s.build_ms = per(counters.build_ns, last.build_ns, built); + s.slot_wait_ms = per(counters.slot_wait_ns, last.slot_wait_ns, ticks); + + History sorted = m_interval_history; + std::sort(sorted.begin(), sorted.begin() + static_cast(m_filled)); + s.p99_interval_ms = m_filled > 0 ? sorted[std::min(m_filled - 1, m_filled * 99 / 100)] : 0.0f; + s.max_interval_ms = m_filled > 0 ? sorted[m_filled - 1] : 0.0f; + + publishPasses(); + m_has_sample = true; + } + + if (first_sample || sample_due) { + m_last_sample = now; + m_last_counters = counters; + m_last_skipped = skipped; + } +} + +auto PerfWindow::phases() const -> std::array { + const Sample& s = m_sample; + const float accounted = + s.gpu_wait_ms + s.record_ms + s.submit_ms + s.upload_ms + s.acquire_ms + s.present_ms + s.frame_wait_ms + s.pacing_ms; + return { + Phase {"GPU wait", s.gpu_wait_ms, perfColor(220, 90, 90)}, + Phase {"Record", s.record_ms, perfColor(90, 150, 230)}, + Phase {"Submit", s.submit_ms, perfColor(160, 110, 220)}, + Phase {"Uploads", s.upload_ms, perfColor(70, 190, 180)}, + Phase {"Acquire", s.acquire_ms, perfColor(235, 150, 60)}, + Phase {"Present", s.present_ms, perfColor(230, 210, 80)}, + Phase {"Game wait", s.frame_wait_ms, perfColor(130, 130, 130)}, + Phase {"Cap sleep", s.pacing_ms, perfColor(70, 130, 80)}, + Phase {"Other", std::max(0.0f, m_interval_ms - accounted), perfColor(80, 80, 80)}, + }; +} + +auto PerfWindow::renderVerdict(bool has_gpu) const -> Verdict { + const Sample& s = m_sample; + const auto share = [this](float ms) { return m_interval_ms > 0.0f ? ms / m_interval_ms : 0.0f; }; + const auto top = [this](bool by_gpu) -> std::string { + const PassStats* pass = heaviestPass(by_gpu); + if (pass == nullptr) { + return {}; + } + return std::format("top: {} {:.2f} ms", pass->name, by_gpu ? pass->perFrame() : pass->cpu_avg * pass->presence); + }; + + const float cpu_work_ms = s.upload_ms + s.record_ms + s.submit_ms; + const float display_ms = s.acquire_ms + s.present_ms; + const float idle_ms = s.frame_wait_ms + s.pacing_ms; + + if (share(s.gpu_wait_ms) >= k_perf_trend || (has_gpu && share(m_gpu_ms) >= 0.85f && share(idle_ms) < k_perf_trend)) { + return {.label = "GPU bound", .bad = true, .detail = top(true)}; + } + if (share(display_ms) >= k_perf_trend) { + return {.label = "Display paced", .warn = true, .detail = std::format("acquire+present {:.0f}%", share(display_ms) * 100.0f)}; + } + if (share(cpu_work_ms) >= 0.6f) { + return {.label = "CPU bound", .bad = true, .detail = top(false)}; + } + if (share(idle_ms) >= k_perf_trend) { + return {.label = "Headroom", .detail = std::format("idle {:.0f}%", share(idle_ms) * 100.0f)}; + } + return {.label = "Balanced", .detail = {}}; +} + +auto PerfWindow::gameVerdict() const -> Verdict { + const Sample& s = m_sample; + const auto cap = static_cast(VulkanRenderer::instance->effectiveFrameRateLimit()); + const bool at_cap = cap > 0.0f && s.render_fps >= cap * 0.95f; + + if (s.skipped_per_second > 0.0f || (!at_cap && s.wait_ratio >= k_perf_trend)) { + return { + .label = "Renderer exhausted", + .bad = true, + .detail = std::format("wait {:.0f}% skipped {:.1f}/s", s.wait_ratio * 100.0f, s.skipped_per_second) + }; + } + if (s.repeated_ratio >= k_perf_trend) { + return {.label = "CPU starved", .warn = true, .detail = std::format("repeated {:.0f}%", s.repeated_ratio * 100.0f)}; + } + if (at_cap) { + return {.label = "Capped", .detail = {}}; + } + return {.label = "Balanced", .detail = {}}; +} + +void PerfWindow::drawFrameGraph(float width, float height) { + const ImVec2 origin = ImGui::GetCursorScreenPos(); + ImGui::InvisibleButton("##perf_frame_graph", ImVec2(width, height)); + const bool hovered = ImGui::IsItemHovered(); + + // Budget and the recent worst case stay on screen + const float target = std::max({m_sample.p99_interval_ms * 1.2f, m_budget_ms * 1.5f, m_gpu_ms * 1.5f, 1.0f}); + m_graph_scale_ms = m_graph_scale_ms <= 0.0f ? target : m_graph_scale_ms + (target - m_graph_scale_ms) * 0.05f; + const float scale = m_graph_scale_ms; + + ImDrawList* draw = ImGui::GetWindowDrawList(); + const ImVec2 end(origin.x + width, origin.y + height); + draw->AddRectFilled(origin, end, k_perf_graph_bg, 3.0f); + + const float bar_width = width / static_cast(k_history); + const auto y_of = [&](float ms) { return end.y - std::clamp(ms / scale, 0.0f, 1.0f) * height; }; + const int offset = historyOffset(); + const auto count = static_cast(m_filled); + // Newest frame at the right edge + const float first_x = end.x - bar_width * static_cast(count); + + std::array gpu_points {}; + int hovered_index = -1; + for (int i = 0; i < count; ++i) { + const size_t index = static_cast((offset + i) % static_cast(k_history)); + const float ms = m_interval_history[index]; + const float x0 = first_x + bar_width * static_cast(i); + const ImU32 color = + ms <= m_budget_ms * 1.05f ? k_perf_bar_good : (ms <= m_budget_ms * 1.5f ? k_perf_bar_warn : k_perf_bar_bad); + draw->AddRectFilled(ImVec2(x0, y_of(ms)), ImVec2(x0 + std::max(bar_width - 0.5f, 1.0f), end.y), color); + gpu_points[static_cast(i)] = ImVec2(x0 + bar_width * 0.5f, y_of(m_gpu_history[index])); + + if (hovered && ImGui::GetIO().MousePos.x >= x0 && ImGui::GetIO().MousePos.x < x0 + bar_width) { + hovered_index = static_cast(index); + draw->AddRectFilled(ImVec2(x0, origin.y), ImVec2(x0 + bar_width, end.y), perfColor(255, 255, 255, 40)); + } + } + if (count > 1 && m_gpu_ms > 0.0f) { + draw->AddPolyline(gpu_points.data(), count, k_perf_gpu_line, ImDrawFlags_None, 1.5f); + } + + const float budget_y = y_of(m_budget_ms); + for (float x = origin.x; x < end.x; x += 8.0f) { + draw->AddLine(ImVec2(x, budget_y), ImVec2(std::min(x + 4.0f, end.x), budget_y), k_perf_budget_line, 1.0f); + } + const std::string budget_label = std::format("{:.1f} ms", m_budget_ms); + draw->AddText(ImVec2(origin.x + 4.0f, budget_y - ImGui::GetTextLineHeight()), k_perf_budget_line, budget_label.c_str()); + const std::string scale_label = std::format("{:.0f} ms", scale); + draw->AddText(ImVec2(origin.x + 4.0f, origin.y + 1.0f), perfColor(255, 255, 255, 90), scale_label.c_str()); + + if (hovered_index >= 0) { + const auto index = static_cast(hovered_index); + ImGui::SetTooltip("Frame %.2f ms\nGPU %.2f ms", m_interval_history[index], m_gpu_history[index]); + } +} + +void PerfWindow::drawPhaseBar(float width, float height) const { + std::vector segments; + for (const Phase& phase : phases()) { + segments.push_back({phase.label, phase.ms, phase.color}); + } + perfStackedBar("##perf_phase_bar", width, height, m_interval_ms, segments); +} + +void PerfWindow::drawSummary(bool has_gpu) { + const Sample& s = m_sample; + auto* renderer = VulkanRenderer::instance; + const auto cap = static_cast(renderer->effectiveFrameRateLimit()); + + const ImVec4 fps_color = m_interval_ms <= m_budget_ms * 1.05f ? k_perf_good + : m_interval_ms <= m_budget_ms * 1.5f ? k_perf_warn + : k_perf_bad; + ImGui::TextColored(fps_color, "%5.1f fps", s.render_fps); + ImGui::SameLine(); + ImGui::Text("%6.2f ms", m_interval_ms); + ImGui::SameLine(); + ImGui::TextDisabled("p99 %.1f max %.1f", s.p99_interval_ms, s.max_interval_ms); + + ImGui::TextDisabled("GPU "); + ImGui::SameLine(); + if (has_gpu) { + ImGui::TextColored(ImVec4(0.43f, 0.67f, 1.0f, 1.0f), "%5.2f ms", m_gpu_ms); + } else { + ImGui::TextDisabled("n/a "); + } + ImGui::SameLine(); + ImGui::TextDisabled(" CPU %5.2f ms budget %.1f ms", s.upload_ms + s.record_ms + s.submit_ms, m_budget_ms); + ImGui::SameLine(); + if (cap > 0.0f) { + ImGui::TextDisabled("(%.0f fps cap%s)", cap, renderer->applicationFocused() ? "" : ", unfocused"); + } else { + ImGui::TextDisabled("(uncapped)"); + } + + const float width = ImGui::GetFontSize() * 30.0f; + drawFrameGraph(width, ImGui::GetFontSize() * 4.0f); + drawPhaseBar(width, ImGui::GetFontSize() * 0.6f); + + const Verdict render = renderVerdict(has_gpu); + perfVerdictText("Render", render.bad ? k_perf_bad : (render.warn ? k_perf_warn : k_perf_good), render.label, render.detail); + const Verdict game = gameVerdict(); + perfVerdictText("Game ", game.bad ? k_perf_bad : (game.warn ? k_perf_warn : k_perf_good), game.label, game.detail); +} + +void PerfWindow::drawFrameTab() { + const Sample& s = m_sample; + ImGui::SeparatorText("Render thread, per frame"); + if (ImGui::BeginTable("##perf_phases", 3, ImGuiTableFlags_SizingFixedFit | ImGuiTableFlags_RowBg)) { + for (const Phase& phase : phases()) { + ImGui::TableNextRow(); + ImGui::TableNextColumn(); + perfSwatch(phase.color); + ImGui::TextUnformatted(phase.label); + ImGui::TableNextColumn(); + ImGui::Text("%6.2f ms", phase.ms); + ImGui::TableNextColumn(); + const float fraction = m_interval_ms > 0.0f ? std::clamp(phase.ms / m_interval_ms, 0.0f, 1.0f) : 0.0f; + const std::string label = std::format("{:.0f}%", fraction * 100.0f); + ImGui::ProgressBar(fraction, ImVec2(ImGui::GetFontSize() * 12.0f, 0.0f), label.c_str()); + } + ImGui::EndTable(); + } + + ImGui::SeparatorText("Game thread"); + if (ImGui::BeginTable("##perf_game", 2, ImGuiTableFlags_SizingFixedFit)) { + const auto row = [](const char* label, const std::string& value) { + ImGui::TableNextRow(); + ImGui::TableNextColumn(); + ImGui::TextDisabled("%s", label); + ImGui::TableNextColumn(); + ImGui::TextUnformatted(value.c_str()); + }; + row("Ticks", std::format("{:.0f} /s", s.tick_rate)); + row("Frames built", std::format("{:.1f} /s {:.2f} ms each", s.game_fps, s.build_ms)); + row("Slot wait", std::format("{:.2f} ms {:.0f}% of the time", s.slot_wait_ms, s.wait_ratio * 100.0f)); + row("Repeated draws", std::format("{:.0f}%", s.repeated_ratio * 100.0f)); + row("Skipped builds", std::format("{:.1f} /s", s.skipped_per_second)); + ImGui::EndTable(); + } + ImGui::TextDisabled( + "Repeated: the renderer drew a frame the game had not rebuilt.\nSkipped: the game built a frame the renderer never drew." + ); +} + +void PerfWindow::drawGpuTab() { + const GpuTimer* timer = VulkanRenderer::instance->gpuTimer(); + if (timer == nullptr || !timer->supported()) { + ImGui::TextDisabled("This device has no timestamp queries"); + return; + } + if (m_passes.empty()) { + ImGui::TextDisabled("No GPU timings yet"); + return; + } + + const bool has_baseline = m_graphics_baseline >= 0.0f; + const float width = ImGui::GetFontSize() * 36.0f; + + ImGui::Text("Graphics queue %.2f ms/frame", m_graphics_avg); + if (has_baseline) { + const float delta = m_graphics_avg - m_graphics_baseline; + ImGui::SameLine(); + ImGui::TextColored(delta <= 0.0f ? k_perf_good : k_perf_bad, "%+.2f ms", delta); + ImGui::SameLine(); + ImGui::TextDisabled("vs baseline %.2f", m_graphics_baseline); + } + ImGui::SameLine(); + if (ImGui::SmallButton("Set baseline")) { + m_graphics_baseline = m_graphics_avg; + for (PassStats& pass : m_passes) { + pass.baseline = pass.perFrame(); + } + } + if (ImGui::IsItemHovered()) { + ImGui::SetTooltip("Remember the current costs; the table then shows each pass's change against them"); + } + if (has_baseline) { + ImGui::SameLine(); + if (ImGui::SmallButton("Clear")) { + m_graphics_baseline = -1.0f; + for (PassStats& pass : m_passes) { + pass.baseline = -1.0f; + } + } + } + + // Queue time left after the top level passes is GPU work outside any scope + constexpr std::array k_pass_colors { + perfColor(90, 150, 230), + perfColor(92, 184, 92), + perfColor(230, 170, 60), + perfColor(160, 110, 220), + perfColor(70, 190, 180), + perfColor(220, 90, 90), + perfColor(230, 210, 80), + perfColor(200, 120, 170), + }; + std::vector segments; + float scoped_ms = 0.0f; + for (const PassStats& pass : m_passes) { + if (pass.depth == 0) { + segments.push_back({pass.name, pass.perFrame(), k_pass_colors[segments.size() % k_pass_colors.size()]}); + scoped_ms += pass.perFrame(); + } + } + const float unscoped_ms = std::max(0.0f, m_graphics_avg - scoped_ms); + segments.push_back({"Unscoped", unscoped_ms, perfColor(70, 70, 70)}); + perfStackedBar("##perf_gpu_bar", width, ImGui::GetFontSize() * 0.9f, m_graphics_avg, segments); + + ImGui::Checkbox("Hide idle passes", &m_hide_idle_passes); + ImGui::SameLine(); + ImGui::TextDisabled("Averaged every 0.5 s. Hover a row for details"); + + enum Column : uint8_t { + pass_column, + gpu_column, + peak_column, + cpu_column, + runs_column, + delta_column + }; + + constexpr ImGuiTableFlags k_flags = ImGuiTableFlags_SizingFixedFit | ImGuiTableFlags_NoHostExtendX | ImGuiTableFlags_RowBg | + ImGuiTableFlags_BordersInnerV | ImGuiTableFlags_Sortable | ImGuiTableFlags_SortTristate | + ImGuiTableFlags_ScrollY; + // Fixed sizes so the auto sized window does not jitter as rows and digits change + const ImVec2 outer(width, ImGui::GetTextLineHeightWithSpacing() * 18.0f); + const int column_count = has_baseline ? 6 : 5; + if (!ImGui::BeginTable("##perf_gpu_passes", column_count, k_flags, outer)) { + return; + } + const float digits = ImGui::CalcTextSize("00.00").x; + ImGui::TableSetupScrollFreeze(0, 1); + ImGui::TableSetupColumn("Pass", ImGuiTableColumnFlags_NoSort | ImGuiTableColumnFlags_WidthStretch, 0.0f, pass_column); + ImGui::TableSetupColumn( + "GPU ms", ImGuiTableColumnFlags_PreferSortDescending | ImGuiTableColumnFlags_WidthFixed, digits, gpu_column + ); + ImGui::TableSetupColumn( + "Peak", ImGuiTableColumnFlags_PreferSortDescending | ImGuiTableColumnFlags_WidthFixed, digits, peak_column + ); + ImGui::TableSetupColumn( + "CPU ms", ImGuiTableColumnFlags_PreferSortDescending | ImGuiTableColumnFlags_WidthFixed, digits, cpu_column + ); + ImGui::TableSetupColumn( + "Runs", + ImGuiTableColumnFlags_PreferSortDescending | ImGuiTableColumnFlags_WidthFixed, + ImGui::CalcTextSize("1 @ 100%").x, + runs_column + ); + if (has_baseline) { + ImGui::TableSetupColumn( + "Change", ImGuiTableColumnFlags_PreferSortDescending | ImGuiTableColumnFlags_WidthFixed, digits * 1.2f, delta_column + ); + } + ImGui::TableHeadersRow(); + + const auto idle = [](const PassStats& pass) { return pass.perFrame() < 0.005f && pass.cpu_avg * pass.presence < 0.005f; }; + std::vector order; + order.reserve(m_passes.size()); + for (size_t i = 0; i < m_passes.size(); ++i) { + if (!m_hide_idle_passes || !idle(m_passes[i])) { + order.push_back(i); + } + } + + // Flat when sorted since nesting only reads right in frame order + bool sorted = false; + if (const ImGuiTableSortSpecs* specs = ImGui::TableGetSortSpecs(); specs != nullptr && specs->SpecsCount > 0) { + const ImGuiTableColumnSortSpecs& spec = specs->Specs[0]; + const bool ascending = spec.SortDirection == ImGuiSortDirection_Ascending; + const auto key = [&](size_t i) -> float { + const PassStats& pass = m_passes[i]; + switch (spec.ColumnUserID) { + case peak_column: return pass.gpu_peak; + case cpu_column: return pass.cpu_avg; + case runs_column: return pass.runs * pass.presence; + case delta_column: return pass.baseline >= 0.0f ? pass.perFrame() - pass.baseline : pass.perFrame(); + default: return pass.perFrame(); + } + }; + std::ranges::stable_sort(order, [&](size_t a, size_t b) { return ascending ? key(a) < key(b) : key(a) > key(b); }); + sorted = true; + } + + for (const size_t i : order) { + const PassStats& pass = m_passes[i]; + const bool is_idle = idle(pass); + const bool absent = pass.presence <= 0.0f; + ImGui::TableNextRow(); + ImGui::PushID(static_cast(i)); + + ImGui::TableNextColumn(); + const int indent = sorted ? 0 : static_cast(pass.depth * 2); + if (is_idle || absent) { + ImGui::TextDisabled("%*s%s", indent, "", pass.name.c_str()); + } else { + ImGui::Text("%*s%s", indent, "", pass.name.c_str()); + } + if (ImGui::IsItemHovered()) { + ImGui::SetTooltip( + "%s\n" + "GPU %.3f ms when it runs, %.3f ms per frame\n" + "Peak %.3f ms (worst frame of the last 0.5 s)\n" + "CPU %.3f ms recording\n" + "Runs %.1f per frame, in %.0f%% of frames", + pass.name.c_str(), + pass.gpu_avg, + pass.perFrame(), + pass.gpu_peak, + pass.cpu_avg, + pass.runs, + pass.presence * 100.0f + ); + } + + ImGui::TableNextColumn(); + ImGui::Text("%5.2f", pass.perFrame()); + + // Peak well above average means a hitch rather than a steady cost + ImGui::TableNextColumn(); + const bool spiky = pass.gpu_peak > 0.25f && pass.gpu_peak > pass.gpu_avg * 2.0f; + if (spiky) { + ImGui::TextColored(k_perf_warn, "%5.2f", pass.gpu_peak); + } else { + ImGui::TextDisabled("%5.2f", pass.gpu_peak); + } + + ImGui::TableNextColumn(); + ImGui::Text("%5.2f", pass.cpu_avg * pass.presence); + + ImGui::TableNextColumn(); + if (absent) { + ImGui::TextDisabled("not run"); + } else if (pass.presence < 0.995f) { + ImGui::Text("%.0f @ %.0f%%", pass.runs, pass.presence * 100.0f); + } else { + ImGui::Text("%.0f", pass.runs); + } + + if (has_baseline) { + ImGui::TableNextColumn(); + if (pass.baseline < 0.0f) { + ImGui::TextColored(k_perf_warn, "new"); + } else { + const float delta = pass.perFrame() - pass.baseline; + if (std::abs(delta) < 0.02f) { + ImGui::TextDisabled(" ="); + } else { + ImGui::TextColored(delta < 0.0f ? k_perf_good : k_perf_bad, "%+.2f", delta); + } + } + } + ImGui::PopID(); + } + + // So the column sums to the queue total + if (!sorted && unscoped_ms >= 0.01f) { + ImGui::TableNextRow(); + ImGui::TableNextColumn(); + ImGui::TextDisabled("Unscoped"); + ImGui::TableNextColumn(); + ImGui::TextDisabled("%5.2f", unscoped_ms); + } + ImGui::EndTable(); +} + +void PerfWindow::drawMemoryTab() { + auto* renderer = VulkanRenderer::instance; + const VulkanCore& core = renderer->getCore(); + + ImGui::SeparatorText("GPU memory heaps"); + if (!core.isMemoryBudgetSupported()) { + ImGui::TextColored(k_perf_warn, "Estimated: VK_EXT_memory_budget is unavailable"); + } + const auto budgets = core.getAllocator().getHeapBudgets(); + const auto properties = core.getPhysicalDevice().getMemoryProperties(); + for (uint32_t heap = 0; heap < properties.memoryHeapCount && heap < budgets.size(); ++heap) { + const auto& budget = budgets[heap]; + if (budget.budget == 0) { + continue; + } + const bool device_local = + (properties.memoryHeaps[heap].flags & vk::MemoryHeapFlagBits::eDeviceLocal) != vk::MemoryHeapFlags {}; + const float fraction = std::clamp(static_cast(budget.usage) / static_cast(budget.budget), 0.0f, 1.0f); + + ImGui::Text("Heap %u %s", heap, device_local ? "device local" : "host"); + ImGui::SameLine(); + ImGui::TextDisabled("%.0f MiB total", perfMiB(properties.memoryHeaps[heap].size)); + + const std::string label = std::format("{:.0f} / {:.0f} MiB", perfMiB(budget.usage), perfMiB(budget.budget)); + ImGui::PushStyleColor(ImGuiCol_PlotHistogram, fraction < 0.75f ? k_perf_good : (fraction < 0.9f ? k_perf_warn : k_perf_bad)); + ImGui::ProgressBar(fraction, ImVec2(ImGui::GetFontSize() * 30.0f, 0.0f), label.c_str()); + ImGui::PopStyleColor(); + + // Usage beyond the VMA blocks is other processes and drivers + const auto& stats = budget.statistics; + const double packing = + stats.blockBytes > 0 ? static_cast(stats.allocationBytes) / static_cast(stats.blockBytes) * 100.0 : 0.0; + ImGui::TextDisabled( + " engine %.1f MiB in %u allocations, %u blocks of %.1f MiB (%.0f%% packed)", + perfMiB(stats.allocationBytes), + stats.allocationCount, + stats.blockCount, + perfMiB(stats.blockBytes), + packing + ); + } + + ImGui::SeparatorText("Uploads"); + const VulkanRenderer::UploadStats uploads = renderer->uploadStats(); + ImGui::Text("Transfer slots %u / %u in flight", uploads.slots_in_flight, uploads.slot_count); + ImGui::Text("Batches pending %zu", uploads.batches_in_flight); + ImGui::Text("Jobs waiting %zu", uploads.waiting_jobs); + const float staging = + uploads.host_budget > 0 + ? std::clamp(static_cast(uploads.host_bytes) / static_cast(uploads.host_budget), 0.0f, 1.0f) + : 0.0f; + const std::string staging_label = + std::format("staging {:.1f} / {:.0f} MiB", perfMiB(uploads.host_bytes), perfMiB(uploads.host_budget)); + ImGui::ProgressBar(staging, ImVec2(ImGui::GetFontSize() * 30.0f, 0.0f), staging_label.c_str()); + + ImGui::SeparatorText("Async compute"); + const AsyncCompute& compute = core.asyncCompute(); + const AsyncCompute::Ticket submitted = compute.lastSubmittedTicket(); + const AsyncCompute::Ticket completed = compute.completedTicket(); + ImGui::Text("Queue family %u", compute.queueFamilyIndex()); + ImGui::Text("Jobs in flight %llu", static_cast(submitted > completed ? submitted - completed : 0)); + ImGui::TextDisabled( + "Tickets: %llu submitted, %llu completed", + static_cast(submitted), + static_cast(completed) + ); +} + +void PerfWindow::drawSceneTab() { + const auto* frame = VulkanRenderer::instance->renderingFrame(); + if (frame == nullptr) { + ImGui::TextDisabled("No frame"); + return; + } + const auto visible_meshes = std::ranges::count_if(frame->mesh_instances, [](const auto& proxy) { return proxy.visible; }); + const auto visible_voxels = std::ranges::count_if(frame->voxel_instances, [](const auto& proxy) { return proxy.visible; }); + + if (ImGui::BeginTable("##perf_scene", 2, ImGuiTableFlags_SizingFixedFit)) { + const auto row = [](const char* label, const std::string& value) { + ImGui::TableNextRow(); + ImGui::TableNextColumn(); + ImGui::TextDisabled("%s", label); + ImGui::TableNextColumn(); + ImGui::TextUnformatted(value.c_str()); + }; + row("Mesh instances", std::format("{} visible of {}", visible_meshes, frame->mesh_instances.size())); + row("Material ranges", std::format("{}", frame->material_ranges.size())); + row("Voxel volumes", std::format("{} visible of {}", visible_voxels, frame->voxel_instances.size())); + row("Lights", std::format("{} of {} submitted", frame->lights.size(), frame->light_stats.submitted)); + ImGui::EndTable(); + } +} + +void PerfWindow::draw() { + ZoneScoped; + auto* renderer = VulkanRenderer::instance; + const GpuTimer* timer = renderer->gpuTimer(); + m_has_gpu = timer != nullptr && timer->last().valid; + + record(timer); + sample(renderer->perfCounters(), renderer->getSkippedBuildCount()); + + const auto cap = static_cast(renderer->effectiveFrameRateLimit()); + m_budget_ms = cap > 0.0f ? 1000.0f / cap : 1000.0f / 60.0f; + m_interval_ms = m_sample.render_fps > 0.0f ? 1000.0f / m_sample.render_fps : 0.0f; + // Published value so the text does not flicker + m_gpu_ms = m_graphics_avg; + + const ImGuiIO& io = ImGui::GetIO(); + ImGui::SetNextWindowPos(ImVec2(io.DisplaySize.x - 12.0f, 12.0f), ImGuiCond_FirstUseEver, ImVec2(1.0f, 0.0f)); + ImGui::SetNextWindowBgAlpha(0.88f); + constexpr ImGuiWindowFlags k_flags = + ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoFocusOnAppearing | ImGuiWindowFlags_NoNav; + + const bool open = ImGui::Begin("Performance", nullptr, k_flags); + if (!open || !m_has_sample) { + if (open) { + ImGui::TextDisabled("Sampling..."); + } + ImGui::End(); + return; + } + + drawSummary(m_has_gpu); + + if (ImGui::SmallButton(m_expanded ? "Hide details" : "Details")) { + m_expanded = !m_expanded; + } + + if (m_expanded && ImGui::BeginTabBar("##perf_tabs")) { + if (ImGui::BeginTabItem("Frame")) { + drawFrameTab(); + ImGui::EndTabItem(); + } + if (ImGui::BeginTabItem("GPU")) { + drawGpuTab(); + ImGui::EndTabItem(); + } + if (ImGui::BeginTabItem("Memory")) { + drawMemoryTab(); + ImGui::EndTabItem(); + } + if (ImGui::BeginTabItem("Scene")) { + drawSceneTab(); + ImGui::EndTabItem(); + } + ImGui::EndTabBar(); + } + + ImGui::End(); +} + +} diff --git a/engine/src/toast/renderer/passes/perf_window.hpp b/engine/src/toast/renderer/passes/perf_window.hpp new file mode 100644 index 000000000..d6261c620 --- /dev/null +++ b/engine/src/toast/renderer/passes/perf_window.hpp @@ -0,0 +1,148 @@ +/// @file perf_window.hpp +/// @author dario +/// @date 09/10/2026 + +#pragma once + +#include "../vulkan_renderer.hpp" + +#include +#include +#include +#include +#include + +namespace renderer { + +class PerfWindow { +public: + /// Render thread only + void draw(); + +private: + static constexpr size_t k_history = 240; + using History = std::array; + + /// Rates and per frame averages over the last sample interval + struct Sample { + float render_fps = 0.0f; + float game_fps = 0.0f; + float tick_rate = 0.0f; + float skipped_per_second = 0.0f; + float repeated_ratio = 0.0f; + float wait_ratio = 0.0f; + + float draw_ms = 0.0f; + float upload_ms = 0.0f; + float gpu_wait_ms = 0.0f; + float acquire_ms = 0.0f; + float record_ms = 0.0f; + float submit_ms = 0.0f; + float present_ms = 0.0f; + float frame_wait_ms = 0.0f; + float pacing_ms = 0.0f; + + float build_ms = 0.0f; + float slot_wait_ms = 0.0f; + + float p99_interval_ms = 0.0f; + float max_interval_ms = 0.0f; + }; + + struct Phase { + const char* label; + float ms; + uint32_t color; + }; + + /// Kept across frames so rows never jump and published once per sample interval + struct PassStats { + std::string name; + uint32_t depth = 0; + std::chrono::steady_clock::time_point last_seen; + + // Accumulated since the last publish + double gpu_sum = 0.0; + double cpu_sum = 0.0; + double gpu_peak_acc = 0.0; + uint32_t runs_sum = 0; + uint32_t frames_run = 0; + + // Published values the table shows + float gpu_avg = 0.0f; ///< per frame it ran + float gpu_peak = 0.0f; ///< worst single frame + float cpu_avg = 0.0f; + float runs = 0.0f; ///< per frame it ran + float presence = 0.0f; ///< share of frames it ran in + float baseline = -1.0f; ///< per frame cost at baseline and negative when unset + + /// Frames it did not run count as zero + [[nodiscard]] + auto perFrame() const -> float { + return gpu_avg * presence; + } + }; + + struct Verdict { + const char* label = ""; + bool bad = false; + bool warn = false; + std::string detail; + }; + + void record(const GpuTimer* timer); + void accumulatePasses(const GpuTimer::Timings& frame); + void publishPasses(); + [[nodiscard]] + auto heaviestPass(bool by_gpu) const -> const PassStats*; + void sample(const VulkanRenderer::PerfCounters& counters, uint32_t skipped); + + [[nodiscard]] + auto phases() const -> std::array; + [[nodiscard]] + auto renderVerdict(bool has_gpu) const -> Verdict; + [[nodiscard]] + auto gameVerdict() const -> Verdict; + + void drawSummary(bool has_gpu); + void drawFrameGraph(float width, float height); + void drawPhaseBar(float width, float height) const; + void drawFrameTab(); + void drawGpuTab(); + void drawMemoryTab(); + void drawSceneTab(); + + [[nodiscard]] + auto historyOffset() const -> int; + + std::chrono::steady_clock::time_point m_last_sample; + VulkanRenderer::PerfCounters m_last_counters {}; + uint32_t m_last_skipped = 0; + bool m_has_sample = false; + Sample m_sample; + + History m_interval_history {}; + History m_gpu_history {}; + /// Frame order with new passes inserted after their predecessor + std::vector m_passes; + uint64_t m_last_gpu_serial = 0; + uint32_t m_window_frames = 0; + double m_graphics_sum = 0.0; + float m_graphics_avg = 0.0f; + float m_graphics_baseline = -1.0f; + size_t m_cursor = 0; + size_t m_filled = 0; + + /// Eased so spikes do not rescale the graph + float m_graph_scale_ms = 0.0f; + /// From the frame rate cap or 60 fps when uncapped + float m_budget_ms = 1000.0f / 60.0f; + float m_interval_ms = 0.0f; + float m_gpu_ms = 0.0f; + + bool m_expanded = false; + bool m_hide_idle_passes = true; + bool m_has_gpu = false; +}; + +} diff --git a/engine/src/toast/renderer/passes/reflection_probe_pass.cpp b/engine/src/toast/renderer/passes/reflection_probe_pass.cpp index d9272583b..1b87c1328 100644 --- a/engine/src/toast/renderer/passes/reflection_probe_pass.cpp +++ b/engine/src/toast/renderer/passes/reflection_probe_pass.cpp @@ -211,9 +211,7 @@ void ReflectionProbePass::captureFace( return; } - m_staging.cube.transition( - cmd, vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits::eTransferWrite, vk::PipelineStageFlagBits::eTransfer - ); + m_staging.cube.transition(cmd, sync::Usage::transfer_dst); vk::ImageBlit blit {}; blit.srcSubresource = vk::ImageSubresourceLayers(vk::ImageAspectFlagBits::eColor, 0, 0, 1); @@ -246,9 +244,7 @@ void ReflectionProbePass::captureIrradianceFace( return; } - m_staging.cube.transition( - cmd, vk::ImageLayout::eTransferDstOptimal, vk::AccessFlagBits::eTransferWrite, vk::PipelineStageFlagBits::eTransfer - ); + m_staging.cube.transition(cmd, sync::Usage::transfer_dst); vk::ImageBlit blit {}; blit.srcSubresource = vk::ImageSubresourceLayers(vk::ImageAspectFlagBits::eColor, 0, 0, 1); @@ -273,9 +269,7 @@ void ReflectionProbePass::projectStagingToSh(vk::CommandBuffer cmd, uint32_t pro return; } - m_staging.cube.transition( - cmd, vk::ImageLayout::eShaderReadOnlyOptimal, vk::AccessFlagBits::eShaderRead, vk::PipelineStageFlagBits::eComputeShader - ); + m_staging.cube.transition(cmd, sync::Usage::compute_sampled); struct ShParams { glm::uvec4 config {0}; @@ -290,18 +284,9 @@ void ReflectionProbePass::projectStagingToSh(vk::CommandBuffer cmd, uint32_t pro cmd.pushConstants(*m_sh_layout.getPipelineLayout(), vk::ShaderStageFlagBits::eAll, 0, sizeof(ShParams), ¶ms); cmd.dispatch(1, 1, 1); - const vk::BufferMemoryBarrier barrier( - vk::AccessFlagBits::eShaderWrite, - vk::AccessFlagBits::eShaderRead, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - **m_sh_buffer, - 0, - VK_WHOLE_SIZE - ); - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eComputeShader, vk::PipelineStageFlagBits::eFragmentShader, {}, nullptr, barrier, nullptr - ); + sync::BarrierBatch barriers; + barriers.buffer(**m_sh_buffer, sync::Usage::compute_storage_write, sync::Usage::fragment_storage_read); + barriers.flush(cmd); } auto ReflectionProbePass::getShBuffer() const -> vk::Buffer { @@ -400,9 +385,12 @@ auto ReflectionProbePass::loadShRange(uint32_t base, uint32_t count, std::string cmd.begin(vk::CommandBufferBeginInfo(vk::CommandBufferUsageFlagBits::eOneTimeSubmit)); const vk::BufferCopy copy(0, static_cast(base) * 4 * sizeof(glm::vec4), coefficient_bytes); cmd.copyBuffer(*staging, **m_sh_buffer, copy); + sync::BarrierBatch barriers; + barriers.buffer(**m_sh_buffer, sync::Usage::transfer_dst, sync::Usage::fragment_storage_read); + barriers.flush(*cmd); cmd.end(); - submitAndWait(device, m_core->getGraphicsQueue(), *cmd); + submitAndWait(device, m_core->getGraphicsQueue(), m_core->graphicsSubmitMutex(), *cmd); TOAST_INFO("Render", "Loaded {} irradiance probe(s) from {}", count, uri); return true; @@ -518,15 +506,8 @@ void ReflectionProbePass::prefilterInto(vk::CommandBuffer cmd, uint32_t probe) { return; } - m_staging.cube.transition( - cmd, vk::ImageLayout::eShaderReadOnlyOptimal, vk::AccessFlagBits::eShaderRead, vk::PipelineStageFlagBits::eFragmentShader - ); - cube.cube.transition( - cmd, - vk::ImageLayout::eColorAttachmentOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::PipelineStageFlagBits::eColorAttachmentOutput - ); + m_staging.cube.transition(cmd, sync::Usage::fragment_sampled); + cube.cube.transition(cmd, sync::Usage::color_attachment); for (uint32_t mip = 0; mip < m_mip_levels; ++mip) { const uint32_t mip_size = std::max(cube.cube.size() >> mip, 1U); @@ -536,9 +517,7 @@ void ReflectionProbePass::prefilterInto(vk::CommandBuffer cmd, uint32_t probe) { } } - cube.cube.transition( - cmd, vk::ImageLayout::eShaderReadOnlyOptimal, vk::AccessFlagBits::eShaderRead, vk::PipelineStageFlagBits::eFragmentShader - ); + cube.cube.transition(cmd, sync::Usage::fragment_sampled); cube.baked = true; TOAST_INFO("Render", "Probe {} baked", probe); } @@ -550,20 +529,13 @@ void ReflectionProbePass::convolveIrradianceInto(vk::CommandBuffer cmd, uint32_t return; } - cube.cube.transition( - cmd, - vk::ImageLayout::eColorAttachmentOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::PipelineStageFlagBits::eColorAttachmentOutput - ); + cube.cube.transition(cmd, sync::Usage::color_attachment); for (uint32_t face = 0; face < 6; ++face) { renderFace(cmd, m_irradiance_pipeline, cube, k_irradiance_size, 0, face, 0.0f); } - cube.cube.transition( - cmd, vk::ImageLayout::eShaderReadOnlyOptimal, vk::AccessFlagBits::eShaderRead, vk::PipelineStageFlagBits::eFragmentShader - ); + cube.cube.transition(cmd, sync::Usage::fragment_sampled); cube.baked = true; } @@ -670,19 +642,7 @@ auto ReflectionProbePass::readbackCube(ProbeCube& cube, std::vector& ou cmd.begin(vk::CommandBufferBeginInfo(vk::CommandBufferUsageFlagBits::eOneTimeSubmit)); - const vk::ImageSubresourceRange range(vk::ImageAspectFlagBits::eColor, 0, cube.cube.mipLevels(), 0, 6); - vk::ImageMemoryBarrier to_src {}; - to_src.oldLayout = cube.cube.layout(); - to_src.newLayout = vk::ImageLayout::eTransferSrcOptimal; - to_src.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - to_src.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - to_src.image = cube.cube.image(); - to_src.subresourceRange = range; - to_src.srcAccessMask = vk::AccessFlagBits::eShaderRead; - to_src.dstAccessMask = vk::AccessFlagBits::eTransferRead; - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eFragmentShader, vk::PipelineStageFlagBits::eTransfer, {}, nullptr, nullptr, to_src - ); + cube.cube.transition(*cmd, sync::Usage::transfer_src); std::vector regions; regions.reserve(cube.cube.mipLevels()); @@ -698,19 +658,15 @@ auto ReflectionProbePass::readbackCube(ProbeCube& cube, std::vector& ou } cmd.copyImageToBuffer(cube.cube.image(), vk::ImageLayout::eTransferSrcOptimal, *staging, regions); - vk::ImageMemoryBarrier back = to_src; - back.oldLayout = vk::ImageLayout::eTransferSrcOptimal; - back.newLayout = vk::ImageLayout::eShaderReadOnlyOptimal; - back.srcAccessMask = vk::AccessFlagBits::eTransferRead; - back.dstAccessMask = vk::AccessFlagBits::eShaderRead; - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eFragmentShader, {}, nullptr, nullptr, back - ); - cube.cube.setLayout(vk::ImageLayout::eShaderReadOnlyOptimal); + // A fence wait alone does not make device writes visible to the host + sync::BarrierBatch barriers; + barriers.buffer(*staging, sync::Usage::transfer_dst, sync::Usage::host_read); + barriers.flush(*cmd); + cube.cube.transition(*cmd, sync::Usage::fragment_sampled); cmd.end(); - submitAndWait(device, m_core->getGraphicsQueue(), *cmd); + submitAndWait(device, m_core->getGraphicsQueue(), m_core->graphicsSubmitMutex(), *cmd); staging.getAllocation().invalidate(0, bytes); const auto* mapped = static_cast(staging.getAllocation().getInfo().pMappedData); @@ -776,11 +732,11 @@ auto ReflectionProbePass::uploadCube(ProbeCube& cube, const std::vector cmd.copyBufferToImage(*staging, cube.cube.image(), vk::ImageLayout::eTransferDstOptimal, regions); recordTransferDstToShaderRead(cmd, cube.cube.image(), range); - cube.cube.setLayout(vk::ImageLayout::eShaderReadOnlyOptimal); + cube.cube.setUsage(sync::Usage::shader_sampled); cmd.end(); - submitAndWait(device, m_core->getGraphicsQueue(), *cmd); + submitAndWait(device, m_core->getGraphicsQueue(), m_core->graphicsSubmitMutex(), *cmd); return true; } @@ -799,28 +755,16 @@ void ReflectionProbePass::recordPre(vk::CommandBuffer cmd, uint32_t frame_index, // Bound cubemaps must not be eUndefined even when unbaked for (auto& probe : m_probes) { if (probe.cube.isReady()) { - probe.cube.transition( - cmd, - vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits::eShaderRead, - vk::PipelineStageFlagBits::eFragmentShader - ); + probe.cube.transition(cmd, sync::Usage::fragment_sampled); } } for (auto& irradiance : m_probe_irradiance) { if (irradiance.cube.isReady()) { - irradiance.cube.transition( - cmd, - vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits::eShaderRead, - vk::PipelineStageFlagBits::eFragmentShader - ); + irradiance.cube.transition(cmd, sync::Usage::fragment_sampled); } } if (m_staging.cube.isReady()) { - m_staging.cube.transition( - cmd, vk::ImageLayout::eShaderReadOnlyOptimal, vk::AccessFlagBits::eShaderRead, vk::PipelineStageFlagBits::eFragmentShader - ); + m_staging.cube.transition(cmd, sync::Usage::fragment_sampled); } if ((m_pending_saves.empty() && m_pending_sh_saves.empty()) || m_core == nullptr) { @@ -886,10 +830,16 @@ void ReflectionProbePass::recordPre(vk::CommandBuffer cmd, uint32_t frame_index, readback.begin(vk::CommandBufferBeginInfo(vk::CommandBufferUsageFlagBits::eOneTimeSubmit)); const vk::BufferCopy copy(static_cast(save.base) * 4 * sizeof(glm::vec4), 0, coefficient_bytes); + sync::BarrierBatch barriers; + barriers.buffer(**m_sh_buffer, sync::Usage::compute_storage_write, sync::Usage::transfer_src); + barriers.flush(*readback); readback.copyBuffer(**m_sh_buffer, *staging, copy); + // A fence wait alone does not make device writes visible to the host + barriers.buffer(*staging, sync::Usage::transfer_dst, sync::Usage::host_read); + barriers.flush(*readback); readback.end(); - submitAndWait(device, m_core->getGraphicsQueue(), *readback); + submitAndWait(device, m_core->getGraphicsQueue(), m_core->graphicsSubmitMutex(), *readback); staging.getAllocation().invalidate(0, coefficient_bytes); ShFileHeader header {}; diff --git a/engine/src/toast/renderer/passes/shadow_pass.cpp b/engine/src/toast/renderer/passes/shadow_pass.cpp index da9196aa1..b58a153ac 100644 --- a/engine/src/toast/renderer/passes/shadow_pass.cpp +++ b/engine/src/toast/renderer/passes/shadow_pass.cpp @@ -122,6 +122,8 @@ ShadowPass::ShadowPass(const VulkanCore& core) : m_core(&core) { config.vertex_attributes = {vertexAttributeDescriptions()[0]}; config.depth_test = true; config.depth_write = true; + // Shadow maps keep standard depth whatever the scene convention + config.depth_compare = vk::CompareOp::eLess; config.depth_bias_constant = 1.5f; config.depth_bias_slope = 3.0f; config.cull_mode = vk::CullModeFlagBits::eNone; @@ -187,7 +189,6 @@ void ShadowPass::createShadowMap( map.resolution = resolution; map.layer_count = layer_count; - map.layout = vk::ImageLayout::eUndefined; vk::ImageCreateInfo image_ci {}; image_ci.imageType = vk::ImageType::e2D; @@ -206,6 +207,7 @@ void ShadowPass::createShadowMap( map.image.emplace(core.getAllocator().createImage(image_ci, allocation_ci)); setDebugName(core, **map.image, std::string(debug_name)); + map.state.reset(**map.image, shadowLayerRange(0, layer_count)); vk::ImageViewCreateInfo array_view_ci {}; array_view_ci.image = **map.image; @@ -352,6 +354,8 @@ void ShadowPass::createVoxelResources(const VulkanCore& core) { config.topology = vk::PrimitiveTopology::eTriangleList; config.depth_test = true; config.depth_write = true; + // Shadow maps keep standard depth whatever the scene convention + config.depth_compare = vk::CompareOp::eLess; for (size_t mask_index = 0; mask_index < m_voxel_pipeline_sets.size(); ++mask_index) { VoxelPipelineSet& set = m_voxel_pipeline_sets[mask_index]; @@ -465,6 +469,18 @@ auto ShadowPass::prepareVoxels(uint32_t frame_index) -> uint32_t { // Every proxy not only camera visible ones since an off screen volume still casts into view const auto count = static_cast(std::min(frame->voxel_instances.size(), k_max_voxel_casters)); + if (frame->voxel_instances.size() > k_max_voxel_casters) { + const auto now = std::chrono::steady_clock::now(); + if (now - m_last_caster_limit_warning > std::chrono::seconds(5)) { + m_last_caster_limit_warning = now; + TOAST_WARN( + "Render", + "ShadowPass: {} voxel casters, only the first {} cast shadows", + frame->voxel_instances.size(), + k_max_voxel_casters + ); + } + } for (uint32_t i = 0; i < count; ++i) { const auto& proxy = frame->voxel_instances[i]; // Shadows never sample motion so whether a real previous transform exists does not matter here @@ -508,7 +524,7 @@ void ShadowPass::recordMap(vk::CommandBuffer cmd, ShadowMap& map, uint32_t frame } // MaterialPass still binds these and an eUndefined image is invalid even unread - if (frame->frame_data.traced_shadow_params.x >= 0.5f && map.layout == vk::ImageLayout::eShaderReadOnlyOptimal) { + if (frame->frame_data.traced_shadow_params.x >= 0.5f && map.state.usage == sync::Usage::fragment_sampled) { return; } @@ -659,27 +675,8 @@ void ShadowPass::recordMap(vk::CommandBuffer cmd, ShadowMap& map, uint32_t frame return; } - // From the tracked layout so skipped groups keep their depth - const vk::ImageMemoryBarrier to_attachment( - map.layout == vk::ImageLayout::eShaderReadOnlyOptimal ? vk::AccessFlagBits::eShaderRead : vk::AccessFlags {}, - vk::AccessFlagBits::eDepthStencilAttachmentWrite, - map.layout, - vk::ImageLayout::eDepthAttachmentOptimal, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - **map.image, - shadowLayerRange(0, map.layer_count) - ); - cmd.pipelineBarrier( - map.layout == vk::ImageLayout::eShaderReadOnlyOptimal ? vk::PipelineStageFlagBits::eFragmentShader - : vk::PipelineStageFlagBits::eTopOfPipe, - vk::PipelineStageFlagBits::eEarlyFragmentTests, - {}, - nullptr, - nullptr, - to_attachment - ); - map.layout = vk::ImageLayout::eDepthAttachmentOptimal; + // From the tracked usage so skipped groups keep their depth + sync::transition(cmd, map.state, sync::Usage::depth_attachment); const vk::Rect2D layer_area({0, 0}, vk::Extent2D {map.resolution, map.resolution}); @@ -844,20 +841,8 @@ void ShadowPass::recordMap(vk::CommandBuffer cmd, ShadowMap& map, uint32_t frame cmd.endRendering(); } - const vk::ImageMemoryBarrier to_sampled( - vk::AccessFlagBits::eDepthStencilAttachmentWrite, - vk::AccessFlagBits::eShaderRead, - vk::ImageLayout::eDepthAttachmentOptimal, - vk::ImageLayout::eShaderReadOnlyOptimal, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - **map.image, - shadowLayerRange(0, map.layer_count) - ); - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eLateFragmentTests, vk::PipelineStageFlagBits::eFragmentShader, {}, nullptr, nullptr, to_sampled - ); - map.layout = vk::ImageLayout::eShaderReadOnlyOptimal; + // Bound in eShaderReadOnlyOptimal by the lighting descriptors + sync::transition(cmd, map.state, sync::Usage::fragment_sampled); } void ShadowPass::recordPre(vk::CommandBuffer cmd, uint32_t frame_index, uint32_t image_index) { diff --git a/engine/src/toast/renderer/passes/shadow_pass.hpp b/engine/src/toast/renderer/passes/shadow_pass.hpp index 9dca23876..28ad1fe6f 100644 --- a/engine/src/toast/renderer/passes/shadow_pass.hpp +++ b/engine/src/toast/renderer/passes/shadow_pass.hpp @@ -14,6 +14,7 @@ #include "../vulkan_pipeline.hpp" #include +#include #include #include #include @@ -102,6 +103,8 @@ class ShadowPass : public IRenderPass { }; static constexpr uint32_t k_max_voxel_casters = 4096; + /// Throttles the over limit warning + std::chrono::steady_clock::time_point m_last_caster_limit_warning {}; struct LayerGroup { uint32_t base_layer = 0; @@ -124,7 +127,7 @@ class ShadowPass : public IRenderPass { std::optional image; std::optional array_view; std::vector groups; - vk::ImageLayout layout = vk::ImageLayout::eUndefined; + sync::ImageState state; uint32_t resolution = 0; uint32_t layer_count = 0; }; diff --git a/engine/src/toast/renderer/passes/voxel_pass.cpp b/engine/src/toast/renderer/passes/voxel_pass.cpp index ae2d444eb..6b9359ef7 100644 --- a/engine/src/toast/renderer/passes/voxel_pass.cpp +++ b/engine/src/toast/renderer/passes/voxel_pass.cpp @@ -274,6 +274,12 @@ auto VoxelPass::prepare(uint32_t frame_index) -> bool { continue; } if (m_draws.size() >= k_max_instances) { + const auto now = std::chrono::steady_clock::now(); + if (now - m_last_limit_warning > std::chrono::seconds(5)) { + m_last_limit_warning = now; + const auto visible = std::ranges::count_if(frame->voxel_instances, [](const auto& p) { return p.visible; }); + TOAST_WARN("Render", "VoxelPass: {} visible volumes, only the first {} are drawn", visible, k_max_instances); + } break; } diff --git a/engine/src/toast/renderer/passes/voxel_pass.hpp b/engine/src/toast/renderer/passes/voxel_pass.hpp index 7bd3c5ee7..97ecaa0c2 100644 --- a/engine/src/toast/renderer/passes/voxel_pass.hpp +++ b/engine/src/toast/renderer/passes/voxel_pass.hpp @@ -5,12 +5,14 @@ */ #pragma once + #include "../render_pass_base.hpp" #include "../scene_descriptor_set.hpp" #include "../shader_layout.hpp" #include "../vulkan_pipeline.hpp" #include +#include #include #include #include @@ -51,6 +53,9 @@ class VoxelPass : public IRenderPass { static constexpr uint32_t k_max_instances = 4096; private: + /// Throttles the over limit warning + std::chrono::steady_clock::time_point m_last_limit_warning {}; + /// Mirrors voxel.slang PushConstants struct PushConstants { uint32_t instance_index = 0; diff --git a/engine/src/toast/renderer/post_process_target.cpp b/engine/src/toast/renderer/post_process_target.cpp index 62c7eb79f..b48b66d67 100644 --- a/engine/src/toast/renderer/post_process_target.cpp +++ b/engine/src/toast/renderer/post_process_target.cpp @@ -21,7 +21,7 @@ void PostProcessTarget::create(const VulkanCore& core, vk::Extent2D extent, vk:: m_view.reset(); m_image.reset(); m_extent = extent; - m_layout = vk::ImageLayout::eUndefined; + m_state = {}; const auto image_ci = colorTargetImageInfo(extent, format); @@ -30,6 +30,7 @@ void PostProcessTarget::create(const VulkanCore& core, vk::Extent2D extent, vk:: m_image.emplace(core.getAllocator().createImage(image_ci, allocation_ci)); setDebugName(core, **m_image, std::format("{} Target", debug_name)); + m_state.reset(**m_image, colorSubresourceRange()); vk::ImageViewCreateInfo view_ci {}; view_ci.image = **m_image; @@ -45,27 +46,7 @@ void PostProcessTarget::beginScope(vk::CommandBuffer cmd) const { return; } - const bool was_sampled = m_layout == vk::ImageLayout::eShaderReadOnlyOptimal; - - const vk::ImageMemoryBarrier to_attachment( - was_sampled ? vk::AccessFlagBits::eShaderRead : vk::AccessFlags {}, - vk::AccessFlagBits::eColorAttachmentWrite, - m_layout, - vk::ImageLayout::eColorAttachmentOptimal, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - **m_image, - colorSubresourceRange() - ); - cmd.pipelineBarrier( - was_sampled ? vk::PipelineStageFlagBits::eFragmentShader : vk::PipelineStageFlagBits::eTopOfPipe, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - {}, - nullptr, - nullptr, - to_attachment - ); - m_layout = vk::ImageLayout::eColorAttachmentOptimal; + sync::transition(cmd, m_state, sync::Usage::color_attachment); vk::RenderingAttachmentInfo attachment {}; attachment.imageView = **m_view; @@ -93,25 +74,8 @@ void PostProcessTarget::endScope(vk::CommandBuffer cmd) const { cmd.endRendering(); - const vk::ImageMemoryBarrier to_sampled( - vk::AccessFlagBits::eColorAttachmentWrite, - vk::AccessFlagBits::eShaderRead, - vk::ImageLayout::eColorAttachmentOptimal, - vk::ImageLayout::eShaderReadOnlyOptimal, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - **m_image, - colorSubresourceRange() - ); - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eColorAttachmentOutput, - vk::PipelineStageFlagBits::eFragmentShader, - {}, - nullptr, - nullptr, - to_sampled - ); - m_layout = vk::ImageLayout::eShaderReadOnlyOptimal; + // Auto exposure reads it from compute too + sync::transition(cmd, m_state, sync::Usage::shader_sampled); } } diff --git a/engine/src/toast/renderer/post_process_target.hpp b/engine/src/toast/renderer/post_process_target.hpp index f263b8d3a..21e723304 100644 --- a/engine/src/toast/renderer/post_process_target.hpp +++ b/engine/src/toast/renderer/post_process_target.hpp @@ -52,7 +52,7 @@ class PostProcessTarget { std::optional m_view; vk::Extent2D m_extent {}; - mutable vk::ImageLayout m_layout = vk::ImageLayout::eUndefined; + mutable sync::ImageState m_state; }; } diff --git a/engine/src/toast/renderer/sdl_output_target.cpp b/engine/src/toast/renderer/sdl_output_target.cpp index 16ea83ab0..a884ab2df 100644 --- a/engine/src/toast/renderer/sdl_output_target.cpp +++ b/engine/src/toast/renderer/sdl_output_target.cpp @@ -4,6 +4,8 @@ #include "sdl_output_target.hpp" +#include "vulkan_common.hpp" + #include #include #include @@ -100,18 +102,13 @@ auto SDLOutputTarget::present(uint32_t image_index, vk::Semaphore render_finishe auto SDLOutputTarget::recordFinalize(vk::CommandBuffer command_buffer, uint32_t image_index) -> void { ZoneScoped; - const vk::ImageMemoryBarrier barrier( - vk::AccessFlagBits::eColorAttachmentWrite, - vk::AccessFlags {}, - vk::ImageLayout::eColorAttachmentOptimal, - vk::ImageLayout::ePresentSrcKHR, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, + // VulkanRenderer records this image as sync::Usage::present afterwards + sync::transition( + command_buffer, m_swapchain->getImage(image_index), - vk::ImageSubresourceRange {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1} - ); - command_buffer.pipelineBarrier( - vk::PipelineStageFlagBits::eColorAttachmentOutput, vk::PipelineStageFlagBits::eBottomOfPipe, {}, {}, {}, barrier + colorSubresourceRange(), + sync::Usage::color_attachment, + sync::Usage::present ); } diff --git a/engine/src/toast/renderer/shader_compiler.cpp b/engine/src/toast/renderer/shader_compiler.cpp index a722e2153..64517d4eb 100644 --- a/engine/src/toast/renderer/shader_compiler.cpp +++ b/engine/src/toast/renderer/shader_compiler.cpp @@ -4,6 +4,7 @@ #include "shader_compiler.hpp" +#include "depth_convention.hpp" #include "slang_vfs.hpp" #include @@ -46,7 +47,9 @@ auto ShaderCompiler::featureHash() -> uint64_t { #else constexpr uint64_t k_build_profile = 0xd3b09c0de5a11e01ull; #endif - return (ray_query_available ? 0x9e3779b97f4a7c15ull : 0x0ull) ^ k_build_profile; + // TOAST_REVERSED_Z changes the SPIR-V so the other convention must read as stale + constexpr uint64_t k_depth_convention = depth::k_reversed_z ? 0x5eed2e7e45edull : 0x0ull; + return (ray_query_available ? 0x9e3779b97f4a7c15ull : 0x0ull) ^ k_build_profile ^ k_depth_convention; } static auto createSession() -> Slang::ComPtr { @@ -94,6 +97,14 @@ static auto createSession() -> Slang::ComPtr { ray_query_entry.value.stringValue1 = ray_query_available ? "1" : "0"; compiler_options.push_back(ray_query_entry); + // Always defined to 0 or 1 + slang::CompilerOptionEntry reversed_z_entry {}; + reversed_z_entry.name = slang::CompilerOptionName::MacroDefine; + reversed_z_entry.value.kind = slang::CompilerOptionValueKind::String; + reversed_z_entry.value.stringValue0 = "TOAST_REVERSED_Z"; + reversed_z_entry.value.stringValue1 = depth::k_reversed_z ? "1" : "0"; + compiler_options.push_back(reversed_z_entry); + slang::SessionDesc session_desc {}; session_desc.targets = slang_targets.data(); session_desc.targetCount = SlangInt(slang_targets.size()); diff --git a/engine/src/toast/renderer/shared_texture_output_target.cpp b/engine/src/toast/renderer/shared_texture_output_target.cpp index 44342d0a4..821a5ddc6 100644 --- a/engine/src/toast/renderer/shared_texture_output_target.cpp +++ b/engine/src/toast/renderer/shared_texture_output_target.cpp @@ -4,6 +4,7 @@ #include "shared_texture_output_target.hpp" +#include "vulkan_common.hpp" #include "vulkan_core.hpp" #include "vulkan_debug.hpp" @@ -121,19 +122,8 @@ void SharedTextureOutputTarget::recordFinalize(vk::CommandBuffer command_buffer, const vk::Image image = **shared.image; const vk::Buffer staging = **shared.staging; - const vk::ImageMemoryBarrier to_transfer( - vk::AccessFlagBits::eColorAttachmentWrite, - vk::AccessFlagBits::eTransferRead, - vk::ImageLayout::eColorAttachmentOptimal, - vk::ImageLayout::eTransferSrcOptimal, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - image, - colorSubresourceRange() - ); - command_buffer.pipelineBarrier( - vk::PipelineStageFlagBits::eColorAttachmentOutput, vk::PipelineStageFlagBits::eTransfer, {}, {}, {}, to_transfer - ); + // VulkanRenderer records this image as sync::Usage::transfer_src afterwards + sync::transition(command_buffer, image, colorSubresourceRange(), sync::Usage::color_attachment, sync::Usage::transfer_src); const vk::BufferImageCopy region( 0, @@ -145,16 +135,9 @@ void SharedTextureOutputTarget::recordFinalize(vk::CommandBuffer command_buffer, ); command_buffer.copyImageToBuffer(image, vk::ImageLayout::eTransferSrcOptimal, staging, region); - const vk::BufferMemoryBarrier to_host( - vk::AccessFlagBits::eTransferWrite, - vk::AccessFlagBits::eHostRead, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - staging, - 0, - VK_WHOLE_SIZE - ); - command_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eHost, {}, {}, to_host, {}); + sync::BarrierBatch to_host; + to_host.buffer(staging, sync::Usage::transfer_dst, sync::Usage::host_read); + to_host.flush(command_buffer); } void SharedTextureOutputTarget::onImageRenderComplete(uint32_t image_index) { diff --git a/engine/src/toast/renderer/voxel_debug.cpp b/engine/src/toast/renderer/voxel_debug.cpp index f3ac7efdf..c87673cd6 100644 --- a/engine/src/toast/renderer/voxel_debug.cpp +++ b/engine/src/toast/renderer/voxel_debug.cpp @@ -1,5 +1,6 @@ #include "voxel_debug.hpp" +#include "depth_convention.hpp" #include "passes/voxel_pass.hpp" #include "voxel_gpu_storage.hpp" @@ -278,7 +279,7 @@ auto probe(const VulkanRenderer::RenderFrame& frame, glm::vec2 cursor) -> std::o const glm::vec4 point = inverse_view_projection * glm::vec4(ndc, depth, 1.0f); return glm::vec3(point) / point.w; }; - const glm::vec3 near_point = unproject(0.0f); + const glm::vec3 near_point = unproject(depth::k_near); // Half depth since an infinite projection puts the far plane at w = 0 const glm::vec3 direction = unproject(0.5f) - near_point; diff --git a/engine/src/toast/renderer/voxel_gpu_storage.cpp b/engine/src/toast/renderer/voxel_gpu_storage.cpp index 5d71bf514..e0d83f1bd 100644 --- a/engine/src/toast/renderer/voxel_gpu_storage.cpp +++ b/engine/src/toast/renderer/voxel_gpu_storage.cpp @@ -174,6 +174,33 @@ void VoxelSceneUpload::build(const VulkanCore& core) { m_packed.reset(); } +namespace { + +/// eConcurrent sections so the hand off is visibility only. General since shaders copies and patches all read them +auto voxelHandoff(sync::QueueHandoff handoff) -> sync::QueueHandoff { + handoff.exclusive = false; + return handoff; +} + +void releaseSections(sync::BarrierBatch& batch, const VoxelGpuStorage& storage, const sync::QueueHandoff& handoff) { + for (size_t i = 0; i < VoxelGpuStorage::k_section_count; ++i) { + batch.release( + storage.buffer(static_cast(i)), + sync::Usage::transfer_dst, + sync::Usage::general, + voxelHandoff(handoff) + ); + } +} + +void acquireSections(sync::BarrierBatch& batch, const VoxelGpuStorage& storage, const sync::QueueHandoff& handoff) { + for (size_t i = 0; i < VoxelGpuStorage::k_section_count; ++i) { + batch.acquire(storage.buffer(static_cast(i)), sync::Usage::general, voxelHandoff(handoff)); + } +} + +} + void VoxelSceneUpload::record(vk::CommandBuffer cmd) { ZoneScoped; if (m_storage->hasFailed()) { @@ -186,9 +213,20 @@ void VoxelSceneUpload::record(vk::CommandBuffer cmd) { } cmd.copyBuffer(*m_staging, **m_storage->m_buffers[i], vk::BufferCopy(m_offsets[i], 0, m_bytes[i])); } + + sync::BarrierBatch barriers; + releaseSections(barriers, *m_storage, m_handoff); + barriers.flush(cmd); + m_released = true; m_storage->markRecorded(); } +void VoxelSceneUpload::recordAcquire(sync::BarrierBatch& batch) { + if (m_released && !m_storage->hasFailed()) { + acquireSections(batch, *m_storage, m_handoff); + } +} + VoxelScenePatchUpload::VoxelScenePatchUpload( std::shared_ptr storage, std::shared_ptr source, voxel::gpu::ScenePatch patch, SectionBytes section_bytes @@ -260,10 +298,9 @@ void VoxelScenePatchUpload::record(vk::CommandBuffer cmd) { }; // The copies that filled the source may still be running and these read it - const vk::MemoryBarrier source_ready(vk::AccessFlagBits::eTransferWrite, vk::AccessFlagBits::eTransferRead); - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eTransfer, {}, source_ready, nullptr, nullptr - ); + sync::BarrierBatch barriers; + barriers.memory(sync::Usage::transfer_dst, sync::Usage::transfer_src); + barriers.flush(cmd); for (const Section section : k_sections) { const vk::DeviceSize bytes = std::min(m_source->size(section), m_storage->size(section)); @@ -273,10 +310,8 @@ void VoxelScenePatchUpload::record(vk::CommandBuffer cmd) { } // The patch overwrites what was just carried over - const vk::MemoryBarrier carried_over(vk::AccessFlagBits::eTransferWrite, vk::AccessFlagBits::eTransferWrite); - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eTransfer, {}, carried_over, nullptr, nullptr - ); + barriers.memory(sync::Usage::transfer_dst, sync::Usage::transfer_dst); + barriers.flush(cmd); const std::array patched { &m_patch.materials, &m_patch.occupancy, &m_patch.grids, &m_patch.coarse @@ -297,9 +332,18 @@ void VoxelScenePatchUpload::record(vk::CommandBuffer cmd) { } cmd.copyBuffer(*m_staging, m_storage->buffer(k_sections[i]), regions); } + releaseSections(barriers, *m_storage, m_handoff); + barriers.flush(cmd); + m_released = true; m_storage->markRecorded(); } +void VoxelScenePatchUpload::recordAcquire(sync::BarrierBatch& batch) { + if (m_released && !m_storage->hasFailed()) { + acquireSections(batch, *m_storage, m_handoff); + } +} + void VoxelScenePatchUpload::finished() { PendingResourceUpload::finished(); diff --git a/engine/src/toast/renderer/voxel_gpu_storage.hpp b/engine/src/toast/renderer/voxel_gpu_storage.hpp index e5f7f325a..f6b67e38e 100644 --- a/engine/src/toast/renderer/voxel_gpu_storage.hpp +++ b/engine/src/toast/renderer/voxel_gpu_storage.hpp @@ -175,6 +175,8 @@ class VoxelSceneUpload : public PendingResourceUpload { void record(vk::CommandBuffer cmd) override; + void recordAcquire(sync::BarrierBatch& batch) override; + auto resource() -> IVulkanResource* override { return m_storage.get(); } private: @@ -182,6 +184,8 @@ class VoxelSceneUpload : public PendingResourceUpload { std::shared_ptr m_packed; + bool m_released = false; + vma::raii::Buffer m_staging = nullptr; std::array m_offsets {}; @@ -208,11 +212,15 @@ class VoxelScenePatchUpload : public PendingResourceUpload { void finished() override; + void recordAcquire(sync::BarrierBatch& batch) override; + auto resource() -> IVulkanResource* override { return m_storage.get(); } private: static constexpr size_t k_patched_sections = 4; + bool m_released = false; + std::shared_ptr m_storage; std::shared_ptr m_source; voxel::gpu::ScenePatch m_patch; diff --git a/engine/src/toast/renderer/vulkan_common.hpp b/engine/src/toast/renderer/vulkan_common.hpp index 27f3e11ef..b7d084b42 100644 --- a/engine/src/toast/renderer/vulkan_common.hpp +++ b/engine/src/toast/renderer/vulkan_common.hpp @@ -1,6 +1,10 @@ #pragma once +#include "gpu_sync.hpp" + #include +#include +#include #include #include #include @@ -51,42 +55,30 @@ inline auto colorSubresourceRange() -> vk::ImageSubresourceRange { return {vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1}; } -inline void submitAndWait(const vk::raii::Device& device, vk::Queue queue, vk::CommandBuffer cmd) { +/// @p queue_mutex guards against the UI thread submitting uploads to the same queue +inline void submitAndWait(const vk::raii::Device& device, vk::Queue queue, std::mutex& queue_mutex, vk::CommandBuffer cmd) { const vk::raii::Fence fence(device, vk::FenceCreateInfo {}); - vk::SubmitInfo submit {}; - submit.commandBufferCount = 1; - submit.pCommandBuffers = &cmd; - queue.submit(submit, *fence); + { + std::scoped_lock queue_lock(queue_mutex); + sync::submit( + queue, + { + .commands = {&cmd, 1}, + .fence = *fence + } + ); + } std::ignore = device.waitForFences(*fence, VK_TRUE, std::numeric_limits::max()); } inline void recordUndefinedToTransferDst(vk::CommandBuffer cmd, vk::Image image, const vk::ImageSubresourceRange& range) { - vk::ImageMemoryBarrier to_dst {}; - to_dst.oldLayout = vk::ImageLayout::eUndefined; - to_dst.newLayout = vk::ImageLayout::eTransferDstOptimal; - to_dst.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - to_dst.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - to_dst.image = image; - to_dst.subresourceRange = range; - to_dst.dstAccessMask = vk::AccessFlagBits::eTransferWrite; - cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTopOfPipe, vk::PipelineStageFlagBits::eTransfer, {}, nullptr, nullptr, to_dst); + sync::transition(cmd, image, range, sync::Usage::none, sync::Usage::transfer_dst); } inline void recordTransferDstToShaderRead(vk::CommandBuffer cmd, vk::Image image, const vk::ImageSubresourceRange& range) { - vk::ImageMemoryBarrier to_read {}; - to_read.oldLayout = vk::ImageLayout::eTransferDstOptimal; - to_read.newLayout = vk::ImageLayout::eShaderReadOnlyOptimal; - to_read.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - to_read.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - to_read.image = image; - to_read.subresourceRange = range; - to_read.srcAccessMask = vk::AccessFlagBits::eTransferWrite; - to_read.dstAccessMask = vk::AccessFlagBits::eShaderRead; - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eFragmentShader, {}, nullptr, nullptr, to_read - ); + sync::transition(cmd, image, range, sync::Usage::transfer_dst, sync::Usage::shader_sampled); } [[nodiscard]] diff --git a/engine/src/toast/renderer/vulkan_core.cpp b/engine/src/toast/renderer/vulkan_core.cpp index 19b83282b..6dce82267 100644 --- a/engine/src/toast/renderer/vulkan_core.cpp +++ b/engine/src/toast/renderer/vulkan_core.cpp @@ -7,8 +7,11 @@ #include "shader_compiler.hpp" #include +#include #include #include +#include +#include #include #include @@ -16,9 +19,11 @@ #include #endif #include +#include #include #include #include +#include #include #if defined(_WIN32) @@ -29,19 +34,72 @@ namespace renderer { namespace { + +/// One file per GPU model +auto vulkanCorePipelineCacheUri(const vk::PhysicalDeviceProperties& props) -> std::string { + return std::format("cache://pipelines/{:04x}-{:04x}.bin", props.vendorID, props.deviceID); +} + +/// Some drivers crash on cache data from another driver instead of rejecting it +auto vulkanCorePipelineCacheMatches(std::span data, const vk::PhysicalDeviceProperties& props) -> bool { + VkPipelineCacheHeaderVersionOne header {}; + if (data.size() < sizeof(header)) { + return false; + } + std::memcpy(&header, data.data(), sizeof(header)); + return header.headerSize >= sizeof(header) && header.headerVersion == VK_PIPELINE_CACHE_HEADER_VERSION_ONE && + header.vendorID == props.vendorID && header.deviceID == props.deviceID && + std::memcmp(header.pipelineCacheUUID, props.pipelineCacheUUID.data(), VK_UUID_SIZE) == 0; +} + constexpr std::size_t k_gigabyte_bytes = 1024ull * 1024ull * 1024ull; constexpr uint32_t k_invalid_queue_family = std::numeric_limits::max(); #ifdef TRACY_ENABLE -// Tracy keys memory pools by name address and MSVC Debug does not pool string literals constexpr auto k_tracy_vram_pool = std::to_array("VRAM"); +constexpr auto k_tracy_plot_used = std::to_array("VRAM used (MiB)"); +constexpr auto k_tracy_plot_budget = std::to_array("VRAM budget (MiB)"); +constexpr auto k_tracy_plot_allocations = std::to_array("VRAM engine allocations (MiB)"); +constexpr auto k_tracy_plot_blocks = std::to_array("VRAM engine blocks (MiB)"); + +struct TracyVramTracker { + struct Block { + VkDeviceSize size = 0; + bool reported = false; + }; + + std::mutex mutex; + std::unordered_map blocks; + /// True once this connection has seen every live block + bool session_synced = false; +}; + +auto tracyVramTracker() -> TracyVramTracker& { + static TracyVramTracker tracker; + return tracker; +} void tracyVmaAllocate(VmaAllocator, uint32_t, VkDeviceMemory memory, VkDeviceSize size, void*) { - TracyAllocN(reinterpret_cast(memory), size, k_tracy_vram_pool.data()); + auto& tracker = tracyVramTracker(); + std::scoped_lock lock(tracker.mutex); + const bool report = tracker.session_synced && TracyIsConnected; + tracker.blocks[memory] = {.size = size, .reported = report}; + if (report) { + TracyAllocN(reinterpret_cast(memory), size, k_tracy_vram_pool.data()); + } } void tracyVmaFree(VmaAllocator, uint32_t, VkDeviceMemory memory, VkDeviceSize, void*) { - TracyFreeN(reinterpret_cast(memory), k_tracy_vram_pool.data()); + auto& tracker = tracyVramTracker(); + std::scoped_lock lock(tracker.mutex); + const auto it = tracker.blocks.find(memory); + if (it == tracker.blocks.end()) { + return; + } + if (it->second.reported && TracyIsConnected) { + TracyFreeN(reinterpret_cast(memory), k_tracy_vram_pool.data()); + } + tracker.blocks.erase(it); } #endif @@ -278,6 +336,18 @@ VulkanCore::VulkanCore( TOAST_INFO("Render", "Debug object names: {}", m_debug_utils_enabled ? "enabled" : "disabled"); vk::InstanceCreateInfo instance_ci({}, &app_info, layers, extensions); + +#ifdef TOAST_SYNC_VALIDATION + // Only ever defined in Debug + const std::array enabled_validation_features {vk::ValidationFeatureEnableEXT::eSynchronizationValidation}; + vk::ValidationFeaturesEXT validation_features {}; + validation_features.setEnabledValidationFeatures(enabled_validation_features); + if (m_validation_enabled) { + instance_ci.pNext = &validation_features; + } + TOAST_INFO("Render", "Synchronization validation: {}", m_validation_enabled ? "enabled" : "disabled"); +#endif + m_instance = vk::raii::Instance(m_context, instance_ci); if (m_validation_enabled && m_debug_utils_enabled) { @@ -294,6 +364,7 @@ VulkanCore::VulkanCore( pickPhysicalDevice(required_device_extensions); createLogicalDeviceAndAllocator(required_device_extensions); + m_pipeline_cache = vk::raii::PipelineCache(m_device, vk::PipelineCacheCreateInfo {}); #if defined(_WIN32) if (HMODULE mod = GetModuleHandleA("renderdoc.dll")) { @@ -529,11 +600,17 @@ void VulkanCore::createLogicalDeviceAndAllocator(std::span re } } - float queue_priority = 1.0f; + // A second queue when transfer falls back to the compute family so uploads never share a VkQueue with AsyncCompute + const bool transfer_in_compute_family = m_transfer_queue_family_index == m_compute_queue_family_index; + m_transfer_queue_index = + transfer_in_compute_family && queue_family_properties[m_compute_queue_family_index].queueCount >= 2 ? 1u : 0u; + + const std::array queue_priorities {1.0f, 1.0f}; std::vector queue_create_infos; queue_create_infos.reserve(unique_queue_families.size()); for (auto family_index : unique_queue_families) { - queue_create_infos.emplace_back(vk::DeviceQueueCreateInfo({}, family_index, 1, &queue_priority)); + const uint32_t count = family_index == m_compute_queue_family_index ? m_transfer_queue_index + 1 : 1; + queue_create_infos.emplace_back(vk::DeviceQueueCreateInfo({}, family_index, count, queue_priorities.data())); } const auto device_features = m_physical_device.getFeatures(); @@ -573,6 +650,10 @@ void VulkanCore::createLogicalDeviceAndAllocator(std::span re "Vulkan 1.2 descriptorBindingVariableDescriptorCount" ); require_feature(supported_vulkan12.bufferDeviceAddress == VK_TRUE, "Vulkan 1.2 bufferDeviceAddress"); + // eDepthAttachmentOptimal and eDepthReadOnlyOptimal are only legal with this enabled + require_feature(supported_vulkan12.separateDepthStencilLayouts == VK_TRUE, "Vulkan 1.2 separateDepthStencilLayouts"); + // AsyncCompute tickets are timeline values + require_feature(supported_vulkan12.timelineSemaphore == VK_TRUE, "Vulkan 1.2 timelineSemaphore"); require_feature(supported_vulkan11.shaderDrawParameters == VK_TRUE, "Vulkan 1.1 shaderDrawParameters"); require_feature(supported_vulkan11.multiview == VK_TRUE, "Vulkan 1.1 multiview"); require_feature(device_features.independentBlend == VK_TRUE, "independentBlend"); @@ -585,6 +666,14 @@ void VulkanCore::createLogicalDeviceAndAllocator(std::span re return std::string_view(ext.extensionName.data()) == VK_EXT_FRAME_BOUNDARY_EXTENSION_NAME; }); + // VMA only estimates heap usage and budget without it + m_memory_budget_supported = std::ranges::any_of(available, [](const vk::ExtensionProperties& ext) { + return std::string_view(ext.extensionName.data()) == VK_EXT_MEMORY_BUDGET_EXTENSION_NAME; + }); + if (m_memory_budget_supported) { + device_extensions.push_back(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME); + } + if (m_frame_boundary_supported) { device_extensions.push_back(VK_EXT_FRAME_BOUNDARY_EXTENSION_NAME); TOAST_INFO("Render", "VK_EXT_frame_boundary available; frame delimiters will be emitted for graphics debuggers"); @@ -671,6 +760,8 @@ void VulkanCore::createLogicalDeviceAndAllocator(std::span re vulkan12_features.descriptorBindingPartiallyBound = VK_TRUE; vulkan12_features.descriptorBindingVariableDescriptorCount = VK_TRUE; vulkan12_features.bufferDeviceAddress = VK_TRUE; + vulkan12_features.separateDepthStencilLayouts = VK_TRUE; + vulkan12_features.timelineSemaphore = VK_TRUE; vulkan11_features.shaderDrawParameters = VK_TRUE; vulkan11_features.multiview = VK_TRUE; @@ -691,7 +782,19 @@ void VulkanCore::createLogicalDeviceAndAllocator(std::span re m_graphics_queue = m_device.getQueue(m_graphics_queue_family_index, 0); m_compute_queue = m_device.getQueue(m_compute_queue_family_index, 0); - m_transfer_queue = m_device.getQueue(m_transfer_queue_family_index, 0); + m_transfer_queue = m_device.getQueue(m_transfer_queue_family_index, m_transfer_queue_index); + if (m_transfer_queue == m_compute_queue) { + TOAST_WARN( + "Render", "Transfer and compute share one queue on this device; uploads and async compute serialize on a shared lock" + ); + } + + m_async_compute = std::make_unique( + m_device, + m_compute_queue_family_index, + m_compute_queue, + m_transfer_queue == m_compute_queue ? &m_transfer_submit_mutex : nullptr + ); vma::AllocatorCreateInfo allocator_ci {}; allocator_ci.vulkanApiVersion = VK_API_VERSION_1_4; @@ -699,6 +802,9 @@ void VulkanCore::createLogicalDeviceAndAllocator(std::span re // VMA asserts without eBufferDeviceAddress allocator_ci.flags = vma::AllocatorCreateFlagBits::eBufferDeviceAddress; + if (m_memory_budget_supported) { + allocator_ci.flags |= vma::AllocatorCreateFlagBits::eExtMemoryBudget; + } #ifdef TRACY_ENABLE static constexpr vma::DeviceMemoryCallbacks tracy_memory_callbacks {&tracyVmaAllocate, &tracyVmaFree, nullptr}; @@ -872,6 +978,12 @@ auto VulkanCore::calculateDeviceScore(const vk::PhysicalDevice& device, std::spa extension_score += 75; } + const bool supports_separate_depth_stencil_layouts = + props.apiVersion >= VK_API_VERSION_1_2 && vulkan12_features.separateDepthStencilLayouts == VK_TRUE; + if (!supports_separate_depth_stencil_layouts) { + required_missing_names.emplace_back("Vulkan 1.2 separate depth stencil layouts"); + } + const bool supports_shader_draw_parameters = props.apiVersion >= VK_API_VERSION_1_1 && vulkan11_features.shaderDrawParameters == VK_TRUE; if (!supports_shader_draw_parameters) { @@ -993,4 +1105,101 @@ auto VulkanCore::checkValidationLayerSupport() -> bool { }) != available_layers.end(); } } + +void VulkanCore::loadPipelineCache() const { + ZoneScoped; + if (!assets::AssetManager::isInitialized()) { + return; + } + + const vk::PhysicalDeviceProperties props = m_physical_device.getProperties(); + const std::string uri = vulkanCorePipelineCacheUri(props); + const auto bytes = assets::AssetManager::get().tryLoadBytes(uri); + if (!bytes.has_value()) { + TOAST_INFO("Render", "No pipeline cache at {}; pipelines compile from scratch this run", uri); + return; + } + if (!vulkanCorePipelineCacheMatches(*bytes, props)) { + TOAST_INFO("Render", "Pipeline cache {} is from another GPU or driver; ignoring it", uri); + return; + } + + try { + const vk::raii::PipelineCache loaded(m_device, vk::PipelineCacheCreateInfo({}, bytes->size(), bytes->data())); + m_pipeline_cache.merge(*loaded); + TOAST_INFO("Render", "Loaded pipeline cache {} ({} KiB)", uri, bytes->size() / 1024); + } catch (const std::exception& e) { TOAST_WARN("Render", "Pipeline cache {} could not be loaded: {}", uri, e.what()); } +} + +void VulkanCore::savePipelineCache() const { + ZoneScoped; + if (!assets::AssetManager::isInitialized() || !*m_pipeline_cache) { + return; + } + + const std::string uri = vulkanCorePipelineCacheUri(m_physical_device.getProperties()); + try { + const std::vector data = m_pipeline_cache.getData(); + if (data.empty()) { + return; + } + if (!assets::AssetManager::get().saveBytes(uri, data)) { + TOAST_WARN("Render", "Pipeline cache could not be written to {}", uri); + return; + } + TOAST_INFO("Render", "Saved pipeline cache {} ({} KiB)", uri, data.size() / 1024); + } catch (const std::exception& e) { TOAST_WARN("Render", "Pipeline cache data could not be read: {}", e.what()); } +} + +void VulkanCore::updateTracyMemory() const { +#ifdef TRACY_ENABLE + ZoneScoped; + auto& tracker = tracyVramTracker(); + { + std::scoped_lock lock(tracker.mutex); + if (!TracyIsConnected) { + // The next connection is a new session that has seen nothing + tracker.session_synced = false; + for (auto& [memory, block] : tracker.blocks) { + block.reported = false; + } + return; + } + if (!tracker.session_synced) { + // Sent per connection since on demand Tracy dropped any earlier config + TracyPlotConfig(k_tracy_plot_used.data(), tracy::PlotFormatType::Number, false, true, 0); + TracyPlotConfig(k_tracy_plot_budget.data(), tracy::PlotFormatType::Number, false, false, 0); + TracyPlotConfig(k_tracy_plot_allocations.data(), tracy::PlotFormatType::Number, false, true, 0); + TracyPlotConfig(k_tracy_plot_blocks.data(), tracy::PlotFormatType::Number, false, false, 0); + for (auto& [memory, block] : tracker.blocks) { + TracyAllocN(reinterpret_cast(memory), block.size, k_tracy_vram_pool.data()); + block.reported = true; + } + tracker.session_synced = true; + } + } + + // Same VMA budgets as the performance window so both tools agree + const auto budgets = m_allocator->getHeapBudgets(); + const auto properties = m_physical_device.getMemoryProperties(); + VkDeviceSize used = 0; + VkDeviceSize budget = 0; + VkDeviceSize allocations = 0; + VkDeviceSize blocks = 0; + for (uint32_t heap = 0; heap < properties.memoryHeapCount && heap < budgets.size(); ++heap) { + if ((properties.memoryHeaps[heap].flags & vk::MemoryHeapFlagBits::eDeviceLocal) == vk::MemoryHeapFlags {}) { + continue; + } + used += budgets[heap].usage; + budget += budgets[heap].budget; + allocations += budgets[heap].statistics.allocationBytes; + blocks += budgets[heap].statistics.blockBytes; + } + constexpr double k_mib = 1024.0 * 1024.0; + TracyPlot(k_tracy_plot_used.data(), static_cast(used) / k_mib); + TracyPlot(k_tracy_plot_budget.data(), static_cast(budget) / k_mib); + TracyPlot(k_tracy_plot_allocations.data(), static_cast(allocations) / k_mib); + TracyPlot(k_tracy_plot_blocks.data(), static_cast(blocks) / k_mib); +#endif +} } diff --git a/engine/src/toast/renderer/vulkan_core.hpp b/engine/src/toast/renderer/vulkan_core.hpp index 692e57644..1e08597e9 100644 --- a/engine/src/toast/renderer/vulkan_core.hpp +++ b/engine/src/toast/renderer/vulkan_core.hpp @@ -4,11 +4,13 @@ #pragma once +#include "async_compute.hpp" #include "vulkan_common.hpp" #include #include #include +#include #include #include #include @@ -93,6 +95,7 @@ class VulkanCore { return m_graphics_queue; } + /// Only AsyncCompute submits to it [[nodiscard]] auto getComputeQueue() const noexcept -> vk::Queue { return m_compute_queue; @@ -132,6 +135,11 @@ class VulkanCore { return (getBufferAddress(buffer) + alignment - 1) & ~(alignment - 1); } + [[nodiscard]] + auto isMemoryBudgetSupported() const noexcept -> bool { + return m_memory_budget_supported; + } + [[nodiscard]] auto isFrameBoundarySupported() const noexcept -> bool { return m_frame_boundary_supported; @@ -167,6 +175,32 @@ class VulkanCore { return m_max_sampler_anisotropy; } + /// Usable from any thread + [[nodiscard]] + auto getPipelineCache() const noexcept -> const vk::raii::PipelineCache& { + return m_pipeline_cache; + } + + /// Before creating pipelines. Missing or stale files are ignored + void loadPipelineCache() const; + + void savePipelineCache() const; + + /// Replays live VRAM blocks to a newly connected Tracy and plots heap usage + void updateTracyMemory() const; + + /// The only way to submit compute work outside a frame + [[nodiscard]] + auto asyncCompute() const noexcept -> AsyncCompute& { + return *m_async_compute; + } + + /// Only contended when AsyncCompute shares the transfer queue + [[nodiscard]] + auto transferSubmitMutex() const noexcept -> std::mutex& { + return m_transfer_submit_mutex; + } + // TODO UI system should submit on the render thread [[nodiscard]] auto graphicsSubmitMutex() const noexcept -> std::mutex& { @@ -202,6 +236,10 @@ class VulkanCore { vk::raii::PhysicalDevice m_physical_device = nullptr; vk::raii::Device m_device = nullptr; + /// Declared after m_device so it is destroyed first + vk::raii::PipelineCache m_pipeline_cache = nullptr; + /// After m_device for the same reason + std::unique_ptr m_async_compute; std::optional m_allocator; @@ -216,12 +254,15 @@ class VulkanCore { float m_max_sampler_anisotropy = 1.0f; mutable std::mutex m_graphics_submit_mutex; + mutable std::mutex m_transfer_submit_mutex; + uint32_t m_transfer_queue_index = 0; RENDERDOC_API_1_6_0* rdoc_api = nullptr; mutable NsightMode m_nsight_mode = NsightMode::none; bool m_frame_boundary_supported = false; + bool m_memory_budget_supported = false; bool m_ray_tracing_supported = false; diff --git a/engine/src/toast/renderer/vulkan_mesh.cpp b/engine/src/toast/renderer/vulkan_mesh.cpp index d7ef34162..1d3448818 100644 --- a/engine/src/toast/renderer/vulkan_mesh.cpp +++ b/engine/src/toast/renderer/vulkan_mesh.cpp @@ -343,43 +343,37 @@ void MeshUpload::record(vk::CommandBuffer cmd) { mesh->recordUpload(cmd, *vertex_staging, 0, vertex_size, vertex_size + index_size); - std::vector barriers = { - vk::BufferMemoryBarrier( - vk::AccessFlagBits::eTransferWrite, - vk::AccessFlags {}, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - mesh->m_vertex_buffer.value(), - 0, - vertex_size - ), - vk::BufferMemoryBarrier( - vk::AccessFlagBits::eTransferWrite, - vk::AccessFlags {}, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - mesh->m_index_buffer.value(), - 0, - mesh->m_index_size - ) + // General since vertex input skinning and BLAS builds read these. eConcurrent so the hand off is visibility only + sync::QueueHandoff handoff = m_handoff; + handoff.exclusive = false; + sync::BarrierBatch barriers; + for (const auto& [buffer, size] : uploadedBuffers()) { + barriers.release(buffer, sync::Usage::transfer_dst, sync::Usage::general, handoff, 0, size); + } + barriers.flush(cmd); + m_released = true; +} + +auto MeshUpload::uploadedBuffers() const -> std::array, 3> { + const auto handle = [](const std::optional& buffer) { + return buffer.has_value() ? **buffer : vk::Buffer {}; + }; + return { + std::pair { handle(mesh->m_vertex_buffer), mesh->m_vertex_size}, + std::pair { handle(mesh->m_index_buffer), mesh->m_index_size}, + std::pair {handle(mesh->m_skin_vertex_buffer), mesh->m_skin_vertex_size}, }; +} - if (mesh->m_skin_vertex_buffer.has_value()) { - barriers.emplace_back( - vk::AccessFlagBits::eTransferWrite, - vk::AccessFlags {}, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - mesh->m_skin_vertex_buffer.value(), - 0, - mesh->m_skin_vertex_size - ); +void MeshUpload::recordAcquire(sync::BarrierBatch& batch) { + if (!m_released || mesh->hasFailed()) { + return; + } + sync::QueueHandoff handoff = m_handoff; + handoff.exclusive = false; + for (const auto& [buffer, size] : uploadedBuffers()) { + batch.acquire(buffer, sync::Usage::general, handoff, 0, size); } - - // No BLAS build here since transfer only families cannot build acceleration structures - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eBottomOfPipe, {}, nullptr, barriers, nullptr - ); } } diff --git a/engine/src/toast/renderer/vulkan_mesh.hpp b/engine/src/toast/renderer/vulkan_mesh.hpp index bed824f01..5938d84cf 100644 --- a/engine/src/toast/renderer/vulkan_mesh.hpp +++ b/engine/src/toast/renderer/vulkan_mesh.hpp @@ -9,10 +9,12 @@ #include "vulkan_common.hpp" #include "vulkan_resource_base.hpp" +#include #include #include #include #include +#include namespace renderer { class VulkanCore; @@ -136,7 +138,16 @@ class MeshUpload : public PendingResourceUpload { void record(vk::CommandBuffer cmd) override; + void recordAcquire(sync::BarrierBatch& batch) override; + auto resource() -> IVulkanResource* override { return mesh; } + +private: + /// Same order in record() and recordAcquire() + [[nodiscard]] + auto uploadedBuffers() const -> std::array, 3>; + + bool m_released = false; }; } diff --git a/engine/src/toast/renderer/vulkan_pipeline.cpp b/engine/src/toast/renderer/vulkan_pipeline.cpp index 529f0773c..4cfe80d5b 100644 --- a/engine/src/toast/renderer/vulkan_pipeline.cpp +++ b/engine/src/toast/renderer/vulkan_pipeline.cpp @@ -203,7 +203,7 @@ auto createGraphicsPipelineImpl( pipeline_ci.layout = pipeline_layout; pipeline_ci.renderPass = nullptr; - auto pipelines = device.createGraphicsPipelines(nullptr, pipeline_ci); + auto pipelines = device.createGraphicsPipelines(core.getPipelineCache(), pipeline_ci); return std::move(pipelines[0]); } @@ -220,7 +220,7 @@ auto createComputePipelineImpl( ); const vk::ComputePipelineCreateInfo pipeline_ci({}, shader_stage_ci, pipeline_layout); - auto pipelines = device.createComputePipelines(nullptr, pipeline_ci); + auto pipelines = device.createComputePipelines(core.getPipelineCache(), pipeline_ci); return std::move(pipelines[0]); } } diff --git a/engine/src/toast/renderer/vulkan_pipeline.hpp b/engine/src/toast/renderer/vulkan_pipeline.hpp index dc52a8d06..04fcd32c4 100644 --- a/engine/src/toast/renderer/vulkan_pipeline.hpp +++ b/engine/src/toast/renderer/vulkan_pipeline.hpp @@ -4,6 +4,7 @@ #pragma once +#include "depth_convention.hpp" #include "vulkan_common.hpp" #include @@ -71,8 +72,8 @@ class VulkanPipeline { bool depth_test = true; bool depth_write = true; - /// Far plane draws like the skybox need eLessOrEqual - vk::CompareOp depth_compare = vk::CompareOp::eLess; + /// Scene convention. Shadow maps set eLess and far plane draws need depth::k_closer_or_equal + vk::CompareOp depth_compare = depth::k_closer; float depth_bias_constant = 0.0f; float depth_bias_slope = 0.0f; diff --git a/engine/src/toast/renderer/vulkan_renderer.cpp b/engine/src/toast/renderer/vulkan_renderer.cpp index 71e03d99c..5625ba9c8 100644 --- a/engine/src/toast/renderer/vulkan_renderer.cpp +++ b/engine/src/toast/renderer/vulkan_renderer.cpp @@ -90,24 +90,6 @@ auto depthAttachmentRange(vk::Format format) -> vk::ImageSubresourceRange { return {aspect, 0, 1, 0, 1}; } -auto transitionImageLayout( - vk::CommandBuffer command_buffer, vk::Image image, vk::ImageLayout old_layout, vk::ImageLayout new_layout, - vk::AccessFlags src_access_mask, vk::AccessFlags dst_access_mask, vk::PipelineStageFlags src_stage_mask, - vk::PipelineStageFlags dst_stage_mask, vk::ImageSubresourceRange subresource_range -) -> void { - const vk::ImageMemoryBarrier barrier( - src_access_mask, - dst_access_mask, - old_layout, - new_layout, - VK_QUEUE_FAMILY_IGNORED, - VK_QUEUE_FAMILY_IGNORED, - image, - subresource_range - ); - command_buffer.pipelineBarrier(src_stage_mask, dst_stage_mask, {}, {}, {}, barrier); -} - struct DirectionalCascadeFit { glm::mat4 view_projection {1.0f}; glm::vec4 cull_sphere {0.0f}; @@ -430,7 +412,7 @@ void uploadSinglePixelTexture( cmd.end(); - submitAndWait(device, core.getGraphicsQueue(), *cmd); + submitAndWait(device, core.getGraphicsQueue(), core.graphicsSubmitMutex(), *cmd); texture.markReady(); } @@ -672,6 +654,7 @@ void VulkanRenderer::stop() { } catch (const std::exception& e) { TOAST_ERROR("Render", "Device wait failed during shutdown, tearing down anyway: {}", e.what()); } + m_core->savePipelineCache(); } m_gpu_timer.reset(); @@ -702,11 +685,12 @@ VulkanRenderer::VulkanRenderer(const VulkanCore& core, std::unique_ptrgetImageCount(); m_images_in_flight.assign(output_image_count, vk::Fence {}); - m_output_image_layouts.assign(output_image_count, vk::ImageLayout::eUndefined); + m_output_image_usage.assign(output_image_count, sync::Usage::none); createDescriptorPool(); @@ -820,15 +804,6 @@ auto VulkanRenderer::createTransferCommandPool() -> void { TOAST_TRACE("Render", "Transfer command pool created (transfer family {})", m_core->getTransferQueueFamilyIndex()); } -auto VulkanRenderer::createComputeCommandPool() -> void { - const vk::CommandPoolCreateInfo pool_ci( - vk::CommandPoolCreateFlagBits::eResetCommandBuffer, m_core->getComputeQueueFamilyIndex() - ); - m_compute_command_pool = vk::raii::CommandPool(m_core->getDevice(), pool_ci); - setDebugName(*m_core, *m_compute_command_pool, "VulkanRenderer ComputeCommandPool"); - TOAST_TRACE("VulkanRenderer", "Compute command pool created (compute family {})", m_core->getComputeQueueFamilyIndex()); -} - auto VulkanRenderer::createUploadRing() -> void { m_upload_slots.clear(); m_upload_slots.resize(k_upload_slots); @@ -855,42 +830,20 @@ auto VulkanRenderer::createFrameContexts() -> void { const vk::SemaphoreCreateInfo semaphore_ci {}; const vk::FenceCreateInfo fence_ci(vk::FenceCreateFlagBits::eSignaled); const vk::CommandBufferAllocateInfo command_buffer_ci(*m_command_pool, vk::CommandBufferLevel::ePrimary, k_frames_in_flight); - const vk::CommandBufferAllocateInfo compute_command_buffer_ci( - *m_compute_command_pool, vk::CommandBufferLevel::ePrimary, k_frames_in_flight - ); auto allocated_command_buffers = m_core->getDevice().allocateCommandBuffers(command_buffer_ci); - auto allocated_compute_command_buffers = m_core->getDevice().allocateCommandBuffers(compute_command_buffer_ci); for (uint32_t frame_index = 0; frame_index < k_frames_in_flight; ++frame_index) { m_frames[frame_index].command_buffer = std::move(allocated_command_buffers[frame_index]); - m_frames[frame_index].compute_command_buffer = std::move(allocated_compute_command_buffers[frame_index]); m_frames[frame_index].image_available = vk::raii::Semaphore(m_core->getDevice(), semaphore_ci); - m_frames[frame_index].compute_to_graphics = vk::raii::Semaphore(m_core->getDevice(), semaphore_ci); m_frames[frame_index].in_flight = vk::raii::Fence(m_core->getDevice(), fence_ci); - m_frames[frame_index].compute_in_flight = vk::raii::Fence(m_core->getDevice(), fence_ci); setDebugName( *m_core, *m_frames[frame_index].command_buffer, std::format("VulkanRenderer Frame[{}] CommandBuffer", frame_index) ); - setDebugName( - *m_core, - *m_frames[frame_index].compute_command_buffer, - std::format("VulkanRenderer Frame[{}] ComputeCommandBuffer", frame_index) - ); setDebugName( *m_core, *m_frames[frame_index].image_available, std::format("VulkanRenderer Frame[{}] ImageAvailable", frame_index) ); - setDebugName( - *m_core, - *m_frames[frame_index].compute_to_graphics, - std::format("VulkanRenderer Frame[{}] ComputeToGraphics", frame_index) - ); setDebugName(*m_core, *m_frames[frame_index].in_flight, std::format("VulkanRenderer Frame[{}] InFlightFence", frame_index)); - setDebugName( - *m_core, - *m_frames[frame_index].compute_in_flight, - std::format("VulkanRenderer Frame[{}] ComputeInFlightFence", frame_index) - ); } TOAST_TRACE("Render", "Frame command buffers created: {}", k_frames_in_flight); @@ -951,7 +904,7 @@ auto VulkanRenderer::createDepthResources() -> void { m_depth_resources.view.emplace(m_core->getDevice(), view_ci); setDebugName(*m_core, **m_depth_resources.view, "VulkanRenderer DepthImageView"); - m_depth_layout = vk::ImageLayout::eUndefined; + m_depth_resources.state.reset(**m_depth_resources.image, depthAttachmentRange(m_depth_format)); TOAST_TRACE( "Render", "Depth resources created at {}x{} with format {}", extent.width, extent.height, vk::to_string(m_depth_format) ); @@ -984,7 +937,7 @@ void VulkanRenderer::createSceneColorResources() { target.view.emplace(m_core->getDevice(), view_ci); setDebugName(*m_core, **target.view, std::format("VulkanRenderer {}View", label)); - target.layout = vk::ImageLayout::eUndefined; + target.state.reset(**target.image, colorSubresourceRange()); TOAST_TRACE("Render", "{} target created at {}x{} with format {}", label, extent.width, extent.height, vk::to_string(format)); }; @@ -1046,7 +999,7 @@ void VulkanRenderer::createDefaultTexture() { } auto VulkanRenderer::selectDepthFormat(const VulkanCore& core) -> vk::Format { - const std::array candidates { + constexpr std::array candidates { vk::Format::eD32Sfloat, vk::Format::eD24UnormS8Uint, vk::Format::eD32SfloatS8Uint, vk::Format::eD16Unorm }; @@ -1230,14 +1183,14 @@ void VulkanRenderer::createDefaultShadowMap() { recordUndefinedToTransferDst(cmd, **m_default_shadow_image, range); - const vk::ClearDepthStencilValue clear_value(1.0f, 0); + constexpr vk::ClearDepthStencilValue clear_value(depth::k_clear, 0); cmd.clearDepthStencilImage(**m_default_shadow_image, vk::ImageLayout::eTransferDstOptimal, clear_value, range); recordTransferDstToShaderRead(cmd, **m_default_shadow_image, range); cmd.end(); - submitAndWait(device, m_core->getGraphicsQueue(), *cmd); + submitAndWait(device, m_core->getGraphicsQueue(), m_core->graphicsSubmitMutex(), *cmd); // Sampler2DArrayShadow needs an array view vk::ImageViewCreateInfo view_ci {}; @@ -1286,14 +1239,14 @@ void VulkanRenderer::createDefaultCubemap() { recordUndefinedToTransferDst(cmd, **m_default_cube_image, range); - const vk::ClearColorValue clear_value(std::array {0.0f, 0.0f, 0.0f, 1.0f}); + constexpr vk::ClearColorValue clear_value(std::array {0.0f, 0.0f, 0.0f, 1.0f}); cmd.clearColorImage(**m_default_cube_image, vk::ImageLayout::eTransferDstOptimal, clear_value, range); recordTransferDstToShaderRead(cmd, **m_default_cube_image, range); cmd.end(); - submitAndWait(device, m_core->getGraphicsQueue(), *cmd); + submitAndWait(device, m_core->getGraphicsQueue(), m_core->graphicsSubmitMutex(), *cmd); vk::ImageViewCreateInfo view_ci {}; view_ci.image = **m_default_cube_image; @@ -1388,18 +1341,11 @@ void VulkanRenderer::recordAccelerationStructureBuilds(FrameContext& frame) { } if (built > 0 || skinned_recorded > 0) { - vk::MemoryBarrier barrier {}; - barrier.srcAccessMask = vk::AccessFlagBits::eAccelerationStructureWriteKHR; - barrier.dstAccessMask = vk::AccessFlagBits::eAccelerationStructureReadKHR; - - frame.command_buffer.pipelineBarrier( - vk::PipelineStageFlagBits::eAccelerationStructureBuildKHR, - vk::PipelineStageFlagBits::eAccelerationStructureBuildKHR | vk::PipelineStageFlagBits::eFragmentShader, - {}, - barrier, - nullptr, - nullptr - ); + // BLAS builds before the TLAS build that references them and before any fragment ray query + sync::BarrierBatch barriers; + barriers.memory(sync::Usage::acceleration_build, sync::Usage::acceleration_build); + barriers.memory(sync::Usage::acceleration_build, sync::Usage::acceleration_trace_fragment); + barriers.flush(*frame.command_buffer); } if (m_ray_tracing_scene != nullptr) { @@ -1491,58 +1437,12 @@ auto VulkanRenderer::recordVoxelScopes( m_scene_color.view.has_value() && m_scene_normal.view.has_value() && m_scene_indirect.view.has_value() && m_scene_motion.view.has_value(); if (m_voxel_pass == nullptr || !m_voxel_pass->isEnabled() || !targets_ready || - m_depth_layout != vk::ImageLayout::eDepthAttachmentOptimal || !m_voxel_pass->prepare(m_current_frame)) { + m_depth_resources.state.usage != sync::Usage::depth_attachment || !m_voxel_pass->prepare(m_current_frame)) { return false; } const vk::CommandBuffer cmd = *frame.command_buffer; - - const auto to_layout = [cmd](SceneColorResources& target, vk::ImageLayout layout) { - if (target.layout == layout) { - return; - } - vk::AccessFlags src_access {}; - vk::PipelineStageFlags src_stage = vk::PipelineStageFlagBits::eTopOfPipe; - if (target.layout == vk::ImageLayout::eColorAttachmentOptimal) { - src_access = vk::AccessFlagBits::eColorAttachmentWrite; - src_stage = vk::PipelineStageFlagBits::eColorAttachmentOutput; - } else if (target.layout == vk::ImageLayout::eShaderReadOnlyOptimal) { - src_access = vk::AccessFlagBits::eShaderRead; - src_stage = vk::PipelineStageFlagBits::eFragmentShader; - } - const bool sampled = layout == vk::ImageLayout::eShaderReadOnlyOptimal; - transitionImageLayout( - cmd, - **target.image, - target.layout, - layout, - src_access, - sampled ? vk::AccessFlags(vk::AccessFlagBits::eShaderRead) - : vk::AccessFlagBits::eColorAttachmentRead | vk::AccessFlagBits::eColorAttachmentWrite, - src_stage, - sampled ? vk::PipelineStageFlagBits::eFragmentShader : vk::PipelineStageFlagBits::eColorAttachmentOutput, - colorSubresourceRange() - ); - target.layout = layout; - }; - - // Scopes give no ordering between a write and the next load - const auto attachment_barrier = [cmd]() { - const vk::MemoryBarrier barrier( - vk::AccessFlagBits::eColorAttachmentWrite | vk::AccessFlagBits::eDepthStencilAttachmentWrite, - vk::AccessFlagBits::eColorAttachmentRead | vk::AccessFlagBits::eColorAttachmentWrite | - vk::AccessFlagBits::eDepthStencilAttachmentRead | vk::AccessFlagBits::eDepthStencilAttachmentWrite - ); - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eColorAttachmentOutput | vk::PipelineStageFlagBits::eLateFragmentTests, - vk::PipelineStageFlagBits::eColorAttachmentOutput | vk::PipelineStageFlagBits::eEarlyFragmentTests | - vk::PipelineStageFlagBits::eLateFragmentTests, - {}, - barrier, - nullptr, - nullptr - ); - }; + sync::BarrierBatch barriers; const auto attachment = [](const SceneColorResources& target, vk::AttachmentLoadOp load, const vk::ClearValue& clear) { vk::RenderingAttachmentInfo info {}; @@ -1557,13 +1457,14 @@ auto VulkanRenderer::recordVoxelScopes( const vk::ClearValue clear_zero(vk::ClearColorValue(std::array {0.0f, 0.0f, 0.0f, 0.0f})); const vk::ClearValue clear_scene(vk::ClearColorValue(std::array {0.0f, 0.0f, 0.0f, 1.0f})); - const auto depth_range = depthAttachmentRange(m_depth_format); - attachment_barrier(); - to_layout(m_voxel_albedo, vk::ImageLayout::eColorAttachmentOptimal); - to_layout(m_voxel_face_normal, vk::ImageLayout::eColorAttachmentOptimal); - to_layout(m_voxel_material, vk::ImageLayout::eColorAttachmentOptimal); - to_layout(m_scene_motion, vk::ImageLayout::eColorAttachmentOptimal); + // Depth stays an attachment but the prepass writes must land before this scope loads them + barriers.image(m_depth_resources.state, sync::Usage::depth_attachment); + barriers.image(m_voxel_albedo.state, sync::Usage::color_attachment); + barriers.image(m_voxel_face_normal.state, sync::Usage::color_attachment); + barriers.image(m_voxel_material.state, sync::Usage::color_attachment); + barriers.image(m_scene_motion.state, sync::Usage::color_attachment); + barriers.flush(cmd); // Order matches voxel.slang FSOutput const std::array gbuffer_attachments { @@ -1578,7 +1479,7 @@ auto VulkanRenderer::recordVoxelScopes( depth_attachment.imageLayout = vk::ImageLayout::eDepthAttachmentOptimal; depth_attachment.loadOp = depth_loaded ? vk::AttachmentLoadOp::eLoad : vk::AttachmentLoadOp::eClear; depth_attachment.storeOp = vk::AttachmentStoreOp::eStore; - depth_attachment.clearValue = vk::ClearDepthStencilValue {1.0f, 0}; + depth_attachment.clearValue = vk::ClearDepthStencilValue {depth::k_clear, 0}; vk::RenderingInfo gbuffer_info {}; gbuffer_info.renderArea = scissor; @@ -1597,24 +1498,14 @@ auto VulkanRenderer::recordVoxelScopes( } cmd.endRendering(); - to_layout(m_voxel_albedo, vk::ImageLayout::eShaderReadOnlyOptimal); - to_layout(m_voxel_face_normal, vk::ImageLayout::eShaderReadOnlyOptimal); - to_layout(m_voxel_material, vk::ImageLayout::eShaderReadOnlyOptimal); - transitionImageLayout( - cmd, - **m_depth_resources.image, - vk::ImageLayout::eDepthAttachmentOptimal, - vk::ImageLayout::eDepthReadOnlyOptimal, - vk::AccessFlagBits::eDepthStencilAttachmentWrite, - vk::AccessFlagBits::eShaderRead, - vk::PipelineStageFlagBits::eLateFragmentTests, - vk::PipelineStageFlagBits::eFragmentShader, - depth_range - ); - m_depth_layout = vk::ImageLayout::eDepthReadOnlyOptimal; - + barriers.image(m_voxel_albedo.state, sync::Usage::fragment_sampled); + barriers.image(m_voxel_face_normal.state, sync::Usage::fragment_sampled); + barriers.image(m_voxel_material.state, sync::Usage::fragment_sampled); + barriers.image(m_depth_resources.state, sync::Usage::depth_sampled); // Rounded normals land in the shared target other passes already read - to_layout(m_scene_normal, vk::ImageLayout::eColorAttachmentOptimal); + barriers.image(m_scene_normal.state, sync::Usage::color_attachment); + barriers.flush(cmd); + const std::array blur_attachments {attachment(m_scene_normal, vk::AttachmentLoadOp::eClear, clear_zero)}; vk::RenderingInfo blur_info {}; @@ -1632,10 +1523,11 @@ auto VulkanRenderer::recordVoxelScopes( m_voxel_pass->recordNormalBlur(cmd, m_current_frame, scissor.extent); } cmd.endRendering(); - to_layout(m_scene_normal, vk::ImageLayout::eShaderReadOnlyOptimal); - to_layout(m_scene_color, vk::ImageLayout::eColorAttachmentOptimal); - to_layout(m_scene_indirect, vk::ImageLayout::eColorAttachmentOptimal); + barriers.image(m_scene_normal.state, sync::Usage::fragment_sampled); + barriers.image(m_scene_color.state, sync::Usage::color_attachment); + barriers.image(m_scene_indirect.state, sync::Usage::color_attachment); + barriers.flush(cmd); // Clears what no voxel covers since the world scope now loads instead const std::array lighting_attachments { @@ -1659,20 +1551,13 @@ auto VulkanRenderer::recordVoxelScopes( } cmd.endRendering(); - to_layout(m_scene_normal, vk::ImageLayout::eColorAttachmentOptimal); - transitionImageLayout( - cmd, - **m_depth_resources.image, - vk::ImageLayout::eDepthReadOnlyOptimal, - vk::ImageLayout::eDepthAttachmentOptimal, - vk::AccessFlagBits::eShaderRead, - vk::AccessFlagBits::eDepthStencilAttachmentRead | vk::AccessFlagBits::eDepthStencilAttachmentWrite, - vk::PipelineStageFlagBits::eFragmentShader, - vk::PipelineStageFlagBits::eEarlyFragmentTests | vk::PipelineStageFlagBits::eLateFragmentTests, - depth_range - ); - m_depth_layout = vk::ImageLayout::eDepthAttachmentOptimal; - attachment_barrier(); + // The world scope loads all of these so writes above must land first + barriers.image(m_scene_normal.state, sync::Usage::color_attachment); + barriers.image(m_scene_color.state, sync::Usage::color_attachment); + barriers.image(m_scene_indirect.state, sync::Usage::color_attachment); + barriers.image(m_scene_motion.state, sync::Usage::color_attachment); + barriers.image(m_depth_resources.state, sync::Usage::depth_attachment); + barriers.flush(cmd); return true; } @@ -1685,6 +1570,19 @@ auto VulkanRenderer::recordFrame(FrameContext& frame, uint32_t image_index) noex m_gpu_timer->beginGraphics(m_current_frame, *frame.command_buffer); } + m_pending_acquires.flush(*frame.command_buffer); + + // Graphics queue since this frame reads it and the compute queue belongs to AsyncCompute + if (!m_compute_passes.empty()) { + const GpuScope scope(m_gpu_timer.get(), m_current_frame, *frame.command_buffer, "Light culling"); + for (auto& pass : m_compute_passes) { + pass->dispatch(*frame.command_buffer, m_current_frame); + } + sync::BarrierBatch compute_done; + compute_done.memory(sync::Usage::compute_storage_write, sync::Usage::fragment_storage_read); + compute_done.flush(*frame.command_buffer); + } + // Before the AS builds which read its output if (m_skinning_pass != nullptr) { const GpuScope scope(m_gpu_timer.get(), m_current_frame, *frame.command_buffer, "Skinning"); @@ -1718,91 +1616,21 @@ auto VulkanRenderer::recordFrame(FrameContext& frame, uint32_t image_index) noex const vk::Rect2D scene_scissor({0, 0}, scene_extent); const vk::Image scene_image = m_scene_color.image ? **m_scene_color.image : VK_NULL_HANDLE; - if (scene_image != VK_NULL_HANDLE) { - const bool was_sampled = m_scene_color.layout == vk::ImageLayout::eShaderReadOnlyOptimal; - transitionImageLayout( - frame.command_buffer, - scene_image, - m_scene_color.layout, - vk::ImageLayout::eColorAttachmentOptimal, - was_sampled ? vk::AccessFlagBits::eShaderRead : vk::AccessFlags {}, - vk::AccessFlagBits::eColorAttachmentWrite, - was_sampled ? vk::PipelineStageFlagBits::eFragmentShader : vk::PipelineStageFlagBits::eTopOfPipe, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - colorSubresourceRange() - ); - m_scene_color.layout = vk::ImageLayout::eColorAttachmentOptimal; - } - const vk::Image normal_image = m_scene_normal.image ? **m_scene_normal.image : VK_NULL_HANDLE; - if (normal_image != VK_NULL_HANDLE && m_scene_normal.layout != vk::ImageLayout::eColorAttachmentOptimal) { - const bool normal_was_sampled = m_scene_normal.layout == vk::ImageLayout::eShaderReadOnlyOptimal; - transitionImageLayout( - frame.command_buffer, - normal_image, - m_scene_normal.layout, - vk::ImageLayout::eColorAttachmentOptimal, - normal_was_sampled ? vk::AccessFlagBits::eShaderRead : vk::AccessFlags {}, - vk::AccessFlagBits::eColorAttachmentWrite, - normal_was_sampled ? vk::PipelineStageFlagBits::eFragmentShader : vk::PipelineStageFlagBits::eTopOfPipe, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - colorSubresourceRange() - ); - m_scene_normal.layout = vk::ImageLayout::eColorAttachmentOptimal; - } - const vk::Image indirect_image = m_scene_indirect.image ? **m_scene_indirect.image : VK_NULL_HANDLE; - if (indirect_image != VK_NULL_HANDLE && m_scene_indirect.layout != vk::ImageLayout::eColorAttachmentOptimal) { - const bool indirect_was_sampled = m_scene_indirect.layout == vk::ImageLayout::eShaderReadOnlyOptimal; - transitionImageLayout( - frame.command_buffer, - indirect_image, - m_scene_indirect.layout, - vk::ImageLayout::eColorAttachmentOptimal, - indirect_was_sampled ? vk::AccessFlagBits::eShaderRead : vk::AccessFlags {}, - vk::AccessFlagBits::eColorAttachmentWrite, - indirect_was_sampled ? vk::PipelineStageFlagBits::eFragmentShader : vk::PipelineStageFlagBits::eTopOfPipe, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - colorSubresourceRange() - ); - m_scene_indirect.layout = vk::ImageLayout::eColorAttachmentOptimal; - } - const vk::Image motion_image = m_scene_motion.image ? **m_scene_motion.image : VK_NULL_HANDLE; - if (motion_image != VK_NULL_HANDLE && m_scene_motion.layout != vk::ImageLayout::eColorAttachmentOptimal) { - const bool motion_was_sampled = m_scene_motion.layout == vk::ImageLayout::eShaderReadOnlyOptimal; - transitionImageLayout( - frame.command_buffer, - motion_image, - m_scene_motion.layout, - vk::ImageLayout::eColorAttachmentOptimal, - motion_was_sampled ? vk::AccessFlagBits::eShaderRead : vk::AccessFlags {}, - vk::AccessFlagBits::eColorAttachmentWrite, - motion_was_sampled ? vk::PipelineStageFlagBits::eFragmentShader : vk::PipelineStageFlagBits::eTopOfPipe, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - colorSubresourceRange() - ); - m_scene_motion.layout = vk::ImageLayout::eColorAttachmentOptimal; - } - - const vk::Image depth_image = m_depth_resources.image ? **m_depth_resources.image : VK_NULL_HANDLE; - if (depth_image != VK_NULL_HANDLE && m_depth_layout != vk::ImageLayout::eDepthAttachmentOptimal) { - transitionImageLayout( - frame.command_buffer, - depth_image, - m_depth_layout, - vk::ImageLayout::eDepthAttachmentOptimal, - vk::AccessFlags {}, - vk::AccessFlagBits::eDepthStencilAttachmentWrite, - vk::PipelineStageFlagBits::eTopOfPipe, - vk::PipelineStageFlagBits::eEarlyFragmentTests | vk::PipelineStageFlagBits::eLateFragmentTests, - depthAttachmentRange(m_depth_format) - ); - m_depth_layout = vk::ImageLayout::eDepthAttachmentOptimal; - } + + // Last frame sampled these and its overlay scope stored depth + sync::BarrierBatch barriers; + barriers.image(m_scene_color.state, sync::Usage::color_attachment); + barriers.image(m_scene_normal.state, sync::Usage::color_attachment); + barriers.image(m_scene_indirect.state, sync::Usage::color_attachment); + barriers.image(m_scene_motion.state, sync::Usage::color_attachment); + barriers.image(m_depth_resources.state, sync::Usage::depth_attachment); + barriers.flush(*frame.command_buffer); const vk::ClearValue clear_color(vk::ClearColorValue(std::array {0.0f, 0.0f, 0.0f, 1.0f})); - const vk::ClearValue clear_depth(vk::ClearDepthStencilValue {1.0f, 0}); + const vk::ClearValue clear_depth(vk::ClearDepthStencilValue {depth::k_clear, 0}); vk::RenderingAttachmentInfo scene_attachment_info {}; scene_attachment_info.imageView = scene_image != VK_NULL_HANDLE ? **m_scene_color.view : nullptr; @@ -1917,19 +1745,11 @@ auto VulkanRenderer::recordFrame(FrameContext& frame, uint32_t image_index) noex frame.command_buffer.endRendering(); + // PROBE and Iradiance capture + if (const auto* capture_frame = renderingFrame(); capture_frame != nullptr && capture_frame->probe_capture_index >= 0 && m_reflection_probe_pass != nullptr && scene_image != VK_NULL_HANDLE) { - transitionImageLayout( - frame.command_buffer, - scene_image, - vk::ImageLayout::eColorAttachmentOptimal, - vk::ImageLayout::eTransferSrcOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::AccessFlagBits::eTransferRead, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - vk::PipelineStageFlagBits::eTransfer, - colorSubresourceRange() - ); + sync::transition(*frame.command_buffer, m_scene_color.state, sync::Usage::transfer_src); m_reflection_probe_pass->captureFace( *frame.command_buffer, @@ -1939,48 +1759,18 @@ auto VulkanRenderer::recordFrame(FrameContext& frame, uint32_t image_index) noex capture_frame->probe_capture_face ); - transitionImageLayout( - frame.command_buffer, - scene_image, - vk::ImageLayout::eTransferSrcOptimal, - vk::ImageLayout::eColorAttachmentOptimal, - vk::AccessFlagBits::eTransferRead, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - colorSubresourceRange() - ); + sync::transition(*frame.command_buffer, m_scene_color.state, sync::Usage::color_attachment); } if (const auto* capture_frame = renderingFrame(); capture_frame != nullptr && capture_frame->irradiance_capture_index >= 0 && m_reflection_probe_pass != nullptr && scene_image != VK_NULL_HANDLE) { - transitionImageLayout( - frame.command_buffer, - scene_image, - vk::ImageLayout::eColorAttachmentOptimal, - vk::ImageLayout::eTransferSrcOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::AccessFlagBits::eTransferRead, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - vk::PipelineStageFlagBits::eTransfer, - colorSubresourceRange() - ); + sync::transition(*frame.command_buffer, m_scene_color.state, sync::Usage::transfer_src); m_reflection_probe_pass->captureIrradianceFace( *frame.command_buffer, scene_image, scene_extent, capture_frame->irradiance_capture_face ); - transitionImageLayout( - frame.command_buffer, - scene_image, - vk::ImageLayout::eTransferSrcOptimal, - vk::ImageLayout::eColorAttachmentOptimal, - vk::AccessFlagBits::eTransferRead, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::PipelineStageFlagBits::eTransfer, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - colorSubresourceRange() - ); + sync::transition(*frame.command_buffer, m_scene_color.state, sync::Usage::color_attachment); if (capture_frame->irradiance_capture_face == 5) { m_reflection_probe_pass->projectStagingToSh( @@ -1989,80 +1779,13 @@ auto VulkanRenderer::recordFrame(FrameContext& frame, uint32_t image_index) noex } } - if (scene_image != VK_NULL_HANDLE) { - transitionImageLayout( - frame.command_buffer, - scene_image, - vk::ImageLayout::eColorAttachmentOptimal, - vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::AccessFlagBits::eShaderRead, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - vk::PipelineStageFlagBits::eFragmentShader, - colorSubresourceRange() - ); - m_scene_color.layout = vk::ImageLayout::eShaderReadOnlyOptimal; - } - - if (normal_image != VK_NULL_HANDLE && m_scene_normal.layout != vk::ImageLayout::eShaderReadOnlyOptimal) { - transitionImageLayout( - frame.command_buffer, - normal_image, - m_scene_normal.layout, - vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::AccessFlagBits::eShaderRead, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - vk::PipelineStageFlagBits::eFragmentShader, - colorSubresourceRange() - ); - m_scene_normal.layout = vk::ImageLayout::eShaderReadOnlyOptimal; - } - - if (indirect_image != VK_NULL_HANDLE && m_scene_indirect.layout != vk::ImageLayout::eShaderReadOnlyOptimal) { - transitionImageLayout( - frame.command_buffer, - indirect_image, - m_scene_indirect.layout, - vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::AccessFlagBits::eShaderRead, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - vk::PipelineStageFlagBits::eFragmentShader, - colorSubresourceRange() - ); - m_scene_indirect.layout = vk::ImageLayout::eShaderReadOnlyOptimal; - } - - if (motion_image != VK_NULL_HANDLE && m_scene_motion.layout != vk::ImageLayout::eShaderReadOnlyOptimal) { - transitionImageLayout( - frame.command_buffer, - motion_image, - m_scene_motion.layout, - vk::ImageLayout::eShaderReadOnlyOptimal, - vk::AccessFlagBits::eColorAttachmentWrite, - vk::AccessFlagBits::eShaderRead, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - vk::PipelineStageFlagBits::eFragmentShader, - colorSubresourceRange() - ); - m_scene_motion.layout = vk::ImageLayout::eShaderReadOnlyOptimal; - } - - if (depth_image != VK_NULL_HANDLE && m_depth_layout != vk::ImageLayout::eDepthReadOnlyOptimal) { - transitionImageLayout( - frame.command_buffer, - depth_image, - m_depth_layout, - vk::ImageLayout::eDepthReadOnlyOptimal, - vk::AccessFlagBits::eDepthStencilAttachmentWrite, - vk::AccessFlagBits::eShaderRead, - vk::PipelineStageFlagBits::eLateFragmentTests, - vk::PipelineStageFlagBits::eFragmentShader, - depthAttachmentRange(m_depth_format) - ); - m_depth_layout = vk::ImageLayout::eDepthReadOnlyOptimal; - } + // Auto exposure reads these from compute too + barriers.image(m_scene_color.state, sync::Usage::shader_sampled); + barriers.image(m_scene_normal.state, sync::Usage::shader_sampled); + barriers.image(m_scene_indirect.state, sync::Usage::shader_sampled); + barriers.image(m_scene_motion.state, sync::Usage::shader_sampled); + barriers.image(m_depth_resources.state, sync::Usage::depth_sampled); + barriers.flush(*frame.command_buffer); const auto* capture_check = renderingFrame(); const bool capture_frame = @@ -2081,61 +1804,14 @@ auto VulkanRenderer::recordFrame(FrameContext& frame, uint32_t image_index) noex } } - if (depth_image != VK_NULL_HANDLE && m_depth_layout != vk::ImageLayout::eDepthAttachmentOptimal) { - transitionImageLayout( - frame.command_buffer, - depth_image, - m_depth_layout, - vk::ImageLayout::eDepthAttachmentOptimal, - vk::AccessFlagBits::eShaderRead, - vk::AccessFlagBits::eDepthStencilAttachmentRead, - vk::PipelineStageFlagBits::eFragmentShader, - vk::PipelineStageFlagBits::eEarlyFragmentTests, - depthAttachmentRange(m_depth_format) - ); - m_depth_layout = vk::ImageLayout::eDepthAttachmentOptimal; - } + // The overlay scope depth tests against the scene and its DontCare store counts as a write + barriers.image(m_depth_resources.state, sync::Usage::depth_attachment); const vk::Image image = m_output_target->getColorImage(image_index); - const vk::ImageLayout previous_layout = m_output_image_layouts.at(image_index); - - vk::AccessFlags src_access_mask {}; - vk::PipelineStageFlags src_stage_mask = vk::PipelineStageFlagBits::eTopOfPipe; - - switch (previous_layout) { - case vk::ImageLayout::eUndefined: - src_access_mask = vk::AccessFlags {}; - src_stage_mask = vk::PipelineStageFlagBits::eTopOfPipe; - break; - case vk::ImageLayout::ePresentSrcKHR: - src_access_mask = vk::AccessFlags {}; - src_stage_mask = vk::PipelineStageFlagBits::eBottomOfPipe; - break; - case vk::ImageLayout::eTransferSrcOptimal: - src_access_mask = vk::AccessFlagBits::eTransferRead; - src_stage_mask = vk::PipelineStageFlagBits::eTransfer; - break; - case vk::ImageLayout::eColorAttachmentOptimal: - src_access_mask = vk::AccessFlagBits::eColorAttachmentWrite; - src_stage_mask = vk::PipelineStageFlagBits::eColorAttachmentOutput; - break; - default: - src_access_mask = vk::AccessFlags {}; - src_stage_mask = vk::PipelineStageFlagBits::eTopOfPipe; - break; - } - - transitionImageLayout( - frame.command_buffer, - image, - previous_layout, - vk::ImageLayout::eColorAttachmentOptimal, - src_access_mask, - vk::AccessFlagBits::eColorAttachmentWrite, - src_stage_mask, - vk::PipelineStageFlagBits::eColorAttachmentOutput, - colorSubresourceRange() - ); + sync::Usage& output_usage = m_output_image_usage.at(image_index); + barriers.image(image, colorSubresourceRange(), output_usage, sync::Usage::color_attachment); + output_usage = sync::Usage::color_attachment; + barriers.flush(*frame.command_buffer); vk::RenderingAttachmentInfo output_attachment_info {}; output_attachment_info.imageView = *m_output_target->getColorAttachment(image_index); @@ -2214,8 +1890,8 @@ auto VulkanRenderer::recordFrame(FrameContext& frame, uint32_t image_index) noex const GpuScope scope(m_gpu_timer.get(), m_current_frame, *frame.command_buffer, "Output finalize"); m_output_target->recordFinalize(frame.command_buffer, image_index); } - m_output_image_layouts.at(image_index) = - m_output_target->usesAcquirePresentSemaphores() ? vk::ImageLayout::ePresentSrcKHR : vk::ImageLayout::eTransferSrcOptimal; + // Must match the last transition IOutputTarget::recordFinalize records + output_usage = m_output_target->usesAcquirePresentSemaphores() ? sync::Usage::present : sync::Usage::transfer_src; TracyVkCollect(m_tracy_vk_ctx, *frame.command_buffer); @@ -2355,9 +2031,9 @@ auto VulkanRenderer::drawFrame(RenderFrame& frame_data) -> void { auto& frame = m_frames[m_current_frame]; std::ignore = m_core->getDevice().waitForFences(*frame.in_flight, true, std::numeric_limits::max()); - std::ignore = m_core->getDevice().waitForFences(*frame.compute_in_flight, true, std::numeric_limits::max()); end_phase(m_perf_gpu_wait_ns); publishCompletedFrames(); + m_core->updateTracyMemory(); if (m_gpu_timer != nullptr && !frame.was_probe_capture) { m_gpu_timer->collect(m_current_frame); @@ -2401,6 +2077,10 @@ auto VulkanRenderer::drawFrame(RenderFrame& frame_data) -> void { for (auto& pass : m_render_passes) { pass->update(m_current_frame, Time::renderDelta()); } + // Before recordFrame dispatches them + for (auto& pass : m_compute_passes) { + pass->update(m_current_frame, Time::renderDelta()); + } } if (const auto* rendering = renderingFrame(); rendering != nullptr) { @@ -2410,64 +2090,35 @@ auto VulkanRenderer::drawFrame(RenderFrame& frame_data) -> void { } recordFrame(frame, image_index); - - // Never conditional since the graphics submit waits on its semaphore - m_core->getDevice().resetFences(*frame.compute_in_flight); - - const vk::CommandBuffer compute_command_buffer = *frame.compute_command_buffer; - compute_command_buffer.reset(); - constexpr vk::CommandBufferBeginInfo compute_begin_info {}; - compute_command_buffer.begin(compute_begin_info); - if (m_gpu_timer != nullptr) { - m_gpu_timer->beginCompute(m_current_frame, compute_command_buffer); - } - for (auto& pass : m_compute_passes) { - pass->update(m_current_frame, Time::renderDelta()); - pass->dispatch(compute_command_buffer, m_current_frame); - } - if (m_gpu_timer != nullptr) { - m_gpu_timer->endCompute(m_current_frame, compute_command_buffer); - } - compute_command_buffer.end(); end_phase(m_perf_record_ns); - const vk::Semaphore compute_to_graphics_semaphore = *frame.compute_to_graphics; - const vk::SubmitInfo compute_submit_info(0, nullptr, nullptr, 1, &compute_command_buffer, 1, &compute_to_graphics_semaphore); - m_core->getComputeQueue().submit(compute_submit_info, *frame.compute_in_flight); - const bool present_sync = m_output_target->usesAcquirePresentSemaphores(); const vk::CommandBuffer command_buffer = *frame.command_buffer; const vk::Semaphore signal_semaphore = *m_render_finished_per_image.at(image_index); const vk::Semaphore wait_semaphore = *frame.image_available; - std::array wait_semaphores {compute_to_graphics_semaphore, wait_semaphore}; - std::array wait_stages { - vk::PipelineStageFlagBits::eFragmentShader, vk::PipelineStageFlagBits::eColorAttachmentOutput - }; - const uint32_t wait_semaphore_count = present_sync ? 2 : 1; + // AsyncCompute work never synchronises with a frame on the GPU + const sync::SemaphoreWait image_wait {wait_semaphore, vk::PipelineStageFlagBits2::eColorAttachmentOutput}; - vk::SubmitInfo submit_info {}; - submit_info.waitSemaphoreCount = wait_semaphore_count; - submit_info.pWaitSemaphores = wait_semaphores.data(); - submit_info.pWaitDstStageMask = wait_stages.data(); + sync::SubmitDesc submit {}; + submit.commands = {&command_buffer, 1}; if (present_sync) { - submit_info.signalSemaphoreCount = 1; - submit_info.pSignalSemaphores = &signal_semaphore; + submit.waits = {&image_wait, 1}; + submit.signals = {&signal_semaphore, 1}; } - submit_info.commandBufferCount = 1; - submit_info.pCommandBuffers = &command_buffer; + submit.fence = *frame.in_flight; vk::FrameBoundaryEXT frame_boundary {}; if (m_core->isFrameBoundarySupported()) { frame_boundary.flags = vk::FrameBoundaryFlagBitsEXT::eFrameEnd; frame_boundary.frameID = m_frame_boundary_counter++; - submit_info.pNext = &frame_boundary; + submit.next = &frame_boundary; } { std::scoped_lock submit_lock(m_core->graphicsSubmitMutex()); - m_core->getGraphicsQueue().submit(submit_info, *frame.in_flight); + sync::submit(m_core->getGraphicsQueue(), submit); } end_phase(m_perf_submit_ns); @@ -2908,7 +2559,6 @@ void VulkanRenderer::fitShadowViews( void VulkanRenderer::finalizeLights(RenderFrame& frame) { ZoneScoped; - using namespace clustered_lighting; auto& lights = frame.lights; auto& meta = m_tick_light_meta; @@ -2931,13 +2581,13 @@ void VulkanRenderer::finalizeLights(RenderFrame& frame) { meta.resize(kept); float light_reach = 0.0f; - if (kept > k_max_lights_per_cluster) { + if (kept > clustered_lighting::k_max_lights_per_cluster) { auto& order = m_tick_light_order; order.resize(kept); std::iota(order.begin(), order.end(), 0U); std::ranges::stable_sort(order, [&meta](uint32_t a, uint32_t b) { return meta[a].importance > meta[b].importance; }); - const size_t final_count = std::min(kept, k_max_lights); + const size_t final_count = std::min(kept, clustered_lighting::k_max_lights); frame.light_stats.truncated = static_cast(kept - final_count); auto& scratch = m_tick_light_scratch; @@ -3213,6 +2863,7 @@ void VulkanRenderer::tick(float time) noexcept { (overlays.surface_unit_grid ? 1u : 0u) | (overlays.surface_voxel_grid ? 2u : 0u) | (overlays.voxel_edges ? 4u : 0u); } + // PROBES SHIT { std::scoped_lock lock(m_reflection_probe_mutex); const auto probe_count = static_cast(std::min(m_reflection_probe_nodes.size(), k_max_reflection_probes)); @@ -3270,7 +2921,7 @@ void VulkanRenderer::tick(float time) noexcept { // Not Y flipped unlike Camera::getProjection() const glm::mat4 capture_projection = - glm::perspectiveRH_ZO(glm::radians(90.0f), 1.0f, m_camera->near_plane, m_camera->far_plane); + depth::perspective(glm::radians(90.0f), 1.0f, m_camera->near_plane, m_camera->far_plane); frame.frame_data.view = capture_view; frame.frame_data.projection = capture_projection; @@ -3442,7 +3093,7 @@ void VulkanRenderer::tick(float time) noexcept { const auto basis = cubeFaceBasis(face); frame.frame_data.view = glm::lookAt(probe_position, probe_position + basis.forward, basis.up); - frame.frame_data.projection = glm::perspectiveRH_ZO(glm::radians(90.0f), 1.0f, m_camera->near_plane, m_camera->far_plane); + frame.frame_data.projection = depth::perspective(glm::radians(90.0f), 1.0f, m_camera->near_plane, m_camera->far_plane); frame.frame_data.view_projection = frame.frame_data.projection * frame.frame_data.view; frame.frame_data.camera_position = probe_position; frame.irradiance_capture_index = probe_index; @@ -3466,6 +3117,7 @@ void VulkanRenderer::tick(float time) noexcept { frame.camera_near = m_camera->near_plane; frame.camera_far = m_camera->far_plane; + // Cameras debug draws { auto& camera_nodes_snapshot = m_tick_camera_nodes; { @@ -3484,12 +3136,15 @@ void VulkanRenderer::tick(float time) noexcept { debug::drawMesh(cameraGizmoMesh(), camera->getWorldTransform()); constexpr float k_camera_frustum_length = 3.0f; - const auto extent = m_output_target->getExtent(); - const float aspect = extent.height > 0 ? static_cast(extent.width) / static_cast(extent.height) : 1.0f; - debug::drawFrustum(*camera, aspect, glm::vec4(1.0f, 1.0f, 0.2f, 1.0f), k_camera_frustum_length); + const auto camera_extent = m_output_target->getExtent(); // TODO this might not always be the case + const float camera_aspect = + camera_extent.height > 0 ? static_cast(camera_extent.width) / static_cast(camera_extent.height) : 1.0f; + debug::drawFrustum(*camera, camera_aspect, glm::vec4(1.0f, 1.0f, 0.2f, 1.0f), k_camera_frustum_length); } } + // Lights shit + glm::vec3 ambient_sum(0.0f); uint32_t directional_count = 0; frame.lights.reserve(light_nodes_snapshot.size()); @@ -3504,7 +3159,7 @@ void VulkanRenderer::tick(float time) noexcept { m_tick_light_meta.clear(); const glm::mat4& frame_view = frame.frame_data.view; - const auto frustum = extractFrustumPlanes(frame.frame_data.view_projection); + const auto frustum = extractFrustumPlanes(frame.frame_data.view_projection, depth::k_reversed_z); struct PunctualSubmit { GpuLight gpu; @@ -3588,6 +3243,7 @@ void VulkanRenderer::tick(float time) noexcept { break; } case toast::LightType::directional: { + // FIXME debug::drawArrow(light->world_position, light->world_position + light->forward() * 2.0f, glm::vec4(light->color(), 1.0f)); if (directional_count >= k_max_directional_lights) { @@ -3609,7 +3265,7 @@ void VulkanRenderer::tick(float time) noexcept { const glm::vec3 world_pos = point->world_position; const glm::vec3 view_pos = glm::vec3(frame_view * glm::vec4(world_pos, 1.0f)); - // FIXME MOVE ELSEWHERE + // FIXME // debug::drawSphere(world_pos, point->attenuation(), glm::vec4(point->color(), 1.0f)); submit_punctual( @@ -3643,6 +3299,7 @@ void VulkanRenderer::tick(float time) noexcept { const float outer = glm::radians(spot->outerRadius()); const float inner = std::min(glm::radians(spot->innerRadius()), outer); + // FIXME debug::drawCone(world_pos, forward, spot->attenuation(), spot->outerRadius(), glm::vec4(spot->color(), 1.0f)); submit_punctual( @@ -3756,7 +3413,7 @@ void VulkanRenderer::tick(float time) noexcept { static std::optional last_state; if (!last_state.has_value() || *last_state != state) { last_state = state; - TOAST_INFO( + TOAST_TRACE( "Render", "Lighting inputs: ambient=({:.3f}, {:.3f}, {:.3f}), {} directional, {} punctual, environment={}, " "{} mesh instances, render mode {}, exposure {:.3f}, {} probes (baked mask {:#x}),", @@ -3775,8 +3432,15 @@ void VulkanRenderer::tick(float time) noexcept { } } - const bool trace_shadows = - m_core->isRayTracingSupported() && tracedShadowsEnabled() && std::ranges::any_of(frame.mesh_instances, isTraceable); + // Voxels are not in the TLAS so tracing would erase their shadows + // TODO remove once voxel visibility can be traced + const bool voxels_block_tracing = !frame.voxel_instances.empty(); + const bool trace_shadows = m_core->isRayTracingSupported() && tracedShadowsEnabled() && !voxels_block_tracing && + std::ranges::any_of(frame.mesh_instances, isTraceable); + if (tracedShadowsEnabled() && voxels_block_tracing && !m_traced_shadows_voxel_fallback_logged) { + TOAST_WARN("Render", "Traced shadows fall back to shadow maps while voxel volumes are drawn"); + } + m_traced_shadows_voxel_fallback_logged = tracedShadowsEnabled() && voxels_block_tracing; frame.frame_data.traced_shadow_params.x = trace_shadows ? 1.0f : 0.0f; if (!trace_shadows) { @@ -3820,7 +3484,7 @@ void VulkanRenderer::tick(float time) noexcept { } const glm::mat4 cull_view_projection = freeze ? m_frozen_cull_view_projection : frame.frame_data.view_projection; - const auto planes = extractFrustumPlanes(cull_view_projection); + const auto planes = extractFrustumPlanes(cull_view_projection, depth::k_reversed_z); uint32_t visible = 0; for (auto& proxy : frame.mesh_instances) { @@ -3953,6 +3617,7 @@ void VulkanRenderer::tick(float time) noexcept { } } + // ANTIALIASING { const bool is_capture = frame.probe_capture_index >= 0 || frame.irradiance_capture_index >= 0; const bool taa = taaEnabled() && !is_capture && frame.render_mode == 0; @@ -4489,7 +4154,7 @@ auto VulkanRenderer::applyResizeInternal(vk::Extent2D extent) -> void { const vk::Extent2D actual_extent = m_output_target->getExtent(); const auto image_count = m_output_target->getImageCount(); m_images_in_flight.assign(image_count, vk::Fence {}); - m_output_image_layouts.assign(image_count, vk::ImageLayout::eUndefined); + m_output_image_usage.assign(image_count, sync::Usage::none); createPerImageSync(); createDepthResources(); createSceneColorResources(); @@ -4553,6 +4218,21 @@ void VulkanRenderer::pumpUploadQueue() { } } +auto VulkanRenderer::uploadStats() -> UploadStats { + UploadStats stats {}; + stats.slot_count = static_cast(m_upload_slots.size()); + stats.slots_in_flight = + static_cast(std::ranges::count_if(m_upload_slots, [](const UploadSlot& slot) { return slot.in_flight; })); + stats.batches_in_flight = m_pending_uploads.size(); + stats.host_bytes = m_upload_host_bytes.load(std::memory_order_relaxed); + stats.host_budget = k_upload_host_budget; + { + std::scoped_lock lock(m_upload_mutex); + stats.waiting_jobs = m_upload_waiting.size(); + } + return stats; +} + void VulkanRenderer::processPendingUploads() { ZoneScoped; const auto& device = m_core->getDevice(); @@ -4567,6 +4247,8 @@ void VulkanRenderer::processPendingUploads() { vk::DeviceSize reclaimed = 0; for (auto& job : oldest_batch.jobs) { + // The fence wait above orders this after the upload release + job->recordAcquire(m_pending_acquires); job->finished(); reclaimed += job->host_bytes; } @@ -4630,6 +4312,7 @@ void VulkanRenderer::flushResourceUploads() { ++job; continue; } + (*job)->setQueueHandoff({m_core->getTransferQueueFamilyIndex(), m_core->getGraphicsQueueFamilyIndex()}); (*job)->record(*slot.command_buffer); recorded.push_back(std::move(*job)); job = jobs_to_flush.erase(job); @@ -4645,8 +4328,17 @@ void VulkanRenderer::flushResourceUploads() { batch.jobs = std::move(recorded); const vk::CommandBuffer raw_transfer_cmd = *slot.command_buffer; - const vk::SubmitInfo submit_info(0, nullptr, nullptr, 1, &raw_transfer_cmd); - m_core->getTransferQueue().submit(submit_info, *slot.fence); + { + // Only contended when AsyncCompute shares the transfer queue + std::scoped_lock transfer_lock(m_core->transferSubmitMutex()); + sync::submit( + m_core->getTransferQueue(), + { + .commands = {&raw_transfer_cmd, 1}, + .fence = *slot.fence + } + ); + } slot.in_flight = true; m_next_upload_slot = (m_next_upload_slot + 1) % k_upload_slots; diff --git a/engine/src/toast/renderer/vulkan_renderer.hpp b/engine/src/toast/renderer/vulkan_renderer.hpp index d3a562e88..a0cad6efc 100644 --- a/engine/src/toast/renderer/vulkan_renderer.hpp +++ b/engine/src/toast/renderer/vulkan_renderer.hpp @@ -6,6 +6,8 @@ #include "compute_pass_base.hpp" #include "debug_draw.hpp" +#include "depth_convention.hpp" +#include "gpu_sync.hpp" #include "gpu_timer.hpp" #include "output_target_base.hpp" #include "post_process_pass_base.hpp" @@ -93,11 +95,8 @@ class VulkanRenderer { struct FrameContext { vk::raii::CommandBuffer command_buffer = nullptr; - vk::raii::CommandBuffer compute_command_buffer = nullptr; vk::raii::Semaphore image_available = nullptr; - vk::raii::Semaphore compute_to_graphics = nullptr; vk::raii::Fence in_flight = nullptr; - vk::raii::Fence compute_in_flight = nullptr; uint32_t last_image_index = 0; bool has_submitted = false; bool was_probe_capture = false; @@ -468,8 +467,7 @@ class VulkanRenderer { void submitFrame() noexcept; - /// Queues a debug line for the current render frame, or for the next one when - /// called from gameplay code before frame construction begins. + /// For the next frame when called before frame construction begins void queueDebugLine(glm::vec3 a, glm::vec3 b, glm::vec4 color); void tick(float time) noexcept; @@ -901,6 +899,19 @@ class VulkanRenderer { }; } + struct UploadStats { + uint32_t slots_in_flight = 0; + uint32_t slot_count = 0; + size_t waiting_jobs = 0; + size_t batches_in_flight = 0; + vk::DeviceSize host_bytes = 0; + vk::DeviceSize host_budget = 0; + }; + + /// @note Render thread only + [[nodiscard]] + auto uploadStats() -> UploadStats; + /// @note Render thread only [[nodiscard]] auto gpuTimer() const noexcept -> const GpuTimer* { @@ -1010,17 +1021,17 @@ class VulkanRenderer { struct DepthResources { std::optional image; std::optional view; + sync::ImageState state; }; struct SceneColorResources { std::optional image; std::optional view; - vk::ImageLayout layout = vk::ImageLayout::eUndefined; + sync::ImageState state; }; void createGraphicsCommandPool(); void createTransferCommandPool(); - void createComputeCommandPool(); void createFrameContexts(); void createPerImageSync(); @@ -1086,7 +1097,6 @@ class VulkanRenderer { vk::raii::CommandPool m_command_pool = nullptr; vk::raii::CommandPool m_transfer_command_pool = nullptr; - vk::raii::CommandPool m_compute_command_pool = nullptr; vk::raii::DescriptorPool m_descriptor_pool = nullptr; std::vector m_upload_slots; @@ -1184,7 +1194,6 @@ class VulkanRenderer { vk::raii::Sampler m_present_sampler = nullptr; std::vector m_present_sets; std::vector m_present_bound_views; - vk::ImageLayout m_depth_layout = vk::ImageLayout::eUndefined; TracyVkCtx m_tracy_vk_ctx = nullptr; std::unique_ptr m_gpu_timer; @@ -1192,7 +1201,10 @@ class VulkanRenderer { std::vector m_frames; std::vector m_render_finished_per_image; std::vector m_images_in_flight; - std::vector m_output_image_layouts; + std::vector m_output_image_usage; + + /// Acquire halves of finished uploads flushed at the start of the next frame + sync::BarrierBatch m_pending_acquires; uint32_t m_current_frame = 0; toast::Camera* m_camera = nullptr; @@ -1205,6 +1217,8 @@ class VulkanRenderer { PostProcessSettings m_post_process_settings; std::atomic_bool m_traced_shadows_enabled {false}; + /// Logs once each time the fallback starts + bool m_traced_shadows_voxel_fallback_logged = false; std::atomic_bool m_taa_enabled {true}; std::atomic m_taa_history_weight {0.9f}; diff --git a/engine/src/toast/renderer/vulkan_resource_base.hpp b/engine/src/toast/renderer/vulkan_resource_base.hpp index 9059cc235..06304a99c 100644 --- a/engine/src/toast/renderer/vulkan_resource_base.hpp +++ b/engine/src/toast/renderer/vulkan_resource_base.hpp @@ -67,7 +67,16 @@ class PendingResourceUpload { } } + /// Before record(). The default keeps one queue and record() ends in a plain barrier + void setQueueHandoff(const sync::QueueHandoff& handoff) noexcept { m_handoff = handoff; } + + /// On the consuming queue after the upload fence signalled. No op for an upload that stayed on one queue + virtual void recordAcquire(sync::BarrierBatch& batch) { (void)batch; } + vk::DeviceSize host_bytes = 0; + +protected: + sync::QueueHandoff m_handoff; }; } diff --git a/engine/src/toast/renderer/vulkan_texture.cpp b/engine/src/toast/renderer/vulkan_texture.cpp index bcc3c4155..8e33a4f44 100644 --- a/engine/src/toast/renderer/vulkan_texture.cpp +++ b/engine/src/toast/renderer/vulkan_texture.cpp @@ -187,35 +187,26 @@ void TextureUpload::record(vk::CommandBuffer cmd) { return; } - vk::Image image_handle = m_texture->getImage(); - - vk::ImageMemoryBarrier barrier {}; - barrier.oldLayout = vk::ImageLayout::eUndefined; - barrier.newLayout = vk::ImageLayout::eTransferDstOptimal; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.image = image_handle; - barrier.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; - barrier.subresourceRange.baseMipLevel = 0; - barrier.subresourceRange.levelCount = m_tex_params.mip_levels; - barrier.subresourceRange.baseArrayLayer = 0; - barrier.subresourceRange.layerCount = m_tex_params.layer_count; - barrier.srcAccessMask = {}; - barrier.dstAccessMask = vk::AccessFlagBits::eTransferWrite; - - cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTopOfPipe, vk::PipelineStageFlagBits::eTransfer, {}, nullptr, nullptr, barrier); + const vk::Image image_handle = m_texture->getImage(); + const vk::ImageSubresourceRange range(vk::ImageAspectFlagBits::eColor, 0, m_tex_params.mip_levels, 0, m_tex_params.layer_count); + + sync::BarrierBatch barriers; + barriers.image(image_handle, range, sync::Usage::none, sync::Usage::transfer_dst); + barriers.flush(cmd); cmd.copyBufferToImage(*m_staging_buffer, image_handle, vk::ImageLayout::eTransferDstOptimal, m_copy_regions); - barrier.oldLayout = vk::ImageLayout::eTransferDstOptimal; - barrier.newLayout = vk::ImageLayout::eShaderReadOnlyOptimal; - barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; - // Transfer queue barrier since that queue has no fragment shader stage - barrier.dstAccessMask = {}; + barriers.release(image_handle, range, sync::Usage::transfer_dst, sync::Usage::shader_sampled, m_handoff); + barriers.flush(cmd); + m_released = true; +} - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eBottomOfPipe, {}, nullptr, nullptr, barrier - ); +void TextureUpload::recordAcquire(sync::BarrierBatch& batch) { + if (!m_released || m_texture->hasFailed()) { + return; + } + const vk::ImageSubresourceRange range(vk::ImageAspectFlagBits::eColor, 0, m_tex_params.mip_levels, 0, m_tex_params.layer_count); + batch.acquire(m_texture->getImage(), range, sync::Usage::transfer_dst, sync::Usage::shader_sampled, m_handoff); } void RawTextureUpload::build(const VulkanCore& core) { @@ -257,21 +248,9 @@ void RawTextureUpload::record(vk::CommandBuffer cmd) { ZoneScoped; const vk::Image image = m_texture->getImage(); - vk::ImageMemoryBarrier barrier {}; - barrier.oldLayout = vk::ImageLayout::eUndefined; - barrier.newLayout = vk::ImageLayout::eTransferDstOptimal; - barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - barrier.image = image; - barrier.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; - barrier.subresourceRange.baseMipLevel = 0; - barrier.subresourceRange.levelCount = 1; - barrier.subresourceRange.baseArrayLayer = 0; - barrier.subresourceRange.layerCount = 1; - barrier.srcAccessMask = {}; - barrier.dstAccessMask = vk::AccessFlagBits::eTransferWrite; - - cmd.pipelineBarrier(vk::PipelineStageFlagBits::eTopOfPipe, vk::PipelineStageFlagBits::eTransfer, {}, nullptr, nullptr, barrier); + sync::BarrierBatch barriers; + barriers.image(image, colorSubresourceRange(), sync::Usage::none, sync::Usage::transfer_dst); + barriers.flush(cmd); vk::BufferImageCopy region {}; region.bufferOffset = 0; @@ -283,14 +262,17 @@ void RawTextureUpload::record(vk::CommandBuffer cmd) { cmd.copyBufferToImage(*m_staging_buffer, image, vk::ImageLayout::eTransferDstOptimal, region); - barrier.oldLayout = vk::ImageLayout::eTransferDstOptimal; - barrier.newLayout = vk::ImageLayout::eShaderReadOnlyOptimal; - barrier.srcAccessMask = vk::AccessFlagBits::eTransferWrite; - - barrier.dstAccessMask = {}; + barriers.release(image, colorSubresourceRange(), sync::Usage::transfer_dst, sync::Usage::shader_sampled, m_handoff); + barriers.flush(cmd); + m_released = true; +} - cmd.pipelineBarrier( - vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eBottomOfPipe, {}, nullptr, nullptr, barrier +void RawTextureUpload::recordAcquire(sync::BarrierBatch& batch) { + if (!m_released || m_texture->hasFailed()) { + return; + } + batch.acquire( + m_texture->getImage(), colorSubresourceRange(), sync::Usage::transfer_dst, sync::Usage::shader_sampled, m_handoff ); } @@ -315,7 +297,16 @@ auto uploadTextureSync(const VulkanCore& core, VulkanTexture& texture, std::vect const vk::raii::Fence fence(device, vk::FenceCreateInfo {}); const vk::CommandBuffer raw_cmd = *cmd; - core.getGraphicsQueue().submit(vk::SubmitInfo(0, nullptr, nullptr, 1, &raw_cmd), *fence); + { + std::scoped_lock queue_lock(core.graphicsSubmitMutex()); + sync::submit( + core.getGraphicsQueue(), + { + .commands = {&raw_cmd, 1}, + .fence = *fence + } + ); + } if (device.waitForFences(*fence, vk::True, std::numeric_limits::max()) != vk::Result::eSuccess) { texture.markFailed(IVulkanResource::UploadState::failed_gpu); return false; diff --git a/engine/src/toast/renderer/vulkan_texture.hpp b/engine/src/toast/renderer/vulkan_texture.hpp index 842dc5b35..739fcb527 100644 --- a/engine/src/toast/renderer/vulkan_texture.hpp +++ b/engine/src/toast/renderer/vulkan_texture.hpp @@ -63,6 +63,7 @@ class TextureUpload : public PendingResourceUpload { void build(const VulkanCore& core) override; void record(vk::CommandBuffer cmd) override; + void recordAcquire(sync::BarrierBatch& batch) override; auto resource() -> IVulkanResource* override { return m_texture; } @@ -75,6 +76,8 @@ class TextureUpload : public PendingResourceUpload { vma::raii::Buffer m_staging_buffer = nullptr; std::vector m_copy_regions; VulkanTexture::Params m_tex_params {}; + /// So recordAcquire() has a release to match + bool m_released = false; }; class RawTextureUpload : public PendingResourceUpload { @@ -92,6 +95,7 @@ class RawTextureUpload : public PendingResourceUpload { void build(const VulkanCore& core) override; void record(vk::CommandBuffer cmd) override; + void recordAcquire(sync::BarrierBatch& batch) override; auto resource() -> IVulkanResource* override { return m_texture; } @@ -103,6 +107,7 @@ class RawTextureUpload : public PendingResourceUpload { vk::Format m_format = vk::Format::eUndefined; std::string m_debug_name; vma::raii::Buffer m_staging_buffer = nullptr; + bool m_released = false; }; auto uploadTextureSync(const VulkanCore& core, VulkanTexture& texture, std::vector data, std::string_view debug_name) diff --git a/engine/src/toast/ui/render/rmlui_renderer_vk.cpp b/engine/src/toast/ui/render/rmlui_renderer_vk.cpp index 06a31637a..0ae39714b 100644 --- a/engine/src/toast/ui/render/rmlui_renderer_vk.cpp +++ b/engine/src/toast/ui/render/rmlui_renderer_vk.cpp @@ -520,18 +520,7 @@ Rml::TextureHandle range.levelCount = 1; range.layerCount = 1; - VkImageMemoryBarrier info_barrier = {}; - info_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; - info_barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; - info_barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - info_barrier.image = p_image; - info_barrier.subresourceRange = range; - info_barrier.srcAccessMask = 0; - info_barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - - vkCmdPipelineBarrier( - p_cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &info_barrier - ); + rmlui_vk_sync::TransitionImage(p_cmd, p_image, range, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); VkBufferImageCopy region = {}; region.bufferOffset = 0; @@ -546,27 +535,8 @@ Rml::TextureHandle vkCmdCopyBufferToImage(p_cmd, cpu_buffer.m_p_vk_buffer, p_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); - VkImageMemoryBarrier info_barrier_shader_read = {}; - info_barrier_shader_read.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; - info_barrier_shader_read.pNext = nullptr; - info_barrier_shader_read.image = p_image; - info_barrier_shader_read.subresourceRange = range; - info_barrier_shader_read.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - info_barrier_shader_read.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; - info_barrier_shader_read.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - info_barrier_shader_read.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; - - vkCmdPipelineBarrier( - p_cmd, - VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, - 0, - 0, - nullptr, - 0, - nullptr, - 1, - &info_barrier_shader_read + rmlui_vk_sync::TransitionImage( + p_cmd, p_image, range, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL ); }); @@ -785,7 +755,7 @@ void RenderInterface_VK::SetViewport(int width, int height) { bool RenderInterface_VK::Initialize( VkInstance instance, VkPhysicalDevice physical_device, VkDevice device, uint32_t queue_family_index, VkQueue queue, - VkFormat color_format, VkFormat depth_stencil_format, std::mutex* submit_mutex + VkFormat color_format, VkFormat depth_stencil_format, std::mutex* submit_mutex, VkPipelineCache pipeline_cache ) { RMLUI_ZoneScopedN("Vulkan - Initialize"); RMLUI_VK_ASSERTMSG(instance && physical_device && device && queue, "you must pass valid Vulkan handles from the engine"); @@ -797,6 +767,7 @@ bool RenderInterface_VK::Initialize( m_color_attachment_format = color_format; m_depth_stencil_attachment_format = depth_stencil_format; m_p_submit_mutex = submit_mutex; + m_p_pipeline_cache = pipeline_cache; VmaVulkanFunctions vulkan_functions = {}; vulkan_functions.vkGetInstanceProcAddr = vkGetInstanceProcAddr; @@ -1161,7 +1132,7 @@ void RenderInterface_VK::Create_Pipelines() noexcept { info.renderPass = VK_NULL_HANDLE; info.subpass = 0; - auto status = vkCreateGraphicsPipelines(m_p_device, nullptr, 1, &info, nullptr, &m_p_pipeline_with_textures); + auto status = vkCreateGraphicsPipelines(m_p_device, m_p_pipeline_cache, 1, &info, nullptr, &m_p_pipeline_with_textures); RMLUI_VK_ASSERTMSG(status == VkResult::VK_SUCCESS, "failed to vkCreateGraphicsPipelines"); info_depth.back.passOp = VK_STENCIL_OP_KEEP; @@ -1174,7 +1145,12 @@ void RenderInterface_VK::Create_Pipelines() noexcept { info_depth.front = info_depth.back; status = vkCreateGraphicsPipelines( - m_p_device, nullptr, 1, &info, nullptr, &m_p_pipeline_stencil_for_regular_geometry_that_applied_to_region_with_textures + m_p_device, + m_p_pipeline_cache, + 1, + &info, + nullptr, + &m_p_pipeline_stencil_for_regular_geometry_that_applied_to_region_with_textures ); RMLUI_VK_ASSERTMSG(status == VkResult::VK_SUCCESS, "failed to vkCreateGraphicsPipelines"); @@ -1189,7 +1165,7 @@ void RenderInterface_VK::Create_Pipelines() noexcept { info_depth.back.reference = 1; info_depth.front = info_depth.back; - status = vkCreateGraphicsPipelines(m_p_device, nullptr, 1, &info, nullptr, &m_p_pipeline_without_textures); + status = vkCreateGraphicsPipelines(m_p_device, m_p_pipeline_cache, 1, &info, nullptr, &m_p_pipeline_without_textures); RMLUI_VK_ASSERTMSG(status == VkResult::VK_SUCCESS, "failed to vkCreateGraphicsPipelines"); info_depth.back.passOp = VK_STENCIL_OP_KEEP; @@ -1202,7 +1178,12 @@ void RenderInterface_VK::Create_Pipelines() noexcept { info_depth.front = info_depth.back; status = vkCreateGraphicsPipelines( - m_p_device, nullptr, 1, &info, nullptr, &m_p_pipeline_stencil_for_regular_geometry_that_applied_to_region_without_textures + m_p_device, + m_p_pipeline_cache, + 1, + &info, + nullptr, + &m_p_pipeline_stencil_for_regular_geometry_that_applied_to_region_without_textures ); RMLUI_VK_ASSERTMSG(status == VkResult::VK_SUCCESS, "failed to vkCreateGraphicsPipelines"); @@ -1217,7 +1198,7 @@ void RenderInterface_VK::Create_Pipelines() noexcept { info_depth.front = info_depth.back; status = vkCreateGraphicsPipelines( - m_p_device, nullptr, 1, &info, nullptr, &m_p_pipeline_stencil_for_region_where_geometry_will_be_drawn + m_p_device, m_p_pipeline_cache, 1, &info, nullptr, &m_p_pipeline_stencil_for_region_where_geometry_will_be_drawn ); RMLUI_VK_ASSERTMSG(status == VkResult::VK_SUCCESS, "failed to vkCreateGraphicsPipelines"); @@ -1225,7 +1206,7 @@ void RenderInterface_VK::Create_Pipelines() noexcept { info_depth.back.passOp = VK_STENCIL_OP_INCREMENT_AND_CLAMP; info_depth.front = info_depth.back; - status = vkCreateGraphicsPipelines(m_p_device, nullptr, 1, &info, nullptr, &m_p_pipeline_clip_write_incr); + status = vkCreateGraphicsPipelines(m_p_device, m_p_pipeline_cache, 1, &info, nullptr, &m_p_pipeline_clip_write_incr); RMLUI_VK_ASSERTMSG(status == VkResult::VK_SUCCESS, "failed to vkCreateGraphicsPipelines"); } diff --git a/engine/src/toast/ui/render/rmlui_renderer_vk.h b/engine/src/toast/ui/render/rmlui_renderer_vk.h index 5f2d857ef..75d6feaee 100644 --- a/engine/src/toast/ui/render/rmlui_renderer_vk.h +++ b/engine/src/toast/ui/render/rmlui_renderer_vk.h @@ -30,6 +30,73 @@ #define RMLUI_VK_ASSERTMSG(statement, msg) static_cast(statement) #endif +namespace rmlui_vk_sync { + +struct LayoutScope { + VkPipelineStageFlags2 stages; + VkAccessFlags2 access; +}; + +inline constexpr VkAccessFlags2 kWriteAccess = VK_ACCESS_2_SHADER_WRITE_BIT | VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT | + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT | + VK_ACCESS_2_HOST_WRITE_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT; + +inline LayoutScope ScopeForLayout(VkImageLayout layout) { + switch (layout) { + case VK_IMAGE_LAYOUT_UNDEFINED: return {VK_PIPELINE_STAGE_2_NONE, VK_ACCESS_2_NONE}; + case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL: + return { + VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT + }; + case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL: + return { + VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT, + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT + }; + case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL: + return {VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT, VK_ACCESS_2_SHADER_SAMPLED_READ_BIT}; + case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: + return {VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_BLIT_BIT, VK_ACCESS_2_TRANSFER_READ_BIT}; + case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: + return { + VK_PIPELINE_STAGE_2_COPY_BIT | VK_PIPELINE_STAGE_2_BLIT_BIT | VK_PIPELINE_STAGE_2_CLEAR_BIT, + VK_ACCESS_2_TRANSFER_WRITE_BIT + }; + default: return {VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT}; + } +} + +/// Only source writes are made available since reads need the execution dependency alone +inline void TransitionImage( + VkCommandBuffer cmd, VkImage image, const VkImageSubresourceRange& range, VkImageLayout old_layout, VkImageLayout new_layout +) { + const LayoutScope src = ScopeForLayout(old_layout); + const LayoutScope dst = ScopeForLayout(new_layout); + + VkImageMemoryBarrier2 barrier = {}; + barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2; + barrier.srcStageMask = src.stages; + barrier.srcAccessMask = src.access & kWriteAccess; + barrier.dstStageMask = dst.stages; + barrier.dstAccessMask = dst.access; + barrier.oldLayout = old_layout; + barrier.newLayout = new_layout; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image; + barrier.subresourceRange = range; + + VkDependencyInfo dependency = {}; + dependency.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO; + dependency.imageMemoryBarrierCount = 1; + dependency.pImageMemoryBarriers = &barrier; + vkCmdPipelineBarrier2(cmd, &dependency); +} + +} // namespace rmlui_vk_sync + class RenderInterface_VK : public Rml::RenderInterface { public: // Matches the engine renderer's frames-in-flight; per-frame resources cycle on this @@ -46,9 +113,11 @@ class RenderInterface_VK : public Rml::RenderInterface { /// Borrows the engine's Vulkan handles; the backend owns only its own resources. /// @param submit_mutex guards graphics queue submission shared with the render thread + /// @param pipeline_cache shared with the engine so UI pipelines are cached across runs bool Initialize( VkInstance instance, VkPhysicalDevice physical_device, VkDevice device, uint32_t queue_family_index, VkQueue queue, - VkFormat color_format, VkFormat depth_stencil_format, std::mutex* submit_mutex + VkFormat color_format, VkFormat depth_stencil_format, std::mutex* submit_mutex, + VkPipelineCache pipeline_cache = VK_NULL_HANDLE ); void Shutdown(); @@ -321,17 +390,14 @@ class RenderInterface_VK : public Rml::RenderInterface { } void Submit() noexcept { - VkSubmitInfo info = {}; + VkCommandBufferSubmitInfo command_info = {}; + command_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO; + command_info.commandBuffer = m_p_command_buffer; - info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; - info.pNext = nullptr; - info.waitSemaphoreCount = 0; - info.signalSemaphoreCount = 0; - info.pSignalSemaphores = nullptr; - info.pWaitSemaphores = nullptr; - info.pWaitDstStageMask = nullptr; - info.pCommandBuffers = &m_p_command_buffer; - info.commandBufferCount = 1; + VkSubmitInfo2 info = {}; + info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2; + info.commandBufferInfoCount = 1; + info.pCommandBufferInfos = &command_info; // The render thread submits on the same queue; queues need external synchronization std::unique_lock lock; @@ -339,9 +405,9 @@ class RenderInterface_VK : public Rml::RenderInterface { lock = std::unique_lock(*m_p_submit_mutex); } - auto status = vkQueueSubmit(m_p_graphics_queue, 1, &info, m_p_fence); + auto status = vkQueueSubmit2(m_p_graphics_queue, 1, &info, m_p_fence); - RMLUI_VK_ASSERTMSG(status == VkResult::VK_SUCCESS, "failed to vkQueueSubmit"); + RMLUI_VK_ASSERTMSG(status == VkResult::VK_SUCCESS, "failed to vkQueueSubmit2"); } private: @@ -645,6 +711,7 @@ class RenderInterface_VK : public Rml::RenderInterface { VkInstance m_p_instance; VkDevice m_p_device; + VkPipelineCache m_p_pipeline_cache = VK_NULL_HANDLE; VkPhysicalDevice m_p_physical_device; VmaAllocator m_p_allocator; // @ the secondary command buffer between BeginRecording/EndRecording diff --git a/engine/src/toast/ui/render/rmlui_renderer_vk_effects.cpp b/engine/src/toast/ui/render/rmlui_renderer_vk_effects.cpp index bbae1ed9e..f1637de6a 100644 --- a/engine/src/toast/ui/render/rmlui_renderer_vk_effects.cpp +++ b/engine/src/toast/ui/render/rmlui_renderer_vk_effects.cpp @@ -290,7 +290,7 @@ void RenderInterface_VK::CreateEffectResources() noexcept { info.layout = m_p_pipeline_layout_effects; VkPipeline p_pipeline = nullptr; - VkResult status = vkCreateGraphicsPipelines(m_p_device, nullptr, 1, &info, nullptr, &p_pipeline); + VkResult status = vkCreateGraphicsPipelines(m_p_device, m_p_pipeline_cache, 1, &info, nullptr, &p_pipeline); RMLUI_VK_ASSERTMSG(status == VK_SUCCESS, "failed to vkCreateGraphicsPipelines (effects)"); return p_pipeline; }; @@ -410,7 +410,7 @@ void RenderInterface_VK::CreateEffectResources() noexcept { info.layout = m_p_pipeline_layout_gradient; VkPipeline p_pipeline = nullptr; - VkResult status = vkCreateGraphicsPipelines(m_p_device, nullptr, 1, &info, nullptr, &p_pipeline); + VkResult status = vkCreateGraphicsPipelines(m_p_device, m_p_pipeline_cache, 1, &info, nullptr, &p_pipeline); RMLUI_VK_ASSERTMSG(status == VK_SUCCESS, "failed to vkCreateGraphicsPipelines (gradient)"); return p_pipeline; }; @@ -665,35 +665,17 @@ void RenderInterface_VK::TransitionEffectImage(effect_image_t& image, VkImageLay const bool is_stencil = image.m_p_image == m_p_current_pool->m_stencil.m_p_image; - VkImageMemoryBarrier barrier = {}; - barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; - barrier.oldLayout = image.m_layout; - barrier.newLayout = new_layout; - barrier.image = image.m_p_image; - barrier.subresourceRange.baseMipLevel = 0; - barrier.subresourceRange.levelCount = 1; - barrier.subresourceRange.baseArrayLayer = 0; - barrier.subresourceRange.layerCount = 1; - barrier.subresourceRange.aspectMask = is_stencil ? (m_depth_stencil_attachment_format == VK_FORMAT_S8_UINT - ? VK_IMAGE_ASPECT_STENCIL_BIT - : (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) - : VK_IMAGE_ASPECT_COLOR_BIT; - barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; - barrier.dstAccessMask = VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_MEMORY_READ_BIT; - - // All-stage barrier - vkCmdPipelineBarrier( - m_p_current_command_buffer, - VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, - VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, - 0, - 0, - nullptr, - 0, - nullptr, - 1, - &barrier - ); + VkImageSubresourceRange range = {}; + range.baseMipLevel = 0; + range.levelCount = 1; + range.baseArrayLayer = 0; + range.layerCount = 1; + range.aspectMask = is_stencil ? (m_depth_stencil_attachment_format == VK_FORMAT_S8_UINT + ? VK_IMAGE_ASPECT_STENCIL_BIT + : (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) + : VK_IMAGE_ASPECT_COLOR_BIT; + + rmlui_vk_sync::TransitionImage(m_p_current_command_buffer, image.m_p_image, range, image.m_layout, new_layout); image.m_layout = new_layout; } @@ -1105,25 +1087,8 @@ Rml::TextureHandle RenderInterface_VK::SaveLayerAsTexture() { range.levelCount = 1; range.layerCount = 1; - VkImageMemoryBarrier to_dst = {}; - to_dst.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; - to_dst.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; - to_dst.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - to_dst.image = p_texture->m_p_vk_image; - to_dst.subresourceRange = range; - to_dst.srcAccessMask = 0; - to_dst.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - vkCmdPipelineBarrier( - m_p_current_command_buffer, - VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, - 0, - nullptr, - 0, - nullptr, - 1, - &to_dst + rmlui_vk_sync::TransitionImage( + m_p_current_command_buffer, p_texture->m_p_vk_image, range, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL ); VkImageCopy region = {}; @@ -1141,25 +1106,12 @@ Rml::TextureHandle RenderInterface_VK::SaveLayerAsTexture() { ®ion ); - VkImageMemoryBarrier to_read = {}; - to_read.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; - to_read.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - to_read.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; - to_read.image = p_texture->m_p_vk_image; - to_read.subresourceRange = range; - to_read.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - to_read.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; - vkCmdPipelineBarrier( + rmlui_vk_sync::TransitionImage( m_p_current_command_buffer, - VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, - 0, - 0, - nullptr, - 0, - nullptr, - 1, - &to_read + p_texture->m_p_vk_image, + range, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL ); VkImageViewCreateInfo info_view = {}; diff --git a/engine/src/toast/ui/ui_system.cpp b/engine/src/toast/ui/ui_system.cpp index b42a50277..8f1ee0278 100644 --- a/engine/src/toast/ui/ui_system.cpp +++ b/engine/src/toast/ui/ui_system.cpp @@ -79,7 +79,8 @@ void UISystem::initializeRenderer(const renderer::VulkanCore& core) { static_cast(core.getGraphicsQueue()), static_cast(UIPass::k_color_format), static_cast(stencil_format), - &core.graphicsSubmitMutex() + &core.graphicsSubmitMutex(), + static_cast(*core.getPipelineCache()) ); if (!ok) { @@ -186,8 +187,7 @@ void UISystem::tick() noexcept { if (m_documents_reload_pending) { m_documents_reload_pending = false; if (m_stylesheet_cache_dirty) { - // RmlUi caches parsed external stylesheets by URI. Rebuilding a document - // alone would otherwise link the old parsed RCSS again. + // RmlUi caches parsed stylesheets by URI so a rebuild alone would link the old RCSS again Rml::Factory::ClearStyleSheetCache(); m_stylesheet_cache_dirty = false; } @@ -339,7 +339,7 @@ auto UISystem::colorFromSchemes(std::string_view name) const -> std::optional +#include #include namespace toast { @@ -88,7 +89,7 @@ auto Camera::getView() const -> glm::mat4 { } auto Camera::getProjection(float aspect) const -> glm::mat4 { - glm::mat4 proj = glm::perspectiveRH_ZO(glm::radians(fov), aspect, near_plane, far_plane); + glm::mat4 proj = renderer::depth::perspective(glm::radians(fov), aspect, near_plane, far_plane); proj[1][1] *= -1.0f; @@ -112,8 +113,8 @@ auto Camera::screenPointToRay(glm::vec2 screen_px, glm::vec2 viewport_size) cons const glm::mat4 inv_view_proj = glm::inverse(getProjection(aspect) * getView()); - glm::vec4 near_point = inv_view_proj * glm::vec4(ndc.x, ndc.y, 0.0f, 1.0f); - glm::vec4 far_point = inv_view_proj * glm::vec4(ndc.x, ndc.y, 1.0f, 1.0f); + glm::vec4 near_point = inv_view_proj * glm::vec4(ndc.x, ndc.y, renderer::depth::k_near, 1.0f); + glm::vec4 far_point = inv_view_proj * glm::vec4(ndc.x, ndc.y, renderer::depth::k_far, 1.0f); near_point /= near_point.w; far_point /= far_point.w; From df2d85c7ae50368c7b0dc6548e3ac3324f87fa52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dar=C3=ADo=20Rodr=C3=ADguez=20Mart=C3=ADnez?= Date: Sat, 10 Oct 2026 15:45:09 +0200 Subject: [PATCH 3/7] tidi --- .../src/toast/renderer/async_compute_task.cpp | 5 +++- .../src/toast/renderer/passes/perf_window.cpp | 23 ++++++++++++------- .../src/toast/renderer/passes/shadow_pass.hpp | 2 +- .../src/toast/renderer/passes/voxel_pass.hpp | 2 +- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/engine/src/toast/renderer/async_compute_task.cpp b/engine/src/toast/renderer/async_compute_task.cpp index 23486f88d..ab5440396 100644 --- a/engine/src/toast/renderer/async_compute_task.cpp +++ b/engine/src/toast/renderer/async_compute_task.cpp @@ -173,7 +173,10 @@ void AsyncComputeTask::writeDescriptors(uint32_t slot) { writes.reserve(m_bindings.size()); for (const Binding& binding : m_bindings) { const Buffer& buffer = m_buffers[binding.id]; - const uint32_t copy = !buffer.ring ? 0 : (binding.copy == Copy::current ? slot : previous); + const uint32_t copy = + !buffer.ring + ? 0 + : (binding.copy == Copy::current ? slot : previous); // NOLINT(readability-avoid-nested-conditional-operator) infos.emplace_back(*buffer.copies[copy], 0, buffer.size); writes.emplace_back(*m_slots[slot].set, binding.binding, 0, 1, vk::DescriptorType::eStorageBuffer, nullptr, &infos.back()); } diff --git a/engine/src/toast/renderer/passes/perf_window.cpp b/engine/src/toast/renderer/passes/perf_window.cpp index e641c713e..dc21de6d5 100644 --- a/engine/src/toast/renderer/passes/perf_window.cpp +++ b/engine/src/toast/renderer/passes/perf_window.cpp @@ -329,7 +329,7 @@ void PerfWindow::drawFrameGraph(float width, float height) { // Budget and the recent worst case stay on screen const float target = std::max({m_sample.p99_interval_ms * 1.2f, m_budget_ms * 1.5f, m_gpu_ms * 1.5f, 1.0f}); - m_graph_scale_ms = m_graph_scale_ms <= 0.0f ? target : m_graph_scale_ms + (target - m_graph_scale_ms) * 0.05f; + m_graph_scale_ms = m_graph_scale_ms <= 0.0f ? target : m_graph_scale_ms + ((target - m_graph_scale_ms) * 0.05f); const float scale = m_graph_scale_ms; ImDrawList* draw = ImGui::GetWindowDrawList(); @@ -337,22 +337,22 @@ void PerfWindow::drawFrameGraph(float width, float height) { draw->AddRectFilled(origin, end, k_perf_graph_bg, 3.0f); const float bar_width = width / static_cast(k_history); - const auto y_of = [&](float ms) { return end.y - std::clamp(ms / scale, 0.0f, 1.0f) * height; }; + const auto y_of = [&](float ms) { return end.y - (std::clamp(ms / scale, 0.0f, 1.0f) * height); }; const int offset = historyOffset(); const auto count = static_cast(m_filled); // Newest frame at the right edge - const float first_x = end.x - bar_width * static_cast(count); + const float first_x = end.x - (bar_width * static_cast(count)); std::array gpu_points {}; int hovered_index = -1; for (int i = 0; i < count; ++i) { const size_t index = static_cast((offset + i) % static_cast(k_history)); const float ms = m_interval_history[index]; - const float x0 = first_x + bar_width * static_cast(i); + const float x0 = first_x + (bar_width * static_cast(i)); const ImU32 color = ms <= m_budget_ms * 1.05f ? k_perf_bar_good : (ms <= m_budget_ms * 1.5f ? k_perf_bar_warn : k_perf_bar_bad); draw->AddRectFilled(ImVec2(x0, y_of(ms)), ImVec2(x0 + std::max(bar_width - 0.5f, 1.0f), end.y), color); - gpu_points[static_cast(i)] = ImVec2(x0 + bar_width * 0.5f, y_of(m_gpu_history[index])); + gpu_points[static_cast(i)] = ImVec2(x0 + (bar_width * 0.5f), y_of(m_gpu_history[index])); if (hovered && ImGui::GetIO().MousePos.x >= x0 && ImGui::GetIO().MousePos.x < x0 + bar_width) { hovered_index = static_cast(index); @@ -391,6 +391,7 @@ void PerfWindow::drawSummary(bool has_gpu) { auto* renderer = VulkanRenderer::instance; const auto cap = static_cast(renderer->effectiveFrameRateLimit()); + // NOLINTNEXTLINE(readability-avoid-nested-conditional-operator) const ImVec4 fps_color = m_interval_ms <= m_budget_ms * 1.05f ? k_perf_good : m_interval_ms <= m_budget_ms * 1.5f ? k_perf_warn : k_perf_bad; @@ -421,9 +422,13 @@ void PerfWindow::drawSummary(bool has_gpu) { drawPhaseBar(width, ImGui::GetFontSize() * 0.6f); const Verdict render = renderVerdict(has_gpu); - perfVerdictText("Render", render.bad ? k_perf_bad : (render.warn ? k_perf_warn : k_perf_good), render.label, render.detail); + perfVerdictText( + "Render", render.bad ? k_perf_bad : (render.warn ? k_perf_warn : k_perf_good), render.label, render.detail + ); // NOLINT(readability-avoid-nested-conditional-operator) const Verdict game = gameVerdict(); - perfVerdictText("Game ", game.bad ? k_perf_bad : (game.warn ? k_perf_warn : k_perf_good), game.label, game.detail); + perfVerdictText( + "Game ", game.bad ? k_perf_bad : (game.warn ? k_perf_warn : k_perf_good), game.label, game.detail + ); // NOLINT(readability-avoid-nested-conditional-operator) } void PerfWindow::drawFrameTab() { @@ -712,7 +717,9 @@ void PerfWindow::drawMemoryTab() { ImGui::TextDisabled("%.0f MiB total", perfMiB(properties.memoryHeaps[heap].size)); const std::string label = std::format("{:.0f} / {:.0f} MiB", perfMiB(budget.usage), perfMiB(budget.budget)); - ImGui::PushStyleColor(ImGuiCol_PlotHistogram, fraction < 0.75f ? k_perf_good : (fraction < 0.9f ? k_perf_warn : k_perf_bad)); + ImGui::PushStyleColor( + ImGuiCol_PlotHistogram, fraction < 0.75f ? k_perf_good : (fraction < 0.9f ? k_perf_warn : k_perf_bad) + ); // NOLINT(readability-avoid-nested-conditional-operator) ImGui::ProgressBar(fraction, ImVec2(ImGui::GetFontSize() * 30.0f, 0.0f), label.c_str()); ImGui::PopStyleColor(); diff --git a/engine/src/toast/renderer/passes/shadow_pass.hpp b/engine/src/toast/renderer/passes/shadow_pass.hpp index 28ad1fe6f..b053762ad 100644 --- a/engine/src/toast/renderer/passes/shadow_pass.hpp +++ b/engine/src/toast/renderer/passes/shadow_pass.hpp @@ -104,7 +104,7 @@ class ShadowPass : public IRenderPass { static constexpr uint32_t k_max_voxel_casters = 4096; /// Throttles the over limit warning - std::chrono::steady_clock::time_point m_last_caster_limit_warning {}; + std::chrono::steady_clock::time_point m_last_caster_limit_warning; struct LayerGroup { uint32_t base_layer = 0; diff --git a/engine/src/toast/renderer/passes/voxel_pass.hpp b/engine/src/toast/renderer/passes/voxel_pass.hpp index 97ecaa0c2..a41eab84f 100644 --- a/engine/src/toast/renderer/passes/voxel_pass.hpp +++ b/engine/src/toast/renderer/passes/voxel_pass.hpp @@ -54,7 +54,7 @@ class VoxelPass : public IRenderPass { private: /// Throttles the over limit warning - std::chrono::steady_clock::time_point m_last_limit_warning {}; + std::chrono::steady_clock::time_point m_last_limit_warning; /// Mirrors voxel.slang PushConstants struct PushConstants { From 44522468f7e9c357fbf01387f4712c381d92647d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dar=C3=ADo=20Rodr=C3=ADguez=20Mart=C3=ADnez?= Date: Sat, 10 Oct 2026 16:36:09 +0200 Subject: [PATCH 4/7] tidi --- .../src/toast/renderer/async_compute_task.cpp | 8 +-- .../src/toast/renderer/passes/perf_window.cpp | 63 ++++++++++++++----- 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/engine/src/toast/renderer/async_compute_task.cpp b/engine/src/toast/renderer/async_compute_task.cpp index ab5440396..a6b37db29 100644 --- a/engine/src/toast/renderer/async_compute_task.cpp +++ b/engine/src/toast/renderer/async_compute_task.cpp @@ -173,10 +173,10 @@ void AsyncComputeTask::writeDescriptors(uint32_t slot) { writes.reserve(m_bindings.size()); for (const Binding& binding : m_bindings) { const Buffer& buffer = m_buffers[binding.id]; - const uint32_t copy = - !buffer.ring - ? 0 - : (binding.copy == Copy::current ? slot : previous); // NOLINT(readability-avoid-nested-conditional-operator) + uint32_t copy = 0; + if (buffer.ring) { + copy = binding.copy == Copy::current ? slot : previous; + } infos.emplace_back(*buffer.copies[copy], 0, buffer.size); writes.emplace_back(*m_slots[slot].set, binding.binding, 0, 1, vk::DescriptorType::eStorageBuffer, nullptr, &infos.back()); } diff --git a/engine/src/toast/renderer/passes/perf_window.cpp b/engine/src/toast/renderer/passes/perf_window.cpp index dc21de6d5..32d10d50f 100644 --- a/engine/src/toast/renderer/passes/perf_window.cpp +++ b/engine/src/toast/renderer/passes/perf_window.cpp @@ -20,7 +20,7 @@ namespace { // Prefixed since unity builds merge this with the other passes -constexpr ImU32 perfColor(int r, int g, int b, int a = 255) { +constexpr auto perfColor(int r, int g, int b, int a = 255) -> ImU32 { return IM_COL32(r, g, b, a); } @@ -31,6 +31,47 @@ const ImVec4 k_perf_bad(1.0f, 0.38f, 0.38f, 1.0f); constexpr ImU32 k_perf_bar_good = perfColor(92, 184, 92); constexpr ImU32 k_perf_bar_warn = perfColor(230, 170, 60); constexpr ImU32 k_perf_bar_bad = perfColor(220, 80, 80); + +enum class PerfLevel : uint8_t { good, warn, bad }; + +/// Good up to @p warn_from and bad past @p bad_from +auto perfLevel(float value, float warn_from, float bad_from) -> PerfLevel { + if (value > bad_from) { + return PerfLevel::bad; + } + if (value > warn_from) { + return PerfLevel::warn; + } + return PerfLevel::good; +} + +auto perfLevel(bool bad, bool warn) -> PerfLevel { + if (bad) { + return PerfLevel::bad; + } + if (warn) { + return PerfLevel::warn; + } + return PerfLevel::good; +} + +auto perfTextColor(PerfLevel level) -> ImVec4 { + switch (level) { + case PerfLevel::bad: return k_perf_bad; + case PerfLevel::warn: return k_perf_warn; + case PerfLevel::good: break; + } + return k_perf_good; +} + +auto perfBarColor(PerfLevel level) -> ImU32 { + switch (level) { + case PerfLevel::bad: return k_perf_bar_bad; + case PerfLevel::warn: return k_perf_bar_warn; + case PerfLevel::good: break; + } + return k_perf_bar_good; +} constexpr ImU32 k_perf_gpu_line = perfColor(110, 170, 255); constexpr ImU32 k_perf_budget_line = perfColor(255, 255, 255, 110); constexpr ImU32 k_perf_graph_bg = perfColor(0, 0, 0, 90); @@ -349,8 +390,7 @@ void PerfWindow::drawFrameGraph(float width, float height) { const size_t index = static_cast((offset + i) % static_cast(k_history)); const float ms = m_interval_history[index]; const float x0 = first_x + (bar_width * static_cast(i)); - const ImU32 color = - ms <= m_budget_ms * 1.05f ? k_perf_bar_good : (ms <= m_budget_ms * 1.5f ? k_perf_bar_warn : k_perf_bar_bad); + const ImU32 color = perfBarColor(perfLevel(ms, m_budget_ms * 1.05f, m_budget_ms * 1.5f)); draw->AddRectFilled(ImVec2(x0, y_of(ms)), ImVec2(x0 + std::max(bar_width - 0.5f, 1.0f), end.y), color); gpu_points[static_cast(i)] = ImVec2(x0 + (bar_width * 0.5f), y_of(m_gpu_history[index])); @@ -391,10 +431,7 @@ void PerfWindow::drawSummary(bool has_gpu) { auto* renderer = VulkanRenderer::instance; const auto cap = static_cast(renderer->effectiveFrameRateLimit()); - // NOLINTNEXTLINE(readability-avoid-nested-conditional-operator) - const ImVec4 fps_color = m_interval_ms <= m_budget_ms * 1.05f ? k_perf_good - : m_interval_ms <= m_budget_ms * 1.5f ? k_perf_warn - : k_perf_bad; + const ImVec4 fps_color = perfTextColor(perfLevel(m_interval_ms, m_budget_ms * 1.05f, m_budget_ms * 1.5f)); ImGui::TextColored(fps_color, "%5.1f fps", s.render_fps); ImGui::SameLine(); ImGui::Text("%6.2f ms", m_interval_ms); @@ -422,13 +459,9 @@ void PerfWindow::drawSummary(bool has_gpu) { drawPhaseBar(width, ImGui::GetFontSize() * 0.6f); const Verdict render = renderVerdict(has_gpu); - perfVerdictText( - "Render", render.bad ? k_perf_bad : (render.warn ? k_perf_warn : k_perf_good), render.label, render.detail - ); // NOLINT(readability-avoid-nested-conditional-operator) + perfVerdictText("Render", perfTextColor(perfLevel(render.bad, render.warn)), render.label, render.detail); const Verdict game = gameVerdict(); - perfVerdictText( - "Game ", game.bad ? k_perf_bad : (game.warn ? k_perf_warn : k_perf_good), game.label, game.detail - ); // NOLINT(readability-avoid-nested-conditional-operator) + perfVerdictText("Game ", perfTextColor(perfLevel(game.bad, game.warn)), game.label, game.detail); } void PerfWindow::drawFrameTab() { @@ -717,9 +750,7 @@ void PerfWindow::drawMemoryTab() { ImGui::TextDisabled("%.0f MiB total", perfMiB(properties.memoryHeaps[heap].size)); const std::string label = std::format("{:.0f} / {:.0f} MiB", perfMiB(budget.usage), perfMiB(budget.budget)); - ImGui::PushStyleColor( - ImGuiCol_PlotHistogram, fraction < 0.75f ? k_perf_good : (fraction < 0.9f ? k_perf_warn : k_perf_bad) - ); // NOLINT(readability-avoid-nested-conditional-operator) + ImGui::PushStyleColor(ImGuiCol_PlotHistogram, perfTextColor(perfLevel(fraction, 0.75f, 0.9f))); ImGui::ProgressBar(fraction, ImVec2(ImGui::GetFontSize() * 30.0f, 0.0f), label.c_str()); ImGui::PopStyleColor(); From 33182d91d92371fbd1bdc75503fde6b5ed0a60c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dar=C3=ADo=20Rodr=C3=ADguez=20Mart=C3=ADnez?= Date: Sat, 10 Oct 2026 16:48:00 +0200 Subject: [PATCH 5/7] fuck off, just pass already --- engine/src/toast/renderer/passes/perf_window.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/engine/src/toast/renderer/passes/perf_window.cpp b/engine/src/toast/renderer/passes/perf_window.cpp index 32d10d50f..4fefab0ae 100644 --- a/engine/src/toast/renderer/passes/perf_window.cpp +++ b/engine/src/toast/renderer/passes/perf_window.cpp @@ -32,7 +32,11 @@ constexpr ImU32 k_perf_bar_good = perfColor(92, 184, 92); constexpr ImU32 k_perf_bar_warn = perfColor(230, 170, 60); constexpr ImU32 k_perf_bar_bad = perfColor(220, 80, 80); -enum class PerfLevel : uint8_t { good, warn, bad }; +enum class PerfLevel : uint8_t { + good, + warn, + bad +}; /// Good up to @p warn_from and bad past @p bad_from auto perfLevel(float value, float warn_from, float bad_from) -> PerfLevel { @@ -72,6 +76,7 @@ auto perfBarColor(PerfLevel level) -> ImU32 { } return k_perf_bar_good; } + constexpr ImU32 k_perf_gpu_line = perfColor(110, 170, 255); constexpr ImU32 k_perf_budget_line = perfColor(255, 255, 255, 110); constexpr ImU32 k_perf_graph_bg = perfColor(0, 0, 0, 90); From 595216073c52298788a16573f76679814c3c3b88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dar=C3=ADo=20Rodr=C3=ADguez=20Mart=C3=ADnez?= Date: Sun, 11 Oct 2026 11:58:03 +0200 Subject: [PATCH 6/7] optimized voxel volume --- engine/assets/shaders/voxel.slang | 66 +++- engine/assets/shaders/voxel_cells.slang | 165 ++++++++++ engine/assets/shaders/voxel_cells.slang.meta | 4 + engine/assets/shaders/voxel_dda.slang | 48 ++- engine/src/toast/renderer/gpu_sync.cpp | 11 +- engine/src/toast/renderer/gpu_sync.hpp | 11 +- engine/src/toast/renderer/gpu_timer.cpp | 38 +-- engine/src/toast/renderer/gpu_timer.hpp | 17 +- .../toast/renderer/passes/material_pass.cpp | 250 +++++++++------ .../toast/renderer/passes/material_pass.hpp | 73 +++-- .../src/toast/renderer/passes/voxel_pass.cpp | 242 +++++++++++++- .../src/toast/renderer/passes/voxel_pass.hpp | 52 ++- .../src/toast/renderer/renderer_settings.cpp | 10 + engine/src/toast/renderer/shader_cache.cpp | 193 ++++++----- engine/src/toast/renderer/shader_cache.hpp | 21 +- engine/src/toast/renderer/shader_compiler.cpp | 101 ++++-- engine/src/toast/renderer/shader_compiler.hpp | 1 + engine/src/toast/renderer/slang_vfs.cpp | 2 +- engine/src/toast/renderer/slang_vfs.hpp | 4 +- .../src/toast/renderer/voxel_gpu_storage.cpp | 299 ++++++++++-------- .../src/toast/renderer/voxel_gpu_storage.hpp | 85 ++--- engine/src/toast/renderer/vulkan_renderer.cpp | 134 +++++--- engine/src/toast/renderer/vulkan_renderer.hpp | 41 ++- 23 files changed, 1341 insertions(+), 527 deletions(-) create mode 100644 engine/assets/shaders/voxel_cells.slang create mode 100644 engine/assets/shaders/voxel_cells.slang.meta diff --git a/engine/assets/shaders/voxel.slang b/engine/assets/shaders/voxel.slang index c28052f6a..2ce180a79 100644 --- a/engine/assets/shaders/voxel.slang +++ b/engine/assets/shaders/voxel.slang @@ -3,38 +3,83 @@ import voxel_gbuffer; import lighting; import depth_convention; +/// Mirrors renderer::VoxelPass::PushConstants struct PushConstants { uint instanceIndex; + uint bucket; + uint mode; uint pad0; - uint pad1; - uint pad2; }; [[vk::push_constant]] PushConstants gPush; +static const uint kDrawVolume = 0u; +static const uint kDrawCells = 1u; + +[[vk::binding(7, 1)]] +StructuredBuffer gVoxelCells; +[[vk::binding(8, 1)]] +StructuredBuffer gVoxelCellArgs; + struct VSOutput { float4 position : SV_Position; [[vk::location(0)]] float3 worldPosition : TEXCOORD0; + + [[vk::location(1)]] + nointerpolation uint instanceIndex : TEXCOORD1; + + [[vk::location(2)]] + nointerpolation int3 boxMin : TEXCOORD2; + + [[vk::location(3)]] + nointerpolation int3 boxMax : TEXCOORD3; }; [shader("vertex")] -VSOutput vertexMain(uint vertexId : SV_VertexID) +VSOutput vertexMain(uint vertexId : SV_VertexID, uint instanceId : SV_InstanceID) { - const VoxelInstance instance = gVoxelInstances[gPush.instanceIndex]; + uint instanceIndex = gPush.instanceIndex; + uint packedCell = 0u; + if (gPush.mode == kDrawCells) + { + const uint2 cell = gVoxelCells[gVoxelCellArgs[kCellOffsetsWord + gPush.bucket] + instanceId]; + instanceIndex = cell.x; + packedCell = cell.y; + } + + const VoxelInstance instance = gVoxelInstances[instanceIndex]; const VolumeRecord record = gVolumeRecords[instance.recordIndex]; - const float3 voxels = float3(record.brickDimsX, record.brickDimsY, record.brickDimsZ) * 8.0; + const int3 dims = int3(record.brickDimsX, record.brickDimsY, record.brickDimsZ) * int(kBrickDim); - const float3 local = kBoxCorners[kBoxIndices[vertexId]] * voxels; + int3 boxMin = int3(0, 0, 0); + int3 boxMax = dims; + if (gPush.mode == kDrawCells) + { + boxMin = int3(unpackCell(packedCell)) * kCoarseVoxels; + boxMax = min(boxMin + kCoarseVoxels, dims); + } + + // A mirrored transform flips winding so swap two corners of each triangle to keep one cull mode + uint corner = vertexId % 3u; + if ((instance.flags & kInstanceMirrored) != 0u && corner != 0u) + { + corner = 3u - corner; + } + const float3 unit = kBoxCorners[kBoxIndices[vertexId - (vertexId % 3u) + corner]]; + const float3 local = float3(boxMin) + unit * float3(boxMax - boxMin); const float4 world = mul(instance.voxelToWorld, float4(local, 1.0)); VSOutput output; output.worldPosition = world.xyz; output.position = mul(gCamera.jitteredViewProjection, world); + output.instanceIndex = instanceIndex; + output.boxMin = boxMin; + output.boxMax = boxMax; return output; } @@ -254,7 +299,7 @@ bool shade(VSOutput input, out FSOutput output, out float depth) output.motion = float4(0.0, 0.0, 0.0, 0.0); depth = input.position.z; - const VoxelInstance instance = gVoxelInstances[gPush.instanceIndex]; + const VoxelInstance instance = gVoxelInstances[input.instanceIndex]; const float3 camera = gCamera.position; // t = 1 is the box surface @@ -267,7 +312,7 @@ bool shade(VSOutput input, out FSOutput output, out float depth) const float nearStep = clipNearDistance(mul(gCamera.viewProjection, float4(worldDirection, 0.0))); const float tMin = nearStep > 0.0 ? max(0.0, -nearOrigin / nearStep) : 0.0; - const RayHit hit = marchRay(instance.recordIndex, origin, direction, tMin, kFloatMax); + const RayHit hit = marchRayBox(instance.recordIndex, origin, direction, tMin, kFloatMax, input.boxMin, input.boxMax); const float3 hitWorld = camera + worldDirection * hit.t; @@ -304,6 +349,11 @@ bool shade(VSOutput input, out FSOutput output, out float depth) const uint renderMode = gCamera.renderModePad.x; if (isVoxelDebugMode(renderMode)) { + // A miss drawn by a debug view sits on the far plane so a box behind it still wins the depth test + if (!hit.hit) + { + depth = kFarDepth; + } return shadeDebug(renderMode, input, instance, origin, direction, hit, normal, toCamera, output); } diff --git a/engine/assets/shaders/voxel_cells.slang b/engine/assets/shaders/voxel_cells.slang new file mode 100644 index 000000000..c6b6ab5c1 --- /dev/null +++ b/engine/assets/shaders/voxel_cells.slang @@ -0,0 +1,165 @@ +// Compacts the non-empty coarse cells of every drawn voxel volume into distance buckets for renderer::VoxelPass +// Three dispatches per frame over one thread per coarse cell +// count each surviving cell adds one to its bucket +// allocate one thread turns the counts into offsets and indirect draw commands +// write each surviving cell takes a slot in its bucket +import voxel_dda; + +/// Mirrors renderer::VoxelPass::CellCullPush +struct CellCullPush +{ + float4x4 viewProjection; + /// xyz world camera position and w the near plane distance + float4 cameraNear; + uint totalCells; + uint instanceCount; + uint phase; + uint pad0; +}; + +[[vk::push_constant]] +CellCullPush gCull; + +[[vk::binding(7, 1)]] +RWStructuredBuffer gCellsOut; +[[vk::binding(8, 1)]] +RWStructuredBuffer gCellArgsOut; + +static const uint kPhaseCount = 0u; +static const uint kPhaseAllocate = 1u; +static const uint kPhaseWrite = 2u; + +static const uint kNoBucket = 0xFFFFFFFFu; + +/// Last instance whose first cell is at or before @p cell +uint instanceOfCell(uint cell) +{ + uint low = 0u; + uint high = gCull.instanceCount; + while (high - low > 1u) + { + const uint middle = (low + high) / 2u; + if (gVoxelInstances[middle].cellBase <= cell) + { + low = middle; + } + else + { + high = middle; + } + } + return low; +} + +/// Largest axis scale so a world distance becomes voxels +float worldToVoxelScale(float4x4 worldToVoxel) +{ + return max(max(length(mul(worldToVoxel, float4(1.0, 0.0, 0.0, 0.0)).xyz), length(mul(worldToVoxel, float4(0.0, 1.0, 0.0, 0.0)).xyz)), + length(mul(worldToVoxel, float4(0.0, 0.0, 1.0, 0.0)).xyz)); +} + +bool outsideFrustum(float4x4 voxelToClip, float3 boxMin, float3 boxMax) +{ + uint outside = 0x1Fu; + for (uint i = 0u; i < 8u; ++i) + { + const float3 corner = lerp(boxMin, boxMax, kBoxCorners[i]); + const float4 clip = mul(voxelToClip, float4(corner, 1.0)); + uint flags = 0u; + flags |= clip.x < -clip.w ? 1u : 0u; + flags |= clip.x > clip.w ? 2u : 0u; + flags |= clip.y < -clip.w ? 4u : 0u; + flags |= clip.y > clip.w ? 8u : 0u; + flags |= clip.w <= 0.0 ? 16u : 0u; + outside &= flags; + } + return outside != 0u; +} + +/// Bucket 0 holds cells the camera or near plane is inside and the rest step by a factor of sqrt 2 in distance +uint classifyCell(uint globalCell, out uint instanceIndex, out uint packed) +{ + instanceIndex = instanceOfCell(globalCell); + packed = 0u; + + const VoxelInstance instance = gVoxelInstances[instanceIndex]; + const VolumeRecord record = gVolumeRecords[instance.recordIndex]; + const uint local = globalCell - instance.cellBase; + const uint3 coarseDims = uint3(record.coarseDimsX, record.coarseDimsY, record.coarseDimsZ); + if (local >= coarseDims.x * coarseDims.y * coarseDims.z) + { + return kNoBucket; + } + + const uint3 cell = uint3(local % coarseDims.x, (local / coarseDims.x) % coarseDims.y, local / (coarseDims.x * coarseDims.y)); + if (!coarseSolid(instance.recordIndex, int3(cell))) + { + return kNoBucket; + } + packed = packCell(cell); + + const int3 dims = int3(record.brickDimsX, record.brickDimsY, record.brickDimsZ) * int(kBrickDim); + const float3 boxMin = float3(int3(cell) * kCoarseVoxels); + const float3 boxMax = float3(min(int3(cell) * kCoarseVoxels + kCoarseVoxels, dims)); + + const float3 camera = mul(instance.worldToVoxel, float4(gCull.cameraNear.xyz, 1.0)).xyz; + const float guard = gCull.cameraNear.w * worldToVoxelScale(instance.worldToVoxel); + if (all(camera >= boxMin - guard) && all(camera <= boxMax + guard)) + { + return 0u; + } + + if (outsideFrustum(mul(gCull.viewProjection, instance.voxelToWorld), boxMin, boxMax)) + { + return kNoBucket; + } + + const float cells = length(camera - 0.5 * (boxMin + boxMax)) / float(kCoarseVoxels); + return 1u + min(uint(2.0 * log2(max(cells, 1.0))), kCellBucketCount - 2u); +} + +[shader("compute")] +[numthreads(64, 1, 1)] +void computeMain(uint3 thread : SV_DispatchThreadID) +{ + if (gCull.phase == kPhaseAllocate) + { + if (thread.x != 0u) + { + return; + } + uint offset = 0u; + for (uint bucket = 0u; bucket < kCellBucketCount; ++bucket) + { + const uint count = gCellArgsOut[kCellCursorsWord + bucket]; + gCellArgsOut[bucket * 4u + 0u] = 36u; + gCellArgsOut[bucket * 4u + 1u] = count; + gCellArgsOut[bucket * 4u + 2u] = 0u; + gCellArgsOut[bucket * 4u + 3u] = 0u; + gCellArgsOut[kCellOffsetsWord + bucket] = offset; + gCellArgsOut[kCellCursorsWord + bucket] = offset; + offset += count; + } + return; + } + + if (thread.x >= gCull.totalCells) + { + return; + } + + uint instanceIndex; + uint packed; + const uint bucket = classifyCell(thread.x, instanceIndex, packed); + if (bucket == kNoBucket) + { + return; + } + + uint slot; + InterlockedAdd(gCellArgsOut[kCellCursorsWord + bucket], 1u, slot); + if (gCull.phase == kPhaseWrite) + { + gCellsOut[slot] = uint2(instanceIndex, packed); + } +} diff --git a/engine/assets/shaders/voxel_cells.slang.meta b/engine/assets/shaders/voxel_cells.slang.meta new file mode 100644 index 000000000..ea58dc7e2 --- /dev/null +++ b/engine/assets/shaders/voxel_cells.slang.meta @@ -0,0 +1,4 @@ +uid = "V0xCellCu1L" +type = "shader" +created_at = "2026-10-11T12:00:00.0000000Z" +modified_at = "2026-10-11T12:00:00.0000000Z" diff --git a/engine/assets/shaders/voxel_dda.slang b/engine/assets/shaders/voxel_dda.slang index 32b92c48d..0c8c8726b 100644 --- a/engine/assets/shaders/voxel_dda.slang +++ b/engine/assets/shaders/voxel_dda.slang @@ -24,11 +24,15 @@ struct VoxelInstance float4x4 previousVoxelToWorld; uint recordIndex; uint highlightRecord; /// this is for selection tint - /// Bit 0 set means previousVoxelToWorld is not a real previous frame + /// Bit 0 no real previous frame and bit 1 a mirrored transform uint flags; - uint pad2; + /// First global coarse cell of this instance in the cell cull dispatch + uint cellBase; }; +static const uint kInstanceNoMotion = 1u; +static const uint kInstanceMirrored = 2u; + [[vk::binding(0, 1)]] StructuredBuffer gBrickMaterials; [[vk::binding(1, 1)]] @@ -67,6 +71,23 @@ static const uint kBoxIndices[36] = { 1, 2, 6, 1, 6, 5, // +X }; +/// Cell args word layout mirrored by renderer::VoxelPass with bucket 0 holding the camera +static const uint kCellBucketCount = 17; +static const uint kCellOffsetsWord = kCellBucketCount * 4; +static const uint kCellCursorsWord = kCellBucketCount * 5; +static const uint kCellArgsWords = kCellBucketCount * 6; + +/// Ten bits per axis +uint packCell(uint3 cell) +{ + return cell.x | (cell.y << 10) | (cell.z << 20); +} + +uint3 unpackCell(uint packed) +{ + return uint3(packed & 0x3FFu, (packed >> 10) & 0x3FFu, packed >> 20); +} + static const uint kTagEmpty = 0; static const uint kTagUniform = 1; static const uint kTagShared = 2; @@ -179,8 +200,15 @@ void leaveCell(float3 origin, float3 inverse, float3 stepSign, int3 cellMin, int axis = nextAxis; } -/// Cells are chosen by integer index not float position since nudging positions skips voxels under grazing rays RayHit marchRay(uint volume, float3 origin, float3 direction, float tMin, float tMax) +{ + const int3 dims = int3(gVolumeRecords[volume].brickDimsX, gVolumeRecords[volume].brickDimsY, gVolumeRecords[volume].brickDimsZ) * int(kBrickDim); + return marchRayBox(volume, origin, direction, tMin, tMax, int3(0, 0, 0), dims); +} + +/// The box must lie inside the volume +/// Cells are chosen by integer index not float position since nudging positions skips voxels under grazing rays +RayHit marchRayBox(uint volume, float3 origin, float3 direction, float tMin, float tMax, int3 boxMin, int3 boxMax) { RayHit result; result.hit = false; @@ -192,12 +220,6 @@ RayHit marchRay(uint volume, float3 origin, float3 direction, float tMin, float result.coarseSteps = 0; result.brickSteps = 0; - // Read ONLY the extents to avoid keeping a full struct alive - const uint dimsX = gVolumeRecords[volume].brickDimsX; - const uint dimsY = gVolumeRecords[volume].brickDimsY; - const uint dimsZ = gVolumeRecords[volume].brickDimsZ; - const int3 dims = int3(dimsX, dimsY, dimsZ) * int(kBrickDim); - float tExit = kFloatMax; float3 entryTimes = float3(-kFloatMax, -kFloatMax, -kFloatMax); float3 inverse = float3(0.0, 0.0, 0.0); @@ -207,7 +229,7 @@ RayHit marchRay(uint volume, float3 origin, float3 direction, float tMin, float { if (direction[a] == 0.0) { - if (origin[a] < 0.0 || origin[a] > float(dims[a])) + if (origin[a] < float(boxMin[a]) || origin[a] > float(boxMax[a])) { return result; } @@ -217,8 +239,8 @@ RayHit marchRay(uint volume, float3 origin, float3 direction, float tMin, float inverse[a] = 1.0 / direction[a]; stepSign[a] = direction[a] > 0.0 ? 1.0 : -1.0; - const float t0 = (0.0 - origin[a]) * inverse[a]; - const float t1 = (float(dims[a]) - origin[a]) * inverse[a]; + const float t0 = (float(boxMin[a]) - origin[a]) * inverse[a]; + const float t1 = (float(boxMax[a]) - origin[a]) * inverse[a]; entryTimes[a] = min(t0, t1); tExit = min(tExit, max(t0, t1)); } @@ -270,7 +292,7 @@ RayHit marchRay(uint volume, float3 origin, float3 direction, float tMin, float if ((crossed & 2u) != 0) voxel.y = int(round(position.y)) - (stepSign.y < 0.0 ? 1 : 0); if ((crossed & 4u) != 0) voxel.z = int(round(position.z)) - (stepSign.z < 0.0 ? 1 : 0); - if (any(uint3(voxel) >= uint3(dims))) return result; + if (any(voxel < boxMin) || any(voxel >= boxMax)) return result; const int3 coarse = voxel >> 5; if (!coarseSolid(volume, coarse)) diff --git a/engine/src/toast/renderer/gpu_sync.cpp b/engine/src/toast/renderer/gpu_sync.cpp index 9574db1d4..6f3dcee78 100644 --- a/engine/src/toast/renderer/gpu_sync.cpp +++ b/engine/src/toast/renderer/gpu_sync.cpp @@ -305,13 +305,18 @@ void submit(vk::Queue queue, const SubmitDesc& desc) { std::vector waits; waits.reserve(desc.waits.size()); for (const auto& wait : desc.waits) { - waits.push_back(vk::SemaphoreSubmitInfo {}.setSemaphore(wait.semaphore).setStageMask(wait.stages)); + waits.push_back(vk::SemaphoreSubmitInfo {}.setSemaphore(wait.semaphore).setValue(wait.value).setStageMask(wait.stages)); } std::vector signals; signals.reserve(desc.signals.size()); - for (const vk::Semaphore semaphore : desc.signals) { - signals.push_back(vk::SemaphoreSubmitInfo {}.setSemaphore(semaphore).setStageMask(vk::PipelineStageFlagBits2::eAllCommands)); + for (const SemaphoreSignal& signal : desc.signals) { + signals.push_back( + vk::SemaphoreSubmitInfo {} + .setSemaphore(signal.semaphore) + .setValue(signal.value) + .setStageMask(vk::PipelineStageFlagBits2::eAllCommands) + ); } std::vector commands; diff --git a/engine/src/toast/renderer/gpu_sync.hpp b/engine/src/toast/renderer/gpu_sync.hpp index fd522e941..214b1cbf0 100644 --- a/engine/src/toast/renderer/gpu_sync.hpp +++ b/engine/src/toast/renderer/gpu_sync.hpp @@ -164,13 +164,20 @@ TOAST_API void transition(vk::CommandBuffer cmd, ImageState& state, Usage next); struct SemaphoreWait { vk::Semaphore semaphore; vk::PipelineStageFlags2 stages; + /// Timeline value and ignored for binary semaphores + uint64_t value = 0; +}; + +struct SemaphoreSignal { + vk::Semaphore semaphore; + /// Timeline value and ignored for binary semaphores + uint64_t value = 0; }; -/// Signals fire once every command in the submit completes struct SubmitDesc { std::span commands; std::span waits; - std::span signals; + std::span signals; vk::Fence fence; /// Chained to SubmitInfo2 such as a FrameBoundaryEXT const void* next = nullptr; diff --git a/engine/src/toast/renderer/gpu_timer.cpp b/engine/src/toast/renderer/gpu_timer.cpp index 4d7e23241..d4a890879 100644 --- a/engine/src/toast/renderer/gpu_timer.cpp +++ b/engine/src/toast/renderer/gpu_timer.cpp @@ -42,9 +42,7 @@ GpuTimer::GpuTimer(const VulkanCore& core, uint32_t frame_slots) { const auto families = physical_device.getQueueFamilyProperties(); const uint32_t graphics_family = core.getGraphicsQueueFamilyIndex(); - const uint32_t compute_family = core.getComputeQueueFamilyIndex(); const uint32_t graphics_bits = graphics_family < families.size() ? families[graphics_family].timestampValidBits : 0; - const uint32_t compute_bits = compute_family < families.size() ? families[compute_family].timestampValidBits : 0; if (period <= 0.0f || graphics_bits == 0) { TOAST_INFO("Render", "No GPU timestamp queries on this device; the performance overlay shows CPU timings only"); @@ -63,9 +61,7 @@ GpuTimer::GpuTimer(const VulkanCore& core, uint32_t frame_slots) { m_ns_per_tick = static_cast(period); m_graphics_mask = validBitsMask(graphics_bits); - m_compute_mask = validBitsMask(compute_bits); m_supported = true; - m_compute_supported = compute_bits > 0; } void GpuTimer::beginGraphics(uint32_t slot, vk::CommandBuffer cmd) { @@ -77,8 +73,7 @@ void GpuTimer::beginGraphics(uint32_t slot, vk::CommandBuffer cmd) { state.depth = 0; state.graphics_written = false; - // The compute pair is reset by the compute command buffer - cmd.resetQueryPool(*m_pool, base(slot), k_queries_per_slot - 2); + cmd.resetQueryPool(*m_pool, base(slot), k_queries_per_slot); cmd.writeTimestamp2(vk::PipelineStageFlagBits2::eNone, *m_pool, base(slot)); } @@ -90,25 +85,6 @@ void GpuTimer::endGraphics(uint32_t slot, vk::CommandBuffer cmd) { m_slots[slot].graphics_written = true; } -void GpuTimer::beginCompute(uint32_t slot, vk::CommandBuffer cmd) { - if (!m_compute_supported || slot >= m_slots.size()) { - return; - } - m_slots[slot].compute_written = false; - - const uint32_t first = base(slot) + k_queries_per_slot - 2; - cmd.resetQueryPool(*m_pool, first, 2); - cmd.writeTimestamp2(vk::PipelineStageFlagBits2::eNone, *m_pool, first); -} - -void GpuTimer::endCompute(uint32_t slot, vk::CommandBuffer cmd) { - if (!m_compute_supported || slot >= m_slots.size()) { - return; - } - cmd.writeTimestamp2(vk::PipelineStageFlagBits2::eAllCommands, *m_pool, base(slot) + k_queries_per_slot - 1); - m_slots[slot].compute_written = true; -} - auto GpuTimer::beginScope(uint32_t slot, vk::CommandBuffer cmd, std::string_view name) -> uint32_t { if (!m_supported || slot >= m_slots.size()) { return k_no_scope; @@ -172,7 +148,6 @@ void GpuTimer::collect(uint32_t slot) { m_frame.valid = true; ++m_serial; m_frame.graphics_ms = ticksToMs(ticks[0], ticks[1], m_graphics_mask); - m_frame.compute_ms = 0.0; m_frame.scopes.clear(); for (uint32_t i = 0; i < scope_count; ++i) { const PendingScope& pending = state.scopes[i]; @@ -192,20 +167,9 @@ void GpuTimer::collect(uint32_t slot) { } } - if (m_compute_supported && state.compute_written) { - state.compute_written = false; - const uint32_t first = base(slot) + k_queries_per_slot - 2; - const auto compute = - m_pool.getResults(first, 2, 2 * sizeof(uint64_t), sizeof(uint64_t), vk::QueryResultFlagBits::e64); - if (compute.result == vk::Result::eSuccess) { - m_frame.compute_ms = ticksToMs(compute.value[0], compute.value[1], m_compute_mask); - } - } - const bool first_frame = !m_smoothed.valid; m_scratch.valid = true; m_scratch.graphics_ms = smooth(m_smoothed.graphics_ms, m_frame.graphics_ms, first_frame); - m_scratch.compute_ms = smooth(m_smoothed.compute_ms, m_frame.compute_ms, first_frame); m_scratch.scopes.clear(); for (const ScopeTiming& sample : m_frame.scopes) { const auto previous = std::ranges::find_if(m_smoothed.scopes, [&sample](const ScopeTiming& timing) { diff --git a/engine/src/toast/renderer/gpu_timer.hpp b/engine/src/toast/renderer/gpu_timer.hpp index f363e1573..c7854ab93 100644 --- a/engine/src/toast/renderer/gpu_timer.hpp +++ b/engine/src/toast/renderer/gpu_timer.hpp @@ -43,7 +43,6 @@ class GpuTimer { struct Timings { bool valid = false; double graphics_ms = 0.0; - double compute_ms = 0.0; std::vector scopes; }; @@ -54,20 +53,13 @@ class GpuTimer { return m_supported; } - [[nodiscard]] - auto computeSupported() const noexcept -> bool { - return m_compute_supported; - } - void beginGraphics(uint32_t slot, vk::CommandBuffer cmd); void endGraphics(uint32_t slot, vk::CommandBuffer cmd); - void beginCompute(uint32_t slot, vk::CommandBuffer cmd); - void endCompute(uint32_t slot, vk::CommandBuffer cmd); auto beginScope(uint32_t slot, vk::CommandBuffer cmd, std::string_view name) -> uint32_t; void endScope(uint32_t slot, vk::CommandBuffer cmd, uint32_t scope); - /// @note Only once the graphics and compute fences of the slot have signalled + /// @note Only once the frame fence of the slot has signalled void collect(uint32_t slot); [[nodiscard]] @@ -98,11 +90,10 @@ class GpuTimer { std::vector scopes; uint32_t depth = 0; bool graphics_written = false; - bool compute_written = false; }; - /// graphics begin and end then a pair per scope then compute begin and end - static constexpr uint32_t k_queries_per_slot = 2 + (2 * k_max_scopes) + 2; + /// Graphics begin and end then a pair per scope + static constexpr uint32_t k_queries_per_slot = 2 + (2 * k_max_scopes); [[nodiscard]] static auto base(uint32_t slot) noexcept -> uint32_t { @@ -116,9 +107,7 @@ class GpuTimer { std::vector m_slots; double m_ns_per_tick = 1.0; uint64_t m_graphics_mask = ~uint64_t {0}; - uint64_t m_compute_mask = ~uint64_t {0}; bool m_supported = false; - bool m_compute_supported = false; bool m_reported_unavailable = false; uint64_t m_serial = 0; diff --git a/engine/src/toast/renderer/passes/material_pass.cpp b/engine/src/toast/renderer/passes/material_pass.cpp index c4055c9f3..5dd0e2e66 100644 --- a/engine/src/toast/renderer/passes/material_pass.cpp +++ b/engine/src/toast/renderer/passes/material_pass.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -20,6 +21,7 @@ #include #include #include +#include #include #include @@ -86,13 +88,15 @@ MaterialPass::MaterialPass( m_name(root_material != nullptr ? root_material->name() : "Material"), m_color_format(color_format), m_depth_format(depth_format), - m_extent(extent), - m_root_runtime(core, root_material) { - rebuildPipeline(); + m_extent(extent) { + startBuild(); } -auto MaterialPass::resolvedAlphaCutoff() -> float { - return resolvedAlphaCutoffOf(m_root_runtime); +MaterialPass::~MaterialPass() { + // The job owns copies of its inputs but must not outlive the core + if (m_pending_build.valid()) { + m_pending_build.wait(); + } } auto MaterialPass::resolvedAlphaCutoffOf(MaterialRuntime& runtime) -> float { @@ -119,61 +123,61 @@ auto MaterialPass::resolvedAlphaCutoffOf(MaterialRuntime& runtime) -> float { return 0.0f; } -void MaterialPass::rebuildPipeline() { - ZoneScoped; - m_instances.clear(); - - m_pipeline.reset(); - - m_root_runtime.rebuild(); - const auto& entries = m_root_runtime.shaderEntries(); - if (entries.empty()) { - TOAST_WARN("Render", "MaterialPass '{}' has no compiled shaders, nothing will be drawn", m_name); - return; - } - if (entries.size() > 1) { - TOAST_WARN("Render", "MaterialPass '{}': multiple shader modules per material not supported yet, using the first", m_name); - } - - m_layout.rebuild(*m_core, m_root_runtime.reflection(), m_name); - - const assets::MaterialSettings settings = m_root_material->settings(); - - VulkanPipeline::Config config; - config.pipeline_type = VulkanPipeline::PipelineType::graphics; - config.debug_name = std::format("MaterialPass ({})", m_name); - config.color_format = m_color_format; - config.depth_format = m_depth_format; - config.extent = m_extent; - config.shader_spirv = entries.front()->spirv; - config.pipeline_layout = *m_layout.getPipelineLayout(); - config.vertex_bindings = {vertexBindingDescription()}; - const auto vertex_attributes = vertexAttributeDescriptions(); - config.vertex_attributes.assign(vertex_attributes.begin(), vertex_attributes.end()); - config.depth_test = settings.depth_test; - config.depth_write = settings.depth_write; - - // eLessOrEqual since the prepass writes the exact same depth - config.depth_compare = depth::k_closer_or_equal; - config.cull_mode = toCullMode(settings.cull_mode); - config.blend_preset = toBlendPreset(settings.blend_mode); - config.extra_color_formats = worldStageExtraColorFormats(); - - // The cutout entry point discards which costs early Z - m_uses_cutout = resolvedAlphaCutoff() > 0.0f; - if (m_uses_cutout) { - config.fragment_entry = "fragmentMainCutout"; - } +auto MaterialPass::build(const BuildInputs& inputs) -> std::shared_ptr { + ZoneScopedN("MaterialPass build"); + try { + auto built = std::make_shared(*inputs.core, inputs.root_material); + const auto& entries = built->runtime.shaderEntries(); + if (entries.empty()) { + TOAST_WARN("Render", "MaterialPass '{}' has no compiled shaders, nothing will be drawn", inputs.name); + return nullptr; + } + if (entries.size() > 1) { + TOAST_WARN( + "Render", "MaterialPass '{}': multiple shader modules per material not supported yet, using the first", inputs.name + ); + } - config.write_extra_color = config.blend_preset == VulkanPipeline::BlendPreset::none; + built->layout.rebuild(*inputs.core, built->runtime.reflection(), inputs.name); + + const assets::MaterialSettings settings = inputs.root_material->settings(); + + VulkanPipeline::Config config; + config.pipeline_type = VulkanPipeline::PipelineType::graphics; + config.debug_name = std::format("MaterialPass ({})", inputs.name); + config.color_format = inputs.color_format; + config.depth_format = inputs.depth_format; + config.extent = inputs.extent; + config.shader_spirv = entries.front()->spirv; + config.pipeline_layout = *built->layout.getPipelineLayout(); + config.vertex_bindings = {vertexBindingDescription()}; + const auto vertex_attributes = vertexAttributeDescriptions(); + config.vertex_attributes.assign(vertex_attributes.begin(), vertex_attributes.end()); + config.depth_test = settings.depth_test; + config.depth_write = settings.depth_write; + + // Or equal since the prepass writes the exact same depth + config.depth_compare = depth::k_closer_or_equal; + config.cull_mode = toCullMode(settings.cull_mode); + config.blend_preset = toBlendPreset(settings.blend_mode); + config.extra_color_formats = worldStageExtraColorFormats(); + + // The cutout entry point discards which costs early Z + built->uses_cutout = resolvedAlphaCutoffOf(built->runtime) > 0.0f; + if (built->uses_cutout) { + config.fragment_entry = "fragmentMainCutout"; + } - m_pipeline.rebuild(*m_core, config); + config.write_extra_color = config.blend_preset == VulkanPipeline::BlendPreset::none; - TOAST_INFO("Render", "MaterialPass '{}' pipeline ready: {}", m_name, m_pipeline.isReady()); + built->pipeline.rebuild(*inputs.core, config); + if (!built->pipeline.isReady()) { + TOAST_ERROR("Render", "MaterialPass '{}' pipeline failed to build", inputs.name); + return nullptr; + } - { - const auto& blobs = m_root_runtime.uniformBlobs(); - for (const auto& binding : m_root_runtime.reflection().bindings) { + const auto& blobs = built->runtime.uniformBlobs(); + for (const auto& binding : built->runtime.reflection().bindings) { if (binding.kind != ShaderBindingKind::uniform_buffer || !binding.engine_semantic.empty()) { continue; } @@ -194,20 +198,87 @@ void MaterialPass::rebuildPipeline() { values += std::format("{}{}={:.3f}", values.empty() ? "" : ", ", member.name, value); } if (!values.empty()) { - TOAST_TRACE("Render", "MaterialPass '{}' resolved scalars: {}", m_name, values); + TOAST_TRACE("Render", "MaterialPass '{}' resolved scalars: {}", inputs.name, values); } } + + return built; + } catch (const std::exception& e) { + TOAST_ERROR("Render", "MaterialPass '{}' build threw: {}", inputs.name, e.what()); + return nullptr; } +} - const auto& set_layouts = m_layout.getDescriptorSetLayouts(); +void MaterialPass::startBuild() { + m_rebuild_pending.store(false, std::memory_order_release); + BuildInputs inputs { + .core = m_core, + .root_material = m_root_material, + .name = m_name, + .color_format = m_color_format, + .depth_format = m_depth_format, + .extent = m_extent, + }; + m_pending_build = toast::ThreadPool::push([inputs = std::move(inputs)] { return build(inputs); }); +} + +void MaterialPass::adopt(std::shared_ptr built) { + ZoneScoped; + if (built == nullptr) { + if (m_built != nullptr) { + TOAST_WARN("Render", "MaterialPass '{}' rebuild failed, keeping the previous pipeline", m_name); + } + return; + } + + const auto& set_layouts = built->layout.getDescriptorSetLayouts(); if (!set_layouts.empty()) { - m_scene_sets.create(*m_core, m_root_runtime.reflection(), *set_layouts[0], m_name); + built->scene_sets.create(*m_core, built->runtime.reflection(), *set_layouts[0], m_name); + } + + // Frames in flight may still bind the old pipeline and its sets + if (m_built != nullptr) { + VulkanRenderer::instance->retire(std::move(m_built)); + } + m_built = std::move(built); + TOAST_INFO("Render", "MaterialPass '{}' pipeline ready", m_name); +} + +void MaterialPass::update(uint32_t frame_index, float dt) { + (void)frame_index; + (void)dt; + + if (m_pending_build.valid() && m_pending_build.wait_for(std::chrono::seconds(0)) == std::future_status::ready) { + adopt(m_pending_build.get()); + } + + if (m_pending_build.valid()) { + return; + } + + if (m_rebuild_pending.load(std::memory_order_acquire)) { + startBuild(); + return; + } + + // Kept while a build runs since that build may have baked the values before they changed + if (m_built != nullptr && m_values_dirty.exchange(false, std::memory_order_acq_rel)) { + for (auto& [material, res] : m_built->instances) { + res.runtime->markValuesDirty(); + } + } +} + +void MaterialPass::finishPendingBuild() { + if (m_pending_build.valid()) { + ZoneScopedN("MaterialPass wait for build"); + adopt(m_pending_build.get()); } } -auto MaterialPass::ensureInstanceResources(assets::Material* material) -> InstanceResources* { +auto MaterialPass::ensureInstanceResources(Built& built, assets::Material* material) -> InstanceResources* { ZoneScoped; - auto [it, inserted] = m_instances.try_emplace(material); + auto [it, inserted] = built.instances.try_emplace(material); InstanceResources& res = it->second; if (!inserted) { return &res; @@ -216,7 +287,7 @@ auto MaterialPass::ensureInstanceResources(assets::Material* material) -> Instan res.runtime = std::make_unique(*m_core, material); // The variant comes from the root material but alphaCutoff is per instance - if (!m_uses_cutout && resolvedAlphaCutoffOf(*res.runtime) > 0.0f) { + if (!built.uses_cutout && resolvedAlphaCutoffOf(*res.runtime) > 0.0f) { TOAST_WARN( "Render", "Material instance '{}' sets alphaCutoff but its root material '{}' does not, so this pass built the " @@ -226,7 +297,7 @@ auto MaterialPass::ensureInstanceResources(assets::Material* material) -> Instan ); } - const auto& layouts = m_layout.getDescriptorSetLayouts(); + const auto& layouts = built.layout.getDescriptorSetLayouts(); if (layouts.size() < 2) { return &res; } @@ -357,33 +428,23 @@ void MaterialPass::record(vk::CommandBuffer cmd, uint32_t frame_index, uint32_t ZoneScoped; (void)image_index; - if (m_rebuild_pending.exchange(false, std::memory_order_acq_rel)) { - m_core->getDevice().waitIdle(); - rebuildPipeline(); - } - - if (m_values_dirty.exchange(false, std::memory_order_acq_rel)) { - for (auto& [material, res] : m_instances) { - res.runtime->markValuesDirty(); - } - } - - if (!m_pipeline.isReady() || !m_scene_sets.get(frame_index)) { + if (m_built == nullptr || !m_built->scene_sets.get(frame_index)) { return; } + Built& built = *m_built; const auto* frame = VulkanRenderer::instance->renderingFrame(); if (frame == nullptr) { return; } - m_scene_sets.updateTlas(frame_index); + built.scene_sets.updateTlas(frame_index); cmd.bindDescriptorSets( vk::PipelineBindPoint::eGraphics, - *m_layout.getPipelineLayout(), + *built.layout.getPipelineLayout(), 0, - std::array {m_scene_sets.get(frame_index)}, + std::array {built.scene_sets.get(frame_index)}, {} ); @@ -395,7 +456,7 @@ void MaterialPass::record(vk::CommandBuffer cmd, uint32_t frame_index, uint32_t }; const auto draw_instance = [&](const VulkanRenderer::MeshInstanceProxy& proxy, uint32_t instance_count = 1) { - drawInstance(cmd, frame_index, proxy, posed_vertices, posed_offset_of(proxy), &bound_material, instance_count); + drawInstance(built, cmd, frame_index, proxy, posed_vertices, posed_offset_of(proxy), &bound_material, instance_count); }; const auto range = @@ -426,9 +487,9 @@ void MaterialPass::record(vk::CommandBuffer cmd, uint32_t frame_index, uint32_t std::ranges::sort(order, [&](uint32_t a, uint32_t b) { return distance_squared(a) > distance_squared(b); }); } - cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, m_pipeline.getPipeline()); + cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, built.pipeline.getPipeline()); - const bool allow_batching = !isBlended() && m_root_runtime.instanceBaseOffset().has_value(); + const bool allow_batching = !isBlended() && built.runtime.instanceBaseOffset().has_value(); for (size_t i = 0; i < order.size();) { const auto& proxy = frame->mesh_instances[order[i]]; @@ -458,10 +519,10 @@ void MaterialPass::record(vk::CommandBuffer cmd, uint32_t frame_index, uint32_t } void MaterialPass::drawInstance( - vk::CommandBuffer cmd, uint32_t frame_index, const VulkanRenderer::MeshInstanceProxy& proxy, vk::Buffer posed_vertices, - uint32_t posed_vertex_offset, assets::Material** bound_material, uint32_t instance_count + Built& built, vk::CommandBuffer cmd, uint32_t frame_index, const VulkanRenderer::MeshInstanceProxy& proxy, + vk::Buffer posed_vertices, uint32_t posed_vertex_offset, assets::Material** bound_material, uint32_t instance_count ) { - InstanceResources* res = ensureInstanceResources(proxy.material != nullptr ? proxy.material : m_root_material); + InstanceResources* res = ensureInstanceResources(built, proxy.material != nullptr ? proxy.material : m_root_material); if (res == nullptr || res->runtime == nullptr) { return; } @@ -475,7 +536,7 @@ void MaterialPass::drawInstance( for (const auto& set : res->sets[frame_index]) { raw_sets.push_back(*set); } - cmd.bindDescriptorSets(vk::PipelineBindPoint::eGraphics, *m_layout.getPipelineLayout(), 1, raw_sets, {}); + cmd.bindDescriptorSets(vk::PipelineBindPoint::eGraphics, *built.layout.getPipelineLayout(), 1, raw_sets, {}); } if (bound_material != nullptr) { *bound_material = res->runtime->material(); @@ -500,7 +561,11 @@ void MaterialPass::drawInstance( std::memcpy(push_data.data() + *joint_offset_offset, &proxy.joint_offset, sizeof(uint32_t)); } cmd.pushConstants( - *m_layout.getPipelineLayout(), vk::ShaderStageFlagBits::eAll, 0, static_cast(push_data.size()), push_data.data() + *built.layout.getPipelineLayout(), + vk::ShaderStageFlagBits::eAll, + 0, + static_cast(push_data.size()), + push_data.data() ); } @@ -513,23 +578,24 @@ void MaterialPass::drawInstance( } void MaterialPass::recordInstance(vk::CommandBuffer cmd, uint32_t frame_index, const VulkanRenderer::MeshInstanceProxy& proxy) { - if (!m_pipeline.isReady() || !m_scene_sets.get(frame_index) || proxy.mesh == nullptr || !proxy.mesh->isReady()) { + if (m_built == nullptr || !m_built->scene_sets.get(frame_index) || proxy.mesh == nullptr || !proxy.mesh->isReady()) { return; } + Built& built = *m_built; - m_scene_sets.updateTlas(frame_index); + built.scene_sets.updateTlas(frame_index); - cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, m_pipeline.getPipeline()); + cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, built.pipeline.getPipeline()); cmd.bindDescriptorSets( vk::PipelineBindPoint::eGraphics, - *m_layout.getPipelineLayout(), + *built.layout.getPipelineLayout(), 0, - std::array {m_scene_sets.get(frame_index)}, + std::array {built.scene_sets.get(frame_index)}, {} ); const vk::Buffer posed_vertices = VulkanRenderer::instance->getPosedVertexBuffer(frame_index); - drawInstance(cmd, frame_index, proxy, posed_vertices, proxy.posed_vertex_offset, nullptr); + drawInstance(built, cmd, frame_index, proxy, posed_vertices, proxy.posed_vertex_offset, nullptr); } } diff --git a/engine/src/toast/renderer/passes/material_pass.hpp b/engine/src/toast/renderer/passes/material_pass.hpp index 9def2fc99..029e4bf09 100644 --- a/engine/src/toast/renderer/passes/material_pass.hpp +++ b/engine/src/toast/renderer/passes/material_pass.hpp @@ -14,6 +14,7 @@ #include "../vulkan_renderer.hpp" #include +#include #include #include #include @@ -36,27 +37,40 @@ class MaterialPass : public IRenderPass { return m_name; } + ~MaterialPass() override; + + MaterialPass(const MaterialPass&) = delete; + auto operator=(const MaterialPass&) -> MaterialPass& = delete; + MaterialPass(MaterialPass&&) = delete; + auto operator=(MaterialPass&&) -> MaterialPass& = delete; + + void update(uint32_t frame_index, float dt) override; + void record(vk::CommandBuffer cmd, uint32_t frame_index, uint32_t image_index) override; void recordInstance(vk::CommandBuffer cmd, uint32_t frame_index, const VulkanRenderer::MeshInstanceProxy& proxy); + /// Render thread and blocks on a running build for frames such as probe captures + void finishPendingBuild(); + [[nodiscard]] auto rootMaterial() const -> assets::Material* { return m_root_material; } + /// Any thread and the old pipeline draws until the new one is built void markShadersDirty() { m_rebuild_pending.store(true, std::memory_order_release); } void markValuesDirty() { m_values_dirty.store(true, std::memory_order_release); } [[nodiscard]] auto isValid() const -> bool { - return m_pipeline.isReady(); + return m_built != nullptr; } [[nodiscard]] auto usesCutout() const noexcept -> bool { - return m_uses_cutout; + return m_built != nullptr && m_built->uses_cutout; } [[nodiscard]] @@ -65,19 +79,9 @@ class MaterialPass : public IRenderPass { } private: - [[nodiscard]] - auto resolvedAlphaCutoff() -> float; - [[nodiscard]] static auto resolvedAlphaCutoffOf(MaterialRuntime& runtime) -> float; - bool m_uses_cutout = false; - - void drawInstance( - vk::CommandBuffer cmd, uint32_t frame_index, const VulkanRenderer::MeshInstanceProxy& proxy, vk::Buffer posed_vertices, - uint32_t posed_vertex_offset, assets::Material** bound_material, uint32_t instance_count = 1 - ); - struct InstanceResources { std::unique_ptr runtime; @@ -92,8 +96,40 @@ class MaterialPass : public IRenderPass { std::vector> bound_views; }; - void rebuildPipeline(); - auto ensureInstanceResources(assets::Material* material) -> InstanceResources*; + struct Built { + Built(const VulkanCore& core, assets::Material* material) : runtime(core, material) { } + + MaterialRuntime runtime; + ShaderLayout layout; + VulkanPipeline pipeline; + bool uses_cutout = false; + + /// Render thread only since they come from the renderer descriptor pool + SceneDescriptorSets scene_sets; + std::unordered_map instances; + }; + + /// Inputs copied so a worker never touches the pass + struct BuildInputs { + const VulkanCore* core = nullptr; + assets::Material* root_material = nullptr; + std::string name; + vk::Format color_format = vk::Format::eUndefined; + vk::Format depth_format = vk::Format::eUndefined; + vk::Extent2D extent; + }; + + /// Worker thread and null when the shaders or pipeline failed + static auto build(const BuildInputs& inputs) -> std::shared_ptr; + void startBuild(); + void adopt(std::shared_ptr built); + + void drawInstance( + Built& built, vk::CommandBuffer cmd, uint32_t frame_index, const VulkanRenderer::MeshInstanceProxy& proxy, + vk::Buffer posed_vertices, uint32_t posed_vertex_offset, assets::Material** bound_material, uint32_t instance_count = 1 + ); + + auto ensureInstanceResources(Built& built, assets::Material* material) -> InstanceResources*; void updateInstanceDescriptors(InstanceResources& res, uint32_t frame_index); const VulkanCore* m_core = nullptr; @@ -106,12 +142,9 @@ class MaterialPass : public IRenderPass { vk::Format m_depth_format = vk::Format::eUndefined; vk::Extent2D m_extent; - MaterialRuntime m_root_runtime; - ShaderLayout m_layout; - VulkanPipeline m_pipeline; - - SceneDescriptorSets m_scene_sets; - std::unordered_map m_instances; + /// Render thread only + std::shared_ptr m_built; + std::future> m_pending_build; std::vector m_push_scratch; std::vector m_draw_order; diff --git a/engine/src/toast/renderer/passes/voxel_pass.cpp b/engine/src/toast/renderer/passes/voxel_pass.cpp index 6b9359ef7..1e9f7e077 100644 --- a/engine/src/toast/renderer/passes/voxel_pass.cpp +++ b/engine/src/toast/renderer/passes/voxel_pass.cpp @@ -1,6 +1,7 @@ #include "voxel_pass.hpp" #include "../descriptor_writer.hpp" +#include "../gpu_sync.hpp" #include "../shader_cache.hpp" #include "../voxel_gpu_storage.hpp" #include "../vulkan_core.hpp" @@ -13,6 +14,7 @@ #include #include #include +#include #include #include @@ -25,6 +27,13 @@ static_assert(sizeof(glm::mat4) == 64); constexpr uint32_t k_scene_set = 0; constexpr uint32_t k_storage_set = 1; constexpr uint32_t k_instance_binding = 6; +constexpr uint32_t k_cells_binding = 7; +constexpr uint32_t k_cell_args_binding = 8; + +constexpr uint32_t k_cell_phase_count = 0; +constexpr uint32_t k_cell_phase_allocate = 1; +constexpr uint32_t k_cell_phase_write = 2; +constexpr uint32_t k_cell_group_size = 64; /// Albedo normal material depth face normal at bindings 0 to 4 of voxel_lighting.slang constexpr uint32_t k_gbuffer_set = 1; @@ -40,6 +49,7 @@ VoxelPass::VoxelPass(const VulkanCore& core, vk::Format scene_format, vk::Format : m_core(&core) { ZoneScoped; static_assert(sizeof(PushConstants) == 16, "PushConstants is mirrored by voxel.slang"); + static_assert(sizeof(CellCullPush) == 96, "CellCullPush is mirrored by voxel_cells.slang"); const auto uid = assets::resolveURI("core://shaders/voxel.slang"); const auto shader = uid.has_value() ? ShaderCache::get().acquire(*uid) : nullptr; @@ -80,9 +90,105 @@ VoxelPass::VoxelPass(const VulkanCore& core, vk::Format scene_format, vk::Format createInstanceBuffers(core); createDescriptors(core, shader->reflection); + createCells(core); createLighting(core, scene_format, extent); createNormalBlur(core, extent); - TOAST_INFO("Render", "VoxelPass ready: dense DDA into a G-buffer, up to {} volumes a frame", k_max_instances); + TOAST_INFO( + "Render", + "VoxelPass ready: DDA into a G-buffer, up to {} volumes a frame, cell boxes {}", + k_max_instances, + m_cell_pipeline.isReady() ? "available" : "unavailable" + ); +} + +void VoxelPass::createCells(const VulkanCore& core) { + ZoneScoped; + const vk::DescriptorPool pool = VulkanRenderer::instance->getDescriptorPoolHandle(); + m_cell_buffers.clear(); + m_cell_buffers.resize(m_storage_sets.size()); + m_cell_sets.clear(); + + const auto uid = assets::resolveURI("core://shaders/voxel_cells.slang"); + const auto shader = uid.has_value() ? ShaderCache::get().acquire(*uid) : nullptr; + if (shader) { + m_cell_layout.rebuild(core, shader->reflection, "VoxelCells"); + const auto& layouts = m_cell_layout.getDescriptorSetLayouts(); + if (layouts.size() > k_storage_set) { + VulkanPipeline::Config config; + config.pipeline_type = VulkanPipeline::PipelineType::compute; + config.debug_name = "VoxelCells"; + config.shader_spirv = shader->spirv; + config.pipeline_layout = *m_cell_layout.getPipelineLayout(); + m_cell_pipeline.rebuild(core, config); + + const vk::DescriptorSetLayout set_layout = *layouts[k_storage_set]; + for (uint32_t i = 0; i < m_storage_sets.size(); ++i) { + auto allocated = core.getDevice().allocateDescriptorSets(vk::DescriptorSetAllocateInfo(pool, 1, &set_layout)); + m_cell_sets.push_back(std::move(allocated[0])); + setDebugName(core, *m_cell_sets.back(), std::format("VoxelCells StorageSet[{}]", i)); + + DescriptorWriter writer; + writer.buffer(*m_cell_sets.back(), k_instance_binding, vk::DescriptorType::eStorageBuffer, *m_instance_buffers[i]); + writer.flush(core.getDevice()); + } + } + } + if (!m_cell_pipeline.isReady()) { + TOAST_WARN( + "Render", "VoxelPass cell boxes unavailable since core://shaders/voxel_cells.slang did not build, drawing whole volumes" + ); + m_cell_sets.clear(); + } + + // The G-buffer set declares the cell bindings either way so they always hold a buffer + for (uint32_t i = 0; i < m_cell_buffers.size(); ++i) { + ensureCellCapacity(i, k_min_cell_capacity); + } +} + +void VoxelPass::ensureCellCapacity(uint32_t frame_index, uint32_t cells) { + CellBuffers& buffers = m_cell_buffers[frame_index]; + if (buffers.cells.has_value() && buffers.capacity >= cells) { + return; + } + ZoneScoped; + + // A quarter of headroom so a growing scene does not reallocate every frame + const uint32_t capacity = std::max({cells + (cells / 4), k_min_cell_capacity, buffers.capacity}); + + vma::AllocationCreateInfo alloc_ci {}; + alloc_ci.usage = vma::MemoryUsage::eAutoPreferDevice; + + // Safe to replace since the fence of this slot was waited on + vk::BufferCreateInfo cells_ci {}; + cells_ci.size = vk::DeviceSize {capacity} * sizeof(glm::uvec2); + cells_ci.usage = vk::BufferUsageFlagBits::eStorageBuffer; + buffers.cells.emplace(m_core->getAllocator().createBuffer(cells_ci, alloc_ci)); + setDebugName(*m_core, **buffers.cells, std::format("VoxelPass Cells[{}]", frame_index)); + buffers.capacity = capacity; + + if (!buffers.args.has_value()) { + vk::BufferCreateInfo args_ci {}; + args_ci.size = vk::DeviceSize {k_cell_args_words} * sizeof(uint32_t); + args_ci.usage = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eIndirectBuffer | + vk::BufferUsageFlagBits::eTransferDst; + buffers.args.emplace(m_core->getAllocator().createBuffer(args_ci, alloc_ci)); + setDebugName(*m_core, **buffers.args, std::format("VoxelPass CellArgs[{}]", frame_index)); + } + + writeCellDescriptors(frame_index); +} + +void VoxelPass::writeCellDescriptors(uint32_t frame_index) { + const CellBuffers& buffers = m_cell_buffers[frame_index]; + DescriptorWriter writer; + writer.buffer(*m_storage_sets[frame_index], k_cells_binding, vk::DescriptorType::eStorageBuffer, **buffers.cells); + writer.buffer(*m_storage_sets[frame_index], k_cell_args_binding, vk::DescriptorType::eStorageBuffer, **buffers.args); + if (frame_index < m_cell_sets.size()) { + writer.buffer(*m_cell_sets[frame_index], k_cells_binding, vk::DescriptorType::eStorageBuffer, **buffers.cells); + writer.buffer(*m_cell_sets[frame_index], k_cell_args_binding, vk::DescriptorType::eStorageBuffer, **buffers.args); + } + writer.flush(m_core->getDevice()); } void VoxelPass::createNormalBlur(const VulkanCore& core, vk::Extent2D extent) { @@ -242,12 +348,17 @@ void VoxelPass::bindStorage(uint32_t frame_index, const std::shared_ptrgetDevice(), *m_storage_sets[frame_index], *storage); + if (frame_index < m_cell_sets.size()) { + writeVoxelStorageDescriptors(m_core->getDevice(), *m_cell_sets[frame_index], *storage); + } m_bound_storage[frame_index] = storage; } auto VoxelPass::prepare(uint32_t frame_index) -> bool { ZoneScoped; m_draws.clear(); + m_cells_active = false; + m_total_cells = 0; if (frame_index >= m_storage_sets.size() || frame_index >= m_instance_buffers.size() || !m_scene_sets.get(frame_index) || !m_lighting_pipeline.isReady() || !m_blur_pipeline.isReady()) { @@ -269,6 +380,8 @@ auto VoxelPass::prepare(uint32_t frame_index) -> bool { return false; } + bool cells_fit = true; + for (const auto& proxy : frame->voxel_instances) { if (!proxy.visible) { continue; @@ -283,14 +396,24 @@ auto VoxelPass::prepare(uint32_t frame_index) -> bool { break; } + const glm::uvec3 coarse = voxel::gpu::coarseDims(proxy.brick_dims); + cells_fit = cells_fit && glm::all(glm::lessThanEqual(coarse, glm::uvec3(k_max_coarse_dim))); + const auto index = static_cast(m_draws.size()); instances[index] = makeVoxelInstance(proxy.model, proxy.inverse_model, proxy.previous_model, proxy.has_previous, proxy.record_index); instances[index].pad0 = proxy.highlight_record; + instances[index].cell_base = m_total_cells; + if (proxy.mirrored) { + instances[index].flags |= k_voxel_instance_mirrored; + } + m_total_cells += coarse.x * coarse.y * coarse.z; + + // voxel.slang flips the winding of mirrored instances so the cull mode only follows the camera m_draws.push_back( Draw { .instance = index, - .cull = proxy.camera_inside != proxy.mirrored ? Cull::front : Cull::back, + .cull = proxy.camera_inside ? Cull::front : Cull::back, .depth = proxy.camera_inside ? Depth::exact : Depth::conservative, } ); @@ -299,9 +422,117 @@ auto VoxelPass::prepare(uint32_t frame_index) -> bool { return false; } allocation.flush(0, sizeof(VoxelInstanceGpu) * m_draws.size()); + + if (VulkanRenderer::instance->voxelCellBoxes() && m_cell_pipeline.isReady()) { + if (cells_fit) { + ensureCellCapacity(frame_index, m_total_cells); + m_cells_active = true; + } else if (const auto now = std::chrono::steady_clock::now(); now - m_last_limit_warning > std::chrono::seconds(5)) { + m_last_limit_warning = now; + TOAST_WARN( + "Render", "VoxelPass: a volume spans more than {} coarse cells on one axis, drawing whole volumes", k_max_coarse_dim + ); + } + } return true; } +void VoxelPass::recordCellCull(vk::CommandBuffer cmd, uint32_t frame_index) { + ZoneScoped; + if (!m_cells_active) { + return; + } + const auto* frame = VulkanRenderer::instance->renderingFrame(); + if (frame == nullptr) { + m_cells_active = false; + return; + } + + const CellBuffers& buffers = m_cell_buffers[frame_index]; + const vk::Buffer args = **buffers.args; + const vk::Buffer cells = **buffers.cells; + + // The count phase accumulates into the cursors + cmd.fillBuffer( + args, vk::DeviceSize {k_cell_cursors_word} * sizeof(uint32_t), vk::DeviceSize {k_cell_bucket_count} * sizeof(uint32_t), 0 + ); + + sync::BarrierBatch barriers; + barriers.buffer(args, sync::Usage::transfer_dst, sync::Usage::compute_storage_write); + barriers.flush(cmd); + + const vk::PipelineLayout layout = *m_cell_layout.getPipelineLayout(); + cmd.bindPipeline(vk::PipelineBindPoint::eCompute, m_cell_pipeline.getPipeline()); + cmd.bindDescriptorSets( + vk::PipelineBindPoint::eCompute, layout, k_storage_set, std::array {*m_cell_sets[frame_index]}, {} + ); + + CellCullPush push { + .view_projection = frame->frame_data.view_projection, + .camera_near = glm::vec4(frame->frame_data.camera_position, frame->near_plane), + .total_cells = m_total_cells, + .instance_count = static_cast(m_draws.size()), + }; + const uint32_t groups = (m_total_cells + k_cell_group_size - 1) / k_cell_group_size; + + for (const uint32_t phase : {k_cell_phase_count, k_cell_phase_allocate, k_cell_phase_write}) { + push.phase = phase; + cmd.pushConstants(layout, vk::ShaderStageFlagBits::eAll, 0, sizeof(CellCullPush), &push); + cmd.dispatch(phase == k_cell_phase_allocate ? 1 : groups, 1, 1); + if (phase != k_cell_phase_write) { + barriers.buffer(args, sync::Usage::compute_storage_write, sync::Usage::compute_storage_write); + barriers.flush(cmd); + } + } + + barriers.buffer(args, sync::Usage::compute_storage_write, sync::Usage::indirect_read); + barriers.buffer(args, sync::Usage::compute_storage_write, sync::Usage::vertex_storage_read); + barriers.buffer(cells, sync::Usage::compute_storage_write, sync::Usage::vertex_storage_read); + barriers.flush(cmd); +} + +void VoxelPass::recordCellDraws(vk::CommandBuffer cmd, uint32_t frame_index) { + const vk::PipelineLayout layout = *m_shader_layout.getPipelineLayout(); + const vk::Buffer args = **m_cell_buffers[frame_index].args; + constexpr auto k_command_stride = static_cast(sizeof(vk::DrawIndirectCommand)); + + bool sets_bound = false; + const auto bind = [&](const VulkanPipeline& pipeline) { + cmd.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline.getPipeline()); + if (!sets_bound) { + cmd.bindDescriptorSets( + vk::PipelineBindPoint::eGraphics, + layout, + k_scene_set, + std::array {m_scene_sets.get(frame_index), *m_storage_sets[frame_index]}, + {} + ); + sets_bound = true; + } + }; + const auto draw_bucket = [&](uint32_t bucket) { + const PushConstants push {.bucket = bucket, .mode = k_draw_cells}; + cmd.pushConstants(layout, vk::ShaderStageFlagBits::eAll, 0, sizeof(PushConstants), &push); + cmd.drawIndirect(args, vk::DeviceSize {bucket} * k_command_stride, 1, k_command_stride); + }; + + // Cells holding the camera are nearest so they go first and seed depth for the rest + const VulkanPipeline& inside = m_pipelines[static_cast(Cull::front)][static_cast(Depth::exact)]; + if (inside.isReady()) { + bind(inside); + draw_bucket(0); + } + + // Front to back by bucket so early depth testing rejects hidden cells before they march + const VulkanPipeline& outside = m_pipelines[static_cast(Cull::back)][static_cast(Depth::conservative)]; + if (outside.isReady()) { + bind(outside); + for (uint32_t bucket = 1; bucket < k_cell_bucket_count; ++bucket) { + draw_bucket(bucket); + } + } +} + void VoxelPass::record(vk::CommandBuffer cmd, uint32_t frame_index, uint32_t image_index) { ZoneScoped; (void)image_index; @@ -310,9 +541,14 @@ void VoxelPass::record(vk::CommandBuffer cmd, uint32_t frame_index, uint32_t ima return; } + m_scene_sets.updateTlas(frame_index); + if (m_cells_active) { + recordCellDraws(cmd, frame_index); + return; + } + const vk::PipelineLayout layout = *m_shader_layout.getPipelineLayout(); const VulkanPipeline* bound = nullptr; - m_scene_sets.updateTlas(frame_index); for (const Draw& draw : m_draws) { const VulkanPipeline& pipeline = m_pipelines[static_cast(draw.cull)][static_cast(draw.depth)]; diff --git a/engine/src/toast/renderer/passes/voxel_pass.hpp b/engine/src/toast/renderer/passes/voxel_pass.hpp index a41eab84f..45c062ee1 100644 --- a/engine/src/toast/renderer/passes/voxel_pass.hpp +++ b/engine/src/toast/renderer/passes/voxel_pass.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -36,6 +37,9 @@ class VoxelPass : public IRenderPass { [[nodiscard]] auto prepare(uint32_t frame_index) -> bool; + /// Outside any rendering scope after prepare() and before record() + void recordCellCull(vk::CommandBuffer cmd, uint32_t frame_index); + /// Inside the voxel G-buffer scope after prepare() void record(vk::CommandBuffer cmd, uint32_t frame_index, uint32_t image_index) override; @@ -59,9 +63,40 @@ class VoxelPass : public IRenderPass { /// Mirrors voxel.slang PushConstants struct PushConstants { uint32_t instance_index = 0; + uint32_t bucket = 0; + uint32_t mode = 0; uint32_t pad0 = 0; - uint32_t pad1 = 0; - uint32_t pad2 = 0; + }; + + /// Mirrors voxel.slang kDrawVolume and kDrawCells + static constexpr uint32_t k_draw_volume = 0; + static constexpr uint32_t k_draw_cells = 1; + + /// Mirrors voxel_cells.slang CellCullPush + struct CellCullPush { + glm::mat4 view_projection {1.0f}; + glm::vec4 camera_near {0.0f}; + uint32_t total_cells = 0; + uint32_t instance_count = 0; + uint32_t phase = 0; + uint32_t pad0 = 0; + }; + + /// Mirrors voxel_dda.slang kCellBucketCount and the word layout after it + static constexpr uint32_t k_cell_bucket_count = 17; + static constexpr uint32_t k_cell_cursors_word = k_cell_bucket_count * 5; + static constexpr uint32_t k_cell_args_words = k_cell_bucket_count * 6; + /// Ten bits per axis in voxel_dda.slang packCell + static constexpr uint32_t k_max_coarse_dim = 1024; + static constexpr uint32_t k_min_cell_capacity = 4096; + + /// Per frame in flight + struct CellBuffers { + /// Instance index and packed cell per surviving cell + std::optional cells; + /// Indirect commands then bucket offsets then cursors + std::optional args; + uint32_t capacity = 0; }; /// Mirrors voxel_lighting.slang LightingPushConstants @@ -76,6 +111,11 @@ class VoxelPass : public IRenderPass { glm::vec4 viewport {0.0f}; }; + void createCells(const VulkanCore& core); + void ensureCellCapacity(uint32_t frame_index, uint32_t cells); + void writeCellDescriptors(uint32_t frame_index); + void recordCellDraws(vk::CommandBuffer cmd, uint32_t frame_index); + enum class Cull : uint8_t { back, front, @@ -117,6 +157,14 @@ class VoxelPass : public IRenderPass { std::vector m_draws; + ShaderLayout m_cell_layout; + VulkanPipeline m_cell_pipeline; + std::vector m_cell_sets; + std::vector m_cell_buffers; + + bool m_cells_active = false; + uint32_t m_total_cells = 0; + ShaderLayout m_lighting_layout; VulkanPipeline m_lighting_pipeline; SceneDescriptorSets m_lighting_scene_sets; diff --git a/engine/src/toast/renderer/renderer_settings.cpp b/engine/src/toast/renderer/renderer_settings.cpp index ed80be799..81d2c9e92 100644 --- a/engine/src/toast/renderer/renderer_settings.cpp +++ b/engine/src/toast/renderer/renderer_settings.cpp @@ -171,6 +171,16 @@ void registerVoxels(VulkanRenderer& renderer) { .step = 0.05} ) .onChange([&renderer](double v) { renderer.setVoxelNormalRounding(static_cast(v)); }); + + toast::settings::declareBool( + "renderer.voxels.cell_boxes", + true, + {.label = "Cell boxes", + .category = "Voxels", + .description = "On draws one box per non-empty 32 voxel cell front to back and marches only inside it. Off " + "draws one box per volume and marches the whole volume, kept to compare the two."} + ) + .onChange([&renderer](bool enabled) { renderer.setVoxelCellBoxes(enabled); }); } void registerEnvironment(VulkanRenderer& renderer) { diff --git a/engine/src/toast/renderer/shader_cache.cpp b/engine/src/toast/renderer/shader_cache.cpp index d618e4d3a..ed2b11f53 100644 --- a/engine/src/toast/renderer/shader_cache.cpp +++ b/engine/src/toast/renderer/shader_cache.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace renderer { @@ -101,20 +102,34 @@ auto ShaderCache::sourceHandleLocked(toast::UID uid) -> assets::Handlesecond; } -auto ShaderCache::isDiskCacheFreshLocked(toast::UID uid, uint64_t source_hash) -> bool { - loadHashIndexLocked(); - - const auto& shaders = m_hash_index["shaders"]; - if (!shaders.is_object() || !shaders.contains(uid.get())) { - return false; +auto ShaderCache::sourceOf(toast::UID uid) -> std::optional { + const std::lock_guard lock(m_mutex); + auto& handle = sourceHandleLocked(uid); + if (!handle.hasValue()) { + return std::nullopt; } + return handle->source(); +} - const auto& entry = shaders[uid.get()]; - if (entry.value("hash", "") != hashToHex(source_hash)) { - return false; +auto ShaderCache::isDiskCacheFresh(toast::UID uid, uint64_t source_hash) -> bool { + nlohmann::json deps; + { + const std::lock_guard lock(m_mutex); + loadHashIndexLocked(); + + const auto& shaders = m_hash_index["shaders"]; + if (!shaders.is_object() || !shaders.contains(uid.get())) { + return false; + } + + const auto& entry = shaders[uid.get()]; + if (entry.value("hash", "") != hashToHex(source_hash)) { + return false; + } + deps = entry.value("deps", nlohmann::json::object()); } - for (const auto& [dep_uri, dep_hash] : entry.value("deps", nlohmann::json::object()).items()) { + for (const auto& [dep_uri, dep_hash] : deps.items()) { auto dep_bytes = assets::AssetManager::get().tryLoadBytes(dep_uri); if (!dep_bytes || hashToHex(fnv1a(dep_bytes->data(), dep_bytes->size())) != dep_hash.get()) { return false; @@ -124,7 +139,7 @@ auto ShaderCache::isDiskCacheFreshLocked(toast::UID uid, uint64_t source_hash) - return true; } -auto ShaderCache::loadFromDiskLocked(toast::UID uid) -> std::shared_ptr { +auto ShaderCache::loadFromDisk(toast::UID uid) -> EntryPtr { ZoneScoped; auto& manager = assets::AssetManager::get(); @@ -155,16 +170,9 @@ auto ShaderCache::loadFromDiskLocked(toast::UID uid) -> std::shared_ptr std::shared_ptr { +auto ShaderCache::compile(toast::UID uid, const std::string& source) -> EntryPtr { ZoneScoped; - auto& source_handle = sourceHandleLocked(uid); - if (!source_handle.hasValue()) { - TOAST_ERROR("Render", "Cannot compile shader {}: asset not found", uid.get()); - return nullptr; - } - const std::string source_uri = assets::AssetManager::getURI(uid); - const std::string& source = source_handle->source(); auto compiled = ShaderCompiler::compile(uid, source, source_uri); if (compiled.spirv.empty()) { @@ -187,8 +195,7 @@ auto ShaderCache::compileLocked(toast::UID uid) -> std::shared_ptr TOAST_ERROR("Render", "Failed to write {}", spirvUri(uid)); } - nlohmann::json deps_json = nlohmann::json::object(); - nlohmann::json cache_json { + const nlohmann::json cache_json { { "format", k_cache_format}, { "shader", uid.get()}, { "source", entry->source_uri}, @@ -201,21 +208,28 @@ auto ShaderCache::compileLocked(toast::UID uid) -> std::shared_ptr TOAST_ERROR("Render", "Failed to write {}", reflectionUri(uid)); } - loadHashIndexLocked(); + nlohmann::json deps_json = nlohmann::json::object(); + std::vector dep_uids; for (const auto& dep_uri : entry->dependencies) { if (auto dep_bytes = manager.tryLoadBytes(dep_uri)) { deps_json[dep_uri] = hashToHex(fnv1a(dep_bytes->data(), dep_bytes->size())); } + if (auto dep_uid = assets::AssetManager::resolveURI(dep_uri)) { + dep_uids.push_back(dep_uid->data()); + } } - m_hash_index["shaders"][uid.get()] = { - {"hash", hashToHex(entry->hash)}, - {"deps", deps_json}, - }; - saveHashIndexLocked(); - for (const auto& dep_uri : entry->dependencies) { - if (auto dep_uid = assets::AssetManager::resolveURI(dep_uri)) { - m_reverse_deps[dep_uid->data()].insert(uid.data()); + { + const std::lock_guard lock(m_mutex); + loadHashIndexLocked(); + m_hash_index["shaders"][uid.get()] = { + {"hash", hashToHex(entry->hash)}, + {"deps", deps_json}, + }; + saveHashIndexLocked(); + + for (const uint64_t dep_uid : dep_uids) { + m_reverse_deps[dep_uid].insert(uid.data()); } } @@ -224,60 +238,82 @@ auto ShaderCache::compileLocked(toast::UID uid) -> std::shared_ptr return entry; } -auto ShaderCache::loadOrCompileLocked(toast::UID uid) -> std::shared_ptr { +auto ShaderCache::loadOrCompile(toast::UID uid) -> EntryPtr { ZoneScoped; - auto& source_handle = sourceHandleLocked(uid); - if (!source_handle.hasValue()) { + const auto source = sourceOf(uid); + if (!source) { TOAST_ERROR("Render", "Unknown shader asset {}", uid.get()); return nullptr; } - const std::string& source = source_handle->source(); - const uint64_t source_hash = sourceHash(source); - - if (isDiskCacheFreshLocked(uid, source_hash)) { - if (auto entry = loadFromDiskLocked(uid)) { + if (isDiskCacheFresh(uid, sourceHash(*source))) { + if (auto entry = loadFromDisk(uid)) { TOAST_TRACE("Render", "Recovered shader {} from disk cache", uid.get()); return entry; } } - return compileLocked(uid); + return compile(uid, *source); } void ShaderCache::compileAllAtStartup() { ZoneScoped; const auto shader_uids = assets::listByType("shader"); - std::lock_guard lock(m_mutex); - size_t compiled = 0; + std::vector> jobs; + jobs.reserve(shader_uids.size()); for (const auto uid : shader_uids) { - if (m_entries.contains(uid.data())) { - continue; - } - if (auto entry = loadOrCompileLocked(uid)) { - m_entries[uid.data()] = std::move(entry); - ++compiled; - } else { + jobs.push_back(toast::ThreadPool::push([this, uid] { + if (acquire(uid) != nullptr) { + return true; + } TOAST_ERROR("Render", "Failed to load or compile shader {}", uid.get()); - } + return false; + })); + } + + size_t loaded = 0; + for (auto& job : jobs) { + loaded += job.get() ? 1 : 0; } - TOAST_INFO("Render", "Shader cache ready: {}/{} shaders loaded", compiled, shader_uids.size()); + TOAST_INFO("Render", "Shader cache ready: {}/{} shaders loaded", loaded, shader_uids.size()); } -auto ShaderCache::acquire(toast::UID uid) -> std::shared_ptr { +auto ShaderCache::acquire(toast::UID uid) -> EntryPtr { ZoneScoped; - std::lock_guard lock(m_mutex); - - if (const auto it = m_entries.find(uid.data()); it != m_entries.end()) { - return it->second; + std::promise promise; + std::shared_future running; + { + const std::lock_guard lock(m_mutex); + if (const auto it = m_entries.find(uid.data()); it != m_entries.end()) { + return it->second; + } + if (const auto it = m_in_flight.find(uid.data()); it != m_in_flight.end()) { + running = it->second; + } else { + m_in_flight.emplace(uid.data(), promise.get_future().share()); + } + } + // The owner compiles inline rather than queueing so waiting here cannot starve the pool + if (running.valid()) { + return running.get(); } - auto entry = loadOrCompileLocked(uid); - if (entry) { - m_entries[uid.data()] = entry; + EntryPtr entry; + try { + entry = loadOrCompile(uid); + } catch (const std::exception& e) { TOAST_ERROR("Render", "Shader {} threw while compiling: {}", uid.get(), e.what()); } + + { + const std::lock_guard lock(m_mutex); + // A hot reload that landed meanwhile is newer + if (entry != nullptr) { + entry = m_entries.try_emplace(uid.data(), entry).first->second; + } + m_in_flight.erase(uid.data()); } + promise.set_value(entry); return entry; } @@ -287,27 +323,36 @@ auto ShaderCache::ensureCompiled(toast::UID uid) -> bool { auto ShaderCache::onShaderSourceReloaded(toast::UID uid) -> bool { ZoneScoped; - std::lock_guard lock(m_mutex); - auto entry = compileLocked(uid); - if (!entry) { + const auto recompile = [this](toast::UID target) -> bool { + const auto source = sourceOf(target); + auto entry = source ? compile(target, *source) : nullptr; + if (!entry) { + return false; + } + { + const std::lock_guard lock(m_mutex); + m_entries[target.data()] = std::move(entry); + } + event::send(target); + return true; + }; + + if (!recompile(uid)) { TOAST_WARN("Render", "Hot reload of shader {} failed, keeping previous SPIR-V", uid.get()); return false; } - m_entries[uid.data()] = std::move(entry); - event::send(uid); - - const auto it = m_reverse_deps.find(uid.data()); - if (it != m_reverse_deps.end()) { - for (const auto dep_hash : it->second) { - auto dep_entry = compileLocked(toast::UID(dep_hash)); - if (dep_entry) { - m_entries[dep_hash] = std::move(dep_entry); - event::send(toast::UID(dep_hash)); - } else { - TOAST_WARN("Render", "Hot reload of dependent shader (of {}) failed, keeping previous SPIR-V", uid.get()); - } + std::vector dependents; + { + const std::lock_guard lock(m_mutex); + if (const auto it = m_reverse_deps.find(uid.data()); it != m_reverse_deps.end()) { + dependents.assign(it->second.begin(), it->second.end()); + } + } + for (const uint64_t dependent : dependents) { + if (!recompile(toast::UID(dependent))) { + TOAST_WARN("Render", "Hot reload of dependent shader (of {}) failed, keeping previous SPIR-V", uid.get()); } } diff --git a/engine/src/toast/renderer/shader_cache.hpp b/engine/src/toast/renderer/shader_cache.hpp index e1d16f6ff..d7b69bffe 100644 --- a/engine/src/toast/renderer/shader_cache.hpp +++ b/engine/src/toast/renderer/shader_cache.hpp @@ -9,8 +9,10 @@ #include "shader_reflection.hpp" #include +#include #include #include +#include #include #include #include @@ -47,6 +49,7 @@ class TOAST_API ShaderCache { void compileAllAtStartup(); + /// Thread safe and concurrent calls for one shader share a compile auto acquire(toast::UID uid) -> std::shared_ptr; auto ensureCompiled(toast::UID uid) -> bool; @@ -56,18 +59,26 @@ class TOAST_API ShaderCache { static auto fnv1a(const void* data, size_t size) -> uint64_t; private: + using EntryPtr = std::shared_ptr; + ShaderCache(); - auto loadOrCompileLocked(toast::UID uid) -> std::shared_ptr; - auto compileLocked(toast::UID uid) -> std::shared_ptr; - auto loadFromDiskLocked(toast::UID uid) -> std::shared_ptr; - auto isDiskCacheFreshLocked(toast::UID uid, uint64_t source_hash) -> bool; + /// Everything below runs without m_mutex held unless named Locked + auto loadOrCompile(toast::UID uid) -> EntryPtr; + auto compile(toast::UID uid, const std::string& source) -> EntryPtr; + static auto loadFromDisk(toast::UID uid) -> EntryPtr; + auto isDiskCacheFresh(toast::UID uid, uint64_t source_hash) -> bool; + /// Copy since a hot reload may replace the asset while a worker compiles + auto sourceOf(toast::UID uid) -> std::optional; + void loadHashIndexLocked(); void saveHashIndexLocked(); auto sourceHandleLocked(toast::UID uid) -> assets::Handle&; + /// Guards every member below and is never held while compiling std::mutex m_mutex; - std::unordered_map> m_entries; + std::unordered_map m_entries; + std::unordered_map> m_in_flight; std::unordered_map> m_sources; std::unordered_map> m_reverse_deps; nlohmann::json m_hash_index; diff --git a/engine/src/toast/renderer/shader_compiler.cpp b/engine/src/toast/renderer/shader_compiler.cpp index 64517d4eb..06d16070d 100644 --- a/engine/src/toast/renderer/shader_compiler.cpp +++ b/engine/src/toast/renderer/shader_compiler.cpp @@ -9,35 +9,96 @@ #include #include +#include +#include +#include #include #include +#include #include #include +#include namespace renderer { -static Slang::ComPtr slang_global_session; +namespace { -static void ensureSlangGlobalSession() { - if (slang_global_session) { - return; +/// Global sessions are not thread safe so compiles lease one and the cap bounds core modules in memory +class SlangGlobalSessionPool { +public: + class Lease { + public: + Lease() : m_session(SlangGlobalSessionPool::get().acquire()) { } + + ~Lease() { SlangGlobalSessionPool::get().release(std::move(m_session)); } + + Lease(const Lease&) = delete; + auto operator=(const Lease&) -> Lease& = delete; + Lease(Lease&&) = delete; + auto operator=(Lease&&) -> Lease& = delete; + + [[nodiscard]] + auto get() const -> slang::IGlobalSession* { + return m_session.get(); + } + + private: + Slang::ComPtr m_session; + }; + + static auto get() -> SlangGlobalSessionPool& { + static SlangGlobalSessionPool pool; + return pool; } - Slang::ComPtr session; - SlangResult res = slang::createGlobalSession(session.writeRef()); - if (SLANG_FAILED(res)) { - TOAST_CRITICAL("Render", "Failed to create Slang global session"); + + auto acquire() -> Slang::ComPtr { + { + std::unique_lock lock(m_mutex); + m_available.wait(lock, [this] { return !m_free.empty() || m_created < m_limit; }); + if (!m_free.empty()) { + auto session = std::move(m_free.back()); + m_free.pop_back(); + return session; + } + ++m_created; + } + + ZoneScopedN("Create Slang global session"); + Slang::ComPtr session; + if (SLANG_FAILED(slang::createGlobalSession(session.writeRef())) || !session) { + TOAST_CRITICAL("Render", "Failed to create Slang global session"); + } + return session; } - slang_global_session = session; -} -static bool ray_query_available = false; + void release(Slang::ComPtr session) { + { + const std::lock_guard lock(m_mutex); + m_free.push_back(std::move(session)); + } + m_available.notify_one(); + } + +private: + SlangGlobalSessionPool() : m_limit(std::clamp(std::thread::hardware_concurrency() / 2, 1, 4)) { } + + std::mutex m_mutex; + std::condition_variable m_available; + std::vector> m_free; + size_t m_created = 0; + size_t m_limit; +}; + +std::atomic_bool ray_query_available = false; + +} void ShaderCompiler::setRayQueryAvailable(bool available) { - ray_query_available = available; + ray_query_available.store(available, std::memory_order_relaxed); } auto ShaderCompiler::isRayQueryAvailable() -> bool { - return ray_query_available; + return ray_query_available.load(std::memory_order_relaxed); } auto ShaderCompiler::featureHash() -> uint64_t { @@ -49,16 +110,15 @@ auto ShaderCompiler::featureHash() -> uint64_t { #endif // TOAST_REVERSED_Z changes the SPIR-V so the other convention must read as stale constexpr uint64_t k_depth_convention = depth::k_reversed_z ? 0x5eed2e7e45edull : 0x0ull; - return (ray_query_available ? 0x9e3779b97f4a7c15ull : 0x0ull) ^ k_build_profile ^ k_depth_convention; + return (ShaderCompiler::isRayQueryAvailable() ? 0x9e3779b97f4a7c15ull : 0x0ull) ^ k_build_profile ^ k_depth_convention; } -static auto createSession() -> Slang::ComPtr { +static auto createSession(slang::IGlobalSession* global_session) -> Slang::ComPtr { ZoneScoped; - ensureSlangGlobalSession(); slang::TargetDesc target {}; target.format = SLANG_SPIRV; - target.profile = slang_global_session->findProfile("spirv_1_6"); + target.profile = global_session->findProfile("spirv_1_6"); std::array slang_targets {target}; std::vector compiler_options; @@ -94,7 +154,7 @@ static auto createSession() -> Slang::ComPtr { ray_query_entry.name = slang::CompilerOptionName::MacroDefine; ray_query_entry.value.kind = slang::CompilerOptionValueKind::String; ray_query_entry.value.stringValue0 = "TOAST_RAY_QUERY"; - ray_query_entry.value.stringValue1 = ray_query_available ? "1" : "0"; + ray_query_entry.value.stringValue1 = ShaderCompiler::isRayQueryAvailable() ? "1" : "0"; compiler_options.push_back(ray_query_entry); // Always defined to 0 or 1 @@ -125,7 +185,7 @@ static auto createSession() -> Slang::ComPtr { session_desc.compilerOptionEntryCount = SlangInt(compiler_options.size()); Slang::ComPtr session; - SlangResult r = slang_global_session->createSession(session_desc, session.writeRef()); + SlangResult r = global_session->createSession(session_desc, session.writeRef()); if (SLANG_FAILED(r) || !session) { TOAST_CRITICAL("Render", "Failed to create Slang compilation session"); } @@ -149,7 +209,8 @@ void logDiagnostics(const Slang::ComPtr& diagnostics, std::string_ auto compileModule(std::string_view module_name, std::string_view source_path, std::string_view source) -> CompiledShaderCode { ZoneScoped; - auto slang_session = createSession(); + const SlangGlobalSessionPool::Lease global_session; + auto slang_session = createSession(global_session.get()); Slang::ComPtr slang_module; Slang::ComPtr slang_diagnostics; diff --git a/engine/src/toast/renderer/shader_compiler.hpp b/engine/src/toast/renderer/shader_compiler.hpp index 591f71ccb..9ce2b50b9 100644 --- a/engine/src/toast/renderer/shader_compiler.hpp +++ b/engine/src/toast/renderer/shader_compiler.hpp @@ -23,6 +23,7 @@ struct CompiledShaderCode { class ShaderCompiler { public: + /// Thread safe static auto compile(toast::UID uid, std::string_view source, std::string_view source_uri) -> CompiledShaderCode; /// @note Call before any compilation diff --git a/engine/src/toast/renderer/slang_vfs.cpp b/engine/src/toast/renderer/slang_vfs.cpp index c49613392..2f6bdbe69 100644 --- a/engine/src/toast/renderer/slang_vfs.cpp +++ b/engine/src/toast/renderer/slang_vfs.cpp @@ -156,7 +156,7 @@ auto SlangVfs::loadFile(const char* path, ISlangBlob** out_blob) -> SlangResult } SlangVfs::Recorder::Recorder() { - TOAST_ASSERT(s_recorder == nullptr, "Render", "A SlangVfs::Recorder is already active; shader compiles must not overlap"); + TOAST_ASSERT(s_recorder == nullptr, "Render", "A SlangVfs::Recorder is already active on this thread"); s_recorder = this; } diff --git a/engine/src/toast/renderer/slang_vfs.hpp b/engine/src/toast/renderer/slang_vfs.hpp index 8b20f4e2c..eacf508a6 100644 --- a/engine/src/toast/renderer/slang_vfs.hpp +++ b/engine/src/toast/renderer/slang_vfs.hpp @@ -31,7 +31,7 @@ class SlangVfs final : public ISlangFileSystem { /** * @brief Records every URI resolved while alive. Slang getDependencyFileCount() is empty for modules loaded from source - * @note Not thread safe + * @note Per thread so compiles on different threads each record their own imports */ class Recorder { public: @@ -57,7 +57,7 @@ class SlangVfs final : public ISlangFileSystem { }; private: - static inline Recorder* s_recorder = nullptr; + static inline thread_local Recorder* s_recorder = nullptr; }; } diff --git a/engine/src/toast/renderer/voxel_gpu_storage.cpp b/engine/src/toast/renderer/voxel_gpu_storage.cpp index e0d83f1bd..7e058c22c 100644 --- a/engine/src/toast/renderer/voxel_gpu_storage.cpp +++ b/engine/src/toast/renderer/voxel_gpu_storage.cpp @@ -5,8 +5,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -26,15 +28,16 @@ constexpr std::array k_secti /// Storage buffers cannot be empty constexpr vk::DeviceSize k_min_section_bytes = sizeof(uint32_t); -/// Also a copy source since the next patch starts from it +constexpr uint32_t k_slot_headroom_divisor = 4; +constexpr uint32_t k_min_slot_headroom = 256; + auto createSectionBuffer(const VulkanCore& core, vk::DeviceSize size, std::string_view name) -> vma::raii::Buffer { const bool concurrent = core.getGraphicsQueueFamilyIndex() != core.getTransferQueueFamilyIndex(); const std::array family_indices {core.getGraphicsQueueFamilyIndex(), core.getTransferQueueFamilyIndex()}; vk::BufferCreateInfo buffer_ci {}; buffer_ci.size = size; - buffer_ci.usage = - vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferDst | vk::BufferUsageFlagBits::eTransferSrc; + buffer_ci.usage = vk::BufferUsageFlagBits::eStorageBuffer | vk::BufferUsageFlagBits::eTransferDst; if (concurrent) { buffer_ci.sharingMode = vk::SharingMode::eConcurrent; buffer_ci.queueFamilyIndexCount = 2; @@ -68,11 +71,147 @@ auto createStagingBuffer(const VulkanCore& core, vk::DeviceSize size) -> vma::ra } +/// Section buffers shared by every version patched in place since the last full upload +class VoxelGpuSections { +public: + std::array, VoxelGpuStorage::k_section_count> buffers; + std::array sizes {}; + + uint32_t slot_capacity = 0; + + auto queuePatch(const VulkanCore& core, const voxel::gpu::ScenePatch& patch) -> std::optional; + void recordPending(vk::CommandBuffer cmd, uint64_t up_to, std::vector>& retired); + +private: + /// Materials then occupancy then grids then coarse + static constexpr size_t k_patched_sections = 4; + + struct Patch { + uint64_t sequence = 0; + std::shared_ptr staging; + std::array, k_patched_sections> regions; + }; + + std::mutex m_mutex; + std::deque m_pending; + uint64_t m_next_sequence = 0; +}; + +auto VoxelGpuSections::queuePatch(const VulkanCore& core, const voxel::gpu::ScenePatch& patch) -> std::optional { + ZoneScoped; + if (patch.pool_slots > slot_capacity) { + return std::nullopt; + } + + const std::array sections { + &patch.materials, &patch.occupancy, &patch.grids, &patch.coarse + }; + + Patch staged; + vk::DeviceSize total = 0; + std::array offsets {}; + for (size_t i = 0; i < sections.size(); ++i) { + offsets[i] = total; + total += sections[i]->words.size() * sizeof(uint32_t); + for (const voxel::gpu::PatchRun& run : sections[i]->runs) { + staged.regions[i].emplace_back( + offsets[i] + (vk::DeviceSize {run.src} * sizeof(uint32_t)), + vk::DeviceSize {run.dst} * sizeof(uint32_t), + vk::DeviceSize {run.count} * sizeof(uint32_t) + ); + } + } + + if (total > 0) { + try { + staged.staging = std::make_shared(createStagingBuffer(core, total)); + } catch (const std::exception& e) { + TOAST_WARN("Render", "Voxel patch staging failed to allocate, uploading the whole scene instead: {}", e.what()); + return std::nullopt; + } + auto* mapped = static_cast(staged.staging->getAllocation().getInfo().pMappedData); + if (mapped == nullptr) { + return std::nullopt; + } + for (size_t i = 0; i < sections.size(); ++i) { + if (!sections[i]->words.empty()) { + std::memcpy(mapped + offsets[i], sections[i]->words.data(), sections[i]->words.size() * sizeof(uint32_t)); + } + } + } + + const std::lock_guard lock(m_mutex); + staged.sequence = ++m_next_sequence; + const uint64_t sequence = staged.sequence; + m_pending.push_back(std::move(staged)); + return sequence; +} + +void VoxelGpuSections::recordPending(vk::CommandBuffer cmd, uint64_t up_to, std::vector>& retired) { + std::vector ready; + { + const std::lock_guard lock(m_mutex); + while (!m_pending.empty() && m_pending.front().sequence <= up_to) { + ready.push_back(std::move(m_pending.front())); + m_pending.pop_front(); + } + } + if (ready.empty()) { + return; + } + ZoneScoped; + + using Section = VoxelGpuStorage::Section; + constexpr std::array k_targets { + Section::materials, Section::occupancy, Section::grids, Section::coarse + }; + + // Frames already submitted may still read the sections and later patches may rewrite the same words + sync::BarrierBatch barriers; + for (const Patch& patch : ready) { + barriers.memory(sync::Usage::general, sync::Usage::transfer_dst); + barriers.flush(cmd); + for (size_t i = 0; i < k_patched_sections; ++i) { + const auto& target = buffers[static_cast(k_targets[i])]; + if (patch.staging != nullptr && !patch.regions[i].empty() && target.has_value()) { + cmd.copyBuffer(**patch.staging, **target, patch.regions[i]); + } + } + if (patch.staging != nullptr) { + retired.push_back(patch.staging); + } + } + barriers.memory(sync::Usage::transfer_dst, sync::Usage::general); + barriers.flush(cmd); +} + VoxelGpuStorage::VoxelGpuStorage(std::vector node_uids, std::vector brick_dims) : m_node_uids(std::move(node_uids)), m_brick_dims(std::move(brick_dims)), + m_sections(std::make_shared()), m_generation(g_voxel_storage_generation.fetch_add(1, std::memory_order_relaxed) + 1) { } +VoxelGpuStorage::VoxelGpuStorage( + std::vector node_uids, std::vector brick_dims, const VoxelGpuStorage& previous, uint64_t patch_sequence +) + : m_node_uids(std::move(node_uids)), + m_brick_dims(std::move(brick_dims)), + m_sections(previous.m_sections), + m_patch_sequence(patch_sequence), + m_generation(g_voxel_storage_generation.fetch_add(1, std::memory_order_relaxed) + 1) { + markReady(); +} + +auto VoxelGpuStorage::queuePatch(const VulkanCore& core, const voxel::gpu::ScenePatch& patch) const -> std::optional { + return isReady() ? m_sections->queuePatch(core, patch) : std::nullopt; +} + +void VoxelGpuStorage::recordPendingPatches(vk::CommandBuffer cmd, std::vector>& retired) const { + if (m_patch_sequence != 0) { + m_sections->recordPending(cmd, m_patch_sequence, retired); + } +} + auto makeVoxelInstance( const glm::mat4& model, const glm::mat4& inverse_model, const glm::mat4& previous_model, bool has_previous, uint32_t record_index @@ -113,10 +252,14 @@ auto VoxelGpuStorage::recordIndexOf(uint64_t node_uid) const -> std::optional vk::Buffer { - const auto& slot = m_buffers[static_cast(section)]; + const auto& slot = m_sections->buffers[static_cast(section)]; return slot.has_value() ? **slot : vk::Buffer {}; } +auto VoxelGpuStorage::size(Section section) const -> vk::DeviceSize { + return m_sections->sizes[static_cast(section)]; +} + VoxelSceneUpload::VoxelSceneUpload(std::shared_ptr storage, std::shared_ptr packed) : m_storage(std::move(storage)), m_packed(std::move(packed)) { } @@ -138,14 +281,28 @@ void VoxelSceneUpload::build(const VulkanCore& core) { } }; + VoxelGpuSections& sections = *m_storage->m_sections; + + // Spare slots so later carves patch in place + const auto slots = static_cast(pool.materials.size() / voxel::gpu::k_material_words_per_brick); + sections.slot_capacity = slots + std::max(slots / k_slot_headroom_divisor, k_min_slot_headroom); + const std::array capacities { + vk::DeviceSize {sections.slot_capacity} * voxel::gpu::k_material_words_per_brick * sizeof(uint32_t), + vk::DeviceSize {sections.slot_capacity} * voxel::gpu::k_occupancy_words_per_brick * sizeof(uint32_t), + sources[2].second, + sources[3].second, + sources[4].second, + sources[5].second, + }; + try { vk::DeviceSize total = 0; for (size_t i = 0; i < sources.size(); ++i) { m_offsets[i] = total; m_bytes[i] = sources[i].second; total += sources[i].second; - m_storage->m_sizes[i] = std::max(sources[i].second, k_min_section_bytes); - m_storage->m_buffers[i].emplace(createSectionBuffer(core, m_storage->m_sizes[i], k_section_names[i])); + sections.sizes[i] = std::max(capacities[i], k_min_section_bytes); + sections.buffers[i].emplace(createSectionBuffer(core, sections.sizes[i], k_section_names[i])); } if (total > 0) { @@ -207,18 +364,18 @@ void VoxelSceneUpload::record(vk::CommandBuffer cmd) { return; } + const VoxelGpuSections& sections = *m_storage->m_sections; for (size_t i = 0; i < VoxelGpuStorage::k_section_count; ++i) { - if (m_bytes[i] == 0 || !m_storage->m_buffers[i].has_value()) { + if (m_bytes[i] == 0 || !sections.buffers[i].has_value()) { continue; } - cmd.copyBuffer(*m_staging, **m_storage->m_buffers[i], vk::BufferCopy(m_offsets[i], 0, m_bytes[i])); + cmd.copyBuffer(*m_staging, **sections.buffers[i], vk::BufferCopy(m_offsets[i], 0, m_bytes[i])); } sync::BarrierBatch barriers; releaseSections(barriers, *m_storage, m_handoff); barriers.flush(cmd); m_released = true; - m_storage->markRecorded(); } void VoxelSceneUpload::recordAcquire(sync::BarrierBatch& batch) { @@ -227,128 +384,4 @@ void VoxelSceneUpload::recordAcquire(sync::BarrierBatch& batch) { } } -VoxelScenePatchUpload::VoxelScenePatchUpload( - std::shared_ptr storage, std::shared_ptr source, voxel::gpu::ScenePatch patch, - SectionBytes section_bytes -) - : m_storage(std::move(storage)), - m_source(std::move(source)), - m_patch(std::move(patch)), - m_section_bytes(section_bytes) { } - -void VoxelScenePatchUpload::build(const VulkanCore& core) { - ZoneScoped; - m_storage->markUploading(); - - const std::array patched { - &m_patch.materials, &m_patch.occupancy, &m_patch.grids, &m_patch.coarse - }; - - try { - for (size_t i = 0; i < VoxelGpuStorage::k_section_count; ++i) { - m_storage->m_sizes[i] = std::max(m_section_bytes[i], k_min_section_bytes); - m_storage->m_buffers[i].emplace(createSectionBuffer(core, m_storage->m_sizes[i], k_section_names[i])); - } - - vk::DeviceSize total = 0; - for (size_t i = 0; i < patched.size(); ++i) { - m_staging_offsets[i] = total; - total += patched[i]->words.size() * sizeof(uint32_t); - } - - if (total > 0) { - m_staging = createStagingBuffer(core, total); - host_bytes = total; - - auto* mapped = static_cast(m_staging.getAllocation().getInfo().pMappedData); - if (mapped == nullptr) { - TOAST_ERROR("Render", "Voxel staging buffer is not mapped; the voxel scene is not uploaded"); - m_storage->markFailed(IVulkanResource::UploadState::failed_gpu); - return; - } - for (size_t i = 0; i < patched.size(); ++i) { - if (!patched[i]->words.empty()) { - std::memcpy(mapped + m_staging_offsets[i], patched[i]->words.data(), patched[i]->words.size() * sizeof(uint32_t)); - } - } - } - } catch (const std::exception& e) { - TOAST_ERROR("Render", "Voxel scene patch failed to allocate: {}", e.what()); - m_storage->markFailed(IVulkanResource::UploadState::failed_gpu); - } -} - -auto VoxelScenePatchUpload::canRecord() const -> bool { - return m_source->isRecorded() || m_source->hasFailed(); -} - -void VoxelScenePatchUpload::record(vk::CommandBuffer cmd) { - ZoneScoped; - if (m_storage->hasFailed()) { - return; - } - if (m_source->hasFailed()) { - m_storage->markFailed(IVulkanResource::UploadState::failed_gpu); - return; - } - - using Section = VoxelGpuStorage::Section; - constexpr std::array k_sections { - Section::materials, Section::occupancy, Section::grids, Section::coarse, Section::palettes, Section::records - }; - - // The copies that filled the source may still be running and these read it - sync::BarrierBatch barriers; - barriers.memory(sync::Usage::transfer_dst, sync::Usage::transfer_src); - barriers.flush(cmd); - - for (const Section section : k_sections) { - const vk::DeviceSize bytes = std::min(m_source->size(section), m_storage->size(section)); - if (bytes > 0 && m_source->buffer(section) && m_storage->buffer(section)) { - cmd.copyBuffer(m_source->buffer(section), m_storage->buffer(section), vk::BufferCopy(0, 0, bytes)); - } - } - - // The patch overwrites what was just carried over - barriers.memory(sync::Usage::transfer_dst, sync::Usage::transfer_dst); - barriers.flush(cmd); - - const std::array patched { - &m_patch.materials, &m_patch.occupancy, &m_patch.grids, &m_patch.coarse - }; - std::vector regions; - for (size_t i = 0; i < patched.size(); ++i) { - if (patched[i]->runs.empty() || !m_storage->buffer(k_sections[i])) { - continue; - } - - regions.clear(); - for (const voxel::gpu::PatchRun& run : patched[i]->runs) { - regions.emplace_back( - m_staging_offsets[i] + (static_cast(run.src) * sizeof(uint32_t)), - static_cast(run.dst) * sizeof(uint32_t), - static_cast(run.count) * sizeof(uint32_t) - ); - } - cmd.copyBuffer(*m_staging, m_storage->buffer(k_sections[i]), regions); - } - releaseSections(barriers, *m_storage, m_handoff); - barriers.flush(cmd); - m_released = true; - m_storage->markRecorded(); -} - -void VoxelScenePatchUpload::recordAcquire(sync::BarrierBatch& batch) { - if (m_released && !m_storage->hasFailed()) { - acquireSections(batch, *m_storage, m_handoff); - } -} - -void VoxelScenePatchUpload::finished() { - PendingResourceUpload::finished(); - - // The copies out of it are done so the previous storage may go - m_source.reset(); -} - } diff --git a/engine/src/toast/renderer/voxel_gpu_storage.hpp b/engine/src/toast/renderer/voxel_gpu_storage.hpp index f6b67e38e..6271a934c 100644 --- a/engine/src/toast/renderer/voxel_gpu_storage.hpp +++ b/engine/src/toast/renderer/voxel_gpu_storage.hpp @@ -9,7 +9,6 @@ #include "vulkan_resource_base.hpp" #include -#include #include #include #include @@ -49,6 +48,9 @@ struct VoxelStorageDebugInfo { std::shared_ptr mirror; }; +class VoxelGpuSections; + +/// Immutable scene version and versions from small carves share buffers patched in place class VoxelGpuStorage : public IVulkanResource { public: /// In voxel_dda.slang binding order @@ -63,8 +65,15 @@ class VoxelGpuStorage : public IVulkanResource { static constexpr size_t k_section_count = 6; + /// New section buffers that a VoxelSceneUpload fills VoxelGpuStorage(std::vector node_uids, std::vector brick_dims); + /// Ready at once over the buffers of @p previous and needs every patch up to @p patch_sequence applied + VoxelGpuStorage( + std::vector node_uids, std::vector brick_dims, const VoxelGpuStorage& previous, + uint64_t patch_sequence + ); + /// Unique per storage so a cache keyed on it notices every upload [[nodiscard]] auto generation() const noexcept -> uint64_t { @@ -88,9 +97,14 @@ class VoxelGpuStorage : public IVulkanResource { auto buffer(Section section) const -> vk::Buffer; [[nodiscard]] - auto size(Section section) const -> vk::DeviceSize { - return m_sizes[static_cast(section)]; - } + auto size(Section section) const -> vk::DeviceSize; + + /// Tick thread on a ready storage and nullopt when the pool outgrew the buffers or staging failed + [[nodiscard]] + auto queuePatch(const VulkanCore& core, const voxel::gpu::ScenePatch& patch) const -> std::optional; + + /// Render thread before anything reads the sections and @p retired must outlive the frame + void recordPendingPatches(vk::CommandBuffer cmd, std::vector>& retired) const; [[nodiscard]] auto nodeUid(uint32_t index) const -> uint64_t { @@ -108,14 +122,6 @@ class VoxelGpuStorage : public IVulkanResource { /// @note Main thread only and before the upload is queued void setChangeHistory(VoxelChangeHistory history) { m_history = std::move(history); } - /// Set once the copies that fill it are recorded so a patch may read from it - void markRecorded() noexcept { m_recorded.store(true, std::memory_order_release); } - - [[nodiscard]] - auto isRecorded() const noexcept -> bool { - return m_recorded.load(std::memory_order_acquire); - } - [[nodiscard]] auto changeHistory() const -> const VoxelChangeHistory& { return m_history; @@ -123,21 +129,20 @@ class VoxelGpuStorage : public IVulkanResource { private: friend class VoxelSceneUpload; - friend class VoxelScenePatchUpload; std::vector m_node_uids; std::vector m_brick_dims; - std::array, k_section_count> m_buffers; - std::array m_sizes {}; + std::shared_ptr m_sections; + + /// Last in place patch this version needs and 0 for none + uint64_t m_patch_sequence = 0; uint64_t m_generation = 0; VoxelStorageDebugInfo m_debug; VoxelChangeHistory m_history; - - std::atomic m_recorded {false}; }; /// Mirrors voxel_dda.slang VoxelInstance @@ -148,7 +153,8 @@ struct VoxelInstanceGpu { uint32_t record_index = 0; uint32_t pad0 = 0; uint32_t flags = 0; - uint32_t pad2 = 0; + /// First global coarse cell of this instance in the VoxelPass cell cull + uint32_t cell_base = 0; }; static_assert(sizeof(VoxelInstanceGpu) == 208, "VoxelInstanceGpu is mirrored by voxel_dda.slang VoxelInstance"); @@ -156,6 +162,9 @@ static_assert(sizeof(VoxelInstanceGpu) == 208, "VoxelInstanceGpu is mirrored by /// Bit 0 of VoxelInstanceGpu::flags with the meaning of VulkanRenderer::k_instance_no_motion inline constexpr uint32_t k_voxel_instance_no_motion = 1u << 0; +/// Bit 1 of VoxelInstanceGpu::flags so voxel.slang flips the box winding +inline constexpr uint32_t k_voxel_instance_mirrored = 1u << 1; + /// Voxel (x y z) occupies [x x + 1] in the space the instance maps from [[nodiscard]] auto makeVoxelInstance( @@ -192,44 +201,4 @@ class VoxelSceneUpload : public PendingResourceUpload { std::array m_bytes {}; }; -/// Copies the previous storage and overwrites what changed so a carve does not pack or send the whole scene -class VoxelScenePatchUpload : public PendingResourceUpload { -public: - using SectionBytes = std::array; - - VoxelScenePatchUpload( - std::shared_ptr storage, std::shared_ptr source, voxel::gpu::ScenePatch patch, - SectionBytes section_bytes - ); - - void build(const VulkanCore& core) override; - - void record(vk::CommandBuffer cmd) override; - - /// The copies read the source so they wait until its own copies are recorded - [[nodiscard]] - auto canRecord() const -> bool override; - - void finished() override; - - void recordAcquire(sync::BarrierBatch& batch) override; - - auto resource() -> IVulkanResource* override { return m_storage.get(); } - -private: - static constexpr size_t k_patched_sections = 4; - - bool m_released = false; - - std::shared_ptr m_storage; - std::shared_ptr m_source; - voxel::gpu::ScenePatch m_patch; - SectionBytes m_section_bytes {}; - - vma::raii::Buffer m_staging = nullptr; - - /// Materials then occupancy then grids then coarse - std::array m_staging_offsets {}; -}; - } diff --git a/engine/src/toast/renderer/vulkan_renderer.cpp b/engine/src/toast/renderer/vulkan_renderer.cpp index 5625ba9c8..95969d4f7 100644 --- a/engine/src/toast/renderer/vulkan_renderer.cpp +++ b/engine/src/toast/renderer/vulkan_renderer.cpp @@ -654,6 +654,9 @@ void VulkanRenderer::stop() { } catch (const std::exception& e) { TOAST_ERROR("Render", "Device wait failed during shutdown, tearing down anyway: {}", e.what()); } + for (auto& frame : m_frames) { + frame.retired.clear(); + } m_core->savePipelineCache(); } @@ -814,11 +817,16 @@ auto VulkanRenderer::createUploadRing() -> void { for (uint32_t i = 0; i < k_upload_slots; ++i) { m_upload_slots[i].command_buffer = std::move(buffers[i]); - m_upload_slots[i].fence = vk::raii::Fence(m_core->getDevice(), vk::FenceCreateInfo {}); setDebugName(*m_core, *m_upload_slots[i].command_buffer, std::format("VulkanRenderer Upload[{}] CommandBuffer", i)); - setDebugName(*m_core, *m_upload_slots[i].fence, std::format("VulkanRenderer Upload[{}] Fence", i)); } + vk::SemaphoreTypeCreateInfo timeline_ci(vk::SemaphoreType::eTimeline, 0); + m_upload_timeline = vk::raii::Semaphore(m_core->getDevice(), vk::SemaphoreCreateInfo({}, &timeline_ci)); + setDebugName(*m_core, *m_upload_timeline, "VulkanRenderer UploadTimeline"); + m_upload_timeline_value = 0; + m_upload_acquired_value = 0; + m_upload_waited_value = 0; + TOAST_TRACE("VulkanRenderer", "Upload command buffer ring created: {} slots", k_upload_slots); } @@ -1442,6 +1450,11 @@ auto VulkanRenderer::recordVoxelScopes( } const vk::CommandBuffer cmd = *frame.command_buffer; + { + TracyVkZone(m_tracy_vk_ctx, cmd, "VoxelCells"); + const GpuScope scope(m_gpu_timer.get(), m_current_frame, cmd, "Voxel cells"); + m_voxel_pass->recordCellCull(cmd, m_current_frame); + } sync::BarrierBatch barriers; const auto attachment = [](const SceneColorResources& target, vk::AttachmentLoadOp load, const vk::ClearValue& clear) { @@ -1572,6 +1585,15 @@ auto VulkanRenderer::recordFrame(FrameContext& frame, uint32_t image_index) noex m_pending_acquires.flush(*frame.command_buffer); + // Small carves patch the voxel sections in place before anything this frame reads them + if (const auto* rendering = renderingFrame(); rendering != nullptr && rendering->voxel_storage != nullptr) { + std::vector> staging; + rendering->voxel_storage->recordPendingPatches(*frame.command_buffer, staging); + for (auto& buffer : staging) { + retire(std::move(buffer)); + } + } + // Graphics queue since this frame reads it and the compute queue belongs to AsyncCompute if (!m_compute_passes.empty()) { const GpuScope scope(m_gpu_timer.get(), m_current_frame, *frame.command_buffer, "Light culling"); @@ -1904,13 +1926,14 @@ auto VulkanRenderer::recordFrame(FrameContext& frame, uint32_t image_index) noex void VulkanRenderer::ensureMaterialPasses(RenderFrame& frame_data) { ZoneScoped; + std::lock_guard lock(m_pass_mutex); if (m_pending_material_pass_clear.exchange(false, std::memory_order_acq_rel)) { - m_core->getDevice().waitIdle(); - std::lock_guard lock(m_pass_mutex); + for (auto& [material, pass] : m_material_passes) { + retire(std::shared_ptr(std::move(pass))); + } m_material_passes.clear(); } - std::lock_guard lock(m_pass_mutex); for (const auto& proxy : frame_data.mesh_instances) { if (proxy.root_material == nullptr || m_material_passes.contains(proxy.root_material)) { continue; @@ -1921,6 +1944,33 @@ void VulkanRenderer::ensureMaterialPasses(RenderFrame& frame_data) { TOAST_INFO("Render", "Created material pass '{}'", pass->name()); m_material_passes.emplace(proxy.root_material, std::move(pass)); } + + // A baked probe must not miss a material that is still building + if (frame_data.probe_capture_index >= 0 || frame_data.irradiance_capture_index >= 0) { + for (auto& [material, pass] : m_material_passes) { + pass->finishPendingBuild(); + } + } +} + +void VulkanRenderer::hideInstancesWithoutPipeline(RenderFrame& frame_data) { + ZoneScoped; + std::lock_guard lock(m_pass_mutex); + for (auto& proxy : frame_data.mesh_instances) { + if (!proxy.visible || proxy.root_material == nullptr) { + continue; + } + const auto it = m_material_passes.find(proxy.root_material); + if (it == m_material_passes.end() || !it->second->isValid()) { + proxy.visible = false; + } + } +} + +void VulkanRenderer::retire(std::shared_ptr object) { + if (object != nullptr) { + m_frames[m_current_frame].retired.push_back(std::move(object)); + } } auto VulkanRenderer::listPasses() -> std::vector { @@ -2032,6 +2082,7 @@ auto VulkanRenderer::drawFrame(RenderFrame& frame_data) -> void { auto& frame = m_frames[m_current_frame]; std::ignore = m_core->getDevice().waitForFences(*frame.in_flight, true, std::numeric_limits::max()); end_phase(m_perf_gpu_wait_ns); + frame.retired.clear(); publishCompletedFrames(); m_core->updateTracyMemory(); @@ -2069,10 +2120,12 @@ auto VulkanRenderer::drawFrame(RenderFrame& frame_data) -> void { ensureMaterialPasses(frame_data); // TODO move outside the render loop + bool any_unbuilt_material = false; { std::lock_guard lock(m_pass_mutex); for (auto& [material, pass] : m_material_passes) { pass->update(m_current_frame, Time::renderDelta()); + any_unbuilt_material = any_unbuilt_material || !pass->isValid(); } for (auto& pass : m_render_passes) { pass->update(m_current_frame, Time::renderDelta()); @@ -2082,6 +2135,10 @@ auto VulkanRenderer::drawFrame(RenderFrame& frame_data) -> void { pass->update(m_current_frame, Time::renderDelta()); } } + // Only while a first build runs so the steady state skips the walk + if (any_unbuilt_material) { + hideInstancesWithoutPipeline(frame_data); + } if (const auto* rendering = renderingFrame(); rendering != nullptr) { frame.was_probe_capture = rendering->probe_capture_index >= 0 || rendering->irradiance_capture_index >= 0; @@ -2099,13 +2156,23 @@ auto VulkanRenderer::drawFrame(RenderFrame& frame_data) -> void { const vk::Semaphore wait_semaphore = *frame.image_available; // AsyncCompute work never synchronises with a frame on the GPU - const sync::SemaphoreWait image_wait {wait_semaphore, vk::PipelineStageFlagBits2::eColorAttachmentOutput}; + std::array waits {}; + uint32_t wait_count = 0; + if (present_sync) { + waits[wait_count++] = {wait_semaphore, vk::PipelineStageFlagBits2::eColorAttachmentOutput}; + } + // Already reached but makes the release to acquire order explicit for validation + if (m_upload_acquired_value > m_upload_waited_value) { + waits[wait_count++] = {*m_upload_timeline, vk::PipelineStageFlagBits2::eAllCommands, m_upload_acquired_value}; + m_upload_waited_value = m_upload_acquired_value; + } + const sync::SemaphoreSignal present_signal {signal_semaphore}; sync::SubmitDesc submit {}; submit.commands = {&command_buffer, 1}; + submit.waits = std::span(waits).first(wait_count); if (present_sync) { - submit.waits = {&image_wait, 1}; - submit.signals = {&signal_semaphore, 1}; + submit.signals = {&present_signal, 1}; } submit.fence = *frame.in_flight; @@ -3495,6 +3562,7 @@ void VulkanRenderer::tick(float time) noexcept { const glm::vec3 eye = frame.frame_data.camera_position; const float near_plane = m_camera->near_plane; + frame.near_plane = near_plane; for (auto& proxy : frame.voxel_instances) { proxy.camera_inside = voxel::containsPoint(proxy.inverse_model, proxy.brick_dims, eye, near_plane); proxy.visible = proxy.camera_inside || sphereInFrustum(planes, proxy.bounds_center, proxy.bounds_radius); @@ -3682,7 +3750,6 @@ void VulkanRenderer::buildVoxelProxies(RenderFrame& frame) { } m_voxel_storage_pending.reset(); m_voxel_patcher.reset(); - m_voxel_latest_storage.reset(); } } @@ -3819,7 +3886,6 @@ void VulkanRenderer::buildVoxelProxies(RenderFrame& frame) { m_voxel_storage_pending.reset(); m_voxel_change_tracker.reset(); m_voxel_patcher.reset(); - m_voxel_latest_storage.reset(); } else { std::vector scene_volumes; std::vector node_uids; @@ -3841,16 +3907,21 @@ void VulkanRenderer::buildVoxelProxies(RenderFrame& frame) { return std::chrono::duration(std::chrono::steady_clock::now() - pack_start).count(); }; + // Small carves patch the shared buffers in place and only a grown pool or a new layout uploads everything std::optional patch; - if (!keep_mirror && m_voxel_latest_storage != nullptr && !m_voxel_latest_storage->hasFailed()) { + if (!keep_mirror && m_voxel_storage != nullptr && m_voxel_storage->isReady()) { patch = m_voxel_patcher.tryPatch(voxel::runtimeBrickPool(), scene_volumes, node_uids, deltas); } - + std::optional patch_sequence; if (patch.has_value()) { + patch_sequence = m_voxel_storage->queuePatch(*m_core, *patch); + } + + if (patch_sequence.has_value()) { debug.pack_ms = elapsed_ms(); voxel_lock.unlock(); - const VoxelStorageDebugInfo& previous = m_voxel_latest_storage->debugInfo(); + const VoxelStorageDebugInfo& previous = m_voxel_storage->debugInfo(); debug.sequence = ++m_voxel_upload_sequence; debug.packed_slots = patch->pool_slots; debug.packed_palettes = previous.packed_palettes; @@ -3870,24 +3941,12 @@ void VulkanRenderer::buildVoxelProxies(RenderFrame& frame) { }) ); - const voxel::gpu::PackedScene& retained = m_voxel_patcher.scene(); - const VoxelScenePatchUpload::SectionBytes section_bytes { - static_cast(patch->pool_slots) * voxel::gpu::k_material_words_per_brick * sizeof(uint32_t), - static_cast(patch->pool_slots) * voxel::gpu::k_occupancy_words_per_brick * sizeof(uint32_t), - retained.grids.size() * sizeof(uint32_t), - retained.coarse.size() * sizeof(uint32_t), - retained.palettes.size() * sizeof(uint32_t), - retained.records.size() * sizeof(voxel::gpu::VolumeRecord), - }; - - auto storage = std::make_shared(std::move(node_uids), std::move(brick_dims)); + auto storage = + std::make_shared(std::move(node_uids), std::move(brick_dims), *m_voxel_storage, *patch_sequence); storage->setDebugInfo(std::move(debug)); storage->setChangeHistory(m_voxel_change_tracker.packed(storage->generation())); - queueResourceUpload( - std::make_unique(storage, m_voxel_latest_storage, std::move(*patch), section_bytes) - ); - m_voxel_storage_pending = storage; - m_voxel_latest_storage = std::move(storage); + m_voxel_storage = std::move(storage); + m_voxel_change_tracker.published(); } else { auto packed = std::make_shared(VoxelPackedScene { .pool = voxel::gpu::packPool(voxel::runtimeBrickPool()), .scene = voxel::gpu::packScene(scene_volumes) @@ -3922,8 +3981,7 @@ void VulkanRenderer::buildVoxelProxies(RenderFrame& frame) { storage->setDebugInfo(std::move(debug)); storage->setChangeHistory(m_voxel_change_tracker.packed(storage->generation())); queueResourceUpload(std::make_unique(storage, std::move(packed))); - m_voxel_storage_pending = storage; - m_voxel_latest_storage = std::move(storage); + m_voxel_storage_pending = std::move(storage); } } } @@ -4235,19 +4293,19 @@ auto VulkanRenderer::uploadStats() -> UploadStats { void VulkanRenderer::processPendingUploads() { ZoneScoped; - const auto& device = m_core->getDevice(); + const uint64_t completed = m_pending_uploads.empty() ? 0 : m_upload_timeline.getCounterValue(); while (!m_pending_uploads.empty()) { auto& oldest_batch = m_pending_uploads.front(); auto& slot = m_upload_slots[oldest_batch.slot]; - if (vkGetFenceStatus(*device, *slot.fence) != VkResult::VK_SUCCESS) { + if (slot.timeline_value > completed) { break; } vk::DeviceSize reclaimed = 0; for (auto& job : oldest_batch.jobs) { - // The fence wait above orders this after the upload release + // The timeline check above orders this after the upload release job->recordAcquire(m_pending_acquires); job->finished(); reclaimed += job->host_bytes; @@ -4255,6 +4313,7 @@ void VulkanRenderer::processPendingUploads() { m_upload_host_bytes.fetch_sub(reclaimed, std::memory_order_acq_rel); slot.in_flight = false; + m_upload_acquired_value = std::max(m_upload_acquired_value, slot.timeline_value); m_pending_uploads.pop(); } @@ -4297,9 +4356,6 @@ void VulkanRenderer::flushResourceUploads() { return; } - const auto& device = m_core->getDevice(); - - device.resetFences(*slot.fence); slot.command_buffer.reset(); slot.command_buffer.begin(vk::CommandBufferBeginInfo(vk::CommandBufferUsageFlagBits::eOneTimeSubmit)); @@ -4328,6 +4384,8 @@ void VulkanRenderer::flushResourceUploads() { batch.jobs = std::move(recorded); const vk::CommandBuffer raw_transfer_cmd = *slot.command_buffer; + slot.timeline_value = ++m_upload_timeline_value; + const sync::SemaphoreSignal done {*m_upload_timeline, slot.timeline_value}; { // Only contended when AsyncCompute shares the transfer queue std::scoped_lock transfer_lock(m_core->transferSubmitMutex()); @@ -4335,7 +4393,7 @@ void VulkanRenderer::flushResourceUploads() { m_core->getTransferQueue(), { .commands = {&raw_transfer_cmd, 1}, - .fence = *slot.fence + .signals = { &done, 1} } ); } diff --git a/engine/src/toast/renderer/vulkan_renderer.hpp b/engine/src/toast/renderer/vulkan_renderer.hpp index a0cad6efc..193287bf3 100644 --- a/engine/src/toast/renderer/vulkan_renderer.hpp +++ b/engine/src/toast/renderer/vulkan_renderer.hpp @@ -102,6 +102,9 @@ class VulkanRenderer { bool was_probe_capture = false; std::vector blas_scratch; + + /// Released once this frame fence was waited on again + std::vector> retired; }; static constexpr uint32_t k_max_directional_lights = 4; @@ -380,6 +383,9 @@ class VulkanRenderer { std::shared_ptr voxel_storage; + /// Camera near plane distance so voxel cells it clips draw with the camera inside variant + float near_plane = 0.0f; + std::vector asset_refs; std::vector lights; @@ -605,6 +611,13 @@ class VulkanRenderer { } /// Edge rounding radius in voxels where 0 keeps the flat face normals + void setVoxelCellBoxes(bool enabled) noexcept { m_voxel_cell_boxes.store(enabled, std::memory_order_relaxed); } + + [[nodiscard]] + auto voxelCellBoxes() const noexcept -> bool { + return m_voxel_cell_boxes.load(std::memory_order_relaxed); + } + void setVoxelNormalRounding(float voxels) noexcept { m_voxel_normal_rounding.store(std::max(voxels, 0.0f), std::memory_order_relaxed); } @@ -912,6 +925,21 @@ class VulkanRenderer { [[nodiscard]] auto uploadStats() -> UploadStats; + /// Signalled by every transfer submit so a submit needing an upload waits on its value + [[nodiscard]] + auto uploadTimeline() const noexcept -> vk::Semaphore { + return *m_upload_timeline; + } + + /// Render thread only + [[nodiscard]] + auto uploadTimelineValue() const noexcept -> uint64_t { + return m_upload_timeline_value; + } + + /// Render thread and frees @p object once the frame being recorded finished on the GPU + void retire(std::shared_ptr object); + /// @note Render thread only [[nodiscard]] auto gpuTimer() const noexcept -> const GpuTimer* { @@ -1054,6 +1082,8 @@ class VulkanRenderer { void createDefaultCubemap(); void ensureMaterialPasses(RenderFrame& frame_data); + /// After the pass updates so the prepass never writes depth the color pass leaves empty + void hideInstancesWithoutPipeline(RenderFrame& frame_data); void recordFrame(FrameContext& frame, uint32_t image_index) noexcept; @@ -1084,7 +1114,8 @@ class VulkanRenderer { struct UploadSlot { vk::raii::CommandBuffer command_buffer = nullptr; - vk::raii::Fence fence = nullptr; + /// m_upload_timeline reaches it once the batch completed + uint64_t timeline_value = 0; bool in_flight = false; }; @@ -1102,6 +1133,12 @@ class VulkanRenderer { std::vector m_upload_slots; uint32_t m_next_upload_slot = 0; + vk::raii::Semaphore m_upload_timeline = nullptr; + uint64_t m_upload_timeline_value = 0; + /// Newest value whose acquires are recorded and newest a graphics submit waited on + uint64_t m_upload_acquired_value = 0; + uint64_t m_upload_waited_value = 0; + static constexpr vk::DeviceSize k_upload_host_budget = 256ull << 20; /// Guarded by m_upload_mutex @@ -1188,6 +1225,7 @@ class VulkanRenderer { SceneColorResources m_voxel_face_normal; std::atomic m_voxel_normal_rounding {0.3f}; + std::atomic_bool m_voxel_cell_boxes {true}; ShaderLayout m_present_layout; VulkanPipeline m_present_pipeline; @@ -1299,7 +1337,6 @@ class VulkanRenderer { /// Describes the newest storage whether published or pending so a patch starts from it VoxelScenePatcher m_voxel_patcher; - std::shared_ptr m_voxel_latest_storage; std::unordered_map m_voxel_previous_models; From d6700a271973bb9ae132aafeb9950593b5a14603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dar=C3=ADo=20Rodr=C3=ADguez=20Mart=C3=ADnez?= Date: Tue, 13 Oct 2026 18:42:06 +0200 Subject: [PATCH 7/7] Tons of fixes, async compute docs and also compute meshed voxels? --- docs/examples/async_compute/debris.slang | 48 +++ .../async_compute/debris_simulation.cpp | 42 +++ .../async_compute/debris_simulation.hpp | 49 ++++ editor/Engine/ToastEngine.cs | 9 + editor/Workspace/MainWindowView.axaml | 4 + editor/Workspace/MainWindowView.axaml.cs | 2 +- editor/Workspace/MainWindowViewModel.cs | 5 + engine/assets/shaders/voxel.slang | 160 ++++++++-- engine/assets/shaders/voxel_cells.slang | 273 ++++++++++++++++-- engine/assets/shaders/voxel_dda.slang | 30 +- engine/ffi/engine.h | 2 + engine/src/toast/engine.cpp | 6 + .../src/toast/renderer/passes/debug_pass.cpp | 18 +- .../toast/renderer/passes/material_pass.cpp | 11 +- .../toast/renderer/passes/material_pass.hpp | 11 +- .../src/toast/renderer/passes/perf_window.cpp | 63 +++- .../src/toast/renderer/passes/perf_window.hpp | 6 + .../renderer/passes/traced_shadow_pass.cpp | 7 +- .../renderer/passes/traced_shadow_pass.hpp | 4 +- .../src/toast/renderer/passes/voxel_pass.cpp | 124 +++++++- .../src/toast/renderer/passes/voxel_pass.hpp | 30 +- .../src/toast/renderer/ray_tracing_scene.cpp | 1 + .../src/toast/renderer/ray_tracing_scene.hpp | 7 + .../src/toast/renderer/renderer_settings.cpp | 12 + .../toast/renderer/scene_descriptor_set.cpp | 7 +- .../toast/renderer/scene_descriptor_set.hpp | 4 +- .../src/toast/renderer/voxel_gpu_storage.cpp | 33 ++- .../toast/renderer/voxel_scene_patcher.cpp | 149 ++++++++-- .../toast/renderer/voxel_scene_patcher.hpp | 22 +- engine/src/toast/renderer/vulkan_core.cpp | 41 ++- engine/src/toast/renderer/vulkan_renderer.cpp | 48 ++- engine/src/toast/renderer/vulkan_renderer.hpp | 19 ++ engine/src/toast/renderer/vulkan_texture.cpp | 4 + engine/src/toast/renderer/vulkan_texture.hpp | 8 + engine/src/toast/voxel/gpu_layout.cpp | 68 ++++- engine/src/toast/voxel/gpu_layout.hpp | 15 +- 36 files changed, 1193 insertions(+), 149 deletions(-) create mode 100644 docs/examples/async_compute/debris.slang create mode 100644 docs/examples/async_compute/debris_simulation.cpp create mode 100644 docs/examples/async_compute/debris_simulation.hpp diff --git a/docs/examples/async_compute/debris.slang b/docs/examples/async_compute/debris.slang new file mode 100644 index 000000000..ad2ce26c5 --- /dev/null +++ b/docs/examples/async_compute/debris.slang @@ -0,0 +1,48 @@ +// Mirrors DebrisSimulation::Params +struct Params +{ + float dt; + uint count; +}; + +[[vk::push_constant]] +Params gParams; + +// xyz position w radius +[[vk::binding(0, 0)]] +StructuredBuffer gPrevious; + +[[vk::binding(1, 0)]] +RWStructuredBuffer gCurrent; + +// xyz velocity w unused +[[vk::binding(2, 0)]] +RWStructuredBuffer gVelocities; + +static const float3 kGravity = float3(0.0, -9.81, 0.0); + +[shader("compute")] +[numthreads(64, 1, 1)] +void integrateMain(uint3 id : SV_DispatchThreadID) +{ + const uint i = id.x; + if (i >= gParams.count) + { + return; + } + + float4 particle = gPrevious[i]; + float3 velocity = gVelocities[i].xyz + kGravity * gParams.dt; + particle.xyz += velocity * gParams.dt; + + // Ground plane bounce so the example stays on screen + if (particle.y < particle.w) + { + particle.y = particle.w; + velocity.y = -velocity.y * 0.4; + velocity.xz *= 0.8; + } + + gCurrent[i] = particle; + gVelocities[i] = float4(velocity, 0.0); +} diff --git a/docs/examples/async_compute/debris_simulation.cpp b/docs/examples/async_compute/debris_simulation.cpp new file mode 100644 index 000000000..14551ebe9 --- /dev/null +++ b/docs/examples/async_compute/debris_simulation.cpp @@ -0,0 +1,42 @@ +/// @file debris_simulation.cpp +/// @author dario +/// @date 10/10/2026 + +#include "debris_simulation.hpp" + +#include + +namespace example { + +namespace { +constexpr vk::DeviceSize k_particle_bytes = sizeof(glm::vec4) * DebrisSimulation::k_max_particles; +constexpr uint32_t k_workgroup_size = 64; +} + +DebrisSimulation::DebrisSimulation() + : AsyncComputeTask({.shader_uri = "core://shaders/debris.slang", .entries = {"integrateMain"}, .debug_name = "Debris"}) { + // The renderer draws positions and gameplay reads them back. Velocities never leave the task + m_positions = addRingBuffer(k_particle_bytes, true); + m_velocities = addPersistentBuffer(k_particle_bytes); + + // Mirrors the bindings in debris.slang + bind(0, m_positions, Copy::previous); + bind(1, m_positions, Copy::current); + bind(2, m_velocities); +} + +void DebrisSimulation::spawn(std::span positions, std::span velocities) { + m_count = static_cast(std::min({positions.size(), velocities.size(), size_t {k_max_particles}})); + upload(m_positions, positions.first(m_count)); + upload(m_velocities, velocities.first(m_count)); +} + +void DebrisSimulation::step(float dt) { + if (m_count == 0) { + return; + } + const Params params {.dt = dt, .count = m_count}; + AsyncComputeTask::step([&](StepContext& ctx) { ctx.dispatch((m_count + k_workgroup_size - 1) / k_workgroup_size, params); }); +} + +} diff --git a/docs/examples/async_compute/debris_simulation.hpp b/docs/examples/async_compute/debris_simulation.hpp new file mode 100644 index 000000000..be0b1b036 --- /dev/null +++ b/docs/examples/async_compute/debris_simulation.hpp @@ -0,0 +1,49 @@ +/// @file debris_simulation.hpp +/// @author dario +/// @date 10/10/2026 + +#pragma once + +#include +#include +#include + +namespace example { + +/// Debris particles integrated on the async compute queue +class DebrisSimulation : public renderer::AsyncComputeTask { +public: + static constexpr uint32_t k_max_particles = 65536; + + DebrisSimulation(); + + /// Physics thread before the first step + void spawn(std::span positions, std::span velocities); + + /// Physics thread. Returns right away + void step(float dt); + + /// Physics thread. Newest finished positions and false until one exists + auto readPositions(std::span out) -> bool { + return read(m_positions, out); + } + + /// For the render pass. xyz position w radius + [[nodiscard]] + auto positionsId() const -> BufferId { + return m_positions; + } + +private: + /// Mirrors debris.slang Params + struct Params { + float dt; + uint32_t count; + }; + + BufferId m_positions = 0; + BufferId m_velocities = 0; + uint32_t m_count = 0; +}; + +} diff --git a/editor/Engine/ToastEngine.cs b/editor/Engine/ToastEngine.cs index 473398c2c..749d16ee7 100644 --- a/editor/Engine/ToastEngine.cs +++ b/editor/Engine/ToastEngine.cs @@ -128,6 +128,12 @@ public static void SetWindowState(bool focused, bool minimized) { toast_set_window_state(focused ? 1 : 0, minimized ? 1 : 0); } + /// The ImGui overlays drawn over the viewport, such as the performance window + public static void SetShowDebugWindows(bool visible) { + if (!IsEngineReady) return; + toast_set_show_debug_windows(visible ? 1 : 0); + } + public string ProjectPath { get; } public string CorePath { get; } @@ -517,6 +523,9 @@ public static void ReloadManifest() { [LibraryImport(EngineLib)] private static partial uint toast_get_max_tick_rate(); + [LibraryImport(EngineLib)] + private static partial void toast_set_show_debug_windows(int visible); + public static void ReloadProjectSettings() { if (IsEngineReady) OnTickThread(toast_reload_project_settings); } diff --git a/editor/Workspace/MainWindowView.axaml b/editor/Workspace/MainWindowView.axaml index 5054a50bf..c8ba582b2 100644 --- a/editor/Workspace/MainWindowView.axaml +++ b/editor/Workspace/MainWindowView.axaml @@ -366,6 +366,10 @@ Header="_Restore" Click="OnMenuRestore" /> + + diff --git a/editor/Workspace/MainWindowView.axaml.cs b/editor/Workspace/MainWindowView.axaml.cs index 95ff1b8f2..6d378c996 100644 --- a/editor/Workspace/MainWindowView.axaml.cs +++ b/editor/Workspace/MainWindowView.axaml.cs @@ -25,7 +25,7 @@ namespace editor.Workspace; public partial class MainWindowView : Window { - private const int StaticWindowMenuItems = 4; + private const int StaticWindowMenuItems = 6; private readonly ToastEngine? m_toast; private readonly Border? m_toastBorder; diff --git a/editor/Workspace/MainWindowViewModel.cs b/editor/Workspace/MainWindowViewModel.cs index 89f5e9c2a..c05aa8ae4 100644 --- a/editor/Workspace/MainWindowViewModel.cs +++ b/editor/Workspace/MainWindowViewModel.cs @@ -52,6 +52,7 @@ public partial class MainWindowViewModel : ViewModelBase, IDisposable { [ObservableProperty] private bool m_historyVisible; [ObservableProperty] private bool m_locksVisible; [ObservableProperty] private bool m_inspectorVisible = true; + [ObservableProperty] private bool m_showDebugWindows = true; [ObservableProperty] private bool m_signalsVisible = true; [ObservableProperty] private bool m_logsVisible = true; [ObservableProperty] private IRootDock m_mainLayout; @@ -375,6 +376,10 @@ partial void OnCurveEditorVisibleChanged(bool value) { ToggleToastTool("Curve", value); } + partial void OnShowDebugWindowsChanged(bool value) { + ToastEngine.SetShowDebugWindows(value); + } + partial void OnTableEditorVisibleChanged(bool value) { ToggleToastTool("Table", value); } diff --git a/engine/assets/shaders/voxel.slang b/engine/assets/shaders/voxel.slang index 2ce180a79..a65b40b41 100644 --- a/engine/assets/shaders/voxel.slang +++ b/engine/assets/shaders/voxel.slang @@ -9,7 +9,7 @@ struct PushConstants uint instanceIndex; uint bucket; uint mode; - uint pad0; + uint quadCapacity; }; [[vk::push_constant]] @@ -219,7 +219,7 @@ float3 applyEditorOverlays(float3 color, VoxelInstance instance, RayHit hit, flo } /// False discards -bool shadeDebug(uint mode, VSOutput input, VoxelInstance instance, float3 origin, float3 direction, RayHit hit, float3 normal, float3 toCamera, inout FSOutput output) +bool shadeDebug(uint mode, float4 pixel, VoxelInstance instance, float3 origin, float3 direction, RayHit hit, float3 normal, float3 toCamera, inout FSOutput output) { // Before any branch on the hit since derivatives need uniform control flow const float3 cellPosition = origin + direction * hit.t; @@ -234,7 +234,7 @@ bool shadeDebug(uint mode, VSOutput input, VoxelInstance instance, float3 origin return false; } - const bool stripe = ((uint(input.position.x) + uint(input.position.y)) / 4u) % 2u == 0u; + const bool stripe = ((uint(pixel.x) + uint(pixel.y)) / 4u) % 2u == 0u; const float light = hit.hit ? 0.55 + 0.45 * saturate(dot(normal, toCamera)) : (stripe ? 0.5 : 0.2); float3 color = float3(0.0, 0.0, 0.0); @@ -291,29 +291,16 @@ bool shadeDebug(uint mode, VSOutput input, VoxelInstance instance, float3 origin return true; } -bool shade(VSOutput input, out FSOutput output, out float depth) +/// Shared by the march and the quads so both write the same G-buffer +bool shadeHit(float4 pixel, VoxelInstance instance, float3 origin, float3 direction, float3 worldDirection, RayHit hit, + out FSOutput output, inout float depth) { output.albedo = float4(0.0, 0.0, 0.0, 0.0); output.normalRoughness = float4(0.0, 0.0, 1.0, 1.0); output.material = float4(0.0, 0.0, 0.0, 0.0); output.motion = float4(0.0, 0.0, 0.0, 0.0); - depth = input.position.z; - const VoxelInstance instance = gVoxelInstances[input.instanceIndex]; const float3 camera = gCamera.position; - - // t = 1 is the box surface - const float3 worldDirection = input.worldPosition - camera; - const float3 origin = mul(instance.worldToVoxel, float4(camera, 1.0)).xyz; - const float3 direction = mul(instance.worldToVoxel, float4(worldDirection, 0.0)).xyz; - - // Never hit in front of the near plane whose clip distance is linear along the ray in either convention - const float nearOrigin = clipNearDistance(mul(gCamera.viewProjection, float4(camera, 1.0))); - const float nearStep = clipNearDistance(mul(gCamera.viewProjection, float4(worldDirection, 0.0))); - const float tMin = nearStep > 0.0 ? max(0.0, -nearOrigin / nearStep) : 0.0; - - const RayHit hit = marchRayBox(instance.recordIndex, origin, direction, tMin, kFloatMax, input.boxMin, input.boxMax); - const float3 hitWorld = camera + worldDirection * hit.t; const float3 cellPosition = origin + direction * hit.t; @@ -354,7 +341,7 @@ bool shade(VSOutput input, out FSOutput output, out float depth) { depth = kFarDepth; } - return shadeDebug(renderMode, input, instance, origin, direction, hit, normal, toCamera, output); + return shadeDebug(renderMode, pixel, instance, origin, direction, hit, normal, toCamera, output); } if (!hit.hit) @@ -373,6 +360,27 @@ bool shade(VSOutput input, out FSOutput output, out float depth) return true; } +bool shade(VSOutput input, out FSOutput output, out float depth) +{ + depth = input.position.z; + + const VoxelInstance instance = gVoxelInstances[input.instanceIndex]; + const float3 camera = gCamera.position; + + // t = 1 is the box surface + const float3 worldDirection = input.worldPosition - camera; + const float3 origin = mul(instance.worldToVoxel, float4(camera, 1.0)).xyz; + const float3 direction = mul(instance.worldToVoxel, float4(worldDirection, 0.0)).xyz; + + // Never hit in front of the near plane whose clip distance is linear along the ray in either convention + const float nearOrigin = clipNearDistance(mul(gCamera.viewProjection, float4(camera, 1.0))); + const float nearStep = clipNearDistance(mul(gCamera.viewProjection, float4(worldDirection, 0.0))); + const float tMin = nearStep > 0.0 ? max(0.0, -nearOrigin / nearStep) : 0.0; + + const RayHit hit = marchRayBox(instance.recordIndex, origin, direction, tMin, kFloatMax, input.boxMin, input.boxMax); + return shadeHit(input.position, instance, origin, direction, worldDirection, hit, output, depth); +} + // The DDA hit is never closer than the rasterized face which keeps early depth testing on [shader("fragment")] #if TOAST_REVERSED_Z @@ -401,4 +409,114 @@ FSOutput fragmentInside(VSOutput input, out float depth : SV_Depth) discard; } return output; -} \ No newline at end of file +} + +[[vk::binding(9, 1)]] +StructuredBuffer gVoxelQuads; + +struct QuadVSOutput +{ + float4 position : SV_Position; + + [[vk::location(0)]] + float3 worldPosition : TEXCOORD0; + + [[vk::location(1)]] + nointerpolation uint instanceIndex : TEXCOORD1; + + [[vk::location(2)]] + nointerpolation uint direction : TEXCOORD2; +}; + +/// Counter clockwise from outside when the plane axes u and v cross to the face normal +static const uint kQuadCorners[6] = { 0, 1, 2, 0, 2, 3 }; +static const uint kQuadCornersFlipped[6] = { 0, 2, 1, 0, 3, 2 }; + +[shader("vertex")] +QuadVSOutput quadVertexMain(uint vertexId : SV_VertexID) +{ + QuadVSOutput output; + output.position = float4(0.0, 0.0, 0.0, 1.0); + output.worldPosition = float3(0.0, 0.0, 0.0); + output.instanceIndex = 0u; + output.direction = 0u; + + // The mesh pass dropped quads past capacity + const uint quadIndex = vertexId / 6u; + if (quadIndex >= gPush.quadCapacity) + { + return output; + } + + const uint2 quad = gVoxelQuads[quadIndex]; + const uint2 cell = gVoxelCells[quad.x]; + const VoxelInstance instance = gVoxelInstances[cell.x]; + + const uint direction = quad.y & 7u; + const uint slice = (quad.y >> 3) & 7u; + const uint u0 = (quad.y >> 6) & 7u; + const uint v0 = (quad.y >> 9) & 7u; + const uint width = ((quad.y >> 12) & 7u) + 1u; + const uint height = ((quad.y >> 15) & 7u) + 1u; + const uint brickInCell = (quad.y >> 18) & 63u; + + const uint axis = direction >> 1; + const bool positive = (direction & 1u) == 0u; + const uint3 brick = unpackCell(cell.y) * kCoarseBricks + uint3(brickInCell & 3u, (brickInCell >> 2) & 3u, brickInCell >> 4); + const float3 brickMin = float3(brick * kBrickDim); + + // u cross v is +x for X faces and -y for Y faces and +z for Z faces + bool flipped = positive == (axis == 1u); + if ((instance.flags & kInstanceMirrored) != 0u) + { + flipped = !flipped; + } + const uint corner = flipped ? kQuadCornersFlipped[vertexId % 6u] : kQuadCorners[vertexId % 6u]; + const float cornerU = (corner == 1u || corner == 2u) ? float(width) : 0.0; + const float cornerV = corner >= 2u ? float(height) : 0.0; + + float3 local = brickMin; + local[axis] += float(slice) + (positive ? 1.0 : 0.0); + local[quadAxisU(axis)] += float(u0) + cornerU; + local[quadAxisV(axis)] += float(v0) + cornerV; + const float4 world = mul(instance.voxelToWorld, float4(local, 1.0)); + + output.position = mul(gCamera.jitteredViewProjection, world); + output.worldPosition = world.xyz; + output.instanceIndex = cell.x; + output.direction = direction; + return output; +} + +[shader("fragment")] +FSOutput quadFragmentMain(QuadVSOutput input) +{ + const VoxelInstance instance = gVoxelInstances[input.instanceIndex]; + const float3 camera = gCamera.position; + const float3 worldDirection = input.worldPosition - camera; + const float3 origin = mul(instance.worldToVoxel, float4(camera, 1.0)).xyz; + const float3 direction = mul(instance.worldToVoxel, float4(worldDirection, 0.0)).xyz; + + const uint axis = input.direction >> 1; + float3 inward = float3(0.0, 0.0, 0.0); + inward[axis] = (input.direction & 1u) == 0u ? -0.5 : 0.5; + + // Greedy quads span materials so the voxel behind the pixel picks its own + RayHit hit; + hit.t = 1.0; + hit.voxel = int3(floor(origin + direction + inward)); + hit.axis = int(axis); + hit.material = voxelMaterialAt(instance.recordIndex, hit.voxel); + hit.hit = hit.material != 0u; + hit.steps = 0u; + hit.coarseSteps = 0u; + hit.brickSteps = 0u; + + FSOutput output; + float depth = input.position.z; + if (!shadeHit(input.position, instance, origin, direction, worldDirection, hit, output, depth)) + { + discard; + } + return output; +} diff --git a/engine/assets/shaders/voxel_cells.slang b/engine/assets/shaders/voxel_cells.slang index c6b6ab5c1..d589ac8f0 100644 --- a/engine/assets/shaders/voxel_cells.slang +++ b/engine/assets/shaders/voxel_cells.slang @@ -1,8 +1,7 @@ -// Compacts the non-empty coarse cells of every drawn voxel volume into distance buckets for renderer::VoxelPass -// Three dispatches per frame over one thread per coarse cell -// count each surviving cell adds one to its bucket -// allocate one thread turns the counts into offsets and indirect draw commands -// write each surviving cell takes a slot in its bucket +// count each surviving cell adds one to its list +// allocate one thread turns the counts into offsets and indirect commands +// write each surviving cell takes a slot in its list +// mesh one group per near cell turns the exposed faces of its bricks into greedy quads import voxel_dda; /// Mirrors renderer::VoxelPass::CellCullPush @@ -14,7 +13,12 @@ struct CellCullPush uint totalCells; uint instanceCount; uint phase; + uint quadCapacity; + /// Cells closer than this many voxels are meshed and 0 marches every cell + float nearVoxels; uint pad0; + uint pad1; + uint pad2; }; [[vk::push_constant]] @@ -24,6 +28,9 @@ CellCullPush gCull; RWStructuredBuffer gCellsOut; [[vk::binding(8, 1)]] RWStructuredBuffer gCellArgsOut; +/// Cell entry index and packQuad bits +[[vk::binding(9, 1)]] +RWStructuredBuffer gQuadsOut; static const uint kPhaseCount = 0u; static const uint kPhaseAllocate = 1u; @@ -76,7 +83,7 @@ bool outsideFrustum(float4x4 voxelToClip, float3 boxMin, float3 boxMax) return outside != 0u; } -/// Bucket 0 holds cells the camera or near plane is inside and the rest step by a factor of sqrt 2 in distance +/// Near cells go to kNearList and camera cells to bucket 0 while the rest step by sqrt 2 in distance uint classifyCell(uint globalCell, out uint instanceIndex, out uint packed) { instanceIndex = instanceOfCell(globalCell); @@ -104,41 +111,66 @@ uint classifyCell(uint globalCell, out uint instanceIndex, out uint packed) const float3 camera = mul(instance.worldToVoxel, float4(gCull.cameraNear.xyz, 1.0)).xyz; const float guard = gCull.cameraNear.w * worldToVoxelScale(instance.worldToVoxel); - if (all(camera >= boxMin - guard) && all(camera <= boxMax + guard)) + const bool inside = all(camera >= boxMin - guard) && all(camera <= boxMax + guard); + if (!inside && outsideFrustum(mul(gCull.viewProjection, instance.voxelToWorld), boxMin, boxMax)) { - return 0u; + return kNoBucket; } - if (outsideFrustum(mul(gCull.viewProjection, instance.voxelToWorld), boxMin, boxMax)) + if (length(camera - clamp(camera, boxMin, boxMax)) < gCull.nearVoxels) { - return kNoBucket; + return kNearList; + } + if (inside) + { + return 0u; } const float cells = length(camera - 0.5 * (boxMin + boxMax)) / float(kCoarseVoxels); return 1u + min(uint(2.0 * log2(max(cells, 1.0))), kCellBucketCount - 2u); } +void allocateLists() +{ + // Read before the loop turns the cursors into offsets + const uint nearCells = gCellArgsOut[kCellCursorsWord + kNearList]; + + uint offset = 0u; + for (uint list = 0u; list < kCellListCount; ++list) + { + const uint count = gCellArgsOut[kCellCursorsWord + list]; + if (list < kCellBucketCount) + { + gCellArgsOut[list * 4u + 0u] = 36u; + gCellArgsOut[list * 4u + 1u] = count; + gCellArgsOut[list * 4u + 2u] = 0u; + gCellArgsOut[list * 4u + 3u] = 0u; + } + gCellArgsOut[kCellOffsetsWord + list] = offset; + gCellArgsOut[kCellCursorsWord + list] = offset; + offset += count; + } + + gCellArgsOut[kNearDispatchWord + 0u] = gCull.nearVoxels > 0.0 ? nearCells : 0u; + gCellArgsOut[kNearDispatchWord + 1u] = 1u; + gCellArgsOut[kNearDispatchWord + 2u] = 1u; + + // The mesh pass accumulates six vertices per quad into the vertex count + gCellArgsOut[kQuadDrawWord + 0u] = 0u; + gCellArgsOut[kQuadDrawWord + 1u] = 1u; + gCellArgsOut[kQuadDrawWord + 2u] = 0u; + gCellArgsOut[kQuadDrawWord + 3u] = 0u; +} + [shader("compute")] [numthreads(64, 1, 1)] void computeMain(uint3 thread : SV_DispatchThreadID) { if (gCull.phase == kPhaseAllocate) { - if (thread.x != 0u) + if (thread.x == 0u) { - return; - } - uint offset = 0u; - for (uint bucket = 0u; bucket < kCellBucketCount; ++bucket) - { - const uint count = gCellArgsOut[kCellCursorsWord + bucket]; - gCellArgsOut[bucket * 4u + 0u] = 36u; - gCellArgsOut[bucket * 4u + 1u] = count; - gCellArgsOut[bucket * 4u + 2u] = 0u; - gCellArgsOut[bucket * 4u + 3u] = 0u; - gCellArgsOut[kCellOffsetsWord + bucket] = offset; - gCellArgsOut[kCellCursorsWord + bucket] = offset; - offset += count; + allocateLists(); } return; } @@ -150,16 +182,203 @@ void computeMain(uint3 thread : SV_DispatchThreadID) uint instanceIndex; uint packed; - const uint bucket = classifyCell(thread.x, instanceIndex, packed); - if (bucket == kNoBucket) + const uint list = classifyCell(thread.x, instanceIndex, packed); + if (list == kNoBucket) { return; } uint slot; - InterlockedAdd(gCellArgsOut[kCellCursorsWord + bucket], 1u, slot); + InterlockedAdd(gCellArgsOut[kCellCursorsWord + list], 1u, slot); if (gCull.phase == kPhaseWrite) { gCellsOut[slot] = uint2(instanceIndex, packed); } } + +/// Grid entry with the empty tag outside the volume +uint brickEntryAt(uint volume, int3 brick) +{ + const int3 dims = int3(gVolumeRecords[volume].brickDimsX, gVolumeRecords[volume].brickDimsY, gVolumeRecords[volume].brickDimsZ); + if (any(brick < 0) || any(brick >= dims)) + { + return kTagEmpty; + } + return gridEntry(volume, uint3(brick)); +} + +/// Occupancy word in the pooled layout so uniform bricks read as solid +uint occupancyWord(uint entry, uint word) +{ + const uint tag = entry & 3u; + if (tag == kTagEmpty) + { + return 0u; + } + if (tag == kTagUniform) + { + return 0xFFFFFFFFu; + } + return gBrickOccupancy[((entry >> 2) << 4) + word]; +} + +/// Eight x bits of row y in slice z +uint occupancyRow(uint occupancy[16], uint z, uint y) +{ + return (occupancy[(z << 1) + (y >> 2)] >> ((y & 3u) << 3)) & 0xFFu; +} + +/// Exposed faces of one slice as rows over v with bits over u +void exposedRows(uint occupancy[16], uint neighbour[16], uint axis, bool positive, uint slice, out uint rows[8]) +{ + for (uint v = 0u; v < 8u; ++v) + { + uint mask = 0u; + if (axis == 2u) + { + const uint current = occupancyRow(occupancy, slice, v); + uint other; + if (positive) + { + other = slice < 7u ? occupancyRow(occupancy, slice + 1u, v) : occupancyRow(neighbour, 0u, v); + } + else + { + other = slice > 0u ? occupancyRow(occupancy, slice - 1u, v) : occupancyRow(neighbour, 7u, v); + } + mask = current & ~other; + } + else if (axis == 1u) + { + const uint current = occupancyRow(occupancy, v, slice); + uint other; + if (positive) + { + other = slice < 7u ? occupancyRow(occupancy, v, slice + 1u) : occupancyRow(neighbour, v, 0u); + } + else + { + other = slice > 0u ? occupancyRow(occupancy, v, slice - 1u) : occupancyRow(neighbour, v, 7u); + } + mask = current & ~other; + } + else + { + for (uint y = 0u; y < 8u; ++y) + { + const uint row = occupancyRow(occupancy, v, y); + uint other; + if (positive) + { + other = slice < 7u ? row >> (slice + 1u) : occupancyRow(neighbour, v, y); + } + else + { + other = slice > 0u ? row >> (slice - 1u) : occupancyRow(neighbour, v, y) >> 7u; + } + mask |= (((row >> slice) & ~other) & 1u) << y; + } + } + rows[v] = mask & 0xFFu; + } +} + +/// Counts the quads when @p base is kNoBucket and writes them from @p base otherwise +uint meshBrick(uint volume, int3 brick, float3 camera, uint occupancy[16], uint entryIndex, uint brickInCell, uint base) +{ + const int3 brickMin = brick * int(kBrickDim); + uint emitted = 0u; + for (uint direction = 0u; direction < 6u; ++direction) + { + const uint axis = direction >> 1; + const bool positive = (direction & 1u) == 0u; + + // Faces of one direction share a normal so a whole brick side facing away is skipped + const float lowest = float(brickMin[axis]) + (positive ? 1.0 : 0.0); + const float highest = float(brickMin[axis]) + (positive ? 8.0 : 7.0); + if ((positive && camera[axis] <= lowest) || (!positive && camera[axis] >= highest)) + { + continue; + } + + int3 step = int3(0, 0, 0); + step[axis] = positive ? 1 : -1; + const uint neighbourEntry = brickEntryAt(volume, brick + step); + uint neighbour[16]; + for (uint word = 0u; word < 16u; ++word) + { + neighbour[word] = occupancyWord(neighbourEntry, word); + } + + for (uint slice = 0u; slice < 8u; ++slice) + { + const float plane = float(brickMin[axis] + int(slice)) + (positive ? 1.0 : 0.0); + if ((positive && camera[axis] <= plane) || (!positive && camera[axis] >= plane)) + { + continue; + } + + uint rows[8]; + exposedRows(occupancy, neighbour, axis, positive, slice, rows); + + for (uint v = 0u; v < 8u; ++v) + { + while (rows[v] != 0u) + { + const uint u = firstbitlow(rows[v]); + const uint width = firstbitlow(~(rows[v] >> u)); + const uint run = ((1u << width) - 1u) << u; + uint height = 1u; + while (v + height < 8u && (rows[v + height] & run) == run) + { + rows[v + height] &= ~run; + ++height; + } + rows[v] &= ~run; + + if (base != kNoBucket && base + emitted < gCull.quadCapacity) + { + gQuadsOut[base + emitted] = uint2(entryIndex, packQuad(direction, slice, u, v, width, height, brickInCell)); + } + ++emitted; + } + } + } + } + return emitted; +} + +/// One group per near cell and one thread per brick +[shader("compute")] +[numthreads(64, 1, 1)] +void meshMain(uint3 group : SV_GroupID, uint3 local : SV_GroupThreadID) +{ + const uint entryIndex = gCellArgsOut[kCellOffsetsWord + kNearList] + group.x; + const uint2 entry = gCellsOut[entryIndex]; + const VoxelInstance instance = gVoxelInstances[entry.x]; + + const uint brickInCell = local.x; + const int3 brick = int3(unpackCell(entry.y) * kCoarseBricks + uint3(brickInCell & 3u, (brickInCell >> 2) & 3u, brickInCell >> 4)); + const uint brickEntry = brickEntryAt(instance.recordIndex, brick); + if ((brickEntry & 3u) == kTagEmpty) + { + return; + } + + uint occupancy[16]; + for (uint word = 0u; word < 16u; ++word) + { + occupancy[word] = occupancyWord(brickEntry, word); + } + + const float3 camera = mul(instance.worldToVoxel, float4(gCull.cameraNear.xyz, 1.0)).xyz; + const uint count = meshBrick(instance.recordIndex, brick, camera, occupancy, entryIndex, brickInCell, kNoBucket); + if (count == 0u) + { + return; + } + + uint vertexBase; + InterlockedAdd(gCellArgsOut[kQuadDrawWord], count * 6u, vertexBase); + meshBrick(instance.recordIndex, brick, camera, occupancy, entryIndex, brickInCell, vertexBase / 6u); +} diff --git a/engine/assets/shaders/voxel_dda.slang b/engine/assets/shaders/voxel_dda.slang index 0c8c8726b..77560afe4 100644 --- a/engine/assets/shaders/voxel_dda.slang +++ b/engine/assets/shaders/voxel_dda.slang @@ -50,6 +50,7 @@ StructuredBuffer gVoxelInstances; static const uint kBrickDim = 8; static const int kCoarseVoxels = 32; // gpu::k_coarse_bricks * k_brick_dim +static const uint kCoarseBricks = 4u; // gpu::k_coarse_bricks static const uint kMaterialWordsPerBrick = 128; static const uint kOccupancyWordsPerBrick = 16; static const uint kMaxMarchSteps = 1024; @@ -71,11 +72,34 @@ static const uint kBoxIndices[36] = { 1, 2, 6, 1, 6, 5, // +X }; -/// Cell args word layout mirrored by renderer::VoxelPass with bucket 0 holding the camera +/// Cell args word layout mirrored by renderer::VoxelPass +/// Draw commands per box bucket with bucket 0 holding the camera then offsets and cursors per list with the near cells last +/// then the mesh dispatch and the quad draw static const uint kCellBucketCount = 17; +static const uint kNearList = kCellBucketCount; +static const uint kCellListCount = kCellBucketCount + 1; static const uint kCellOffsetsWord = kCellBucketCount * 4; -static const uint kCellCursorsWord = kCellBucketCount * 5; -static const uint kCellArgsWords = kCellBucketCount * 6; +static const uint kCellCursorsWord = kCellOffsetsWord + kCellListCount; +static const uint kNearDispatchWord = kCellCursorsWord + kCellListCount; +static const uint kQuadDrawWord = kNearDispatchWord + 3; +static const uint kCellArgsWords = kQuadDrawWord + 4; + +/// Bits are direction 3 slice 3 u 3 v 3 width-1 3 height-1 3 and brick in cell 6 +uint packQuad(uint direction, uint slice, uint u, uint v, uint width, uint height, uint brickInCell) +{ + return direction | (slice << 3) | (u << 6) | (v << 9) | ((width - 1u) << 12) | ((height - 1u) << 15) | (brickInCell << 18); +} + +/// Face plane axes per normal axis as rows over v with bits over u +uint quadAxisU(uint axis) +{ + return axis == 0u ? 1u : 0u; +} + +uint quadAxisV(uint axis) +{ + return axis == 2u ? 1u : 2u; +} /// Ten bits per axis uint packCell(uint3 cell) diff --git a/engine/ffi/engine.h b/engine/ffi/engine.h index edf2d8233..70f8a30b2 100644 --- a/engine/ffi/engine.h +++ b/engine/ffi/engine.h @@ -115,6 +115,8 @@ TOAST_C_API void toast_reload_project_settings(void) NOEXCEPT; /// @return Maximum ticks per second from the project settings, 0 is uncapped TOAST_C_API uint32_t toast_get_max_tick_rate(void) NOEXCEPT; +TOAST_C_API void toast_set_show_debug_windows(int visible) NOEXCEPT; + /// @brief Tells the renderer whether the host application is focused and whether it is minimized /// @param focused 0 holds rendering at 30 fps (or the configured cap when lower), 1 restores the configured cap /// @param minimized 1 stops rendering entirely until called again with 0 diff --git a/engine/src/toast/engine.cpp b/engine/src/toast/engine.cpp index d1e6262a5..e4445df89 100644 --- a/engine/src/toast/engine.cpp +++ b/engine/src/toast/engine.cpp @@ -1107,6 +1107,12 @@ auto toast_get_max_tick_rate() noexcept -> uint32_t { return toast::ProjectSettings::maxTickRate(); } +void toast_set_show_debug_windows(int visible) noexcept { + if (auto* vk_renderer = renderer::VulkanRenderer::instance; vk_renderer != nullptr) { + vk_renderer->setDebugWindowsVisible(visible != 0); + } +} + void toast_set_window_state(int focused, int minimized) noexcept { auto* vk_renderer = renderer::VulkanRenderer::instance; if (vk_renderer == nullptr) { diff --git a/engine/src/toast/renderer/passes/debug_pass.cpp b/engine/src/toast/renderer/passes/debug_pass.cpp index c59e3f97b..d7373de46 100644 --- a/engine/src/toast/renderer/passes/debug_pass.cpp +++ b/engine/src/toast/renderer/passes/debug_pass.cpp @@ -423,17 +423,21 @@ void DebugPass::update(uint32_t frame_index, float dt) { ImGui_ImplVulkan_NewFrame(); ImGui::NewFrame(); - if (!m_perf) { - m_perf = std::make_unique(); + const bool show_windows = VulkanRenderer::instance->debugWindowsVisible(); + if (show_windows) { + if (!m_perf) { + m_perf = std::make_unique(); + } + m_perf->draw(); } - m_perf->draw(); if (!m_voxels) { m_voxels = std::make_unique(); } m_voxels->update(*frame); - if (m_editor_panels && ImGui::Begin("Toast Debug")) { + const bool editor_panels = m_editor_panels && show_windows; + if (editor_panels && ImGui::Begin("Toast Debug")) { { const uint32_t dropped = VulkanRenderer::instance->getDroppedFrameCount(); const uint32_t out_of_order = VulkanRenderer::instance->getOutOfOrderFrameCount(); @@ -845,11 +849,13 @@ void DebugPass::update(uint32_t frame_index, float dt) { ImGui::TextColored(ImVec4(1.0f, 0.2f, 0.2f, 1.0f), "Red = 8+ lights/cluster"); } } - if (m_editor_panels) { + if (editor_panels) { ImGui::End(); } - m_voxels->drawOverlay(*frame); + if (show_windows) { + m_voxels->drawOverlay(*frame); + } if (frame->render_mode == 1 && m_cluster_lighting_pass != nullptr) { m_cluster_lighting_pass->requestGridReadback(); diff --git a/engine/src/toast/renderer/passes/material_pass.cpp b/engine/src/toast/renderer/passes/material_pass.cpp index 5dd0e2e66..d3f9ac223 100644 --- a/engine/src/toast/renderer/passes/material_pass.cpp +++ b/engine/src/toast/renderer/passes/material_pass.cpp @@ -342,7 +342,7 @@ auto MaterialPass::ensureInstanceResources(Built& built, assets::Material* mater std::format("{} MaterialSet{} frame{} ({})", m_name, set_index, frame, material->name()) ); } - res.bound_views[frame].assign(res.runtime->textureSlots().size(), vk::ImageView {}); + res.bound_views[frame].assign(res.runtime->textureSlots().size(), InstanceResources::BoundView {}); DescriptorWriter writer; for (const auto& ubo : res.ubo_buffers) { @@ -376,7 +376,7 @@ void MaterialPass::updateInstanceDescriptors(InstanceResources& res, uint32_t fr const auto& slots = res.runtime->textureSlots(); if (res.bound_views[frame_index].size() != slots.size()) { - res.bound_views[frame_index].assign(slots.size(), vk::ImageView {}); + res.bound_views[frame_index].assign(slots.size(), InstanceResources::BoundView {}); } for (size_t i = 0; i < slots.size(); ++i) { @@ -396,8 +396,10 @@ void MaterialPass::updateInstanceDescriptors(InstanceResources& res, uint32_t fr vk::Sampler sampler = slot.sampler ? slot.sampler : VulkanRenderer::instance->getDefaultSampler(); const VulkanTexture* gpu_texture = slot.texture.hasValue() ? &slot.texture->gpuTexture() : nullptr; + uint64_t generation = 0; if (gpu_texture != nullptr && gpu_texture->isReady() && gpu_texture->getView()) { view = gpu_texture->getView(); + generation = gpu_texture->generation(); warnIfWrongColorSpace(slot, *gpu_texture); } else if ( const vk::ImageView failsafe = @@ -407,7 +409,8 @@ void MaterialPass::updateInstanceDescriptors(InstanceResources& res, uint32_t fr sampler = VulkanRenderer::instance->getFailsafeSampler(); } - if (res.bound_views[frame_index][i] == view) { + const InstanceResources::BoundView bound {.view = view, .generation = generation}; + if (res.bound_views[frame_index][i] == bound) { continue; } @@ -420,7 +423,7 @@ void MaterialPass::updateInstanceDescriptors(InstanceResources& res, uint32_t fr *res.sets[frame_index][slot.set - 1], slot.binding, 0, 1, vk::DescriptorType::eCombinedImageSampler, &image_info ); device.updateDescriptorSets(write, {}); - res.bound_views[frame_index][i] = view; + res.bound_views[frame_index][i] = bound; } } diff --git a/engine/src/toast/renderer/passes/material_pass.hpp b/engine/src/toast/renderer/passes/material_pass.hpp index 029e4bf09..e2f16e9f0 100644 --- a/engine/src/toast/renderer/passes/material_pass.hpp +++ b/engine/src/toast/renderer/passes/material_pass.hpp @@ -93,7 +93,16 @@ class MaterialPass : public IRenderPass { std::vector ubo_buffers; std::vector> sets; - std::vector> bound_views; + + /// View plus texture generation since a recreated view can reuse a destroyed handle value + struct BoundView { + vk::ImageView view; + uint64_t generation = 0; + + auto operator==(const BoundView&) const -> bool = default; + }; + + std::vector> bound_views; }; struct Built { diff --git a/engine/src/toast/renderer/passes/perf_window.cpp b/engine/src/toast/renderer/passes/perf_window.cpp index 4fefab0ae..5c3d76ef4 100644 --- a/engine/src/toast/renderer/passes/perf_window.cpp +++ b/engine/src/toast/renderer/passes/perf_window.cpp @@ -459,7 +459,8 @@ void PerfWindow::drawSummary(bool has_gpu) { ImGui::TextDisabled("(uncapped)"); } - const float width = ImGui::GetFontSize() * 30.0f; + const float width = + m_expanded ? std::max(ImGui::GetContentRegionAvail().x, ImGui::GetFontSize() * 20.0f) : ImGui::GetFontSize() * 30.0f; drawFrameGraph(width, ImGui::GetFontSize() * 4.0f); drawPhaseBar(width, ImGui::GetFontSize() * 0.6f); @@ -521,7 +522,7 @@ void PerfWindow::drawGpuTab() { } const bool has_baseline = m_graphics_baseline >= 0.0f; - const float width = ImGui::GetFontSize() * 36.0f; + const float width = std::max(ImGui::GetContentRegionAvail().x, ImGui::GetFontSize() * 20.0f); ImGui::Text("Graphics queue %.2f ms/frame", m_graphics_avg); if (has_baseline) { @@ -562,16 +563,23 @@ void PerfWindow::drawGpuTab() { perfColor(230, 210, 80), perfColor(200, 120, 170), }; + constexpr ImU32 k_unscoped_color = perfColor(70, 70, 70); std::vector segments; + // Nested passes share the color of the top level pass they run under + std::vector row_colors(m_passes.size(), k_unscoped_color); + ImU32 scope_color = k_unscoped_color; float scoped_ms = 0.0f; - for (const PassStats& pass : m_passes) { + for (size_t i = 0; i < m_passes.size(); ++i) { + const PassStats& pass = m_passes[i]; if (pass.depth == 0) { - segments.push_back({pass.name, pass.perFrame(), k_pass_colors[segments.size() % k_pass_colors.size()]}); + scope_color = k_pass_colors[segments.size() % k_pass_colors.size()]; + segments.push_back({pass.name, pass.perFrame(), scope_color}); scoped_ms += pass.perFrame(); } + row_colors[i] = scope_color; } const float unscoped_ms = std::max(0.0f, m_graphics_avg - scoped_ms); - segments.push_back({"Unscoped", unscoped_ms, perfColor(70, 70, 70)}); + segments.push_back({"Unscoped", unscoped_ms, k_unscoped_color}); perfStackedBar("##perf_gpu_bar", width, ImGui::GetFontSize() * 0.9f, m_graphics_avg, segments); ImGui::Checkbox("Hide idle passes", &m_hide_idle_passes); @@ -590,8 +598,8 @@ void PerfWindow::drawGpuTab() { constexpr ImGuiTableFlags k_flags = ImGuiTableFlags_SizingFixedFit | ImGuiTableFlags_NoHostExtendX | ImGuiTableFlags_RowBg | ImGuiTableFlags_BordersInnerV | ImGuiTableFlags_Sortable | ImGuiTableFlags_SortTristate | ImGuiTableFlags_ScrollY; - // Fixed sizes so the auto sized window does not jitter as rows and digits change - const ImVec2 outer(width, ImGui::GetTextLineHeightWithSpacing() * 18.0f); + // Takes the height the window has left so resizing it shows more rows + const ImVec2 outer(0.0f, std::max(ImGui::GetContentRegionAvail().y, ImGui::GetTextLineHeightWithSpacing() * 8.0f)); const int column_count = has_baseline ? 6 : 5; if (!ImGui::BeginTable("##perf_gpu_passes", column_count, k_flags, outer)) { return; @@ -657,6 +665,7 @@ void PerfWindow::drawGpuTab() { ImGui::PushID(static_cast(i)); ImGui::TableNextColumn(); + perfSwatch(row_colors[i]); const int indent = sorted ? 0 : static_cast(pass.depth * 2); if (is_idle || absent) { ImGui::TextDisabled("%*s%s", indent, "", pass.name.c_str()); @@ -724,6 +733,7 @@ void PerfWindow::drawGpuTab() { if (!sorted && unscoped_ms >= 0.01f) { ImGui::TableNextRow(); ImGui::TableNextColumn(); + perfSwatch(k_unscoped_color); ImGui::TextDisabled("Unscoped"); ImGui::TableNextColumn(); ImGui::TextDisabled("%5.2f", unscoped_ms); @@ -842,10 +852,42 @@ void PerfWindow::draw() { const ImGuiIO& io = ImGui::GetIO(); ImGui::SetNextWindowPos(ImVec2(io.DisplaySize.x - 12.0f, 12.0f), ImGuiCond_FirstUseEver, ImVec2(1.0f, 0.0f)); ImGui::SetNextWindowBgAlpha(0.88f); - constexpr ImGuiWindowFlags k_flags = - ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoFocusOnAppearing | ImGuiWindowFlags_NoNav; - const bool open = ImGui::Begin("Performance", nullptr, k_flags); + // Fits its content collapsed and the user sizes it with the details open + ImGuiWindowFlags flags = ImGuiWindowFlags_NoFocusOnAppearing | ImGuiWindowFlags_NoNav; + if (!m_expanded) { + flags |= ImGuiWindowFlags_AlwaysAutoResize; + } else { + const float font = ImGui::GetFontSize(); + ImGui::SetNextWindowSizeConstraints(ImVec2(font * 24.0f, font * 12.0f), ImVec2(io.DisplaySize.x, io.DisplaySize.y)); + if (m_apply_expanded_size) { + m_apply_expanded_size = false; + const float width = std::min(m_expanded_width > 0.0f ? m_expanded_width : font * 38.0f, io.DisplaySize.x); + const float height = std::min(m_expanded_height > 0.0f ? m_expanded_height : font * 34.0f, io.DisplaySize.y); + ImGui::SetNextWindowSize(ImVec2(width, height), ImGuiCond_Always); + ImGui::SetNextWindowPos( + ImVec2( + std::clamp(m_window_x, 0.0f, std::max(0.0f, io.DisplaySize.x - width)), + std::clamp(m_window_y, 0.0f, std::max(0.0f, io.DisplaySize.y - height)) + ), + ImGuiCond_Always + ); + m_expanded_width = width; + m_expanded_height = height; + } + } + + const bool open = ImGui::Begin("Performance", nullptr, flags); + if (open) { + const ImVec2 pos = ImGui::GetWindowPos(); + m_window_x = pos.x; + m_window_y = pos.y; + if (m_expanded) { + const ImVec2 size = ImGui::GetWindowSize(); + m_expanded_width = size.x; + m_expanded_height = size.y; + } + } if (!open || !m_has_sample) { if (open) { ImGui::TextDisabled("Sampling..."); @@ -858,6 +900,7 @@ void PerfWindow::draw() { if (ImGui::SmallButton(m_expanded ? "Hide details" : "Details")) { m_expanded = !m_expanded; + m_apply_expanded_size = m_expanded; } if (m_expanded && ImGui::BeginTabBar("##perf_tabs")) { diff --git a/engine/src/toast/renderer/passes/perf_window.hpp b/engine/src/toast/renderer/passes/perf_window.hpp index d6261c620..68b61c5d9 100644 --- a/engine/src/toast/renderer/passes/perf_window.hpp +++ b/engine/src/toast/renderer/passes/perf_window.hpp @@ -141,6 +141,12 @@ class PerfWindow { float m_gpu_ms = 0.0f; bool m_expanded = false; + /// Applied once when the details open so the window can be resized after that + bool m_apply_expanded_size = false; + float m_expanded_width = 0.0f; + float m_expanded_height = 0.0f; + float m_window_x = 0.0f; + float m_window_y = 0.0f; bool m_hide_idle_passes = true; bool m_has_gpu = false; }; diff --git a/engine/src/toast/renderer/passes/traced_shadow_pass.cpp b/engine/src/toast/renderer/passes/traced_shadow_pass.cpp index 36d1c534d..813bff9dc 100644 --- a/engine/src/toast/renderer/passes/traced_shadow_pass.cpp +++ b/engine/src/toast/renderer/passes/traced_shadow_pass.cpp @@ -86,7 +86,7 @@ void TracedShadowPass::createResources(const VulkanCore& core) { m_descriptor_sets.clear(); m_bound_views.assign(VulkanRenderer::k_frames_in_flight, vk::ImageView {}); - m_bound_acceleration_structures.assign(VulkanRenderer::k_frames_in_flight, vk::AccelerationStructureKHR {}); + m_bound_acceleration_structures.assign(VulkanRenderer::k_frames_in_flight, {}); for (uint32_t i = 0; i < VulkanRenderer::k_frames_in_flight; ++i) { const vk::DescriptorSetAllocateInfo alloc_info(pool, 1, &set_layout); auto allocated = device.allocateDescriptorSets(alloc_info); @@ -131,7 +131,8 @@ auto TracedShadowPass::record(vk::CommandBuffer cmd, uint32_t frame_index, vk::I return source_view; } - if (m_bound_views[frame_index] != source_view || m_bound_acceleration_structures[frame_index] != tlas) { + const std::pair bound_tlas {tlas, scene->getAccelerationStructureGeneration(frame_index)}; + if (m_bound_views[frame_index] != source_view || m_bound_acceleration_structures[frame_index] != bound_tlas) { constexpr auto k_sampler = vk::DescriptorType::eCombinedImageSampler; const vk::DescriptorSet set = *m_descriptor_sets[frame_index]; @@ -142,7 +143,7 @@ auto TracedShadowPass::record(vk::CommandBuffer cmd, uint32_t frame_index, vk::I .accelerationStructure(set, 3, tlas); writer.flush(m_core->getDevice()); m_bound_views[frame_index] = source_view; - m_bound_acceleration_structures[frame_index] = tlas; + m_bound_acceleration_structures[frame_index] = bound_tlas; } m_target.beginScope(cmd); diff --git a/engine/src/toast/renderer/passes/traced_shadow_pass.hpp b/engine/src/toast/renderer/passes/traced_shadow_pass.hpp index ab7d1a3f7..a366017c0 100644 --- a/engine/src/toast/renderer/passes/traced_shadow_pass.hpp +++ b/engine/src/toast/renderer/passes/traced_shadow_pass.hpp @@ -14,6 +14,7 @@ #include #include +#include #include namespace renderer { @@ -55,7 +56,8 @@ class TracedShadowPass : public IPostProcessPass { std::vector m_descriptor_sets; std::vector m_bound_views; - std::vector m_bound_acceleration_structures; + /// Handle plus generation since a recreated structure can reuse a destroyed handle value + std::vector> m_bound_acceleration_structures; PostProcessTarget m_target; }; diff --git a/engine/src/toast/renderer/passes/voxel_pass.cpp b/engine/src/toast/renderer/passes/voxel_pass.cpp index 1e9f7e077..0453796cb 100644 --- a/engine/src/toast/renderer/passes/voxel_pass.cpp +++ b/engine/src/toast/renderer/passes/voxel_pass.cpp @@ -29,6 +29,8 @@ constexpr uint32_t k_storage_set = 1; constexpr uint32_t k_instance_binding = 6; constexpr uint32_t k_cells_binding = 7; constexpr uint32_t k_cell_args_binding = 8; +constexpr uint32_t k_quads_binding = 9; +constexpr uint32_t k_vertices_per_quad = 6; constexpr uint32_t k_cell_phase_count = 0; constexpr uint32_t k_cell_phase_allocate = 1; @@ -49,7 +51,7 @@ VoxelPass::VoxelPass(const VulkanCore& core, vk::Format scene_format, vk::Format : m_core(&core) { ZoneScoped; static_assert(sizeof(PushConstants) == 16, "PushConstants is mirrored by voxel.slang"); - static_assert(sizeof(CellCullPush) == 96, "CellCullPush is mirrored by voxel_cells.slang"); + static_assert(sizeof(CellCullPush) == 112, "CellCullPush is mirrored by voxel_cells.slang"); const auto uid = assets::resolveURI("core://shaders/voxel.slang"); const auto shader = uid.has_value() ? ShaderCache::get().acquire(*uid) : nullptr; @@ -88,6 +90,13 @@ VoxelPass::VoxelPass(const VulkanCore& core, vk::Format scene_format, vk::Format } } + // No SV_Depth so near quads keep early depth testing + config.cull_mode = vk::CullModeFlagBits::eBack; + config.vertex_entry = "quadVertexMain"; + config.fragment_entry = "quadFragmentMain"; + config.debug_name = "VoxelPass Quads"; + m_quad_pipeline.rebuild(core, config); + createInstanceBuffers(core); createDescriptors(core, shader->reflection); createCells(core); @@ -121,6 +130,10 @@ void VoxelPass::createCells(const VulkanCore& core) { config.pipeline_layout = *m_cell_layout.getPipelineLayout(); m_cell_pipeline.rebuild(core, config); + config.debug_name = "VoxelMesh"; + config.compute_entry = "meshMain"; + m_mesh_pipeline.rebuild(core, config); + const vk::DescriptorSetLayout set_layout = *layouts[k_storage_set]; for (uint32_t i = 0; i < m_storage_sets.size(); ++i) { auto allocated = core.getDevice().allocateDescriptorSets(vk::DescriptorSetAllocateInfo(pool, 1, &set_layout)); @@ -143,7 +156,47 @@ void VoxelPass::createCells(const VulkanCore& core) { // The G-buffer set declares the cell bindings either way so they always hold a buffer for (uint32_t i = 0; i < m_cell_buffers.size(); ++i) { ensureCellCapacity(i, k_min_cell_capacity); + ensureQuadCapacity(i, k_min_quad_capacity); + } +} + +void VoxelPass::ensureQuadCapacity(uint32_t frame_index, uint32_t quads) { + CellBuffers& buffers = m_cell_buffers[frame_index]; + if (buffers.quads.has_value() && buffers.quad_capacity >= quads) { + return; + } + ZoneScoped; + + const uint32_t capacity = std::max({quads + (quads / 4), k_min_quad_capacity, buffers.quad_capacity}); + + vma::AllocationCreateInfo alloc_ci {}; + alloc_ci.usage = vma::MemoryUsage::eAutoPreferDevice; + + // Safe to replace since the fence of this slot was waited on + vk::BufferCreateInfo quads_ci {}; + quads_ci.size = vk::DeviceSize {capacity} * sizeof(glm::uvec2); + quads_ci.usage = vk::BufferUsageFlagBits::eStorageBuffer; + buffers.quads.emplace(m_core->getAllocator().createBuffer(quads_ci, alloc_ci)); + setDebugName(*m_core, **buffers.quads, std::format("VoxelPass Quads[{}]", frame_index)); + buffers.quad_capacity = capacity; + + if (!buffers.readback.has_value()) { + vk::BufferCreateInfo readback_ci {}; + readback_ci.size = sizeof(uint32_t); + readback_ci.usage = vk::BufferUsageFlagBits::eTransferDst; + + vma::AllocationCreateInfo readback_alloc {}; + readback_alloc.usage = vma::MemoryUsage::eAuto; + readback_alloc.flags = vma::AllocationCreateFlagBits::eMapped | vma::AllocationCreateFlagBits::eHostAccessRandom; + buffers.readback.emplace(m_core->getAllocator().createBuffer(readback_ci, readback_alloc)); + setDebugName(*m_core, **buffers.readback, std::format("VoxelPass QuadReadback[{}]", frame_index)); + if (auto* mapped = static_cast(buffers.readback->getAllocation().getInfo().pMappedData); mapped != nullptr) { + *mapped = 0; + buffers.readback->getAllocation().flush(0, sizeof(uint32_t)); + } } + + writeCellDescriptors(frame_index); } void VoxelPass::ensureCellCapacity(uint32_t frame_index, uint32_t cells) { @@ -181,12 +234,20 @@ void VoxelPass::ensureCellCapacity(uint32_t frame_index, uint32_t cells) { void VoxelPass::writeCellDescriptors(uint32_t frame_index) { const CellBuffers& buffers = m_cell_buffers[frame_index]; + // The cell and quad buffers come up one after the other at creation + if (!buffers.cells.has_value() || !buffers.quads.has_value()) { + return; + } + DescriptorWriter writer; - writer.buffer(*m_storage_sets[frame_index], k_cells_binding, vk::DescriptorType::eStorageBuffer, **buffers.cells); - writer.buffer(*m_storage_sets[frame_index], k_cell_args_binding, vk::DescriptorType::eStorageBuffer, **buffers.args); + const auto write = [&](vk::DescriptorSet set) { + writer.buffer(set, k_cells_binding, vk::DescriptorType::eStorageBuffer, **buffers.cells); + writer.buffer(set, k_cell_args_binding, vk::DescriptorType::eStorageBuffer, **buffers.args); + writer.buffer(set, k_quads_binding, vk::DescriptorType::eStorageBuffer, **buffers.quads); + }; + write(*m_storage_sets[frame_index]); if (frame_index < m_cell_sets.size()) { - writer.buffer(*m_cell_sets[frame_index], k_cells_binding, vk::DescriptorType::eStorageBuffer, **buffers.cells); - writer.buffer(*m_cell_sets[frame_index], k_cell_args_binding, vk::DescriptorType::eStorageBuffer, **buffers.args); + write(*m_cell_sets[frame_index]); } writer.flush(m_core->getDevice()); } @@ -359,6 +420,7 @@ auto VoxelPass::prepare(uint32_t frame_index) -> bool { m_draws.clear(); m_cells_active = false; m_total_cells = 0; + m_near_voxels = 0.0f; if (frame_index >= m_storage_sets.size() || frame_index >= m_instance_buffers.size() || !m_scene_sets.get(frame_index) || !m_lighting_pipeline.isReady() || !m_blur_pipeline.isReady()) { @@ -427,6 +489,21 @@ auto VoxelPass::prepare(uint32_t frame_index) -> bool { if (cells_fit) { ensureCellCapacity(frame_index, m_total_cells); m_cells_active = true; + + // Grows from what this slot meshed last time so a spike costs at most one frame of missing quads + CellBuffers& buffers = m_cell_buffers[frame_index]; + const auto& readback = buffers.readback->getAllocation(); + readback.invalidate(0, sizeof(uint32_t)); + if (const auto* vertices = static_cast(readback.getInfo().pMappedData); vertices != nullptr) { + ensureQuadCapacity(frame_index, *vertices / k_vertices_per_quad); + } + + // A voxel covers at least this many pixels closer than near_voxels voxels whatever the volume scale + const float raster_pixels = VulkanRenderer::instance->voxelRasterPixels(); + const float height = static_cast(VulkanRenderer::instance->getOutputTarget().getExtent().height); + m_near_voxels = raster_pixels > 0.0f && m_mesh_pipeline.isReady() && m_quad_pipeline.isReady() + ? std::abs(frame->frame_data.projection[1][1]) * height / (2.0f * raster_pixels) + : 0.0f; } else if (const auto now = std::chrono::steady_clock::now(); now - m_last_limit_warning > std::chrono::seconds(5)) { m_last_limit_warning = now; TOAST_WARN( @@ -454,7 +531,7 @@ void VoxelPass::recordCellCull(vk::CommandBuffer cmd, uint32_t frame_index) { // The count phase accumulates into the cursors cmd.fillBuffer( - args, vk::DeviceSize {k_cell_cursors_word} * sizeof(uint32_t), vk::DeviceSize {k_cell_bucket_count} * sizeof(uint32_t), 0 + args, vk::DeviceSize {k_cell_cursors_word} * sizeof(uint32_t), vk::DeviceSize {k_cell_list_count} * sizeof(uint32_t), 0 ); sync::BarrierBatch barriers; @@ -472,6 +549,8 @@ void VoxelPass::recordCellCull(vk::CommandBuffer cmd, uint32_t frame_index) { .camera_near = glm::vec4(frame->frame_data.camera_position, frame->near_plane), .total_cells = m_total_cells, .instance_count = static_cast(m_draws.size()), + .quad_capacity = buffers.quad_capacity, + .near_voxels = m_near_voxels, }; const uint32_t groups = (m_total_cells + k_cell_group_size - 1) / k_cell_group_size; @@ -485,9 +564,29 @@ void VoxelPass::recordCellCull(vk::CommandBuffer cmd, uint32_t frame_index) { } } + const vk::Buffer quads = **buffers.quads; + if (m_near_voxels > 0.0f) { + barriers.buffer(args, sync::Usage::compute_storage_write, sync::Usage::indirect_read); + barriers.buffer(args, sync::Usage::compute_storage_write, sync::Usage::compute_storage_write); + barriers.buffer(cells, sync::Usage::compute_storage_write, sync::Usage::compute_storage_read); + barriers.flush(cmd); + + cmd.bindPipeline(vk::PipelineBindPoint::eCompute, m_mesh_pipeline.getPipeline()); + cmd.pushConstants(layout, vk::ShaderStageFlagBits::eAll, 0, sizeof(CellCullPush), &push); + cmd.dispatchIndirect(args, vk::DeviceSize {k_near_dispatch_word} * sizeof(uint32_t)); + + barriers.buffer(args, sync::Usage::compute_storage_write, sync::Usage::transfer_src); + barriers.flush(cmd); + cmd.copyBuffer( + args, **buffers.readback, vk::BufferCopy(vk::DeviceSize {k_quad_draw_word} * sizeof(uint32_t), 0, sizeof(uint32_t)) + ); + barriers.buffer(**buffers.readback, sync::Usage::transfer_dst, sync::Usage::host_read); + } + barriers.buffer(args, sync::Usage::compute_storage_write, sync::Usage::indirect_read); barriers.buffer(args, sync::Usage::compute_storage_write, sync::Usage::vertex_storage_read); barriers.buffer(cells, sync::Usage::compute_storage_write, sync::Usage::vertex_storage_read); + barriers.buffer(quads, sync::Usage::compute_storage_write, sync::Usage::vertex_storage_read); barriers.flush(cmd); } @@ -516,6 +615,14 @@ void VoxelPass::recordCellDraws(vk::CommandBuffer cmd, uint32_t frame_index) { cmd.drawIndirect(args, vk::DeviceSize {bucket} * k_command_stride, 1, k_command_stride); }; + // Near quads first so their depth rejects the far boxes behind them before they march + if (m_near_voxels > 0.0f) { + bind(m_quad_pipeline); + const PushConstants push {.mode = k_draw_cells, .quad_capacity = m_cell_buffers[frame_index].quad_capacity}; + cmd.pushConstants(layout, vk::ShaderStageFlagBits::eAll, 0, sizeof(PushConstants), &push); + cmd.drawIndirect(args, vk::DeviceSize {k_quad_draw_word} * sizeof(uint32_t), 1, k_command_stride); + } + // Cells holding the camera are nearest so they go first and seed depth for the rest const VulkanPipeline& inside = m_pipelines[static_cast(Cull::front)][static_cast(Depth::exact)]; if (inside.isReady()) { @@ -631,6 +738,11 @@ void VoxelPass::recordLighting(vk::CommandBuffer cmd, uint32_t frame_index) { cmd.draw(3, 1, 0, 0); } +void VoxelPass::onTargetsRecreated() { + std::ranges::fill(m_bound_gbuffer, std::array {}); + std::ranges::fill(m_bound_blur, std::array {}); +} + void VoxelPass::recordNormalBlur(vk::CommandBuffer cmd, uint32_t frame_index, vk::Extent2D viewport) { ZoneScoped; if (!m_blur_pipeline.isReady() || frame_index >= m_blur_sets.size()) { diff --git a/engine/src/toast/renderer/passes/voxel_pass.hpp b/engine/src/toast/renderer/passes/voxel_pass.hpp index 45c062ee1..1b54675d4 100644 --- a/engine/src/toast/renderer/passes/voxel_pass.hpp +++ b/engine/src/toast/renderer/passes/voxel_pass.hpp @@ -49,6 +49,9 @@ class VoxelPass : public IRenderPass { /// Writes scene colour and indirect while reading the G-buffer and depth void recordLighting(vk::CommandBuffer cmd, uint32_t frame_index); + /// After the G-buffer and depth targets are recreated since a new view can reuse a destroyed handle value + void onTargetsRecreated(); + [[nodiscard]] auto name() const -> std::string_view override { return "Voxels"; @@ -65,7 +68,7 @@ class VoxelPass : public IRenderPass { uint32_t instance_index = 0; uint32_t bucket = 0; uint32_t mode = 0; - uint32_t pad0 = 0; + uint32_t quad_capacity = 0; }; /// Mirrors voxel.slang kDrawVolume and kDrawCells @@ -79,13 +82,22 @@ class VoxelPass : public IRenderPass { uint32_t total_cells = 0; uint32_t instance_count = 0; uint32_t phase = 0; + uint32_t quad_capacity = 0; + float near_voxels = 0.0f; uint32_t pad0 = 0; + uint32_t pad1 = 0; + uint32_t pad2 = 0; }; /// Mirrors voxel_dda.slang kCellBucketCount and the word layout after it static constexpr uint32_t k_cell_bucket_count = 17; - static constexpr uint32_t k_cell_cursors_word = k_cell_bucket_count * 5; - static constexpr uint32_t k_cell_args_words = k_cell_bucket_count * 6; + static constexpr uint32_t k_cell_list_count = k_cell_bucket_count + 1; + static constexpr uint32_t k_cell_offsets_word = k_cell_bucket_count * 4; + static constexpr uint32_t k_cell_cursors_word = k_cell_offsets_word + k_cell_list_count; + static constexpr uint32_t k_near_dispatch_word = k_cell_cursors_word + k_cell_list_count; + static constexpr uint32_t k_quad_draw_word = k_near_dispatch_word + 3; + static constexpr uint32_t k_cell_args_words = k_quad_draw_word + 4; + static constexpr uint32_t k_min_quad_capacity = 65536; /// Ten bits per axis in voxel_dda.slang packCell static constexpr uint32_t k_max_coarse_dim = 1024; static constexpr uint32_t k_min_cell_capacity = 4096; @@ -94,9 +106,14 @@ class VoxelPass : public IRenderPass { struct CellBuffers { /// Instance index and packed cell per surviving cell std::optional cells; - /// Indirect commands then bucket offsets then cursors + /// Indirect commands then list offsets and cursors then the mesh dispatch and the quad draw std::optional args; uint32_t capacity = 0; + /// Cell entry index and packed quad per meshed face run + std::optional quads; + uint32_t quad_capacity = 0; + /// Quad draw vertex count copied back for growing the quad buffer + std::optional readback; }; /// Mirrors voxel_lighting.slang LightingPushConstants @@ -113,6 +130,7 @@ class VoxelPass : public IRenderPass { void createCells(const VulkanCore& core); void ensureCellCapacity(uint32_t frame_index, uint32_t cells); + void ensureQuadCapacity(uint32_t frame_index, uint32_t quads); void writeCellDescriptors(uint32_t frame_index); void recordCellDraws(vk::CommandBuffer cmd, uint32_t frame_index); @@ -159,11 +177,15 @@ class VoxelPass : public IRenderPass { ShaderLayout m_cell_layout; VulkanPipeline m_cell_pipeline; + VulkanPipeline m_mesh_pipeline; + VulkanPipeline m_quad_pipeline; std::vector m_cell_sets; std::vector m_cell_buffers; bool m_cells_active = false; uint32_t m_total_cells = 0; + /// Cells closer than this many voxels are meshed this frame and 0 marches all of them + float m_near_voxels = 0.0f; ShaderLayout m_lighting_layout; VulkanPipeline m_lighting_pipeline; diff --git a/engine/src/toast/renderer/ray_tracing_scene.cpp b/engine/src/toast/renderer/ray_tracing_scene.cpp index f8bff2797..dd1020a69 100644 --- a/engine/src/toast/renderer/ray_tracing_scene.cpp +++ b/engine/src/toast/renderer/ray_tracing_scene.cpp @@ -187,6 +187,7 @@ void RayTracingScene::build(vk::CommandBuffer cmd, uint32_t frame_index, bool tr as_ci.size = sizes.accelerationStructureSize; as_ci.type = vk::AccelerationStructureTypeKHR::eTopLevel; frame.tlas = vk::raii::AccelerationStructureKHR(device, as_ci); + ++frame.tlas_generation; } } diff --git a/engine/src/toast/renderer/ray_tracing_scene.hpp b/engine/src/toast/renderer/ray_tracing_scene.hpp index 063230a51..9c3f62e16 100644 --- a/engine/src/toast/renderer/ray_tracing_scene.hpp +++ b/engine/src/toast/renderer/ray_tracing_scene.hpp @@ -37,6 +37,12 @@ class RayTracingScene { [[nodiscard]] auto getAccelerationStructure(uint32_t frame_index) const -> vk::AccelerationStructureKHR; + /// Bumped whenever the structure of @p frame_index is recreated since the new one can reuse the old handle value + [[nodiscard]] + auto getAccelerationStructureGeneration(uint32_t frame_index) const -> uint64_t { + return frame_index < m_frames.size() ? m_frames[frame_index].tlas_generation : 0; + } + [[nodiscard]] auto getInstanceCount() const noexcept -> uint32_t { return static_cast(m_instances.size()); @@ -52,6 +58,7 @@ class RayTracingScene { std::optional tlas_buffer; std::optional scratch; vk::raii::AccelerationStructureKHR tlas = nullptr; + uint64_t tlas_generation = 0; uint32_t built_instances = 0; std::vector built; diff --git a/engine/src/toast/renderer/renderer_settings.cpp b/engine/src/toast/renderer/renderer_settings.cpp index 81d2c9e92..d0bc2f075 100644 --- a/engine/src/toast/renderer/renderer_settings.cpp +++ b/engine/src/toast/renderer/renderer_settings.cpp @@ -181,6 +181,18 @@ void registerVoxels(VulkanRenderer& renderer) { "draws one box per volume and marches the whole volume, kept to compare the two."} ) .onChange([&renderer](bool enabled) { renderer.setVoxelCellBoxes(enabled); }); + + toast::settings::declareFloat( + "renderer.voxels.raster_pixels", + 4.0, + {.label = "Raster voxel size", + .category = "Voxels", + .description = "Cells whose voxels cover at least this many pixels are meshed into quads and rasterized", + .min = 0.0, + .max = 32.0, + .step = 0.5} + ) + .onChange([&renderer](double pixels) { renderer.setVoxelRasterPixels(static_cast(pixels)); }); } void registerEnvironment(VulkanRenderer& renderer) { diff --git a/engine/src/toast/renderer/scene_descriptor_set.cpp b/engine/src/toast/renderer/scene_descriptor_set.cpp index 74960e170..3c365b6ab 100644 --- a/engine/src/toast/renderer/scene_descriptor_set.cpp +++ b/engine/src/toast/renderer/scene_descriptor_set.cpp @@ -31,7 +31,7 @@ void SceneDescriptorSets::create( m_sets.clear(); m_sets.reserve(VulkanRenderer::k_frames_in_flight); m_scene_binding.reset(); - m_bound_tlas.assign(VulkanRenderer::k_frames_in_flight, vk::AccelerationStructureKHR {}); + m_bound_tlas.assign(VulkanRenderer::k_frames_in_flight, {}); for (uint32_t i = 0; i < VulkanRenderer::k_frames_in_flight; ++i) { const vk::DescriptorSetAllocateInfo alloc_info(pool, 1, &layout); @@ -179,9 +179,10 @@ void SceneDescriptorSets::updateTlas(uint32_t frame_index) { const auto* scene = VulkanRenderer::instance->getRayTracingScene(); const vk::AccelerationStructureKHR tlas = scene != nullptr ? scene->getAccelerationStructure(frame_index) : nullptr; + const std::pair bound {tlas, scene != nullptr ? scene->getAccelerationStructureGeneration(frame_index) : 0}; // A descriptor cannot be written with a null structure - if (!tlas || tlas == m_bound_tlas[frame_index]) { + if (!tlas || bound == m_bound_tlas[frame_index]) { return; } @@ -189,7 +190,7 @@ void SceneDescriptorSets::updateTlas(uint32_t frame_index) { writer.accelerationStructure(*m_sets[frame_index], *m_scene_binding, tlas); writer.flush(m_core->getDevice()); - m_bound_tlas[frame_index] = tlas; + m_bound_tlas[frame_index] = bound; } } diff --git a/engine/src/toast/renderer/scene_descriptor_set.hpp b/engine/src/toast/renderer/scene_descriptor_set.hpp index 8dafe7e15..231db1c89 100644 --- a/engine/src/toast/renderer/scene_descriptor_set.hpp +++ b/engine/src/toast/renderer/scene_descriptor_set.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include namespace renderer { @@ -43,7 +44,8 @@ class SceneDescriptorSets { std::optional m_scene_binding; - std::vector m_bound_tlas; + /// Handle plus generation since a recreated structure can reuse a destroyed handle value + std::vector> m_bound_tlas; }; } diff --git a/engine/src/toast/renderer/voxel_gpu_storage.cpp b/engine/src/toast/renderer/voxel_gpu_storage.cpp index 7e058c22c..085845e11 100644 --- a/engine/src/toast/renderer/voxel_gpu_storage.cpp +++ b/engine/src/toast/renderer/voxel_gpu_storage.cpp @@ -31,6 +31,9 @@ constexpr vk::DeviceSize k_min_section_bytes = sizeof(uint32_t); constexpr uint32_t k_slot_headroom_divisor = 4; constexpr uint32_t k_min_slot_headroom = 256; +/// Spare bytes past the packed grids coarse palettes and records for volumes that spawn later +constexpr std::array k_min_append_headroom {64 * 1024, 4 * 1024, 64 * 1024, 16 * 1024}; + auto createSectionBuffer(const VulkanCore& core, vk::DeviceSize size, std::string_view name) -> vma::raii::Buffer { const bool concurrent = core.getGraphicsQueueFamilyIndex() != core.getTransferQueueFamilyIndex(); const std::array family_indices {core.getGraphicsQueueFamilyIndex(), core.getTransferQueueFamilyIndex()}; @@ -83,8 +86,8 @@ class VoxelGpuSections { void recordPending(vk::CommandBuffer cmd, uint64_t up_to, std::vector>& retired); private: - /// Materials then occupancy then grids then coarse - static constexpr size_t k_patched_sections = 4; + /// Every section in VoxelGpuStorage::Section order + static constexpr size_t k_patched_sections = VoxelGpuStorage::k_section_count; struct Patch { uint64_t sequence = 0; @@ -99,12 +102,17 @@ class VoxelGpuSections { auto VoxelGpuSections::queuePatch(const VulkanCore& core, const voxel::gpu::ScenePatch& patch) -> std::optional { ZoneScoped; - if (patch.pool_slots > slot_capacity) { + using Section = VoxelGpuStorage::Section; + const auto fits = [this](Section section, vk::DeviceSize bytes) { return bytes <= sizes[static_cast(section)]; }; + if (patch.pool_slots > slot_capacity || !fits(Section::grids, vk::DeviceSize {patch.grid_words} * sizeof(uint32_t)) || + !fits(Section::coarse, vk::DeviceSize {patch.coarse_words} * sizeof(uint32_t)) || + !fits(Section::palettes, vk::DeviceSize {patch.palette_words} * sizeof(uint32_t)) || + !fits(Section::records, vk::DeviceSize {patch.record_count} * sizeof(voxel::gpu::VolumeRecord))) { return std::nullopt; } const std::array sections { - &patch.materials, &patch.occupancy, &patch.grids, &patch.coarse + &patch.materials, &patch.occupancy, &patch.grids, &patch.coarse, &patch.palettes, &patch.records }; Patch staged; @@ -161,18 +169,13 @@ void VoxelGpuSections::recordPending(vk::CommandBuffer cmd, uint64_t up_to, std: } ZoneScoped; - using Section = VoxelGpuStorage::Section; - constexpr std::array k_targets { - Section::materials, Section::occupancy, Section::grids, Section::coarse - }; - // Frames already submitted may still read the sections and later patches may rewrite the same words sync::BarrierBatch barriers; for (const Patch& patch : ready) { barriers.memory(sync::Usage::general, sync::Usage::transfer_dst); barriers.flush(cmd); for (size_t i = 0; i < k_patched_sections; ++i) { - const auto& target = buffers[static_cast(k_targets[i])]; + const auto& target = buffers[i]; if (patch.staging != nullptr && !patch.regions[i].empty() && target.has_value()) { cmd.copyBuffer(**patch.staging, **target, patch.regions[i]); } @@ -283,16 +286,16 @@ void VoxelSceneUpload::build(const VulkanCore& core) { VoxelGpuSections& sections = *m_storage->m_sections; - // Spare slots so later carves patch in place + // Spare room so later carves and spawns patch in place const auto slots = static_cast(pool.materials.size() / voxel::gpu::k_material_words_per_brick); sections.slot_capacity = slots + std::max(slots / k_slot_headroom_divisor, k_min_slot_headroom); const std::array capacities { vk::DeviceSize {sections.slot_capacity} * voxel::gpu::k_material_words_per_brick * sizeof(uint32_t), vk::DeviceSize {sections.slot_capacity} * voxel::gpu::k_occupancy_words_per_brick * sizeof(uint32_t), - sources[2].second, - sources[3].second, - sources[4].second, - sources[5].second, + sources[2].second + std::max(sources[2].second / k_slot_headroom_divisor, k_min_append_headroom[0]), + sources[3].second + std::max(sources[3].second / k_slot_headroom_divisor, k_min_append_headroom[1]), + sources[4].second + std::max(sources[4].second / k_slot_headroom_divisor, k_min_append_headroom[2]), + sources[5].second + std::max(sources[5].second / k_slot_headroom_divisor, k_min_append_headroom[3]), }; try { diff --git a/engine/src/toast/renderer/voxel_scene_patcher.cpp b/engine/src/toast/renderer/voxel_scene_patcher.cpp index 690ccd2b1..db65eaf08 100644 --- a/engine/src/toast/renderer/voxel_scene_patcher.cpp +++ b/engine/src/toast/renderer/voxel_scene_patcher.cpp @@ -5,64 +5,179 @@ #include "voxel_scene_patcher.hpp" #include +#include +#include +#include +#include #include namespace renderer { +namespace { + +auto voxelPatchRecordDims(const voxel::gpu::VolumeRecord& record) -> glm::uvec3 { + return {record.brick_dims_x, record.brick_dims_y, record.brick_dims_z}; +} + +auto voxelPatchGridWords(const voxel::gpu::VolumeRecord& record) -> uint64_t { + return uint64_t {record.brick_dims_x} * record.brick_dims_y * record.brick_dims_z; +} + +} + void VoxelScenePatcher::adopt(const voxel::gpu::PackedScene& scene, uint32_t pool_slots, std::vector node_uids) { m_scene = scene; m_pool_slots = pool_slots; m_node_uids = std::move(node_uids); + m_dead_grid_words = 0; m_valid = true; } +auto VoxelScenePatcher::paletteMatches(const voxel::gpu::VolumeRecord& record, const voxel::Palette& palette) const -> bool { + const size_t base = size_t {record.palette_offset} * 4u; + return record.max_emissive == palette.max_emissive && base + voxel::gpu::k_palette_words <= m_scene.palettes.size() && + std::memcmp(m_scene.palettes.data() + base, palette.entries.data(), voxel::gpu::k_palette_words * sizeof(uint32_t)) == 0; +} + +auto VoxelScenePatcher::findOrAppendPalette(const voxel::Palette& palette) -> uint32_t { + using voxel::gpu::k_palette_words; + for (size_t base = 0; base + k_palette_words <= m_scene.palettes.size(); base += k_palette_words) { + if (std::memcmp(m_scene.palettes.data() + base, palette.entries.data(), k_palette_words * sizeof(uint32_t)) == 0) { + return static_cast(base / 4u); + } + } + const size_t base = m_scene.palettes.size(); + m_scene.palettes.resize(base + k_palette_words); + std::memcpy(m_scene.palettes.data() + base, palette.entries.data(), k_palette_words * sizeof(uint32_t)); + return static_cast(base / 4u); +} + auto VoxelScenePatcher::tryPatch( const voxel::BrickPool& pool, std::span volumes, std::span node_uids, std::span deltas -) -> std::optional { +) -> std::optional { using namespace voxel::gpu; + ZoneScoped; - // Same volumes in the same order since a record is found by its index - if (!m_valid || deltas.size() != volumes.size() || !std::ranges::equal(node_uids, m_node_uids)) { + if (!m_valid || deltas.size() != volumes.size() || node_uids.size() != volumes.size()) { return std::nullopt; } + std::unordered_map live_record; + for (uint32_t record = 0; record < m_node_uids.size(); ++record) { + if (m_node_uids[record] != 0) { + live_record.emplace(m_node_uids[record], record); + } + } + + // Decide before touching the retained copy so a bail out leaves it as the GPU holds it + std::vector> kept_record(volumes.size()); + std::vector record_kept(m_node_uids.size(), false); uint64_t dirty_bricks = 0; - for (const VolumeDelta& delta : deltas) { - if (delta.unknown) { - return std::nullopt; + uint64_t appended_grid_words = 0; + for (size_t i = 0; i < volumes.size(); ++i) { + const glm::uvec3 dims = volumes[i].volume->brickDims(); + const uint64_t bricks = uint64_t {dims.x} * dims.y * dims.z; + const auto found = live_record.find(node_uids[i]); + if (found != live_record.end()) { + const VolumeRecord& record = m_scene.records[found->second]; + if (voxelPatchRecordDims(record) == dims && paletteMatches(record, *volumes[i].palette)) { + kept_record[i] = found->second; + record_kept[found->second] = true; + dirty_bricks += deltas[i].unknown ? bricks : deltas[i].bricks.size(); + continue; + } } - dirty_bricks += delta.bricks.size(); + dirty_bricks += bricks; + appended_grid_words += bricks; + } + + uint64_t dead_grid_words = m_dead_grid_words; + for (uint32_t record = 0; record < m_node_uids.size(); ++record) { + if (m_node_uids[record] != 0 && !record_kept[record]) { + dead_grid_words += voxelPatchGridWords(m_scene.records[record]); + } + } + + // Dead records outweighing live ones are compacted by a full pack + const uint64_t grid_words = m_scene.grids.size() + appended_grid_words; + if (dead_grid_words * 2 > grid_words) { + return std::nullopt; } // A dirty brick costs its slot and a grid word and a full pack costs the whole pool constexpr uint64_t k_words_per_dirty_brick = k_material_words_per_brick + k_occupancy_words_per_brick + 1; - const uint64_t full_words = - (static_cast(m_pool_slots) * (k_material_words_per_brick + k_occupancy_words_per_brick)) + m_scene.grids.size(); + const uint64_t full_words = (uint64_t {m_pool_slots} * (k_material_words_per_brick + k_occupancy_words_per_brick)) + grid_words; if (dirty_bricks * k_words_per_dirty_brick * 2 > full_words) { return std::nullopt; } - if (!layoutMatches(m_scene, packLayout(volumes))) { - return std::nullopt; + for (uint32_t record = 0; record < m_node_uids.size(); ++record) { + if (!record_kept[record]) { + m_node_uids[record] = 0; + } } + m_dead_grid_words = dead_grid_words; + const auto first_new_palette_word = static_cast(m_scene.palettes.size()); + std::vector appended; + std::vector by_record(m_scene.records.size()); std::vector dirty; - for (size_t i = 0; i < deltas.size(); ++i) { - if (!deltas[i].bricks.empty()) { - dirty.push_back({.volume = static_cast(i), .bricks = deltas[i].bricks}); + std::vector> every_brick; + every_brick.reserve(volumes.size()); + + for (size_t i = 0; i < volumes.size(); ++i) { + if (kept_record[i].has_value()) { + by_record[*kept_record[i]] = volumes[i]; + if (deltas[i].unknown) { + const glm::uvec3 dims = volumes[i].volume->brickDims(); + auto& all = every_brick.emplace_back(dims.x * dims.y * dims.z); + std::iota(all.begin(), all.end(), 0u); + dirty.push_back({.volume = *kept_record[i], .bricks = all}); + } else if (!deltas[i].bricks.empty()) { + dirty.push_back({.volume = *kept_record[i], .bricks = deltas[i].bricks}); + } + continue; } + + const glm::uvec3 dims = volumes[i].volume->brickDims(); + const glm::uvec3 coarse = coarseDims(dims); + VolumeRecord record; + record.brick_dims_x = dims.x; + record.brick_dims_y = dims.y; + record.brick_dims_z = dims.z; + record.grid_offset = static_cast(m_scene.grids.size()); + record.coarse_dims_x = coarse.x; + record.coarse_dims_y = coarse.y; + record.coarse_dims_z = coarse.z; + record.coarse_offset = static_cast(m_scene.coarse.size()); + record.palette_offset = findOrAppendPalette(*volumes[i].palette); + record.max_emissive = volumes[i].palette->max_emissive; + + m_scene.grids.resize(m_scene.grids.size() + (dims.x * dims.y * dims.z), 0u); + m_scene.coarse.resize(m_scene.coarse.size() + (((coarse.x * coarse.y * coarse.z) + 31u) / 32u), 0u); + appended.push_back(static_cast(m_scene.records.size())); + m_scene.records.push_back(record); + m_node_uids.push_back(node_uids[i]); + by_record.push_back(volumes[i]); } - ScenePatch patch = computePatch(m_scene, m_pool_slots, pool, volumes, dirty); - m_pool_slots = patch.pool_slots; - return patch; + Patched out; + out.patch = computePatch(m_scene, m_pool_slots, pool, by_record, dirty, appended, first_new_palette_word); + m_pool_slots = out.patch.pool_slots; + out.record_uids = m_node_uids; + out.record_dims.reserve(m_scene.records.size()); + for (const VolumeRecord& record : m_scene.records) { + out.record_dims.push_back(voxelPatchRecordDims(record)); + } + return out; } void VoxelScenePatcher::reset() { m_scene = {}; m_pool_slots = 0; m_node_uids.clear(); + m_dead_grid_words = 0; m_valid = false; } diff --git a/engine/src/toast/renderer/voxel_scene_patcher.hpp b/engine/src/toast/renderer/voxel_scene_patcher.hpp index 7af163ad0..2fb6e4b79 100644 --- a/engine/src/toast/renderer/voxel_scene_patcher.hpp +++ b/engine/src/toast/renderer/voxel_scene_patcher.hpp @@ -7,6 +7,7 @@ #include "voxel_change_history.hpp" #include +#include #include #include #include @@ -15,18 +16,25 @@ namespace renderer { -/// Remembers what the GPU copy of the voxel scene holds and turns small changes into patches of it +/// Records stay in place so a removed volume leaves a dead record and a new one is appended class VoxelScenePatcher { public: + struct Patched { + voxel::gpu::ScenePatch patch; + /// Per record and 0 for a dead one + std::vector record_uids; + std::vector record_dims; + }; + /// A full pack is what the GPU will hold so later patches build on it void adopt(const voxel::gpu::PackedScene& scene, uint32_t pool_slots, std::vector node_uids); - /// @returns nullopt when only a full pack can describe the change + /// @returns nullopt when only a full pack is worth it [[nodiscard]] auto tryPatch( const voxel::BrickPool& pool, std::span volumes, std::span node_uids, std::span deltas - ) -> std::optional; + ) -> std::optional; void reset(); @@ -46,9 +54,17 @@ class VoxelScenePatcher { } private: + [[nodiscard]] + auto paletteMatches(const voxel::gpu::VolumeRecord& record, const voxel::Palette& palette) const -> bool; + + /// In palette entries like VolumeRecord::palette_offset + auto findOrAppendPalette(const voxel::Palette& palette) -> uint32_t; + voxel::gpu::PackedScene m_scene; uint32_t m_pool_slots = 0; std::vector m_node_uids; + /// Grid words of dead records so enough of them makes a full pack compact the sections + uint64_t m_dead_grid_words = 0; bool m_valid = false; }; diff --git a/engine/src/toast/renderer/vulkan_core.cpp b/engine/src/toast/renderer/vulkan_core.cpp index 6dce82267..772e5e957 100644 --- a/engine/src/toast/renderer/vulkan_core.cpp +++ b/engine/src/toast/renderer/vulkan_core.cpp @@ -20,8 +20,10 @@ #endif #include #include +#include #include #include +#include #include #include #include @@ -52,6 +54,25 @@ auto vulkanCorePipelineCacheMatches(std::span data, const vk::Phy std::memcmp(header.pipelineCacheUUID, props.pipelineCacheUUID.data(), VK_UUID_SIZE) == 0; } +/// Injected before main so it is known before the instance exists +auto vulkanCoreAttachedGraphicsDebugger() -> std::optional { +#if defined(_WIN32) + if (GetModuleHandleA("renderdoc.dll") != nullptr) { + return "RenderDoc"; + } + if (GetModuleHandleW(NGFX_GRAPHICS_CAPTURE_INJECTION_LIB_NAME) != nullptr || + GetModuleHandleW(NGFX_GPU_TRACE_INJECTION_LIB_NAME) != nullptr) { + return "Nsight Graphics"; + } +#elif defined(__linux__) + if (void* mod = dlopen("librenderdoc.so", RTLD_NOLOAD | RTLD_LAZY); mod != nullptr) { + dlclose(mod); + return "RenderDoc"; + } +#endif + return std::nullopt; +} + constexpr std::size_t k_gigabyte_bytes = 1024ull * 1024ull * 1024ull; constexpr uint32_t k_invalid_queue_family = std::numeric_limits::max(); @@ -313,14 +334,20 @@ VulkanCore::VulkanCore( m_validation_enabled = false; #endif - TOAST_INFO("Render", "Validation layers: {}", m_validation_enabled ? "enabled" : "disabled"); - TOAST_TRACE("Render", "Required instance extensions: {}", joinRequiredExtensions(required_instance_extensions)); - TOAST_TRACE("Render", "Required device extensions: {}", joinRequiredExtensions(required_device_extensions)); - #if defined(_WIN32) initializeNsightActivity(); #endif + // Capture tools record the instance and crash capturing or replaying it with validation chained in + if (const auto debugger = vulkanCoreAttachedGraphicsDebugger(); debugger.has_value() && m_validation_enabled) { + m_validation_enabled = false; + TOAST_WARN("Render", "{} is attached so validation layers stay off", *debugger); + } + + TOAST_INFO("Render", "Validation layers: {}", m_validation_enabled ? "enabled" : "disabled"); + TOAST_TRACE("Render", "Required instance extensions: {}", joinRequiredExtensions(required_instance_extensions)); + TOAST_TRACE("Render", "Required device extensions: {}", joinRequiredExtensions(required_device_extensions)); + vk::ApplicationInfo app_info("SUPER DUPER TOASTY GAME", 1, "TOAST ENGINE", 1, VK_API_VERSION_1_4); std::vector layers; @@ -797,7 +824,11 @@ void VulkanCore::createLogicalDeviceAndAllocator(std::span re ); vma::AllocatorCreateInfo allocator_ci {}; - allocator_ci.vulkanApiVersion = VK_API_VERSION_1_4; + // Never above the device or VMA calls entry points that capture tools capping the version lack + const uint32_t device_version = m_physical_device.getProperties().apiVersion; + allocator_ci.vulkanApiVersion = std::min( + VK_API_VERSION_1_4, VK_MAKE_API_VERSION(0, VK_API_VERSION_MAJOR(device_version), VK_API_VERSION_MINOR(device_version), 0) + ); allocator_ci.physicalDevice = *m_physical_device; // VMA asserts without eBufferDeviceAddress diff --git a/engine/src/toast/renderer/vulkan_renderer.cpp b/engine/src/toast/renderer/vulkan_renderer.cpp index 95969d4f7..66e7f94e3 100644 --- a/engine/src/toast/renderer/vulkan_renderer.cpp +++ b/engine/src/toast/renderer/vulkan_renderer.cpp @@ -1191,7 +1191,8 @@ void VulkanRenderer::createDefaultShadowMap() { recordUndefinedToTransferDst(cmd, **m_default_shadow_image, range); - constexpr vk::ClearDepthStencilValue clear_value(depth::k_clear, 0); + // Shadow maps keep standard depth so 1 is the far plane and every less-or-equal compare reads as lit + constexpr vk::ClearDepthStencilValue clear_value(1.0f, 0); cmd.clearDepthStencilImage(**m_default_shadow_image, vk::ImageLayout::eTransferDstOptimal, clear_value, range); recordTransferDstToShaderRead(cmd, **m_default_shadow_image, range); @@ -3907,32 +3908,47 @@ void VulkanRenderer::buildVoxelProxies(RenderFrame& frame) { return std::chrono::duration(std::chrono::steady_clock::now() - pack_start).count(); }; - // Small carves patch the shared buffers in place and only a grown pool or a new layout uploads everything - std::optional patch; + // Carves spawns and despawns patch the shared buffers in place and only running out of spare room uploads + // everything again + std::optional patched; if (!keep_mirror && m_voxel_storage != nullptr && m_voxel_storage->isReady()) { - patch = m_voxel_patcher.tryPatch(voxel::runtimeBrickPool(), scene_volumes, node_uids, deltas); + patched = m_voxel_patcher.tryPatch(voxel::runtimeBrickPool(), scene_volumes, node_uids, deltas); } std::optional patch_sequence; - if (patch.has_value()) { - patch_sequence = m_voxel_storage->queuePatch(*m_core, *patch); + if (patched.has_value()) { + patch_sequence = m_voxel_storage->queuePatch(*m_core, patched->patch); } if (patch_sequence.has_value()) { debug.pack_ms = elapsed_ms(); voxel_lock.unlock(); + const voxel::gpu::ScenePatch& patch = patched->patch; const VoxelStorageDebugInfo& previous = m_voxel_storage->debugInfo(); debug.sequence = ++m_voxel_upload_sequence; - debug.packed_slots = patch->pool_slots; - debug.packed_palettes = previous.packed_palettes; + debug.packed_slots = patch.pool_slots; + debug.packed_palettes = patch.palette_words / voxel::gpu::k_palette_words; + + // Debug rows follow records and a dead record keeps an empty name + std::unordered_map names; + for (size_t i = 0; i < node_uids.size(); ++i) { + names.emplace(node_uids[i], std::move(debug.node_names[i])); + } + debug.node_names.assign(patched->record_uids.size(), std::string {}); + for (size_t i = 0; i < patched->record_uids.size(); ++i) { + if (const auto name = names.find(patched->record_uids[i]); name != names.end()) { + debug.node_names[i] = std::move(name->second); + } + } debug.bricks = previous.bricks; - for (size_t i = 0; i < debug.bricks.size() && i < patch->tags.size(); ++i) { + debug.bricks.resize(patched->record_uids.size()); + for (size_t i = 0; i < debug.bricks.size() && i < patch.tags.size(); ++i) { const auto apply = [](uint32_t count, int32_t delta) { return static_cast(static_cast(count) + delta); }; - debug.bricks[i].uniform = apply(debug.bricks[i].uniform, patch->tags[i].uniform); - debug.bricks[i].shared = apply(debug.bricks[i].shared, patch->tags[i].shared); - debug.bricks[i].owned = apply(debug.bricks[i].owned, patch->tags[i].owned); + debug.bricks[i].uniform = apply(debug.bricks[i].uniform, patch.tags[i].uniform); + debug.bricks[i].shared = apply(debug.bricks[i].shared, patch.tags[i].shared); + debug.bricks[i].owned = apply(debug.bricks[i].owned, patch.tags[i].owned); } debug.patched = true; debug.patched_bricks = static_cast( @@ -3941,8 +3957,9 @@ void VulkanRenderer::buildVoxelProxies(RenderFrame& frame) { }) ); - auto storage = - std::make_shared(std::move(node_uids), std::move(brick_dims), *m_voxel_storage, *patch_sequence); + auto storage = std::make_shared( + std::move(patched->record_uids), std::move(patched->record_dims), *m_voxel_storage, *patch_sequence + ); storage->setDebugInfo(std::move(debug)); storage->setChangeHistory(m_voxel_change_tracker.packed(storage->generation())); m_voxel_storage = std::move(storage); @@ -4223,6 +4240,9 @@ auto VulkanRenderer::applyResizeInternal(vk::Extent2D extent) -> void { pass->onResize(actual_extent); } } + if (m_voxel_pass != nullptr) { + m_voxel_pass->onTargetsRecreated(); + } std::ranges::fill(m_present_bound_views, vk::ImageView {}); m_current_frame = 0; } catch (const std::exception& e) { TOAST_CRITICAL("Render", "Failed to recreate output target on resize: {}", e.what()); } diff --git a/engine/src/toast/renderer/vulkan_renderer.hpp b/engine/src/toast/renderer/vulkan_renderer.hpp index 193287bf3..36aee8d7c 100644 --- a/engine/src/toast/renderer/vulkan_renderer.hpp +++ b/engine/src/toast/renderer/vulkan_renderer.hpp @@ -530,6 +530,13 @@ class VulkanRenderer { return m_clamp_to_simulation.load(std::memory_order_relaxed); } + void setDebugWindowsVisible(bool visible) noexcept { m_debug_windows_visible.store(visible, std::memory_order_relaxed); } + + [[nodiscard]] + auto debugWindowsVisible() const noexcept -> bool { + return m_debug_windows_visible.load(std::memory_order_relaxed); + } + void setApplicationFocused(bool focused) noexcept { m_application_focused.store(focused, std::memory_order_relaxed); } [[nodiscard]] @@ -613,6 +620,16 @@ class VulkanRenderer { /// Edge rounding radius in voxels where 0 keeps the flat face normals void setVoxelCellBoxes(bool enabled) noexcept { m_voxel_cell_boxes.store(enabled, std::memory_order_relaxed); } + /// Smallest on screen voxel size in pixels that is rasterized instead of marched and 0 marches everything + void setVoxelRasterPixels(float pixels) noexcept { + m_voxel_raster_pixels.store(std::max(pixels, 0.0f), std::memory_order_relaxed); + } + + [[nodiscard]] + auto voxelRasterPixels() const noexcept -> float { + return m_voxel_raster_pixels.load(std::memory_order_relaxed); + } + [[nodiscard]] auto voxelCellBoxes() const noexcept -> bool { return m_voxel_cell_boxes.load(std::memory_order_relaxed); @@ -1226,6 +1243,7 @@ class VulkanRenderer { std::atomic m_voxel_normal_rounding {0.3f}; std::atomic_bool m_voxel_cell_boxes {true}; + std::atomic m_voxel_raster_pixels {4.0f}; ShaderLayout m_present_layout; VulkanPipeline m_present_pipeline; @@ -1472,6 +1490,7 @@ class VulkanRenderer { std::atomic m_frame_rate_limit_hz {0.0}; std::atomic_bool m_application_focused {true}; + std::atomic_bool m_debug_windows_visible {true}; std::atomic_bool m_rendering_paused {false}; std::atomic_bool m_clamp_to_simulation {true}; }; diff --git a/engine/src/toast/renderer/vulkan_texture.cpp b/engine/src/toast/renderer/vulkan_texture.cpp index 8e33a4f44..daaab74a7 100644 --- a/engine/src/toast/renderer/vulkan_texture.cpp +++ b/engine/src/toast/renderer/vulkan_texture.cpp @@ -6,6 +6,7 @@ #include "vulkan_debug.hpp" +#include #include #include #include @@ -70,6 +71,9 @@ void VulkanTexture::create(const VulkanCore& core, Params params, std::string_vi m_image_view = vk::raii::ImageView(core.getDevice(), view_ci); setDebugName(core, *m_image_view, name + " View"); + + static std::atomic next_generation {0}; + m_generation = next_generation.fetch_add(1, std::memory_order_relaxed) + 1; } void VulkanTexture::destroy() { diff --git a/engine/src/toast/renderer/vulkan_texture.hpp b/engine/src/toast/renderer/vulkan_texture.hpp index 739fcb527..ccc0e9ed3 100644 --- a/engine/src/toast/renderer/vulkan_texture.hpp +++ b/engine/src/toast/renderer/vulkan_texture.hpp @@ -7,6 +7,7 @@ #include "vulkan_common.hpp" #include "vulkan_resource_base.hpp" +#include #include #include @@ -42,10 +43,17 @@ class VulkanTexture : public IVulkanResource { return m_params.format; } + /// Unique per create() so a descriptor cache notices a recreated view that reuses a destroyed handle value + [[nodiscard]] + auto generation() const noexcept -> uint64_t { + return m_generation; + } + private: std::optional m_image; vk::raii::ImageView m_image_view = nullptr; Params m_params; + uint64_t m_generation = 0; }; class TextureUpload : public PendingResourceUpload { diff --git a/engine/src/toast/voxel/gpu_layout.cpp b/engine/src/toast/voxel/gpu_layout.cpp index f32aaa493..9d6253927 100644 --- a/engine/src/toast/voxel/gpu_layout.cpp +++ b/engine/src/toast/voxel/gpu_layout.cpp @@ -193,7 +193,7 @@ auto layoutMatches(const PackedScene& retained, const SceneLayout& layout) -> bo auto computePatch( PackedScene& retained, uint32_t retained_pool_slots, const BrickPool& pool, std::span volumes, - std::span dirty + std::span dirty, std::span appended, uint32_t first_new_palette_word ) -> ScenePatch { ZoneScoped; @@ -205,6 +205,41 @@ auto computePatch( std::vector grid_words; std::vector coarse_words; std::vector cells; + std::vector record_words; + + // The GPU holds nothing for these yet so every word is written including empty ones + for (const uint32_t index : appended) { + const Volume& volume = *volumes[index].volume; + const VolumeRecord& record = retained.records[index]; + const glm::uvec3 dims(record.brick_dims_x, record.brick_dims_y, record.brick_dims_z); + const glm::uvec3 coarse(record.coarse_dims_x, record.coarse_dims_y, record.coarse_dims_z); + TagDelta& tags = patch.tags[index]; + + const uint32_t bricks = dims.x * dims.y * dims.z; + for (uint32_t i = 0; i < bricks; ++i) { + const BrickEntry entry = volume.entryAt(glm::ivec3(brickOfIndex(dims, i))); + retained.grids[record.grid_offset + i] = entry.value; + grid_words.push_back(record.grid_offset + i); + bump(tags, entry.tag(), 1); + if (entry.isPooled()) { + slots.push_back(entry.payload()); + } + if (entry.tag() != BrickTag::empty) { + const uint32_t cell = gridIndex(coarse, brickOfIndex(dims, i) / k_coarse_bricks); + retained.coarse[record.coarse_offset + (cell / 32u)] |= 1u << (cell % 32u); + } + } + + const uint32_t coarse_count = ((coarse.x * coarse.y * coarse.z) + 31u) / 32u; + for (uint32_t i = 0; i < coarse_count; ++i) { + coarse_words.push_back(record.coarse_offset + i); + } + + constexpr uint32_t k_record_words = sizeof(VolumeRecord) / sizeof(uint32_t); + for (uint32_t i = 0; i < k_record_words; ++i) { + record_words.push_back((index * k_record_words) + i); + } + } for (const VolumeDirty& entry : dirty) { const Volume& volume = *volumes[entry.volume].volume; @@ -262,6 +297,23 @@ auto computePatch( appendWords(patch.grids, grid_words, retained.grids); appendWords(patch.coarse, coarse_words, retained.coarse); + if (!record_words.empty()) { + std::vector records(retained.records.size() * (sizeof(VolumeRecord) / sizeof(uint32_t))); + std::memcpy(records.data(), retained.records.data(), records.size() * sizeof(uint32_t)); + appendWords(patch.records, record_words, records); + } + + std::vector palette_words; + for (uint32_t i = first_new_palette_word; i < retained.palettes.size(); ++i) { + palette_words.push_back(i); + } + appendWords(patch.palettes, palette_words, retained.palettes); + + patch.grid_words = static_cast(retained.grids.size()); + patch.coarse_words = static_cast(retained.coarse.size()); + patch.palette_words = static_cast(retained.palettes.size()); + patch.record_count = static_cast(retained.records.size()); + appendSlots(patch.materials, slots, k_material_words_per_brick, [&pool](uint32_t slot, uint32_t* out) { assert(pool.isValid(slot)); std::memcpy(out, pool.material(slot).data(), k_brick_material_bytes); @@ -287,14 +339,26 @@ void applyPatch(PackedPool& pool, PackedScene& scene, const ScenePatch& patch) { std::copy_n(section.words.begin() + run.src, run.count, destination.begin() + run.dst); } }; + scene.grids.resize(std::max(scene.grids.size(), patch.grid_words), 0u); + scene.coarse.resize(std::max(scene.coarse.size(), patch.coarse_words), 0u); + scene.palettes.resize(std::max(scene.palettes.size(), patch.palette_words), 0u); + scene.records.resize(std::max(scene.records.size(), patch.record_count)); + apply(pool.materials, patch.materials); apply(pool.occupancy, patch.occupancy); apply(scene.grids, patch.grids); apply(scene.coarse, patch.coarse); + apply(scene.palettes, patch.palettes); + + std::vector records(scene.records.size() * (sizeof(VolumeRecord) / sizeof(uint32_t))); + std::memcpy(records.data(), scene.records.data(), records.size() * sizeof(uint32_t)); + apply(records, patch.records); + std::memcpy(scene.records.data(), records.data(), records.size() * sizeof(uint32_t)); } auto patchWords(const ScenePatch& patch) -> size_t { - return patch.materials.words.size() + patch.occupancy.words.size() + patch.grids.words.size() + patch.coarse.words.size(); + return patch.materials.words.size() + patch.occupancy.words.size() + patch.grids.words.size() + patch.coarse.words.size() + + patch.palettes.words.size() + patch.records.words.size(); } auto sampleMaterial(const PackedPool& pool, const PackedScene& scene, uint32_t volume, glm::ivec3 voxel) -> uint8_t { diff --git a/engine/src/toast/voxel/gpu_layout.hpp b/engine/src/toast/voxel/gpu_layout.hpp index e0b7cfe0d..2787d342e 100644 --- a/engine/src/toast/voxel/gpu_layout.hpp +++ b/engine/src/toast/voxel/gpu_layout.hpp @@ -117,10 +117,19 @@ struct ScenePatch { /// Slots the pool sections hold afterwards and they only ever grow uint32_t pool_slots = 0; + /// Section sizes afterwards in words or records + uint32_t grid_words = 0; + uint32_t coarse_words = 0; + uint32_t palette_words = 0; + uint32_t record_count = 0; + PatchSection materials; PatchSection occupancy; PatchSection grids; PatchSection coarse; + PatchSection palettes; + /// VolumeRecord words + PatchSection records; /// Per scene volume std::vector tags; @@ -156,11 +165,13 @@ TOAST_API auto packLayout(std::span volumes) -> SceneLayout; [[nodiscard]] TOAST_API auto layoutMatches(const PackedScene& retained, const SceneLayout& layout) -> bool; -/// @pre layoutMatches and leaves retained grids and coarse as the GPU holds them after the patch +/// Leaves retained as the GPU holds it afterwards +/// @p appended records are already in retained with zeroed grid and coarse ranges and palette words from +/// @p first_new_palette_word on are new [[nodiscard]] TOAST_API auto computePatch( PackedScene& retained, uint32_t retained_pool_slots, const BrickPool& pool, std::span volumes, - std::span dirty + std::span dirty, std::span appended = {}, uint32_t first_new_palette_word = UINT32_MAX ) -> ScenePatch; /// Applies a patch the way the GPU copies do