From 624184fff3d2e25f483b0d8993f85a18df46fd10 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Sun, 22 Mar 2026 19:25:07 +0100 Subject: [PATCH 01/37] Reduce Wave preprocess overhead and update DXC pointer --- 3rdparty/dxc/dxc | 2 +- include/nbl/asset/utils/IShaderCompiler.h | 17 +- .../hlsl/bxdf/reflection/beckmann.hlsl | 2 - .../nbl/builtin/hlsl/bxdf/reflection/ggx.hlsl | 2 - .../hlsl/bxdf/transmission/beckmann.hlsl | 4 +- .../builtin/hlsl/bxdf/transmission/ggx.hlsl | 3 - .../nbl/builtin/hlsl/emulated/float64_t.hlsl | 1 - .../nbl/builtin/hlsl/member_test_macros.hlsl | 7 +- .../builtin/hlsl/path_tracing/concepts.hlsl | 52 +++- .../hlsl/path_tracing/unidirectional.hlsl | 4 +- include/nbl/builtin/hlsl/tgmath.hlsl | 4 - include/nbl/video/CJITIncludeLoader.h | 2 +- src/nbl/asset/utils/CWaveStringResolver.cpp | 129 +++++---- src/nbl/asset/utils/IShaderCompiler.cpp | 108 ++++++-- src/nbl/asset/utils/includeResolutionCommon.h | 36 +++ src/nbl/asset/utils/waveContext.h | 253 ++++++++++++++++-- src/nbl/video/CJITIncludeLoader.cpp | 2 +- 17 files changed, 510 insertions(+), 118 deletions(-) create mode 100644 src/nbl/asset/utils/includeResolutionCommon.h diff --git a/3rdparty/dxc/dxc b/3rdparty/dxc/dxc index d76c7890b1..8466c63aaf 160000 --- a/3rdparty/dxc/dxc +++ b/3rdparty/dxc/dxc @@ -1 +1 @@ -Subproject commit d76c7890b19ce0b344ee0ce116dbc1c92220ccea +Subproject commit 8466c63aaf8f577948b5dfeb7f92b244c7bb9062 diff --git a/include/nbl/asset/utils/IShaderCompiler.h b/include/nbl/asset/utils/IShaderCompiler.h index f3cfe07132..c43890d161 100644 --- a/include/nbl/asset/utils/IShaderCompiler.h +++ b/include/nbl/asset/utils/IShaderCompiler.h @@ -17,6 +17,9 @@ #include "nbl/builtin/hlsl/enums.hlsl" #include +#include +#include +#include namespace nbl::asset { @@ -39,7 +42,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted explicit inline operator bool() const {return !absolutePath.empty();} }; - virtual found_t getInclude(const system::path& searchPath, const std::string& includeName) const = 0; + virtual found_t getInclude(const system::path& searchPath, const std::string& includeName, bool needHash = true) const = 0; }; class NBL_API2 IIncludeGenerator : public core::IReferenceCounted @@ -65,10 +68,13 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted public: CFileSystemIncludeLoader(core::smart_refctd_ptr&& system); - IIncludeLoader::found_t getInclude(const system::path& searchPath, const std::string& includeName) const override; + IIncludeLoader::found_t getInclude(const system::path& searchPath, const std::string& includeName, bool needHash = true) const override; protected: core::smart_refctd_ptr m_system; + mutable std::mutex m_cacheMutex; + mutable std::unordered_map m_cache; + mutable std::unordered_set m_missingCache; }; class NBL_API2 CIncludeFinder : public core::IReferenceCounted @@ -79,12 +85,12 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted // ! includes within <> // @param requestingSourceDir: the directory where the incude was requested // @param includeName: the string within <> of the include preprocessing directive - IIncludeLoader::found_t getIncludeStandard(const system::path& requestingSourceDir, const std::string& includeName) const; + IIncludeLoader::found_t getIncludeStandard(const system::path& requestingSourceDir, const std::string& includeName, bool needHash = true) const; // ! includes within "" // @param requestingSourceDir: the directory where the incude was requested // @param includeName: the string within "" of the include preprocessing directive - IIncludeLoader::found_t getIncludeRelative(const system::path& requestingSourceDir, const std::string& includeName) const; + IIncludeLoader::found_t getIncludeRelative(const system::path& requestingSourceDir, const std::string& includeName, bool needHash = true) const; inline core::smart_refctd_ptr getDefaultFileSystemLoader() const { return m_defaultFileSystemLoader; } @@ -93,7 +99,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted void addGenerator(const core::smart_refctd_ptr& generator); protected: - IIncludeLoader::found_t trySearchPaths(const std::string& includeName) const; + IIncludeLoader::found_t trySearchPaths(const std::string& includeName, bool needHash) const; IIncludeLoader::found_t tryIncludeGenerators(const std::string& includeName) const; @@ -137,6 +143,7 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted std::span extraDefines = {}; E_SPIRV_VERSION targetSpirvVersion = E_SPIRV_VERSION::ESV_1_6; bool depfile = false; + bool preserveComments = false; system::path depfilePath = {}; std::function onPartialOutputOnFailure = {}; }; diff --git a/include/nbl/builtin/hlsl/bxdf/reflection/beckmann.hlsl b/include/nbl/builtin/hlsl/bxdf/reflection/beckmann.hlsl index cb7743e02d..65db32f336 100644 --- a/include/nbl/builtin/hlsl/bxdf/reflection/beckmann.hlsl +++ b/include/nbl/builtin/hlsl/bxdf/reflection/beckmann.hlsl @@ -4,9 +4,7 @@ #ifndef _NBL_BUILTIN_HLSL_BXDF_REFLECTION_BECKMANN_INCLUDED_ #define _NBL_BUILTIN_HLSL_BXDF_REFLECTION_BECKMANN_INCLUDED_ -#include "nbl/builtin/hlsl/bxdf/common.hlsl" #include "nbl/builtin/hlsl/bxdf/bxdf_traits.hlsl" -#include "nbl/builtin/hlsl/sampling/cos_weighted_spheres.hlsl" #include "nbl/builtin/hlsl/bxdf/ndf/beckmann.hlsl" #include "nbl/builtin/hlsl/bxdf/base/cook_torrance_base.hlsl" diff --git a/include/nbl/builtin/hlsl/bxdf/reflection/ggx.hlsl b/include/nbl/builtin/hlsl/bxdf/reflection/ggx.hlsl index 0f49d0be43..a984a14b3f 100644 --- a/include/nbl/builtin/hlsl/bxdf/reflection/ggx.hlsl +++ b/include/nbl/builtin/hlsl/bxdf/reflection/ggx.hlsl @@ -4,9 +4,7 @@ #ifndef _NBL_BUILTIN_HLSL_BXDF_REFLECTION_GGX_INCLUDED_ #define _NBL_BUILTIN_HLSL_BXDF_REFLECTION_GGX_INCLUDED_ -#include "nbl/builtin/hlsl/bxdf/common.hlsl" #include "nbl/builtin/hlsl/bxdf/bxdf_traits.hlsl" -#include "nbl/builtin/hlsl/sampling/cos_weighted_spheres.hlsl" #include "nbl/builtin/hlsl/bxdf/ndf/ggx.hlsl" #include "nbl/builtin/hlsl/bxdf/base/cook_torrance_base.hlsl" diff --git a/include/nbl/builtin/hlsl/bxdf/transmission/beckmann.hlsl b/include/nbl/builtin/hlsl/bxdf/transmission/beckmann.hlsl index 8c61692c5c..b911968d16 100644 --- a/include/nbl/builtin/hlsl/bxdf/transmission/beckmann.hlsl +++ b/include/nbl/builtin/hlsl/bxdf/transmission/beckmann.hlsl @@ -4,10 +4,8 @@ #ifndef _NBL_BUILTIN_HLSL_BXDF_TRANSMISSION_BECKMANN_INCLUDED_ #define _NBL_BUILTIN_HLSL_BXDF_TRANSMISSION_BECKMANN_INCLUDED_ -#include "nbl/builtin/hlsl/bxdf/common.hlsl" #include "nbl/builtin/hlsl/bxdf/bxdf_traits.hlsl" -#include "nbl/builtin/hlsl/sampling/cos_weighted_spheres.hlsl" -#include "nbl/builtin/hlsl/bxdf/reflection.hlsl" +#include "nbl/builtin/hlsl/bxdf/ndf/beckmann.hlsl" #include "nbl/builtin/hlsl/bxdf/base/cook_torrance_base.hlsl" namespace nbl diff --git a/include/nbl/builtin/hlsl/bxdf/transmission/ggx.hlsl b/include/nbl/builtin/hlsl/bxdf/transmission/ggx.hlsl index cdd4483c7f..a095d5fdba 100644 --- a/include/nbl/builtin/hlsl/bxdf/transmission/ggx.hlsl +++ b/include/nbl/builtin/hlsl/bxdf/transmission/ggx.hlsl @@ -4,10 +4,7 @@ #ifndef _NBL_BUILTIN_HLSL_BXDF_TRANSMISSION_GGX_INCLUDED_ #define _NBL_BUILTIN_HLSL_BXDF_TRANSMISSION_GGX_INCLUDED_ -#include "nbl/builtin/hlsl/bxdf/common.hlsl" #include "nbl/builtin/hlsl/bxdf/bxdf_traits.hlsl" -#include "nbl/builtin/hlsl/sampling/cos_weighted_spheres.hlsl" -#include "nbl/builtin/hlsl/bxdf/reflection.hlsl" #include "nbl/builtin/hlsl/bxdf/base/cook_torrance_base.hlsl" namespace nbl diff --git a/include/nbl/builtin/hlsl/emulated/float64_t.hlsl b/include/nbl/builtin/hlsl/emulated/float64_t.hlsl index da32fab7b0..54c07ddb27 100644 --- a/include/nbl/builtin/hlsl/emulated/float64_t.hlsl +++ b/include/nbl/builtin/hlsl/emulated/float64_t.hlsl @@ -2,7 +2,6 @@ #define _NBL_BUILTIN_HLSL_EMULATED_FLOAT64_T_HLSL_INCLUDED_ #include -#include namespace nbl { diff --git a/include/nbl/builtin/hlsl/member_test_macros.hlsl b/include/nbl/builtin/hlsl/member_test_macros.hlsl index 7579fb0fa2..556c6a463e 100644 --- a/include/nbl/builtin/hlsl/member_test_macros.hlsl +++ b/include/nbl/builtin/hlsl/member_test_macros.hlsl @@ -5,7 +5,10 @@ #define _NBL_BUILTIN_HLSL_MEMBER_TEST_MACROS_INCLUDED_ #include -#include +#include +#include +#include +#include #ifdef __HLSL_VERSION @@ -123,4 +126,4 @@ GENERATE_METHOD_TESTER(set) #endif -#endif \ No newline at end of file +#endif diff --git a/include/nbl/builtin/hlsl/path_tracing/concepts.hlsl b/include/nbl/builtin/hlsl/path_tracing/concepts.hlsl index 25ca98772c..24a0e86455 100644 --- a/include/nbl/builtin/hlsl/path_tracing/concepts.hlsl +++ b/include/nbl/builtin/hlsl/path_tracing/concepts.hlsl @@ -5,7 +5,6 @@ #define _NBL_BUILTIN_HLSL_PATH_TRACING_CONCEPTS_INCLUDED_ #include -#include namespace nbl { @@ -15,6 +14,17 @@ namespace path_tracing { namespace concepts { +namespace impl +{ +template +struct DummyRayInteraction +{ + using vector3_type = Vector3; + + vector3_type getN() NBL_CONST_MEMBER_FUNC; + bool isMaterialBSDF() NBL_CONST_MEMBER_FUNC; +}; +} #define NBL_CONCEPT_NAME RandGenerator #define NBL_CONCEPT_TPLT_PRM_KINDS (typename) @@ -38,7 +48,7 @@ NBL_CONCEPT_END( #define NBL_CONCEPT_TPLT_PRM_NAMES (T) #define NBL_CONCEPT_PARAM_0 (ray, T) #define NBL_CONCEPT_PARAM_1 (v, typename T::vector3_type) -#define NBL_CONCEPT_PARAM_2 (interaction, bxdf::surface_interactions::SIsotropic, typename T::spectral_type>) +#define NBL_CONCEPT_PARAM_2 (interaction, impl::DummyRayInteraction) #define NBL_CONCEPT_PARAM_3 (scalar, typename T::scalar_type) #define NBL_CONCEPT_PARAM_4 (color, typename T::spectral_type) NBL_CONCEPT_BEGIN(5) @@ -52,7 +62,7 @@ NBL_CONCEPT_END( ((NBL_CONCEPT_REQ_TYPE)(T::vector3_type)) ((NBL_CONCEPT_REQ_TYPE)(T::spectral_type)) ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((ray.init(v/*origin*/, v/*direction*/)), ::nbl::hlsl::is_same_v, void)) - ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((ray.template setInteraction, typename T::spectral_type> >(interaction)), ::nbl::hlsl::is_same_v, void)) + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((ray.setInteraction(interaction)), ::nbl::hlsl::is_same_v, void)) ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((ray.initPayload()), ::nbl::hlsl::is_same_v, void)) ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((ray.shouldDoMIS()), ::nbl::hlsl::is_same_v, bool)) ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((ray.foundEmissiveMIS(scalar)), ::nbl::hlsl::is_same_v, typename T::scalar_type)) @@ -70,6 +80,21 @@ NBL_CONCEPT_END( #undef ray #include +#define NBL_CONCEPT_NAME RaySetInteraction +#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)(typename) +#define NBL_CONCEPT_TPLT_PRM_NAMES (RayT)(Interaction) +#define NBL_CONCEPT_PARAM_0 (ray, RayT) +#define NBL_CONCEPT_PARAM_1 (interaction, Interaction) +NBL_CONCEPT_BEGIN(2) +#define ray NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0 +#define interaction NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 +NBL_CONCEPT_END( + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((ray.setInteraction(interaction)), ::nbl::hlsl::is_same_v, void)) +); +#undef interaction +#undef ray +#include + #define NBL_CONCEPT_NAME RayGenerator #define NBL_CONCEPT_TPLT_PRM_KINDS (typename) #define NBL_CONCEPT_TPLT_PRM_NAMES (T) @@ -124,6 +149,7 @@ NBL_CONCEPT_END( ((NBL_CONCEPT_REQ_TYPE)(T::scene_type)) ((NBL_CONCEPT_REQ_TYPE)(T::ray_type)) ((NBL_CONCEPT_REQ_TYPE)(T::object_handle_type)) + ((NBL_CONCEPT_REQ_TYPE)(T::anisotropic_interaction_type)) ((NBL_CONCEPT_REQ_TYPE)(T::closest_hit_type)) ((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(IntersectorClosestHit, typename T::closest_hit_type)) ((NBL_CONCEPT_REQ_TYPE_ALIAS_CONCEPT)(Ray, typename T::ray_type)) @@ -136,6 +162,26 @@ NBL_CONCEPT_END( #undef intersect #include +#define NBL_CONCEPT_NAME UnidirectionalInteractionContract +#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)(typename)(typename) +#define NBL_CONCEPT_TPLT_PRM_NAMES (RayT)(IntersectorT)(MaterialSystemT) +#define NBL_CONCEPT_PARAM_0 (ray, RayT) +#define NBL_CONCEPT_PARAM_1 (hit, typename IntersectorT::closest_hit_type) +#define NBL_CONCEPT_PARAM_2 (interaction, typename MaterialSystemT::anisotropic_interaction_type) +NBL_CONCEPT_BEGIN(3) +#define ray NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0 +#define hit NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 +#define interaction NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_2 +NBL_CONCEPT_END( + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((hit.getInteraction()), ::nbl::hlsl::is_same_v, typename IntersectorT::anisotropic_interaction_type)) + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((hit.getInteraction()), ::nbl::hlsl::is_same_v, typename MaterialSystemT::anisotropic_interaction_type)) + ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((ray.setInteraction(interaction)), ::nbl::hlsl::is_same_v, void)) +); +#undef interaction +#undef hit +#undef ray +#include + #define NBL_CONCEPT_NAME BxdfNode #define NBL_CONCEPT_TPLT_PRM_KINDS (typename) #define NBL_CONCEPT_TPLT_PRM_NAMES (T) diff --git a/include/nbl/builtin/hlsl/path_tracing/unidirectional.hlsl b/include/nbl/builtin/hlsl/path_tracing/unidirectional.hlsl index 43e4cb124e..505b366b0f 100644 --- a/include/nbl/builtin/hlsl/path_tracing/unidirectional.hlsl +++ b/include/nbl/builtin/hlsl/path_tracing/unidirectional.hlsl @@ -5,11 +5,8 @@ #define _NBL_BUILTIN_HLSL_PATH_TRACING_UNIDIRECTIONAL_INCLUDED_ #include -#include #include #include -#include -#include #include namespace nbl @@ -22,6 +19,7 @@ namespace path_tracing template && concepts::Ray && concepts::Intersector && concepts::MaterialSystem && + concepts::UnidirectionalInteractionContract && concepts::NextEventEstimator && concepts::Accumulator && concepts::Scene) struct Unidirectional diff --git a/include/nbl/builtin/hlsl/tgmath.hlsl b/include/nbl/builtin/hlsl/tgmath.hlsl index c569d34f85..40ad48c13c 100644 --- a/include/nbl/builtin/hlsl/tgmath.hlsl +++ b/include/nbl/builtin/hlsl/tgmath.hlsl @@ -7,12 +7,8 @@ #include #include #include -#include #include #include -#include -#include -#include #include // C++ headers diff --git a/include/nbl/video/CJITIncludeLoader.h b/include/nbl/video/CJITIncludeLoader.h index 3b341631f4..04be1ea60a 100644 --- a/include/nbl/video/CJITIncludeLoader.h +++ b/include/nbl/video/CJITIncludeLoader.h @@ -20,7 +20,7 @@ class NBL_API2 CJITIncludeLoader : public asset::IShaderCompiler::IIncludeLoader m_includes["nbl/builtin/hlsl/jit/device_capabilities.hlsl"] = collectDeviceCaps(limits,features); } - found_t getInclude(const system::path& searchPath, const std::string& includeName) const override; + found_t getInclude(const system::path& searchPath, const std::string& includeName, bool needHash = true) const override; protected: template diff --git a/src/nbl/asset/utils/CWaveStringResolver.cpp b/src/nbl/asset/utils/CWaveStringResolver.cpp index 8da0e828ec..744c103818 100644 --- a/src/nbl/asset/utils/CWaveStringResolver.cpp +++ b/src/nbl/asset/utils/CWaveStringResolver.cpp @@ -57,13 +57,11 @@ constexpr size_t kWaveFailureLogTokenPreviewMaxChars = 160ull; struct WaveRenderProgress { core::string output; - std::optional previousPosition = std::nullopt; + std::string previousFile; + int previousLine = 0; + bool hasPreviousToken = false; bool previousWasExplicitWhitespace = false; size_t emittedTokenCount = 0ull; - std::string lastTokenFile; - int lastTokenLine = 0; - int lastTokenColumn = 0; - std::string lastTokenValue; }; std::string getLineSnippet(std::string_view text, const int lineNo) @@ -218,11 +216,6 @@ std::string makeWaveFailureContext( stream << "\n emitted_output_bytes: " << renderProgress.output.size(); stream << "\n emitted_output_lines: " << countLogicalLines(renderProgress.output); stream << "\n emitted_token_count: " << renderProgress.emittedTokenCount; - if (!renderProgress.lastTokenFile.empty()) - stream << "\n last_emitted_token_location: " << nbl::wave::detail::escape_control_chars(renderProgress.lastTokenFile) << ':' << renderProgress.lastTokenLine << ':' << renderProgress.lastTokenColumn; - if (!renderProgress.lastTokenValue.empty()) - stream << "\n last_emitted_token_value: " << truncateEscapedPreview(nbl::wave::detail::escape_control_chars(renderProgress.lastTokenValue), kWaveFailureLogTokenPreviewMaxChars); - const auto snippet = getLineSnippet(code, lineNo); if (!snippet.empty() && fileName && preprocessOptions.sourceIdentifier == fileName) { @@ -248,64 +241,90 @@ bool isWhitespaceLikeToken(const TokenT& token) return id == T_NEWLINE || id == T_GENERATEDNEWLINE || id == T_CONTLINE || IS_CATEGORY(token, WhiteSpaceTokenType); } -template -std::string tokenValueToString(const TokenT& token) -{ - const auto& value = token.get_value(); - return std::string(value.data(), value.size()); -} - void renderPreprocessedOutput(nbl::wave::context& context, WaveRenderProgress& renderProgress) { using namespace boost::wave; util::insert_whitespace_detection whitespace(true); - - for (auto it = context.begin(); it != context.end(); ++it) + auto& perfStats = nbl::wave::detail::perf_stats(); + auto it = context.begin(); + const auto end = context.end(); + while (it != end) { + std::optional loopBodyTimer; + if (perfStats.enabled) + loopBodyTimer.emplace(perfStats.loopBodyTime); + const auto& token = *it; const auto id = token_id(token); - if (id == T_EOF || id == T_EOI) - continue; + if (id != T_EOF && id != T_EOI) + { + std::optional tokenTimer; + if (perfStats.enabled) + tokenTimer.emplace(perfStats.tokenHandlingTime); - const auto explicitWhitespace = isWhitespaceLikeToken(token); - const auto& position = token.get_position(); - const auto value = tokenValueToString(token); + const auto explicitWhitespace = isWhitespaceLikeToken(token); + const auto& position = token.get_position(); + const auto& value = token.get_value(); - if (renderProgress.previousPosition.has_value() && !explicitWhitespace) - { - const auto movedToNewLogicalLine = - position.get_file() != renderProgress.previousPosition->get_file() || - position.get_line() > renderProgress.previousPosition->get_line(); + const auto currentLine = position.get_line(); + const auto& currentFile = position.get_file(); - if (movedToNewLogicalLine) + if (renderProgress.hasPreviousToken && !explicitWhitespace) { - if (renderProgress.output.empty() || renderProgress.output.back() != '\n') + bool movedToNewLogicalLine = currentLine > renderProgress.previousLine; + if (!movedToNewLogicalLine) { - renderProgress.output.push_back('\n'); - whitespace.shift_tokens(T_NEWLINE); + movedToNewLogicalLine = + renderProgress.previousFile.size() != currentFile.size() || + !std::equal(currentFile.begin(), currentFile.end(), renderProgress.previousFile.begin()); } - } - else if (!renderProgress.previousWasExplicitWhitespace && whitespace.must_insert(id, value)) - { - if (renderProgress.output.empty() || (renderProgress.output.back() != ' ' && renderProgress.output.back() != '\n' && renderProgress.output.back() != '\r' && renderProgress.output.back() != '\t')) + + if (movedToNewLogicalLine) { - renderProgress.output.push_back(' '); - whitespace.shift_tokens(T_SPACE); + if (renderProgress.output.empty() || renderProgress.output.back() != '\n') + { + renderProgress.output.push_back('\n'); + whitespace.shift_tokens(T_NEWLINE); + } } + else if (!renderProgress.previousWasExplicitWhitespace && whitespace.must_insert(id, value)) + { + if (renderProgress.output.empty() || (renderProgress.output.back() != ' ' && renderProgress.output.back() != '\n' && renderProgress.output.back() != '\r' && renderProgress.output.back() != '\t')) + { + renderProgress.output.push_back(' '); + whitespace.shift_tokens(T_SPACE); + } + } + } + + renderProgress.output.append(value.data(), value.size()); + whitespace.shift_tokens(id); + if (!renderProgress.hasPreviousToken || + renderProgress.previousFile.size() != currentFile.size() || + !std::equal(currentFile.begin(), currentFile.end(), renderProgress.previousFile.begin())) + { + renderProgress.previousFile.assign(currentFile.c_str(), currentFile.size()); } + renderProgress.previousLine = currentLine; + renderProgress.hasPreviousToken = true; + renderProgress.previousWasExplicitWhitespace = explicitWhitespace; + ++renderProgress.emittedTokenCount; + + if (tokenTimer.has_value()) + tokenTimer.reset(); } - renderProgress.output += value; - whitespace.shift_tokens(id); - renderProgress.previousPosition = position; - renderProgress.previousWasExplicitWhitespace = explicitWhitespace; - const auto& file = position.get_file(); - renderProgress.lastTokenFile.assign(file.c_str(), file.size()); - renderProgress.lastTokenLine = position.get_line(); - renderProgress.lastTokenColumn = position.get_column(); - renderProgress.lastTokenValue = value; - ++renderProgress.emittedTokenCount; + if (loopBodyTimer.has_value()) + loopBodyTimer.reset(); + + if (perfStats.enabled) + { + nbl::wave::detail::ScopedPerfTimer iteratorAdvanceTimer(perfStats.iteratorAdvanceTime); + ++it; + } + else + ++it; } } @@ -331,6 +350,8 @@ std::string preprocessImpl( }; try { + const auto totalBegin = std::chrono::steady_clock::now(); + nbl::wave::detail::reset_perf_stats(); context.set_caching(withCaching); context.add_macro_definition("__HLSL_VERSION"); context.add_macro_definition("__SPIRV_MAJOR_VERSION__=" + std::to_string(IShaderCompiler::getSpirvMajor(preprocessOptions.targetSpirvVersion))); @@ -347,7 +368,14 @@ std::string preprocessImpl( activeMacroDefinition.clear(); phase = "expanding translation unit"; - renderPreprocessedOutput(context, renderProgress); + { + nbl::wave::detail::ScopedPerfTimer renderTimer(nbl::wave::detail::perf_stats().renderTime); + renderPreprocessedOutput(context, renderProgress); + } + auto& perfStats = nbl::wave::detail::perf_stats(); + perfStats.outputBytes = renderProgress.output.size(); + perfStats.emittedTokenCount = renderProgress.emittedTokenCount; + perfStats.totalPreprocessTime = std::chrono::steady_clock::now() - totalBegin; } catch (boost::wave::preprocess_exception& e) { @@ -384,6 +412,7 @@ std::string preprocessImpl( } post(context); + nbl::wave::detail::dump_perf_stats(); return std::move(renderProgress.output); } diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index 372e877e21..434a5bf416 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -2,6 +2,7 @@ // This file is part of the "Nabla Engine". // For conditions of distribution and use, see copyright notice in nabla.h #include "nbl/asset/utils/IShaderCompiler.h" +#include "includeResolutionCommon.h" #include "nbl/asset/utils/shadercUtils.h" #include @@ -600,22 +601,53 @@ core::vector IShaderCompiler::IIncludeGenerator::parseArgumentsFrom IShaderCompiler::CFileSystemIncludeLoader::CFileSystemIncludeLoader(core::smart_refctd_ptr&& system) : m_system(std::move(system)) {} -auto IShaderCompiler::CFileSystemIncludeLoader::getInclude(const system::path& searchPath, const std::string& includeName) const -> found_t +auto IShaderCompiler::CFileSystemIncludeLoader::getInclude(const system::path& searchPath, const std::string& includeName, bool needHash) const -> found_t { - system::path path = searchPath / includeName; - if (std::filesystem::exists(path)) - path = std::filesystem::canonical(path); + system::path path = (searchPath / includeName).lexically_normal(); + + const auto cacheKey = path.generic_string(); + if (!cacheKey.empty()) + { + std::lock_guard lock(m_cacheMutex); + const auto found = m_cache.find(cacheKey); + if (found != m_cache.end()) + { + if (needHash && found->second.hash == core::blake3_hash_t{}) + { + core::blake3_hasher hasher; + hasher.update(reinterpret_cast(found->second.contents.data()), found->second.contents.size() * (sizeof(char) / sizeof(uint8_t))); + found->second.hash = static_cast(hasher); + } + return found->second; + } + if (m_missingCache.contains(cacheKey)) + return {}; + } core::smart_refctd_ptr f; { system::ISystem::future_t> future; m_system->createFile(future, path.c_str(), system::IFile::ECF_READ); if (!future.wait()) + { + if (!cacheKey.empty()) + { + std::lock_guard lock(m_cacheMutex); + m_missingCache.insert(cacheKey); + } return {}; + } future.acquire().move_into(f); } if (!f) + { + if (!cacheKey.empty()) + { + std::lock_guard lock(m_cacheMutex); + m_missingCache.insert(cacheKey); + } return {}; + } const size_t size = f->getSize(); std::string contents(size, '\0'); @@ -624,7 +656,22 @@ auto IShaderCompiler::CFileSystemIncludeLoader::getInclude(const system::path& s const bool success = bool(succ); assert(success); - return { f->getFileName(),std::move(contents) }; + found_t retVal = { f->getFileName(),std::move(contents) }; + if (needHash) + { + core::blake3_hasher hasher; + hasher.update(reinterpret_cast(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); + retVal.hash = static_cast(hasher); + } + + if (!cacheKey.empty()) + { + std::lock_guard lock(m_cacheMutex); + m_missingCache.erase(cacheKey); + m_cache.insert_or_assign(cacheKey, retVal); + } + + return retVal; } namespace @@ -644,49 +691,68 @@ std::string normalizeIncludeLookupName(const std::string& includeName) return includeName.substr(1ull); } + } IShaderCompiler::CIncludeFinder::CIncludeFinder(core::smart_refctd_ptr&& system) : m_defaultFileSystemLoader(core::make_smart_refctd_ptr(std::move(system))) { - addSearchPath("", m_defaultFileSystemLoader); } // ! includes within <> // @param requestingSourceDir: the directory where the incude was requested // @param includeName: the string within <> of the include preprocessing directive // @param -auto IShaderCompiler::CIncludeFinder::getIncludeStandard(const system::path& requestingSourceDir, const std::string& includeName) const -> IIncludeLoader::found_t +auto IShaderCompiler::CIncludeFinder::getIncludeStandard(const system::path& requestingSourceDir, const std::string& includeName, bool needHash) const -> IIncludeLoader::found_t { const auto lookupName = normalizeIncludeLookupName(includeName); IShaderCompiler::IIncludeLoader::found_t retVal; if (auto contents = tryIncludeGenerators(lookupName)) retVal = std::move(contents); - else if (auto contents = trySearchPaths(lookupName)) + else if (auto contents = trySearchPaths(lookupName, needHash)) retVal = std::move(contents); - else retVal = m_defaultFileSystemLoader->getInclude(requestingSourceDir.string(), lookupName); + else retVal = m_defaultFileSystemLoader->getInclude(requestingSourceDir.string(), lookupName, needHash); - core::blake3_hasher hasher; - hasher.update(reinterpret_cast(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); - retVal.hash = static_cast(hasher); + if (needHash && retVal && retVal.hash == core::blake3_hash_t{}) + { + core::blake3_hasher hasher; + hasher.update(reinterpret_cast(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); + retVal.hash = static_cast(hasher); + } return retVal; } // ! includes within "" // @param requestingSourceDir: the directory where the incude was requested // @param includeName: the string within "" of the include preprocessing directive -auto IShaderCompiler::CIncludeFinder::getIncludeRelative(const system::path& requestingSourceDir, const std::string& includeName) const -> IIncludeLoader::found_t +auto IShaderCompiler::CIncludeFinder::getIncludeRelative(const system::path& requestingSourceDir, const std::string& includeName, bool needHash) const -> IIncludeLoader::found_t { const auto lookupName = normalizeIncludeLookupName(includeName); IShaderCompiler::IIncludeLoader::found_t retVal; - if (auto contents = m_defaultFileSystemLoader->getInclude(requestingSourceDir.string(), lookupName)) - retVal = std::move(contents); - else retVal = std::move(trySearchPaths(lookupName)); + if (asset::detail::isGloballyResolvedIncludeName(lookupName)) + { + if (auto contents = tryIncludeGenerators(lookupName)) + retVal = std::move(contents); + else if (auto contents = trySearchPaths(lookupName, needHash)) + retVal = std::move(contents); + else retVal = m_defaultFileSystemLoader->getInclude(requestingSourceDir.string(), lookupName, needHash); + } + else + { + if (auto contents = m_defaultFileSystemLoader->getInclude(requestingSourceDir.string(), lookupName, needHash)) + retVal = std::move(contents); + else if (auto contents = tryIncludeGenerators(lookupName)) + retVal = std::move(contents); + else retVal = std::move(trySearchPaths(lookupName, needHash)); + } - core::blake3_hasher hasher; - hasher.update(reinterpret_cast(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); - retVal.hash = static_cast(hasher); + if (needHash && retVal && retVal.hash == core::blake3_hash_t{}) + { + core::blake3_hasher hasher; + hasher.update(reinterpret_cast(retVal.contents.data()), retVal.contents.size() * (sizeof(char) / sizeof(uint8_t))); + retVal.hash = static_cast(hasher); + } return retVal; } @@ -713,10 +779,10 @@ void IShaderCompiler::CIncludeFinder::addGenerator(const core::smart_refctd_ptr< m_generators.insert(found, generatorToAdd); } -auto IShaderCompiler::CIncludeFinder::trySearchPaths(const std::string& includeName) const -> IIncludeLoader::found_t +auto IShaderCompiler::CIncludeFinder::trySearchPaths(const std::string& includeName, bool needHash) const -> IIncludeLoader::found_t { for (const auto& itr : m_loaders) - if (auto contents = itr.loader->getInclude(itr.searchPath, includeName)) + if (auto contents = itr.loader->getInclude(itr.searchPath, includeName, needHash)) return contents; return {}; } diff --git a/src/nbl/asset/utils/includeResolutionCommon.h b/src/nbl/asset/utils/includeResolutionCommon.h new file mode 100644 index 0000000000..dfe9a28684 --- /dev/null +++ b/src/nbl/asset/utils/includeResolutionCommon.h @@ -0,0 +1,36 @@ +// Copyright (C) 2018-2026 - DevSH Graphics Programming Sp. z O.O. +// This file is part of the "Nabla Engine". +// For conditions of distribution and use, see copyright notice in nabla.h +#ifndef _NBL_ASSET_INCLUDE_RESOLUTION_COMMON_H_INCLUDED_ +#define _NBL_ASSET_INCLUDE_RESOLUTION_COMMON_H_INCLUDED_ + +#include + +namespace nbl::asset::detail +{ +inline bool isGloballyResolvedIncludeName(std::string_view includeName) +{ + constexpr std::string_view globalPrefixes[] = { + "nbl/", + "nbl\\", + "boost/", + "boost\\", + "glm/", + "glm\\", + "spirv/", + "spirv\\", + "Imath/", + "Imath\\" + }; + + for (const auto prefix : globalPrefixes) + { + if (includeName.rfind(prefix, 0ull) == 0ull) + return true; + } + + return false; +} +} + +#endif diff --git a/src/nbl/asset/utils/waveContext.h b/src/nbl/asset/utils/waveContext.h index 36f9d4ea99..4b9a263add 100644 --- a/src/nbl/asset/utils/waveContext.h +++ b/src/nbl/asset/utils/waveContext.h @@ -8,10 +8,16 @@ #include #include #include +#include +#include +#include +#include #include +#include #include #include "nbl/asset/utils/IShaderCompiler.h" +#include "includeResolutionCommon.h" namespace nbl::wave { @@ -21,6 +27,153 @@ using namespace boost::wave::util; namespace detail { +struct PerfStats +{ + bool enabled = false; + bool includeDetailsEnabled = false; + uint64_t includeRequests = 0ull; + uint64_t includeLookupCount = 0ull; + uint64_t includeResolutionCacheSkips = 0ull; + uint64_t postLoadPragmaSkips = 0ull; + std::chrono::nanoseconds includeLookupTime = std::chrono::nanoseconds::zero(); + std::chrono::nanoseconds tokenHandlingTime = std::chrono::nanoseconds::zero(); + std::chrono::nanoseconds iteratorAdvanceTime = std::chrono::nanoseconds::zero(); + std::chrono::nanoseconds loopBodyTime = std::chrono::nanoseconds::zero(); + std::chrono::nanoseconds renderTime = std::chrono::nanoseconds::zero(); + std::chrono::nanoseconds totalPreprocessTime = std::chrono::nanoseconds::zero(); + size_t outputBytes = 0ull; + uint64_t emittedTokenCount = 0ull; + std::unordered_map requestedIncludeSpellingCounts; + std::unordered_map resolvedIncludePathCounts; +}; + +inline PerfStats& perf_stats() +{ + static PerfStats stats = []() + { + PerfStats value; + value.enabled = std::getenv("NBL_WAVE_PROFILE") != nullptr; + value.includeDetailsEnabled = std::getenv("NBL_WAVE_PROFILE_INCLUDES") != nullptr; + return value; + }(); + return stats; +} + +inline void reset_perf_stats() +{ + auto& stats = perf_stats(); + const bool enabled = stats.enabled; + const bool includeDetailsEnabled = stats.includeDetailsEnabled; + stats = {}; + stats.enabled = enabled; + stats.includeDetailsEnabled = includeDetailsEnabled; +} + +class ScopedPerfTimer +{ + public: + explicit ScopedPerfTimer(std::chrono::nanoseconds& target) : m_target(target), m_begin(std::chrono::steady_clock::now()) {} + ~ScopedPerfTimer() + { + m_target += std::chrono::steady_clock::now() - m_begin; + } + + private: + std::chrono::nanoseconds& m_target; + std::chrono::steady_clock::time_point m_begin; +}; + +inline void dump_perf_stats() +{ + const auto& stats = perf_stats(); + if (!stats.enabled) + return; + + const auto to_ms = [](const std::chrono::nanoseconds value) -> double + { + return std::chrono::duration(value).count(); + }; + + std::fprintf( + stderr, + "[wave-profile] total_ms=%.3f include_lookup_ms=%.3f token_handling_ms=%.3f iterator_advance_ms=%.3f loop_body_ms=%.3f render_ms=%.3f include_requests=%llu include_lookups=%llu resolution_cache_skips=%llu postload_pragma_skips=%llu emitted_tokens=%llu output_bytes=%zu\n", + to_ms(stats.totalPreprocessTime), + to_ms(stats.includeLookupTime), + to_ms(stats.tokenHandlingTime), + to_ms(stats.iteratorAdvanceTime), + to_ms(stats.loopBodyTime), + to_ms(stats.renderTime), + static_cast(stats.includeRequests), + static_cast(stats.includeLookupCount), + static_cast(stats.includeResolutionCacheSkips), + static_cast(stats.postLoadPragmaSkips), + static_cast(stats.emittedTokenCount), + stats.outputBytes + ); + + if (!stats.includeDetailsEnabled) + return; + + auto dumpTopCounts = [](const char* const label, const std::unordered_map& counts) + { + if (counts.empty()) + return; + + std::vector> entries; + entries.reserve(counts.size()); + for (const auto& [name, count] : counts) + entries.emplace_back(name, count); + + std::sort(entries.begin(), entries.end(), [](const auto& lhs, const auto& rhs) + { + if (lhs.second != rhs.second) + return lhs.second > rhs.second; + return lhs.first < rhs.first; + }); + + constexpr size_t kMaxEntries = 24ull; + const auto limit = std::min(entries.size(), kMaxEntries); + for (size_t i = 0ull; i < limit; ++i) + { + const auto& entry = entries[i]; + std::fprintf(stderr, "[wave-profile] %s[%zu]=%llu %s\n", label, i, static_cast(entry.second), entry.first.c_str()); + } + }; + + dumpTopCounts("requested_include", stats.requestedIncludeSpellingCounts); + dumpTopCounts("resolved_include", stats.resolvedIncludePathCounts); +} + +struct LanguageFlagConfig +{ + bool preserveComments = false; + bool enableCpp20 = true; + bool preferPpNumbers = true; + bool emitLineDirectives = true; + bool includeGuardDetection = true; + bool emitPragmaDirectives = true; +}; + +inline boost::wave::language_support make_language_flags(const LanguageFlagConfig& config) +{ + auto flags = boost::wave::language_support(); + if (config.enableCpp20) + flags = boost::wave::language_support(flags | support_cpp20); // C++20 lexer mode. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L56-L59 + if (config.preferPpNumbers) + flags = boost::wave::language_support(flags | support_option_prefer_pp_numbers); // Prefer pp-number lexing before retokenization. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L71 + if (config.preserveComments) + flags = boost::wave::language_support(flags | support_option_preserve_comments); // Keep comments in the token stream. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L67 + if (config.emitLineDirectives) + flags = boost::wave::language_support(flags | support_option_emit_line_directives); // Emit #line directives in the output. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L72 + if (config.includeGuardDetection) + flags = boost::wave::language_support(flags | support_option_include_guard_detection); // Let Wave short-circuit classic include guards. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/libs/wave/include/boost/wave/language_support.hpp#L239 + if (config.emitPragmaDirectives) + flags = boost::wave::language_support(flags | support_option_emit_pragma_directives); // Keep pragma directives in the output. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L74 + // support_option_emit_contnewlines // Emit escaped line continuations. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L65 + // support_option_insert_whitespace // Let Wave inject separator whitespace. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L66 + return flags; +} + inline std::string escape_control_chars(std::string_view text) { static constexpr char hex[] = "0123456789ABCDEF"; @@ -81,7 +234,7 @@ struct load_to_string final template static void init_iterators(IterContextT& iter_ctx, PositionT const& act_pos, boost::wave::language_support language) { - iter_ctx.instring = iter_ctx.ctx.get_located_include_content(); + iter_ctx.instring = iter_ctx.ctx.take_located_include_content(); if (!iter_ctx.instring.empty() && iter_ctx.instring.back() != '\n' && iter_ctx.instring.back() != '\r') iter_ctx.instring.push_back('\n'); @@ -99,7 +252,7 @@ struct load_to_string final struct preprocessing_hooks final : public boost::wave::context_policies::default_preprocessing_hooks { preprocessing_hooks(const nbl::asset::IShaderCompiler::SPreprocessorOptions& _preprocessOptions) - : m_includeFinder(_preprocessOptions.includeFinder), m_logger(_preprocessOptions.logger), m_pragmaStage(nbl::asset::IShader::E_SHADER_STAGE::ESS_UNKNOWN), m_dxc_compile_flags_override() + : m_includeFinder(_preprocessOptions.includeFinder), m_logger(_preprocessOptions.logger), m_preserveComments(_preprocessOptions.preserveComments), m_pragmaStage(nbl::asset::IShader::E_SHADER_STAGE::ESS_UNKNOWN), m_dxc_compile_flags_override() { hash_token_occurences = 0; } @@ -205,9 +358,9 @@ struct preprocessing_hooks final : public boost::wave::context_policies::default return false; } - const asset::IShaderCompiler::CIncludeFinder* m_includeFinder; system::logger_opt_ptr m_logger; + bool m_preserveComments; asset::IShader::E_SHADER_STAGE m_pragmaStage; int hash_token_occurences; std::vector m_dxc_compile_flags_override; @@ -250,15 +403,7 @@ class context : private boost::noncopyable , current_filename(fname) , current_relative_filename(fname) , macros(*this_()) - , language(language_support( - support_cpp20 // C++20 lexer mode. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L56-L59 - | support_option_prefer_pp_numbers // Prefer pp-number lexing before retokenization. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L71 - | support_option_preserve_comments // Keep comments in the token stream. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L67 - | support_option_emit_line_directives // Emit #line directives in the output. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L72 - | support_option_emit_pragma_directives // Keep pragma directives in the output. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L74 -// | support_option_emit_contnewlines // Emit escaped line continuations. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L65 -// | support_option_insert_whitespace // Let Wave inject separator whitespace. https://github.com/Devsh-Graphics-Programming/wave/blob/e02cda69e4d070fd9b16a39282d6b5c717cb3da4/include/boost/wave/language_support.hpp#L66 - )) + , language(detail::make_language_flags(detail::LanguageFlagConfig{.preserveComments = hooks_.m_preserveComments})) , hooks(hooks_) { macros.init_predefined_macros(fname); @@ -414,6 +559,10 @@ class context : private boost::noncopyable { return located_include_content; } + core::string take_located_include_content() + { + return std::move(located_include_content); + } // Nabla Additions End #if !defined(BOOST_NO_MEMBER_TEMPLATE_FRIENDS) @@ -512,6 +661,19 @@ class context : private boost::noncopyable { return pragma_once_headers.contains(filename_); } + bool has_cached_include_resolution(std::string_view includeName, bool is_system, std::string& absolutePath) const + { + const auto found = include_resolution_cache.find(make_include_resolution_key(includeName, is_system)); + if (found == include_resolution_cache.end()) + return false; + + absolutePath = found->second; + return true; + } + void cache_include_resolution(std::string_view includeName, bool is_system, const std::string& absolutePath) + { + include_resolution_cache.insert_or_assign(make_include_resolution_key(includeName, is_system), absolutePath); + } bool add_pragma_once_header(std::string const& filename_, std::string const& guard_name) { get_hooks().detected_include_guard(derived(), filename_, guard_name); @@ -558,6 +720,7 @@ class context : private boost::noncopyable #if BOOST_WAVE_SUPPORT_PRAGMA_ONCE != 0 std::unordered_set pragma_once_headers; #endif + std::unordered_map include_resolution_cache; // Cache Additions bool cachingRequested = false; std::vector dependencies = {}; @@ -568,6 +731,25 @@ class context : private boost::noncopyable macromap_type macros; // map of defined macros const boost::wave::language_support language; // supported language/extensions preprocessing_hooks hooks; // hook policy instance + + std::string make_include_resolution_key(std::string_view includeName, bool is_system) const + { + std::string key; + const bool globallyResolved = is_system || asset::detail::isGloballyResolvedIncludeName(includeName); + if (!globallyResolved) + { + const auto currentDirString = current_dir.generic_string(); + key.reserve(currentDirString.size() + includeName.size() + 3ull); + key.append(currentDirString); + key.push_back('\n'); + } + else + key.reserve(includeName.size() + 2ull); + key.push_back(globallyResolved ? 'G' : 'R'); + key.push_back('\n'); + key.append(includeName.data(), includeName.size()); + return key; + } }; } @@ -588,19 +770,49 @@ template<> inline bool boost::wave::impl::pp_iterator_functorgetDefaultFileSystemLoader()->getInclude(nbl::system::path{}, cachedAbsolutePath, needHash); + standardInclude = is_system; + } + } - if (includeFinder) + if (!result && includeFinder) { + nbl::wave::detail::ScopedPerfTimer lookupTimer(perfStats.includeLookupTime); + if (perfStats.enabled) + ++perfStats.includeLookupCount; if (is_system) { - result = includeFinder->getIncludeStandard(ctx.get_current_directory(), file_path); + result = includeFinder->getIncludeStandard(ctx.get_current_directory(), file_path, needHash); standardInclude = true; } else { - result = includeFinder->getIncludeRelative(ctx.get_current_directory(), file_path); + result = includeFinder->getIncludeRelative(ctx.get_current_directory(), file_path, needHash); standardInclude = false; } } - else { + else if (!result) { const auto escapedPath = nbl::wave::detail::escape_control_chars(file_path); ctx.get_hooks().m_logger.log("Pre-processor error: Include finder not assigned, preprocessor will not include file %s", nbl::system::ILogger::ELL_ERROR, escapedPath.c_str()); return false; @@ -618,9 +830,18 @@ template<> inline bool boost::wave::impl::pp_iterator_functor found_t +auto CJITIncludeLoader::getInclude(const system::path& searchPath, const std::string& includeName, bool) const -> found_t { assert(searchPath=="nbl/builtin/hlsl/jit"); From 555684d18626008154be5ca524ab9b239444b781 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Sun, 22 Mar 2026 23:22:44 +0100 Subject: [PATCH 02/37] Advance DXC to latest unroll-devshFixes --- 3rdparty/dxc/dxc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/dxc/dxc b/3rdparty/dxc/dxc index 8466c63aaf..118cade274 160000 --- a/3rdparty/dxc/dxc +++ b/3rdparty/dxc/dxc @@ -1 +1 @@ -Subproject commit 8466c63aaf8f577948b5dfeb7f92b244c7bb9062 +Subproject commit 118cade2746a156d17fb219a7e16e2d9d433742e From 03ad12b4cc651719220442592e1aacff6530cc80 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Mon, 23 Mar 2026 05:43:58 +0100 Subject: [PATCH 03/37] Restore default include search path for builtin HLSL --- src/nbl/asset/utils/IShaderCompiler.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nbl/asset/utils/IShaderCompiler.cpp b/src/nbl/asset/utils/IShaderCompiler.cpp index 434a5bf416..8ef764e4a6 100644 --- a/src/nbl/asset/utils/IShaderCompiler.cpp +++ b/src/nbl/asset/utils/IShaderCompiler.cpp @@ -697,6 +697,7 @@ std::string normalizeIncludeLookupName(const std::string& includeName) IShaderCompiler::CIncludeFinder::CIncludeFinder(core::smart_refctd_ptr&& system) : m_defaultFileSystemLoader(core::make_smart_refctd_ptr(std::move(system))) { + addSearchPath("", m_defaultFileSystemLoader); } // ! includes within <> From ac0289dda98b1046000873b0b3ffedb06356be53 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Mon, 23 Mar 2026 10:42:38 +0100 Subject: [PATCH 04/37] Advance DXC to latest unroll-devshFixes --- 3rdparty/dxc/dxc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/dxc/dxc b/3rdparty/dxc/dxc index 118cade274..07f06e9d48 160000 --- a/3rdparty/dxc/dxc +++ b/3rdparty/dxc/dxc @@ -1 +1 @@ -Subproject commit 118cade2746a156d17fb219a7e16e2d9d433742e +Subproject commit 07f06e9d48807ef8e7cabc41ae6acdeb26c68c09 From 8e3c301685a29e7dde322048ac0199c35df1dc4c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 11:45:40 +0100 Subject: [PATCH 05/37] Promote NSC channel ac0289dda98b1046000873b0b3ffedb06356be53 (#1028) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- tools/nsc/manifests/nsc-windows-x64-release.tag | 2 +- .../exe/tools/nsc/bin/build-info.json.dvc | 4 ++-- .../nsc-windows-x64-release/exe/tools/nsc/bin/nsc.exe.dvc | 2 +- .../runtime/nbl/3rdparty/dxc/dxcompiler.dll.dvc | 4 ++-- .../nsc-windows-x64-release/runtime/nbl/Nabla.dll.dvc | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/nsc/manifests/nsc-windows-x64-release.tag b/tools/nsc/manifests/nsc-windows-x64-release.tag index cd8d651439..0e2bf783fa 100644 --- a/tools/nsc/manifests/nsc-windows-x64-release.tag +++ b/tools/nsc/manifests/nsc-windows-x64-release.tag @@ -1 +1 @@ -nsc-windows-x64-release-66da590b3f06b586f69bdb522bad2f2eebf11b6f +nsc-windows-x64-release-ac0289dda98b1046000873b0b3ffedb06356be53 diff --git a/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/build-info.json.dvc b/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/build-info.json.dvc index 2e26e723f6..aa682377ac 100644 --- a/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/build-info.json.dvc +++ b/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/build-info.json.dvc @@ -1,5 +1,5 @@ outs: -- md5: 3422c063e9f0078b4efae5aa374e12c6 - size: 1286 +- md5: 0313edc75aedd79a1ec94ab90feb5881 + size: 1326 hash: md5 path: build-info.json diff --git a/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/nsc.exe.dvc b/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/nsc.exe.dvc index 976c2e55fb..38b1766b5a 100644 --- a/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/nsc.exe.dvc +++ b/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/nsc.exe.dvc @@ -1,5 +1,5 @@ outs: -- md5: e8859f963019b7c7dd0fd815e625e4ee +- md5: 159813ebd3546457ad48e3a310b3f693 size: 256512 hash: md5 path: nsc.exe diff --git a/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/3rdparty/dxc/dxcompiler.dll.dvc b/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/3rdparty/dxc/dxcompiler.dll.dvc index 3b22d29bd8..b1f8ff3713 100644 --- a/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/3rdparty/dxc/dxcompiler.dll.dvc +++ b/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/3rdparty/dxc/dxcompiler.dll.dvc @@ -1,5 +1,5 @@ outs: -- md5: bcdd137482f6fd4a3b55da0884978d58 - size: 21367296 +- md5: ec2f20db14ac9272795ba5ad970423b8 + size: 21578752 hash: md5 path: dxcompiler.dll diff --git a/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/Nabla.dll.dvc b/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/Nabla.dll.dvc index 5ad31c5b16..c74a077ab8 100644 --- a/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/Nabla.dll.dvc +++ b/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/Nabla.dll.dvc @@ -1,5 +1,5 @@ outs: -- md5: 5707af6c5ca1d82db41e877d075af6b2 - size: 29018624 +- md5: 219186a91cf612f2ab0f9dd22278206f + size: 29173760 hash: md5 path: Nabla.dll From fe4a5280abd281a6b309fa04c06141359fb8035c Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Mon, 23 Mar 2026 12:49:03 +0100 Subject: [PATCH 06/37] Update examples_tests to local unroll --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 8f045a1c27..adf0db4f96 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 8f045a1c27a198f8542456378f865032765378b8 +Subproject commit adf0db4f96e56330b58100adceff172e2f640da6 From f195565cdf9ca828194d5156230663f5c76769e7 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Mon, 23 Mar 2026 16:03:36 +0100 Subject: [PATCH 07/37] Update EX31 examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index adf0db4f96..13f092092d 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit adf0db4f96e56330b58100adceff172e2f640da6 +Subproject commit 13f092092d31101b1cf82cbf4442f8666cdfc1ff From 697cfcfdd27c6264ecc1b833dd78f7abcccf1877 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Mon, 23 Mar 2026 17:26:51 +0100 Subject: [PATCH 08/37] Wire path tracer pipeline cache --- examples_tests | 2 +- include/nbl/ext/FullScreenTriangle/FullScreenTriangle.h | 3 ++- src/nbl/ext/FullScreenTriangle/CFullScreenTriangle.cpp | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/examples_tests b/examples_tests index 13f092092d..6fec0e4035 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 13f092092d31101b1cf82cbf4442f8666cdfc1ff +Subproject commit 6fec0e40352023965da651a364df3a0c229a3152 diff --git a/include/nbl/ext/FullScreenTriangle/FullScreenTriangle.h b/include/nbl/ext/FullScreenTriangle/FullScreenTriangle.h index 597ebdbd4e..39013417dc 100644 --- a/include/nbl/ext/FullScreenTriangle/FullScreenTriangle.h +++ b/include/nbl/ext/FullScreenTriangle/FullScreenTriangle.h @@ -24,7 +24,8 @@ struct ProtoPipeline final const video::IGPURenderpass* renderpass, const uint32_t subpassIx=0, asset::SBlendParams blendParams = {}, - const hlsl::SurfaceTransform::FLAG_BITS swapchainTransform=hlsl::SurfaceTransform::FLAG_BITS::IDENTITY_BIT + const hlsl::SurfaceTransform::FLAG_BITS swapchainTransform=hlsl::SurfaceTransform::FLAG_BITS::IDENTITY_BIT, + video::IGPUPipelineCache* pipelineCache = nullptr ); core::smart_refctd_ptr m_vxShader; diff --git a/src/nbl/ext/FullScreenTriangle/CFullScreenTriangle.cpp b/src/nbl/ext/FullScreenTriangle/CFullScreenTriangle.cpp index fd5411c2ab..58b1f2ea84 100644 --- a/src/nbl/ext/FullScreenTriangle/CFullScreenTriangle.cpp +++ b/src/nbl/ext/FullScreenTriangle/CFullScreenTriangle.cpp @@ -84,7 +84,8 @@ smart_refctd_ptr ProtoPipeline::createPipeline( const IGPURenderpass* renderpass, const uint32_t subpassIx, SBlendParams blendParams, - const hlsl::SurfaceTransform::FLAG_BITS swapchainTransform) + const hlsl::SurfaceTransform::FLAG_BITS swapchainTransform, + IGPUPipelineCache* pipelineCache) { if (!renderpass || !bool(*this) || hlsl::bitCount(swapchainTransform) != 1) return nullptr; @@ -116,7 +117,7 @@ smart_refctd_ptr ProtoPipeline::createPipeline( }; params[0].renderpass = renderpass; - if (!device->createGraphicsPipelines(nullptr, params, &m_retval)) + if (!device->createGraphicsPipelines(pipelineCache, params, &m_retval)) return nullptr; } return m_retval; From fad9d56e4773b073463fd9b969c3cece03d1b827 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Mon, 23 Mar 2026 17:33:12 +0100 Subject: [PATCH 09/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 6fec0e4035..25d3f00972 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 6fec0e40352023965da651a364df3a0c229a3152 +Subproject commit 25d3f00972e786e4641a06a1abe2a65f952743d6 From a0b65da93ee84c6e3931b61af61dd06a7b1f27eb Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Mon, 23 Mar 2026 22:03:00 +0100 Subject: [PATCH 10/37] Add SPIR-V trimmer fast path --- .../nbl/asset/utils/ISPIRVEntryPointTrimmer.h | 2 +- .../asset/utils/ISPIRVEntryPointTrimmer.cpp | 116 +++++++++++------- 2 files changed, 72 insertions(+), 46 deletions(-) diff --git a/include/nbl/asset/utils/ISPIRVEntryPointTrimmer.h b/include/nbl/asset/utils/ISPIRVEntryPointTrimmer.h index a2e24dabab..1a1f6b857c 100644 --- a/include/nbl/asset/utils/ISPIRVEntryPointTrimmer.h +++ b/include/nbl/asset/utils/ISPIRVEntryPointTrimmer.h @@ -10,7 +10,7 @@ namespace nbl::asset { -class ISPIRVEntryPointTrimmer final : public core::IReferenceCounted +class NBL_API2 ISPIRVEntryPointTrimmer final : public core::IReferenceCounted { public: ISPIRVEntryPointTrimmer(); diff --git a/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp b/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp index 6695c78e96..3ee3f18e35 100644 --- a/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp +++ b/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp @@ -31,27 +31,6 @@ ISPIRVEntryPointTrimmer::ISPIRVEntryPointTrimmer() m_optimizer = core::make_smart_refctd_ptr(std::span(optimizationPasses)); } -// This is for debugging temporarily. will be reworked after finish testing -static void printCapabilities(const uint32_t* spirv, uint32_t spirvDwordCount,nbl::system::logger_opt_ptr logger) -{ - spvtools::SpirvTools core(SPIRV_VERSION); - std::string disassembly; - core.Disassemble(spirv, spirvDwordCount, &disassembly, SPV_BINARY_TO_TEXT_OPTION_NO_HEADER); - std::stringstream ss(disassembly); - std::string to; - const auto stringsToFind = std::array{ "OpCapability", "= OpFunction","OpFunctionEnd", "OpSpecConstant", "=OpType"}; - while(std::getline(ss, to, '\n')){ - if (to.size() > 1 && to.back() == ',') continue; - for (const auto& stringToFind: stringsToFind) - { - if (to.find(stringToFind) != std::string::npos) - { - logger.log("%s", nbl::system::ILogger::ELL_DEBUG, to.c_str()); - } - } - } -} - static bool validate(const uint32_t* binary, uint32_t binarySize, nbl::system::logger_opt_ptr logger) { auto msgConsumer = [&logger](spv_message_level_t level, const char* src, const spv_position_t& pos, const char* msg) @@ -98,18 +77,6 @@ ISPIRVEntryPointTrimmer::Result ISPIRVEntryPointTrimmer::trim(const ICPUBuffer* }; } - auto foundEntryPoint = 0; - - const bool isInputSpirvValid = validate(spirv, spirvDwordCount, logger); - if (!isInputSpirvValid) - { - logger.log("SPIR-V is not valid", system::ILogger::ELL_ERROR); - return Result{ - nullptr, - false - }; - } - auto getHlslShaderStage = [](spv::ExecutionModel executionModel) -> hlsl::ShaderStage { switch (executionModel) @@ -149,6 +116,77 @@ ISPIRVEntryPointTrimmer::Result ISPIRVEntryPointTrimmer::trim(const ICPUBuffer* return { length, opcode }; }; + { + auto probeOffset = HEADER_SIZE; + auto totalEntryPoints = 0u; + auto matchingEntryPoints = 0u; + auto validFastPath = (spirvDwordCount >= HEADER_SIZE); + + while (validFastPath && probeOffset < spirvDwordCount) + { + const auto instruction = spirv[probeOffset]; + const auto [length, opcode] = parse_instruction(instruction); + if (length == 0u || (probeOffset + length) > spirvDwordCount) + { + validFastPath = false; + break; + } + if (opcode == spv::OpEntryPoint) + break; + probeOffset += length; + } + + while (validFastPath && probeOffset < spirvDwordCount) + { + const auto curOffset = probeOffset; + const auto instruction = spirv[curOffset]; + const auto [length, opcode] = parse_instruction(instruction); + if (length == 0u || (probeOffset + length) > spirvDwordCount) + { + validFastPath = false; + break; + } + if (opcode != spv::OpEntryPoint) + break; + probeOffset += length; + ++totalEntryPoints; + + const auto curExecutionModel = static_cast(spirv[curOffset + 1]); + const auto curEntryPointName = std::string_view(reinterpret_cast(spirv + curOffset + 3)); + const auto entryPoint = EntryPoint{ + .name = curEntryPointName, + .stage = getHlslShaderStage(curExecutionModel), + }; + if (entryPoint.stage == hlsl::ESS_UNKNOWN) + { + validFastPath = false; + break; + } + if (entryPoints.contains(entryPoint)) + ++matchingEntryPoints; + } + + if (validFastPath && totalEntryPoints == entryPoints.size() && matchingEntryPoints == entryPoints.size()) + { + return { + .spirv = nullptr, + .isSuccess = true, + }; + } + } + + auto foundEntryPoint = 0; + + const bool isInputSpirvValid = validate(spirv, spirvDwordCount, logger); + if (!isInputSpirvValid) + { + logger.log("SPIR-V is not valid", system::ILogger::ELL_ERROR); + return Result{ + nullptr, + false + }; + } + // Keep in mind about this layout while reading all the code below: https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#LogicalLayout // skip until entry point @@ -244,18 +282,6 @@ ISPIRVEntryPointTrimmer::Result ISPIRVEntryPointTrimmer::trim(const ICPUBuffer* auto trimmedSpirv = m_optimizer->optimize(minimizedSpirv.data(), minimizedSpirv.size(), logger); -#ifdef _NBL_DEBUG - logger.log("Before stripping capabilities:", nbl::system::ILogger::ELL_DEBUG); - printCapabilities(spirv, spirvDwordCount, logger); - logger.log("\n", nbl::system::ILogger::ELL_DEBUG); - - const auto* trimmedSpirvBuffer = static_cast(trimmedSpirv->getPointer()); - const auto trimmedSpirvDwordCount = trimmedSpirv->getSize() / 4; - logger.log("After stripping capabilities:", nbl::system::ILogger::ELL_DEBUG); - printCapabilities(trimmedSpirvBuffer, trimmedSpirvDwordCount, logger); - logger.log("\n", nbl::system::ILogger::ELL_DEBUG); -#endif - return { .spirv = std::move(trimmedSpirv), .isSuccess = true, From 9515bdda3296bab56af2b34773b385959dceb251 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Mon, 23 Mar 2026 22:03:10 +0100 Subject: [PATCH 11/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 25d3f00972..3b9b2f32ca 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 25d3f00972e786e4641a06a1abe2a65f952743d6 +Subproject commit 3b9b2f32cab4be51007e3b5874a5be2089396bf1 From 939de4f02438064c6a8b01b1e86572de5cee6adc Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 06:20:17 +0100 Subject: [PATCH 12/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 3b9b2f32ca..f966e190b1 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 3b9b2f32cab4be51007e3b5874a5be2089396bf1 +Subproject commit f966e190b1b0a7c54a28f5a36035c206ba568e99 From dd5180b0a97c1fdab819a885f8e2c40973bb2e33 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 07:23:13 +0100 Subject: [PATCH 13/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index f966e190b1..6b5ff686fb 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit f966e190b1b0a7c54a28f5a36035c206ba568e99 +Subproject commit 6b5ff686fbb5d2f2c5aecf988398e407f4814e60 From 8d3e66d2e79865c594824a1c8c1778d919a6e790 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 07:30:57 +0100 Subject: [PATCH 14/37] Trim manifest whitespace and update examples pointer --- examples_tests | 2 +- .../exe/tools/nsc/bin/build-info.json.dvc | 4 ++-- .../nsc-windows-x64-release/exe/tools/nsc/bin/nsc.exe.dvc | 2 +- .../runtime/nbl/3rdparty/dxc/dxcompiler.dll.dvc | 4 ++-- .../nsc-windows-x64-release/runtime/nbl/Nabla.dll.dvc | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples_tests b/examples_tests index 6b5ff686fb..200cb4a670 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 6b5ff686fbb5d2f2c5aecf988398e407f4814e60 +Subproject commit 200cb4a670b69667d03ce7f58427016e3a7054c3 diff --git a/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/build-info.json.dvc b/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/build-info.json.dvc index aa682377ac..0b3eb66f37 100644 --- a/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/build-info.json.dvc +++ b/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/build-info.json.dvc @@ -1,5 +1,5 @@ outs: -- md5: 0313edc75aedd79a1ec94ab90feb5881 - size: 1326 +- md5: 0313edc75aedd79a1ec94ab90feb5881 + size: 1326 hash: md5 path: build-info.json diff --git a/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/nsc.exe.dvc b/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/nsc.exe.dvc index 38b1766b5a..11c5dee4da 100644 --- a/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/nsc.exe.dvc +++ b/tools/nsc/manifests/nsc-windows-x64-release/exe/tools/nsc/bin/nsc.exe.dvc @@ -1,5 +1,5 @@ outs: -- md5: 159813ebd3546457ad48e3a310b3f693 +- md5: 159813ebd3546457ad48e3a310b3f693 size: 256512 hash: md5 path: nsc.exe diff --git a/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/3rdparty/dxc/dxcompiler.dll.dvc b/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/3rdparty/dxc/dxcompiler.dll.dvc index b1f8ff3713..0edd7b4136 100644 --- a/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/3rdparty/dxc/dxcompiler.dll.dvc +++ b/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/3rdparty/dxc/dxcompiler.dll.dvc @@ -1,5 +1,5 @@ outs: -- md5: ec2f20db14ac9272795ba5ad970423b8 - size: 21578752 +- md5: ec2f20db14ac9272795ba5ad970423b8 + size: 21578752 hash: md5 path: dxcompiler.dll diff --git a/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/Nabla.dll.dvc b/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/Nabla.dll.dvc index c74a077ab8..aa5654f0ec 100644 --- a/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/Nabla.dll.dvc +++ b/tools/nsc/manifests/nsc-windows-x64-release/runtime/nbl/Nabla.dll.dvc @@ -1,5 +1,5 @@ outs: -- md5: 219186a91cf612f2ab0f9dd22278206f - size: 29173760 +- md5: 219186a91cf612f2ab0f9dd22278206f + size: 29173760 hash: md5 path: Nabla.dll From cba61133a3bf969e49e1e9009ba8e721877089f7 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 08:52:19 +0100 Subject: [PATCH 15/37] Clean up shader review leftovers --- .../builtin/hlsl/path_tracing/concepts.hlsl | 15 ---------- src/nbl/asset/utils/CWaveStringResolver.cpp | 29 ------------------- src/nbl/asset/utils/waveContext.h | 7 ++++- 3 files changed, 6 insertions(+), 45 deletions(-) diff --git a/include/nbl/builtin/hlsl/path_tracing/concepts.hlsl b/include/nbl/builtin/hlsl/path_tracing/concepts.hlsl index 24a0e86455..140a800c81 100644 --- a/include/nbl/builtin/hlsl/path_tracing/concepts.hlsl +++ b/include/nbl/builtin/hlsl/path_tracing/concepts.hlsl @@ -80,21 +80,6 @@ NBL_CONCEPT_END( #undef ray #include -#define NBL_CONCEPT_NAME RaySetInteraction -#define NBL_CONCEPT_TPLT_PRM_KINDS (typename)(typename) -#define NBL_CONCEPT_TPLT_PRM_NAMES (RayT)(Interaction) -#define NBL_CONCEPT_PARAM_0 (ray, RayT) -#define NBL_CONCEPT_PARAM_1 (interaction, Interaction) -NBL_CONCEPT_BEGIN(2) -#define ray NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_0 -#define interaction NBL_CONCEPT_PARAM_T NBL_CONCEPT_PARAM_1 -NBL_CONCEPT_END( - ((NBL_CONCEPT_REQ_EXPR_RET_TYPE)((ray.setInteraction(interaction)), ::nbl::hlsl::is_same_v, void)) -); -#undef interaction -#undef ray -#include - #define NBL_CONCEPT_NAME RayGenerator #define NBL_CONCEPT_TPLT_PRM_KINDS (typename) #define NBL_CONCEPT_TPLT_PRM_NAMES (T) diff --git a/src/nbl/asset/utils/CWaveStringResolver.cpp b/src/nbl/asset/utils/CWaveStringResolver.cpp index 744c103818..29ddf4ce55 100644 --- a/src/nbl/asset/utils/CWaveStringResolver.cpp +++ b/src/nbl/asset/utils/CWaveStringResolver.cpp @@ -8,35 +8,6 @@ options remain and there is no mismatch, we force agressive inlining and optimizations mostly regardless build configuration by default */ -/* - Arek leaving thoughts, TODO: - - in NBL_WAVE_STRING_RESOLVER_TU_DEBUG_OPTIMISATION mode enabled -> here in this TU do - - #define _ITERATOR_DEBUG_LEVEL 0 - #define _HAS_ITERATOR_DEBUGGING 0 - - and allow Nabla to mismatch debug iterator *on purpose* by - - #define _ALLOW_ITERATOR_DEBUG_LEVEL_MISMATCH - - in Debug/RWDI - - then make preprocess full C API with raw in/out pointers and bytes out pointer, - with mismtach we must be very careful about memory ownership as STL stuff will have - different struct layouts and its easy to make a crash, we will have extra memcpy and - deallocation but as a trade each config will have almost the same preprocessing perf - which matters for our NSC integration - - then we can think to make use of existing shader cache and maybe consider HLSL PCH - which NSC would inject into each input - - NOTE: this approach allows to do all in single Nabla module, no extra proxy/fake shared DLL needed! - NOTE: yep I know I have currently a callback for which context size will differ accross TUs afterwards but will think about it - - or ignore it and take care of NSC special target creating global HLSL PCH injected into each registered input -*/ - #include "nabla.h" #include #include diff --git a/src/nbl/asset/utils/waveContext.h b/src/nbl/asset/utils/waveContext.h index 4b9a263add..147d6d53e0 100644 --- a/src/nbl/asset/utils/waveContext.h +++ b/src/nbl/asset/utils/waveContext.h @@ -403,7 +403,12 @@ class context : private boost::noncopyable , current_filename(fname) , current_relative_filename(fname) , macros(*this_()) - , language(detail::make_language_flags(detail::LanguageFlagConfig{.preserveComments = hooks_.m_preserveComments})) + , language([&hooks_] + { + auto config = detail::LanguageFlagConfig{}; + config.preserveComments = hooks_.m_preserveComments; + return detail::make_language_flags(config); + }()) , hooks(hooks_) { macros.init_predefined_macros(fname); From 476a5bfb3012b5814675784a9e34fbf09602690d Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 08:55:07 +0100 Subject: [PATCH 16/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 200cb4a670..b78500cba0 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 200cb4a670b69667d03ce7f58427016e3a7054c3 +Subproject commit b78500cba07327e9d3f6219be6a42c8b7c44ee35 From d986945228c2a50fb88390aa2fdfbbd0da054ffe Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 10:25:24 +0100 Subject: [PATCH 17/37] Cache validated SPIR-V hashes --- examples_tests | 2 +- .../nbl/asset/utils/ISPIRVEntryPointTrimmer.h | 5 ++ .../asset/utils/ISPIRVEntryPointTrimmer.cpp | 62 ++++++++++++++++--- 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/examples_tests b/examples_tests index b78500cba0..1a64e825a8 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit b78500cba07327e9d3f6219be6a42c8b7c44ee35 +Subproject commit 1a64e825a85eb4fe44176caa61bbb8743683107c diff --git a/include/nbl/asset/utils/ISPIRVEntryPointTrimmer.h b/include/nbl/asset/utils/ISPIRVEntryPointTrimmer.h index 1a1f6b857c..1ccde512a6 100644 --- a/include/nbl/asset/utils/ISPIRVEntryPointTrimmer.h +++ b/include/nbl/asset/utils/ISPIRVEntryPointTrimmer.h @@ -7,6 +7,8 @@ #include "nbl/system/ILogger.h" +#include + namespace nbl::asset { @@ -46,6 +48,7 @@ class NBL_API2 ISPIRVEntryPointTrimmer final : public core::IReferenceCounted }; Result trim(const ICPUBuffer* spirvBuffer, const core::set& entryPoints, system::logger_opt_ptr logger = nullptr) const; + bool ensureValidated(const ICPUBuffer* spirvBuffer, system::logger_opt_ptr logger = nullptr) const; inline core::smart_refctd_ptr trim(const IShader* shader, const core::set& entryPoints, system::logger_opt_ptr logger = nullptr) const { @@ -72,6 +75,8 @@ class NBL_API2 ISPIRVEntryPointTrimmer final : public core::IReferenceCounted private: core::smart_refctd_ptr m_optimizer; + mutable std::mutex m_validationCacheMutex; + mutable core::unordered_set m_validatedSpirvHashes; }; } diff --git a/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp b/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp index 3ee3f18e35..c6503e3481 100644 --- a/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp +++ b/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp @@ -8,7 +8,9 @@ using namespace nbl::asset; -static constexpr spv_target_env SPIRV_VERSION = spv_target_env::SPV_ENV_UNIVERSAL_1_6; +// Why are we validating Universal instead of a Vulkan environment? +// Trimming works on generic SPIR-V before the Vulkan backend chooses its environment. +static constexpr spv_target_env SPIRV_VALIDATION_ENV = spv_target_env::SPV_ENV_UNIVERSAL_1_6; ISPIRVEntryPointTrimmer::ISPIRVEntryPointTrimmer() { @@ -55,7 +57,7 @@ static bool validate(const uint32_t* binary, uint32_t binarySize, nbl::system::l logger.log(location, lvl, msg); }; - spvtools::SpirvTools core(SPIRV_VERSION); + spvtools::SpirvTools core(SPIRV_VALIDATION_ENV); core.SetMessageConsumer(msgConsumer); spvtools::ValidatorOptions validatorOptions; // Nabla use Scalar block layout, we skip this validation to work around this and to save time @@ -63,6 +65,31 @@ static bool validate(const uint32_t* binary, uint32_t binarySize, nbl::system::l return core.Validate(binary, binarySize, validatorOptions); } +bool ISPIRVEntryPointTrimmer::ensureValidated(const ICPUBuffer* spirvBuffer, system::logger_opt_ptr logger) const +{ + auto contentHash = spirvBuffer->getContentHash(); + if (contentHash == ICPUBuffer::INVALID_HASH) + contentHash = spirvBuffer->computeContentHash(); + + { + std::lock_guard lock(m_validationCacheMutex); + if (m_validatedSpirvHashes.contains(contentHash)) + return true; + } + + const auto* spirv = static_cast(spirvBuffer->getPointer()); + const auto spirvDwordCount = spirvBuffer->getSize() / 4u; + if (!validate(spirv, spirvDwordCount, logger)) + return false; + + { + std::lock_guard lock(m_validationCacheMutex); + m_validatedSpirvHashes.insert(contentHash); + } + + return true; +} + ISPIRVEntryPointTrimmer::Result ISPIRVEntryPointTrimmer::trim(const ICPUBuffer* spirvBuffer, const core::set& entryPoints, system::logger_opt_ptr logger) const { const auto* spirv = static_cast(spirvBuffer->getPointer()); @@ -175,18 +202,17 @@ ISPIRVEntryPointTrimmer::Result ISPIRVEntryPointTrimmer::trim(const ICPUBuffer* } } - auto foundEntryPoint = 0; - - const bool isInputSpirvValid = validate(spirv, spirvDwordCount, logger); - if (!isInputSpirvValid) + if (!ensureValidated(spirvBuffer, logger)) { logger.log("SPIR-V is not valid", system::ILogger::ELL_ERROR); return Result{ - nullptr, - false + .spirv = nullptr, + .isSuccess = false, }; } + auto foundEntryPoint = 0; + // Keep in mind about this layout while reading all the code below: https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#LogicalLayout // skip until entry point @@ -278,9 +304,25 @@ ISPIRVEntryPointTrimmer::Result ISPIRVEntryPointTrimmer::trim(const ICPUBuffer* minimizedSpirv.insert(minimizedSpirv.end(), spirv + offset, spirv + spirvDwordCount); - assert(validate(minimizedSpirv.data(), minimizedSpirv.size(), logger)); - auto trimmedSpirv = m_optimizer->optimize(minimizedSpirv.data(), minimizedSpirv.size(), logger); + if (!trimmedSpirv) + { + logger.log("Failed to optimize trimmed SPIR-V", system::ILogger::ELL_ERROR); + return { + .spirv = nullptr, + .isSuccess = false, + }; + } + + trimmedSpirv->setContentHash(trimmedSpirv->computeContentHash()); + if (!ensureValidated(trimmedSpirv.get(), logger)) + { + logger.log("Trimmed SPIR-V is not valid", system::ILogger::ELL_ERROR); + return { + .spirv = nullptr, + .isSuccess = false, + }; + } return { .spirv = std::move(trimmedSpirv), From 5ecde9af1a2bc50462dcf4b8bf0cc1b8d1745d6d Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 11:28:31 +0100 Subject: [PATCH 18/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 1a64e825a8..db1e1cf679 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 1a64e825a85eb4fe44176caa61bbb8743683107c +Subproject commit db1e1cf679d4d74423d4cfc7edf9134e06f39239 From 1ede3de70723eccdd22d58f98e404e63d0bdc105 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 12:33:55 +0100 Subject: [PATCH 19/37] Tighten final shader cleanup --- examples_tests | 2 +- src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples_tests b/examples_tests index db1e1cf679..20587db819 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit db1e1cf679d4d74423d4cfc7edf9134e06f39239 +Subproject commit 20587db819f8ab19554f2872c7a44149c99f9fa4 diff --git a/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp b/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp index c6503e3481..8ec28f5222 100644 --- a/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp +++ b/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp @@ -78,7 +78,7 @@ bool ISPIRVEntryPointTrimmer::ensureValidated(const ICPUBuffer* spirvBuffer, sys } const auto* spirv = static_cast(spirvBuffer->getPointer()); - const auto spirvDwordCount = spirvBuffer->getSize() / 4u; + const auto spirvDwordCount = spirvBuffer->getSize() / sizeof(uint32_t); if (!validate(spirv, spirvDwordCount, logger)) return false; @@ -93,7 +93,7 @@ bool ISPIRVEntryPointTrimmer::ensureValidated(const ICPUBuffer* spirvBuffer, sys ISPIRVEntryPointTrimmer::Result ISPIRVEntryPointTrimmer::trim(const ICPUBuffer* spirvBuffer, const core::set& entryPoints, system::logger_opt_ptr logger) const { const auto* spirv = static_cast(spirvBuffer->getPointer()); - const auto spirvDwordCount = spirvBuffer->getSize() / 4; + const auto spirvDwordCount = spirvBuffer->getSize() / sizeof(uint32_t); if (entryPoints.empty()) { From 758f7c8a43d481ce1a99fb005eaafc4b0457a509 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 12:42:40 +0100 Subject: [PATCH 20/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 20587db819..63a0cb9351 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 20587db819f8ab19554f2872c7a44149c99f9fa4 +Subproject commit 63a0cb935125bbe7005d9ece342f4df479914ebb From 8745660c8aaffdbeee465896bc2ea9f6ba3cea47 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 17:23:26 +0100 Subject: [PATCH 21/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 63a0cb9351..4a68db651f 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 63a0cb935125bbe7005d9ece342f4df479914ebb +Subproject commit 4a68db651f95cb1f9785ada468d4a515d3f9a8f0 From 94a501fc1dfd86c6855b9ee11eb0d6fc4963cee3 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 17:47:41 +0100 Subject: [PATCH 22/37] Mark generated NSC headers correctly --- cmake/common.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/common.cmake b/cmake/common.cmake index e70994dcdb..041904471c 100755 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1308,6 +1308,7 @@ struct DeviceConfigCaps target_sources(${IMPL_TARGET} PUBLIC ${INCLUDE_FILE}) set_source_files_properties(${INCLUDE_FILE} PROPERTIES + GENERATED ON HEADER_FILE_ONLY ON VS_TOOL_OVERRIDE None ) From b1f28c08c794dbba93ff1afdef04bb6f2d540e28 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 18:32:43 +0100 Subject: [PATCH 23/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 4a68db651f..704ef5fa6c 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 4a68db651f95cb1f9785ada468d4a515d3f9a8f0 +Subproject commit 704ef5fa6cd5e2e749b5001c2d586c280fa00311 From 4b444b6b0c4db0b907692540260972ff8ddf93f3 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 19:44:14 +0100 Subject: [PATCH 24/37] Revert "Mark generated NSC headers correctly" This reverts commit 94a501fc1dfd86c6855b9ee11eb0d6fc4963cee3. --- cmake/common.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/common.cmake b/cmake/common.cmake index 041904471c..e70994dcdb 100755 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -1308,7 +1308,6 @@ struct DeviceConfigCaps target_sources(${IMPL_TARGET} PUBLIC ${INCLUDE_FILE}) set_source_files_properties(${INCLUDE_FILE} PROPERTIES - GENERATED ON HEADER_FILE_ONLY ON VS_TOOL_OVERRIDE None ) From f4b0aedbdce7919bf3bfcb10f21659f4ba507185 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 19:44:25 +0100 Subject: [PATCH 25/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 704ef5fa6c..3fd1ae6640 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 704ef5fa6cd5e2e749b5001c2d586c280fa00311 +Subproject commit 3fd1ae6640f0fcd06371afadf9b345ac56077b27 From 01794c57ee30d267a91cd25e82e9cbab9edd70f5 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 21:15:59 +0100 Subject: [PATCH 26/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 3fd1ae6640..bfaa4a2b57 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 3fd1ae6640f0fcd06371afadf9b345ac56077b27 +Subproject commit bfaa4a2b57ce3207bc5ea384194137c030ee4420 From 4a0c2e24ec86361e3a357f78ca5a1160166a3f57 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 22:11:47 +0100 Subject: [PATCH 27/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index bfaa4a2b57..615ae45d25 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit bfaa4a2b57ce3207bc5ea384194137c030ee4420 +Subproject commit 615ae45d25d7b2aea584659e3f96255692fe7347 From 02f04db2b265cdd8cf0626af61b52b83f5f00c74 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Tue, 24 Mar 2026 23:23:29 +0100 Subject: [PATCH 28/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 615ae45d25..caaf16ce78 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 615ae45d25d7b2aea584659e3f96255692fe7347 +Subproject commit caaf16ce78086ae5854b5e78da9d06ab758106c5 From 3ae2b26254edc09c1e1ef95fb1d7e05e06875bc1 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Wed, 25 Mar 2026 00:59:15 +0100 Subject: [PATCH 29/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index caaf16ce78..079b6e4664 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit caaf16ce78086ae5854b5e78da9d06ab758106c5 +Subproject commit 079b6e4664459785e9913a72b37245976d601373 From fcae99102a97e50b0a865677dc6d91d77af13a3c Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Wed, 25 Mar 2026 01:27:20 +0100 Subject: [PATCH 30/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 079b6e4664..fc1ae322fd 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 079b6e4664459785e9913a72b37245976d601373 +Subproject commit fc1ae322fd43b867ff183ec58d24c328f48f5581 From c8af81b2f261f9e653f04545dd4792b0730af00f Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Wed, 25 Mar 2026 01:58:32 +0100 Subject: [PATCH 31/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index fc1ae322fd..8424767aad 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit fc1ae322fd43b867ff183ec58d24c328f48f5581 +Subproject commit 8424767aad75c5d6fd4ffa15a79a983ce3f53734 From 3541a9d24d57132609f0291f4cef89cf08827007 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Wed, 25 Mar 2026 05:26:17 +0100 Subject: [PATCH 32/37] Update path tracer examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 8424767aad..018757b8f7 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 8424767aad75c5d6fd4ffa15a79a983ce3f53734 +Subproject commit 018757b8f7031922b181e3aaae4453d7ec02863e From 87237712ffb45b08ee3af3dc79c9d40468aef14d Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Wed, 25 Mar 2026 06:14:22 +0100 Subject: [PATCH 33/37] Validate SPIR-V once per blob --- examples_tests | 2 +- .../nbl/asset/utils/ISPIRVEntryPointTrimmer.h | 1 + .../asset/utils/ISPIRVEntryPointTrimmer.cpp | 34 +++++++++++++------ 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/examples_tests b/examples_tests index 018757b8f7..a3ef3600f3 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 018757b8f7031922b181e3aaae4453d7ec02863e +Subproject commit a3ef3600f3afa7ad2caefb1ab7b36a997ea9e1af diff --git a/include/nbl/asset/utils/ISPIRVEntryPointTrimmer.h b/include/nbl/asset/utils/ISPIRVEntryPointTrimmer.h index 1ccde512a6..fceea0aac3 100644 --- a/include/nbl/asset/utils/ISPIRVEntryPointTrimmer.h +++ b/include/nbl/asset/utils/ISPIRVEntryPointTrimmer.h @@ -49,6 +49,7 @@ class NBL_API2 ISPIRVEntryPointTrimmer final : public core::IReferenceCounted Result trim(const ICPUBuffer* spirvBuffer, const core::set& entryPoints, system::logger_opt_ptr logger = nullptr) const; bool ensureValidated(const ICPUBuffer* spirvBuffer, system::logger_opt_ptr logger = nullptr) const; + void markValidated(const ICPUBuffer* spirvBuffer) const; inline core::smart_refctd_ptr trim(const IShader* shader, const core::set& entryPoints, system::logger_opt_ptr logger = nullptr) const { diff --git a/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp b/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp index 8ec28f5222..8c3b72e08b 100644 --- a/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp +++ b/src/nbl/asset/utils/ISPIRVEntryPointTrimmer.cpp @@ -65,11 +65,17 @@ static bool validate(const uint32_t* binary, uint32_t binarySize, nbl::system::l return core.Validate(binary, binarySize, validatorOptions); } -bool ISPIRVEntryPointTrimmer::ensureValidated(const ICPUBuffer* spirvBuffer, system::logger_opt_ptr logger) const +static nbl::core::blake3_hash_t getContentHash(const ICPUBuffer* spirvBuffer) { auto contentHash = spirvBuffer->getContentHash(); if (contentHash == ICPUBuffer::INVALID_HASH) contentHash = spirvBuffer->computeContentHash(); + return contentHash; +} + +bool ISPIRVEntryPointTrimmer::ensureValidated(const ICPUBuffer* spirvBuffer, system::logger_opt_ptr logger) const +{ + const auto contentHash = getContentHash(spirvBuffer); { std::lock_guard lock(m_validationCacheMutex); @@ -84,12 +90,18 @@ bool ISPIRVEntryPointTrimmer::ensureValidated(const ICPUBuffer* spirvBuffer, sys { std::lock_guard lock(m_validationCacheMutex); - m_validatedSpirvHashes.insert(contentHash); + m_validatedSpirvHashes.emplace(contentHash); } return true; } +void ISPIRVEntryPointTrimmer::markValidated(const ICPUBuffer* spirvBuffer) const +{ + std::lock_guard lock(m_validationCacheMutex); + m_validatedSpirvHashes.emplace(getContentHash(spirvBuffer)); +} + ISPIRVEntryPointTrimmer::Result ISPIRVEntryPointTrimmer::trim(const ICPUBuffer* spirvBuffer, const core::set& entryPoints, system::logger_opt_ptr logger) const { const auto* spirv = static_cast(spirvBuffer->getPointer()); @@ -143,6 +155,15 @@ ISPIRVEntryPointTrimmer::Result ISPIRVEntryPointTrimmer::trim(const ICPUBuffer* return { length, opcode }; }; + if (!ensureValidated(spirvBuffer, logger)) + { + logger.log("SPIR-V is not valid", system::ILogger::ELL_ERROR); + return Result{ + .spirv = nullptr, + .isSuccess = false, + }; + } + { auto probeOffset = HEADER_SIZE; auto totalEntryPoints = 0u; @@ -202,15 +223,6 @@ ISPIRVEntryPointTrimmer::Result ISPIRVEntryPointTrimmer::trim(const ICPUBuffer* } } - if (!ensureValidated(spirvBuffer, logger)) - { - logger.log("SPIR-V is not valid", system::ILogger::ELL_ERROR); - return Result{ - .spirv = nullptr, - .isSuccess = false, - }; - } - auto foundEntryPoint = 0; // Keep in mind about this layout while reading all the code below: https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#LogicalLayout From 52ae40bf4bdebdd66da32c5d27598dd8344ecaab Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Wed, 25 Mar 2026 06:50:54 +0100 Subject: [PATCH 34/37] Update EX31 examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index a3ef3600f3..339cb3efa1 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit a3ef3600f3afa7ad2caefb1ab7b36a997ea9e1af +Subproject commit 339cb3efa11aa5f5454fdf6f3fc0fb89fa9a28c5 From 6476500519b4378fec94a61e452e4d287ce2c20d Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Wed, 25 Mar 2026 07:17:03 +0100 Subject: [PATCH 35/37] Update EX31 examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 339cb3efa1..04863f9cdd 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 339cb3efa11aa5f5454fdf6f3fc0fb89fa9a28c5 +Subproject commit 04863f9cddcdbcdaea990d8086c2e5200a673b66 From f5f036e488db23b6fdb2fb71e7c198f37447ee37 Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Wed, 25 Mar 2026 09:09:13 +0100 Subject: [PATCH 36/37] Update EX31 examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 04863f9cdd..7b6c1540b1 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 04863f9cddcdbcdaea990d8086c2e5200a673b66 +Subproject commit 7b6c1540b1bac6af168b32419a0102369455961d From e545d3787abe18c8dd31d46865bafb42dca8b13b Mon Sep 17 00:00:00 2001 From: Arkadiusz Lachowicz Date: Wed, 25 Mar 2026 11:46:18 +0100 Subject: [PATCH 37/37] Update EX31 examples pointer --- examples_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples_tests b/examples_tests index 7b6c1540b1..520e26fb52 160000 --- a/examples_tests +++ b/examples_tests @@ -1 +1 @@ -Subproject commit 7b6c1540b1bac6af168b32419a0102369455961d +Subproject commit 520e26fb5225072a8cdc9d47b9c8077c0065f8c2