From df7f7b168f059e7366646d9159b0725910542cdf Mon Sep 17 00:00:00 2001 From: Jake Stevens Date: Tue, 3 Nov 2026 09:46:26 -0800 Subject: [PATCH] Support exclusive raw-logit output dumps (#24505) Summary: Add optional `--dump_output` to the native classification runner for independent full-output validation. Write the original FP32 logits before reference comparison, including mismatches and non-finite outputs, without altering their bits or numerical acceptance. The runner loads `--categories` and `--expected` before creating the dump, so an input error exits 2 without leaving a file, and a missing reference cannot be satisfied by the runner's own dump. As a result, a bad `--expected` path now fails before the top-5 and argmax are printed, even without `--dump_output`. `write_floats` removes an incomplete file when the write or close fails. The LLM runner's `--dump_first_logits` now uses it instead of its own copy of the same logic. This change was written with help from Claude Code. Differential Revision: D129375473 --- .ci/scripts/test_native_llm_runner.py | 12 +- backends/native/README.md | 3 +- .../examples/hf_llm/runtime/LlmRunner.cpp | 17 +-- .../examples/runner_utils/runner_utils.cpp | 37 +++++- .../examples/runner_utils/runner_utils.h | 7 ++ .../runner_utils/test_runner_utils.cpp | 105 ++++++++++++++++++ .../examples/vision/ClassificationRunner.cpp | 16 ++- 7 files changed, 173 insertions(+), 24 deletions(-) diff --git a/.ci/scripts/test_native_llm_runner.py b/.ci/scripts/test_native_llm_runner.py index 9fe9eadf638..db882625c43 100644 --- a/.ci/scripts/test_native_llm_runner.py +++ b/.ci/scripts/test_native_llm_runner.py @@ -67,6 +67,7 @@ def main() -> None: prompt = "abc" unreachable_eos_id = config.vocab_size tokens = [config.bos_token_id, *prompt.encode()] + expected_logits = model(torch.tensor([tokens]), use_cache=False).logits[0, -1] expected = [] # Recompute the full prefix without a cache for an independent reference. for _ in range(max_new_tokens): @@ -75,6 +76,7 @@ def main() -> None: expected.append(token) tokens.append(token) + logits_dump = root / "first_logits.bin" result = subprocess.run( [ str(args.runner.resolve()), @@ -87,6 +89,7 @@ def main() -> None: f"--max_new_tokens={max_new_tokens}", f"--max_seq_length={max_seq_length}", "--print_token_ids", + f"--dump_first_logits={logits_dump}", ], capture_output=True, text=True, @@ -102,7 +105,14 @@ def main() -> None: actual = [int(token) for token in match[1].split(",")] if actual != expected: raise AssertionError(f"Generated tokens {actual} != eager {expected}") - print(f"GGML HF LLM runner matched all {max_new_tokens} eager token IDs") + actual_logits = torch.frombuffer( + bytearray(logits_dump.read_bytes()), dtype=torch.float32 + ) + torch.testing.assert_close(actual_logits, expected_logits, atol=1e-3, rtol=1e-3) + print( + f"GGML HF LLM runner matched all {max_new_tokens} eager token IDs " + f"and {config.vocab_size} initial logits" + ) if __name__ == "__main__": diff --git a/backends/native/README.md b/backends/native/README.md index 9ce1b69d277..4237f161705 100644 --- a/backends/native/README.md +++ b/backends/native/README.md @@ -217,7 +217,8 @@ runs `native_cpu_engine_test` and builds `run_classification_cpu`. The GGML job also runs `native_llm_runner_utils_test` and executes `run_hf_llm_ggml` on a tiny seeded Hugging Face Llama with a generated byte tokenizer. Its eight generated token IDs must match eager PyTorch, covering prefill and KV-cache decoding without -downloading a model. The GGML job requires the Python export dependencies +downloading a model, and its `--dump_first_logits` output must match the eager +prefill logits within 1e-3. The GGML job requires the Python export dependencies installed by the `Test Native Engines` workflow. The Vulkan job builds `run_classification` and `run_hf_llm_vulkan`; its tests do not require a GPU. diff --git a/backends/native/examples/hf_llm/runtime/LlmRunner.cpp b/backends/native/examples/hf_llm/runtime/LlmRunner.cpp index b1efd5d90f1..280f139363a 100644 --- a/backends/native/examples/hf_llm/runtime/LlmRunner.cpp +++ b/backends/native/examples/hf_llm/runtime/LlmRunner.cpp @@ -202,20 +202,9 @@ Generation generate( executable.get_output( logits_output, logits.data(), logits.size(), ptn::kFloat); if (position == 0 && !FLAGS_dump_first_logits.empty()) { - FILE* output = std::fopen(FLAGS_dump_first_logits.c_str(), "wbx"); - if (output == nullptr) { - throw std::runtime_error( - "cannot create new logits dump: " + FLAGS_dump_first_logits); - } - const float* row = logits.data() + logits_numel - vocabulary_size; - const size_t written = - std::fwrite(row, sizeof(float), vocabulary_size, output); - const int close_status = std::fclose(output); - if (written != vocabulary_size || close_status != 0) { - std::remove(FLAGS_dump_first_logits.c_str()); - throw std::runtime_error( - "failed to write complete initial logits dump"); - } + ptn::runner_utils::write_floats( + FLAGS_dump_first_logits, + {logits.data() + logits_numel - vocabulary_size, vocabulary_size}); } if (FLAGS_print_top_logits) { const float* row = logits.data() + logits_numel - vocabulary_size; diff --git a/backends/native/examples/runner_utils/runner_utils.cpp b/backends/native/examples/runner_utils/runner_utils.cpp index 30fb140acd1..fe054ae01c6 100644 --- a/backends/native/examples/runner_utils/runner_utils.cpp +++ b/backends/native/examples/runner_utils/runner_utils.cpp @@ -35,6 +35,20 @@ std::vector read_floats(const std::string& path) { return out; } +void write_floats(const std::string& path, std::span values) { + FILE* file = std::fopen(path.c_str(), "wbx"); + if (!file) { + throw std::runtime_error("cannot create " + path); + } + const size_t written = + std::fwrite(values.data(), sizeof(float), values.size(), file); + const int closed = std::fclose(file); + if (written != values.size() || closed != 0) { + std::remove(path.c_str()); + throw std::runtime_error("cannot write " + path); + } +} + std::vector read_lines(const std::string& path) { std::vector lines; std::ifstream file(path); @@ -73,9 +87,9 @@ std::vector top_k(const std::vector& values, int k) { return order; } -bool compare_floats( - const std::vector& actual, - const std::vector& expected, +bool can_compare_floats( + size_t actual_size, + size_t expected_size, double rtol, double atol) { if (!std::isfinite(rtol) || !std::isfinite(atol) || rtol < 0 || atol < 0) { @@ -87,12 +101,23 @@ bool compare_floats( atol); return false; } - if (actual.size() != expected.size()) { + if (actual_size != expected_size) { std::fprintf( stderr, "error: reference has %zu values, model produced %zu\n", - expected.size(), - actual.size()); + expected_size, + actual_size); + return false; + } + return true; +} + +bool compare_floats( + const std::vector& actual, + const std::vector& expected, + double rtol, + double atol) { + if (!can_compare_floats(actual.size(), expected.size(), rtol, atol)) { return false; } size_t failures = 0; diff --git a/backends/native/examples/runner_utils/runner_utils.h b/backends/native/examples/runner_utils/runner_utils.h index a4dc7a4b73c..3fcd5836df4 100644 --- a/backends/native/examples/runner_utils/runner_utils.h +++ b/backends/native/examples/runner_utils/runner_utils.h @@ -7,18 +7,25 @@ #pragma once #include +#include #include #include namespace ptn::runner_utils { std::vector read_floats(const std::string& path); +void write_floats(const std::string& path, std::span values); std::vector read_lines(const std::string& path); int64_t numel_of(const std::vector& sizes); std::string shape_str(const std::vector& sizes); std::vector top_k(const std::vector& values, int k); +bool can_compare_floats( + size_t actual_size, + size_t expected_size, + double rtol, + double atol); bool compare_floats( const std::vector& actual, const std::vector& expected, diff --git a/backends/native/examples/runner_utils/test_runner_utils.cpp b/backends/native/examples/runner_utils/test_runner_utils.cpp index c4ce576660f..419ce958993 100644 --- a/backends/native/examples/runner_utils/test_runner_utils.cpp +++ b/backends/native/examples/runner_utils/test_runner_utils.cpp @@ -6,15 +6,120 @@ #include +#include +#include +#include +#include #include +#include +#include +#include #include #include +#if !defined(_WIN32) +#include +#endif + namespace ptn::runner_utils { namespace { +class RunnerOutputTest : public ::testing::Test { + protected: + void SetUp() override { + directory_ = std::filesystem::path(::testing::TempDir()) / + ("runner_output_" + std::to_string(std::random_device{}())); + owns_directory_ = std::filesystem::create_directory(directory_); + ASSERT_TRUE(owns_directory_); + path_ = (directory_ / "output.bin").string(); + } + + void TearDown() override { + if (owns_directory_) { + std::filesystem::remove_all(directory_); + } + } + + std::filesystem::path directory_; + std::string path_; + bool owns_directory_ = false; +}; + // cppcheck-suppress-begin syntaxError +TEST_F(RunnerOutputTest, WriteFloatsPreservesAllBits) { + const std::vector bits{ + 0x00000000, 0x80000000, 0x3f800000, 0xc0400000, 0x7f800000, 0x7fc12345}; + std::vector values; + values.reserve(bits.size()); + for (const uint32_t value : bits) { + values.push_back(std::bit_cast(value)); + } + + write_floats(path_, values); + + EXPECT_EQ(std::filesystem::file_size(path_), bits.size() * sizeof(float)); + std::vector actual; + for (const float value : read_floats(path_)) { + actual.push_back(std::bit_cast(value)); + } + EXPECT_EQ(actual, bits); +} + +TEST_F(RunnerOutputTest, WriteFloatsDoesNotOverwriteExistingFile) { + const std::vector original{1.0f, -3.0f}; + write_floats(path_, original); + + EXPECT_THROW( + write_floats(path_, std::vector{4.0f}), std::runtime_error); + + EXPECT_EQ(read_floats(path_), original); +} + +TEST_F(RunnerOutputTest, WriteFloatsCreatesEmptyOutput) { + write_floats(path_, {}); + + EXPECT_EQ(std::filesystem::file_size(path_), 0); + EXPECT_TRUE(read_floats(path_).empty()); +} + +TEST_F(RunnerOutputTest, WriteFloatsReportsInvalidOutputPath) { + const std::string missing = (directory_ / "missing" / "output.bin").string(); + + EXPECT_THROW( + write_floats(missing, std::vector{1.0f}), std::runtime_error); +} + +#if !defined(_WIN32) +TEST_F(RunnerOutputTest, WriteFloatsRemovesIncompleteOutput) { + const std::vector values(1024, 1.0f); + rlimit original{}; + ASSERT_EQ(getrlimit(RLIMIT_FSIZE, &original), 0); + rlimit limited = original; + limited.rlim_cur = sizeof(float); + const auto previous_handler = std::signal(SIGXFSZ, SIG_IGN); + ASSERT_EQ(setrlimit(RLIMIT_FSIZE, &limited), 0); + + bool threw = false; + try { + write_floats(path_, values); + } catch (const std::runtime_error&) { + threw = true; + } + setrlimit(RLIMIT_FSIZE, &original); + std::signal(SIGXFSZ, previous_handler); + + EXPECT_TRUE(threw); + EXPECT_FALSE(std::filesystem::exists(path_)); +} +#endif + +TEST(RunnerUtilsTest, CanCompareFloatsChecksSizesAndTolerances) { + EXPECT_TRUE(can_compare_floats(2, 2, 1e-3, 0.0)); + EXPECT_FALSE(can_compare_floats(1, 2, 1e-3, 1e-3)); + EXPECT_FALSE(can_compare_floats(1, 1, -1e-3, 1e-3)); +} + TEST(RunnerUtilsTest, CompareFloatsUsesAbsoluteAndRelativeTolerance) { EXPECT_TRUE(compare_floats({1.0f, 100.0f}, {1.0005f, 100.05f}, 1e-3, 1e-3)); EXPECT_FALSE(compare_floats({1.0f, 100.0f}, {1.1f, 100.0f}, 1e-3, 1e-3)); diff --git a/backends/native/examples/vision/ClassificationRunner.cpp b/backends/native/examples/vision/ClassificationRunner.cpp index 298ea1cbd50..339251a86ab 100644 --- a/backends/native/examples/vision/ClassificationRunner.cpp +++ b/backends/native/examples/vision/ClassificationRunner.cpp @@ -28,6 +28,7 @@ DEFINE_string( "Reference logits to compare against, raw float32."); DEFINE_string(categories, "", "Newline-separated class labels, for display."); DEFINE_string(method, "", "Method to run. Defaults to the first one."); +DEFINE_string(dump_output, "", "Write raw float32 logits to a new file."); DEFINE_double(rtol, 1e-3, "Relative tolerance for the logits comparison."); DEFINE_double(atol, 1e-3, "Absolute tolerance for the logits comparison."); namespace { @@ -57,6 +58,7 @@ void print_top5( namespace ptn { +// cppcheck-suppress unusedFunction int run_classification( const std::function()>& make_host, const std::function& constant_bytes) { @@ -156,6 +158,18 @@ int run_classification( if (!FLAGS_categories.empty()) { labels = ptn::runner_utils::read_lines(FLAGS_categories); } + const std::vector reference = FLAGS_expected.empty() + ? std::vector{} + : ptn::runner_utils::read_floats(FLAGS_expected); + if (!FLAGS_expected.empty() && + !ptn::runner_utils::can_compare_floats( + logits.size(), reference.size(), FLAGS_rtol, FLAGS_atol)) { + return 2; + } + if (!FLAGS_dump_output.empty()) { + ptn::runner_utils::write_floats(FLAGS_dump_output, logits); + } + print_top5(logits, labels); const int argmax = ptn::runner_utils::top_k(logits, 1).front(); std::printf( @@ -165,8 +179,6 @@ int run_classification( std::printf("result: ran (no reference given)\n"); return 0; } - const std::vector reference = - ptn::runner_utils::read_floats(FLAGS_expected); const bool pass = ptn::runner_utils::compare_floats( logits, reference, FLAGS_rtol, FLAGS_atol); std::printf("result: %s\n", pass ? "PASS" : "FAIL");