From b0d522f6b03287c99cbff03f091a75e576651bae Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 13:30:15 +0000 Subject: [PATCH] Ingest string-bench results into the compression tables string-bench's Vortex file suite now writes v3 ingest records via a new --ingest-jsonl flag, and develop-bench ingests them into the existing compression_sizes and compression_times tables. Each (column, encoder) run emits one size row and encode (write) / decode (read) timing rows, with dataset `strings/` and the encoder in dataset_variant so the rows never collide with compress-bench keys. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniBE69Jeaf1h8ZkLq7QB Signed-off-by: Joe Isaacs --- .github/workflows/develop-bench.yml | 4 +-- benchmarks/string-bench/src/main.rs | 12 ++++++++ benchmarks/string-bench/src/serialized.rs | 37 +++++++++++++++++++++++ scripts/string-split.py | 3 ++ 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/.github/workflows/develop-bench.yml b/.github/workflows/develop-bench.yml index 7776f16f4f3..87d4c21cf0f 100644 --- a/.github/workflows/develop-bench.yml +++ b/.github/workflows/develop-bench.yml @@ -90,7 +90,7 @@ jobs: v4_ingest: true - id: string-bench name: String Encoding - v4_ingest: false + v4_ingest: true steps: - uses: runs-on/action@v2 if: github.repository == 'vortex-data/vortex' @@ -200,8 +200,6 @@ jobs: # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live # benchmarks website for suites that emit v4 records; a failure here fails the job. - # string-bench does not emit results.ingest.jsonl, so these steps are disabled for it - # and its results are not stored. # Also gated on the ingest-role ARN var (the assume-role input that MUST exist for # OIDC to succeed). post-ingest.py mints the RDS IAM token internally (boto3) from # the assumed GitHubBenchmarkIngestRole; sslmode=verify-full validates the cert. diff --git a/benchmarks/string-bench/src/main.rs b/benchmarks/string-bench/src/main.rs index 67aa3e491cb..5d3946ef18c 100644 --- a/benchmarks/string-bench/src/main.rs +++ b/benchmarks/string-bench/src/main.rs @@ -128,6 +128,10 @@ struct Args { /// Write output to this file instead of stdout. #[arg(short, long)] output_path: Option, + /// Also write the Vortex file suite results as JSONL ingest records for the + /// benchmarks database to this path. + #[arg(long)] + ingest_jsonl: Option, /// Enable verbose (debug-level) logging. #[arg(short, long)] verbose: bool, @@ -230,6 +234,14 @@ async fn run(args: Args) -> Result<()> { } progress.finish(); + if let Some(path) = &args.ingest_jsonl { + let records: Vec<_> = serialized_results + .iter() + .flat_map(SerializedResult::v3_records) + .collect(); + vortex_bench::v3::write_jsonl_to_path(path, &records)?; + } + let mut writer = create_output_writer(&args.display_format, args.output_path, BENCHMARK_ID)?; match args.display_format { DisplayFormat::Table => { diff --git a/benchmarks/string-bench/src/serialized.rs b/benchmarks/string-bench/src/serialized.rs index 77354b8839f..efb3817552f 100644 --- a/benchmarks/string-bench/src/serialized.rs +++ b/benchmarks/string-bench/src/serialized.rs @@ -38,7 +38,11 @@ use vortex::file::WriteStrategyBuilder; use vortex::layout::LayoutStrategy; use vortex::session::VortexSession; use vortex_bench::Format; +use vortex_bench::compress::CompressOp; +use vortex_bench::measurements::CompressionTimingMeasurement; use vortex_bench::measurements::CustomUnitMeasurement; +use vortex_bench::v3; +use vortex_bench::v3::V3Record; use vortex_btrblocks::Scheme; use vortex_btrblocks::SchemeExt; use vortex_btrblocks::SchemeId; @@ -156,6 +160,39 @@ impl SerializedResult { ms(format!("read/{suffix}"), &self.read_runs), ] } + + /// Emit the size, write (`encode`) and read (`decode`) results as records for the + /// shared `compression_sizes` and `compression_times` tables. + /// + /// The dataset is namespaced under `strings/` and the encoder goes in + /// `dataset_variant`, so these rows never share a key with `compress-bench` rows. + pub fn v3_records(&self) -> Vec { + let dataset = format!("strings/{}", self.name); + let variant = Some(self.encoder.as_str()); + let time = |op, runs: &[Duration]| { + let timing = CompressionTimingMeasurement { + name: format!("{dataset}/{}", self.encoder), + format: Format::OnDiskVortex, + time: median(runs), + }; + let all_runtimes_ns = runs + .iter() + .map(|d| u64::try_from(d.as_nanos()).unwrap_or(u64::MAX)) + .collect(); + v3::compression_time_record(&timing, &dataset, variant, op, all_runtimes_ns) + }; + vec![ + v3::compression_size_record( + &dataset, + variant, + Format::OnDiskVortex, + self.file_bytes, + self.uncompressed_bytes, + ), + time(CompressOp::Compress, &self.write_runs), + time(CompressOp::Decompress, &self.read_runs), + ] + } } /// Build the file writer strategy that forces one configured string scheme while diff --git a/scripts/string-split.py b/scripts/string-split.py index 63f94922fe8..83453ee6564 100755 --- a/scripts/string-split.py +++ b/scripts/string-split.py @@ -59,6 +59,8 @@ def run_combinations() -> None: "gh-json", "-o", str(PARTS_DIR / f"{i}.gh.json"), + "--ingest-jsonl", + str(PARTS_DIR / f"{i}.ingest.jsonl"), ] print("+", " ".join(args), flush=True) subprocess.run(args, check=True) @@ -79,6 +81,7 @@ def merge(pattern: str, out_path: str) -> None: def main() -> None: run_combinations() merge(f"{PARTS_DIR}/*.gh.json", "results.json") + merge(f"{PARTS_DIR}/*.ingest.jsonl", "results.ingest.jsonl") if __name__ == "__main__":