From f3a1c12fe4bc98124b9221125fafec08138a8829 Mon Sep 17 00:00:00 2001 From: Maxwell Elliott Date: Thu, 15 Oct 2026 12:21:30 +0000 Subject: [PATCH] perf: add nightly bazel-diff vs Tango comparison harness Add tools/perf/tango_compare.py, a stdlib-only harness that benchmarks bazel-diff against Uber's Tango on identical workloads. It builds Tango's example server and client from source at a pinned commit, generates synthetic genrule workspaces (small/medium/large/xlarge presets, custom sizes, or a real repository), and times both tools end to end on the same base..head pair: cold (fresh Bazel servers), warm (servers alive, Tango with -bypass-cache) and diff-only. Tool order alternates per iteration. The impacted label sets are cross-checked and a markdown summary plus a JSON metrics report are written. The pure helpers (Tango's id-mapped JSON stream, Go duration strings, label diffing, spec parsing, summary rendering) are covered by a py_test under //tools/perf. The harness itself is not a py_binary, like the serve harnesses, because it can shell out to `bazel build`. A new perf-tango-compare workflow runs the four presets nightly and on dispatch (workloads, iterations, Tango ref, pool size, quick profile inputs), publishing the summary as a step summary and a 90-day artifact. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FUbHuckDDMM9DEvfiZo6xo --- .github/workflows/perf-tango-compare.yml | 126 +++ tools/perf/BUILD | 22 + tools/perf/README.md | 103 +++ tools/perf/tango_compare.py | 1025 ++++++++++++++++++++++ tools/perf/tango_compare_test.py | 292 ++++++ 5 files changed, 1568 insertions(+) create mode 100644 .github/workflows/perf-tango-compare.yml create mode 100644 tools/perf/BUILD create mode 100644 tools/perf/README.md create mode 100755 tools/perf/tango_compare.py create mode 100644 tools/perf/tango_compare_test.py diff --git a/.github/workflows/perf-tango-compare.yml b/.github/workflows/perf-tango-compare.yml new file mode 100644 index 00000000..980685fb --- /dev/null +++ b/.github/workflows/perf-tango-compare.yml @@ -0,0 +1,126 @@ +name: "Perf: bazel-diff vs Tango" + +# Nightly performance comparison of bazel-diff against Uber's Tango (https://github.com/uber/tango) +# on identical workloads. Runs tools/perf/tango_compare.py, which builds the Tango example +# server/client (go build, pinned commit), generates synthetic genrule workspaces of several sizes, +# and for each one times both tools end to end (checkout -> bazel query -> hash -> diff) cold (fresh +# Bazel servers), warm (servers alive) and diff-only, cross-checking that the two tools report the +# same impacted targets. See tools/perf/README.md for the methodology. +# +# Every run writes a markdown step summary and a 90-day metrics artifact, so trends can be read +# across nights, and a Tango commit can be compared on demand via the tango_ref input. Scheduled +# runs pin Tango to the commit in the harness so a Tango change never silently moves the baseline. +# Note: scheduled + dispatched workflows run from the default branch, so this must land on master +# before the cron fires. +on: + workflow_dispatch: + inputs: + workloads: + description: "Comma-separated synthetic presets (tiny, small, medium, large, xlarge)" + type: string + default: "small,medium,large,xlarge" + iterations: + description: "Timed iterations per workload" + type: string + default: "3" + tango_ref: + description: "uber/tango commit/branch/tag to build (empty = the harness's pinned commit)" + type: string + default: "" + tango_pool_size: + description: "Tango worker pool size (1 = sequential like bazel-diff; 2 = parallel revisions)" + type: string + default: "1" + quick: + description: "Smoke profile: tiny workload, one iteration (overrides workloads/iterations)" + type: boolean + default: false + schedule: + # Nightly at 03:45 UTC, offset from the serve crons (30 6, 30 8, 0 */12) and the format/badge + # crons (12:00-13:00) so the jobs never contend for runners. + - cron: "45 3 * * *" + +permissions: + contents: read + +jobs: + TangoCompare: + runs-on: ubuntu-latest + # The default legs x 3 iterations x (cold + warm) x 2 tools, plus the bazel-diff Rust build and + # the Tango Go build, fit well inside an hour on a 4-core runner (the cold numbers are dominated + # by Bazel server start-up, not graph size). The timeout is generous so a slow build or a runner + # hiccup yields data, not a kill. + timeout-minutes: 120 + steps: + - name: Setup Go environment + # Tango's go.mod declares go 1.25; the harness also sets GOTOOLCHAIN=auto as a fallback. + uses: actions/setup-go@v5 + with: + go-version: '>=1.25.0' + cache: false + id: go + - name: Setup Bazelisk + # Retries guard against transient proxy.golang.org/sum.golang.org module-fetch errors. + run: | + for i in 1 2 3; do + go install github.com/bazelbuild/bazelisk@latest && break + echo "bazelisk install failed (attempt $i); retrying in 10s..." + sleep 10 + done + - uses: actions/checkout@v4 + - name: Build bazel-diff + # Built here rather than by the harness so the Rust build has its own step and log. The + # .bazelrc lint aspects are skipped: this is a measurement binary, CI lints it elsewhere. + run: > + ~/go/bin/bazelisk build //src:bazel-diff -c opt + --output_groups=-clippy_checks,-rustfmt_checks + --enable_bzlmod=true --enable_workspace=false + - name: Run comparison harness + # Dispatch inputs reach the harness through the environment and a bash array, never by + # interpolating them into the command line. Scheduled runs see empty inputs and take the + # defaults. --bazel points both tools (and the harness's own `bazel shutdown`s) at bazelisk; + # each workspace's .bazelversion selects the Bazel release, so both tools run the same one. + env: + IN_WORKLOADS: ${{ inputs.workloads || 'small,medium,large,xlarge' }} + IN_ITERATIONS: ${{ inputs.iterations || '3' }} + IN_TANGO_REF: ${{ inputs.tango_ref }} + IN_POOL: ${{ inputs.tango_pool_size || '1' }} + IN_QUICK: ${{ inputs.quick && 'true' || 'false' }} + run: | + args=() + if [ "$IN_QUICK" = "true" ]; then + args+=(--quick) + else + IFS=',' read -ra WLS <<< "$IN_WORKLOADS" + for w in "${WLS[@]}"; do + w="${w//[[:space:]]/}" + [ -n "$w" ] && args+=(--workload "$w") + done + args+=(--iterations "$IN_ITERATIONS") + fi + [ -n "$IN_TANGO_REF" ] && args+=(--tango-ref "$IN_TANGO_REF") + args+=(--tango-pool-size "$IN_POOL") + echo "harness args: ${args[*]}" + python3 tools/perf/tango_compare.py \ + --bazel "$HOME/go/bin/bazelisk" \ + --bazel-diff bazel-bin/src/bazel-diff \ + --skip-build \ + --metrics-out tango-compare-metrics.json \ + --summary-out tango-compare-summary.md \ + "${args[@]}" + - name: Publish step summary + if: always() + run: | + if [ -f tango-compare-summary.md ]; then + cat tango-compare-summary.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: Upload metrics artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: tango-compare-${{ github.run_id }} + path: | + tango-compare-metrics.json + tango-compare-summary.md + if-no-files-found: warn + retention-days: 90 diff --git a/tools/perf/BUILD b/tools/perf/BUILD new file mode 100644 index 00000000..a950b88a --- /dev/null +++ b/tools/perf/BUILD @@ -0,0 +1,22 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +# tango_compare.py is intentionally NOT a py_binary, for the same reason as the serve harnesses in +# //tools: by default it shells out to `bazel build //src:bazel-diff`, which would deadlock on the +# output-base lock if the harness were itself launched via `bazel run`. Run it directly: +# `python3 tools/perf/tango_compare.py` (see its module docstring and README.md). Python stdlib only. +# +# Its pure helpers (output decoding, timing math, summary rendering, CLI parsing) never shell out, so +# a py_test over them is safe and runs in CI with the rest of //tools/.... +py_library( + name = "tango_compare_lib", + srcs = ["tango_compare.py"], + imports = ["."], + visibility = ["//visibility:private"], +) + +py_test( + name = "tango_compare_test", + srcs = ["tango_compare_test.py"], + python_version = "PY3", + deps = [":tango_compare_lib"], +) diff --git a/tools/perf/README.md b/tools/perf/README.md new file mode 100644 index 00000000..230d346e --- /dev/null +++ b/tools/perf/README.md @@ -0,0 +1,103 @@ +# bazel-diff vs Tango performance harness + +[`tango_compare.py`](tango_compare.py) benchmarks bazel-diff against +[Uber's Tango](https://github.com/uber/tango) (Target Analyzer in Go) on the same workspaces and +commit pairs, and cross-checks that both tools report the same impacted targets. The +[`perf-tango-compare` workflow](../../.github/workflows/perf-tango-compare.yml) runs it nightly +and publishes the comparison as a step summary plus a 90-day metrics artifact. + +## What is compared + +Both tools compute impacted targets the same way: check out each revision, `bazel query` the whole +graph (`deps(//...:all-targets)`, streamed proto), hash every target, diff the two hash maps. The +harness drives each tool through its own natural end-to-end flow on one workspace and one +`base..head` pair: + +| | bazel-diff | Tango | +|---|---|---| +| Flow | `git checkout base` → `generate-hashes` → `git checkout head` → `generate-hashes --depEdgesFile` → `get-impacted-targets` (the README recipe) | one `get-changed-targets` RPC to Tango's example server, which leases a worker clone, checks out and queries each revision, hashes, and compares | +| Timed as | wall time of the five stages (each also reported) | wall time seen by Tango's example client; the server's own `total_duration` is pulled from its log alongside | +| Includes | its two checkouts | its two checkouts | + +Each flow's one-off setup (Tango cloning its origin and worker, both Bazel servers fetching +`bazel_tools` & co.) happens in an untimed warm-up. Then, per iteration: + +- **cold** -- `bazel shutdown` for both tools' servers, then run each. Fresh server start plus full + graph load: what a CI job pays per PR today. +- **warm** -- run each again with the servers alive (incremental re-analysis after the checkouts). + Tango runs with `-bypass-cache` so it recomputes rather than serving its cached result. +- **diff only** -- no Bazel on either side: Tango with its result cache enabled (a treehash lookup) + next to bazel-diff's `get-impacted-targets` over the hash files already on disk. Informational. + +Tool order alternates every iteration so neither tool consistently runs into a page cache the other +just warmed. Medians are reported; the ratio is Tango / bazel-diff, so above `1.00x` means +bazel-diff was faster. + +The impacted sets from the last iteration are compared label by label. The tools' semantics are +close enough that the synthetic workloads agree exactly; on real repositories differences (external +repository handling, generated files) are reported, not failed, unless `--require-match` is given. + +## Workloads + +Synthetic workspaces come from [`tools/firecracker/bench/gen_project.py`](../firecracker/bench/gen_project.py): +layered genrule DAGs with real source files and no external toolchains, so query time is graph +load, not fetching. Presets (`--workload`): + +| Preset | Packages | genrules / package | Rule targets | +|---|---:|---:|---:| +| `tiny` | 60 | 3 | 240 (smoke tests only) | +| `small` | 400 | 4 | 2,000 | +| `medium` | 1,500 | 4 | 7,500 | +| `large` | 4,000 | 4 | 20,000 | +| `xlarge` | 10,000 | 4 | 50,000 | + +"Rule targets" counts genrules plus one filegroup per package; the query also sees +`sources_per_package` (3) source files and the genrule outputs per package, plus `bazel_tools`, +so `xlarge` is a ~130k-target `bazel query`. With no workload flags the harness runs the four +nightly presets. +Custom sizes: `--synthetic name:packages:targets_per_package[:layers]`. A real repository: +`--repo-url URL --repo-from HEAD~1 --repo-to HEAD [--repo-clone-args --depth=50]`. + +## Running locally + +```bash +# Build bazel-diff, clone+build Tango at the pinned commit, run the four presets x 3 iterations. +python3 tools/perf/tango_compare.py --bazel ~/go/bin/bazelisk + +# Smoke profile: tiny workload, one iteration, reusing an existing binary and Tango checkout. +python3 tools/perf/tango_compare.py --quick --skip-build --bazel-diff bazel-bin/src/bazel-diff \ + --tango-dir ~/src/tango + +# Keep everything on disk and write the report files the workflow publishes. +python3 tools/perf/tango_compare.py --workload medium --iterations 5 \ + --workdir /tmp/tc --keep-workdir --metrics-out metrics.json --summary-out summary.md +``` + +Requirements: `git`, `go` 1.25+ (or any Go with `GOTOOLCHAIN=auto`, the default the harness sets, +so the right toolchain is fetched), a `bazel`/`bazelisk` binary, Python 3.9+. No third-party +Python. Tango's example server hard-codes `127.0.0.1:8081`; the port must be free. + +Tango is pinned to `TANGO_PINNED_REF` in the script so a Tango change never silently moves the +baseline. Compare against another Tango with `--tango-ref ` (the workflow exposes +this as a dispatch input). `--tango-pool-size 2` lets Tango analyse the two revisions in parallel on +two workers -- an architectural advantage it has over bazel-diff's sequential flow -- at the cost of +twice the Bazel servers; the default of 1 keeps the comparison sequential-vs-sequential. + +## Output + +`--summary-out` is GitHub-flavored markdown (the workflow appends it to `$GITHUB_STEP_SUMMARY`): +the per-workload timing table, bazel-diff's cold stage breakdown, the impacted-set agreement table, +and any failures. `--metrics-out` is JSON with the environment (versions, SHAs of both tools, CPU and +memory), per-phase medians/min/max, every raw sample (bazel-diff stages, Tango server-side +durations), and the full label diff when the tools disagree. + +Exit status is non-zero when any run fails, and with `--require-match` also when the tools disagree. + +## Unit tests + +```bash +bazel test //tools/perf:tango_compare_test +``` + +The harness is not a `py_binary` (it builds bazel-diff with `bazel build`, which would deadlock under +`bazel run`); the test covers the pure helpers between raw tool output and the summary. diff --git a/tools/perf/tango_compare.py b/tools/perf/tango_compare.py new file mode 100755 index 00000000..7fc51ce8 --- /dev/null +++ b/tools/perf/tango_compare.py @@ -0,0 +1,1025 @@ +#!/usr/bin/env python3 +"""Performance harness comparing bazel-diff against Uber's Tango on identical workloads. + +Both tools answer the same question -- "which Bazel targets are impacted between two git +revisions?" -- with the same recipe: check out each revision, run `bazel query` over the whole +graph, hash every target, and diff the two hash maps. This harness runs both tools on the same +workspace and commit pair, times each end-to-end, and cross-checks that the impacted sets agree. + +What is measured, per workload and iteration (the Bazel server state is what distinguishes the +phases; both tools run against the same bazelisk and the workload's own .bazelversion): + + cold `bazel shutdown` is issued to both tools' Bazel servers first, so each pays a fresh + server start plus a full graph load. This is what a CI job pays on every PR today. + warm Run again immediately, servers alive: incremental re-analysis after the checkouts. + Tango is asked to `-bypass-cache` so it recomputes both graphs instead of serving the + result it cached during the cold run. + cached Tango with its result cache enabled (a treehash lookup, no bazel) next to bazel-diff's + `get-impacted-targets` over the hash files already on disk (a diff, no bazel). Neither + number involves Bazel; they are reported as informational "diff only" rows. + +bazel-diff's flow is `git checkout BASE; generate-hashes; git checkout HEAD; generate-hashes +--depEdgesFile; get-impacted-targets` (the README's recipe, run in-process by this harness, each +stage timed). Tango's flow is one `get-changed-targets` RPC against its example server, which +leases a worker clone, checks out and queries each revision, hashes, and compares -- timed as the +client sees it, with the server's own `total_duration` pulled from its log when available. Each +flow includes its own checkouts, so the totals are comparable. Tool order alternates every +iteration so neither tool systematically benefits from the other having just warmed the page cache. + +Tango is built from source (`go build`) at a pinned commit so a Tango change never silently moves +the baseline; pass --tango-ref to compare against another commit or branch, or --tango-dir to reuse +a checkout. Tango's example server hard-codes 127.0.0.1:8081 and reads example/tango-config.yaml +from its working directory, so the harness runs it from a scratch directory it writes that file +into; the port must be free. + +Workloads are the synthetic genrule workspaces from tools/firecracker/bench/gen_project.py +(hermetic, size-controlled, no external toolchains), selected by preset name (--workload tiny|small| +medium|large) or spelled out (--synthetic name:packages:targets_per_package[:layers]). A real +repository can be used instead (--repo-url with --repo-from/--repo-to), in which case the impacted +set is whatever the two tools say about those two real commits. + +Usage: + tools/perf/tango_compare.py --bazel ~/go/bin/bazelisk # build bazel-diff, then run + tools/perf/tango_compare.py --bazel-diff bazel-bin/src/bazel-diff --skip-build --workload small + tools/perf/tango_compare.py --quick # tiny workload, 1 iteration + tools/perf/tango_compare.py --workload medium --iterations 5 --metrics-out m.json --summary-out s.md + tools/perf/tango_compare.py --synthetic huge:20000:5:40 --iterations 2 + tools/perf/tango_compare.py --repo-url https://github.com/bazelbuild/bazel-skylib.git \ + --repo-from HEAD~1 --repo-to HEAD --workload skylib + +With no workload flags the four nightly presets run (small, medium, large, xlarge), 3 iterations each. + +Exit status is non-zero when either tool fails on any run, and additionally (with --require-match) +when the two tools disagree on an impacted set. Requires git, go (1.25+, or GOTOOLCHAIN=auto so the +go command fetches it), and a bazel/bazelisk binary. Python stdlib only. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import platform +import re +import shutil +import socket +import statistics +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +GEN_PROJECT = REPO_ROOT / "tools" / "firecracker" / "bench" / "gen_project.py" +# Overridable (BAZEL_DIFF_LAUNCHER) like the serve harnesses, so a downloaded release asset can be +# measured without building. +LAUNCHER = Path(os.environ.get("BAZEL_DIFF_LAUNCHER", str(REPO_ROOT / "bazel-bin" / "src" / "bazel-diff"))) +BUILD_TARGET = "//src:bazel-diff" + +TANGO_REPO_URL = "https://github.com/uber/tango.git" +# uber/tango main as of 2026-10-09 ("fix(tgb): drop per-column zstd expansion cap (#332)"). Bump +# deliberately: a Tango change moves the baseline, and the nightly trend should say which side moved. +TANGO_PINNED_REF = "41480a27e50d3a86dc25cd4d773bbe1b60474cdb" +# Hard-coded in tango's example/main.go; the example server takes no address flag. +TANGO_PORT = 8081 +TANGO_SERVER_STARTUP_TIMEOUT_S = 60.0 + +# packages x targets_per_package genrules (+1 filegroup per package, +sources_per_package source +# files). `tiny` is for smoke tests and the unit-test fixtures; the other four are the nightly legs. +SYNTHETIC_PRESETS: dict[str, dict[str, int]] = { + "tiny": {"packages": 60, "targets_per_package": 3, "layers": 6}, + "small": {"packages": 400, "targets_per_package": 4, "layers": 20}, + "medium": {"packages": 1500, "targets_per_package": 4, "layers": 40}, + "large": {"packages": 4000, "targets_per_package": 4, "layers": 40}, + "xlarge": {"packages": 10000, "targets_per_package": 4, "layers": 40}, +} + +PHASES = ("cold", "warm") + +VERBOSE = False + + +def log(msg: str) -> None: + print(msg, flush=True) + + +def vlog(msg: str) -> None: + if VERBOSE: + print(f" | {msg}", flush=True) + + +# ---------------------------------------------------------------------------------------------- +# Pure helpers (unit-tested in tango_compare_test.py; nothing here shells out) +# ---------------------------------------------------------------------------------------------- + +_GO_DURATION_UNITS = { + "h": 3600.0, + "m": 60.0, + "s": 1.0, + "ms": 1e-3, + "us": 1e-6, + "µs": 1e-6, # U+00B5 MICRO SIGN, what Go's time.Duration.String emits + "μs": 1e-6, # U+03BC GREEK SMALL LETTER MU, in case a log was transcoded + "ns": 1e-9, +} +_GO_DURATION_RE = re.compile(r"(\d+(?:\.\d+)?)(h|ms|m|s|us|µs|μs|ns)") + + +def parse_go_duration(text: str) -> float | None: + """Seconds for a Go `time.Duration` string such as `11.265275041s`, `1m2.5s`, `110.849µs`. + + Returns None for anything that is not entirely made of Go duration components (so a stray + log field can never be mistaken for a timing).""" + s = text.strip() + if not s: + return None + sign = 1.0 + if s[0] in "+-": + sign = -1.0 if s[0] == "-" else 1.0 + s = s[1:] + if s == "0": + return 0.0 + pos = 0 + total = 0.0 + while pos < len(s): + m = _GO_DURATION_RE.match(s, pos) + if m is None: + return None + total += float(m.group(1)) * _GO_DURATION_UNITS[m.group(2)] + pos = m.end() + return sign * total + + +def parse_tango_client_output(stdout: str) -> dict[str, Any]: + """Resolve the changed-target labels from the tango example client's stdout. + + The client prints one JSON object per streamed message: `{"changed_targets": [...]}` packets + (targets referenced by int32 id; gogo's `omitempty` drops an id of 0 and a distance of 0) and + `{"target_id_mapping": {"": "