From 7d899ecd20250c75b1387988d90da1e6eac23642 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 21 Aug 2026 09:58:04 +0200 Subject: [PATCH 1/5] ci(memtrack): benchmark memtrack's own tracking overhead `codspeed-memtrack track` pays a fixed cost per invocation (BPF program load plus uprobe/uretprobe attaches) on top of the tracked command, and nothing measured it so far, so wall-clock regressions in that overhead went unnoticed. Add a walltime config with three exec targets covering distinct workloads (read-only, allocation-heavy, I/O-heavy) and a CI job that runs them with the CLI and memtrack built from source. --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++++++ crates/memtrack/codspeed.yml | 25 +++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 crates/memtrack/codspeed.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8fe75ea..b9fc7478 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,6 +153,32 @@ jobs: mode: ${{ matrix.mode }} run: cargo codspeed run -p runner-shared + memtrack-benchmarks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: true + + - uses: ./.github/actions/install-rust + - uses: ./.github/actions/install-bpf-deps + + - name: Install memtrack + run: | + cargo install --path crates/memtrack --locked + + - name: Grant memtrack file capabilities + run: cargo r -- setup --mode memory + + - name: Build the codspeed CLI + run: cargo build --release + + - name: Prepare memtrack output directory + run: mkdir -p /tmp/codspeed-memtrack-bench + + - name: Run memtrack walltime benchmarks + run: ./target/release/codspeed --config crates/memtrack/codspeed.yml run -m walltime + check: runs-on: ubuntu-latest if: always() @@ -164,6 +190,7 @@ jobs: - macos-basic-run-test - bpf-tests - benchmarks + - memtrack-benchmarks steps: - uses: re-actors/alls-green@release/v1 with: diff --git a/crates/memtrack/codspeed.yml b/crates/memtrack/codspeed.yml new file mode 100644 index 00000000..477a8ac3 --- /dev/null +++ b/crates/memtrack/codspeed.yml @@ -0,0 +1,25 @@ +$schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/schemas/codspeed.schema.json + +# Walltime benchmarks measuring codspeed-memtrack's own overhead (eBPF probe +# attach + tracking) across a few representative workloads, not the memory +# usage of the tracked command. +# +# The warmup/max times are generous because a single tracked run already pays a +# fixed BPF load + uprobe attach cost, which is far above the defaults tuned +# for near-instant commands. +options: + warmup-time: "5s" + max-time: "60s" + +benchmarks: + # Read-only, low-allocation baseline. + - name: "memtrack track ls" + exec: codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench + + # Allocation- and I/O-heavy: many small file reads. + - name: "memtrack track tar" + exec: codspeed-memtrack track "tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench + + # I/O-heavy with minimal allocation. + - name: "memtrack track dd" + exec: codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64" --output /tmp/codspeed-memtrack-bench From 8ab188b6b84e62010c33609a39c99ea2f3833a3c Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 21 Aug 2026 12:35:52 +0200 Subject: [PATCH 2/5] ci(memtrack): discard tracked command output The listing and dd's stderr were captured into the runner log once per round, which made the uploaded log 4.2 MB of noise. The tracked command string is run through `bash -c`, so a redirect inside it works. --- crates/memtrack/codspeed.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/memtrack/codspeed.yml b/crates/memtrack/codspeed.yml index 477a8ac3..9b57b897 100644 --- a/crates/memtrack/codspeed.yml +++ b/crates/memtrack/codspeed.yml @@ -12,9 +12,11 @@ options: max-time: "60s" benchmarks: - # Read-only, low-allocation baseline. + # Read-only, low-allocation baseline. The tracked command string is run + # through `bash -c`, so output can be redirected away: otherwise every round + # dumps the whole listing into the runner log. - name: "memtrack track ls" - exec: codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench + exec: codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu > /dev/null" --output /tmp/codspeed-memtrack-bench # Allocation- and I/O-heavy: many small file reads. - name: "memtrack track tar" @@ -22,4 +24,4 @@ benchmarks: # I/O-heavy with minimal allocation. - name: "memtrack track dd" - exec: codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64" --output /tmp/codspeed-memtrack-bench + exec: codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null" --output /tmp/codspeed-memtrack-bench From 41ee10bf612356405bc5db927f4a4efb882862ce Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 24 Aug 2026 16:01:52 +0200 Subject: [PATCH 3/5] ci(memtrack): compare rmap tracking overhead Run each walltime workload with rmap disabled and enabled so the rmap contribution can be measured directly. --- crates/memtrack/codspeed.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/crates/memtrack/codspeed.yml b/crates/memtrack/codspeed.yml index 9b57b897..e3fbd449 100644 --- a/crates/memtrack/codspeed.yml +++ b/crates/memtrack/codspeed.yml @@ -1,8 +1,9 @@ $schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/schemas/codspeed.schema.json -# Walltime benchmarks measuring codspeed-memtrack's own overhead (eBPF probe -# attach + tracking) across a few representative workloads, not the memory -# usage of the tracked command. +# Walltime benchmarks measure codspeed-memtrack's own overhead across a few +# representative workloads, not the memory usage of the tracked command. Each +# workload runs both RSS-only and RSS+rmap variants so rmap's overhead is +# directly comparable. # # The warmup/max times are generous because a single tracked run already pays a # fixed BPF load + uprobe attach cost, which is far above the defaults tuned @@ -16,12 +17,21 @@ benchmarks: # through `bash -c`, so output can be redirected away: otherwise every round # dumps the whole listing into the runner log. - name: "memtrack track ls" - exec: codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu > /dev/null" --output /tmp/codspeed-memtrack-bench + exec: CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu > /dev/null" --output /tmp/codspeed-memtrack-bench # Allocation- and I/O-heavy: many small file reads. - name: "memtrack track tar" - exec: codspeed-memtrack track "tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench + exec: CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench # I/O-heavy with minimal allocation. - name: "memtrack track dd" - exec: codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null" --output /tmp/codspeed-memtrack-bench + exec: CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null" --output /tmp/codspeed-memtrack-bench + + - name: "memtrack track ls (rmap)" + exec: CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu > /dev/null" --output /tmp/codspeed-memtrack-bench + + - name: "memtrack track tar (rmap)" + exec: CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench + + - name: "memtrack track dd (rmap)" + exec: CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null" --output /tmp/codspeed-memtrack-bench From 83254153f2126c7807b9a33eedca8a66611b1488 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 24 Aug 2026 16:14:10 +0200 Subject: [PATCH 4/5] fix(memtrack): launch rmap benchmarks through env Exec harness treats the first token as the executable, so invoke env to apply the rmap setting before starting memtrack. --- crates/memtrack/codspeed.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/memtrack/codspeed.yml b/crates/memtrack/codspeed.yml index e3fbd449..68b8aecd 100644 --- a/crates/memtrack/codspeed.yml +++ b/crates/memtrack/codspeed.yml @@ -17,21 +17,21 @@ benchmarks: # through `bash -c`, so output can be redirected away: otherwise every round # dumps the whole listing into the runner log. - name: "memtrack track ls" - exec: CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu > /dev/null" --output /tmp/codspeed-memtrack-bench + exec: env CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu > /dev/null" --output /tmp/codspeed-memtrack-bench # Allocation- and I/O-heavy: many small file reads. - name: "memtrack track tar" - exec: CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench + exec: env CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench # I/O-heavy with minimal allocation. - name: "memtrack track dd" - exec: CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null" --output /tmp/codspeed-memtrack-bench + exec: env CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null" --output /tmp/codspeed-memtrack-bench - name: "memtrack track ls (rmap)" - exec: CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu > /dev/null" --output /tmp/codspeed-memtrack-bench + exec: env CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu > /dev/null" --output /tmp/codspeed-memtrack-bench - name: "memtrack track tar (rmap)" - exec: CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench + exec: env CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench - name: "memtrack track dd (rmap)" - exec: CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null" --output /tmp/codspeed-memtrack-bench + exec: env CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null" --output /tmp/codspeed-memtrack-bench From a2d6c6fa9450f3834c8405fc5152c65a12fccee8 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 25 Aug 2026 14:48:22 +0200 Subject: [PATCH 5/5] ci(memtrack): pre-create benchmark write targets The tar benchmark's first measured round created a multi-GB archive from scratch, so its reported value was dominated by one-off disk allocation instead of tracking overhead. On a runner this turned a single warmup round into ~95s and made the RSS+rmap variant (which ran later and overwrote the already-allocated file) look 18x faster. Interleave the rmap variants so each pair runs back to back over the same page-cache state, and create the archives before measurement so every round measures steady-state overwrite I/O. --- .github/workflows/ci.yml | 7 +++++++ crates/memtrack/codspeed.yml | 24 ++++++++++++++---------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9fc7478..af4e0d59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,6 +176,13 @@ jobs: - name: Prepare memtrack output directory run: mkdir -p /tmp/codspeed-memtrack-bench + # The write benchmarks overwrite these archives every round; creating + # them inside a measured round would time one-off disk allocation. + - name: Pre-create benchmark archives + run: | + tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu + dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null + - name: Run memtrack walltime benchmarks run: ./target/release/codspeed --config crates/memtrack/codspeed.yml run -m walltime diff --git a/crates/memtrack/codspeed.yml b/crates/memtrack/codspeed.yml index 68b8aecd..4fb539af 100644 --- a/crates/memtrack/codspeed.yml +++ b/crates/memtrack/codspeed.yml @@ -2,12 +2,16 @@ $schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/s # Walltime benchmarks measure codspeed-memtrack's own overhead across a few # representative workloads, not the memory usage of the tracked command. Each -# workload runs both RSS-only and RSS+rmap variants so rmap's overhead is -# directly comparable. +# workload's RSS-only and RSS+rmap variants run back to back so rmap's overhead +# is directly comparable. # # The warmup/max times are generous because a single tracked run already pays a # fixed BPF load + uprobe attach cost, which is far above the defaults tuned # for near-instant commands. +# +# The write workloads overwrite archives pre-created by CI before measurement: +# a first-time multi-GB allocation on the runner disk would dominate one +# variant's warmup round purely from I/O ordering, not tracking overhead. options: warmup-time: "5s" max-time: "60s" @@ -19,19 +23,19 @@ benchmarks: - name: "memtrack track ls" exec: env CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu > /dev/null" --output /tmp/codspeed-memtrack-bench - # Allocation- and I/O-heavy: many small file reads. - - name: "memtrack track tar" - exec: env CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench + - name: "memtrack track ls (rmap)" + exec: env CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu > /dev/null" --output /tmp/codspeed-memtrack-bench # I/O-heavy with minimal allocation. - name: "memtrack track dd" exec: env CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null" --output /tmp/codspeed-memtrack-bench - - name: "memtrack track ls (rmap)" - exec: env CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu > /dev/null" --output /tmp/codspeed-memtrack-bench + - name: "memtrack track dd (rmap)" + exec: env CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null" --output /tmp/codspeed-memtrack-bench + + # Allocation- and I/O-heavy: many small file reads. + - name: "memtrack track tar" + exec: env CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench - name: "memtrack track tar (rmap)" exec: env CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench - - - name: "memtrack track dd (rmap)" - exec: env CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null" --output /tmp/codspeed-memtrack-bench