Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,39 @@ jobs:
mode: ${{ matrix.mode }}
run: cargo codspeed run -p runner-shared

memtrack-benchmarks:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true

- uses: ./.github/actions/install-rust
- uses: ./.github/actions/install-bpf-deps

- name: Install memtrack
run: |
cargo install --path crates/memtrack --locked

- name: Grant memtrack file capabilities
run: cargo r -- setup --mode memory

- name: Build the codspeed CLI
run: cargo build --release

- name: Prepare memtrack output directory
run: mkdir -p /tmp/codspeed-memtrack-bench

# The write benchmarks overwrite these archives every round; creating
# them inside a measured round would time one-off disk allocation.
- name: Pre-create benchmark archives
run: |
tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu
dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null

- name: Run memtrack walltime benchmarks
run: ./target/release/codspeed --config crates/memtrack/codspeed.yml run -m walltime

check:
runs-on: ubuntu-latest
if: always()
Expand All @@ -164,6 +197,7 @@ jobs:
- macos-basic-run-test
- bpf-tests
- benchmarks
- memtrack-benchmarks
steps:
- uses: re-actors/alls-green@release/v1
with:
Expand Down
41 changes: 41 additions & 0 deletions crates/memtrack/codspeed.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
$schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/schemas/codspeed.schema.json

# Walltime benchmarks measure codspeed-memtrack's own overhead across a few
# representative workloads, not the memory usage of the tracked command. Each
# workload's RSS-only and RSS+rmap variants run back to back so rmap's overhead
# is directly comparable.
#
# The warmup/max times are generous because a single tracked run already pays a
# fixed BPF load + uprobe attach cost, which is far above the defaults tuned
# for near-instant commands.
#
# The write workloads overwrite archives pre-created by CI before measurement:
# a first-time multi-GB allocation on the runner disk would dominate one
# variant's warmup round purely from I/O ordering, not tracking overhead.
options:
warmup-time: "5s"
max-time: "60s"

benchmarks:
# Read-only, low-allocation baseline. The tracked command string is run
# through `bash -c`, so output can be redirected away: otherwise every round
# dumps the whole listing into the runner log.
- name: "memtrack track ls"
exec: env CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu > /dev/null" --output /tmp/codspeed-memtrack-bench

- name: "memtrack track ls (rmap)"
exec: env CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "ls -la /usr/lib/x86_64-linux-gnu > /dev/null" --output /tmp/codspeed-memtrack-bench

# I/O-heavy with minimal allocation.
- name: "memtrack track dd"
exec: env CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null" --output /tmp/codspeed-memtrack-bench

- name: "memtrack track dd (rmap)"
exec: env CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "dd if=/dev/zero of=/tmp/memtrack-bench-dd.bin bs=1M count=64 2> /dev/null" --output /tmp/codspeed-memtrack-bench

# Allocation- and I/O-heavy: many small file reads.
- name: "memtrack track tar"
exec: env CODSPEED_MEMTRACK_TRACK_RMAP=0 codspeed-memtrack track "tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench

- name: "memtrack track tar (rmap)"
exec: env CODSPEED_MEMTRACK_TRACK_RMAP=1 codspeed-memtrack track "tar -cf /tmp/memtrack-bench.tar /usr/lib/x86_64-linux-gnu" --output /tmp/codspeed-memtrack-bench