From 078f965b9e515e8aa779478703e22b3db20c077d Mon Sep 17 00:00:00 2001 From: Arnaud Becheler <8360330+Becheler@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:26:16 +0200 Subject: [PATCH 1/3] infra: add a boostdeps counting bot --- .github/scripts/dep_table.py | 110 ++++++++++++++++++ .../workflows/dependency-count-comment.yml | 61 ++++++++++ .github/workflows/dependency-report.yml | 32 +++++ 3 files changed, 203 insertions(+) create mode 100644 .github/scripts/dep_table.py create mode 100644 .github/workflows/dependency-count-comment.yml create mode 100644 .github/workflows/dependency-report.yml diff --git a/.github/scripts/dep_table.py b/.github/scripts/dep_table.py new file mode 100644 index 000000000..1561273f3 --- /dev/null +++ b/.github/scripts/dep_table.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Print a markdown table of Boost dependency changes between two CI log archives. + +Usage: dep_table.py BASELINE.zip AFTER.zip + +Each archive is a GitHub Actions run-log zip. The `deps` CI job prints two +marker-delimited boostdep reports into its log: + + ===DEP-BRIEF-START=== boostdep --brief graph ===DEP-BRIEF-END=== + ===DEP-PRIMARY-START=== boostdep graph (primary) ===DEP-PRIMARY-END=== + +From those it derives two metrics and prints their delta vs the baseline: + - the set of transitive Boost modules graph depends on (from --brief), and + - the header-inclusion weight of each direct dependency (from the primary + report: how many distinct boost/graph/* headers pull that dependency in). +Weights are the live signal, they drop toward zero as coupling is removed; +the module count is the coarse target that only moves when a dep hits zero. +""" +import re +import sys +import zipfile + +BRIEF = ("===DEP-BRIEF-START===", "===DEP-BRIEF-END===") +PRIMARY = ("===DEP-PRIMARY-START===", "===DEP-PRIMARY-END===") + + +def read_logs(zip_path): + """Concatenate the top-level per-job .txt logs from a run-log archive.""" + out = [] + with zipfile.ZipFile(zip_path) as z: + for entry in z.namelist(): + if "/" in entry or not entry.endswith(".txt"): + continue + out.append(z.read(entry).decode("utf-8", "replace")) + return "\n".join(out) + + +def block(text, markers): + m = re.search(re.escape(markers[0]) + r"(.*?)" + re.escape(markers[1]), text, re.S) + return m.group(1) if m else "" + + +def parse_brief(text): + """--brief output -> set of module names (one bare name per line).""" + mods = set() + for line in text.splitlines(): + s = line.strip() + if not s or s.startswith("#") or s.lower().startswith("brief dependency"): + continue + if re.fullmatch(r"[A-Za-z0-9_.-]+", s): + mods.add(s) + return mods + + +def parse_weights(text): + """Primary report -> {dependency: number of distinct boost/graph headers that pull it in}.""" + weights, cur = {}, None + for line in text.splitlines(): + head = re.match(r"^([A-Za-z0-9_.-]+):\s*$", line) # "module:" at column 0 + if head: + cur = head.group(1) + weights.setdefault(cur, set()) + continue + frm = re.match(r"^\s+from\s+(.+?)\s*$", line) + if frm and cur is not None: + weights[cur].add(frm.group(1)) + return {k: len(v) for k, v in weights.items()} + + +def main(base_zip, pr_zip): + base, pr = read_logs(base_zip), read_logs(pr_zip) + base_mods = parse_brief(block(base, BRIEF)) + pr_mods = parse_brief(block(pr, BRIEF)) + base_w = parse_weights(block(base, PRIMARY)) + pr_w = parse_weights(block(pr, PRIMARY)) + + # Header-inclusion weights: the live signal. Show only rows that changed, + # most-reduced first. + print("**Header-inclusion weights** (graph headers pulling each direct dependency in):") + print() + changed = [] + for dep in base_w.keys() | pr_w.keys(): + b, p = base_w.get(dep, 0), pr_w.get(dep, 0) + if b != p: + changed.append((p - b, dep, b, p)) + if changed: + print("| Dependency | develop | PR | Δ |") + print("|-----|--------:|---:|----:|") + for d, dep, b, p in sorted(changed): # reductions (negative delta) first + print(f"| {dep} | {b} | {p} | {d:+d} |") + else: + print("_No header-inclusion-weight changes._") + print() + + # Transitive module set: the coarse target. + added = sorted(pr_mods - base_mods) + removed = sorted(base_mods - pr_mods) + nb, np_ = len(base_mods), len(pr_mods) + diff = np_ - nb + print(f"**Transitive Boost modules:** {nb} → {np_} ({f'{diff:+d}' if diff else '0'})") + if added: + print(f"- added: {', '.join(added)}") + if removed: + print(f"- removed: {', '.join(removed)}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit("usage: dep_table.py BASELINE.zip AFTER.zip") + main(sys.argv[1], sys.argv[2]) diff --git a/.github/workflows/dependency-count-comment.yml b/.github/workflows/dependency-count-comment.yml new file mode 100644 index 000000000..ddf49b0a1 --- /dev/null +++ b/.github/workflows/dependency-count-comment.yml @@ -0,0 +1,61 @@ +name: dependency-count-comment + +on: + workflow_run: + workflows: ["dependency-report"] + types: [completed] + +permissions: + actions: read + pull-requests: write + contents: read + +jobs: + comment: + if: github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + PR_RUN: ${{ github.event.workflow_run.id }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + steps: + - uses: actions/checkout@v4 + + - name: Download this PR run's log archive + run: gh api "repos/$REPO/actions/runs/$PR_RUN/logs" > pr-logs.zip + + - name: Download latest successful develop run's log archive + run: | + DEV=$(gh run list --workflow dependency-report --branch develop --status success \ + --limit 1 --json databaseId --jq '.[0].databaseId') + if [ -z "$DEV" ]; then echo "no develop baseline run found" >&2; exit 1; fi + echo "DEV_RUN=$DEV" >> "$GITHUB_ENV" + gh api "repos/$REPO/actions/runs/$DEV/logs" > base-logs.zip + + - name: Generate table + run: | + { + echo "Boost dependency footprint vs \`develop\` (auto-generated)." + echo "PR run \`$PR_RUN\` vs develop run \`$DEV_RUN\` (\`${HEAD_SHA:0:10}\`)." + echo + python3 .github/scripts/dep_table.py base-logs.zip pr-logs.zip + } > table.md + + - name: Resolve PR number + id: pr + run: | + num=$(gh api "repos/$REPO/pulls?state=open&per_page=100" \ + --jq "map(select(.head.sha==\"$HEAD_SHA\"))[0].number") + if [ -z "$num" ]; then + echo "no open PR found with head $HEAD_SHA" >&2 + exit 1 + fi + echo "num=$num" >> "$GITHUB_OUTPUT" + + - name: Post or update sticky comment + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: dependency-counts + number: ${{ steps.pr.outputs.num }} + path: table.md diff --git a/.github/workflows/dependency-report.yml b/.github/workflows/dependency-report.yml new file mode 100644 index 000000000..21b12978b --- /dev/null +++ b/.github/workflows/dependency-report.yml @@ -0,0 +1,32 @@ +name: dependency-report + +# Computes graph's Boost dependency footprint with boostdep and prints a +# marker-delimited report into the job log, on every push and PR. The +# dependency-count-comment workflow reads these logs (this PR's run and the +# latest develop run) and posts the diff as a PR comment. Kept separate from +# CI so ci.yml is untouched. + +on: [ push, pull_request ] + +jobs: + deps: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Set up Boost tree and build boostdep + run: | + git clone -b develop --depth 1 https://github.com/boostorg/boost.git ../boost-root + cd ../boost-root + cp -r $GITHUB_WORKSPACE/* libs/graph + git submodule update --init tools/boostdep + python tools/boostdep/depinst/depinst.py --git_args "--jobs 3" graph + c++ -std=c++14 -O2 tools/boostdep/src/boostdep.cpp -o boostdep + - name: Boostdep dependency report + working-directory: ../boost-root + run: | + echo "===DEP-BRIEF-START===" + ./boostdep --boost-root . --track-sources --no-track-tests --brief graph + echo "===DEP-BRIEF-END===" + echo "===DEP-PRIMARY-START===" + ./boostdep --boost-root . --track-sources --no-track-tests graph + echo "===DEP-PRIMARY-END===" From abf032dabb4b6814b1aa28859d80fb1a7b647b5e Mon Sep 17 00:00:00 2001 From: Arnaud Becheler <8360330+Becheler@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:39:09 +0200 Subject: [PATCH 2/3] fix: boostdep needs boost headers and compiled filesystem --- .github/workflows/dependency-report.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dependency-report.yml b/.github/workflows/dependency-report.yml index 21b12978b..c788d5e06 100644 --- a/.github/workflows/dependency-report.yml +++ b/.github/workflows/dependency-report.yml @@ -19,14 +19,24 @@ jobs: cd ../boost-root cp -r $GITHUB_WORKSPACE/* libs/graph git submodule update --init tools/boostdep + # depinst pulls graph's deps; also ensure boostdep's own dep + # (Boost.Filesystem) is present since boostdep links against it. python tools/boostdep/depinst/depinst.py --git_args "--jobs 3" graph - c++ -std=c++14 -O2 tools/boostdep/src/boostdep.cpp -o boostdep + python tools/boostdep/depinst/depinst.py --git_args "--jobs 3" filesystem + ./bootstrap.sh + ./b2 headers + # boostdep #includes and links the compiled + # library, so build it with b2 (a bare c++ compile can't resolve that). + ./b2 tools/boostdep/build - name: Boostdep dependency report working-directory: ../boost-root run: | + BOOSTDEP=$(find "$PWD" -type f -name boostdep -perm -u+x 2>/dev/null | grep -v '/src/' | head -1) + if [ -z "$BOOSTDEP" ]; then echo "boostdep executable not found after build" >&2; exit 1; fi + echo "using boostdep: $BOOSTDEP" echo "===DEP-BRIEF-START===" - ./boostdep --boost-root . --track-sources --no-track-tests --brief graph + "$BOOSTDEP" --boost-root . --track-sources --no-track-tests --brief graph echo "===DEP-BRIEF-END===" echo "===DEP-PRIMARY-START===" - ./boostdep --boost-root . --track-sources --no-track-tests graph + "$BOOSTDEP" --boost-root . --track-sources --no-track-tests graph echo "===DEP-PRIMARY-END===" From 848786f2ef41d6a78fdb3e0a498bdbe4af141c4d Mon Sep 17 00:00:00 2001 From: Arnaud Becheler <8360330+Becheler@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:55:50 +0200 Subject: [PATCH 3/3] fix: minor fix tested against real logs --- .github/scripts/dep_table.py | 85 ++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 32 deletions(-) diff --git a/.github/scripts/dep_table.py b/.github/scripts/dep_table.py index 1561273f3..0073be3ca 100644 --- a/.github/scripts/dep_table.py +++ b/.github/scripts/dep_table.py @@ -3,16 +3,18 @@ Usage: dep_table.py BASELINE.zip AFTER.zip -Each archive is a GitHub Actions run-log zip. The `deps` CI job prints two -marker-delimited boostdep reports into its log: +Each archive is a GitHub Actions run-log zip. The `deps` job of the +dependency-report workflow prints two marker-delimited boostdep reports into +its log: ===DEP-BRIEF-START=== boostdep --brief graph ===DEP-BRIEF-END=== ===DEP-PRIMARY-START=== boostdep graph (primary) ===DEP-PRIMARY-END=== From those it derives two metrics and prints their delta vs the baseline: - - the set of transitive Boost modules graph depends on (from --brief), and + - the set of transitive Boost modules graph depends on (from --brief: + Primary + Secondary sections = the full transitive closure), and - the header-inclusion weight of each direct dependency (from the primary - report: how many distinct boost/graph/* headers pull that dependency in). + report: how many distinct boost/graph files pull that dependency in). Weights are the live signal, they drop toward zero as coupling is removed; the module count is the coarse target that only moves when a dep hits zero. """ @@ -22,61 +24,80 @@ BRIEF = ("===DEP-BRIEF-START===", "===DEP-BRIEF-END===") PRIMARY = ("===DEP-PRIMARY-START===", "===DEP-PRIMARY-END===") +# Module names as boostdep prints them, e.g. "numeric~conversion". +MODULE = re.compile(r"[A-Za-z0-9_.~-]+") +TIMESTAMP = re.compile(r"^\d{4}-\d\d-\d\dT[\d:.]+Z\s?") # GitHub log line prefix +ANSI = re.compile(r"\x1b\[[0-9;]*m") -def read_logs(zip_path): - """Concatenate the top-level per-job .txt logs from a run-log archive.""" - out = [] +def read_clean_lines(zip_path): + """Top-level per-job logs, with GitHub timestamp and ANSI prefixes stripped.""" + lines = [] with zipfile.ZipFile(zip_path) as z: for entry in z.namelist(): if "/" in entry or not entry.endswith(".txt"): continue - out.append(z.read(entry).decode("utf-8", "replace")) - return "\n".join(out) - - -def block(text, markers): - m = re.search(re.escape(markers[0]) + r"(.*?)" + re.escape(markers[1]), text, re.S) - return m.group(1) if m else "" + for ln in z.read(entry).decode("utf-8", "replace").splitlines(): + lines.append(TIMESTAMP.sub("", ANSI.sub("", ln))) + return lines + + +def block(lines, markers): + """Lines strictly between the marker lines, matched exactly. + + Exact matching is what skips the echoed `echo ""` command lines + GitHub prepends to a run step, so we capture boostdep's real stdout. + """ + start, end = markers + out, capturing = [], False + for ln in lines: + s = ln.strip() + if s == start: + capturing = True + continue + if s == end and capturing: + break + if capturing: + out.append(ln) + return out -def parse_brief(text): +def parse_brief(lines): """--brief output -> set of module names (one bare name per line).""" mods = set() - for line in text.splitlines(): - s = line.strip() + for ln in lines: + s = ln.strip() if not s or s.startswith("#") or s.lower().startswith("brief dependency"): continue - if re.fullmatch(r"[A-Za-z0-9_.-]+", s): + if MODULE.fullmatch(s): mods.add(s) return mods -def parse_weights(text): - """Primary report -> {dependency: number of distinct boost/graph headers that pull it in}.""" +def parse_weights(lines): + """Primary report -> {dependency: number of distinct graph files that pull it in}.""" weights, cur = {}, None - for line in text.splitlines(): - head = re.match(r"^([A-Za-z0-9_.-]+):\s*$", line) # "module:" at column 0 + for ln in lines: + head = re.match(r"^([A-Za-z0-9_.~-]+):\s*$", ln) # "module:" at column 0 if head: cur = head.group(1) weights.setdefault(cur, set()) continue - frm = re.match(r"^\s+from\s+(.+?)\s*$", line) + frm = re.match(r"^\s+from\s+(.+?)\s*$", ln) if frm and cur is not None: weights[cur].add(frm.group(1)) return {k: len(v) for k, v in weights.items()} def main(base_zip, pr_zip): - base, pr = read_logs(base_zip), read_logs(pr_zip) - base_mods = parse_brief(block(base, BRIEF)) - pr_mods = parse_brief(block(pr, BRIEF)) - base_w = parse_weights(block(base, PRIMARY)) - pr_w = parse_weights(block(pr, PRIMARY)) - - # Header-inclusion weights: the live signal. Show only rows that changed, - # most-reduced first. - print("**Header-inclusion weights** (graph headers pulling each direct dependency in):") + bl, pl = read_clean_lines(base_zip), read_clean_lines(pr_zip) + base_mods = parse_brief(block(bl, BRIEF)) + pr_mods = parse_brief(block(pl, BRIEF)) + base_w = parse_weights(block(bl, PRIMARY)) + pr_w = parse_weights(block(pl, PRIMARY)) + + # Header-inclusion weights: the live signal. Only rows that changed, most-reduced first. + print("**Header-inclusion weights** (graph files pulling each direct dependency in):") print() changed = [] for dep in base_w.keys() | pr_w.keys():