From b4ec96ef1a1f38869949ebaf6e60107a721a5f50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Loipf=C3=BChrer?= Date: Mon, 10 Aug 2026 22:16:07 +0200 Subject: [PATCH] feat(cli): improve ci self package times by allowing build tree and env caching Changes the meaning of the --incremental flag to always reuse the buildtree even if the underlying build environment has changed. --- .dockerignore | 2 ++ .github/ci/Dockerfile.build-deps | 13 +++++++ .github/workflows/ci.yaml | 56 ++++++++++++++++++++++------- .github/workflows/push_on_main.yaml | 4 +++ docs/usage/build.md | 2 ++ packages/debmagic/src/build/mod.rs | 10 +++--- 6 files changed, 70 insertions(+), 17 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/ci/Dockerfile.build-deps diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5874b8c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.git +target diff --git a/.github/ci/Dockerfile.build-deps b/.github/ci/Dockerfile.build-deps new file mode 100644 index 0000000..0c4c48b --- /dev/null +++ b/.github/ci/Dockerfile.build-deps @@ -0,0 +1,13 @@ +# Pre-baked build environment for the package_ourself CI job: debmagic's base +# tooling plus this repo's Build-Depends, so the in-container +# `apt-get build-dep` only tops up archive drift. Rebuilt whenever +# debian/control, this file, or the weekly freshness bucket changes +# (see the tag key in .github/workflows/ci.yaml). +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +WORKDIR /src +COPY debian /src/debian +RUN apt-get update \ + && apt-get install -y dpkg-dev python3 \ + && apt-get -y build-dep . \ + && rm -rf /var/lib/apt/lists/* diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index daf4294..a57d447 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -137,25 +137,55 @@ jobs: # integration / self packaging tests package_ourself: runs-on: ubuntu-latest + permissions: + contents: read + # Push of the pre-built CI image is additionally gated to main below. + packages: write steps: - uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - name: Install uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 - with: - enable-cache: true - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Install the project - run: uv sync --locked --all-extras --dev - - name: Install debmagic (cli) - run: uv pip install packages/debmagic + - name: Cache the debmagic build tree + uses: actions/cache@v4 + with: + path: /tmp/debmagic + key: debmagic-build-${{ hashFiles('Cargo.lock', 'debian/changelog', 'debian/control', 'debian/rules') }} + restore-keys: debmagic-build- + - name: Prune stale build trees from the restored cache + run: | + mkdir -p /tmp/debmagic + current="$(dpkg-parsechangelog -SSource)-$(dpkg-parsechangelog -SVersion)" + find /tmp/debmagic -mindepth 1 -maxdepth 1 -type d ! -name "$current" -exec rm -rf {} + + - name: Resolve CI image tag + id: image + run: | + codename="$(dpkg-parsechangelog -SDistribution)" + key="$(cat debian/control .github/ci/Dockerfile.build-deps | sha256sum | cut -c1-12)-$(date -u +%G-W%V)" + echo "tag=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/debmagic-ci:${codename}-${key}" >> "$GITHUB_OUTPUT" + echo "codename=${codename}" >> "$GITHUB_OUTPUT" + - name: Pull or build the CI image + run: | + if ! docker pull "${{ steps.image.outputs.tag }}"; then + docker build \ + --build-arg "BASE_IMAGE=docker.io/debian:${{ steps.image.outputs.codename }}" \ + --tag "${{ steps.image.outputs.tag }}" \ + --file .github/ci/Dockerfile.build-deps \ + . + echo "image_built=true" >> "$GITHUB_ENV" + fi - name: Run Debmagic build on ourself - run: uv run debmagic build binary --driver=docker + run: | + cargo run --locked -p debmagic -- build binary \ + --driver=docker \ + --persistent \ + --incremental \ + --driver-docker-base-image="${{ steps.image.outputs.tag }}" + - name: Push the CI image + if: github.ref == 'refs/heads/main' && env.image_built == 'true' + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username "${{ github.actor }}" --password-stdin + docker push "${{ steps.image.outputs.tag }}" # TODO: integration tests currently don't work in the CI since they require running apt source on debian trixie -> CI runs on ubuntu # integration-tests: diff --git a/.github/workflows/push_on_main.yaml b/.github/workflows/push_on_main.yaml index 53a1b33..5ed183f 100644 --- a/.github/workflows/push_on_main.yaml +++ b/.github/workflows/push_on_main.yaml @@ -7,5 +7,9 @@ on: jobs: ci: + permissions: + contents: read + # package_ourself pushes the pre-built CI image to ghcr.io on main. + packages: write uses: ./.github/workflows/ci.yaml secrets: inherit diff --git a/docs/usage/build.md b/docs/usage/build.md index 488f745..89d3096 100644 --- a/docs/usage/build.md +++ b/docs/usage/build.md @@ -108,6 +108,8 @@ debmagic build binary --driver lxd --persistent \ Use `--incremental` to retain the environment and synchronize only source changes while preserving generated files and unchanged source inodes. This flag implies `--persistent`, and cannot be combined with `--clean yes`. +The preserved build tree is kept even when the environment itself is *not* reused (e.g. a fresh CI runner where the tree was restored from a cache). + ## Selecting a distro/release diff --git a/packages/debmagic/src/build/mod.rs b/packages/debmagic/src/build/mod.rs index 8dbf94c..d6729b1 100644 --- a/packages/debmagic/src/build/mod.rs +++ b/packages/debmagic/src/build/mod.rs @@ -291,14 +291,16 @@ fn prepare_build_env(intent: &BuildIntent, target: &PackageTarget) -> anyhow::Re &intent.driver_overrides, ) .context(format!("failed to create {:?} build driver", intent.driver))?; - if !intent.config.incremental - || !source_manifest_path(&build_config).is_file() - || !build.driver.reused_environment() - { + if !intent.config.incremental || !source_manifest_path(&build_config).is_file() { build .driver .reset_build_root() .context("failed to reset persistent build directory")?; + } else if !build.driver.reused_environment() { + // A fresh environment (e.g. a new CI runner with a restored build + // tree) keeps incremental outputs; cargo's own fingerprinting + // discards whatever the new toolchain/archive state invalidates. + println!("Keeping incremental build tree in a fresh build environment"); } build_config .create_dirs()