From 52f95849d8e5bb108cf09783c2274cc8ed48d94a Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 1 Oct 2026 12:05:04 -0700 Subject: [PATCH 1/3] ci: key opam cache on resolved pins; skip setup-ocaml on warm switch --- .github/workflows/ci.yml | 95 +++++++++++++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1113eff..eb2cf69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,11 +24,63 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # pin-depends float on `#main`: resolve their current commits so the + # opam cache key changes when a pinned repo moves. Without this the + # restored switch is always stale (its pins predate upstream HEAD), + # `opam install` rebuilds them on every run, and the rebuilt result is + # never cached because the save step is skipped on cache hits. + - name: Resolve pinned dependency commits + id: pins + run: | + export GIT_TERMINAL_PROMPT=0 + sha=$( + grep -hoE 'git\+[^"]+#[^"]+' ./*.opam | sort -u | while IFS= read -r pin; do + repo="${pin%%#*}"; ref="${pin##*#}" + git ls-remote "${repo#git+}" "$ref" || echo "unresolved $pin" + done | shasum | awk '{print $1}' + ) + echo "resolved pins sha: $sha" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + + - name: Restore opam switch cache + id: opam-cache + uses: actions/cache/restore@v4 + with: + path: ~/.opam + key: ${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }}-opam-${{ hashFiles('*.opam') }}-${{ steps.pins.outputs.sha }} + restore-keys: | + ${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }}-opam- + + # Any restore (exact or restore-keys fallback) yields a complete opam + # root + switch; only the opam binary is needed. This skips + # ocaml/setup-ocaml, whose unconditional `opam update` costs ~1m30s on + # cache hits. + - name: Check for restored opam environment + id: opam-env + run: | + if [ -f "$HOME/.opam/config" ]; then + echo "restored=true" >> "$GITHUB_OUTPUT" + else + echo "restored=false" >> "$GITHUB_OUTPUT" + fi + + - name: Install opam binary + if: steps.opam-env.outputs.restored == 'true' + run: | + mkdir -p ~/.local/bin + curl -fsSL https://github.com/ocaml/opam/releases/download/2.6.0/opam-2.6.0-x86_64-linux -o ~/.local/bin/opam + chmod +x ~/.local/bin/opam + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + switch="$(opam switch list --short | head -n1)" + echo "OPAMSWITCH=$switch" >> "$GITHUB_ENV" + eval "$(opam env --switch="$switch")" + opam --version && ocaml -version + - name: Set up OCaml + if: steps.opam-env.outputs.restored != 'true' uses: ocaml/setup-ocaml@v3 with: ocaml-compiler: ${{ matrix.ocaml-compiler }} - dune-cache: true - name: Set up Java uses: actions/setup-java@v4 @@ -65,17 +117,17 @@ jobs: - name: Install system dependencies run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev pkg-config - - name: Restore opam switch cache - id: opam-cache - uses: actions/cache/restore@v4 - with: - path: ~/.opam - key: ${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }}-opam-${{ hashFiles('*.opam') }} - restore-keys: | - ${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }}-opam- - + # On an exact switch-cache hit this only verifies and re-fetches the + # (already matching) pins; on a partial restore it rebuilds just the + # moved pins; on a miss it installs everything. The retry covers a + # stale opam-repository snapshot in a restored root failing to resolve + # newly released versions. - name: Install OCaml dependencies - run: opam install . --deps-only --with-test -y + run: | + opam install . --deps-only --with-test -y || { + opam update -y + opam install . --deps-only --with-test -y + } - name: Save opam switch cache if: steps.opam-cache.outputs.cache-hit != 'true' @@ -84,6 +136,20 @@ jobs: path: ~/.opam key: ${{ steps.opam-cache.outputs.cache-primary-key }} + - name: Restore dune build cache + uses: actions/cache/restore@v4 + with: + path: ~/.cache/dune + key: dune-cache-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + dune-cache-${{ github.ref_name }}- + dune-cache-main- + + - name: Enable dune shared cache + run: | + mkdir -p ~/.config/dune + printf '(lang dune 3.0)\n(cache enabled)\n' > ~/.config/dune/config + - name: Cache upstream DataScript id: cache-upstream-datascript uses: actions/cache@v4 @@ -122,3 +188,10 @@ jobs: env: UPSTREAM_DATASCRIPT_JS: ${{ github.workspace }}/_ci/datascript/release-js/datascript.js run: opam exec -- script/benchmark_vs_cljs.sh + + - name: Save dune build cache + if: always() + uses: actions/cache/save@v4 + with: + path: ~/.cache/dune + key: dune-cache-${{ github.ref_name }}-${{ github.sha }} From 308f377a03644974bbce9df813e6e6560b84d106 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 1 Oct 2026 12:08:29 -0700 Subject: [PATCH 2/3] ci: cache the local _opam switch (deps live there, not ~/.opam) --- .github/workflows/ci.yml | 44 ++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb2cf69..91c73e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,23 +42,32 @@ jobs: echo "resolved pins sha: $sha" echo "sha=$sha" >> "$GITHUB_OUTPUT" - - name: Restore opam switch cache - id: opam-cache + # setup-ocaml creates a *local* switch: the compiler and installed + # packages live in `_opam/` (~1GB+), while `~/.opam` is only the opam + # root skeleton (~40MB) — caching `~/.opam` alone never restores deps. + - name: Restore opam root + id: opam-root uses: actions/cache/restore@v4 with: path: ~/.opam - key: ${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }}-opam-${{ hashFiles('*.opam') }}-${{ steps.pins.outputs.sha }} + key: opam-root-${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }} + + - name: Restore opam switch + id: opam-switch + uses: actions/cache/restore@v4 + with: + path: _opam + key: opam-switch-${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }}-${{ hashFiles('*.opam') }}-${{ steps.pins.outputs.sha }} restore-keys: | - ${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }}-opam- + opam-switch-${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }}- - # Any restore (exact or restore-keys fallback) yields a complete opam - # root + switch; only the opam binary is needed. This skips - # ocaml/setup-ocaml, whose unconditional `opam update` costs ~1m30s on - # cache hits. + # A restored root + local switch is already complete; only the opam + # binary is needed. This skips ocaml/setup-ocaml, whose unconditional + # `opam update` costs ~1m30s on cache hits. - name: Check for restored opam environment id: opam-env run: | - if [ -f "$HOME/.opam/config" ]; then + if [ -f "$HOME/.opam/config" ] && [ -d "_opam" ]; then echo "restored=true" >> "$GITHUB_OUTPUT" else echo "restored=false" >> "$GITHUB_OUTPUT" @@ -71,9 +80,7 @@ jobs: curl -fsSL https://github.com/ocaml/opam/releases/download/2.6.0/opam-2.6.0-x86_64-linux -o ~/.local/bin/opam chmod +x ~/.local/bin/opam echo "$HOME/.local/bin" >> "$GITHUB_PATH" - switch="$(opam switch list --short | head -n1)" - echo "OPAMSWITCH=$switch" >> "$GITHUB_ENV" - eval "$(opam env --switch="$switch")" + eval "$(opam env)" opam --version && ocaml -version - name: Set up OCaml @@ -129,12 +136,19 @@ jobs: opam install . --deps-only --with-test -y } - - name: Save opam switch cache - if: steps.opam-cache.outputs.cache-hit != 'true' + - name: Save opam root + if: steps.opam-root.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: path: ~/.opam - key: ${{ steps.opam-cache.outputs.cache-primary-key }} + key: opam-root-${{ runner.os }}-ocaml-${{ matrix.ocaml-compiler }} + + - name: Save opam switch + if: steps.opam-switch.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: _opam + key: ${{ steps.opam-switch.outputs.cache-primary-key }} - name: Restore dune build cache uses: actions/cache/restore@v4 From f3515799e2a58ac7ecda0a11119efd3c339922ba Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 1 Oct 2026 12:17:44 -0700 Subject: [PATCH 3/3] ci: re-run (test_perf timing flake)