From 7933a3e4687cff18c811dff18e1a8d8bed024c9e Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 14 Nov 2025 18:07:23 +0000 Subject: [PATCH 01/20] Created persist-dir-cache. --- .../composite/persist-dir-cache/action.yml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/composite/persist-dir-cache/action.yml diff --git a/.github/workflows/composite/persist-dir-cache/action.yml b/.github/workflows/composite/persist-dir-cache/action.yml new file mode 100644 index 0000000000..5331bd3922 --- /dev/null +++ b/.github/workflows/composite/persist-dir-cache/action.yml @@ -0,0 +1,42 @@ +name: "persist-dir-cache" +description: "Use caching to persist a directory across workflow runs" + +inputs: + cache_path: + description: "path to the directory to cache" + required: true + +env: + CACHE_BUILD: 0 + CACHE_WEEKS: 2 + +runs: + using: "composite" + steps: + - id: define-cache-period + run: echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${{ env.CACHE_WEEKS }})" >> $GITHUB_OUTPUT + + - id: sanitise-cache-path + run: echo "SANITISED_PATH=$(echo '${{ inputs.cache_path }}' | sed 's|/|_|g')" >> $GITHUB_OUTPUT + + - id: define-cache-key + run: | + OS="${{ runner.os }}" + DIR_PATH="${{ steps.sanitise-cache-path.outputs.SANITISED_PATH }}" + PERIOD="${{ steps.define-cache-period.outputs.CACHE_PERIOD }}" + BUILD="${{ env.CACHE_BUILD }}" + CACHE_KEY="${OS}-${DIR_PATH}_cached-p${PERIOD}-b${BUILD}" + echo "CACHE_KEY=$CACHE_KEY" >> $GITHUB_ENV + + - id: restore-dir-cache + uses: actions/cache/restore@v4 + with: + path: ${{ inputs.cache_path }} + key: ${{ env.CACHE_KEY }} + + post: + - id: save-dir-cache + uses: actions/cache/save@v4 + with: + path: ${{ inputs.cache_path }} + key: ${{ env.CACHE_KEY }} From e7b2fdfb768a3101e07b3e355bf74015a0f71d89 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 17 Nov 2025 13:58:14 +0000 Subject: [PATCH 02/20] Refactor conda-pkg-cache. --- .github/workflows/ci-tests.yml | 6 ++-- .github/workflows/ci-wheels.yml | 6 ++-- .../composite/conda-pkg-cache/action.yml | 32 ++++++++++--------- .../composite/persist-dir-cache/action.yml | 20 ++++++------ 4 files changed, 31 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 5df55143bd..eefb2199b3 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -73,12 +73,10 @@ jobs: env_name: ${{ env.ENV_NAME }} version: ${{ env.IRIS_TEST_DATA_VERSION }} + # Note: this cache is shared across workflows, so sets its own cache period + # and build number. - name: "conda package cache" uses: ./.github/workflows/composite/conda-pkg-cache - with: - cache_build: 6 - cache_period: ${{ env.CACHE_PERIOD }} - env_name: ${{ env.ENV_NAME }} - name: "conda install" uses: conda-incubator/setup-miniconda@v3 diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index 7d7eea830b..d84e87f3d3 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -74,12 +74,10 @@ jobs: echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> ${GITHUB_ENV} echo "LOCK_FILE=requirements/locks/py$(echo ${{ matrix.python-version }} | tr -d '.')-linux-64.lock" >> ${GITHUB_ENV} + # Note: this cache is shared across workflows, so sets its own cache period + # and build number. - name: "conda package cache" uses: ./.github/workflows/composite/conda-pkg-cache - with: - cache_build: 0 - cache_period: ${{ env.CACHE_PERIOD }} - env_name: ${{ env.ENV_NAME }} - name: "conda install" uses: conda-incubator/setup-miniconda@v3 diff --git a/.github/workflows/composite/conda-pkg-cache/action.yml b/.github/workflows/composite/conda-pkg-cache/action.yml index 48c4470e44..98b7e0734c 100644 --- a/.github/workflows/composite/conda-pkg-cache/action.yml +++ b/.github/workflows/composite/conda-pkg-cache/action.yml @@ -1,22 +1,24 @@ name: "conda-pkg-cache" -description: "cache the conda environment packages" +description: "standardise conda package cache to ~/conda_pkgs_dir and cache across workflow runs" -inputs: - cache_build: - description: "conda environment cache build number" - required: false - default: "0" - cache_period: - description: "conda environment cache timestamp" - required: true - env_name: - description: "environment name" - required: true +env: + # Lets us manually bump the cache to rebuild. + CACHE_BUILD: 0 + # Maximum cache period (in weeks) before forcing a cache refresh. + CACHE_WEEKS: 2 runs: using: "composite" steps: - - uses: actions/cache@v4 + - id: set-pkg-dir + run: conda config --add pkgs_dirs ~/conda_pkgs_dir + + - id: set-cache-period + run: echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${{ env.CACHE_WEEKS }})" >> $GITHUB_OUTPUT + + - id: conda-pkg-cache + uses: ./.github/workflows/composite/persist-dir-cache with: - path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-pkgs-${{ inputs.env_name }}-p${{ inputs.cache_period }}-b${{ inputs.cache_build }} + cache_path: ~/conda_pkgs_dir + cache_build: ${{ env.CACHE_BUILD }} + cache_period: ${{ steps.set-cache-period.outputs.CACHE_PERIOD }} diff --git a/.github/workflows/composite/persist-dir-cache/action.yml b/.github/workflows/composite/persist-dir-cache/action.yml index 5331bd3922..078214d1a9 100644 --- a/.github/workflows/composite/persist-dir-cache/action.yml +++ b/.github/workflows/composite/persist-dir-cache/action.yml @@ -5,17 +5,17 @@ inputs: cache_path: description: "path to the directory to cache" required: true - -env: - CACHE_BUILD: 0 - CACHE_WEEKS: 2 + cache_build: + description: "persist dir cache build number" + required: false + default: "0" + cache_period: + description: "persist dir cache timestamp" + required: true runs: using: "composite" steps: - - id: define-cache-period - run: echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${{ env.CACHE_WEEKS }})" >> $GITHUB_OUTPUT - - id: sanitise-cache-path run: echo "SANITISED_PATH=$(echo '${{ inputs.cache_path }}' | sed 's|/|_|g')" >> $GITHUB_OUTPUT @@ -23,9 +23,9 @@ runs: run: | OS="${{ runner.os }}" DIR_PATH="${{ steps.sanitise-cache-path.outputs.SANITISED_PATH }}" - PERIOD="${{ steps.define-cache-period.outputs.CACHE_PERIOD }}" - BUILD="${{ env.CACHE_BUILD }}" - CACHE_KEY="${OS}-${DIR_PATH}_cached-p${PERIOD}-b${BUILD}" + PERIOD="${{ inputs.cache_period }}" + BUILD="${{ inputs.cache_build }}" + CACHE_KEY="${DIR_PATH}_cached-${OS}-p${PERIOD}-b${BUILD}" echo "CACHE_KEY=$CACHE_KEY" >> $GITHUB_ENV - id: restore-dir-cache From 4ae11cd7b3ad27d4b43d572bfd5d7fc659973c50 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 17 Nov 2025 14:28:38 +0000 Subject: [PATCH 03/20] More sensible Conda caching in all workflows. --- .github/workflows/benchmarks_run.yml | 14 +++----- .github/workflows/benchmarks_validate.yml | 17 +++------ .github/workflows/ci-tests.yml | 16 ++------- .github/workflows/ci-wheels.yml | 16 ++------- .../composite/conda-env-cache/action.yml | 35 ------------------- .../workflows/composite/nox-cache/action.yml | 22 ------------ 6 files changed, 12 insertions(+), 108 deletions(-) delete mode 100644 .github/workflows/composite/conda-env-cache/action.yml delete mode 100644 .github/workflows/composite/nox-cache/action.yml diff --git a/.github/workflows/benchmarks_run.yml b/.github/workflows/benchmarks_run.yml index c91e7841a1..36e7c5e0fb 100644 --- a/.github/workflows/benchmarks_run.yml +++ b/.github/workflows/benchmarks_run.yml @@ -66,7 +66,6 @@ jobs: IRIS_TEST_DATA_PATH: benchmarks/iris-test-data IRIS_TEST_DATA_VERSION: "2.28" # Lets us manually bump the cache to rebuild - ENV_CACHE_BUILD: "0" TEST_DATA_CACHE_BUILD: "2" steps: @@ -79,15 +78,10 @@ jobs: - name: Install run dependencies run: pip install asv nox!=2025.05.01 - - name: Cache environment directories - id: cache-env-dir - uses: actions/cache@v4 - with: - path: | - .nox - benchmarks/.asv/env - $CONDA/pkgs - key: ${{ runner.os }}-${{ hashFiles('requirements/') }}-${{ env.ENV_CACHE_BUILD }} + # Note: this cache is shared across workflows, so sets its own cache period + # and build number. + - name: Cache Conda package directory + uses: ./.github/workflows/composite/conda-pkg-cache - name: Cache test data directory id: cache-test-data diff --git a/.github/workflows/benchmarks_validate.yml b/.github/workflows/benchmarks_validate.yml index e3f090b32c..26930b501d 100644 --- a/.github/workflows/benchmarks_validate.yml +++ b/.github/workflows/benchmarks_validate.yml @@ -21,10 +21,6 @@ jobs: validate: runs-on: ubuntu-latest - env: - # Lets us manually bump the cache to rebuild - ENV_CACHE_BUILD: "0" - steps: - name: Checkout repo uses: actions/checkout@v5 @@ -34,15 +30,10 @@ jobs: - name: Install run dependencies run: pip install asv nox!=2025.05.01 - - name: Cache environment directories - id: cache-env-dir - uses: actions/cache@v4 - with: - path: | - .nox - benchmarks/.asv/env - $CONDA/pkgs - key: ${{ runner.os }}-${{ hashFiles('requirements/') }}-${{ env.ENV_CACHE_BUILD }} + # Note: this cache is shared across workflows, so sets its own cache period + # and build number. + - name: Cache Conda package directory + uses: ./.github/workflows/composite/conda-pkg-cache - name: Validate setup run: nox -s benchmarks -- validate diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index eefb2199b3..1f22319332 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -86,13 +86,8 @@ jobs: activate-environment: ${{ env.ENV_NAME }} auto-update-conda: false - - name: "conda environment cache" - uses: ./.github/workflows/composite/conda-env-cache - with: - cache_build: 6 - cache_period: ${{ env.CACHE_PERIOD }} - env_name: ${{ env.ENV_NAME }} - install_packages: "cartopy nox pip" + - name: "install cartopy + nox + pip" + run: conda install --quiet --name ${{ env.ENV_NAME }} cartopy nox pip - name: "conda info" run: | @@ -106,13 +101,6 @@ jobs: cache_period: ${{ env.CACHE_PERIOD }} env_name: ${{ env.ENV_NAME }} - - name: "nox cache" - uses: ./.github/workflows/composite/nox-cache - with: - cache_build: 6 - env_name: ${{ env.ENV_NAME }} - lock_file: ${{ env.LOCK_FILE }} - # TODO: drop use of site.cfg and explicit use of mplrc - name: "iris configure" env: diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index d84e87f3d3..c7339ee0ea 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -88,20 +88,8 @@ jobs: auto-update-conda: false use-only-tar-bz2: true - - name: "conda environment cache" - uses: ./.github/workflows/composite/conda-env-cache - with: - cache_build: 0 - cache_period: ${{ env.CACHE_PERIOD }} - env_name: ${{ env.ENV_NAME }} - install_packages: "nox pip" - - - name: "nox cache" - uses: ./.github/workflows/composite/nox-cache - with: - cache_build: 1 - env_name: ${{ env.ENV_NAME }} - lock_file: ${{ env.LOCK_FILE }} + - name: "install nox + pip" + run: conda install --quiet --name ${{ env.ENV_NAME }} nox pip - name: "nox install and test wheel" env: diff --git a/.github/workflows/composite/conda-env-cache/action.yml b/.github/workflows/composite/conda-env-cache/action.yml deleted file mode 100644 index 15eaaec63c..0000000000 --- a/.github/workflows/composite/conda-env-cache/action.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "conda-env-cache" -description: "create and cache the conda environment" - -# -# Assumes the environment contains the following variables: -# - CONDA -# -inputs: - cache_build: - description: "conda environment cache build number" - required: false - default: "0" - cache_period: - description: "conda environment cache timestamp" - required: true - env_name: - description: "environment name" - required: true - install_packages: - description: "conda packages to install into environment" - required: true - -runs: - using: "composite" - steps: - - uses: actions/cache@v4 - id: conda-env-cache - with: - path: ${{ env.CONDA }}/envs/${{ inputs.env_name }} - key: ${{ runner.os }}-conda-env-${{ inputs.env_name }}-p${{ inputs.cache_period }}-b${{ inputs.cache_build }} - - - if: steps.conda-env-cache.outputs.cache-hit != 'true' - shell: bash - run: | - conda install --quiet --name ${{ inputs.env_name }} ${{ inputs.install_packages }} diff --git a/.github/workflows/composite/nox-cache/action.yml b/.github/workflows/composite/nox-cache/action.yml deleted file mode 100644 index 00387331e7..0000000000 --- a/.github/workflows/composite/nox-cache/action.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "nox cache" -description: "cache the nox test environments" - -inputs: - cache_build: - description: "nox cache build number" - required: false - default: "0" - env_name: - description: "environment name" - required: true - lock_file: - description: "conda-lock environment requirements filename" - required: true - -runs: - using: "composite" - steps: - - uses: actions/cache@v4 - with: - path: ${{ github.workspace }}/.nox - key: ${{ runner.os }}-nox-${{ inputs.env_name }}-s${{ matrix.session }}-py${{ matrix.python-version }}-b${{ inputs.cache_build }}-${{ hashFiles(inputs.lock_file) }} From 49fdf42882df2890b3e4c3b87183a50f9ae5bb0a Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 17 Nov 2025 15:30:11 +0000 Subject: [PATCH 04/20] Include missing shell keys. --- .github/workflows/composite/conda-pkg-cache/action.yml | 2 ++ .github/workflows/composite/persist-dir-cache/action.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/composite/conda-pkg-cache/action.yml b/.github/workflows/composite/conda-pkg-cache/action.yml index 98b7e0734c..2aa5a74daf 100644 --- a/.github/workflows/composite/conda-pkg-cache/action.yml +++ b/.github/workflows/composite/conda-pkg-cache/action.yml @@ -11,9 +11,11 @@ runs: using: "composite" steps: - id: set-pkg-dir + shell: bash run: conda config --add pkgs_dirs ~/conda_pkgs_dir - id: set-cache-period + shell: bash run: echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${{ env.CACHE_WEEKS }})" >> $GITHUB_OUTPUT - id: conda-pkg-cache diff --git a/.github/workflows/composite/persist-dir-cache/action.yml b/.github/workflows/composite/persist-dir-cache/action.yml index 078214d1a9..a794b9b85d 100644 --- a/.github/workflows/composite/persist-dir-cache/action.yml +++ b/.github/workflows/composite/persist-dir-cache/action.yml @@ -17,9 +17,11 @@ runs: using: "composite" steps: - id: sanitise-cache-path + shell: bash run: echo "SANITISED_PATH=$(echo '${{ inputs.cache_path }}' | sed 's|/|_|g')" >> $GITHUB_OUTPUT - id: define-cache-key + shell: bash run: | OS="${{ runner.os }}" DIR_PATH="${{ steps.sanitise-cache-path.outputs.SANITISED_PATH }}" From 0d55c0600a72f41e0e441e9710b25c00635b2783 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 17 Nov 2025 16:32:05 +0000 Subject: [PATCH 05/20] New approach that is Python specific and copes with unavailable post: syntax. --- .github/workflows/ci-tests.yml | 2 ++ .github/workflows/ci-wheels.yml | 2 ++ .../composite/conda-pkg-cache/action.yml | 6 +++++ .../composite/persist-dir-cache/action.yml | 27 +++++++++++-------- lib/iris/tests/test_coding_standards.py | 2 ++ 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1f22319332..6f33fad585 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -77,6 +77,8 @@ jobs: # and build number. - name: "conda package cache" uses: ./.github/workflows/composite/conda-pkg-cache + with: + python_version: ${{ matrix.python-version }} - name: "conda install" uses: conda-incubator/setup-miniconda@v3 diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index c7339ee0ea..968361ea87 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -78,6 +78,8 @@ jobs: # and build number. - name: "conda package cache" uses: ./.github/workflows/composite/conda-pkg-cache + with: + python_version: ${{ matrix.python-version }} - name: "conda install" uses: conda-incubator/setup-miniconda@v3 diff --git a/.github/workflows/composite/conda-pkg-cache/action.yml b/.github/workflows/composite/conda-pkg-cache/action.yml index 2aa5a74daf..2376c9432d 100644 --- a/.github/workflows/composite/conda-pkg-cache/action.yml +++ b/.github/workflows/composite/conda-pkg-cache/action.yml @@ -1,6 +1,12 @@ name: "conda-pkg-cache" description: "standardise conda package cache to ~/conda_pkgs_dir and cache across workflow runs" +inputs: + python_version: + description: "conda package cache python version" + required: false + default: "3.13" + env: # Lets us manually bump the cache to rebuild. CACHE_BUILD: 0 diff --git a/.github/workflows/composite/persist-dir-cache/action.yml b/.github/workflows/composite/persist-dir-cache/action.yml index a794b9b85d..51e54c62bf 100644 --- a/.github/workflows/composite/persist-dir-cache/action.yml +++ b/.github/workflows/composite/persist-dir-cache/action.yml @@ -12,6 +12,10 @@ inputs: cache_period: description: "persist dir cache timestamp" required: true + key_suffix: + description: "optional suffix to append to the cache key" + required: false + default: "" runs: using: "composite" @@ -27,18 +31,19 @@ runs: DIR_PATH="${{ steps.sanitise-cache-path.outputs.SANITISED_PATH }}" PERIOD="${{ inputs.cache_period }}" BUILD="${{ inputs.cache_build }}" - CACHE_KEY="${DIR_PATH}_cached-${OS}-p${PERIOD}-b${BUILD}" + SUFFIX="${{ inputs.key_suffix }}" + CACHE_KEY="${DIR_PATH}_cached-${OS}-p${PERIOD}-b${BUILD}-${SUFFIX}" echo "CACHE_KEY=$CACHE_KEY" >> $GITHUB_ENV - - id: restore-dir-cache - uses: actions/cache/restore@v4 - with: - path: ${{ inputs.cache_path }} - key: ${{ env.CACHE_KEY }} - - post: - - id: save-dir-cache - uses: actions/cache/save@v4 + - id: dir-cache + uses: actions/cache@v4 with: path: ${{ inputs.cache_path }} - key: ${{ env.CACHE_KEY }} + # This pattern allows continual updating of the cache_path with each new + # commit. The most recent cache matching CACHE_KEY will be used, while + # any updates will create a new cache entry specific to the commit SHA, + # which will then be available to the next run as the new most-recent + # cache. + key: ${{ env.CACHE_KEY }}-{{ github.sha }} + restore-keys: | + ${{ env.CACHE_KEY }} diff --git a/lib/iris/tests/test_coding_standards.py b/lib/iris/tests/test_coding_standards.py index f51a531721..13abceb858 100644 --- a/lib/iris/tests/test_coding_standards.py +++ b/lib/iris/tests/test_coding_standards.py @@ -83,6 +83,7 @@ def test_python_versions(): nox_file = root_dir / "noxfile.py" ci_wheels_file = workflows_dir / "ci-wheels.yml" ci_tests_file = workflows_dir / "ci-tests.yml" + ci_pkg_cache_file = workflows_dir / "composite" / "conda-pkg-cache" / "action.yml" benchmark_runner_file = benchmarks_dir / "bm_runner.py" text_searches: List[Tuple[Path, str]] = [ @@ -107,6 +108,7 @@ def test_python_versions(): f'{" " * 8}session: ["doctest", "gallery"]' ), ), + (ci_pkg_cache_file, f'default: "{latest_supported}"'), (benchmark_runner_file, f'python_version = "{latest_supported}"'), ] From 40a0f529eb5de7edd901972fb3b265338422b80f Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 17 Nov 2025 16:35:51 +0000 Subject: [PATCH 06/20] Use Python version as key_suffix. --- .github/workflows/composite/conda-pkg-cache/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/composite/conda-pkg-cache/action.yml b/.github/workflows/composite/conda-pkg-cache/action.yml index 2376c9432d..671b22eb8d 100644 --- a/.github/workflows/composite/conda-pkg-cache/action.yml +++ b/.github/workflows/composite/conda-pkg-cache/action.yml @@ -30,3 +30,4 @@ runs: cache_path: ~/conda_pkgs_dir cache_build: ${{ env.CACHE_BUILD }} cache_period: ${{ steps.set-cache-period.outputs.CACHE_PERIOD }} + key_suffix: "py${{ inputs.python_version }}" From 2d18e4674a9ecbc8a8f7758972a34e38d2e88314 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 17 Nov 2025 16:38:38 +0000 Subject: [PATCH 07/20] Minor corrections. --- .github/workflows/composite/conda-pkg-cache/action.yml | 2 +- .github/workflows/composite/persist-dir-cache/action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/composite/conda-pkg-cache/action.yml b/.github/workflows/composite/conda-pkg-cache/action.yml index 671b22eb8d..717e305451 100644 --- a/.github/workflows/composite/conda-pkg-cache/action.yml +++ b/.github/workflows/composite/conda-pkg-cache/action.yml @@ -9,7 +9,7 @@ inputs: env: # Lets us manually bump the cache to rebuild. - CACHE_BUILD: 0 + CACHE_BUILD: "0" # Maximum cache period (in weeks) before forcing a cache refresh. CACHE_WEEKS: 2 diff --git a/.github/workflows/composite/persist-dir-cache/action.yml b/.github/workflows/composite/persist-dir-cache/action.yml index 51e54c62bf..ae521754b5 100644 --- a/.github/workflows/composite/persist-dir-cache/action.yml +++ b/.github/workflows/composite/persist-dir-cache/action.yml @@ -44,6 +44,6 @@ runs: # any updates will create a new cache entry specific to the commit SHA, # which will then be available to the next run as the new most-recent # cache. - key: ${{ env.CACHE_KEY }}-{{ github.sha }} + key: ${{ env.CACHE_KEY }}-${{ github.sha }} restore-keys: | ${{ env.CACHE_KEY }} From d286c762788315df7cb866d0be1a6588252ad9a5 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 17 Nov 2025 16:45:30 +0000 Subject: [PATCH 08/20] Fix CACHE_BUILD and CACHE_WEEKS handling. --- .../workflows/composite/conda-pkg-cache/action.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/composite/conda-pkg-cache/action.yml b/.github/workflows/composite/conda-pkg-cache/action.yml index 717e305451..f0aa2ef88e 100644 --- a/.github/workflows/composite/conda-pkg-cache/action.yml +++ b/.github/workflows/composite/conda-pkg-cache/action.yml @@ -8,8 +8,6 @@ inputs: default: "3.13" env: - # Lets us manually bump the cache to rebuild. - CACHE_BUILD: "0" # Maximum cache period (in weeks) before forcing a cache refresh. CACHE_WEEKS: 2 @@ -22,12 +20,17 @@ runs: - id: set-cache-period shell: bash - run: echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${{ env.CACHE_WEEKS }})" >> $GITHUB_OUTPUT + env: + CACHE_WEEKS: ${{ env.CACHE_WEEKS }} + run: echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> $GITHUB_OUTPUT - id: conda-pkg-cache uses: ./.github/workflows/composite/persist-dir-cache + env: + CACHE_BUILD: ${{ env.CACHE_BUILD }} with: cache_path: ~/conda_pkgs_dir - cache_build: ${{ env.CACHE_BUILD }} + # Lets us manually bump the cache to rebuild. + cache_build: 0 cache_period: ${{ steps.set-cache-period.outputs.CACHE_PERIOD }} key_suffix: "py${{ inputs.python_version }}" From 210f953e024c559a3011c204ac928267cb4f985f Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 17 Nov 2025 16:49:40 +0000 Subject: [PATCH 09/20] More fixes for CACHE_WEEKS. --- .github/workflows/composite/conda-pkg-cache/action.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/composite/conda-pkg-cache/action.yml b/.github/workflows/composite/conda-pkg-cache/action.yml index f0aa2ef88e..b4ac914414 100644 --- a/.github/workflows/composite/conda-pkg-cache/action.yml +++ b/.github/workflows/composite/conda-pkg-cache/action.yml @@ -7,10 +7,6 @@ inputs: required: false default: "3.13" -env: - # Maximum cache period (in weeks) before forcing a cache refresh. - CACHE_WEEKS: 2 - runs: using: "composite" steps: @@ -21,7 +17,8 @@ runs: - id: set-cache-period shell: bash env: - CACHE_WEEKS: ${{ env.CACHE_WEEKS }} + # Maximum cache period (in weeks) before forcing a cache refresh. + CACHE_WEEKS: 2 run: echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> $GITHUB_OUTPUT - id: conda-pkg-cache From fea533a93b57c59ea3f699d82db3d3a68a7d5d76 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 17 Nov 2025 17:04:07 +0000 Subject: [PATCH 10/20] Work around actions/runner#2009 . --- .github/workflows/composite/persist-dir-cache/action.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/composite/persist-dir-cache/action.yml b/.github/workflows/composite/persist-dir-cache/action.yml index ae521754b5..831df36212 100644 --- a/.github/workflows/composite/persist-dir-cache/action.yml +++ b/.github/workflows/composite/persist-dir-cache/action.yml @@ -20,6 +20,12 @@ inputs: runs: using: "composite" steps: + # Work around bug where inputs.cache_path is unavailable for the post-cache + # step. https://github.com/actions/runner/issues/2009 + - id: store-path + shell: bash + run: echo "CACHE_PATH=${{ inputs.cache_path }}" >> $GITHUB_ENV + - id: sanitise-cache-path shell: bash run: echo "SANITISED_PATH=$(echo '${{ inputs.cache_path }}' | sed 's|/|_|g')" >> $GITHUB_OUTPUT @@ -38,7 +44,7 @@ runs: - id: dir-cache uses: actions/cache@v4 with: - path: ${{ inputs.cache_path }} + path: ${{ env.CACHE_PATH }} # This pattern allows continual updating of the cache_path with each new # commit. The most recent cache matching CACHE_KEY will be used, while # any updates will create a new cache entry specific to the commit SHA, From 8e3ffc721143afd4de6a67d31ad5fd9274c2d1f0 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 17 Nov 2025 17:37:28 +0000 Subject: [PATCH 11/20] More traditional hashing solution. --- .../composite/conda-pkg-cache/action.yml | 33 +++++++---- .../composite/persist-dir-cache/action.yml | 55 ------------------- 2 files changed, 21 insertions(+), 67 deletions(-) delete mode 100644 .github/workflows/composite/persist-dir-cache/action.yml diff --git a/.github/workflows/composite/conda-pkg-cache/action.yml b/.github/workflows/composite/conda-pkg-cache/action.yml index b4ac914414..4719bb6aa9 100644 --- a/.github/workflows/composite/conda-pkg-cache/action.yml +++ b/.github/workflows/composite/conda-pkg-cache/action.yml @@ -10,10 +10,6 @@ inputs: runs: using: "composite" steps: - - id: set-pkg-dir - shell: bash - run: conda config --add pkgs_dirs ~/conda_pkgs_dir - - id: set-cache-period shell: bash env: @@ -21,13 +17,26 @@ runs: CACHE_WEEKS: 2 run: echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> $GITHUB_OUTPUT - - id: conda-pkg-cache - uses: ./.github/workflows/composite/persist-dir-cache + - id: define-cache-key + shell: bash env: - CACHE_BUILD: ${{ env.CACHE_BUILD }} - with: - cache_path: ~/conda_pkgs_dir # Lets us manually bump the cache to rebuild. - cache_build: 0 - cache_period: ${{ steps.set-cache-period.outputs.CACHE_PERIOD }} - key_suffix: "py${{ inputs.python_version }}" + CACHE_BUILD: "0" + run: | + OS="${{ runner.os }}" + PERIOD="${{ steps.set-cache-period.outputs.CACHE_PERIOD }}" + BUILD="${{ env.CACHE_BUILD }}" + PY="${{ inputs.python_version }}" + REQ="${{ hashFiles('requirements/locks/**') }}" + CACHE_KEY="$conda_pkgs_dir-${OS}-p${PERIOD}-b${BUILD}-py${PY}-${REQ}" + echo "CACHE_KEY=$CACHE_KEY" >> $GITHUB_ENV + + - id: set-pkg-dir + shell: bash + run: conda config --add pkgs_dirs ~/conda_pkgs_dir + + - id: conda-pkg-cache + uses: actions/cache@v4 + with: + path: ~/conda_pkgs_dir + key: ${{ env.CACHE_KEY }} diff --git a/.github/workflows/composite/persist-dir-cache/action.yml b/.github/workflows/composite/persist-dir-cache/action.yml deleted file mode 100644 index 831df36212..0000000000 --- a/.github/workflows/composite/persist-dir-cache/action.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: "persist-dir-cache" -description: "Use caching to persist a directory across workflow runs" - -inputs: - cache_path: - description: "path to the directory to cache" - required: true - cache_build: - description: "persist dir cache build number" - required: false - default: "0" - cache_period: - description: "persist dir cache timestamp" - required: true - key_suffix: - description: "optional suffix to append to the cache key" - required: false - default: "" - -runs: - using: "composite" - steps: - # Work around bug where inputs.cache_path is unavailable for the post-cache - # step. https://github.com/actions/runner/issues/2009 - - id: store-path - shell: bash - run: echo "CACHE_PATH=${{ inputs.cache_path }}" >> $GITHUB_ENV - - - id: sanitise-cache-path - shell: bash - run: echo "SANITISED_PATH=$(echo '${{ inputs.cache_path }}' | sed 's|/|_|g')" >> $GITHUB_OUTPUT - - - id: define-cache-key - shell: bash - run: | - OS="${{ runner.os }}" - DIR_PATH="${{ steps.sanitise-cache-path.outputs.SANITISED_PATH }}" - PERIOD="${{ inputs.cache_period }}" - BUILD="${{ inputs.cache_build }}" - SUFFIX="${{ inputs.key_suffix }}" - CACHE_KEY="${DIR_PATH}_cached-${OS}-p${PERIOD}-b${BUILD}-${SUFFIX}" - echo "CACHE_KEY=$CACHE_KEY" >> $GITHUB_ENV - - - id: dir-cache - uses: actions/cache@v4 - with: - path: ${{ env.CACHE_PATH }} - # This pattern allows continual updating of the cache_path with each new - # commit. The most recent cache matching CACHE_KEY will be used, while - # any updates will create a new cache entry specific to the commit SHA, - # which will then be available to the next run as the new most-recent - # cache. - key: ${{ env.CACHE_KEY }}-${{ github.sha }} - restore-keys: | - ${{ env.CACHE_KEY }} From 1f0a11321f6ca36284322992e9e247b0bd4177d1 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 17 Nov 2025 17:39:16 +0000 Subject: [PATCH 12/20] Fix typo. --- .github/workflows/composite/conda-pkg-cache/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/composite/conda-pkg-cache/action.yml b/.github/workflows/composite/conda-pkg-cache/action.yml index 4719bb6aa9..2feece7da2 100644 --- a/.github/workflows/composite/conda-pkg-cache/action.yml +++ b/.github/workflows/composite/conda-pkg-cache/action.yml @@ -28,7 +28,7 @@ runs: BUILD="${{ env.CACHE_BUILD }}" PY="${{ inputs.python_version }}" REQ="${{ hashFiles('requirements/locks/**') }}" - CACHE_KEY="$conda_pkgs_dir-${OS}-p${PERIOD}-b${BUILD}-py${PY}-${REQ}" + CACHE_KEY="conda_pkgs_dir-${OS}-p${PERIOD}-b${BUILD}-py${PY}-${REQ}" echo "CACHE_KEY=$CACHE_KEY" >> $GITHUB_ENV - id: set-pkg-dir From 45dc2735b304e6fb7249cd49320071079852e892 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Mon, 17 Nov 2025 17:52:51 +0000 Subject: [PATCH 13/20] Dummy commit to demonstrate Conda package caching. --- .github/workflows/composite/conda-pkg-cache/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/composite/conda-pkg-cache/action.yml b/.github/workflows/composite/conda-pkg-cache/action.yml index 2feece7da2..616c76df96 100644 --- a/.github/workflows/composite/conda-pkg-cache/action.yml +++ b/.github/workflows/composite/conda-pkg-cache/action.yml @@ -13,7 +13,7 @@ runs: - id: set-cache-period shell: bash env: - # Maximum cache period (in weeks) before forcing a cache refresh. + # Maximum cache period (in weeks) before forcing a cache refresh.. CACHE_WEEKS: 2 run: echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> $GITHUB_OUTPUT From 20d3b372a8213b0b4c8c7bf3dc65b93891aa9281 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 28 Nov 2025 10:03:31 +0000 Subject: [PATCH 14/20] Align GHA Python versions for better caching. --- .github/workflows/ci-tests.yml | 3 +++ .github/workflows/ci-wheels.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 6f33fad585..9cb036031d 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -87,6 +87,9 @@ jobs: channels: conda-forge activate-environment: ${{ env.ENV_NAME }} auto-update-conda: false + # This aligns the top-level env with the env that Nox will create + # below - meaning optimal use of package caching. + python-version: ${{ matrix.python-version }} - name: "install cartopy + nox + pip" run: conda install --quiet --name ${{ env.ENV_NAME }} cartopy nox pip diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index 968361ea87..a0ab79d1d8 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -89,6 +89,9 @@ jobs: activate-environment: ${{ env.ENV_NAME }} auto-update-conda: false use-only-tar-bz2: true + # This aligns the top-level env with the env that Nox will create + # below - meaning optimal use of package caching. + python-version: ${{ matrix.python-version }} - name: "install nox + pip" run: conda install --quiet --name ${{ env.ENV_NAME }} nox pip From 7635adb27310385c7c6dee16149dc2c6af5962b2 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 28 Nov 2025 10:15:41 +0000 Subject: [PATCH 15/20] Remove defaults channel from ci-wheels. --- .github/workflows/ci-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index a0ab79d1d8..c5d08c3bd6 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -85,7 +85,7 @@ jobs: uses: conda-incubator/setup-miniconda@v3 with: miniforge-version: latest - channels: conda-forge,defaults + channels: conda-forge activate-environment: ${{ env.ENV_NAME }} auto-update-conda: false use-only-tar-bz2: true From 20ed16cff8a5e74613f020574693039111b031c0 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 28 Nov 2025 10:22:21 +0000 Subject: [PATCH 16/20] No longer need to use miniforge. --- .github/workflows/ci-tests.yml | 1 - .github/workflows/ci-wheels.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 9cb036031d..7bad98c946 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -83,7 +83,6 @@ jobs: - name: "conda install" uses: conda-incubator/setup-miniconda@v3 with: - miniforge-version: latest channels: conda-forge activate-environment: ${{ env.ENV_NAME }} auto-update-conda: false diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index c5d08c3bd6..920ed085dc 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -84,7 +84,6 @@ jobs: - name: "conda install" uses: conda-incubator/setup-miniconda@v3 with: - miniforge-version: latest channels: conda-forge activate-environment: ${{ env.ENV_NAME }} auto-update-conda: false From 0a8cbcf45d04744491d50b269664c77a9dc87df0 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 28 Nov 2025 10:27:07 +0000 Subject: [PATCH 17/20] Revert "No longer need to use miniforge." This reverts commit 20ed16cff8a5e74613f020574693039111b031c0. --- .github/workflows/ci-tests.yml | 1 + .github/workflows/ci-wheels.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 7bad98c946..9cb036031d 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -83,6 +83,7 @@ jobs: - name: "conda install" uses: conda-incubator/setup-miniconda@v3 with: + miniforge-version: latest channels: conda-forge activate-environment: ${{ env.ENV_NAME }} auto-update-conda: false diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index 920ed085dc..c5d08c3bd6 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -84,6 +84,7 @@ jobs: - name: "conda install" uses: conda-incubator/setup-miniconda@v3 with: + miniforge-version: latest channels: conda-forge activate-environment: ${{ env.ENV_NAME }} auto-update-conda: false From 7b780a14c6b97fc09bef8421ce43ca8f98a1268d Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 28 Nov 2025 10:28:42 +0000 Subject: [PATCH 18/20] ci-wheels Nox minimum pin. --- .github/workflows/ci-wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index c5d08c3bd6..45e4fcaba1 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -94,7 +94,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: "install nox + pip" - run: conda install --quiet --name ${{ env.ENV_NAME }} nox pip + # Nox minimum pin is just to help avoid a solver bug. + run: conda install --quiet --name ${{ env.ENV_NAME }} "nox>=2025" pip - name: "nox install and test wheel" env: From 6d3d00f668748745a9b22f2d4a8f0952f2bf5e18 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 28 Nov 2025 11:29:28 +0000 Subject: [PATCH 19/20] Revert "ci-wheels Nox minimum pin." This reverts commit 7b780a14c6b97fc09bef8421ce43ca8f98a1268d. --- .github/workflows/ci-wheels.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index 45e4fcaba1..c5d08c3bd6 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -94,8 +94,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: "install nox + pip" - # Nox minimum pin is just to help avoid a solver bug. - run: conda install --quiet --name ${{ env.ENV_NAME }} "nox>=2025" pip + run: conda install --quiet --name ${{ env.ENV_NAME }} nox pip - name: "nox install and test wheel" env: From 957dc9f92a23c41b06d240319da7bec90f936827 Mon Sep 17 00:00:00 2001 From: Martin Yeo Date: Fri, 28 Nov 2025 11:31:52 +0000 Subject: [PATCH 20/20] Remove bz2 only line - this was a legacy workaround, no longer necessary. --- .github/workflows/ci-wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-wheels.yml b/.github/workflows/ci-wheels.yml index c5d08c3bd6..24342c3db4 100644 --- a/.github/workflows/ci-wheels.yml +++ b/.github/workflows/ci-wheels.yml @@ -88,7 +88,6 @@ jobs: channels: conda-forge activate-environment: ${{ env.ENV_NAME }} auto-update-conda: false - use-only-tar-bz2: true # This aligns the top-level env with the env that Nox will create # below - meaning optimal use of package caching. python-version: ${{ matrix.python-version }}