From bb62ce6f2a58d35800732c2a62d9a049e4bcd4ac Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Thu, 13 Nov 2025 12:57:17 -0600
Subject: [PATCH 1/7] feat: add CI job for default container build without args

---
 .gitlab-ci.yml | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3a2caec6..576f925f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -343,6 +343,36 @@ base:
         let attempts=$attempts+1 ;
       done
 
+base:no-args:
+  extends: .build
+  stage: base
+  needs:
+    - version
+  script:
+    - attempts=0
+    - nocache=""
+    - while !
+      docker buildx build --push ${BUILD_OPTIONS} ${nocache}
+                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-amd64
+                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-amd64
+                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-amd64
+                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-amd64
+                   --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-amd64,mode=max
+                   --file containers/debian/Dockerfile
+                   containers/debian
+                   2>&1 | tee build.log
+      ; do
+        if grep "unknown blob" build.log ; then
+          nocache="--no-cache" ;
+        else
+          exit 1 ;
+        fi ;
+        if test ${attempts} -ge 1 ; then
+          echo "Failed to build on second attempt!" ;
+          exit 1 ;
+        fi ;
+        let attempts=$attempts+1 ;
+      done
 
 eic:
   parallel:
@@ -496,6 +526,38 @@ eic:
         let attempts=$attempts+1 ;
       done
 
+eic:noargs:
+  extends: .build
+  stage: eic
+  needs:
+    - version
+    - base
+  script:
+    - attempts=0
+    - nocache=""
+    - while !
+      docker buildx build ${BUILD_OPTIONS} ${nocache}
+                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}${ENV}-${CI_COMMIT_REF_SLUG}-noargs-amd64
+                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}${ENV}-${CI_COMMIT_REF_SLUG}-noargs-amd64
+                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}${ENV}-${CI_DEFAULT_BRANCH_SLUG}-noargs-amd64
+                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}${ENV}-${CI_DEFAULT_BRANCH_SLUG}-noargs-amd64
+                   --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}${ENV}-${CI_COMMIT_REF_SLUG}-noargs-amd64,mode=max
+                   --file containers/eic/Dockerfile
+                   containers/eic
+                   2>&1 | tee build.log
+      ; do
+        if grep "unknown blob" build.log ; then
+          nocache="--no-cache" ;
+        else
+          exit 1 ;
+        fi ;
+        if test ${attempts} -ge 1 ; then
+          echo "Failed to build on second attempt!" ;
+          exit 1 ;
+        fi ;
+        let attempts=$attempts+1 ;
+      done
+
 user_spack_environment:
   stage: benchmarks
   needs:

From cfa3372600d44da9c712c0b3633044413e240232 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Sun, 16 Nov 2025 11:22:25 -0600
Subject: [PATCH 2/7] fix: add noargs to debian layer cache

---
 .gitlab-ci.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 576f925f..812e2b4d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -353,11 +353,11 @@ base:no-args:
     - nocache=""
     - while !
       docker buildx build --push ${BUILD_OPTIONS} ${nocache}
-                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-amd64
-                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-amd64
-                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-amd64
-                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-amd64
-                   --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-amd64,mode=max
+                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-noargs-amd64
+                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-noargs-amd64
+                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-noargs-amd64
+                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-noargs-amd64
+                   --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-noargs-amd64,mode=max
                    --file containers/debian/Dockerfile
                    containers/debian
                    2>&1 | tee build.log

From 59f4d4e154ac1a88ce2b885b4949514bc0d94eca Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Sun, 16 Nov 2025 11:29:35 -0600
Subject: [PATCH 3/7] fix: don't push without tag

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 812e2b4d..9c91e3d0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -352,7 +352,7 @@ base:no-args:
     - attempts=0
     - nocache=""
     - while !
-      docker buildx build --push ${BUILD_OPTIONS} ${nocache}
+      docker buildx build ${BUILD_OPTIONS} ${nocache}
                    --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-noargs-amd64
                    --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-noargs-amd64
                    --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-noargs-amd64

From 020c7e2f6e766d48dd7cc316f1f5902c41f37b56 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Sun, 16 Nov 2025 11:42:26 -0600
Subject: [PATCH 4/7] fix: rename jobs to base:noargs; cache from/to
 buildcache:base-noargs

---
 .gitlab-ci.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9c91e3d0..6a058545 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -343,7 +343,7 @@ base:
         let attempts=$attempts+1 ;
       done
 
-base:no-args:
+base:noargs:
   extends: .build
   stage: base
   needs:
@@ -353,11 +353,11 @@ base:no-args:
     - nocache=""
     - while !
       docker buildx build ${BUILD_OPTIONS} ${nocache}
-                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-noargs-amd64
-                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-noargs-amd64
-                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-noargs-amd64
-                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG}-noargs-amd64
-                   --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG}-noargs-amd64,mode=max
+                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:base-noargs-${CI_COMMIT_REF_SLUG}-amd64
+                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:base-noargs-${CI_COMMIT_REF_SLUG}-amd64
+                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:base-noargs-${CI_DEFAULT_BRANCH_SLUG}-amd64
+                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:base-noargs-${CI_DEFAULT_BRANCH_SLUG}-amd64
+                   --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:base-noargs-${CI_COMMIT_REF_SLUG}-amd64,mode=max
                    --file containers/debian/Dockerfile
                    containers/debian
                    2>&1 | tee build.log

From d929b7b207e708b2521b874c37573205f9b344f0 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Sun, 16 Nov 2025 11:38:02 -0600
Subject: [PATCH 5/7] fix: ensure ~/.spack exists; update docs

---
 containers/debian/Dockerfile |  3 ++-
 containers/debian/README.md  | 10 ++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/containers/debian/Dockerfile b/containers/debian/Dockerfile
index ee4583ee..af9398af 100644
--- a/containers/debian/Dockerfile
+++ b/containers/debian/Dockerfile
@@ -237,6 +237,7 @@ target=${target[${TARGETPLATFORM}]}
 spack config --scope site add "packages:all:require:[target=${target}]"
 spack config --scope site add "packages:all:target:[${target}]"
 spack config blame packages
+mkdir -p $HOME/.spack/
 spack config --scope user add "config:suppress_gpg_warnings:true"
 spack config --scope user add "config:build_jobs:${jobs}"
 spack config --scope user add "config:db_lock_timeout:${jobs}00"
@@ -278,4 +279,4 @@ set -e
 git clone --filter=tree:0 https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT}
 git -C ${EICSPACK_ROOT} checkout ${EICSPACK_VERSION}
 spack repo add --scope site "${EICSPACK_ROOT}"
-EOF
\ No newline at end of file
+EOF
diff --git a/containers/debian/README.md b/containers/debian/README.md
index 46aaf43b..34bf9cbb 100644
--- a/containers/debian/README.md
+++ b/containers/debian/README.md
@@ -3,3 +3,13 @@ To build the container in this directory:
 ```
 docker buildx build -f Dockerfile .
 ```
+
+**Note:** The minimal build command shown above will assume default values for all build arguments. For this container, that means 'develop' versions for all spack repositories:
+- `SPACK_VERSION=develop`
+- `SPACKPACKAGES_VERSION=develop`
+- `KEY4HEPSPACK_VERSION=main`
+- `EICSPACK_VERSION=develop`
+
+**Important:** Docker layer caching will not automatically update the previous checkout of these 'develop' versions. To ensure you have the latest commits, you may need to use `--no-cache` or rebuild without cache.
+
+For specific operations as used in CI builds (including custom build arguments and cache management), please refer to the GitHub and GitLab CI workflows in this repository.

From eb7054a0e31d7165353d171e42576b6823356d51 Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Sun, 16 Nov 2025 11:44:08 -0600
Subject: [PATCH 6/7] fix: cache from/to buildcache:eic-noargs

---
 .gitlab-ci.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6a058545..7c77d1d7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -537,11 +537,11 @@ eic:noargs:
     - nocache=""
     - while !
       docker buildx build ${BUILD_OPTIONS} ${nocache}
-                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}${ENV}-${CI_COMMIT_REF_SLUG}-noargs-amd64
-                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}${ENV}-${CI_COMMIT_REF_SLUG}-noargs-amd64
-                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}${ENV}-${CI_DEFAULT_BRANCH_SLUG}-noargs-amd64
-                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:${BUILD_IMAGE}${ENV}-${CI_DEFAULT_BRANCH_SLUG}-noargs-amd64
-                   --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:${BUILD_IMAGE}${ENV}-${CI_COMMIT_REF_SLUG}-noargs-amd64,mode=max
+                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:eic-noargs-${CI_COMMIT_REF_SLUG}-amd64
+                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:eic-noargs--${CI_COMMIT_REF_SLUG}-amd64
+                   --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:eic-noargs--${CI_DEFAULT_BRANCH_SLUG}-amd64
+                   --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:eic-noargs--${CI_DEFAULT_BRANCH_SLUG}-amd64
+                   --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:eic-noargs--${CI_COMMIT_REF_SLUG}-amd64,mode=max
                    --file containers/eic/Dockerfile
                    containers/eic
                    2>&1 | tee build.log

From f8dcdc5803e670303ad59c520af4b42bb323a89a Mon Sep 17 00:00:00 2001
From: Wouter Deconinck <wdconinc@gmail.com>
Date: Sun, 16 Nov 2025 12:31:46 -0600
Subject: [PATCH 7/7] fix: add docs on required build-context argument

---
 .gitlab-ci.yml            |  1 +
 containers/eic/Dockerfile |  4 ++++
 containers/eic/README.md  | 13 +++++++++++++
 3 files changed, 18 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7c77d1d7..696b6d42 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -542,6 +542,7 @@ eic:noargs:
                    --cache-from type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:eic-noargs--${CI_DEFAULT_BRANCH_SLUG}-amd64
                    --cache-from type=registry,ref=${GH_REGISTRY}/${GH_REGISTRY_USER}/buildcache:eic-noargs--${CI_DEFAULT_BRANCH_SLUG}-amd64
                    --cache-to type=registry,ref=${CI_REGISTRY}/${CI_PROJECT_PATH}/buildcache:eic-noargs--${CI_COMMIT_REF_SLUG}-amd64,mode=max
+                   --build-context spack-environment=spack-environment
                    --file containers/eic/Dockerfile
                    containers/eic
                    2>&1 | tee build.log
diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile
index 4e0c5422..b6869259 100644
--- a/containers/eic/Dockerfile
+++ b/containers/eic/Dockerfile
@@ -41,6 +41,8 @@ ARG TARGETPLATFORM
 LABEL org.opencontainers.image.title="Electron-Ion Collider build concretization image (default configuration, $TARGETPLATFORM)"
 
 ## Copy our default environment
+## Note that a named build context is *required* for this build:
+## e.g. --build-context spack-environment=../../spack-environment
 COPY --from=spack-environment . /opt/spack-environment/
 ARG ENV=xl
 ENV SPACK_ENV=/opt/spack-environment/${ENV}
@@ -114,6 +116,8 @@ ARG TARGETPLATFORM
 LABEL org.opencontainers.image.title="Electron-Ion Collider runtime concretization image (default configuration, $TARGETPLATFORM)"
 
 ## Copy our default environment
+## Note that a named build context is *required* for this build:
+## e.g. --build-context spack-environment=../../spack-environment
 COPY --from=spack-environment . /opt/spack-environment/
 ARG ENV=xl
 ENV SPACK_ENV=/opt/spack-environment/${ENV}
diff --git a/containers/eic/README.md b/containers/eic/README.md
index 859d7466..30c4f0d2 100644
--- a/containers/eic/README.md
+++ b/containers/eic/README.md
@@ -3,3 +3,16 @@ To build the container in this directory:
 ```
 docker buildx build -f Dockerfile --build-context spack-environment=../../spack-environment .
 ```
+
+**Note:** The minimal build command shown above will assume default values for all build arguments. For this container, that means 'develop' versions for all spack repositories:
+- `BUILDER_IMAGE=debian_stable_base`
+- `RUNTIME_IMAGE=debian_stable_base`
+- `ENV=xl`
+- `BENCHMARK_COM_VERSION=master`
+- `BENCHMARK_DET_VERSION=master`
+- `BENCHMARK_REC_VERSION=master`
+- `BENCHMARK_PHY_VERSION=master`
+
+**Important:** Docker layer caching will not automatically update the previous checkout of these 'master' versions. To ensure you have the latest commits, you may need to use `--no-cache` or rebuild without cache.
+
+For specific operations as used in CI builds (including custom build arguments and cache management), please refer to the GitHub and GitLab CI workflows in this repository.