Skip to content

Containers and ec2 changes for 2.1 release #1954

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eap merged 5 commits into from
Mar 13, 2026
Merged

Containers and ec2 changes for 2.1 release #1954

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
06a09ee
A batch of changes from containers and ec2
eap Mar 12, 2026
b3be5ea
Docker fixes
eap Mar 12, 2026
1f8734e
Remove qt and alphabetize
eap Mar 13, 2026
d3e9168
Merge branch 'release/2.1' into feature/container-21release
eap Mar 13, 2026
272b880
notest fix ec2 packages from merge
eap Mar 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions .dockerignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# .dockerignore

# Log files.
log.*
*.log

# spack-stack environments and builds.
envs/
.env/
build/
cache/

# Misc.
.vscode
.Trashes
.github/
venv/
.venv/

39 changes: 30 additions & 9 deletions configs/sites/tier1/container/Dockerfile.gcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,6 @@ RUN set -euo pipefail; \
# External dependencies.
libcurl4-openssl-dev \
libmysqlclient-dev \
libqt5svg5-dev \
qt5-qmake \
qt5dxcb-plugin \
qtbase5-dev \
zstd \
# Shell tools, source retrieval and networking.
sed \
Expand Down Expand Up @@ -72,10 +68,15 @@ FROM ubuntu_base AS builder
ENV SPACK_STACK_DIR=/opt/spack-stack \
SPACK_ROOT=/opt/spack-stack/spack


# Copy spack-stack from build context. Note the build context must be the
# root of the spack-stack repository (see the README.md for details).
COPY . ${SPACK_STACK_DIR}
# Do not copy the dockerfiles into the container.
COPY --exclude=configs/sites/tier1/container/Dockerfile.oneapi \
--exclude=configs/sites/tier1/container/Dockerfile.gcc \
--exclude=configs/sites/tier1/container/packages_oneapi.yaml \
--exclude=envs/** \
--exclude=cache/** \
. ${SPACK_STACK_DIR}

# Create spack-stack environment
WORKDIR /opt/spack-stack
Expand All @@ -89,12 +90,19 @@ RUN mkdir -p /tmp/spack-stack && \
--compiler $COMPILER && \
cd ${SPACK_STACK_DIR}/envs/container && \
spack env activate . && \
spack concretize 2>&1 | tee log.concretize && \
spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install && \
spack concretize 2>&1 | tee log.concretize

# Build the spack environment.
RUN source setup.sh && cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \
spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install

# Setup modules and meta-modules.
RUN source setup.sh && cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \
spack module tcl refresh -y && \
spack stack setup-meta-modules && \
# Save output of spack find.
spack find 2>&1 | tee /opt/spack-software/spack_find.out && \
cp ./log.concretize /opt/spack-software/log.concretize && \
spack clean --all

## ---------- Create runtime container ----------
Expand All @@ -118,7 +126,20 @@ RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \
&& echo "export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe" >> /etc/spack_container_rc.sh \
&& echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \
&& echo "source /etc/profile.d/modules.sh" >> /etc/spack_container_rc.sh \
&& echo "module use /opt/spack-software/modules/Core" >> /etc/spack_container_rc.sh \
&& echo "module use /opt/spack-software/modulefiles" >> /etc/spack_container_rc.sh \
&& echo "module use /opt/spack-software/modulefiles/Core" >> /etc/spack_container_rc.sh \
&& echo "# Bash function to load modules necessary to build the JEDI bundle." >> /etc/spack_container_rc.sh \
&& echo "load_jedi_bundle_env() {" >> /etc/spack_container_rc.sh \
&& echo " module purge" >> /etc/spack_container_rc.sh \
&& echo " module load stack-gcc" >> /etc/spack_container_rc.sh \
&& echo " module load stack-openmpi" >> /etc/spack_container_rc.sh \
&& echo " module load base-env" >> /etc/spack_container_rc.sh \
&& echo " module load jedi-mpas-env" >> /etc/spack_container_rc.sh \
&& echo " module load py-fortranformat" >> /etc/spack_container_rc.sh \
&& echo " module load jedi-fv3-env" >> /etc/spack_container_rc.sh \
&& echo " module load ewok-env" >> /etc/spack_container_rc.sh \
&& echo " module load ip" >> /etc/spack_container_rc.sh \
&& echo "}" >> /etc/spack_container_rc.sh \ \
&& echo "source /etc/spack_container_rc.sh" >> /etc/bash.bashrc \
&& printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig \
&& mkdir /root/.pmix \
Expand Down
39 changes: 30 additions & 9 deletions configs/sites/tier1/container/Dockerfile.oneapi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,6 @@ RUN set -euo pipefail; \
# External dependencies.
libcurl4-openssl-dev \
libmysqlclient-dev \
libqt5svg5-dev \
qt5-qmake \
qt5dxcb-plugin \
qtbase5-dev \
zstd \
# Shell tools, source retrieval and networking.
sed \
Expand Down Expand Up @@ -91,7 +87,13 @@ ENV SPACK_STACK_DIR=/opt/spack-stack \

# Copy spack-stack from build context. Note the build context must be the
# root of the spack-stack repository (see the README.md for details).
COPY . ${SPACK_STACK_DIR}
# Do not copy the dockerfiles into the container.
COPY --exclude=configs/sites/tier1/container/Dockerfile.oneapi \
--exclude=configs/sites/tier1/container/Dockerfile.gcc \
--exclude=configs/sites/tier1/container/packages_gcc.yaml \
--exclude=envs/** \
--exclude=cache/** \
. ${SPACK_STACK_DIR}

# Create spack-stack environment
WORKDIR /opt/spack-stack
Expand All @@ -105,12 +107,19 @@ RUN mkdir -p /tmp/spack-stack && \
--compiler oneapi && \
cd ${SPACK_STACK_DIR}/envs/container && \
spack env activate . && \
spack concretize 2>&1 | tee log.concretize && \
spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install && \
spack concretize 2>&1 | tee log.concretize

# Build the spack environment.
RUN source setup.sh && cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \
spack install --fail-fast -j ${BUILD_JOBS} 2>&1 | tee log.install

# Setup modules and meta-modules.
RUN source setup.sh && cd ${SPACK_STACK_DIR}/envs/container && spack env activate . && \
spack module tcl refresh -y && \
spack stack setup-meta-modules && \
# Save output of spack find.
spack find 2>&1 | tee /opt/spack-software/spack_find.out && \
cp ./log.concretize /opt/spack-software/log.concretize && \
spack clean --all

## ---------- Create runtime container ----------
Expand All @@ -129,10 +138,22 @@ RUN echo "ulimit -s unlimited" > /etc/spack_container_rc.sh \
&& echo "export CXX=icpx" >> /etc/spack_container_rc.sh \
&& echo "export FC=ifx" >> /etc/spack_container_rc.sh \
&& echo "# TCL module path for the spack-stack environment." >> /etc/spack_container_rc.sh \
&& echo "export MODULEPATH=/opt/spack-software/modules/Core" >> /etc/spack_container_rc.sh \
&& echo "source /etc/profile.d/modules.sh" >> /etc/spack_container_rc.sh \
&& echo "module use /opt/intel/oneapi/modulefiles" >> /etc/spack_container_rc.sh \
&& echo "module use /opt/spack-software/modules/Core" >> /etc/spack_container_rc.sh \
&& echo "module use /opt/spack-software/modulefiles" >> /etc/spack_container_rc.sh \
&& echo "module use /opt/spack-software/modulefiles/Core" >> /etc/spack_container_rc.sh \
&& echo "# Bash function to load modules necessary to build the JEDI bundle." >> /etc/spack_container_rc.sh \
&& echo "load_jedi_bundle_env() {" >> /etc/spack_container_rc.sh \
&& echo " module purge" >> /etc/spack_container_rc.sh \
&& echo " module load stack-intel-oneapi-compilers" >> /etc/spack_container_rc.sh \
&& echo " module load stack-intel-oneapi-mpi" >> /etc/spack_container_rc.sh \
&& echo " module load base-env" >> /etc/spack_container_rc.sh \
&& echo " module load jedi-mpas-env" >> /etc/spack_container_rc.sh \
&& echo " module load py-fortranformat" >> /etc/spack_container_rc.sh \
&& echo " module load jedi-fv3-env" >> /etc/spack_container_rc.sh \
&& echo " module load ewok-env" >> /etc/spack_container_rc.sh \
&& echo " module load ip" >> /etc/spack_container_rc.sh \
&& echo "}" >> /etc/spack_container_rc.sh \ \
&& echo "source /etc/spack_container_rc.sh" >> /etc/bash.bashrc \
&& printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig \
&& mkdir /root/.pmix
Expand Down
63 changes: 40 additions & 23 deletions configs/sites/tier1/container/packages.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,30 +1,54 @@
packages:
all:
target: [core2]
target: [x86_64_v3]
awscli-v2:
require:
- "@2.22.4"
cairo:
variants: +pic
ewok-env:
require:
- +ecflow
fontconfig:
variants: +pic
met:
variants: +python +grib2 +graphics +lidar2nc +modis
parallelio:
require:
- +pnetcdf
pixman:
variants: +pic
py-awscrt:
require:
- "@0.19.19"
py-botocore:
require:
- "@1.34.162"
py-boto3:
require:
- "@1.34.162"
py-netcdf4:
require:
- "@1.7.2"

# System packages installed during the container build.
grep:
externals:
- spec: grep@3.11
prefix: /usr
findutils:
autoconf:
externals:
- spec: findutils@4.9.0
- spec: autoconf@2.71
prefix: /usr
coreutils:
externals:
- spec: coreutils@9.4
prefix: /usr
autoconf:
externals:
- spec: autoconf@2.71
prefix: /usr
diffutils:
buildable: false
externals:
- spec: diffutils@3.10
prefix: /usr
findutils:
externals:
- spec: findutils@4.9.0
prefix: /usr
git:
buildable: false
externals:
Expand All @@ -35,24 +59,17 @@ packages:
externals:
- spec: git-lfs@3.4.1
prefix: /usr
llvm:
buildable: false
externals:
- spec: llvm@14.0.6
prefix: /usr
qt:
buildable: false
grep:
externals:
- spec: qt@5.15.3
- spec: grep@3.11
prefix: /usr
version: [5.15.3]
wget:
llvm:
buildable: false
externals:
- spec: wget@1.21.4
- spec: llvm@14.0.6
prefix: /usr
perl:
buildable: false
externals:
- spec: perl@5.38.2
prefix: /usr
prefix: /usr
4 changes: 2 additions & 2 deletions configs/sites/tier1/container/packages_oneapi.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ packages:
modules:
- umf/1.0.2
- tbb/2022.3
- compiler-rt/2025.3.0
- compiler/2025.3.0
- compiler-rt/2025.3.2
- compiler/2025.3.2
extra_attributes:
compilers:
c: /opt/intel/oneapi/compiler/2025.3/bin/icx
Expand Down
30 changes: 26 additions & 4 deletions configs/sites/tier2/aws-ubuntu2404/packages.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,25 @@
packages:
# Package constraints.
awscli-v2:
require:
- "@2.22.4"
ewok-env:
require:
- +ecflow
py-awscrt:
require:
- "@0.19.19"
py-botocore:
require:
- "@1.34.162"
py-boto3:
require:
- "@1.34.162"
py-netcdf4:
require:
- "@1.7.2"

# System packages.
autoconf:
externals:
- spec: autoconf@2.71
Expand Down Expand Up @@ -31,10 +52,6 @@ packages:
externals:
- spec: gawk@5.2.1
prefix: /usr
gettext:
externals:
- spec: gettext@0.21
prefix: /usr
git:
externals:
- spec: git@2.43.0~tcltk
Expand All @@ -44,9 +61,14 @@ packages:
- spec: git-lfs@3.4.1
prefix: /usr
gmake:
buildable: false
externals:
- spec: gmake@4.3
prefix: /usr
grep:
externals:
- spec: grep@3.11
prefix: /usr
groff:
externals:
- spec: groff@1.23.0
Expand Down
Loading