diff --git a/.dockerignore b/.dockerignore index 09b3588..26b0a5e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,3 +14,8 @@ packages packages-docker runtime.*.liblouis/runtimes LibLouis.NET.Tables/tables + +# Not needed inside the image, and excluding them means editing the build definition does not +# invalidate every COPY . /source and force a full recompile of every target. +Dockerfile +.dockerignore diff --git a/Dockerfile b/Dockerfile index 828c6fb..bdd59c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# The pinned platform below is deliberate, see the comment on the FROM line. +# The pinned platforms below are deliberate, see the comment on the first FROM. # check=skip=FromPlatformFlagConstDisallowed # Builds the native liblouis binaries for the Linux and Windows runtime identifiers, packs one @@ -8,33 +8,43 @@ # native-macos CI job. The managed packages are not built here either, because building them # resolves runtime.liblouis, which depends on the macOS packages this container cannot produce. # -# The gcc and llvm-mingw targets are separate stages on purpose. llvm-mingw also ships -# i686-w64-mingw32-gcc and x86_64-w64-mingw32-gcc, so having it installed alongside the Ubuntu cross -# compilers means it can displace them and silently change which toolchain builds win-x86 and -# win-x64. Keeping it out of that stage entirely makes the mistake impossible rather than merely -# documented. BuildKit also builds the independent stages concurrently, so the wall clock is the -# slowest stage rather than the sum. +# Compiling and packing are separate stages, each on an image chosen for the job. +# +# The compilers do not need a .NET SDK: dotnet appears exactly once in the native build, to pack an +# already-compiled binary into a .nupkg. Building C on a dotnet/sdk image meant apt-get installing a +# toolchain onto an image picked for something else, and pulling in packages the build never uses - +# which is how a 404 on linux-libc-dev, a dependency of build-essential, once failed CI. +# +# So: toolchain images compile and stage binaries, and the SDK image packs whatever it finds. The +# SDK stage installs one package, llvm, because packing verifies the binaries it is handed and the +# SDK image ships no object readers - see the comment on that stage. # # --platform is pinned because the cross toolchain package names below only exist for amd64. On an # Apple Silicon machine this runs under emulation: slower, but it works. -FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:8.0-jammy AS base -LABEL org.opencontainers.image.source=https://github.com/Notalib/LibLouis.NET/ + + +# The five targets Ubuntu has cross compilers for. +FROM --platform=linux/amd64 ubuntu:noble AS gcc-build # Retried, because a single apt-get run is a coin flip against archive.ubuntu.com: the index and # the pool are not updated atomically, so a package version can be listed after it has been removed -# and the fetch 404s. That is what it did. Each attempt refreshes the index first, since a newer -# index is usually what resolves it. The explicit ok check matters: without it a loop that never -# succeeds still falls through and the layer builds with nothing installed. +# and the fetch 404s. Each attempt refreshes the index first, since a newer index is usually what +# resolves it. The explicit ok check matters: without it a loop that never succeeds still falls +# through and the layer builds with nothing installed. RUN set -eu; \ ok=0; \ for attempt in 1 2 3; do \ - if apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ + if apt-get update && apt-get install -y --no-install-recommends \ build-essential \ ca-certificates \ curl \ m4 \ - xz-utils \ - llvm; then \ + gcc-i686-linux-gnu \ + gcc-aarch64-linux-gnu \ + gcc-mingw-w64-i686 \ + gcc-mingw-w64-x86-64 \ + libc6-dev-i386-cross \ + libc6-dev-arm64-cross; then \ ok=1; break; \ fi; \ echo "apt attempt $attempt failed, retrying" >&2; \ @@ -42,27 +52,54 @@ RUN set -eu; \ done; \ [ "$ok" = 1 ] || exit 1; \ rm -rf /var/lib/apt/lists/* +# The cross gcc packages above only Recommend their target libc, so with --no-install-recommends +# they would install a compiler that cannot link. libc6-dev-*-cross are named explicitly rather +# than dropping the flag, so the requirement is visible. -ENV PACKAGE_OUTPUT_DIR=/packages +ENV SKIP_PACK=1 WORKDIR /source +COPY . /source +RUN sh ./build/build_runtime_packages.sh gcc -# The five targets Ubuntu has cross compilers for. -FROM base AS gcc-targets +# win-arm64. Ubuntu has no aarch64 mingw-w64 cross compiler, so this uses llvm-mingw, taken from +# the image its own author publishes and pinned to a dated release rather than downloaded and +# checksummed by hand. The image already carries make, m4, curl and the toolchain on PATH, so this +# stage installs nothing. +# +# It is a separate stage from the gcc targets, and that separation is load bearing: llvm-mingw also +# ships i686-w64-mingw32-gcc and x86_64-w64-mingw32-gcc, so having it on PATH alongside the Ubuntu +# cross compilers silently takes over the win-x86 and win-x64 builds. That is not hypothetical - it +# happened, and it broke win-x86, because clang treats the -Wincompatible-pointer-types that gnulib +# trips on mingw as an error where gcc only warns. +FROM --platform=linux/amd64 mstorsjo/llvm-mingw:20260616 AS llvm-build + +ENV SKIP_PACK=1 +WORKDIR /source +COPY . /source +RUN sh ./build/build_runtime_packages.sh llvm -# The cross gcc packages only Recommend their target libc, so with --no-install-recommends they -# install a compiler that cannot link. Named explicitly rather than dropping the flag, so the -# requirement is visible. Retried for the same reason as the base stage. + +# Packs what the toolchain stages produced, and the metapackage, which needs no native binary at +# all. Installs nothing: dotnet pack is the only thing this stage does. +# +# The SDK version barely affects the output here - these packages are netstandard2.0 metadata around +# an already-compiled binary, with IncludeBuildOutput off - but .NET 8 goes out of support in +# November 2026, and there is no reason for the build to be the thing still on it. +FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:10.0-noble AS pack +LABEL org.opencontainers.image.source=https://github.com/Notalib/LibLouis.NET/ + +ENV PACKAGE_OUTPUT_DIR=/packages +WORKDIR /source + +# llvm is not here to compile anything: packing verifies each staged binary first, and the SDK image +# carries no object readers at all - not even binutils. verify_native_binary.sh prefers the llvm +# tools anyway, and for the Windows targets it requires them, since binutils cannot read aarch64 PE. +# Retried for the same reason as the toolchain stages above. RUN set -eu; \ ok=0; \ for attempt in 1 2 3; do \ - if apt-get update && apt-get install -y --no-install-recommends \ - gcc-i686-linux-gnu \ - gcc-aarch64-linux-gnu \ - gcc-mingw-w64-i686 \ - gcc-mingw-w64-x86-64 \ - libc6-dev-i386-cross \ - libc6-dev-arm64-cross; then \ + if apt-get update && apt-get install -y --no-install-recommends llvm; then \ ok=1; break; \ fi; \ echo "apt attempt $attempt failed, retrying" >&2; \ @@ -72,42 +109,18 @@ RUN set -eu; \ rm -rf /var/lib/apt/lists/* COPY . /source -RUN sh ./build/build_runtime_packages.sh gcc - -# win-arm64. Ubuntu has no aarch64 mingw-w64 cross compiler, so this stage uses the prebuilt -# llvm-mingw toolchain, pinned by digest: an unpinned toolchain would silently change what the -# published binaries were built with. Bump both values together when moving to a newer release. -FROM base AS llvm-targets +COPY --from=gcc-build /source/runtime.linux-x86.liblouis/runtimes /source/runtime.linux-x86.liblouis/runtimes +COPY --from=gcc-build /source/runtime.linux-x64.liblouis/runtimes /source/runtime.linux-x64.liblouis/runtimes +COPY --from=gcc-build /source/runtime.linux-arm64.liblouis/runtimes /source/runtime.linux-arm64.liblouis/runtimes +COPY --from=gcc-build /source/runtime.win-x86.liblouis/runtimes /source/runtime.win-x86.liblouis/runtimes +COPY --from=gcc-build /source/runtime.win-x64.liblouis/runtimes /source/runtime.win-x64.liblouis/runtimes +COPY --from=llvm-build /source/runtime.win-arm64.liblouis/runtimes /source/runtime.win-arm64.liblouis/runtimes -ARG LLVM_MINGW_VERSION=20260616 -ARG LLVM_MINGW_SHA256=534b92e067b22a6b4441f48ae9240a3341b17825d04d577eab0cf85c44b4deda -RUN set -eu; \ - archive="llvm-mingw-${LLVM_MINGW_VERSION}-ucrt-ubuntu-22.04-x86_64.tar.xz"; \ - curl -fL -o "/tmp/$archive" \ - "https://github.com/mstorsjo/llvm-mingw/releases/download/${LLVM_MINGW_VERSION}/$archive"; \ - echo "${LLVM_MINGW_SHA256} /tmp/$archive" | sha256sum --check; \ - mkdir -p /opt/llvm-mingw; \ - tar xf "/tmp/$archive" -C /opt/llvm-mingw --strip-components=1; \ - rm "/tmp/$archive" - -# The build script prepends this to PATH for the targets that need it. Left off PATH here so there -# is exactly one mechanism selecting the toolchain, in the script, where it is visible. -ENV LLVM_MINGW_BIN=/opt/llvm-mingw/bin - -COPY . /source -RUN sh ./build/build_runtime_packages.sh llvm - - -# The metapackage is pure metadata and needs no toolchain at all. -FROM base AS metapackage - -COPY . /source -RUN sh ./build/build_metapackage.sh +RUN sh ./build/pack_runtime_packages.sh && \ + sh ./build/build_metapackage.sh # `docker build --output=packages .` exports just the .nupkg files into ./packages. FROM scratch -COPY --from=gcc-targets /packages/* / -COPY --from=llvm-targets /packages/* / -COPY --from=metapackage /packages/* / +COPY --from=pack /packages/* / diff --git a/PACKAGING.md b/PACKAGING.md index 8b41840..77b68a0 100644 --- a/PACKAGING.md +++ b/PACKAGING.md @@ -94,13 +94,32 @@ NUGET_LOCAL_FEED=$PWD/packages sh build/build_managed_packages.sh ### Cross-compilation Linux and Windows binaries are cross-compiled in the container defined by `Dockerfile`. The -toolchain package names there only exist for amd64, so the image is pinned to `linux/amd64` and -runs under emulation on Apple Silicon. +toolchain package names there only exist for amd64, so the images are pinned to `linux/amd64` and +run under emulation on Apple Silicon. -`win-arm64` is the exception: Ubuntu has no aarch64 mingw-w64 gcc, so the image installs the -[llvm-mingw](https://github.com/mstorsjo/llvm-mingw) toolchain, pinned by SHA-256. It provides -`aarch64-w64-mingw32-gcc` driver wrappers, so the build script treats it like the other Windows -targets. Bump `LLVM_MINGW_VERSION` and `LLVM_MINGW_SHA256` together. +Compiling and packing are separate stages, on images chosen for the job: + +| Stage | Image | Installs | +| --- | --- | --- | +| `gcc-build` | `ubuntu:noble` | the cross toolchains | +| `llvm-build` | `mstorsjo/llvm-mingw` | nothing | +| `pack` | `dotnet/sdk` | nothing | + +dotnet appears exactly once in the native build, to pack an already-compiled binary. Compiling C on +a `dotnet/sdk` image meant apt-getting a toolchain onto an image chosen for something else, and +pulling in packages the build never uses — which is how a 404 on `linux-libc-dev`, a dependency of +`build-essential`, once failed CI. The toolchain stages now leave binaries staged under +`runtime..liblouis/runtimes/`, and `build/pack_runtime_packages.sh` packs whatever it finds +rather than a list that could fall out of step. Finding nothing is an error. + +`win-arm64` is the exception among the Windows targets: Ubuntu has no aarch64 mingw-w64 gcc, so it +uses [llvm-mingw](https://github.com/mstorsjo/llvm-mingw), taken from the image its own author +publishes and pinned to a dated release. That image already carries `make`, `m4`, `curl` and the +toolchain on `PATH`. + +The `Dockerfile` is excluded from the build context. It is not needed inside any image, and +excluding it means editing the build definition does not invalidate every `COPY . /source` and +force a full recompile of every target. llvm-mingw lives in its own container stage, and that separation is load bearing. It also ships `i686-w64-mingw32-gcc` and `x86_64-w64-mingw32-gcc`, so merely having it on `PATH` alongside the diff --git a/build/build_runtime_packages.sh b/build/build_runtime_packages.sh index 3b1c688..211aa89 100755 --- a/build/build_runtime_packages.sh +++ b/build/build_runtime_packages.sh @@ -102,7 +102,11 @@ build_runtime_nuget() { make distclean ) - pack_runtime_package "$rid" + # SKIP_PACK leaves the binary staged without packing it, for a container stage that has a + # toolchain but no .NET SDK. A later stage packs what this one produced. + if [ -z "${SKIP_PACK:-}" ]; then + pack_runtime_package "$rid" + fi } if [ "$group" = "gcc" ] || [ "$group" = "all" ]; then diff --git a/build/pack_runtime_packages.sh b/build/pack_runtime_packages.sh new file mode 100755 index 0000000..2c6650a --- /dev/null +++ b/build/pack_runtime_packages.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Packs a runtime package for every RID that has a native binary staged. +# +# Compiling and packing happen in different container stages: the toolchain images have no .NET SDK +# and the SDK image has no cross compilers. The build stages leave binaries staged under +# runtime..liblouis/runtimes/, and this packs whatever it finds. +# +# Packing whatever is present, rather than a list kept here, means the list cannot fall out of step +# with the RIDs the build stages actually produce. Finding nothing is an error: a stage that copied +# no binaries would otherwise produce an empty set of packages and look like a success. + +. "$(dirname "$0")/common.sh" + +packed=0 + +for project in "$REPO_ROOT"/runtime.*.liblouis; do + rid=$(basename "$project" | sed 's/^runtime\.\(.*\)\.liblouis$/\1/') + + # The metapackage has no RID of its own and no native payload. + if [ "$rid" = "liblouis" ]; then + continue + fi + + if [ -z "$(find "$project/runtimes" -type f 2>/dev/null | head -n 1)" ]; then + continue + fi + + pack_runtime_package "$rid" + packed=$((packed + 1)) +done + +if [ "$packed" -eq 0 ]; then + echo "pack_runtime_packages: no staged native binaries found under runtime.*.liblouis/runtimes/" >&2 + exit 1 +fi + +echo "==> Packed $packed runtime package(s) into $PACKAGE_OUTPUT_DIR"