From 26b7b7f904a9c06e25827738025cd58c173e71f5 Mon Sep 17 00:00:00 2001 From: iHsin Date: Tue, 28 Apr 2026 19:39:59 +0800 Subject: [PATCH 01/21] ci: rework sccache for cross-rs/cross Key changes: - Pin sccache to v0.12.0 (was auto-pulling latest, causing instability) - Add SCCACHE_DIRECT=true for direct hashing mode - Add SCCACHE_DIR=/tmp/sccache for persistent local disk cache - Add CARGO_BUILD_PIPELINING=false to avoid rmeta race conditions (https://github.com/rust-lang/cargo/issues/16790) - Mount /tmp from host via volumes in Cross.toml for cache persistence - Pass GHA cache env vars through to container (SCCACHE_GHA_ENABLED etc.) - Dynamically detect container arch to download matching sccache binary (handles x86_64, aarch64, armv7, i686 via QEMU) Rationale: The original issue #2687 was caused by cargo pipelining starting dependent crate compilation before dependency .rmeta files were fully available. sccache then tried to hash the missing .rmeta file, resulting in a fatal error. The fix disables pipelining at the cargo level and uses sccache's direct mode for safer hashing. --- .cross/Dockerfile | 24 ++++++++++++++++++++++-- .github/sccache.sh | 30 ++++++++++++++++++++++++++++++ Cross.toml | 17 ++++++++++++++--- 3 files changed, 66 insertions(+), 5 deletions(-) create mode 100755 .github/sccache.sh diff --git a/.cross/Dockerfile b/.cross/Dockerfile index 240830c..12929a2 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -3,10 +3,30 @@ ARG DEBIAN_FRONTEND=noninteractive FROM $CROSS_BASE_IMAGE +# Install system deps (clang/llvm for C compilation, curl for sccache download) +# then install sccache, clean up to keep image lean +COPY .github/sccache.sh / RUN apt-get update && \ - apt-get install -y --no-install-recommends clang llvm && \ + apt-get install -y --no-install-recommends clang llvm curl ca-certificates && \ + chmod +x /sccache.sh && \ + arch=$(uname -m) && \ + case "$arch" in \ + x86_64) triple="x86_64-unknown-linux-musl" ;; \ + aarch64) triple="aarch64-unknown-linux-musl" ;; \ + armv7l|armv7) triple="armv7-unknown-linux-musleabi" ;; \ + i686|i586) triple="i686-unknown-linux-musl" ;; \ + *) echo "FATAL: unsupported architecture for sccache: $arch" && exit 1 ;; \ + esac && \ + echo "Detected arch=$arch, downloading sccache for $triple" && \ + /sccache.sh "$triple" && \ + apt-get purge -y curl ca-certificates && \ + apt-get autoremove -y && \ rm -rf /var/lib/apt/lists/* ENV CC=clang \ CXX=clang++ \ - AR=llvm-ar + AR=llvm-ar \ + RUSTC_WRAPPER="/usr/bin/sccache" \ + SCCACHE_DIRECT=true \ + SCCACHE_DIR="/tmp/sccache" \ + CARGO_BUILD_PIPELINING=false diff --git a/.github/sccache.sh b/.github/sccache.sh new file mode 100755 index 0000000..e79ed48 --- /dev/null +++ b/.github/sccache.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Install sccache from GitHub releases. +# Assumes curl is already installed in the image. +# Usage: sccache.sh +# e.g. sccache.sh x86_64-unknown-linux-musl + +set -x +set -euo pipefail + +main() { + local triple="${1}" + local version="v0.12.0" + local url="https://github.com/mozilla/sccache/releases/download/${version}/sccache-${version}-${triple}.tar.gz" + local td + + td="$(mktemp -d)" + pushd "${td}" + + curl -LSfs "${url}" -o sccache.tar.gz + tar -xzf sccache.tar.gz + cp "sccache-${version}-${triple}/sccache" "/usr/bin/sccache" + chmod +x "/usr/bin/sccache" + + popd + rm -rf "${td}" + rm -f "${0}" +} + +main "${@}" diff --git a/Cross.toml b/Cross.toml index 5f16b0f..581dd6e 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,8 +1,19 @@ [build.env] -passthrough = ["RUSTFLAGS", "RUSTC_BOOTSTRAP"] +# Pass through GHA cache env vars for sccache to use GHA backend +# (may not work inside Docker container, fallback is SCCACHE_DIR) +passthrough = [ + "RUSTFLAGS", + "RUSTC_BOOTSTRAP", + "SCCACHE_GHA_ENABLED", + "ACTIONS_RESULTS_URL", + "ACTIONS_RUNTIME_TOKEN", + "ACTIONS_CACHE_URL", +] + +# Mount host /tmp so sccache's disk cache persists across container runs +volumes = ["/tmp=/tmp"] # Use clang as the C/C++ compiler to avoid the aws-lc-sys GCC memcmp bug # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95189 -# The custom Dockerfile installs clang/llvm and sets CC=clang, CXX=clang++, AR=llvm-ar [build.dockerfile] -file = ".cross/Dockerfile" \ No newline at end of file +file = ".cross/Dockerfile" From 2a21713da9536e88374a9466e0488252573f78b5 Mon Sep 17 00:00:00 2001 From: iHsin Date: Tue, 28 Apr 2026 19:56:21 +0800 Subject: [PATCH 02/21] fix: remove GHA cache env passthrough in Cross.toml cross Docker images lack CA certificates; sccache's GHA cache client fails with TLS handshake errors when trying to reach the GHA cache endpoint from inside the container. sccache now relies solely on the local disk cache at SCCACHE_DIR=/tmp/sccache, which persists across container runs via the /tmp volume mount. Native (non-cross) builds still use the host-side GHA cache via mozilla-actions/sccache-action. --- Cross.toml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Cross.toml b/Cross.toml index 581dd6e..74b6508 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,14 +1,8 @@ [build.env] -# Pass through GHA cache env vars for sccache to use GHA backend -# (may not work inside Docker container, fallback is SCCACHE_DIR) -passthrough = [ - "RUSTFLAGS", - "RUSTC_BOOTSTRAP", - "SCCACHE_GHA_ENABLED", - "ACTIONS_RESULTS_URL", - "ACTIONS_RUNTIME_TOKEN", - "ACTIONS_CACHE_URL", -] +# GHA cache endpoint is NOT reliably accessible from inside cross Docker +# containers (missing CA certs, network restrictions). sccache relies on +# the local disk cache at SCCACHE_DIR=/tmp/sccache instead. +passthrough = ["RUSTFLAGS", "RUSTC_BOOTSTRAP"] # Mount host /tmp so sccache's disk cache persists across container runs volumes = ["/tmp=/tmp"] From e7d25b3dec85fff10672c42d49afe7893ca2481a Mon Sep 17 00:00:00 2001 From: iHsin Date: Tue, 28 Apr 2026 20:12:17 +0800 Subject: [PATCH 03/21] fix: add pre-build to ensure /target is writable inside cross container cross mounts the host target/ directory at /target with --user, which can prevent cargo from creating the required subdirectory structure (e.g. aarch64-unknown-linux-gnu/debug/deps/). The pre-build step ensures /target exists and is world-writable. --- Cross.toml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Cross.toml b/Cross.toml index 74b6508..2872387 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,7 +1,15 @@ +[build] +# cross mounts host target/ at /target with --user, which may prevent +# cargo from creating subdirectories. Pre-create them before cargo runs. +pre-build = [ + "mkdir -p /target", + "chmod 777 /target", +] + [build.env] # GHA cache endpoint is NOT reliably accessible from inside cross Docker -# containers (missing CA certs, network restrictions). sccache relies on -# the local disk cache at SCCACHE_DIR=/tmp/sccache instead. +# containers (missing CA certs). sccache relies on the local disk cache +# at SCCACHE_DIR=/tmp/sccache instead. passthrough = ["RUSTFLAGS", "RUSTC_BOOTSTRAP"] # Mount host /tmp so sccache's disk cache persists across container runs From dfe406499c1bd3933f09415d44dc36a2a742a34e Mon Sep 17 00:00:00 2001 From: iHsin Date: Tue, 28 Apr 2026 20:29:39 +0800 Subject: [PATCH 04/21] chore: strip Cross.toml back to minimal to isolate the /target volume issue Remove pre-build and volumes from Cross.toml to debug whether the container write permission error is caused by our Dockerfile changes (sccache, env vars) or by Cross.toml additions (volumes, pre-build). --- .cross/Dockerfile | 8 ++++---- Cross.toml | 14 -------------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/.cross/Dockerfile b/.cross/Dockerfile index 12929a2..6ed9277 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -3,11 +3,11 @@ ARG DEBIAN_FRONTEND=noninteractive FROM $CROSS_BASE_IMAGE -# Install system deps (clang/llvm for C compilation, curl for sccache download) -# then install sccache, clean up to keep image lean +# Install system deps (clang/llvm for C compilation, curl + ca-certificates for sccache download and GHA cache TLS) COPY .github/sccache.sh / RUN apt-get update && \ - apt-get install -y --no-install-recommends clang llvm curl ca-certificates && \ + apt-get install -y --no-install-recommends \ + clang llvm curl ca-certificates && \ chmod +x /sccache.sh && \ arch=$(uname -m) && \ case "$arch" in \ @@ -19,7 +19,7 @@ RUN apt-get update && \ esac && \ echo "Detected arch=$arch, downloading sccache for $triple" && \ /sccache.sh "$triple" && \ - apt-get purge -y curl ca-certificates && \ + apt-get purge -y curl && \ apt-get autoremove -y && \ rm -rf /var/lib/apt/lists/* diff --git a/Cross.toml b/Cross.toml index 2872387..1df2934 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,20 +1,6 @@ -[build] -# cross mounts host target/ at /target with --user, which may prevent -# cargo from creating subdirectories. Pre-create them before cargo runs. -pre-build = [ - "mkdir -p /target", - "chmod 777 /target", -] - [build.env] -# GHA cache endpoint is NOT reliably accessible from inside cross Docker -# containers (missing CA certs). sccache relies on the local disk cache -# at SCCACHE_DIR=/tmp/sccache instead. passthrough = ["RUSTFLAGS", "RUSTC_BOOTSTRAP"] -# Mount host /tmp so sccache's disk cache persists across container runs -volumes = ["/tmp=/tmp"] - # Use clang as the C/C++ compiler to avoid the aws-lc-sys GCC memcmp bug # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95189 [build.dockerfile] From e837ffcaa71a2a2741e6a2f512ab229d7c7c2bd2 Mon Sep 17 00:00:00 2001 From: iHsin Date: Tue, 28 Apr 2026 20:41:25 +0800 Subject: [PATCH 05/21] chore: minimal Dockerfile identical to original next branch Test whether any custom Dockerfile at all causes the /target volume write issue, or if it's specific to sccache-related changes. --- .cross/Dockerfile | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/.cross/Dockerfile b/.cross/Dockerfile index 6ed9277..240830c 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -3,30 +3,10 @@ ARG DEBIAN_FRONTEND=noninteractive FROM $CROSS_BASE_IMAGE -# Install system deps (clang/llvm for C compilation, curl + ca-certificates for sccache download and GHA cache TLS) -COPY .github/sccache.sh / RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - clang llvm curl ca-certificates && \ - chmod +x /sccache.sh && \ - arch=$(uname -m) && \ - case "$arch" in \ - x86_64) triple="x86_64-unknown-linux-musl" ;; \ - aarch64) triple="aarch64-unknown-linux-musl" ;; \ - armv7l|armv7) triple="armv7-unknown-linux-musleabi" ;; \ - i686|i586) triple="i686-unknown-linux-musl" ;; \ - *) echo "FATAL: unsupported architecture for sccache: $arch" && exit 1 ;; \ - esac && \ - echo "Detected arch=$arch, downloading sccache for $triple" && \ - /sccache.sh "$triple" && \ - apt-get purge -y curl && \ - apt-get autoremove -y && \ + apt-get install -y --no-install-recommends clang llvm && \ rm -rf /var/lib/apt/lists/* ENV CC=clang \ CXX=clang++ \ - AR=llvm-ar \ - RUSTC_WRAPPER="/usr/bin/sccache" \ - SCCACHE_DIRECT=true \ - SCCACHE_DIR="/tmp/sccache" \ - CARGO_BUILD_PIPELINING=false + AR=llvm-ar From e20d0c62f6df7724a6dcfbdd6dcbcecbdde0a432 Mon Sep 17 00:00:00 2001 From: iHsin Date: Tue, 28 Apr 2026 20:53:07 +0800 Subject: [PATCH 06/21] test: install sccache binary only (no RUSTC_WRAPPER) --- .cross/Dockerfile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.cross/Dockerfile b/.cross/Dockerfile index 240830c..8d9d6e1 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -3,8 +3,22 @@ ARG DEBIAN_FRONTEND=noninteractive FROM $CROSS_BASE_IMAGE +COPY .github/sccache.sh / RUN apt-get update && \ - apt-get install -y --no-install-recommends clang llvm && \ + apt-get install -y --no-install-recommends clang llvm curl ca-certificates && \ + chmod +x /sccache.sh && \ + arch=$(uname -m) && \ + case "$arch" in \ + x86_64) triple="x86_64-unknown-linux-musl" ;; \ + aarch64) triple="aarch64-unknown-linux-musl" ;; \ + armv7l|armv7) triple="armv7-unknown-linux-musleabi" ;; \ + i686|i586) triple="i686-unknown-linux-musl" ;; \ + *) echo "FATAL: unsupported architecture for sccache: $arch" && exit 1 ;; \ + esac && \ + echo "Detected arch=$arch, downloading sccache for $triple" && \ + /sccache.sh "$triple" && \ + apt-get purge -y curl ca-certificates && \ + apt-get autoremove -y && \ rm -rf /var/lib/apt/lists/* ENV CC=clang \ From a210aef961f585b63a54e35dd8e58e5f90be08c4 Mon Sep 17 00:00:00 2001 From: iHsin Date: Tue, 28 Apr 2026 21:01:33 +0800 Subject: [PATCH 07/21] test: add RUSTC_WRAPPER only (no SCCACHE_DIRECT/DIR/CARGO vars) --- .cross/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cross/Dockerfile b/.cross/Dockerfile index 8d9d6e1..170e516 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -23,4 +23,5 @@ RUN apt-get update && \ ENV CC=clang \ CXX=clang++ \ - AR=llvm-ar + AR=llvm-ar \ + RUSTC_WRAPPER="/usr/bin/sccache" From 8c16bfa5993b9c8a6d5de20f2c964708ea9d92f0 Mon Sep 17 00:00:00 2001 From: iHsin Date: Tue, 28 Apr 2026 21:08:40 +0800 Subject: [PATCH 08/21] test: add RUSTC_WRAPPER + SCCACHE_DIRECT=true SCCACHE_DIRECT=true skips sccache's dep-info pre-pass (which writes to a temp directory without deps/ subdir), going straight to the real rustc invocation which handles directory creation correctly. --- .cross/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.cross/Dockerfile b/.cross/Dockerfile index 170e516..b6e40f9 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -24,4 +24,7 @@ RUN apt-get update && \ ENV CC=clang \ CXX=clang++ \ AR=llvm-ar \ - RUSTC_WRAPPER="/usr/bin/sccache" + RUSTC_WRAPPER="/usr/bin/sccache" \ + SCCACHE_DIRECT=true \ + SCCACHE_DIR="/tmp/sccache" \ + CARGO_BUILD_PIPELINING=false From cb52e14c65f1f32b224efb27f6908d5b2e6a49f6 Mon Sep 17 00:00:00 2001 From: iHsin Date: Tue, 28 Apr 2026 21:17:19 +0800 Subject: [PATCH 09/21] test: add SCCACHE_START_SERVER=0 --- .cross/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.cross/Dockerfile b/.cross/Dockerfile index b6e40f9..961dc50 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -25,6 +25,7 @@ ENV CC=clang \ CXX=clang++ \ AR=llvm-ar \ RUSTC_WRAPPER="/usr/bin/sccache" \ + SCCACHE_START_SERVER=0 \ SCCACHE_DIRECT=true \ SCCACHE_DIR="/tmp/sccache" \ CARGO_BUILD_PIPELINING=false From 0eab8cceb01e50ccaaa31dbd7ce2f4129a83d2f9 Mon Sep 17 00:00:00 2001 From: iHsin Date: Tue, 28 Apr 2026 21:22:22 +0800 Subject: [PATCH 10/21] test: upgrade sccache to v0.14.0 --- .github/sccache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/sccache.sh b/.github/sccache.sh index e79ed48..90f9feb 100755 --- a/.github/sccache.sh +++ b/.github/sccache.sh @@ -10,7 +10,7 @@ set -euo pipefail main() { local triple="${1}" - local version="v0.12.0" + local version="v0.14.0" local url="https://github.com/mozilla/sccache/releases/download/${version}/sccache-${version}-${triple}.tar.gz" local td From 98584f1c486bdc40f0951f54da98f275c55877e0 Mon Sep 17 00:00:00 2001 From: iHsin Date: Wed, 29 Apr 2026 00:00:38 +0800 Subject: [PATCH 11/21] ci: add sccache for cross-rs/cross with proper debugging Add sccache v0.14.0 to the cross Docker image for diagnostic use. RUSTC_WRAPPER is deliberately NOT set because sccache's server mode interferes with cross's --user Docker setup - rustc fails to create output directories under /target (Docker volume mount). Key findings from debugging (next 8 CI runs): - Plain Dockerfile (clang/llvm only) -> PASS - sccache binary installed, no RUSTC_WRAPPER -> PASS - RUSTC_WRAPPER=/usr/bin/sccache added -> ALL cross targets FAIL (rustc: "error writing dependencies to /target/...d: No such file") - SCCACHE_DIRECT=true, SCCACHE_START_SERVER=0, v0.12.0 -> all FAIL - v0.14.0 (same as host) -> all FAIL - The issue is consistent across all archs (x86_64, aarch64, armv7, i686) - Non-cross targets (Windows, macOS) work fine with host sccache Native builds continue to use the host sccache via mozilla-actions/sccache-action@v0.0.9 (GHA cache backend). Cross builds rely on cargo's fingerprint tracking + Swatinem/rust-cache for incremental compilation across CI runs. --- .cross/Dockerfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.cross/Dockerfile b/.cross/Dockerfile index 961dc50..eaf61e3 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -3,6 +3,12 @@ ARG DEBIAN_FRONTEND=noninteractive FROM $CROSS_BASE_IMAGE +# Install clang/llvm for C compilation (aws-lc-sys GCC memcmp bug workaround) +# Also install sccache binary for manual cache ops and future use. +# RUSTC_WRAPPER is deliberately NOT set: inside cross containers with --user, +# sccache's server mode interferes with rustc's ability to create output +# directories under /target (which is a Docker volume mount owned by root). +# Native (non-cross) builds use the host sccache via mozilla-actions/sccache-action. COPY .github/sccache.sh / RUN apt-get update && \ apt-get install -y --no-install-recommends clang llvm curl ca-certificates && \ @@ -23,9 +29,4 @@ RUN apt-get update && \ ENV CC=clang \ CXX=clang++ \ - AR=llvm-ar \ - RUSTC_WRAPPER="/usr/bin/sccache" \ - SCCACHE_START_SERVER=0 \ - SCCACHE_DIRECT=true \ - SCCACHE_DIR="/tmp/sccache" \ - CARGO_BUILD_PIPELINING=false + AR=llvm-ar From 7f492b0f7926d7ef5432172f507d01193699d240 Mon Sep 17 00:00:00 2001 From: iHsin Date: Wed, 29 Apr 2026 00:15:25 +0800 Subject: [PATCH 12/21] ci: integrate sccache with GHA cache backend via wrapper script - sccache v0.14.0 installed inside cross container - RUSTC_WRAPPER points to sccache-wrapper.sh instead of sccache directly - Wrapper pre-creates --out-dir/deps before delegating to sccache, fixing the directory creation issue with cross's --user Docker setup - SCCACHE_GHA_ENABLED + ACTIONS_* env vars passed through Cross.toml so container sccache uses GHA cache API (persistent across CI runs) - ca-certificates kept installed for GHA cache TLS verification - SCCACHE_DIRECT=true for faster direct-mode hashing - CARGO_BUILD_PIPELINING=false to avoid cargo-level pipelining races --- .cross/Dockerfile | 22 +++++++++++----------- .github/sccache-wrapper.sh | 31 +++++++++++++++++++++++++++++++ Cross.toml | 11 ++++++++++- 3 files changed, 52 insertions(+), 12 deletions(-) create mode 100644 .github/sccache-wrapper.sh diff --git a/.cross/Dockerfile b/.cross/Dockerfile index eaf61e3..5b68523 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -3,16 +3,12 @@ ARG DEBIAN_FRONTEND=noninteractive FROM $CROSS_BASE_IMAGE -# Install clang/llvm for C compilation (aws-lc-sys GCC memcmp bug workaround) -# Also install sccache binary for manual cache ops and future use. -# RUSTC_WRAPPER is deliberately NOT set: inside cross containers with --user, -# sccache's server mode interferes with rustc's ability to create output -# directories under /target (which is a Docker volume mount owned by root). -# Native (non-cross) builds use the host sccache via mozilla-actions/sccache-action. -COPY .github/sccache.sh / +# Install system deps +COPY .github/sccache.sh .github/sccache-wrapper.sh / RUN apt-get update && \ - apt-get install -y --no-install-recommends clang llvm curl ca-certificates && \ - chmod +x /sccache.sh && \ + apt-get install -y --no-install-recommends \ + clang llvm curl ca-certificates && \ + chmod +x /sccache.sh /sccache-wrapper.sh && \ arch=$(uname -m) && \ case "$arch" in \ x86_64) triple="x86_64-unknown-linux-musl" ;; \ @@ -23,10 +19,14 @@ RUN apt-get update && \ esac && \ echo "Detected arch=$arch, downloading sccache for $triple" && \ /sccache.sh "$triple" && \ - apt-get purge -y curl ca-certificates && \ + # Keep ca-certificates for GHA cache TLS; purge curl only + apt-get purge -y curl && \ apt-get autoremove -y && \ rm -rf /var/lib/apt/lists/* ENV CC=clang \ CXX=clang++ \ - AR=llvm-ar + AR=llvm-ar \ + RUSTC_WRAPPER="/sccache-wrapper.sh" \ + SCCACHE_DIRECT=true \ + CARGO_BUILD_PIPELINING=false diff --git a/.github/sccache-wrapper.sh b/.github/sccache-wrapper.sh new file mode 100644 index 0000000..c0d7daa --- /dev/null +++ b/.github/sccache-wrapper.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# sccache wrapper for cross-rs/cross containers +# +# Problem: sccache inside cross Docker containers with --user fails when +# rustc tries to write .d files to /target/.../deps/ because the output +# directory structure may not exist (Docker volume mount + --user prevents +# sccache from creating deep paths in a single mkdir call). +# +# This wrapper pre-creates the output directory tree before delegating +# to sccache, matching what rustc would normally do on its own. + +set -euo pipefail + +# Find --out-dir argument and ensure it exists with deps/ subdirectory +while [ $# -gt 0 ]; do + case "$1" in + --out-dir) + if [ -n "${2:-}" ]; then + dir="$2" + mkdir -p "$dir/deps" + fi + ;; + --out-dir=*) + dir="${1#*=}" + mkdir -p "$dir/deps" + ;; + esac + shift +done + +exec /usr/bin/sccache "$@" diff --git a/Cross.toml b/Cross.toml index 1df2934..030fb47 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,5 +1,14 @@ [build.env] -passthrough = ["RUSTFLAGS", "RUSTC_BOOTSTRAP"] +# Pass through GHA cache API env vars so sccache inside the cross +# container can use GHA cache as its backend (instead of local disk). +passthrough = [ + "RUSTFLAGS", + "RUSTC_BOOTSTRAP", + "SCCACHE_GHA_ENABLED", + "ACTIONS_RESULTS_URL", + "ACTIONS_RUNTIME_TOKEN", + "ACTIONS_CACHE_URL", +] # Use clang as the C/C++ compiler to avoid the aws-lc-sys GCC memcmp bug # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95189 From 5544ba46d26c407351be909f45a4c7da220068aa Mon Sep 17 00:00:00 2001 From: iHsin Date: Wed, 29 Apr 2026 00:20:06 +0800 Subject: [PATCH 13/21] fix: sccache wrapper now handles all /target/ paths, not just --out-dir --- .github/sccache-wrapper.sh | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/.github/sccache-wrapper.sh b/.github/sccache-wrapper.sh index c0d7daa..6bb62df 100644 --- a/.github/sccache-wrapper.sh +++ b/.github/sccache-wrapper.sh @@ -1,31 +1,22 @@ #!/bin/bash # sccache wrapper for cross-rs/cross containers # -# Problem: sccache inside cross Docker containers with --user fails when -# rustc tries to write .d files to /target/.../deps/ because the output -# directory structure may not exist (Docker volume mount + --user prevents -# sccache from creating deep paths in a single mkdir call). -# -# This wrapper pre-creates the output directory tree before delegating -# to sccache, matching what rustc would normally do on its own. +# Pre-creates output directories under /target before delegating to +# sccache. cross runs with --user + Docker volume mount at /target, +# which can prevent sccache/rustc from creating the necessary directory +# tree. We scan args for /target/ paths and mkdir -p their parent to +# ensure rustc can write there. set -euo pipefail -# Find --out-dir argument and ensure it exists with deps/ subdirectory -while [ $# -gt 0 ]; do - case "$1" in - --out-dir) - if [ -n "${2:-}" ]; then - dir="$2" - mkdir -p "$dir/deps" - fi - ;; - --out-dir=*) - dir="${1#*=}" - mkdir -p "$dir/deps" - ;; - esac - shift +for arg in "$@"; do + # Match any arg containing /target/ (handles standalone paths, + # --emit=dep-info=/target/..., dep-info=/target/..., etc.) + if [[ "$arg" == *"/target/"* ]]; then + # Strip everything before /target/ to get the absolute path + local tgt="${arg#*/target/}" + mkdir -p "/target/$(dirname "$tgt")" 2>/dev/null || true + fi done exec /usr/bin/sccache "$@" From 58aef62a5d4b88a1ab39e6abb7e03073976abe68 Mon Sep 17 00:00:00 2001 From: iHsin Date: Wed, 29 Apr 2026 00:27:53 +0800 Subject: [PATCH 14/21] revert: remove sccache wrapper, restore working state All attempts to use RUSTC_WRAPPER inside cross containers fail due to cross's --user + Docker volume mount at /target. Wrapper scripts that pre-create directory trees also don't help - the issue is deeper than just missing directories. sccache binary is installed for manual/diagnostic use. GHA cache env vars are passed through so sccache can use GHA API when invoked explicitly. No RUSTC_WRAPPER set - cross compiles natively without sccache (host sccache handles non-cross builds). --- .cross/Dockerfile | 11 ++++------- .github/sccache-wrapper.sh | 22 ---------------------- 2 files changed, 4 insertions(+), 29 deletions(-) delete mode 100644 .github/sccache-wrapper.sh diff --git a/.cross/Dockerfile b/.cross/Dockerfile index 5b68523..43db27a 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -4,11 +4,11 @@ ARG DEBIAN_FRONTEND=noninteractive FROM $CROSS_BASE_IMAGE # Install system deps -COPY .github/sccache.sh .github/sccache-wrapper.sh / +COPY .github/sccache.sh / RUN apt-get update && \ apt-get install -y --no-install-recommends \ clang llvm curl ca-certificates && \ - chmod +x /sccache.sh /sccache-wrapper.sh && \ + chmod +x /sccache.sh && \ arch=$(uname -m) && \ case "$arch" in \ x86_64) triple="x86_64-unknown-linux-musl" ;; \ @@ -19,14 +19,11 @@ RUN apt-get update && \ esac && \ echo "Detected arch=$arch, downloading sccache for $triple" && \ /sccache.sh "$triple" && \ - # Keep ca-certificates for GHA cache TLS; purge curl only + # Keep ca-certificates for GHA cache TLS apt-get purge -y curl && \ apt-get autoremove -y && \ rm -rf /var/lib/apt/lists/* ENV CC=clang \ CXX=clang++ \ - AR=llvm-ar \ - RUSTC_WRAPPER="/sccache-wrapper.sh" \ - SCCACHE_DIRECT=true \ - CARGO_BUILD_PIPELINING=false + AR=llvm-ar diff --git a/.github/sccache-wrapper.sh b/.github/sccache-wrapper.sh deleted file mode 100644 index 6bb62df..0000000 --- a/.github/sccache-wrapper.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# sccache wrapper for cross-rs/cross containers -# -# Pre-creates output directories under /target before delegating to -# sccache. cross runs with --user + Docker volume mount at /target, -# which can prevent sccache/rustc from creating the necessary directory -# tree. We scan args for /target/ paths and mkdir -p their parent to -# ensure rustc can write there. - -set -euo pipefail - -for arg in "$@"; do - # Match any arg containing /target/ (handles standalone paths, - # --emit=dep-info=/target/..., dep-info=/target/..., etc.) - if [[ "$arg" == *"/target/"* ]]; then - # Strip everything before /target/ to get the absolute path - local tgt="${arg#*/target/}" - mkdir -p "/target/$(dirname "$tgt")" 2>/dev/null || true - fi -done - -exec /usr/bin/sccache "$@" From 383a0607dd0daef4e01c7289970ed5b4fa64378c Mon Sep 17 00:00:00 2001 From: iHsin Date: Wed, 29 Apr 2026 00:45:23 +0800 Subject: [PATCH 15/21] fix: run cross as root via CROSS_CONTAINER_UID=0 to fix sccache sccache's RUSTC_WRAPPER fails in cross containers because cross runs with --user and the /target Docker volume mount prevents directory creation. Solution: set CROSS_CONTAINER_UID=0 and CROSS_CONTAINER_GID=0 in the CI workflow env so cross runs the container as root. This allows sccache (and rustc) to create the full /target directory tree without permission issues. --- .cross/Dockerfile | 6 +++++- .github/workflows/ci.yml | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.cross/Dockerfile b/.cross/Dockerfile index 43db27a..61ab2e6 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -26,4 +26,8 @@ RUN apt-get update && \ ENV CC=clang \ CXX=clang++ \ - AR=llvm-ar + AR=llvm-ar \ + RUSTC_WRAPPER="/usr/bin/sccache" \ + SCCACHE_DIRECT=true \ + SCCACHE_DIR="/tmp/sccache" \ + CARGO_BUILD_PIPELINING=false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 982a2b3..cabbc70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,9 @@ env: RUST_LOG: "wind=TRACE" RUST_TOOLCHAIN: "stable" RUSTC_BOOTSTRAP: "1" + # Run cross containers as root so sccache can create /target subdirs + CROSS_CONTAINER_UID: "0" + CROSS_CONTAINER_GID: "0" jobs: build: From 7c233860c146b70dbb416fabe1775da3a512419c Mon Sep 17 00:00:00 2001 From: iHsin Date: Wed, 29 Apr 2026 01:00:27 +0800 Subject: [PATCH 16/21] revert: remove CROSS_CONTAINER_UID and wrapper - not supported CROSS_CONTAINER_UID=0 env var does not exist in cross v0.2.5 (checked source at commit 65fe72b). The feature was added in cross v1 (PR #543) but cross v0.2.5 is a v2 reimplementation. Revert to working state: sccache installed, GHA cache env vars passed through, no RUSTC_WRAPPER. cross targets compile natively without sccache (cargo incremental + Swatinem/rust-cache handles cross-CI caching). --- .cross/Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.cross/Dockerfile b/.cross/Dockerfile index 61ab2e6..657d37f 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -19,15 +19,10 @@ RUN apt-get update && \ esac && \ echo "Detected arch=$arch, downloading sccache for $triple" && \ /sccache.sh "$triple" && \ - # Keep ca-certificates for GHA cache TLS apt-get purge -y curl && \ apt-get autoremove -y && \ rm -rf /var/lib/apt/lists/* ENV CC=clang \ CXX=clang++ \ - AR=llvm-ar \ - RUSTC_WRAPPER="/usr/bin/sccache" \ - SCCACHE_DIRECT=true \ - SCCACHE_DIR="/tmp/sccache" \ - CARGO_BUILD_PIPELINING=false + AR=llvm-ar From b9dcd7e1635a98db00886640e3eb8763cc702e91 Mon Sep 17 00:00:00 2001 From: iHsin Date: Wed, 29 Apr 2026 01:09:43 +0800 Subject: [PATCH 17/21] feat: sccache host-client via shared Unix Domain Socket sccache on the HOST runs as the server (started during 'cargo install cross' compilation). sccache in the cross container acts as a client connecting to the host's server via a shared Unix Domain Socket at /tmp/sccache-server.sock, mounted via Docker volume. This avoids the cross --user permission issue entirely because: - Server runs on the host with full filesystem access and GHA cache (ghac) - Client in the container just sends compilation requests over UDS - No file I/O from sccache inside the container at all Key changes: - CI workflow: set SCCACHE_SERVER_UDS=/tmp/sccache-server.sock - Cross.toml: mount /tmp, pass through GHA cache env vars - Dockerfile: RUSTC_WRAPPER points to container sccache (client mode) with SCCACHE_SERVER_UDS set to the shared socket --- .cross/Dockerfile | 7 ++++++- .github/workflows/ci.yml | 6 +++--- Cross.toml | 9 +++++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.cross/Dockerfile b/.cross/Dockerfile index 657d37f..ac8db2c 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -25,4 +25,9 @@ RUN apt-get update && \ ENV CC=clang \ CXX=clang++ \ - AR=llvm-ar + AR=llvm-ar \ + RUSTC_WRAPPER="/usr/bin/sccache" \ + SCCACHE_DIRECT=true \ + SCCACHE_SERVER_UDS="/tmp/sccache-server.sock" \ + SCCACHE_IDLE_TIMEOUT="0" \ + CARGO_BUILD_PIPELINING=false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cabbc70..2afe051 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,9 +20,9 @@ env: RUST_LOG: "wind=TRACE" RUST_TOOLCHAIN: "stable" RUSTC_BOOTSTRAP: "1" - # Run cross containers as root so sccache can create /target subdirs - CROSS_CONTAINER_UID: "0" - CROSS_CONTAINER_GID: "0" + # sccache host-client via shared Unix Domain Socket + SCCACHE_SERVER_UDS: "/tmp/sccache-server.sock" + SCCACHE_IDLE_TIMEOUT: "0" jobs: build: diff --git a/Cross.toml b/Cross.toml index 030fb47..21ac44b 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,6 +1,7 @@ [build.env] -# Pass through GHA cache API env vars so sccache inside the cross -# container can use GHA cache as its backend (instead of local disk). +# Pass through GHA cache API env vars so the HOST's sccache server +# (which the container's sccache client connects to via shared UDS) +# can use GHA cache as its backend. passthrough = [ "RUSTFLAGS", "RUSTC_BOOTSTRAP", @@ -10,6 +11,10 @@ passthrough = [ "ACTIONS_CACHE_URL", ] +# Mount /tmp so the sccache Unix Domain Socket is shared between +# the host server and the container client. +volumes = ["/tmp=/tmp"] + # Use clang as the C/C++ compiler to avoid the aws-lc-sys GCC memcmp bug # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95189 [build.dockerfile] From 35945b3437db6c60e8ffa2332dd780721e6b5d13 Mon Sep 17 00:00:00 2001 From: iHsin Date: Wed, 29 Apr 2026 01:27:05 +0800 Subject: [PATCH 18/21] revert: back to working state - sccache installed, no RUSTC_WRAPPER All attempts to use RUSTC_WRAPPER inside cross containers failed: 1. RUSTC_WRAPPER=sccache directly -> all cross targets fail 2. + SCCACHE_DIRECT, +SCCACHE_START_SERVER=0 -> same failure 3. + v0.14.0 (same as host) -> same failure 4. + sccache wrapper (mkdir deps/) -> same failure 5. CROSS_CONTAINER_UID=0 (not in cross v0.2.5) -> not supported 6. Host-client UDS (sccache server can't access GHA cache) -> fails The working approach: sccache binary installed for manual use, no RUSTC_WRAPPER. cross compiles natively (cargo incremental + Swatinem/rust-cache handles caching across CI runs). Native builds use host sccache via mozilla-actions/sccache-action. --- .cross/Dockerfile | 7 +------ .github/workflows/ci.yml | 3 --- Cross.toml | 16 +--------------- 3 files changed, 2 insertions(+), 24 deletions(-) diff --git a/.cross/Dockerfile b/.cross/Dockerfile index ac8db2c..657d37f 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -25,9 +25,4 @@ RUN apt-get update && \ ENV CC=clang \ CXX=clang++ \ - AR=llvm-ar \ - RUSTC_WRAPPER="/usr/bin/sccache" \ - SCCACHE_DIRECT=true \ - SCCACHE_SERVER_UDS="/tmp/sccache-server.sock" \ - SCCACHE_IDLE_TIMEOUT="0" \ - CARGO_BUILD_PIPELINING=false + AR=llvm-ar diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2afe051..982a2b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,9 +20,6 @@ env: RUST_LOG: "wind=TRACE" RUST_TOOLCHAIN: "stable" RUSTC_BOOTSTRAP: "1" - # sccache host-client via shared Unix Domain Socket - SCCACHE_SERVER_UDS: "/tmp/sccache-server.sock" - SCCACHE_IDLE_TIMEOUT: "0" jobs: build: diff --git a/Cross.toml b/Cross.toml index 21ac44b..1df2934 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,19 +1,5 @@ [build.env] -# Pass through GHA cache API env vars so the HOST's sccache server -# (which the container's sccache client connects to via shared UDS) -# can use GHA cache as its backend. -passthrough = [ - "RUSTFLAGS", - "RUSTC_BOOTSTRAP", - "SCCACHE_GHA_ENABLED", - "ACTIONS_RESULTS_URL", - "ACTIONS_RUNTIME_TOKEN", - "ACTIONS_CACHE_URL", -] - -# Mount /tmp so the sccache Unix Domain Socket is shared between -# the host server and the container client. -volumes = ["/tmp=/tmp"] +passthrough = ["RUSTFLAGS", "RUSTC_BOOTSTRAP"] # Use clang as the C/C++ compiler to avoid the aws-lc-sys GCC memcmp bug # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95189 From 6fd587683db13a60dcaa8011d0eb7a038d4682d3 Mon Sep 17 00:00:00 2001 From: iHsin Date: Wed, 29 Apr 2026 01:29:07 +0800 Subject: [PATCH 19/21] test: sccache with SCCACHE_SERVER_UDS, no GHA passthrough - Use SCCACHE_SERVER_UDS=/tmp/sccache.sock for explicit server socket - SCCACHE_DIR=/tmp/sccache for local disk cache (no GHA) - /tmp volume mount so socket/cache persist across runs - No GHA cache env passthrough (was causing server startup failure) --- .cross/Dockerfile | 7 ++++++- Cross.toml | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.cross/Dockerfile b/.cross/Dockerfile index 657d37f..34fa06f 100644 --- a/.cross/Dockerfile +++ b/.cross/Dockerfile @@ -25,4 +25,9 @@ RUN apt-get update && \ ENV CC=clang \ CXX=clang++ \ - AR=llvm-ar + AR=llvm-ar \ + RUSTC_WRAPPER="/usr/bin/sccache" \ + SCCACHE_DIRECT=true \ + SCCACHE_SERVER_UDS="/tmp/sccache.sock" \ + SCCACHE_DIR="/tmp/sccache" \ + CARGO_BUILD_PIPELINING=false diff --git a/Cross.toml b/Cross.toml index 1df2934..03a6a69 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,5 +1,6 @@ [build.env] passthrough = ["RUSTFLAGS", "RUSTC_BOOTSTRAP"] +volumes = ["/tmp=/tmp"] # Use clang as the C/C++ compiler to avoid the aws-lc-sys GCC memcmp bug # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95189 From 3755f6a30be9ebfc91fccbd85c5e4337668657da Mon Sep 17 00:00:00 2001 From: iHsin Date: Wed, 29 Apr 2026 02:08:00 +0800 Subject: [PATCH 20/21] ci: use workflows@feat/sccache-uds-server (host sccache server) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 982a2b3..0c6d1fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ env: jobs: build: name: Build - uses: rust-proxy/workflows/.github/workflows/rust-build.yml@main + uses: rust-proxy/workflows/.github/workflows/rust-build.yml@feat/sccache-uds-server with: rust-toolchain: "stable" packages: "wind,tuic-server,tuic-client" From 5b81362a6861412d6ffdfda4f5927bbaa9e94555 Mon Sep 17 00:00:00 2001 From: iHsin Date: Wed, 29 Apr 2026 02:12:52 +0800 Subject: [PATCH 21/21] revert: back to main workflows (host UDS server can't access container paths) The host sccache server and container client can't share a UDS socket because they're in different mount namespaces - the host server can't resolve the container's source file paths. Container uses its own on-demand sccache server with local disk cache via /tmp volume mount. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c6d1fb..982a2b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ env: jobs: build: name: Build - uses: rust-proxy/workflows/.github/workflows/rust-build.yml@feat/sccache-uds-server + uses: rust-proxy/workflows/.github/workflows/rust-build.yml@main with: rust-toolchain: "stable" packages: "wind,tuic-server,tuic-client"