diff --git a/.github/workflows/cross-arch-build.yml b/.github/workflows/cross-arch-build.yml index 29327fce46..e93f9e7fbd 100644 --- a/.github/workflows/cross-arch-build.yml +++ b/.github/workflows/cross-arch-build.yml @@ -54,16 +54,44 @@ jobs: - name: Verify build binary run: ./build/seid version --long + # Detect PRs that change the static-build inputs so they run the full static + # job below even when targeting main; otherwise a change to the static build + # only gets exercised once it reaches a release branch. + static-paths: + name: "Static build inputs changed?" + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + outputs: + changed: ${{ steps.diff.outputs.changed }} + steps: + - id: diff + env: + GH_TOKEN: ${{ github.token }} + run: | + PATTERN='^(scripts/(build-static|boot-smoke|check-libwasmvm-static)\.sh|Makefile|third_party/alpine-gcc10-libgcc/|\.goreleaser\.yaml|\.github/workflows/cross-arch-build\.yml)' + FILES=$(gh api "repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" --paginate -q '.[].filename') + if echo "$FILES" | grep -qE "$PATTERN"; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + echo "static-relevant files: $(echo "$FILES" | grep -cE "$PATTERN" || true)" + linux-amd64-static: name: "Linux AMD64 (static)" runs-on: ubuntu-latest - # The static seid is a release artefact, so only build it for release work, - # not on every PR: PRs into release/**, pushes to release/**, or merge-queue - # entries targeting release/**. + needs: [static-paths] + # The static seid is a release artefact: build it for release work (PRs into + # release/**, pushes to release/**, merge-queue entries targeting release/**) + # and for any PR that changes the static-build inputs themselves. The + # !cancelled() keeps this job eligible when static-paths is skipped (push and + # merge_group events). if: >- + !cancelled() && ( startsWith(github.base_ref, 'release/') || startsWith(github.ref, 'refs/heads/release/') || - startsWith(github.event.merge_group.base_ref, 'refs/heads/release/') + startsWith(github.event.merge_group.base_ref, 'refs/heads/release/') || + needs.static-paths.outputs.changed == 'true' ) steps: - name: Checkout code # See: https://github.com/actions/checkout/releases/tag/v7.0.0 @@ -93,6 +121,12 @@ jobs: - name: Smoke test run: ./build/seid version --long + # Repeated boots: the gcc>=12 unwind b-tree crash killed ~70% of boots at the + # genesis wasm store, so a single boot (let alone `seid version`, which never + # touches the wasm VM) can pass on luck. See scripts/boot-smoke.sh. + - name: Boot smoke (repeated) + run: bash scripts/boot-smoke.sh ./build/seid 8 + - name: Upload artifact for inspection uses: actions/upload-artifact@v5 with: diff --git a/.github/workflows/uci-release-publish.yml b/.github/workflows/uci-release-publish.yml index a0a2ffa29d..7cc9e2893c 100644 --- a/.github/workflows/uci-release-publish.yml +++ b/.github/workflows/uci-release-publish.yml @@ -3,6 +3,11 @@ run-name: UCI / Release Publish on: push: + # Note: `paths` filters do not apply to tag pushes, so this workflow also fires + # when a version tag is pushed manually. That is the normal release path here: + # the "Restrict tagging" ruleset blocks this workflow's token from creating tags, + # so the publish job fails on branch pushes and a maintainer pushes the tag by + # hand. The goreleaser job below handles that path explicitly. paths: [ 'version.json' ] workflow_dispatch: @@ -10,8 +15,11 @@ permissions: contents: write concurrency: - group: ${{ github.workflow }}-${{ github.sha }} - cancel-in-progress: true + # Keyed per event+ref, never cancelled: a version.json branch push and the + # follow-up manual tag push share the same SHA, and an in-flight release build + # must not be cancelled by the later event. + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: false jobs: publish: @@ -21,10 +29,17 @@ jobs: goreleaser: name: GoReleaser needs: publish - if: needs.publish.outputs.release_created == 'true' + # Build and attach when this run created the release, or when a maintainer + # pushed a version tag by hand (release_created is false on that path because + # the tag already exists; publish then no-ops and this job attaches the + # binaries). Re-pushing an existing tag re-appends the goreleaser notes block + # to the release body, so treat this as run-once per tag. + if: >- + needs.publish.outputs.release_created == 'true' || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'v')) uses: sei-protocol/uci/.github/workflows/goreleaser-release.yml@v0.0.11 with: - ref: ${{ needs.publish.outputs.version }} + ref: ${{ github.ref_type == 'tag' && github.ref_name || needs.publish.outputs.version }} go-version: '1.25.6' # Repo submodules are Solidity test libs (forge-std/openzeppelin/solmate); the # seid binary build doesn't use them, so skip the fetch. diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 9eb7682cf2..f6f14ebc59 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -7,6 +7,9 @@ before: - go mod download # build-static.sh self-verifies (libwasmvm archives present + output is static). - bash scripts/build-static.sh + # Boot the binary before packaging: catches crash-at-first-wasm-use defects that + # neither a successful link nor `seid version` can (see scripts/boot-smoke.sh). + - bash scripts/boot-smoke.sh build/seid 4 builds: # The static seid is built in Alpine (musl-native) by the `before:` hook above diff --git a/Makefile b/Makefile index 085bf60949..72bfb330a9 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,10 @@ ifeq ($(firstword $(sort go1.23 $(shell go env GOVERSION))), go1.23) ldflags += -checklinkname=0 endif ifeq ($(LINK_STATICALLY),true) - ldflags += -linkmode=external -extldflags "-Wl,-z,muldefs -static" + # STATIC_EXTRA_LDFLAGS lets the static build inject linker search paths, e.g. + # scripts/build-static.sh points it at the pinned pre-gcc-12 libgcc (see + # third_party/alpine-gcc10-libgcc/README.md). + ldflags += -linkmode=external -extldflags "-Wl,-z,muldefs -static $(STATIC_EXTRA_LDFLAGS)" endif ldflags += $(LDFLAGS) ldflags := $(strip $(ldflags)) diff --git a/scripts/boot-smoke.sh b/scripts/boot-smoke.sh new file mode 100755 index 0000000000..aa03393770 --- /dev/null +++ b/scripts/boot-smoke.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Boot the given seid binary N times against throwaway single-node genesis homes and +# assert every boot gets through the genesis wasm store (the first wasmer JIT compile) +# and completes the ABCI handshake. +# +# Why repeated boots: the class of defect this guards against crashes probabilistically +# at first wasm use (the gcc>=12 unwind b-tree bug killed ~70% of boots, so any single +# boot check can pass on luck). N clean boots make a lucky pass vanishingly unlikely. +# +# Why "handshake completed" is the success marker: a lone node cannot leave blocksync on +# this codebase (blocksync's IsCaughtUp requires more than one peer), so block production +# is not observable single-node. The handshake only completes after the genesis wasm +# StoreCode calls succeed, which is exactly the code path that crashes. +# +# Linux-only (runs the linux/amd64 binary natively; uses GNU timeout). +# +# Usage: boot-smoke.sh [boots] +set -euo pipefail + +BIN=${1:?usage: boot-smoke.sh [boots]} +BOOTS=${2:-8} +CHAIN_ID=boot-smoke-1 + +for i in $(seq 1 "$BOOTS"); do + H=$(mktemp -d) + "$BIN" init smoke --chain-id "$CHAIN_ID" --home "$H" >/dev/null 2>&1 + sed -i 's/"stake"/"usei"/g' "$H/config/genesis.json" + "$BIN" keys add val --keyring-backend test --home "$H" >/dev/null 2>&1 + ADDR=$("$BIN" keys show val -a --keyring-backend test --home "$H") + "$BIN" add-genesis-account "$ADDR" 100000000000000usei --home "$H" >/dev/null + "$BIN" gentx val 10000000000000usei --chain-id "$CHAIN_ID" --keyring-backend test --home "$H" >/dev/null 2>&1 + "$BIN" collect-gentxs --home "$H" >/dev/null 2>&1 + + LOG="$H/start.log" + timeout -k 5 25 "$BIN" start --home "$H" >"$LOG" 2>&1 || true + + if grep -qE "SIGSEGV|SIGILL|SIGBUS|panic:" "$LOG"; then + echo "boot-smoke: boot $i/$BOOTS CRASHED:" >&2 + tail -25 "$LOG" >&2 + exit 1 + fi + if ! grep -q "Completed ABCI Handshake" "$LOG"; then + echo "boot-smoke: boot $i/$BOOTS did not reach the ABCI handshake:" >&2 + tail -25 "$LOG" >&2 + exit 1 + fi + echo "boot-smoke: boot $i/$BOOTS ok" + rm -rf "$H" +done +echo "boot-smoke: all $BOOTS boots clean" diff --git a/scripts/build-static.sh b/scripts/build-static.sh index a96930c2c8..6c99e06877 100755 --- a/scripts/build-static.sh +++ b/scripts/build-static.sh @@ -9,15 +9,35 @@ # absent in musl) and zig cc rejects the -z muldefs flag needed for the libwasmvm # v152/v155 archives; Alpine's GNU ld + musl links cleanly. --platform linux/amd64 is a # no-op on amd64 CI and forces the right arch on Apple Silicon for local runs. +# +# The link takes libgcc from third_party/alpine-gcc10-libgcc instead of the build +# image's toolchain: gcc >= 12's unwind-frame registry (a lock-free b-tree) corrupts +# under wasmer's JIT frame registration and SIGSEGVs at the genesis wasm store on most +# boots, so the static binary must carry the pre-b-tree registry. See that directory's +# README.md for the full story and provenance. The nm assertion below keeps a toolchain +# upgrade from silently reintroducing the b-tree. set -euo pipefail # Fail fast if the required static libwasmvm archives are missing. bash "$(dirname "$0")/check-libwasmvm-static.sh" -docker run --rm --platform linux/amd64 -v "$PWD":/src -w /src golang:1.25.6-alpine sh -c ' - apk add --no-cache build-base git && - git config --global --add safe.directory /src && - LINK_STATICALLY=true BUILD_TAGS=muslc LEDGER_ENABLED=false make build' +LIBGCC_DIR="third_party/alpine-gcc10-libgcc" + +docker run --rm --platform linux/amd64 -v "$PWD":/src -w /src golang:1.25.6-alpine@sha256:98e6cffc31ccc44c7c15d83df1d69891efee8115a5bb7ede2bf30a38af3e3c92 sh -c ' + set -e + apk add --no-cache build-base git + git config --global --add safe.directory /src + printf "%s %s\n%s %s\n" \ + d3e066fafde74d53a89d48f2ceb9ed9934249a5d450e281edd22947a829469d8 '"$LIBGCC_DIR"'/libgcc.a \ + d14c9973a735909e11a863b0c850300bfd3aa683ef4689cbe76a53139766ed79 '"$LIBGCC_DIR"'/libgcc_eh.a \ + | sha256sum -c - + LINK_STATICALLY=true BUILD_TAGS=muslc LEDGER_ENABLED=false \ + STATIC_EXTRA_LDFLAGS="-L/src/'"$LIBGCC_DIR"'" make build + if nm build/seid | grep -q version_lock_lock_exclusive; then + echo "build-static: ERROR: binary contains the gcc>=12 unwind b-tree (libgcc pin not applied)" >&2 + exit 1 + fi + echo "build-static: pre-b-tree unwinder confirmed (no version_lock symbols)"' # Assert the output really is statically linked, so a regression fails here rather than # shipping a dynamically-linked binary advertised as static. diff --git a/third_party/alpine-gcc10-libgcc/README.md b/third_party/alpine-gcc10-libgcc/README.md new file mode 100644 index 0000000000..c2968375f0 --- /dev/null +++ b/third_party/alpine-gcc10-libgcc/README.md @@ -0,0 +1,41 @@ +# Pinned pre-gcc-12 libgcc for the static seid build + +`libgcc.a` and `libgcc_eh.a` from Alpine 3.15's gcc 10.3.1 package. The static +build (`scripts/build-static.sh`) links these ahead of the build image's own +libgcc via `STATIC_EXTRA_LDFLAGS=-L`. + +## Why + +gcc >= 12 replaced libgcc's DWARF unwind-frame registry with a lock-free +b-tree (`libgcc/unwind-dw2-btree.h`). Wasmer registers unwind frames for every +JIT-compiled wasm module (`__register_frame`, called from +`wasmer_compiler::engine::unwind::systemv::UnwindRegistry::publish`), and under +that registration pattern the b-tree corrupts and SIGSEGVs (`btree_insert` +walks a null/garbage node pointer). A statically linked musl `seid` gets the +registry from the build image's toolchain (current Alpine ships gcc 15), which +made every static binary crash on ~70% of boots at the genesis wasm store. +Dynamically linked glibc builds use the shared `libgcc_s` runtime path and +never hit this; upstream wasmd's static binaries link a pre-b-tree libgcc, +which is what this pin restores. + +`scripts/build-static.sh` verifies these archives' checksums before the build +and asserts the final binary contains no `version_lock_lock_exclusive` symbol, +so a toolchain upgrade cannot silently reintroduce the b-tree registry. + +## Provenance + +Extracted from +`https://dl-cdn.alpinelinux.org/alpine/v3.15/main/x86_64/gcc-10.3.1_git20211027-r0.apk` + +- apk sha256: `dbbe8d585eb1f8fdb5815168944d442f28c79fbc9be98bba7cfffaff1e5c10bb` +- libgcc.a sha256: `d3e066fafde74d53a89d48f2ceb9ed9934249a5d450e281edd22947a829469d8` +- libgcc_eh.a sha256: `d14c9973a735909e11a863b0c850300bfd3aa683ef4689cbe76a53139766ed79` + +To re-derive: + +```sh +wget https://dl-cdn.alpinelinux.org/alpine/v3.15/main/x86_64/gcc-10.3.1_git20211027-r0.apk +tar -xzf gcc-10.3.1_git20211027-r0.apk \ + usr/lib/gcc/x86_64-alpine-linux-musl/10.3.1/libgcc.a \ + usr/lib/gcc/x86_64-alpine-linux-musl/10.3.1/libgcc_eh.a +``` diff --git a/third_party/alpine-gcc10-libgcc/libgcc.a b/third_party/alpine-gcc10-libgcc/libgcc.a new file mode 100644 index 0000000000..6b14c02875 Binary files /dev/null and b/third_party/alpine-gcc10-libgcc/libgcc.a differ diff --git a/third_party/alpine-gcc10-libgcc/libgcc_eh.a b/third_party/alpine-gcc10-libgcc/libgcc_eh.a new file mode 100644 index 0000000000..6b50aac33a Binary files /dev/null and b/third_party/alpine-gcc10-libgcc/libgcc_eh.a differ