From 8639a9f899aee85dfcd4f6c9f1a54065bdef5c86 Mon Sep 17 00:00:00 2001 From: Carlos Arilla Date: Fri, 28 Aug 2026 14:44:00 +0200 Subject: [PATCH 1/5] Bundle extensions as one multi-call binary and gate the command list at release Prepare for extension bundling Co-Authored-By: Claude --- .github/workflows/ci.yml | 72 ++++ .gitignore | 8 + .goreleaser.yaml | 16 +- CLAUDE.md | 2 +- Makefile | 7 +- bundled/extensions.version | 21 + cmd/extension.go | 41 +- docs/extensions-authoring.md | 16 + docs/extensions-bundling.md | 203 +++++++++ internal/extension/bundle.go | 101 +++++ internal/extension/bundle_test.go | 360 ++++++++++++++++ internal/extension/descriptions.go | 38 +- internal/extension/exec.go | 5 + internal/extension/extension.go | 25 +- internal/extension/resolve.go | 67 ++- internal/validate/validate.go | 36 ++ internal/validate/validate_test.go | 47 +++ .../design.md | 5 +- .../tasks.md | 43 +- scripts/add-bundled-to-npm.sh | 88 ++++ scripts/check-bundled-packaging-sync.sh | 80 ++++ scripts/check-descriptions.sh | 146 +++++++ scripts/fetch-bundled-extensions.sh | 372 +++++++++++++++++ scripts/test-scripts.sh | 15 + scripts/tests/add-bundled-to-npm_test.sh | 139 +++++++ .../check-bundled-packaging-sync_test.sh | 157 +++++++ scripts/tests/check-descriptions_test.sh | 211 ++++++++++ .../tests/fetch-bundled-extensions_test.sh | 388 ++++++++++++++++++ scripts/tests/lib.sh | 95 +++++ .../__snapshots__/extension_bundle_test.snap | 52 +++ test/integration/extension_bundle_test.go | 132 ++++++ test/integration/extension_test.go | 25 ++ .../test-samples/extensions/lstk-ref/main.go | 11 + 33 files changed, 2964 insertions(+), 60 deletions(-) create mode 100644 bundled/extensions.version create mode 100644 docs/extensions-bundling.md create mode 100644 internal/extension/bundle.go create mode 100644 internal/extension/bundle_test.go create mode 100755 scripts/add-bundled-to-npm.sh create mode 100755 scripts/check-bundled-packaging-sync.sh create mode 100755 scripts/check-descriptions.sh create mode 100755 scripts/fetch-bundled-extensions.sh create mode 100755 scripts/test-scripts.sh create mode 100644 scripts/tests/add-bundled-to-npm_test.sh create mode 100755 scripts/tests/check-bundled-packaging-sync_test.sh create mode 100644 scripts/tests/check-descriptions_test.sh create mode 100755 scripts/tests/fetch-bundled-extensions_test.sh create mode 100644 scripts/tests/lib.sh create mode 100644 test/integration/__snapshots__/extension_bundle_test.snap create mode 100644 test/integration/extension_bundle_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50d93be9..2e2d3701 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,6 +122,15 @@ jobs: version: "~> v2" args: check + # `goreleaser check` only validates config syntax, so it cannot see that + # packaging bundled extensions and downloading them have to land + # together. This does, and fails the PR instead of the release. + - name: Check bundled packaging is in step with the download + run: scripts/check-bundled-packaging-sync.sh + + - name: Test release scripts + run: make test-scripts + test-unit: name: Unit Tests runs-on: ubuntu-latest @@ -319,6 +328,43 @@ jobs: exit 1 fi + # Bundled extensions. Downloads the multi-call binary for every platform + # and the descriptions file from the private extensions repo, verifies + # them against its checksum manifest, and stages them under bundled/ for + # GoReleaser and the npm step below. The version file says `latest`, so + # the script resolves that to one concrete tag here, and every later step + # uses that tag. On a re-run of an already-published lstk release the tag + # is read back from that release's notes instead of re-resolved, so the + # re-run ships the same extension binaries the original did. + - name: Fetch bundled extensions + id: bundle + run: | + recorded="$(gh release view "${GITHUB_REF_NAME}" --json body --jq .body 2>/dev/null \ + | sed -n 's/^Bundled extensions: \([^ ]*\).*/\1/p' | head -n1 || true)" + if [ -n "${recorded}" ]; then + echo "Release ${GITHUB_REF_NAME} already records bundle ${recorded}; pinning to it." + export LSTK_EXTENSIONS_TAG="${recorded}" + fi + scripts/fetch-bundled-extensions.sh | tee fetch.log + tag="$(sed -n 's/^Resolved extensions bundle: \([^ ]*\) .*/\1/p' fetch.log)" + [ -n "${tag}" ] || { echo "could not determine the resolved bundle tag"; exit 1; } + commit="$(GH_TOKEN="${LSTK_EXTENSIONS_READ_TOKEN}" gh api "repos/${LSTK_EXTENSIONS_REPO}/commits/${tag}" --jq .sha)" + echo "tag=${tag}" >> "${GITHUB_OUTPUT}" + echo "commit=${commit}" >> "${GITHUB_OUTPUT}" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LSTK_EXTENSIONS_READ_TOKEN: ${{ secrets.LSTK_EXTENSIONS_READ_TOKEN }} + LSTK_EXTENSIONS_REPO: localstack/lstk-bundled-extensions + + # A command described in lstk-extensions.toml that the bundle does not + # provide (per bundle-commands.txt, which the fetch step records from the + # archives' lstk- aliases) would show in help and fail when run; a + # binary with no command list would be unreachable. Either fails the + # release. Descriptions and the list are the same on every platform, so + # one platform dir is enough. + - name: Check descriptions match the bundled binary + run: scripts/check-descriptions.sh bundled/linux_amd64 + - name: Run GoReleaser uses: goreleaser/goreleaser-action@v7 with: @@ -356,6 +402,14 @@ jobs: - name: Install signal-forwarding launcher run: cp npm/launcher.js dist/npm/lstk/index.js + # The launcher execs the Go binary from the platform package, so that is + # where lstk's bundled dir resolves to and where the bundled extensions + # must live — not the wrapper. The script also registers them in each + # package's `files` allowlist: the publisher emits "files": [], which npm + # packs as package.json + bin only, so a bare copy would be dropped. + - name: Add bundled extensions to the npm platform packages + run: scripts/add-bundled-to-npm.sh dist/npm bundled + - name: Publish to NPM run: | for dir in dist/npm/lstk-*/ dist/npm/lstk/; do @@ -363,3 +417,21 @@ jobs: done env: NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} + + # Permanent record of which bundle this release shipped. Job logs expire; + # the release notes do not, and they are what someone investigating a + # bundled-extension bug months later will read. Idempotent so a re-run + # does not append a second line. + - name: Record the bundle in the release notes + run: | + gh release view "${GITHUB_REF_NAME}" --json body --jq .body > notes.md + if grep -q '^Bundled extensions: ' notes.md; then + echo "Release notes already record the bundle." + exit 0 + fi + printf '\n\n---\n\nBundled extensions: %s (commit %s)\n' "${TAG}" "${COMMIT}" >> notes.md + gh release edit "${GITHUB_REF_NAME}" --notes-file notes.md + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ steps.bundle.outputs.tag }} + COMMIT: ${{ steps.bundle.outputs.commit }} diff --git a/.gitignore b/.gitignore index 3b0c8cee..b49bf4f4 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,11 @@ test/integration/test-samples/**/.terraform/ test/integration/test-samples/**/.terraform.lock.hcl test/integration/test-samples/**/*.tfstate test/integration/test-samples/**/*.tfstate.* + +# Bundled-extension staging tree, populated by +# scripts/fetch-bundled-extensions.sh at release-build time: per-platform +# binaries in bundled/_/ and bundled/lstk-extensions.toml. Only the +# version file is tracked; downloaded artifacts must never be committed. +# It deliberately lives outside dist/, which `goreleaser --clean` wipes. +/bundled/* +!/bundled/extensions.version diff --git a/.goreleaser.yaml b/.goreleaser.yaml index eecb5d78..df42fa54 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -44,6 +44,17 @@ archives: files: - completions/* - manpages/* + # LocalStack's bundled extensions: one multi-call binary per platform plus + # the descriptions file, at the archive root next to lstk. Staged under + # bundled/ by scripts/fetch-bundled-extensions.sh, which the release job + # runs first; a local snapshot build needs it too (see + # docs/extensions-bundling.md). A glob matching nothing fails the build. + - src: "bundled/{{ .Os }}_{{ .Arch }}/bundled-extensions*" + strip_parent: true + info: + mode: 0o755 + - src: bundled/lstk-extensions.toml + strip_parent: true checksum: name_template: checksums.txt @@ -71,5 +82,8 @@ homebrew_casks: post: install: | if OS.mac? - system_command "/usr/bin/xattr", args: ["-dr", "com.apple.quarantine", "#{staged_path}/lstk"] + # The whole staged dir, not only lstk: the bundled extensions binary + # sits next to it and would otherwise be blocked by Gatekeeper on its + # first run. + system_command "/usr/bin/xattr", args: ["-dr", "com.apple.quarantine", "#{staged_path}"] end diff --git a/CLAUDE.md b/CLAUDE.md index 0d416630..b3286ccd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -170,7 +170,7 @@ Shared plumbing lives in `cmd/proxy.go` (`leadingFlags`, `stripLeadingProxyFlags # Extensions -lstk supports Git-style extensions: when `lstk ` is not a built-in command or alias, lstk resolves and execs an external `lstk-` executable, forwarding arguments verbatim and propagating the exit code. Built-ins always win. Resolution order is built-ins → bundled dir (the directory of the symlink-resolved lstk executable) → `PATH`; there is no manifest. Runtime context is conveyed via `LSTK_EXT_API_VERSION` and `LSTK_EXT_CONTEXT` (JSON: `configDir`, optional `authToken`, `nonInteractive`, `json`, optional `sessionId` — lstk's telemetry session id, omitted when telemetry is disabled, so an extension's own telemetry can join lstk's `ext:` event — optional `machineId` — lstk's anonymized machine id (the prepared hash), omitted alongside `sessionId` when telemetry is disabled, so an extension reports the same machine without re-deriving it — optional `endpointUrl` — the resolved `--endpoint-url`/`LSTK_ENDPOINT_URL`/`AWS_ENDPOINT_URL` value, conveyed verbatim and unvalidated (dispatch never rejects or probes it, unlike the built-ins' `rejectEndpointURL`), omitted when no source is set — and an `emulators` array, which stays local-Docker discovery and is independent of `endpointUrl`) — see `extension.Context`/`Environ` in `internal/extension/context.go`; dispatch and help listing are in `cmd/extension.go`. Automated distribution/co-update of bundled extensions is deferred to the `add-bundled-extension-distribution` change. See [extensions-authoring.md](docs/extensions-authoring.md) for the author-facing contract. +lstk supports Git-style extensions: when `lstk ` is not a built-in command or alias, lstk resolves and execs an external `lstk-` executable, forwarding arguments verbatim and propagating the exit code. Built-ins always win. Resolution order is built-ins → bundled dir (the directory of the symlink-resolved lstk executable) → `PATH`; there is no manifest. Runtime context is conveyed via `LSTK_EXT_API_VERSION` and `LSTK_EXT_CONTEXT` (JSON: `configDir`, optional `authToken`, `nonInteractive`, `json`, optional `sessionId` — lstk's telemetry session id, omitted when telemetry is disabled, so an extension's own telemetry can join lstk's `ext:` event — optional `machineId` — lstk's anonymized machine id (the prepared hash), omitted alongside `sessionId` when telemetry is disabled, so an extension reports the same machine without re-deriving it — optional `endpointUrl` — the resolved `--endpoint-url`/`LSTK_ENDPOINT_URL`/`AWS_ENDPOINT_URL` value, conveyed verbatim and unvalidated (dispatch never rejects or probes it, unlike the built-ins' `rejectEndpointURL`), omitted when no source is set — and an `emulators` array, which stays local-Docker discovery and is independent of `endpointUrl`) — see `extension.Context`/`Environ` in `internal/extension/context.go`; dispatch and help listing are in `cmd/extension.go`. LocalStack's own bundled extensions ship as **one multi-call binary**, `bundled-extensions` (`extension.BundledBinaryName`), next to `lstk`, plus the hand-authored `lstk-extensions.toml`; that file is load-bearing for the bundle — it is the only record of which commands the binary provides — so `extension.LoadBundle` (`internal/extension/bundle.go`) hard-fails on a missing/malformed/empty one when the binary is present, whereas `LoadDescriptions` (help text only) still degrades quietly. `Resolver.Resolve` consults the bundle first for described names and execs it with `argv[0]` = `lstk-` (`Extension.Argv0`, applied in `Invoke`), then standalone `lstk-` files in the bundled dir, then `PATH`; a broken bundle is reported by `Resolve` only when nothing else provides the name, and skipped (logged) by `List` so help never breaks. `LoadBundle` also rejects a toml key that is not a dispatchable name (`validate.ExtensionName`, the same rule the release gate applies), and `Resolver.List` attaches each bundled entry's help `Description` so `cmd` renders from one parse of the file. The argv[0] contract is uniform: a standalone `lstk-` file is invoked under its own base name, which is the same `lstk-` a bundle-provided command receives. Distribution is automated in the release job: `scripts/fetch-bundled-extensions.sh` downloads and checksum-verifies the bundle selected by `bundled/extensions.version` from the private extensions repo, `scripts/check-descriptions.sh` gates the toml against the binary's own command list (`bundled/bundle-commands.txt`, which the fetch script records from the `lstk-` alias entries every bundle archive must carry, and which is never packaged), so a described command the binary cannot dispatch fails the release, `.goreleaser.yaml` packages both at the archive root (the cask inherits them; a release-job step copies them into each npm **platform** package), and the resolved bundle tag is recorded in the release notes. `scripts/check-bundled-packaging-sync.sh` runs on every PR to keep the packaging and download halves from merging separately. Bash tests for these scripts: `make test-scripts` (`scripts/tests/`). Set-wise co-update on the binary channel (`internal/update`) is still pending in the `add-bundled-extension-distribution` change. See [extensions-authoring.md](docs/extensions-authoring.md) for the author-facing contract and [extensions-bundling.md](docs/extensions-bundling.md) for the release pipeline and on-disk layout per channel. # Signal Forwarding to Wrapped Tools diff --git a/Makefile b/Makefile index 7c2262d8..748638c5 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ endif BUILD_DIR=bin export CGO_ENABLED=0 -.PHONY: build clean test test-integration lint govulncheck mock-generate otel +.PHONY: build clean test test-integration test-scripts lint govulncheck mock-generate otel # Always invoke `go build` and let Go's build cache handle incrementality; a # file target on bin/lstk would be skipped when the binary exists, even with @@ -23,6 +23,11 @@ test: test-integration: build @RUN="$(RUN)" ./scripts/test-integration.sh +# Bash suites for the release helper scripts under scripts/. They only ever run +# on the Linux release runner, so a bash suite is the faithful test here. +test-scripts: + @./scripts/test-scripts.sh + otel: docker compose -f docker-compose.tracing.yaml up -d diff --git a/bundled/extensions.version b/bundled/extensions.version new file mode 100644 index 00000000..a2f1ab78 --- /dev/null +++ b/bundled/extensions.version @@ -0,0 +1,21 @@ +# Which extensions bundle this lstk release ships. +# +# One value line (blank lines and #-comments are ignored), in one of two forms: +# +# latest Take the newest published release of the private extensions +# repository. This is the default: there is no routine bump to +# remember, and a release can never go silently stale. The release +# job resolves it to a concrete tag ONCE and records that tag in +# the published release notes, so an lstk version still maps to +# exactly one bundle. +# +# v0.1.0 Lock this build to that exact release tag of the private +# extensions repository. Use it to hold a bad bundle back. To +# re-run an already-published lstk release against the bundle it +# originally shipped, do not edit this file — pass that release's +# recorded tag to scripts/fetch-bundled-extensions.sh instead +# (--tag / LSTK_EXTENSIONS_TAG). +# +# This is the only tracked file under bundled/. The downloaded binaries and +# descriptions file are staged alongside it and are gitignored. +latest diff --git a/cmd/extension.go b/cmd/extension.go index ebfdda4d..38b8109e 100644 --- a/cmd/extension.go +++ b/cmd/extension.go @@ -54,7 +54,15 @@ func dispatchExtension(ctx context.Context, cfg *env.Env, tel *telemetry.Client, }) return output.NewSilentError(fmt.Errorf("unknown command %q for lstk", name)) } - return err + // Anything else is a broken bundled install (the multi-call binary is + // present but its command list is not loadable) — an lstk problem, not + // an unknown command, so it gets the styled error and a way out. + output.NewPlainSink(os.Stderr).Emit(output.ErrorEvent{ + Title: "bundled extensions are not usable", + Summary: err.Error(), + Actions: []output.ErrorAction{{Label: "Reinstall lstk to restore them:", Value: "lstk update"}}, + }) + return output.NewSilentError(err) } emulators := resolveEmulators(ctx, cfg, logger) @@ -153,33 +161,32 @@ func emulatorCandidates() []config.ContainerConfig { } // registerExtensionHelp wires an "extensions" template function that renders the -// Extensions section of `lstk --help`. It scans the bundled dir + PATH for -// `lstk-*` executables (de-duplicated, bundled wins) and attaches descriptions -// for bundled extensions from the hand-authored descriptions file; PATH and -// custom extensions, and bundled names missing from the file, are name-only. -// Rendering never executes an extension. A scan happens on each help render so -// freshly installed extensions appear without restarting. +// Extensions section of `lstk --help`. It lists the bundle's commands, then +// `lstk-*` executables in the bundled dir and on PATH (de-duplicated, bundled +// wins); bundled entries come with the description Resolver.List attached from +// the hand-authored descriptions file, while PATH and custom extensions, and +// bundled names missing from the file, are name-only. Rendering never executes +// an extension. A scan happens on each help render so freshly installed +// extensions appear without restarting. func registerExtensionHelp(logger log.Logger) { cobra.AddTemplateFunc("extensions", func(namePadding int) string { - resolver := extension.NewResolver(logger) - list := resolver.List() + list := extension.NewResolver(logger).List() if len(list) == 0 { return "" } - descriptions := extension.LoadDescriptions(resolver.BundledDir, logger) - return formatExtensionList(list, descriptions, namePadding) + return formatExtensionList(list, namePadding) }) } // formatExtensionList renders the extension help lines so they align with the // command sections above them. It mirrors Cobra's own scheme (see the usage // template's "{{rpad .Name .NamePadding}} {{.Short}}"): each name is right-padded -// to namePadding, then a single space, then its description (bundled extensions -// only, from the descriptions file). namePadding is the root command's +// to namePadding, then a single space, then its Description (set for bundled +// extensions only). namePadding is the root command's // .NamePadding, so the description column matches the Commands/Tools sections; a // name longer than namePadding widens its own row exactly as Cobra's per-row // rpad does. Lines are sorted by name (List already sorts). -func formatExtensionList(list []extension.Extension, descriptions map[string]string, namePadding int) string { +func formatExtensionList(list []extension.Extension, namePadding int) string { width := namePadding for _, ext := range list { if len(ext.Name) > width { @@ -189,11 +196,7 @@ func formatExtensionList(list []extension.Extension, descriptions map[string]str var b strings.Builder for _, ext := range list { - desc := "" - if ext.Bundled { - desc = descriptions[ext.Name] - } - if desc != "" { + if desc := ext.Description; desc != "" { fmt.Fprintf(&b, " %-*s %s\n", width, ext.Name, desc) } else { fmt.Fprintf(&b, " %s\n", ext.Name) diff --git a/docs/extensions-authoring.md b/docs/extensions-authoring.md index d927c6ed..9a8c57fc 100644 --- a/docs/extensions-authoring.md +++ b/docs/extensions-authoring.md @@ -5,6 +5,7 @@ lstk supports Git-style extensions. When you run `lstk ` and `` is n ## The contract at a glance - **Name it `lstk-`** and put it on `PATH`. `lstk ...` will run it; `lstk help` will list it. +- **You are invoked as `lstk-`.** Your `argv[0]` is the base name `lstk-` (plus `.exe` when your file carries that suffix on Windows), never a full path, and it is the same whether lstk found you on `PATH`, next to its own binary, or inside LocalStack's bundle. Strip a trailing `.exe` before comparing. A `` starts with a letter or digit and uses only letters, digits, hyphens and underscores. - **Your arguments are forwarded verbatim.** Everything after `` is yours — lstk does not parse it. Define and parse your own flags however you like, including flags that happen to share a name with an lstk global flag. - **lstk's global flags are consumed before the name.** `lstk --non-interactive --foo` runs your extension with just `--foo`; the resolved global state reaches you via environment variables (below), not on your command line. - **Exit code and streams pass through.** Your exit status becomes lstk's exit status, and your stdin/stdout/stderr are wired straight to the terminal. @@ -111,6 +112,21 @@ Reuse `machineId` rather than deriving your own: it is already the final hashed **Absence is ambiguous, by design.** Both fields are omitted when lstk's telemetry is disabled — a disabled lstk computes neither, so they always appear and disappear together — and both are also absent on an lstk released before they existed. You cannot tell those two cases apart, so don't try. Treat absence as "no correlation available" and carry on: generate or derive your own ids if you need them, and never make either field a hard requirement. +## How LocalStack's bundled extensions differ + +Everything above applies to bundled extensions too, with one mechanical +difference: they are not separate `lstk-` files. LocalStack ships one +multi-call binary, `bundled-extensions`, next to `lstk`, and lstk executes it +with `argv[0]` set to `lstk-` for whichever command was requested. The +list of commands it provides is the descriptions file `lstk-extensions.toml` +beside it; a name that file does not list is never handed to the bundle. Read +`os.Args[0]` (or your language's equivalent) to find out which extension you are +being asked to be. The value is exactly `lstk-`, with no path and no +`.exe`, and lstk only ever hands the bundle a name the toml lists, so a lookup +miss inside the binary means the toml and the binary disagree: report it +loudly rather than guessing. See [extensions-bundling.md](extensions-bundling.md) for how +the bundle is built and shipped. + ## Help descriptions `lstk --help` lists installed extensions by command name. One-line descriptions are shown **only for extensions LocalStack bundles with lstk**, from a static descriptions file LocalStack ships with them. Third-party and `PATH`-installed extensions are listed by name only (the same as Git's `git help -a`). lstk never executes an extension to render help, so listing is always side-effect-free. diff --git a/docs/extensions-bundling.md b/docs/extensions-bundling.md new file mode 100644 index 00000000..936cf14b --- /dev/null +++ b/docs/extensions-bundling.md @@ -0,0 +1,203 @@ +# Bundled extensions: how they are built, shipped and found + +This page is for whoever touches the release pipeline or debugs a bundled +extension in the field. It assumes no prior context. The author-facing +contract for writing an extension is in [extensions-authoring.md](extensions-authoring.md). + +## What ships + +LocalStack's own extensions (for example `lstk doctor`) are closed source and +ship **inside every lstk release** as two files placed next to the `lstk` +binary: + +| File | What it is | +| --- | --- | +| `bundled-extensions` (`.exe` on Windows) | One multi-call binary providing every bundled extension. It decides which extension to be from the name it is invoked as (`argv[0]`), the way busybox and git do. | +| `lstk-extensions.toml` | A flat TOML table, `name = "one-line description"`, hand-written in the private extensions repository. For the bundle this file is load-bearing: it is the only record of which commands the binary provides. | + +lstk never learns about bundled commands from directory contents. `lstk ` +reads the toml, sees `name` listed, and executes `bundled-extensions` with +`argv[0]` set to `lstk-`, forwarding the arguments and the usual +`LSTK_EXT_API_VERSION` / `LSTK_EXT_CONTEXT` runtime context. A name the toml +does not list is not handed to the bundle; lstk falls through to standalone +`lstk-` files and then `PATH`, exactly as for third-party extensions. +Every key in the toml must be a dispatchable command name (a letter or digit +first, then letters, digits, hyphens and underscores); lstk refuses to load a +bundle whose toml breaks that rule, and the release gate applies the same rule, +so a file that ships always loads. + +Why one binary rather than a copy per name: copies cost roughly 30 MB per +extension in every archive and npm package, and symlink aliases do not survive +the tar/zip extractors or Windows. One binary has none of those problems. The +trade is that the toml must be present and correct whenever the binary is, +which the release gate below enforces and the runtime treats as a broken +install if violated. + +## Where the files live on disk + +lstk looks in one place: the directory of its own symlink-resolved executable +(`extension.BundledDir`). Each install method lands the two files there +without any layout work of its own. + +| Channel | Directory | How it gets there | +| --- | --- | --- | +| Binary archive (`curl` + `tar`) | Wherever the user extracted the archive; the files sit at the archive root next to `lstk`. | GoReleaser `archives.files` entries in `.goreleaser.yaml`. | +| Homebrew | The cask's Caskroom staged directory, e.g. `/opt/homebrew/Caskroom/lstk//`. `bin/lstk` is a symlink into it; lstk resolves the link. | The cask stages the whole archive. Only `lstk` is symlinked into `bin`; the bundle is found via the directory, never via `PATH`. The post-install hook strips the macOS quarantine attribute from the **whole** staged directory so the bundle runs without a Gatekeeper prompt. | +| npm | The **platform** package, e.g. `node_modules/@localstack/lstk-darwin-arm64/`, not the `@localstack/lstk` wrapper. The launcher execs the Go binary from there, so that is where lstk's bundled dir resolves to. | `scripts/add-bundled-to-npm.sh` copies the files into each `dist/npm/lstk--/` directory before `npm publish`, translating Node's platform names (`win32` → `windows`, `x64` → `amd64`), **and** adds them to that package's `files` allowlist. The publisher generates `"files": []`, which npm reads as "only `package.json` and the `bin` entry", so a plain copy would be silently dropped at publish. | + +## The release pipeline + +Everything happens in the `release` job of `.github/workflows/ci.yml`, in this +order. Every step failing fails the release. + +1. **Select the bundle.** `bundled/extensions.version` (the only tracked file + under `bundled/`) says which release of the private extensions repository to + take. It says `latest` by default: the newest published bundle, with no + routine bump to remember. Set it to an explicit tag (`v0.3.1`) to hold a + build to one bundle. +2. **Fetch and verify.** `scripts/fetch-bundled-extensions.sh` resolves + `latest` to a concrete tag **once**, prints it, downloads that tag's release + assets with `gh release download`, and verifies every asset against the + `checksums.txt` published in the same release. A missing manifest, an + unlisted asset or a mismatching hash aborts. It then unpacks each platform + archive and stages only two members: the binary as + `bundled/_/bundled-extensions[.exe]` (mode 0755) and the + descriptions file as `bundled/lstk-extensions.toml` (taken once; every + archive must carry an identical copy). The `lstk-` alias entries in + the archives are not staged, but their names are recorded, sorted, in + `bundled/bundle-commands.txt`: they are the bundle's own declaration of + which commands the binary answers to. An archive with no aliases, or whose + list differs from another platform's, aborts. It fails if any of lstk's six target platforms + (`linux`/`darwin`/`windows` × `amd64`/`arm64`) has no archive. A platform can be exempted only by listing it in + `UNSUPPORTED_PLATFORMS` at the top of the script, so a gap is always a + visible choice. +3. **Gate the pairing.** `scripts/check-descriptions.sh bundled/linux_amd64` + reads the command names from the toml (left-hand side only; values are + never parsed) and compares them with the binary's own list in + `bundle-commands.txt`. It fails if commands are described but there is no + binary, if there is a binary but no commands are described, if the command + list is missing or empty, or if the toml describes a command the bundle + does not provide (lstk would exec the binary under a name it does not + answer to). A command the bundle provides but the toml omits only warns: + lstk will not expose it. Descriptions and the list are the same on every + platform, so one directory is enough. +4. **Package.** GoReleaser adds the staged files to each archive at the root. + The cask inherits them. `scripts/add-bundled-to-npm.sh` copies them into + the platform packages and registers them in each package's `files`. +5. **Record.** After publishing, the job appends + `Bundled extensions: (commit )` to the GitHub release notes. + Job logs expire; release notes do not. This line is how you answer "which + extensions build does this customer have?" from an lstk version number. + +The credential is `LSTK_EXTENSIONS_READ_TOKEN`, a repository secret holding a +fine-grained personal access token with **read-only Contents** access to the +private extensions repository and nothing else. It is deliberately not +`PRO_ACCESS_TOKEN`: the release should hold no more access than it needs, and +a read-only token rotates independently. + +### Re-running a published release + +`latest` re-resolves on every invocation, so a naive re-run of an +already-published tag could ship different extension binaries under a version +already in the wild. The fetch step guards against this: before resolving, it +reads the existing GitHub release for the lstk tag being built and, if its +notes already carry a `Bundled extensions:` line, pins the fetch to that tag. +To do the same by hand, pass the recorded tag explicitly: + +```bash +LSTK_EXTENSIONS_READ_TOKEN=... scripts/fetch-bundled-extensions.sh --tag v0.3.1 +``` + +## What the private repository must publish + +Each tagged release of `localstack/lstk-bundled-extensions` ships: + +- one archive per lstk target platform, named + `bundled-extensions___.tar.gz` (`.zip` for Windows), + containing at its root the multi-call binary `bundled-extensions` (`.exe` on + Windows), `lstk-extensions.toml`, and one `lstk-` alias entry (a + symlink on Unix, a copy on Windows) for every command the binary answers + to. The aliases are never installed: lstk dispatches by `argv[0]` and does + not need them on disk. They are required all the same, because they are the + binary's own statement of its command list, and the release gate verifies + the toml against exactly that list, so a described command the binary cannot + dispatch is caught before it ships; +- the same `lstk-extensions.toml` in every archive, hand-authored, describing + every command the binary provides — a described command with no + implementation would show in `lstk --help` and fail when run; +- `checksums.txt` with a SHA-256 line for every archive. + +The extensions team owns the descriptions text. lstk only validates that the +file and the binary agree. + +## Local snapshot builds + +Since the `bundled/` entries in `.goreleaser.yaml` are live, `goreleaser` fails +on an empty staging tree. Stage a bundle first, either for real: + +```bash +LSTK_EXTENSIONS_READ_TOKEN= scripts/fetch-bundled-extensions.sh +goreleaser release --snapshot --clean +``` + +or, without access to the private repository, with placeholders: + +```bash +scripts/fetch-bundled-extensions.sh --stub +goreleaser release --snapshot --clean +``` + +`--stub` writes a tiny shell-script stand-in named `bundled-extensions` for +every platform, plus a toml describing one placeholder `doctor` command, and +prints a banner: **artifacts built from a stub bundle must never be +released.** + +`scripts/check-bundled-packaging-sync.sh` runs on every PR next to +`goreleaser check`. It fails if `.goreleaser.yaml` references `bundled/` while +the release job has no fetch step, or the reverse. `goreleaser check` cannot +catch this itself because it only validates config syntax and never looks at +the filesystem. Both halves must land in the same PR. + +The bash suites for all four scripts run with `make test-scripts`. + +## Updates + +**Homebrew and npm** replace the whole package directory on `lstk update`, so +the binary, the bundle and the toml are always replaced together, including +renames and removals. + +**Binary channel** (`lstk update` downloading an archive itself): set-wise +replacement of `lstk` + `bundled-extensions` + `lstk-extensions.toml` is +pending (section 1 of the `add-bundled-extension-distribution` change). Until +it lands, the in-the-field updater replaces only `lstk`; the other two files +must be extracted from the archive by hand. Updating never deletes standalone +`lstk-` files a user placed next to the binary. + +## Diagnosing a broken install + +If `bundled-extensions` is present but `lstk-extensions.toml` is missing, +unreadable or empty, lstk cannot know which commands the binary provides. +Running a command the bundle would normally provide reports "bundled +extensions are not usable" with the reason, instead of "unknown command"; +`lstk --help` still renders, without the bundled entries. Reinstalling from +the same channel restores the pair. +The same happens when the file names a command that cannot be dispatched (a +space, a path character, a leading hyphen): the whole bundle is reported as +unusable rather than partially loaded, because the file is a release artifact +and an inconsistency in it is a release bug, not a per-entry condition. + +## Release-candidate checklist + +Run on the first bundling release and after any packaging change. On each +channel: + +- Fresh install (`curl` + `tar`; `brew install localstack/tap/lstk`; + `npm install -g @localstack/lstk`), then `lstk ` runs immediately and + `lstk --help` lists it with its description. +- On macOS, a bundled command runs with no Gatekeeper prompt from a genuinely + downloaded archive. +- Homebrew has created a `bin` symlink for `lstk` only; the bundle is not + symlinked. +- Starting from the previously released version, `lstk update` succeeds and + the installed bundle matches the new lstk. +- The published release notes carry the `Bundled extensions:` line. diff --git a/internal/extension/bundle.go b/internal/extension/bundle.go new file mode 100644 index 00000000..c9a0cf49 --- /dev/null +++ b/internal/extension/bundle.go @@ -0,0 +1,101 @@ +package extension + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + + "github.com/localstack/lstk/internal/log" + "github.com/localstack/lstk/internal/validate" +) + +// BundledBinaryName is the file name of the multi-call binary that provides +// every LocalStack-bundled extension. It ships next to lstk (in BundledDir) +// as one binary rather than one lstk- copy per extension, and dispatches +// on the name it is invoked as: lstk execs it with argv[0] set to +// "lstk-" (see Invoke), the busybox/git approach. +// +// This is what makes the bundle deliverable at all. Per-name copies would cost +// ~30 MB per extension in every archive and npm package; symlink aliases are +// dropped by the tar extractor, materialized as text files by the zip one, and +// need elevation to create on Windows. A single binary has none of those +// problems, at the cost of making the descriptions file load-bearing: it is the +// only record of which commands the binary provides, so LoadBundle treats a +// missing or unreadable one as a hard error rather than degrading to an empty +// set the way LoadDescriptions does for help rendering. +const BundledBinaryName = "bundled-extensions" + +// Bundle is the installed multi-call bundle: the path to its binary and the +// commands it provides with their help descriptions, taken from the +// descriptions file. +type Bundle struct { + Path string + descriptions map[string]string +} + +// Provides reports whether the bundle provides the given extension command. +func (b *Bundle) Provides(name string) bool { + _, ok := b.descriptions[name] + return ok +} + +// Names returns the bundle's extension command names, sorted. +func (b *Bundle) Names() []string { + names := make([]string, 0, len(b.descriptions)) + for name := range b.descriptions { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// Description returns the one-line help description of a bundled command, or +// "" when the bundle does not provide it. +func (b *Bundle) Description(name string) string { + return b.descriptions[name] +} + +// LoadBundle returns the multi-call bundle installed in dir, or (nil, nil) +// when dir has no BundledBinaryName executable — the pre-bundling shape, and +// any user who has only lstk- files there. When the binary IS present, +// the descriptions file becomes mandatory: a missing, unreadable, malformed, +// or empty one is returned as an error, because without it lstk cannot know +// which commands the binary answers to, and silently reporting "unknown +// command" would hide a broken install. +// +// Every key must also be a dispatchable command name (validate.ExtensionName, +// the rule the release gate applies to the same file). A key that fails it +// would make lstk exec the binary under an argv[0] nothing answers to, so it is +// reported as a broken bundle rather than skipped: the file is LocalStack's own +// release artifact, and an inconsistency in it is a release bug to surface, not +// a per-entry condition to tolerate. +func LoadBundle(dir string, logger log.Logger) (*Bundle, error) { + if dir == "" { + return nil, nil + } + path := findExecutable(dir, BundledBinaryName) + if path == "" { + return nil, nil + } + + descPath := filepath.Join(dir, DescriptionsFileName) + descriptions, err := readDescriptions(descPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("bundled extensions binary %s is installed but its command list %s is missing; reinstall lstk to restore it", path, descPath) + } + return nil, fmt.Errorf("bundled extensions command list %s is not usable: %w", descPath, err) + } + if len(descriptions) == 0 { + return nil, fmt.Errorf("bundled extensions command list %s describes no commands", descPath) + } + for name := range descriptions { + if err := validate.ExtensionName(name); err != nil { + return nil, fmt.Errorf("bundled extensions command list %s has an invalid command name %q: %w", descPath, name, err) + } + } + logger.Info("extension: bundle at %s provides %d command(s)", path, len(descriptions)) + return &Bundle{Path: path, descriptions: descriptions}, nil +} diff --git a/internal/extension/bundle_test.go b/internal/extension/bundle_test.go new file mode 100644 index 00000000..1b6d86e9 --- /dev/null +++ b/internal/extension/bundle_test.go @@ -0,0 +1,360 @@ +package extension + +import ( + "os" + "path/filepath" + goruntime "runtime" + "strings" + "testing" + + "github.com/localstack/lstk/internal/log" +) + +// writeBundle installs the multi-call bundled binary in dir plus a descriptions +// file naming the given commands, which is the on-disk shape a release ships. +func writeBundle(t *testing.T, dir string, names ...string) string { + t.Helper() + path := writeExe(t, dir, BundledBinaryName) + var b strings.Builder + for _, name := range names { + b.WriteString(name + " = \"Description of " + name + "\"\n") + } + if err := os.WriteFile(filepath.Join(dir, DescriptionsFileName), []byte(b.String()), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func TestLoadBundleReadsCommandsFromDescriptions(t *testing.T) { + dir := t.TempDir() + path := writeBundle(t, dir, "doctor", "deploy") + + bundle, err := LoadBundle(dir, log.Nop()) + if err != nil { + t.Fatalf("LoadBundle: %v", err) + } + if bundle == nil { + t.Fatal("expected a bundle") + } + if bundle.Path != path { + t.Fatalf("path = %q, want %q", bundle.Path, path) + } + if !bundle.Provides("doctor") || !bundle.Provides("deploy") { + t.Fatalf("expected doctor and deploy, got %v", bundle.Names()) + } + if bundle.Provides("nope") { + t.Fatal("expected an undescribed name not to be provided") + } +} + +func TestLoadBundleAbsentWhenNoBinary(t *testing.T) { + dir := t.TempDir() + // A descriptions file with no bundled binary is the pre-bundling shape. + if err := os.WriteFile(filepath.Join(dir, DescriptionsFileName), []byte("doctor = \"x\"\n"), 0o644); err != nil { + t.Fatal(err) + } + bundle, err := LoadBundle(dir, log.Nop()) + if err != nil { + t.Fatalf("LoadBundle: %v", err) + } + if bundle != nil { + t.Fatalf("expected no bundle, got %+v", bundle) + } +} + +func TestLoadBundleEmptyDir(t *testing.T) { + bundle, err := LoadBundle("", log.Nop()) + if err != nil || bundle != nil { + t.Fatalf("expected no bundle and no error, got %+v / %v", bundle, err) + } +} + +// The descriptions file is the only record of which commands the bundled binary +// provides, so unlike the lenient help path it must not degrade to "no +// extensions" when the binary is present. +func TestLoadBundleMissingDescriptionsIsHardError(t *testing.T) { + dir := t.TempDir() + writeExe(t, dir, BundledBinaryName) + + if _, err := LoadBundle(dir, log.Nop()); err == nil { + t.Fatal("expected an error when the descriptions file is missing") + } +} + +func TestLoadBundleMalformedDescriptionsIsHardError(t *testing.T) { + dir := t.TempDir() + writeExe(t, dir, BundledBinaryName) + if err := os.WriteFile(filepath.Join(dir, DescriptionsFileName), []byte("not = valid = toml ="), 0o644); err != nil { + t.Fatal(err) + } + if _, err := LoadBundle(dir, log.Nop()); err == nil { + t.Fatal("expected an error for a malformed descriptions file") + } +} + +func TestLoadBundleEmptyDescriptionsIsHardError(t *testing.T) { + dir := t.TempDir() + writeExe(t, dir, BundledBinaryName) + if err := os.WriteFile(filepath.Join(dir, DescriptionsFileName), []byte("\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := LoadBundle(dir, log.Nop()); err == nil { + t.Fatal("expected an error when the bundle describes no commands") + } +} + +func TestResolveBundledMultiCallSetsArgv0(t *testing.T) { + dir := t.TempDir() + path := writeBundle(t, dir, "doctor") + t.Setenv("PATH", t.TempDir()) + + r := &Resolver{BundledDir: dir, logger: log.Nop()} + ext, err := r.Resolve("doctor") + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if ext.Path != path { + t.Fatalf("path = %q, want the bundled binary %q", ext.Path, path) + } + if !ext.Bundled { + t.Fatal("expected the extension to be marked bundled") + } + // argv[0] is the whole mechanism: it is how the one binary knows which + // extension it is being asked to be. + if want := NamePrefix + "doctor"; ext.Argv0 != want { + t.Fatalf("argv0 = %q, want %q", ext.Argv0, want) + } +} + +func TestResolveBundledMultiCallWinsOverPath(t *testing.T) { + dir := t.TempDir() + pathDir := t.TempDir() + bundlePath := writeBundle(t, dir, "doctor") + writeExe(t, pathDir, "lstk-doctor") + t.Setenv("PATH", pathDir) + + r := &Resolver{BundledDir: dir, logger: log.Nop()} + ext, err := r.Resolve("doctor") + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if ext.Path != bundlePath { + t.Fatalf("expected the bundle to win, got %q", ext.Path) + } +} + +// An undescribed name is not part of the bundled set, so resolution must fall +// through to PATH rather than handing an unknown command to the bundle. +func TestResolveUndescribedNameFallsThroughToPath(t *testing.T) { + dir := t.TempDir() + pathDir := t.TempDir() + writeBundle(t, dir, "doctor") + writeExe(t, pathDir, "lstk-other") + t.Setenv("PATH", pathDir) + + r := &Resolver{BundledDir: dir, logger: log.Nop()} + ext, err := r.Resolve("other") + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if ext.Bundled { + t.Fatalf("expected the PATH extension, got bundled %+v", ext) + } +} + +func TestResolveUndescribedNameNotFound(t *testing.T) { + dir := t.TempDir() + writeBundle(t, dir, "doctor") + t.Setenv("PATH", t.TempDir()) + + r := &Resolver{BundledDir: dir, logger: log.Nop()} + if _, err := r.Resolve("other"); err != ErrNotFound { + t.Fatalf("expected ErrNotFound, got %v", err) + } +} + +// A shipped bundle whose descriptions file cannot be read is a broken install, +// not an absent extension: reporting "unknown command" would hide it. +func TestResolveBrokenBundleReturnsError(t *testing.T) { + dir := t.TempDir() + writeExe(t, dir, BundledBinaryName) + t.Setenv("PATH", t.TempDir()) + + r := &Resolver{BundledDir: dir, logger: log.Nop()} + _, err := r.Resolve("doctor") + if err == nil || err == ErrNotFound { + t.Fatalf("expected a descriptive error, got %v", err) + } +} + +func TestListIncludesBundledMultiCallCommands(t *testing.T) { + dir := t.TempDir() + pathDir := t.TempDir() + writeBundle(t, dir, "doctor", "deploy") + writeExe(t, pathDir, "lstk-hello") + t.Setenv("PATH", pathDir) + + r := &Resolver{BundledDir: dir, logger: log.Nop()} + list := r.List() + + if len(list) != 3 { + t.Fatalf("expected 3 extensions, got %d: %+v", len(list), list) + } + // Sorted by name: deploy, doctor, hello. + for i, want := range []string{"deploy", "doctor", "hello"} { + if list[i].Name != want { + t.Fatalf("list[%d].Name = %q, want %q", i, list[i].Name, want) + } + } + if !list[0].Bundled || !list[1].Bundled || list[2].Bundled { + t.Fatalf("unexpected bundled flags: %+v", list) + } + if list[0].Path != list[1].Path { + t.Fatal("expected both bundled commands to point at the one binary") + } +} + +// The bundled binary itself is not an extension named "bundled-extensions", and +// the descriptions file is not an extension named "extensions". +func TestListDoesNotListBundleArtifactsAsExtensions(t *testing.T) { + dir := t.TempDir() + writeBundle(t, dir, "doctor") + t.Setenv("PATH", t.TempDir()) + + r := &Resolver{BundledDir: dir, logger: log.Nop()} + for _, ext := range r.List() { + if ext.Name != "doctor" { + t.Fatalf("unexpected extension listed: %+v", ext) + } + } +} + +// Help rendering must never fail on account of a broken bundle, so List +// degrades where Resolve reports. +func TestListDegradesOnBrokenBundle(t *testing.T) { + dir := t.TempDir() + pathDir := t.TempDir() + writeExe(t, dir, BundledBinaryName) + writeExe(t, pathDir, "lstk-hello") + t.Setenv("PATH", pathDir) + + r := &Resolver{BundledDir: dir, logger: log.Nop()} + list := r.List() + if len(list) != 1 || list[0].Name != "hello" { + t.Fatalf("expected only the PATH extension, got %+v", list) + } +} + +// Manually placed lstk- files in the install directory keep working +// alongside the bundle, which is how the mechanism shipped before bundling. +func TestResolveManuallyPlacedBundledFileStillWorks(t *testing.T) { + dir := t.TempDir() + writeBundle(t, dir, "doctor") + manual := writeExe(t, dir, "lstk-manual") + t.Setenv("PATH", t.TempDir()) + + r := &Resolver{BundledDir: dir, logger: log.Nop()} + ext, err := r.Resolve("manual") + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if ext.Path != manual { + t.Fatalf("path = %q, want %q", ext.Path, manual) + } + if ext.Argv0 != execBase("lstk-manual") { + t.Fatalf("argv0 = %q, want the file's own name", ext.Argv0) + } +} + +func execBase(base string) string { + if goruntime.GOOS == "windows" { + return base + ".exe" + } + return base +} + +// A toml key that cannot be an extension command name (spaces, path +// separators, shell metacharacters, a leading hyphen) can never be dispatched: +// lstk would build an argv[0] no binary answers to. A bundle describing one is +// broken, not partially usable, and the rule mirrors the release gate's so a +// toml that passes the gate always loads. TOML quoted keys make such names +// syntactically valid, which is why the check cannot be left to the parser. +func TestLoadBundleInvalidCommandNameIsHardError(t *testing.T) { + for _, name := range []string{"doc tor", "../doctor", "a;b", "-doctor", "doctor.exe", ""} { + dir := t.TempDir() + writeExe(t, dir, BundledBinaryName) + body := "doctor = \"ok\"\n\"" + name + "\" = \"bad\"\n" + if err := os.WriteFile(filepath.Join(dir, DescriptionsFileName), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if _, err := LoadBundle(dir, log.Nop()); err == nil { + t.Errorf("LoadBundle accepted invalid command name %q", name) + } else if !strings.Contains(err.Error(), name) && name != "" { + t.Errorf("error for %q does not name the offending key: %v", name, err) + } + } +} + +func TestLoadBundleCarriesDescriptions(t *testing.T) { + dir := t.TempDir() + writeBundle(t, dir, "doctor") + + bundle, err := LoadBundle(dir, log.Nop()) + if err != nil { + t.Fatalf("LoadBundle: %v", err) + } + if got := bundle.Description("doctor"); got != "Description of doctor" { + t.Fatalf("Description(doctor) = %q", got) + } + if got := bundle.Description("nope"); got != "" { + t.Fatalf("Description(nope) = %q, want empty", got) + } +} + +// Help shows descriptions for bundled extensions only. List carries them on +// each entry so the caller renders from one read of the descriptions file +// instead of parsing it a second time. +func TestListAttachesDescriptionsToBundledEntries(t *testing.T) { + dir := t.TempDir() + pathDir := t.TempDir() + writeBundle(t, dir, "doctor") + writeExe(t, pathDir, "lstk-hello") + t.Setenv("PATH", pathDir) + + byName := map[string]Extension{} + for _, ext := range (&Resolver{BundledDir: dir, logger: log.Nop()}).List() { + byName[ext.Name] = ext + } + if got := byName["doctor"].Description; got != "Description of doctor" { + t.Fatalf("bundled description = %q", got) + } + if got := byName["hello"].Description; got != "" { + t.Fatalf("PATH extension must be name-only, got description %q", got) + } +} + +// Without a bundle binary, standalone lstk- files in the bundled dir +// still take their descriptions from the file (the pre-bundling shape), while a +// PATH extension stays name-only even when the file happens to describe it. +func TestListStandaloneBundledFileDescribedFromFile(t *testing.T) { + dir := t.TempDir() + pathDir := t.TempDir() + writeExe(t, dir, "lstk-deploy") + writeExe(t, pathDir, "lstk-hello") + body := "deploy = \"Deploy to LocalStack\"\nhello = \"Not for PATH extensions\"\n" + if err := os.WriteFile(filepath.Join(dir, DescriptionsFileName), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", pathDir) + + byName := map[string]Extension{} + for _, ext := range (&Resolver{BundledDir: dir, logger: log.Nop()}).List() { + byName[ext.Name] = ext + } + if got := byName["deploy"].Description; got != "Deploy to LocalStack" { + t.Fatalf("standalone bundled description = %q", got) + } + if got := byName["hello"].Description; got != "" { + t.Fatalf("PATH extension must be name-only, got description %q", got) + } +} diff --git a/internal/extension/descriptions.go b/internal/extension/descriptions.go index ec97e308..b0e524a9 100644 --- a/internal/extension/descriptions.go +++ b/internal/extension/descriptions.go @@ -1,6 +1,8 @@ package extension import ( + "errors" + "fmt" "os" "path/filepath" @@ -16,29 +18,47 @@ import ( // Its TOML body is a flat table of name = "description" entries, e.g.: // // deploy = "Deploy your application to LocalStack" +// +// When the multi-call bundle (BundledBinaryName) is installed the same file is +// also the record of which commands that binary provides; see LoadBundle for +// the stricter contract that implies. const DescriptionsFileName = "lstk-extensions.toml" +// readDescriptions parses the descriptions file at path into a +// name → description map. A read error is returned as-is so callers can test +// for os.ErrNotExist; a parse error is wrapped with the path. It is the single +// parser behind both LoadBundle (strict) and LoadDescriptions (lenient), so the +// two can never disagree on what the file says. +func readDescriptions(path string) (map[string]string, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + descriptions := map[string]string{} + if err := toml.Unmarshal(data, &descriptions); err != nil { + return nil, fmt.Errorf("parsing %s: %w", path, err) + } + return descriptions, nil +} + // LoadDescriptions reads the bundled descriptions file from dir and returns a // map of extension command name to one-line description. A missing or unreadable // file degrades to an empty map without error, so help rendering never fails on // account of descriptions. dir is the bundled-extensions directory; an empty dir -// yields an empty map. +// yields an empty map. It serves the pre-bundling shape (standalone lstk- +// files next to lstk); with the multi-call bundle installed, Resolver.List takes +// descriptions from the Bundle it already loaded instead. func LoadDescriptions(dir string, logger log.Logger) map[string]string { if dir == "" { return map[string]string{} } path := filepath.Join(dir, DescriptionsFileName) - data, err := os.ReadFile(path) + descriptions, err := readDescriptions(path) if err != nil { - if !os.IsNotExist(err) { - logger.Info("extension: could not read descriptions file %s: %v", path, err) + if !errors.Is(err, os.ErrNotExist) { + logger.Info("extension: could not load descriptions file %s: %v", path, err) } return map[string]string{} } - descriptions := map[string]string{} - if err := toml.Unmarshal(data, &descriptions); err != nil { - logger.Info("extension: could not parse descriptions file %s: %v", path, err) - return map[string]string{} - } return descriptions } diff --git a/internal/extension/exec.go b/internal/extension/exec.go index 66dc3c0e..bcb9c007 100644 --- a/internal/extension/exec.go +++ b/internal/extension/exec.go @@ -47,6 +47,11 @@ func Invoke(ctx context.Context, ext *Extension, args []string, runCtx Context) } cmd := exec.CommandContext(ctx, ext.Path, args...) + // exec.Command sets Args[0] to the path it was given; the multi-call bundle + // dispatches on argv[0] instead, so tell it which extension to be. + if ext.Argv0 != "" { + cmd.Args[0] = ext.Argv0 + } cmd.Env = envv cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout diff --git a/internal/extension/extension.go b/internal/extension/extension.go index 928b5cd1..aed95f22 100644 --- a/internal/extension/extension.go +++ b/internal/extension/extension.go @@ -7,6 +7,8 @@ // binary that never touches the core repository. package extension +import "path/filepath" + // APIVersion is the integer version of the LSTK_EXT_* runtime-context contract // that this lstk implements. It is exposed to extensions as // LSTK_EXT_API_VERSION. Bump it only when a variable is removed or repurposed; @@ -21,13 +23,26 @@ const NamePrefix = "lstk-" // the "lstk-" prefix) and the absolute path to the executable that provides it. // Bundled reports whether it was resolved from the bundled-extensions directory // (which ships with lstk and takes precedence over PATH) rather than from PATH. +// +// Argv0 is the name the executable is invoked as (its argv[0]). For a +// standalone lstk- file it is simply the file's own base name; for a +// command provided by the multi-call bundle (BundledBinaryName) it is +// "lstk-", which is how that one binary learns which extension to be. +// +// Description is the one-line help text for a bundled extension, taken from the +// descriptions file beside it. It is empty for PATH extensions (help lists them +// name-only) and for bundled files the file does not describe. Resolver.List +// fills it in; Resolve leaves it empty, since dispatch never renders it. type Extension struct { - Name string - Path string - Bundled bool + Name string + Path string + Bundled bool + Argv0 string + Description string } -// NewExtension returns an Extension for the given command name and executable path. +// NewExtension returns an Extension for the given command name and executable +// path, invoked under its own file name. func NewExtension(name, path string, bundled bool) *Extension { - return &Extension{Name: name, Path: path, Bundled: bundled} + return &Extension{Name: name, Path: path, Bundled: bundled, Argv0: filepath.Base(path)} } diff --git a/internal/extension/resolve.go b/internal/extension/resolve.go index f6d9b8c9..dd8cb203 100644 --- a/internal/extension/resolve.go +++ b/internal/extension/resolve.go @@ -55,13 +55,28 @@ func BundledDir(logger log.Logger) string { return filepath.Dir(resolved) } -// Resolve returns the extension for the given command name, searching the -// bundled directory first and then PATH. It returns ErrNotFound when no -// matching executable exists anywhere. +// Resolve returns the extension for the given command name, searching in +// order: the multi-call bundle in the bundled directory (for the commands its +// descriptions file lists), standalone lstk- files in the bundled +// directory, then PATH. It returns ErrNotFound when no match exists anywhere. +// +// A bundle whose descriptions file cannot be loaded does not block extensions +// found elsewhere, but when nothing else provides the name that load error is +// returned instead of ErrNotFound: the user has the binary installed and asked +// for a command it very likely provides, and "unknown command" would hide the +// broken install from them. func (r *Resolver) Resolve(name string) (*Extension, error) { base := NamePrefix + name + var bundleErr error if r.BundledDir != "" { + bundle, err := LoadBundle(r.BundledDir, r.logger) + switch { + case err != nil: + bundleErr = err + case bundle != nil && bundle.Provides(name): + return &Extension{Name: name, Path: bundle.Path, Bundled: true, Argv0: base}, nil + } if path := findExecutable(r.BundledDir, base); path != "" { return NewExtension(name, path, true), nil } @@ -71,17 +86,29 @@ func (r *Resolver) Resolve(name string) (*Extension, error) { return NewExtension(name, path, false), nil } + if bundleErr != nil { + return nil, bundleErr + } return nil, ErrNotFound } -// List returns the extensions resolvable from the bundled directory and PATH, -// de-duplicated by command name with bundled-then-PATH precedence (so a bundled -// extension shadows a same-named PATH executable), sorted by command name. It -// never executes an extension. +// List returns the extensions resolvable from the bundle, the bundled directory +// and PATH, de-duplicated by command name with that same precedence (so a +// bundled extension shadows a same-named PATH executable), sorted by command +// name. It never executes an extension. A bundle that fails to load is logged +// and skipped rather than failing the listing, so help rendering never breaks +// on account of it; Resolve is where that failure is reported. +// +// Bundled entries carry their help Description. With a bundle installed the +// descriptions come from the load that already parsed the file; without one +// (or with a broken one) the lenient LoadDescriptions serves the standalone +// lstk- files in the bundled dir, the pre-bundling shape. PATH entries +// are always name-only, whatever the file says. func (r *Resolver) List() []Extension { seen := map[string]struct{}{} var found []Extension + describe := func(string) string { return "" } add := func(dir string, bundled bool) { for _, name := range scanDir(dir) { if _, ok := seen[name]; ok { @@ -89,11 +116,35 @@ func (r *Resolver) List() []Extension { } seen[name] = struct{}{} path := findExecutable(dir, NamePrefix+name) - found = append(found, Extension{Name: name, Path: path, Bundled: bundled}) + ext := NewExtension(name, path, bundled) + if bundled { + ext.Description = describe(name) + } + found = append(found, *ext) } } if r.BundledDir != "" { + bundle, err := LoadBundle(r.BundledDir, r.logger) + if err != nil { + r.logger.Info("extension: skipping bundle in help listing: %v", err) + } + if bundle != nil { + describe = bundle.Description + for _, name := range bundle.Names() { + seen[name] = struct{}{} + found = append(found, Extension{ + Name: name, + Path: bundle.Path, + Bundled: true, + Argv0: NamePrefix + name, + Description: bundle.Description(name), + }) + } + } else { + descriptions := LoadDescriptions(r.BundledDir, r.logger) + describe = func(name string) string { return descriptions[name] } + } add(r.BundledDir, true) } for _, dir := range pathDirs() { diff --git a/internal/validate/validate.go b/internal/validate/validate.go index 0eb67936..954a982f 100644 --- a/internal/validate/validate.go +++ b/internal/validate/validate.go @@ -215,3 +215,39 @@ func AuthToken(value string) error { } return nil } + +// extensionNameRegexp matches an extension command name as lstk dispatches it: +// `lstk ` runs the executable `lstk-`, or hands the bundled +// multi-call binary argv[0] "lstk-". The first character must be a letter +// or digit so a name can never read as a flag, and the rest is limited to +// letters, digits, hyphens, and underscores. It is the same rule the release +// gate (scripts/check-descriptions.sh) applies to lstk-extensions.toml keys, so +// a descriptions file that passes the gate always loads and vice versa. +var extensionNameRegexp = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`) + +// ExtensionName validates an extension command name taken from the bundled +// descriptions file (lstk-extensions.toml). Although that file ships with lstk +// rather than being typed by a user, TOML quoted keys make any string a valid +// key, so the parser cannot be relied on to reject an undispatchable name. Like +// PodName it runs ordered deny-checks so the most specific reason wins, then the +// allow-list. The 64-character cap is a local sanity limit, not a contract. +func ExtensionName(value string) error { + const field = "extension name" + switch { + case value == "": + return newError(field, RuleEmpty, "must not be empty") + case containsControlChars(value): + return newError(field, RuleControlChars, "contains control characters") + case strings.Contains(value, "%"): + return newError(field, RuleEncoding, "contains percent-encoding") + case strings.ContainsAny(value, "/?#\\"): + return newError(field, RuleEmbedded, "contains path or query characters (/, \\, ?, #)") + case strings.ContainsAny(value, shellMetaChars): + return newError(field, RuleMetachars, "contains shell metacharacters") + case len(value) > 64: + return newError(field, RuleRange, "must be 64 characters or fewer") + case !extensionNameRegexp.MatchString(value): + return newError(field, RuleFormat, "must start with a letter or digit and use only letters, digits, hyphens, and underscores") + } + return nil +} diff --git a/internal/validate/validate_test.go b/internal/validate/validate_test.go index 4a4e6e3d..62a27d11 100644 --- a/internal/validate/validate_test.go +++ b/internal/validate/validate_test.go @@ -255,3 +255,50 @@ func TestServiceList(t *testing.T) { }) } } + +func TestExtensionName(t *testing.T) { + t.Parallel() + tests := []struct { + name string + value string + wantErr bool + wantRule string + }{ + {"simple", "doctor", false, ""}, + {"hyphenated", "deploy-app", false, ""}, + {"underscored", "snake_case", false, ""}, + {"leading digit", "2fa", false, ""}, + {"single char", "a", false, ""}, + {"maximum length", strings.Repeat("a", 64), false, ""}, + {"too long", strings.Repeat("a", 65), true, RuleRange}, + {"empty", "", true, RuleEmpty}, + {"control char", "doc\x00tor", true, RuleControlChars}, + {"percent encoding", "doc%20tor", true, RuleEncoding}, + {"path traversal", "../doctor", true, RuleEmbedded}, + {"slash", "a/b", true, RuleEmbedded}, + {"shell metachar", "a;rm", true, RuleMetachars}, + {"space", "doc tor", true, RuleFormat}, + {"leading hyphen", "-doctor", true, RuleFormat}, + {"leading underscore", "_doctor", true, RuleFormat}, + {"dot", "doc.tor", true, RuleFormat}, + {"exe suffix", "doctor.exe", true, RuleFormat}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := ExtensionName(tt.value) + if (err != nil) != tt.wantErr { + t.Fatalf("ExtensionName(%q) error = %v, wantErr %v", tt.value, err, tt.wantErr) + } + if tt.wantRule != "" { + var ve *Error + if !errors.As(err, &ve) { + t.Fatalf("ExtensionName(%q) error is not *Error: %T", tt.value, err) + } + if ve.Rule != tt.wantRule { + t.Errorf("ExtensionName(%q) Rule = %q, want %q", tt.value, ve.Rule, tt.wantRule) + } + } + }) + } +} diff --git a/openspec/changes/add-bundled-extension-distribution/design.md b/openspec/changes/add-bundled-extension-distribution/design.md index e6a2fa27..fb0a01c4 100644 --- a/openspec/changes/add-bundled-extension-distribution/design.md +++ b/openspec/changes/add-bundled-extension-distribution/design.md @@ -83,7 +83,10 @@ The `archives.files` entries land commented until the private pull is wired, the The first bundling release ships with the smallest viable bundle (a single extension) and is verified against the release-candidate checklist in `docs/extensions-bundling.md` — fresh install and upgrade-from-previous on all three channels — before further extensions are added to the bundle. -### Decision 7: Bundled binary layout — OPEN, needs a call before implementation +### Decision 7: Bundled binary layout — RESOLVED: (b), one multi-call binary + +**Resolution (2026-08-27, DPX-692):** option (b). The bundle ships as a single binary named `bundled-extensions` next to `lstk`, plus `lstk-extensions.toml`. lstk takes the bundled command list from the descriptions file and execs the one binary with `argv[0]` set to `lstk-` (`Extension.Argv0`, honoured in `extension.Invoke`). The runtime changes this implies are in `internal/extension/bundle.go` (`LoadBundle`, `BundledBinaryName`) and the bundle branch of `Resolver.Resolve`/`List`: the bundle is consulted first for described names, then standalone `lstk-` files in the bundled dir (manual placement keeps working), then PATH. When the binary is present, a missing/unreadable/empty descriptions file is a hard error surfaced by `Resolve` when nothing else provides the name — never a silent "unknown command" — while `List` logs and skips it so help never breaks. Release-side, `scripts/check-descriptions.sh` enforces the same pairing (described-but-no-binary and binary-but-nothing-described both fail), and `.goreleaser.yaml` packages `bundled-extensions*` + the toml. Since the binary cannot be executed cross-platform at release time, its command list is taken from the `lstk-` alias entries the bundle archives carry: the fetch script records them as `bundled/bundle-commands.txt` (never packaged) and the gate fails on a described name that list lacks, which makes the aliases a required part of the private repo's release convention rather than an optional convenience. The same name rule (`validate.ExtensionName`) is applied by `LoadBundle`, so a toml that passes the gate always loads. The fetch script unpacks the private repo's actual release convention — one `bundled-extensions___.tar.gz`/`.zip` per platform containing the binary, the toml and `lstk-` alias entries — and stages only the binary and the toml, recording the aliases' names rather than the aliases themselves (symlinks would be dropped by lstk's tar extractor and copies would triple the payload). The section-1 updater work and the section-6 test plan should be read with "the set" = `lstk`, `bundled-extensions`, `lstk-extensions.toml`, and "complete" = the binary present alongside a loadable descriptions file. The original analysis follows. + **What this blocks, and what it does not.** Decision 7 gates section 5 of `tasks.md` (turning packaging on) and the parts of the test plan that name individual on-disk files, because both have to know what the payload looks like. It does not gate sections 1 to 3: the set-wise updater, the descriptions check and the fetch script are all written against "whatever the archive contains" and can be built and merged first. Leaving it open therefore holds nothing up, and the input it is waiting for (whether the bundle really is one binary, and how big it is) arrives naturally once the doctor extension exists. diff --git a/openspec/changes/add-bundled-extension-distribution/tasks.md b/openspec/changes/add-bundled-extension-distribution/tasks.md index 272942a0..1cc569ae 100644 --- a/openspec/changes/add-bundled-extension-distribution/tasks.md +++ b/openspec/changes/add-bundled-extension-distribution/tasks.md @@ -16,8 +16,9 @@ Today `internal/update/extract.go` extracts the downloaded archive and replaces The descriptions file `lstk-extensions.toml` is a flat TOML table (`deploy = "One-line description"`), hand-written in the private extensions repo. If it describes an extension that we didn't actually ship a binary for, users would see help text for a command that doesn't work. This script makes that a release-blocking error. -- [ ] 2.1 Re-introduce `scripts/check-descriptions.sh` (plain bash, same style as `scripts/test-integration.sh`). Input: a directory containing the downloaded extension binaries and the toml. Behavior: read the names on the left-hand side of each `name = "…"` line (only the names — never parse the values, so a weird description string can't break the script); for each name, check an executable file `lstk-` exists in that directory; if any is missing, print which ones and exit non-zero (this fails the release). The reverse case — a binary present but not described — only prints a warning, because lstk's help intentionally falls back to showing such extensions name-only. -- [ ] 2.2 Test the script against fixture directories (a small test script or make target creating temp dirs): described-but-missing binary → fails and names it; described-and-present → passes; binary-without-description → warns but passes; empty or absent toml → passes (nothing is described, nothing to check). +- [x] 2.1 Re-introduce `scripts/check-descriptions.sh` (plain bash, same style as `scripts/test-integration.sh`). Input: a directory containing the downloaded extension binaries and the toml. Behavior: read the names on the left-hand side of each `name = "…"` line (only the names — never parse the values, so a weird description string can't break the script); for each name, check an executable file `lstk-` exists in that directory; if any is missing, print which ones and exit non-zero (this fails the release). The reverse case — a binary present but not described — only prints a warning, because lstk's help intentionally falls back to showing such extensions name-only. +- [x] 2.2 Test the script against fixture directories (a small test script or make target creating temp dirs): described-but-missing binary → fails and names it; described-and-present → passes; binary-without-description → warns but passes; empty or absent toml → passes (nothing is described, nothing to check). +- [x] 2.3 Verify the described names against the bundle's own command list, not just the binary's presence. The fetch script records the `lstk-` alias entries every archive carries into `bundled/bundle-commands.txt` (aborting on an archive with none, or on lists that differ between platforms), and `check-descriptions.sh` fails when the toml describes a command that list lacks and warns on the reverse. Covered in both bash suites; the list is never packaged. Note: the check runs once per release, against the Linux/amd64 download directory only. Descriptions are the same for every OS, and on Linux the binaries have plain names with no `.exe`, so one directory is enough. @@ -25,31 +26,31 @@ Note: the check runs once per release, against the Linux/amd64 download director The extension binaries are never committed to this repo. Instead, a one-line version file says which bundle to take — `latest` normally, so there is nothing to maintain, or an explicit tag when we deliberately want to hold a build to one bundle. A script resolves that to a concrete tag and downloads it at release-build time. Resolving happens once per build and the answer is recorded, so a release version always maps to exactly one bundle even if the release job is re-run (design Decision 2). -- [ ] 3.1 Add the version file `bundled/extensions.version` containing a single line: either `latest` (the default we ship) or an explicit release tag of the private extensions repo (e.g. `v0.1.0`). Document both forms in the file itself, since it is the only place someone will look. Add `.gitignore` rules so ONLY this file is tracked: the downloaded binaries land in `bundled/_/` folders and the toml at `bundled/lstk-extensions.toml`, and none of that may ever be committed. (Why the staging folder is `bundled/` at the repo root and not inside `dist/`: the release runs `goreleaser --clean`, which deletes `dist/` before building — it would wipe the downloads.) -- [ ] 3.2 Add `scripts/fetch-bundled-extensions.sh`. What it does, in order: read the version file and, if it says `latest`, resolve it to the concrete tag of the newest published release and print the resolved tag (every later step uses the resolved tag, never `latest` again, so one build can't mix two bundles); accept an already-resolved tag via an env var or flag so a re-run of a published release can be pointed back at the bundle it originally shipped; download that tag's release assets from the private extensions repo with `gh release download` (repo name configurable via an env var, with a sensible default); verify every downloaded file against the `checksums.txt` that the private repo publishes in the same release — abort loudly if the manifest is missing or any hash doesn't match; then arrange the files into the layout the rest of the pipeline expects: binaries at `bundled/_/lstk-` (with `.exe` for Windows), executable bit set, and the descriptions file at `bundled/lstk-extensions.toml`. -- [ ] 3.3 Make the script fail — listing exactly what's missing — if any of lstk's six target platforms (`linux`/`darwin`/`windows` × `amd64`/`arm64`) has no binary for a bundled extension. A platform can be exempted by adding it to an `UNSUPPORTED_PLATFORMS` list at the top of the script, so skipping a platform is always a visible, deliberate choice. Without this check, a missing binary would surface later as a confusing "glob matched nothing" error inside GoReleaser. -- [ ] 3.4 Add a `--stub` flag that skips the download entirely and writes placeholder files into the same layout. This exists for contributors without access to the private repo who want to run a local `goreleaser` snapshot build (which fails if `bundled/` is empty once section 5 is merged). Print an unmissable banner that stub output must never be released. -- [ ] 3.5 When run without a token, fail with a message that says which secret/env var is needed and mentions `--stub` as the alternative for local builds. +- [x] 3.1 Add the version file `bundled/extensions.version` containing a single line: either `latest` (the default we ship) or an explicit release tag of the private extensions repo (e.g. `v0.1.0`). Document both forms in the file itself, since it is the only place someone will look. Add `.gitignore` rules so ONLY this file is tracked: the downloaded binaries land in `bundled/_/` folders and the toml at `bundled/lstk-extensions.toml`, and none of that may ever be committed. (Why the staging folder is `bundled/` at the repo root and not inside `dist/`: the release runs `goreleaser --clean`, which deletes `dist/` before building — it would wipe the downloads.) +- [x] 3.2 Add `scripts/fetch-bundled-extensions.sh`. What it does, in order: read the version file and, if it says `latest`, resolve it to the concrete tag of the newest published release and print the resolved tag (every later step uses the resolved tag, never `latest` again, so one build can't mix two bundles); accept an already-resolved tag via an env var or flag so a re-run of a published release can be pointed back at the bundle it originally shipped; download that tag's release assets from the private extensions repo with `gh release download` (repo name configurable via an env var, with a sensible default); verify every downloaded file against the `checksums.txt` that the private repo publishes in the same release — abort loudly if the manifest is missing or any hash doesn't match; then arrange the files into the layout the rest of the pipeline expects: binaries at `bundled/_/lstk-` (with `.exe` for Windows), executable bit set, and the descriptions file at `bundled/lstk-extensions.toml`. +- [x] 3.3 Make the script fail — listing exactly what's missing — if any of lstk's six target platforms (`linux`/`darwin`/`windows` × `amd64`/`arm64`) has no binary for a bundled extension. A platform can be exempted by adding it to an `UNSUPPORTED_PLATFORMS` list at the top of the script, so skipping a platform is always a visible, deliberate choice. Without this check, a missing binary would surface later as a confusing "glob matched nothing" error inside GoReleaser. +- [x] 3.4 Add a `--stub` flag that skips the download entirely and writes placeholder files into the same layout. This exists for contributors without access to the private repo who want to run a local `goreleaser` snapshot build (which fails if `bundled/` is empty once section 5 is merged). Print an unmissable banner that stub output must never be released. +- [x] 3.5 When run without a token, fail with a message that says which secret/env var is needed and mentions `--stub` as the alternative for local builds. ## 4. The private repo side, and wiring the download into the release (cross-team) -- [ ] 4.1 Agree with the owners of the private extensions repo on what their releases must contain, and write it down in `docs/extensions-bundling.md`: each tagged release ships one binary per extension per platform, named `lstk-__` (plus `.exe` for Windows), the hand-written `lstk-extensions.toml`, and a `checksums.txt` covering every asset. They own the descriptions text; we only validate it. +- [x] 4.1 Agree with the owners of the private extensions repo on what their releases must contain, and write it down in `docs/extensions-bundling.md`: each tagged release ships one binary per extension per platform, named `lstk-__` (plus `.exe` for Windows), the hand-written `lstk-extensions.toml`, and a `checksums.txt` covering every asset. They own the descriptions text; we only validate it. - [ ] 4.2 Create the credential the release uses to download from the private repo: a fine-grained personal access token with **read-only** access to **only** that repo, stored as a repository/organization secret (e.g. `LSTK_EXTENSIONS_READ_TOKEN`). Deliberately not reusing `PRO_ACCESS_TOKEN` — the release should not hold broader access than it needs, and a read-only token can be rotated independently. -- [ ] 4.3 In `.github/workflows/ci.yml`, add two steps to the `release` job before the GoReleaser step: run `scripts/fetch-bundled-extensions.sh` (with the secret), then `scripts/check-descriptions.sh bundled/linux_amd64`. Either failing must fail the release. -- [ ] 4.4 Record which bundle a release shipped, permanently. In the `release` job, take the tag resolved by 3.2 (and the bundle's commit hash) and append it to the published GitHub release notes. The job log is not enough — GitHub expires logs, and this is exactly the information someone needs months later when a customer reports a bug in a bundled extension. Document in `docs/extensions-bundling.md` how to read it back, and how to re-run a published release against its original bundle by passing that tag to the fetch script. -- [ ] 4.5 Add a CI check that fails when the packaging half and the download half are out of step, so a future PR cannot get this wrong the way section 5 warns about. It should fail if `.goreleaser.yaml` has live (uncommented) `files:` entries pointing at `bundled/` while the `release` job in `.github/workflows/ci.yml` has no `fetch-bundled-extensions.sh` step — and fail in the opposite direction too. Run it on every PR alongside `goreleaser check`, which cannot catch this itself because it only validates config syntax and never looks at the filesystem. +- [x] 4.3 In `.github/workflows/ci.yml`, add two steps to the `release` job before the GoReleaser step: run `scripts/fetch-bundled-extensions.sh` (with the secret), then `scripts/check-descriptions.sh bundled/linux_amd64`. Either failing must fail the release. +- [x] 4.4 Record which bundle a release shipped, permanently. In the `release` job, take the tag resolved by 3.2 (and the bundle's commit hash) and append it to the published GitHub release notes. The job log is not enough — GitHub expires logs, and this is exactly the information someone needs months later when a customer reports a bug in a bundled extension. Document in `docs/extensions-bundling.md` how to read it back, and how to re-run a published release against its original bundle by passing that tag to the fetch script. +- [x] 4.5 Add a CI check that fails when the packaging half and the download half are out of step, so a future PR cannot get this wrong the way section 5 warns about. It should fail if `.goreleaser.yaml` has live (uncommented) `files:` entries pointing at `bundled/` while the `release` job in `.github/workflows/ci.yml` has no `fetch-bundled-extensions.sh` step — and fail in the opposite direction too. Run it on every PR alongside `goreleaser check`, which cannot catch this itself because it only validates config syntax and never looks at the filesystem. ## 5. Turn on packaging in all three install channels (one PR, together with 4.3) ⚠️ These changes reference the `bundled/` folder that only exists after the fetch script has run. GoReleaser fails a release when a `files:` pattern matches nothing, and the PR-level `goreleaser check` job won't catch it (it only checks config syntax, it never looks at the filesystem). This section is also gated on design Decision 7, which decides what the payload actually is. So this section must merge in the same PR as the CI wiring in 4.3 — enabling one without the other breaks every release until reverted. Task 4.5 adds a CI check that enforces this, so it fails as a red build on the PR rather than as a broken release. -- [ ] 5.1 Binary archives — in `.goreleaser.yaml`, add the bundled files to `archives.files` so they end up next to `lstk` at the archive root: +- [x] 5.1 Binary archives — in `.goreleaser.yaml`, add the bundled files to `archives.files` so they end up next to `lstk` at the archive root: `{ src: "bundled/{{ .Os }}_{{ .Arch }}/lstk-*", strip_parent: true, info: { mode: 0o755 } }` and `{ src: "bundled/lstk-extensions.toml", strip_parent: true }`. (`strip_parent` drops the `bundled/linux_amd64/` folder prefix so the files sit at the root; the explicit `mode` keeps them executable regardless of how the download step left them.) -- [ ] 5.2 Homebrew — the cask needs no layout work at all: it stages the whole archive into the Caskroom, and lstk finds the extensions there automatically. Two things to do anyway: change the post-install hook in `.goreleaser.yaml` from de-quarantining only `#{staged_path}/lstk` to the whole staged directory (`xattr -dr com.apple.quarantine "#{staged_path}"`) — without this, macOS Gatekeeper blocks the first run of every bundled extension — and double-check the generated cask still symlinks only `lstk` into `bin` (extensions must stay un-symlinked; they're found via the bundled dir, not PATH). -- [ ] 5.3 npm — the real Go binary lives in the platform package (`@localstack/lstk-darwin-arm64` etc.), not in the `@localstack/lstk` wrapper, so that's where the extensions must go. The npm build tool can't add per-platform files, so add a step to the release job right after the existing "Install signal-forwarding launcher" step: for each `dist/npm/lstk--/` directory, copy in the matching `bundled/_/lstk-*` files and the toml. Node and Go name platforms differently — translate `win32`→`windows` and `x64`→`amd64` (`darwin`, `linux`, `arm64` are the same in both). Don't touch the wrapper package, its `bin` entry, or the launcher. -- [ ] 5.4 Document how to run a local snapshot build after this lands (fetch with a token, or `--stub`) in `docs/extensions-bundling.md`, and leave a one-line comment next to the new `.goreleaser.yaml` entries pointing there. +- [x] 5.2 Homebrew — the cask needs no layout work at all: it stages the whole archive into the Caskroom, and lstk finds the extensions there automatically. Two things to do anyway: change the post-install hook in `.goreleaser.yaml` from de-quarantining only `#{staged_path}/lstk` to the whole staged directory (`xattr -dr com.apple.quarantine "#{staged_path}"`) — without this, macOS Gatekeeper blocks the first run of every bundled extension — and double-check the generated cask still symlinks only `lstk` into `bin` (extensions must stay un-symlinked; they're found via the bundled dir, not PATH). +- [x] 5.3 npm — the real Go binary lives in the platform package (`@localstack/lstk-darwin-arm64` etc.), not in the `@localstack/lstk` wrapper, so that's where the extensions must go. The npm build tool can't add per-platform files, so add a step to the release job right after the existing "Install signal-forwarding launcher" step: for each `dist/npm/lstk--/` directory, copy in the matching `bundled/_/lstk-*` files and the toml. Node and Go name platforms differently — translate `win32`→`windows` and `x64`→`amd64` (`darwin`, `linux`, `arm64` are the same in both). Don't touch the wrapper package, its `bin` entry, or the launcher. +- [x] 5.4 Document how to run a local snapshot build after this lands (fetch with a token, or `--stub`) in `docs/extensions-bundling.md`, and leave a one-line comment next to the new `.goreleaser.yaml` entries pointing there. ## 6. Test plan for the update path @@ -115,6 +116,18 @@ Groups 6.2–6.6 are automated (unit tests in `internal/update/extract_test.go`, - [ ] 6.11 Exit criteria — the first bundling release does not go out until: every case in 6.2–6.6 passes on Linux, macOS and Windows in CI; 6.7 passes on all three channels; 6.10 has been executed by hand against the real release candidate and signed off; and the transition case (6.4) has been verified against a genuinely published previous release rather than a locally built stand-in. +## 8. Runtime: the multi-call bundle (consequence of design Decision 7 = option b) + +Decided during DPX-692: the bundle ships as one binary, `bundled-extensions`, not one `lstk-` copy per extension. That makes `lstk-extensions.toml` the list of bundled commands, so the runtime has to read it rather than the directory. Sections 2 and 5 above are implemented against this shape; section 1 and the section-6 test plan should read "the set" as `lstk` + `bundled-extensions` + `lstk-extensions.toml`. + +- [x] 8.1 `internal/extension/bundle.go`: `BundledBinaryName` and `LoadBundle(dir)`, returning the binary path plus the command names from the toml. No binary → `(nil, nil)` (pre-bundling shape). Binary present but toml missing, unreadable, malformed or empty → error, because the file is the only record of which commands exist. `LoadDescriptions` (help text) keeps degrading quietly. +- [x] 8.2 `Extension.Argv0`, defaulting to the file's own base name in `NewExtension`; `Invoke` sets `cmd.Args[0]` to it. Bundled commands get `lstk-`, which is how the one binary knows which extension to be. +- [x] 8.3 `Resolver.Resolve`: bundle first for described names, then standalone `lstk-` in the bundled dir (manual placement keeps working), then PATH. A bundle load error is returned only when nothing else provides the name, so `lstk hello` from PATH still works next to a broken bundle while `lstk doctor` reports the broken install instead of "unknown command". `Resolver.List` adds the bundle's names (bundled, one shared path), logging and skipping a broken bundle so help never fails. +- [x] 8.4 `cmd/extension.go`: a non-`ErrNotFound` resolve error is rendered as a styled `ErrorEvent` ("bundled extensions are not usable", with `lstk update` as the action) and returned silent. +- [x] 8.5 Tests: unit (`internal/extension/bundle_test.go`) and e2e (`test/integration/extension_bundle_test.go`, backed by a new `argv0` mode in the reference extension) covering argv[0] dispatch, bundle-beats-PATH, help listing with descriptions and no phantom `bundled-extensions`/`extensions` rows, undescribed name → unknown command with the bundle never executed, binary-without-toml → error naming the toml, and the runtime context still conveyed. +- [x] 8.6 Docs: `docs/extensions-authoring.md` gains a "How LocalStack's bundled extensions differ" section (read `argv[0]`); `docs/extensions-bundling.md` and the CLAUDE.md Extensions section describe the shape; design Decision 7 records the resolution. +- [x] 8.7 Make the runtime and the gate agree on what a command name is: `validate.ExtensionName` (a letter or digit first, then letters, digits, hyphens, underscores) is applied to every toml key by `LoadBundle`, so an undispatchable key is a broken bundle rather than a silent miss. `Resolver.List` attaches each bundled entry's help `Description`, removing the second parse of the toml in `cmd`. The argv[0] contract is uniform (`lstk-` for standalone files too) and pinned by `TestExtensionArgv0IsExtensionName`. + ## 7. Documentation - [ ] 7.1 Re-introduce `docs/extensions-bundling.md` covering, for a reader who knows none of the context: where the bundled files live on disk for each install method; the release pipeline (pin file → download + checksum verify → descriptions check → packaging); the private repo's release-asset convention (4.1); how the pin gets bumped (4.4); how to do local snapshot builds (`--stub`); what an update does on each channel, the exact safety guarantee from 1.5, and why the binary channel never deletes extensions; how to roll back; and the release-candidate checklist (6.10). diff --git a/scripts/add-bundled-to-npm.sh b/scripts/add-bundled-to-npm.sh new file mode 100755 index 00000000..e1472431 --- /dev/null +++ b/scripts/add-bundled-to-npm.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# +# Adds the bundled extensions to every npm PLATFORM package. +# +# The npm wrapper (@localstack/lstk) only holds the launcher; the real Go +# binary lives in the platform package (@localstack/lstk--) and the +# launcher execs it from there. lstk resolves its bundled-extensions directory +# from its own executable's location, so that platform directory is where +# `bundled-extensions` and `lstk-extensions.toml` must live. +# +# goreleaser-npm-publisher has no per-platform extra-files option, so this runs +# on its dist/npm output before `npm publish`. Two details it has to get right: +# +# * Node and Go name platforms differently: win32 -> windows, x64 -> amd64 +# (darwin, linux and arm64 are the same in both). +# * The generated package.json carries "files": [], which npm packs as +# package.json + the bin entry and nothing else. Copying alone would be +# silently dropped at publish, so the copied names are appended to that +# allowlist as well. +# +# Usage: +# scripts/add-bundled-to-npm.sh +set -euo pipefail + +die() { + echo "add-bundled-to-npm: $*" >&2 + exit 1 +} + +usage() { + sed -n '2,23p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' >&2 + exit 1 +} + +[ $# -eq 2 ] || usage +NPM_DIR="$1" +BUNDLED_DIR="$2" +TOML="${BUNDLED_DIR}/lstk-extensions.toml" + +[ -d "${NPM_DIR}" ] || die "no such directory: ${NPM_DIR}" +[ -d "${BUNDLED_DIR}" ] || die "no such directory: ${BUNDLED_DIR}" +[ -f "${TOML}" ] || die "no descriptions file at ${TOML}; run scripts/fetch-bundled-extensions.sh first" +command -v node >/dev/null 2>&1 || die "node is required to edit package.json" + +# Appends names to the package.json "files" array, de-duplicated, preserving +# everything else. Done in node so the JSON is rewritten faithfully. +register_files() { + local pkg_json="$1"; shift + node -e ' + const fs = require("fs"); + const [file, ...names] = process.argv.slice(1); + const pkg = JSON.parse(fs.readFileSync(file, "utf8")); + const files = Array.isArray(pkg.files) ? pkg.files : []; + for (const name of names) if (!files.includes(name)) files.push(name); + pkg.files = files; + fs.writeFileSync(file, JSON.stringify(pkg, null, 2) + "\n"); + ' "${pkg_json}" "$@" +} + +count=0 +for dir in "${NPM_DIR}"/lstk-*/; do + [ -d "${dir}" ] || continue + pkg="$(basename "${dir}")" + cpu="${pkg##*-}" + os="${pkg#lstk-}" + os="${os%-*}" + case "${os}" in win32) goos=windows ;; *) goos="${os}" ;; esac + case "${cpu}" in x64) goarch=amd64 ;; *) goarch="${cpu}" ;; esac + src="${BUNDLED_DIR}/${goos}_${goarch}" + [ -d "${src}" ] || die "no staged bundle for ${pkg} at ${src}" + + added="" + for file in "${src}"/bundled-extensions*; do + [ -e "${file}" ] || die "no bundled-extensions binary in ${src} for ${pkg}" + cp -p "${file}" "${dir}" + added="${added} $(basename "${file}")" + done + cp "${TOML}" "${dir}" + added="${added} lstk-extensions.toml" + + # shellcheck disable=SC2086 # deliberate word splitting of the collected names + register_files "${dir}/package.json" ${added} + echo "${pkg}: added${added}" + count=$((count + 1)) +done + +[ "${count}" -gt 0 ] || die "no platform packages found under ${NPM_DIR} (expected lstk-- directories)" +echo "Bundled extensions added to ${count} platform package(s)." diff --git a/scripts/check-bundled-packaging-sync.sh b/scripts/check-bundled-packaging-sync.sh new file mode 100755 index 00000000..39d83ab6 --- /dev/null +++ b/scripts/check-bundled-packaging-sync.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# +# Fails when the two halves of bundled-extension packaging are out of step. +# +# The packaging half (.goreleaser.yaml referencing bundled/) and the download +# half (the release job running fetch-bundled-extensions.sh) must land in the +# same PR. Packaging without the download makes every release fail on a glob +# that matches nothing; the download without packaging silently ships nothing. +# +# `goreleaser check` cannot catch either direction: it validates config syntax +# and never looks at the filesystem or at the workflow. So this runs beside it +# on every PR, where it costs a red build instead of a broken release. +# +# Usage: +# scripts/check-bundled-packaging-sync.sh [goreleaser.yaml] [ci-workflow.yml] +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +GORELEASER_FILE="${1:-${REPO_ROOT}/.goreleaser.yaml}" +WORKFLOW_FILE="${2:-${REPO_ROOT}/.github/workflows/ci.yml}" + +FETCH_SCRIPT="fetch-bundled-extensions.sh" +STAGING_DIR="bundled/" + +die() { + echo "check-bundled-packaging-sync: $*" >&2 + exit 1 +} + +[ -f "${GORELEASER_FILE}" ] || die "no such file: ${GORELEASER_FILE}" +[ -f "${WORKFLOW_FILE}" ] || die "no such file: ${WORKFLOW_FILE}" + +# Drops comment lines so a commented-out entry never counts as live. YAML has +# no block comments, so line-wise is exact here. +uncommented() { + sed -e 's/^[[:space:]]*//' "$1" | grep -v '^#' || true +} + +# The steps of the `release` job only — a fetch step in some other job does not +# populate the staging tree for the release. +release_job_steps() { + awk ' + /^ release:[[:space:]]*$/ { in_job = 1; next } + in_job && /^ [A-Za-z_][A-Za-z0-9_-]*:/ { in_job = 0 } + in_job { print } + ' "$1" | sed -e 's/^[[:space:]]*//' | grep -v '^#' || true +} + +packaging_live=0 +if uncommented "${GORELEASER_FILE}" | grep -q -- "${STAGING_DIR}"; then + packaging_live=1 +fi + +fetch_wired=0 +if release_job_steps "${WORKFLOW_FILE}" | grep -q -- "${FETCH_SCRIPT}"; then + fetch_wired=1 +fi + +if [ "${packaging_live}" -eq 1 ] && [ "${fetch_wired}" -eq 0 ]; then + die "$(basename "${GORELEASER_FILE}") packages files from ${STAGING_DIR}, but the + release job in $(basename "${WORKFLOW_FILE}") never runs ${FETCH_SCRIPT}. + Nothing would populate ${STAGING_DIR}, so every release would fail on a glob + that matches no files. Add the fetch step, or comment the packaging entries + back out." +fi + +if [ "${packaging_live}" -eq 0 ] && [ "${fetch_wired}" -eq 1 ]; then + die "the release job in $(basename "${WORKFLOW_FILE}") runs ${FETCH_SCRIPT}, but + $(basename "${GORELEASER_FILE}") packages nothing from ${STAGING_DIR}. + The bundle would be downloaded and then silently dropped. Add the packaging + entries, or remove the fetch step." +fi + +if [ "${packaging_live}" -eq 1 ]; then + echo "In step: bundled extensions are downloaded by the release job and packaged." +else + echo "In step: bundled-extension packaging is not enabled, and nothing downloads it." +fi diff --git a/scripts/check-descriptions.sh b/scripts/check-descriptions.sh new file mode 100755 index 00000000..1512a46d --- /dev/null +++ b/scripts/check-descriptions.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# +# Release gate: the descriptions file, the bundled extensions binary and the +# binary's own command list must agree. +# +# LocalStack's bundled extensions ship as one multi-call binary, +# `bundled-extensions`, and lstk learns which commands it provides from +# lstk-extensions.toml (a flat table of `name = "one-line description"`) and +# execs the binary with argv[0] set to `lstk-`. That makes the file +# load-bearing in both directions: a described name the binary does not answer +# to is a command that shows in help and fails when run. The binary's side of +# the story is bundle-commands.txt, which scripts/fetch-bundled-extensions.sh +# records from the `lstk-` alias entries the bundle archives carry. +# +# * commands described but no binary -> FAIL (help would list commands +# that cannot run) +# * binary present but nothing described -> FAIL (lstk could never reach it; +# the runtime treats this as a +# broken install) +# * binary present but no command list -> FAIL (nothing to verify the +# descriptions against) +# * described but the bundle lacks it -> FAIL (lstk would exec the binary +# under a name it does not answer to) +# * provided but not described -> warn (unreachable through lstk; +# the bundle's own inconsistency) +# * neither binary nor descriptions -> pass (nothing is bundled) +# * everything agrees -> pass; the command names are +# printed for the release log +# +# Only the left-hand names are read from the toml, never the description +# values, so no description string can break this check. Descriptions and the +# command list are identical on every platform, so the release runs this once +# against one platform directory. +# +# Usage: +# scripts/check-descriptions.sh [descriptions-file] [commands-file] +# +# e.g. bundled/linux_amd64, as staged by +# scripts/fetch-bundled-extensions.sh +# [descriptions-file] defaults to /../lstk-extensions.toml +# [commands-file] defaults to /../bundle-commands.txt +set -euo pipefail + +BUNDLED_BINARY="bundled-extensions" +DESCRIPTIONS_FILE="lstk-extensions.toml" +COMMANDS_FILE="bundle-commands.txt" + +die() { + echo "check-descriptions: $*" >&2 + exit 1 +} + +usage() { + sed -n '2,42p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' >&2 + exit 1 +} + +[ $# -ge 1 ] && [ $# -le 3 ] || usage +PLATFORM_DIR="$1" +TOML="${2:-${PLATFORM_DIR}/../${DESCRIPTIONS_FILE}}" +COMMANDS="${3:-${PLATFORM_DIR}/../${COMMANDS_FILE}}" + +[ -d "${PLATFORM_DIR}" ] || die "no such directory: ${PLATFORM_DIR}" + +# The binary, if present, under either spelling. +binary="" +for candidate in "${PLATFORM_DIR}/${BUNDLED_BINARY}" "${PLATFORM_DIR}/${BUNDLED_BINARY}.exe"; do + if [ -e "${candidate}" ]; then + binary="${candidate}" + break + fi +done + +# The described command names: the bare identifier left of the first `=` on +# each non-comment line. Values are never looked at. The name rule is the same +# one lstk applies at load time (validate.ExtensionName), so a file that passes +# here always loads. +names="" +invalid="" +if [ -f "${TOML}" ]; then + while IFS= read -r line; do + line="${line%%#*}" + case "${line}" in *=*) ;; *) continue ;; esac + key="${line%%=*}" + key="$(echo "${key}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + [ -n "${key}" ] || continue + if echo "${key}" | grep -Eq '^[A-Za-z0-9][A-Za-z0-9_-]*$'; then + names="${names}${key} +" + else + invalid="${invalid} ${key} +" + fi + done < "${TOML}" +fi + +if [ -n "${invalid}" ]; then + echo "check-descriptions: ${TOML} contains invalid command names:" >&2 + printf '%s' "${invalid}" >&2 + die "command names must match ^[A-Za-z0-9][A-Za-z0-9_-]*$" +fi + +# Standalone lstk- files are not how the bundle ships. They still work +# (lstk resolves them from its directory) but carry no description, so flag +# them rather than fail. +for stray in "${PLATFORM_DIR}"/lstk-*; do + [ -e "${stray}" ] || continue + echo "Warning: standalone extension binary $(basename "${stray}") in ${PLATFORM_DIR} is not part of the bundle and will show name-only in help." +done + +if [ -z "${binary}" ]; then + if [ -n "${names}" ]; then + echo "check-descriptions: ${TOML} describes commands but ${PLATFORM_DIR} has no ${BUNDLED_BINARY} binary to provide them:" >&2 + printf '%s' "${names}" | sed 's/^/ /' >&2 + die "either add the bundle binary to the release or remove these entries" + fi + echo "Nothing bundled: no ${BUNDLED_BINARY} binary and no described commands." + exit 0 +fi + +[ -x "${binary}" ] || die "${binary} is not executable" +[ -f "${TOML}" ] || die "${binary} is present but ${TOML} is missing; lstk cannot know which commands the bundle provides without ${DESCRIPTIONS_FILE}" +[ -n "${names}" ] || die "${binary} is present but ${TOML} describes no commands; the bundle would be unreachable" +[ -f "${COMMANDS}" ] || die "${binary} is present but its command list ${COMMANDS} is missing; scripts/fetch-bundled-extensions.sh records it from the bundle's lstk- alias entries" + +provided="$(grep -v '^[[:space:]]*$' "${COMMANDS}" || true)" +[ -n "${provided}" ] || die "${binary} is present but its command list ${COMMANDS} is empty; the bundle declares no commands" + +unprovided="" +for name in ${names}; do + echo "${provided}" | grep -qx -- "${name}" || unprovided="${unprovided} ${name} +" +done +if [ -n "${unprovided}" ]; then + echo "check-descriptions: ${TOML} describes commands that $(basename "${binary}") does not provide:" >&2 + printf '%s' "${unprovided}" >&2 + die "lstk would exec the bundle under a name it does not answer to; fix the descriptions file or the bundle" +fi + +for name in ${provided}; do + printf '%s' "${names}" | grep -qx -- "${name}" \ + || echo "Warning: the bundle provides ${name} but ${TOML} does not describe it, so lstk will not expose it." +done + +count="$(printf '%s' "${names}" | grep -c . || true)" +echo "Bundle $(basename "${binary}") provides ${count} described command(s): $(printf '%s' "${names}" | tr '\n' ' ')" diff --git a/scripts/fetch-bundled-extensions.sh b/scripts/fetch-bundled-extensions.sh new file mode 100755 index 00000000..05c7b0e0 --- /dev/null +++ b/scripts/fetch-bundled-extensions.sh @@ -0,0 +1,372 @@ +#!/usr/bin/env bash +# +# Stages LocalStack's bundled extensions for a release build. +# +# Downloads one extensions bundle from the private extensions repository's +# release assets, verifies every asset against that release's checksum +# manifest, and arranges the contents under bundled/ in the layout the +# packaging step consumes: +# +# bundled/_/bundled-extensions[.exe] one per platform +# bundled/lstk-extensions.toml os/arch-independent +# bundled/bundle-commands.txt os/arch-independent +# +# The private repository publishes one archive per platform, +# `bundled-extensions___.tar.gz` (`.zip` for Windows), each +# containing the multi-call binary `bundled-extensions[.exe]` and the +# descriptions file `lstk-extensions.toml`, plus a `checksums.txt` covering the +# archives, and `lstk-` alias entries for every command the binary +# answers to (symlinks on Unix, copies on Windows). The binary and the toml are +# staged as they are. The aliases are not: lstk dispatches to the one binary by +# argv[0] and never needs them on disk. Their names are, though, recorded in +# bundle-commands.txt, because they are the bundle's own declaration of which +# commands it provides, and scripts/check-descriptions.sh verifies the toml +# against that list. An archive with no aliases is rejected for that reason. +# +# Which bundle is taken comes from bundled/extensions.version — `latest` by +# default. `latest` is resolved to a concrete tag exactly once here and +# printed; every later step in the release uses that tag, so one build can +# never mix two bundles. Re-running an already-published release must pass that +# release's recorded tag back in via --tag / LSTK_EXTENSIONS_TAG, because +# `latest` re-resolves on every invocation. +# +# Usage: +# scripts/fetch-bundled-extensions.sh [--tag ] [--stub] +# +# --tag Use this bundle tag instead of resolving the version file. +# --stub Skip the download entirely and write placeholder files into +# the same layout, for local snapshot builds by contributors +# without access to the private repository. +# +# Environment: +# LSTK_EXTENSIONS_READ_TOKEN Read-only token scoped to the private +# extensions repository. Required (not --stub). +# LSTK_EXTENSIONS_REPO Private repository (default below). +# LSTK_EXTENSIONS_TAG Same as --tag. +# LSTK_BUNDLED_DIR Staging tree (default: /bundled). +# LSTK_UNSUPPORTED_PLATFORMS Overrides UNSUPPORTED_PLATFORMS below. +# LSTK_BUNDLED_STUB_BINARIES --stub only: binary names to fabricate. +set -euo pipefail + +# The platforms lstk itself is built for (.goreleaser.yaml `builds`). Every +# bundled extension must have a binary for each of them. +TARGET_PLATFORMS="linux_amd64 linux_arm64 darwin_amd64 darwin_arm64 windows_amd64 windows_arm64" + +# Platforms the bundle is knowingly not built for. Adding one here is how a gap +# becomes a deliberate, reviewable choice instead of a release failure; leaving +# it empty is what makes an accidental gap loud. +UNSUPPORTED_PLATFORMS="${LSTK_UNSUPPORTED_PLATFORMS-}" + +REPO="${LSTK_EXTENSIONS_REPO:-localstack/lstk-bundled-extensions}" +BUNDLED_BINARY="bundled-extensions" +DESCRIPTIONS_FILE="lstk-extensions.toml" +COMMANDS_FILE="bundle-commands.txt" +MANIFEST_FILE="checksums.txt" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +BUNDLED_DIR="${LSTK_BUNDLED_DIR:-${REPO_ROOT}/bundled}" +VERSION_FILE="${BUNDLED_DIR}/extensions.version" + +die() { + echo "fetch-bundled-extensions: $*" >&2 + exit 1 +} + +sha256_of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +is_unsupported_platform() { + local candidate="$1" platform + for platform in $(echo "${UNSUPPORTED_PLATFORMS}" | tr ',' ' '); do + [ "${platform}" = "${candidate}" ] && return 0 + done + return 1 +} + +# The version file is documentation as much as configuration, so comments and +# blank lines are ignored and exactly one value line is expected. +read_version_file() { + [ -f "${VERSION_FILE}" ] || die "no version file at ${VERSION_FILE} (it selects which extensions bundle to ship)" + local values + values="$(sed -e 's/#.*//' -e 's/[[:space:]]*$//' -e 's/^[[:space:]]*//' "${VERSION_FILE}" | grep -v '^$' || true)" + [ -n "${values}" ] || die "${VERSION_FILE} has no value line; expected 'latest' or a release tag" + [ "$(echo "${values}" | wc -l | tr -d ' ')" -eq 1 ] || die "${VERSION_FILE} has more than one value line; expected exactly one" + echo "${values}" +} + +# Wipes the staging tree without touching the one tracked file in it, so a +# re-run never merges a previous bundle's binaries into the current one. +reset_staging_tree() { + mkdir -p "${BUNDLED_DIR}" + find "${BUNDLED_DIR}" -mindepth 1 -maxdepth 1 ! -name "$(basename "${VERSION_FILE}")" -exec rm -rf {} + +} + +# Reads the platform out of an archive asset name: +# `bundled-extensions_v2026.08.19_windows_amd64.zip` -> `windows amd64 zip`. +# Returns non-zero when the name is not a platform archive. +split_asset_name() { + local base="$1" stem kind arch rest os + case "${base}" in + *.tar.gz) stem="${base%.tar.gz}"; kind="tar.gz" ;; + *.zip) stem="${base%.zip}"; kind="zip" ;; + *) return 1 ;; + esac + case "${stem}" in *_*_*) ;; *) return 1 ;; esac + arch="${stem##*_}" + rest="${stem%_*}" + os="${rest##*_}" + case " ${TARGET_PLATFORMS} " in *" ${os}_${arch} "*) ;; *) return 1 ;; esac + echo "${os} ${arch} ${kind}" +} + +verify_checksums() { + local dir="$1" + local manifest="${dir}/${MANIFEST_FILE}" + [ -f "${manifest}" ] || die "the bundle publishes no ${MANIFEST_FILE}; refusing to stage unverified binaries" + local file base expected actual count=0 + for file in "${dir}"/*; do + base="$(basename "${file}")" + [ "${base}" = "${MANIFEST_FILE}" ] && continue + # `*name` is the binary-mode form some sha256sum implementations emit. + expected="$(awk -v f="${base}" '$2 == f || $2 == "*" f { print $1; exit }' "${manifest}")" + [ -n "${expected}" ] || die "asset ${base} is not listed in ${MANIFEST_FILE}" + actual="$(sha256_of "${file}")" + [ "${actual}" = "${expected}" ] || die "checksum mismatch for ${base}: manifest says ${expected}, downloaded file is ${actual}" + count=$((count + 1)) + done + echo "Verified ${count} asset(s) against ${MANIFEST_FILE}." +} + +# Unpacks one archive into an empty directory. Symlinked alias entries in a +# tarball come out as symlinks; alias_names reads them by name only, and the +# regular-file lookups in stage_assets never pick them up. +extract_archive() { + local archive="$1" kind="$2" dest="$3" + mkdir -p "${dest}" + case "${kind}" in + tar.gz) tar xzf "${archive}" -C "${dest}" ;; + zip) unzip -q -o "${archive}" -d "${dest}" ;; + esac +} + +# The command names an unpacked archive declares through its lstk- alias +# entries (any entry type: symlinks in tarballs, copies in zips), one per +# line, sorted. The descriptions file is excluded by name. +alias_names() { + find "$1" -mindepth 1 -maxdepth 1 -name "lstk-*" ! -name "${DESCRIPTIONS_FILE}" -exec basename {} \; \ + | sed -e "s/^lstk-//" -e "s/\.exe$//" | grep -v "^$" | sort -u || true +} + +stage_assets() { + local dir="$1" file base parsed os arch kind ext unpacked binary toml staged=0 + local toml_staged="${BUNDLED_DIR}/${DESCRIPTIONS_FILE}" + local commands commands_staged="${BUNDLED_DIR}/${COMMANDS_FILE}" + for file in "${dir}"/*; do + base="$(basename "${file}")" + [ "${base}" = "${MANIFEST_FILE}" ] && continue + if ! parsed="$(split_asset_name "${base}")"; then + echo "Note: ignoring release asset that is not a platform archive: ${base}" + continue + fi + # shellcheck disable=SC2086 # deliberate word splitting of the parsed tuple + set -- ${parsed} + os="$1"; arch="$2"; kind="$3" + ext="" + [ "${os}" = "windows" ] && ext=".exe" + + unpacked="${dir}/.unpacked/${os}_${arch}" + extract_archive "${file}" "${kind}" "${unpacked}" + + binary="$(find "${unpacked}" -type f -name "${BUNDLED_BINARY}${ext}" | head -n1)" + [ -n "${binary}" ] || die "${base} contains no ${BUNDLED_BINARY}${ext}" + mkdir -p "${BUNDLED_DIR}/${os}_${arch}" + cp "${binary}" "${BUNDLED_DIR}/${os}_${arch}/${BUNDLED_BINARY}${ext}" + chmod 0755 "${BUNDLED_DIR}/${os}_${arch}/${BUNDLED_BINARY}${ext}" + staged=$((staged + 1)) + + # The descriptions file is os/arch-independent: take it from the first + # archive and insist every other archive agrees, since a bundle whose + # platforms describe different commands is a bug in the bundle. + toml="$(find "${unpacked}" -type f -name "${DESCRIPTIONS_FILE}" | head -n1)" + [ -n "${toml}" ] || die "${base} contains no ${DESCRIPTIONS_FILE}" + if [ -f "${toml_staged}" ]; then + cmp -s "${toml}" "${toml_staged}" || die "${DESCRIPTIONS_FILE} in ${base} differs from the one in an earlier archive of the same bundle" + else + cp "${toml}" "${toml_staged}" + fi + + # The alias entries are the bundle's own statement of which commands the + # binary answers to. They are not staged (lstk never needs them on disk) + # but their names are, so the descriptions gate can verify the toml + # against them. Like the toml, they must agree across platforms. + commands="$(alias_names "${unpacked}")" + [ -n "${commands}" ] || die "${base} carries no lstk- alias entries, so the bundle's command list cannot be verified against ${DESCRIPTIONS_FILE}" + if [ -f "${commands_staged}" ]; then + [ "${commands}" = "$(cat "${commands_staged}")" ] || die "the command list (lstk- alias entries) in ${base} differs from the one in an earlier archive of the same bundle" + else + printf "%s\n" "${commands}" > "${commands_staged}" + fi + done + [ -f "${toml_staged}" ] || die "the bundle publishes no ${DESCRIPTIONS_FILE}" + echo "Staged ${staged} platform binaries into ${BUNDLED_DIR}." +} + +# The bundle must exist for every non-exempt target platform. Checking here +# turns a gap into a named error at pull time; left to GoReleaser it surfaces +# as an unexplained "glob matched nothing" during the release. +check_platform_coverage() { + local names platform name file suffix missing="" + names="$(find "${BUNDLED_DIR}" -mindepth 2 -maxdepth 2 -type f -exec basename {} \; \ + | sed 's/\.exe$//' | sort -u)" + if [ -z "${names}" ]; then + echo "Warning: the bundle staged no extension binaries." + return 0 + fi + for platform in ${TARGET_PLATFORMS}; do + if is_unsupported_platform "${platform}"; then + echo "Note: ${platform} is listed as unsupported; skipping its coverage check." + continue + fi + suffix="" + case "${platform}" in windows_*) suffix=".exe" ;; esac + for name in ${names}; do + file="${BUNDLED_DIR}/${platform}/${name}${suffix}" + [ -f "${file}" ] || missing="${missing} ${name} for ${platform} +" + done + done + if [ -n "${missing}" ]; then + echo "fetch-bundled-extensions: the bundle has no binary for:" >&2 + printf '%s' "${missing}" >&2 + die "add the missing platforms to the bundle, or list them in UNSUPPORTED_PLATFORMS" + fi + echo "Platform coverage complete for: $(echo "${names}" | tr '\n' ' ')" +} + +write_stub_bundle() { + # Defaults to the real layout so `goreleaser --snapshot` works out of the + # box; the override exists for experiments with standalone lstk- files. + local binaries="${LSTK_BUNDLED_STUB_BINARIES:-${BUNDLED_BINARY}}" + local platform name suffix key + reset_staging_tree + for platform in ${TARGET_PLATFORMS}; do + is_unsupported_platform "${platform}" && continue + suffix="" + case "${platform}" in windows_*) suffix=".exe" ;; esac + mkdir -p "${BUNDLED_DIR}/${platform}" + for name in ${binaries}; do + printf '#!/bin/sh\necho "stub %s for %s"\n' "${name}" "${platform}" \ + > "${BUNDLED_DIR}/${platform}/${name}${suffix}" + chmod 0755 "${BUNDLED_DIR}/${platform}/${name}${suffix}" + done + done + { + echo "# Stub descriptions file written by fetch-bundled-extensions.sh --stub." + echo "# The real one is hand-authored in the private extensions repository." + for name in ${binaries}; do + case "${name}" in + "${BUNDLED_BINARY}") + # The multi-call bundle: describe one placeholder command so lstk + # has something to dispatch to it. + echo "doctor = \"Stub bundled command (local build only)\"" + ;; + lstk-*) + key="${name#lstk-}" + echo "${key} = \"Stub description for ${key} (local build only)\"" + ;; + esac + done + } > "${BUNDLED_DIR}/${DESCRIPTIONS_FILE}" + { + for name in ${binaries}; do + case "${name}" in + "${BUNDLED_BINARY}") echo "doctor" ;; + lstk-*) echo "${name#lstk-}" ;; + esac + done + } | sort -u > "${BUNDLED_DIR}/${COMMANDS_FILE}" + + cat >&2 <<'BANNER' + + ########################################################## + # # + # STUB BUNDLE: placeholder files, not real extensions. # + # # + # For local snapshot builds only. Artifacts built from # + # this bundle must never be released. # + # # + ########################################################## + +BANNER +} + +STUB=0 +TAG="${LSTK_EXTENSIONS_TAG-}" +while [ $# -gt 0 ]; do + case "$1" in + --stub) STUB=1 ;; + --tag) + [ $# -ge 2 ] || die "--tag needs a value" + TAG="$2" + shift + ;; + --tag=*) TAG="${1#--tag=}" ;; + -h|--help) + sed -n '2,45p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) die "unknown argument: $1" ;; + esac + shift +done + +if [ "${STUB}" -eq 1 ]; then + write_stub_bundle + echo "Staged a stub bundle into ${BUNDLED_DIR}." + exit 0 +fi + +if [ -z "${LSTK_EXTENSIONS_READ_TOKEN-}" ]; then + die "LSTK_EXTENSIONS_READ_TOKEN is not set. + It must be a read-only token scoped to ${REPO}; in CI it comes from the + repository secret of the same name. For a local snapshot build without + access to that repository, re-run with --stub instead." +fi +command -v gh >/dev/null 2>&1 || die "the GitHub CLI (gh) is required to download the bundle" +command -v unzip >/dev/null 2>&1 || die "unzip is required to unpack the Windows bundle archives" + +if [ -z "${TAG}" ]; then + VERSION="$(read_version_file)" + if [ "${VERSION}" = "latest" ]; then + TAG="$(GH_TOKEN="${LSTK_EXTENSIONS_READ_TOKEN}" gh release view \ + --repo "${REPO}" --json tagName --jq .tagName)" \ + || die "could not resolve 'latest' to a release of ${REPO}" + [ -n "${TAG}" ] || die "${REPO} has no published release to resolve 'latest' to" + else + TAG="${VERSION}" + fi +fi + +# Printed, not just logged: this is the one line every later release step and +# the published release notes need in order to pin the build to one bundle. +echo "Resolved extensions bundle: ${TAG} (${REPO})" + +DOWNLOAD_DIR="$(mktemp -d)" +trap 'rm -rf "${DOWNLOAD_DIR}"' EXIT + +GH_TOKEN="${LSTK_EXTENSIONS_READ_TOKEN}" gh release download "${TAG}" \ + --repo "${REPO}" --dir "${DOWNLOAD_DIR}" --clobber \ + || die "could not download release ${TAG} from ${REPO}" + +verify_checksums "${DOWNLOAD_DIR}" +reset_staging_tree +stage_assets "${DOWNLOAD_DIR}" +check_platform_coverage + +echo "Bundle ${TAG} staged in ${BUNDLED_DIR}." diff --git a/scripts/test-scripts.sh b/scripts/test-scripts.sh new file mode 100755 index 00000000..d31cc51c --- /dev/null +++ b/scripts/test-scripts.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Runs the bash test suites for the release helper scripts under scripts/. +# These scripts only ever run on the Linux release runner, so a bash suite is +# the faithful test here; lstk's own behavior is covered by the Go suites. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +status=0 +for suite in "${SCRIPT_DIR}"/tests/*_test.sh; do + [ -e "${suite}" ] || continue + bash "${suite}" || status=1 + echo +done +exit "${status}" diff --git a/scripts/tests/add-bundled-to-npm_test.sh b/scripts/tests/add-bundled-to-npm_test.sh new file mode 100644 index 00000000..f7e1460b --- /dev/null +++ b/scripts/tests/add-bundled-to-npm_test.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Tests for scripts/add-bundled-to-npm.sh — the release step that copies the +# bundled extensions into each npm PLATFORM package and registers them in that +# package's `files` allowlist. Fixtures mirror goreleaser-npm-publisher's +# dist/npm layout: platform dirs named lstk-- with a +# package.json carrying "files": [], plus the lstk wrapper dir. +set -euo pipefail + +SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/tests/lib.sh +. "${SUITE_DIR}/lib.sh" + +ADD="${SUITE_DIR}/../add-bundled-to-npm.sh" +NPM_PLATFORMS="darwin-arm64 darwin-x64 linux-arm64 linux-x64 win32-arm64 win32-x64" +GO_PLATFORMS="darwin_arm64 darwin_amd64 linux_arm64 linux_amd64 windows_arm64 windows_amd64" + +# Fresh workspace with a full staged bundle and a full dist/npm tree. +# Sets WORK, BUNDLED, NPM. +setup_workspace() { + WORK="$(mktemp -d)" + BUNDLED="${WORK}/bundled" + NPM="${WORK}/dist/npm" + local platform + for platform in ${GO_PLATFORMS}; do + mkdir -p "${BUNDLED}/${platform}" + case "${platform}" in + windows_*) echo "bin ${platform}" > "${BUNDLED}/${platform}/bundled-extensions.exe" ;; + *) echo "bin ${platform}" > "${BUNDLED}/${platform}/bundled-extensions" ;; + esac + chmod 0755 "${BUNDLED}/${platform}"/bundled-extensions* + done + echo 'doctor = "Check the local setup"' > "${BUNDLED}/lstk-extensions.toml" + + for platform in ${NPM_PLATFORMS}; do + mkdir -p "${NPM}/lstk-${platform}" + local bin="lstk" + case "${platform}" in win32-*) bin="lstk.exe" ;; esac + echo "lstk binary" > "${NPM}/lstk-${platform}/${bin}" + printf '{\n "name": "@localstack/lstk-%s",\n "version": "0.1.0",\n "bin": {\n "lstk": "%s"\n },\n "files": []\n}\n' \ + "${platform}" "${bin}" > "${NPM}/lstk-${platform}/package.json" + done + mkdir -p "${NPM}/lstk" + echo "launcher" > "${NPM}/lstk/index.js" + printf '{\n "name": "@localstack/lstk",\n "version": "0.1.0",\n "bin": {\n "lstk": "index.js"\n },\n "files": []\n}\n' > "${NPM}/lstk/package.json" +} + +files_field() { + node -e 'const p=JSON.parse(require("fs").readFileSync(process.argv[1]));console.log((p.files||[]).join(" "))' "$1" +} + +echo "== add-bundled-to-npm.sh ==" + +begin_test "copies the matching platform binary and the toml into every platform package" +setup_workspace +run_script "${ADD}" "${NPM}" "${BUNDLED}" +assert_ok +assert_file_exists "${NPM}/lstk-darwin-arm64/bundled-extensions" +assert_file_contains "${NPM}/lstk-darwin-arm64/bundled-extensions" "darwin_arm64" +assert_file_exists "${NPM}/lstk-linux-x64/bundled-extensions" +assert_file_contains "${NPM}/lstk-linux-x64/bundled-extensions" "linux_amd64" +assert_file_exists "${NPM}/lstk-win32-x64/bundled-extensions.exe" +assert_file_contains "${NPM}/lstk-win32-x64/bundled-extensions.exe" "windows_amd64" +assert_file_exists "${NPM}/lstk-win32-arm64/bundled-extensions.exe" +assert_file_contains "${NPM}/lstk-win32-arm64/bundled-extensions.exe" "windows_arm64" +assert_file_exists "${NPM}/lstk-darwin-x64/lstk-extensions.toml" + +begin_test "the copied binary keeps its executable bit" +setup_workspace +run_script "${ADD}" "${NPM}" "${BUNDLED}" +assert_ok +assert_executable "${NPM}/lstk-linux-arm64/bundled-extensions" + +begin_test "registers the files in each platform package's files allowlist" +setup_workspace +run_script "${ADD}" "${NPM}" "${BUNDLED}" +assert_ok +LAST_OUTPUT="$(files_field "${NPM}/lstk-darwin-arm64/package.json")" +assert_output_contains "bundled-extensions" +assert_output_contains "lstk-extensions.toml" +LAST_OUTPUT="$(files_field "${NPM}/lstk-win32-x64/package.json")" +assert_output_contains "bundled-extensions.exe" +assert_output_contains "lstk-extensions.toml" + +begin_test "npm would actually pack the registered files" +setup_workspace +run_script "${ADD}" "${NPM}" "${BUNDLED}" +assert_ok +run_script npm pack --dry-run "${NPM}/lstk-darwin-arm64" +assert_ok +assert_output_contains "bundled-extensions" +assert_output_contains "lstk-extensions.toml" +assert_output_contains " lstk" + +begin_test "leaves the wrapper package untouched" +setup_workspace +before="$(cat "${NPM}/lstk/package.json")" +run_script "${ADD}" "${NPM}" "${BUNDLED}" +assert_ok +assert_file_absent "${NPM}/lstk/bundled-extensions" +assert_file_absent "${NPM}/lstk/lstk-extensions.toml" +[ "$(cat "${NPM}/lstk/package.json")" = "${before}" ] || fail "wrapper package.json was modified" +assert_file_contains "${NPM}/lstk/index.js" "launcher" + +begin_test "fails naming the package when its platform has no staged bundle" +setup_workspace +rm -rf "${BUNDLED}/windows_arm64" +run_script "${ADD}" "${NPM}" "${BUNDLED}" +assert_fails +assert_output_contains "lstk-win32-arm64" +assert_output_contains "windows_arm64" + +begin_test "fails when the toml is missing" +setup_workspace +rm "${BUNDLED}/lstk-extensions.toml" +run_script "${ADD}" "${NPM}" "${BUNDLED}" +assert_fails +assert_output_contains "lstk-extensions.toml" + +begin_test "is idempotent: a second run does not duplicate files entries" +setup_workspace +run_script "${ADD}" "${NPM}" "${BUNDLED}" +run_script "${ADD}" "${NPM}" "${BUNDLED}" +assert_ok +count="$(files_field "${NPM}/lstk-linux-x64/package.json" | tr ' ' '\n' | grep -c '^bundled-extensions$' || true)" +[ "${count}" -eq 1 ] || fail "expected one bundled-extensions entry, got ${count}" + +begin_test "fails when there are no platform packages at all" +setup_workspace +rm -rf "${NPM}"/lstk-* +run_script "${ADD}" "${NPM}" "${BUNDLED}" +assert_fails +assert_output_contains "no platform packages" + +begin_test "missing arguments print usage and fail" +run_script "${ADD}" +assert_fails +assert_output_contains "sage" + +finish_suite diff --git a/scripts/tests/check-bundled-packaging-sync_test.sh b/scripts/tests/check-bundled-packaging-sync_test.sh new file mode 100755 index 00000000..976a30ca --- /dev/null +++ b/scripts/tests/check-bundled-packaging-sync_test.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# Tests for scripts/check-bundled-packaging-sync.sh — the guard that keeps the +# packaging half (.goreleaser.yaml) and the download half (the release job in +# ci.yml) from being merged separately. Both fixtures are passed in as file +# paths, so the suite never depends on the repo's current wiring. +set -euo pipefail + +SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/tests/lib.sh +. "${SUITE_DIR}/lib.sh" + +CHECK="${SUITE_DIR}/../check-bundled-packaging-sync.sh" + +write_goreleaser() { + local path="$1" mode="$2" + case "${mode}" in + live) + cat > "${path}" <<'YAML' +archives: + - id: lstk + files: + - completions/* + - src: "bundled/{{ .Os }}_{{ .Arch }}/lstk-*" + strip_parent: true +YAML + ;; + commented) + cat > "${path}" <<'YAML' +archives: + - id: lstk + files: + - completions/* + # - src: "bundled/{{ .Os }}_{{ .Arch }}/lstk-*" + # strip_parent: true +YAML + ;; + absent) + cat > "${path}" <<'YAML' +archives: + - id: lstk + files: + - completions/* +YAML + ;; + esac +} + +write_workflow() { + local path="$1" mode="$2" + case "${mode}" in + fetch) + cat > "${path}" <<'YAML' +jobs: + test-unit: + steps: + - run: make test + release: + steps: + - name: Fetch bundled extensions + run: scripts/fetch-bundled-extensions.sh + - name: Run GoReleaser + run: goreleaser release --clean +YAML + ;; + none) + cat > "${path}" <<'YAML' +jobs: + test-unit: + steps: + - run: make test + release: + steps: + - name: Run GoReleaser + run: goreleaser release --clean +YAML + ;; + commented) + cat > "${path}" <<'YAML' +jobs: + release: + steps: + # - name: Fetch bundled extensions + # run: scripts/fetch-bundled-extensions.sh + - name: Run GoReleaser + run: goreleaser release --clean +YAML + ;; + other-job) + cat > "${path}" <<'YAML' +jobs: + some-other-job: + steps: + - run: scripts/fetch-bundled-extensions.sh + release: + steps: + - name: Run GoReleaser + run: goreleaser release --clean +YAML + ;; + esac +} + +WORK="$(mktemp -d)" +GOR="${WORK}/goreleaser.yaml" +WF="${WORK}/ci.yml" + +echo "== check-bundled-packaging-sync.sh ==" + +begin_test "both halves absent: in step, passes" +write_goreleaser "${GOR}" absent +write_workflow "${WF}" none +run_script "${CHECK}" "${GOR}" "${WF}" +assert_ok + +begin_test "both halves present: in step, passes" +write_goreleaser "${GOR}" live +write_workflow "${WF}" fetch +run_script "${CHECK}" "${GOR}" "${WF}" +assert_ok + +begin_test "packaging without the fetch step fails" +write_goreleaser "${GOR}" live +write_workflow "${WF}" none +run_script "${CHECK}" "${GOR}" "${WF}" +assert_fails +assert_output_contains "fetch-bundled-extensions.sh" + +begin_test "the fetch step without packaging fails" +write_goreleaser "${GOR}" absent +write_workflow "${WF}" fetch +run_script "${CHECK}" "${GOR}" "${WF}" +assert_fails +assert_output_contains "bundled/" + +begin_test "a commented-out packaging entry does not count as live" +write_goreleaser "${GOR}" commented +write_workflow "${WF}" none +run_script "${CHECK}" "${GOR}" "${WF}" +assert_ok + +begin_test "a commented-out fetch step does not count as wired" +write_goreleaser "${GOR}" live +write_workflow "${WF}" commented +run_script "${CHECK}" "${GOR}" "${WF}" +assert_fails + +begin_test "a fetch step in another job does not satisfy the release job" +write_goreleaser "${GOR}" live +write_workflow "${WF}" other-job +run_script "${CHECK}" "${GOR}" "${WF}" +assert_fails + +begin_test "the repo's own files are in step" +run_script "${CHECK}" +assert_ok + +finish_suite diff --git a/scripts/tests/check-descriptions_test.sh b/scripts/tests/check-descriptions_test.sh new file mode 100644 index 00000000..c5c11432 --- /dev/null +++ b/scripts/tests/check-descriptions_test.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# Tests for scripts/check-descriptions.sh — the release gate that keeps the +# descriptions file and the multi-call bundled binary in agreement. Fixtures are +# built in temp dirs mirroring the staging layout the fetch script produces: +# a platform dir holding the binary, and the toml plus the bundle's own command +# list (bundle-commands.txt) one level up. +set -euo pipefail + +SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/tests/lib.sh +. "${SUITE_DIR}/lib.sh" + +CHECK="${SUITE_DIR}/../check-descriptions.sh" + +# Fresh staging tree. Sets STAGE (the bundled/ root) and PLATFORM_DIR. +setup_stage() { + STAGE="$(mktemp -d)" + PLATFORM_DIR="${STAGE}/linux_amd64" + mkdir -p "${PLATFORM_DIR}" +} + +write_binary() { + echo "fake" > "${PLATFORM_DIR}/${1:-bundled-extensions}" + chmod 0755 "${PLATFORM_DIR}/${1:-bundled-extensions}" +} + +write_toml() { + printf '%s' "$1" > "${STAGE}/lstk-extensions.toml" +} + +# The command list the fetch script records from the bundle's own lstk- +# alias entries: what the binary actually answers to. +write_commands() { + : > "${STAGE}/bundle-commands.txt" + for name in "$@"; do + echo "${name}" >> "${STAGE}/bundle-commands.txt" + done +} + +echo "== check-descriptions.sh ==" + +begin_test "binary, descriptions and command list agree: passes and lists the commands" +setup_stage +write_binary +write_commands doctor deploy +write_toml 'doctor = "Check the local setup" +deploy = "Deploy to LocalStack" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_ok +assert_output_contains "doctor" +assert_output_contains "deploy" +assert_output_lacks "Warning" + +begin_test "a described command the bundle does not provide fails, naming it" +setup_stage +write_binary +write_commands doctor +write_toml 'doctor = "Check the local setup" +deploy = "Deploy to LocalStack" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_fails +assert_output_contains "deploy" +assert_output_contains "does not provide" + +begin_test "a bundle command that is not described warns but passes" +setup_stage +write_binary +write_commands doctor deploy +write_toml 'doctor = "Check the local setup" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_ok +assert_output_contains "Warning" +assert_output_contains "deploy" + +begin_test "a bundled binary with no command list fails, naming the file" +setup_stage +write_binary +write_toml 'doctor = "Check the local setup" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_fails +assert_output_contains "bundle-commands.txt" + +begin_test "a bundled binary with an empty command list fails" +setup_stage +write_binary +write_commands +write_toml 'doctor = "Check the local setup" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_fails +assert_output_contains "bundle-commands.txt" + +begin_test "described commands with no bundled binary fail, naming them" +setup_stage +write_toml 'doctor = "Check the local setup" +deploy = "Deploy to LocalStack" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_fails +assert_output_contains "bundled-extensions" +assert_output_contains "doctor" +assert_output_contains "deploy" + +begin_test "a bundled binary with no descriptions file fails" +setup_stage +write_binary +write_commands doctor +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_fails +assert_output_contains "lstk-extensions.toml" + +begin_test "a bundled binary with an empty descriptions file fails" +setup_stage +write_binary +write_commands doctor +write_toml '# nothing described yet +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_fails +assert_output_contains "no commands" + +begin_test "nothing bundled at all passes" +setup_stage +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_ok + +begin_test "an empty descriptions file with no binary passes" +setup_stage +write_toml '' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_ok + +begin_test "the Windows binary name is accepted" +setup_stage +write_binary bundled-extensions.exe +write_commands doctor +write_toml 'doctor = "Check the local setup" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_ok + +begin_test "a non-executable binary fails" +setup_stage +echo "fake" > "${PLATFORM_DIR}/bundled-extensions" +chmod 0644 "${PLATFORM_DIR}/bundled-extensions" +write_commands doctor +write_toml 'doctor = "Check the local setup" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_fails +assert_output_contains "executable" + +begin_test "only the left-hand names are read, never the values" +setup_stage +write_binary +write_commands doctor +# A hostile description: quotes, an equals sign, a fake key on the same line. +write_toml 'doctor = "a = b \"quoted\" evil = \"x\"" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_ok +assert_output_contains "doctor" +assert_output_lacks "evil" + +begin_test "an invalid command name fails" +setup_stage +write_binary +write_commands doctor +write_toml 'doc tor = "spaces are not a command" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_fails +assert_output_contains "doc tor" + +begin_test "a stray standalone lstk- binary warns but passes" +setup_stage +write_binary +write_binary lstk-legacy +write_commands doctor +write_toml 'doctor = "Check the local setup" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_ok +assert_output_contains "Warning" +assert_output_contains "lstk-legacy" + +begin_test "explicit toml and command-list paths override the default locations" +setup_stage +write_binary +OTHER_DIR="$(mktemp -d)" +printf 'doctor = "x"\n' > "${OTHER_DIR}/other.toml" +printf 'doctor\n' > "${OTHER_DIR}/other-commands.txt" +run_script "${CHECK}" "${PLATFORM_DIR}" "${OTHER_DIR}/other.toml" "${OTHER_DIR}/other-commands.txt" +assert_ok +assert_output_contains "doctor" + +begin_test "a missing platform directory fails and names it" +run_script "${CHECK}" "/nonexistent/linux_amd64" +assert_fails +assert_output_contains "/nonexistent/linux_amd64" + +begin_test "no argument prints usage and fails" +run_script "${CHECK}" +assert_fails +assert_output_contains "sage" + +finish_suite diff --git a/scripts/tests/fetch-bundled-extensions_test.sh b/scripts/tests/fetch-bundled-extensions_test.sh new file mode 100755 index 00000000..920d6619 --- /dev/null +++ b/scripts/tests/fetch-bundled-extensions_test.sh @@ -0,0 +1,388 @@ +#!/usr/bin/env bash +# Tests for scripts/fetch-bundled-extensions.sh. +# +# The private extensions repository is mocked with a fake `gh` on PATH, so the +# suite never needs the real repo or a credential. Fixtures reproduce what that +# repo actually publishes: one archive per platform holding the multi-call +# binary, the descriptions file and lstk- alias entries, plus a +# checksums.txt over the archives. Every assertion is about what the script +# leaves on disk and what it prints — the two things the release consumes. +set -euo pipefail + +SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/tests/lib.sh +. "${SUITE_DIR}/lib.sh" + +FETCH="${SUITE_DIR}/../fetch-bundled-extensions.sh" +PLATFORMS="linux_amd64 linux_arm64 darwin_amd64 darwin_arm64 windows_amd64 windows_arm64" + +sha256_of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +# Builds a fixture release for the given tag: per platform, a tar.gz (zip for +# Windows) containing bundled-extensions[.exe], lstk-extensions.toml and an +# lstk- alias per entry in ALIASES (default "doctor"; a symlink in the +# tarballs, a copy in the zips — exactly what the private repo's goreleaser +# emits), plus a checksums.txt over the archives. TOML_BODY overrides the +# descriptions file for every platform. +make_release_assets() { + local dir="$1" tag="${2:-v1.4.0}" + local toml_body="${TOML_BODY-doctor = \"Fake doctor description\" +}" + mkdir -p "${dir}" + local platform work + for platform in ${PLATFORMS}; do + work="$(mktemp -d)" + printf '%s' "${toml_body}" > "${work}/lstk-extensions.toml" + case "${platform}" in + windows_*) + echo "fake bundle binary for ${platform}" > "${work}/bundled-extensions.exe" + for alias in ${ALIASES-doctor}; do cp "${work}/bundled-extensions.exe" "${work}/lstk-${alias}.exe"; done + ( cd "${work}" && zip -q -r "${dir}/bundled-extensions_${tag}_${platform}.zip" . ) + ;; + *) + echo "fake bundle binary for ${platform}" > "${work}/bundled-extensions" + chmod 0755 "${work}/bundled-extensions" + for alias in ${ALIASES-doctor}; do ( cd "${work}" && ln -s bundled-extensions "lstk-${alias}" ); done + ( cd "${work}" && tar czf "${dir}/bundled-extensions_${tag}_${platform}.tar.gz" . ) + ;; + esac + rm -rf "${work}" + done + ( cd "${dir}" && for asset in *; do + [ "${asset}" = "checksums.txt" ] && continue + echo "$(sha256_of "${asset}") ${asset}" + done > checksums.txt ) +} + +# Rewrites checksums.txt after a fixture was modified (used by tests that want +# a valid manifest for a deliberately altered set of assets). +refresh_manifest() { + ( cd "$1" && for asset in *; do + [ "${asset}" = "checksums.txt" ] && continue + echo "$(sha256_of "${asset}") ${asset}" + done > checksums.txt ) +} + +# A `gh` stand-in covering the two subcommands the fetch script uses. +install_fake_gh() { + local bindir="$1" assets="$2" latest_tag="$3" + mkdir -p "${bindir}" + cat > "${bindir}/gh" <> "${bindir}/gh.log" +if [ "\$1" = "release" ] && [ "\$2" = "view" ]; then + echo "${latest_tag}" + exit 0 +fi +if [ "\$1" = "release" ] && [ "\$2" = "download" ]; then + dest="" + while [ \$# -gt 0 ]; do + if [ "\$1" = "--dir" ]; then dest="\$2"; fi + shift + done + mkdir -p "\$dest" + cp "${assets}"/* "\$dest"/ + exit 0 +fi +echo "fake gh: unsupported invocation: \$*" >&2 +exit 1 +FAKE + chmod +x "${bindir}/gh" +} + +# Fresh workspace: a bundled dir with a version file, a fake gh on PATH, and a +# fixture release. Sets BUNDLED, BINDIR and ASSETS for the calling test. +setup_workspace() { + local version_value="${1:-latest}" latest_tag="${2:-v1.4.0}" + WORK="$(mktemp -d)" + BUNDLED="${WORK}/bundled" + BINDIR="${WORK}/bin" + ASSETS="${WORK}/assets" + mkdir -p "${BUNDLED}" + echo "${version_value}" > "${BUNDLED}/extensions.version" + make_release_assets "${ASSETS}" "${latest_tag}" + install_fake_gh "${BINDIR}" "${ASSETS}" "${latest_tag}" + export LSTK_BUNDLED_DIR="${BUNDLED}" + export LSTK_EXTENSIONS_READ_TOKEN="fake-token" + export LSTK_EXTENSIONS_REPO="localstack/fake-extensions" + export PATH="${BINDIR}:${ORIGINAL_PATH}" + unset LSTK_EXTENSIONS_TAG || true +} + +ORIGINAL_PATH="${PATH}" + +echo "== fetch-bundled-extensions.sh ==" + +begin_test "fails without a token, naming the variable and the --stub alternative" +setup_workspace +unset LSTK_EXTENSIONS_READ_TOKEN +run_script "${FETCH}" +assert_fails +assert_output_contains "LSTK_EXTENSIONS_READ_TOKEN" +assert_output_contains "--stub" + +begin_test "--stub stages the real layout for every platform without a token or gh" +setup_workspace +unset LSTK_EXTENSIONS_READ_TOKEN +export PATH="${ORIGINAL_PATH}" +run_script "${FETCH}" --stub +assert_ok +for platform in ${PLATFORMS}; do + suffix="" + case "${platform}" in windows_*) suffix=".exe" ;; esac + assert_executable "${BUNDLED}/${platform}/bundled-extensions${suffix}" +done +assert_file_exists "${BUNDLED}/lstk-extensions.toml" +# The stub toml must describe at least one command, or the descriptions gate +# (and lstk itself) would reject the pairing. +assert_file_contains "${BUNDLED}/lstk-extensions.toml" "doctor" + +begin_test "--stub prints an unmissable never-release banner" +setup_workspace +run_script "${FETCH}" --stub +assert_ok +assert_output_contains "STUB" +assert_output_contains "must never be released" + +begin_test "--stub honours an explicit binary list" +setup_workspace +LSTK_BUNDLED_STUB_BINARIES="lstk-doctor" run_script "${FETCH}" --stub +assert_ok +assert_executable "${BUNDLED}/linux_amd64/lstk-doctor" +assert_file_absent "${BUNDLED}/linux_amd64/bundled-extensions" +assert_file_contains "${BUNDLED}/lstk-extensions.toml" "doctor" + +begin_test "--stub output passes the descriptions gate" +setup_workspace +run_script "${FETCH}" --stub +run_script "${SUITE_DIR}/../check-descriptions.sh" "${BUNDLED}/linux_amd64" +assert_ok + +begin_test "resolves 'latest' to a concrete tag and prints it" +setup_workspace latest v2.1.3 +run_script "${FETCH}" +assert_ok +assert_output_contains "v2.1.3" + +begin_test "downloads the resolved tag rather than 'latest'" +setup_workspace latest v2.1.3 +run_script "${FETCH}" +assert_ok +assert_file_contains "${BINDIR}/gh.log" "release download v2.1.3" +run_script grep -c "release download latest" "${BINDIR}/gh.log" +assert_fails + +begin_test "an explicit tag in the version file is used without resolving" +setup_workspace v0.9.1 v0.9.1 +run_script "${FETCH}" +assert_ok +assert_file_contains "${BINDIR}/gh.log" "release download v0.9.1" +run_script grep -c "release view" "${BINDIR}/gh.log" +assert_fails + +begin_test "--tag overrides the version file" +setup_workspace latest v0.5.0 +run_script "${FETCH}" --tag v0.5.0 +assert_ok +assert_file_contains "${BINDIR}/gh.log" "release download v0.5.0" +run_script grep -c "release view" "${BINDIR}/gh.log" +assert_fails + +begin_test "LSTK_EXTENSIONS_TAG overrides the version file" +setup_workspace latest v0.4.2 +LSTK_EXTENSIONS_TAG=v0.4.2 run_script "${FETCH}" +assert_ok +assert_file_contains "${BINDIR}/gh.log" "release download v0.4.2" +run_script grep -c "release view" "${BINDIR}/gh.log" +assert_fails + +begin_test "unpacks each archive into its platform dir with the executable bit, toml at the root" +setup_workspace +run_script "${FETCH}" +assert_ok +for platform in ${PLATFORMS}; do + suffix="" + case "${platform}" in windows_*) suffix=".exe" ;; esac + assert_executable "${BUNDLED}/${platform}/bundled-extensions${suffix}" + assert_file_contains "${BUNDLED}/${platform}/bundled-extensions${suffix}" "${platform}" +done +assert_file_exists "${BUNDLED}/lstk-extensions.toml" +assert_file_contains "${BUNDLED}/lstk-extensions.toml" "doctor" +assert_file_absent "${BUNDLED}/linux_amd64/lstk-extensions.toml" +assert_file_absent "${BUNDLED}/linux_amd64/checksums.txt" + +begin_test "lstk- alias entries in the archives are not staged" +setup_workspace +run_script "${FETCH}" +assert_ok +assert_file_absent "${BUNDLED}/linux_amd64/lstk-doctor" +assert_file_absent "${BUNDLED}/windows_amd64/lstk-doctor.exe" + +begin_test "the staged tree passes the descriptions gate" +setup_workspace +run_script "${FETCH}" +run_script "${SUITE_DIR}/../check-descriptions.sh" "${BUNDLED}/linux_amd64" +assert_ok + +begin_test "a checksum mismatch aborts the fetch and names the asset" +setup_workspace +echo "tampered" >> "${ASSETS}/bundled-extensions_v1.4.0_linux_amd64.tar.gz" +run_script "${FETCH}" +assert_fails +assert_output_contains "bundled-extensions_v1.4.0_linux_amd64.tar.gz" +assert_file_absent "${BUNDLED}/linux_amd64/bundled-extensions" + +begin_test "a missing checksum manifest aborts the fetch" +setup_workspace +rm "${ASSETS}/checksums.txt" +run_script "${FETCH}" +assert_fails +assert_output_contains "checksums.txt" + +begin_test "an asset absent from the manifest aborts the fetch" +setup_workspace +cp "${ASSETS}/bundled-extensions_v1.4.0_linux_amd64.tar.gz" "${ASSETS}/bundled-extensions_v1.4.0_linux_386.tar.gz" +run_script "${FETCH}" +assert_fails +assert_output_contains "bundled-extensions_v1.4.0_linux_386.tar.gz" + +begin_test "an archive without the bundled binary aborts the fetch" +setup_workspace +work="$(mktemp -d)" +printf 'doctor = "x"\n' > "${work}/lstk-extensions.toml" +( cd "${work}" && tar czf "${ASSETS}/bundled-extensions_v1.4.0_linux_amd64.tar.gz" . ) +refresh_manifest "${ASSETS}" +run_script "${FETCH}" +assert_fails +assert_output_contains "bundled-extensions_v1.4.0_linux_amd64.tar.gz" +assert_output_contains "contains no bundled-extensions" + +begin_test "an archive without the descriptions file aborts the fetch" +setup_workspace +work="$(mktemp -d)" +echo "bin" > "${work}/bundled-extensions" +( cd "${work}" && tar czf "${ASSETS}/bundled-extensions_v1.4.0_darwin_arm64.tar.gz" . ) +refresh_manifest "${ASSETS}" +run_script "${FETCH}" +assert_fails +assert_output_contains "contains no lstk-extensions.toml" + +begin_test "descriptions differing between platforms abort the fetch" +setup_workspace +work="$(mktemp -d)" +echo "bin" > "${work}/bundled-extensions" +printf 'deploy = "a different command list"\n' > "${work}/lstk-extensions.toml" +( cd "${work}" && tar czf "${ASSETS}/bundled-extensions_v1.4.0_linux_arm64.tar.gz" . ) +refresh_manifest "${ASSETS}" +run_script "${FETCH}" +assert_fails +assert_output_contains "differs" + +begin_test "a platform with no archive fails and names the platform" +setup_workspace +rm "${ASSETS}/bundled-extensions_v1.4.0_darwin_arm64.tar.gz" +refresh_manifest "${ASSETS}" +run_script "${FETCH}" +assert_fails +assert_output_contains "darwin_arm64" +assert_output_contains "bundled-extensions" + +begin_test "UNSUPPORTED_PLATFORMS exempts a platform from the coverage check" +setup_workspace +rm "${ASSETS}/bundled-extensions_v1.4.0_darwin_arm64.tar.gz" +refresh_manifest "${ASSETS}" +LSTK_UNSUPPORTED_PLATFORMS="darwin_arm64" run_script "${FETCH}" +assert_ok +assert_file_absent "${BUNDLED}/darwin_arm64/bundled-extensions" + +begin_test "a non-archive extra asset is ignored with a note" +setup_workspace +echo "notes" > "${ASSETS}/RELEASE_NOTES.md" +refresh_manifest "${ASSETS}" +run_script "${FETCH}" +assert_ok +assert_output_contains "RELEASE_NOTES.md" +assert_file_absent "${BUNDLED}/RELEASE_NOTES.md" + +begin_test "a missing version file fails and names it" +setup_workspace +rm "${BUNDLED}/extensions.version" +run_script "${FETCH}" +assert_fails +assert_output_contains "extensions.version" + +begin_test "the version file ignores comments and blank lines" +setup_workspace latest v0.7.7 +printf '# which bundle to ship\n\nv0.7.7\n' > "${BUNDLED}/extensions.version" +run_script "${FETCH}" +assert_ok +assert_file_contains "${BINDIR}/gh.log" "release download v0.7.7" + +begin_test "a stale staging tree is replaced rather than merged into" +setup_workspace +mkdir -p "${BUNDLED}/linux_amd64" +echo stale > "${BUNDLED}/linux_amd64/lstk-removed" +run_script "${FETCH}" +assert_ok +assert_file_absent "${BUNDLED}/linux_amd64/lstk-removed" +assert_file_exists "${BUNDLED}/extensions.version" + +begin_test "records the bundle's own command list from its alias entries, sorted" +setup_workspace +ALIASES="doctor deploy" TOML_BODY='doctor = "x" +deploy = "y" +' make_release_assets "${ASSETS}" +run_script "${FETCH}" +assert_ok +assert_file_exists "${BUNDLED}/bundle-commands.txt" +run_script cat "${BUNDLED}/bundle-commands.txt" +[ "${LAST_OUTPUT}" = "deploy +doctor" ] || fail "expected the sorted alias names, got: ${LAST_OUTPUT}" + +begin_test "alias entries differing between platforms abort the fetch" +setup_workspace +work="$(mktemp -d)" +echo "bin" > "${work}/bundled-extensions" +chmod 0755 "${work}/bundled-extensions" +printf 'doctor = "Fake doctor description"\n' > "${work}/lstk-extensions.toml" +( cd "${work}" && ln -s bundled-extensions lstk-doctor && ln -s bundled-extensions lstk-extra ) +( cd "${work}" && tar czf "${ASSETS}/bundled-extensions_v1.4.0_linux_arm64.tar.gz" . ) +refresh_manifest "${ASSETS}" +run_script "${FETCH}" +assert_fails +assert_output_contains "linux_arm64" +assert_output_contains "command list" + +begin_test "an archive with no alias entries aborts the fetch, naming the archive" +setup_workspace +work="$(mktemp -d)" +echo "bin" > "${work}/bundled-extensions" +chmod 0755 "${work}/bundled-extensions" +printf 'doctor = "Fake doctor description"\n' > "${work}/lstk-extensions.toml" +( cd "${work}" && tar czf "${ASSETS}/bundled-extensions_v1.4.0_darwin_amd64.tar.gz" . ) +refresh_manifest "${ASSETS}" +run_script "${FETCH}" +assert_fails +assert_output_contains "bundled-extensions_v1.4.0_darwin_amd64.tar.gz" +assert_output_contains "alias" + +begin_test "the command list is not packaged next to the binaries" +setup_workspace +run_script "${FETCH}" +assert_ok +assert_file_absent "${BUNDLED}/linux_amd64/bundle-commands.txt" + +begin_test "--stub records a command list matching its descriptions" +setup_workspace +run_script "${FETCH}" --stub +assert_ok +assert_file_contains "${BUNDLED}/bundle-commands.txt" "doctor" + +finish_suite diff --git a/scripts/tests/lib.sh b/scripts/tests/lib.sh new file mode 100644 index 00000000..2836bcf8 --- /dev/null +++ b/scripts/tests/lib.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Minimal assertion helpers for the release-script tests. Kept bash 3.2 +# compatible (no associative arrays, namerefs or mapfile) so the suite runs on +# a stock macOS shell as well as on the Linux release runner. + +TESTS_RUN=0 +TESTS_FAILED=0 +CURRENT_TEST="" +CURRENT_FAILED=0 + +# Captured by run_script for the assertions below. +LAST_STATUS=0 +LAST_OUTPUT="" + +fail() { + # Counted once per test, however many assertions in it fail. + if [ "${CURRENT_FAILED}" -eq 0 ]; then + CURRENT_FAILED=1 + TESTS_FAILED=$((TESTS_FAILED + 1)) + fi + echo " FAIL: ${CURRENT_TEST}" + echo " $1" + if [ -n "${LAST_OUTPUT}" ]; then + echo " --- captured output ---" + echo "${LAST_OUTPUT}" | sed 's/^/ /' + echo " -----------------------" + fi +} + +begin_test() { + CURRENT_TEST="$1" + CURRENT_FAILED=0 + TESTS_RUN=$((TESTS_RUN + 1)) + LAST_STATUS=0 + LAST_OUTPUT="" +} + +# Runs a command, capturing stdout+stderr and the exit status instead of +# aborting the suite. Every assertion below reads what this recorded. +run_script() { + set +e + LAST_OUTPUT="$("$@" 2>&1)" + LAST_STATUS=$? + set -e +} + +assert_ok() { + [ "${LAST_STATUS}" -eq 0 ] || fail "expected success, got exit status ${LAST_STATUS}" +} + +assert_fails() { + [ "${LAST_STATUS}" -ne 0 ] || fail "expected a non-zero exit status, got success" +} + +assert_output_contains() { + case "${LAST_OUTPUT}" in + *"$1"*) ;; + *) fail "expected output to contain: $1" ;; + esac +} + +assert_output_lacks() { + case "${LAST_OUTPUT}" in + *"$1"*) fail "expected output NOT to contain: $1" ;; + esac +} + +assert_file_exists() { + [ -f "$1" ] || fail "expected file to exist: $1" +} + +assert_file_absent() { + [ ! -e "$1" ] || fail "expected file NOT to exist: $1" +} + +assert_executable() { + [ -x "$1" ] || fail "expected file to be executable: $1" +} + +assert_file_contains() { + if [ ! -f "$1" ]; then + fail "expected file to exist: $1" + return + fi + grep -q -- "$2" "$1" || fail "expected $1 to contain: $2" +} + +finish_suite() { + echo + if [ "${TESTS_FAILED}" -gt 0 ]; then + echo "${TESTS_FAILED}/${TESTS_RUN} test(s) failed in $(basename "$0")" + exit 1 + fi + echo "${TESTS_RUN}/${TESTS_RUN} test(s) passed in $(basename "$0")" +} diff --git a/test/integration/__snapshots__/extension_bundle_test.snap b/test/integration/__snapshots__/extension_bundle_test.snap new file mode 100644 index 00000000..280b110d --- /dev/null +++ b/test/integration/__snapshots__/extension_bundle_test.snap @@ -0,0 +1,52 @@ +Snapshots created by internal/snap. UPDATE_SNAPS=true go test rewrites +this file. + +[TestBundledMultiCallHelpListsDescribedCommands_1] +Usage: lstk [options] [command] + +LSTK - LocalStack command-line interface + +Commands: + completion Generate the autocompletion script for the specified shell + config Manage configuration + help Help about any command + load Load a snapshot into the running emulator + login Manage login + logout Remove stored authentication credentials + logs Show emulator logs + reset Reset emulator state + restart Restart emulator + save Save a snapshot of the emulator state + setup Set up emulator CLI integration + snapshot Manage emulator snapshots + start Start emulator + status Show emulator status and deployed resources + stop Stop emulator + update Update lstk to the latest version + volume Manage emulator volume + +Tools: + aws Run AWS CLI commands against LocalStack + az Run Azure CLI commands against LocalStack + cdk Run AWS CDK against LocalStack + sam Run the AWS SAM CLI against LocalStack + terraform Run Terraform against LocalStack + +Extensions: + deploy Deploy to LocalStack + doctor Check the local setup + hello + +Options: + --config string Path to config file + --endpoint-url string Target an existing, externally-managed emulator at this URL + -h, --help Show help + --json Output in JSON format (only supported by some commands) + --no-snapshot Skip auto-loading the configured snapshot for this run + --non-interactive Disable interactive mode + --persist Persist emulator state across restarts + --snapshot string Snapshot REF to load after start (overrides config for this run) + --timeout duration Maximum time to wait for the emulator to become ready (overrides LSTK_STARTUP_TIMEOUT; 0 uses the default) + -t, --type string Emulator type to start (aws, snowflake, azure) + -v, --version Show version +--- diff --git a/test/integration/extension_bundle_test.go b/test/integration/extension_bundle_test.go new file mode 100644 index 00000000..33e38f50 --- /dev/null +++ b/test/integration/extension_bundle_test.go @@ -0,0 +1,132 @@ +package integration_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/localstack/lstk/internal/snap" + "github.com/stretchr/testify/require" +) + +// LocalStack's bundled extensions ship as ONE multi-call binary, +// `bundled-extensions`, next to lstk; the descriptions file lists the commands +// it provides, and lstk execs it with argv[0] set to `lstk-` so the binary +// knows which extension to be. These tests install the reference extension +// under that name and assert the user-visible contract: `lstk ` runs it +// under the right argv[0], help lists every described command, and a command +// the file does not describe is not handed to the bundle. + +// installMultiCallBundle places the reference extension in dir as the multi-call +// bundled binary and writes a descriptions file naming the given commands. +func installMultiCallBundle(t *testing.T, dir string, descriptions string) string { + t.Helper() + path := filepath.Join(dir, execName("bundled-extensions")) + copyExecutable(t, referenceExtensionBinary(t), path) + require.NoError(t, os.WriteFile(filepath.Join(dir, "lstk-extensions.toml"), []byte(descriptions), 0o644)) + return path +} + +func TestBundledMultiCallDispatchesWithArgv0(t *testing.T) { + t.Parallel() + bundleDir := t.TempDir() + lstkBin := installLstkBundle(t, bundleDir) + installMultiCallBundle(t, bundleDir, "doctor = \"Check the local setup\"\ndeploy = \"Deploy to LocalStack\"\n") + + tmpHome := t.TempDir() + environ := envWithPath(tmpHome, t.TempDir()) + + stdout, stderr, err := runBinary(t, t.TempDir(), environ, lstkBin, "doctor", "argv0") + require.NoError(t, err, stderr) + require.Contains(t, stdout, "ARGS=[argv0]") + require.Contains(t, stdout, "ARGV0=lstk-doctor", "the bundle must be told which extension to be via argv[0]") + + // The same binary, a different name. + stdout, stderr, err = runBinary(t, t.TempDir(), environ, lstkBin, "deploy", "argv0") + require.NoError(t, err, stderr) + require.Contains(t, stdout, "ARGV0=lstk-deploy") +} + +func TestBundledMultiCallRunsTheBundledBinary(t *testing.T) { + t.Parallel() + bundleDir := t.TempDir() + lstkBin := installLstkBundle(t, bundleDir) + bundlePath := installMultiCallBundle(t, bundleDir, "doctor = \"Check the local setup\"\n") + + // A same-named lstk-doctor on PATH must lose to the bundle. + extDir := t.TempDir() + installExtension(t, extDir, "doctor") + + tmpHome := t.TempDir() + stdout, stderr, err := runBinary(t, t.TempDir(), envWithPath(tmpHome, extDir), lstkBin, "doctor") + require.NoError(t, err, stderr) + resolvedBundle, err := filepath.EvalSymlinks(bundlePath) + require.NoError(t, err) + require.Contains(t, stdout, "SELF="+resolvedBundle, "expected the bundled binary to run, not the PATH one") +} + +func TestBundledMultiCallHelpListsDescribedCommands(t *testing.T) { + t.Parallel() + bundleDir := t.TempDir() + lstkBin := installLstkBundle(t, bundleDir) + installMultiCallBundle(t, bundleDir, "doctor = \"Check the local setup\"\ndeploy = \"Deploy to LocalStack\"\n") + + extDir := t.TempDir() + installExtension(t, extDir, "hello") // PATH-only, name-only in help + + tmpHome := t.TempDir() + stdout, stderr, err := runBinary(t, t.TempDir(), envWithPath(tmpHome, extDir), lstkBin, "--help") + require.NoError(t, err, stderr) + // Pins: both bundled commands with their descriptions, the PATH one + // name-only, no `bundled-extensions` or `extensions` phantom entries, and + // no ARGS= line (help never executes anything). + snap.Match(t, stdout) +} + +func TestBundledMultiCallUndescribedCommandIsUnknown(t *testing.T) { + t.Parallel() + bundleDir := t.TempDir() + lstkBin := installLstkBundle(t, bundleDir) + installMultiCallBundle(t, bundleDir, "doctor = \"Check the local setup\"\n") + + tmpHome := t.TempDir() + // `other` is not in the descriptions file, so it must not reach the bundle; + // with nothing on PATH either, it is an unknown command. + stdout, stderr, err := runBinary(t, t.TempDir(), envWithPath(tmpHome, t.TempDir()), lstkBin, "other") + requireExitCode(t, 1, err) + require.NotContains(t, stdout, "ARGS=", "the bundle must not have been executed") + require.Contains(t, stderr, `unknown command "other"`) +} + +func TestBundledMultiCallBinaryWithoutDescriptionsIsAnError(t *testing.T) { + t.Parallel() + bundleDir := t.TempDir() + lstkBin := installLstkBundle(t, bundleDir) + // The binary shipped but its descriptions file did not: a broken install. + // That must surface as an error, not as "unknown command". + copyExecutable(t, referenceExtensionBinary(t), filepath.Join(bundleDir, execName("bundled-extensions"))) + + tmpHome := t.TempDir() + stdout, stderr, err := runBinary(t, t.TempDir(), envWithPath(tmpHome, t.TempDir()), lstkBin, "doctor") + require.Error(t, err) + require.NotContains(t, stdout, "ARGS=") + require.NotContains(t, stderr, "unknown command") + require.Contains(t, stderr, "lstk-extensions.toml") +} + +func TestBundledMultiCallContextConveyed(t *testing.T) { + t.Parallel() + bundleDir := t.TempDir() + lstkBin := installLstkBundle(t, bundleDir) + installMultiCallBundle(t, bundleDir, "doctor = \"Check the local setup\"\n") + + tmpHome := t.TempDir() + environ := append(envWithPath(tmpHome, t.TempDir()), "DOCKER_HOST=tcp://127.0.0.1:1") + stdout, stderr, err := runBinary(t, t.TempDir(), environ, lstkBin, "--non-interactive", "doctor", "--foo") + require.NoError(t, err, stderr) + // Same contract as a standalone extension: args forwarded, lstk's own flag + // consumed and conveyed, API version and context present. + require.Contains(t, stdout, "ARGS=[--foo]") + require.Contains(t, stdout, "NON_INTERACTIVE=true") + require.Contains(t, stdout, "API_VERSION=1") +} diff --git a/test/integration/extension_test.go b/test/integration/extension_test.go index 91d8b689..1b0b3bbe 100644 --- a/test/integration/extension_test.go +++ b/test/integration/extension_test.go @@ -641,3 +641,28 @@ func TestExtensionBundledWinsOverPath(t *testing.T) { require.NoError(t, err) snap.Match(t, helpOut) } + +// TestExtensionArgv0IsExtensionName pins the argv[0] half of the contract for +// standalone extensions: resolved from PATH or from the bundled dir alike, the +// program is invoked as `lstk-` (its own file name), the same value a +// bundle-provided command receives from the multi-call dispatch. That is what +// lets one extension binary be shipped either way without a code change. +func TestExtensionArgv0IsExtensionName(t *testing.T) { + t.Parallel() + extDir := t.TempDir() + installExtension(t, extDir, "hello") + + tmpHome := t.TempDir() + environ := append(envWithPath(tmpHome, extDir), "DOCKER_HOST=tcp://127.0.0.1:1") + stdout, stderr, err := runLstk(t, testContext(t), t.TempDir(), environ, "hello", "argv0") + require.NoError(t, err, stderr) + require.Contains(t, stdout, "ARGV0=lstk-hello") + + // Same for a standalone file placed next to the lstk binary. + bundleDir := t.TempDir() + lstkBin := installLstkBundle(t, bundleDir) + installExtension(t, bundleDir, "deploy") + stdout, stderr, err = runBinary(t, t.TempDir(), envWithPath(tmpHome, t.TempDir()), lstkBin, "deploy", "argv0") + require.NoError(t, err, stderr) + require.Contains(t, stdout, "ARGV0=lstk-deploy") +} diff --git a/test/integration/test-samples/extensions/lstk-ref/main.go b/test/integration/test-samples/extensions/lstk-ref/main.go index 1f608aa0..42e502db 100644 --- a/test/integration/test-samples/extensions/lstk-ref/main.go +++ b/test/integration/test-samples/extensions/lstk-ref/main.go @@ -11,6 +11,10 @@ // // (default) Echo the received args and decoded context, then exit 0. // exit N Echo, then exit with status N (for exit-code propagation tests). +// argv0 Echo, then print the name the binary was invoked as. Backs the +// multi-call bundle tests: lstk execs the one bundled binary with +// Args[0] set to lstk-, which is how a real bundle selects +// which extension to be. // auth Perform a stubbed self-authorization: succeed (exit 0) only when // the conveyed context carries an auth token, otherwise refuse // (exit 13). A real extension would verify the token server-side @@ -30,7 +34,9 @@ import ( "fmt" "os" "os/signal" + "path/filepath" "strconv" + "strings" "syscall" "time" ) @@ -79,6 +85,11 @@ func run(args []string) int { return 0 } switch args[0] { + case "argv0": + // Base name only, with any .exe stripped, so the value is identical on + // every platform regardless of how lstk spelled the path. + fmt.Printf("ARGV0=%s\n", strings.TrimSuffix(filepath.Base(os.Args[0]), ".exe")) + return 0 case "exit": if len(args) < 2 { fmt.Fprintln(os.Stderr, "lstk-ref: exit requires a status code") From 1a4bc45b810e60283183e2cbb4db1984fee30630 Mon Sep 17 00:00:00 2001 From: Carlos Arilla Date: Mon, 31 Aug 2026 17:43:29 +0200 Subject: [PATCH 2/5] We are keeping symslinks when possible, evven when they are not required. They do a cosmetic function and allow to potentially test the command without lstk. --- .goreleaser.yaml | 7 +++- CLAUDE.md | 2 +- docs/extensions-authoring.md | 4 +- docs/extensions-bundling.md | 33 +++++++++++---- .../design.md | 2 +- .../tasks.md | 2 +- scripts/check-descriptions.sh | 15 +++++-- scripts/fetch-bundled-extensions.sh | 35 +++++++++++++--- scripts/tests/check-descriptions_test.sh | 11 +++++ .../tests/fetch-bundled-extensions_test.sh | 23 ++++++++++- scripts/tests/lib.sh | 12 ++++++ test/integration/extension_bundle_test.go | 40 +++++++++++++++++++ 12 files changed, 162 insertions(+), 24 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index df42fa54..19f70a20 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -49,7 +49,12 @@ archives: # bundled/ by scripts/fetch-bundled-extensions.sh, which the release job # runs first; a local snapshot build needs it too (see # docs/extensions-bundling.md). A glob matching nothing fails the build. - - src: "bundled/{{ .Os }}_{{ .Arch }}/bundled-extensions*" + # + # The whole platform directory is taken, not just the binary, so the + # lstk- alias symlinks staged beside it (Unix only) ride along — + # goreleaser preserves symlinks in tar.gz. They are a convenience for + # running an extension straight from a shell; lstk never resolves them. + - src: "bundled/{{ .Os }}_{{ .Arch }}/*" strip_parent: true info: mode: 0o755 diff --git a/CLAUDE.md b/CLAUDE.md index b3286ccd..5f4c61b6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -170,7 +170,7 @@ Shared plumbing lives in `cmd/proxy.go` (`leadingFlags`, `stripLeadingProxyFlags # Extensions -lstk supports Git-style extensions: when `lstk ` is not a built-in command or alias, lstk resolves and execs an external `lstk-` executable, forwarding arguments verbatim and propagating the exit code. Built-ins always win. Resolution order is built-ins → bundled dir (the directory of the symlink-resolved lstk executable) → `PATH`; there is no manifest. Runtime context is conveyed via `LSTK_EXT_API_VERSION` and `LSTK_EXT_CONTEXT` (JSON: `configDir`, optional `authToken`, `nonInteractive`, `json`, optional `sessionId` — lstk's telemetry session id, omitted when telemetry is disabled, so an extension's own telemetry can join lstk's `ext:` event — optional `machineId` — lstk's anonymized machine id (the prepared hash), omitted alongside `sessionId` when telemetry is disabled, so an extension reports the same machine without re-deriving it — optional `endpointUrl` — the resolved `--endpoint-url`/`LSTK_ENDPOINT_URL`/`AWS_ENDPOINT_URL` value, conveyed verbatim and unvalidated (dispatch never rejects or probes it, unlike the built-ins' `rejectEndpointURL`), omitted when no source is set — and an `emulators` array, which stays local-Docker discovery and is independent of `endpointUrl`) — see `extension.Context`/`Environ` in `internal/extension/context.go`; dispatch and help listing are in `cmd/extension.go`. LocalStack's own bundled extensions ship as **one multi-call binary**, `bundled-extensions` (`extension.BundledBinaryName`), next to `lstk`, plus the hand-authored `lstk-extensions.toml`; that file is load-bearing for the bundle — it is the only record of which commands the binary provides — so `extension.LoadBundle` (`internal/extension/bundle.go`) hard-fails on a missing/malformed/empty one when the binary is present, whereas `LoadDescriptions` (help text only) still degrades quietly. `Resolver.Resolve` consults the bundle first for described names and execs it with `argv[0]` = `lstk-` (`Extension.Argv0`, applied in `Invoke`), then standalone `lstk-` files in the bundled dir, then `PATH`; a broken bundle is reported by `Resolve` only when nothing else provides the name, and skipped (logged) by `List` so help never breaks. `LoadBundle` also rejects a toml key that is not a dispatchable name (`validate.ExtensionName`, the same rule the release gate applies), and `Resolver.List` attaches each bundled entry's help `Description` so `cmd` renders from one parse of the file. The argv[0] contract is uniform: a standalone `lstk-` file is invoked under its own base name, which is the same `lstk-` a bundle-provided command receives. Distribution is automated in the release job: `scripts/fetch-bundled-extensions.sh` downloads and checksum-verifies the bundle selected by `bundled/extensions.version` from the private extensions repo, `scripts/check-descriptions.sh` gates the toml against the binary's own command list (`bundled/bundle-commands.txt`, which the fetch script records from the `lstk-` alias entries every bundle archive must carry, and which is never packaged), so a described command the binary cannot dispatch fails the release, `.goreleaser.yaml` packages both at the archive root (the cask inherits them; a release-job step copies them into each npm **platform** package), and the resolved bundle tag is recorded in the release notes. `scripts/check-bundled-packaging-sync.sh` runs on every PR to keep the packaging and download halves from merging separately. Bash tests for these scripts: `make test-scripts` (`scripts/tests/`). Set-wise co-update on the binary channel (`internal/update`) is still pending in the `add-bundled-extension-distribution` change. See [extensions-authoring.md](docs/extensions-authoring.md) for the author-facing contract and [extensions-bundling.md](docs/extensions-bundling.md) for the release pipeline and on-disk layout per channel. +lstk supports Git-style extensions: when `lstk ` is not a built-in command or alias, lstk resolves and execs an external `lstk-` executable, forwarding arguments verbatim and propagating the exit code. Built-ins always win. Resolution order is built-ins → bundled dir (the directory of the symlink-resolved lstk executable) → `PATH`; there is no manifest. Runtime context is conveyed via `LSTK_EXT_API_VERSION` and `LSTK_EXT_CONTEXT` (JSON: `configDir`, optional `authToken`, `nonInteractive`, `json`, optional `sessionId` — lstk's telemetry session id, omitted when telemetry is disabled, so an extension's own telemetry can join lstk's `ext:` event — optional `machineId` — lstk's anonymized machine id (the prepared hash), omitted alongside `sessionId` when telemetry is disabled, so an extension reports the same machine without re-deriving it — optional `endpointUrl` — the resolved `--endpoint-url`/`LSTK_ENDPOINT_URL`/`AWS_ENDPOINT_URL` value, conveyed verbatim and unvalidated (dispatch never rejects or probes it, unlike the built-ins' `rejectEndpointURL`), omitted when no source is set — and an `emulators` array, which stays local-Docker discovery and is independent of `endpointUrl`) — see `extension.Context`/`Environ` in `internal/extension/context.go`; dispatch and help listing are in `cmd/extension.go`. LocalStack's own bundled extensions ship as **one multi-call binary**, `bundled-extensions` (`extension.BundledBinaryName`), next to `lstk`, plus the hand-authored `lstk-extensions.toml`; that file is load-bearing for the bundle — it is the only record of which commands the binary provides — so `extension.LoadBundle` (`internal/extension/bundle.go`) hard-fails on a missing/malformed/empty one when the binary is present, whereas `LoadDescriptions` (help text only) still degrades quietly. `Resolver.Resolve` consults the bundle first for described names and execs it with `argv[0]` = `lstk-` (`Extension.Argv0`, applied in `Invoke`), then standalone `lstk-` files in the bundled dir, then `PATH`; a broken bundle is reported by `Resolve` only when nothing else provides the name, and skipped (logged) by `List` so help never breaks. `LoadBundle` also rejects a toml key that is not a dispatchable name (`validate.ExtensionName`, the same rule the release gate applies), and `Resolver.List` attaches each bundled entry's help `Description` so `cmd` renders from one parse of the file. The argv[0] contract is uniform: a standalone `lstk-` file is invoked under its own base name, which is the same `lstk-` a bundle-provided command receives. Distribution is automated in the release job: `scripts/fetch-bundled-extensions.sh` downloads and checksum-verifies the bundle selected by `bundled/extensions.version` from the private extensions repo, `scripts/check-descriptions.sh` gates the toml against the binary's own command list (`bundled/bundle-commands.txt`, which the fetch script records from the `lstk-` alias entries every bundle archive must carry, and which is never packaged), so a described command the binary cannot dispatch fails the release; the fetch script also re-creates those aliases as relative symlinks beside the staged binary (not on Windows, where extractors materialize them as junk text files) so a packaged install can run `lstk-` directly — goreleaser preserves symlinks in tar.gz, `npm pack` drops them, and lstk itself never resolves them, `.goreleaser.yaml` packages both at the archive root (the cask inherits them; a release-job step copies them into each npm **platform** package), and the resolved bundle tag is recorded in the release notes. `scripts/check-bundled-packaging-sync.sh` runs on every PR to keep the packaging and download halves from merging separately. Bash tests for these scripts: `make test-scripts` (`scripts/tests/`). Set-wise co-update on the binary channel (`internal/update`) is still pending in the `add-bundled-extension-distribution` change. See [extensions-authoring.md](docs/extensions-authoring.md) for the author-facing contract and [extensions-bundling.md](docs/extensions-bundling.md) for the release pipeline and on-disk layout per channel. # Signal Forwarding to Wrapped Tools diff --git a/docs/extensions-authoring.md b/docs/extensions-authoring.md index 9a8c57fc..ac8f2aa2 100644 --- a/docs/extensions-authoring.md +++ b/docs/extensions-authoring.md @@ -124,7 +124,9 @@ beside it; a name that file does not list is never handed to the bundle. Read being asked to be. The value is exactly `lstk-`, with no path and no `.exe`, and lstk only ever hands the bundle a name the toml lists, so a lookup miss inside the binary means the toml and the binary disagree: report it -loudly rather than guessing. See [extensions-bundling.md](extensions-bundling.md) for how +loudly rather than guessing. Where the install channel allows it a release also +places an `lstk-` symlink to the binary, so the same command can be run +straight from a shell; that is a convenience only, and lstk never resolves it. See [extensions-bundling.md](extensions-bundling.md) for how the bundle is built and shipped. ## Help descriptions diff --git a/docs/extensions-bundling.md b/docs/extensions-bundling.md index 936cf14b..f3140949 100644 --- a/docs/extensions-bundling.md +++ b/docs/extensions-bundling.md @@ -63,8 +63,9 @@ order. Every step failing fails the release. archive and stages only two members: the binary as `bundled/_/bundled-extensions[.exe]` (mode 0755) and the descriptions file as `bundled/lstk-extensions.toml` (taken once; every - archive must carry an identical copy). The `lstk-` alias entries in - the archives are not staged, but their names are recorded, sorted, in + archive must carry an identical copy). It re-creates one `lstk-` + symlink to the binary per command, except on Windows (see "Running an + extension directly" below), and records the names, sorted, in `bundled/bundle-commands.txt`: they are the bundle's own declaration of which commands the binary answers to. An archive with no aliases, or whose list differs from another platform's, aborts. It fails if any of lstk's six target platforms @@ -117,11 +118,10 @@ Each tagged release of `localstack/lstk-bundled-extensions` ships: containing at its root the multi-call binary `bundled-extensions` (`.exe` on Windows), `lstk-extensions.toml`, and one `lstk-` alias entry (a symlink on Unix, a copy on Windows) for every command the binary answers - to. The aliases are never installed: lstk dispatches by `argv[0]` and does - not need them on disk. They are required all the same, because they are the - binary's own statement of its command list, and the release gate verifies - the toml against exactly that list, so a described command the binary cannot - dispatch is caught before it ships; + to. They serve two purposes: they are the binary's own statement of its + command list, which the release gate checks the toml against, and they are + re-created in the packaged install so a command can be run directly (see + below); - the same `lstk-extensions.toml` in every archive, hand-authored, describing every command the binary provides — a described command with no implementation would show in `lstk --help` and fail when run; @@ -173,6 +173,25 @@ it lands, the in-the-field updater replaces only `lstk`; the other two files must be extracted from the archive by hand. Updating never deletes standalone `lstk-` files a user placed next to the binary. +## Running an extension directly + +Where the channel allows it, a release also carries one `lstk-` symlink +to `bundled-extensions` per command, so `lstk-doctor` can be run straight from +a shell. That is a convenience for trying an extension on its own; lstk never +resolves those links. It dispatches to the binary by `argv[0]` and takes its +command list from the toml, so a channel without them behaves identically. + +| Channel | Aliases | Why | +| --- | --- | --- | +| Binary archive (tar.gz) | Yes | goreleaser preserves symlinks in tar.gz. | +| Homebrew cask | Yes | It stages the same archive. | +| Windows (zip) | No | Most Windows extractors turn a zip symlink into a small text file holding the target's name, which is worse than absent. | +| npm | No | `npm pack` silently drops symlinks from the published tarball. | + +`lstk update` on the binary channel does not currently re-create them: its +extractor skips symlink entries. Whatever a fresh install put there is left +alone, so an updated install keeps the links it already had. + ## Diagnosing a broken install If `bundled-extensions` is present but `lstk-extensions.toml` is missing, diff --git a/openspec/changes/add-bundled-extension-distribution/design.md b/openspec/changes/add-bundled-extension-distribution/design.md index fb0a01c4..e3b97030 100644 --- a/openspec/changes/add-bundled-extension-distribution/design.md +++ b/openspec/changes/add-bundled-extension-distribution/design.md @@ -85,7 +85,7 @@ The first bundling release ships with the smallest viable bundle (a single exten ### Decision 7: Bundled binary layout — RESOLVED: (b), one multi-call binary -**Resolution (2026-08-27, DPX-692):** option (b). The bundle ships as a single binary named `bundled-extensions` next to `lstk`, plus `lstk-extensions.toml`. lstk takes the bundled command list from the descriptions file and execs the one binary with `argv[0]` set to `lstk-` (`Extension.Argv0`, honoured in `extension.Invoke`). The runtime changes this implies are in `internal/extension/bundle.go` (`LoadBundle`, `BundledBinaryName`) and the bundle branch of `Resolver.Resolve`/`List`: the bundle is consulted first for described names, then standalone `lstk-` files in the bundled dir (manual placement keeps working), then PATH. When the binary is present, a missing/unreadable/empty descriptions file is a hard error surfaced by `Resolve` when nothing else provides the name — never a silent "unknown command" — while `List` logs and skips it so help never breaks. Release-side, `scripts/check-descriptions.sh` enforces the same pairing (described-but-no-binary and binary-but-nothing-described both fail), and `.goreleaser.yaml` packages `bundled-extensions*` + the toml. Since the binary cannot be executed cross-platform at release time, its command list is taken from the `lstk-` alias entries the bundle archives carry: the fetch script records them as `bundled/bundle-commands.txt` (never packaged) and the gate fails on a described name that list lacks, which makes the aliases a required part of the private repo's release convention rather than an optional convenience. The same name rule (`validate.ExtensionName`) is applied by `LoadBundle`, so a toml that passes the gate always loads. The fetch script unpacks the private repo's actual release convention — one `bundled-extensions___.tar.gz`/`.zip` per platform containing the binary, the toml and `lstk-` alias entries — and stages only the binary and the toml, recording the aliases' names rather than the aliases themselves (symlinks would be dropped by lstk's tar extractor and copies would triple the payload). The section-1 updater work and the section-6 test plan should be read with "the set" = `lstk`, `bundled-extensions`, `lstk-extensions.toml`, and "complete" = the binary present alongside a loadable descriptions file. The original analysis follows. +**Resolution (2026-08-27, DPX-692):** option (b). The bundle ships as a single binary named `bundled-extensions` next to `lstk`, plus `lstk-extensions.toml`. lstk takes the bundled command list from the descriptions file and execs the one binary with `argv[0]` set to `lstk-` (`Extension.Argv0`, honoured in `extension.Invoke`). The runtime changes this implies are in `internal/extension/bundle.go` (`LoadBundle`, `BundledBinaryName`) and the bundle branch of `Resolver.Resolve`/`List`: the bundle is consulted first for described names, then standalone `lstk-` files in the bundled dir (manual placement keeps working), then PATH. When the binary is present, a missing/unreadable/empty descriptions file is a hard error surfaced by `Resolve` when nothing else provides the name — never a silent "unknown command" — while `List` logs and skips it so help never breaks. Release-side, `scripts/check-descriptions.sh` enforces the same pairing (described-but-no-binary and binary-but-nothing-described both fail), and `.goreleaser.yaml` packages `bundled-extensions*` + the toml. Since the binary cannot be executed cross-platform at release time, its command list is taken from the `lstk-` alias entries the bundle archives carry: the fetch script records them as `bundled/bundle-commands.txt` (never packaged) and the gate fails on a described name that list lacks, which makes the aliases a required part of the private repo's release convention rather than an optional convenience. The same name rule (`validate.ExtensionName`) is applied by `LoadBundle`, so a toml that passes the gate always loads. The fetch script unpacks the private repo's actual release convention — one `bundled-extensions___.tar.gz`/`.zip` per platform containing the binary, the toml and `lstk-` alias entries — and stages the binary, the toml, and re-created relative alias symlinks (Unix only), recording the alias names for the gate. The aliases are a convenience for running a command directly, never a resolution path: verified empirically, goreleaser preserves symlinks in tar.gz (so the binary channel and the cask carry them) while `npm pack` drops them and a Windows zip symlink is re-created by most extractors as a text file holding the target name, so those two channels ship without them and behave identically. The section-1 updater work and the section-6 test plan should be read with "the set" = `lstk`, `bundled-extensions`, `lstk-extensions.toml`, and "complete" = the binary present alongside a loadable descriptions file. The original analysis follows. **What this blocks, and what it does not.** Decision 7 gates section 5 of `tasks.md` (turning packaging on) and the parts of the test plan that name individual on-disk files, because both have to know what the payload looks like. It does not gate sections 1 to 3: the set-wise updater, the descriptions check and the fetch script are all written against "whatever the archive contains" and can be built and merged first. Leaving it open therefore holds nothing up, and the input it is waiting for (whether the bundle really is one binary, and how big it is) arrives naturally once the doctor extension exists. diff --git a/openspec/changes/add-bundled-extension-distribution/tasks.md b/openspec/changes/add-bundled-extension-distribution/tasks.md index 1cc569ae..1e582084 100644 --- a/openspec/changes/add-bundled-extension-distribution/tasks.md +++ b/openspec/changes/add-bundled-extension-distribution/tasks.md @@ -18,7 +18,7 @@ The descriptions file `lstk-extensions.toml` is a flat TOML table (`deploy = "On - [x] 2.1 Re-introduce `scripts/check-descriptions.sh` (plain bash, same style as `scripts/test-integration.sh`). Input: a directory containing the downloaded extension binaries and the toml. Behavior: read the names on the left-hand side of each `name = "…"` line (only the names — never parse the values, so a weird description string can't break the script); for each name, check an executable file `lstk-` exists in that directory; if any is missing, print which ones and exit non-zero (this fails the release). The reverse case — a binary present but not described — only prints a warning, because lstk's help intentionally falls back to showing such extensions name-only. - [x] 2.2 Test the script against fixture directories (a small test script or make target creating temp dirs): described-but-missing binary → fails and names it; described-and-present → passes; binary-without-description → warns but passes; empty or absent toml → passes (nothing is described, nothing to check). -- [x] 2.3 Verify the described names against the bundle's own command list, not just the binary's presence. The fetch script records the `lstk-` alias entries every archive carries into `bundled/bundle-commands.txt` (aborting on an archive with none, or on lists that differ between platforms), and `check-descriptions.sh` fails when the toml describes a command that list lacks and warns on the reverse. Covered in both bash suites; the list is never packaged. +- [x] 2.3 Verify the described names against the bundle's own command list, not just the binary's presence. The fetch script records the `lstk-` alias entries every archive carries into `bundled/bundle-commands.txt` (aborting on an archive with none, or on lists that differ between platforms), and `check-descriptions.sh` fails when the toml describes a command that list lacks and warns on the reverse. Covered in both bash suites; the list is never packaged. The fetch script also re-creates the aliases as relative symlinks beside the staged binary so a packaged install can run `lstk-` directly; Windows is skipped (extractors materialize zip symlinks as junk text files) and `npm pack` drops them, so those channels ship without and behave identically — lstk never resolves them. Note: the check runs once per release, against the Linux/amd64 download directory only. Descriptions are the same for every OS, and on Linux the binaries have plain names with no `.exe`, so one directory is enough. diff --git a/scripts/check-descriptions.sh b/scripts/check-descriptions.sh index 1512a46d..434e734c 100755 --- a/scripts/check-descriptions.sh +++ b/scripts/check-descriptions.sh @@ -100,11 +100,18 @@ if [ -n "${invalid}" ]; then die "command names must match ^[A-Za-z0-9][A-Za-z0-9_-]*$" fi -# Standalone lstk- files are not how the bundle ships. They still work -# (lstk resolves them from its directory) but carry no description, so flag -# them rather than fail. +# lstk- entries beside the binary are either the bundle's own aliases +# (symlinks to it, staged so the command can also be run straight from a shell) +# or a genuinely standalone extension. The first are expected and silent; the +# second still works — lstk resolves it from its directory — but carries no +# description, so it is flagged rather than failed. for stray in "${PLATFORM_DIR}"/lstk-*; do - [ -e "${stray}" ] || continue + [ -e "${stray}" ] || [ -L "${stray}" ] || continue + if [ -L "${stray}" ]; then + case "$(readlink "${stray}")" in + "${BUNDLED_BINARY}"|"${BUNDLED_BINARY}.exe") continue ;; + esac + fi echo "Warning: standalone extension binary $(basename "${stray}") in ${PLATFORM_DIR} is not part of the bundle and will show name-only in help." done diff --git a/scripts/fetch-bundled-extensions.sh b/scripts/fetch-bundled-extensions.sh index 05c7b0e0..7c363719 100755 --- a/scripts/fetch-bundled-extensions.sh +++ b/scripts/fetch-bundled-extensions.sh @@ -16,12 +16,21 @@ # containing the multi-call binary `bundled-extensions[.exe]` and the # descriptions file `lstk-extensions.toml`, plus a `checksums.txt` covering the # archives, and `lstk-` alias entries for every command the binary -# answers to (symlinks on Unix, copies on Windows). The binary and the toml are -# staged as they are. The aliases are not: lstk dispatches to the one binary by -# argv[0] and never needs them on disk. Their names are, though, recorded in -# bundle-commands.txt, because they are the bundle's own declaration of which -# commands it provides, and scripts/check-descriptions.sh verifies the toml -# against that list. An archive with no aliases is rejected for that reason. +# answers to (symlinks on Unix, copies on Windows). +# +# The binary and the toml are staged as they are. The aliases are re-created as +# relative symlinks next to the binary, so a packaged install can also run +# `lstk-doctor` straight from a shell — useful for testing an extension on its +# own. lstk itself never resolves them: it dispatches to the one binary by +# argv[0] and takes its command list from the toml, so a channel that cannot +# carry symlinks loses only that convenience. Windows is skipped deliberately: +# a zip symlink is re-created by most Windows extractors as a small text file +# holding the target's name, which is worse than absent. +# +# The alias names are also recorded in bundle-commands.txt, because they are the +# bundle's own declaration of which commands it provides, and +# scripts/check-descriptions.sh verifies the toml against that list. An archive +# with no aliases is rejected for that reason. # # Which bundle is taken comes from bundled/extensions.version — `latest` by # default. `latest` is resolved to a concrete tag exactly once here and @@ -59,6 +68,7 @@ UNSUPPORTED_PLATFORMS="${LSTK_UNSUPPORTED_PLATFORMS-}" REPO="${LSTK_EXTENSIONS_REPO:-localstack/lstk-bundled-extensions}" BUNDLED_BINARY="bundled-extensions" +NAME_PREFIX="lstk-" DESCRIPTIONS_FILE="lstk-extensions.toml" COMMANDS_FILE="bundle-commands.txt" MANIFEST_FILE="checksums.txt" @@ -212,6 +222,15 @@ stage_assets() { else printf "%s\n" "${commands}" > "${commands_staged}" fi + + # Re-create the aliases rather than copying the archive's own entries: the + # target is written relative and bare so it resolves wherever the pair is + # unpacked, whatever the archive happened to contain. + if [ "${os}" != "windows" ]; then + for name in ${commands}; do + ln -sf "${BUNDLED_BINARY}" "${BUNDLED_DIR}/${os}_${arch}/${NAME_PREFIX}${name}" + done + fi done [ -f "${toml_staged}" ] || die "the bundle publishes no ${DESCRIPTIONS_FILE}" echo "Staged ${staged} platform binaries into ${BUNDLED_DIR}." @@ -264,6 +283,10 @@ write_stub_bundle() { printf '#!/bin/sh\necho "stub %s for %s"\n' "${name}" "${platform}" \ > "${BUNDLED_DIR}/${platform}/${name}${suffix}" chmod 0755 "${BUNDLED_DIR}/${platform}/${name}${suffix}" + # Mirror the real layout so a snapshot build exercises the same shape. + if [ "${name}" = "${BUNDLED_BINARY}" ] && [ "${suffix}" = "" ]; then + ln -sf "${BUNDLED_BINARY}" "${BUNDLED_DIR}/${platform}/${NAME_PREFIX}doctor" + fi done done { diff --git a/scripts/tests/check-descriptions_test.sh b/scripts/tests/check-descriptions_test.sh index c5c11432..ce7973aa 100644 --- a/scripts/tests/check-descriptions_test.sh +++ b/scripts/tests/check-descriptions_test.sh @@ -176,6 +176,17 @@ run_script "${CHECK}" "${PLATFORM_DIR}" assert_fails assert_output_contains "doc tor" +begin_test "alias symlinks to the bundle are expected and do not warn" +setup_stage +write_binary +( cd "${PLATFORM_DIR}" && ln -s bundled-extensions lstk-doctor ) +write_commands doctor +write_toml 'doctor = "Check the local setup" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_ok +assert_output_lacks "Warning" + begin_test "a stray standalone lstk- binary warns but passes" setup_stage write_binary diff --git a/scripts/tests/fetch-bundled-extensions_test.sh b/scripts/tests/fetch-bundled-extensions_test.sh index 920d6619..5a2ca42a 100755 --- a/scripts/tests/fetch-bundled-extensions_test.sh +++ b/scripts/tests/fetch-bundled-extensions_test.sh @@ -218,12 +218,31 @@ assert_file_contains "${BUNDLED}/lstk-extensions.toml" "doctor" assert_file_absent "${BUNDLED}/linux_amd64/lstk-extensions.toml" assert_file_absent "${BUNDLED}/linux_amd64/checksums.txt" -begin_test "lstk- alias entries in the archives are not staged" +begin_test "alias entries are staged as symlinks beside the binary on unix" +setup_workspace +run_script "${FETCH}" +assert_ok +for platform in linux_amd64 linux_arm64 darwin_amd64 darwin_arm64; do + assert_symlink_to "${BUNDLED}/${platform}/lstk-doctor" "bundled-extensions" +done + +begin_test "alias entries are skipped on Windows, where extraction would junk them" setup_workspace run_script "${FETCH}" assert_ok -assert_file_absent "${BUNDLED}/linux_amd64/lstk-doctor" assert_file_absent "${BUNDLED}/windows_amd64/lstk-doctor.exe" +assert_file_absent "${BUNDLED}/windows_arm64/lstk-doctor.exe" +assert_executable "${BUNDLED}/windows_amd64/bundled-extensions.exe" + +begin_test "every described command gets an alias" +setup_workspace +ALIASES="doctor deploy" TOML_BODY='doctor = "x" +deploy = "y" +' make_release_assets "${ASSETS}" +run_script "${FETCH}" +assert_ok +assert_symlink_to "${BUNDLED}/linux_amd64/lstk-doctor" "bundled-extensions" +assert_symlink_to "${BUNDLED}/linux_amd64/lstk-deploy" "bundled-extensions" begin_test "the staged tree passes the descriptions gate" setup_workspace diff --git a/scripts/tests/lib.sh b/scripts/tests/lib.sh index 2836bcf8..b46dc71c 100644 --- a/scripts/tests/lib.sh +++ b/scripts/tests/lib.sh @@ -93,3 +93,15 @@ finish_suite() { fi echo "${TESTS_RUN}/${TESTS_RUN} test(s) passed in $(basename "$0")" } + +# Asserts that path is a symlink whose target is exactly want. Relative targets +# are compared verbatim: an absolute or ../-prefixed target would not survive +# packaging, so the exact string is the thing under test. +assert_symlink_to() { + if [ ! -L "$1" ]; then + fail "expected a symlink at: $1" + return + fi + got="$(readlink "$1")" + [ "${got}" = "$2" ] || fail "expected $1 -> $2, got -> ${got}" +} diff --git a/test/integration/extension_bundle_test.go b/test/integration/extension_bundle_test.go index 33e38f50..86953d4b 100644 --- a/test/integration/extension_bundle_test.go +++ b/test/integration/extension_bundle_test.go @@ -3,6 +3,8 @@ package integration_test import ( "os" "path/filepath" + "runtime" + "strings" "testing" "github.com/localstack/lstk/internal/snap" @@ -130,3 +132,41 @@ func TestBundledMultiCallContextConveyed(t *testing.T) { require.Contains(t, stdout, "NON_INTERACTIVE=true") require.Contains(t, stdout, "API_VERSION=1") } + +// Releases stage an lstk- symlink next to the bundle for every command it +// provides, so a user can also run `lstk-doctor` straight from a shell. lstk +// must stay indifferent to them: it dispatches through the bundle by argv[0] +// and takes its command list from the descriptions file, so an alias must not +// produce a second help entry, and removing one must change nothing. This is +// what lets channels that cannot carry symlinks (npm, Windows zip) ship without +// them and behave identically. +func TestBundledAliasSymlinksAreInertForLstk(t *testing.T) { + t.Parallel() + if runtime.GOOS == "windows" { + t.Skip("aliases are deliberately not staged on Windows") + } + bundleDir := t.TempDir() + lstkBin := installLstkBundle(t, bundleDir) + bundlePath := installMultiCallBundle(t, bundleDir, "doctor = \"Check the local setup\"\ndeploy = \"Deploy to LocalStack\"\n") + for _, name := range []string{"doctor", "deploy"} { + require.NoError(t, os.Symlink("bundled-extensions", filepath.Join(bundleDir, "lstk-"+name))) + } + + tmpHome := t.TempDir() + environ := envWithPath(tmpHome, t.TempDir()) + + // Each command appears exactly once, with its description — not twice, and + // not as a name-only row shadowing the described one. + stdout, stderr, err := runBinary(t, t.TempDir(), environ, lstkBin, "--help") + require.NoError(t, err, stderr) + require.Equal(t, 1, strings.Count(stdout, "doctor Check the local setup")) + require.Equal(t, 1, strings.Count(stdout, "deploy Deploy to LocalStack")) + + // Dispatch still goes through the bundle under the right argv[0]. + stdout, stderr, err = runBinary(t, t.TempDir(), environ, lstkBin, "doctor", "argv0") + require.NoError(t, err, stderr) + require.Contains(t, stdout, "ARGV0=lstk-doctor") + resolved, err := filepath.EvalSymlinks(bundlePath) + require.NoError(t, err) + require.Contains(t, stdout, "SELF="+resolved) +} From c0cbe897079e8897a7b5106ae43a20e3fb61f573 Mon Sep 17 00:00:00 2001 From: Carlos Arilla Date: Tue, 1 Sep 2026 10:33:58 +0200 Subject: [PATCH 3/5] fix npm issue found in testing --- scripts/add-bundled-to-npm.sh | 38 ++++++++++----- scripts/tests/add-bundled-to-npm_test.sh | 62 +++++++++++++++--------- 2 files changed, 65 insertions(+), 35 deletions(-) diff --git a/scripts/add-bundled-to-npm.sh b/scripts/add-bundled-to-npm.sh index e1472431..e7737693 100755 --- a/scripts/add-bundled-to-npm.sh +++ b/scripts/add-bundled-to-npm.sh @@ -11,8 +11,11 @@ # goreleaser-npm-publisher has no per-platform extra-files option, so this runs # on its dist/npm output before `npm publish`. Two details it has to get right: # -# * Node and Go name platforms differently: win32 -> windows, x64 -> amd64 -# (darwin, linux and arm64 are the same in both). +# * The publisher slugifies its output directory names +# (dist/npm/lstk-darwin-arm-64-v-8-0), so a directory name cannot be parsed +# back into a platform. The package.json inside carries the authoritative +# name, @localstack/lstk__, which maps straight onto the +# staging directory - so that is what this reads. # * The generated package.json carries "files": [], which npm packs as # package.json + the bin entry and nothing else. Copying alone would be # silently dropped at publish, so the copied names are appended to that @@ -28,7 +31,7 @@ die() { } usage() { - sed -n '2,23p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' >&2 + sed -n '2,25p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' >&2 exit 1 } @@ -57,17 +60,28 @@ register_files() { ' "${pkg_json}" "$@" } +# Reads the "name" field of a package.json. +package_name() { + node -e ' + const fs = require("fs"); + const pkg = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); + process.stdout.write(pkg.name || ""); + ' "$1" +} + count=0 -for dir in "${NPM_DIR}"/lstk-*/; do - [ -d "${dir}" ] || continue +for dir in "${NPM_DIR}"/*/; do + [ -f "${dir}package.json" ] || continue pkg="$(basename "${dir}")" - cpu="${pkg##*-}" - os="${pkg#lstk-}" - os="${os%-*}" - case "${os}" in win32) goos=windows ;; *) goos="${os}" ;; esac - case "${cpu}" in x64) goarch=amd64 ;; *) goarch="${cpu}" ;; esac - src="${BUNDLED_DIR}/${goos}_${goarch}" - [ -d "${src}" ] || die "no staged bundle for ${pkg} at ${src}" + name="$(package_name "${dir}package.json")" + # Platform packages are @localstack/lstk__; the wrapper is a + # bare @localstack/lstk and carries no binary, so it is skipped here. + case "${name}" in + */lstk_*) goplatform="${name##*/lstk_}" ;; + *) continue ;; + esac + src="${BUNDLED_DIR}/${goplatform}" + [ -d "${src}" ] || die "no staged bundle for ${name} (${pkg}) at ${src}" added="" for file in "${src}"/bundled-extensions*; do diff --git a/scripts/tests/add-bundled-to-npm_test.sh b/scripts/tests/add-bundled-to-npm_test.sh index f7e1460b..41b1783b 100644 --- a/scripts/tests/add-bundled-to-npm_test.sh +++ b/scripts/tests/add-bundled-to-npm_test.sh @@ -11,7 +11,21 @@ SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" . "${SUITE_DIR}/lib.sh" ADD="${SUITE_DIR}/../add-bundled-to-npm.sh" -NPM_PLATFORMS="darwin-arm64 darwin-x64 linux-arm64 linux-x64 win32-arm64 win32-x64" +# The publisher slugifies its output directory names (lstk-darwin-arm-64-v-8-0) +# while the package.json inside carries the authoritative Go-style name +# (@localstack/lstk_darwin_arm64). Fixtures reproduce both, because parsing the +# directory name is exactly the bug these tests exist to prevent. +NPM_DIRS="lstk-darwin-arm-64-v-8-0 lstk-darwin-amd-64-v-1 lstk-linux-arm-64-v-8-0 lstk-linux-amd-64-v-1 lstk-windows-arm-64-v-8-0 lstk-windows-amd-64-v-1" +dir_to_goplatform() { + case "$1" in + lstk-darwin-arm-64*) echo darwin_arm64 ;; + lstk-darwin-amd-64*) echo darwin_amd64 ;; + lstk-linux-arm-64*) echo linux_arm64 ;; + lstk-linux-amd-64*) echo linux_amd64 ;; + lstk-windows-arm-64*) echo windows_arm64 ;; + lstk-windows-amd-64*) echo windows_amd64 ;; + esac +} GO_PLATFORMS="darwin_arm64 darwin_amd64 linux_arm64 linux_amd64 windows_arm64 windows_amd64" # Fresh workspace with a full staged bundle and a full dist/npm tree. @@ -31,13 +45,15 @@ setup_workspace() { done echo 'doctor = "Check the local setup"' > "${BUNDLED}/lstk-extensions.toml" - for platform in ${NPM_PLATFORMS}; do - mkdir -p "${NPM}/lstk-${platform}" - local bin="lstk" - case "${platform}" in win32-*) bin="lstk.exe" ;; esac - echo "lstk binary" > "${NPM}/lstk-${platform}/${bin}" - printf '{\n "name": "@localstack/lstk-%s",\n "version": "0.1.0",\n "bin": {\n "lstk": "%s"\n },\n "files": []\n}\n' \ - "${platform}" "${bin}" > "${NPM}/lstk-${platform}/package.json" + local d goplat bin + for d in ${NPM_DIRS}; do + goplat="$(dir_to_goplatform "${d}")" + mkdir -p "${NPM}/${d}" + bin="lstk" + case "${goplat}" in windows_*) bin="lstk.exe" ;; esac + echo "lstk binary" > "${NPM}/${d}/${bin}" + printf '{\n "name": "@localstack/lstk_%s",\n "version": "0.1.0",\n "bin": {\n "lstk_%s": "%s"\n },\n "files": []\n}\n' \ + "${goplat}" "${goplat}" "${bin}" > "${NPM}/${d}/package.json" done mkdir -p "${NPM}/lstk" echo "launcher" > "${NPM}/lstk/index.js" @@ -54,30 +70,30 @@ begin_test "copies the matching platform binary and the toml into every platform setup_workspace run_script "${ADD}" "${NPM}" "${BUNDLED}" assert_ok -assert_file_exists "${NPM}/lstk-darwin-arm64/bundled-extensions" -assert_file_contains "${NPM}/lstk-darwin-arm64/bundled-extensions" "darwin_arm64" -assert_file_exists "${NPM}/lstk-linux-x64/bundled-extensions" -assert_file_contains "${NPM}/lstk-linux-x64/bundled-extensions" "linux_amd64" -assert_file_exists "${NPM}/lstk-win32-x64/bundled-extensions.exe" -assert_file_contains "${NPM}/lstk-win32-x64/bundled-extensions.exe" "windows_amd64" -assert_file_exists "${NPM}/lstk-win32-arm64/bundled-extensions.exe" -assert_file_contains "${NPM}/lstk-win32-arm64/bundled-extensions.exe" "windows_arm64" -assert_file_exists "${NPM}/lstk-darwin-x64/lstk-extensions.toml" +assert_file_exists "${NPM}/lstk-darwin-arm-64-v-8-0/bundled-extensions" +assert_file_contains "${NPM}/lstk-darwin-arm-64-v-8-0/bundled-extensions" "darwin_arm64" +assert_file_exists "${NPM}/lstk-linux-amd-64-v-1/bundled-extensions" +assert_file_contains "${NPM}/lstk-linux-amd-64-v-1/bundled-extensions" "linux_amd64" +assert_file_exists "${NPM}/lstk-windows-amd-64-v-1/bundled-extensions.exe" +assert_file_contains "${NPM}/lstk-windows-amd-64-v-1/bundled-extensions.exe" "windows_amd64" +assert_file_exists "${NPM}/lstk-windows-arm-64-v-8-0/bundled-extensions.exe" +assert_file_contains "${NPM}/lstk-windows-arm-64-v-8-0/bundled-extensions.exe" "windows_arm64" +assert_file_exists "${NPM}/lstk-darwin-amd-64-v-1/lstk-extensions.toml" begin_test "the copied binary keeps its executable bit" setup_workspace run_script "${ADD}" "${NPM}" "${BUNDLED}" assert_ok -assert_executable "${NPM}/lstk-linux-arm64/bundled-extensions" +assert_executable "${NPM}/lstk-linux-arm-64-v-8-0/bundled-extensions" begin_test "registers the files in each platform package's files allowlist" setup_workspace run_script "${ADD}" "${NPM}" "${BUNDLED}" assert_ok -LAST_OUTPUT="$(files_field "${NPM}/lstk-darwin-arm64/package.json")" +LAST_OUTPUT="$(files_field "${NPM}/lstk-darwin-arm-64-v-8-0/package.json")" assert_output_contains "bundled-extensions" assert_output_contains "lstk-extensions.toml" -LAST_OUTPUT="$(files_field "${NPM}/lstk-win32-x64/package.json")" +LAST_OUTPUT="$(files_field "${NPM}/lstk-windows-amd-64-v-1/package.json")" assert_output_contains "bundled-extensions.exe" assert_output_contains "lstk-extensions.toml" @@ -85,7 +101,7 @@ begin_test "npm would actually pack the registered files" setup_workspace run_script "${ADD}" "${NPM}" "${BUNDLED}" assert_ok -run_script npm pack --dry-run "${NPM}/lstk-darwin-arm64" +run_script npm pack --dry-run "${NPM}/lstk-darwin-arm-64-v-8-0" assert_ok assert_output_contains "bundled-extensions" assert_output_contains "lstk-extensions.toml" @@ -106,7 +122,7 @@ setup_workspace rm -rf "${BUNDLED}/windows_arm64" run_script "${ADD}" "${NPM}" "${BUNDLED}" assert_fails -assert_output_contains "lstk-win32-arm64" +assert_output_contains "lstk-windows-arm-64-v-8-0" assert_output_contains "windows_arm64" begin_test "fails when the toml is missing" @@ -121,7 +137,7 @@ setup_workspace run_script "${ADD}" "${NPM}" "${BUNDLED}" run_script "${ADD}" "${NPM}" "${BUNDLED}" assert_ok -count="$(files_field "${NPM}/lstk-linux-x64/package.json" | tr ' ' '\n' | grep -c '^bundled-extensions$' || true)" +count="$(files_field "${NPM}/lstk-linux-amd-64-v-1/package.json" | tr ' ' '\n' | grep -c '^bundled-extensions$' || true)" [ "${count}" -eq 1 ] || fail "expected one bundled-extensions entry, got ${count}" begin_test "fails when there are no platform packages at all" From cd20329e300aed879fb4909f085c4c3e1d667e66 Mon Sep 17 00:00:00 2001 From: Carlos Arilla Date: Wed, 2 Sep 2026 13:33:12 +0200 Subject: [PATCH 4/5] Drop bundled-extension symlinks and ask the binary for its commands Co-Authored-By: Claude --- .github/workflows/ci.yml | 10 +- .goreleaser.yaml | 10 +- docs/extensions-authoring.md | 14 +- docs/extensions-bundling.md | 184 ++++++++++++++---- scripts/add-bundled-to-npm.sh | 4 +- scripts/check-descriptions.sh | 85 +++++--- scripts/fetch-bundled-extensions.sh | 85 ++------ scripts/tests/check-descriptions_test.sh | 118 ++++++----- .../tests/fetch-bundled-extensions_test.sh | 102 +++++----- scripts/tests/lib.sh | 11 -- test/integration/extension_bundle_test.go | 15 +- 11 files changed, 370 insertions(+), 268 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d932b4ae..6a19aaf5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -357,11 +357,11 @@ jobs: LSTK_EXTENSIONS_REPO: localstack/lstk-bundled-extensions # A command described in lstk-extensions.toml that the bundle does not - # provide (per bundle-commands.txt, which the fetch step records from the - # archives' lstk- aliases) would show in help and fail when run; a - # binary with no command list would be unreachable. Either fails the - # release. Descriptions and the list are the same on every platform, so - # one platform dir is enough. + # provide would show in help and fail when run; a binary whose commands + # are all undescribed would be unreachable. Either fails the release. The + # bundle's side of the comparison comes from asking it — the gate runs + # `bundled-extensions list` — so it has to be pointed at the runner's own + # platform directory. - name: Check descriptions match the bundled binary run: scripts/check-descriptions.sh bundled/linux_amd64 diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 19f70a20..cccde895 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -50,11 +50,11 @@ archives: # runs first; a local snapshot build needs it too (see # docs/extensions-bundling.md). A glob matching nothing fails the build. # - # The whole platform directory is taken, not just the binary, so the - # lstk- alias symlinks staged beside it (Unix only) ride along — - # goreleaser preserves symlinks in tar.gz. They are a convenience for - # running an extension straight from a shell; lstk never resolves them. - - src: "bundled/{{ .Os }}_{{ .Arch }}/*" + # The glob is the binary alone (bare on Unix, .exe on Windows). Nothing + # else in the staged platform directory is shipped: lstk finds bundled + # commands through lstk-extensions.toml and dispatches by argv[0], so a + # per-command file at the archive root would serve no purpose. + - src: "bundled/{{ .Os }}_{{ .Arch }}/bundled-extensions*" strip_parent: true info: mode: 0o755 diff --git a/docs/extensions-authoring.md b/docs/extensions-authoring.md index ac8f2aa2..3706d20b 100644 --- a/docs/extensions-authoring.md +++ b/docs/extensions-authoring.md @@ -124,10 +124,16 @@ beside it; a name that file does not list is never handed to the bundle. Read being asked to be. The value is exactly `lstk-`, with no path and no `.exe`, and lstk only ever hands the bundle a name the toml lists, so a lookup miss inside the binary means the toml and the binary disagree: report it -loudly rather than guessing. Where the install channel allows it a release also -places an `lstk-` symlink to the binary, so the same command can be run -straight from a shell; that is a convenience only, and lstk never resolves it. See [extensions-bundling.md](extensions-bundling.md) for how -the bundle is built and shipped. +loudly rather than guessing. + +The binary is the only copy on disk — there is no `lstk-` file per +command — so it is reachable only as `lstk `. It nonetheless sits in the +install directory next to `lstk`, where somebody will eventually find it and +run it directly. Handle that: when `LSTK_EXT_API_VERSION` is unset there is no +runtime context to work from, so print a short message saying the binary is +part of lstk and naming the command to use instead, then exit non-zero. Do not +fall back to defaults. See [extensions-bundling.md](extensions-bundling.md) +for how the bundle is built and shipped. ## Help descriptions diff --git a/docs/extensions-bundling.md b/docs/extensions-bundling.md index f3140949..145e446b 100644 --- a/docs/extensions-bundling.md +++ b/docs/extensions-bundling.md @@ -33,6 +33,25 @@ trade is that the toml must be present and correct whenever the binary is, which the release gate below enforces and the runtime treats as a broken install if violated. +## How `lstk ` is resolved + +In order, first match wins: + +1. **A built-in command or alias** (`lstk start`, `lstk aws`, …). Built-ins + always win, so a bundle can never shadow one. +2. **The bundle.** If `bundled-extensions` sits next to lstk and + `lstk-extensions.toml` lists ``, lstk runs that binary with `argv[0]` + set to `lstk-`. +3. **A standalone `lstk-` executable**, first next to lstk, then on + `PATH` — the third-party extension path, unchanged by bundling. + +If nothing matches, `lstk ` reports an unknown command exactly as it did +before bundling existed. The bundle is consulted only through the toml, so a +name the toml does not list is indistinguishable from a typo — which is the +intent. The one case that is not "unknown command" is a bundle that is +installed but whose toml cannot be read; see +[Diagnosing a broken install](#diagnosing-a-broken-install). + ## Where the files live on disk lstk looks in one place: the directory of its own symlink-resolved executable @@ -43,7 +62,7 @@ without any layout work of its own. | --- | --- | --- | | Binary archive (`curl` + `tar`) | Wherever the user extracted the archive; the files sit at the archive root next to `lstk`. | GoReleaser `archives.files` entries in `.goreleaser.yaml`. | | Homebrew | The cask's Caskroom staged directory, e.g. `/opt/homebrew/Caskroom/lstk//`. `bin/lstk` is a symlink into it; lstk resolves the link. | The cask stages the whole archive. Only `lstk` is symlinked into `bin`; the bundle is found via the directory, never via `PATH`. The post-install hook strips the macOS quarantine attribute from the **whole** staged directory so the bundle runs without a Gatekeeper prompt. | -| npm | The **platform** package, e.g. `node_modules/@localstack/lstk-darwin-arm64/`, not the `@localstack/lstk` wrapper. The launcher execs the Go binary from there, so that is where lstk's bundled dir resolves to. | `scripts/add-bundled-to-npm.sh` copies the files into each `dist/npm/lstk--/` directory before `npm publish`, translating Node's platform names (`win32` → `windows`, `x64` → `amd64`), **and** adds them to that package's `files` allowlist. The publisher generates `"files": []`, which npm reads as "only `package.json` and the `bin` entry", so a plain copy would be silently dropped at publish. | +| npm | The **platform** package, e.g. `node_modules/@localstack/lstk_darwin_arm64/` (underscores; the wrapper `@localstack/lstk` holds only the launcher). The launcher execs the Go binary from there, so that is where lstk's bundled dir resolves to. | `scripts/add-bundled-to-npm.sh` copies the files into each platform package under `dist/npm/` before `npm publish`, **and** adds them to that package's `files` allowlist. The publisher generates `"files": []`, which npm reads as "only `package.json` and the `bin` entry", so a plain copy would be silently dropped at publish. Its output directories are slugified (`dist/npm/lstk-darwin-arm-64-v-8-0`) and cannot be parsed back into a platform, so the script reads the authoritative `@localstack/lstk__` name from each `package.json` instead. | ## The release pipeline @@ -63,25 +82,28 @@ order. Every step failing fails the release. archive and stages only two members: the binary as `bundled/_/bundled-extensions[.exe]` (mode 0755) and the descriptions file as `bundled/lstk-extensions.toml` (taken once; every - archive must carry an identical copy). It re-creates one `lstk-` - symlink to the binary per command, except on Windows (see "Running an - extension directly" below), and records the names, sorted, in - `bundled/bundle-commands.txt`: they are the bundle's own declaration of - which commands the binary answers to. An archive with no aliases, or whose - list differs from another platform's, aborts. It fails if any of lstk's six target platforms + archive must carry an identical copy, which is what lets the next step + verify one file rather than six). Nothing else is staged, and nothing else + reaches a released package. It fails if any of lstk's six target platforms (`linux`/`darwin`/`windows` × `amd64`/`arm64`) has no archive. A platform can be exempted only by listing it in `UNSUPPORTED_PLATFORMS` at the top of the script, so a gap is always a visible choice. 3. **Gate the pairing.** `scripts/check-descriptions.sh bundled/linux_amd64` reads the command names from the toml (left-hand side only; values are - never parsed) and compares them with the binary's own list in - `bundle-commands.txt`. It fails if commands are described but there is no - binary, if there is a binary but no commands are described, if the command - list is missing or empty, or if the toml describes a command the bundle - does not provide (lstk would exec the binary under a name it does not - answer to). A command the bundle provides but the toml omits only warns: - lstk will not expose it. Descriptions and the list are the same on every - platform, so one directory is enough. + never parsed) and compares them with the bundle's own answer: it runs + `bundled-extensions list`, which prints one bare command name per line. It + fails if commands are described but there is no binary, if there is a + binary but no commands are described, if `list` fails or prints nothing or + prints something that is not a command name, or if the toml describes a + command the bundle does not provide (lstk would exec the binary under a + name it does not answer to). A command the bundle provides but the toml + omits only warns: lstk will not expose it. + + Because it runs the binary, the gate can only be pointed at a platform + directory the host can execute — `linux_amd64` on the release runner — so + only that platform's binary is interrogated. A bundle whose platforms + disagree with each other is outside what this can see; the identical-toml + check in step 2 is what makes one directory a fair sample. 4. **Package.** GoReleaser adds the staged files to each archive at the root. The cask inherits them. `scripts/add-bundled-to-npm.sh` copies them into the platform packages and registers them in each package's `files`. @@ -116,17 +138,30 @@ Each tagged release of `localstack/lstk-bundled-extensions` ships: - one archive per lstk target platform, named `bundled-extensions___.tar.gz` (`.zip` for Windows), containing at its root the multi-call binary `bundled-extensions` (`.exe` on - Windows), `lstk-extensions.toml`, and one `lstk-` alias entry (a - symlink on Unix, a copy on Windows) for every command the binary answers - to. They serve two purposes: they are the binary's own statement of its - command list, which the release gate checks the toml against, and they are - re-created in the packaged install so a command can be run directly (see - below); + Windows) and `lstk-extensions.toml`. Anything else in the archive is ignored; + only those two files are staged and shipped; +- a `list` subcommand on the binary: `bundled-extensions list` prints the + commands it provides, one bare name per line and nothing else (each matching + `^[A-Za-z0-9][A-Za-z0-9_-]*$`), exiting zero. This is the binary's own + statement of what it answers to, and the release gate checks the toml + against it. A different output shape fails the release rather than being + parsed loosely, so that a change here surfaces as a red build and not as a + silently shorter command list; - the same `lstk-extensions.toml` in every archive, hand-authored, describing every command the binary provides — a described command with no implementation would show in `lstk --help` and fail when run; - `checksums.txt` with a SHA-256 line for every archive. +The binary must also **fail helpfully when it is run outside lstk**. lstk sets +`LSTK_EXT_API_VERSION` and `LSTK_EXT_CONTEXT` before executing it; run from a +shell, neither is present and the binary has no config directory, no auth +token and no emulator list to work with. `bundled-extensions` sits in the +install directory next to `lstk`, so someone will eventually find it and run +it. When `LSTK_EXT_API_VERSION` is unset it should print a short message +saying it is part of lstk and naming the command to use instead (for example +`lstk doctor`), and exit non-zero — not crash, and not half-run against +defaults. + The extensions team owns the descriptions text. lstk only validates that the file and the binary agree. @@ -152,6 +187,13 @@ every platform, plus a toml describing one placeholder `doctor` command, and prints a banner: **artifacts built from a stub bundle must never be released.** +It exists because the `bundled/` entries in `.goreleaser.yaml` are live: with +an empty staging tree `goreleaser` fails outright, so without `--stub` nobody +without a private-repo credential could run a snapshot build or work on the +packaging at all. It makes the packaging path runnable — it is not a way to +test the extensions, and the real bundle is what the release-candidate +checklist below exercises. + `scripts/check-bundled-packaging-sync.sh` runs on every PR next to `goreleaser check`. It fails if `.goreleaser.yaml` references `bundled/` while the release job has no fetch step, or the reverse. `goreleaser check` cannot @@ -160,6 +202,80 @@ the filesystem. Both halves must land in the same PR. The bash suites for all four scripts run with `make test-scripts`. +## Testing a release candidate before publishing + +A snapshot build produces the same artifacts the real release publishes, so all +three channels can be installed and exercised before anything reaches GitHub, +Homebrew or npm. Stage a bundle and build once: + +```bash +LSTK_EXTENSIONS_READ_TOKEN= scripts/fetch-bundled-extensions.sh +goreleaser release --snapshot --clean +``` + +Use the real bundle here, not `--stub`: the stub only proves the packaging path +runs, and says nothing about the extensions themselves. + +### Binary archive + +The archives in `dist/` are the ones the release uploads, so there is nothing +further to do. + +```bash +mkdir -p /tmp/lstk-rc && tar xzf dist/lstk__darwin_arm64.tar.gz -C /tmp/lstk-rc +/tmp/lstk-rc/lstk doctor +``` + +### npm + +Run the three steps the release job runs, in order: + +```bash +npx --yes goreleaser-npm-publisher@1.5.0 build --project . --prefix @localstack \ + --license Apache-2.0 \ + --description "LocalStack CLI v2 - Start and manage LocalStack emulators" \ + --files README.md LICENSE +cp npm/launcher.js dist/npm/lstk/index.js +scripts/add-bundled-to-npm.sh dist/npm bundled +``` + +Then **pack the packages before installing them**. `ls dist/npm/` and pick the +slug for your own platform: + +```bash +npm pack --pack-destination /tmp/lstk-tgz \ + ./dist/npm/lstk-darwin-arm-64-v-8-0 ./dist/npm/lstk +mkdir -p /tmp/lstk-npm && cd /tmp/lstk-npm +npm install --no-save /tmp/lstk-tgz/*.tgz +./node_modules/.bin/lstk doctor +``` + +Installing the directories directly (`npm install ./dist/npm/lstk`) does not +work: npm symlinks a local directory, so the launcher's `__dirname` resolves +outside the install tree and it reports `no prebuilt binary found for +`. That is an artifact of installing from a path, not a packaging +fault — after `npm pack` the layout is identical to a registry install. + +### Homebrew cask + +GoReleaser writes the cask it would push to the tap at +`dist/homebrew/Casks/lstk.rb`, with `sha256` values already matching the local +archives. Two changes make it installable: point the urls at the local files, +and put it in a tap, because Homebrew refuses a loose `.rb` path. + +```bash +sed "s#https://github.com/localstack/lstk/releases/download/v[^/]*/#file://${PWD}/dist/#" \ + dist/homebrew/Casks/lstk.rb > /tmp/lstk.rb +brew tap-new localstack/rc-test +cp /tmp/lstk.rb "$(brew --repository)/Library/Taps/localstack/homebrew-rc-test/Casks/" +brew install --cask localstack/rc-test/lstk +``` + +Undo with `brew uninstall --cask localstack/rc-test/lstk` followed by +`brew untap localstack/rc-test`. This is the only way to check the two cask +properties that matter before release: that `lstk` alone is symlinked into +`bin`, and that a bundled command runs with no Gatekeeper prompt. + ## Updates **Homebrew and npm** replace the whole package directory on `lstk update`, so @@ -173,25 +289,6 @@ it lands, the in-the-field updater replaces only `lstk`; the other two files must be extracted from the archive by hand. Updating never deletes standalone `lstk-` files a user placed next to the binary. -## Running an extension directly - -Where the channel allows it, a release also carries one `lstk-` symlink -to `bundled-extensions` per command, so `lstk-doctor` can be run straight from -a shell. That is a convenience for trying an extension on its own; lstk never -resolves those links. It dispatches to the binary by `argv[0]` and takes its -command list from the toml, so a channel without them behaves identically. - -| Channel | Aliases | Why | -| --- | --- | --- | -| Binary archive (tar.gz) | Yes | goreleaser preserves symlinks in tar.gz. | -| Homebrew cask | Yes | It stages the same archive. | -| Windows (zip) | No | Most Windows extractors turn a zip symlink into a small text file holding the target's name, which is worse than absent. | -| npm | No | `npm pack` silently drops symlinks from the published tarball. | - -`lstk update` on the binary channel does not currently re-create them: its -extractor skips symlink entries. Whatever a fresh install put there is left -alone, so an updated install keeps the links it already had. - ## Diagnosing a broken install If `bundled-extensions` is present but `lstk-extensions.toml` is missing, @@ -207,8 +304,11 @@ and an inconsistency in it is a release bug, not a per-entry condition. ## Release-candidate checklist -Run on the first bundling release and after any packaging change. On each -channel: +Run on the first bundling release and after any packaging change. Do it against +a real release candidate before publishing, using the locally built artifacts +from [Testing a release candidate before +publishing](#testing-a-release-candidate-before-publishing); the fresh-install +commands below are the published equivalents. On each channel: - Fresh install (`curl` + `tar`; `brew install localstack/tap/lstk`; `npm install -g @localstack/lstk`), then `lstk ` runs immediately and diff --git a/scripts/add-bundled-to-npm.sh b/scripts/add-bundled-to-npm.sh index e7737693..4936fb53 100755 --- a/scripts/add-bundled-to-npm.sh +++ b/scripts/add-bundled-to-npm.sh @@ -3,7 +3,7 @@ # Adds the bundled extensions to every npm PLATFORM package. # # The npm wrapper (@localstack/lstk) only holds the launcher; the real Go -# binary lives in the platform package (@localstack/lstk--) and the +# binary lives in the platform package (@localstack/lstk__) and the # launcher execs it from there. lstk resolves its bundled-extensions directory # from its own executable's location, so that platform directory is where # `bundled-extensions` and `lstk-extensions.toml` must live. @@ -98,5 +98,5 @@ for dir in "${NPM_DIR}"/*/; do count=$((count + 1)) done -[ "${count}" -gt 0 ] || die "no platform packages found under ${NPM_DIR} (expected lstk-- directories)" +[ "${count}" -gt 0 ] || die "no platform packages found under ${NPM_DIR} (expected @localstack/lstk__ packages)" echo "Bundled extensions added to ${count} platform package(s)." diff --git a/scripts/check-descriptions.sh b/scripts/check-descriptions.sh index 434e734c..9a78f81e 100755 --- a/scripts/check-descriptions.sh +++ b/scripts/check-descriptions.sh @@ -9,15 +9,17 @@ # execs the binary with argv[0] set to `lstk-`. That makes the file # load-bearing in both directions: a described name the binary does not answer # to is a command that shows in help and fails when run. The binary's side of -# the story is bundle-commands.txt, which scripts/fetch-bundled-extensions.sh -# records from the `lstk-` alias entries the bundle archives carry. +# the story comes from the binary itself: `bundled-extensions list` prints the +# commands it provides, one bare name per line. Asking it is the only +# authoritative answer — the toml is hand-written, and nothing else on disk +# records what the binary answers to. # # * commands described but no binary -> FAIL (help would list commands # that cannot run) # * binary present but nothing described -> FAIL (lstk could never reach it; # the runtime treats this as a # broken install) -# * binary present but no command list -> FAIL (nothing to verify the +# * `list` fails or prints nothing -> FAIL (nothing to verify the # descriptions against) # * described but the bundle lacks it -> FAIL (lstk would exec the binary # under a name it does not answer to) @@ -28,22 +30,28 @@ # printed for the release log # # Only the left-hand names are read from the toml, never the description -# values, so no description string can break this check. Descriptions and the -# command list are identical on every platform, so the release runs this once -# against one platform directory. +# values, so no description string can break this check. +# +# The binary has to run, so this can only be pointed at a platform directory +# the host can execute — the release runner checks bundled/linux_amd64. That +# also means only that platform's binary is interrogated; a bundle whose +# platforms disagree with each other is not something this can see. The toml is +# identical in every archive (the fetch script insists on it), so one directory +# is the practical unit of verification. # # Usage: -# scripts/check-descriptions.sh [descriptions-file] [commands-file] +# scripts/check-descriptions.sh [descriptions-file] # # e.g. bundled/linux_amd64, as staged by # scripts/fetch-bundled-extensions.sh # [descriptions-file] defaults to /../lstk-extensions.toml -# [commands-file] defaults to /../bundle-commands.txt set -euo pipefail BUNDLED_BINARY="bundled-extensions" DESCRIPTIONS_FILE="lstk-extensions.toml" -COMMANDS_FILE="bundle-commands.txt" +# The subcommand the bundle answers with its command list. +LIST_COMMAND="list" +NAME_RULE='^[A-Za-z0-9][A-Za-z0-9_-]*$' die() { echo "check-descriptions: $*" >&2 @@ -51,14 +59,13 @@ die() { } usage() { - sed -n '2,42p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' >&2 + sed -n '2,47p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' >&2 exit 1 } -[ $# -ge 1 ] && [ $# -le 3 ] || usage +[ $# -ge 1 ] && [ $# -le 2 ] || usage PLATFORM_DIR="$1" TOML="${2:-${PLATFORM_DIR}/../${DESCRIPTIONS_FILE}}" -COMMANDS="${3:-${PLATFORM_DIR}/../${COMMANDS_FILE}}" [ -d "${PLATFORM_DIR}" ] || die "no such directory: ${PLATFORM_DIR}" @@ -84,7 +91,7 @@ if [ -f "${TOML}" ]; then key="${line%%=*}" key="$(echo "${key}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" [ -n "${key}" ] || continue - if echo "${key}" | grep -Eq '^[A-Za-z0-9][A-Za-z0-9_-]*$'; then + if echo "${key}" | grep -Eq "${NAME_RULE}"; then names="${names}${key} " else @@ -97,21 +104,15 @@ fi if [ -n "${invalid}" ]; then echo "check-descriptions: ${TOML} contains invalid command names:" >&2 printf '%s' "${invalid}" >&2 - die "command names must match ^[A-Za-z0-9][A-Za-z0-9_-]*$" + die "command names must match ${NAME_RULE}" fi -# lstk- entries beside the binary are either the bundle's own aliases -# (symlinks to it, staged so the command can also be run straight from a shell) -# or a genuinely standalone extension. The first are expected and silent; the -# second still works — lstk resolves it from its directory — but carries no +# The staging tree holds the bundle binary and nothing else per platform, so an +# lstk- file here is a genuinely standalone extension somebody added. It +# still works — lstk resolves it from its directory — but carries no # description, so it is flagged rather than failed. for stray in "${PLATFORM_DIR}"/lstk-*; do [ -e "${stray}" ] || [ -L "${stray}" ] || continue - if [ -L "${stray}" ]; then - case "$(readlink "${stray}")" in - "${BUNDLED_BINARY}"|"${BUNDLED_BINARY}.exe") continue ;; - esac - fi echo "Warning: standalone extension binary $(basename "${stray}") in ${PLATFORM_DIR} is not part of the bundle and will show name-only in help." done @@ -128,10 +129,42 @@ fi [ -x "${binary}" ] || die "${binary} is not executable" [ -f "${TOML}" ] || die "${binary} is present but ${TOML} is missing; lstk cannot know which commands the bundle provides without ${DESCRIPTIONS_FILE}" [ -n "${names}" ] || die "${binary} is present but ${TOML} describes no commands; the bundle would be unreachable" -[ -f "${COMMANDS}" ] || die "${binary} is present but its command list ${COMMANDS} is missing; scripts/fetch-bundled-extensions.sh records it from the bundle's lstk- alias entries" +# Ask the bundle which commands it provides. A non-zero exit is fatal: without +# its answer there is nothing to verify the descriptions against, and shipping +# an unverified pair is exactly what this gate exists to prevent. The most +# likely cause in practice is pointing this at a platform directory the host +# cannot execute (a windows_* dir on Linux, say). +if ! listed="$("${binary}" "${LIST_COMMAND}" 2>&1)"; then + echo "check-descriptions: ${binary} ${LIST_COMMAND} failed:" >&2 + printf '%s\n' "${listed}" | sed 's/^/ /' >&2 + die "cannot read the bundle's command list; run this against a platform directory this machine can execute" +fi + +# One bare command name per line. Anything else is rejected rather than +# best-guess parsed: this list decides what lstk will dispatch, so a format +# change in the bundle must surface here and not as a silently shorter list. +provided="" +malformed="" +while IFS= read -r line; do + line="$(printf '%s' "${line}" | tr -d '\r' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + [ -n "${line}" ] || continue + if echo "${line}" | grep -Eq "${NAME_RULE}"; then + provided="${provided}${line} +" + else + malformed="${malformed} ${line} +" + fi +done <&2 + printf '%s' "${malformed}" >&2 + die "expected one bare command name per line, each matching ${NAME_RULE}" +fi +[ -n "${provided}" ] || die "${binary} ${LIST_COMMAND} printed no commands; the bundle declares nothing for lstk to dispatch" unprovided="" for name in ${names}; do diff --git a/scripts/fetch-bundled-extensions.sh b/scripts/fetch-bundled-extensions.sh index 7c363719..ee56f5a9 100755 --- a/scripts/fetch-bundled-extensions.sh +++ b/scripts/fetch-bundled-extensions.sh @@ -9,28 +9,22 @@ # # bundled/_/bundled-extensions[.exe] one per platform # bundled/lstk-extensions.toml os/arch-independent -# bundled/bundle-commands.txt os/arch-independent # # The private repository publishes one archive per platform, # `bundled-extensions___.tar.gz` (`.zip` for Windows), each # containing the multi-call binary `bundled-extensions[.exe]` and the # descriptions file `lstk-extensions.toml`, plus a `checksums.txt` covering the -# archives, and `lstk-` alias entries for every command the binary -# answers to (symlinks on Unix, copies on Windows). +# archives. # -# The binary and the toml are staged as they are. The aliases are re-created as -# relative symlinks next to the binary, so a packaged install can also run -# `lstk-doctor` straight from a shell — useful for testing an extension on its -# own. lstk itself never resolves them: it dispatches to the one binary by -# argv[0] and takes its command list from the toml, so a channel that cannot -# carry symlinks loses only that convenience. Windows is skipped deliberately: -# a zip symlink is re-created by most Windows extractors as a small text file -# holding the target's name, which is worse than absent. +# Only those two files are staged, and only they reach an lstk package: lstk +# dispatches to the one binary by argv[0] and takes its command list from the +# toml, so a per-command file on disk would serve no purpose on any channel. # -# The alias names are also recorded in bundle-commands.txt, because they are the -# bundle's own declaration of which commands it provides, and -# scripts/check-descriptions.sh verifies the toml against that list. An archive -# with no aliases is rejected for that reason. +# Which commands the binary actually provides is not this script's business — +# `bundled-extensions list` answers that, and scripts/check-descriptions.sh +# asks it. What this script does guarantee is that every archive carries an +# identical toml, so that answer can be checked against one file rather than +# six. # # Which bundle is taken comes from bundled/extensions.version — `latest` by # default. `latest` is resolved to a concrete tag exactly once here and @@ -68,9 +62,7 @@ UNSUPPORTED_PLATFORMS="${LSTK_UNSUPPORTED_PLATFORMS-}" REPO="${LSTK_EXTENSIONS_REPO:-localstack/lstk-bundled-extensions}" BUNDLED_BINARY="bundled-extensions" -NAME_PREFIX="lstk-" DESCRIPTIONS_FILE="lstk-extensions.toml" -COMMANDS_FILE="bundle-commands.txt" MANIFEST_FILE="checksums.txt" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -153,9 +145,9 @@ verify_checksums() { echo "Verified ${count} asset(s) against ${MANIFEST_FILE}." } -# Unpacks one archive into an empty directory. Symlinked alias entries in a -# tarball come out as symlinks; alias_names reads them by name only, and the -# regular-file lookups in stage_assets never pick them up. +# Unpacks one archive into an empty directory. Whatever else it holds is +# ignored: stage_assets looks up the binary and the toml by name and copies only +# those two. extract_archive() { local archive="$1" kind="$2" dest="$3" mkdir -p "${dest}" @@ -165,18 +157,9 @@ extract_archive() { esac } -# The command names an unpacked archive declares through its lstk- alias -# entries (any entry type: symlinks in tarballs, copies in zips), one per -# line, sorted. The descriptions file is excluded by name. -alias_names() { - find "$1" -mindepth 1 -maxdepth 1 -name "lstk-*" ! -name "${DESCRIPTIONS_FILE}" -exec basename {} \; \ - | sed -e "s/^lstk-//" -e "s/\.exe$//" | grep -v "^$" | sort -u || true -} - stage_assets() { local dir="$1" file base parsed os arch kind ext unpacked binary toml staged=0 local toml_staged="${BUNDLED_DIR}/${DESCRIPTIONS_FILE}" - local commands commands_staged="${BUNDLED_DIR}/${COMMANDS_FILE}" for file in "${dir}"/*; do base="$(basename "${file}")" [ "${base}" = "${MANIFEST_FILE}" ] && continue @@ -210,27 +193,6 @@ stage_assets() { else cp "${toml}" "${toml_staged}" fi - - # The alias entries are the bundle's own statement of which commands the - # binary answers to. They are not staged (lstk never needs them on disk) - # but their names are, so the descriptions gate can verify the toml - # against them. Like the toml, they must agree across platforms. - commands="$(alias_names "${unpacked}")" - [ -n "${commands}" ] || die "${base} carries no lstk- alias entries, so the bundle's command list cannot be verified against ${DESCRIPTIONS_FILE}" - if [ -f "${commands_staged}" ]; then - [ "${commands}" = "$(cat "${commands_staged}")" ] || die "the command list (lstk- alias entries) in ${base} differs from the one in an earlier archive of the same bundle" - else - printf "%s\n" "${commands}" > "${commands_staged}" - fi - - # Re-create the aliases rather than copying the archive's own entries: the - # target is written relative and bare so it resolves wherever the pair is - # unpacked, whatever the archive happened to contain. - if [ "${os}" != "windows" ]; then - for name in ${commands}; do - ln -sf "${BUNDLED_BINARY}" "${BUNDLED_DIR}/${os}_${arch}/${NAME_PREFIX}${name}" - done - fi done [ -f "${toml_staged}" ] || die "the bundle publishes no ${DESCRIPTIONS_FILE}" echo "Staged ${staged} platform binaries into ${BUNDLED_DIR}." @@ -280,13 +242,16 @@ write_stub_bundle() { case "${platform}" in windows_*) suffix=".exe" ;; esac mkdir -p "${BUNDLED_DIR}/${platform}" for name in ${binaries}; do - printf '#!/bin/sh\necho "stub %s for %s"\n' "${name}" "${platform}" \ - > "${BUNDLED_DIR}/${platform}/${name}${suffix}" - chmod 0755 "${BUNDLED_DIR}/${platform}/${name}${suffix}" - # Mirror the real layout so a snapshot build exercises the same shape. - if [ "${name}" = "${BUNDLED_BINARY}" ] && [ "${suffix}" = "" ]; then - ln -sf "${BUNDLED_BINARY}" "${BUNDLED_DIR}/${platform}/${NAME_PREFIX}doctor" + if [ "${name}" = "${BUNDLED_BINARY}" ]; then + # The stub answers `list` like the real bundle does, so the descriptions + # gate can be run against a stub tree. + printf '#!/bin/sh\nif [ "$1" = "list" ]; then echo "doctor"; exit 0; fi\necho "stub %s for %s"\n' \ + "${name}" "${platform}" > "${BUNDLED_DIR}/${platform}/${name}${suffix}" + else + printf '#!/bin/sh\necho "stub %s for %s"\n' "${name}" "${platform}" \ + > "${BUNDLED_DIR}/${platform}/${name}${suffix}" fi + chmod 0755 "${BUNDLED_DIR}/${platform}/${name}${suffix}" done done { @@ -306,14 +271,6 @@ write_stub_bundle() { esac done } > "${BUNDLED_DIR}/${DESCRIPTIONS_FILE}" - { - for name in ${binaries}; do - case "${name}" in - "${BUNDLED_BINARY}") echo "doctor" ;; - lstk-*) echo "${name#lstk-}" ;; - esac - done - } | sort -u > "${BUNDLED_DIR}/${COMMANDS_FILE}" cat >&2 <<'BANNER' diff --git a/scripts/tests/check-descriptions_test.sh b/scripts/tests/check-descriptions_test.sh index ce7973aa..8b16ea19 100644 --- a/scripts/tests/check-descriptions_test.sh +++ b/scripts/tests/check-descriptions_test.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash # Tests for scripts/check-descriptions.sh — the release gate that keeps the # descriptions file and the multi-call bundled binary in agreement. Fixtures are -# built in temp dirs mirroring the staging layout the fetch script produces: -# a platform dir holding the binary, and the toml plus the bundle's own command -# list (bundle-commands.txt) one level up. +# built in temp dirs mirroring the staging layout the fetch script produces: a +# platform dir holding the binary, and the toml one level up. The binary is a +# stub script answering `list`, which is where the gate gets the bundle's own +# command list. set -euo pipefail SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -19,6 +20,7 @@ setup_stage() { mkdir -p "${PLATFORM_DIR}" } +# A plain non-bundle file in the platform dir, e.g. a standalone extension. write_binary() { echo "fake" > "${PLATFORM_DIR}/${1:-bundled-extensions}" chmod 0755 "${PLATFORM_DIR}/${1:-bundled-extensions}" @@ -28,21 +30,33 @@ write_toml() { printf '%s' "$1" > "${STAGE}/lstk-extensions.toml" } -# The command list the fetch script records from the bundle's own lstk- -# alias entries: what the binary actually answers to. -write_commands() { - : > "${STAGE}/bundle-commands.txt" - for name in "$@"; do - echo "${name}" >> "${STAGE}/bundle-commands.txt" - done +# The bundle binary, answering `list` with the given command names — the +# bundle's own statement of what it provides. +write_bundle() { + { + echo '#!/bin/sh' + echo 'if [ "$1" = "list" ]; then' + for name in "$@"; do + echo " echo '${name}'" + done + echo ' exit 0' + echo 'fi' + echo 'echo "stub bundle"' + } > "${PLATFORM_DIR}/bundled-extensions" + chmod 0755 "${PLATFORM_DIR}/bundled-extensions" +} + +# A bundle binary whose `list` fails or prints something unusable. +write_broken_bundle() { + printf '#!/bin/sh\n%s\n' "$1" > "${PLATFORM_DIR}/bundled-extensions" + chmod 0755 "${PLATFORM_DIR}/bundled-extensions" } echo "== check-descriptions.sh ==" begin_test "binary, descriptions and command list agree: passes and lists the commands" setup_stage -write_binary -write_commands doctor deploy +write_bundle doctor deploy write_toml 'doctor = "Check the local setup" deploy = "Deploy to LocalStack" ' @@ -54,8 +68,7 @@ assert_output_lacks "Warning" begin_test "a described command the bundle does not provide fails, naming it" setup_stage -write_binary -write_commands doctor +write_bundle doctor write_toml 'doctor = "Check the local setup" deploy = "Deploy to LocalStack" ' @@ -66,8 +79,7 @@ assert_output_contains "does not provide" begin_test "a bundle command that is not described warns but passes" setup_stage -write_binary -write_commands doctor deploy +write_bundle doctor deploy write_toml 'doctor = "Check the local setup" ' run_script "${CHECK}" "${PLATFORM_DIR}" @@ -75,24 +87,44 @@ assert_ok assert_output_contains "Warning" assert_output_contains "deploy" -begin_test "a bundled binary with no command list fails, naming the file" +begin_test "a bundle whose list command fails is fatal, showing its own output" setup_stage -write_binary +write_broken_bundle 'echo "boom" >&2; exit 3' write_toml 'doctor = "Check the local setup" ' run_script "${CHECK}" "${PLATFORM_DIR}" assert_fails -assert_output_contains "bundle-commands.txt" +assert_output_contains "boom" +assert_output_contains "cannot read the bundle" -begin_test "a bundled binary with an empty command list fails" +begin_test "a bundle that lists nothing fails" setup_stage -write_binary -write_commands +write_bundle write_toml 'doctor = "Check the local setup" ' run_script "${CHECK}" "${PLATFORM_DIR}" assert_fails -assert_output_contains "bundle-commands.txt" +assert_output_contains "printed no commands" + +begin_test "a bundle listing something that is not a command name fails, naming it" +setup_stage +write_broken_bundle 'echo "doctor - Check the local setup"' +write_toml 'doctor = "Check the local setup" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_fails +assert_output_contains "doctor - Check the local setup" +assert_output_contains "one bare command name per line" + +begin_test "blank lines and surrounding whitespace in the list are tolerated" +setup_stage +write_broken_bundle 'printf "doctor \n\n deploy\n"' +write_toml 'doctor = "Check the local setup" +deploy = "Deploy to LocalStack" +' +run_script "${CHECK}" "${PLATFORM_DIR}" +assert_ok +assert_output_lacks "Warning" begin_test "described commands with no bundled binary fail, naming them" setup_stage @@ -107,16 +139,14 @@ assert_output_contains "deploy" begin_test "a bundled binary with no descriptions file fails" setup_stage -write_binary -write_commands doctor +write_bundle doctor run_script "${CHECK}" "${PLATFORM_DIR}" assert_fails assert_output_contains "lstk-extensions.toml" begin_test "a bundled binary with an empty descriptions file fails" setup_stage -write_binary -write_commands doctor +write_bundle doctor write_toml '# nothing described yet ' run_script "${CHECK}" "${PLATFORM_DIR}" @@ -134,20 +164,22 @@ write_toml '' run_script "${CHECK}" "${PLATFORM_DIR}" assert_ok -begin_test "the Windows binary name is accepted" +begin_test "the Windows binary name is recognised as the bundle" setup_stage write_binary bundled-extensions.exe -write_commands doctor write_toml 'doctor = "Check the local setup" ' run_script "${CHECK}" "${PLATFORM_DIR}" -assert_ok +# Recognised, then run — which is as far as this gets on a host that cannot +# execute it. The release checks a platform directory it can run. +assert_fails +assert_output_contains "bundled-extensions.exe" +assert_output_contains "a platform directory this machine can execute" begin_test "a non-executable binary fails" setup_stage -echo "fake" > "${PLATFORM_DIR}/bundled-extensions" +write_bundle doctor chmod 0644 "${PLATFORM_DIR}/bundled-extensions" -write_commands doctor write_toml 'doctor = "Check the local setup" ' run_script "${CHECK}" "${PLATFORM_DIR}" @@ -156,8 +188,7 @@ assert_output_contains "executable" begin_test "only the left-hand names are read, never the values" setup_stage -write_binary -write_commands doctor +write_bundle doctor # A hostile description: quotes, an equals sign, a fake key on the same line. write_toml 'doctor = "a = b \"quoted\" evil = \"x\"" ' @@ -168,19 +199,16 @@ assert_output_lacks "evil" begin_test "an invalid command name fails" setup_stage -write_binary -write_commands doctor +write_bundle doctor write_toml 'doc tor = "spaces are not a command" ' run_script "${CHECK}" "${PLATFORM_DIR}" assert_fails assert_output_contains "doc tor" -begin_test "alias symlinks to the bundle are expected and do not warn" +begin_test "a bundle staged on its own passes with no warnings" setup_stage -write_binary -( cd "${PLATFORM_DIR}" && ln -s bundled-extensions lstk-doctor ) -write_commands doctor +write_bundle doctor write_toml 'doctor = "Check the local setup" ' run_script "${CHECK}" "${PLATFORM_DIR}" @@ -189,9 +217,8 @@ assert_output_lacks "Warning" begin_test "a stray standalone lstk- binary warns but passes" setup_stage -write_binary write_binary lstk-legacy -write_commands doctor +write_bundle doctor write_toml 'doctor = "Check the local setup" ' run_script "${CHECK}" "${PLATFORM_DIR}" @@ -199,13 +226,12 @@ assert_ok assert_output_contains "Warning" assert_output_contains "lstk-legacy" -begin_test "explicit toml and command-list paths override the default locations" +begin_test "an explicit toml path overrides the default location" setup_stage -write_binary +write_bundle doctor OTHER_DIR="$(mktemp -d)" printf 'doctor = "x"\n' > "${OTHER_DIR}/other.toml" -printf 'doctor\n' > "${OTHER_DIR}/other-commands.txt" -run_script "${CHECK}" "${PLATFORM_DIR}" "${OTHER_DIR}/other.toml" "${OTHER_DIR}/other-commands.txt" +run_script "${CHECK}" "${PLATFORM_DIR}" "${OTHER_DIR}/other.toml" assert_ok assert_output_contains "doctor" diff --git a/scripts/tests/fetch-bundled-extensions_test.sh b/scripts/tests/fetch-bundled-extensions_test.sh index 5a2ca42a..f90e3300 100755 --- a/scripts/tests/fetch-bundled-extensions_test.sh +++ b/scripts/tests/fetch-bundled-extensions_test.sh @@ -24,11 +24,27 @@ sha256_of() { fi } +# Writes a stand-in bundle binary that answers `list` with COMMANDS (default +# "doctor"), the way the real one does. +write_fake_bundle() { + { + echo '#!/bin/sh' + echo 'if [ "$1" = "list" ]; then' + for name in ${COMMANDS-doctor}; do + echo " echo '${name}'" + done + echo ' exit 0' + echo 'fi' + echo "echo 'fake bundle binary for $2'" + } > "$1" + chmod 0755 "$1" +} + # Builds a fixture release for the given tag: per platform, a tar.gz (zip for -# Windows) containing bundled-extensions[.exe], lstk-extensions.toml and an -# lstk- alias per entry in ALIASES (default "doctor"; a symlink in the -# tarballs, a copy in the zips — exactly what the private repo's goreleaser -# emits), plus a checksums.txt over the archives. TOML_BODY overrides the +# Windows) containing bundled-extensions[.exe] and lstk-extensions.toml, plus a +# checksums.txt over the archives. ALIASES adds lstk- entries the way the +# private repo used to emit them (a symlink in the tarballs, a copy in the +# zips), so the tests can prove they are ignored. TOML_BODY overrides the # descriptions file for every platform. make_release_assets() { local dir="$1" tag="${2:-v1.4.0}" @@ -41,14 +57,13 @@ make_release_assets() { printf '%s' "${toml_body}" > "${work}/lstk-extensions.toml" case "${platform}" in windows_*) - echo "fake bundle binary for ${platform}" > "${work}/bundled-extensions.exe" - for alias in ${ALIASES-doctor}; do cp "${work}/bundled-extensions.exe" "${work}/lstk-${alias}.exe"; done + write_fake_bundle "${work}/bundled-extensions.exe" "${platform}" + for alias in ${ALIASES-}; do cp "${work}/bundled-extensions.exe" "${work}/lstk-${alias}.exe"; done ( cd "${work}" && zip -q -r "${dir}/bundled-extensions_${tag}_${platform}.zip" . ) ;; *) - echo "fake bundle binary for ${platform}" > "${work}/bundled-extensions" - chmod 0755 "${work}/bundled-extensions" - for alias in ${ALIASES-doctor}; do ( cd "${work}" && ln -s bundled-extensions "lstk-${alias}" ); done + write_fake_bundle "${work}/bundled-extensions" "${platform}" + for alias in ${ALIASES-}; do ( cd "${work}" && ln -s bundled-extensions "lstk-${alias}" ); done ( cd "${work}" && tar czf "${dir}/bundled-extensions_${tag}_${platform}.tar.gz" . ) ;; esac @@ -218,31 +233,29 @@ assert_file_contains "${BUNDLED}/lstk-extensions.toml" "doctor" assert_file_absent "${BUNDLED}/linux_amd64/lstk-extensions.toml" assert_file_absent "${BUNDLED}/linux_amd64/checksums.txt" -begin_test "alias entries are staged as symlinks beside the binary on unix" +begin_test "the archive's alias entries are never staged, on any platform" setup_workspace run_script "${FETCH}" assert_ok for platform in linux_amd64 linux_arm64 darwin_amd64 darwin_arm64; do - assert_symlink_to "${BUNDLED}/${platform}/lstk-doctor" "bundled-extensions" + assert_file_absent "${BUNDLED}/${platform}/lstk-doctor" + assert_executable "${BUNDLED}/${platform}/bundled-extensions" done - -begin_test "alias entries are skipped on Windows, where extraction would junk them" -setup_workspace -run_script "${FETCH}" -assert_ok assert_file_absent "${BUNDLED}/windows_amd64/lstk-doctor.exe" assert_file_absent "${BUNDLED}/windows_arm64/lstk-doctor.exe" assert_executable "${BUNDLED}/windows_amd64/bundled-extensions.exe" -begin_test "every described command gets an alias" +begin_test "a multi-command bundle still stages only the binary and the toml" setup_workspace -ALIASES="doctor deploy" TOML_BODY='doctor = "x" +COMMANDS="doctor deploy" TOML_BODY='doctor = "x" deploy = "y" ' make_release_assets "${ASSETS}" run_script "${FETCH}" assert_ok -assert_symlink_to "${BUNDLED}/linux_amd64/lstk-doctor" "bundled-extensions" -assert_symlink_to "${BUNDLED}/linux_amd64/lstk-deploy" "bundled-extensions" +assert_executable "${BUNDLED}/linux_amd64/bundled-extensions" +assert_file_exists "${BUNDLED}/lstk-extensions.toml" +assert_file_absent "${BUNDLED}/linux_amd64/lstk-doctor" +assert_file_absent "${BUNDLED}/linux_amd64/lstk-deploy" begin_test "the staged tree passes the descriptions gate" setup_workspace @@ -353,55 +366,34 @@ assert_ok assert_file_absent "${BUNDLED}/linux_amd64/lstk-removed" assert_file_exists "${BUNDLED}/extensions.version" -begin_test "records the bundle's own command list from its alias entries, sorted" +begin_test "an archive still carrying lstk- entries stages neither them nor a command list" setup_workspace -ALIASES="doctor deploy" TOML_BODY='doctor = "x" -deploy = "y" -' make_release_assets "${ASSETS}" +ALIASES="doctor deploy" make_release_assets "${ASSETS}" run_script "${FETCH}" assert_ok -assert_file_exists "${BUNDLED}/bundle-commands.txt" -run_script cat "${BUNDLED}/bundle-commands.txt" -[ "${LAST_OUTPUT}" = "deploy -doctor" ] || fail "expected the sorted alias names, got: ${LAST_OUTPUT}" - -begin_test "alias entries differing between platforms abort the fetch" -setup_workspace -work="$(mktemp -d)" -echo "bin" > "${work}/bundled-extensions" -chmod 0755 "${work}/bundled-extensions" -printf 'doctor = "Fake doctor description"\n' > "${work}/lstk-extensions.toml" -( cd "${work}" && ln -s bundled-extensions lstk-doctor && ln -s bundled-extensions lstk-extra ) -( cd "${work}" && tar czf "${ASSETS}/bundled-extensions_v1.4.0_linux_arm64.tar.gz" . ) -refresh_manifest "${ASSETS}" -run_script "${FETCH}" -assert_fails -assert_output_contains "linux_arm64" -assert_output_contains "command list" +assert_file_absent "${BUNDLED}/linux_amd64/lstk-doctor" +assert_file_absent "${BUNDLED}/linux_amd64/lstk-deploy" +assert_file_absent "${BUNDLED}/bundle-commands.txt" -begin_test "an archive with no alias entries aborts the fetch, naming the archive" +begin_test "an archive with no lstk- entries is fine; the binary declares its own commands" setup_workspace work="$(mktemp -d)" -echo "bin" > "${work}/bundled-extensions" -chmod 0755 "${work}/bundled-extensions" +write_fake_bundle "${work}/bundled-extensions" darwin_amd64 printf 'doctor = "Fake doctor description"\n' > "${work}/lstk-extensions.toml" ( cd "${work}" && tar czf "${ASSETS}/bundled-extensions_v1.4.0_darwin_amd64.tar.gz" . ) refresh_manifest "${ASSETS}" run_script "${FETCH}" -assert_fails -assert_output_contains "bundled-extensions_v1.4.0_darwin_amd64.tar.gz" -assert_output_contains "alias" - -begin_test "the command list is not packaged next to the binaries" -setup_workspace -run_script "${FETCH}" assert_ok -assert_file_absent "${BUNDLED}/linux_amd64/bundle-commands.txt" +assert_executable "${BUNDLED}/darwin_amd64/bundled-extensions" -begin_test "--stub records a command list matching its descriptions" +begin_test "--stub produces a bundle that answers list and passes the gate" setup_workspace run_script "${FETCH}" --stub assert_ok -assert_file_contains "${BUNDLED}/bundle-commands.txt" "doctor" +run_script "${BUNDLED}/linux_amd64/bundled-extensions" list +assert_ok +assert_output_contains "doctor" +run_script "${SUITE_DIR}/../check-descriptions.sh" "${BUNDLED}/linux_amd64" +assert_ok finish_suite diff --git a/scripts/tests/lib.sh b/scripts/tests/lib.sh index b46dc71c..9c19ff5f 100644 --- a/scripts/tests/lib.sh +++ b/scripts/tests/lib.sh @@ -94,14 +94,3 @@ finish_suite() { echo "${TESTS_RUN}/${TESTS_RUN} test(s) passed in $(basename "$0")" } -# Asserts that path is a symlink whose target is exactly want. Relative targets -# are compared verbatim: an absolute or ../-prefixed target would not survive -# packaging, so the exact string is the thing under test. -assert_symlink_to() { - if [ ! -L "$1" ]; then - fail "expected a symlink at: $1" - return - fi - got="$(readlink "$1")" - [ "${got}" = "$2" ] || fail "expected $1 -> $2, got -> ${got}" -} diff --git a/test/integration/extension_bundle_test.go b/test/integration/extension_bundle_test.go index 86953d4b..cc8a657b 100644 --- a/test/integration/extension_bundle_test.go +++ b/test/integration/extension_bundle_test.go @@ -133,17 +133,16 @@ func TestBundledMultiCallContextConveyed(t *testing.T) { require.Contains(t, stdout, "API_VERSION=1") } -// Releases stage an lstk- symlink next to the bundle for every command it -// provides, so a user can also run `lstk-doctor` straight from a shell. lstk -// must stay indifferent to them: it dispatches through the bundle by argv[0] -// and takes its command list from the descriptions file, so an alias must not -// produce a second help entry, and removing one must change nothing. This is -// what lets channels that cannot carry symlinks (npm, Windows zip) ship without -// them and behave identically. +// A release ships the bundle binary and the descriptions file and nothing else, +// but a user can still drop an lstk- link to the binary next to it, and +// installs made before aliases were dropped will have some. lstk must be +// indifferent to them: it takes its command list from the descriptions file and +// dispatches by argv[0], so such a link must not add a second help entry or +// change what runs. func TestBundledAliasSymlinksAreInertForLstk(t *testing.T) { t.Parallel() if runtime.GOOS == "windows" { - t.Skip("aliases are deliberately not staged on Windows") + t.Skip("os.Symlink needs Developer Mode or elevation on Windows") } bundleDir := t.TempDir() lstkBin := installLstkBundle(t, bundleDir) From aac6d59fb25ed0fa1bd4feee59c6b5c97c68f405 Mon Sep 17 00:00:00 2001 From: Carlos Arilla Date: Thu, 3 Sep 2026 11:53:30 +0200 Subject: [PATCH 5/5] Move the bundled-extension scripts into their own directory and other minor doc fixes --- .github/workflows/ci.yml | 8 ++-- .gitignore | 7 +-- .goreleaser.yaml | 7 +-- CLAUDE.md | 2 +- Makefile | 4 +- bundled/extensions.version | 26 +++++------ docs/extensions-bundling.md | 38 ++++++++-------- internal/validate/validate.go | 5 ++- .../design.md | 8 ++-- .../proposal.md | 4 +- .../tasks.md | 6 +-- .../add-bundled-to-npm.sh | 4 +- .../check-bundled-packaging-sync.sh | 4 +- .../check-descriptions.sh | 4 +- .../fetch-bundled-extensions.sh | 6 +-- .../{ => bundled-extensions}/test-scripts.sh | 3 +- .../tests/add-bundled-to-npm_test.sh | 2 +- .../check-bundled-packaging-sync_test.sh | 2 +- .../tests/check-descriptions_test.sh | 2 +- .../tests/fetch-bundled-extensions_test.sh | 2 +- scripts/{ => bundled-extensions}/tests/lib.sh | 0 .../__snapshots__/extension_bundle_test.snap | 43 ------------------- test/integration/extension_bundle_test.go | 33 ++++++++++++-- 23 files changed, 106 insertions(+), 114 deletions(-) rename scripts/{ => bundled-extensions}/add-bundled-to-npm.sh (96%) rename scripts/{ => bundled-extensions}/check-bundled-packaging-sync.sh (95%) rename scripts/{ => bundled-extensions}/check-descriptions.sh (97%) rename scripts/{ => bundled-extensions}/fetch-bundled-extensions.sh (98%) rename scripts/{ => bundled-extensions}/test-scripts.sh (80%) rename scripts/{ => bundled-extensions}/tests/add-bundled-to-npm_test.sh (99%) rename scripts/{ => bundled-extensions}/tests/check-bundled-packaging-sync_test.sh (98%) rename scripts/{ => bundled-extensions}/tests/check-descriptions_test.sh (99%) rename scripts/{ => bundled-extensions}/tests/fetch-bundled-extensions_test.sh (99%) rename scripts/{ => bundled-extensions}/tests/lib.sh (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a19aaf5..91f3ef2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,7 +126,7 @@ jobs: # packaging bundled extensions and downloading them have to land # together. This does, and fails the PR instead of the release. - name: Check bundled packaging is in step with the download - run: scripts/check-bundled-packaging-sync.sh + run: scripts/bundled-extensions/check-bundled-packaging-sync.sh - name: Test release scripts run: make test-scripts @@ -345,7 +345,7 @@ jobs: echo "Release ${GITHUB_REF_NAME} already records bundle ${recorded}; pinning to it." export LSTK_EXTENSIONS_TAG="${recorded}" fi - scripts/fetch-bundled-extensions.sh | tee fetch.log + scripts/bundled-extensions/fetch-bundled-extensions.sh | tee fetch.log tag="$(sed -n 's/^Resolved extensions bundle: \([^ ]*\) .*/\1/p' fetch.log)" [ -n "${tag}" ] || { echo "could not determine the resolved bundle tag"; exit 1; } commit="$(GH_TOKEN="${LSTK_EXTENSIONS_READ_TOKEN}" gh api "repos/${LSTK_EXTENSIONS_REPO}/commits/${tag}" --jq .sha)" @@ -363,7 +363,7 @@ jobs: # `bundled-extensions list` — so it has to be pointed at the runner's own # platform directory. - name: Check descriptions match the bundled binary - run: scripts/check-descriptions.sh bundled/linux_amd64 + run: scripts/bundled-extensions/check-descriptions.sh bundled/linux_amd64 - name: Run GoReleaser uses: goreleaser/goreleaser-action@v7 @@ -408,7 +408,7 @@ jobs: # package's `files` allowlist: the publisher emits "files": [], which npm # packs as package.json + bin only, so a bare copy would be dropped. - name: Add bundled extensions to the npm platform packages - run: scripts/add-bundled-to-npm.sh dist/npm bundled + run: scripts/bundled-extensions/add-bundled-to-npm.sh dist/npm bundled - name: Publish to NPM run: | diff --git a/.gitignore b/.gitignore index b49bf4f4..2a1dad9f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,9 +14,10 @@ test/integration/test-samples/**/*.tfstate test/integration/test-samples/**/*.tfstate.* # Bundled-extension staging tree, populated by -# scripts/fetch-bundled-extensions.sh at release-build time: per-platform -# binaries in bundled/_/ and bundled/lstk-extensions.toml. Only the -# version file is tracked; downloaded artifacts must never be committed. +# scripts/bundled-extensions/fetch-bundled-extensions.sh at release-build +# time: per-platform binaries in bundled/_/ and +# bundled/lstk-extensions.toml. Only the version file is tracked; downloaded +# artifacts must never be committed. # It deliberately lives outside dist/, which `goreleaser --clean` wipes. /bundled/* !/bundled/extensions.version diff --git a/.goreleaser.yaml b/.goreleaser.yaml index cccde895..d34870fc 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -46,9 +46,10 @@ archives: - manpages/* # LocalStack's bundled extensions: one multi-call binary per platform plus # the descriptions file, at the archive root next to lstk. Staged under - # bundled/ by scripts/fetch-bundled-extensions.sh, which the release job - # runs first; a local snapshot build needs it too (see - # docs/extensions-bundling.md). A glob matching nothing fails the build. + # bundled/ by scripts/bundled-extensions/fetch-bundled-extensions.sh, + # which the release job runs first; a local snapshot build needs it too + # (see docs/extensions-bundling.md). A glob matching nothing fails the + # build. # # The glob is the binary alone (bare on Unix, .exe on Windows). Nothing # else in the staged platform directory is shipped: lstk finds bundled diff --git a/CLAUDE.md b/CLAUDE.md index 5f4c61b6..cd333a0d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -170,7 +170,7 @@ Shared plumbing lives in `cmd/proxy.go` (`leadingFlags`, `stripLeadingProxyFlags # Extensions -lstk supports Git-style extensions: when `lstk ` is not a built-in command or alias, lstk resolves and execs an external `lstk-` executable, forwarding arguments verbatim and propagating the exit code. Built-ins always win. Resolution order is built-ins → bundled dir (the directory of the symlink-resolved lstk executable) → `PATH`; there is no manifest. Runtime context is conveyed via `LSTK_EXT_API_VERSION` and `LSTK_EXT_CONTEXT` (JSON: `configDir`, optional `authToken`, `nonInteractive`, `json`, optional `sessionId` — lstk's telemetry session id, omitted when telemetry is disabled, so an extension's own telemetry can join lstk's `ext:` event — optional `machineId` — lstk's anonymized machine id (the prepared hash), omitted alongside `sessionId` when telemetry is disabled, so an extension reports the same machine without re-deriving it — optional `endpointUrl` — the resolved `--endpoint-url`/`LSTK_ENDPOINT_URL`/`AWS_ENDPOINT_URL` value, conveyed verbatim and unvalidated (dispatch never rejects or probes it, unlike the built-ins' `rejectEndpointURL`), omitted when no source is set — and an `emulators` array, which stays local-Docker discovery and is independent of `endpointUrl`) — see `extension.Context`/`Environ` in `internal/extension/context.go`; dispatch and help listing are in `cmd/extension.go`. LocalStack's own bundled extensions ship as **one multi-call binary**, `bundled-extensions` (`extension.BundledBinaryName`), next to `lstk`, plus the hand-authored `lstk-extensions.toml`; that file is load-bearing for the bundle — it is the only record of which commands the binary provides — so `extension.LoadBundle` (`internal/extension/bundle.go`) hard-fails on a missing/malformed/empty one when the binary is present, whereas `LoadDescriptions` (help text only) still degrades quietly. `Resolver.Resolve` consults the bundle first for described names and execs it with `argv[0]` = `lstk-` (`Extension.Argv0`, applied in `Invoke`), then standalone `lstk-` files in the bundled dir, then `PATH`; a broken bundle is reported by `Resolve` only when nothing else provides the name, and skipped (logged) by `List` so help never breaks. `LoadBundle` also rejects a toml key that is not a dispatchable name (`validate.ExtensionName`, the same rule the release gate applies), and `Resolver.List` attaches each bundled entry's help `Description` so `cmd` renders from one parse of the file. The argv[0] contract is uniform: a standalone `lstk-` file is invoked under its own base name, which is the same `lstk-` a bundle-provided command receives. Distribution is automated in the release job: `scripts/fetch-bundled-extensions.sh` downloads and checksum-verifies the bundle selected by `bundled/extensions.version` from the private extensions repo, `scripts/check-descriptions.sh` gates the toml against the binary's own command list (`bundled/bundle-commands.txt`, which the fetch script records from the `lstk-` alias entries every bundle archive must carry, and which is never packaged), so a described command the binary cannot dispatch fails the release; the fetch script also re-creates those aliases as relative symlinks beside the staged binary (not on Windows, where extractors materialize them as junk text files) so a packaged install can run `lstk-` directly — goreleaser preserves symlinks in tar.gz, `npm pack` drops them, and lstk itself never resolves them, `.goreleaser.yaml` packages both at the archive root (the cask inherits them; a release-job step copies them into each npm **platform** package), and the resolved bundle tag is recorded in the release notes. `scripts/check-bundled-packaging-sync.sh` runs on every PR to keep the packaging and download halves from merging separately. Bash tests for these scripts: `make test-scripts` (`scripts/tests/`). Set-wise co-update on the binary channel (`internal/update`) is still pending in the `add-bundled-extension-distribution` change. See [extensions-authoring.md](docs/extensions-authoring.md) for the author-facing contract and [extensions-bundling.md](docs/extensions-bundling.md) for the release pipeline and on-disk layout per channel. +lstk supports Git-style extensions: when `lstk ` is not a built-in command or alias, lstk resolves and execs an external `lstk-` executable, forwarding arguments verbatim and propagating the exit code. Built-ins always win. Resolution order is built-ins → bundled dir (the directory of the symlink-resolved lstk executable) → `PATH`; there is no manifest. Runtime context is conveyed via `LSTK_EXT_API_VERSION` and `LSTK_EXT_CONTEXT` (JSON: `configDir`, optional `authToken`, `nonInteractive`, `json`, optional `sessionId` — lstk's telemetry session id, omitted when telemetry is disabled, so an extension's own telemetry can join lstk's `ext:` event — optional `machineId` — lstk's anonymized machine id (the prepared hash), omitted alongside `sessionId` when telemetry is disabled, so an extension reports the same machine without re-deriving it — optional `endpointUrl` — the resolved `--endpoint-url`/`LSTK_ENDPOINT_URL`/`AWS_ENDPOINT_URL` value, conveyed verbatim and unvalidated (dispatch never rejects or probes it, unlike the built-ins' `rejectEndpointURL`), omitted when no source is set — and an `emulators` array, which stays local-Docker discovery and is independent of `endpointUrl`) — see `extension.Context`/`Environ` in `internal/extension/context.go`; dispatch and help listing are in `cmd/extension.go`. LocalStack's own bundled extensions ship as **one multi-call binary**, `bundled-extensions`, next to `lstk`, plus an `lstk-extensions.toml` listing the commands it provides; lstk execs it with `argv[0]` = `lstk-`. The invariants are on `extension.BundledBinaryName`, `LoadBundle` and `Extension.Argv0`; the release pipeline (helper scripts in `scripts/bundled-extensions/`, `make test-scripts`) is in [extensions-bundling.md](docs/extensions-bundling.md) and the author-facing contract in [extensions-authoring.md](docs/extensions-authoring.md). Set-wise co-update (`internal/update`) is still pending in the `add-bundled-extension-distribution` change. # Signal Forwarding to Wrapped Tools diff --git a/Makefile b/Makefile index 748638c5..14f08b8d 100644 --- a/Makefile +++ b/Makefile @@ -23,10 +23,10 @@ test: test-integration: build @RUN="$(RUN)" ./scripts/test-integration.sh -# Bash suites for the release helper scripts under scripts/. They only ever run +# Bash suites for the bundled-extension release helper scripts. They only ever run # on the Linux release runner, so a bash suite is the faithful test here. test-scripts: - @./scripts/test-scripts.sh + @./scripts/bundled-extensions/test-scripts.sh otel: docker compose -f docker-compose.tracing.yaml up -d diff --git a/bundled/extensions.version b/bundled/extensions.version index a2f1ab78..632eba74 100644 --- a/bundled/extensions.version +++ b/bundled/extensions.version @@ -2,19 +2,21 @@ # # One value line (blank lines and #-comments are ignored), in one of two forms: # -# latest Take the newest published release of the private extensions -# repository. This is the default: there is no routine bump to -# remember, and a release can never go silently stale. The release -# job resolves it to a concrete tag ONCE and records that tag in -# the published release notes, so an lstk version still maps to -# exactly one bundle. +# latest Take the newest published release of the private extensions +# repository. This is the default: there is no routine bump to +# remember, and a release can never go silently stale. The +# release job resolves it to a concrete tag ONCE and records +# that tag in the published release notes, so an lstk version +# still maps to exactly one bundle. # -# v0.1.0 Lock this build to that exact release tag of the private -# extensions repository. Use it to hold a bad bundle back. To -# re-run an already-published lstk release against the bundle it -# originally shipped, do not edit this file — pass that release's -# recorded tag to scripts/fetch-bundled-extensions.sh instead -# (--tag / LSTK_EXTENSIONS_TAG). +# v2026.08.19 Lock this build to that exact release tag of the private +# extensions repository (that repository tags by date, not +# semver). Use it to hold a bad bundle back. To re-run an +# already-published lstk release against the bundle it +# originally shipped, do not edit this file — pass that +# release's recorded tag to +# scripts/bundled-extensions/fetch-bundled-extensions.sh instead +# (--tag / LSTK_EXTENSIONS_TAG). # # This is the only tracked file under bundled/. The downloaded binaries and # descriptions file are staged alongside it and are gitignored. diff --git a/docs/extensions-bundling.md b/docs/extensions-bundling.md index 145e446b..2e5d6f2f 100644 --- a/docs/extensions-bundling.md +++ b/docs/extensions-bundling.md @@ -62,7 +62,7 @@ without any layout work of its own. | --- | --- | --- | | Binary archive (`curl` + `tar`) | Wherever the user extracted the archive; the files sit at the archive root next to `lstk`. | GoReleaser `archives.files` entries in `.goreleaser.yaml`. | | Homebrew | The cask's Caskroom staged directory, e.g. `/opt/homebrew/Caskroom/lstk//`. `bin/lstk` is a symlink into it; lstk resolves the link. | The cask stages the whole archive. Only `lstk` is symlinked into `bin`; the bundle is found via the directory, never via `PATH`. The post-install hook strips the macOS quarantine attribute from the **whole** staged directory so the bundle runs without a Gatekeeper prompt. | -| npm | The **platform** package, e.g. `node_modules/@localstack/lstk_darwin_arm64/` (underscores; the wrapper `@localstack/lstk` holds only the launcher). The launcher execs the Go binary from there, so that is where lstk's bundled dir resolves to. | `scripts/add-bundled-to-npm.sh` copies the files into each platform package under `dist/npm/` before `npm publish`, **and** adds them to that package's `files` allowlist. The publisher generates `"files": []`, which npm reads as "only `package.json` and the `bin` entry", so a plain copy would be silently dropped at publish. Its output directories are slugified (`dist/npm/lstk-darwin-arm-64-v-8-0`) and cannot be parsed back into a platform, so the script reads the authoritative `@localstack/lstk__` name from each `package.json` instead. | +| npm | The **platform** package, e.g. `node_modules/@localstack/lstk_darwin_arm64/` (underscores; the wrapper `@localstack/lstk` holds only the launcher). The launcher execs the Go binary from there, so that is where lstk's bundled dir resolves to. | `scripts/bundled-extensions/add-bundled-to-npm.sh` copies the files into each platform package under `dist/npm/` before `npm publish`, **and** adds them to that package's `files` allowlist. The publisher generates `"files": []`, which npm reads as "only `package.json` and the `bin` entry", so a plain copy would be silently dropped at publish. Its output directories are slugified (`dist/npm/lstk-darwin-arm-64-v-8-0`) and cannot be parsed back into a platform, so the script reads the authoritative `@localstack/lstk__` name from each `package.json` instead. | ## The release pipeline @@ -72,10 +72,11 @@ order. Every step failing fails the release. 1. **Select the bundle.** `bundled/extensions.version` (the only tracked file under `bundled/`) says which release of the private extensions repository to take. It says `latest` by default: the newest published bundle, with no - routine bump to remember. Set it to an explicit tag (`v0.3.1`) to hold a + routine bump to remember. Set it to an explicit tag (`v2026.08.19`) to hold a build to one bundle. -2. **Fetch and verify.** `scripts/fetch-bundled-extensions.sh` resolves - `latest` to a concrete tag **once**, prints it, downloads that tag's release +2. **Fetch and verify.** + `scripts/bundled-extensions/fetch-bundled-extensions.sh` resolves `latest` + to a concrete tag **once**, prints it, downloads that tag's release assets with `gh release download`, and verifies every asset against the `checksums.txt` published in the same release. A missing manifest, an unlisted asset or a mismatching hash aborts. It then unpacks each platform @@ -88,7 +89,8 @@ order. Every step failing fails the release. (`linux`/`darwin`/`windows` × `amd64`/`arm64`) has no archive. A platform can be exempted only by listing it in `UNSUPPORTED_PLATFORMS` at the top of the script, so a gap is always a visible choice. -3. **Gate the pairing.** `scripts/check-descriptions.sh bundled/linux_amd64` +3. **Gate the pairing.** + `scripts/bundled-extensions/check-descriptions.sh bundled/linux_amd64` reads the command names from the toml (left-hand side only; values are never parsed) and compares them with the bundle's own answer: it runs `bundled-extensions list`, which prints one bare command name per line. It @@ -105,8 +107,9 @@ order. Every step failing fails the release. disagree with each other is outside what this can see; the identical-toml check in step 2 is what makes one directory a fair sample. 4. **Package.** GoReleaser adds the staged files to each archive at the root. - The cask inherits them. `scripts/add-bundled-to-npm.sh` copies them into - the platform packages and registers them in each package's `files`. + The cask inherits them. `scripts/bundled-extensions/add-bundled-to-npm.sh` + copies them into the platform packages and registers them in each + package's `files`. 5. **Record.** After publishing, the job appends `Bundled extensions: (commit )` to the GitHub release notes. Job logs expire; release notes do not. This line is how you answer "which @@ -128,7 +131,7 @@ notes already carry a `Bundled extensions:` line, pins the fetch to that tag. To do the same by hand, pass the recorded tag explicitly: ```bash -LSTK_EXTENSIONS_READ_TOKEN=... scripts/fetch-bundled-extensions.sh --tag v0.3.1 +LSTK_EXTENSIONS_READ_TOKEN=... scripts/bundled-extensions/fetch-bundled-extensions.sh --tag v2026.08.19 ``` ## What the private repository must publish @@ -171,14 +174,14 @@ Since the `bundled/` entries in `.goreleaser.yaml` are live, `goreleaser` fails on an empty staging tree. Stage a bundle first, either for real: ```bash -LSTK_EXTENSIONS_READ_TOKEN= scripts/fetch-bundled-extensions.sh +LSTK_EXTENSIONS_READ_TOKEN= scripts/bundled-extensions/fetch-bundled-extensions.sh goreleaser release --snapshot --clean ``` or, without access to the private repository, with placeholders: ```bash -scripts/fetch-bundled-extensions.sh --stub +scripts/bundled-extensions/fetch-bundled-extensions.sh --stub goreleaser release --snapshot --clean ``` @@ -194,11 +197,12 @@ packaging at all. It makes the packaging path runnable — it is not a way to test the extensions, and the real bundle is what the release-candidate checklist below exercises. -`scripts/check-bundled-packaging-sync.sh` runs on every PR next to -`goreleaser check`. It fails if `.goreleaser.yaml` references `bundled/` while -the release job has no fetch step, or the reverse. `goreleaser check` cannot -catch this itself because it only validates config syntax and never looks at -the filesystem. Both halves must land in the same PR. +`scripts/bundled-extensions/check-bundled-packaging-sync.sh` runs on every PR +next to `goreleaser check`. It fails if `.goreleaser.yaml` references +`bundled/` while the release job has no fetch step, or the reverse. +`goreleaser check` cannot catch this itself because it only validates config +syntax and never looks at the filesystem. Both halves must land in the same +PR. The bash suites for all four scripts run with `make test-scripts`. @@ -209,7 +213,7 @@ three channels can be installed and exercised before anything reaches GitHub, Homebrew or npm. Stage a bundle and build once: ```bash -LSTK_EXTENSIONS_READ_TOKEN= scripts/fetch-bundled-extensions.sh +LSTK_EXTENSIONS_READ_TOKEN= scripts/bundled-extensions/fetch-bundled-extensions.sh goreleaser release --snapshot --clean ``` @@ -236,7 +240,7 @@ npx --yes goreleaser-npm-publisher@1.5.0 build --project . --prefix @localstack --description "LocalStack CLI v2 - Start and manage LocalStack emulators" \ --files README.md LICENSE cp npm/launcher.js dist/npm/lstk/index.js -scripts/add-bundled-to-npm.sh dist/npm bundled +scripts/bundled-extensions/add-bundled-to-npm.sh dist/npm bundled ``` Then **pack the packages before installing them**. `ls dist/npm/` and pick the diff --git a/internal/validate/validate.go b/internal/validate/validate.go index 954a982f..d782845c 100644 --- a/internal/validate/validate.go +++ b/internal/validate/validate.go @@ -221,8 +221,9 @@ func AuthToken(value string) error { // multi-call binary argv[0] "lstk-". The first character must be a letter // or digit so a name can never read as a flag, and the rest is limited to // letters, digits, hyphens, and underscores. It is the same rule the release -// gate (scripts/check-descriptions.sh) applies to lstk-extensions.toml keys, so -// a descriptions file that passes the gate always loads and vice versa. +// gate (scripts/bundled-extensions/check-descriptions.sh) applies to +// lstk-extensions.toml keys, so a descriptions file that passes the gate always +// loads and vice versa. var extensionNameRegexp = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]*$`) // ExtensionName validates an extension command name taken from the bundled diff --git a/openspec/changes/add-bundled-extension-distribution/design.md b/openspec/changes/add-bundled-extension-distribution/design.md index e3b97030..5c67a853 100644 --- a/openspec/changes/add-bundled-extension-distribution/design.md +++ b/openspec/changes/add-bundled-extension-distribution/design.md @@ -27,7 +27,7 @@ A single release-time staging tree is the source for every channel: `bundled/__[.exe]`), `lstk-extensions.toml`, and a `checksums.txt` manifest covering them. -This repo carries a **version file**, `bundled/extensions.version`, whose default value is `latest`: each lstk release takes whatever the newest published bundle is, so there is no routine bump to remember and no way to go silently stale. Setting it to an explicit release tag locks that build to one bundle — the escape hatch for holding a bad bundle back, or for rebuilding an older release with the bundle it originally shipped. `scripts/fetch-bundled-extensions.sh` reads the file, downloads the assets (`gh release download`), **verifies each against the bundle's `checksums.txt`**, and stages them under `bundled/` with canonical names. It hard-fails when any lstk target platform has no matching asset (subject to an explicit not-supported allowlist), so platform gaps surface at pull time, not as an empty-glob failure inside GoReleaser. +This repo carries a **version file**, `bundled/extensions.version`, whose default value is `latest`: each lstk release takes whatever the newest published bundle is, so there is no routine bump to remember and no way to go silently stale. Setting it to an explicit release tag locks that build to one bundle — the escape hatch for holding a bad bundle back, or for rebuilding an older release with the bundle it originally shipped. `scripts/bundled-extensions/fetch-bundled-extensions.sh` reads the file, downloads the assets (`gh release download`), **verifies each against the bundle's `checksums.txt`**, and stages them under `bundled/` with canonical names. It hard-fails when any lstk target platform has no matching asset (subject to an explicit not-supported allowlist), so platform gaps surface at pull time, not as an empty-glob failure inside GoReleaser. **`latest` is resolved once per release build, and the result is recorded permanently.** The release job resolves it to a concrete tag in its first step and every later step uses that tag, so a single build can never mix two bundles. The resolved tag and the bundle's commit hash are written into the **published release notes** — not only the job log, which GitHub expires — which is what makes "which extensions build does this customer have?" answerable from a version number alone. It is also what a re-run of an already-published tag is pointed at: `latest` re-resolves on every invocation, so without the recorded tag a re-run of `v0.5.2` would publish different extension binaries under a version already in the wild. @@ -39,7 +39,7 @@ The credential is a **dedicated fine-grained read-only PAT** (contents: read on ### Decision 3: Hand-authored descriptions file, release-validated by a shell script -The descriptions file (`lstk-extensions.toml`) is hand-authored in the private extensions repository and shipped as-is; the open-source repo never generates it. A release-time bash script, `scripts/check-descriptions.sh` (consistent with `scripts/test-integration.sh`), extracts the described command names — the bare left-hand identifiers of the flat `name = "…"` table (`^[[:space:]]*([A-Za-z0-9][A-Za-z0-9_-]*)[[:space:]]*=`); values are never parsed — and fails the release if any described name has no corresponding executable `lstk-` in the staged dir. A staged binary with no description warns but passes (help degrades to name-only, per the `extension-bundling` spec). +The descriptions file (`lstk-extensions.toml`) is hand-authored in the private extensions repository and shipped as-is; the open-source repo never generates it. A release-time bash script, `scripts/bundled-extensions/check-descriptions.sh` (consistent with `scripts/test-integration.sh`), extracts the described command names — the bare left-hand identifiers of the flat `name = "…"` table (`^[[:space:]]*([A-Za-z0-9][A-Za-z0-9_-]*)[[:space:]]*=`); values are never parsed — and fails the release if any described name has no corresponding executable `lstk-` in the staged dir. A staged binary with no description warns but passes (help degrades to name-only, per the `extension-bundling` spec). **Validation targets a single, host-native staging dir** — descriptions are os/arch-independent, so the check runs once against the release runner's own platform staging dir (`linux_amd64`), where binaries are bare `lstk-` with no `.exe`/PATHEXT ambiguity. @@ -79,13 +79,13 @@ The transition release (the first that ships bundled extensions) must be reachab ### Decision 6: Release gating and local builds -The `archives.files` entries land commented until the private pull is wired, then pull + payload are enabled **in one PR**: the PR-level `goreleaser check` job only validates config syntax (globs are not evaluated), but `goreleaser release/build` fails on a glob with zero matches, so the entries must never be live without the staging step that populates `bundled/`. After enabling, local snapshot builds require `scripts/fetch-bundled-extensions.sh` first; its `--stub` mode stages placeholder files for local, never-published builds so contributors without the private-repo credential can still run `goreleaser` locally. The staging tree is gitignored (only the pin file is tracked); it deliberately does not live under `dist/`, which `goreleaser --clean` wipes at startup. +The `archives.files` entries land commented until the private pull is wired, then pull + payload are enabled **in one PR**: the PR-level `goreleaser check` job only validates config syntax (globs are not evaluated), but `goreleaser release/build` fails on a glob with zero matches, so the entries must never be live without the staging step that populates `bundled/`. After enabling, local snapshot builds require `scripts/bundled-extensions/fetch-bundled-extensions.sh` first; its `--stub` mode stages placeholder files for local, never-published builds so contributors without the private-repo credential can still run `goreleaser` locally. The staging tree is gitignored (only the pin file is tracked); it deliberately does not live under `dist/`, which `goreleaser --clean` wipes at startup. The first bundling release ships with the smallest viable bundle (a single extension) and is verified against the release-candidate checklist in `docs/extensions-bundling.md` — fresh install and upgrade-from-previous on all three channels — before further extensions are added to the bundle. ### Decision 7: Bundled binary layout — RESOLVED: (b), one multi-call binary -**Resolution (2026-08-27, DPX-692):** option (b). The bundle ships as a single binary named `bundled-extensions` next to `lstk`, plus `lstk-extensions.toml`. lstk takes the bundled command list from the descriptions file and execs the one binary with `argv[0]` set to `lstk-` (`Extension.Argv0`, honoured in `extension.Invoke`). The runtime changes this implies are in `internal/extension/bundle.go` (`LoadBundle`, `BundledBinaryName`) and the bundle branch of `Resolver.Resolve`/`List`: the bundle is consulted first for described names, then standalone `lstk-` files in the bundled dir (manual placement keeps working), then PATH. When the binary is present, a missing/unreadable/empty descriptions file is a hard error surfaced by `Resolve` when nothing else provides the name — never a silent "unknown command" — while `List` logs and skips it so help never breaks. Release-side, `scripts/check-descriptions.sh` enforces the same pairing (described-but-no-binary and binary-but-nothing-described both fail), and `.goreleaser.yaml` packages `bundled-extensions*` + the toml. Since the binary cannot be executed cross-platform at release time, its command list is taken from the `lstk-` alias entries the bundle archives carry: the fetch script records them as `bundled/bundle-commands.txt` (never packaged) and the gate fails on a described name that list lacks, which makes the aliases a required part of the private repo's release convention rather than an optional convenience. The same name rule (`validate.ExtensionName`) is applied by `LoadBundle`, so a toml that passes the gate always loads. The fetch script unpacks the private repo's actual release convention — one `bundled-extensions___.tar.gz`/`.zip` per platform containing the binary, the toml and `lstk-` alias entries — and stages the binary, the toml, and re-created relative alias symlinks (Unix only), recording the alias names for the gate. The aliases are a convenience for running a command directly, never a resolution path: verified empirically, goreleaser preserves symlinks in tar.gz (so the binary channel and the cask carry them) while `npm pack` drops them and a Windows zip symlink is re-created by most extractors as a text file holding the target name, so those two channels ship without them and behave identically. The section-1 updater work and the section-6 test plan should be read with "the set" = `lstk`, `bundled-extensions`, `lstk-extensions.toml`, and "complete" = the binary present alongside a loadable descriptions file. The original analysis follows. +**Resolution (2026-08-27, DPX-692):** option (b). The bundle ships as a single binary named `bundled-extensions` next to `lstk`, plus `lstk-extensions.toml`. lstk takes the bundled command list from the descriptions file and execs the one binary with `argv[0]` set to `lstk-` (`Extension.Argv0`, honoured in `extension.Invoke`). The runtime changes this implies are in `internal/extension/bundle.go` (`LoadBundle`, `BundledBinaryName`) and the bundle branch of `Resolver.Resolve`/`List`: the bundle is consulted first for described names, then standalone `lstk-` files in the bundled dir (manual placement keeps working), then PATH. When the binary is present, a missing/unreadable/empty descriptions file is a hard error surfaced by `Resolve` when nothing else provides the name — never a silent "unknown command" — while `List` logs and skips it so help never breaks. Release-side, `scripts/bundled-extensions/check-descriptions.sh` enforces the same pairing (described-but-no-binary and binary-but-nothing-described both fail), and `.goreleaser.yaml` packages `bundled-extensions*` + the toml. Since the binary cannot be executed cross-platform at release time, its command list is taken from the `lstk-` alias entries the bundle archives carry: the fetch script records them as `bundled/bundle-commands.txt` (never packaged) and the gate fails on a described name that list lacks, which makes the aliases a required part of the private repo's release convention rather than an optional convenience. The same name rule (`validate.ExtensionName`) is applied by `LoadBundle`, so a toml that passes the gate always loads. The fetch script unpacks the private repo's actual release convention — one `bundled-extensions___.tar.gz`/`.zip` per platform containing the binary, the toml and `lstk-` alias entries — and stages the binary, the toml, and re-created relative alias symlinks (Unix only), recording the alias names for the gate. The aliases are a convenience for running a command directly, never a resolution path: verified empirically, goreleaser preserves symlinks in tar.gz (so the binary channel and the cask carry them) while `npm pack` drops them and a Windows zip symlink is re-created by most extractors as a text file holding the target name, so those two channels ship without them and behave identically. The section-1 updater work and the section-6 test plan should be read with "the set" = `lstk`, `bundled-extensions`, `lstk-extensions.toml`, and "complete" = the binary present alongside a loadable descriptions file. The original analysis follows. **What this blocks, and what it does not.** Decision 7 gates section 5 of `tasks.md` (turning packaging on) and the parts of the test plan that name individual on-disk files, because both have to know what the payload looks like. It does not gate sections 1 to 3: the set-wise updater, the descriptions check and the fetch script are all written against "whatever the archive contains" and can be built and merged first. Leaving it open therefore holds nothing up, and the input it is waiting for (whether the bundle really is one binary, and how big it is) arrives naturally once the doctor extension exists. diff --git a/openspec/changes/add-bundled-extension-distribution/proposal.md b/openspec/changes/add-bundled-extension-distribution/proposal.md index c89b35e0..f9bb8f26 100644 --- a/openspec/changes/add-bundled-extension-distribution/proposal.md +++ b/openspec/changes/add-bundled-extension-distribution/proposal.md @@ -11,7 +11,7 @@ The runtime half already exists and is not touched by this change: `extension.Bu - Homebrew: automatic via the cask's Caskroom staging of the whole archive (lstk ships as a **cask**, not a formula — no libexec involved); the cask's post-install quarantine hook is widened from the single `lstk` binary to the whole staged directory; - npm: bundled files are copied into each **platform package** (`@localstack/lstk--`), where the real binary lives — not the wrapper package — via a post-processing step in the release job. - **Pull the prebuilt closed-source bundled binaries from the private extensions repository's releases** into the release build context, **selected by a version file in this repo** (`bundled/extensions.version` — `latest` by default, settable to an explicit tag to lock a build down), resolved to a concrete tag once per build and recorded in the published release notes, checksum-verified against the private release's manifest, authenticated with a dedicated read-only token, without exposing source in the public repository. -- **Ship the hand-authored descriptions file** (`lstk-extensions.toml`), owned by LocalStack's private extensions repository, and **validate it at release time** (`scripts/check-descriptions.sh`) so a described-but-missing extension is a release-blocking error. +- **Ship the hand-authored descriptions file** (`lstk-extensions.toml`), owned by LocalStack's private extensions repository, and **validate it at release time** (`scripts/bundled-extensions/check-descriptions.sh`) so a described-but-missing extension is a release-blocking error. - **Update the `lstk`/`lstk-*` set as one unit** in `internal/update` for the self-managed binary channel (stage `.lstk-new` siblings, then rename, lstk last); Homebrew and npm replace the whole package — and therefore the whole set — via their package managers. - **Guarantee update continuity**: `lstk update` keeps working for every existing install across the transition — a pre-bundling lstk updates cleanly into the first bundling release on all three channels (Homebrew and npm especially, where the updater shells out to the package manager), and a bundling lstk updates cleanly from an archive that carries no extensions (rollback). An archive carrying no extensions is a valid archive and must not fail an update — but when an archive does carry them they are not optional: the update installs the complete set or fails, and never reports success with a partial one. @@ -27,7 +27,7 @@ The runtime half already exists and is not touched by this change: `extension.Bu ## Impact -- **Touched code**: `internal/update/extract.go` (+ re-introduced `extract_test.go`) — set-wise stage-then-commit replacement; `.goreleaser.yaml` — archive payload entries and the cask quarantine hook; `.github/workflows/ci.yml` release job — private pull step, descriptions validation step, npm platform-package copy step; new `scripts/fetch-bundled-extensions.sh` and `scripts/check-descriptions.sh`; new `bundled/extensions.version` version file (+ `.gitignore` entries for the staging dirs). +- **Touched code**: `internal/update/extract.go` (+ re-introduced `extract_test.go`) — set-wise stage-then-commit replacement; `.goreleaser.yaml` — archive payload entries and the cask quarantine hook; `.github/workflows/ci.yml` release job — private pull step, descriptions validation step, npm platform-package copy step; new `scripts/bundled-extensions/fetch-bundled-extensions.sh` and `scripts/bundled-extensions/check-descriptions.sh`; new `bundled/extensions.version` version file (+ `.gitignore` entries for the staging dirs). - **Packaging/release**: the binary archives gain `lstk-*` + `lstk-extensions.toml` at the root; the Homebrew cask inherits them via archive staging (hook widened); the npm platform packages gain them via post-processing; the release workflow pulls the resolved private release's binaries with a repository/organization secret (dedicated read-only PAT). - **Docs**: re-introduce `docs/extensions-bundling.md` (on-disk layout per channel, the release pipeline, how the bundle version is resolved and recorded, local snapshot builds, update semantics and guarantees, rollback); update the CLAUDE.md Extensions section. - **External dependencies/services**: the private extensions repository publishing tagged releases (per-platform binaries, `lstk-extensions.toml`, `checksums.txt`) and a release-time read-only credential to download them. diff --git a/openspec/changes/add-bundled-extension-distribution/tasks.md b/openspec/changes/add-bundled-extension-distribution/tasks.md index 1e582084..60e23678 100644 --- a/openspec/changes/add-bundled-extension-distribution/tasks.md +++ b/openspec/changes/add-bundled-extension-distribution/tasks.md @@ -16,7 +16,7 @@ Today `internal/update/extract.go` extracts the downloaded archive and replaces The descriptions file `lstk-extensions.toml` is a flat TOML table (`deploy = "One-line description"`), hand-written in the private extensions repo. If it describes an extension that we didn't actually ship a binary for, users would see help text for a command that doesn't work. This script makes that a release-blocking error. -- [x] 2.1 Re-introduce `scripts/check-descriptions.sh` (plain bash, same style as `scripts/test-integration.sh`). Input: a directory containing the downloaded extension binaries and the toml. Behavior: read the names on the left-hand side of each `name = "…"` line (only the names — never parse the values, so a weird description string can't break the script); for each name, check an executable file `lstk-` exists in that directory; if any is missing, print which ones and exit non-zero (this fails the release). The reverse case — a binary present but not described — only prints a warning, because lstk's help intentionally falls back to showing such extensions name-only. +- [x] 2.1 Re-introduce `scripts/bundled-extensions/check-descriptions.sh` (plain bash, same style as `scripts/test-integration.sh`). Input: a directory containing the downloaded extension binaries and the toml. Behavior: read the names on the left-hand side of each `name = "…"` line (only the names — never parse the values, so a weird description string can't break the script); for each name, check an executable file `lstk-` exists in that directory; if any is missing, print which ones and exit non-zero (this fails the release). The reverse case — a binary present but not described — only prints a warning, because lstk's help intentionally falls back to showing such extensions name-only. - [x] 2.2 Test the script against fixture directories (a small test script or make target creating temp dirs): described-but-missing binary → fails and names it; described-and-present → passes; binary-without-description → warns but passes; empty or absent toml → passes (nothing is described, nothing to check). - [x] 2.3 Verify the described names against the bundle's own command list, not just the binary's presence. The fetch script records the `lstk-` alias entries every archive carries into `bundled/bundle-commands.txt` (aborting on an archive with none, or on lists that differ between platforms), and `check-descriptions.sh` fails when the toml describes a command that list lacks and warns on the reverse. Covered in both bash suites; the list is never packaged. The fetch script also re-creates the aliases as relative symlinks beside the staged binary so a packaged install can run `lstk-` directly; Windows is skipped (extractors materialize zip symlinks as junk text files) and `npm pack` drops them, so those channels ship without and behave identically — lstk never resolves them. @@ -27,7 +27,7 @@ Note: the check runs once per release, against the Linux/amd64 download director The extension binaries are never committed to this repo. Instead, a one-line version file says which bundle to take — `latest` normally, so there is nothing to maintain, or an explicit tag when we deliberately want to hold a build to one bundle. A script resolves that to a concrete tag and downloads it at release-build time. Resolving happens once per build and the answer is recorded, so a release version always maps to exactly one bundle even if the release job is re-run (design Decision 2). - [x] 3.1 Add the version file `bundled/extensions.version` containing a single line: either `latest` (the default we ship) or an explicit release tag of the private extensions repo (e.g. `v0.1.0`). Document both forms in the file itself, since it is the only place someone will look. Add `.gitignore` rules so ONLY this file is tracked: the downloaded binaries land in `bundled/_/` folders and the toml at `bundled/lstk-extensions.toml`, and none of that may ever be committed. (Why the staging folder is `bundled/` at the repo root and not inside `dist/`: the release runs `goreleaser --clean`, which deletes `dist/` before building — it would wipe the downloads.) -- [x] 3.2 Add `scripts/fetch-bundled-extensions.sh`. What it does, in order: read the version file and, if it says `latest`, resolve it to the concrete tag of the newest published release and print the resolved tag (every later step uses the resolved tag, never `latest` again, so one build can't mix two bundles); accept an already-resolved tag via an env var or flag so a re-run of a published release can be pointed back at the bundle it originally shipped; download that tag's release assets from the private extensions repo with `gh release download` (repo name configurable via an env var, with a sensible default); verify every downloaded file against the `checksums.txt` that the private repo publishes in the same release — abort loudly if the manifest is missing or any hash doesn't match; then arrange the files into the layout the rest of the pipeline expects: binaries at `bundled/_/lstk-` (with `.exe` for Windows), executable bit set, and the descriptions file at `bundled/lstk-extensions.toml`. +- [x] 3.2 Add `scripts/bundled-extensions/fetch-bundled-extensions.sh`. What it does, in order: read the version file and, if it says `latest`, resolve it to the concrete tag of the newest published release and print the resolved tag (every later step uses the resolved tag, never `latest` again, so one build can't mix two bundles); accept an already-resolved tag via an env var or flag so a re-run of a published release can be pointed back at the bundle it originally shipped; download that tag's release assets from the private extensions repo with `gh release download` (repo name configurable via an env var, with a sensible default); verify every downloaded file against the `checksums.txt` that the private repo publishes in the same release — abort loudly if the manifest is missing or any hash doesn't match; then arrange the files into the layout the rest of the pipeline expects: binaries at `bundled/_/lstk-` (with `.exe` for Windows), executable bit set, and the descriptions file at `bundled/lstk-extensions.toml`. - [x] 3.3 Make the script fail — listing exactly what's missing — if any of lstk's six target platforms (`linux`/`darwin`/`windows` × `amd64`/`arm64`) has no binary for a bundled extension. A platform can be exempted by adding it to an `UNSUPPORTED_PLATFORMS` list at the top of the script, so skipping a platform is always a visible, deliberate choice. Without this check, a missing binary would surface later as a confusing "glob matched nothing" error inside GoReleaser. - [x] 3.4 Add a `--stub` flag that skips the download entirely and writes placeholder files into the same layout. This exists for contributors without access to the private repo who want to run a local `goreleaser` snapshot build (which fails if `bundled/` is empty once section 5 is merged). Print an unmissable banner that stub output must never be released. - [x] 3.5 When run without a token, fail with a message that says which secret/env var is needed and mentions `--stub` as the alternative for local builds. @@ -36,7 +36,7 @@ The extension binaries are never committed to this repo. Instead, a one-line ver - [x] 4.1 Agree with the owners of the private extensions repo on what their releases must contain, and write it down in `docs/extensions-bundling.md`: each tagged release ships one binary per extension per platform, named `lstk-__` (plus `.exe` for Windows), the hand-written `lstk-extensions.toml`, and a `checksums.txt` covering every asset. They own the descriptions text; we only validate it. - [ ] 4.2 Create the credential the release uses to download from the private repo: a fine-grained personal access token with **read-only** access to **only** that repo, stored as a repository/organization secret (e.g. `LSTK_EXTENSIONS_READ_TOKEN`). Deliberately not reusing `PRO_ACCESS_TOKEN` — the release should not hold broader access than it needs, and a read-only token can be rotated independently. -- [x] 4.3 In `.github/workflows/ci.yml`, add two steps to the `release` job before the GoReleaser step: run `scripts/fetch-bundled-extensions.sh` (with the secret), then `scripts/check-descriptions.sh bundled/linux_amd64`. Either failing must fail the release. +- [x] 4.3 In `.github/workflows/ci.yml`, add two steps to the `release` job before the GoReleaser step: run `scripts/bundled-extensions/fetch-bundled-extensions.sh` (with the secret), then `scripts/bundled-extensions/check-descriptions.sh bundled/linux_amd64`. Either failing must fail the release. - [x] 4.4 Record which bundle a release shipped, permanently. In the `release` job, take the tag resolved by 3.2 (and the bundle's commit hash) and append it to the published GitHub release notes. The job log is not enough — GitHub expires logs, and this is exactly the information someone needs months later when a customer reports a bug in a bundled extension. Document in `docs/extensions-bundling.md` how to read it back, and how to re-run a published release against its original bundle by passing that tag to the fetch script. - [x] 4.5 Add a CI check that fails when the packaging half and the download half are out of step, so a future PR cannot get this wrong the way section 5 warns about. It should fail if `.goreleaser.yaml` has live (uncommented) `files:` entries pointing at `bundled/` while the `release` job in `.github/workflows/ci.yml` has no `fetch-bundled-extensions.sh` step — and fail in the opposite direction too. Run it on every PR alongside `goreleaser check`, which cannot catch this itself because it only validates config syntax and never looks at the filesystem. diff --git a/scripts/add-bundled-to-npm.sh b/scripts/bundled-extensions/add-bundled-to-npm.sh similarity index 96% rename from scripts/add-bundled-to-npm.sh rename to scripts/bundled-extensions/add-bundled-to-npm.sh index 4936fb53..53713adb 100755 --- a/scripts/add-bundled-to-npm.sh +++ b/scripts/bundled-extensions/add-bundled-to-npm.sh @@ -22,7 +22,7 @@ # allowlist as well. # # Usage: -# scripts/add-bundled-to-npm.sh +# scripts/bundled-extensions/add-bundled-to-npm.sh set -euo pipefail die() { @@ -42,7 +42,7 @@ TOML="${BUNDLED_DIR}/lstk-extensions.toml" [ -d "${NPM_DIR}" ] || die "no such directory: ${NPM_DIR}" [ -d "${BUNDLED_DIR}" ] || die "no such directory: ${BUNDLED_DIR}" -[ -f "${TOML}" ] || die "no descriptions file at ${TOML}; run scripts/fetch-bundled-extensions.sh first" +[ -f "${TOML}" ] || die "no descriptions file at ${TOML}; run scripts/bundled-extensions/fetch-bundled-extensions.sh first" command -v node >/dev/null 2>&1 || die "node is required to edit package.json" # Appends names to the package.json "files" array, de-duplicated, preserving diff --git a/scripts/check-bundled-packaging-sync.sh b/scripts/bundled-extensions/check-bundled-packaging-sync.sh similarity index 95% rename from scripts/check-bundled-packaging-sync.sh rename to scripts/bundled-extensions/check-bundled-packaging-sync.sh index 39d83ab6..06787b73 100755 --- a/scripts/check-bundled-packaging-sync.sh +++ b/scripts/bundled-extensions/check-bundled-packaging-sync.sh @@ -12,11 +12,11 @@ # on every PR, where it costs a red build instead of a broken release. # # Usage: -# scripts/check-bundled-packaging-sync.sh [goreleaser.yaml] [ci-workflow.yml] +# scripts/bundled-extensions/check-bundled-packaging-sync.sh [goreleaser.yaml] [ci-workflow.yml] set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" GORELEASER_FILE="${1:-${REPO_ROOT}/.goreleaser.yaml}" WORKFLOW_FILE="${2:-${REPO_ROOT}/.github/workflows/ci.yml}" diff --git a/scripts/check-descriptions.sh b/scripts/bundled-extensions/check-descriptions.sh similarity index 97% rename from scripts/check-descriptions.sh rename to scripts/bundled-extensions/check-descriptions.sh index 9a78f81e..6c0fa386 100755 --- a/scripts/check-descriptions.sh +++ b/scripts/bundled-extensions/check-descriptions.sh @@ -40,10 +40,10 @@ # is the practical unit of verification. # # Usage: -# scripts/check-descriptions.sh [descriptions-file] +# scripts/bundled-extensions/check-descriptions.sh [descriptions-file] # # e.g. bundled/linux_amd64, as staged by -# scripts/fetch-bundled-extensions.sh +# scripts/bundled-extensions/fetch-bundled-extensions.sh # [descriptions-file] defaults to /../lstk-extensions.toml set -euo pipefail diff --git a/scripts/fetch-bundled-extensions.sh b/scripts/bundled-extensions/fetch-bundled-extensions.sh similarity index 98% rename from scripts/fetch-bundled-extensions.sh rename to scripts/bundled-extensions/fetch-bundled-extensions.sh index ee56f5a9..92f0e2c4 100755 --- a/scripts/fetch-bundled-extensions.sh +++ b/scripts/bundled-extensions/fetch-bundled-extensions.sh @@ -21,7 +21,7 @@ # toml, so a per-command file on disk would serve no purpose on any channel. # # Which commands the binary actually provides is not this script's business — -# `bundled-extensions list` answers that, and scripts/check-descriptions.sh +# `bundled-extensions list` answers that, and scripts/bundled-extensions/check-descriptions.sh # asks it. What this script does guarantee is that every archive carries an # identical toml, so that answer can be checked against one file rather than # six. @@ -34,7 +34,7 @@ # `latest` re-resolves on every invocation. # # Usage: -# scripts/fetch-bundled-extensions.sh [--tag ] [--stub] +# scripts/bundled-extensions/fetch-bundled-extensions.sh [--tag ] [--stub] # # --tag Use this bundle tag instead of resolving the version file. # --stub Skip the download entirely and write placeholder files into @@ -66,7 +66,7 @@ DESCRIPTIONS_FILE="lstk-extensions.toml" MANIFEST_FILE="checksums.txt" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" BUNDLED_DIR="${LSTK_BUNDLED_DIR:-${REPO_ROOT}/bundled}" VERSION_FILE="${BUNDLED_DIR}/extensions.version" diff --git a/scripts/test-scripts.sh b/scripts/bundled-extensions/test-scripts.sh similarity index 80% rename from scripts/test-scripts.sh rename to scripts/bundled-extensions/test-scripts.sh index d31cc51c..d8064105 100755 --- a/scripts/test-scripts.sh +++ b/scripts/bundled-extensions/test-scripts.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Runs the bash test suites for the release helper scripts under scripts/. +# Runs the bash test suites for the bundled-extension release helper scripts +# in this directory. # These scripts only ever run on the Linux release runner, so a bash suite is # the faithful test here; lstk's own behavior is covered by the Go suites. set -euo pipefail diff --git a/scripts/tests/add-bundled-to-npm_test.sh b/scripts/bundled-extensions/tests/add-bundled-to-npm_test.sh similarity index 99% rename from scripts/tests/add-bundled-to-npm_test.sh rename to scripts/bundled-extensions/tests/add-bundled-to-npm_test.sh index 41b1783b..fea4d48c 100644 --- a/scripts/tests/add-bundled-to-npm_test.sh +++ b/scripts/bundled-extensions/tests/add-bundled-to-npm_test.sh @@ -7,7 +7,7 @@ set -euo pipefail SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=scripts/tests/lib.sh +# shellcheck source=scripts/bundled-extensions/tests/lib.sh . "${SUITE_DIR}/lib.sh" ADD="${SUITE_DIR}/../add-bundled-to-npm.sh" diff --git a/scripts/tests/check-bundled-packaging-sync_test.sh b/scripts/bundled-extensions/tests/check-bundled-packaging-sync_test.sh similarity index 98% rename from scripts/tests/check-bundled-packaging-sync_test.sh rename to scripts/bundled-extensions/tests/check-bundled-packaging-sync_test.sh index 976a30ca..393d226c 100755 --- a/scripts/tests/check-bundled-packaging-sync_test.sh +++ b/scripts/bundled-extensions/tests/check-bundled-packaging-sync_test.sh @@ -6,7 +6,7 @@ set -euo pipefail SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=scripts/tests/lib.sh +# shellcheck source=scripts/bundled-extensions/tests/lib.sh . "${SUITE_DIR}/lib.sh" CHECK="${SUITE_DIR}/../check-bundled-packaging-sync.sh" diff --git a/scripts/tests/check-descriptions_test.sh b/scripts/bundled-extensions/tests/check-descriptions_test.sh similarity index 99% rename from scripts/tests/check-descriptions_test.sh rename to scripts/bundled-extensions/tests/check-descriptions_test.sh index 8b16ea19..7abfd176 100644 --- a/scripts/tests/check-descriptions_test.sh +++ b/scripts/bundled-extensions/tests/check-descriptions_test.sh @@ -8,7 +8,7 @@ set -euo pipefail SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=scripts/tests/lib.sh +# shellcheck source=scripts/bundled-extensions/tests/lib.sh . "${SUITE_DIR}/lib.sh" CHECK="${SUITE_DIR}/../check-descriptions.sh" diff --git a/scripts/tests/fetch-bundled-extensions_test.sh b/scripts/bundled-extensions/tests/fetch-bundled-extensions_test.sh similarity index 99% rename from scripts/tests/fetch-bundled-extensions_test.sh rename to scripts/bundled-extensions/tests/fetch-bundled-extensions_test.sh index f90e3300..493a5406 100755 --- a/scripts/tests/fetch-bundled-extensions_test.sh +++ b/scripts/bundled-extensions/tests/fetch-bundled-extensions_test.sh @@ -10,7 +10,7 @@ set -euo pipefail SUITE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=scripts/tests/lib.sh +# shellcheck source=scripts/bundled-extensions/tests/lib.sh . "${SUITE_DIR}/lib.sh" FETCH="${SUITE_DIR}/../fetch-bundled-extensions.sh" diff --git a/scripts/tests/lib.sh b/scripts/bundled-extensions/tests/lib.sh similarity index 100% rename from scripts/tests/lib.sh rename to scripts/bundled-extensions/tests/lib.sh diff --git a/test/integration/__snapshots__/extension_bundle_test.snap b/test/integration/__snapshots__/extension_bundle_test.snap index 280b110d..fb02afc2 100644 --- a/test/integration/__snapshots__/extension_bundle_test.snap +++ b/test/integration/__snapshots__/extension_bundle_test.snap @@ -2,51 +2,8 @@ Snapshots created by internal/snap. UPDATE_SNAPS=true go test rewrites this file. [TestBundledMultiCallHelpListsDescribedCommands_1] -Usage: lstk [options] [command] - -LSTK - LocalStack command-line interface - -Commands: - completion Generate the autocompletion script for the specified shell - config Manage configuration - help Help about any command - load Load a snapshot into the running emulator - login Manage login - logout Remove stored authentication credentials - logs Show emulator logs - reset Reset emulator state - restart Restart emulator - save Save a snapshot of the emulator state - setup Set up emulator CLI integration - snapshot Manage emulator snapshots - start Start emulator - status Show emulator status and deployed resources - stop Stop emulator - update Update lstk to the latest version - volume Manage emulator volume - -Tools: - aws Run AWS CLI commands against LocalStack - az Run Azure CLI commands against LocalStack - cdk Run AWS CDK against LocalStack - sam Run the AWS SAM CLI against LocalStack - terraform Run Terraform against LocalStack - Extensions: deploy Deploy to LocalStack doctor Check the local setup hello - -Options: - --config string Path to config file - --endpoint-url string Target an existing, externally-managed emulator at this URL - -h, --help Show help - --json Output in JSON format (only supported by some commands) - --no-snapshot Skip auto-loading the configured snapshot for this run - --non-interactive Disable interactive mode - --persist Persist emulator state across restarts - --snapshot string Snapshot REF to load after start (overrides config for this run) - --timeout duration Maximum time to wait for the emulator to become ready (overrides LSTK_STARTUP_TIMEOUT; 0 uses the default) - -t, --type string Emulator type to start (aws, snowflake, azure) - -v, --version Show version --- diff --git a/test/integration/extension_bundle_test.go b/test/integration/extension_bundle_test.go index cc8a657b..fbfdab8f 100644 --- a/test/integration/extension_bundle_test.go +++ b/test/integration/extension_bundle_test.go @@ -79,10 +79,35 @@ func TestBundledMultiCallHelpListsDescribedCommands(t *testing.T) { tmpHome := t.TempDir() stdout, stderr, err := runBinary(t, t.TempDir(), envWithPath(tmpHome, extDir), lstkBin, "--help") require.NoError(t, err, stderr) - // Pins: both bundled commands with their descriptions, the PATH one - // name-only, no `bundled-extensions` or `extensions` phantom entries, and - // no ARGS= line (help never executes anything). - snap.Match(t, stdout) + // Pins: both bundled commands with their descriptions and the PATH one + // name-only. Only the Extensions section is snapshotted; the rest of the + // help text belongs to commands and flags this test says nothing about. + snap.Match(t, helpSection(t, stdout, "Extensions:")) + // The bundle binary itself is not a command, and help never executes it. + require.NotContains(t, stdout, "bundled-extensions") + require.NotContains(t, stdout, "ARGS=") +} + +// helpSection returns one section of `lstk --help`: the header line and the +// indented entries under it, up to the blank line that ends it. Tests pin a +// section rather than the whole help text, so that adding a command or a flag +// elsewhere in lstk does not rewrite an extensions snapshot. +func helpSection(t *testing.T, help, header string) string { + t.Helper() + lines := strings.Split(strings.ReplaceAll(help, "\r\n", "\n"), "\n") + start := -1 + for i, line := range lines { + if line == header { + start = i + break + } + } + require.NotEqual(t, -1, start, "no %q section in help output:\n%s", header, help) + end := start + 1 + for end < len(lines) && strings.TrimSpace(lines[end]) != "" { + end++ + } + return strings.Join(lines[start:end], "\n") } func TestBundledMultiCallUndescribedCommandIsUnknown(t *testing.T) {