From 9b474150b122f19ad9e1259574d0b1ee7c57398c Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Thu, 28 May 2026 13:40:27 +0200 Subject: [PATCH] PoC: embed Viceroy directly into the fastly binary This allows shipping Viceroy inside the fastly binary so the common path becomes pure local extraction. No more downloads. The embedded binary lives behind a `viceroy_embed` build tag. Without the tag, fastly builds and behaves exactly as before. On first run the binary is decompressed. It's embedded as ztd-compressed to avoid bloating the binary too much if viceroy is not used - Each per-platform release archive grows by 8-10 MB. We already require klauspost/compress so no new dependencies are required. Extraction failures or version mismatches fall through to the existing download flow untouched, as do --viceroy-path and FASTLY_VICEROY_USE_PATH. The pinned Viceroy version is the contents of pkg/embedded/viceroy/VICEROY_VERSION, currently 0.18.0. Bumping it is a one-line edit followed by a checksums refresh. "make build-embedded" builds a CLI executable with Viceroy embedded. --- .goreleaser.yml | 5 + Makefile | 14 ++ go.mod | 2 +- pkg/commands/compute/serve.go | 34 +++ pkg/commands/compute/serve_embed_test.go | 103 +++++++++ pkg/commands/compute/serve_test.go | 8 + pkg/commands/version/root.go | 10 +- pkg/commands/version/version_embed_test.go | 58 +++++ pkg/commands/version/version_test.go | 2 + pkg/embedded/viceroy/.gitignore | 3 + pkg/embedded/viceroy/VICEROY_VERSION | 1 + pkg/embedded/viceroy/checksums.txt | 12 + pkg/embedded/viceroy/embed_darwin_amd64.go | 10 + pkg/embedded/viceroy/embed_darwin_arm64.go | 10 + pkg/embedded/viceroy/embed_linux_amd64.go | 10 + pkg/embedded/viceroy/embed_linux_arm64.go | 10 + pkg/embedded/viceroy/embed_test.go | 161 +++++++++++++ pkg/embedded/viceroy/embed_unsupported.go | 7 + pkg/embedded/viceroy/embed_windows_amd64.go | 10 + pkg/embedded/viceroy/extract.go | 100 ++++++++ pkg/embedded/viceroy/noembed_test.go | 21 ++ pkg/embedded/viceroy/viceroy.go | 37 +++ pkg/embedded/viceroy/viceroy_noembed.go | 7 + pkg/embedded/viceroy/viceroy_test.go | 16 ++ pkg/mock/versioner.go | 8 +- scripts/fetch-viceroy.sh | 238 ++++++++++++++++++++ 26 files changed, 893 insertions(+), 4 deletions(-) create mode 100644 pkg/commands/compute/serve_embed_test.go create mode 100644 pkg/commands/version/version_embed_test.go create mode 100644 pkg/embedded/viceroy/.gitignore create mode 100644 pkg/embedded/viceroy/VICEROY_VERSION create mode 100644 pkg/embedded/viceroy/checksums.txt create mode 100644 pkg/embedded/viceroy/embed_darwin_amd64.go create mode 100644 pkg/embedded/viceroy/embed_darwin_arm64.go create mode 100644 pkg/embedded/viceroy/embed_linux_amd64.go create mode 100644 pkg/embedded/viceroy/embed_linux_arm64.go create mode 100644 pkg/embedded/viceroy/embed_test.go create mode 100644 pkg/embedded/viceroy/embed_unsupported.go create mode 100644 pkg/embedded/viceroy/embed_windows_amd64.go create mode 100644 pkg/embedded/viceroy/extract.go create mode 100644 pkg/embedded/viceroy/noembed_test.go create mode 100644 pkg/embedded/viceroy/viceroy.go create mode 100644 pkg/embedded/viceroy/viceroy_noembed.go create mode 100644 pkg/embedded/viceroy/viceroy_test.go create mode 100755 scripts/fetch-viceroy.sh diff --git a/.goreleaser.yml b/.goreleaser.yml index eda2343e4..a4c52266f 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -14,11 +14,16 @@ before: hooks: - go mod tidy - go mod download + # Stage the embedded Viceroy assets for every supported platform. + # The viceroy_embed build tag (set on each per-platform build below) + # pulls these in via //go:embed. + - ./scripts/fetch-viceroy.sh --all # https://goreleaser.com/customization/builds/ builds: - <<: &build_defaults main: ./cmd/fastly + tags: [viceroy_embed] ldflags: - -s -w -X "github.com/fastly/cli/pkg/revision.AppVersion=v{{ .Version }}" - -X "github.com/fastly/cli/pkg/revision.GitCommit={{ .ShortCommit }}" diff --git a/Makefile b/Makefile index 358b047cd..d18f1a232 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,20 @@ SHELL := /usr/bin/env bash -o pipefail ## Set the shell to use for finding Go fi build: config ## Compile program (CGO disabled) CGO_ENABLED=0 $(GO_BIN) build $(GO_ARGS) ./cmd/fastly +# Requires curl, tar, zstd, shasum on PATH. +.PHONY: embed-viceroy +embed-viceroy: ## Stage embedded Viceroy asset for the host platform + @./scripts/fetch-viceroy.sh --host + +.PHONY: embed-viceroy-all +embed-viceroy-all: ## Stage embedded Viceroy assets for all supported platforms + @./scripts/fetch-viceroy.sh --all + +# Run `make embed-viceroy` first to stage the asset. +.PHONY: build-embedded +build-embedded: config ## Compile program with embedded Viceroy + CGO_ENABLED=0 $(GO_BIN) build -tags viceroy_embed $(GO_ARGS) ./cmd/fastly + ## Allows overriding go executable. GO_BIN ?= go ## Enables support for tools such as https://github.com/rakyll/gotest diff --git a/go.mod b/go.mod index 3941905c4..c88a321f4 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect - github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/compress v1.18.6 github.com/klauspost/pgzip v1.2.6 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-runewidth v0.0.23 // indirect diff --git a/pkg/commands/compute/serve.go b/pkg/commands/compute/serve.go index 66fe3f72e..346091a85 100644 --- a/pkg/commands/compute/serve.go +++ b/pkg/commands/compute/serve.go @@ -34,6 +34,7 @@ import ( "github.com/fastly/cli/pkg/argparser" "github.com/fastly/cli/pkg/check" + "github.com/fastly/cli/pkg/embedded/viceroy" fsterr "github.com/fastly/cli/pkg/errors" fstexec "github.com/fastly/cli/pkg/exec" "github.com/fastly/cli/pkg/filesystem" @@ -375,6 +376,20 @@ func (c *ServeCommand) GetViceroy(spinner text.Spinner, out io.Writer, manifestP return filepath.Abs(path) } + // Prefer the embedded binary when its version satisfies the manifest + // pin and the user is not forcing a refresh; this short-circuits the + // exec(viceroy --version) probe and download flow below. + if viceroy.Supported() && !c.ForceCheckViceroyLatest && embeddedVersionMatches(c.ViceroyVersioner.RequestedVersion(), viceroy.Version()) { + extracted, extractErr := viceroy.Extract(github.InstallDir) + if extractErr == nil { + if c.Globals.Verbose() { + text.Info(out, "Using embedded Viceroy v%s: %s\n\n", viceroy.Version(), extracted) + } + return extracted, nil + } + c.Globals.ErrLog.Add(extractErr) + } + bin = filepath.Join(github.InstallDir, c.ViceroyVersioner.BinaryName()) // NOTE: When checking if Viceroy is installed we don't use @@ -434,6 +449,25 @@ func (c *ServeCommand) GetViceroy(spinner text.Spinner, out io.Writer, manifestP return bin, nil } +// embeddedVersionMatches reports whether a fastly.toml `viceroy_version` +// pin is satisfied by the embedded Viceroy. An empty pin matches; a pin +// that fails to parse does not, so the regular download path surfaces +// the parse error to the user. +func embeddedVersionMatches(pinned, embedded string) bool { + if pinned == "" { + return true + } + p, err := semver.Parse(strings.TrimPrefix(pinned, "v")) + if err != nil { + return false + } + e, err := semver.Parse(strings.TrimPrefix(embedded, "v")) + if err != nil { + return false + } + return p.Equals(e) +} + // checkViceroyEnvVar indicates if the CLI should use a Viceroy binary exposed // on the user's $PATH. func checkViceroyEnvVar(value string) bool { diff --git a/pkg/commands/compute/serve_embed_test.go b/pkg/commands/compute/serve_embed_test.go new file mode 100644 index 000000000..77c9f6fcb --- /dev/null +++ b/pkg/commands/compute/serve_embed_test.go @@ -0,0 +1,103 @@ +//go:build viceroy_embed + +package compute_test + +import ( + "bytes" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/fastly/cli/pkg/argparser" + "github.com/fastly/cli/pkg/commands/compute" + "github.com/fastly/cli/pkg/config" + "github.com/fastly/cli/pkg/embedded/viceroy" + fsterr "github.com/fastly/cli/pkg/errors" + "github.com/fastly/cli/pkg/github" + "github.com/fastly/cli/pkg/global" + "github.com/fastly/cli/pkg/mock" + "github.com/fastly/cli/pkg/testutil" + "github.com/fastly/cli/pkg/text" +) + +func TestGetViceroyUsesEmbedded(t *testing.T) { + if !viceroy.Supported() { + t.Skipf("no embedded Viceroy for %s/%s; skipping", runtime.GOOS, runtime.GOARCH) + } + + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + rootdir := testutil.NewEnv(testutil.EnvOpts{ + T: t, + Dirs: []string{"install"}, + Write: []testutil.FileIO{ + {Src: "", Dst: config.FileName}, + }, + }) + installDir := filepath.Join(rootdir, "install") + configPath := filepath.Join(rootdir, config.FileName) + defer os.RemoveAll(rootdir) + + if err := os.Chdir(rootdir); err != nil { + t.Fatal(err) + } + defer func() { _ = os.Chdir(wd) }() + + github.InstallDir = installDir + + var out bytes.Buffer + + // DownloadOK=false makes any accidental fallback to the download path + // surface as a test failure. + av := mock.AssetVersioner{ + AssetVersion: viceroy.Version(), + BinaryFilename: "viceroy", + DownloadOK: false, + } + + var file config.File + if err := file.Read("example", strings.NewReader("yes"), &out, fsterr.MockLog{}, false); err != nil { + t.Fatal(err) + } + + spinner, err := text.NewSpinner(&out) + if err != nil { + t.Fatal(err) + } + + c := &compute.ServeCommand{ + Base: argparser.Base{ + Globals: &global.Data{ + Config: file, + ConfigPath: configPath, + ErrLog: fsterr.MockLog{}, + }, + }, + ViceroyVersioner: av, + } + + bin, err := c.GetViceroy(spinner, &out, "fastly.toml") + if err != nil { + t.Fatalf("GetViceroy() error = %v", err) + } + + wantDir := filepath.Join(installDir, "viceroy-embedded", "v"+viceroy.Version()) + if filepath.Dir(bin) != wantDir { + t.Errorf("extracted binary in unexpected dir: got %q, want under %q", bin, wantDir) + } + + if info, err := os.Stat(bin); err != nil { + t.Fatalf("extracted binary missing: %v", err) + } else if info.Size() == 0 { + t.Error("extracted binary is empty") + } + + if strings.Contains(out.String(), "Fetching Viceroy release") { + t.Errorf("embedded path unexpectedly triggered a download: %s", out.String()) + } +} diff --git a/pkg/commands/compute/serve_test.go b/pkg/commands/compute/serve_test.go index 09a3aa644..be7ec0339 100644 --- a/pkg/commands/compute/serve_test.go +++ b/pkg/commands/compute/serve_test.go @@ -10,6 +10,7 @@ import ( "github.com/fastly/cli/pkg/argparser" "github.com/fastly/cli/pkg/commands/compute" "github.com/fastly/cli/pkg/config" + "github.com/fastly/cli/pkg/embedded/viceroy" fsterr "github.com/fastly/cli/pkg/errors" "github.com/fastly/cli/pkg/github" "github.com/fastly/cli/pkg/global" @@ -71,11 +72,18 @@ func TestGetViceroy(t *testing.T) { var out bytes.Buffer + // Pin a version that cannot match the embedded one so this test always + // exercises the download path, regardless of whether the binary was + // built with the viceroy_embed tag. av := mock.AssetVersioner{ AssetVersion: "1.2.3", BinaryFilename: viceroyBinName, DownloadOK: true, DownloadedFile: binPath, + Requested: "0.0.0-test", + } + if av.Requested == viceroy.Version() { + t.Fatalf("test sentinel version %q collides with embedded version", av.Requested) } var file config.File diff --git a/pkg/commands/version/root.go b/pkg/commands/version/root.go index 00fc44297..07e0686c7 100644 --- a/pkg/commands/version/root.go +++ b/pkg/commands/version/root.go @@ -9,6 +9,7 @@ import ( "time" "github.com/fastly/cli/pkg/argparser" + "github.com/fastly/cli/pkg/embedded/viceroy" "github.com/fastly/cli/pkg/github" "github.com/fastly/cli/pkg/global" "github.com/fastly/cli/pkg/revision" @@ -39,7 +40,12 @@ func (c *RootCommand) Exec(_ io.Reader, out io.Writer) error { fmt.Fprintf(out, "Fastly CLI version %s (%s)\n", revision.AppVersion, revision.GitCommit) fmt.Fprintf(out, "Built with %s (%s)\n", revision.GoVersion, Now().Format("2006-01-02")) - viceroy := filepath.Join(github.InstallDir, c.Globals.Versioners.Viceroy.BinaryName()) + viceroyBin := filepath.Join(github.InstallDir, c.Globals.Versioners.Viceroy.BinaryName()) + if viceroy.Supported() { + if extracted, err := viceroy.Extract(github.InstallDir); err == nil { + viceroyBin = extracted + } + } // gosec flagged this: // G204 (CWE-78): Subprocess launched with variable // Disabling as we lookup the binary in a trusted location. For this to be a @@ -47,7 +53,7 @@ func (c *RootCommand) Exec(_ io.Reader, out io.Writer) error { // attacker could swap the actual viceroy executable for something malicious. /* #nosec */ // nosemgrep - command := exec.Command(viceroy, "--version") + command := exec.Command(viceroyBin, "--version") if stdoutStderr, err := command.CombinedOutput(); err == nil { fmt.Fprintf(out, "Viceroy version: %s", stdoutStderr) } diff --git a/pkg/commands/version/version_embed_test.go b/pkg/commands/version/version_embed_test.go new file mode 100644 index 000000000..ffe7b5582 --- /dev/null +++ b/pkg/commands/version/version_embed_test.go @@ -0,0 +1,58 @@ +//go:build viceroy_embed + +package version_test + +import ( + "bytes" + "fmt" + "io" + "runtime" + "strings" + "testing" + "time" + + "github.com/fastly/cli/pkg/app" + "github.com/fastly/cli/pkg/commands/version" + "github.com/fastly/cli/pkg/embedded/viceroy" + "github.com/fastly/cli/pkg/github" + "github.com/fastly/cli/pkg/global" + "github.com/fastly/cli/pkg/testutil" +) + +func TestVersionUsesEmbeddedViceroy(t *testing.T) { + if !viceroy.Supported() { + t.Skipf("no embedded Viceroy for %s/%s; skipping", runtime.GOOS, runtime.GOARCH) + } + if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { + t.Skip("skipping non-unix variants") + } + + rootdir := testutil.NewEnv(testutil.EnvOpts{T: t}) + orgInstallDir := github.InstallDir + github.InstallDir = rootdir + defer func() { github.InstallDir = orgInstallDir }() + + version.Now = func() (t time.Time) { return t } + + var stdout bytes.Buffer + args := testutil.SplitArgs("version") + opts := testutil.MockGlobalData(args, &stdout) + opts.Versioners = global.Versioners{ + Viceroy: github.New(github.Opts{Org: "fastly", Repo: "viceroy", Binary: "viceroy"}), + } + app.Init = func(_ []string, _ io.Reader) (*global.Data, error) { return opts, nil } + if err := app.Run(args, nil); err != nil { + t.Fatal(err) + } + + var mockTime time.Time + want := strings.Join([]string{ + "Fastly CLI version v0.0.0-unknown (unknown)", + fmt.Sprintf("Built with go version %s %s/%s (%s)", runtime.Version(), runtime.GOOS, runtime.GOARCH, mockTime.Format("2006-01-02")), + fmt.Sprintf("Viceroy version: viceroy %s", viceroy.Version()), + "", + }, "\n") + if stdout.String() != want { + t.Errorf("unexpected output:\n got: %q\nwant: %q", stdout.String(), want) + } +} diff --git a/pkg/commands/version/version_test.go b/pkg/commands/version/version_test.go index 779bbaa20..22009c704 100644 --- a/pkg/commands/version/version_test.go +++ b/pkg/commands/version/version_test.go @@ -1,3 +1,5 @@ +//go:build !viceroy_embed + package version_test import ( diff --git a/pkg/embedded/viceroy/.gitignore b/pkg/embedded/viceroy/.gitignore new file mode 100644 index 000000000..834c52917 --- /dev/null +++ b/pkg/embedded/viceroy/.gitignore @@ -0,0 +1,3 @@ +# Embedded Viceroy assets are downloaded at build time by +# scripts/fetch-viceroy.sh and are not checked in. +assets/*.zst diff --git a/pkg/embedded/viceroy/VICEROY_VERSION b/pkg/embedded/viceroy/VICEROY_VERSION new file mode 100644 index 000000000..66333910a --- /dev/null +++ b/pkg/embedded/viceroy/VICEROY_VERSION @@ -0,0 +1 @@ +0.18.0 diff --git a/pkg/embedded/viceroy/checksums.txt b/pkg/embedded/viceroy/checksums.txt new file mode 100644 index 000000000..5323e986f --- /dev/null +++ b/pkg/embedded/viceroy/checksums.txt @@ -0,0 +1,12 @@ +# SHA-256 checksums of the raw Viceroy executables, pre-compression. +# Format matches `sha256sum`. Refresh via: +# scripts/fetch-viceroy.sh --all --refresh-checksums +# Building with -tags viceroy_embed verifies the downloaded asset matches. +# +# Pinned Viceroy version: see VICEROY_VERSION. + +20c6790dacd5ca86d599a5c2f0ca2083dc609b8ad87708504482e3a31c02b087 viceroy_darwin_amd64 +8957c60ef2059688b58f7581ec55b1ed6c7128ed0aac25d0f81d64b52f62da02 viceroy_darwin_arm64 +e2173c8799f73850e657e3722ce5000d3338ccdf6246790688a6b8ae8e53b896 viceroy_linux_amd64 +ef4c8d71eaabd7729c4848b246f501fdf4d3f703c1f57f3e964d81e02d38bc7d viceroy_linux_arm64 +1f98b49e3114865ed79538d3dd86e7bb2d2d08da8ad4658d4cb925166a211817 viceroy_windows_amd64 diff --git a/pkg/embedded/viceroy/embed_darwin_amd64.go b/pkg/embedded/viceroy/embed_darwin_amd64.go new file mode 100644 index 000000000..c7c86b264 --- /dev/null +++ b/pkg/embedded/viceroy/embed_darwin_amd64.go @@ -0,0 +1,10 @@ +//go:build viceroy_embed && darwin && amd64 + +package viceroy + +import _ "embed" + +//go:embed assets/viceroy_darwin_amd64.zst +var binaryZstd []byte + +const platformSupported = true diff --git a/pkg/embedded/viceroy/embed_darwin_arm64.go b/pkg/embedded/viceroy/embed_darwin_arm64.go new file mode 100644 index 000000000..f14d4f0ce --- /dev/null +++ b/pkg/embedded/viceroy/embed_darwin_arm64.go @@ -0,0 +1,10 @@ +//go:build viceroy_embed && darwin && arm64 + +package viceroy + +import _ "embed" + +//go:embed assets/viceroy_darwin_arm64.zst +var binaryZstd []byte + +const platformSupported = true diff --git a/pkg/embedded/viceroy/embed_linux_amd64.go b/pkg/embedded/viceroy/embed_linux_amd64.go new file mode 100644 index 000000000..2deaaacd9 --- /dev/null +++ b/pkg/embedded/viceroy/embed_linux_amd64.go @@ -0,0 +1,10 @@ +//go:build viceroy_embed && linux && amd64 + +package viceroy + +import _ "embed" + +//go:embed assets/viceroy_linux_amd64.zst +var binaryZstd []byte + +const platformSupported = true diff --git a/pkg/embedded/viceroy/embed_linux_arm64.go b/pkg/embedded/viceroy/embed_linux_arm64.go new file mode 100644 index 000000000..04ee58e33 --- /dev/null +++ b/pkg/embedded/viceroy/embed_linux_arm64.go @@ -0,0 +1,10 @@ +//go:build viceroy_embed && linux && arm64 + +package viceroy + +import _ "embed" + +//go:embed assets/viceroy_linux_arm64.zst +var binaryZstd []byte + +const platformSupported = true diff --git a/pkg/embedded/viceroy/embed_test.go b/pkg/embedded/viceroy/embed_test.go new file mode 100644 index 000000000..03cc17209 --- /dev/null +++ b/pkg/embedded/viceroy/embed_test.go @@ -0,0 +1,161 @@ +//go:build viceroy_embed + +package viceroy + +import ( + "os" + "path/filepath" + "runtime" + "sync" + "testing" +) + +func onlyOnSupportedPlatform(t *testing.T) { + t.Helper() + if !Supported() { + t.Skipf("no embedded Viceroy for %s/%s; skipping", runtime.GOOS, runtime.GOARCH) + } +} + +func TestExtractCreatesExecutableFile(t *testing.T) { + onlyOnSupportedPlatform(t) + + dir := t.TempDir() + path, err := Extract(dir) + if err != nil { + t.Fatalf("Extract() error = %v", err) + } + + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat extracted file: %v", err) + } + if info.Size() == 0 { + t.Error("extracted file is empty") + } + + if runtime.GOOS != "windows" { + if info.Mode().Perm()&0o111 == 0 { + t.Errorf("extracted file is not executable: mode=%v", info.Mode()) + } + } + + expectedSuffix := binaryName() + if filepath.Base(path) != expectedSuffix { + t.Errorf("extracted basename = %q, want %q", filepath.Base(path), expectedSuffix) + } + + wantDir := filepath.Join(dir, extractSubdir, "v"+Version()) + if filepath.Dir(path) != wantDir { + t.Errorf("extracted dir = %q, want %q", filepath.Dir(path), wantDir) + } +} + +func TestExtractIdempotent(t *testing.T) { + onlyOnSupportedPlatform(t) + + dir := t.TempDir() + first, err := Extract(dir) + if err != nil { + t.Fatalf("first Extract() error = %v", err) + } + infoBefore, err := os.Stat(first) + if err != nil { + t.Fatal(err) + } + + second, err := Extract(dir) + if err != nil { + t.Fatalf("second Extract() error = %v", err) + } + if first != second { + t.Errorf("second Extract() returned different path: %q vs %q", first, second) + } + + infoAfter, err := os.Stat(second) + if err != nil { + t.Fatal(err) + } + if !infoBefore.ModTime().Equal(infoAfter.ModTime()) { + t.Errorf("Extract() rewrote the file on second call: before=%v after=%v", + infoBefore.ModTime(), infoAfter.ModTime()) + } +} + +func TestExtractConcurrent(t *testing.T) { + onlyOnSupportedPlatform(t) + + dir := t.TempDir() + const N = 8 + + var wg sync.WaitGroup + results := make([]string, N) + errs := make([]error, N) + wg.Add(N) + for i := 0; i < N; i++ { + go func(i int) { + defer wg.Done() + results[i], errs[i] = Extract(dir) + }(i) + } + wg.Wait() + + for i, err := range errs { + if err != nil { + t.Errorf("goroutine %d: Extract() error = %v", i, err) + } + } + for i := 1; i < N; i++ { + if results[i] != results[0] { + t.Errorf("Extract() paths diverged: %q vs %q", results[0], results[i]) + } + } +} + +func TestExtractLeavesStaleSiblingVersionsAlone(t *testing.T) { + onlyOnSupportedPlatform(t) + + dir := t.TempDir() + stale := filepath.Join(dir, extractSubdir, "v0.0.0-stale") + if err := os.MkdirAll(stale, 0o755); err != nil { + t.Fatal(err) + } + staleMarker := filepath.Join(stale, "marker") + if err := os.WriteFile(staleMarker, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + + if _, err := Extract(dir); err != nil { + t.Fatalf("Extract() error = %v", err) + } + + if _, err := os.Stat(stale); err != nil { + t.Errorf("Extract() removed an unrelated sibling directory: %v", err) + } +} + +func TestExtractRejectsEmptyExistingFile(t *testing.T) { + onlyOnSupportedPlatform(t) + + dir := t.TempDir() + versionDir := filepath.Join(dir, extractSubdir, "v"+Version()) + if err := os.MkdirAll(versionDir, 0o755); err != nil { + t.Fatal(err) + } + binPath := filepath.Join(versionDir, binaryName()) + if err := os.WriteFile(binPath, nil, 0o755); err != nil { + t.Fatal(err) + } + + path, err := Extract(dir) + if err != nil { + t.Fatalf("Extract() error = %v", err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Size() == 0 { + t.Error("Extract() returned a zero-byte file instead of overwriting it") + } +} diff --git a/pkg/embedded/viceroy/embed_unsupported.go b/pkg/embedded/viceroy/embed_unsupported.go new file mode 100644 index 000000000..4b9660688 --- /dev/null +++ b/pkg/embedded/viceroy/embed_unsupported.go @@ -0,0 +1,7 @@ +//go:build viceroy_embed && !(darwin && amd64) && !(darwin && arm64) && !(linux && amd64) && !(linux && arm64) && !(windows && amd64) + +package viceroy + +const platformSupported = false + +var binaryZstd []byte diff --git a/pkg/embedded/viceroy/embed_windows_amd64.go b/pkg/embedded/viceroy/embed_windows_amd64.go new file mode 100644 index 000000000..629538b4a --- /dev/null +++ b/pkg/embedded/viceroy/embed_windows_amd64.go @@ -0,0 +1,10 @@ +//go:build viceroy_embed && windows && amd64 + +package viceroy + +import _ "embed" + +//go:embed assets/viceroy_windows_amd64.zst +var binaryZstd []byte + +const platformSupported = true diff --git a/pkg/embedded/viceroy/extract.go b/pkg/embedded/viceroy/extract.go new file mode 100644 index 000000000..6f0114fc0 --- /dev/null +++ b/pkg/embedded/viceroy/extract.go @@ -0,0 +1,100 @@ +package viceroy + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/klauspost/compress/zstd" + + fstruntime "github.com/fastly/cli/pkg/runtime" +) + +const extractSubdir = "viceroy-embedded" + +func binaryName() string { + if fstruntime.Windows { + return "viceroy.exe" + } + return "viceroy" +} + +// Extract decompresses the embedded Viceroy binary into installDir and +// returns the absolute path to the executable. If a non-empty file +// already exists at the expected versioned location, Extract is a +// no-op and returns the existing path. The function is safe to call +// concurrently from multiple processes thanks to a temp-file + atomic +// rename. +// +// Extract returns ErrUnsupported when Supported reports false. Old +// versioned directories from a previous CLI install are intentionally +// left in place: a separate cleanup path (rather than this hot one) +// should handle them, to avoid removing a binary that another fastly +// process may still be exec'ing. +func Extract(installDir string) (string, error) { + if !Supported() { + return "", ErrUnsupported + } + + versionDir := filepath.Join(installDir, extractSubdir, "v"+Version()) + binPath := filepath.Join(versionDir, binaryName()) + + if info, err := os.Stat(binPath); err == nil && !info.IsDir() && info.Size() > 0 { + return binPath, nil + } + + if err := os.MkdirAll(versionDir, 0o755); err != nil { + return "", fmt.Errorf("viceroy: create extract dir: %w", err) + } + + tmpPath, err := writeTempBinary(versionDir) + if err != nil { + return "", err + } + + if err := os.Rename(tmpPath, binPath); err != nil { + _ = os.Remove(tmpPath) + return "", fmt.Errorf("viceroy: install extracted binary: %w", err) + } + return binPath, nil +} + +func writeTempBinary(dir string) (string, error) { + f, err := os.CreateTemp(dir, "."+binaryName()+".tmp.*") + if err != nil { + return "", fmt.Errorf("viceroy: create temp file: %w", err) + } + tmpPath := f.Name() + + dec, err := zstd.NewReader(nil) + if err != nil { + f.Close() + os.Remove(tmpPath) + return "", fmt.Errorf("viceroy: init zstd: %w", err) + } + defer dec.Close() + + src, err := dec.DecodeAll(binaryZstd, nil) + if err != nil { + f.Close() + os.Remove(tmpPath) + return "", fmt.Errorf("viceroy: decompress: %w", err) + } + + if _, err := f.Write(src); err != nil { + f.Close() + os.Remove(tmpPath) + return "", fmt.Errorf("viceroy: write temp file: %w", err) + } + if err := f.Close(); err != nil { + os.Remove(tmpPath) + return "", fmt.Errorf("viceroy: close temp file: %w", err) + } + + if err := os.Chmod(tmpPath, 0o755); err != nil { + os.Remove(tmpPath) + return "", fmt.Errorf("viceroy: chmod temp file: %w", err) + } + + return tmpPath, nil +} diff --git a/pkg/embedded/viceroy/noembed_test.go b/pkg/embedded/viceroy/noembed_test.go new file mode 100644 index 000000000..bd2700518 --- /dev/null +++ b/pkg/embedded/viceroy/noembed_test.go @@ -0,0 +1,21 @@ +//go:build !viceroy_embed + +package viceroy + +import ( + "errors" + "testing" +) + +func TestNoEmbedSupported(t *testing.T) { + if Supported() { + t.Fatal("Supported() = true in a build without -tags viceroy_embed") + } +} + +func TestNoEmbedExtractReturnsErrUnsupported(t *testing.T) { + _, err := Extract(t.TempDir()) + if !errors.Is(err, ErrUnsupported) { + t.Fatalf("Extract() error = %v, want ErrUnsupported", err) + } +} diff --git a/pkg/embedded/viceroy/viceroy.go b/pkg/embedded/viceroy/viceroy.go new file mode 100644 index 000000000..73478a0f4 --- /dev/null +++ b/pkg/embedded/viceroy/viceroy.go @@ -0,0 +1,37 @@ +// Package viceroy carries an optional Viceroy executable embedded directly +// into the fastly CLI binary. When built with the `viceroy_embed` build tag +// and an asset is available for the target platform, the binary is extracted +// to disk on first use and exec'd from there. Without the tag, or on a +// platform that has no shipped Viceroy asset, the package reports +// Supported() == false and callers should fall back to downloading Viceroy +// at runtime. +package viceroy + +import ( + _ "embed" + "errors" + "strings" +) + +//go:embed VICEROY_VERSION +var versionRaw string + +// ErrUnsupported is returned by Extract when this build does not carry a +// Viceroy binary for the current platform. +var ErrUnsupported = errors.New("viceroy: no embedded binary for this platform") + +// Version reports the Viceroy version pinned at CLI build time. It returns +// the contents of pkg/embedded/viceroy/VICEROY_VERSION regardless of whether +// an asset is actually embedded for the current platform, so callers can use +// it for logging and comparison without first consulting Supported. +func Version() string { + return strings.TrimSpace(versionRaw) +} + +// Supported reports whether this build carries a usable embedded Viceroy +// for the current GOOS/GOARCH. It returns false when the binary was built +// without -tags viceroy_embed, or when the tag is set but no asset exists +// for the target platform. +func Supported() bool { + return platformSupported && len(binaryZstd) > 0 +} diff --git a/pkg/embedded/viceroy/viceroy_noembed.go b/pkg/embedded/viceroy/viceroy_noembed.go new file mode 100644 index 000000000..76562557f --- /dev/null +++ b/pkg/embedded/viceroy/viceroy_noembed.go @@ -0,0 +1,7 @@ +//go:build !viceroy_embed + +package viceroy + +const platformSupported = false + +var binaryZstd []byte diff --git a/pkg/embedded/viceroy/viceroy_test.go b/pkg/embedded/viceroy/viceroy_test.go new file mode 100644 index 000000000..37c06d0a1 --- /dev/null +++ b/pkg/embedded/viceroy/viceroy_test.go @@ -0,0 +1,16 @@ +package viceroy + +import ( + "strings" + "testing" +) + +func TestVersionNonEmpty(t *testing.T) { + v := Version() + if v == "" { + t.Fatal("Version() returned empty string") + } + if strings.ContainsAny(v, " \t\n\r") { + t.Errorf("Version() = %q contains whitespace", v) + } +} diff --git a/pkg/mock/versioner.go b/pkg/mock/versioner.go index 97c9d5993..c37ec21ab 100644 --- a/pkg/mock/versioner.go +++ b/pkg/mock/versioner.go @@ -9,6 +9,9 @@ type AssetVersioner struct { DownloadOK bool DownloadedFile string InstallFilePath string + // Requested controls the value returned by RequestedVersion. An empty + // string mimics "not pinned in fastly.toml". + Requested string } // BinaryName implements github.Versioner interface. @@ -26,6 +29,9 @@ func (av AssetVersioner) DownloadLatest() (string, error) { // DownloadVersion implements github.Versioner interface. func (av AssetVersioner) DownloadVersion(_ string) (string, error) { + if av.DownloadOK { + return av.DownloadedFile, nil + } return "", nil } @@ -46,7 +52,7 @@ func (av AssetVersioner) LatestVersion() (string, error) { // RequestedVersion implements github.Versioner interface. func (av AssetVersioner) RequestedVersion() (version string) { - return "" + return av.Requested } // SetRequestedVersion implements github.Versioner interface. diff --git a/scripts/fetch-viceroy.sh b/scripts/fetch-viceroy.sh new file mode 100755 index 000000000..66299b6fa --- /dev/null +++ b/scripts/fetch-viceroy.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +# Fetch the pinned Viceroy release assets and prepare them for embedding +# into the fastly CLI binary. +# +# Usage: +# scripts/fetch-viceroy.sh # fetch for host platform only +# scripts/fetch-viceroy.sh --host # same as default +# scripts/fetch-viceroy.sh --all # fetch every supported platform +# scripts/fetch-viceroy.sh --refresh-checksums [--host|--all] +# +# The script reads the desired version from pkg/embedded/viceroy/VICEROY_VERSION, +# downloads the matching upstream asset, verifies its SHA-256 against +# pkg/embedded/viceroy/checksums.txt (unless --refresh-checksums is set, in +# which case the file is rewritten), compresses the executable with zstd, +# and writes it to pkg/embedded/viceroy/assets/viceroy__.zst. + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +pkg_dir="${repo_root}/pkg/embedded/viceroy" +assets_dir="${pkg_dir}/assets" +version_file="${pkg_dir}/VICEROY_VERSION" +checksums_file="${pkg_dir}/checksums.txt" + +if [ ! -f "${version_file}" ]; then + echo "fetch-viceroy: missing ${version_file}" >&2 + exit 1 +fi + +viceroy_version="$(tr -d '[:space:]' < "${version_file}")" +if [ -z "${viceroy_version}" ]; then + echo "fetch-viceroy: VICEROY_VERSION is empty" >&2 + exit 1 +fi + +mode="host" +refresh_checksums=0 +for arg in "$@"; do + case "${arg}" in + --host) mode="host" ;; + --all) mode="all" ;; + --refresh-checksums) refresh_checksums=1 ;; + -h|--help) + sed -n '2,16p' "$0" + exit 0 ;; + *) + echo "fetch-viceroy: unknown argument: ${arg}" >&2 + exit 1 ;; + esac +done + +# Supported (os, arch) pairs. Keep in sync with pkg/embedded/viceroy/embed_*.go. +platforms="darwin amd64 +darwin arm64 +linux amd64 +linux arm64 +windows amd64" + +# Pick the platforms to process based on --host / --all. +if [ "${mode}" = "all" ]; then + selected_platforms="${platforms}" +else + host_os="$(uname -s | tr '[:upper:]' '[:lower:]')" + case "${host_os}" in + darwin) host_os="darwin" ;; + linux) host_os="linux" ;; + mingw*|msys*|cygwin*|windows*) host_os="windows" ;; + *) + echo "fetch-viceroy: unsupported host OS: ${host_os}" >&2 + exit 1 ;; + esac + + host_arch="$(uname -m)" + case "${host_arch}" in + x86_64|amd64) host_arch="amd64" ;; + arm64|aarch64) host_arch="arm64" ;; + *) + echo "fetch-viceroy: unsupported host arch: ${host_arch}" >&2 + exit 1 ;; + esac + + selected_platforms="$(echo "${platforms}" | grep -E "^${host_os} ${host_arch}$" || true)" + if [ -z "${selected_platforms}" ]; then + echo "fetch-viceroy: no embedded Viceroy ships for ${host_os}/${host_arch}" >&2 + exit 1 + fi +fi + +mkdir -p "${assets_dir}" + +for tool in curl tar shasum zstd awk; do + if ! command -v "${tool}" >/dev/null 2>&1; then + echo "fetch-viceroy: missing required tool: ${tool}" >&2 + exit 1 + fi +done + +# Discover the upstream download URL via the same metadata endpoint +# the runtime uses (pkg/github/github.go:23). The endpoint returns the +# URL of the latest release; we substitute the version string to pin +# to the version requested in VICEROY_VERSION, mirroring +# Asset.DownloadVersion at pkg/github/github.go:128. This keeps the +# build-time fetch in lockstep with the runtime download path if the +# upstream URL template ever changes. +meta_url_for() { + local os="$1" arch="$2" + local endpoint="https://developer.fastly.com/api/internal/releases/meta/viceroy/${os}/${arch}" + curl -fsSL "${endpoint}" +} + +sha256() { + shasum -a 256 "$1" | awk '{print $1}' +} + +lookup_checksum() { + local label="$1" + [ -f "${checksums_file}" ] || return 0 + awk -v label="${label}" ' + { + sub(/#.*/, "") + if (NF < 2) next + if ($2 == label) { print $1; exit } + } + ' "${checksums_file}" +} + +workdir="$(mktemp -d)" +trap 'rm -rf "${workdir}"' EXIT + +# Accumulate (label, digest) pairs to write when --refresh-checksums is set. +new_sums_file="${workdir}/new_sums.txt" +: > "${new_sums_file}" + +while IFS=' ' read -r os arch; do + [ -z "${os}" ] && continue + + meta="$(meta_url_for "${os}" "${arch}" || true)" + if [ -z "${meta}" ]; then + echo "fetch-viceroy: failed to fetch release metadata for ${os}/${arch}" >&2 + exit 1 + fi + latest_version="$(echo "${meta}" | awk -F'"' '/"version":/ { for (i = 1; i <= NF; i++) if ($i == "version") { print $(i+2); exit } }')" + latest_url="$(echo "${meta}" | awk -F'"' '/"url":/ { for (i = 1; i <= NF; i++) if ($i == "url") { print $(i+2); exit } }')" + if [ -z "${latest_version}" ] || [ -z "${latest_url}" ]; then + echo "fetch-viceroy: malformed metadata response for ${os}/${arch}" >&2 + exit 1 + fi + + # Pin to the version requested in VICEROY_VERSION by substituting the + # latest version string in the URL. Matches the runtime's + # Asset.DownloadVersion behaviour. + url="$(echo "${latest_url}" | awk -v from="${latest_version}" -v to="${viceroy_version}" '{gsub(from, to); print}')" + archive_name="$(basename "${url}")" + + echo "fetch-viceroy: downloading ${os}/${arch} v${viceroy_version}" + archive_path="${workdir}/${archive_name}" + if ! curl -fsSL "${url}" -o "${archive_path}"; then + echo "fetch-viceroy: download failed: ${url}" >&2 + exit 1 + fi + + extract_dir="${workdir}/${os}_${arch}" + mkdir -p "${extract_dir}" + tar -xzf "${archive_path}" -C "${extract_dir}" + + bin_name="viceroy" + [ "${os}" = "windows" ] && bin_name="viceroy.exe" + bin_path="${extract_dir}/${bin_name}" + if [ ! -f "${bin_path}" ]; then + bin_path="$(find "${extract_dir}" -type f -name "${bin_name}" -print -quit)" + fi + if [ -z "${bin_path}" ] || [ ! -f "${bin_path}" ]; then + echo "fetch-viceroy: viceroy binary not found in ${archive_name}" >&2 + exit 1 + fi + + asset_label="viceroy_${os}_${arch}" + digest="$(sha256 "${bin_path}")" + + if [ "${refresh_checksums}" -eq 1 ]; then + printf '%s %s\n' "${digest}" "${asset_label}" >> "${new_sums_file}" + else + expected="$(lookup_checksum "${asset_label}")" + if [ -z "${expected}" ]; then + echo "fetch-viceroy: no checksum on file for ${asset_label}. Re-run with --refresh-checksums and review the diff." >&2 + exit 1 + fi + if [ "${digest}" != "${expected}" ]; then + echo "fetch-viceroy: checksum mismatch for ${asset_label}" >&2 + echo " expected: ${expected}" >&2 + echo " got: ${digest}" >&2 + exit 1 + fi + fi + + zstd -q -19 --rm -f "${bin_path}" -o "${assets_dir}/${asset_label}.zst" + echo "fetch-viceroy: wrote ${assets_dir}/${asset_label}.zst" +done <<< "${selected_platforms}" + +# Rewrite checksums.txt when requested. Preserve any platform entries that +# weren't part of this run (so --host --refresh-checksums updates one line +# without losing the others). +if [ "${refresh_checksums}" -eq 1 ]; then + merged_file="${workdir}/merged_sums.txt" + : > "${merged_file}" + + if [ -f "${checksums_file}" ]; then + awk ' + { + line = $0 + sub(/#.*/, "") + if (NF >= 2) { + print $1 " " $2 + } + } + ' "${checksums_file}" >> "${merged_file}" + fi + + while IFS= read -r line; do + digest="$(echo "${line}" | awk '{print $1}')" + label="$(echo "${line}" | awk '{print $2}')" + grep -v " ${label}\$" "${merged_file}" > "${merged_file}.tmp" || true + mv "${merged_file}.tmp" "${merged_file}" + printf '%s %s\n' "${digest}" "${label}" >> "${merged_file}" + done < "${new_sums_file}" + + { + echo "# SHA-256 checksums of the raw Viceroy executables, pre-compression." + echo "# Format matches \`sha256sum\`. Refresh via:" + echo "# scripts/fetch-viceroy.sh --all --refresh-checksums" + echo "# Building with -tags viceroy_embed verifies the downloaded asset matches." + echo "#" + echo "# Pinned Viceroy version: see VICEROY_VERSION." + echo + sort -k2 "${merged_file}" | awk '{printf "%s %s\n", $1, $2}' + } > "${checksums_file}" + echo "fetch-viceroy: refreshed ${checksums_file}" +fi