PoC: embed Viceroy directly into the fastly binary

.goreleaser.yml

@@ -14,11 +14,16 @@ before:
   hooks:
     - go mod tidy
     - go mod download
+    # Stage the embedded Viceroy assets for every supported platform.
+    # The viceroy_embed build tag (set on each per-platform build below)
+    # pulls these in via //go:embed.
+    - ./scripts/fetch-viceroy.sh --all
 
 # https://goreleaser.com/customization/builds/
 builds:
   - <<: &build_defaults
       main: ./cmd/fastly
+      tags: [viceroy_embed]
       ldflags:
         - -s -w -X "github.com/fastly/cli/pkg/revision.AppVersion=v{{ .Version }}"
         - -X "github.com/fastly/cli/pkg/revision.GitCommit={{ .ShortCommit }}"

Makefile

@@ -11,6 +11,20 @@ SHELL := /usr/bin/env bash -o pipefail ## Set the shell to use for finding Go fi
 build: config ## Compile program (CGO disabled)
 	CGO_ENABLED=0 $(GO_BIN) build $(GO_ARGS) ./cmd/fastly
 
+# Requires curl, tar, zstd, shasum on PATH.
+.PHONY: embed-viceroy
+embed-viceroy: ## Stage embedded Viceroy asset for the host platform
+	@./scripts/fetch-viceroy.sh --host
+
+.PHONY: embed-viceroy-all
+embed-viceroy-all: ## Stage embedded Viceroy assets for all supported platforms
+	@./scripts/fetch-viceroy.sh --all
+
+# Run `make embed-viceroy` first to stage the asset.
+.PHONY: build-embedded
+build-embedded: config ## Compile program with embedded Viceroy
+	CGO_ENABLED=0 $(GO_BIN) build -tags viceroy_embed $(GO_ARGS) ./cmd/fastly
+
 ## Allows overriding go executable.
 GO_BIN ?= go
 ## Enables support for tools such as https://github.com/rakyll/gotest

go.mod

@@ -70,7 +70,7 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
-	github.com/klauspost/compress v1.18.6 // indirect
+	github.com/klauspost/compress v1.18.6
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-runewidth v0.0.23 // indirect

pkg/commands/compute/serve.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/fastly/cli/pkg/argparser"
 	"github.com/fastly/cli/pkg/check"
+	"github.com/fastly/cli/pkg/embedded/viceroy"
 	fsterr "github.com/fastly/cli/pkg/errors"
 	fstexec "github.com/fastly/cli/pkg/exec"
 	"github.com/fastly/cli/pkg/filesystem"
@@ -375,6 +376,20 @@ func (c *ServeCommand) GetViceroy(spinner text.Spinner, out io.Writer, manifestP
 		return filepath.Abs(path)
 	}
 
+	// Prefer the embedded binary when its version satisfies the manifest
+	// pin and the user is not forcing a refresh; this short-circuits the
+	// exec(viceroy --version) probe and download flow below.
+	if viceroy.Supported() && !c.ForceCheckViceroyLatest && embeddedVersionMatches(c.ViceroyVersioner.RequestedVersion(), viceroy.Version()) {
+		extracted, extractErr := viceroy.Extract(github.InstallDir)
+		if extractErr == nil {
+			if c.Globals.Verbose() {
+				text.Info(out, "Using embedded Viceroy v%s: %s\n\n", viceroy.Version(), extracted)
+			}
+			return extracted, nil
+		}
+		c.Globals.ErrLog.Add(extractErr)
+	}
+
 	bin = filepath.Join(github.InstallDir, c.ViceroyVersioner.BinaryName())
 
 	// NOTE: When checking if Viceroy is installed we don't use
@@ -434,6 +449,25 @@ func (c *ServeCommand) GetViceroy(spinner text.Spinner, out io.Writer, manifestP
 	return bin, nil
 }
 
+// embeddedVersionMatches reports whether a fastly.toml `viceroy_version`
+// pin is satisfied by the embedded Viceroy. An empty pin matches; a pin
+// that fails to parse does not, so the regular download path surfaces
+// the parse error to the user.
+func embeddedVersionMatches(pinned, embedded string) bool {
+	if pinned == "" {
+		return true
+	}
+	p, err := semver.Parse(strings.TrimPrefix(pinned, "v"))
+	if err != nil {
+		return false
+	}
+	e, err := semver.Parse(strings.TrimPrefix(embedded, "v"))
+	if err != nil {
+		return false
+	}
+	return p.Equals(e)
+}
+
 // checkViceroyEnvVar indicates if the CLI should use a Viceroy binary exposed
 // on the user's $PATH.
 func checkViceroyEnvVar(value string) bool {

pkg/commands/compute/serve_embed_test.go

@@ -0,0 +1,103 @@
+//go:build viceroy_embed
+
+package compute_test
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/fastly/cli/pkg/argparser"
+	"github.com/fastly/cli/pkg/commands/compute"
+	"github.com/fastly/cli/pkg/config"
+	"github.com/fastly/cli/pkg/embedded/viceroy"
+	fsterr "github.com/fastly/cli/pkg/errors"
+	"github.com/fastly/cli/pkg/github"
+	"github.com/fastly/cli/pkg/global"
+	"github.com/fastly/cli/pkg/mock"
+	"github.com/fastly/cli/pkg/testutil"
+	"github.com/fastly/cli/pkg/text"
+)
+
+func TestGetViceroyUsesEmbedded(t *testing.T) {
+	if !viceroy.Supported() {
+		t.Skipf("no embedded Viceroy for %s/%s; skipping", runtime.GOOS, runtime.GOARCH)
+	}
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rootdir := testutil.NewEnv(testutil.EnvOpts{
+		T:    t,
+		Dirs: []string{"install"},
+		Write: []testutil.FileIO{
+			{Src: "", Dst: config.FileName},
+		},
+	})
+	installDir := filepath.Join(rootdir, "install")
+	configPath := filepath.Join(rootdir, config.FileName)
+	defer os.RemoveAll(rootdir)
+
+	if err := os.Chdir(rootdir); err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = os.Chdir(wd) }()
+
+	github.InstallDir = installDir
+
+	var out bytes.Buffer
+
+	// DownloadOK=false makes any accidental fallback to the download path
+	// surface as a test failure.
+	av := mock.AssetVersioner{
+		AssetVersion:   viceroy.Version(),
+		BinaryFilename: "viceroy",
+		DownloadOK:     false,
+	}
+
+	var file config.File
+	if err := file.Read("example", strings.NewReader("yes"), &out, fsterr.MockLog{}, false); err != nil {
+		t.Fatal(err)
+	}
+
+	spinner, err := text.NewSpinner(&out)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := &compute.ServeCommand{
+		Base: argparser.Base{
+			Globals: &global.Data{
+				Config:     file,
+				ConfigPath: configPath,
+				ErrLog:     fsterr.MockLog{},
+			},
+		},
+		ViceroyVersioner: av,
+	}
+
+	bin, err := c.GetViceroy(spinner, &out, "fastly.toml")
+	if err != nil {
+		t.Fatalf("GetViceroy() error = %v", err)
+	}
+
+	wantDir := filepath.Join(installDir, "viceroy-embedded", "v"+viceroy.Version())
+	if filepath.Dir(bin) != wantDir {
+		t.Errorf("extracted binary in unexpected dir: got %q, want under %q", bin, wantDir)
+	}
+
+	if info, err := os.Stat(bin); err != nil {
+		t.Fatalf("extracted binary missing: %v", err)
+	} else if info.Size() == 0 {
+		t.Error("extracted binary is empty")
+	}
+
+	if strings.Contains(out.String(), "Fetching Viceroy release") {
+		t.Errorf("embedded path unexpectedly triggered a download: %s", out.String())
+	}
+}

pkg/commands/compute/serve_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/fastly/cli/pkg/argparser"
 	"github.com/fastly/cli/pkg/commands/compute"
 	"github.com/fastly/cli/pkg/config"
+	"github.com/fastly/cli/pkg/embedded/viceroy"
 	fsterr "github.com/fastly/cli/pkg/errors"
 	"github.com/fastly/cli/pkg/github"
 	"github.com/fastly/cli/pkg/global"
@@ -71,11 +72,18 @@ func TestGetViceroy(t *testing.T) {
 
 	var out bytes.Buffer
 
+	// Pin a version that cannot match the embedded one so this test always
+	// exercises the download path, regardless of whether the binary was
+	// built with the viceroy_embed tag.
 	av := mock.AssetVersioner{
 		AssetVersion:   "1.2.3",
 		BinaryFilename: viceroyBinName,
 		DownloadOK:     true,
 		DownloadedFile: binPath,
+		Requested:      "0.0.0-test",
+	}
+	if av.Requested == viceroy.Version() {
+		t.Fatalf("test sentinel version %q collides with embedded version", av.Requested)
 	}
 
 	var file config.File

pkg/commands/version/root.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/fastly/cli/pkg/argparser"
+	"github.com/fastly/cli/pkg/embedded/viceroy"
 	"github.com/fastly/cli/pkg/github"
 	"github.com/fastly/cli/pkg/global"
 	"github.com/fastly/cli/pkg/revision"
@@ -39,15 +40,20 @@ func (c *RootCommand) Exec(_ io.Reader, out io.Writer) error {
 	fmt.Fprintf(out, "Fastly CLI version %s (%s)\n", revision.AppVersion, revision.GitCommit)
 	fmt.Fprintf(out, "Built with %s (%s)\n", revision.GoVersion, Now().Format("2006-01-02"))
 
-	viceroy := filepath.Join(github.InstallDir, c.Globals.Versioners.Viceroy.BinaryName())
+	viceroyBin := filepath.Join(github.InstallDir, c.Globals.Versioners.Viceroy.BinaryName())
+	if viceroy.Supported() {
+		if extracted, err := viceroy.Extract(github.InstallDir); err == nil {
+			viceroyBin = extracted
+		}
+	}
 	// gosec flagged this:
 	// G204 (CWE-78): Subprocess launched with variable
 	// Disabling as we lookup the binary in a trusted location. For this to be a
 	// concern the user would need to have an already compromised system where an
 	// attacker could swap the actual viceroy executable for something malicious.
 	/* #nosec */
 	// nosemgrep
-	command := exec.Command(viceroy, "--version")
+	command := exec.Command(viceroyBin, "--version")
 	if stdoutStderr, err := command.CombinedOutput(); err == nil {
 		fmt.Fprintf(out, "Viceroy version: %s", stdoutStderr)
 	}

pkg/commands/version/version_embed_test.go

@@ -0,0 +1,58 @@
+//go:build viceroy_embed
+
+package version_test
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/fastly/cli/pkg/app"
+	"github.com/fastly/cli/pkg/commands/version"
+	"github.com/fastly/cli/pkg/embedded/viceroy"
+	"github.com/fastly/cli/pkg/github"
+	"github.com/fastly/cli/pkg/global"
+	"github.com/fastly/cli/pkg/testutil"
+)
+
+func TestVersionUsesEmbeddedViceroy(t *testing.T) {
+	if !viceroy.Supported() {
+		t.Skipf("no embedded Viceroy for %s/%s; skipping", runtime.GOOS, runtime.GOARCH)
+	}
+	if runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
+		t.Skip("skipping non-unix variants")
+	}
+
+	rootdir := testutil.NewEnv(testutil.EnvOpts{T: t})
+	orgInstallDir := github.InstallDir
+	github.InstallDir = rootdir
+	defer func() { github.InstallDir = orgInstallDir }()
+
+	version.Now = func() (t time.Time) { return t }
+
+	var stdout bytes.Buffer
+	args := testutil.SplitArgs("version")
+	opts := testutil.MockGlobalData(args, &stdout)
+	opts.Versioners = global.Versioners{
+		Viceroy: github.New(github.Opts{Org: "fastly", Repo: "viceroy", Binary: "viceroy"}),
+	}
+	app.Init = func(_ []string, _ io.Reader) (*global.Data, error) { return opts, nil }
+	if err := app.Run(args, nil); err != nil {
+		t.Fatal(err)
+	}
+
+	var mockTime time.Time
+	want := strings.Join([]string{
+		"Fastly CLI version v0.0.0-unknown (unknown)",
+		fmt.Sprintf("Built with go version %s %s/%s (%s)", runtime.Version(), runtime.GOOS, runtime.GOARCH, mockTime.Format("2006-01-02")),
+		fmt.Sprintf("Viceroy version: viceroy %s", viceroy.Version()),
+		"",
+	}, "\n")
+	if stdout.String() != want {
+		t.Errorf("unexpected output:\n got: %q\nwant: %q", stdout.String(), want)
+	}
+}

pkg/commands/version/version_test.go

@@ -1,3 +1,5 @@
+//go:build !viceroy_embed
+
 package version_test
 
 import (

pkg/embedded/viceroy/.gitignore

@@ -0,0 +1,3 @@
+# Embedded Viceroy assets are downloaded at build time by
+# scripts/fetch-viceroy.sh and are not checked in.
+assets/*.zst

pkg/embedded/viceroy/VICEROY_VERSION

@@ -0,0 +1 @@
+0.18.0

pkg/embedded/viceroy/checksums.txt

@@ -0,0 +1,12 @@
+# SHA-256 checksums of the raw Viceroy executables, pre-compression.
+# Format matches `sha256sum`. Refresh via:
+#   scripts/fetch-viceroy.sh --all --refresh-checksums
+# Building with -tags viceroy_embed verifies the downloaded asset matches.
+#
+# Pinned Viceroy version: see VICEROY_VERSION.
+
+20c6790dacd5ca86d599a5c2f0ca2083dc609b8ad87708504482e3a31c02b087  viceroy_darwin_amd64
+8957c60ef2059688b58f7581ec55b1ed6c7128ed0aac25d0f81d64b52f62da02  viceroy_darwin_arm64
+e2173c8799f73850e657e3722ce5000d3338ccdf6246790688a6b8ae8e53b896  viceroy_linux_amd64
+ef4c8d71eaabd7729c4848b246f501fdf4d3f703c1f57f3e964d81e02d38bc7d  viceroy_linux_arm64
+1f98b49e3114865ed79538d3dd86e7bb2d2d08da8ad4658d4cb925166a211817  viceroy_windows_amd64

pkg/embedded/viceroy/embed_darwin_amd64.go

@@ -0,0 +1,10 @@
+//go:build viceroy_embed && darwin && amd64
+
+package viceroy
+
+import _ "embed"
+
+//go:embed assets/viceroy_darwin_amd64.zst
+var binaryZstd []byte
+
+const platformSupported = true

pkg/embedded/viceroy/embed_darwin_arm64.go

@@ -0,0 +1,10 @@
+//go:build viceroy_embed && darwin && arm64
+
+package viceroy
+
+import _ "embed"
+
+//go:embed assets/viceroy_darwin_arm64.zst
+var binaryZstd []byte
+
+const platformSupported = true

pkg/embedded/viceroy/embed_linux_amd64.go

@@ -0,0 +1,10 @@
+//go:build viceroy_embed && linux && amd64
+
+package viceroy
+
+import _ "embed"
+
+//go:embed assets/viceroy_linux_amd64.zst
+var binaryZstd []byte
+
+const platformSupported = true

pkg/embedded/viceroy/embed_linux_arm64.go

@@ -0,0 +1,10 @@
+//go:build viceroy_embed && linux && arm64
+
+package viceroy
+
+import _ "embed"
+
+//go:embed assets/viceroy_linux_arm64.zst
+var binaryZstd []byte
+
+const platformSupported = true