Fix Buildx compatibility preflight

README.md

@@ -52,8 +52,8 @@ retained when another workload changes. A relative bind source remains tied to
 its release directory and is recreated safely; an unchanged absolute host bind
 can be retained.
 
-You administer Linux, SSH access, and Docker. Onebox owns the generated
-application runtime inside that boundary.
+You administer Linux, SSH access, and Docker with the Buildx plugin. Onebox
+owns the generated application runtime inside that boundary.
 
 ## Quick start
 

cmd/ob/preflight.go

@@ -18,8 +18,9 @@ func addPreflightCommand(root *cobra.Command, g *globalFlags) {
 		Use:   "preflight",
 		Short: "ask the server whether this project could be deployed (changes nothing)",
 		Long: "Render the project locally, then ask the server what would stand in the way:\n" +
-			"a missing container runtime, a base path this account cannot write, a derived\n" +
-			"name already held by something Onebox does not own, a missing ingress network.\n\n" +
+			"a missing container runtime, a missing or incompatible Docker Buildx image\n" +
+			"resolver, a base path this account cannot write, a derived name already held\n" +
+			"by something Onebox does not own, or a missing ingress network.\n\n" +
 			"Every problem is reported at once rather than the first one, and nothing is\n" +
 			"created, renamed or removed.",
 		RunE: func(cmd *cobra.Command, _ []string) error {

e2e/server_test.go

@@ -22,17 +22,11 @@ func (s *server) requireDocker(t *testing.T) {
 	}
 	// Docker's own packages, not Ubuntu's.
 	//
-	// `apt install docker.io docker-buildx` looks equivalent and is not: that
-	// buildx (0.30.1-0ubuntu1) accepts `--format` on `imagetools inspect` and
-	// ignores it, printing the human-readable manifest and exiting 0. ob pins
-	// every workload image through exactly that command
-	// (internal/engine/plan.go), so on such a host no deploy can succeed — and
-	// the error it produces blames the registry.
-	//
 	// This installs what an operator following Docker's instructions gets,
-	// which is the configuration the product is really used in. The Ubuntu
-	// packaging is a real gap and belongs in an issue, not in a fixture that
-	// quietly avoids it.
+	// including the Compose and Buildx plugins documented as host requirements.
+	// Some packaged Buildx clients ignore a plain Go template while honoring
+	// its JSON template form; image resolution uses that compatible form and
+	// validates the result before treating it as a digest.
 	s.run(t, strings.Join([]string{
 		"set -e",
 		"export DEBIAN_FRONTEND=noninteractive",

internal/app/buildx.go

@@ -0,0 +1,47 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"strings"
+)
+
+// BuildxCapabilityCommand is deliberately local-only. Preflight must prove
+// that the installed client can format an image manifest without contacting a
+// registry (and consuming pull quota) or changing the host.
+const BuildxCapabilityCommand = "docker buildx imagetools inspect --help"
+
+// BuildxRemedy is shared by preflight and planning so a client-side failure is
+// never presented as a registry problem.
+const BuildxRemedy = "install or upgrade the Docker Buildx plugin on the server, then rerun ob preflight"
+
+// BuildxCapabilityError means the server was reachable but its local Docker
+// client cannot perform the digest-resolution operation planning requires.
+type BuildxCapabilityError struct {
+	Detail string
+}
+
+func (err *BuildxCapabilityError) Error() string { return err.Detail }
+
+// CheckBuildxDigestSupport verifies the local CLI surface used by PinImages.
+// It does not inspect an image and therefore never contacts a registry.
+func CheckBuildxDigestSupport(ctx context.Context, run Runner) (string, error) {
+	res, err := run.Run(ctx, BuildxCapabilityCommand)
+	if err != nil {
+		return "", err
+	}
+	output := strings.TrimSpace(strings.Join([]string{res.Stdout, res.Stderr}, "\n"))
+	if res.ExitCode != 0 {
+		detail := strings.TrimSpace(firstLine(output))
+		if detail == "" {
+			detail = fmt.Sprintf("docker buildx imagetools inspect --help exited with status %d", res.ExitCode)
+		}
+		return "", &BuildxCapabilityError{Detail: "Docker Buildx is unavailable: " + detail}
+	}
+	if !strings.Contains(output, "--format") {
+		return "", &BuildxCapabilityError{
+			Detail: "Docker Buildx is incompatible: imagetools inspect does not advertise --format support",
+		}
+	}
+	return "imagetools inspect --format available", nil
+}

internal/app/preflight.go

@@ -1,14 +1,15 @@
 package app
 
 import (
+	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 
-	"bytes"
 	"github.com/labstack/onebox/internal/shellquote"
 	"github.com/labstack/onebox/internal/transport"
 
@@ -97,12 +98,30 @@ func (r *Resolved) Preflight(ctx context.Context, run Runner) (*Report, error) {
 		Detail: "docker " + strings.TrimSpace(res.Stdout),
 	})
 
-	// 2. The base path. Checked without creating anything: preflight that
+	// 2. The image resolver. This only reads the local help output: using an
+	// image as the probe would spend registry quota before planning begins.
+	buildxDetail, err := CheckBuildxDigestSupport(ctx, run)
+	if err != nil {
+		var capabilityErr *BuildxCapabilityError
+		if !errors.As(err, &capabilityErr) {
+			return nil, errf("server_unreachable", "", "ob doctor",
+				"cannot verify Docker Buildx on the server: %v", err)
+		}
+		report.Checks = append(report.Checks, Check{
+			Name: "image resolver", Detail: capabilityErr.Error(), Remedy: BuildxRemedy,
+		})
+	} else {
+		report.Checks = append(report.Checks, Check{
+			Name: "image resolver", OK: true, Detail: buildxDetail,
+		})
+	}
+
+	// 3. The base path. Checked without creating anything: preflight that
 	// mutates is not preflight.
 	report.Checks = append(report.Checks, basePathCheck(ctx, run, n.BasePath))
 	report.Checks = append(report.Checks, hostOwnerCheck(ctx, run, n.HostOwnerPath(), p.Name, r.Env))
 
-	// 3. Name collisions. One listing per resource kind rather than one command
+	// 4. Name collisions. One listing per resource kind rather than one command
 	// per name — a project with twenty derived names should not cost twenty
 	// round trips.
 	owned, err := ownedNames(ctx, run, p, r.Env)
@@ -111,7 +130,7 @@ func (r *Resolved) Preflight(ctx context.Context, run Runner) (*Report, error) {
 	}
 	report.Checks = append(report.Checks, collisionChecks(p.Name, p.All(r.Env), owned)...)
 
-	// 4. The ingress network, which the proxy owns and this project only joins.
+	// 5. The ingress network, which the proxy owns and this project only joins.
 	if p.Proxy.Kind != "none" && p.routesAnywhere() {
 		report.Checks = append(report.Checks, networkCheck(ctx, run, p.Proxy.Network))
 	}

internal/app/preflight_test.go

@@ -42,6 +42,7 @@ func (f *fakeRunner) Run(_ context.Context, cmd string) (transport.Result, error
 func healthyRunner() *fakeRunner {
 	return &fakeRunner{answers: map[string]transport.Result{
 		"docker version": {Stdout: "27.1.1\n"},
+		"docker buildx imagetools inspect --help": {Stdout: "Usage: docker buildx imagetools inspect [OPTIONS] NAME\n      --format string\n"},
 		"/_host/owner":   {Stdout: "ledger\n"},
 		"docker ps":      {Stdout: ""},
 		"docker volume":  {Stdout: ""},
@@ -93,6 +94,48 @@ func TestPreflightPassesOnAReadyHost(t *testing.T) {
 	if !rep.OK() {
 		t.Fatalf("expected a ready host to pass: %+v", rep.Failures())
 	}
+	found := false
+	for _, check := range rep.Checks {
+		if check.Name == "image resolver" && check.OK {
+			found = true
+		}
+	}
+	if !found {
+		t.Fatalf("compatible Buildx was not reported: %+v", rep.Checks)
+	}
+}
+
+func TestPreflightRejectsMissingBuildxWithSpecificRemedy(t *testing.T) {
+	run := healthyRunner()
+	run.answers["docker buildx imagetools inspect --help"] = transport.Result{
+		ExitCode: 1, Stderr: "docker: 'buildx' is not a docker command.\n",
+	}
+
+	rep := preflight(t, run, preflightProject)
+	for _, failure := range rep.Failures() {
+		if failure.Name == "image resolver" {
+			if !strings.Contains(failure.Detail, "Buildx is unavailable") || failure.Remedy != BuildxRemedy {
+				t.Fatalf("missing Buildx diagnosis = %+v", failure)
+			}
+			return
+		}
+	}
+	t.Fatalf("missing Buildx passed preflight: %+v", rep.Checks)
+}
+
+func TestPreflightRejectsBuildxWithoutDigestFormatSupport(t *testing.T) {
+	run := healthyRunner()
+	run.answers["docker buildx imagetools inspect --help"] = transport.Result{
+		Stdout: "Usage: docker buildx imagetools inspect [OPTIONS] NAME\n",
+	}
+
+	rep := preflight(t, run, preflightProject)
+	for _, failure := range rep.Failures() {
+		if failure.Name == "image resolver" && strings.Contains(failure.Detail, "does not advertise --format") {
+			return
+		}
+	}
+	t.Fatalf("incompatible Buildx passed preflight: %+v", rep.Checks)
 }
 
 // TestPreflightOnlyReads is the promise the phase is named for. A preflight
@@ -108,6 +151,9 @@ func TestPreflightOnlyReads(t *testing.T) {
 		"mkdir", "rm ", "touch ", "docker rename", "> ",
 	}
 	for _, cmd := range run.ran {
+		if strings.Contains(cmd, "imagetools inspect") && cmd != BuildxCapabilityCommand {
+			t.Errorf("preflight contacted a registry during its Buildx probe: %q", cmd)
+		}
 		for _, m := range mutating {
 			if strings.Contains(cmd, m) {
 				t.Errorf("preflight ran a mutating command: %q", cmd)
@@ -117,6 +163,15 @@ func TestPreflightOnlyReads(t *testing.T) {
 	if len(run.ran) == 0 {
 		t.Fatal("preflight asked the target nothing")
 	}
+	foundCapabilityProbe := false
+	for _, cmd := range run.ran {
+		if cmd == BuildxCapabilityCommand {
+			foundCapabilityProbe = true
+		}
+	}
+	if !foundCapabilityProbe {
+		t.Fatalf("preflight did not run the local-only Buildx probe: %v", run.ran)
+	}
 }
 
 // TestPreflightReportsEveryProblem: a caller should see all of it at once

internal/engine/plan.go

@@ -224,6 +224,7 @@ func (e *Engine) PinImages(ctx context.Context) (map[string]string, error) {
 	}
 	sort.Strings(services)
 	resolved := map[string]string{}
+	buildxChecked := false
 	for _, svc := range services {
 		s, ok := e.Compose.Services[svc]
 		if !ok {
@@ -242,18 +243,34 @@ func (e *Engine) PinImages(ctx context.Context) (map[string]string, error) {
 			pins[svc] = pinned
 			continue
 		}
-		res, err := e.T.Run(ctx, "docker buildx imagetools inspect "+q(s.Image)+" --format '{{.Manifest.Digest}}'")
+		if !buildxChecked {
+			if _, err := app.CheckBuildxDigestSupport(ctx, e.T); err != nil {
+				var capabilityErr *app.BuildxCapabilityError
+				if errors.As(err, &capabilityErr) {
+					return nil, imageResolutionError(svc, s.Image, capabilityErr.Error()+"; "+app.BuildxRemedy)
+				}
+				return nil, imageResolutionError(svc, s.Image, "could not verify Docker Buildx capability: "+err.Error())
+			}
+			buildxChecked = true
+		}
+		res, err := e.T.Run(ctx, "docker buildx imagetools inspect "+q(s.Image)+" --format '{{json .Manifest.Digest}}'")
 		if err != nil {
 			return nil, imageResolutionError(svc, s.Image, err.Error())
 		}
-		digest := strings.TrimSpace(res.Stdout)
-		if res.ExitCode != 0 || !digestRe.MatchString(digest) {
+		if res.ExitCode != 0 {
 			detail := strings.TrimSpace(res.Stderr)
 			if detail == "" {
-				detail = fmt.Sprintf("registry inspection returned exit %d without a valid sha256 digest", res.ExitCode)
+				detail = fmt.Sprintf("registry inspection failed with exit %d", res.ExitCode)
+			} else {
+				detail = "registry inspection failed: " + detail
 			}
 			return nil, imageResolutionError(svc, s.Image, detail)
 		}
+		var digest string
+		if err := json.Unmarshal([]byte(strings.TrimSpace(res.Stdout)), &digest); err != nil || !digestRe.MatchString(digest) {
+			return nil, imageResolutionError(svc, s.Image,
+				"Docker Buildx accepted --format but returned incompatible digest output; "+app.BuildxRemedy)
+		}
 		pinned, err := imageref.WithDigest(s.Image, digest)
 		if err != nil {
 			return nil, imageResolutionError(svc, s.Image, err.Error())

internal/engine/plan_test.go

@@ -30,8 +30,10 @@ func planFake() *transport.Fake {
 			return transport.Result{Stdout: "\n"}, true
 		case strings.Contains(cmd, "{{.Image}}"):
 			return transport.Result{Stdout: "sha256:aaaa\n"}, true
+		case cmd == app.BuildxCapabilityCommand:
+			return transport.Result{Stdout: "Usage: docker buildx imagetools inspect [OPTIONS] NAME\n      --format string\n"}, true
 		case strings.Contains(cmd, "imagetools inspect"):
-			return transport.Result{Stdout: "sha256:" + strings.Repeat("ab", 32) + "\n"}, true
+			return transport.Result{Stdout: `"sha256:` + strings.Repeat("ab", 32) + `"` + "\n"}, true
 		case strings.Contains(cmd, "cat "):
 			return transport.Result{Stdout: "services: {}\n"}, true
 		}
@@ -77,8 +79,11 @@ func TestPinImagesRewritesToDigest(t *testing.T) {
 	}
 	inspections := 0
 	for _, command := range f.Commands {
-		if strings.Contains(command, "imagetools inspect") {
+		if strings.Contains(command, "imagetools inspect") && command != app.BuildxCapabilityCommand {
 			inspections++
+			if !strings.Contains(command, "{{json .Manifest.Digest}}") {
+				t.Fatalf("registry inspection did not use the compatible JSON template: %q", command)
+			}
 		}
 	}
 	if inspections != 1 {
@@ -90,8 +95,8 @@ func TestPinImagesFailsClosedWhenRegistryCannotResolveDigest(t *testing.T) {
 	f := planFake()
 	base := f.Dynamic
 	f.Dynamic = func(cmd string) (transport.Result, bool) {
-		if strings.Contains(cmd, "imagetools inspect") {
-			return transport.Result{ExitCode: 1, Stderr: "buildx: not found"}, true
+		if strings.Contains(cmd, "imagetools inspect") && cmd != app.BuildxCapabilityCommand {
+			return transport.Result{ExitCode: 1, Stderr: "unauthorized: registry credentials rejected"}, true
 		}
 		return base(cmd)
 	}
@@ -106,6 +111,50 @@ func TestPinImagesFailsClosedWhenRegistryCannotResolveDigest(t *testing.T) {
 		!strings.Contains(resolution.ResolvingCommand, "ob plan --image "+resolution.Workload+"=") {
 		t.Fatalf("typed resolution error = %#v", resolution)
 	}
+	if !strings.Contains(resolution.Detail, "registry credentials rejected") || strings.Contains(resolution.Detail, "Buildx") {
+		t.Fatalf("registry failure was misdiagnosed: %#v", resolution)
+	}
+}
+
+func TestPinImagesFailsClosedWhenBuildxIsMissing(t *testing.T) {
+	f := planFake()
+	base := f.Dynamic
+	f.Dynamic = func(cmd string) (transport.Result, bool) {
+		if cmd == app.BuildxCapabilityCommand {
+			return transport.Result{ExitCode: 1, Stderr: "docker: 'buildx' is not a docker command"}, true
+		}
+		return base(cmd)
+	}
+	e := New(testConfig(), testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep})
+	_, err := e.PinImages(context.Background())
+	var resolution *ImageResolutionError
+	if !errors.As(err, &resolution) || !strings.Contains(resolution.Detail, "Buildx is unavailable") ||
+		!strings.Contains(resolution.Detail, app.BuildxRemedy) {
+		t.Fatalf("missing Buildx error = %#v (%v)", resolution, err)
+	}
+	for _, command := range f.Commands {
+		if strings.Contains(command, "imagetools inspect") && command != app.BuildxCapabilityCommand {
+			t.Fatalf("missing Buildx still contacted a registry: %q", command)
+		}
+	}
+}
+
+func TestPinImagesRejectsSuccessfulInspectWithIncompatibleOutput(t *testing.T) {
+	f := planFake()
+	base := f.Dynamic
+	f.Dynamic = func(cmd string) (transport.Result, bool) {
+		if strings.Contains(cmd, "imagetools inspect") && cmd != app.BuildxCapabilityCommand {
+			return transport.Result{Stdout: "Name: ghcr.io/x/app:latest\nMediaType: application/vnd.oci.image.index.v1+json\n"}, true
+		}
+		return base(cmd)
+	}
+	e := New(testConfig(), testProject(t), f, Options{Out: &bytes.Buffer{}, Sleep: noSleep})
+	_, err := e.PinImages(context.Background())
+	var resolution *ImageResolutionError
+	if !errors.As(err, &resolution) ||
+		!strings.Contains(resolution.Detail, "Buildx accepted --format but returned incompatible digest output") {
+		t.Fatalf("incompatible successful output error = %#v (%v)", resolution, err)
+	}
 }
 
 func TestPinImagesFailsClosedForBuildOnlyRuntimeService(t *testing.T) {

internal/onebox/service_test.go

@@ -96,8 +96,10 @@ func serviceFake() *transport.Fake {
 				return transport.Result{Stdout: "healthy\n"}, true
 			case strings.Contains(cmd, "docker inspect") && strings.Contains(cmd, "{{.Image}}"):
 				return transport.Result{Stdout: "sha256:" + strings.Repeat("ef", 32) + "\n"}, true
+			case cmd == app.BuildxCapabilityCommand:
+				return transport.Result{Stdout: "Usage: docker buildx imagetools inspect [OPTIONS] NAME\n      --format string\n"}, true
 			case strings.Contains(cmd, "docker buildx imagetools inspect"):
-				return transport.Result{Stdout: digest + "\n"}, true
+				return transport.Result{Stdout: `"` + digest + `"` + "\n"}, true
 			case strings.Contains(cmd, "cat ") && strings.Contains(cmd, "compose.yaml"):
 				return transport.Result{Stdout: "services:\n  web:\n    image: ghcr.io/example/app:v0\n    environment:\n      SECRET_TOKEN: live-secret\n"}, true
 			case strings.Contains(cmd, "find . -type f"):

site/src/content/docs/reference/cli.mdx

@@ -775,8 +775,9 @@ Global Flags:
 
 ```
 Render the project locally, then ask the server what would stand in the way:
-a missing container runtime, a base path this account cannot write, a derived
-name already held by something Onebox does not own, a missing ingress network.
+a missing container runtime, a missing or incompatible Docker Buildx image
+resolver, a base path this account cannot write, a derived name already held
+by something Onebox does not own, or a missing ingress network.
 
 Every problem is reported at once rather than the first one, and nothing is
 created, renamed or removed.