-
Notifications
You must be signed in to change notification settings - Fork 23
Add authenticated host capabilities endpoint #338
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
13 commits
Select commit
Hold shift + click to select a range
56530f9
Add authenticated host capabilities endpoint
rgarcia 75b1c61
Redesign capabilities endpoint around multi-runtime registry enumeration
rgarcia 1c227c6
Make capability reporting launch-truthful and config-aware
rgarcia 4973698
Address round-2 review: probe Rosetta availability, check firecracker…
rgarcia 1555733
Address round-3 review: config-seeded default fallback, full QEMU lau…
rgarcia d2f5f90
Address round-4 review: truthful VZ fork capability, registry lock, b…
rgarcia b4a60e8
Kill the QEMU version probe's whole process group on timeout
rgarcia 3c3eb90
Kill the QEMU version probe's process group on every completion path
rgarcia 29baddc
Retry probe-script tests on the fork/exec ETXTBSY race
rgarcia 456c7bc
Deterministic probe tests: cancel on observed pids, retry exec ETXTBS…
rgarcia f86c289
Tighten QEMU launch prerequisites
rgarcia 4fb112d
Address capabilities review follow-ups
rgarcia d8248f2
Keep capability schema unchanged
rgarcia File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,233 @@ | ||
| package api | ||
|
|
||
| import ( | ||
| "context" | ||
| "runtime" | ||
| "sync" | ||
|
|
||
| "github.com/kernel/hypeman/lib/hypervisor" | ||
| "github.com/kernel/hypeman/lib/images" | ||
| "github.com/kernel/hypeman/lib/logger" | ||
| "github.com/kernel/hypeman/lib/network" | ||
| "github.com/kernel/hypeman/lib/oapi" | ||
| ) | ||
|
|
||
| // Server-level feature IDs: API surfaces this server exposes regardless of | ||
| // which runtime backs an instance. Per-runtime feature IDs are owned by | ||
| // lib/hypervisor (hypervisor.Capabilities.FeatureIDs) so adding a runtime | ||
| // capability never requires touching this handler. | ||
| const ( | ||
| featureInstances = "instances" | ||
| featureImages = "images" | ||
| featureBuilds = "builds" | ||
| featureVolumes = "volumes" | ||
| featureIngress = "ingress" | ||
| featureExec = "exec" | ||
| featureLogs = "logs" | ||
| featureDevices = "devices" | ||
| featureRosettaEmulation = "rosetta-emulation" | ||
| ) | ||
|
|
||
| // apiVersion is the API contract version from the embedded OpenAPI document. | ||
| // The decoded spec is cached: decoding it per request is needlessly expensive. | ||
| var apiVersion = sync.OnceValue(func() string { | ||
| spec, err := oapi.GetSwagger() | ||
| if err != nil || spec.Info == nil { | ||
| return "unknown" | ||
| } | ||
| return spec.Info.Version | ||
| }) | ||
|
|
||
| // defaultHypervisorProvider is the narrow accessor this handler needs from | ||
| // the instance manager. The concrete instances manager implements it; it is | ||
| // type-asserted rather than added to instances.Manager so alternate Manager | ||
| // implementations (mocks, wrappers) compiled against the public module keep | ||
| // building without a new method. | ||
| type defaultHypervisorProvider interface { | ||
| DefaultHypervisor() hypervisor.Type | ||
| } | ||
|
|
||
| // GetCapabilities reports host, runtime, network, and image capabilities. | ||
| func (s *ApiService) GetCapabilities(ctx context.Context, _ oapi.GetCapabilitiesRequestObject) (oapi.GetCapabilitiesResponseObject, error) { | ||
| log := logger.FromContext(ctx) | ||
|
|
||
| // Resolve the default runtime the way launches do. Prefer the manager's | ||
| // own effective default via the optional accessor; when the manager does | ||
| // not expose it (a wrapper embedding instances.Manager hides the concrete | ||
| // manager's extra method), fall back to the configured default the manager | ||
| // was constructed from — launches still route through the wrapped manager, | ||
| // so a hardcoded fallback would misreport a Firecracker/QEMU default as | ||
| // cloud-hypervisor. Only an empty (unconfigured) value normalizes to the | ||
| // compile-time default, mirroring lib/instances.NewManagerWithConfigE. | ||
| defaultRuntime := hypervisor.Type(s.Config.Hypervisor.Default) | ||
| if defaultRuntime == "" { | ||
| defaultRuntime = hypervisor.TypeCloudHypervisor | ||
| } | ||
| if p, ok := s.InstanceManager.(defaultHypervisorProvider); ok { | ||
| defaultRuntime = p.DefaultHypervisor() | ||
| } | ||
|
|
||
| // The capability registry is platform-gated at registration time, so its | ||
| // contents are exactly the runtimes this build supports on this host — | ||
| // including ones added after this handler was written. Capabilities are | ||
| // resolved per request so configuration applied after init (for example a | ||
| // pinned cloud-hypervisor version) remains visible. Backends may cache host | ||
| // launch-readiness probes when those prerequisites are startup state. | ||
| registered := hypervisor.RegisteredRuntimes() | ||
| runtimes := make([]oapi.CapabilitiesRuntime, 0, len(registered)) | ||
| defaultRegistered := false | ||
| defaultAvailable := false | ||
| for _, rt := range registered { | ||
| available := rt.Available() | ||
| if rt.Type == defaultRuntime { | ||
| defaultRegistered = true | ||
| defaultAvailable = available | ||
| } | ||
| if !available { | ||
| log.WarnContext(ctx, "runtime is registered but missing launch prerequisites", | ||
| "runtime", string(rt.Type), "error", rt.LaunchErr) | ||
| } | ||
| runtimes = append(runtimes, oapi.CapabilitiesRuntime{ | ||
| Name: string(rt.Type), | ||
| Available: available, | ||
| Features: rt.Capabilities.FeatureIDs(), | ||
| }) | ||
| } | ||
| if !defaultAvailable { | ||
| // A registered default is supported on this platform but missing a | ||
| // prerequisite, so ordinary launches will fail and operators need to | ||
| // act. An unregistered default simply cannot run on this platform; log | ||
| // that expected configuration mismatch at info level. | ||
| if defaultRegistered { | ||
| log.WarnContext(ctx, "configured default runtime is missing launch prerequisites", | ||
| "runtime", string(defaultRuntime), "host_os", runtime.GOOS, "host_arch", runtime.GOARCH) | ||
| } else { | ||
| log.InfoContext(ctx, "configured default runtime cannot run on this platform", | ||
| "runtime", string(defaultRuntime), "host_os", runtime.GOOS, "host_arch", runtime.GOARCH) | ||
| } | ||
| } | ||
|
|
||
| networkCaps, err := s.networkCapabilities(ctx) | ||
| if err != nil { | ||
| log.ErrorContext(ctx, "failed to resolve network capabilities", "error", err) | ||
| return oapi.GetCapabilities500JSONResponse{ | ||
| Code: "internal_error", | ||
| Message: "failed to resolve network capabilities", | ||
| }, nil | ||
| } | ||
|
|
||
| emulation := emulationAvailable(runtime.GOOS, runtime.GOARCH, rosettaInstalled()) | ||
|
|
||
| resp := oapi.Capabilities{ | ||
| Server: oapi.CapabilitiesServer{ | ||
| Version: s.Config.Version, | ||
| ApiVersion: apiVersion(), | ||
| }, | ||
| Host: oapi.CapabilitiesHost{ | ||
| Os: runtime.GOOS, | ||
| Arch: runtime.GOARCH, | ||
| }, | ||
| DefaultRuntime: oapi.CapabilitiesDefaultRuntime{ | ||
| Name: string(defaultRuntime), | ||
| Available: defaultAvailable, | ||
| }, | ||
| Runtimes: runtimes, | ||
| Network: *networkCaps, | ||
| Images: oapi.CapabilitiesImages{ | ||
| Platforms: imagePlatforms(runtime.GOARCH, emulation), | ||
| DefaultPlatform: images.HostPlatformString(), | ||
| }, | ||
| Features: serverFeatures(runtime.GOOS, emulation), | ||
| } | ||
|
|
||
| return oapi.GetCapabilities200JSONResponse(resp), nil | ||
| } | ||
|
|
||
| // networkCapabilities resolves the guest networking model and the | ||
| // guest-visible host gateway from the network manager's effective default | ||
| // network. Gateway and subnet are omitted (not serialized as empty strings) | ||
| // when no default network has been resolved. | ||
| func (s *ApiService) networkCapabilities(ctx context.Context) (*oapi.CapabilitiesNetwork, error) { | ||
| caps := &oapi.CapabilitiesNetwork{ | ||
| Model: oapiNetworkModel(network.NetworkModel()), | ||
| GuestToGuest: false, | ||
| } | ||
| if s.NetworkManager == nil { | ||
| return caps, nil | ||
| } | ||
| nw, err := s.NetworkManager.EffectiveDefaultNetwork() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if nw == nil { | ||
| return caps, nil | ||
| } | ||
| if nw.Gateway != "" { | ||
| gateway := nw.Gateway | ||
| caps.Gateway = &gateway | ||
| } | ||
| if nw.Subnet != "" { | ||
| subnet := nw.Subnet | ||
| caps.Subnet = &subnet | ||
| } | ||
| caps.GuestToGuest = network.GuestToGuestEnabled(nw) | ||
| return caps, nil | ||
| } | ||
|
|
||
| // oapiNetworkModel maps the network package's typed model onto the API enum, | ||
| // keeping lib/network free of oapi dependencies. | ||
| func oapiNetworkModel(m network.Model) oapi.CapabilitiesNetworkModel { | ||
| switch m { | ||
| case network.ModelBridge: | ||
| return oapi.Bridge | ||
| case network.ModelNAT: | ||
| return oapi.Nat | ||
| } | ||
| // A new network.Model must also be added to the OpenAPI enum; surface it | ||
| // verbatim rather than misreporting it as a known model. | ||
| return oapi.CapabilitiesNetworkModel(m) | ||
| } | ||
|
|
||
| // emulationAvailable reports whether the host can boot images built for the | ||
| // other CPU architecture right now. Only Apple Silicon macOS hosts qualify | ||
| // (vz with Rosetta), and only when the Rosetta availability probe — the same | ||
| // Virtualization.framework check the vz-shim enforces at launch — reports it | ||
| // installed. Platform eligibility alone (darwin/arm64) is deliberately not | ||
| // enough: a macOS 11/12 host or one without Rosetta installed would advertise | ||
| // launches that lib/hypervisor/vz rejects. | ||
| func emulationAvailable(goos, goarch string, rosettaInstalled bool) bool { | ||
| return goos == "darwin" && goarch == "arm64" && rosettaInstalled | ||
| } | ||
|
|
||
| // imagePlatforms returns the image platforms (os/arch) the host can run: the | ||
| // host-native Linux guest platform, plus Rosetta-emulated linux/amd64 on | ||
| // Apple Silicon with Rosetta installed. | ||
| func imagePlatforms(goarch string, emulation bool) []string { | ||
| platforms := []string{"linux/" + goarch} | ||
| if emulation { | ||
| platforms = append(platforms, "linux/amd64") | ||
| } | ||
| return platforms | ||
| } | ||
|
|
||
| // serverFeatures builds the server-level feature ID list: API surfaces that | ||
| // are always present plus host-platform conditionals. | ||
| func serverFeatures(goos string, emulation bool) []string { | ||
| features := []string{ | ||
| featureInstances, | ||
| featureImages, | ||
| featureBuilds, | ||
| featureVolumes, | ||
| featureIngress, | ||
| featureExec, | ||
| featureLogs, | ||
| } | ||
| // Device (GPU/PCI) passthrough management is only available on Linux hosts. | ||
| if goos == "linux" { | ||
| features = append(features, featureDevices) | ||
| } | ||
| if emulation { | ||
| features = append(features, featureRosettaEmulation) | ||
| } | ||
| return features | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| //go:build darwin | ||
|
|
||
| package api | ||
|
|
||
| import ( | ||
| "runtime" | ||
| "slices" | ||
| "testing" | ||
|
|
||
| "github.com/kernel/hypeman/lib/hypervisor" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| // TestGetCapabilitiesRosettaTracksProbe pins that Rosetta emulation reporting | ||
| // follows the live Virtualization.framework availability probe on this host | ||
| // — the same check the vz-shim enforces at launch — rather than treating | ||
| // every Apple Silicon host as emulation-capable: a host without Rosetta | ||
| // installed (or on macOS < 13) must advertise neither the rosetta-emulation | ||
| // feature nor the linux/amd64 image platform. | ||
| func TestGetCapabilitiesRosettaTracksProbe(t *testing.T) { | ||
| t.Parallel() | ||
| if runtime.GOARCH != "arm64" { | ||
| t.Skipf("rosetta emulation exists only on Apple Silicon (GOARCH=%s)", runtime.GOARCH) | ||
| } | ||
| svc := newTestService(t) | ||
| svc.NetworkManager = &stubCapabilitiesNetworkManager{} | ||
|
|
||
| caps := getCapabilities(t, svc) | ||
|
|
||
| want := rosettaInstalled() | ||
| require.Equal(t, want, slices.Contains(caps.Features, "rosetta-emulation"), | ||
| "rosetta-emulation feature must track the launch-path availability probe") | ||
| require.Equal(t, want, slices.Contains(caps.Images.Platforms, "linux/amd64"), | ||
| "linux/amd64 image platform must track the launch-path availability probe") | ||
| } | ||
|
|
||
| // TestRegisteredRuntimesDarwin pins the macOS registration boundary: only vz | ||
| // can genuinely launch VMs on macOS (the Linux backends require KVM and | ||
| // kernel AF_VSOCK), so the capability registry contains exactly vz even | ||
| // though the cloud-hypervisor and qemu packages are linked into the binary. | ||
| func TestRegisteredRuntimesDarwin(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| registered := hypervisor.RegisteredRuntimes() | ||
| require.Len(t, registered, 1) | ||
| require.Equal(t, hypervisor.TypeVZ, registered[0].Type) | ||
|
|
||
| for _, linuxOnly := range []hypervisor.Type{ | ||
| hypervisor.TypeCloudHypervisor, | ||
| hypervisor.TypeFirecracker, | ||
| hypervisor.TypeQEMU, | ||
| hypervisor.TypeQEMUMicroVM, | ||
| } { | ||
| _, ok := hypervisor.CapabilitiesForType(linuxOnly) | ||
| require.False(t, ok, "%s must not register capabilities on macOS", linuxOnly) | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| //go:build linux | ||
|
|
||
| package api | ||
|
|
||
| import ( | ||
| "runtime" | ||
| "testing" | ||
|
|
||
| "github.com/kernel/hypeman/lib/hypervisor" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| // TestRegisteredRuntimesLinux pins the Linux registration boundary: the | ||
| // capability registry contains exactly the runtimes launchable on this host, | ||
| // in deterministic sorted order. qemu-microvm appears only where its x86 | ||
| // board exists, and vz never registers off macOS. | ||
| func TestRegisteredRuntimesLinux(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| expected := []string{"cloud-hypervisor", "firecracker", "qemu"} | ||
| if runtime.GOARCH == "amd64" { | ||
| expected = append(expected, "qemu-microvm") | ||
| } | ||
|
|
||
| names := make([]string, 0, len(expected)) | ||
| for _, rt := range hypervisor.RegisteredRuntimes() { | ||
| names = append(names, string(rt.Type)) | ||
| } | ||
| require.Equal(t, expected, names) | ||
|
|
||
| _, vzRegistered := hypervisor.CapabilitiesForType(hypervisor.TypeVZ) | ||
| require.False(t, vzRegistered, "vz must not register capabilities on Linux") | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| //go:build darwin && arm64 | ||
|
|
||
| package api | ||
|
|
||
| import "github.com/Code-Hex/vz/v3" | ||
|
|
||
| // rosettaInstalled reports whether Rosetta translation for Linux guests is | ||
| // installed and usable right now, using the same Virtualization.framework | ||
| // probe the vz-shim enforces when a launch requests Rosetta | ||
| // (cmd/vz-shim/rosetta_arm64.go): NotInstalled (Rosetta missing) and | ||
| // NotSupported (macOS < 13) both fail launches, so capability reporting must | ||
| // not advertise emulation in either state. Evaluated per request, so | ||
| // installing Rosetta (softwareupdate --install-rosetta) is reflected without | ||
| // a restart. cmd/api already links Virtualization.framework on macOS | ||
| // (checkHypervisorAccess), so this adds no build or runtime requirement. | ||
| var rosettaInstalled = func() bool { | ||
| return vz.LinuxRosettaDirectoryShareAvailability() == vz.LinuxRosettaAvailabilityInstalled | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| //go:build !(darwin && arm64) | ||
|
|
||
| package api | ||
|
|
||
| // rosettaInstalled is never true off Apple Silicon macOS: Rosetta emulation | ||
| // exists only under vz. Kept as a var with the same shape as the | ||
| // darwin/arm64 probe so the handler code is identical on every platform. | ||
| var rosettaInstalled = func() bool { return false } |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.