diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 13835a8386..9227a332e5 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -639,6 +639,8 @@ jobs: install-kind: false requires-secret: false install-microsandbox: true + test-timeout: 1200s + job-timeout-minutes: 25 # Snapshot tests @@ -943,35 +945,36 @@ jobs: --crun-path /usr/local/bin/crun \ --bootstrap-timeout 270s - - name: get microsandbox latest version + - name: cache microsandbox release bundle (Linux) if: matrix.install-microsandbox == true && runner.os == 'Linux' - id: msb-version - run: | - version=$(curl -fsSL https://api.github.com/repos/superradcompany/microsandbox/releases/latest | grep '"tag_name"' | head -1 | sed 's/.*"tag_name":[[:space:]]*"\([^"]*\)".*/\1/') - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: cache microsandbox (Linux) - if: matrix.install-microsandbox == true && runner.os == 'Linux' - id: msb-cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: | - ~/.microsandbox - ~/.local/bin/msb - ~/.local/bin/microsandbox - key: ${{ runner.os }}-${{ runner.arch }}-microsandbox-${{ steps.msb-version.outputs.version }} - - - name: Install microsandbox (Linux) - if: matrix.install-microsandbox == true && runner.os == 'Linux' && steps.msb-cache.outputs.cache-hit != 'true' - run: | - curl -fsSL https://install.microsandbox.dev | sh + path: ${{ runner.temp }}/microsandbox-linux-x86_64-v0.7.7.tar.gz + key: ${{ runner.os }}-${{ runner.arch }}-microsandbox-bundle-v0.7.7 - - name: configure microsandbox (Linux) + - name: install checksum-pinned microsandbox (Linux) if: matrix.install-microsandbox == true && runner.os == 'Linux' + timeout-minutes: 5 run: | + archive="${RUNNER_TEMP}/microsandbox-linux-x86_64-v0.7.7.tar.gz" + if [ ! -f "$archive" ]; then + curl -fsSL --retry 3 -o "$archive" \ + https://github.com/superradcompany/microsandbox/releases/download/v0.7.7/microsandbox-linux-x86_64.tar.gz + fi + echo "b3cc4a5e3f52dfdd938a6f67ac4a9a959ddfe304bab56de4964044b8613f01bb $archive" | sha256sum -c - + staging="$(mktemp -d)" + trap 'rm -rf "$staging"' EXIT + tar -xzf "$archive" -C "$staging" + mkdir -p "$HOME/.microsandbox/bin" "$HOME/.microsandbox/lib" "$HOME/.local/bin" + install -m 755 "$staging/msb" "$HOME/.microsandbox/bin/msb" + install -m 644 "$staging/libkrunfw.so.5.6.1" "$HOME/.microsandbox/lib/" + ln -sf libkrunfw.so.5.6.1 "$HOME/.microsandbox/lib/libkrunfw.so.5" + ln -sf libkrunfw.so.5 "$HOME/.microsandbox/lib/libkrunfw.so" + ln -sf "$HOME/.microsandbox/bin/msb" "$HOME/.local/bin/msb" echo "$HOME/.local/bin" >> "$GITHUB_PATH" - # The e2e test skips gracefully if KVM is unavailable on the runner. - "$HOME/.local/bin/msb" doctor || true + "$HOME/.local/bin/msb" --version + sudo test -r /dev/kvm + sudo test -w /dev/kvm - name: remove docker if: matrix.label == 'docker-install' && (matrix.requires-secret == false || needs.can-read-secret.outputs.secret-set == 'true') @@ -1011,6 +1014,7 @@ jobs: GH_CREDENTIAL_USERNAME: x-access-token TEST_TIMEOUT: ${{ matrix.test-timeout || '1500s' }} FLAKE_ATTEMPTS: ${{ matrix.flake-attempts || '1' }} + DEVSY_REQUIRE_MICROSANDBOX: ${{ matrix.install-microsandbox == true && 'true' || 'false' }} run: | if [ "${{ runner.os }}" == "Linux" ]; then if [ "${{ matrix.install-podman || '' }}" = "rootless" ]; then @@ -1027,6 +1031,7 @@ jobs: ./e2e.test -test.v -ginkgo.v -test.timeout "${TEST_TIMEOUT}" -ginkgo.fail-on-empty -ginkgo.flake-attempts="${FLAKE_ATTEMPTS}" --ginkgo.label-filter="${{ matrix.label }}" else sudo \ + DEVSY_REQUIRE_MICROSANDBOX="${DEVSY_REQUIRE_MICROSANDBOX}" \ GH_USERNAME="${GH_USERNAME}" \ GH_ACCESS_TOKEN="${GH_ACCESS_TOKEN}" \ GH_CREDENTIAL_USERNAME="${GH_CREDENTIAL_USERNAME}" \ diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md index b5fef46da7..c9f63bd7e5 100644 --- a/THIRD_PARTY_LICENSES.md +++ b/THIRD_PARTY_LICENSES.md @@ -125,7 +125,7 @@ To regenerate this file after changing dependencies, run `task cli:licenses`. | [github.com/devsy-org/agentapi](https://github.com/devsy-org/agentapi) | `v1.0.1` | MPL-2.0 | | [github.com/devsy-org/api](https://github.com/devsy-org/api) | `v1.1.0` | MPL-2.0 | | [github.com/devsy-org/apiserver](https://github.com/devsy-org/apiserver) | `v1.5.4` | Apache-2.0 | -| [github.com/devsy-org/devsy-runtime-sdk](https://github.com/devsy-org/devsy-runtime-sdk) | `v1.4.0` | MPL-2.0 | +| [github.com/devsy-org/devsy-runtime-sdk](https://github.com/devsy-org/devsy-runtime-sdk) | `v1.5.2` | MPL-2.0 | | [github.com/devsy-org/ssh](https://github.com/devsy-org/ssh) | `v1.2.9` | BSD-3-Clause | | [github.com/distribution/reference](https://github.com/distribution/reference) | `v0.6.0` | Apache-2.0 | | [github.com/docker/cli](https://github.com/docker/cli) | `v29.8.0+incompatible` | Apache-2.0 | diff --git a/cmd/internal/agentworkspace/binaries_test.go b/cmd/internal/agentworkspace/binaries_test.go new file mode 100644 index 0000000000..57a14e9523 --- /dev/null +++ b/cmd/internal/agentworkspace/binaries_test.go @@ -0,0 +1,356 @@ +package agentworkspace + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/devsy-org/devsy/pkg/agent" + "github.com/devsy-org/devsy/pkg/agent/tunnel" + "github.com/devsy-org/devsy/pkg/compress" + "github.com/devsy-org/devsy/pkg/config" + devcontainerconfig "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/devsy-org/devsy/pkg/ssh" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +func TestExistingContentPreparesAgentBinaries(t *testing.T) { + for _, tc := range []struct { + name string + validChecksum bool + }{{"success", true}, {"checksum failure", false}} { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(config.EnvHome, t.TempDir()) + payload := []byte("workspace-runtime-fixture") + server := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(payload) + }), + ) + t.Cleanup(server.Close) + info := existingContentRuntime(t, server.URL, payload) + marker := filepath.Join(info.ContentFolder, "user-data") + require.NoError(t, os.WriteFile(marker, []byte("preserved"), 0o600)) + if !tc.validChecksum { + info.Agent.Binaries["RUNTIME"][0].Checksum = hex.EncodeToString( + make([]byte, sha256.Size), + ) + } + exists, err := InitContentFolder(context.Background(), info) + require.True(t, exists) + if tc.validChecksum { + require.NoError(t, err) + data, err := fs.ReadFile(os.DirFS(info.Origin), "binaries/runtime/runtime-fixture") + require.NoError(t, err) + require.Equal(t, payload, data) + } else { + require.ErrorContains(t, err, "checksum") + } + data, err := fs.ReadFile(os.DirFS(info.ContentFolder), "user-data") + require.NoError(t, err) + require.Equal(t, "preserved", string(data)) + }) + } +} + +func existingContentRuntime(t *testing.T, url string, payload []byte) *provider.AgentWorkspaceInfo { + t.Helper() + home := t.TempDir() + origin := filepath.Join(home, "contexts", config.DefaultContext, "workspaces", "binary-test") + require.NoError(t, os.MkdirAll(origin, 0o750)) + sum := sha256.Sum256(payload) + return &provider.AgentWorkspaceInfo{ + Origin: origin, ContentFolder: t.TempDir(), + Workspace: &provider.Workspace{Context: config.DefaultContext, ID: "binary-test"}, + Agent: provider.ProviderAgentConfig{ + DataPath: home, + Binaries: map[string][]*provider.ProviderBinary{"RUNTIME": {{ + OS: runtime.GOOS, Arch: runtime.GOARCH, Path: url, Name: "runtime-fixture", + Checksum: hex.EncodeToString(sum[:]), + }}}, + }, + } +} + +func TestBinaryPreparationFailureCleanup(t *testing.T) { + for _, existing := range []bool{true, false} { + t.Run(fmt.Sprintf("existing=%t", existing), func(t *testing.T) { + info, sshConfig := binaryCleanupWorkspace(t, existing) + err := prepareWorkspace( + context.Background(), + prepareWorkspaceParams{workspaceInfo: info}, + ) + require.ErrorContains(t, err, "checksum") + initErr := fmt.Errorf("initialize workspace: %w", err) + cmd := &UpCmd{} + require.ErrorIs(t, cmd.handleInitError(initErr, info), initErr) + + if !existing { + require.NoDirExists(t, info.Origin) + require.NoDirExists(t, info.ContentFolder) + data, err := fs.ReadFile( + os.DirFS(filepath.Dir(info.Workspace.SSHConfigPath)), + "ssh_config", + ) + require.NoError(t, err) + require.NotContains(t, string(data), "binary-test") + return + } + for _, file := range []struct{ dir, name, want string }{ + {info.Origin, provider.WorkspaceConfigFile, "workspace record"}, + {info.ContentFolder, "user-data", "preserved"}, + {filepath.Dir(info.Workspace.SSHConfigPath), "ssh_config", string(sshConfig)}, + } { + data, err := fs.ReadFile(os.DirFS(file.dir), file.name) + require.NoError(t, err) + require.Equal(t, file.want, string(data)) + } + }) + } +} + +func TestLocalFolderBinaryFailurePreservesUserContent(t *testing.T) { + for _, existing := range []bool{false, true} { + for _, managedPath := range []bool{false, true} { + t.Run( + fmt.Sprintf("existing=%t/managed_path=%t", existing, managedPath), + func(t *testing.T) { + info, sshConfig := binaryCleanupWorkspace(t, true) + info.WorkspaceWasExisting = existing + if !managedPath { + info.ContentFolder = t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join( + info.ContentFolder, + "user-data", + ), + []byte("preserved"), + 0o600, + )) + } + info.Workspace.Source.LocalFolder = info.ContentFolder + err := prepareWorkspace( + context.Background(), + prepareWorkspaceParams{workspaceInfo: info}, + ) + require.ErrorContains(t, err, "checksum") + require.ErrorIs(t, (&UpCmd{}).handleInitError(err, info), err) + data, readErr := os.ReadFile(filepath.Join(info.ContentFolder, "user-data")) + require.NoError(t, readErr) + require.Equal(t, "preserved", string(data)) + data, readErr = os.ReadFile(info.Workspace.SSHConfigPath) + require.NoError(t, readErr) + if existing { + record, recordErr := os.ReadFile( + filepath.Join(info.Origin, provider.WorkspaceConfigFile), + ) + require.NoError(t, recordErr) + require.Equal(t, "workspace record", string(record)) + require.Equal(t, sshConfig, data) + } else { + require.NoDirExists(t, info.Origin) + require.NotContains(t, string(data), "binary-test") + } + }, + ) + } + } +} + +func binaryCleanupWorkspace(t *testing.T, existing bool) (*provider.AgentWorkspaceInfo, []byte) { + t.Helper() + home := t.TempDir() + t.Setenv(config.EnvHome, home) + payload := []byte("invalid-checksum-runtime") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(payload) + })) + t.Cleanup(server.Close) + info := existingContentRuntime(t, server.URL, payload) + info.Agent.DataPath = home + var err error + info.Origin, err = provider.GetWorkspaceDir(info.Workspace.Context, info.Workspace.ID) + require.NoError(t, err) + info.ContentFolder, err = provider.GetWorkspaceContentDir( + info.Workspace.Context, + info.Workspace.ID, + ) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(info.Origin, 0o750)) + require.NoError( + t, + os.WriteFile( + filepath.Join(info.Origin, provider.WorkspaceConfigFile), + []byte("workspace record"), + 0o600, + ), + ) + if existing { + require.NoError(t, os.MkdirAll(info.ContentFolder, 0o750)) + require.NoError( + t, + os.WriteFile( + filepath.Join(info.ContentFolder, "user-data"), + []byte("preserved"), + 0o600, + ), + ) + } + info.Workspace.SSHConfigPath = filepath.Join(t.TempDir(), "ssh_config") + require.NoError(t, ssh.ConfigureSSHConfig(ssh.SSHConfigParams{ + SSHConfigPath: info.Workspace.SSHConfigPath, + Context: info.Workspace.Context, Workspace: info.Workspace.ID, User: "root", + })) + sshConfig, err := fs.ReadFile( + os.DirFS(filepath.Dir(info.Workspace.SSHConfigPath)), + "ssh_config", + ) + require.NoError(t, err) + info.Agent.Binaries["RUNTIME"][0].Checksum = hex.EncodeToString(make([]byte, sha256.Size)) + return info, sshConfig +} + +func TestReusedWorkspaceWithoutContentPreservesState(t *testing.T) { + info, sshConfig := binaryCleanupWorkspace(t, false) + info.Workspace.UID = "same-workspace-uid" + info.Agent.Local = config.BoolTrue + data, err := json.Marshal(info) + require.NoError(t, err) + require.NoError( + t, + os.WriteFile(filepath.Join(info.Origin, provider.WorkspaceConfigFile), data, 0o600), + ) + encoded, err := compress.Compress(string(data)) + require.NoError(t, err) + shouldExit, loaded, err := agent.WriteWorkspaceInfoAndDeleteOld( + encoded, + func(*provider.AgentWorkspaceInfo) error { + t.Fatal("same-UID workspace must not be replaced") + return nil + }, + ) + require.NoError(t, err) + require.False(t, shouldExit) + require.NotNil(t, loaded) + record, err := fs.ReadFile(os.DirFS(loaded.Origin), provider.WorkspaceConfigFile) + require.NoError(t, err) + err = prepareWorkspace(context.Background(), prepareWorkspaceParams{workspaceInfo: loaded}) + require.ErrorContains(t, err, "checksum") + cmd := &UpCmd{} + require.ErrorIs(t, cmd.handleInitError(err, loaded), err) + got, err := fs.ReadFile(os.DirFS(loaded.Origin), provider.WorkspaceConfigFile) + require.NoError(t, err) + require.Equal(t, record, got) + got, err = fs.ReadFile(os.DirFS(filepath.Dir(loaded.Workspace.SSHConfigPath)), "ssh_config") + require.NoError(t, err) + require.Equal(t, sshConfig, got) + require.NoDirExists(t, loaded.ContentFolder) +} + +func TestSourcePreparationFailureCleanupAndRetry(t *testing.T) { + for _, tc := range []struct { + name string + existing, fallbackConfig bool + }{ + {name: "new content"}, + {name: "existing content", existing: true}, + {name: "fallback config", fallbackConfig: true}, + } { + t.Run(tc.name, func(t *testing.T) { + info, sshConfig := binaryCleanupWorkspace(t, tc.existing) + info.Agent.Binaries = nil + info.WorkspaceWasExisting = true + info.Workspace.Source.LocalFolder = "host-source" + info.CLIOptions.Recreate = tc.existing || tc.fallbackConfig + if tc.fallbackConfig { + info.LastDevContainerConfig = &devcontainerconfig.DevContainerConfigWithPath{ + Path: ".devcontainer.json", Config: &devcontainerconfig.DevContainerConfig{}, + } + } + uploadErr := errors.New("source stream interrupted") + var archive bytes.Buffer + tw := tar.NewWriter(&archive) + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: "source.txt", Mode: 0o600, Size: 6, + })) + _, err := tw.Write([]byte("source")) + require.NoError(t, err) + require.NoError(t, tw.Close()) + client := &sourceRetryClient{archive: archive.Bytes(), firstError: uploadErr} + params := prepareWorkspaceParams{ + workspaceInfo: info, client: client, logger: workspaceTestLogger{}, + } + err = prepareWorkspace(context.Background(), params) + require.ErrorIs(t, err, uploadErr) + require.ErrorIs(t, (&UpCmd{}).handleInitError(err, info), uploadErr) + for _, file := range []struct{ dir, name, want string }{ + {info.Origin, provider.WorkspaceConfigFile, "workspace record"}, + {filepath.Dir(info.Workspace.SSHConfigPath), "ssh_config", string(sshConfig)}, + } { + data, err := fs.ReadFile(os.DirFS(file.dir), file.name) + require.NoError(t, err) + require.Equal(t, file.want, string(data)) + } + if tc.existing { + data, err := fs.ReadFile(os.DirFS(info.ContentFolder), "user-data") + require.NoError(t, err) + require.Equal(t, "preserved", string(data)) + } else { + require.NoDirExists(t, info.ContentFolder) + } + require.NoError(t, prepareWorkspace(context.Background(), params)) + require.Equal(t, 2, client.calls) + data, err := fs.ReadFile(os.DirFS(info.ContentFolder), "source.txt") + require.NoError(t, err) + require.Equal(t, "source", string(data)) + }) + } +} + +type sourceRetryClient struct { + tunnel.TunnelClient + archive []byte + firstError error + calls int +} + +func (c *sourceRetryClient) StreamWorkspace( + context.Context, *tunnel.Empty, ...grpc.CallOption, +) (grpc.ServerStreamingClient[tunnel.Chunk], error) { + c.calls++ + if c.calls == 1 { + // A complete file arrives before the stream fails reading the next header. + return &sourceRetryStream{data: c.archive[:1024], err: c.firstError}, nil + } + return &sourceRetryStream{data: c.archive, err: io.EOF}, nil +} + +type sourceRetryStream struct { + grpc.ClientStream + data []byte + err error +} + +func (s *sourceRetryStream) Recv() (*tunnel.Chunk, error) { + if len(s.data) == 0 { + return nil, s.err + } + data := s.data + s.data = nil + return &tunnel.Chunk{Content: data}, nil +} diff --git a/cmd/internal/agentworkspace/up.go b/cmd/internal/agentworkspace/up.go index 3314c412f3..81c19a6445 100644 --- a/cmd/internal/agentworkspace/up.go +++ b/cmd/internal/agentworkspace/up.go @@ -126,12 +126,23 @@ func (cmd *UpCmd) handleInitError( err error, workspaceInfo *provider.AgentWorkspaceInfo, ) error { + if workspaceInfo.WorkspaceWasExisting { + return err + } + preserveContent := false + if _, existing := errors.AsType[*existingContentPreparationError](err); existing { + if workspaceInfo.ContentFolder != workspaceInfo.Workspace.Source.LocalFolder { + return err + } + preserveContent = true + } deleteErr := clientimplementation.DeleteWorkspaceFolder( clientimplementation.DeleteWorkspaceFolderParams{ Context: workspaceInfo.Workspace.Context, WorkspaceID: workspaceInfo.Workspace.ID, SSHConfigPath: workspaceInfo.Workspace.SSHConfigPath, SSHConfigIncludePath: workspaceInfo.Workspace.SSHConfigIncludePath, + PreserveContent: preserveContent, }, ) if deleteErr != nil { @@ -233,37 +244,65 @@ func CreateRunner( ) } +// InitContentFolder reports whether content or a fallback configuration permits +// source reuse; that result does not indicate ownership of the content directory. func InitContentFolder( ctx context.Context, workspaceInfo *provider.AgentWorkspaceInfo, ) (bool, error) { + state, err := initContentFolder(ctx, workspaceInfo) + return state.skipSource, err +} + +type contentFolderState struct { + existed bool + skipSource bool +} + +func initContentFolder( + ctx context.Context, + workspaceInfo *provider.AgentWorkspaceInfo, +) (contentFolderState, error) { exists, err := contentFolderExists(workspaceInfo.ContentFolder) + state := contentFolderState{existed: exists, skipSource: exists} if err != nil { - return false, err + return state, err } if exists { - return true, nil + if err := downloadWorkspaceBinaries(ctx, workspaceInfo); err != nil { + // Initialization cleanup must not own content that predates this attempt. + return state, &existingContentPreparationError{cause: err} + } + return state, nil } if err := createContentFolder(workspaceInfo.ContentFolder); err != nil { - return false, err + return state, err } if err := downloadWorkspaceBinaries(ctx, workspaceInfo); err != nil { _ = os.RemoveAll(workspaceInfo.ContentFolder) - return false, err + return state, err } if workspaceInfo.LastDevContainerConfig != nil { if err := ensureLastDevContainerJson(workspaceInfo); err != nil { log.Errorf("ensure devcontainer.json: %v", err) } - return true, nil + state.skipSource = true } - return false, nil + return state, nil +} + +type existingContentPreparationError struct { + cause error } +func (e *existingContentPreparationError) Error() string { return e.cause.Error() } + +func (e *existingContentPreparationError) Unwrap() error { return e.cause } + func contentFolderExists(path string) (bool, error) { _, err := os.Stat(path) if err == nil { @@ -288,6 +327,9 @@ func downloadWorkspaceBinaries( ctx context.Context, workspaceInfo *provider.AgentWorkspaceInfo, ) error { + if len(workspaceInfo.Agent.Binaries) == 0 { + return nil + } binariesDir, err := agent.GetAgentBinariesDir( workspaceInfo.Agent.DataPath, workspaceInfo.Workspace.Context, @@ -579,11 +621,11 @@ func prepareWorkspace(ctx context.Context, params prepareWorkspaceParams) error ) } - exists, err := InitContentFolder(ctx, params.workspaceInfo) + folder, err := initContentFolder(ctx, params.workspaceInfo) if err != nil { return err } - if exists && !params.workspaceInfo.CLIOptions.Recreate { + if folder.skipSource && !params.workspaceInfo.CLIOptions.Recreate { params.logger.Debugf("workspace exists, skip downloading") return nil } @@ -595,9 +637,18 @@ func prepareWorkspace(ctx context.Context, params prepareWorkspaceParams) error case params.workspaceInfo.CLIOptions.Recreate: phase = status.PhaseRebuildingWorkspace } - return prepareWorkspaceWithStatus(ctx, params.reporter, phase, func(ctx context.Context) error { - return prepareWorkspaceSource(ctx, params, exists) + err = prepareWorkspaceWithStatus(ctx, params.reporter, phase, func(ctx context.Context) error { + return prepareWorkspaceSource(ctx, params, folder.skipSource) }) + if err != nil && !folder.existed { + // An unfinished folder must not make the next attempt skip source preparation. + if cleanupErr := os.RemoveAll(params.workspaceInfo.ContentFolder); cleanupErr != nil { + return errors.Join( + err, fmt.Errorf("remove unfinished workspace content: %w", cleanupErr), + ) + } + } + return err } func prepareWorkspaceWithStatus( diff --git a/cmd/internal/container_tunnel.go b/cmd/internal/container_tunnel.go index dbb6410ef3..01294b4951 100644 --- a/cmd/internal/container_tunnel.go +++ b/cmd/internal/container_tunnel.go @@ -101,11 +101,12 @@ func (cmd *ContainerTunnelCmd) Run(cobraCtx context.Context) error { return agent.Tunnel(ctx, agent.TunnelOptions{ Exec: func(ctx context.Context, req agent.ExecRequest) error { return runner.Command(ctx, devcontainer.CommandParams{ - User: req.User, - Command: req.Command, - Stdin: req.Stdin, - Stdout: req.Stdout, - Stderr: req.Stderr, + User: req.User, + Command: req.Command, + Stdin: req.Stdin, + Stdout: req.Stdout, + Stderr: req.Stderr, + RawStdout: true, }) }, RuntimeHealthCheck: containerRuntimeHealthCheck(workspaceInfo), diff --git a/cmd/workspace/ssh.go b/cmd/workspace/ssh.go index 9302f82746..3b8f9d2245 100644 --- a/cmd/workspace/ssh.go +++ b/cmd/workspace/ssh.go @@ -542,7 +542,7 @@ func (cmd *SSHCmd) runInteractiveTunnelSession( TermMode: cmd.TermMode, InstallTerminfo: cmd.InstallTerminfo, }, - Exec: func(ctx context.Context, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + Exec: func(ctx context.Context, stdin io.Reader, stdout io.Writer, _ io.Writer) error { if cmd.SSHKeepAliveInterval != DisableSSHKeepAlive { go startSSHKeepAlive(ctx, params.containerClient, cmd.SSHKeepAliveInterval) } @@ -551,11 +551,12 @@ func (cmd *SSHCmd) runInteractiveTunnelSession( Command: params.command, Stdin: stdin, Stdout: stdout, - Stderr: stderr, + Stderr: params.writer, EnvVars: params.envVars, }) }, - Stderr: params.writer, + // Guest stderr is user data; only the outer helper emits JSON diagnostics. + Stderr: os.Stderr, }) } diff --git a/e2e/tests/up/provider_microsandbox.go b/e2e/tests/up/provider_microsandbox.go index 3a363864ac..43041f58e0 100644 --- a/e2e/tests/up/provider_microsandbox.go +++ b/e2e/tests/up/provider_microsandbox.go @@ -1,178 +1,244 @@ package up import ( + "bytes" "context" "encoding/json" + "errors" "fmt" "os" "os/exec" "path/filepath" "runtime" "strings" + "time" + "github.com/blang/semver/v4" "github.com/devsy-org/devsy/e2e/framework" "github.com/devsy-org/devsy/pkg/devcontainer" + "github.com/devsy-org/devsy/pkg/docker" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" ) const osLinux = "linux" -// skipIfNoMicrosandbox skips when the microsandbox runtime or hardware -// virtualization is unavailable, mirroring how other providers guard on their -// runtime being present. -func skipIfNoMicrosandbox() { - if _, err := exec.LookPath("msb"); err != nil { - ginkgo.Skip("microsandbox runtime (msb) not found on PATH") - } +func skipIfNoMicrosandbox(ctx context.Context) { + checkMicrosandboxVersion(ctx) switch { case runtime.GOOS == osLinux: kvm, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) if err != nil { - ginkgo.Skip("microsandbox requires KVM (/dev/kvm not accessible)") + microsandboxUnavailable("microsandbox requires KVM (/dev/kvm not accessible)") } _ = kvm.Close() case runtime.GOOS == "darwin" && runtime.GOARCH == "arm64": // Apple silicon supports microsandbox via the hypervisor framework. default: - ginkgo.Skip("microsandbox requires Apple silicon or Linux with KVM") + microsandboxUnavailable("microsandbox requires Apple silicon or Linux with KVM") } } +func checkMicrosandboxVersion(ctx context.Context) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if _, err := exec.LookPath("msb"); err != nil { + microsandboxUnavailable("microsandbox runtime (msb) not found on PATH") + } + output, err := exec.CommandContext(ctx, "msb", "--version").Output() + if err != nil { + microsandboxUnavailable("microsandbox runtime (msb) cannot execute: " + err.Error()) + } + version, err := semver.ParseTolerant( + strings.TrimPrefix(strings.TrimSpace(string(output)), "msb "), + ) + if err != nil || version.LT(semver.Version{Major: 0, Minor: 7, Patch: 7}) { + microsandboxUnavailable( + "microsandbox parity requires msb v0.7.7 or newer: " + string(output), + ) + } +} + +func microsandboxUnavailable(reason string) { + if os.Getenv("DEVSY_REQUIRE_MICROSANDBOX") == "true" { + ginkgo.Fail(reason) + } + ginkgo.Skip(reason) +} + var _ = ginkgo.Describe( "testing up command for microsandbox provider", ginkgo.Label("up-provider-microsandbox"), func() { var initialDir string - ginkgo.BeforeEach(func() { - skipIfNoMicrosandbox() + ginkgo.BeforeEach(func(ctx context.Context) { + skipIfNoMicrosandbox(ctx) var err error initialDir, err = os.Getwd() framework.ExpectNoError(err) }) - ginkgo.It("runs devsy in a microsandbox microVM", func(ctx context.Context) { - f := framework.NewDefaultFramework(initialDir + "/bin") - tempDir, err := framework.CopyToTempDir("tests/up/testdata/microsandbox") - framework.ExpectNoError(err) - ginkgo.DeferCleanup(framework.CleanupTempDir, initialDir, tempDir) + ginkgo.DescribeTable("runs the shared lifecycle and ownership contract", + func(ctx context.Context, source, name string) { + ginkgo.GinkgoT().Setenv("DEVSY_HOME", ginkgo.GinkgoT().TempDir()) + ginkgo.GinkgoT().Setenv("DEVSY_CONFIG", "") + f := framework.NewDefaultFramework(initialDir + "/bin") + tempDir, err := framework.CopyToTempDir("tests/up/testdata/microsandbox") + framework.ExpectNoError(err) + ginkgo.DeferCleanup(framework.CleanupTempDir, initialDir, tempDir) - _ = f.DevsyProviderDelete(ctx, "microsandbox") - err = f.DevsyProviderAdd(ctx, "microsandbox") - framework.ExpectNoError(err) - ginkgo.DeferCleanup(func(cleanupCtx context.Context) { - err := f.DevsyProviderDelete(cleanupCtx, "microsandbox") + err = f.DevsyProviderAdd(ctx, source, "--name", name) framework.ExpectNoError(err) - }) + ginkgo.DeferCleanup(func(cleanupCtx context.Context) { + err := f.DevsyProviderDelete(cleanupCtx, name) + framework.ExpectNoError(err) + }) + ginkgo.DeferCleanup(f.CleanupWorkspace, tempDir) - // full up: boots the microVM, streams in the agent, opens the tunnel - err = f.DevsyUp(ctx, tempDir, "--devcontainer", ".devcontainer.json") - framework.ExpectNoError(err) - ginkgo.DeferCleanup(f.DevsyWorkspaceDelete, tempDir) + // full up: boots the microVM, streams in the agent, opens the tunnel + err = f.DevsyUp(ctx, tempDir, "--devcontainer", ".devcontainer.json") + framework.ExpectNoError(err) - // the workspace is reachable over SSH - err = f.DevsySSHEchoTestString(ctx, tempDir) - framework.ExpectNoError(err) + // the workspace is reachable over SSH + err = f.DevsySSHEchoTestString(ctx, tempDir) + framework.ExpectNoError(err) + assertMicrosandboxSSHStreams(ctx, f, tempDir) - workspacePath := filepath.Join("/workspaces", filepath.Base(tempDir)) - _, err = f.DevsySSHOnce(ctx, tempDir, fmt.Sprintf( - "umask 022; : > %s/guest-created.txt && mkdir %s/guest-created-dir", - workspacePath, workspacePath, - )) - framework.ExpectNoError(err) - for name, mode := range map[string]os.FileMode{"guest-created.txt": 0o644, "guest-created-dir": 0o755} { - info, err := os.Stat(filepath.Join(tempDir, name)) + workspacePath := filepath.Join("/workspaces", filepath.Base(tempDir)) + _, err = f.DevsySSHOnce(ctx, tempDir, fmt.Sprintf( + "umask 022; : > %s/guest-created.txt && mkdir %s/guest-created-dir", + workspacePath, workspacePath, + )) framework.ExpectNoError(err) - gomega.Expect(info.Mode().Perm()).To(gomega.Equal(mode)) - } + for name, mode := range map[string]os.FileMode{"guest-created.txt": 0o644, "guest-created-dir": 0o755} { + info, err := os.Stat(filepath.Join(tempDir, name)) + framework.ExpectNoError(err) + gomega.Expect(info.Mode().Perm()).To(gomega.Equal(mode)) + } - workspace, err := f.FindWorkspace(ctx, tempDir) - framework.ExpectNoError(err) - sandbox := "devsy-" + devcontainer.GetRunnerIDFromWorkspace(workspace) - createdAt := microsandboxCreationTime(ctx, sandbox) - // No --user override: verify the runtime's default execution identity. - // #nosec G204 -- fixed command and Devsy-generated sandbox name - workloadUser, err := exec.CommandContext( - ctx, - "msb", - "exec", - "--stream", - sandbox, - "--", - "id", - "-u", - ).Output() - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(string(workloadUser))).To(gomega.Equal("0")) + workspace, err := f.FindWorkspace(ctx, tempDir) + framework.ExpectNoError(err) + sandbox := "devsy-" + devcontainer.GetRunnerIDFromWorkspace(workspace) + createdAt := microsandboxCreationTime(ctx, sandbox) + // No --user override: verify the runtime's default execution identity. + // #nosec G204 -- fixed command and Devsy-generated sandbox name + workloadUser, err := exec.CommandContext( + ctx, + "msb", + "exec", + "--stream", + sandbox, + "--", + "id", + "-u", + ).Output() + framework.ExpectNoError(err) + gomega.Expect(strings.TrimSpace(string(workloadUser))).To(gomega.Equal("0")) - developer, err := f.DevsySSHOnce(ctx, tempDir, "id -un") - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(developer)).To(gomega.Equal("vscode")) - assertMicrosandboxHostEntries(ctx, f, tempDir, "fresh") - err = f.DevsyWorkspaceStop(ctx, tempDir) - framework.ExpectNoError(err) - err = f.DevsyUp(ctx, tempDir) - framework.ExpectNoError(err) - gomega.Expect(microsandboxCreationTime(ctx, sandbox)).To(gomega.Equal(createdAt)) - assertMicrosandboxHostEntries(ctx, f, tempDir, "restarted") - err = f.DevsyUpRecreate(ctx, tempDir) - framework.ExpectNoError(err) - gomega.Expect(microsandboxCreationTime(ctx, sandbox)).NotTo(gomega.Equal(createdAt)) - assertMicrosandboxHostEntries(ctx, f, tempDir, "recreated") - - previousCreation := microsandboxCreationTime(ctx, sandbox) - // #nosec G204 -- fixed command and Devsy-generated sandbox name - _, err = exec.CommandContext( - ctx, "msb", "exec", "--stream", sandbox, "--", "sh", "-c", - "echo preserved > /root/recreation-preserved.txt", - ). - Output() - framework.ExpectNoError(err) + developer, err := f.DevsySSHOnce(ctx, tempDir, "id -un") + framework.ExpectNoError(err) + gomega.Expect(strings.TrimSpace(developer)).To(gomega.Equal("vscode")) + assertMicrosandboxHostEntries(ctx, f, tempDir, "fresh") + err = f.DevsyWorkspaceStop(ctx, tempDir) + framework.ExpectNoError(err) + err = f.DevsyUp(ctx, tempDir) + framework.ExpectNoError(err) + gomega.Expect(microsandboxCreationTime(ctx, sandbox)).To(gomega.Equal(createdAt)) + assertMicrosandboxHostEntries(ctx, f, tempDir, "restarted") + err = f.DevsyUpRecreate(ctx, tempDir) + framework.ExpectNoError(err) + gomega.Expect(microsandboxCreationTime(ctx, sandbox)).NotTo(gomega.Equal(createdAt)) + assertMicrosandboxHostEntries(ctx, f, tempDir, "recreated") - configPath := filepath.Join(tempDir, ".devcontainer.json") - // #nosec G304 -- configuration copied into the test-owned temporary directory - data, err := os.ReadFile(configPath) - framework.ExpectNoError(err) - var devConfig map[string]any - framework.ExpectNoError(json.Unmarshal(data, &devConfig)) - devConfig["remoteUser"] = "root" - data, err = json.Marshal(devConfig) - framework.ExpectNoError(err) - framework.ExpectNoError(os.WriteFile(configPath, data, 0o600)) - - stdout, stderr, err := f.DevsyUpStreams( - ctx, tempDir, "--devcontainer", ".devcontainer.json", - ) - gomega.Expect(err).To(gomega.HaveOccurred()) - gomega.Expect(stdout + stderr).To(gomega.ContainSubstring("--recreate")) - gomega.Expect(microsandboxCreationTime(ctx, sandbox)).To(gomega.Equal(previousCreation)) - // #nosec G204 -- fixed command and Devsy-generated sandbox name - preserved, err := exec.CommandContext( - ctx, - "msb", - "exec", - "--stream", - sandbox, - "--", - "cat", - "/root/recreation-preserved.txt", - ). - Output() - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(string(preserved))).To(gomega.Equal("preserved")) - err = f.DevsyUp(ctx, tempDir, "--devcontainer", ".devcontainer.json", "--recreate") - framework.ExpectNoError(err) - gomega.Expect(microsandboxCreationTime(ctx, sandbox)). - NotTo(gomega.Equal(previousCreation)) - developer, err = f.DevsySSHOnce(ctx, tempDir, "id -u") - framework.ExpectNoError(err) - gomega.Expect(strings.TrimSpace(developer)).To(gomega.Equal("0")) - assertMicrosandboxHostEntries(ctx, f, tempDir, "identity-changed") - }, ginkgo.SpecTimeout(framework.TimeoutModerate())) + previousCreation := microsandboxCreationTime(ctx, sandbox) + // #nosec G204 -- fixed command and Devsy-generated sandbox name + _, err = exec.CommandContext( + ctx, "msb", "exec", "--stream", sandbox, "--", "sh", "-c", + "echo preserved > /root/recreation-preserved.txt", + ). + Output() + framework.ExpectNoError(err) + + configPath := filepath.Join(tempDir, ".devcontainer.json") + // #nosec G304 -- configuration copied into the test-owned temporary directory + data, err := os.ReadFile(configPath) + framework.ExpectNoError(err) + var devConfig map[string]any + framework.ExpectNoError(json.Unmarshal(data, &devConfig)) + devConfig["remoteUser"] = "root" + data, err = json.Marshal(devConfig) + framework.ExpectNoError(err) + framework.ExpectNoError(os.WriteFile(configPath, data, 0o600)) + + stdout, stderr, err := f.DevsyUpStreams( + ctx, tempDir, "--devcontainer", ".devcontainer.json", + ) + gomega.Expect(err).To(gomega.HaveOccurred()) + gomega.Expect(stdout + stderr).To(gomega.ContainSubstring("--recreate")) + gomega.Expect(microsandboxCreationTime(ctx, sandbox)). + To(gomega.Equal(previousCreation)) + // #nosec G204 -- fixed command and Devsy-generated sandbox name + preserved, err := exec.CommandContext( + ctx, + "msb", + "exec", + "--stream", + sandbox, + "--", + "cat", + "/root/recreation-preserved.txt", + ). + Output() + framework.ExpectNoError(err) + gomega.Expect(strings.TrimSpace(string(preserved))).To(gomega.Equal("preserved")) + err = f.DevsyUp(ctx, tempDir, "--devcontainer", ".devcontainer.json", "--recreate") + framework.ExpectNoError(err) + gomega.Expect(microsandboxCreationTime(ctx, sandbox)). + NotTo(gomega.Equal(previousCreation)) + developer, err = f.DevsySSHOnce(ctx, tempDir, "id -u") + framework.ExpectNoError(err) + gomega.Expect(strings.TrimSpace(developer)).To(gomega.Equal("0")) + assertMicrosandboxHostEntries(ctx, f, tempDir, "identity-changed") + framework.ExpectNoError(f.DevsyWorkspaceDelete(ctx, tempDir)) + // #nosec G204 -- fixed command and test-owned sandbox name + output, err := exec.CommandContext(ctx, "msb", "list", "--format", "json").Output() + framework.ExpectNoError(err) + gomega.Expect(string(output)).NotTo(gomega.ContainSubstring(sandbox)) + }, + ginkgo.Entry("built-in", "microsandbox", "microsandbox-builtin-parity", + ginkgo.SpecTimeout(framework.TimeoutLong())), + ginkgo.Entry( + "external v0.1.5", + "github.com/devsy-org/devsy-provider-microsandbox@v0.1.5", + "microsandbox-external-parity", + ginkgo.SpecTimeout(framework.TimeoutLong()), + ), + ) }, ) +func assertMicrosandboxSSHStreams(ctx context.Context, f *framework.Framework, workspace string) { + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + payload := bytes.Repeat([]byte{0, 1, 10, 13, 27, 127, 128, 255}, 128*1024) + var stdout, stderr bytes.Buffer + // #nosec G204 -- test binary, test-owned workspace, and fixed guest command + command := exec.CommandContext(ctx, filepath.Join(f.DevsyBinDir, f.DevsyBinName), + "workspace", "ssh", workspace, "--command", "cat; printf parity-stderr >&2; exit 23") + docker.PrepareForGroupCancellation(command) + command.WaitDelay = 30 * time.Second + command.Stdin = bytes.NewReader(payload) + command.Stdout, command.Stderr = &stdout, &stderr + err := command.Run() + var exit *exec.ExitError + gomega.Expect(errors.As(err, &exit)). + To(gomega.BeTrue(), "expected guest exit 23: %v; stderr: %s", err, stderr.String()) + gomega.Expect(exit.ExitCode()).To(gomega.Equal(23)) + gomega.Expect(stdout.Bytes()).To(gomega.Equal(payload)) + gomega.Expect(stderr.String()).To(gomega.ContainSubstring("parity-stderr")) +} + // Entries are created after setup so recursive workspace chown cannot mask fallback ownership. func assertMicrosandboxHostEntries( ctx context.Context, diff --git a/go.mod b/go.mod index 5cb904f858..51a67c52fc 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/devsy-org/agentapi v1.0.1 github.com/devsy-org/api v1.1.0 github.com/devsy-org/apiserver v1.5.4 - github.com/devsy-org/devsy-runtime-sdk v1.4.0 + github.com/devsy-org/devsy-runtime-sdk v1.5.2 github.com/devsy-org/ssh v1.2.9 github.com/distribution/reference v0.6.0 github.com/docker/cli v29.8.0+incompatible diff --git a/go.sum b/go.sum index 678ef9b1dd..0ab3c4c089 100644 --- a/go.sum +++ b/go.sum @@ -416,8 +416,8 @@ github.com/devsy-org/api v1.1.0 h1:l7T9k7RVwatwN4lxeDTF3iN6EYmfGZgR3ZTJMDGha1M= github.com/devsy-org/api v1.1.0/go.mod h1:mAZklKdnywJYiXDReBLte/H+3m69z6G7RHB3n1lI53Q= github.com/devsy-org/apiserver v1.5.4 h1:/bEPrSRSlfii2QHxc9qAiXlQawNJfIaPLraR3bYKHxg= github.com/devsy-org/apiserver v1.5.4/go.mod h1:sDFCTjCN13wAHhno4KX0Z756gCI8ttW/c7NwTzo16K0= -github.com/devsy-org/devsy-runtime-sdk v1.4.0 h1:dJqPJrKxJVmgWT9r1vnZ08rqZTxshyazj9eM+fZG8k4= -github.com/devsy-org/devsy-runtime-sdk v1.4.0/go.mod h1:MiBP/fiY83DAS0TueEiJZSHYZ1vq85UR4KJTMfHIM+E= +github.com/devsy-org/devsy-runtime-sdk v1.5.2 h1:nPQ3HyIrEgqlAU/a7bvFovm+VB9cdryUrejKKX0O2LQ= +github.com/devsy-org/devsy-runtime-sdk v1.5.2/go.mod h1:MiBP/fiY83DAS0TueEiJZSHYZ1vq85UR4KJTMfHIM+E= github.com/devsy-org/ssh v1.2.9 h1:KHqX1xAplGFanm0FMSAojtiC9nV/UFxUeP/5jU5quak= github.com/devsy-org/ssh v1.2.9/go.mod h1:Uff10+cSSDZk3bG07u5D9+eQ8GMGqsgE70WCUJWcHv4= github.com/devsy-org/tailscale v1.102.2 h1:9SB6htvO+HmG8alal8WGCshHapfHR7dUFRSMYiFIzIM= diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index dca37e2208..b2cd28410c 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -241,8 +241,11 @@ func handleStaleWorkspace( deleteWorkspace func(*provider2.AgentWorkspaceInfo) error, ) (string, error) { oldWorkspaceInfo, _ := ParseAgentWorkspaceInfo(workspaceConfig) - if oldWorkspaceInfo == nil || - oldWorkspaceInfo.Workspace.UID == workspaceInfo.Workspace.UID { + if oldWorkspaceInfo == nil { + return workspaceDir, nil + } + if oldWorkspaceInfo.Workspace.UID == workspaceInfo.Workspace.UID { + workspaceInfo.WorkspaceWasExisting = true return workspaceDir, nil } diff --git a/pkg/agent/ownership_test.go b/pkg/agent/ownership_test.go new file mode 100644 index 0000000000..39f69f7312 --- /dev/null +++ b/pkg/agent/ownership_test.go @@ -0,0 +1,74 @@ +package agent + +import ( + "encoding/json" + "os" + "testing" + + "github.com/devsy-org/devsy/pkg/compress" + "github.com/devsy-org/devsy/pkg/config" + "github.com/devsy-org/devsy/pkg/provider" + "github.com/stretchr/testify/require" +) + +func TestWorkspaceInitializationOwnership(t *testing.T) { + for _, tc := range []struct { + name, oldUID string + existing bool + deletions int + }{ + {name: "new workspace"}, + {name: "same UID", oldUID: "current-uid", existing: true}, + {name: "replaced UID", oldUID: "previous-uid", deletions: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + home := t.TempDir() + info := &provider.AgentWorkspaceInfo{ + Workspace: &provider.Workspace{ + Context: config.DefaultContext, + ID: "ownership", + UID: "current-uid", + }, + Agent: provider.ProviderAgentConfig{DataPath: home, Local: config.BoolTrue}, + } + origin, err := CreateAgentWorkspaceDir(home, info.Workspace.Context, info.Workspace.ID) + require.NoError(t, err) + if tc.oldUID != "" { + old := provider.CloneAgentWorkspaceInfo(info) + old.Workspace.UID = tc.oldUID + old.Origin = origin + require.NoError(t, PersistAgentWorkspaceInfo(old)) + } + data, err := json.Marshal(info) + require.NoError(t, err) + var input map[string]any + require.NoError(t, json.Unmarshal(data, &input)) + input["WorkspaceWasExisting"] = true + data, err = json.Marshal(input) + require.NoError(t, err) + encoded, err := compress.Compress(string(data)) + require.NoError(t, err) + deletions := 0 + shouldExit, loaded, err := WriteWorkspaceInfoAndDeleteOld( + encoded, + func(old *provider.AgentWorkspaceInfo) error { + deletions++ + require.Equal(t, tc.oldUID, old.Workspace.UID) + return os.RemoveAll(old.Origin) + }, + ) + require.NoError(t, err) + require.False(t, shouldExit) + require.Equal(t, tc.deletions, deletions) + require.Equal(t, tc.existing, loaded.WorkspaceWasExisting) + require.Equal( + t, + tc.existing, + provider.CloneAgentWorkspaceInfo(loaded).WorkspaceWasExisting, + ) + data, err = json.Marshal(loaded) + require.NoError(t, err) + require.NotContains(t, string(data), "WorkspaceWasExisting") + }) + } +} diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index 8df2be23ae..9e95d41e2a 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -978,6 +978,8 @@ type DeleteWorkspaceFolderParams struct { WorkspaceID string SSHConfigPath string SSHConfigIncludePath string + // PreserveContent leaves the content directory intact when only managed records are owned. + PreserveContent bool } func DeleteWorkspaceFolder(params DeleteWorkspaceFolderParams) error { @@ -989,6 +991,9 @@ func DeleteWorkspaceFolder(params DeleteWorkspaceFolderParams) error { return err } + if params.PreserveContent { + return nil + } return removeWorkspaceContent(params.Context, params.WorkspaceID) } diff --git a/pkg/devcontainer/command_test.go b/pkg/devcontainer/command_test.go new file mode 100644 index 0000000000..fe728b3d4a --- /dev/null +++ b/pkg/devcontainer/command_test.go @@ -0,0 +1,58 @@ +package devcontainer + +import ( + "bytes" + "context" + "testing" + + "github.com/devsy-org/devsy/pkg/driver" + "github.com/devsy-org/devsy/pkg/secrets" + "github.com/devsy-org/devsy/pkg/subprocess" + "github.com/stretchr/testify/require" +) + +type protocolCommandDriver struct { + mockDriver + output *bytes.Buffer + payload []byte + outputBeforeExit []byte +} + +func (d *protocolCommandDriver) CommandDevContainer( + _ context.Context, + params *driver.CommandParams, +) error { + redacted := &subprocess.StreamingRedactingWriter{ + Next: params.Stdout, Redactor: secrets.NewRedactor([]string{"TOKEN=private-canary-value"}), + } + output := params.Stdout + if !params.RawStdout { + output = redacted + } + // A peer must receive the complete packet before it can send its reply. + if _, err := output.Write(d.payload); err != nil { + return err + } + d.outputBeforeExit = bytes.Clone(d.output.Bytes()) + return redacted.Flush() +} + +func TestCommandPreservesProtocolOutputBeforeExit(t *testing.T) { + var output bytes.Buffer + d := &protocolCommandDriver{output: &output, payload: []byte{0, 255, 'p'}} + r := newTestRunner(d) + require.NoError( + t, + r.Command(context.Background(), CommandParams{Stdout: &output, RawStdout: true}), + ) + require.Equal(t, []byte{0, 255, 'p'}, d.outputBeforeExit, + "buffering a secret prefix in a binary packet stalls the peer's next handshake step") +} + +func TestCommandKeepsTextRedactionByDefault(t *testing.T) { + var output bytes.Buffer + d := &protocolCommandDriver{output: &output, payload: []byte("private-canary-value\n")} + r := newTestRunner(d) + require.NoError(t, r.Command(context.Background(), CommandParams{Stdout: &output})) + require.Equal(t, "***\n", output.String()) +} diff --git a/pkg/devcontainer/reuse_preflight_test.go b/pkg/devcontainer/reuse_preflight_test.go new file mode 100644 index 0000000000..d538d6b55a --- /dev/null +++ b/pkg/devcontainer/reuse_preflight_test.go @@ -0,0 +1,136 @@ +package devcontainer + +import ( + "context" + "errors" + "testing" + + "github.com/devsy-org/devsy/pkg/devcontainer/config" + "github.com/devsy-org/devsy/pkg/devcontainer/metadata" + "github.com/devsy-org/devsy/pkg/driver" + "github.com/stretchr/testify/require" +) + +const reuseCurrentUser = "requested-user" + +type reusePreflightMockDriver struct { + *provisioningPreflightMockDriver + err error + workspaceID, remoteUser string + calls int + unsupported bool +} + +func (d *reusePreflightMockDriver) SupportsReusePreflight() bool { return !d.unsupported } + +func (d *reusePreflightMockDriver) ReusePreflight(_ context.Context, id, user string) error { + d.calls++ + d.workspaceID, d.remoteUser = id, user + return d.err +} + +func (*reusePreflightMockDriver) RecreateMode() driver.RecreateMode { return driver.RecreateDelete } + +func TestReusePreflightFailurePreservesExistingWorkspace(t *testing.T) { + sentinel := errors.New("ownership changed; rerun with --recreate") + d := &reusePreflightMockDriver{ + provisioningPreflightMockDriver: &provisioningPreflightMockDriver{ + mockDriver: &mockDriver{}, + }, + err: sentinel, + } + p := recreateResolveParams() + p.options.Recreate = false + p.parsedConfig.Config.RemoteUser = reuseCurrentUser + p.substitutionContext = &config.SubstitutionContext{} + details := runningContainerDetails() + details.Config.Labels[metadata.CreationConfigLabel] = stringTrue + details.Config.Labels[metadata.ImageMetadataLabel] = `[{"remoteUser":"old-user"}]` + r := newTestRunner(d) + _, err := r.resolveContainer(context.Background(), p, details) + require.ErrorIs(t, err, sentinel) + require.Equal(t, r.id, d.workspaceID) + require.Equal(t, reuseCurrentUser, d.remoteUser) + require.False(t, p.options.Recreate) + require.False(t, d.provisioningCalled) + require.False(t, d.stopCalled) + require.False(t, d.deleteCalled) + require.Equal(t, testStatusRunning, string(details.State.Status)) +} + +func TestReusePreflightSkippedForCreation(t *testing.T) { + d := &reusePreflightMockDriver{ + provisioningPreflightMockDriver: &provisioningPreflightMockDriver{ + mockDriver: &mockDriver{}, + }, + err: errors.New("must not validate reuse"), + } + r := newTestRunner(d) + p := recreateResolveParams() + require.NoError( + t, + r.applyDriverRecreateRequirement(context.Background(), runningContainerDetails(), p), + ) + p.options.Recreate = false + require.NoError(t, r.applyDriverRecreateRequirement(context.Background(), nil, p)) + require.Zero(t, d.calls) +} + +func TestReusePreflightUnsupportedPreservesCreationIdentity(t *testing.T) { + d := &reusePreflightMockDriver{ + provisioningPreflightMockDriver: &provisioningPreflightMockDriver{ + mockDriver: &mockDriver{}, + }, + unsupported: true, + } + r := newTestRunner(d) + p := recreateResolveParams() + p.options.Recreate = false + p.parsedConfig.Config.RemoteUser = reuseCurrentUser + p.substitutionContext = &config.SubstitutionContext{} + details := runningContainerDetails() + details.Config.Labels[overlayStructureLabel] = structuralSignature(&config.DevContainerConfig{}) + details.Config.Labels[metadata.ImageMetadataLabel] = `[{"remoteUser":"old-user"}]` + merged, err := r.mergeExistingContainerConfig(context.Background(), details, p) + require.NoError(t, err) + require.Equal(t, "old-user", merged.RemoteUser) + require.NoError(t, r.applyDriverRecreateRequirement(context.Background(), details, p)) + require.Zero(t, d.calls) + require.False(t, p.options.Recreate) +} + +func TestReusePreflightRefreshesDeveloperIdentity(t *testing.T) { + const featureUser = "feature-user" + for _, tc := range []struct{ name, label, value, want string }{ + {"creation marker", metadata.CreationConfigLabel, stringTrue, featureUser}, + {"legacy managed workspace", overlayStructureLabel, structuralSignature(&config.DevContainerConfig{}), featureUser}, + {"unmarked image metadata", "", "", "old-user"}, + } { + t.Run(tc.name, func(t *testing.T) { + d := &reusePreflightMockDriver{ + provisioningPreflightMockDriver: &provisioningPreflightMockDriver{ + mockDriver: &mockDriver{}, + }, + } + r := newTestRunner(d) + p := recreateResolveParams() + p.options.Recreate = false + p.parsedConfig.Config.RemoteUser = "" + p.substitutionContext = &config.SubstitutionContext{} + details := runningContainerDetails() + if tc.label != "" { + details.Config.Labels[tc.label] = tc.value + } + details.Config.Labels[metadata.ImageMetadataLabel] = `[{"remoteUser":"feature-user"},{"remoteUser":"old-user"}]` + require.NoError(t, r.applyDriverRecreateRequirement(context.Background(), details, p)) + require.Equal(t, tc.want, d.remoteUser) + merged, err := r.mergeExistingContainerConfig(context.Background(), details, p) + require.NoError(t, err) + require.Equal(t, tc.want, merged.RemoteUser) + p.parsedConfig.Config.RemoteUser = reuseCurrentUser + merged, err = r.mergeExistingContainerConfig(context.Background(), details, p) + require.NoError(t, err) + require.Equal(t, reuseCurrentUser, merged.RemoteUser) + }) + } +} diff --git a/pkg/devcontainer/run.go b/pkg/devcontainer/run.go index 0485136b81..b701ad970f 100644 --- a/pkg/devcontainer/run.go +++ b/pkg/devcontainer/run.go @@ -44,6 +44,8 @@ type CommandParams struct { Stdin io.Reader Stdout io.Writer Stderr io.Writer + // RawStdout keeps binary protocols out of text redaction and its suffix buffer. + RawStdout bool } // UpOptions configures a single Up invocation. @@ -257,6 +259,7 @@ func (r *runner) Command(ctx context.Context, params CommandParams) error { Stdin: params.Stdin, Stdout: params.Stdout, Stderr: params.Stderr, + RawStdout: params.RawStdout, }) } diff --git a/pkg/devcontainer/single.go b/pkg/devcontainer/single.go index 284fc6c4cb..0deb36bc6a 100644 --- a/pkg/devcontainer/single.go +++ b/pkg/devcontainer/single.go @@ -312,7 +312,7 @@ func (r *runner) mergeExistingContainerConfig( containerDetails *config.ContainerDetails, p *resolveParams, ) (*config.MergedDevContainerConfig, error) { - if _, ok := r.driver.(driver.RecreateRequiredDriver); ok { + if r.needsCurrentContainerIdentity() { return r.currentContainerIdentity(ctx, containerDetails, p) } imageMetadataConfig, err := metadata.GetImageMetadataFromContainer( @@ -824,7 +824,7 @@ func (r *runner) runContainer( ) runOptions.AllowRecreate = p.options.Recreate runOptions.Env = r.addExtraEnvVars(runOptions.Env) - if _, ok := r.driver.(driver.RecreateRequiredDriver); ok { + if r.needsCurrentContainerIdentity() { runOptions.Labels = append(runOptions.Labels, metadata.CreationConfigLabel+"="+stringTrue) } @@ -1212,7 +1212,7 @@ func (r *runner) applyDriverRecreateRequirement( if details == nil || p.options.Recreate { return nil } - if _, ok := r.driver.(driver.RecreateRequiredDriver); !ok { + if !r.needsCurrentContainerIdentity() { return nil } merged, err := r.currentContainerIdentity(ctx, details, p) @@ -1223,10 +1223,25 @@ func (r *runner) applyDriverRecreateRequirement( if containerUser == "" { containerUser = details.Config.Labels[config.UserLabel] } + remoteUser := effectiveRemoteUser(merged, containerUser) + if preflight, ok := r.driver.(driver.ReusePreflightDriver); ok && + preflight.SupportsReusePreflight() { + if err := preflight.ReusePreflight(ctx, r.id, remoteUser); err != nil { + return err + } + } + return r.scheduleDriverRecreation(details, remoteUser, p) +} + +func (r *runner) scheduleDriverRecreation( + details *config.ContainerDetails, + remoteUser string, + p *resolveParams, +) error { required, reason := driver.DriverRequiresRecreate( r.driver, details, - effectiveRemoteUser(merged, containerUser), + remoteUser, ) if !required { return nil @@ -1237,6 +1252,14 @@ func (r *runner) applyDriverRecreateRequirement( return r.scheduleContainerRecreation(p, reason) } +func (r *runner) needsCurrentContainerIdentity() bool { + if _, migration := r.driver.(driver.RecreateRequiredDriver); migration { + return true + } + preflight, ok := r.driver.(driver.ReusePreflightDriver) + return ok && preflight.SupportsReusePreflight() +} + func (r *runner) currentContainerIdentity( ctx context.Context, details *config.ContainerDetails, @@ -1246,8 +1269,10 @@ func (r *runner) currentContainerIdentity( if err != nil { return nil, err } - if details.Config.Labels[metadata.CreationConfigLabel] == stringTrue && - len(imageMetadata.Config) > 0 { + // Older managed workspaces have a structural signature but no creation marker. + hasCreationConfig := details.Config.Labels[metadata.CreationConfigLabel] == stringTrue || + details.Config.Labels[overlayStructureLabel] != "" + if hasCreationConfig && len(imageMetadata.Config) > 0 { imageMetadata.Config = imageMetadata.Config[:len(imageMetadata.Config)-1] } imageMetadata.Config = append( diff --git a/pkg/driver/external/capabilities.go b/pkg/driver/external/capabilities.go index 1912d5cf51..bab90304a8 100644 --- a/pkg/driver/external/capabilities.go +++ b/pkg/driver/external/capabilities.go @@ -14,6 +14,7 @@ var ( _ driver.MountDeliveryDriver = (*Host)(nil) _ driver.WorkspaceChowner = (*Host)(nil) _ driver.RecreatePolicyDriver = (*Host)(nil) + _ driver.ReusePreflightDriver = (*Host)(nil) ) func (h *Host) SupportsMountType(kind string) bool { @@ -29,6 +30,11 @@ func (h *Host) RequiresWorkspaceChown() bool { return h.info.Capabilities.RequiresWorkspaceChown } +// SupportsReusePreflight reflects the negotiated optional RPC. +func (h *Host) SupportsReusePreflight() bool { + return h.info.Capabilities.ReusePreflight +} + func (h *Host) RecreateMode() driver.RecreateMode { if h.info.Capabilities.RecreateMode == runtimev1.RecreateMode_RECREATE_MODE_DELETE { return driver.RecreateDelete diff --git a/pkg/driver/external/lifecycle.go b/pkg/driver/external/lifecycle.go index 6895a486a5..01bb66cc66 100644 --- a/pkg/driver/external/lifecycle.go +++ b/pkg/driver/external/lifecycle.go @@ -33,6 +33,22 @@ func (h *Host) ProvisioningPreflight(ctx context.Context) error { }) } +func (h *Host) ReusePreflight(ctx context.Context, workspaceID, remoteUser string) error { + if !h.SupportsReusePreflight() { + return ctx.Err() + } + if workspaceID == "" || remoteUser == "" { + return errors.New("runtime reuse preflight requires workspace ID and remote user") + } + return h.forWorkspace(workspaceID, nil). + call(ctx, "ReusePreflight", func(client runtimev1.RuntimeDriverClient) error { + _, err := client.ReusePreflight(ctx, &runtimev1.ReusePreflightRequest{ + WorkspaceId: workspaceID, RemoteUser: remoteUser, + }) + return err + }) +} + func (h *Host) FindDevContainer( ctx context.Context, workspaceID string, diff --git a/pkg/driver/external/reuse_test.go b/pkg/driver/external/reuse_test.go new file mode 100644 index 0000000000..d4623b82eb --- /dev/null +++ b/pkg/driver/external/reuse_test.go @@ -0,0 +1,47 @@ +package external + +import ( + "context" + + "github.com/devsy-org/devsy-runtime-sdk/conformance/fake" + "github.com/devsy-org/devsy-runtime-sdk/runtimev1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *HostSuite) TestReusePreflightAcrossFreshProcesses() { + host := s.host(fake.Normal) + ctx := context.Background() + s.Require().NoError(host.RunImage(ctx, &runtimev1.RunImageRequest{ + WorkspaceId: fixtureWorkspace, + Image: fixtureImage, + User: fixtureProcessUser, + RemoteUser: fixtureRemoteUser, + })) + s.Require().NoError(host.StopDevContainer(ctx, fixtureWorkspace)) + before, err := host.FindDevContainer(ctx, fixtureWorkspace) + s.Require().NoError(err) + s.NoError(host.ReusePreflight(ctx, fixtureWorkspace, fixtureRemoteUser)) + err = host.ReusePreflight(ctx, fixtureWorkspace, fixtureProcessUser) + s.Equal(codes.FailedPrecondition, status.Code(err)) + s.ErrorContains(err, "--recreate") + after, err := host.FindDevContainer(ctx, fixtureWorkspace) + s.Require().NoError(err) + s.Equal(before, after) + s.Equal(codes.NotFound, status.Code(host.ReusePreflight(ctx, "absent", fixtureProcessUser))) + s.ErrorContains(host.ReusePreflight(ctx, "", fixtureProcessUser), "workspace ID") + s.ErrorContains(host.ReusePreflight(ctx, fixtureWorkspace, ""), "remote user") +} + +func (s *HostSuite) TestReusePreflightCapabilityNegotiation() { + host := s.host(fake.Normal) + s.True(host.SupportsReusePreflight()) + host.info.Capabilities.ReusePreflight = false + s.False(host.SupportsReusePreflight()) + // An invalid executable proves the optional RPC never launches a runtime. + host.config.Binaries[fixtureKey][0].Checksum = "invalid" + s.NoError(host.ReusePreflight(context.Background(), fixtureWorkspace, fixtureProcessUser)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + s.ErrorIs(host.ReusePreflight(ctx, fixtureWorkspace, fixtureProcessUser), context.Canceled) +} diff --git a/pkg/driver/types.go b/pkg/driver/types.go index 0ab7c13e9a..590fe9588c 100644 --- a/pkg/driver/types.go +++ b/pkg/driver/types.go @@ -236,6 +236,14 @@ type RecreateRequiredDriver interface { RequiresRecreate(details *config.ContainerDetails, remoteUser string) (bool, string) } +// ReusePreflightDriver validates an existing workspace against its resolved developer identity. +// It must not mutate the workspace. An error aborts reuse without scheduling recreation. +// Calls require SupportsReusePreflight to report true. +type ReusePreflightDriver interface { + SupportsReusePreflight() bool + ReusePreflight(ctx context.Context, workspaceID, remoteUser string) error +} + func DriverRequiresRecreate( d Driver, details *config.ContainerDetails, diff --git a/pkg/provider/env.go b/pkg/provider/env.go index 7156169edb..5cdbec128a 100644 --- a/pkg/provider/env.go +++ b/pkg/provider/env.go @@ -201,6 +201,7 @@ func CloneAgentWorkspaceInfo(agentWorkspaceInfo *AgentWorkspaceInfo) *AgentWorks ret := &AgentWorkspaceInfo{} _ = json.Unmarshal(out, ret) ret.Origin = agentWorkspaceInfo.Origin + ret.WorkspaceWasExisting = agentWorkspaceInfo.WorkspaceWasExisting ret.Workspace = CloneWorkspace(agentWorkspaceInfo.Workspace) ret.Machine = CloneMachine(agentWorkspaceInfo.Machine) return ret diff --git a/pkg/provider/workspace.go b/pkg/provider/workspace.go index 9c8eddb8a6..e7bec6a881 100644 --- a/pkg/provider/workspace.go +++ b/pkg/provider/workspace.go @@ -209,6 +209,10 @@ type AgentWorkspaceInfo struct { // Origin holds the folder where this config was loaded from Origin string `json:"-"` + // WorkspaceWasExisting records same-UID reuse during agent initialization. + // It is local ownership state, never supplied or persisted through JSON. + WorkspaceWasExisting bool `json:"-"` + // InjectTimeout specifies how long to wait for the agent to be injected into the dev container InjectTimeout time.Duration `json:"injectTimeout,omitempty"` diff --git a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx index 507928f581..e359969b9c 100644 --- a/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx +++ b/sites/docs-devsy-sh/content/docs/developing-providers/runtime-protocol.mdx @@ -7,11 +7,11 @@ description: Contract for external runtime driver authors using the Devsy Runtim Providers select this protocol with `agent.driver: external`. See the [external driver](./driver.mdx#external-driver). -The canonical [protobuf schema](https://github.com/devsy-org/devsy-runtime-sdk/blob/main/proto/devsy/runtime/v1/runtime.proto) is maintained in the [Devsy Runtime SDK](https://github.com/devsy-org/devsy-runtime-sdk). The module path is `github.com/devsy-org/devsy-runtime-sdk`; the logical plugin name is `devsy-runtime`. HashiCorp application protocol 1 and Info API major 1/minor 1 are separate version checks. Same-major newer minor versions are accepted. Unknown mount/recreate enum values are rejected because they control host behavior. +The canonical [protobuf schema](https://github.com/devsy-org/devsy-runtime-sdk/blob/main/proto/devsy/runtime/v1/runtime.proto) is maintained in the [Devsy Runtime SDK](https://github.com/devsy-org/devsy-runtime-sdk). The module path is `github.com/devsy-org/devsy-runtime-sdk`; the logical plugin name is `devsy-runtime`. HashiCorp application protocol 1 and Info API major 1/minor 2 are separate version checks. Same-major newer minor versions are accepted. Unknown mount/recreate enum values are rejected because they control host behavior. ## Runtime boundary -Info, Preflight, ProvisioningPreflight, Find, TargetArchitecture, RunImage, Start, Stop, Delete, Exec, and Logs are the v1 RPC surface. Runtime state persists in the backend across plugin processes. Plugins do not own image build/tag/push, registry credentials, Compose, IDE configuration, snapshots, provider machine lifecycle, or updates. +Info, Preflight, ProvisioningPreflight, ReusePreflight, Find, TargetArchitecture, RunImage, Start, Stop, Delete, Exec, and Logs are the v1 RPC surface. Runtime state persists in the backend across plugin processes. Plugins do not own image build/tag/push, registry credentials, Compose, IDE configuration, snapshots, provider machine lifecycle, or updates. RunImage receives resolved intent. Its empty response acknowledges completion; Find queries state. `image_built_locally` is an image-origin hint, not permission to build. Optional privileged/init flags distinguish absent from explicit false. Environment and mounts may contain secrets and must not appear in diagnostic logs. @@ -27,6 +27,8 @@ Find returns `found=false` for ordinary absence, without a NotFound RPC error. A Start on an already running workspace succeeds; missing returns NotFound. Stop on an already stopped workspace succeeds; missing may return NotFound. Delete normalizes missing state to success for cleanup. Provisioning compatibility checks must precede destructive teardown. +API 1.2 adds optional `capabilities.reuse_preflight`. Before reusing a workspace, the host calls ReusePreflight with its workspace ID and current resolved developer identity. The runtime validates its own creation-time contract, such as mount policy or ownership, without starting, stopping, deleting, or modifying the workspace. An incompatible contract returns structured FailedPrecondition with explicit `--recreate` guidance. The host propagates that error without scheduling replacement, preserving the existing VM. Missing workspaces return NotFound; backend, permission, and context failures remain errors. When the capability is absent, hosts skip the RPC and retain their existing identity-resolution behavior. Explicit recreation follows the separate provisioning checks and negotiated stop/delete policy. + ## Exec and Logs The first client frame is exactly one ExecStart containing argv. Later frames contain stdin bytes or exactly one CloseStdin; the client then closes its send side. Data after CloseStdin, repeated Start, unset payloads and empty stdin data frames, and unexpected EOF before CloseStdin are InvalidArgument. v1 Devsy callers use `tty=false`; runtimes reject unsupported TTY requests. @@ -42,3 +44,17 @@ Use canonical gRPC status and attach RuntimeError details for stable categories, The plugin binary is trusted provider code. The host resolves it from checksum-verified `Agent.Binaries`, rather than PATH discovery, and runs it through the runtime supervisor. The magic cookie is not a security boundary. For SDK usage and development commands, see the [SDK README](https://github.com/devsy-org/devsy-runtime-sdk#readme). + +## MicroSandbox parity gate + +The `up-provider-microsandbox` E2E label runs the same lifecycle and ownership scenario against the built-in provider and the external v0.1.5 release, each with isolated Devsy configuration. CI pins MicroSandbox v0.7.7 by checksum and requires access to KVM; unavailable virtualization fails this job instead of producing a passing skipped test. Each provider scenario has a ten-minute deadline and the CI job has a 25-minute deadline. + +The shared scenario exercises agent delivery, SSH, a 1 MiB binary stdin/stdout round trip with separate stderr and a nonzero guest exit, root workload versus developer identity, bind-mount ownership and mode mirroring, stop/start, recreation, rejection of an identity change without recreation while preserving VM-local data, and deletion of the VM. + +Run it on Linux with KVM or Apple silicon after installing MicroSandbox v0.7.7 or newer: + +```sh +DEVSY_REQUIRE_MICROSANDBOX=true task cli:test:e2e:suite -- up-provider-microsandbox +``` + +This baseline does not establish complete parity. Resource limits, hotplug ceilings, storage, ephemeral roots, egress denial, named volumes, tmpfs, alternate mount policies, locally built images, logs, cancellation, and runtime compatibility failures still require coverage before replacing the built-in provider. A green baseline alone does not authorize that cutover.