From 53146d2a7b779b13c1e283ccd36c3e109e04bf8d Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 18 Aug 2026 01:17:23 +0000 Subject: [PATCH 01/12] ci: add apt timeout Signed-off-by: Samuel K --- .github/workflows/pr-ci.yml | 15 +++++++++++++-- pkg/dockerinstall/debian.go | 7 +++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 5ff7db45f..684a00272 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -723,6 +723,7 @@ jobs: - name: cache apt packages (podman/runc) if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux' + timeout: 10m run: | for i in $(seq 1 60); do if sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; then @@ -733,8 +734,18 @@ jobs: fi done mkdir -p "${{ runner.temp }}/apt-archives" - sudo apt-get -o Dir::Cache::Archives="${{ runner.temp }}/apt-archives" update - sudo apt-get -o Dir::Cache::Archives="${{ runner.temp }}/apt-archives" install -y podman runc + sudo apt-get \ + -o Acquire::http::Timeout="30" \ + -o Acquire::https::Timeout="30" \ + -o Acquire::Retries="3" \ + -o Dir::Cache::Archives="${{ runner.temp }}/apt-archives" \ + update + sudo apt-get \ + -o Acquire::http::Timeout="30" \ + -o Acquire::https::Timeout="30" \ + -o Acquire::Retries="3" \ + -o Dir::Cache::Archives="${{ runner.temp }}/apt-archives" \ + install -y podman runc sudo rm -f "${{ runner.temp }}/apt-archives/lock" sudo rm -rf "${{ runner.temp }}/apt-archives/partial" diff --git a/pkg/dockerinstall/debian.go b/pkg/dockerinstall/debian.go index 7a10829bf..5738ffa7a 100644 --- a/pkg/dockerinstall/debian.go +++ b/pkg/dockerinstall/debian.go @@ -65,8 +65,11 @@ func (i *DebianInstaller) setupRepo(shC string) error { ) cmds := []string{ - "apt-get update -qq >/dev/null", - fmt.Sprintf("DEBIAN_FRONTEND=noninteractive apt-get install -y -qq %s >/dev/null", preReqs), + "apt-get -o Acquire::http::Timeout=\"30\" -o Acquire::https::Timeout=\"30\" -o Acquire::Retries=\"3\" update -qq >/dev/null", + fmt.Sprintf( + "DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::http::Timeout=\"30\" -o Acquire::https::Timeout=\"30\" -o Acquire::Retries=\"3\" install -y -qq %s >/dev/null", + preReqs, + ), "mkdir -p /etc/apt/keyrings && chmod -R 0755 /etc/apt/keyrings", fmt.Sprintf( "curl -fsSL \"%s/linux/%s/gpg\" | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg", From c98eca721a827ed8324e6df751ebfedb657827d9 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 18 Aug 2026 01:21:00 +0000 Subject: [PATCH 02/12] fix: timeout-minutes field name Signed-off-by: Samuel K --- .github/workflows/pr-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 684a00272..db646fdaa 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -723,7 +723,7 @@ jobs: - name: cache apt packages (podman/runc) if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux' - timeout: 10m + timeout-minutes: 10 run: | for i in $(seq 1 60); do if sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; then From 33b46f460cad7ccfaf16772a23c8472f2743bbb1 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 18 Aug 2026 01:30:28 +0000 Subject: [PATCH 03/12] fix: linting errors Signed-off-by: Samuel K --- pkg/dockerinstall/debian.go | 42 +++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/pkg/dockerinstall/debian.go b/pkg/dockerinstall/debian.go index 5738ffa7a..cc42fd7ed 100644 --- a/pkg/dockerinstall/debian.go +++ b/pkg/dockerinstall/debian.go @@ -64,20 +64,40 @@ func (i *DebianInstaller) setupRepo(shC string) error { i.opts.channel, ) + aptUpdateCmd := strings.Join([]string{ + "apt-get", + "-o Acquire::http::Timeout=\"30\"", + "-o Acquire::https::Timeout=\"30\"", + "-o Acquire::Retries=\"3\"", + "update -qq >/dev/null", + }, " ") + + aptInstallCmd := strings.Join([]string{ + "DEBIAN_FRONTEND=noninteractive", + "apt-get", + "-o Acquire::http::Timeout=\"30\"", + "-o Acquire::https::Timeout=\"30\"", + "-o Acquire::Retries=\"3\"", + "install -y -qq", + preReqs, + ">/dev/null", + }, " ") + + cmdKeyringDir := "mkdir -p /etc/apt/keyrings && chmod -R 0755 /etc/apt/keyrings" + + cmdDockerGPG := fmt.Sprintf( + "curl -fsSL \"%s/linux/%s/gpg\" | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg", + i.opts.downloadURL, i.distro.ID, + ) + cmds := []string{ - "apt-get -o Acquire::http::Timeout=\"30\" -o Acquire::https::Timeout=\"30\" -o Acquire::Retries=\"3\" update -qq >/dev/null", - fmt.Sprintf( - "DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::http::Timeout=\"30\" -o Acquire::https::Timeout=\"30\" -o Acquire::Retries=\"3\" install -y -qq %s >/dev/null", - preReqs, - ), - "mkdir -p /etc/apt/keyrings && chmod -R 0755 /etc/apt/keyrings", - fmt.Sprintf( - "curl -fsSL \"%s/linux/%s/gpg\" | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg", - i.opts.downloadURL, i.distro.ID, - ), + aptUpdateCmd, + aptInstallCmd, + cmdKeyringDir, + cmdDockerGPG, "chmod a+r /etc/apt/keyrings/docker.gpg", fmt.Sprintf("echo %q > /etc/apt/sources.list.d/docker.list", aptRepo), - "apt-get update -qq >/dev/null", + aptUpdateCmd, } return i.executor.RunCommandsWithRetry(shC, cmds, DefaultTimeout) From d81a6ba71fde054fc376f9bd986bccfc592d0981 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 18 Aug 2026 01:31:48 +0000 Subject: [PATCH 04/12] style: update variable naming Signed-off-by: Samuel K --- pkg/dockerinstall/debian.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/dockerinstall/debian.go b/pkg/dockerinstall/debian.go index cc42fd7ed..1c7bfac13 100644 --- a/pkg/dockerinstall/debian.go +++ b/pkg/dockerinstall/debian.go @@ -64,7 +64,7 @@ func (i *DebianInstaller) setupRepo(shC string) error { i.opts.channel, ) - aptUpdateCmd := strings.Join([]string{ + cmdAptUpdate := strings.Join([]string{ "apt-get", "-o Acquire::http::Timeout=\"30\"", "-o Acquire::https::Timeout=\"30\"", @@ -72,7 +72,7 @@ func (i *DebianInstaller) setupRepo(shC string) error { "update -qq >/dev/null", }, " ") - aptInstallCmd := strings.Join([]string{ + cmdAptInstall := strings.Join([]string{ "DEBIAN_FRONTEND=noninteractive", "apt-get", "-o Acquire::http::Timeout=\"30\"", @@ -91,13 +91,13 @@ func (i *DebianInstaller) setupRepo(shC string) error { ) cmds := []string{ - aptUpdateCmd, - aptInstallCmd, + cmdAptUpdate, + cmdAptInstall, cmdKeyringDir, cmdDockerGPG, "chmod a+r /etc/apt/keyrings/docker.gpg", fmt.Sprintf("echo %q > /etc/apt/sources.list.d/docker.list", aptRepo), - aptUpdateCmd, + cmdAptUpdate, } return i.executor.RunCommandsWithRetry(shC, cmds, DefaultTimeout) From be1eb0145bbf7df42356c6f0d024cd689bcca1cb Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 18 Aug 2026 01:46:15 +0000 Subject: [PATCH 05/12] refactor: apt timeout via context + wait.PollUntilContextTimeout Replace the duplicated apt-get flag strings with a single aptFlags constant, and rewrite the dpkg-lock retry loop in Executor to use context.Context and k8s.io/apimachinery's wait.PollUntilContextTimeout instead of a hand-rolled time.Sleep loop, matching the pattern used elsewhere in this repo (pkg/docker, pkg/apple, e2e/framework). Threads ctx through Installer.Install, DebianInstaller, Executor, and the docker install call chain in cmd/internal/agentworkspace/up.go so cancellation propagates to the underlying exec.CommandContext calls. Signed-off-by: Samuel K --- cmd/internal/agentworkspace/up.go | 14 ++-- pkg/dockerinstall/constants.go | 4 ++ pkg/dockerinstall/debian.go | 45 ++++++------- pkg/dockerinstall/executor.go | 107 +++++++++++++++++++----------- pkg/dockerinstall/install.go | 5 +- pkg/dockerinstall/installer.go | 7 +- 6 files changed, 109 insertions(+), 73 deletions(-) diff --git a/cmd/internal/agentworkspace/up.go b/cmd/internal/agentworkspace/up.go index c3af1f26c..852503d45 100644 --- a/cmd/internal/agentworkspace/up.go +++ b/cmd/internal/agentworkspace/up.go @@ -331,7 +331,7 @@ func (w *workspaceInitializer) initialize(ctx context.Context) error { log.Warnf("failed to set up docker/git credentials (continuing without them): %v", err) } - dockerErrChan := w.installDockerAsync() + dockerErrChan := w.installDockerAsync(ctx) if err := w.prepareWorkspaceContent(ctx); err != nil { return err @@ -400,7 +400,7 @@ type dockerInstallResult struct { err error } -func (w *workspaceInitializer) installDockerAsync() <-chan dockerInstallResult { +func (w *workspaceInitializer) installDockerAsync(ctx context.Context) <-chan dockerInstallResult { resultChan := make(chan dockerInstallResult, 1) go func() { @@ -410,14 +410,14 @@ func (w *workspaceInitializer) installDockerAsync() <-chan dockerInstallResult { return } - dockerPath, err := w.ensureDockerInstalled() + dockerPath, err := w.ensureDockerInstalled(ctx) resultChan <- dockerInstallResult{path: dockerPath, err: err} }() return resultChan } -func (w *workspaceInitializer) ensureDockerInstalled() (string, error) { +func (w *workspaceInitializer) ensureDockerInstalled(ctx context.Context) (string, error) { dockerCmd := w.getDockerCommand() if command.Exists(dockerCmd) { @@ -444,7 +444,7 @@ func (w *workspaceInitializer) ensureDockerInstalled() (string, error) { } log.Debug("attempting to install docker") - dockerPath, err := installDocker() + dockerPath, err := installDocker(ctx) log.Debugf("docker installation path=%q, err=%v", dockerPath, err) return dockerPath, err } @@ -784,11 +784,11 @@ func prepareImage(workspaceDir, image string) error { // installDocker installs Docker and returns the path to the docker binary. // This function assumes docker does not already exist - the caller should check first. -func installDocker() (dockerPath string, err error) { +func installDocker(ctx context.Context) (dockerPath string, err error) { writer := log.Writer(log.LevelInfo) defer func() { _ = writer.Close() }() log.Debug("installing Docker") - return dockerinstall.Install(writer, writer) + return dockerinstall.Install(ctx, writer, writer) } func configureDockerDaemon(ctx context.Context) error { diff --git a/pkg/dockerinstall/constants.go b/pkg/dockerinstall/constants.go index 5c5462d0a..e45fa7ec3 100644 --- a/pkg/dockerinstall/constants.go +++ b/pkg/dockerinstall/constants.go @@ -13,6 +13,10 @@ const ( DeprecationDelay = 10 * time.Second RetryDelay = 10 * time.Second + // Apt options. + AptTimeoutSeconds = 30 + AptRetries = 3 + // Paths. DefaultDownloadURL = "https://download.docker.com" DefaultRepoFile = "docker-ce.repo" diff --git a/pkg/dockerinstall/debian.go b/pkg/dockerinstall/debian.go index 1c7bfac13..d756a16c2 100644 --- a/pkg/dockerinstall/debian.go +++ b/pkg/dockerinstall/debian.go @@ -1,11 +1,19 @@ package dockerinstall import ( + "context" "fmt" "os/exec" "strings" ) +// aptFlags configures apt-get to fail fast on transient network issues +// instead of hanging indefinitely, retrying a bounded number of times first. +var aptFlags = fmt.Sprintf( + `-o Acquire::http::Timeout="%d" -o Acquire::https::Timeout="%d" -o Acquire::Retries="%d"`, + AptTimeoutSeconds, AptTimeoutSeconds, AptRetries, +) + type DebianInstaller struct { distro *Distro opts *InstallOptions @@ -20,8 +28,8 @@ func NewDebianInstaller(distro *Distro, opts *InstallOptions) *DebianInstaller { } } -func (i *DebianInstaller) Install(shC string) error { - if err := i.setupRepo(shC); err != nil { +func (i *DebianInstaller) Install(ctx context.Context, shC string) error { + if err := i.setupRepo(ctx, shC); err != nil { return err } @@ -32,17 +40,18 @@ func (i *DebianInstaller) Install(shC string) error { pkgs := BuildPackageList(i.opts.version, pkgVersion, cliPkgVersion) installCmd := fmt.Sprintf( - "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends %s >/dev/null", + "DEBIAN_FRONTEND=noninteractive apt-get %s install -y -qq --no-install-recommends %s >/dev/null", + aptFlags, pkgs, ) - if err := i.executor.RunWithRetry(shC, installCmd, DefaultTimeout); err != nil { + if err := i.executor.RunWithRetry(ctx, shC, installCmd, DefaultTimeout); err != nil { return err } return nil } -func (i *DebianInstaller) setupRepo(shC string) error { +func (i *DebianInstaller) setupRepo(ctx context.Context, shC string) error { preReqs := "apt-transport-https ca-certificates curl" if !commandExists("gpg") { preReqs += " gnupg" @@ -64,24 +73,12 @@ func (i *DebianInstaller) setupRepo(shC string) error { i.opts.channel, ) - cmdAptUpdate := strings.Join([]string{ - "apt-get", - "-o Acquire::http::Timeout=\"30\"", - "-o Acquire::https::Timeout=\"30\"", - "-o Acquire::Retries=\"3\"", - "update -qq >/dev/null", - }, " ") - - cmdAptInstall := strings.Join([]string{ - "DEBIAN_FRONTEND=noninteractive", - "apt-get", - "-o Acquire::http::Timeout=\"30\"", - "-o Acquire::https::Timeout=\"30\"", - "-o Acquire::Retries=\"3\"", - "install -y -qq", - preReqs, - ">/dev/null", - }, " ") + cmdAptUpdate := fmt.Sprintf("apt-get %s update -qq >/dev/null", aptFlags) + + cmdAptInstall := fmt.Sprintf( + "DEBIAN_FRONTEND=noninteractive apt-get %s install -y -qq %s >/dev/null", + aptFlags, preReqs, + ) cmdKeyringDir := "mkdir -p /etc/apt/keyrings && chmod -R 0755 /etc/apt/keyrings" @@ -100,7 +97,7 @@ func (i *DebianInstaller) setupRepo(shC string) error { cmdAptUpdate, } - return i.executor.RunCommandsWithRetry(shC, cmds, DefaultTimeout) + return i.executor.RunCommandsWithRetry(ctx, shC, cmds, DefaultTimeout) } func (i *DebianInstaller) findVersions() (string, string, error) { diff --git a/pkg/dockerinstall/executor.go b/pkg/dockerinstall/executor.go index 31814c605..4d982f674 100644 --- a/pkg/dockerinstall/executor.go +++ b/pkg/dockerinstall/executor.go @@ -1,12 +1,15 @@ package dockerinstall import ( + "context" "fmt" "io" "os" "os/exec" "strings" "time" + + "k8s.io/apimachinery/pkg/util/wait" ) type Executor struct { @@ -17,7 +20,7 @@ func NewExecutor(opts *InstallOptions) *Executor { return &Executor{opts: opts} } -func (e *Executor) Run(shC, cmdStr string) error { +func (e *Executor) Run(ctx context.Context, shC, cmdStr string) error { fprintln(e.opts.stdout, cmdStr) if e.opts.dryRun { @@ -26,65 +29,90 @@ func (e *Executor) Run(shC, cmdStr string) error { switch { case strings.HasPrefix(shC, "sudo"): - return e.runCommand(exec.Command("sudo", "-E", "sh", "-c", cmdStr)) + //nolint:gosec // G204: cmdStr built internally from constants and opts, not external input + return e.runCommand(exec.CommandContext(ctx, "sudo", "-E", "sh", "-c", cmdStr)) case strings.HasPrefix(shC, "su"): - return e.runCommand(exec.Command("su", "-c", cmdStr)) + //nolint:gosec // G204: cmdStr built internally from constants and opts, not external input + return e.runCommand(exec.CommandContext(ctx, "su", "-c", cmdStr)) case shC == ShellEcho: return nil default: - return e.runCommand(exec.Command("sh", "-c", cmdStr)) + //nolint:gosec // G204: cmdStr built internally from constants and opts, not external input + return e.runCommand(exec.CommandContext(ctx, "sh", "-c", cmdStr)) } } -func (e *Executor) RunWithRetry(shC, cmdStr string, timeout time.Duration) error { - start := time.Now() - for { - fprintln(e.opts.stdout, fmt.Sprintf("running command: %s", cmdStr)) - - stderrBuf := &strings.Builder{} - err := e.runWithStderrCapture(shC, cmdStr, stderrBuf) - - if err == nil { - fprintln(e.opts.stdout, "command succeeded") - return nil - } - - stderrStr := stderrBuf.String() - isDpkgLock := strings.Contains(stderrStr, "Could not get lock") || - strings.Contains(stderrStr, "/var/lib/dpkg/lock") - - if !isDpkgLock { - return err - } - - if time.Since(start) >= timeout { - return fmt.Errorf("timeout waiting for dpkg lock after %v: %w", timeout, err) - } +// isDpkgLockError reports whether stderr indicates another process (e.g. +// unattended-upgrades) is holding the dpkg lock, a transient condition worth +// retrying rather than a real command failure. +func isDpkgLockError(stderr string) bool { + return strings.Contains(stderr, "Could not get lock") || + strings.Contains(stderr, "/var/lib/dpkg/lock") +} - fprintln(e.opts.stderr, "waiting for dpkg lock to be released") - time.Sleep(RetryDelay) +// RunWithRetry runs cmdStr, retrying at RetryDelay intervals while it fails +// with a dpkg lock error, up to timeout. Any other error returns immediately. +func (e *Executor) RunWithRetry( + ctx context.Context, + shC, cmdStr string, + timeout time.Duration, +) error { + var lastErr error + pollErr := wait.PollUntilContextTimeout( + ctx, RetryDelay, timeout, true, + func(ctx context.Context) (bool, error) { + fprintln(e.opts.stdout, fmt.Sprintf("running command: %s", cmdStr)) + + stderrBuf := &strings.Builder{} + err := e.runWithStderrCapture(ctx, shC, cmdStr, stderrBuf) + if err == nil { + fprintln(e.opts.stdout, "command succeeded") + return true, nil + } + + if !isDpkgLockError(stderrBuf.String()) { + return true, err + } + + lastErr = err + fprintln(e.opts.stderr, "waiting for dpkg lock to be released") + return false, nil + }, + ) + if pollErr != nil && lastErr != nil { + return fmt.Errorf("timeout waiting for dpkg lock after %v: %w", timeout, lastErr) } + return pollErr } -func (e *Executor) RunCommands(shC string, cmds []string) error { +func (e *Executor) RunCommands(ctx context.Context, shC string, cmds []string) error { for _, cmd := range cmds { - if err := e.Run(shC, cmd); err != nil { + if err := e.Run(ctx, shC, cmd); err != nil { return err } } return nil } -func (e *Executor) RunCommandsWithRetry(shC string, cmds []string, timeout time.Duration) error { +func (e *Executor) RunCommandsWithRetry( + ctx context.Context, + shC string, + cmds []string, + timeout time.Duration, +) error { for _, cmd := range cmds { - if err := e.RunWithRetry(shC, cmd, timeout); err != nil { + if err := e.RunWithRetry(ctx, shC, cmd, timeout); err != nil { return err } } return nil } -func (e *Executor) runWithStderrCapture(shC, cmdStr string, stderrBuf *strings.Builder) error { +func (e *Executor) runWithStderrCapture( + ctx context.Context, + shC, cmdStr string, + stderrBuf *strings.Builder, +) error { fprintln(e.opts.stdout, cmdStr) if e.opts.dryRun { @@ -94,13 +122,16 @@ func (e *Executor) runWithStderrCapture(shC, cmdStr string, stderrBuf *strings.B var cmd *exec.Cmd switch { case strings.HasPrefix(shC, "sudo"): - cmd = exec.Command("sudo", "-E", "sh", "-c", cmdStr) + //nolint:gosec // G204: cmdStr built internally from constants and opts, not external input + cmd = exec.CommandContext(ctx, "sudo", "-E", "sh", "-c", cmdStr) case strings.HasPrefix(shC, "su"): - cmd = exec.Command("su", "-c", cmdStr) + //nolint:gosec // G204: cmdStr built internally from constants and opts, not external input + cmd = exec.CommandContext(ctx, "su", "-c", cmdStr) case shC == ShellEcho: return nil default: - cmd = exec.Command("sh", "-c", cmdStr) + //nolint:gosec // G204: cmdStr built internally from constants and opts, not external input + cmd = exec.CommandContext(ctx, "sh", "-c", cmdStr) } cmd.Stdout = e.opts.stdout diff --git a/pkg/dockerinstall/install.go b/pkg/dockerinstall/install.go index bde2b8614..48aca205f 100644 --- a/pkg/dockerinstall/install.go +++ b/pkg/dockerinstall/install.go @@ -1,6 +1,7 @@ package dockerinstall import ( + "context" "fmt" "io" "os" @@ -8,7 +9,7 @@ import ( "strings" ) -func Install(stdout, stderr io.Writer) (string, error) { +func Install(ctx context.Context, stdout, stderr io.Writer) (string, error) { opts := &InstallOptions{ channel: getEnv("CHANNEL", ChannelStable), version: getEnv("VERSION", ""), @@ -47,7 +48,7 @@ func Install(stdout, stderr io.Writer) (string, error) { if installer == nil { return "", fmt.Errorf("unsupported distribution: %s", distro.ID) } - if err := installer.Install(shC); err != nil { + if err := installer.Install(ctx, shC); err != nil { return "", fmt.Errorf("docker installation failed: %w", err) } diff --git a/pkg/dockerinstall/installer.go b/pkg/dockerinstall/installer.go index 853d0c523..7495c9675 100644 --- a/pkg/dockerinstall/installer.go +++ b/pkg/dockerinstall/installer.go @@ -1,6 +1,9 @@ package dockerinstall -import "io" +import ( + "context" + "io" +) type InstallOptions struct { channel string @@ -13,5 +16,5 @@ type InstallOptions struct { } type Installer interface { - Install(shC string) error + Install(ctx context.Context, shC string) error } From 9d94a34e9aaa68fb9687bbd6056ebcf9f93ba4b5 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 18 Aug 2026 02:49:28 +0000 Subject: [PATCH 06/12] ci: force apt IPv4 to avoid ipv6 connect hangs The 'cache apt packages (podman/runc)' step was hanging for the full 10-minute step timeout with zero apt output, not the expected 'Failed to fetch'/retry cycle. Acquire::http::Timeout and Acquire::Retries only bound a completed, failed transfer attempt; they don't reliably bound a stalled TCP connect. GitHub-hosted Ubuntu runners commonly have a broken/black-holed IPv6 route to archive.ubuntu.com, so apt's IPv6 connection attempt hangs until the OS-level connect timeout, well past the 10-minute job step timeout. Force IPv4 on both apt-get invocations in that step so the IPv6 connect attempt is skipped entirely. Signed-off-by: Samuel K --- .github/workflows/pr-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index db646fdaa..87535330d 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -735,12 +735,14 @@ jobs: done mkdir -p "${{ runner.temp }}/apt-archives" sudo apt-get \ + -o Acquire::ForceIPv4="true" \ -o Acquire::http::Timeout="30" \ -o Acquire::https::Timeout="30" \ -o Acquire::Retries="3" \ -o Dir::Cache::Archives="${{ runner.temp }}/apt-archives" \ update sudo apt-get \ + -o Acquire::ForceIPv4="true" \ -o Acquire::http::Timeout="30" \ -o Acquire::https::Timeout="30" \ -o Acquire::Retries="3" \ From 5813051f574f1a507be7e5b45087fb665ee19055 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 18 Aug 2026 03:06:06 +0000 Subject: [PATCH 07/12] ci: install podman/runc via Homebrew instead of apt Replaces the apt-get install of podman/runc (subject to Ubuntu mirror IPv6/getaddrinfo hangs, see prior commit) with Homebrew, which is pre-installed on the ubuntu-latest runner image and fetches bottles from ghcr.io rather than going through Ubuntu's mirror-selection layer. Adjustments required by the switch: - sudo's default secure_path excludes Homebrew's prefix; add a sudoers.d drop-in so rootful podman (run via sudo) can still exec runc/conmon/crun/fuse-overlayfs. - containers.conf now also pins engine.runtimes.runc to the Homebrew path explicitly, as defense in depth. - Homebrew's podman formula does not ship a podman.socket systemd unit (unlike the apt package), so the rootful step now starts 'podman system service' directly via 'sudo -b' instead of 'systemctl enable --now podman.socket'. - apt-archives cache/env vars replaced with a Homebrew download cache (HOMEBREW_CACHE); the unused APT_CACHE_DIR env vars on the rootless/rootful steps are dropped. Signed-off-by: Samuel K --- .github/workflows/pr-ci.yml | 64 +++++++++++++++---------------------- 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 87535330d..228fafaa1 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -714,65 +714,51 @@ jobs: image: kindest/node:v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a skipClusterLogsExport: true - - name: cache apt packages (podman/runc) + - name: cache Homebrew downloads (podman/runc) if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: ${{ runner.temp }}/apt-archives - key: apt-podman-runc-${{ runner.os }}-v1 + path: ${{ runner.temp }}/homebrew-cache + key: brew-podman-runc-${{ runner.os }}-${{ runner.arch }}-v1 - - name: cache apt packages (podman/runc) + - name: install podman/runc (Homebrew) if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux' timeout-minutes: 10 + env: + HOMEBREW_CACHE: ${{ runner.temp }}/homebrew-cache + HOMEBREW_NO_AUTO_UPDATE: "1" + HOMEBREW_NO_INSTALL_CLEANUP: "1" run: | - for i in $(seq 1 60); do - if sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; then - echo "waiting for dpkg lock to be released ($i/60)" - sleep 2 - else - break - fi - done - mkdir -p "${{ runner.temp }}/apt-archives" - sudo apt-get \ - -o Acquire::ForceIPv4="true" \ - -o Acquire::http::Timeout="30" \ - -o Acquire::https::Timeout="30" \ - -o Acquire::Retries="3" \ - -o Dir::Cache::Archives="${{ runner.temp }}/apt-archives" \ - update - sudo apt-get \ - -o Acquire::ForceIPv4="true" \ - -o Acquire::http::Timeout="30" \ - -o Acquire::https::Timeout="30" \ - -o Acquire::Retries="3" \ - -o Dir::Cache::Archives="${{ runner.temp }}/apt-archives" \ - install -y podman runc - - sudo rm -f "${{ runner.temp }}/apt-archives/lock" - sudo rm -rf "${{ runner.temp }}/apt-archives/partial" + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH" + brew install runc podman + + # sudo's default secure_path excludes Homebrew's prefix, but rootful + # podman runs as root (via sudo) and needs it on PATH to exec runc, + # conmon, crun, and fuse-overlayfs. + echo "Defaults secure_path=\"$HOMEBREW_PREFIX/bin:$HOMEBREW_PREFIX/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"" \ + | sudo tee /etc/sudoers.d/homebrew-path >/dev/null + sudo visudo -cf /etc/sudoers.d/homebrew-path sudo mkdir -p /etc/containers - printf '[engine]\nruntime = "runc"\n' | sudo tee /etc/containers/containers.conf + printf '[engine]\nruntime = "runc"\n\n[engine.runtimes]\nrunc = ["%s/bin/runc"]\n' "$HOMEBREW_PREFIX" \ + | sudo tee /etc/containers/containers.conf - name: Install Podman (Linux rootless) if: matrix.install-podman == 'rootless' && runner.os == 'Linux' - env: - APT_CACHE_DIR: ${{ runner.temp }}/apt-archives run: | podman info podman run --rm busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d echo "podman runtime preflight OK" - name: Install Podman (Linux rootful) if: matrix.install-podman == 'rootful' && runner.os == 'Linux' - env: - APT_CACHE_DIR: ${{ runner.temp }}/apt-archives run: | - sudo systemctl enable --now podman.socket + sudo mkdir -p /run/podman + sudo -b podman system service --time=0 unix:///run/podman/podman.sock \ + > /tmp/podman-service.log 2>&1 if ! timeout 30 bash -c 'until sudo podman info >/dev/null 2>&1; do sleep 1; done'; then - echo "::error::podman.socket did not become ready within 30s" - sudo systemctl status podman.socket --no-pager || true - sudo journalctl -u podman.socket --no-pager -n 100 || true + echo "::error::podman service did not become ready within 30s" + cat /tmp/podman-service.log || true exit 1 fi echo "DOCKER_HOST=unix:///run/podman/podman.sock" >> "$GITHUB_ENV" From c3c02b933d58e9a9b1790586aefecc2c0b157836 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 18 Aug 2026 03:08:33 +0000 Subject: [PATCH 08/12] ci: use podman's default crun runtime instead of runc crun is already a Homebrew podman dependency; drop the separate runc install and the containers.conf override that forced runtime=runc, letting podman use its default. Also drops the explanatory comment on the sudoers secure_path drop-in (still needed either way, since crun/conmon/fuse-overlayfs live under the Homebrew prefix too). Signed-off-by: Samuel K --- .github/workflows/pr-ci.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 228fafaa1..3f1b09c05 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -714,14 +714,14 @@ jobs: image: kindest/node:v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a skipClusterLogsExport: true - - name: cache Homebrew downloads (podman/runc) + - name: cache Homebrew downloads (podman) if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/homebrew-cache - key: brew-podman-runc-${{ runner.os }}-${{ runner.arch }}-v1 + key: brew-podman-${{ runner.os }}-${{ runner.arch }}-v1 - - name: install podman/runc (Homebrew) + - name: install podman (Homebrew) if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux' timeout-minutes: 10 env: @@ -731,19 +731,12 @@ jobs: run: | eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH" - brew install runc podman + brew install podman - # sudo's default secure_path excludes Homebrew's prefix, but rootful - # podman runs as root (via sudo) and needs it on PATH to exec runc, - # conmon, crun, and fuse-overlayfs. echo "Defaults secure_path=\"$HOMEBREW_PREFIX/bin:$HOMEBREW_PREFIX/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"" \ | sudo tee /etc/sudoers.d/homebrew-path >/dev/null sudo visudo -cf /etc/sudoers.d/homebrew-path - sudo mkdir -p /etc/containers - printf '[engine]\nruntime = "runc"\n\n[engine.runtimes]\nrunc = ["%s/bin/runc"]\n' "$HOMEBREW_PREFIX" \ - | sudo tee /etc/containers/containers.conf - - name: Install Podman (Linux rootless) if: matrix.install-podman == 'rootless' && runner.os == 'Linux' run: | From c12c28b0295a1dd6f114dde591a5b3a72267bb0d Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 18 Aug 2026 03:20:12 +0000 Subject: [PATCH 09/12] ci: fix shellcheck SC2024 in podman rootful service startup 'sudo cmd > file' redirects in the caller's (unprivileged) shell, not under sudo, which is what shellcheck's SC2024 flags. Move the redirect inside the sudo'd command via 'sudo -b sh -c ...' so the logfile write happens under the same sudo invocation. Signed-off-by: Samuel K --- .github/workflows/pr-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 3f1b09c05..fda696f78 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -747,8 +747,7 @@ jobs: if: matrix.install-podman == 'rootful' && runner.os == 'Linux' run: | sudo mkdir -p /run/podman - sudo -b podman system service --time=0 unix:///run/podman/podman.sock \ - > /tmp/podman-service.log 2>&1 + sudo -b sh -c 'podman system service --time=0 unix:///run/podman/podman.sock > /tmp/podman-service.log 2>&1' if ! timeout 30 bash -c 'until sudo podman info >/dev/null 2>&1; do sleep 1; done'; then echo "::error::podman service did not become ready within 30s" cat /tmp/podman-service.log || true From 676deb0a1b2d9d6b2902dc90ef84fbe436ea5dcd Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 18 Aug 2026 03:52:21 +0000 Subject: [PATCH 10/12] fix(automations): isolate generator test writes in a sandbox dir TestGenerateWritesAllAgents/TestCheckIsIdempotent/TestToolchainPins/ TestToolchainDownloadsAreBounded ran the automations generator with cmd.Dir pointed at the real repo checkout, so it read/wrote the real .agents/agents/*/agent.md files as a side effect of running go test. goreleaser's build pre-hooks run 'go test github.com/devsy-org/devsy github.com/devsy-org/devsy/cmd github.com/devsy-org/devsy/cmd/ci github.com/devsy-org/devsy/cmd/completion github.com/devsy-org/devsy/cmd/config github.com/devsy-org/devsy/cmd/context github.com/devsy-org/devsy/cmd/env github.com/devsy-org/devsy/cmd/feature github.com/devsy-org/devsy/cmd/flags github.com/devsy-org/devsy/cmd/ide github.com/devsy-org/devsy/cmd/internal github.com/devsy-org/devsy/cmd/internal/agentcontainer github.com/devsy-org/devsy/cmd/internal/agentworkspace github.com/devsy-org/devsy/cmd/machine github.com/devsy-org/devsy/cmd/mcp github.com/devsy-org/devsy/cmd/pro github.com/devsy-org/devsy/cmd/pro/cluster github.com/devsy-org/devsy/cmd/pro/completion github.com/devsy-org/devsy/cmd/pro/daemon github.com/devsy-org/devsy/cmd/pro/flags github.com/devsy-org/devsy/cmd/pro/project github.com/devsy-org/devsy/cmd/pro/proutil github.com/devsy-org/devsy/cmd/pro/provider github.com/devsy-org/devsy/cmd/pro/provider/create github.com/devsy-org/devsy/cmd/pro/provider/get github.com/devsy-org/devsy/cmd/pro/provider/list github.com/devsy-org/devsy/cmd/pro/provider/update github.com/devsy-org/devsy/cmd/pro/provider/watch github.com/devsy-org/devsy/cmd/pro/template github.com/devsy-org/devsy/cmd/pro/user github.com/devsy-org/devsy/cmd/pro/workspace github.com/devsy-org/devsy/cmd/provider github.com/devsy-org/devsy/cmd/secrets github.com/devsy-org/devsy/cmd/snapshot github.com/devsy-org/devsy/cmd/template github.com/devsy-org/devsy/cmd/update github.com/devsy-org/devsy/cmd/workspace github.com/devsy-org/devsy/cmd/workspace/up github.com/devsy-org/devsy/e2e github.com/devsy-org/devsy/e2e/framework github.com/devsy-org/devsy/e2e/tests/build github.com/devsy-org/devsy/e2e/tests/ci github.com/devsy-org/devsy/e2e/tests/configapply github.com/devsy-org/devsy/e2e/tests/configread github.com/devsy-org/devsy/e2e/tests/context github.com/devsy-org/devsy/e2e/tests/delivery github.com/devsy-org/devsy/e2e/tests/dockerinstall github.com/devsy-org/devsy/e2e/tests/down github.com/devsy-org/devsy/e2e/tests/exec github.com/devsy-org/devsy/e2e/tests/extends github.com/devsy-org/devsy/e2e/tests/extends-up github.com/devsy-org/devsy/e2e/tests/feature github.com/devsy-org/devsy/e2e/tests/ide github.com/devsy-org/devsy/e2e/tests/integration github.com/devsy-org/devsy/e2e/tests/logs github.com/devsy-org/devsy/e2e/tests/machine github.com/devsy-org/devsy/e2e/tests/machineprovider github.com/devsy-org/devsy/e2e/tests/mcp github.com/devsy-org/devsy/e2e/tests/outdated github.com/devsy-org/devsy/e2e/tests/provider github.com/devsy-org/devsy/e2e/tests/rename github.com/devsy-org/devsy/e2e/tests/runusercommands github.com/devsy-org/devsy/e2e/tests/self github.com/devsy-org/devsy/e2e/tests/snapshot github.com/devsy-org/devsy/e2e/tests/ssh github.com/devsy-org/devsy/e2e/tests/template github.com/devsy-org/devsy/e2e/tests/tunnel github.com/devsy-org/devsy/e2e/tests/up github.com/devsy-org/devsy/e2e/tests/up-docker-compose github.com/devsy-org/devsy/e2e/tests/up-features github.com/devsy-org/devsy/e2e/tests/upgrade github.com/devsy-org/devsy/hack/automations github.com/devsy-org/devsy/hack/gen_github_app_jwt github.com/devsy-org/devsy/hack/homebrew_cask github.com/devsy-org/devsy/hack/homebrew_formula github.com/devsy-org/devsy/hack/licenses github.com/devsy-org/devsy/hack/merge_mac_metadata github.com/devsy-org/devsy/hack/pro github.com/devsy-org/devsy/hack/release_artifacts github.com/devsy-org/devsy/hack/rewrite_manifest_urls github.com/devsy-org/devsy/hack/sign_commit github.com/devsy-org/devsy/pkg/agent github.com/devsy-org/devsy/pkg/agent/delivery github.com/devsy-org/devsy/pkg/agent/snapshot github.com/devsy-org/devsy/pkg/agent/tunnel github.com/devsy-org/devsy/pkg/agent/tunnelserver github.com/devsy-org/devsy/pkg/apple github.com/devsy-org/devsy/pkg/client github.com/devsy-org/devsy/pkg/client/clientimplementation github.com/devsy-org/devsy/pkg/client/clientimplementation/daemonclient github.com/devsy-org/devsy/pkg/client/proxycmd github.com/devsy-org/devsy/pkg/clierr github.com/devsy-org/devsy/pkg/clihelp github.com/devsy-org/devsy/pkg/command github.com/devsy-org/devsy/pkg/compose github.com/devsy-org/devsy/pkg/compress github.com/devsy-org/devsy/pkg/config github.com/devsy-org/devsy/pkg/copy github.com/devsy-org/devsy/pkg/credentials github.com/devsy-org/devsy/pkg/daemon/agent github.com/devsy-org/devsy/pkg/daemon/local github.com/devsy-org/devsy/pkg/daemon/platform github.com/devsy-org/devsy/pkg/devcontainer github.com/devsy-org/devsy/pkg/devcontainer/build github.com/devsy-org/devsy/pkg/devcontainer/buildkit github.com/devsy-org/devsy/pkg/devcontainer/config github.com/devsy-org/devsy/pkg/devcontainer/crane github.com/devsy-org/devsy/pkg/devcontainer/feature github.com/devsy-org/devsy/pkg/devcontainer/graph github.com/devsy-org/devsy/pkg/devcontainer/metadata github.com/devsy-org/devsy/pkg/devcontainer/setup github.com/devsy-org/devsy/pkg/devcontainer/sshtunnel github.com/devsy-org/devsy/pkg/devsyconfig github.com/devsy-org/devsy/pkg/docker github.com/devsy-org/devsy/pkg/dockercredentials github.com/devsy-org/devsy/pkg/dockerfile github.com/devsy-org/devsy/pkg/dockerinstall github.com/devsy-org/devsy/pkg/dotfiles github.com/devsy-org/devsy/pkg/download github.com/devsy-org/devsy/pkg/driver github.com/devsy-org/devsy/pkg/driver/apple github.com/devsy-org/devsy/pkg/driver/custom github.com/devsy-org/devsy/pkg/driver/docker github.com/devsy-org/devsy/pkg/driver/drivercreate github.com/devsy-org/devsy/pkg/driver/kubernetes github.com/devsy-org/devsy/pkg/driver/kubernetes/throttledlogger github.com/devsy-org/devsy/pkg/driver/microsandbox github.com/devsy-org/devsy/pkg/encoding github.com/devsy-org/devsy/pkg/envfile github.com/devsy-org/devsy/pkg/exitcode github.com/devsy-org/devsy/pkg/extract github.com/devsy-org/devsy/pkg/file github.com/devsy-org/devsy/pkg/flags github.com/devsy-org/devsy/pkg/flags/names github.com/devsy-org/devsy/pkg/flatpak github.com/devsy-org/devsy/pkg/git github.com/devsy-org/devsy/pkg/gitcredentials github.com/devsy-org/devsy/pkg/gitsshsigning github.com/devsy-org/devsy/pkg/gpg github.com/devsy-org/devsy/pkg/hash github.com/devsy-org/devsy/pkg/http github.com/devsy-org/devsy/pkg/id github.com/devsy-org/devsy/pkg/ide github.com/devsy-org/devsy/pkg/ide/codeserver github.com/devsy-org/devsy/pkg/ide/fleet github.com/devsy-org/devsy/pkg/ide/ideparse github.com/devsy-org/devsy/pkg/ide/jetbrains github.com/devsy-org/devsy/pkg/ide/jupyter github.com/devsy-org/devsy/pkg/ide/marimo github.com/devsy-org/devsy/pkg/ide/opener github.com/devsy-org/devsy/pkg/ide/openvscode github.com/devsy-org/devsy/pkg/ide/rstudio github.com/devsy-org/devsy/pkg/ide/vscode github.com/devsy-org/devsy/pkg/ide/vscodeweb github.com/devsy-org/devsy/pkg/ide/zed github.com/devsy-org/devsy/pkg/image github.com/devsy-org/devsy/pkg/inject github.com/devsy-org/devsy/pkg/language github.com/devsy-org/devsy/pkg/log github.com/devsy-org/devsy/pkg/machineid github.com/devsy-org/devsy/pkg/netstat github.com/devsy-org/devsy/pkg/open github.com/devsy-org/devsy/pkg/options github.com/devsy-org/devsy/pkg/options/resolver github.com/devsy-org/devsy/pkg/output github.com/devsy-org/devsy/pkg/platform github.com/devsy-org/devsy/pkg/platform/annotations github.com/devsy-org/devsy/pkg/platform/client github.com/devsy-org/devsy/pkg/platform/form github.com/devsy-org/devsy/pkg/platform/kube github.com/devsy-org/devsy/pkg/platform/parameters github.com/devsy-org/devsy/pkg/platform/project github.com/devsy-org/devsy/pkg/platform/remotecommand github.com/devsy-org/devsy/pkg/port github.com/devsy-org/devsy/pkg/provider github.com/devsy-org/devsy/pkg/pty github.com/devsy-org/devsy/pkg/pty/ptytest github.com/devsy-org/devsy/pkg/random github.com/devsy-org/devsy/pkg/scanner github.com/devsy-org/devsy/pkg/secrets github.com/devsy-org/devsy/pkg/selfupdate github.com/devsy-org/devsy/pkg/sharedfile github.com/devsy-org/devsy/pkg/shell github.com/devsy-org/devsy/pkg/snapshot github.com/devsy-org/devsy/pkg/ssh github.com/devsy-org/devsy/pkg/ssh/agent github.com/devsy-org/devsy/pkg/ssh/server github.com/devsy-org/devsy/pkg/ssh/server/port github.com/devsy-org/devsy/pkg/status github.com/devsy-org/devsy/pkg/stdio github.com/devsy-org/devsy/pkg/survey github.com/devsy-org/devsy/pkg/table github.com/devsy-org/devsy/pkg/task github.com/devsy-org/devsy/pkg/telemetry github.com/devsy-org/devsy/pkg/telemetry/analytics github.com/devsy-org/devsy/pkg/telemetry/distinctid github.com/devsy-org/devsy/pkg/template github.com/devsy-org/devsy/pkg/terminal github.com/devsy-org/devsy/pkg/theme github.com/devsy-org/devsy/pkg/token github.com/devsy-org/devsy/pkg/ts github.com/devsy-org/devsy/pkg/tunnel github.com/devsy-org/devsy/pkg/types github.com/devsy-org/devsy/pkg/util github.com/devsy-org/devsy/pkg/util/goleaktest github.com/devsy-org/devsy/pkg/util/hash github.com/devsy-org/devsy/pkg/util/iojoin github.com/devsy-org/devsy/pkg/version github.com/devsy-org/devsy/pkg/workspace github.com/devsy-org/devsy/providers -race -short' once per goarch target within a build id (e.g. linux/amd64 and linux/arm64 for the devsy-linux id), and goreleaser builds those targets in parallel by default. Two concurrent go test invocations both mutating the same real files raced: one process's partial write could be read mid-write by the other, producing spurious 'missing ...' assertion failures (observed in CI, unreproducible locally since a single go test run never overlaps itself). Copy hack/automations/agents.yaml and the existing .agents/agents tree into a per-test t.TempDir() sandbox and run/inspect the generator there instead of the real checkout. Verified 4 concurrent go test invocations now all pass consistently, and the real .agents/ tree is untouched by running the suite. Signed-off-by: Samuel K --- hack/automations/main_test.go | 92 +++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 14 deletions(-) diff --git a/hack/automations/main_test.go b/hack/automations/main_test.go index 3ef441fd2..1237b3a14 100644 --- a/hack/automations/main_test.go +++ b/hack/automations/main_test.go @@ -1,6 +1,7 @@ package main import ( + "io/fs" "os" "os/exec" "path/filepath" @@ -8,14 +9,74 @@ import ( "testing" ) -func runGenerator(t *testing.T, args ...string) string { +// newSandbox copies the generator's real inputs (agents.yaml and the existing +// .agents/agents tree) into an isolated temp directory and returns its path. +// The generator writes agent.md files relative to its working directory, and +// goreleaser's build hooks run this test suite once per target arch in +// parallel; without isolation, concurrent runs race on the same real +// .agents/agents/*/agent.md files in the checkout. +func newSandbox(t *testing.T) string { + t.Helper() + root := repoRoot(t) + dir := t.TempDir() + + copyFile(t, + filepath.Join(root, "hack", "automations", "agents.yaml"), + filepath.Join(dir, "hack", "automations", "agents.yaml")) + copyTree(t, + filepath.Join(root, ".agents", "agents"), + filepath.Join(dir, ".agents", "agents")) + + return dir +} + +func copyFile(t *testing.T, src, dst string) { + t.Helper() + b, err := os.ReadFile(src) + if err != nil { + t.Fatalf("read %s: %v", src, err) + } + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(dst), err) + } + if err := os.WriteFile(dst, b, 0o644); err != nil { + t.Fatalf("write %s: %v", dst, err) + } +} + +func copyTree(t *testing.T, src, dst string) { + t.Helper() + err := filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + target := filepath.Join(dst, rel) + if d.IsDir() { + return os.MkdirAll(target, 0o755) + } + b, err := os.ReadFile(path) + if err != nil { + return err + } + return os.WriteFile(target, b, 0o644) + }) + if err != nil { + t.Fatalf("copy tree %s: %v", src, err) + } +} + +func runGenerator(t *testing.T, dir string, args ...string) string { t.Helper() bin := filepath.Join(t.TempDir(), "automations") if out, err := exec.Command("go", "build", "-o", bin, ".").CombinedOutput(); err != nil { t.Fatalf("go build: %v\n%s", err, out) } cmd := exec.Command(bin, args...) - cmd.Dir = repoRoot(t) + cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { t.Logf("generator output:\n%s", out) @@ -32,13 +93,13 @@ func repoRoot(t *testing.T) string { return filepath.Dir(filepath.Dir(wd)) } -func agentFilePath(t *testing.T, id string) string { - t.Helper() - return filepath.Join(repoRoot(t), ".agents", "agents", id, "agent.md") +func agentFilePath(dir, id string) string { + return filepath.Join(dir, ".agents", "agents", id, "agent.md") } func TestGenerateWritesAllAgents(t *testing.T) { - out := runGenerator(t) + dir := newSandbox(t) + out := runGenerator(t, dir) if !strings.Contains(out, "wrote") { t.Fatalf("expected write output, got:\n%s", out) } @@ -51,7 +112,7 @@ func TestGenerateWritesAllAgents(t *testing.T) { pythonAgents := map[string]bool{"agent-analytics": true} for _, id := range ids { - p := agentFilePath(t, id) + p := agentFilePath(dir, id) b, err := os.ReadFile(p) if err != nil { t.Errorf("read %s: %v", p, err) @@ -87,16 +148,18 @@ func TestGenerateWritesAllAgents(t *testing.T) { } func TestCheckIsIdempotent(t *testing.T) { - runGenerator(t) - out := runGenerator(t, "-check") + dir := newSandbox(t) + runGenerator(t, dir) + out := runGenerator(t, dir, "-check") if strings.Contains(out, "DRIFT") { t.Fatalf("freshly generated files report drift:\n%s", out) } } func TestToolchainPins(t *testing.T) { - runGenerator(t) - goAgent := agentFilePath(t, "pkg-container") + dir := newSandbox(t) + runGenerator(t, dir) + goAgent := agentFilePath(dir, "pkg-container") b, err := os.ReadFile(goAgent) if err != nil { t.Fatal(err) @@ -107,7 +170,7 @@ func TestToolchainPins(t *testing.T) { t.Errorf("pkg-container: missing pinned %q", want) } } - ciAgent := agentFilePath(t, "ci-optimizer") + ciAgent := agentFilePath(dir, "ci-optimizer") cb, err := os.ReadFile(ciAgent) if err != nil { t.Fatal(err) @@ -118,9 +181,10 @@ func TestToolchainPins(t *testing.T) { } func TestToolchainDownloadsAreBounded(t *testing.T) { - runGenerator(t) + dir := newSandbox(t) + runGenerator(t, dir) for _, id := range []string{"ui-polish", "pkg-container", "agent-analytics"} { - p := agentFilePath(t, id) + p := agentFilePath(dir, id) b, err := os.ReadFile(p) if err != nil { continue From 862b54d99f0238c96068173e0d669f5f94fd131a Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 18 Aug 2026 04:06:18 +0000 Subject: [PATCH 11/12] fix: address PR review feedback - hack/automations/main_test.go: remove explanatory doc comment on newSandbox per reviewer request. - .github/workflows/pr-ci.yml: the podman rootful readiness loop checked 'sudo podman info', which always succeeds against the local engine regardless of whether the backgrounded 'podman system service' has bound the socket yet. Probe 'sudo podman --remote --url unix:///run/podman/podman.sock info' instead so readiness reflects the actual DOCKER_HOST endpoint later steps depend on. - cmd/internal/agentworkspace/up.go: lowercase the 'installing docker' log message per repo logging conventions. - pkg/dockerinstall/executor.go: RunWithRetry discarded the caller's context error (Canceled/DeadlineExceeded) in favor of the dpkg-lock timeout message whenever lastErr was set. Check ctx.Err() first so caller cancellation/deadline is reported accurately; only wrap lastErr into the dpkg-lock timeout message when the poll failure is our own internal per-call timeout, not the caller's context. Added executor_test.go covering context-cancellation precedence, dpkg-lock timeout wrapping, immediate non-lock-error return, and the success path (previously untested). Signed-off-by: Samuel K --- .github/workflows/pr-ci.yml | 4 +- cmd/internal/agentworkspace/up.go | 2 +- hack/automations/main_test.go | 6 --- pkg/dockerinstall/executor.go | 9 ++++- pkg/dockerinstall/executor_test.go | 63 ++++++++++++++++++++++++++++++ 5 files changed, 73 insertions(+), 11 deletions(-) create mode 100644 pkg/dockerinstall/executor_test.go diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index fda696f78..ddfce224d 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -748,13 +748,13 @@ jobs: run: | sudo mkdir -p /run/podman sudo -b sh -c 'podman system service --time=0 unix:///run/podman/podman.sock > /tmp/podman-service.log 2>&1' - if ! timeout 30 bash -c 'until sudo podman info >/dev/null 2>&1; do sleep 1; done'; then + if ! timeout 30 bash -c 'until sudo podman --remote --url unix:///run/podman/podman.sock info >/dev/null 2>&1; do sleep 1; done'; then echo "::error::podman service did not become ready within 30s" cat /tmp/podman-service.log || true exit 1 fi echo "DOCKER_HOST=unix:///run/podman/podman.sock" >> "$GITHUB_ENV" - sudo podman info + sudo podman --remote --url unix:///run/podman/podman.sock info sudo podman run --rm busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d echo "podman runtime preflight OK" - name: get microsandbox latest version diff --git a/cmd/internal/agentworkspace/up.go b/cmd/internal/agentworkspace/up.go index 852503d45..d368978e4 100644 --- a/cmd/internal/agentworkspace/up.go +++ b/cmd/internal/agentworkspace/up.go @@ -787,7 +787,7 @@ func prepareImage(workspaceDir, image string) error { func installDocker(ctx context.Context) (dockerPath string, err error) { writer := log.Writer(log.LevelInfo) defer func() { _ = writer.Close() }() - log.Debug("installing Docker") + log.Debug("installing docker") return dockerinstall.Install(ctx, writer, writer) } diff --git a/hack/automations/main_test.go b/hack/automations/main_test.go index 1237b3a14..3e9c266e1 100644 --- a/hack/automations/main_test.go +++ b/hack/automations/main_test.go @@ -9,12 +9,6 @@ import ( "testing" ) -// newSandbox copies the generator's real inputs (agents.yaml and the existing -// .agents/agents tree) into an isolated temp directory and returns its path. -// The generator writes agent.md files relative to its working directory, and -// goreleaser's build hooks run this test suite once per target arch in -// parallel; without isolation, concurrent runs race on the same real -// .agents/agents/*/agent.md files in the checkout. func newSandbox(t *testing.T) string { t.Helper() root := repoRoot(t) diff --git a/pkg/dockerinstall/executor.go b/pkg/dockerinstall/executor.go index 4d982f674..13ba07f51 100644 --- a/pkg/dockerinstall/executor.go +++ b/pkg/dockerinstall/executor.go @@ -79,8 +79,13 @@ func (e *Executor) RunWithRetry( return false, nil }, ) - if pollErr != nil && lastErr != nil { - return fmt.Errorf("timeout waiting for dpkg lock after %v: %w", timeout, lastErr) + if pollErr != nil { + if err := ctx.Err(); err != nil { + return err + } + if lastErr != nil { + return fmt.Errorf("timeout waiting for dpkg lock after %v: %w", timeout, lastErr) + } } return pollErr } diff --git a/pkg/dockerinstall/executor_test.go b/pkg/dockerinstall/executor_test.go new file mode 100644 index 000000000..4fab206f1 --- /dev/null +++ b/pkg/dockerinstall/executor_test.go @@ -0,0 +1,63 @@ +package dockerinstall + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + "time" +) + +func newTestExecutor() *Executor { + return NewExecutor(&InstallOptions{stdout: &bytes.Buffer{}, stderr: &bytes.Buffer{}}) +} + +const dpkgLockCmd = `echo "E: Could not get lock /var/lib/dpkg/lock" >&2; exit 1` + +func TestRunWithRetry_ContextCancellationTakesPrecedence(t *testing.T) { + e := newTestExecutor() + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(50 * time.Millisecond) + cancel() + }() + + err := e.RunWithRetry(ctx, "sh", dpkgLockCmd, 5*time.Second) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} + +func TestRunWithRetry_DpkgLockTimeoutWrapsLastError(t *testing.T) { + e := newTestExecutor() + + err := e.RunWithRetry(context.Background(), "sh", dpkgLockCmd, 200*time.Millisecond) + if err == nil || !strings.Contains(err.Error(), "timeout waiting for dpkg lock") { + t.Fatalf("expected dpkg lock timeout error, got %v", err) + } + if !strings.Contains(err.Error(), "exit status") { + t.Fatalf("expected wrapped command error in message, got %v", err) + } +} + +func TestRunWithRetry_NonLockErrorReturnsImmediately(t *testing.T) { + e := newTestExecutor() + + start := time.Now() + err := e.RunWithRetry(context.Background(), "sh", "exit 1", 5*time.Second) + if err == nil { + t.Fatal("expected error") + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("expected immediate return without retrying, took %v", elapsed) + } +} + +func TestRunWithRetry_SucceedsOnSuccess(t *testing.T) { + e := newTestExecutor() + + if err := e.RunWithRetry(context.Background(), "sh", "exit 0", 5*time.Second); err != nil { + t.Fatalf("expected success, got %v", err) + } +} From 24dbd2f0e071b351309d38046df3c667d6562345 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Tue, 18 Aug 2026 04:51:43 +0000 Subject: [PATCH 12/12] ci: install podman directly from mgoltzsche/podman-static, not Homebrew Homebrew's podman 6.1.0 bottle had two runtime-default mismatches that Homebrew's own build/test process doesn't catch: - Rootless podman tried to use the rootful graphroot (/var/lib/containers/storage) as a non-root user and got denied. - Rootful podman's bundled netavark couldn't validate an iptables firewall backend. Switch to mgoltzsche/podman-static (like the existing Windows step already downloads podman's official MSI directly rather than via a package manager): a static tarball bundling podman + conmon + crun + runc + netavark + aardvark-dns + pasta + fuse-overlayfs + correct per-UID storage config, built and tested specifically for this standalone-host use case. Installs to /usr/local/{bin,lib}, which is already on both Ubuntu's default sudo secure_path and systemd's default service PATH, so the Homebrew-specific secure_path sudoers drop-in is no longer needed. Ships proper system podman.socket/podman.service units, so the rootful step goes back to 'systemctl enable --now podman.socket' (matching the pattern used before any of this session's apt/brew changes) instead of a hand-backgrounded 'podman system service'. The --remote --url readiness probe from the prior review round is kept, since it's still correct for whichever backend runs the socket. Downloaded from GitHub Releases (like the Windows MSI), sidestepping the Ubuntu apt-mirror IPv6/DNS hang entirely rather than working around it. SHA256-pinned like the Windows MSI step. Signed-off-by: Samuel K --- .github/workflows/pr-ci.yml | 44 ++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index ddfce224d..b69972a12 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -714,28 +714,35 @@ jobs: image: kindest/node:v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a skipClusterLogsExport: true - - name: cache Homebrew downloads (podman) + - name: cache podman installer (Linux) if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux' + id: podman-cache-linux uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - path: ${{ runner.temp }}/homebrew-cache - key: brew-podman-${{ runner.os }}-${{ runner.arch }}-v1 + path: ${{ runner.temp }}/podman-linux-amd64.tar.gz + key: ${{ runner.os }}-${{ runner.arch }}-podman-static-v5.8.2 - - name: install podman (Homebrew) + - name: install podman (Linux) if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux' - timeout-minutes: 10 - env: - HOMEBREW_CACHE: ${{ runner.temp }}/homebrew-cache - HOMEBREW_NO_AUTO_UPDATE: "1" - HOMEBREW_NO_INSTALL_CLEANUP: "1" + timeout-minutes: 5 run: | - eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" - echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH" - brew install podman + archive="${{ runner.temp }}/podman-linux-amd64.tar.gz" + if [ "${{ steps.podman-cache-linux.outputs.cache-hit }}" != "true" ]; then + curl -fsSL -o "$archive" \ + https://github.com/mgoltzsche/podman-static/releases/download/v5.8.2/podman-linux-amd64.tar.gz + fi + + expectedHash="228b9adf1ba3585d1d72f0a4bb9a669de5ea806d13884b2c43b5e65601a1a580" + actualHash=$(sha256sum "$archive" | cut -d' ' -f1) + if [ "$actualHash" != "$expectedHash" ]; then + echo "::error::SHA256 mismatch for podman-static archive! Expected: $expectedHash, Got: $actualHash" + exit 1 + fi + + tar -xzf "$archive" -C "${{ runner.temp }}" + sudo cp -r "${{ runner.temp }}/podman-linux-amd64/usr" "${{ runner.temp }}/podman-linux-amd64/etc" / - echo "Defaults secure_path=\"$HOMEBREW_PREFIX/bin:$HOMEBREW_PREFIX/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"" \ - | sudo tee /etc/sudoers.d/homebrew-path >/dev/null - sudo visudo -cf /etc/sudoers.d/homebrew-path + command -v newuidmap >/dev/null || sudo apt-get install -y uidmap - name: Install Podman (Linux rootless) if: matrix.install-podman == 'rootless' && runner.os == 'Linux' @@ -746,11 +753,12 @@ jobs: - name: Install Podman (Linux rootful) if: matrix.install-podman == 'rootful' && runner.os == 'Linux' run: | - sudo mkdir -p /run/podman - sudo -b sh -c 'podman system service --time=0 unix:///run/podman/podman.sock > /tmp/podman-service.log 2>&1' + sudo systemctl daemon-reload + sudo systemctl enable --now podman.socket if ! timeout 30 bash -c 'until sudo podman --remote --url unix:///run/podman/podman.sock info >/dev/null 2>&1; do sleep 1; done'; then echo "::error::podman service did not become ready within 30s" - cat /tmp/podman-service.log || true + sudo systemctl status podman.socket --no-pager || true + sudo journalctl -u podman.socket --no-pager -n 100 || true exit 1 fi echo "DOCKER_HOST=unix:///run/podman/podman.sock" >> "$GITHUB_ENV"