ci: add apt timeout

.github/workflows/pr-ci.yml

@@ -714,56 +714,55 @@ jobs:
           image: kindest/node:v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a
           skipClusterLogsExport: true
 
-      - name: cache apt packages (podman/runc)
+      - name: cache podman installer (Linux)
         if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux'
+        id: podman-cache-linux
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
-          path: ${{ runner.temp }}/apt-archives
-          key: apt-podman-runc-${{ runner.os }}-v1
+          path: ${{ runner.temp }}/podman-linux-amd64.tar.gz
+          key: ${{ runner.os }}-${{ runner.arch }}-podman-static-v5.8.2
 
-      - name: cache apt packages (podman/runc)
+      - name: install podman (Linux)
         if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux'
+        timeout-minutes: 5
         run: |
-          for i in $(seq 1 60); do
-            if sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; then
-              echo "waiting for dpkg lock to be released ($i/60)"
-              sleep 2
-            else
-              break
-            fi
-          done
-          mkdir -p "${{ runner.temp }}/apt-archives"
-          sudo apt-get -o Dir::Cache::Archives="${{ runner.temp }}/apt-archives" update
-          sudo apt-get -o Dir::Cache::Archives="${{ runner.temp }}/apt-archives" install -y podman runc
+          archive="${{ runner.temp }}/podman-linux-amd64.tar.gz"
+          if [ "${{ steps.podman-cache-linux.outputs.cache-hit }}" != "true" ]; then
+            curl -fsSL -o "$archive" \
+              https://github.com/mgoltzsche/podman-static/releases/download/v5.8.2/podman-linux-amd64.tar.gz
+          fi
 
-          sudo rm -f "${{ runner.temp }}/apt-archives/lock"
-          sudo rm -rf "${{ runner.temp }}/apt-archives/partial"
+          expectedHash="228b9adf1ba3585d1d72f0a4bb9a669de5ea806d13884b2c43b5e65601a1a580"
+          actualHash=$(sha256sum "$archive" | cut -d' ' -f1)
+          if [ "$actualHash" != "$expectedHash" ]; then
+            echo "::error::SHA256 mismatch for podman-static archive! Expected: $expectedHash, Got: $actualHash"
+            exit 1
+          fi
 
-          sudo mkdir -p /etc/containers
-          printf '[engine]\nruntime = "runc"\n' | sudo tee /etc/containers/containers.conf
+          tar -xzf "$archive" -C "${{ runner.temp }}"
+          sudo cp -r "${{ runner.temp }}/podman-linux-amd64/usr" "${{ runner.temp }}/podman-linux-amd64/etc" /
+
+          command -v newuidmap >/dev/null || sudo apt-get install -y uidmap
 
       - name: Install Podman (Linux rootless)
         if: matrix.install-podman == 'rootless' && runner.os == 'Linux'
-        env:
-          APT_CACHE_DIR: ${{ runner.temp }}/apt-archives
         run: |
           podman info
           podman run --rm busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d echo "podman runtime preflight OK"
 
       - name: Install Podman (Linux rootful)
         if: matrix.install-podman == 'rootful' && runner.os == 'Linux'
-        env:
-          APT_CACHE_DIR: ${{ runner.temp }}/apt-archives
         run: |
+          sudo systemctl daemon-reload
           sudo systemctl enable --now podman.socket
-          if ! timeout 30 bash -c 'until sudo podman info >/dev/null 2>&1; do sleep 1; done'; then
-            echo "::error::podman.socket did not become ready within 30s"
+          if ! timeout 30 bash -c 'until sudo podman --remote --url unix:///run/podman/podman.sock info >/dev/null 2>&1; do sleep 1; done'; then
+            echo "::error::podman service did not become ready within 30s"
             sudo systemctl status podman.socket --no-pager || true
             sudo journalctl -u podman.socket --no-pager -n 100 || true
             exit 1
           fi
           echo "DOCKER_HOST=unix:///run/podman/podman.sock" >> "$GITHUB_ENV"
-          sudo podman info
+          sudo podman --remote --url unix:///run/podman/podman.sock info
           sudo podman run --rm busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d echo "podman runtime preflight OK"
 
       - name: get microsandbox latest version

cmd/internal/agentworkspace/up.go

@@ -331,7 +331,7 @@ func (w *workspaceInitializer) initialize(ctx context.Context) error {
 		log.Warnf("failed to set up docker/git credentials (continuing without them): %v", err)
 	}
 
-	dockerErrChan := w.installDockerAsync()
+	dockerErrChan := w.installDockerAsync(ctx)
 
 	if err := w.prepareWorkspaceContent(ctx); err != nil {
 		return err
@@ -400,7 +400,7 @@ type dockerInstallResult struct {
 	err  error
 }
 
-func (w *workspaceInitializer) installDockerAsync() <-chan dockerInstallResult {
+func (w *workspaceInitializer) installDockerAsync(ctx context.Context) <-chan dockerInstallResult {
 	resultChan := make(chan dockerInstallResult, 1)
 
 	go func() {
@@ -410,14 +410,14 @@ func (w *workspaceInitializer) installDockerAsync() <-chan dockerInstallResult {
 			return
 		}
 
-		dockerPath, err := w.ensureDockerInstalled()
+		dockerPath, err := w.ensureDockerInstalled(ctx)
 		resultChan <- dockerInstallResult{path: dockerPath, err: err}
 	}()
 
 	return resultChan
 }
 
-func (w *workspaceInitializer) ensureDockerInstalled() (string, error) {
+func (w *workspaceInitializer) ensureDockerInstalled(ctx context.Context) (string, error) {
 	dockerCmd := w.getDockerCommand()
 
 	if command.Exists(dockerCmd) {
@@ -444,7 +444,7 @@ func (w *workspaceInitializer) ensureDockerInstalled() (string, error) {
 	}
 
 	log.Debug("attempting to install docker")
-	dockerPath, err := installDocker()
+	dockerPath, err := installDocker(ctx)
 	log.Debugf("docker installation path=%q, err=%v", dockerPath, err)
 	return dockerPath, err
 }
@@ -784,11 +784,11 @@ func prepareImage(workspaceDir, image string) error {
 
 // installDocker installs Docker and returns the path to the docker binary.
 // This function assumes docker does not already exist - the caller should check first.
-func installDocker() (dockerPath string, err error) {
+func installDocker(ctx context.Context) (dockerPath string, err error) {
 	writer := log.Writer(log.LevelInfo)
 	defer func() { _ = writer.Close() }()
-	log.Debug("installing Docker")
-	return dockerinstall.Install(writer, writer)
+	log.Debug("installing docker")
+	return dockerinstall.Install(ctx, writer, writer)
 }
 
 func configureDockerDaemon(ctx context.Context) error {

hack/automations/main_test.go

@@ -1,21 +1,76 @@
 package main
 
 import (
+	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
 )
 
-func runGenerator(t *testing.T, args ...string) string {
+func newSandbox(t *testing.T) string {
+	t.Helper()
+	root := repoRoot(t)
+	dir := t.TempDir()
+
+	copyFile(t,
+		filepath.Join(root, "hack", "automations", "agents.yaml"),
+		filepath.Join(dir, "hack", "automations", "agents.yaml"))
+	copyTree(t,
+		filepath.Join(root, ".agents", "agents"),
+		filepath.Join(dir, ".agents", "agents"))
+
+	return dir
+}
+
+func copyFile(t *testing.T, src, dst string) {
+	t.Helper()
+	b, err := os.ReadFile(src)
+	if err != nil {
+		t.Fatalf("read %s: %v", src, err)
+	}
+	if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
+		t.Fatalf("mkdir %s: %v", filepath.Dir(dst), err)
+	}
+	if err := os.WriteFile(dst, b, 0o644); err != nil {
+		t.Fatalf("write %s: %v", dst, err)
+	}
+}
+
+func copyTree(t *testing.T, src, dst string) {
+	t.Helper()
+	err := filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		rel, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+		target := filepath.Join(dst, rel)
+		if d.IsDir() {
+			return os.MkdirAll(target, 0o755)
+		}
+		b, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		return os.WriteFile(target, b, 0o644)
+	})
+	if err != nil {
+		t.Fatalf("copy tree %s: %v", src, err)
+	}
+}
+
+func runGenerator(t *testing.T, dir string, args ...string) string {
 	t.Helper()
 	bin := filepath.Join(t.TempDir(), "automations")
 	if out, err := exec.Command("go", "build", "-o", bin, ".").CombinedOutput(); err != nil {
 		t.Fatalf("go build: %v\n%s", err, out)
 	}
 	cmd := exec.Command(bin, args...)
-	cmd.Dir = repoRoot(t)
+	cmd.Dir = dir
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		t.Logf("generator output:\n%s", out)
@@ -32,13 +87,13 @@ func repoRoot(t *testing.T) string {
 	return filepath.Dir(filepath.Dir(wd))
 }
 
-func agentFilePath(t *testing.T, id string) string {
-	t.Helper()
-	return filepath.Join(repoRoot(t), ".agents", "agents", id, "agent.md")
+func agentFilePath(dir, id string) string {
+	return filepath.Join(dir, ".agents", "agents", id, "agent.md")
 }
 
 func TestGenerateWritesAllAgents(t *testing.T) {
-	out := runGenerator(t)
+	dir := newSandbox(t)
+	out := runGenerator(t, dir)
 	if !strings.Contains(out, "wrote") {
 		t.Fatalf("expected write output, got:\n%s", out)
 	}
@@ -51,7 +106,7 @@ func TestGenerateWritesAllAgents(t *testing.T) {
 
 	pythonAgents := map[string]bool{"agent-analytics": true}
 	for _, id := range ids {
-		p := agentFilePath(t, id)
+		p := agentFilePath(dir, id)
 		b, err := os.ReadFile(p)
 		if err != nil {
 			t.Errorf("read %s: %v", p, err)
@@ -87,16 +142,18 @@ func TestGenerateWritesAllAgents(t *testing.T) {
 }
 
 func TestCheckIsIdempotent(t *testing.T) {
-	runGenerator(t)
-	out := runGenerator(t, "-check")
+	dir := newSandbox(t)
+	runGenerator(t, dir)
+	out := runGenerator(t, dir, "-check")
 	if strings.Contains(out, "DRIFT") {
 		t.Fatalf("freshly generated files report drift:\n%s", out)
 	}
 }
 
 func TestToolchainPins(t *testing.T) {
-	runGenerator(t)
-	goAgent := agentFilePath(t, "pkg-container")
+	dir := newSandbox(t)
+	runGenerator(t, dir)
+	goAgent := agentFilePath(dir, "pkg-container")
 	b, err := os.ReadFile(goAgent)
 	if err != nil {
 		t.Fatal(err)
@@ -107,7 +164,7 @@ func TestToolchainPins(t *testing.T) {
 			t.Errorf("pkg-container: missing pinned %q", want)
 		}
 	}
-	ciAgent := agentFilePath(t, "ci-optimizer")
+	ciAgent := agentFilePath(dir, "ci-optimizer")
 	cb, err := os.ReadFile(ciAgent)
 	if err != nil {
 		t.Fatal(err)
@@ -118,9 +175,10 @@ func TestToolchainPins(t *testing.T) {
 }
 
 func TestToolchainDownloadsAreBounded(t *testing.T) {
-	runGenerator(t)
+	dir := newSandbox(t)
+	runGenerator(t, dir)
 	for _, id := range []string{"ui-polish", "pkg-container", "agent-analytics"} {
-		p := agentFilePath(t, id)
+		p := agentFilePath(dir, id)
 		b, err := os.ReadFile(p)
 		if err != nil {
 			continue

pkg/dockerinstall/constants.go

@@ -13,6 +13,10 @@ const (
 	DeprecationDelay    = 10 * time.Second
 	RetryDelay          = 10 * time.Second
 
+	// Apt options.
+	AptTimeoutSeconds = 30
+	AptRetries        = 3
+
 	// Paths.
 	DefaultDownloadURL = "https://download.docker.com"
 	DefaultRepoFile    = "docker-ce.repo"

pkg/dockerinstall/debian.go

@@ -1,11 +1,19 @@
 package dockerinstall
 
 import (
+	"context"
 	"fmt"
 	"os/exec"
 	"strings"
 )
 
+// aptFlags configures apt-get to fail fast on transient network issues
+// instead of hanging indefinitely, retrying a bounded number of times first.
+var aptFlags = fmt.Sprintf(
+	`-o Acquire::http::Timeout="%d" -o Acquire::https::Timeout="%d" -o Acquire::Retries="%d"`,
+	AptTimeoutSeconds, AptTimeoutSeconds, AptRetries,
+)
+
 type DebianInstaller struct {
 	distro   *Distro
 	opts     *InstallOptions
@@ -20,8 +28,8 @@ func NewDebianInstaller(distro *Distro, opts *InstallOptions) *DebianInstaller {
 	}
 }
 
-func (i *DebianInstaller) Install(shC string) error {
-	if err := i.setupRepo(shC); err != nil {
+func (i *DebianInstaller) Install(ctx context.Context, shC string) error {
+	if err := i.setupRepo(ctx, shC); err != nil {
 		return err
 	}
 
@@ -32,17 +40,18 @@ func (i *DebianInstaller) Install(shC string) error {
 	pkgs := BuildPackageList(i.opts.version, pkgVersion, cliPkgVersion)
 
 	installCmd := fmt.Sprintf(
-		"DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends %s >/dev/null",
+		"DEBIAN_FRONTEND=noninteractive apt-get %s install -y -qq --no-install-recommends %s >/dev/null",
+		aptFlags,
 		pkgs,
 	)
-	if err := i.executor.RunWithRetry(shC, installCmd, DefaultTimeout); err != nil {
+	if err := i.executor.RunWithRetry(ctx, shC, installCmd, DefaultTimeout); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (i *DebianInstaller) setupRepo(shC string) error {
+func (i *DebianInstaller) setupRepo(ctx context.Context, shC string) error {
 	preReqs := "apt-transport-https ca-certificates curl"
 	if !commandExists("gpg") {
 		preReqs += " gnupg"
@@ -64,20 +73,31 @@ func (i *DebianInstaller) setupRepo(shC string) error {
 		i.opts.channel,
 	)
 
+	cmdAptUpdate := fmt.Sprintf("apt-get %s update -qq >/dev/null", aptFlags)
+
+	cmdAptInstall := fmt.Sprintf(
+		"DEBIAN_FRONTEND=noninteractive apt-get %s install -y -qq %s >/dev/null",
+		aptFlags, preReqs,
+	)
+
+	cmdKeyringDir := "mkdir -p /etc/apt/keyrings && chmod -R 0755 /etc/apt/keyrings"
+
+	cmdDockerGPG := fmt.Sprintf(
+		"curl -fsSL \"%s/linux/%s/gpg\" | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg",
+		i.opts.downloadURL, i.distro.ID,
+	)
+
 	cmds := []string{
-		"apt-get update -qq >/dev/null",
-		fmt.Sprintf("DEBIAN_FRONTEND=noninteractive apt-get install -y -qq %s >/dev/null", preReqs),
-		"mkdir -p /etc/apt/keyrings && chmod -R 0755 /etc/apt/keyrings",
-		fmt.Sprintf(
-			"curl -fsSL \"%s/linux/%s/gpg\" | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg",
-			i.opts.downloadURL, i.distro.ID,
-		),
+		cmdAptUpdate,
+		cmdAptInstall,
+		cmdKeyringDir,
+		cmdDockerGPG,
 		"chmod a+r /etc/apt/keyrings/docker.gpg",
 		fmt.Sprintf("echo %q > /etc/apt/sources.list.d/docker.list", aptRepo),
-		"apt-get update -qq >/dev/null",
+		cmdAptUpdate,
 	}
 
-	return i.executor.RunCommandsWithRetry(shC, cmds, DefaultTimeout)
+	return i.executor.RunCommandsWithRetry(ctx, shC, cmds, DefaultTimeout)
 }
 
 func (i *DebianInstaller) findVersions() (string, string, error) {