From 903aaf661b7457f7538915dcb39f193179674a2e Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Mon, 17 Aug 2026 06:52:43 +0000 Subject: [PATCH 1/4] gha: run integration tests rootless Use the user Podman socket instead of the system socket. Write container config to ~/.config/containers/ and set CONTAINER_HOST via GITHUB_ENV so all integration test steps pick it up without sudo. Assisted-by: AI Signed-off-by: Alice Frosi --- .github/actions/setup-bink/action.yml | 38 +++++++++++++++---------- .github/workflows/integration-tests.yml | 34 ++++++++++------------ 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/.github/actions/setup-bink/action.yml b/.github/actions/setup-bink/action.yml index ccfcc70..fe4c3e0 100644 --- a/.github/actions/setup-bink/action.yml +++ b/.github/actions/setup-bink/action.yml @@ -60,24 +60,33 @@ runs: shell: bash run: | podman --version - sudo mkdir -p /etc/containers - echo '{"defaultAction":"SCMP_ACT_ALLOW"}' | sudo tee /etc/containers/seccomp.json - printf '[containers]\napparmor_profile = "unconfined"\nseccomp_profile = "/etc/containers/seccomp.json"\n' | sudo tee /etc/containers/containers.conf - grep -q '^root:' /etc/subuid || echo 'root:100000:65536' | sudo tee -a /etc/subuid - grep -q '^root:' /etc/subgid || echo 'root:100000:65536' | sudo tee -a /etc/subgid - printf 'unqualified-search-registries = ["docker.io"]\n' | sudo tee /etc/containers/registries.conf - sudo systemctl start podman.socket - sudo podman info --format '{{.Store.GraphRoot}}' + mkdir -p ~/.config/containers + echo '{"defaultAction":"SCMP_ACT_ALLOW"}' > ~/.config/containers/seccomp.json + printf '[containers]\napparmor_profile = "unconfined"\nseccomp_profile = "%s/.config/containers/seccomp.json"\n' "$HOME" > ~/.config/containers/containers.conf + USER_NAME=$(id -un) + grep -q "^${USER_NAME}:" /etc/subuid || echo "${USER_NAME}:100000:65536" | sudo tee -a /etc/subuid + grep -q "^${USER_NAME}:" /etc/subgid || echo "${USER_NAME}:100000:65536" | sudo tee -a /etc/subgid + printf 'unqualified-search-registries = ["docker.io"]\n' > ~/.config/containers/registries.conf + systemctl --user enable --now podman.socket + echo "CONTAINER_HOST=unix:///run/user/$(id -u)/podman/podman.sock" >> "$GITHUB_ENV" + podman info --format '{{.Store.GraphRoot}}' - name: Build bink binary shell: bash - run: sudo make build-bink + run: make build-bink + + - name: Build cluster and DNS images + shell: bash + run: | + make build-cluster-image + make build-dns-image + - name: Verify prerequisites shell: bash run: | test -f ./bink - sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" + podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" df -h / free -h @@ -105,9 +114,9 @@ runs: shell: bash run: | for f in /tmp/podman-image-cache/*.tar; do - sudo podman load -i "$f" + podman load -i "$f" done - sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" + podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" - name: Pre-pull container images if: steps.image-cache.outputs.cache-hit != 'true' @@ -115,11 +124,10 @@ runs: run: | mkdir -p /tmp/podman-image-cache for img in $BINK_IMAGES $EXTERNAL_IMAGES; do - sudo podman pull "$img" + podman pull "$img" name=$(echo "$img" | sed 's|[/:]|_|g') - sudo podman save -o "/tmp/podman-image-cache/${name}.tar" "$img" + podman save -o "/tmp/podman-image-cache/${name}.tar" "$img" done - sudo chown -R $(id -u):$(id -g) /tmp/podman-image-cache - name: Save image cache if: steps.image-cache.outputs.cache-hit != 'true' diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 7ee03d0..2c9e11e 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -175,7 +175,7 @@ jobs: - name: Load node image if: needs.build-node-images.result == 'success' - run: sudo podman load -i node-image.tar + run: podman load -i node-image.tar - name: Setup bink uses: ./.github/actions/setup-bink @@ -184,10 +184,8 @@ jobs: node-image: ${{ needs.build-node-images.result == 'skipped' && format('ghcr.io/bootc-dev/bink/node:v{0}-fedora-{1}-disk', needs.supported-versions.outputs.default-version, needs.supported-versions.outputs.fedora-version) || '' }} - name: Run integration tests - run: sudo make test-integration TEST_PROCS=2 + run: make test-integration TEST_PROCS=2 timeout-minutes: 90 - env: - CONTAINER_HOST: unix:///run/podman/podman.sock - name: Collect logs if: failure() @@ -203,9 +201,9 @@ jobs: - name: Cleanup test clusters if: always() run: | - sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ - xargs -r sudo podman rm -f 2>/dev/null || true - sudo podman volume prune -f 2>/dev/null || true + podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ + xargs -r podman rm -f 2>/dev/null || true + podman volume prune -f 2>/dev/null || true integration-tests-composefs: needs: [changes, supported-versions, build-node-images] @@ -226,7 +224,7 @@ jobs: - name: Load composefs node image if: needs.build-node-images.result == 'success' - run: sudo podman load -i node-image-composefs.tar + run: podman load -i node-image-composefs.tar - name: Setup bink uses: ./.github/actions/setup-bink @@ -235,10 +233,9 @@ jobs: node-image: ${{ needs.build-node-images.result == 'skipped' && format('ghcr.io/bootc-dev/bink/node:v{0}-fedora-{1}-disk-composefs', needs.supported-versions.outputs.default-version, needs.supported-versions.outputs.fedora-version) || '' }} - name: Run composefs integration tests - run: sudo make test-integration-composefs + run: make test-integration-composefs timeout-minutes: 90 env: - CONTAINER_HOST: unix:///run/podman/podman.sock BINK_NODE_IMAGE: ghcr.io/bootc-dev/bink/node:v${{ needs.supported-versions.outputs.default-version }}-fedora-${{ needs.supported-versions.outputs.fedora-version }}-disk-composefs - name: Collect logs @@ -255,9 +252,9 @@ jobs: - name: Cleanup test clusters if: always() run: | - sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ - xargs -r sudo podman rm -f 2>/dev/null || true - sudo podman volume prune -f 2>/dev/null || true + podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ + xargs -r podman rm -f 2>/dev/null || true + podman volume prune -f 2>/dev/null || true integration-tests-k8s-versions: needs: [changes, supported-versions, build-node-images] @@ -285,7 +282,7 @@ jobs: - name: Load node image if: needs.build-node-images.result == 'success' - run: sudo podman load -i node-image.tar + run: podman load -i node-image.tar - name: Setup bink uses: ./.github/actions/setup-bink @@ -294,10 +291,9 @@ jobs: node-image: ${{ needs.build-node-images.result == 'skipped' && format('ghcr.io/bootc-dev/bink/node:v{0}-fedora-{1}-disk', matrix.kube-minor, needs.supported-versions.outputs.fedora-version) || '' }} - name: Run integration tests (K8s ${{ matrix.kube-minor }}) - run: sudo make test-integration GINKGO_FOCUS="should create and initialize a complete Kubernetes cluster" + run: make test-integration GINKGO_FOCUS="should create and initialize a complete Kubernetes cluster" timeout-minutes: 90 env: - CONTAINER_HOST: unix:///run/podman/podman.sock BINK_NODE_IMAGE: ${{ env.BINK_NODE_IMAGE }} - name: Collect logs @@ -314,9 +310,9 @@ jobs: - name: Cleanup test clusters if: always() run: | - sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ - xargs -r sudo podman rm -f 2>/dev/null || true - sudo podman volume prune -f 2>/dev/null || true + podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ + xargs -r podman rm -f 2>/dev/null || true + podman volume prune -f 2>/dev/null || true push-node-images: needs: [supported-versions, build-node-images, integration-tests, integration-tests-composefs, integration-tests-k8s-versions] From d417457aa37e709a35650bf16c7078266f28ebda Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Mon, 17 Aug 2026 11:09:05 +0000 Subject: [PATCH 2/4] test: add capabilities for nested podman in images test The verify container needs SYS_ADMIN, /dev/fuse, and disabled SELinux to run podman inside a rootless container, matching the populator container configuration. Assisted-by: AI Signed-off-by: Alice Frosi --- test/integration/images_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/integration/images_test.go b/test/integration/images_test.go index d55b669..14df188 100644 --- a/test/integration/images_test.go +++ b/test/integration/images_test.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + specs "github.com/opencontainers/runtime-spec/specs-go" "go.podman.io/podman/v6/pkg/specgen" "github.com/bootc-dev/bink/internal/cluster" @@ -125,6 +126,9 @@ var _ = Describe("Cluster Images Volume", Serial, func() { Name: volumeName, Dest: "/var/lib/containers/storage", }}, + CapAdd: []string{"SYS_ADMIN"}, + Devices: []specs.LinuxDevice{{Path: "/dev/fuse"}}, + SelinuxOpts: []string{"disable"}, }) Expect(err).ToNot(HaveOccurred()) From 80e39262ba324b7c8986eb8239aa42c65dd35e89 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Mon, 17 Aug 2026 07:29:28 +0000 Subject: [PATCH 3/4] gha: fix AppArmor interference with VMs in CI Disable the AppArmor service and run aa-teardown before and after apt-get installs, since packages like passt reload their profiles in enforce mode in postinst scripts. Also set security_driver = "none" in qemu.conf so that libvirt's QEMU driver does not load per-VM AppArmor profiles when starting domains. Assisted-by: AI Signed-off-by: Alice Frosi --- .github/actions/setup-bink/action.yml | 6 +++++- containerfiles/cluster-image/qemu.conf | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-bink/action.yml b/.github/actions/setup-bink/action.yml index fe4c3e0..0049f9b 100644 --- a/.github/actions/setup-bink/action.yml +++ b/.github/actions/setup-bink/action.yml @@ -21,7 +21,10 @@ runs: - name: Configure kernel for nested containers shell: bash run: | + sudo systemctl disable --now apparmor 2>/dev/null || true sudo aa-teardown 2>/dev/null || true + echo "AppArmor status after teardown:" + sudo aa-status 2>/dev/null || echo "(aa-status failed)" sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 @@ -49,6 +52,8 @@ runs: libbtrfs-dev \ libdevmapper-dev \ pkg-config + # passt's postinst reloads its AppArmor profile in enforce mode; remove it + sudo aa-teardown 2>/dev/null || true - name: Set up KVM shell: bash @@ -81,7 +86,6 @@ runs: make build-cluster-image make build-dns-image - - name: Verify prerequisites shell: bash run: | diff --git a/containerfiles/cluster-image/qemu.conf b/containerfiles/cluster-image/qemu.conf index 3c832f5..f36387d 100644 --- a/containerfiles/cluster-image/qemu.conf +++ b/containerfiles/cluster-image/qemu.conf @@ -8,6 +8,7 @@ dynamic_ownership = 1 remember_owner = 0 namespaces = [ ] cgroup_controllers = [ ] +security_driver = "none" # virtiofsd configuration - use wrapper for container compatibility virtiofsd_path = "/usr/local/bin/virtiofsd-wrapper" From c8ced1015dacbf7f6a00f130023de516b8e3c6f9 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Mon, 17 Aug 2026 07:31:01 +0000 Subject: [PATCH 4/4] gha: improve CI log collection Collect audit log and full system journal on failure. Fix the script to work with rootless Podman by removing sudo from podman commands. Use direct redirects instead of sudo tee for host logs. Assisted-by: AI Signed-off-by: Alice Frosi --- .github/collect-logs.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/collect-logs.sh b/.github/collect-logs.sh index 4e9231d..30ef76f 100755 --- a/.github/collect-logs.sh +++ b/.github/collect-logs.sh @@ -2,6 +2,8 @@ # Collect logs from bink test containers for CI debugging # Writes per-container log files to $LOG_DIR (default: /tmp/bink-logs) +set -x + LOG_DIR="${LOG_DIR:-/tmp/bink-logs}" mkdir -p "$LOG_DIR" @@ -9,17 +11,17 @@ SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o Connect ssh_vm() { local ctr="$1" cmd="$2" - sudo podman exec "$ctr" ssh $SSH_OPTS core@localhost "$cmd" 2>&1 + podman exec "$ctr" ssh $SSH_OPTS core@localhost "$cmd" 2>&1 } -sudo podman ps -a 2>/dev/null > "$LOG_DIR/podman-ps.txt" || true +podman ps -a 2>/dev/null > "$LOG_DIR/podman-ps.txt" || true -for ctr in $(sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' 2>/dev/null); do +for ctr in $(podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' 2>/dev/null); do echo "Collecting logs for $ctr" dir="$LOG_DIR/$ctr" mkdir -p "$dir" - sudo podman logs "$ctr" > "$dir/container.log" 2>&1 || true + podman logs "$ctr" > "$dir/container.log" 2>&1 || true ssh_vm "$ctr" "sudo journalctl -n 200 --no-pager" > "$dir/journal.log" || echo "(VM not reachable)" > "$dir/journal.log" ssh_vm "$ctr" "sudo journalctl -u kubelet -n 100 --no-pager" > "$dir/kubelet.log" || true ssh_vm "$ctr" "sudo journalctl -u crio -n 100 --no-pager" > "$dir/crio.log" || true @@ -27,9 +29,11 @@ for ctr in $(sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names} ssh_vm "$ctr" "sudo dmesg" > "$dir/dmesg.log" || true done -df -h | sudo tee "$LOG_DIR/disk.txt" > /dev/null || true -free -h | sudo tee "$LOG_DIR/memory.txt" > /dev/null || true -sudo dmesg | tail -100 | sudo tee "$LOG_DIR/host-dmesg.txt" > /dev/null || true +df -h > "$LOG_DIR/disk.txt" 2>&1 || true +free -h > "$LOG_DIR/memory.txt" 2>&1 || true +sudo dmesg > "$LOG_DIR/host-dmesg.txt" 2>&1 || true +sudo journalctl --no-pager > "$LOG_DIR/host-journal.log" 2>&1 || true +sudo cp /var/log/audit/audit.log "$LOG_DIR/audit.log" 2>/dev/null || true for bcvk_dir in /tmp/bcvk-logs*; do [ -d "$bcvk_dir" ] || continue