diff --git a/.github/actions/setup-bink/action.yml b/.github/actions/setup-bink/action.yml index ccfcc70..0049f9b 100644 --- a/.github/actions/setup-bink/action.yml +++ b/.github/actions/setup-bink/action.yml @@ -21,7 +21,10 @@ runs: - name: Configure kernel for nested containers shell: bash run: | + sudo systemctl disable --now apparmor 2>/dev/null || true sudo aa-teardown 2>/dev/null || true + echo "AppArmor status after teardown:" + sudo aa-status 2>/dev/null || echo "(aa-status failed)" sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 @@ -49,6 +52,8 @@ runs: libbtrfs-dev \ libdevmapper-dev \ pkg-config + # passt's postinst reloads its AppArmor profile in enforce mode; remove it + sudo aa-teardown 2>/dev/null || true - name: Set up KVM shell: bash @@ -60,24 +65,32 @@ runs: shell: bash run: | podman --version - sudo mkdir -p /etc/containers - echo '{"defaultAction":"SCMP_ACT_ALLOW"}' | sudo tee /etc/containers/seccomp.json - printf '[containers]\napparmor_profile = "unconfined"\nseccomp_profile = "/etc/containers/seccomp.json"\n' | sudo tee /etc/containers/containers.conf - grep -q '^root:' /etc/subuid || echo 'root:100000:65536' | sudo tee -a /etc/subuid - grep -q '^root:' /etc/subgid || echo 'root:100000:65536' | sudo tee -a /etc/subgid - printf 'unqualified-search-registries = ["docker.io"]\n' | sudo tee /etc/containers/registries.conf - sudo systemctl start podman.socket - sudo podman info --format '{{.Store.GraphRoot}}' + mkdir -p ~/.config/containers + echo '{"defaultAction":"SCMP_ACT_ALLOW"}' > ~/.config/containers/seccomp.json + printf '[containers]\napparmor_profile = "unconfined"\nseccomp_profile = "%s/.config/containers/seccomp.json"\n' "$HOME" > ~/.config/containers/containers.conf + USER_NAME=$(id -un) + grep -q "^${USER_NAME}:" /etc/subuid || echo "${USER_NAME}:100000:65536" | sudo tee -a /etc/subuid + grep -q "^${USER_NAME}:" /etc/subgid || echo "${USER_NAME}:100000:65536" | sudo tee -a /etc/subgid + printf 'unqualified-search-registries = ["docker.io"]\n' > ~/.config/containers/registries.conf + systemctl --user enable --now podman.socket + echo "CONTAINER_HOST=unix:///run/user/$(id -u)/podman/podman.sock" >> "$GITHUB_ENV" + podman info --format '{{.Store.GraphRoot}}' - name: Build bink binary shell: bash - run: sudo make build-bink + run: make build-bink + + - name: Build cluster and DNS images + shell: bash + run: | + make build-cluster-image + make build-dns-image - name: Verify prerequisites shell: bash run: | test -f ./bink - sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" + podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" df -h / free -h @@ -105,9 +118,9 @@ runs: shell: bash run: | for f in /tmp/podman-image-cache/*.tar; do - sudo podman load -i "$f" + podman load -i "$f" done - sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" + podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" - name: Pre-pull container images if: steps.image-cache.outputs.cache-hit != 'true' @@ -115,11 +128,10 @@ runs: run: | mkdir -p /tmp/podman-image-cache for img in $BINK_IMAGES $EXTERNAL_IMAGES; do - sudo podman pull "$img" + podman pull "$img" name=$(echo "$img" | sed 's|[/:]|_|g') - sudo podman save -o "/tmp/podman-image-cache/${name}.tar" "$img" + podman save -o "/tmp/podman-image-cache/${name}.tar" "$img" done - sudo chown -R $(id -u):$(id -g) /tmp/podman-image-cache - name: Save image cache if: steps.image-cache.outputs.cache-hit != 'true' diff --git a/.github/collect-logs.sh b/.github/collect-logs.sh index 4e9231d..30ef76f 100755 --- a/.github/collect-logs.sh +++ b/.github/collect-logs.sh @@ -2,6 +2,8 @@ # Collect logs from bink test containers for CI debugging # Writes per-container log files to $LOG_DIR (default: /tmp/bink-logs) +set -x + LOG_DIR="${LOG_DIR:-/tmp/bink-logs}" mkdir -p "$LOG_DIR" @@ -9,17 +11,17 @@ SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o Connect ssh_vm() { local ctr="$1" cmd="$2" - sudo podman exec "$ctr" ssh $SSH_OPTS core@localhost "$cmd" 2>&1 + podman exec "$ctr" ssh $SSH_OPTS core@localhost "$cmd" 2>&1 } -sudo podman ps -a 2>/dev/null > "$LOG_DIR/podman-ps.txt" || true +podman ps -a 2>/dev/null > "$LOG_DIR/podman-ps.txt" || true -for ctr in $(sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' 2>/dev/null); do +for ctr in $(podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' 2>/dev/null); do echo "Collecting logs for $ctr" dir="$LOG_DIR/$ctr" mkdir -p "$dir" - sudo podman logs "$ctr" > "$dir/container.log" 2>&1 || true + podman logs "$ctr" > "$dir/container.log" 2>&1 || true ssh_vm "$ctr" "sudo journalctl -n 200 --no-pager" > "$dir/journal.log" || echo "(VM not reachable)" > "$dir/journal.log" ssh_vm "$ctr" "sudo journalctl -u kubelet -n 100 --no-pager" > "$dir/kubelet.log" || true ssh_vm "$ctr" "sudo journalctl -u crio -n 100 --no-pager" > "$dir/crio.log" || true @@ -27,9 +29,11 @@ for ctr in $(sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names} ssh_vm "$ctr" "sudo dmesg" > "$dir/dmesg.log" || true done -df -h | sudo tee "$LOG_DIR/disk.txt" > /dev/null || true -free -h | sudo tee "$LOG_DIR/memory.txt" > /dev/null || true -sudo dmesg | tail -100 | sudo tee "$LOG_DIR/host-dmesg.txt" > /dev/null || true +df -h > "$LOG_DIR/disk.txt" 2>&1 || true +free -h > "$LOG_DIR/memory.txt" 2>&1 || true +sudo dmesg > "$LOG_DIR/host-dmesg.txt" 2>&1 || true +sudo journalctl --no-pager > "$LOG_DIR/host-journal.log" 2>&1 || true +sudo cp /var/log/audit/audit.log "$LOG_DIR/audit.log" 2>/dev/null || true for bcvk_dir in /tmp/bcvk-logs*; do [ -d "$bcvk_dir" ] || continue diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 7ee03d0..2c9e11e 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -175,7 +175,7 @@ jobs: - name: Load node image if: needs.build-node-images.result == 'success' - run: sudo podman load -i node-image.tar + run: podman load -i node-image.tar - name: Setup bink uses: ./.github/actions/setup-bink @@ -184,10 +184,8 @@ jobs: node-image: ${{ needs.build-node-images.result == 'skipped' && format('ghcr.io/bootc-dev/bink/node:v{0}-fedora-{1}-disk', needs.supported-versions.outputs.default-version, needs.supported-versions.outputs.fedora-version) || '' }} - name: Run integration tests - run: sudo make test-integration TEST_PROCS=2 + run: make test-integration TEST_PROCS=2 timeout-minutes: 90 - env: - CONTAINER_HOST: unix:///run/podman/podman.sock - name: Collect logs if: failure() @@ -203,9 +201,9 @@ jobs: - name: Cleanup test clusters if: always() run: | - sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ - xargs -r sudo podman rm -f 2>/dev/null || true - sudo podman volume prune -f 2>/dev/null || true + podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ + xargs -r podman rm -f 2>/dev/null || true + podman volume prune -f 2>/dev/null || true integration-tests-composefs: needs: [changes, supported-versions, build-node-images] @@ -226,7 +224,7 @@ jobs: - name: Load composefs node image if: needs.build-node-images.result == 'success' - run: sudo podman load -i node-image-composefs.tar + run: podman load -i node-image-composefs.tar - name: Setup bink uses: ./.github/actions/setup-bink @@ -235,10 +233,9 @@ jobs: node-image: ${{ needs.build-node-images.result == 'skipped' && format('ghcr.io/bootc-dev/bink/node:v{0}-fedora-{1}-disk-composefs', needs.supported-versions.outputs.default-version, needs.supported-versions.outputs.fedora-version) || '' }} - name: Run composefs integration tests - run: sudo make test-integration-composefs + run: make test-integration-composefs timeout-minutes: 90 env: - CONTAINER_HOST: unix:///run/podman/podman.sock BINK_NODE_IMAGE: ghcr.io/bootc-dev/bink/node:v${{ needs.supported-versions.outputs.default-version }}-fedora-${{ needs.supported-versions.outputs.fedora-version }}-disk-composefs - name: Collect logs @@ -255,9 +252,9 @@ jobs: - name: Cleanup test clusters if: always() run: | - sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ - xargs -r sudo podman rm -f 2>/dev/null || true - sudo podman volume prune -f 2>/dev/null || true + podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ + xargs -r podman rm -f 2>/dev/null || true + podman volume prune -f 2>/dev/null || true integration-tests-k8s-versions: needs: [changes, supported-versions, build-node-images] @@ -285,7 +282,7 @@ jobs: - name: Load node image if: needs.build-node-images.result == 'success' - run: sudo podman load -i node-image.tar + run: podman load -i node-image.tar - name: Setup bink uses: ./.github/actions/setup-bink @@ -294,10 +291,9 @@ jobs: node-image: ${{ needs.build-node-images.result == 'skipped' && format('ghcr.io/bootc-dev/bink/node:v{0}-fedora-{1}-disk', matrix.kube-minor, needs.supported-versions.outputs.fedora-version) || '' }} - name: Run integration tests (K8s ${{ matrix.kube-minor }}) - run: sudo make test-integration GINKGO_FOCUS="should create and initialize a complete Kubernetes cluster" + run: make test-integration GINKGO_FOCUS="should create and initialize a complete Kubernetes cluster" timeout-minutes: 90 env: - CONTAINER_HOST: unix:///run/podman/podman.sock BINK_NODE_IMAGE: ${{ env.BINK_NODE_IMAGE }} - name: Collect logs @@ -314,9 +310,9 @@ jobs: - name: Cleanup test clusters if: always() run: | - sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ - xargs -r sudo podman rm -f 2>/dev/null || true - sudo podman volume prune -f 2>/dev/null || true + podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ + xargs -r podman rm -f 2>/dev/null || true + podman volume prune -f 2>/dev/null || true push-node-images: needs: [supported-versions, build-node-images, integration-tests, integration-tests-composefs, integration-tests-k8s-versions] diff --git a/containerfiles/cluster-image/qemu.conf b/containerfiles/cluster-image/qemu.conf index 3c832f5..f36387d 100644 --- a/containerfiles/cluster-image/qemu.conf +++ b/containerfiles/cluster-image/qemu.conf @@ -8,6 +8,7 @@ dynamic_ownership = 1 remember_owner = 0 namespaces = [ ] cgroup_controllers = [ ] +security_driver = "none" # virtiofsd configuration - use wrapper for container compatibility virtiofsd_path = "/usr/local/bin/virtiofsd-wrapper" diff --git a/test/integration/images_test.go b/test/integration/images_test.go index d55b669..14df188 100644 --- a/test/integration/images_test.go +++ b/test/integration/images_test.go @@ -12,6 +12,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + specs "github.com/opencontainers/runtime-spec/specs-go" "go.podman.io/podman/v6/pkg/specgen" "github.com/bootc-dev/bink/internal/cluster" @@ -125,6 +126,9 @@ var _ = Describe("Cluster Images Volume", Serial, func() { Name: volumeName, Dest: "/var/lib/containers/storage", }}, + CapAdd: []string{"SYS_ADMIN"}, + Devices: []specs.LinuxDevice{{Path: "/dev/fuse"}}, + SelinuxOpts: []string{"disable"}, }) Expect(err).ToNot(HaveOccurred())