From 986b1975940c91b431b7b83afeae3926ccb1ac9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Sat, 1 Aug 2026 17:06:55 +0300 Subject: [PATCH 1/3] fix(ami): bump packer build volume from 16GB to 20GB test-ami-nix (15, amd64/arm64) has been intermittently failing with "No space left on device" during StepProvision, reproduced on unrelated PRs (#2329, #2332) while PG17/orioledb-17 build fine on the same 16GB build volume - the PG15 build closure sits close enough to that limit to tip over depending on small variance in build/cache state. --- amazon-amd64-nix.pkr.hcl | 2 +- amazon-arm64-nix.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/amazon-amd64-nix.pkr.hcl b/amazon-amd64-nix.pkr.hcl index 4d91ac23b..74d6fbb57 100644 --- a/amazon-amd64-nix.pkr.hcl +++ b/amazon-amd64-nix.pkr.hcl @@ -144,7 +144,7 @@ source "amazon-ebssurrogate" "source" { launch_block_device_mappings { device_name = "/dev/${var.build-vol}" delete_on_termination = true - volume_size = 16 + volume_size = 20 volume_type = "gp3" omit_from_artifact = true } diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 1f6064ef3..6ea21381e 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -144,7 +144,7 @@ source "amazon-ebssurrogate" "source" { launch_block_device_mappings { device_name = "/dev/${var.build-vol}" delete_on_termination = true - volume_size = 16 + volume_size = 20 volume_type = "gp3" omit_from_artifact = true } From aab2d6d96af0b1c16d94d510acab73199ff90c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Sat, 1 Aug 2026 19:12:42 +0300 Subject: [PATCH 2/3] fix(ami): size the builder instance's own root volume, not just the surrogate/build volumes The prior 16->20GB bump targeted /dev/xvdc (the scratch build-vol) and left the actual constraint untouched: the amazon-ebssurrogate builder instance boots from its own root disk (source Ubuntu AMI's default /dev/sda1), which is where ansible/apt provisioning actually runs before the chroot into the surrogate root at /mnt. Confirmed via CI logs - "No space left on device" was raised inside ansible's own Python logging flush while writing /tmp/ansible.log, unrelated to xvdf/xvdh/xvdc sizing. Adds an explicit launch_block_device_mappings entry for /dev/sda1. --- amazon-amd64-nix.pkr.hcl | 14 ++++++++++++++ amazon-arm64-nix.pkr.hcl | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/amazon-amd64-nix.pkr.hcl b/amazon-amd64-nix.pkr.hcl index 74d6fbb57..a83fe7569 100644 --- a/amazon-amd64-nix.pkr.hcl +++ b/amazon-amd64-nix.pkr.hcl @@ -124,6 +124,20 @@ source "amazon-ebssurrogate" "source" { } ena_support = true + + # Builder instance's own boot root (from the source Ubuntu AMI, device + # /dev/sda1) - separate from the surrogate root (/dev/xvdf) being + # assembled below. Ansible/apt provisioning runs directly on this disk + # before the chroot into /mnt, and the default Ubuntu AMI root size was + # too small, causing "No space left on device" failures during + # provisioning (reproduced independent of the surrogate/build-vol sizes). + launch_block_device_mappings { + device_name = "/dev/sda1" + delete_on_termination = true + volume_size = 16 + volume_type = "gp3" + } + launch_block_device_mappings { device_name = "/dev/xvdf" delete_on_termination = true diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 6ea21381e..450755fde 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -124,6 +124,20 @@ source "amazon-ebssurrogate" "source" { } ena_support = true + + # Builder instance's own boot root (from the source Ubuntu AMI, device + # /dev/sda1) - separate from the surrogate root (/dev/xvdf) being + # assembled below. Ansible/apt provisioning runs directly on this disk + # before the chroot into /mnt, and the default Ubuntu AMI root size was + # too small, causing "No space left on device" failures during + # provisioning (reproduced independent of the surrogate/build-vol sizes). + launch_block_device_mappings { + device_name = "/dev/sda1" + delete_on_termination = true + volume_size = 16 + volume_type = "gp3" + } + launch_block_device_mappings { device_name = "/dev/xvdf" delete_on_termination = true From 0c3657bac95084013f0f3a2fe6bbc556308bd5e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Sun, 2 Aug 2026 02:34:03 +0300 Subject: [PATCH 3/3] revert: drop /dev/sda1 mapping, it broke previously-passing AMI builds On this HVM AMI, /dev/sda1 aliases to the same underlying root device as ami_root_device's /dev/xvda, so sizing it to 16GB actually grew the *output* AMI's root snapshot to 16GB - breaking test-ami-nix for every Postgres version (including 17/orioledb-17, which were passing before) since the test harness launches instances with an 8GB root, which can no longer satisfy that snapshot: "Volume of size 8GB is smaller than snapshot ... expect size >= 16GB". Back to just the xvdc bump (16->20GB) from the prior commit, which is at least harmless even though it didn't fix the original ENOSPC error. Root cause is still open: ansible's own log flush hits ENOSPC on the builder instance's own root during provisioning, and that disk isn't one of the three currently-configured volumes (xvdf/xvdh/xvdc). --- amazon-amd64-nix.pkr.hcl | 14 -------------- amazon-arm64-nix.pkr.hcl | 14 -------------- 2 files changed, 28 deletions(-) diff --git a/amazon-amd64-nix.pkr.hcl b/amazon-amd64-nix.pkr.hcl index a83fe7569..74d6fbb57 100644 --- a/amazon-amd64-nix.pkr.hcl +++ b/amazon-amd64-nix.pkr.hcl @@ -124,20 +124,6 @@ source "amazon-ebssurrogate" "source" { } ena_support = true - - # Builder instance's own boot root (from the source Ubuntu AMI, device - # /dev/sda1) - separate from the surrogate root (/dev/xvdf) being - # assembled below. Ansible/apt provisioning runs directly on this disk - # before the chroot into /mnt, and the default Ubuntu AMI root size was - # too small, causing "No space left on device" failures during - # provisioning (reproduced independent of the surrogate/build-vol sizes). - launch_block_device_mappings { - device_name = "/dev/sda1" - delete_on_termination = true - volume_size = 16 - volume_type = "gp3" - } - launch_block_device_mappings { device_name = "/dev/xvdf" delete_on_termination = true diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 450755fde..6ea21381e 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -124,20 +124,6 @@ source "amazon-ebssurrogate" "source" { } ena_support = true - - # Builder instance's own boot root (from the source Ubuntu AMI, device - # /dev/sda1) - separate from the surrogate root (/dev/xvdf) being - # assembled below. Ansible/apt provisioning runs directly on this disk - # before the chroot into /mnt, and the default Ubuntu AMI root size was - # too small, causing "No space left on device" failures during - # provisioning (reproduced independent of the surrogate/build-vol sizes). - launch_block_device_mappings { - device_name = "/dev/sda1" - delete_on_termination = true - volume_size = 16 - volume_type = "gp3" - } - launch_block_device_mappings { device_name = "/dev/xvdf" delete_on_termination = true