From 25a16d226f0c51669a4c2cd13ffe9219069917b8 Mon Sep 17 00:00:00 2001 From: calvix <7136358+calvix@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:01:21 +0200 Subject: [PATCH 1/2] kvm: fill the template's holes before taking its LUKS clone snapshot An encrypted root is a librbd CoW clone of a plaintext template carrying a LUKS2 header. librbd serves the ranges the clone's own objects do not hold from that parent, as plaintext, which is what makes the inherited filesystem readable. That stops the moment an object exists in the clone: one 4 KiB guest write copies up the whole 4 MiB object, and the ranges inside it that the parent never materialised are from then on read through the crypto layer - where zeros decrypted with AES-XTS are not zeros. Before taking the snapshot encrypted roots are cloned from, rewrite the template image from its own cloudstack-base-snap with qemu-img convert -S 0, so the snapshot has no holes left. The snapshot gets a new name, -luks2, so templates prepared before get a dense one too; roots already cloned from the old -luks snapshot keep it. If another host prepares the same template at the same time, its protected snapshot is used instead of failing the deploy. QemuImg grows setWriteZeroRanges() for the -S 0 this needs, since qemu-img skips the source's zero ranges by default. --- .../kvm/storage/LibvirtStorageAdaptor.java | 19 ++++++++-- .../apache/cloudstack/utils/qemu/QemuImg.java | 14 ++++++++ .../cloudstack/utils/rbd/RbdEncryption.java | 36 +++++++++++++++++++ .../cloudstack/utils/qemu/QemuImgTest.java | 35 ++++++++++++++++++ .../utils/rbd/RbdEncryptionTest.java | 30 ++++++++++++++++ 5 files changed, 131 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 2c642efb9cfb..4745b74bc288 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -1573,7 +1573,8 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, */ private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KVMStoragePool destPool, String newUuid, KVMPhysicalDisk disk, byte[] passphrase) { - String luksReservedSnapshotName = rbdTemplateSnapName + "-luks"; + // not "-luks": templates prepared before carry a snapshot of that name that still has holes + String luksReservedSnapshotName = rbdTemplateSnapName + "-luks2"; Rados radosConnection = null; IoCTX ioContext = null; Rbd rbdClient = null; @@ -1596,8 +1597,20 @@ private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KV } if (!luksSnapshotExists) { templateImage.resize(template.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES); - templateImage.snapCreate(luksReservedSnapshotName); - templateImage.snapProtect(luksReservedSnapshotName); + // fill the template's holes first: an encrypted clone reads a hole in its parent as + // garbage once a guest write copies up the object around it + new RbdEncryption().copyTemplateDense(destPool.getSourceHost(), destPool.getSourcePort(), + destPool.getAuthUserName(), destPool.getAuthSecret(), destPool.getSourceDir(), + template.getName(), rbdTemplateSnapName); + try { + templateImage.snapCreate(luksReservedSnapshotName); + templateImage.snapProtect(luksReservedSnapshotName); + } catch (RbdException e) { + // fine if another host prepared the same template at the same time + if (!templateImage.snapIsProtected(luksReservedSnapshotName)) { + throw e; + } + } logger.debug("Prepared LUKS-reserved template snapshot {}@{}", template.getName(), luksReservedSnapshotName); } rbdClient.clone(template.getName(), luksReservedSnapshotName, ioContext, newUuid, RBD_FEATURES, rbdOrder); diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java index 6761dd4dc2b9..c911a1d01f4e 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java @@ -66,6 +66,7 @@ public class QemuImg { private String cloudQemuImgPath = "cloud-qemu-img"; private long timeout; private boolean skipZero = false; + private boolean writeZeroRanges = false; private boolean skipTargetVolumeCreation = false; private boolean noCache = false; private long version; @@ -522,6 +523,11 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, QemuI script.add("-n"); } + if (writeZeroRanges) { + script.add("-S"); + script.add("0"); + } + if (destImageOpts == null) { script.add("-O"); script.add(destFile.getFormat().toString()); @@ -1025,6 +1031,14 @@ public void setSkipZero(boolean skipZero) { this.skipZero = skipZero; } + /** + * Make convert write the source's zero ranges to the destination ({@code -S 0}) instead of + * leaving them unallocated. + */ + public void setWriteZeroRanges(boolean writeZeroRanges) { + this.writeZeroRanges = writeZeroRanges; + } + public void setSkipTargetVolumeCreation(boolean skipTargetVolumeCreation) { this.skipTargetVolumeCreation = skipTargetVolumeCreation; } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java index a23ce93d2b82..bca6d00a361b 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java @@ -250,6 +250,42 @@ public void importTemplate(String srcRbdPool, String srcRbdImage, } } + /** + * Rewrite an RBD image from one of its snapshots, writing out the zero ranges too ({@code -S 0}), + * so the image has no holes. Used on a template before taking the snapshot encrypted roots are + * cloned from: a hole in that parent reads as garbage through the clone's encryption once a + * guest write copies up the object around it. + */ + public void copyTemplateDense(String monHost, int monPort, String authUser, String authSecret, + String cephPool, String image, String srcSnapshot) { + Path conf = null; + Path keyring = null; + try { + final FileAttribute ownerOnly = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------")); + keyring = Files.createTempFile("cs-ceph-", ".keyring", ownerOnly); + Files.writeString(keyring, "[client." + authUser + "]\n\tkey = " + authSecret + "\n"); + conf = Files.createTempFile("cs-ceph-", ".conf", ownerOnly); + Files.writeString(conf, "[global]\nmon_host = " + monSpec(monHost, monPort) + "\nkeyring = " + keyring + "\n"); + + QemuImgFile srcQemuFile = new QemuImgFile(cephPool + "/" + image + "@" + srcSnapshot, QemuImg.PhysicalDiskFormat.RAW); + Map srcParams = rbdImageOptions(cephPool, image, conf.toString(), authUser); + srcParams.put("snapshot", srcSnapshot); + QemuImageOptions srcImageOpts = new QemuImageOptions(srcParams); + srcImageOpts.setImageOptsFlag(true); + QemuImageOptions destImageOpts = new QemuImageOptions(rbdImageOptions(cephPool, image, conf.toString(), authUser)); + + QemuImg qemu = createQemuImg(); + qemu.setWriteZeroRanges(true); + qemu.convertIntoExistingTarget(srcQemuFile, null, null, srcImageOpts, destImageOpts, false); + logger.debug("Rewrote RBD image {}/{} from @{} without holes", cephPool, image, srcSnapshot); + } catch (IOException | QemuImgException | LibvirtException ex) { + throw new CloudRuntimeException(String.format("Failed to rewrite RBD image %s/%s from @%s", cephPool, image, srcSnapshot), ex); + } finally { + deleteQuietly(conf); + deleteQuietly(keyring); + } + } + /** * Seam for unit tests; {@link QemuImg} probes the qemu version through libvirt on construction. */ diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/qemu/QemuImgTest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/qemu/QemuImgTest.java index 55f8adb541e4..ffc155cbae8c 100644 --- a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/qemu/QemuImgTest.java +++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/qemu/QemuImgTest.java @@ -342,6 +342,41 @@ public void testCreateWithBackingFileOfOtherFormat() throws QemuImgException, Li assertEquals(PhysicalDiskFormat.RAW.toString(), info.get(QemuImg.BACKING_FILE_FORMAT)); } + @Test + public void testConvertWriteZeroRangesAllocatesTheWholeDestination() throws QemuImgException, LibvirtException { + // qemu-img skips the source's zero ranges by default, so the destination keeps its holes. + // That is wrong for an image whose unallocated ranges are not read as zeros - an RBD image + // with a librbd LUKS header, cloned, is the case setWriteZeroRanges exists for. + String srcPath = "/tmp/" + UUID.randomUUID() + ".raw"; + String sparsePath = "/tmp/" + UUID.randomUUID() + ".raw"; + String densePath = "/tmp/" + UUID.randomUUID() + ".raw"; + long size = 16L * 1024 * 1024; + + QemuImgFile srcFile = new QemuImgFile(srcPath, size, PhysicalDiskFormat.RAW); + QemuImg qemu = new QemuImg(0); + qemu.create(srcFile); + // data in the first MiB only: the remaining 15 MiB of the source are zeros + Script.runSimpleBashScript(String.format("dd if=/dev/urandom of=%s bs=1M count=1 conv=notrunc 2>/dev/null", srcPath)); + + qemu.convert(srcFile, new QemuImgFile(sparsePath, PhysicalDiskFormat.RAW)); + qemu.setWriteZeroRanges(true); + qemu.convert(srcFile, new QemuImgFile(densePath, PhysicalDiskFormat.RAW)); + + long apparent = Long.parseLong(Script.runSimpleBashScript(String.format("stat -c %%s %s", densePath))); + long sparseAllocated = Long.parseLong(Script.runSimpleBashScript(String.format("du --block-size=1 %s | cut -f1", sparsePath))); + long denseAllocated = Long.parseLong(Script.runSimpleBashScript(String.format("du --block-size=1 %s | cut -f1", densePath))); + + assertEquals(size, apparent); + assertTrue("the default convert should have left the zero ranges unallocated, allocated " + sparseAllocated, + sparseAllocated < size); + assertTrue("-S 0 should have written every zero range, allocated " + denseAllocated, + denseAllocated >= size); + + assertTrue(new File(srcPath).delete()); + assertTrue(new File(sparsePath).delete()); + assertTrue(new File(densePath).delete()); + } + @Test public void testConvertBasic() throws QemuImgException, LibvirtException { long srcSize = 20480; diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/rbd/RbdEncryptionTest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/rbd/RbdEncryptionTest.java index e4c977c676f1..9839188618df 100644 --- a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/rbd/RbdEncryptionTest.java +++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/rbd/RbdEncryptionTest.java @@ -143,6 +143,36 @@ public void importTemplateFromFileSourceForcesSourceFormat() throws Exception { Assert.assertTrue(src, src.contains("file.filename=/tmp/tmpl.qcow2")); } + @Test + public void copyTemplateDenseWritesZeroRangesAndLeavesTheCopyPlaintext() throws Exception { + RbdEncryption spy = Mockito.spy(new RbdEncryption()); + QemuImg qemuImg = Mockito.mock(QemuImg.class); + Mockito.doReturn(qemuImg).when(spy).createQemuImg(); + + spy.copyTemplateDense("1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "tmpl", "cloudstack-base-snap"); + + // without -S 0 the template keeps its holes + Mockito.verify(qemuImg).setWriteZeroRanges(true); + + ArgumentCaptor srcOpts = ArgumentCaptor.forClass(QemuImageOptions.class); + ArgumentCaptor destOpts = ArgumentCaptor.forClass(QemuImageOptions.class); + Mockito.verify(qemuImg).convertIntoExistingTarget(Mockito.any(QemuImgFile.class), Mockito.isNull(), + Mockito.isNull(), srcOpts.capture(), destOpts.capture(), Mockito.eq(false)); + + String src = String.join(" ", srcOpts.getValue().toCommandFlag()); + Assert.assertTrue(src, src.startsWith("--image-opts ")); + Assert.assertTrue(src, src.contains("pool=cloudstack")); + Assert.assertTrue(src, src.contains("image=tmpl")); + Assert.assertTrue(src, src.contains("snapshot=cloudstack-base-snap")); + + String dest = String.join(" ", destOpts.getValue().toCommandFlag(QemuImg.TARGET_IMAGE_OPTS_FLAG)); + Assert.assertTrue(dest, dest.startsWith(QemuImg.TARGET_IMAGE_OPTS_FLAG + " ")); + Assert.assertTrue(dest, dest.contains("image=tmpl")); + // the template is written to itself, not to a snapshot, and stays plaintext + Assert.assertFalse(dest, dest.contains("snapshot=")); + Assert.assertFalse(dest, dest.contains("encrypt.")); + } + @Test public void formatRejectsEmptyPassphrase() { Assert.assertThrows(CloudRuntimeException.class, () -> rbdEncryption.format( From 0944c47529a90e2afe525b9266adecddf9963fcf Mon Sep 17 00:00:00 2001 From: calvix <7136358+calvix@users.noreply.github.com> Date: Tue, 10 Nov 2026 10:42:43 +0100 Subject: [PATCH 2/2] kvm: prepare the base snapshot before dense encrypted-root cloning Create the base snapshot for encrypted-first deployments and pass the caller timeout to the dense template rewrite. Cover old snapshot upgrades, reuse, failure and retry, and duplicate-snapshot handling. --- .../kvm/storage/LibvirtStorageAdaptor.java | 88 ++++++++---- .../cloudstack/utils/rbd/RbdEncryption.java | 12 +- .../storage/LibvirtStorageAdaptorTest.java | 135 ++++++++++++++++++ .../utils/rbd/RbdEncryptionTest.java | 11 +- 4 files changed, 210 insertions(+), 36 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 4745b74bc288..37145056f2a2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -1401,7 +1401,7 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, && srcPool.getSourceHost().equals(destPool.getSourceHost()) && srcPool.getSourceDir().equals(destPool.getSourceDir()); if (sameClusterRbd) { - return createEncryptedRootCoWClone(template, destPool, newUuid, disk, passphrase); + return createEncryptedRootCoWClone(template, destPool, newUuid, disk, passphrase, timeout); } return createEncryptedRootFullCopy(srcPool, template, destPool, newUuid, disk, passphrase); } @@ -1562,6 +1562,64 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, return disk; } + /** + * Create and protect a snapshot of a template image. Fine if another host created it at the same time. + */ + private void createProtectedSnapshot(RbdImage templateImage, String snapshotName) throws RbdException { + try { + templateImage.snapCreate(snapshotName); + templateImage.snapProtect(snapshotName); + } catch (RbdException e) { + if (!templateImage.snapIsProtected(snapshotName)) { + throw e; + } + } + } + + /** + * Prepare the template snapshot encrypted roots are cloned from, once per template per pool: grow the + * template by {@link #LUKS2_HEADER_RESERVE_BYTES}, rewrite it without holes and take the snapshot. + * Nothing to do if the snapshot exists. If this fails before the snapshot is taken, the next deploy + * from the template starts over. + * + * @param timeout limit for the rewrite of the template, which writes its whole virtual size + */ + void prepareLuksTemplateSnapshot(RbdImage templateImage, KVMPhysicalDisk template, KVMStoragePool destPool, + String luksSnapshotName, long timeout) throws RbdException { + boolean luksSnapshotExists = false; + boolean baseSnapshotExists = false; + for (RbdSnapInfo snapshotInfo : templateImage.snapList()) { + if (luksSnapshotName.equals(snapshotInfo.name)) { + luksSnapshotExists = true; + } else if (rbdTemplateSnapName.equals(snapshotInfo.name)) { + baseSnapshotExists = true; + } + } + if (luksSnapshotExists) { + return; + } + // the rewrite below reads the template from its base snapshot, which only the plaintext clone path + // creates: take it here when the first deploy from this template is encrypted + if (!baseSnapshotExists) { + createProtectedSnapshot(templateImage, rbdTemplateSnapName); + } + templateImage.resize(template.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES); + // fill the template's holes first: an encrypted clone reads a hole in its parent as + // garbage once a guest write copies up the object around it + createRbdEncryption().copyTemplateDense(destPool.getSourceHost(), destPool.getSourcePort(), + destPool.getAuthUserName(), destPool.getAuthSecret(), destPool.getSourceDir(), + template.getName(), rbdTemplateSnapName, timeout); + createProtectedSnapshot(templateImage, luksSnapshotName); + logger.debug("Prepared LUKS-reserved template snapshot {}@{}", template.getName(), luksSnapshotName); + } + + /** + * Seam for unit tests. + */ + protected RbdEncryption createRbdEncryption() { + return new RbdEncryption(); + } + /** * Option A (thin CoW encrypted root), used when the template already lives on the same RBD cluster * as the destination pool. Per the Ceph "Image Encryption" clone recipe: grow the template base to @@ -1572,7 +1630,7 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, * @return the encrypted CoW clone, or {@code null} if the Ceph operations failed */ private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KVMStoragePool destPool, - String newUuid, KVMPhysicalDisk disk, byte[] passphrase) { + String newUuid, KVMPhysicalDisk disk, byte[] passphrase, long timeout) { // not "-luks": templates prepared before carry a snapshot of that name that still has holes String luksReservedSnapshotName = rbdTemplateSnapName + "-luks2"; Rados radosConnection = null; @@ -1588,31 +1646,7 @@ private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KV ioContext = radosConnection.ioCtxCreate(destPool.getSourceDir()); rbdClient = new Rbd(ioContext); templateImage = rbdClient.open(template.getName()); - boolean luksSnapshotExists = false; - for (RbdSnapInfo snapshotInfo : templateImage.snapList()) { - if (luksReservedSnapshotName.equals(snapshotInfo.name)) { - luksSnapshotExists = true; - break; - } - } - if (!luksSnapshotExists) { - templateImage.resize(template.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES); - // fill the template's holes first: an encrypted clone reads a hole in its parent as - // garbage once a guest write copies up the object around it - new RbdEncryption().copyTemplateDense(destPool.getSourceHost(), destPool.getSourcePort(), - destPool.getAuthUserName(), destPool.getAuthSecret(), destPool.getSourceDir(), - template.getName(), rbdTemplateSnapName); - try { - templateImage.snapCreate(luksReservedSnapshotName); - templateImage.snapProtect(luksReservedSnapshotName); - } catch (RbdException e) { - // fine if another host prepared the same template at the same time - if (!templateImage.snapIsProtected(luksReservedSnapshotName)) { - throw e; - } - } - logger.debug("Prepared LUKS-reserved template snapshot {}@{}", template.getName(), luksReservedSnapshotName); - } + prepareLuksTemplateSnapshot(templateImage, template, destPool, luksReservedSnapshotName, timeout); rbdClient.clone(template.getName(), luksReservedSnapshotName, ioContext, newUuid, RBD_FEATURES, rbdOrder); } catch (RadosException | RbdException e) { logger.error("Failed to create encrypted CoW clone {}: {}", newUuid, e.getMessage()); diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java index bca6d00a361b..8be8a76025b1 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java @@ -239,7 +239,7 @@ public void importTemplate(String srcRbdPool, String srcRbdImage, secretParams.put(QemuObject.ObjectParameter.FILE, passFile.toString()); QemuObject luksSecret = new QemuObject(QemuObject.ObjectType.SECRET, secretParams); - QemuImg qemu = createQemuImg(); + QemuImg qemu = createQemuImg(0); qemu.convertIntoExistingTarget(srcQemuFile, null, List.of(luksSecret), srcImageOpts, destImageOpts, forceSourceFormat); logger.debug("Imported template into encrypted RBD image {}", imageSpec); } catch (IOException | QemuImgException | LibvirtException ex) { @@ -255,9 +255,11 @@ public void importTemplate(String srcRbdPool, String srcRbdImage, * so the image has no holes. Used on a template before taking the snapshot encrypted roots are * cloned from: a hole in that parent reads as garbage through the clone's encryption once a * guest write copies up the object around it. + * + * @param timeout qemu-img timeout in milliseconds; the copy writes the whole virtual size of the image */ public void copyTemplateDense(String monHost, int monPort, String authUser, String authSecret, - String cephPool, String image, String srcSnapshot) { + String cephPool, String image, String srcSnapshot, long timeout) { Path conf = null; Path keyring = null; try { @@ -274,7 +276,7 @@ public void copyTemplateDense(String monHost, int monPort, String authUser, Stri srcImageOpts.setImageOptsFlag(true); QemuImageOptions destImageOpts = new QemuImageOptions(rbdImageOptions(cephPool, image, conf.toString(), authUser)); - QemuImg qemu = createQemuImg(); + QemuImg qemu = createQemuImg(timeout); qemu.setWriteZeroRanges(true); qemu.convertIntoExistingTarget(srcQemuFile, null, null, srcImageOpts, destImageOpts, false); logger.debug("Rewrote RBD image {}/{} from @{} without holes", cephPool, image, srcSnapshot); @@ -289,8 +291,8 @@ public void copyTemplateDense(String monHost, int monPort, String authUser, Stri /** * Seam for unit tests; {@link QemuImg} probes the qemu version through libvirt on construction. */ - protected QemuImg createQemuImg() throws QemuImgException, LibvirtException { - return new QemuImg(0); + protected QemuImg createQemuImg(long timeout) throws QemuImgException, LibvirtException { + return new QemuImg(timeout); } /** qemu image options addressing an RBD image (as used with --image-opts / --target-image-opts). */ diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java index 9b8e1fd2d748..86805b98f663 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java @@ -22,16 +22,21 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.never; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.UUID; +import org.apache.cloudstack.utils.rbd.RbdEncryption; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.libvirt.Connect; import org.libvirt.StoragePool; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.MockedConstruction; import org.mockito.MockedStatic; @@ -40,6 +45,9 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import com.ceph.rbd.RbdException; +import com.ceph.rbd.RbdImage; +import com.ceph.rbd.jna.RbdSnapInfo; import com.cloud.hypervisor.kvm.resource.LibvirtConnection; import com.cloud.hypervisor.kvm.resource.LibvirtStoragePoolDef; import com.cloud.storage.Storage; @@ -347,4 +355,131 @@ public void testCLVMPoolDetailsSecureZeroFillDefault() { assert !secureZeroEnabled; } + private static final String BASE_SNAP = "cloudstack-base-snap"; + private static final String LUKS_SNAP = "cloudstack-base-snap-luks2"; + private static final long TEMPLATE_SIZE = 4L << 30; + private static final long TIMEOUT = 10800000L; + + private RbdImage templateImageWithSnapshots(String... names) throws RbdException { + RbdImage image = Mockito.mock(RbdImage.class); + List snaps = new ArrayList<>(); + for (String name : names) { + RbdSnapInfo info = new RbdSnapInfo(); + info.name = name; + snaps.add(info); + } + Mockito.when(image.snapList()).thenReturn(snaps); + return image; + } + + private KVMPhysicalDisk rbdTemplate() { + KVMPhysicalDisk template = Mockito.mock(KVMPhysicalDisk.class); + Mockito.lenient().when(template.getName()).thenReturn("tmpl"); + Mockito.lenient().when(template.getVirtualSize()).thenReturn(TEMPLATE_SIZE); + return template; + } + + private KVMStoragePool rbdPool() { + KVMStoragePool pool = Mockito.mock(KVMStoragePool.class); + Mockito.lenient().when(pool.getSourceHost()).thenReturn("1.2.3.4"); + Mockito.lenient().when(pool.getSourcePort()).thenReturn(6789); + Mockito.lenient().when(pool.getAuthUserName()).thenReturn("cloudstack"); + Mockito.lenient().when(pool.getAuthSecret()).thenReturn("secret"); + Mockito.lenient().when(pool.getSourceDir()).thenReturn("cloudstack"); + return pool; + } + + private LibvirtStorageAdaptor adaptorWith(RbdEncryption rbdEncryption) { + LibvirtStorageAdaptor adaptor = Mockito.spy(new LibvirtStorageAdaptor(null)); + Mockito.doReturn(rbdEncryption).when(adaptor).createRbdEncryption(); + return adaptor; + } + + @Test + public void prepareLuksTemplateSnapshotCreatesBaseSnapshotWhenFirstDeployIsEncrypted() throws Exception { + RbdImage image = templateImageWithSnapshots(); + RbdEncryption rbdEncryption = Mockito.mock(RbdEncryption.class); + + adaptorWith(rbdEncryption).prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT); + + // the base snapshot must exist, and still hold the template's original size and data, before the rewrite reads it + InOrder order = Mockito.inOrder(image, rbdEncryption); + order.verify(image).snapCreate(BASE_SNAP); + order.verify(image).snapProtect(BASE_SNAP); + order.verify(image).resize(TEMPLATE_SIZE + (16L << 20)); + order.verify(rbdEncryption).copyTemplateDense("1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "tmpl", BASE_SNAP, TIMEOUT); + order.verify(image).snapCreate(LUKS_SNAP); + order.verify(image).snapProtect(LUKS_SNAP); + } + + @Test + public void prepareLuksTemplateSnapshotUpgradesOldLuksSnapshotUsingExistingBase() throws Exception { + RbdImage image = templateImageWithSnapshots(BASE_SNAP, "cloudstack-base-snap-luks"); + RbdEncryption rbdEncryption = Mockito.mock(RbdEncryption.class); + + adaptorWith(rbdEncryption).prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT); + + Mockito.verify(image, never()).snapCreate(BASE_SNAP); + Mockito.verify(rbdEncryption).copyTemplateDense("1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "tmpl", BASE_SNAP, TIMEOUT); + Mockito.verify(image).snapCreate(LUKS_SNAP); + } + + @Test + public void prepareLuksTemplateSnapshotDoesNothingWhenAlreadyPrepared() throws Exception { + RbdImage image = templateImageWithSnapshots(BASE_SNAP, LUKS_SNAP); + RbdEncryption rbdEncryption = Mockito.mock(RbdEncryption.class); + + adaptorWith(rbdEncryption).prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT); + + Mockito.verify(image, never()).resize(anyLong()); + Mockito.verify(image, never()).snapCreate(anyString()); + Mockito.verifyNoInteractions(rbdEncryption); + } + + @Test + public void prepareLuksTemplateSnapshotTakesNoSnapshotWhenRewriteFailsAndRetryStartsOver() throws Exception { + RbdImage image = templateImageWithSnapshots(); + RbdEncryption rbdEncryption = Mockito.mock(RbdEncryption.class); + Mockito.doThrow(new CloudRuntimeException("qemu-img failed")).doNothing().when(rbdEncryption) + .copyTemplateDense(anyString(), Mockito.anyInt(), anyString(), anyString(), anyString(), anyString(), anyString(), anyLong()); + LibvirtStorageAdaptor adaptor = adaptorWith(rbdEncryption); + + Assert.assertThrows(CloudRuntimeException.class, + () -> adaptor.prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT)); + // a snapshot of a half-rewritten template would be cloned from forever + Mockito.verify(image, never()).snapCreate(LUKS_SNAP); + + // next deploy: the base snapshot is there now, the rewrite runs again and the snapshot is taken + RbdSnapInfo baseSnapshot = new RbdSnapInfo(); + baseSnapshot.name = BASE_SNAP; + Mockito.when(image.snapList()).thenReturn(List.of(baseSnapshot)); + adaptor.prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT); + Mockito.verify(image, Mockito.times(1)).snapCreate(BASE_SNAP); + Mockito.verify(rbdEncryption, Mockito.times(2)).copyTemplateDense("1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "tmpl", BASE_SNAP, TIMEOUT); + Mockito.verify(image).snapCreate(LUKS_SNAP); + } + + @Test + public void prepareLuksTemplateSnapshotAcceptsAlreadyProtectedDuplicateSnapshots() throws Exception { + RbdImage image = templateImageWithSnapshots(); + RbdEncryption rbdEncryption = Mockito.mock(RbdEncryption.class); + Mockito.doThrow(new RbdException("File exists", -17)).when(image).snapCreate(anyString()); + Mockito.when(image.snapIsProtected(anyString())).thenReturn(true); + + adaptorWith(rbdEncryption).prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT); + + Mockito.verify(rbdEncryption).copyTemplateDense("1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "tmpl", BASE_SNAP, TIMEOUT); + } + + @Test + public void prepareLuksTemplateSnapshotFailsWhenSnapshotCannotBeTaken() throws Exception { + RbdImage image = templateImageWithSnapshots(BASE_SNAP); + RbdEncryption rbdEncryption = Mockito.mock(RbdEncryption.class); + Mockito.doThrow(new RbdException("No space left", -28)).when(image).snapCreate(LUKS_SNAP); + Mockito.when(image.snapIsProtected(LUKS_SNAP)).thenReturn(false); + LibvirtStorageAdaptor adaptor = adaptorWith(rbdEncryption); + + Assert.assertThrows(RbdException.class, + () -> adaptor.prepareLuksTemplateSnapshot(image, rbdTemplate(), rbdPool(), LUKS_SNAP, TIMEOUT)); + } } diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/rbd/RbdEncryptionTest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/rbd/RbdEncryptionTest.java index 9839188618df..66106a4573bc 100644 --- a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/rbd/RbdEncryptionTest.java +++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/rbd/RbdEncryptionTest.java @@ -90,7 +90,7 @@ public void buildResizeScriptShrinkPassesAllowShrink() { public void importTemplateFromRbdSourceConvertsThroughQemuImg() throws Exception { RbdEncryption spy = Mockito.spy(new RbdEncryption()); QemuImg qemuImg = Mockito.mock(QemuImg.class); - Mockito.doReturn(qemuImg).when(spy).createQemuImg(); + Mockito.doReturn(qemuImg).when(spy).createQemuImg(Mockito.anyLong()); spy.importTemplate("srcpool", "srcimg", null, null, "1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "dst", "passphrase".getBytes(StandardCharsets.UTF_8), CryptSetup.LuksType.LUKS2); @@ -128,7 +128,7 @@ public void importTemplateFromRbdSourceConvertsThroughQemuImg() throws Exception public void importTemplateFromFileSourceForcesSourceFormat() throws Exception { RbdEncryption spy = Mockito.spy(new RbdEncryption()); QemuImg qemuImg = Mockito.mock(QemuImg.class); - Mockito.doReturn(qemuImg).when(spy).createQemuImg(); + Mockito.doReturn(qemuImg).when(spy).createQemuImg(Mockito.anyLong()); spy.importTemplate(null, null, "/tmp/tmpl.qcow2", "QCOW2", "1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "dst", "passphrase".getBytes(StandardCharsets.UTF_8), CryptSetup.LuksType.LUKS2); @@ -147,9 +147,12 @@ public void importTemplateFromFileSourceForcesSourceFormat() throws Exception { public void copyTemplateDenseWritesZeroRangesAndLeavesTheCopyPlaintext() throws Exception { RbdEncryption spy = Mockito.spy(new RbdEncryption()); QemuImg qemuImg = Mockito.mock(QemuImg.class); - Mockito.doReturn(qemuImg).when(spy).createQemuImg(); + Mockito.doReturn(qemuImg).when(spy).createQemuImg(Mockito.anyLong()); - spy.copyTemplateDense("1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "tmpl", "cloudstack-base-snap"); + spy.copyTemplateDense("1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "tmpl", "cloudstack-base-snap", 10800000L); + + // the copy writes the whole template: it must get the caller's timeout, not qemu-img's default of 1 hour + Mockito.verify(spy).createQemuImg(10800000L); // without -S 0 the template keeps its holes Mockito.verify(qemuImg).setWriteZeroRanges(true);