From 59b2676bdb94e2bb9777769b71736f8692cea4bb Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 15:50:25 +0000 Subject: [PATCH 01/30] fix(os): omit built-in FUSE module package --- .../meta-dstack/recipes-core/images/dstack-rootfs-base.inc | 1 - 1 file changed, 1 deletion(-) diff --git a/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc index 388dc0849..e767d38d1 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc +++ b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-rootfs-base.inc @@ -26,7 +26,6 @@ IMAGE_INSTALL = "\ dstack-zfs \ dstack-sysbox \ kernel-module-tun \ - kernel-module-fuse \ kernel-module-br-netfilter \ kernel-module-xt-mark \ kernel-module-xt-connmark \ From d89918f1c494ac7b2b0783574715ce4c6d4f6721 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 15:51:23 +0000 Subject: [PATCH 02/30] fix(os): fail multi-flavor builds on first error --- os/yocto/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/os/yocto/Makefile b/os/yocto/Makefile index 9f0128cf4..94dac8489 100644 --- a/os/yocto/Makefile +++ b/os/yocto/Makefile @@ -20,7 +20,7 @@ all: dist -include $(wildcard mk.d/*.mk) dist: images - $(foreach flavor,$(FLAVORS),./mkimage.sh --dist-name $(call flavor_to_dist,$(flavor)) --flavor $(flavor);) + set -e; $(foreach flavor,$(FLAVORS),./mkimage.sh --dist-name $(call flavor_to_dist,$(flavor)) --flavor $(flavor);) # Build common artifacts (shared across all flavors) # dstack-guest is built here first to warm sstate/downloads and avoid concurrent @@ -30,17 +30,17 @@ images-common: # Build flavor-specific artifacts using multiconfig (serial to avoid deadlock warnings) images-flavors: - $(foreach flavor,$(FLAVORS),bitbake mc:$(flavor):dstack-rootfs mc:$(flavor):dstack-uki;) + set -e; $(foreach flavor,$(FLAVORS),bitbake mc:$(flavor):dstack-rootfs mc:$(flavor):dstack-uki;) images: images-common images-flavors clean: bitbake -c cleansstate virtual/kernel dstack-initramfs dstack-ovmf - $(foreach flavor,$(FLAVORS),bitbake -c cleansstate mc:$(flavor):dstack-rootfs mc:$(flavor):dstack-uki;) + set -e; $(foreach flavor,$(FLAVORS),bitbake -c cleansstate mc:$(flavor):dstack-rootfs mc:$(flavor):dstack-uki;) clean-dstack: bitbake -c cleansstate dstack-guest - $(foreach flavor,$(FLAVORS),bitbake -c cleansstate mc:$(flavor):dstack-rootfs;) + set -e; $(foreach flavor,$(FLAVORS),bitbake -c cleansstate mc:$(flavor):dstack-rootfs;) clean-initrd: bitbake -c cleansstate dstack-initramfs From 26cc02ca117a4a1aa714336814481e5a5c5398a8 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 17:10:35 +0000 Subject: [PATCH 03/30] fix(simulator): wait for GCP vTPM readiness --- dstack/tee-simulator/src/tpm.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs index 70b2ea00b..aa58a1574 100644 --- a/dstack/tee-simulator/src/tpm.rs +++ b/dstack/tee-simulator/src/tpm.rs @@ -115,7 +115,22 @@ pub fn start_gcp_vtpm(runtime_dir: &Path, config: &TeeSimulatorConfig) -> Result } thread::sleep(Duration::from_millis(20)); } - command("tpm2_startup", &["-c"])?; + let mut startup_error = None; + for _ in 0..100 { + match command("tpm2_startup", &["-c"]) { + Ok(()) => { + startup_error = None; + break; + } + Err(error) => { + startup_error = Some(error); + thread::sleep(Duration::from_millis(20)); + } + } + } + if let Some(error) = startup_error { + return Err(error).context("GCP vTPM did not become ready"); + } replay_fixture_event_log()?; let template_with_size = state_dir.join("ak.tpm2b-public"); From dac6797709c6bec0a32b881022b56cb0b8f8a960 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 17:25:41 +0000 Subject: [PATCH 04/30] fix(os): install TPM device TCTI for simulator --- .../recipes-core/dstack-tee-simulator/dstack-tee-simulator.bb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/dstack-tee-simulator.bb b/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/dstack-tee-simulator.bb index 8380e7dcb..4e0d27297 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/dstack-tee-simulator.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/dstack-tee-simulator/dstack-tee-simulator.bb @@ -13,7 +13,7 @@ DSTACK_CORE_SRC ?= "${DSTACK_MONOREPO_ROOT}/dstack" S = "${UNPACKDIR}/repo/dstack" DEPENDS += "rsync-native cmake-native" -RDEPENDS:${PN} += "dstack-guest fuse3-utils swtpm tpm2-tools openssl" +RDEPENDS:${PN} += "dstack-guest fuse3-utils swtpm tpm2-tools libtss2-tcti-device openssl" do_unpack[depends] += "rsync-native:do_populate_sysroot" # aws-lc-sys cannot detect this Yocto cross build reliably with its default From bba596f79a0590f58d04898ff5b40782fef43594 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 17:37:29 +0000 Subject: [PATCH 05/30] fix(simulator): expose GCP TPM event log --- dstack/tee-simulator/src/tpm.rs | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs index 70b2ea00b..88cc929d3 100644 --- a/dstack/tee-simulator/src/tpm.rs +++ b/dstack/tee-simulator/src/tpm.rs @@ -6,6 +6,7 @@ //! template and certificate NV indices consumed by `tpm-attest`. use std::{ + ffi::CString, io::{Read, Write}, os::{ fd::{AsRawFd, FromRawFd}, @@ -117,6 +118,7 @@ pub fn start_gcp_vtpm(runtime_dir: &Path, config: &TeeSimulatorConfig) -> Result } command("tpm2_startup", &["-c"])?; replay_fixture_event_log()?; + install_fixture_event_log()?; let template_with_size = state_dir.join("ak.tpm2b-public"); let generated_public = state_dir.join("ak.public"); @@ -213,6 +215,45 @@ fn replay_fixture_event_log() -> Result<()> { Ok(()) } +fn install_fixture_event_log() -> Result<()> { + let security_root = Path::new("/sys/kernel/security"); + let event_log = security_root.join("tpm0/binary_bios_measurements"); + if event_log.exists() { + return Ok(()); + } + let tpm_dir = event_log.parent().context("TPM event log has no parent")?; + if let Err(error) = fs_err::create_dir_all(tpm_dir) { + if !matches!(error.raw_os_error(), Some(libc::EPERM) | Some(libc::EACCES)) { + return Err(error).context("failed to create simulated TPM event-log directory"); + } + let source = CString::new("dstack-tee-simulator")?; + let target = CString::new("/sys/kernel/security")?; + let fstype = CString::new("tmpfs")?; + let data = CString::new("mode=0755")?; + let rc = unsafe { + libc::mount( + source.as_ptr(), + target.as_ptr(), + fstype.as_ptr(), + libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC, + data.as_ptr().cast(), + ) + }; + if rc != 0 { + return Err(std::io::Error::last_os_error()) + .context("failed to mount simulated securityfs shadow"); + } + fs_err::create_dir_all(tpm_dir) + .context("failed to create simulated TPM event-log directory")?; + } + fs_err::write( + event_log, + include_bytes!("../../cc-eventlog/samples/tpm_eventlog.bin"), + ) + .context("failed to install simulated TPM event log")?; + Ok(()) +} + fn create_tpm_device_node() -> Result<()> { if Path::new("/dev/tpm0").exists() { return Ok(()); From 32cd940f6c222ef0b286f7a7ad4c5ad7c6ee4a75 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 17:48:33 +0000 Subject: [PATCH 06/30] fix(simulator): shadow securityfs for GCP event log --- dstack/tee-simulator/src/tpm.rs | 44 ++++++++++++++++----------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs index 88cc929d3..2f5746a04 100644 --- a/dstack/tee-simulator/src/tpm.rs +++ b/dstack/tee-simulator/src/tpm.rs @@ -222,30 +222,28 @@ fn install_fixture_event_log() -> Result<()> { return Ok(()); } let tpm_dir = event_log.parent().context("TPM event log has no parent")?; - if let Err(error) = fs_err::create_dir_all(tpm_dir) { - if !matches!(error.raw_os_error(), Some(libc::EPERM) | Some(libc::EACCES)) { - return Err(error).context("failed to create simulated TPM event-log directory"); - } - let source = CString::new("dstack-tee-simulator")?; - let target = CString::new("/sys/kernel/security")?; - let fstype = CString::new("tmpfs")?; - let data = CString::new("mode=0755")?; - let rc = unsafe { - libc::mount( - source.as_ptr(), - target.as_ptr(), - fstype.as_ptr(), - libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC, - data.as_ptr().cast(), - ) - }; - if rc != 0 { - return Err(std::io::Error::last_os_error()) - .context("failed to mount simulated securityfs shadow"); - } - fs_err::create_dir_all(tpm_dir) - .context("failed to create simulated TPM event-log directory")?; + // securityfs does not permit userspace to create a synthetic TPM event + // log hierarchy. Shadow it in this development-only guest before + // publishing the fixture that was replayed into the simulated PCRs. + let source = CString::new("dstack-tee-simulator")?; + let target = CString::new("/sys/kernel/security")?; + let fstype = CString::new("tmpfs")?; + let data = CString::new("mode=0755")?; + let rc = unsafe { + libc::mount( + source.as_ptr(), + target.as_ptr(), + fstype.as_ptr(), + libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC, + data.as_ptr().cast(), + ) + }; + if rc != 0 { + return Err(std::io::Error::last_os_error()) + .context("failed to mount simulated securityfs shadow"); } + fs_err::create_dir_all(tpm_dir) + .context("failed to create TPM event-log directory in securityfs shadow")?; fs_err::write( event_log, include_bytes!("../../cc-eventlog/samples/tpm_eventlog.bin"), From d255482ca6b7b8287c244fbf8b9aada8d852580e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 18:04:28 +0000 Subject: [PATCH 07/30] test(simulator): log NitroTPM vendor commands --- dstack/tee-simulator/src/tpm.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs index 70b2ea00b..6b9892681 100644 --- a/dstack/tee-simulator/src/tpm.rs +++ b/dstack/tee-simulator/src/tpm.rs @@ -371,6 +371,9 @@ fn proxy_tpm_commands( command.truncate(size); anyhow::ensure!(command.len() >= 10, "truncated TPM command"); let code = read_be_u32(&command[6..10], "TPM command code")?; + if code >= 0x2000_0000 { + eprintln!("NitroTPM proxy received vendor command 0x{code:08x}"); + } let response = if code == TPM2_CC_AWS_NSM_REQUEST { let template = nv_write .as_ref() From 3de12a003d66015cf5266376e7ad955cd606145c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 18:16:18 +0000 Subject: [PATCH 08/30] fix(simulator): advertise NitroTPM vendor command --- dstack/tee-simulator/src/tpm.rs | 75 ++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs index 6b9892681..6cc0b20d6 100644 --- a/dstack/tee-simulator/src/tpm.rs +++ b/dstack/tee-simulator/src/tpm.rs @@ -17,7 +17,7 @@ use std::{ time::Duration, }; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result, bail}; use aws_nitro_enclaves_nsm_api::api::{Request as NsmRequest, Response as NsmResponse}; use dstack_types::TeeSimulatorConfig; use mock_attestation::{nsm::NsmGenerator, parse_seed, server::MockCollateralState}; @@ -28,7 +28,9 @@ const TPM2_CC_NV_WRITE: u32 = 0x0000_0137; const TPM2_CC_NV_DEFINE_SPACE: u32 = 0x0000_012a; const TPM2_CC_NV_READ: u32 = 0x0000_014e; const TPM2_CC_NV_READ_PUBLIC: u32 = 0x0000_0169; +const TPM2_CC_GET_CAPABILITY: u32 = 0x0000_017a; const TPM2_CC_AWS_NSM_REQUEST: u32 = 0x2000_0001; +const TPM2_CAP_COMMANDS: u32 = 2; const VTPM_PROXY_IOC_NEW_DEV: libc::c_ulong = 0xc014_a100; const VTPM_PROXY_FLAG_TPM2: u32 = 1; @@ -409,7 +411,9 @@ fn proxy_tpm_commands( nv_write = Some(template); } } - transact(backend, &command)? + let mut response = transact(backend, &command)?; + advertise_nsm_vendor_command(code, &mut response)?; + response }; proxy .write_all(&response) @@ -417,6 +421,43 @@ fn proxy_tpm_commands( } } +fn advertise_nsm_vendor_command(code: u32, response: &mut Vec) -> Result<()> { + if code != TPM2_CC_GET_CAPABILITY || response.len() < 19 { + return Ok(()); + } + let response_code = read_be_u32(&response[6..10], "TPM response code")?; + let capability = read_be_u32(&response[11..15], "TPM capability")?; + if response_code != 0 || capability != TPM2_CAP_COMMANDS || response[10] != 0 { + return Ok(()); + } + let count = read_be_u32(&response[15..19], "TPM command attribute count")? as usize; + let attributes_end = 19usize + .checked_add(count.checked_mul(4).context("TPM command count overflow")?) + .context("TPM command attributes overflow")?; + anyhow::ensure!( + response.len() >= attributes_end, + "truncated TPM command attributes" + ); + if response[19..attributes_end].chunks_exact(4).any(|value| { + read_be_u32(value, "TPM command attributes") + .is_ok_and(|attributes| attributes == TPM2_CC_AWS_NSM_REQUEST) + }) { + return Ok(()); + } + let insertion = response[19..attributes_end] + .chunks_exact(4) + .position(|value| { + read_be_u32(value, "TPM command attributes") + .is_ok_and(|attributes| attributes & 0x2000_ffff > TPM2_CC_AWS_NSM_REQUEST) + }) + .map_or(attributes_end, |index| 19 + index * 4); + response.splice(insertion..insertion, TPM2_CC_AWS_NSM_REQUEST.to_be_bytes()); + response[15..19].copy_from_slice(&((count + 1) as u32).to_be_bytes()); + let response_size = response.len() as u32; + response[2..6].copy_from_slice(&response_size.to_be_bytes()); + Ok(()) +} + fn parse_nv_write(command: &[u8]) -> Result> { // sessions header + auth handle + NV index + authorizationSize if command.len() < 24 { @@ -519,3 +560,33 @@ fn transact(stream: &mut UnixStream, command: &[u8]) -> Result> { fn tpm_success_response() -> Vec { [0x80, 0x01, 0, 0, 0, 10, 0, 0, 0, 0].to_vec() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn get_capability_advertises_nsm_vendor_command() { + let mut response = Vec::new(); + response.extend_from_slice(&0x8001u16.to_be_bytes()); + response.extend_from_slice(&23u32.to_be_bytes()); + response.extend_from_slice(&0u32.to_be_bytes()); + response.push(0); + response.extend_from_slice(&TPM2_CAP_COMMANDS.to_be_bytes()); + response.extend_from_slice(&1u32.to_be_bytes()); + response.extend_from_slice(&0x2000_1000u32.to_be_bytes()); + + advertise_nsm_vendor_command(TPM2_CC_GET_CAPABILITY, &mut response).unwrap(); + + assert_eq!(read_be_u32(&response[2..6], "size").unwrap(), 27); + assert_eq!(read_be_u32(&response[15..19], "count").unwrap(), 2); + assert_eq!( + read_be_u32(&response[19..23], "vendor command").unwrap(), + TPM2_CC_AWS_NSM_REQUEST + ); + assert_eq!( + read_be_u32(&response[23..27], "existing command").unwrap(), + 0x2000_1000 + ); + } +} From 36b223cfc0df7c8f21f764055ede37cbaa792f82 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 18:16:34 +0000 Subject: [PATCH 09/30] style(simulator): apply repository rustfmt --- dstack/tee-simulator/src/tpm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs index 6cc0b20d6..b401dfbfd 100644 --- a/dstack/tee-simulator/src/tpm.rs +++ b/dstack/tee-simulator/src/tpm.rs @@ -17,7 +17,7 @@ use std::{ time::Duration, }; -use anyhow::{Context, Result, bail}; +use anyhow::{bail, Context, Result}; use aws_nitro_enclaves_nsm_api::api::{Request as NsmRequest, Response as NsmResponse}; use dstack_types::TeeSimulatorConfig; use mock_attestation::{nsm::NsmGenerator, parse_seed, server::MockCollateralState}; From 3e3f3fef1503fa81485a62fcfc7e424c3bf667f5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 18:56:27 +0000 Subject: [PATCH 10/30] fix(simulator): report live NitroTPM PCRs --- dstack/tee-simulator/src/tpm.rs | 36 ++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs index b401dfbfd..3710ca57d 100644 --- a/dstack/tee-simulator/src/tpm.rs +++ b/dstack/tee-simulator/src/tpm.rs @@ -380,7 +380,7 @@ fn proxy_tpm_commands( let template = nv_write .as_ref() .context("NitroTPM vendor command without an NV request")?; - nsm_response = Some(handle_nsm_vendor_command(generator, template)?); + nsm_response = Some(handle_nsm_vendor_command(generator, template, backend)?); tpm_success_response() } else if code == TPM2_CC_NV_READ && nsm_response.is_some() { nv_read_response( @@ -486,14 +486,15 @@ fn parse_nv_write(command: &[u8]) -> Result> { fn handle_nsm_vendor_command( generator: &NsmGenerator, template: &NvWriteTemplate, + backend: &mut UnixStream, ) -> Result> { let request: NsmRequest = serde_cbor::from_slice(&template.request)?; let response = match request { NsmRequest::Attestation { user_data, .. } => { let pcrs = [4u16, 7, 8, 12, 14] .into_iter() - .map(|i| (i, vec![0; 48])) - .collect(); + .map(|index| Ok((index, read_sha384_pcr(backend, index)?))) + .collect::>()?; let document = generator.attest_with_pcrs( user_data.as_ref().map(|v| v.as_slice()).unwrap_or_default(), pcrs, @@ -505,6 +506,35 @@ fn handle_nsm_vendor_command( Ok(serde_cbor::to_vec(&response)?) } +fn read_sha384_pcr(backend: &mut UnixStream, index: u16) -> Result> { + anyhow::ensure!(index < 24, "invalid PCR index {index}"); + let mut command = Vec::with_capacity(20); + command.extend_from_slice(&0x8001u16.to_be_bytes()); + command.extend_from_slice(&20u32.to_be_bytes()); + command.extend_from_slice(&0x0000_017eu32.to_be_bytes()); + command.extend_from_slice(&1u32.to_be_bytes()); + command.extend_from_slice(&0x000cu16.to_be_bytes()); + command.push(3); + let mut selection = [0u8; 3]; + selection[index as usize / 8] = 1 << (index % 8); + command.extend_from_slice(&selection); + + let response = transact(backend, &command)?; + anyhow::ensure!(response.len() >= 30, "truncated TPM PCR_Read response"); + anyhow::ensure!( + read_be_u32(&response[6..10], "TPM PCR_Read response code")? == 0, + "TPM PCR_Read failed" + ); + anyhow::ensure!( + read_be_u32(&response[24..28], "TPM PCR digest count")? == 1, + "unexpected TPM PCR digest count" + ); + let size = read_be_u16(&response[28..30], "TPM PCR digest size")? as usize; + anyhow::ensure!(size == 48, "unexpected SHA-384 PCR size {size}"); + anyhow::ensure!(response.len() >= 30 + size, "truncated TPM PCR digest"); + Ok(response[30..30 + size].to_vec()) +} + fn set_nv_public_size(response: &mut [u8], size: usize) -> Result<()> { anyhow::ensure!(response.len() >= 26, "truncated NV_ReadPublic response"); let policy_size_pos = 22; From 750a75e10d8e2cc27299f8a05f9c465d223a89ba Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 19:08:33 +0000 Subject: [PATCH 11/30] chore(simulator): remove NitroTPM debug output --- dstack/tee-simulator/src/tpm.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs index 3710ca57d..e127703d5 100644 --- a/dstack/tee-simulator/src/tpm.rs +++ b/dstack/tee-simulator/src/tpm.rs @@ -373,9 +373,6 @@ fn proxy_tpm_commands( command.truncate(size); anyhow::ensure!(command.len() >= 10, "truncated TPM command"); let code = read_be_u32(&command[6..10], "TPM command code")?; - if code >= 0x2000_0000 { - eprintln!("NitroTPM proxy received vendor command 0x{code:08x}"); - } let response = if code == TPM2_CC_AWS_NSM_REQUEST { let template = nv_write .as_ref() From 5786cdce45db895e230ca9a96d6307b81b0b8d9a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 18:30:54 +0000 Subject: [PATCH 12/30] fix(vmm): generate simulated SEV-SNP mr_config --- dstack/vmm/src/app.rs | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 690b7c7af..0583a6ce2 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -283,6 +283,20 @@ pub(crate) enum PullStatus { Failed(String), } +fn needs_mr_config_v3( + manifest: &Manifest, + platform: crate::config::CvmPlatform, + use_mrconfigid: bool, + has_key_provider_id: bool, +) -> bool { + manifest.simulated_tee == Some(dstack_types::TeeVariant::DstackAmdSevSnp) + || (!manifest.no_tee + && (platform == crate::config::CvmPlatform::AmdSevSnp + || (platform == crate::config::CvmPlatform::Tdx + && use_mrconfigid + && has_key_provider_id))) +} + #[derive(Clone)] pub struct App { pub config: Arc, @@ -1087,11 +1101,12 @@ impl App { let app_compose = work_dir .app_compose() .context("Failed to get app compose")?; - let use_mr_config_v3 = !manifest.no_tee - && (platform == crate::config::CvmPlatform::AmdSevSnp - || (platform == crate::config::CvmPlatform::Tdx - && cfg.cvm.use_mrconfigid - && !app_compose.key_provider_id.is_empty())); + let use_mr_config_v3 = needs_mr_config_v3( + &manifest, + platform, + cfg.cvm.use_mrconfigid, + !app_compose.key_provider_id.is_empty(), + ); let mr_config = if use_mr_config_v3 { Some( work_dir @@ -1742,6 +1757,14 @@ mod tests { } } + #[test] + fn simulated_sev_snp_requires_mr_config_even_without_tee() { + let mut manifest = test_manifest(2048); + manifest.no_tee = true; + manifest.simulated_tee = Some(dstack_types::TeeVariant::DstackAmdSevSnp); + assert!(needs_mr_config_v3(&manifest, CvmPlatform::Tdx, true, false)); + } + fn dummy_tdx_measurement_document() -> TdxOsImageMeasurementDocument { let measurement = TdxOsImageMeasurement { image: TdxImageMeasurement { From 09b417159378990fe865bf91c99c1e77e4c42528 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 18:31:17 +0000 Subject: [PATCH 13/30] fix(vmm): pass cloud image measurements to guests --- dstack/vmm/src/app.rs | 75 ++++++++++++++++++++++---------- dstack/vmm/src/app/image.rs | 45 +++++++++++++++++-- dstack/vmm/src/app/qemu.rs | 86 ++++++++++++++++++++++--------------- 3 files changed, 145 insertions(+), 61 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 690b7c7af..843faf5d5 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -4,7 +4,7 @@ use crate::config::{Config, Networking, ProcessAnnotation, Protocol}; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result, bail}; use bon::Builder; use dstack_kms_rpc::kms_client::KmsClient; use dstack_types::mr_config::MrConfigV3; @@ -1394,6 +1394,9 @@ fn make_vm_config( let platform = cfg.cvm.resolved_platform(); let is_amd_sev_snp = platform == crate::config::CvmPlatform::AmdSevSnp && !manifest.no_tee; let is_tdx = platform == crate::config::CvmPlatform::Tdx && !manifest.no_tee; + let is_gcp_tdx = manifest.simulated_tee == Some(dstack_types::TeeVariant::DstackGcpTdx); + let is_aws_nitro_tpm = + manifest.simulated_tee == Some(dstack_types::TeeVariant::DstackAwsNitroTpm); let tdx_attestation_variant = if is_tdx { tdx_attestation_variant_from_requirements(requirements).unwrap_or_else(|| { cfg.cvm @@ -1444,6 +1447,24 @@ fn make_vm_config( let num_nics = resolved_networks(manifest, &cfg.cvm).len() as u32; let num_verity_volumes = manifest.volumes.len() as u32; let swtpm = manifest.swtpm; + let gcp_measurement = if is_gcp_tdx { + Some( + image + .gcp_measurement + .clone() + .context("GCP TDX image is missing measurement.gcp.cbor measurement material")?, + ) + } else { + None + }; + let aws_measurement = + if is_aws_nitro_tpm { + Some(image.aws_measurement.clone().context( + "AWS NitroTPM image is missing measurement.aws.cbor measurement material", + )?) + } else { + None + }; let mut config = serde_json::to_value(dstack_types::VmConfig { os_image_hash, cpu_count: effective_vcpus, @@ -1464,8 +1485,8 @@ fn make_vm_config( ovmf_variant: image.info.ovmf_variant, tdx_attestation_variant, tdx_measurement, - gcp_measurement: None, - aws_measurement: None, + gcp_measurement, + aws_measurement, })?; // For backward compatibility config["spec_version"] = serde_json::Value::from(1); @@ -1504,7 +1525,7 @@ pub(crate) fn needs_swtpm( mod tests { use super::*; use crate::config::{ - load_config_figment, CvmPlatform, Networking, NetworkingMode, TdxAttestationVariantConfig, + CvmPlatform, Networking, NetworkingMode, TdxAttestationVariantConfig, load_config_figment, }; use dstack_types::{ TdxImageMeasurement, TdxMrtdCandidates, TdxOsImageMeasurement, @@ -1576,25 +1597,31 @@ mod tests { #[test] fn gpu_config_has_gpus_only_when_resolved_gpu_list_is_non_empty() { assert!(!GpuConfig::default().has_gpus()); - assert!(!GpuConfig { - attach_mode: AttachMode::All, - ..Default::default() - } - .has_gpus()); - assert!(!GpuConfig { - bridges: vec![GpuSpec { - slot: "0000:01:00.0".into(), - }], - ..Default::default() - } - .has_gpus()); - assert!(GpuConfig { - gpus: vec![GpuSpec { - slot: "0000:02:00.0".into(), - }], - ..Default::default() - } - .has_gpus()); + assert!( + !GpuConfig { + attach_mode: AttachMode::All, + ..Default::default() + } + .has_gpus() + ); + assert!( + !GpuConfig { + bridges: vec![GpuSpec { + slot: "0000:01:00.0".into(), + }], + ..Default::default() + } + .has_gpus() + ); + assert!( + GpuConfig { + gpus: vec![GpuSpec { + slot: "0000:02:00.0".into(), + }], + ..Default::default() + } + .has_gpus() + ); } #[test] @@ -1794,6 +1821,8 @@ mod tests { digest: Some(hex_of(0xaa, 32)), tdx_measurement, sev_measurement: None, + gcp_measurement: None, + aws_measurement: None, } } diff --git a/dstack/vmm/src/app/image.rs b/dstack/vmm/src/app/image.rs index 40f7df4fd..d3508222d 100644 --- a/dstack/vmm/src/app/image.rs +++ b/dstack/vmm/src/app/image.rs @@ -6,13 +6,16 @@ use fs_err as fs; use path_absolutize::Absolutize; use std::path::{Path, PathBuf}; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result, bail}; use dstack_types::{ - SevOsImageMeasurementDocument, TdxOsImageMeasurementDocument, SNP_MEASUREMENT_FILENAME, - TDX_MEASUREMENT_FILENAME, + AwsOsImageMeasurementDocument, GCP_MEASUREMENT_FILENAME, GcpOsImageMeasurementDocument, + SNP_MEASUREMENT_FILENAME, SevOsImageMeasurementDocument, TDX_MEASUREMENT_FILENAME, + TdxOsImageMeasurementDocument, }; use serde::{Deserialize, Serialize}; +const AWS_MEASUREMENT_FILENAME: &str = "measurement.aws.cbor"; + #[derive(Debug, Serialize, Deserialize)] pub struct ImageInfo { pub cmdline: Option, @@ -79,6 +82,10 @@ pub struct Image { pub tdx_measurement: Option, /// AMD SEV-SNP no-image-download measurement material. pub sev_measurement: Option, + /// GCP TDX no-image-download measurement material. + pub gcp_measurement: Option, + /// AWS NitroTPM no-image-download measurement material. + pub aws_measurement: Option, } impl Image { @@ -148,6 +155,18 @@ impl Image { )), _ => None, }; + let gcp_measurement = load_measurement_document( + &base_path, + &sha256sum, + GCP_MEASUREMENT_FILENAME, + GcpOsImageMeasurementDocument::new, + )?; + let aws_measurement = load_measurement_document( + &base_path, + &sha256sum, + AWS_MEASUREMENT_FILENAME, + AwsOsImageMeasurementDocument::new, + )?; if info.version.is_empty() { // Older images does not have version field. Fallback to the version of the image folder name info.version = guess_version(&base_path).unwrap_or_default(); @@ -163,6 +182,8 @@ impl Image { digest, tdx_measurement, sev_measurement, + gcp_measurement, + aws_measurement, } .ensure_exists() } @@ -198,6 +219,24 @@ impl Image { } } +fn load_measurement_document( + base_path: &Path, + checksum_file: &Option>, + filename: &str, + constructor: impl FnOnce(Vec, Vec) -> T, +) -> Result> { + let path = base_path.join(filename); + if !path.exists() { + return Ok(None); + } + let Some(checksum_file) = checksum_file else { + return Ok(None); + }; + let measurement = + fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; + Ok(Some(constructor(checksum_file.clone(), measurement))) +} + fn guess_version(base_path: &Path) -> Option { // name pattern: dstack-dev-0.2.3 or dstack-0.2.3 let basename = base_path.file_name()?.to_str()?.to_string(); diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index a68be81df..6f697ca80 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -4,20 +4,20 @@ //! QEMU launch preparation and command construction. use super::{ - effective_vcpu_count, + GpuConfig, VmWorkDir, effective_vcpu_count, host_share::create_shared_disk, hugepage_numa_nodes, image::Image, mr_config::{snp_host_data, tdx_mr_config_id}, network::{mac_address_for_vm_index, resolved_networks, validate_resolved_networks}, - pci_numa_node, round_up, GpuConfig, VmWorkDir, + pci_numa_node, round_up, }; use crate::{ app::Manifest, config::{CvmConfig, CvmPlatform, Networking, NetworkingMode, ProcessAnnotation}, vm_launcher::{ChildCommand, LaunchSpec}, }; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result, bail}; use bon::Builder; use dstack_types::shared_filenames::HOST_SHARED_DISK_LABEL; use fs_err as fs; @@ -966,17 +966,17 @@ mod tests { use std::path::PathBuf; use rocket::figment::{ - providers::{Format, Toml}, Figment, + providers::{Format, Toml}, }; use super::{ - amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, virtio_pci_device, PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, VmConfig, + amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, virtio_pci_device, }; use crate::app::image::{Image, ImageInfo}; - use crate::app::{needs_swtpm, GpuConfig, Manifest, PortMapping, VmVolume, VmWorkDir}; - use crate::config::{Config, CvmPlatform, Protocol, DEFAULT_CONFIG}; + use crate::app::{GpuConfig, Manifest, PortMapping, VmVolume, VmWorkDir, needs_swtpm}; + use crate::config::{Config, CvmPlatform, DEFAULT_CONFIG, Protocol}; use dstack_types::{KeyProviderKind, TeeVariant}; #[test] @@ -1088,6 +1088,8 @@ mod tests { digest: None, tdx_measurement: None, sev_measurement: None, + gcp_measurement: None, + aws_measurement: None, }, cid: 100, workdir: PathBuf::from("/does-not-exist/vm-1"), @@ -1120,28 +1122,36 @@ mod tests { .unwrap(); assert_eq!(process.command, "/not-installed/qemu-system-x86_64"); - assert!(process - .args - .windows(2) - .any(|args| args == ["-machine", "q35,kernel-irqchip=split,hpet=off"])); - assert!(process - .args - .windows(2) - .any(|args| args == ["-kernel", "/does-not-exist/kernel"])); - assert!(process - .args - .windows(2) - .any(|args| args == ["-append", "console=hvc0"])); + assert!( + process + .args + .windows(2) + .any(|args| args == ["-machine", "q35,kernel-irqchip=split,hpet=off"]) + ); + assert!( + process + .args + .windows(2) + .any(|args| args == ["-kernel", "/does-not-exist/kernel"]) + ); + assert!( + process + .args + .windows(2) + .any(|args| args == ["-append", "console=hvc0"]) + ); assert!(process.args.windows(2).any(|args| { args == [ "-drive", "file=/does-not-exist/volume.img,if=none,id=vol0,format=raw,readonly=on", ] })); - assert!(process - .args - .iter() - .any(|arg| { arg == "virtio-blk-pci,drive=vol0" })); + assert!( + process + .args + .iter() + .any(|arg| { arg == "virtio-blk-pci,drive=vol0" }) + ); let volume_position = process .args .iter() @@ -1164,14 +1174,18 @@ mod tests { assert!(netdevs[0].contains("hostfwd=tcp:127.0.0.1:18080-:8080")); assert!(netdevs[1].contains("user,id=net1")); assert!(!netdevs[1].contains("hostfwd=")); - assert!(process - .args - .iter() - .any(|arg| arg.contains("virtio-net-pci,netdev=net0"))); - assert!(process - .args - .iter() - .any(|arg| arg.contains("virtio-net-pci,netdev=net1"))); + assert!( + process + .args + .iter() + .any(|arg| arg.contains("virtio-net-pci,netdev=net0")) + ); + assert!( + process + .args + .iter() + .any(|arg| arg.contains("virtio-net-pci,netdev=net1")) + ); prepared.swtpm_socket = Some(PathBuf::from("/does-not-exist/vm-1/swtpm/swtpm.sock")); let process = QemuCommandBuilder { @@ -1188,9 +1202,11 @@ mod tests { "socket,id=chrtpm,path=/does-not-exist/vm-1/swtpm/swtpm.sock", ] })); - assert!(process - .args - .windows(2) - .any(|args| args == ["-tpmdev", "emulator,id=tpm0,chardev=chrtpm"])); + assert!( + process + .args + .windows(2) + .any(|args| args == ["-tpmdev", "emulator,id=tpm0,chardev=chrtpm"]) + ); } } From 8b1a25b09a7f37c794ef8167d477182152fe1320 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 18:31:17 +0000 Subject: [PATCH 14/30] style(vmm): apply repository rustfmt --- dstack/vmm/src/app.rs | 48 ++++++++++----------- dstack/vmm/src/app/image.rs | 8 ++-- dstack/vmm/src/app/qemu.rs | 84 ++++++++++++++++--------------------- 3 files changed, 60 insertions(+), 80 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 843faf5d5..7d1bf99e6 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -4,7 +4,7 @@ use crate::config::{Config, Networking, ProcessAnnotation, Protocol}; -use anyhow::{Context, Result, bail}; +use anyhow::{bail, Context, Result}; use bon::Builder; use dstack_kms_rpc::kms_client::KmsClient; use dstack_types::mr_config::MrConfigV3; @@ -1525,7 +1525,7 @@ pub(crate) fn needs_swtpm( mod tests { use super::*; use crate::config::{ - CvmPlatform, Networking, NetworkingMode, TdxAttestationVariantConfig, load_config_figment, + load_config_figment, CvmPlatform, Networking, NetworkingMode, TdxAttestationVariantConfig, }; use dstack_types::{ TdxImageMeasurement, TdxMrtdCandidates, TdxOsImageMeasurement, @@ -1597,31 +1597,25 @@ mod tests { #[test] fn gpu_config_has_gpus_only_when_resolved_gpu_list_is_non_empty() { assert!(!GpuConfig::default().has_gpus()); - assert!( - !GpuConfig { - attach_mode: AttachMode::All, - ..Default::default() - } - .has_gpus() - ); - assert!( - !GpuConfig { - bridges: vec![GpuSpec { - slot: "0000:01:00.0".into(), - }], - ..Default::default() - } - .has_gpus() - ); - assert!( - GpuConfig { - gpus: vec![GpuSpec { - slot: "0000:02:00.0".into(), - }], - ..Default::default() - } - .has_gpus() - ); + assert!(!GpuConfig { + attach_mode: AttachMode::All, + ..Default::default() + } + .has_gpus()); + assert!(!GpuConfig { + bridges: vec![GpuSpec { + slot: "0000:01:00.0".into(), + }], + ..Default::default() + } + .has_gpus()); + assert!(GpuConfig { + gpus: vec![GpuSpec { + slot: "0000:02:00.0".into(), + }], + ..Default::default() + } + .has_gpus()); } #[test] diff --git a/dstack/vmm/src/app/image.rs b/dstack/vmm/src/app/image.rs index d3508222d..6b7cab2fb 100644 --- a/dstack/vmm/src/app/image.rs +++ b/dstack/vmm/src/app/image.rs @@ -6,11 +6,11 @@ use fs_err as fs; use path_absolutize::Absolutize; use std::path::{Path, PathBuf}; -use anyhow::{Context, Result, bail}; +use anyhow::{bail, Context, Result}; use dstack_types::{ - AwsOsImageMeasurementDocument, GCP_MEASUREMENT_FILENAME, GcpOsImageMeasurementDocument, - SNP_MEASUREMENT_FILENAME, SevOsImageMeasurementDocument, TDX_MEASUREMENT_FILENAME, - TdxOsImageMeasurementDocument, + AwsOsImageMeasurementDocument, GcpOsImageMeasurementDocument, SevOsImageMeasurementDocument, + TdxOsImageMeasurementDocument, GCP_MEASUREMENT_FILENAME, SNP_MEASUREMENT_FILENAME, + TDX_MEASUREMENT_FILENAME, }; use serde::{Deserialize, Serialize}; diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 6f697ca80..f84b44fc8 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -4,20 +4,20 @@ //! QEMU launch preparation and command construction. use super::{ - GpuConfig, VmWorkDir, effective_vcpu_count, + effective_vcpu_count, host_share::create_shared_disk, hugepage_numa_nodes, image::Image, mr_config::{snp_host_data, tdx_mr_config_id}, network::{mac_address_for_vm_index, resolved_networks, validate_resolved_networks}, - pci_numa_node, round_up, + pci_numa_node, round_up, GpuConfig, VmWorkDir, }; use crate::{ app::Manifest, config::{CvmConfig, CvmPlatform, Networking, NetworkingMode, ProcessAnnotation}, vm_launcher::{ChildCommand, LaunchSpec}, }; -use anyhow::{Context, Result, bail}; +use anyhow::{bail, Context, Result}; use bon::Builder; use dstack_types::shared_filenames::HOST_SHARED_DISK_LABEL; use fs_err as fs; @@ -966,17 +966,17 @@ mod tests { use std::path::PathBuf; use rocket::figment::{ - Figment, providers::{Format, Toml}, + Figment, }; use super::{ - PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, VmConfig, amd_sev_snp_memory_backend_arg, parse_amd_sev_snp_qmp_capabilities, virtio_pci_device, + PreparedQemuLaunch, PreparedVolume, QemuCommandBuilder, VmConfig, }; use crate::app::image::{Image, ImageInfo}; - use crate::app::{GpuConfig, Manifest, PortMapping, VmVolume, VmWorkDir, needs_swtpm}; - use crate::config::{Config, CvmPlatform, DEFAULT_CONFIG, Protocol}; + use crate::app::{needs_swtpm, GpuConfig, Manifest, PortMapping, VmVolume, VmWorkDir}; + use crate::config::{Config, CvmPlatform, Protocol, DEFAULT_CONFIG}; use dstack_types::{KeyProviderKind, TeeVariant}; #[test] @@ -1122,36 +1122,28 @@ mod tests { .unwrap(); assert_eq!(process.command, "/not-installed/qemu-system-x86_64"); - assert!( - process - .args - .windows(2) - .any(|args| args == ["-machine", "q35,kernel-irqchip=split,hpet=off"]) - ); - assert!( - process - .args - .windows(2) - .any(|args| args == ["-kernel", "/does-not-exist/kernel"]) - ); - assert!( - process - .args - .windows(2) - .any(|args| args == ["-append", "console=hvc0"]) - ); + assert!(process + .args + .windows(2) + .any(|args| args == ["-machine", "q35,kernel-irqchip=split,hpet=off"])); + assert!(process + .args + .windows(2) + .any(|args| args == ["-kernel", "/does-not-exist/kernel"])); + assert!(process + .args + .windows(2) + .any(|args| args == ["-append", "console=hvc0"])); assert!(process.args.windows(2).any(|args| { args == [ "-drive", "file=/does-not-exist/volume.img,if=none,id=vol0,format=raw,readonly=on", ] })); - assert!( - process - .args - .iter() - .any(|arg| { arg == "virtio-blk-pci,drive=vol0" }) - ); + assert!(process + .args + .iter() + .any(|arg| { arg == "virtio-blk-pci,drive=vol0" })); let volume_position = process .args .iter() @@ -1174,18 +1166,14 @@ mod tests { assert!(netdevs[0].contains("hostfwd=tcp:127.0.0.1:18080-:8080")); assert!(netdevs[1].contains("user,id=net1")); assert!(!netdevs[1].contains("hostfwd=")); - assert!( - process - .args - .iter() - .any(|arg| arg.contains("virtio-net-pci,netdev=net0")) - ); - assert!( - process - .args - .iter() - .any(|arg| arg.contains("virtio-net-pci,netdev=net1")) - ); + assert!(process + .args + .iter() + .any(|arg| arg.contains("virtio-net-pci,netdev=net0"))); + assert!(process + .args + .iter() + .any(|arg| arg.contains("virtio-net-pci,netdev=net1"))); prepared.swtpm_socket = Some(PathBuf::from("/does-not-exist/vm-1/swtpm/swtpm.sock")); let process = QemuCommandBuilder { @@ -1202,11 +1190,9 @@ mod tests { "socket,id=chrtpm,path=/does-not-exist/vm-1/swtpm/swtpm.sock", ] })); - assert!( - process - .args - .windows(2) - .any(|args| args == ["-tpmdev", "emulator,id=tpm0,chardev=chrtpm"]) - ); + assert!(process + .args + .windows(2) + .any(|args| args == ["-tpmdev", "emulator,id=tpm0,chardev=chrtpm"])); } } From 7c75a221519bd068257dceba750f3900a6af0853 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 24 Jul 2026 18:46:38 +0000 Subject: [PATCH 15/30] fix(simulator): retry interrupted vTPM reads --- dstack/tee-simulator/src/tpm.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs index 70b2ea00b..43c3c44db 100644 --- a/dstack/tee-simulator/src/tpm.rs +++ b/dstack/tee-simulator/src/tpm.rs @@ -359,6 +359,7 @@ fn proxy_tpm_commands( let size = loop { match proxy.read(&mut command) { Ok(size) => break size, + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, Err(error) if error.raw_os_error() == Some(libc::EPIPE) => { thread::sleep(Duration::from_millis(10)); } From f01de7e2a0694bf7ac03773ad3f5a9d0be8458ae Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 31 Jul 2026 08:47:31 +0000 Subject: [PATCH 16/30] fix(simulator): support current FUSE soname --- dstack/tee-simulator/src/nsm.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dstack/tee-simulator/src/nsm.rs b/dstack/tee-simulator/src/nsm.rs index c0ef2de28..bc4d44835 100644 --- a/dstack/tee-simulator/src/nsm.rs +++ b/dstack/tee-simulator/src/nsm.rs @@ -86,7 +86,12 @@ fn cuse() -> &'static CuseApi { } unsafe fn load_cuse() -> Result { - let library = Box::leak(Box::new(libloading::Library::new("libfuse3.so.3")?)); + // FUSE 3.18 bumped the shared-library SONAME to 4. Keep the older SONAME + // fallback so development binaries also run on distributions that still + // ship the previous ABI. + let library = libloading::Library::new("libfuse3.so.4") + .or_else(|_| libloading::Library::new("libfuse3.so.3"))?; + let library = Box::leak(Box::new(library)); Ok(CuseApi { main: *library.get(b"cuse_lowlevel_main\0")?, reply_open: *library.get(b"fuse_reply_open\0")?, From edc8c712189d5387f7da767698d2e1b41e44d468 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 31 Jul 2026 11:01:49 +0000 Subject: [PATCH 17/30] refactor(simulator): use safe securityfs mount wrapper --- dstack/tee-simulator/Cargo.toml | 1 + dstack/tee-simulator/src/tpm.rs | 29 +++++++++++------------------ 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/dstack/tee-simulator/Cargo.toml b/dstack/tee-simulator/Cargo.toml index b235d5467..18ed41738 100644 --- a/dstack/tee-simulator/Cargo.toml +++ b/dstack/tee-simulator/Cargo.toml @@ -21,6 +21,7 @@ dstack-types.workspace = true dstack-mr.workspace = true fuser.workspace = true libc.workspace = true +nix = { workspace = true, features = ["mount"] } sd-notify.workspace = true sha2.workspace = true tracing.workspace = true diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs index 2f5746a04..bc142ef9f 100644 --- a/dstack/tee-simulator/src/tpm.rs +++ b/dstack/tee-simulator/src/tpm.rs @@ -6,7 +6,6 @@ //! template and certificate NV indices consumed by `tpm-attest`. use std::{ - ffi::CString, io::{Read, Write}, os::{ fd::{AsRawFd, FromRawFd}, @@ -225,23 +224,17 @@ fn install_fixture_event_log() -> Result<()> { // securityfs does not permit userspace to create a synthetic TPM event // log hierarchy. Shadow it in this development-only guest before // publishing the fixture that was replayed into the simulated PCRs. - let source = CString::new("dstack-tee-simulator")?; - let target = CString::new("/sys/kernel/security")?; - let fstype = CString::new("tmpfs")?; - let data = CString::new("mode=0755")?; - let rc = unsafe { - libc::mount( - source.as_ptr(), - target.as_ptr(), - fstype.as_ptr(), - libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC, - data.as_ptr().cast(), - ) - }; - if rc != 0 { - return Err(std::io::Error::last_os_error()) - .context("failed to mount simulated securityfs shadow"); - } + let flags = nix::mount::MsFlags::MS_NOSUID + | nix::mount::MsFlags::MS_NODEV + | nix::mount::MsFlags::MS_NOEXEC; + nix::mount::mount( + Some("dstack-tee-simulator"), + security_root, + Some("tmpfs"), + flags, + Some("mode=0755"), + ) + .context("failed to mount simulated securityfs shadow")?; fs_err::create_dir_all(tpm_dir) .context("failed to create TPM event-log directory in securityfs shadow")?; fs_err::write( From ba398bde0ba02ed61f0f88e153e8cda5e2d174b4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 31 Jul 2026 11:02:14 +0000 Subject: [PATCH 18/30] build(simulator): record safe mount dependency --- dstack/Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index a0167f769..207388a0c 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2161,6 +2161,7 @@ dependencies = [ "libc", "libloading", "mock-attestation", + "nix 0.29.0", "nsm-qvl", "pem", "reqwest", From 16ff3c6b3491afb7d233881f775384e583c1a182 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 31 Jul 2026 11:03:02 +0000 Subject: [PATCH 19/30] refactor(simulator): use safe descriptor flag wrappers --- dstack/tee-simulator/Cargo.toml | 2 +- dstack/tee-simulator/src/tpm.rs | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/dstack/tee-simulator/Cargo.toml b/dstack/tee-simulator/Cargo.toml index 18ed41738..07411d488 100644 --- a/dstack/tee-simulator/Cargo.toml +++ b/dstack/tee-simulator/Cargo.toml @@ -21,7 +21,7 @@ dstack-types.workspace = true dstack-mr.workspace = true fuser.workspace = true libc.workspace = true -nix = { workspace = true, features = ["mount"] } +nix = { workspace = true, features = ["fs", "mount"] } sd-notify.workspace = true sha2.workspace = true tracing.workspace = true diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs index bc142ef9f..45dc4559d 100644 --- a/dstack/tee-simulator/src/tpm.rs +++ b/dstack/tee-simulator/src/tpm.rs @@ -293,12 +293,15 @@ pub fn run_nitro_vtpm(runtime_dir: &Path, config: &TeeSimulatorConfig) -> Result let generator = NsmGenerator::from_seed(parse_seed(seed)?)?; let (mut simulator, swtpm_stream) = UnixStream::pair()?; let swtpm_fd = swtpm_stream.as_raw_fd(); - let flags = unsafe { libc::fcntl(swtpm_fd, libc::F_GETFD) }; - anyhow::ensure!(flags >= 0, "failed to get swtpm socket flags"); - anyhow::ensure!( - unsafe { libc::fcntl(swtpm_fd, libc::F_SETFD, flags & !libc::FD_CLOEXEC) } >= 0, - "failed to make swtpm socket inheritable" + let flags = nix::fcntl::FdFlag::from_bits_truncate( + nix::fcntl::fcntl(swtpm_fd, nix::fcntl::FcntlArg::F_GETFD) + .context("failed to get swtpm socket flags")?, ); + nix::fcntl::fcntl( + swtpm_fd, + nix::fcntl::FcntlArg::F_SETFD(flags - nix::fcntl::FdFlag::FD_CLOEXEC), + ) + .context("failed to make swtpm socket inheritable")?; let state_dir = std::env::temp_dir().join(format!("dstack-nitro-swtpm-{}", std::process::id())); fs_err::create_dir_all(&state_dir)?; From 40ec54ba888f55a9c1e8af3d8596ce837d67802e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 31 Jul 2026 12:07:06 +0000 Subject: [PATCH 20/30] refactor(tpm2): support stream transports --- dstack/tpm2/src/commands.rs | 8 +++++++ dstack/tpm2/src/device.rs | 45 ++++++++++++++++++++++++++----------- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/dstack/tpm2/src/commands.rs b/dstack/tpm2/src/commands.rs index a64dc3a49..85f266a65 100644 --- a/dstack/tpm2/src/commands.rs +++ b/dstack/tpm2/src/commands.rs @@ -8,6 +8,7 @@ use anyhow::{Context, Result}; use std::collections::HashSet; +use std::io::{Read, Write}; use tracing::debug; use super::constants::*; @@ -32,6 +33,13 @@ impl TpmContext { Ok(Self { device }) } + /// Create a TPM context over an already connected byte stream. + pub fn from_stream(stream: impl Read + Write + 'static, name: impl Into) -> Self { + Self { + device: TpmDevice::from_stream(stream, name), + } + } + /// Get the device path pub fn device_path(&self) -> &str { self.device.path() diff --git a/dstack/tpm2/src/device.rs b/dstack/tpm2/src/device.rs index c6cce8a63..2d5163032 100644 --- a/dstack/tpm2/src/device.rs +++ b/dstack/tpm2/src/device.rs @@ -7,7 +7,7 @@ //! Provides low-level communication with TPM devices via /dev/tpmrm0 or /dev/tpm0. use anyhow::{bail, Context, Result}; -use std::fs::{File, OpenOptions}; +use std::fs::OpenOptions; use std::io::{Read, Write}; use std::path::Path; @@ -18,8 +18,12 @@ use super::marshal::*; const TPM_MAX_COMMAND_SIZE: usize = 4096; /// TPM device handle +trait ReadWrite: Read + Write {} + +impl ReadWrite for T {} + pub struct TpmDevice { - file: File, + transport: Box, path: String, } @@ -36,11 +40,19 @@ impl TpmDevice { .with_context(|| format!("failed to open TPM device: {}", device_path))?; Ok(Self { - file, + transport: Box::new(file), path: device_path.to_string(), }) } + /// Create a TPM device over an already connected byte stream. + pub fn from_stream(stream: impl Read + Write + 'static, name: impl Into) -> Self { + Self { + transport: Box::new(stream), + path: name.into(), + } + } + /// Detect and open the default TPM device pub fn detect() -> Result { if Path::new("/dev/tpmrm0").exists() { @@ -59,19 +71,26 @@ impl TpmDevice { /// Send a command to the TPM and receive the response pub fn transmit(&mut self, command: &[u8]) -> Result> { - // Write command - self.file + self.transport .write_all(command) .context("failed to write TPM command")?; - // Read response - let mut response = vec![0u8; TPM_MAX_COMMAND_SIZE]; - let n = self - .file - .read(&mut response) - .context("failed to read TPM response")?; - - response.truncate(n); + // Read the fixed header first so stream transports cannot return a + // partial response and leave bytes for the next transaction. + let mut header = [0u8; 10]; + self.transport + .read_exact(&mut header) + .context("failed to read TPM response header")?; + let response_size = u32::from_be_bytes(header[2..6].try_into().unwrap()) as usize; + if !(header.len()..=TPM_MAX_COMMAND_SIZE).contains(&response_size) { + bail!("invalid TPM response size: {response_size}"); + } + let mut response = Vec::with_capacity(response_size); + response.extend_from_slice(&header); + response.resize(response_size, 0); + self.transport + .read_exact(&mut response[header.len()..]) + .context("failed to read TPM response body")?; Ok(response) } From e8e3c0e0bf806c538d9ce310fe05f23c01acd50e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 31 Jul 2026 12:08:19 +0000 Subject: [PATCH 21/30] feat(tpm2): edit command capability responses --- dstack/tpm2/src/device.rs | 77 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/dstack/tpm2/src/device.rs b/dstack/tpm2/src/device.rs index 2d5163032..9dd2c244a 100644 --- a/dstack/tpm2/src/device.rs +++ b/dstack/tpm2/src/device.rs @@ -294,6 +294,17 @@ impl TpmResponse { } } + /// Serialize this response back to TPM wire format. + pub fn to_bytes(&self) -> Vec { + let size = 10 + self.data.len(); + let mut response = Vec::with_capacity(size); + response.extend_from_slice(&self.tag.to_u16().to_be_bytes()); + response.extend_from_slice(&(size as u32).to_be_bytes()); + response.extend_from_slice(&self.response_code.to_be_bytes()); + response.extend_from_slice(&self.data); + response + } + /// Get a response buffer for parsing the data pub fn data_buffer(&self) -> ResponseBuffer<'_> { ResponseBuffer::new(&self.data) @@ -309,10 +320,76 @@ impl TpmResponse { } } +/// Add a command to a successful TPM_CAP_COMMANDS response. +/// +/// Responses for other capabilities, TPM errors, and paginated command lists +/// are returned unchanged. +pub fn add_command_capability(response: &[u8], command_code: u32) -> Result> { + let mut response = TpmResponse::parse(response)?; + if !response.is_success() { + return Ok(response.to_bytes()); + } + let mut buf = response.data_buffer(); + let more_data = buf.get_u8()? != 0; + let capability = buf.get_u32()?; + if capability != TpmCap::Commands as u32 || more_data { + return Ok(response.to_bytes()); + } + let count = buf.get_u32()? as usize; + let mut attributes = Vec::with_capacity(count + 1); + for _ in 0..count { + attributes.push(buf.get_u32()?); + } + if attributes.contains(&command_code) { + return Ok(response.to_bytes()); + } + const COMMAND_INDEX_AND_VENDOR_MASK: u32 = 0x2000_ffff; + let sort_key = command_code & COMMAND_INDEX_AND_VENDOR_MASK; + let insertion = attributes + .iter() + .position(|attributes| attributes & COMMAND_INDEX_AND_VENDOR_MASK > sort_key) + .unwrap_or(attributes.len()); + attributes.insert(insertion, command_code); + + let mut data = Vec::with_capacity(9 + attributes.len() * 4); + data.push(u8::from(more_data)); + data.extend_from_slice(&(TpmCap::Commands as u32).to_be_bytes()); + data.extend_from_slice(&(attributes.len() as u32).to_be_bytes()); + for attributes in attributes { + data.extend_from_slice(&attributes.to_be_bytes()); + } + response.data = data; + Ok(response.to_bytes()) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn adds_a_vendor_command_to_command_capabilities() { + let response = TpmResponse { + tag: TpmSt::NoSessions, + response_code: 0, + data: [ + &[0], + &(TpmCap::Commands as u32).to_be_bytes(), + &1u32.to_be_bytes(), + &0x2000_1000u32.to_be_bytes(), + ] + .concat(), + } + .to_bytes(); + let response = add_command_capability(&response, 0x2000_0001).unwrap(); + let response = TpmResponse::parse(&response).unwrap(); + let mut data = response.data_buffer(); + assert_eq!(data.get_u8().unwrap(), 0); + assert_eq!(data.get_u32().unwrap(), TpmCap::Commands as u32); + assert_eq!(data.get_u32().unwrap(), 2); + assert_eq!(data.get_u32().unwrap(), 0x2000_0001); + assert_eq!(data.get_u32().unwrap(), 0x2000_1000); + } + #[test] fn test_command_builder() { let mut cmd = TpmCommand::new(TpmCc::GetRandom); From 9af0419cdf6fb62b20fd0dea95eae18e5f18701b Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 31 Jul 2026 12:09:43 +0000 Subject: [PATCH 22/30] refactor(simulator): use typed TPM APIs --- dstack/Cargo.lock | 1 + dstack/tee-simulator/Cargo.toml | 1 + dstack/tee-simulator/src/tpm.rs | 118 ++++---------------------------- dstack/tpm2/src/device.rs | 16 ++--- dstack/tpm2/src/lib.rs | 2 +- 5 files changed, 26 insertions(+), 112 deletions(-) diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index a0167f769..5b507b518 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2173,6 +2173,7 @@ dependencies = [ "tokio", "tpm-qvl", "tpm-types", + "tpm2", "tracing", "tracing-subscriber", ] diff --git a/dstack/tee-simulator/Cargo.toml b/dstack/tee-simulator/Cargo.toml index b235d5467..aaea193b1 100644 --- a/dstack/tee-simulator/Cargo.toml +++ b/dstack/tee-simulator/Cargo.toml @@ -23,6 +23,7 @@ fuser.workspace = true libc.workspace = true sd-notify.workspace = true sha2.workspace = true +tpm2.workspace = true tracing.workspace = true tracing-subscriber.workspace = true serde_json.workspace = true diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs index e127703d5..ca6e43c80 100644 --- a/dstack/tee-simulator/src/tpm.rs +++ b/dstack/tee-simulator/src/tpm.rs @@ -17,20 +17,15 @@ use std::{ time::Duration, }; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result, bail}; use aws_nitro_enclaves_nsm_api::api::{Request as NsmRequest, Response as NsmResponse}; use dstack_types::TeeSimulatorConfig; use mock_attestation::{nsm::NsmGenerator, parse_seed, server::MockCollateralState}; +use tpm2::{TpmAlgId, TpmCc, TpmContext, add_command_capability}; const AK_ECC_CERT: &str = "0x01c10002"; const AK_ECC_TEMPLATE: &str = "0x01c10003"; -const TPM2_CC_NV_WRITE: u32 = 0x0000_0137; -const TPM2_CC_NV_DEFINE_SPACE: u32 = 0x0000_012a; -const TPM2_CC_NV_READ: u32 = 0x0000_014e; -const TPM2_CC_NV_READ_PUBLIC: u32 = 0x0000_0169; -const TPM2_CC_GET_CAPABILITY: u32 = 0x0000_017a; const TPM2_CC_AWS_NSM_REQUEST: u32 = 0x2000_0001; -const TPM2_CAP_COMMANDS: u32 = 2; const VTPM_PROXY_IOC_NEW_DEV: libc::c_ulong = 0xc014_a100; const VTPM_PROXY_FLAG_TPM2: u32 = 1; @@ -379,12 +374,12 @@ fn proxy_tpm_commands( .context("NitroTPM vendor command without an NV request")?; nsm_response = Some(handle_nsm_vendor_command(generator, template, backend)?); tpm_success_response() - } else if code == TPM2_CC_NV_READ && nsm_response.is_some() { + } else if code == TpmCc::NvRead.to_u32() && nsm_response.is_some() { nv_read_response( &command, nsm_response.as_deref().context("missing NSM response")?, )? - } else if code == TPM2_CC_NV_READ_PUBLIC && nsm_response.is_some() { + } else if code == TpmCc::NvReadPublic.to_u32() && nsm_response.is_some() { let mut response = transact(backend, &command)?; set_nv_public_size( &mut response, @@ -399,11 +394,11 @@ fn proxy_tpm_commands( // single NV index at 2 KiB. Define the backing index at that limit; // the proxy virtualizes its public size and reads after the vendor // command, while swtpm still handles its lifecycle and auth setup. - if code == TPM2_CC_NV_DEFINE_SPACE && command.ends_with(&8192u16.to_be_bytes()) { + if code == TpmCc::NvDefineSpace.to_u32() && command.ends_with(&8192u16.to_be_bytes()) { let end = command.len(); command[end - 2..].copy_from_slice(&2048u16.to_be_bytes()); } - if code == TPM2_CC_NV_WRITE { + if code == TpmCc::NvWrite.to_u32() { if let Some(template) = parse_nv_write(&command)? { nv_write = Some(template); } @@ -419,39 +414,9 @@ fn proxy_tpm_commands( } fn advertise_nsm_vendor_command(code: u32, response: &mut Vec) -> Result<()> { - if code != TPM2_CC_GET_CAPABILITY || response.len() < 19 { - return Ok(()); - } - let response_code = read_be_u32(&response[6..10], "TPM response code")?; - let capability = read_be_u32(&response[11..15], "TPM capability")?; - if response_code != 0 || capability != TPM2_CAP_COMMANDS || response[10] != 0 { - return Ok(()); - } - let count = read_be_u32(&response[15..19], "TPM command attribute count")? as usize; - let attributes_end = 19usize - .checked_add(count.checked_mul(4).context("TPM command count overflow")?) - .context("TPM command attributes overflow")?; - anyhow::ensure!( - response.len() >= attributes_end, - "truncated TPM command attributes" - ); - if response[19..attributes_end].chunks_exact(4).any(|value| { - read_be_u32(value, "TPM command attributes") - .is_ok_and(|attributes| attributes == TPM2_CC_AWS_NSM_REQUEST) - }) { - return Ok(()); + if code == TpmCc::GetCapability.to_u32() { + *response = add_command_capability(response, TPM2_CC_AWS_NSM_REQUEST)?; } - let insertion = response[19..attributes_end] - .chunks_exact(4) - .position(|value| { - read_be_u32(value, "TPM command attributes") - .is_ok_and(|attributes| attributes & 0x2000_ffff > TPM2_CC_AWS_NSM_REQUEST) - }) - .map_or(attributes_end, |index| 19 + index * 4); - response.splice(insertion..insertion, TPM2_CC_AWS_NSM_REQUEST.to_be_bytes()); - response[15..19].copy_from_slice(&((count + 1) as u32).to_be_bytes()); - let response_size = response.len() as u32; - response[2..6].copy_from_slice(&response_size.to_be_bytes()); Ok(()) } @@ -488,9 +453,15 @@ fn handle_nsm_vendor_command( let request: NsmRequest = serde_cbor::from_slice(&template.request)?; let response = match request { NsmRequest::Attestation { user_data, .. } => { + let mut tpm = TpmContext::from_stream( + backend + .try_clone() + .context("failed to clone NitroTPM stream")?, + "NitroTPM backend", + ); let pcrs = [4u16, 7, 8, 12, 14] .into_iter() - .map(|index| Ok((index, read_sha384_pcr(backend, index)?))) + .map(|index| Ok((index, tpm.pcr_read_single(index.into(), TpmAlgId::Sha384)?))) .collect::>()?; let document = generator.attest_with_pcrs( user_data.as_ref().map(|v| v.as_slice()).unwrap_or_default(), @@ -503,35 +474,6 @@ fn handle_nsm_vendor_command( Ok(serde_cbor::to_vec(&response)?) } -fn read_sha384_pcr(backend: &mut UnixStream, index: u16) -> Result> { - anyhow::ensure!(index < 24, "invalid PCR index {index}"); - let mut command = Vec::with_capacity(20); - command.extend_from_slice(&0x8001u16.to_be_bytes()); - command.extend_from_slice(&20u32.to_be_bytes()); - command.extend_from_slice(&0x0000_017eu32.to_be_bytes()); - command.extend_from_slice(&1u32.to_be_bytes()); - command.extend_from_slice(&0x000cu16.to_be_bytes()); - command.push(3); - let mut selection = [0u8; 3]; - selection[index as usize / 8] = 1 << (index % 8); - command.extend_from_slice(&selection); - - let response = transact(backend, &command)?; - anyhow::ensure!(response.len() >= 30, "truncated TPM PCR_Read response"); - anyhow::ensure!( - read_be_u32(&response[6..10], "TPM PCR_Read response code")? == 0, - "TPM PCR_Read failed" - ); - anyhow::ensure!( - read_be_u32(&response[24..28], "TPM PCR digest count")? == 1, - "unexpected TPM PCR digest count" - ); - let size = read_be_u16(&response[28..30], "TPM PCR digest size")? as usize; - anyhow::ensure!(size == 48, "unexpected SHA-384 PCR size {size}"); - anyhow::ensure!(response.len() >= 30 + size, "truncated TPM PCR digest"); - Ok(response[30..30 + size].to_vec()) -} - fn set_nv_public_size(response: &mut [u8], size: usize) -> Result<()> { anyhow::ensure!(response.len() >= 26, "truncated NV_ReadPublic response"); let policy_size_pos = 22; @@ -587,33 +529,3 @@ fn transact(stream: &mut UnixStream, command: &[u8]) -> Result> { fn tpm_success_response() -> Vec { [0x80, 0x01, 0, 0, 0, 10, 0, 0, 0, 0].to_vec() } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn get_capability_advertises_nsm_vendor_command() { - let mut response = Vec::new(); - response.extend_from_slice(&0x8001u16.to_be_bytes()); - response.extend_from_slice(&23u32.to_be_bytes()); - response.extend_from_slice(&0u32.to_be_bytes()); - response.push(0); - response.extend_from_slice(&TPM2_CAP_COMMANDS.to_be_bytes()); - response.extend_from_slice(&1u32.to_be_bytes()); - response.extend_from_slice(&0x2000_1000u32.to_be_bytes()); - - advertise_nsm_vendor_command(TPM2_CC_GET_CAPABILITY, &mut response).unwrap(); - - assert_eq!(read_be_u32(&response[2..6], "size").unwrap(), 27); - assert_eq!(read_be_u32(&response[15..19], "count").unwrap(), 2); - assert_eq!( - read_be_u32(&response[19..23], "vendor command").unwrap(), - TPM2_CC_AWS_NSM_REQUEST - ); - assert_eq!( - read_be_u32(&response[23..27], "existing command").unwrap(), - 0x2000_1000 - ); - } -} diff --git a/dstack/tpm2/src/device.rs b/dstack/tpm2/src/device.rs index 9dd2c244a..d35765cfa 100644 --- a/dstack/tpm2/src/device.rs +++ b/dstack/tpm2/src/device.rs @@ -6,7 +6,7 @@ //! //! Provides low-level communication with TPM devices via /dev/tpmrm0 or /dev/tpm0. -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result, bail}; use std::fs::OpenOptions; use std::io::{Read, Write}; use std::path::Path; @@ -371,13 +371,13 @@ mod tests { let response = TpmResponse { tag: TpmSt::NoSessions, response_code: 0, - data: [ - &[0], - &(TpmCap::Commands as u32).to_be_bytes(), - &1u32.to_be_bytes(), - &0x2000_1000u32.to_be_bytes(), - ] - .concat(), + data: { + let mut data = vec![0]; + data.extend_from_slice(&(TpmCap::Commands as u32).to_be_bytes()); + data.extend_from_slice(&1u32.to_be_bytes()); + data.extend_from_slice(&0x2000_1000u32.to_be_bytes()); + data + }, } .to_bytes(); let response = add_command_capability(&response, 0x2000_0001).unwrap(); diff --git a/dstack/tpm2/src/lib.rs b/dstack/tpm2/src/lib.rs index ab5db337e..0b1d0d66f 100644 --- a/dstack/tpm2/src/lib.rs +++ b/dstack/tpm2/src/lib.rs @@ -44,6 +44,6 @@ pub use constants::*; pub use types::*; // Re-export device for advanced usage -pub use device::{TpmCommand, TpmDevice, TpmResponse}; +pub use device::{TpmCommand, TpmDevice, TpmResponse, add_command_capability}; pub use marshal::{CommandBuffer, Marshal, ResponseBuffer, Unmarshal}; pub use session::AuthSession; From dfee3797387cca130f7cca618612e8ac3fbe1b6f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 31 Jul 2026 12:10:29 +0000 Subject: [PATCH 23/30] fix(tpm2): preserve device response reads --- dstack/tee-simulator/src/tpm.rs | 4 +- dstack/tpm2/src/device.rs | 66 ++++++++++++++++++++------------- dstack/tpm2/src/lib.rs | 2 +- 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/dstack/tee-simulator/src/tpm.rs b/dstack/tee-simulator/src/tpm.rs index ca6e43c80..62d28876b 100644 --- a/dstack/tee-simulator/src/tpm.rs +++ b/dstack/tee-simulator/src/tpm.rs @@ -17,11 +17,11 @@ use std::{ time::Duration, }; -use anyhow::{Context, Result, bail}; +use anyhow::{bail, Context, Result}; use aws_nitro_enclaves_nsm_api::api::{Request as NsmRequest, Response as NsmResponse}; use dstack_types::TeeSimulatorConfig; use mock_attestation::{nsm::NsmGenerator, parse_seed, server::MockCollateralState}; -use tpm2::{TpmAlgId, TpmCc, TpmContext, add_command_capability}; +use tpm2::{add_command_capability, TpmAlgId, TpmCc, TpmContext}; const AK_ECC_CERT: &str = "0x01c10002"; const AK_ECC_TEMPLATE: &str = "0x01c10003"; diff --git a/dstack/tpm2/src/device.rs b/dstack/tpm2/src/device.rs index d35765cfa..024bab5fa 100644 --- a/dstack/tpm2/src/device.rs +++ b/dstack/tpm2/src/device.rs @@ -7,7 +7,7 @@ //! Provides low-level communication with TPM devices via /dev/tpmrm0 or /dev/tpm0. use anyhow::{Context, Result, bail}; -use std::fs::OpenOptions; +use std::fs::{File, OpenOptions}; use std::io::{Read, Write}; use std::path::Path; @@ -22,8 +22,13 @@ trait ReadWrite: Read + Write {} impl ReadWrite for T {} +enum TpmTransport { + Device(File), + Stream(Box), +} + pub struct TpmDevice { - transport: Box, + transport: TpmTransport, path: String, } @@ -40,7 +45,7 @@ impl TpmDevice { .with_context(|| format!("failed to open TPM device: {}", device_path))?; Ok(Self { - transport: Box::new(file), + transport: TpmTransport::Device(file), path: device_path.to_string(), }) } @@ -48,7 +53,7 @@ impl TpmDevice { /// Create a TPM device over an already connected byte stream. pub fn from_stream(stream: impl Read + Write + 'static, name: impl Into) -> Self { Self { - transport: Box::new(stream), + transport: TpmTransport::Stream(Box::new(stream)), path: name.into(), } } @@ -69,29 +74,40 @@ impl TpmDevice { &self.path } - /// Send a command to the TPM and receive the response + /// Send a command to the TPM and receive the response. pub fn transmit(&mut self, command: &[u8]) -> Result> { - self.transport - .write_all(command) - .context("failed to write TPM command")?; - - // Read the fixed header first so stream transports cannot return a - // partial response and leave bytes for the next transaction. - let mut header = [0u8; 10]; - self.transport - .read_exact(&mut header) - .context("failed to read TPM response header")?; - let response_size = u32::from_be_bytes(header[2..6].try_into().unwrap()) as usize; - if !(header.len()..=TPM_MAX_COMMAND_SIZE).contains(&response_size) { - bail!("invalid TPM response size: {response_size}"); + match &mut self.transport { + TpmTransport::Device(file) => { + file.write_all(command) + .context("failed to write TPM command")?; + let mut response = vec![0u8; TPM_MAX_COMMAND_SIZE]; + let size = file + .read(&mut response) + .context("failed to read TPM response")?; + response.truncate(size); + Ok(response) + } + TpmTransport::Stream(stream) => { + stream + .write_all(command) + .context("failed to write TPM command")?; + let mut header = [0u8; 10]; + stream + .read_exact(&mut header) + .context("failed to read TPM response header")?; + let response_size = u32::from_be_bytes(header[2..6].try_into().unwrap()) as usize; + if !(header.len()..=TPM_MAX_COMMAND_SIZE).contains(&response_size) { + bail!("invalid TPM response size: {response_size}"); + } + let mut response = Vec::with_capacity(response_size); + response.extend_from_slice(&header); + response.resize(response_size, 0); + stream + .read_exact(&mut response[header.len()..]) + .context("failed to read TPM response body")?; + Ok(response) + } } - let mut response = Vec::with_capacity(response_size); - response.extend_from_slice(&header); - response.resize(response_size, 0); - self.transport - .read_exact(&mut response[header.len()..]) - .context("failed to read TPM response body")?; - Ok(response) } /// Execute a TPM command and parse the response diff --git a/dstack/tpm2/src/lib.rs b/dstack/tpm2/src/lib.rs index 0b1d0d66f..ac670d9a1 100644 --- a/dstack/tpm2/src/lib.rs +++ b/dstack/tpm2/src/lib.rs @@ -44,6 +44,6 @@ pub use constants::*; pub use types::*; // Re-export device for advanced usage -pub use device::{TpmCommand, TpmDevice, TpmResponse, add_command_capability}; +pub use device::{add_command_capability, TpmCommand, TpmDevice, TpmResponse}; pub use marshal::{CommandBuffer, Marshal, ResponseBuffer, Unmarshal}; pub use session::AuthSession; From beee917f2847dcdfeeea7b9520b00f421c8556b4 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 31 Jul 2026 12:12:05 +0000 Subject: [PATCH 24/30] test(tpm2): cover stream response framing --- dstack/tpm2/src/device.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/dstack/tpm2/src/device.rs b/dstack/tpm2/src/device.rs index 024bab5fa..d12e80282 100644 --- a/dstack/tpm2/src/device.rs +++ b/dstack/tpm2/src/device.rs @@ -382,6 +382,34 @@ pub fn add_command_capability(response: &[u8], command_code: u32) -> Result Date: Fri, 31 Jul 2026 12:36:52 +0000 Subject: [PATCH 25/30] fix(tpm2): satisfy strict CI checks --- dstack/tpm2/src/device.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dstack/tpm2/src/device.rs b/dstack/tpm2/src/device.rs index d12e80282..32cea0aa2 100644 --- a/dstack/tpm2/src/device.rs +++ b/dstack/tpm2/src/device.rs @@ -6,7 +6,7 @@ //! //! Provides low-level communication with TPM devices via /dev/tpmrm0 or /dev/tpm0. -use anyhow::{Context, Result, bail}; +use anyhow::{bail, Context, Result}; use std::fs::{File, OpenOptions}; use std::io::{Read, Write}; use std::path::Path; @@ -95,7 +95,8 @@ impl TpmDevice { stream .read_exact(&mut header) .context("failed to read TPM response header")?; - let response_size = u32::from_be_bytes(header[2..6].try_into().unwrap()) as usize; + let response_size = + u32::from_be_bytes([header[2], header[3], header[4], header[5]]) as usize; if !(header.len()..=TPM_MAX_COMMAND_SIZE).contains(&response_size) { bail!("invalid TPM response size: {response_size}"); } From 47dc2d2a1768ee91de34ef25be8637109925d32d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 31 Jul 2026 13:34:57 +0000 Subject: [PATCH 26/30] refactor(vmm): select MR config version explicitly --- dstack/vmm/src/app.rs | 80 +++++++++++++++++---------------- dstack/vmm/src/app/mr_config.rs | 57 ++++++++++++++++++++++- dstack/vmm/src/one_shot.rs | 21 ++------- 3 files changed, 100 insertions(+), 58 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 0583a6ce2..dad8f74ef 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -283,20 +283,6 @@ pub(crate) enum PullStatus { Failed(String), } -fn needs_mr_config_v3( - manifest: &Manifest, - platform: crate::config::CvmPlatform, - use_mrconfigid: bool, - has_key_provider_id: bool, -) -> bool { - manifest.simulated_tee == Some(dstack_types::TeeVariant::DstackAmdSevSnp) - || (!manifest.no_tee - && (platform == crate::config::CvmPlatform::AmdSevSnp - || (platform == crate::config::CvmPlatform::Tdx - && use_mrconfigid - && has_key_provider_id))) -} - #[derive(Clone)] pub struct App { pub config: Arc, @@ -1097,28 +1083,12 @@ impl App { let manifest = work_dir.manifest().context("Failed to read manifest")?; let cfg = &self.config; let compose_hash = sha256_file(shared_dir.join(APP_COMPOSE))?; - let platform = cfg.cvm.resolved_platform(); let app_compose = work_dir .app_compose() .context("Failed to get app compose")?; - let use_mr_config_v3 = needs_mr_config_v3( - &manifest, - platform, - cfg.cvm.use_mrconfigid, - !app_compose.key_provider_id.is_empty(), - ); - let mr_config = if use_mr_config_v3 { - Some( - work_dir - .prepare_mr_config_v3( - &app_compose, - manifest.gpus.as_ref().is_some_and(GpuConfig::has_gpus), - ) - .context("Failed to prepare mr_config")?, - ) - } else { - None - }; + let mr_config = work_dir + .prepare_mr_config(&manifest, &cfg.cvm, &app_compose) + .context("Failed to prepare mr_config")?; let sys_config_str = make_sys_config( cfg, &manifest, @@ -1517,6 +1487,7 @@ pub(crate) fn needs_swtpm( #[cfg(test)] mod tests { + use super::mr_config::{mr_config_version, MrConfigVersion}; use super::*; use crate::config::{ load_config_figment, CvmPlatform, Networking, NetworkingMode, TdxAttestationVariantConfig, @@ -1758,11 +1729,44 @@ mod tests { } #[test] - fn simulated_sev_snp_requires_mr_config_even_without_tee() { - let mut manifest = test_manifest(2048); - manifest.no_tee = true; - manifest.simulated_tee = Some(dstack_types::TeeVariant::DstackAmdSevSnp); - assert!(needs_mr_config_v3(&manifest, CvmPlatform::Tdx, true, false)); + fn selects_mr_config_version_for_each_tee_mode() -> Result<()> { + let manifest = test_manifest(2048); + assert_eq!( + mr_config_version(&manifest, CvmPlatform::AmdSevSnp, false, false)?, + Some(MrConfigVersion::V3) + ); + assert_eq!( + mr_config_version(&manifest, CvmPlatform::Tdx, false, false)?, + None + ); + assert_eq!( + mr_config_version(&manifest, CvmPlatform::Tdx, true, false)?, + Some(MrConfigVersion::V1) + ); + assert_eq!( + mr_config_version(&manifest, CvmPlatform::Tdx, true, true)?, + Some(MrConfigVersion::V3) + ); + assert_eq!( + mr_config_version(&manifest, CvmPlatform::Tdx, false, true) + .err() + .map(|error| error.to_string()), + Some("key provider ID requires MrConfigV3, but use_mrconfigid is disabled".to_string()) + ); + + let mut no_tee = manifest.clone(); + no_tee.no_tee = true; + assert_eq!( + mr_config_version(&no_tee, CvmPlatform::AmdSevSnp, true, true)?, + None + ); + + no_tee.simulated_tee = Some(dstack_types::TeeVariant::DstackAmdSevSnp); + assert_eq!( + mr_config_version(&no_tee, CvmPlatform::Tdx, false, false)?, + Some(MrConfigVersion::V3) + ); + Ok(()) } fn dummy_tdx_measurement_document() -> TdxOsImageMeasurementDocument { diff --git a/dstack/vmm/src/app/mr_config.rs b/dstack/vmm/src/app/mr_config.rs index 434d8a1d1..d607255d0 100644 --- a/dstack/vmm/src/app/mr_config.rs +++ b/dstack/vmm/src/app/mr_config.rs @@ -11,7 +11,37 @@ use dstack_types::{gpu_policy_hash, AppCompose}; use fs_err as fs; use sha2::{Digest, Sha256}; -use super::VmWorkDir; +use super::{GpuConfig, Manifest, VmWorkDir}; +use crate::config::{CvmConfig, CvmPlatform}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum MrConfigVersion { + V1, + V3, +} + +pub(super) fn mr_config_version( + manifest: &Manifest, + platform: CvmPlatform, + use_mrconfigid: bool, + has_key_provider_id: bool, +) -> Result> { + if manifest.simulated_tee == Some(dstack_types::TeeVariant::DstackAmdSevSnp) { + return Ok(Some(MrConfigVersion::V3)); + } + if manifest.no_tee { + return Ok(None); + } + match platform { + CvmPlatform::AmdSevSnp => Ok(Some(MrConfigVersion::V3)), + CvmPlatform::Tdx if has_key_provider_id && !use_mrconfigid => { + bail!("key provider ID requires MrConfigV3, but use_mrconfigid is disabled") + } + CvmPlatform::Tdx if !use_mrconfigid => Ok(None), + CvmPlatform::Tdx if has_key_provider_id => Ok(Some(MrConfigVersion::V3)), + CvmPlatform::Tdx => Ok(Some(MrConfigVersion::V1)), + } +} pub(super) fn tdx_mr_config_id(workdir: &VmWorkDir, app_compose: &AppCompose) -> Result { if let Some(document) = workdir @@ -60,7 +90,30 @@ pub(super) fn snp_host_data(workdir: &VmWorkDir) -> Result { } impl VmWorkDir { - pub fn prepare_mr_config_v3(&self, app_compose: &AppCompose, has_gpus: bool) -> Result { + pub(crate) fn prepare_mr_config( + &self, + manifest: &Manifest, + config: &CvmConfig, + app_compose: &AppCompose, + ) -> Result> { + let version = mr_config_version( + manifest, + config.resolved_platform(), + config.use_mrconfigid, + !app_compose.key_provider_id.is_empty(), + )?; + match version { + Some(MrConfigVersion::V3) => self + .prepare_mr_config_v3( + app_compose, + manifest.gpus.as_ref().is_some_and(GpuConfig::has_gpus), + ) + .map(Some), + Some(MrConfigVersion::V1) | None => Ok(None), + } + } + + fn prepare_mr_config_v3(&self, app_compose: &AppCompose, has_gpus: bool) -> Result { let compose_hash = self .app_compose_hash() .context("failed to get compose hash")?; diff --git a/dstack/vmm/src/one_shot.rs b/dstack/vmm/src/one_shot.rs index ecbf9b83f..f3698681a 100644 --- a/dstack/vmm/src/one_shot.rs +++ b/dstack/vmm/src/one_shot.rs @@ -242,24 +242,9 @@ Compose file content (first 200 chars): let app_compose = vm_work_dir .app_compose() .context("Failed to get app compose")?; - let platform = config.cvm.resolved_platform(); - let use_mr_config_v3 = !manifest.no_tee - && (platform == crate::config::CvmPlatform::AmdSevSnp - || (platform == crate::config::CvmPlatform::Tdx - && config.cvm.use_mrconfigid - && !app_compose.key_provider_id.is_empty())); - let mr_config = if use_mr_config_v3 { - Some( - vm_work_dir - .prepare_mr_config_v3( - &app_compose, - manifest.gpus.as_ref().is_some_and(|gpus| gpus.has_gpus()), - ) - .context("Failed to prepare mr_config")?, - ) - } else { - None - }; + let mr_config = vm_work_dir + .prepare_mr_config(&manifest, &config.cvm, &app_compose) + .context("Failed to prepare mr_config")?; let sys_config_str = make_sys_config( &config, &manifest, From 6a71d79e29e8b688227f51f85eaf45bfd8d18593 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 3 Aug 2026 08:18:39 +0000 Subject: [PATCH 27/30] feat(attestation): allow unbound V3 app identity --- dstack/dstack-mr/src/sev.rs | 4 ++- dstack/dstack-types/src/mr_config.rs | 14 ++++++++- .../src/system_setup/config_id_verifier.rs | 31 ++++++++++++++++++- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/dstack/dstack-mr/src/sev.rs b/dstack/dstack-mr/src/sev.rs index 34dd2ab53..03ac24164 100644 --- a/dstack/dstack-mr/src/sev.rs +++ b/dstack/dstack-mr/src/sev.rs @@ -911,7 +911,9 @@ pub fn validate_mr_config(mr_config: &MrConfigV3) -> Result<()> { if mr_config.version != 3 { bail!("mr_config version must be 3"); } - ensure_len("mr_config.app_id", &mr_config.app_id, 20)?; + if !mr_config.app_id.is_empty() { + ensure_len("mr_config.app_id", &mr_config.app_id, 20)?; + } ensure_len("mr_config.compose_hash", &mr_config.compose_hash, 32)?; if let Some(gpu_policy_hash) = &mr_config.gpu_policy_hash { ensure_len("mr_config.gpu_policy_hash", gpu_policy_hash, 32)?; diff --git a/dstack/dstack-types/src/mr_config.rs b/dstack/dstack-types/src/mr_config.rs index 846f4f746..0de9b9f22 100644 --- a/dstack/dstack-types/src/mr_config.rs +++ b/dstack/dstack-types/src/mr_config.rs @@ -105,7 +105,8 @@ impl From for MrConfigDocumentError { pub struct MrConfigV3 { #[serde(default = "mr_config_v3_version")] pub version: u8, - #[serde(with = "hex_bytes")] + /// Optional application identity pin. An empty value leaves app_id unbound. + #[serde(default, with = "hex_bytes")] pub app_id: Vec, #[serde(with = "hex_bytes")] pub compose_hash: Vec, @@ -235,6 +236,17 @@ mod tests { Ok(()) } + #[test] + fn mr_config_v3_defaults_missing_app_id_to_empty() -> Result<(), Box> { + let config = MrConfigV3::from_document( + r#"{"compose_hash":"2222222222222222222222222222222222222222222222222222222222222222","key_provider":"none"}"#, + )?; + + assert!(config.app_id.is_empty()); + assert!(config.to_canonical_json().contains(r#""app_id":"""#)); + Ok(()) + } + #[test] fn mr_config_v3_generates_jcs_but_hashes_document_bytes() -> Result<(), Box> { let config = MrConfigV3::new( diff --git a/dstack/dstack-util/src/system_setup/config_id_verifier.rs b/dstack/dstack-util/src/system_setup/config_id_verifier.rs index 72cf28755..12ea01137 100644 --- a/dstack/dstack-util/src/system_setup/config_id_verifier.rs +++ b/dstack/dstack-util/src/system_setup/config_id_verifier.rs @@ -168,7 +168,7 @@ fn verify_mr_config_v3_document( bail!("Invalid mr_config gpu_policy_hash"); } } - if mr_config.app_id.as_slice() != local.app_id { + if !mr_config.app_id.is_empty() && mr_config.app_id.as_slice() != local.app_id { bail!("Invalid mr_config app_id"); } if mr_config.instance_id.as_slice() != local.instance_id { @@ -256,6 +256,35 @@ mod tests { } } + #[test] + fn mr_config_v3_skips_app_id_check_when_field_is_missing() -> Result<()> { + let compose_hash = [0x22u8; 32]; + let gpu_policy_hash = [0x55u8; 32]; + let app_id = [0x11u8; 20]; + let instance_id = [0x44u8; 20]; + let key_provider_id = [0x33u8; 32]; + let document = MrConfigV3::new( + Vec::new(), + compose_hash.to_vec(), + Some(gpu_policy_hash.to_vec()), + KeyProviderKind::Kms, + key_provider_id.to_vec(), + instance_id.to_vec(), + ) + .to_canonical_json(); + let local = LocalMrConfigValues { + compose_hash: &compose_hash, + gpu_policy_hash: &gpu_policy_hash, + app_id: &app_id, + instance_id: &instance_id, + key_provider: KeyProviderKind::Kms, + key_provider_id: &key_provider_id, + }; + + verify_mr_config_v3_document(&document, local)?; + Ok(()) + } + #[test] fn mr_config_v3_document_must_match_expected_gpu_policy_hash() { let compose_hash = [0x22u8; 32]; From 7bf644ad6557932ab7a7de03d569b037d28a3f4f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 3 Aug 2026 09:49:49 +0000 Subject: [PATCH 28/30] refactor(attestation): omit unset MR config fields --- dstack/Cargo.lock | 14 +++++++++++ dstack/Cargo.toml | 1 + dstack/dstack-attest/src/attestation.rs | 6 ++--- dstack/dstack-attest/tests/sev_snp_verify.rs | 2 +- dstack/dstack-mr/src/sev.rs | 8 +++---- dstack/dstack-types/Cargo.toml | 1 + dstack/dstack-types/src/mr_config.rs | 24 ++++++++++--------- .../src/system_setup/config_id_verifier.rs | 18 +++++++++----- dstack/kms/src/main_service/amd_attest.rs | 6 ++--- dstack/vmm/src/app.rs | 2 +- 10 files changed, 53 insertions(+), 29 deletions(-) diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index a0167f769..63b05e3d9 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2189,6 +2189,7 @@ dependencies = [ "serde-human-bytes", "serde_jcs", "serde_json", + "serde_with", "sha2 0.10.9", "sha3", "size-parser", @@ -6945,9 +6946,22 @@ dependencies = [ "schemars 1.2.1", "serde_core", "serde_json", + "serde_with_macros", "time", ] +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" diff --git a/dstack/Cargo.toml b/dstack/Cargo.toml index 848809b03..f32315326 100644 --- a/dstack/Cargo.toml +++ b/dstack/Cargo.toml @@ -170,6 +170,7 @@ scale = { version = "3.7.4", package = "parity-scale-codec", features = [ ] } serde = { version = "1.0.228", features = ["derive"], default-features = false } serde-human-bytes = "0.1.2" +serde_with = "3.14.0" semver = "1.0.28" serde_jcs = "0.2.0" rmp-serde = "1.3.1" diff --git a/dstack/dstack-attest/src/attestation.rs b/dstack/dstack-attest/src/attestation.rs index a3b5b323c..8c78d1867 100644 --- a/dstack/dstack-attest/src/attestation.rs +++ b/dstack/dstack-attest/src/attestation.rs @@ -1547,7 +1547,7 @@ struct Mrs { fn key_provider_info_from_mr_config(mr_config: &MrConfigV3) -> Result> { serde_json::to_vec(&KeyProviderInfo::new( mr_config.key_provider_name().to_string(), - hex::encode(&mr_config.key_provider_id), + hex::encode(mr_config.key_provider_id.as_deref().unwrap_or_default()), )) .context("Failed to serialize key provider info") } @@ -1607,8 +1607,8 @@ fn decode_app_info_sev_snp( let mrs = decode_mr_sev_snp(&parsed.measurement, &parsed.host_data); Ok(AppInfo { - app_id: mr_config.app_id, - instance_id: mr_config.instance_id, + app_id: mr_config.app_id.unwrap_or_default(), + instance_id: mr_config.instance_id.unwrap_or_default(), device_id: sha256(parsed.chip_id).to_vec(), mr_system: mrs.mr_system, mr_aggregated: mrs.mr_aggregated, diff --git a/dstack/dstack-attest/tests/sev_snp_verify.rs b/dstack/dstack-attest/tests/sev_snp_verify.rs index 13a87759a..38a19145c 100644 --- a/dstack/dstack-attest/tests/sev_snp_verify.rs +++ b/dstack/dstack-attest/tests/sev_snp_verify.rs @@ -102,7 +102,7 @@ fn verify_sev_snp_attestation_bin() { ); // The HOST_DATA-bound app identity is recovered from the mr_config document. assert_eq!( - hex::encode(&binding.mr_config.app_id), + hex::encode(binding.mr_config.app_id.as_deref().unwrap_or_default()), "86e59625be93207bc2351c4d1bba20037cec8e16", "mr_config app_id bound by HOST_DATA" ); diff --git a/dstack/dstack-mr/src/sev.rs b/dstack/dstack-mr/src/sev.rs index 03ac24164..59ff62f58 100644 --- a/dstack/dstack-mr/src/sev.rs +++ b/dstack/dstack-mr/src/sev.rs @@ -911,15 +911,15 @@ pub fn validate_mr_config(mr_config: &MrConfigV3) -> Result<()> { if mr_config.version != 3 { bail!("mr_config version must be 3"); } - if !mr_config.app_id.is_empty() { - ensure_len("mr_config.app_id", &mr_config.app_id, 20)?; + if let Some(app_id) = mr_config.app_id.as_deref() { + ensure_len("mr_config.app_id", app_id, 20)?; } ensure_len("mr_config.compose_hash", &mr_config.compose_hash, 32)?; if let Some(gpu_policy_hash) = &mr_config.gpu_policy_hash { ensure_len("mr_config.gpu_policy_hash", gpu_policy_hash, 32)?; } - if !mr_config.instance_id.is_empty() { - ensure_len("mr_config.instance_id", &mr_config.instance_id, 20)?; + if let Some(instance_id) = mr_config.instance_id.as_deref() { + ensure_len("mr_config.instance_id", instance_id, 20)?; } Ok(()) } diff --git a/dstack/dstack-types/Cargo.toml b/dstack/dstack-types/Cargo.toml index 526d5192b..5dba0358c 100644 --- a/dstack/dstack-types/Cargo.toml +++ b/dstack/dstack-types/Cargo.toml @@ -16,6 +16,7 @@ or-panic.workspace = true scale = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] } serde-human-bytes.workspace = true +serde_with.workspace = true serde_jcs.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/dstack/dstack-types/src/mr_config.rs b/dstack/dstack-types/src/mr_config.rs index 0de9b9f22..996a68695 100644 --- a/dstack/dstack-types/src/mr_config.rs +++ b/dstack/dstack-types/src/mr_config.rs @@ -5,6 +5,7 @@ use or_panic::ResultOrPanic; use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; +use serde_with::skip_serializing_none; use sha2::Sha256; use sha3::{Digest, Keccak256}; use std::{error::Error, fmt}; @@ -100,25 +101,26 @@ impl From for MrConfigDocumentError { } } +#[skip_serializing_none] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct MrConfigV3 { #[serde(default = "mr_config_v3_version")] pub version: u8, - /// Optional application identity pin. An empty value leaves app_id unbound. + /// Optional application identity pin. #[serde(default, with = "hex_bytes")] - pub app_id: Vec, + pub app_id: Option>, #[serde(with = "hex_bytes")] pub compose_hash: Vec, /// Hash of the raw application GPU policy. GPU launches populate it; /// non-GPU and historical v3 launch documents omit it. - #[serde(default, skip_serializing_if = "Option::is_none", with = "hex_bytes")] + #[serde(default, with = "hex_bytes")] pub gpu_policy_hash: Option>, pub key_provider: KeyProviderKind, #[serde(default, with = "hex_bytes")] - pub key_provider_id: Vec, + pub key_provider_id: Option>, #[serde(default, with = "hex_bytes")] - pub instance_id: Vec, + pub instance_id: Option>, } impl MrConfigV3 { @@ -132,12 +134,12 @@ impl MrConfigV3 { ) -> Self { Self { version: mr_config_v3_version(), - app_id, + app_id: (!app_id.is_empty()).then_some(app_id), compose_hash, gpu_policy_hash, key_provider, - key_provider_id, - instance_id, + key_provider_id: (!key_provider_id.is_empty()).then_some(key_provider_id), + instance_id: (!instance_id.is_empty()).then_some(instance_id), } } @@ -204,7 +206,7 @@ mod tests { vec![0x44; 20], ); let mut changed = config.clone(); - changed.app_id[0] ^= 0xff; + changed.app_id.as_mut().expect("app_id is set")[0] ^= 0xff; assert_ne!(config.to_snp_host_data(), changed.to_snp_host_data()); assert_eq!(config.to_snp_host_data().len(), 32); @@ -242,8 +244,8 @@ mod tests { r#"{"compose_hash":"2222222222222222222222222222222222222222222222222222222222222222","key_provider":"none"}"#, )?; - assert!(config.app_id.is_empty()); - assert!(config.to_canonical_json().contains(r#""app_id":"""#)); + assert!(config.app_id.is_none()); + assert!(!config.to_canonical_json().contains("app_id")); Ok(()) } diff --git a/dstack/dstack-util/src/system_setup/config_id_verifier.rs b/dstack/dstack-util/src/system_setup/config_id_verifier.rs index 12ea01137..02c905e57 100644 --- a/dstack/dstack-util/src/system_setup/config_id_verifier.rs +++ b/dstack/dstack-util/src/system_setup/config_id_verifier.rs @@ -168,17 +168,23 @@ fn verify_mr_config_v3_document( bail!("Invalid mr_config gpu_policy_hash"); } } - if !mr_config.app_id.is_empty() && mr_config.app_id.as_slice() != local.app_id { - bail!("Invalid mr_config app_id"); + if let Some(app_id) = mr_config.app_id.as_deref() { + if app_id != local.app_id { + bail!("Invalid mr_config app_id"); + } } - if mr_config.instance_id.as_slice() != local.instance_id { - bail!("Invalid mr_config instance_id"); + if let Some(instance_id) = mr_config.instance_id.as_deref() { + if instance_id != local.instance_id { + bail!("Invalid mr_config instance_id"); + } } if mr_config.key_provider != local.key_provider { bail!("Invalid mr_config key_provider"); } - if mr_config.key_provider_id.as_slice() != local.key_provider_id { - bail!("Invalid mr_config key_provider_id"); + if let Some(key_provider_id) = mr_config.key_provider_id.as_deref() { + if key_provider_id != local.key_provider_id { + bail!("Invalid mr_config key_provider_id"); + } } Ok(mr_config) } diff --git a/dstack/kms/src/main_service/amd_attest.rs b/dstack/kms/src/main_service/amd_attest.rs index 59387c895..a9a943131 100644 --- a/dstack/kms/src/main_service/amd_attest.rs +++ b/dstack/kms/src/main_service/amd_attest.rs @@ -113,9 +113,9 @@ fn build_amd_snp_boot_info_with_tcb_status( mr_aggregated, os_image_hash: os_image_hash.to_vec(), mr_system, - app_id: mr_config.app_id.clone(), + app_id: mr_config.app_id.clone().unwrap_or_default(), compose_hash: mr_config.compose_hash.clone(), - instance_id: mr_config.instance_id.clone(), + instance_id: mr_config.instance_id.clone().unwrap_or_default(), device_id: verified_chip_id.to_vec(), key_provider_info, tcb_status: tcb_status.to_string(), @@ -182,7 +182,7 @@ fn parse_measurement_input_from_vm_config(vm_config: &str) -> Result Result> { serde_json::to_vec(&KeyProviderInfo::new( mr_config.key_provider_name().to_string(), - hex::encode(&mr_config.key_provider_id), + hex::encode(mr_config.key_provider_id.as_deref().unwrap_or_default()), )) .context("failed to serialize key provider info") } diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index dad8f74ef..47ebe3698 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -2139,7 +2139,7 @@ mod tests { sys_config["nvidia_attestation_proxy_url"], "http://10.0.2.2:8090" ); - assert_eq!(parsed_mr_config.app_id, vec![0x11; 20]); + assert_eq!(parsed_mr_config.app_id, Some(vec![0x11; 20])); assert_eq!(parsed_mr_config.compose_hash, vec![0x22; 32]); assert_eq!(parsed_mr_config.gpu_policy_hash, None); assert_eq!(vm_config["mr_config"], sys_config["mr_config"]); From c4ea8110da14453c50ec57fe83e7a841c7c674cd Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 30 Jul 2026 20:02:43 -0700 Subject: [PATCH 29/30] feat(compose): support multiple init scripts --- docs/attestation-tdx.md | 14 +- ...s-attested-instance-security-evaluation.md | 2 +- docs/security/cvm-boundaries.md | 4 +- docs/security/security-model.md | 11 + docs/tutorials/attestation-verification.md | 6 +- dstack/dstack-attest/src/attestation.rs | 80 +++++++ .../snapshots/nitro_verify__app_info.snap | 5 +- dstack/dstack-types/src/lib.rs | 215 +++++++++++++++++- dstack/dstack-types/src/mr_config.rs | 33 +++ dstack/dstack-util/src/system_setup.rs | 31 +++ .../src/system_setup/config_id_verifier.rs | 86 +++++++ dstack/guest-agent/src/rpc_service.rs | 1 + dstack/verifier/src/types.rs | 1 + dstack/vmm/src/app/mr_config.rs | 63 ++++- .../vmm/ui/src/components/CreateVmDialog.ts | 9 +- .../vmm/ui/src/components/UpdateVmDialog.ts | 9 +- dstack/vmm/ui/src/composables/useVmManager.ts | 49 +++- os/common/rootfs/dstack-prepare.sh | 22 +- 18 files changed, 611 insertions(+), 30 deletions(-) diff --git a/docs/attestation-tdx.md b/docs/attestation-tdx.md index ae6798ca3..85674c6e7 100644 --- a/docs/attestation-tdx.md +++ b/docs/attestation-tdx.md @@ -31,7 +31,19 @@ MRTD, RTMR0, RTMR1, and RTMR2 can be pre-calculated from the built image (given RTMR3 differs as it contains runtime information like compose hash and instance id. Verify this by replaying the event log - if the calculated RTMR3 matches the quote's RTMR3, the event log information is valid. Then verify the compose hash, key provider, and other event log details match expectations. -For a GPU launch, `compose-hash` is followed by `gpu-policy-hash` and, after successful NVIDIA attestation and policy evaluation, `gpu-attestation`. The `gpu-policy-hash` payload is `SHA-256(JCS(requirements.gpu_policy))`, using `{}` when the field is omitted. The `gpu-attestation` payload is JSON containing the verified device count, CC/DevTools state, and `evidence_sha256`. +After `compose-hash`, each configured init script produces an ordered +`init-script-hash` event whose payload is the SHA-256 digest of the exact UTF-8 +script bytes. The event order matches the script array order. These events let +an infrastructure provider contribute initialization code approved by +multiple parties and let each party verify its code independently without +reconstructing the complete compose document. + +For a GPU launch, any `init-script-hash` events are followed by +`gpu-policy-hash` and, after successful NVIDIA attestation and policy +evaluation, `gpu-attestation`. The `gpu-policy-hash` payload is +`SHA-256(JCS(requirements.gpu_policy))`, using `{}` when the field is omitted. +The `gpu-attestation` payload is JSON containing the verified device count, +CC/DevTools state, and `evidence_sha256`. The guest-agent `GpuInfo` API returns the complete `nvattest` JSON captured during boot. It is not trustworthy by itself. After verifying the TDX quote and replaying the event log to RTMR3, hash the exact UTF-8 bytes of `GpuInfo.attestation` and require the result to equal the `gpu-attestation` event's `evidence_sha256`. See [GPU Security for AI Workloads](./security/security-model.md#gpu-security-for-ai-workloads) for the event schema, ordering, Rego example, and platform differences. diff --git a/docs/aws-attested-instance-security-evaluation.md b/docs/aws-attested-instance-security-evaluation.md index d772f3022..4f3ca9682 100644 --- a/docs/aws-attested-instance-security-evaluation.md +++ b/docs/aws-attested-instance-security-evaluation.md @@ -35,7 +35,7 @@ NitroTPM path meets them today. | P1 | Verifiable platform root of trust | TDX/SNP/Nitro quote verification against vendor root; debug rejected; TCB surfaced | NitroTPM Attestation Documents are verified against the AWS Nitro Attestation PKI, including document timestamp sanity. AWS exposes no TDX/SNP-style TCB advisory field, so a verified attestation is normalized to `tcbStatus = "UpToDate"` and passes the standard authorization gate unchanged. | | P2 | Reproducible or independently computable base image measurement | meta-dstack rebuild plus `dstack-mr` computes `MRTD`/`RTMR0-2` | The unified `os/build.sh` flow emits the AWS image archive with `sha256sum.txt`, `digest.txt`, and `measurement.aws.cbor`; its output directory also contains the reference-PCR side-car `aws-pcrs.json`. `os_image_hash = sha256(sha256sum.txt)` is the same identity used on all platforms; the verifier recomputes it from the downloaded image directory (`dstack/verifier/src/verification.rs`). The hardening audit script `os/yocto/tools/aws/audit-aws-ec2-image-hardening.sh` checks the image for operator mutation channels. | | P3 | Boot command line and root filesystem integrity are measured | `RTMR1/2`, rootfs hash, dm-verity, measured initrd/cmdline | The UKI commits kernel, initrd, and embedded cmdline into `PCR4`; the rootfs is dm-verity-protected. `VmConfig.aws_measurement` is required and must bind `boot_pcr_digest = sha256(PCR4||PCR7||PCR12)` to the attested PCRs, so `PCR12` (external cmdline) is always part of the bound digest — a missing-PCR12 bypass is not expressible. Enforced in guest quote generation (`dstack/dstack-attest/src/attestation.rs`), `verify_os_image_hash_for_aws_nitro_tpm` (`dstack/verifier/src/verification.rs`), and the KMS pipeline via the same verifier check. | -| P4 | Runtime application identity is cryptographically bound | RTMR3 `compose-hash`, `app-id`, `instance-id`, `key-provider`; event log replay | SHA384 `PCR14` event-log replay is the authoritative binding (RTMR3 analogue; non-resettable). Launch events: `system-preparing`, `app-id`, `compose-hash`, `instance-id`, `boot-mr-done`, `key-provider`, `storage-fs`, `system-ready` (`dstack/dstack-util/src/system_setup.rs`). `dstack-attest`, `dstack-verifier`, and KMS reject missing/mismatched PCR14 and bad replay. Optionally, the guest extends the raw `MrConfig` V2 `config_id` into `PCR8` once (`PCR8 = sha384(0^48 || config_id)`) so a lightweight third-party verifier can check compose hash + key provider without event-log replay; dstack's own verifier and KMS do not check PCR8. | +| P4 | Runtime application identity is cryptographically bound | RTMR3 `compose-hash`, `app-id`, `instance-id`, `key-provider`; event log replay | SHA384 `PCR14` event-log replay is the authoritative binding (RTMR3 analogue; non-resettable). Launch events: `system-preparing`, `app-id`, `compose-hash`, zero or more ordered `init-script-hash` events, `instance-id`, `boot-mr-done`, `key-provider`, `storage-fs`, `system-ready` (`dstack/dstack-util/src/system_setup.rs`). GPU launches also include `gpu-policy-hash` and `gpu-attestation` before `instance-id`. `dstack-attest`, `dstack-verifier`, and KMS reject missing/mismatched PCR14 and bad replay. Optionally, the guest extends the raw `MrConfig` V2 `config_id` into `PCR8` once (`PCR8 = sha384(0^48 || config_id)`) so a lightweight third-party verifier can check compose hash + key provider without event-log replay; dstack's own verifier and KMS do not check PCR8. | | P5 | Challenge/liveness and caller key binding | `report_data` challenge or RA-TLS public key hash in quote | RA-TLS binds `report_data` to the TLS certificate public key. KMS key release is bound to the live RA-TLS handshake; external `/verify` callers supply and check their own `report_data` challenge. The low-level NitroTPM document verifier also rejects stale or far-future document timestamps. | | P6 | Secret release only to attested code | dstack KMS verifies attestation, checks auth policy, derives per-app keys | dstack KMS verifies the NitroTPM attestation, runs the same `verify_os_image_hash_for_aws_nitro_tpm` binding check as the verifier, builds `BootInfo` from verified boot PCRs plus PCR14 launch events, and checks auth policy before deriving app keys (`dstack/kms/src/main_service.rs`). AWS NitroTPM key release is gated behind the opt-in `aws_nitro_tpm_key_release` flag (default false in `kms.toml`). | | P7 | Key-release policy is not controlled by the untrusted account admin | KMS runs inside TEE; policy from auth API/contracts; KMS identity measured | Satisfied with dstack KMS or another verifiable secret authority outside the untrusted AWS account admin's control. A NitroTPM-backed dstack KMS keeps root material out of account-admin snapshots and clones. The policy backend (auth-simple in a trusted control plane, or on-chain `DstackKms`/`DstackApp`) must be outside the workload account admin's control. Same-account AWS KMS fails this property if the admin can change key policy, create grants, or call secret-bearing operations through a policy they control. | diff --git a/docs/security/cvm-boundaries.md b/docs/security/cvm-boundaries.md index ae668efbe..a47b833f2 100644 --- a/docs/security/cvm-boundaries.md +++ b/docs/security/cvm-boundaries.md @@ -41,11 +41,13 @@ This is the main configuration file for the application in JSON format: | no_instance_id | 0.4.2 | boolean | Disable instance ID generation | | secure_time | 0.5.0 | boolean | Whether secure time is enabled | | pre_launch_script | 0.4.0 | string | Prelaunch bash script that runs before execute `docker compose up` | -| init_script | 0.5.5 | string | Bash script that executed prior to dockerd startup | +| init_script | 0.5.5 (string), 0.6.0 (string[]) | string or string[] | Up to 5 Bash scripts executed in order prior to dockerd startup; a string is treated as a one-element array. Multiple scripts require string `manifest_version: "3"` so older guests fail closed. MrConfigV3 binds the hashes only for manifest v3. | | storage_fs | 0.5.5 | string | Filesystem type for the data disk of the CVM. Supported values: "zfs", "ext4". default to "zfs". **ZFS:** Ensures filesystem integrity with built-in data protection features. **ext4:** Provides better performance for database applications with lower overhead and faster I/O operations, but no strong integrity protection. | | swap_size | 0.5.5 | string/integer | The linux swap size. default to 0. Can be in byte or human-readable format (e.g., "1G", "256M"). | | key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". GCP vTPM and AWS EC2 NitroTPM are part of their platform trust models. The Dstack platform can use VMM-managed swtpm for seal/unseal and restart persistence, but it offers no protection against the host and is intentionally not accepted by remote verifiers. | +The five-script limit bounds runtime-event-log and MrConfigV3 growth while +allowing several independently approved infrastructure initialization stages. The hash of this file content is extended as the dstack `compose-hash` launch event. On TDX-family platforms the launch event is measured into RTMR3. On AWS NitroTPM it is measured into non-resettable SHA384 PCR14 before the `system-ready` launch boundary. Remote verifiers extract and replay this event during attestation. diff --git a/docs/security/security-model.md b/docs/security/security-model.md index a3614548e..63158b1dc 100644 --- a/docs/security/security-model.md +++ b/docs/security/security-model.md @@ -98,6 +98,16 @@ The policy must define the boolean rule `data.policy.nv_match`. Its `input` is t After measuring `compose-hash`, dstack enters the GPU setup gate and JCS-canonicalizes the original `requirements.gpu_policy` JSON value, then measures its SHA-256 digest in a `gpu-policy-hash` event. When the field is absent—including when `requirements` itself is absent—both parsing and measurement use the default empty object `{}`. Thus an omitted policy and an explicit `{}` have the same digest, while any explicitly present field, including an explicit default value, changes the digest. MrConfigV3 GPU launches also carry this digest as the optional `gpu_policy_hash` field; non-GPU launches omit it for compatibility. When the field is present, the guest compares it with the digest computed from app-compose; when it is absent, the guest skips this MrConfigV3 check. The MrConfigV3 document is bound by TDX `MR_CONFIG_ID` or SEV-SNP `HOST_DATA`, so the host cannot substitute a different GPU policy when this optional binding is present without changing the platform launch identity. The typed policy used for enforcement applies omitted-field defaults and rejects unknown fields. If GPU attestation is enabled and NVIDIA GPUs are present, dstack attests them and applies the basic settings and optional Rego policy before setting the GPU ready state. When no attestation claims are produced—because no GPU is attached or `gpu_policy.attest_gpu` is false—Rego is still evaluated with an empty array as `input` before any ready-state transition. This lets an application reject a launch whose attested GPU count is wrong. A false, undefined, malformed, or non-boolean Rego result stops boot before key provisioning. +Up to five init scripts may be configured. Their ordered SHA-256 digests are +measured as `init-script-hash` runtime events on platforms with a quoted +runtime register. MrConfigV3 also carries the ordered digest list; on SEV-SNP, +the signed report's `HOST_DATA` binds the exact canonical MrConfigV3 document. +An omitted `init_script_hashes` field disables this check, +while an explicit empty list requires app-compose to contain no init scripts. +When present, the guest compares the list with hashes computed from app-compose +before continuing boot. Current VMMs serialize the field explicitly for +manifest v3 launches, including `init_script_hashes: []` when the list is empty. + The policy digest is remotely verifiable on each supported platform, but through different carriers: - **TDX:** `gpu-policy-hash` contains the raw 32-byte digest and is measured into RTMR3. Replay the event log and compare the result with the quote's RTMR3, then compare the event payload with the expected digest. When an MrConfigV3 document includes `gpu_policy_hash`, TDX `MR_CONFIG_ID` additionally binds that field. @@ -114,6 +124,7 @@ For a successful TDX GPU launch, the GPU-relevant RTMR3 event order is: ```text compose-hash +init-script-hash (zero or more, in configured order) gpu-policy-hash gpu-attestation instance-id diff --git a/docs/tutorials/attestation-verification.md b/docs/tutorials/attestation-verification.md index b9661de2f..ea1cb1180 100644 --- a/docs/tutorials/attestation-verification.md +++ b/docs/tutorials/attestation-verification.md @@ -521,6 +521,7 @@ These are the standard events you'll see in the log: | `system-preparing` | System initialization marker | Always present | | `app-id` | Application identifier | Should match your app name | | `compose-hash` | SHA-256 of docker compose config | Should match `tcb_info.compose_hash` | +| `init-script-hash` | SHA-256 of one init script; repeated in configured order (maximum 5) | Should match the independently approved script bytes | | `gpu-policy-hash` | SHA-256 of the JCS-canonicalized GPU policy (default `{}`) | Should match the expected `requirements.gpu_policy` digest | | `gpu-attestation` | Verified GPU state and digest of the boot-time `nvattest` JSON | Required for an attested GPU launch; verify as described below | | `instance-id` | Unique instance identifier | Should match `instance_id` from response | @@ -531,8 +532,9 @@ These are the standard events you'll see in the log: | `storage-fs` | Storage filesystem type | Storage configuration | | `system-ready` | System ready marker | Always present at end | -For a successful GPU launch, the relevant order is `compose-hash`, -`gpu-policy-hash`, `gpu-attestation`, `instance-id`, and `boot-mr-done`. +For a successful GPU launch, the relevant order is `compose-hash`, any +`init-script-hash` events, `gpu-policy-hash`, `gpu-attestation`, `instance-id`, +and `boot-mr-done`. After replaying the log to the quote's RTMR3, decode the JSON payload of `gpu-attestation` and compare its `evidence_sha256` with the SHA-256 digest of the exact UTF-8 `GpuInfo.attestation` string. `GpuInfo` reads the result saved diff --git a/dstack/dstack-attest/src/attestation.rs b/dstack/dstack-attest/src/attestation.rs index 8c78d1867..111b6920d 100644 --- a/dstack/dstack-attest/src/attestation.rs +++ b/dstack/dstack-attest/src/attestation.rs @@ -390,6 +390,19 @@ fn find_event_payload(runtime_events: &[RuntimeEvent], event: &str) -> Result Vec> { + runtime_events + .iter() + .take_while(|event| event.event != "system-ready") + .filter(|event| event.event == name) + .map(|event| event.payload.clone()) + .collect() +} + fn decode_vm_config_with_fallback(config: &str, fallback_config: &str) -> Result { let config = if config.is_empty() { fallback_config @@ -890,6 +903,7 @@ impl AttestationV1 { key_provider_info, os_image_hash, compose_hash, + init_script_hashes: Some(find_event_payloads(runtime_events, "init-script-hash")), } }; @@ -1615,6 +1629,7 @@ fn decode_app_info_sev_snp( key_provider_info, os_image_hash, compose_hash: mr_config.compose_hash, + init_script_hashes: mr_config.init_script_hashes, }) } @@ -1937,6 +1952,10 @@ impl Attestation { key_provider_info, os_image_hash, compose_hash, + init_script_hashes: Some(find_event_payloads( + &self.runtime_events, + "init-script-hash", + )), } }; @@ -2027,6 +2046,12 @@ impl Attestation { self.find_event(event).map(|event| event.payload) } + /// SHA-256 payloads of all measured init scripts, in execution order. + /// Application-emitted events after `system-ready` are excluded. + pub fn decode_init_script_hashes(&self) -> Vec> { + find_event_payloads(&self.runtime_events, "init-script-hash") + } + fn find_event_hex_payload(&self, event: &str) -> Result { self.find_event(event) .map(|event| hex::encode(&event.payload)) @@ -2491,12 +2516,52 @@ pub struct AppInfo { /// Key provider info #[serde(with = "hex_bytes")] pub key_provider_info: Vec, + /// Optional SHA-256 pins for init scripts, in execution order. `None` + /// means the evidence did not bind this field. On SEV-SNP, `Some(vec![])` + /// explicitly binds an empty script list. On TDX and Nitro it only means + /// that no `init-script-hash` events were measured before `system-ready`; + /// pre-0.6.0 images emit no such events even when they run an init script. + #[serde(default, with = "dstack_types::init_script_hashes::option")] + pub init_script_hashes: Option>>, } #[cfg(test)] mod tests { use super::*; + #[test] + fn app_info_defaults_missing_init_script_hashes() { + let app_info: AppInfo = serde_json::from_value(serde_json::json!({ + "app_id": "", + "compose_hash": "", + "instance_id": "", + "device_id": "", + "mr_system": "0000000000000000000000000000000000000000000000000000000000000000", + "mr_aggregated": "0000000000000000000000000000000000000000000000000000000000000000", + "os_image_hash": "", + "key_provider_info": "" + })) + .unwrap(); + assert!(app_info.init_script_hashes.is_none()); + } + + #[test] + fn app_info_preserves_explicit_empty_init_script_hashes() { + let app_info: AppInfo = serde_json::from_value(serde_json::json!({ + "app_id": "", + "compose_hash": "", + "instance_id": "", + "device_id": "", + "mr_system": "0000000000000000000000000000000000000000000000000000000000000000", + "mr_aggregated": "0000000000000000000000000000000000000000000000000000000000000000", + "os_image_hash": "", + "key_provider_info": "", + "init_script_hashes": [] + })) + .unwrap(); + assert_eq!(app_info.init_script_hashes, Some(Vec::new())); + } + #[test] fn external_trust_anchor_requires_explicit_insecure_opt_in() { let config = AttestationVerifierConfig { @@ -2826,6 +2891,21 @@ mod tests { RuntimeEvent::new(event, payload, EventLogVersion::V1) } + #[test] + fn init_script_hashes_exclude_application_events_after_system_ready() { + let events = vec![ + v1_event("init-script-hash".into(), vec![0x11; 32]), + v1_event("init-script-hash".into(), vec![0x22; 32]), + v1_event("system-ready".into(), Vec::new()), + v1_event("init-script-hash".into(), vec![0xff; 32]), + ]; + + assert_eq!( + find_event_payloads(&events, "init-script-hash"), + vec![vec![0x11; 32], vec![0x22; 32]] + ); + } + #[test] fn nitro_pcrs_from_verified_extracts_0_1_2() { let mut map = std::collections::BTreeMap::new(); diff --git a/dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap b/dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap index cc08badca..d62ea8cc0 100644 --- a/dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap +++ b/dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap @@ -1,6 +1,6 @@ --- source: dstack-attest/tests/nitro_verify.rs -assertion_line: 25 +assertion_line: 29 expression: app_info_str --- { @@ -11,5 +11,6 @@ expression: app_info_str "mr_system": "1894b0b29e94a9db16e88a2914f0923e52bb16c08bf3bb484df786a147e2eb79", "mr_aggregated": "1894b0b29e94a9db16e88a2914f0923e52bb16c08bf3bb484df786a147e2eb79", "os_image_hash": "1894b0b29e94a9db16e88a2914f0923e52bb16c08bf3bb484df786a147e2eb79", - "key_provider_info": "" + "key_provider_info": "", + "init_script_hashes": [] } diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 141ed3bb9..55b81984f 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -10,6 +10,89 @@ use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; use size_parser::human_size; +/// Bound event-log growth and MrConfigV3 size while supporting independent +/// infrastructure-provider initialization stages. +pub const MAX_INIT_SCRIPTS: usize = 5; + +pub mod init_script_hashes { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use serde_human_bytes::ByteBuf; + + pub fn serialize(values: &[Vec], serializer: S) -> Result + where + S: Serializer, + { + values + .iter() + .cloned() + .map(ByteBuf::from) + .collect::>() + .serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> + where + D: Deserializer<'de>, + { + let values = Vec::::deserialize(deserializer)?; + if values.len() > super::MAX_INIT_SCRIPTS { + return Err(serde::de::Error::custom(format!( + "init_script_hashes supports at most {} hashes", + super::MAX_INIT_SCRIPTS + ))); + } + if values.iter().any(|value| value.len() != 32) { + return Err(serde::de::Error::custom( + "each init_script_hash must be 32 bytes", + )); + } + Ok(values.into_iter().map(ByteBuf::into_vec).collect()) + } + + pub mod option { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use serde_human_bytes::ByteBuf; + + pub fn serialize(values: &Option>>, serializer: S) -> Result + where + S: Serializer, + { + values + .as_ref() + .map(|values| { + values + .iter() + .cloned() + .map(ByteBuf::from) + .collect::>() + }) + .serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result>>, D::Error> + where + D: Deserializer<'de>, + { + Option::>::deserialize(deserializer)? + .map(|values| { + if values.len() > super::super::MAX_INIT_SCRIPTS { + return Err(serde::de::Error::custom(format!( + "init_script_hashes supports at most {} hashes", + super::super::MAX_INIT_SCRIPTS + ))); + } + if values.iter().any(|value| value.len() != 32) { + return Err(serde::de::Error::custom( + "each init_script_hash must be 32 bytes", + )); + } + Ok(values.into_iter().map(ByteBuf::into_vec).collect()) + }) + .transpose() + } + } +} + /// Identifies which OVMF flavour the guest image was built with. /// /// Only the pre-202505 OVMF measurement layout is supported. @@ -131,6 +214,17 @@ pub struct AppCompose { pub snapshotter: Option, #[serde(default)] pub docker_compose_file: Option, + /// Bash scripts executed before the application runner starts. + /// + /// A single string is accepted for backward compatibility and is treated + /// as a one-element list. + #[serde( + default, + deserialize_with = "deserialize_init_scripts", + serialize_with = "serialize_init_scripts", + skip_serializing_if = "Vec::is_empty" + )] + pub init_script: Vec, #[serde(default)] pub public_logs: bool, #[serde(default)] @@ -139,7 +233,11 @@ pub struct AppCompose { pub public_tcbinfo: bool, #[serde(default)] pub kms_enabled: bool, - #[serde(deserialize_with = "deserialize_gateway_enabled", flatten)] + #[serde( + deserialize_with = "deserialize_gateway_enabled", + serialize_with = "serialize_gateway_enabled", + flatten + )] pub gateway_enabled: bool, #[serde(default)] pub local_key_provider_enabled: bool, @@ -178,6 +276,41 @@ pub struct AppCompose { pub verity_volumes: Vec, } +fn deserialize_init_scripts<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum InitScripts { + One(String), + Many(Vec), + } + + let scripts = match Option::::deserialize(deserializer)? { + None => Vec::new(), + Some(InitScripts::One(script)) => vec![script], + Some(InitScripts::Many(scripts)) => scripts, + }; + if scripts.len() > MAX_INIT_SCRIPTS { + return Err(serde::de::Error::custom(format!( + "init_script supports at most {MAX_INIT_SCRIPTS} scripts" + ))); + } + Ok(scripts) +} + +fn serialize_init_scripts(scripts: &[String], serializer: S) -> Result +where + S: serde::Serializer, +{ + if let [script] = scripts { + serializer.serialize_str(script) + } else { + scripts.serialize(serializer) + } +} + /// A pre-baked, read-only dm-verity volume attached to the CVM. #[derive(Deserialize, Serialize, Debug, Clone)] pub struct VerityVolume { @@ -490,6 +623,21 @@ where Ok(value.gateway_enabled || value.tproxy_enabled) } +fn serialize_gateway_enabled(enabled: &bool, serializer: S) -> Result +where + S: serde::Serializer, +{ + #[derive(Serialize)] + struct GatewayEnabled { + gateway_enabled: bool, + } + + GatewayEnabled { + gateway_enabled: *enabled, + } + .serialize(serializer) +} + #[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum KeyProviderKind { @@ -568,6 +716,71 @@ mod app_compose_tests { })) } + #[test] + fn init_script_accepts_string_array_and_null() { + assert!(parse_compose(serde_json::json!(2)) + .unwrap() + .init_script + .is_empty()); + + let single: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": "echo one" + })) + .unwrap(); + assert_eq!(single.init_script, ["echo one"]); + assert_eq!( + serde_json::to_value(&single).unwrap()["init_script"], + "echo one" + ); + + let multiple: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": ["echo one", "echo two"] + })) + .unwrap(); + assert_eq!(multiple.init_script, ["echo one", "echo two"]); + assert_eq!( + serde_json::to_value(&multiple).unwrap()["init_script"], + serde_json::json!(["echo one", "echo two"]) + ); + + let null: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": null + })) + .unwrap(); + assert!(null.init_script.is_empty()); + assert!(serde_json::to_value(&null) + .unwrap() + .get("init_script") + .is_none()); + + assert!(serde_json::from_value::(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": ["echo one", 2] + })) + .is_err()); + + assert!(serde_json::from_value::(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": ["1", "2", "3", "4", "5", "6"] + })) + .unwrap_err() + .to_string() + .contains("at most 5")); + } + #[test] fn manifest_version_accepts_string_versions() { let compose = parse_compose(serde_json::json!("3")).unwrap(); diff --git a/dstack/dstack-types/src/mr_config.rs b/dstack/dstack-types/src/mr_config.rs index 996a68695..ce38f4e7c 100644 --- a/dstack/dstack-types/src/mr_config.rs +++ b/dstack/dstack-types/src/mr_config.rs @@ -121,6 +121,10 @@ pub struct MrConfigV3 { pub key_provider_id: Option>, #[serde(default, with = "hex_bytes")] pub instance_id: Option>, + /// Optional SHA-256 pins for init scripts, in execution order. An omitted + /// field disables this check; an empty list requires no init scripts. + #[serde(default, with = "crate::init_script_hashes::option")] + pub init_script_hashes: Option>>, } impl MrConfigV3 { @@ -140,9 +144,15 @@ impl MrConfigV3 { key_provider, key_provider_id: (!key_provider_id.is_empty()).then_some(key_provider_id), instance_id: (!instance_id.is_empty()).then_some(instance_id), + init_script_hashes: None, } } + pub fn with_init_script_hashes(mut self, init_script_hashes: Vec>) -> Self { + self.init_script_hashes = Some(init_script_hashes); + self + } + pub fn to_snp_host_data(&self) -> [u8; 32] { Self::snp_host_data_from_document(&self.to_canonical_json()) } @@ -249,6 +259,29 @@ mod tests { Ok(()) } + #[test] + fn mr_config_v3_binds_ordered_init_script_hashes() -> Result<(), Box> { + let config = MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + None, + KeyProviderKind::None, + Vec::new(), + vec![0x44; 20], + ) + .with_init_script_hashes(vec![vec![0xaa; 32], vec![0xbb; 32]]); + let document = config.to_canonical_json(); + let decoded = MrConfigV3::from_document(&document)?; + + assert_eq!(decoded.init_script_hashes, config.init_script_hashes); + let reordered = MrConfigV3 { + init_script_hashes: Some(vec![vec![0xbb; 32], vec![0xaa; 32]]), + ..config.clone() + }; + assert_ne!(config.to_snp_host_data(), reordered.to_snp_host_data()); + Ok(()) + } + #[test] fn mr_config_v3_generates_jcs_but_hashes_document_bytes() -> Result<(), Box> { let config = MrConfigV3::new( diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 6f258d34b..8cd68b4b1 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -785,6 +785,11 @@ fn verify_manifest_feature_requirements(app_compose: &AppCompose) -> Result<()> "nerdctl-compose requires manifest_version >= {MANIFEST_VERSION_3}; use string manifest_version \"{MANIFEST_VERSION_3}\" so older guests fail closed" ); } + if app_compose.init_script.len() > 1 && manifest_version < MANIFEST_VERSION_3 { + bail!( + "multiple init scripts require manifest_version >= {MANIFEST_VERSION_3}; use string manifest_version \"{MANIFEST_VERSION_3}\" so older guests fail closed" + ); + } if app_compose.runner != "nerdctl-compose" && app_compose.snapshotter.is_some() { bail!("snapshotter is only supported by the nerdctl-compose runner"); } @@ -1969,6 +1974,7 @@ struct AppInfo { instance_info: InstanceInfo, compose_hash: [u8; 32], gpu_policy_hash: [u8; 32], + init_script_hashes: Vec>, } struct Stage0<'a> { @@ -2586,6 +2592,16 @@ impl<'a> Stage0<'a> { emit_runtime_event("system-preparing", &[])?; emit_runtime_event("app-id", &instance_info.app_id)?; emit_runtime_event("compose-hash", &compose_hash)?; + let init_script_hashes: Vec> = self + .shared + .app_compose + .init_script + .iter() + .map(|script| sha256(script.as_bytes()).to_vec()) + .collect(); + for script_hash in &init_script_hashes { + emit_runtime_event("init-script-hash", script_hash)?; + } let gpu_policy_hash = self .measure_gpu() .await @@ -2619,6 +2635,7 @@ impl<'a> Stage0<'a> { instance_info, compose_hash, gpu_policy_hash, + init_script_hashes, }) } @@ -2626,6 +2643,7 @@ impl<'a> Stage0<'a> { config_id_verifier::verify_mr_config_id( &app_info.compose_hash, &app_info.gpu_policy_hash, + &app_info.init_script_hashes, &app_info .instance_info .app_id @@ -3162,6 +3180,19 @@ fn test_nerdctl_compose_requires_v3_manifest() { verify_manifest_feature_requirements(&app_compose).unwrap(); } +#[test] +fn test_multiple_init_scripts_require_v3_manifest() { + let mut app_compose = test_app_compose(serde_json::json!(2), None, None); + app_compose.init_script = vec!["echo one".into(), "echo two".into()]; + let err = verify_manifest_feature_requirements(&app_compose).unwrap_err(); + assert!(err + .to_string() + .contains("multiple init scripts require manifest_version")); + + app_compose.manifest_version = "3".into(); + verify_manifest_feature_requirements(&app_compose).unwrap(); +} + #[test] fn test_snapshotter_is_rejected_for_other_runners() { let mut app_compose = test_app_compose(serde_json::json!("3"), None, None); diff --git a/dstack/dstack-util/src/system_setup/config_id_verifier.rs b/dstack/dstack-util/src/system_setup/config_id_verifier.rs index 02c905e57..89f888b8c 100644 --- a/dstack/dstack-util/src/system_setup/config_id_verifier.rs +++ b/dstack/dstack-util/src/system_setup/config_id_verifier.rs @@ -15,6 +15,7 @@ use tracing::info; struct LocalMrConfigValues<'a> { compose_hash: &'a [u8; 32], gpu_policy_hash: &'a [u8; 32], + init_script_hashes: &'a [Vec], app_id: &'a [u8; 20], instance_id: &'a [u8], key_provider: KeyProviderKind, @@ -67,6 +68,7 @@ fn read_snp_host_data() -> Result<[u8; 32]> { pub fn verify_mr_config_id( compose_hash: &[u8; 32], gpu_policy_hash: &[u8; 32], + init_script_hashes: &[Vec], app_id: &[u8; 20], instance_id: &[u8], key_provider: KeyProviderKind, @@ -76,6 +78,7 @@ pub fn verify_mr_config_id( let local = LocalMrConfigValues { compose_hash, gpu_policy_hash, + init_script_hashes, app_id, instance_id, key_provider, @@ -168,6 +171,11 @@ fn verify_mr_config_v3_document( bail!("Invalid mr_config gpu_policy_hash"); } } + if let Some(init_script_hashes) = mr_config.init_script_hashes.as_deref() { + if init_script_hashes != local.init_script_hashes { + bail!("Invalid mr_config init_script_hashes"); + } + } if let Some(app_id) = mr_config.app_id.as_deref() { if app_id != local.app_id { bail!("Invalid mr_config app_id"); @@ -221,6 +229,7 @@ mod tests { let local = LocalMrConfigValues { compose_hash: &compose_hash, gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &[], app_id: &app_id, instance_id: &instance_id, key_provider: KeyProviderKind::Kms, @@ -250,6 +259,7 @@ mod tests { let local = LocalMrConfigValues { compose_hash: &compose_hash, gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &[], app_id: &wrong_app_id, instance_id: &instance_id, key_provider: KeyProviderKind::Kms, @@ -281,6 +291,7 @@ mod tests { let local = LocalMrConfigValues { compose_hash: &compose_hash, gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &[], app_id: &app_id, instance_id: &instance_id, key_provider: KeyProviderKind::Kms, @@ -311,6 +322,7 @@ mod tests { let local = LocalMrConfigValues { compose_hash: &compose_hash, gpu_policy_hash: &wrong_gpu_policy_hash, + init_script_hashes: &[], app_id: &app_id, instance_id: &instance_id, key_provider: KeyProviderKind::Kms, @@ -344,6 +356,7 @@ mod tests { let local = LocalMrConfigValues { compose_hash: &compose_hash, gpu_policy_hash: &actual_gpu_policy_hash, + init_script_hashes: &[], app_id: &app_id, instance_id: &instance_id, key_provider: KeyProviderKind::Kms, @@ -352,4 +365,77 @@ mod tests { verify_tdx_mr_config_id_value(mr_config.to_tdx_mr_config_id(), Some(&document), local) } + + #[test] + fn mr_config_v3_document_rejects_mismatched_init_script_hashes() { + let compose_hash = [0x22u8; 32]; + let gpu_policy_hash = [0x55u8; 32]; + let app_id = [0x11u8; 20]; + let instance_id = [0x44u8; 20]; + let declared_hashes = vec![vec![0xaau8; 32]]; + let actual_hashes = vec![vec![0xbbu8; 32]]; + let document = MrConfigV3::new( + app_id.to_vec(), + compose_hash.to_vec(), + None, + KeyProviderKind::None, + Vec::new(), + instance_id.to_vec(), + ) + .with_init_script_hashes(declared_hashes) + .to_canonical_json(); + let local = LocalMrConfigValues { + compose_hash: &compose_hash, + gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &actual_hashes, + app_id: &app_id, + instance_id: &instance_id, + key_provider: KeyProviderKind::None, + key_provider_id: &[], + }; + + assert!(verify_mr_config_v3_document(&document, local) + .unwrap_err() + .to_string() + .contains("Invalid mr_config init_script_hashes")); + } + + #[test] + fn mr_config_v3_document_skips_init_script_check_when_field_is_missing() { + let compose_hash = [0x22u8; 32]; + let gpu_policy_hash = [0x55u8; 32]; + let app_id = [0x11u8; 20]; + let instance_id = [0x44u8; 20]; + let mut document = serde_json::to_value(MrConfigV3::new( + app_id.to_vec(), + compose_hash.to_vec(), + None, + KeyProviderKind::None, + Vec::new(), + instance_id.to_vec(), + )) + .unwrap(); + document + .as_object_mut() + .unwrap() + .remove("init_script_hashes"); + let local_without_scripts = LocalMrConfigValues { + compose_hash: &compose_hash, + gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &[], + app_id: &app_id, + instance_id: &instance_id, + key_provider: KeyProviderKind::None, + key_provider_id: &[], + }; + + verify_mr_config_v3_document(&document.to_string(), local_without_scripts).unwrap(); + + let actual_hashes = vec![vec![0xaau8; 32]]; + let local_with_script = LocalMrConfigValues { + init_script_hashes: &actual_hashes, + ..local_without_scripts + }; + verify_mr_config_v3_document(&document.to_string(), local_with_script).unwrap(); + } } diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index e2719786e..3a52ec1f6 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -739,6 +739,7 @@ mod tests { runner: String::new(), snapshotter: None, docker_compose_file: None, + init_script: Vec::new(), public_logs: false, public_sysinfo: false, public_tcbinfo: false, diff --git a/dstack/verifier/src/types.rs b/dstack/verifier/src/types.rs index 1d4851526..34665d27a 100644 --- a/dstack/verifier/src/types.rs +++ b/dstack/verifier/src/types.rs @@ -203,6 +203,7 @@ mod tests { mr_aggregated: [0x66; 32], os_image_hash: vec![0x77; 32], key_provider_info: br#"{"name":"tpm","id":"aws-test"}"#.to_vec(), + init_script_hashes: Some(Vec::new()), }; let boot_info = PolicyBootInfo::from_app_info( diff --git a/dstack/vmm/src/app/mr_config.rs b/dstack/vmm/src/app/mr_config.rs index d607255d0..e9aa83539 100644 --- a/dstack/vmm/src/app/mr_config.rs +++ b/dstack/vmm/src/app/mr_config.rs @@ -14,6 +14,18 @@ use sha2::{Digest, Sha256}; use super::{GpuConfig, Manifest, VmWorkDir}; use crate::config::{CvmConfig, CvmPlatform}; +fn bind_init_script_hashes(mut mr_config: MrConfigV3, app_compose: &AppCompose) -> MrConfigV3 { + if app_compose.manifest_version_u32().unwrap_or_default() >= 3 { + let hashes = app_compose + .init_script + .iter() + .map(|script| Sha256::digest(script.as_bytes()).to_vec()) + .collect(); + mr_config = mr_config.with_init_script_hashes(hashes); + } + mr_config +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum MrConfigVersion { V1, @@ -167,14 +179,61 @@ impl VmWorkDir { ) .context("failed to write instance info")?; - Ok(MrConfigV3::new( + // Manifest v3 is the fail-closed capability signal: older guests + // reject its string version instead of seeing an unknown MrConfigV3 + // field generated by a newer VMM. + let mr_config = MrConfigV3::new( app_id, compose_hash.to_vec(), gpu_policy_hash, app_compose.key_provider(), app_compose.key_provider_id.clone(), instance_id, + ); + Ok(bind_init_script_hashes(mr_config, app_compose).to_canonical_json()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use dstack_types::KeyProviderKind; + + fn app_compose(manifest_version: serde_json::Value) -> AppCompose { + serde_json::from_value(serde_json::json!({ + "manifest_version": manifest_version, + "name": "test", + "runner": "docker-compose" + })) + .unwrap() + } + + fn base_mr_config() -> MrConfigV3 { + MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + None, + KeyProviderKind::None, + Vec::new(), + vec![0x33; 20], ) - .to_canonical_json()) + } + + #[test] + fn manifest_v2_omits_init_script_hashes() { + let document = + bind_init_script_hashes(base_mr_config(), &app_compose(serde_json::json!(2))) + .to_canonical_json(); + let document: serde_json::Value = serde_json::from_str(&document).unwrap(); + assert!(document.get("init_script_hashes").is_none()); + } + + #[test] + fn manifest_v3_includes_empty_init_script_hashes() { + let document = + bind_init_script_hashes(base_mr_config(), &app_compose(serde_json::json!("3"))) + .to_canonical_json(); + let document: serde_json::Value = serde_json::from_str(&document).unwrap(); + assert_eq!(document["init_script_hashes"], serde_json::json!([])); } } diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index b484602f1..82653a62d 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -108,10 +108,15 @@ const CreateVmDialogComponent = {
-
diff --git a/dstack/vmm/ui/src/components/UpdateVmDialog.ts b/dstack/vmm/ui/src/components/UpdateVmDialog.ts index f994f0487..5dc9568b1 100644 --- a/dstack/vmm/ui/src/components/UpdateVmDialog.ts +++ b/dstack/vmm/ui/src/components/UpdateVmDialog.ts @@ -92,10 +92,15 @@ const UpdateVmDialogComponent = {
-