From c4ea8110da14453c50ec57fe83e7a841c7c674cd Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Thu, 30 Jul 2026 20:02:43 -0700 Subject: [PATCH] feat(compose): support multiple init scripts --- docs/attestation-tdx.md | 14 +- ...s-attested-instance-security-evaluation.md | 2 +- docs/security/cvm-boundaries.md | 4 +- docs/security/security-model.md | 11 + docs/tutorials/attestation-verification.md | 6 +- dstack/dstack-attest/src/attestation.rs | 80 +++++++ .../snapshots/nitro_verify__app_info.snap | 5 +- dstack/dstack-types/src/lib.rs | 215 +++++++++++++++++- dstack/dstack-types/src/mr_config.rs | 33 +++ dstack/dstack-util/src/system_setup.rs | 31 +++ .../src/system_setup/config_id_verifier.rs | 86 +++++++ dstack/guest-agent/src/rpc_service.rs | 1 + dstack/verifier/src/types.rs | 1 + dstack/vmm/src/app/mr_config.rs | 63 ++++- .../vmm/ui/src/components/CreateVmDialog.ts | 9 +- .../vmm/ui/src/components/UpdateVmDialog.ts | 9 +- dstack/vmm/ui/src/composables/useVmManager.ts | 49 +++- os/common/rootfs/dstack-prepare.sh | 22 +- 18 files changed, 611 insertions(+), 30 deletions(-) diff --git a/docs/attestation-tdx.md b/docs/attestation-tdx.md index ae6798ca3..85674c6e7 100644 --- a/docs/attestation-tdx.md +++ b/docs/attestation-tdx.md @@ -31,7 +31,19 @@ MRTD, RTMR0, RTMR1, and RTMR2 can be pre-calculated from the built image (given RTMR3 differs as it contains runtime information like compose hash and instance id. Verify this by replaying the event log - if the calculated RTMR3 matches the quote's RTMR3, the event log information is valid. Then verify the compose hash, key provider, and other event log details match expectations. -For a GPU launch, `compose-hash` is followed by `gpu-policy-hash` and, after successful NVIDIA attestation and policy evaluation, `gpu-attestation`. The `gpu-policy-hash` payload is `SHA-256(JCS(requirements.gpu_policy))`, using `{}` when the field is omitted. The `gpu-attestation` payload is JSON containing the verified device count, CC/DevTools state, and `evidence_sha256`. +After `compose-hash`, each configured init script produces an ordered +`init-script-hash` event whose payload is the SHA-256 digest of the exact UTF-8 +script bytes. The event order matches the script array order. These events let +an infrastructure provider contribute initialization code approved by +multiple parties and let each party verify its code independently without +reconstructing the complete compose document. + +For a GPU launch, any `init-script-hash` events are followed by +`gpu-policy-hash` and, after successful NVIDIA attestation and policy +evaluation, `gpu-attestation`. The `gpu-policy-hash` payload is +`SHA-256(JCS(requirements.gpu_policy))`, using `{}` when the field is omitted. +The `gpu-attestation` payload is JSON containing the verified device count, +CC/DevTools state, and `evidence_sha256`. The guest-agent `GpuInfo` API returns the complete `nvattest` JSON captured during boot. It is not trustworthy by itself. After verifying the TDX quote and replaying the event log to RTMR3, hash the exact UTF-8 bytes of `GpuInfo.attestation` and require the result to equal the `gpu-attestation` event's `evidence_sha256`. See [GPU Security for AI Workloads](./security/security-model.md#gpu-security-for-ai-workloads) for the event schema, ordering, Rego example, and platform differences. diff --git a/docs/aws-attested-instance-security-evaluation.md b/docs/aws-attested-instance-security-evaluation.md index d772f3022..4f3ca9682 100644 --- a/docs/aws-attested-instance-security-evaluation.md +++ b/docs/aws-attested-instance-security-evaluation.md @@ -35,7 +35,7 @@ NitroTPM path meets them today. | P1 | Verifiable platform root of trust | TDX/SNP/Nitro quote verification against vendor root; debug rejected; TCB surfaced | NitroTPM Attestation Documents are verified against the AWS Nitro Attestation PKI, including document timestamp sanity. AWS exposes no TDX/SNP-style TCB advisory field, so a verified attestation is normalized to `tcbStatus = "UpToDate"` and passes the standard authorization gate unchanged. | | P2 | Reproducible or independently computable base image measurement | meta-dstack rebuild plus `dstack-mr` computes `MRTD`/`RTMR0-2` | The unified `os/build.sh` flow emits the AWS image archive with `sha256sum.txt`, `digest.txt`, and `measurement.aws.cbor`; its output directory also contains the reference-PCR side-car `aws-pcrs.json`. `os_image_hash = sha256(sha256sum.txt)` is the same identity used on all platforms; the verifier recomputes it from the downloaded image directory (`dstack/verifier/src/verification.rs`). The hardening audit script `os/yocto/tools/aws/audit-aws-ec2-image-hardening.sh` checks the image for operator mutation channels. | | P3 | Boot command line and root filesystem integrity are measured | `RTMR1/2`, rootfs hash, dm-verity, measured initrd/cmdline | The UKI commits kernel, initrd, and embedded cmdline into `PCR4`; the rootfs is dm-verity-protected. `VmConfig.aws_measurement` is required and must bind `boot_pcr_digest = sha256(PCR4||PCR7||PCR12)` to the attested PCRs, so `PCR12` (external cmdline) is always part of the bound digest — a missing-PCR12 bypass is not expressible. Enforced in guest quote generation (`dstack/dstack-attest/src/attestation.rs`), `verify_os_image_hash_for_aws_nitro_tpm` (`dstack/verifier/src/verification.rs`), and the KMS pipeline via the same verifier check. | -| P4 | Runtime application identity is cryptographically bound | RTMR3 `compose-hash`, `app-id`, `instance-id`, `key-provider`; event log replay | SHA384 `PCR14` event-log replay is the authoritative binding (RTMR3 analogue; non-resettable). Launch events: `system-preparing`, `app-id`, `compose-hash`, `instance-id`, `boot-mr-done`, `key-provider`, `storage-fs`, `system-ready` (`dstack/dstack-util/src/system_setup.rs`). `dstack-attest`, `dstack-verifier`, and KMS reject missing/mismatched PCR14 and bad replay. Optionally, the guest extends the raw `MrConfig` V2 `config_id` into `PCR8` once (`PCR8 = sha384(0^48 || config_id)`) so a lightweight third-party verifier can check compose hash + key provider without event-log replay; dstack's own verifier and KMS do not check PCR8. | +| P4 | Runtime application identity is cryptographically bound | RTMR3 `compose-hash`, `app-id`, `instance-id`, `key-provider`; event log replay | SHA384 `PCR14` event-log replay is the authoritative binding (RTMR3 analogue; non-resettable). Launch events: `system-preparing`, `app-id`, `compose-hash`, zero or more ordered `init-script-hash` events, `instance-id`, `boot-mr-done`, `key-provider`, `storage-fs`, `system-ready` (`dstack/dstack-util/src/system_setup.rs`). GPU launches also include `gpu-policy-hash` and `gpu-attestation` before `instance-id`. `dstack-attest`, `dstack-verifier`, and KMS reject missing/mismatched PCR14 and bad replay. Optionally, the guest extends the raw `MrConfig` V2 `config_id` into `PCR8` once (`PCR8 = sha384(0^48 || config_id)`) so a lightweight third-party verifier can check compose hash + key provider without event-log replay; dstack's own verifier and KMS do not check PCR8. | | P5 | Challenge/liveness and caller key binding | `report_data` challenge or RA-TLS public key hash in quote | RA-TLS binds `report_data` to the TLS certificate public key. KMS key release is bound to the live RA-TLS handshake; external `/verify` callers supply and check their own `report_data` challenge. The low-level NitroTPM document verifier also rejects stale or far-future document timestamps. | | P6 | Secret release only to attested code | dstack KMS verifies attestation, checks auth policy, derives per-app keys | dstack KMS verifies the NitroTPM attestation, runs the same `verify_os_image_hash_for_aws_nitro_tpm` binding check as the verifier, builds `BootInfo` from verified boot PCRs plus PCR14 launch events, and checks auth policy before deriving app keys (`dstack/kms/src/main_service.rs`). AWS NitroTPM key release is gated behind the opt-in `aws_nitro_tpm_key_release` flag (default false in `kms.toml`). | | P7 | Key-release policy is not controlled by the untrusted account admin | KMS runs inside TEE; policy from auth API/contracts; KMS identity measured | Satisfied with dstack KMS or another verifiable secret authority outside the untrusted AWS account admin's control. A NitroTPM-backed dstack KMS keeps root material out of account-admin snapshots and clones. The policy backend (auth-simple in a trusted control plane, or on-chain `DstackKms`/`DstackApp`) must be outside the workload account admin's control. Same-account AWS KMS fails this property if the admin can change key policy, create grants, or call secret-bearing operations through a policy they control. | diff --git a/docs/security/cvm-boundaries.md b/docs/security/cvm-boundaries.md index ae668efbe..a47b833f2 100644 --- a/docs/security/cvm-boundaries.md +++ b/docs/security/cvm-boundaries.md @@ -41,11 +41,13 @@ This is the main configuration file for the application in JSON format: | no_instance_id | 0.4.2 | boolean | Disable instance ID generation | | secure_time | 0.5.0 | boolean | Whether secure time is enabled | | pre_launch_script | 0.4.0 | string | Prelaunch bash script that runs before execute `docker compose up` | -| init_script | 0.5.5 | string | Bash script that executed prior to dockerd startup | +| init_script | 0.5.5 (string), 0.6.0 (string[]) | string or string[] | Up to 5 Bash scripts executed in order prior to dockerd startup; a string is treated as a one-element array. Multiple scripts require string `manifest_version: "3"` so older guests fail closed. MrConfigV3 binds the hashes only for manifest v3. | | storage_fs | 0.5.5 | string | Filesystem type for the data disk of the CVM. Supported values: "zfs", "ext4". default to "zfs". **ZFS:** Ensures filesystem integrity with built-in data protection features. **ext4:** Provides better performance for database applications with lower overhead and faster I/O operations, but no strong integrity protection. | | swap_size | 0.5.5 | string/integer | The linux swap size. default to 0. Can be in byte or human-readable format (e.g., "1G", "256M"). | | key_provider | 0.5.6 | string | Key provider type. Supported values: "none", "kms", "local", "tpm". GCP vTPM and AWS EC2 NitroTPM are part of their platform trust models. The Dstack platform can use VMM-managed swtpm for seal/unseal and restart persistence, but it offers no protection against the host and is intentionally not accepted by remote verifiers. | +The five-script limit bounds runtime-event-log and MrConfigV3 growth while +allowing several independently approved infrastructure initialization stages. The hash of this file content is extended as the dstack `compose-hash` launch event. On TDX-family platforms the launch event is measured into RTMR3. On AWS NitroTPM it is measured into non-resettable SHA384 PCR14 before the `system-ready` launch boundary. Remote verifiers extract and replay this event during attestation. diff --git a/docs/security/security-model.md b/docs/security/security-model.md index a3614548e..63158b1dc 100644 --- a/docs/security/security-model.md +++ b/docs/security/security-model.md @@ -98,6 +98,16 @@ The policy must define the boolean rule `data.policy.nv_match`. Its `input` is t After measuring `compose-hash`, dstack enters the GPU setup gate and JCS-canonicalizes the original `requirements.gpu_policy` JSON value, then measures its SHA-256 digest in a `gpu-policy-hash` event. When the field is absent—including when `requirements` itself is absent—both parsing and measurement use the default empty object `{}`. Thus an omitted policy and an explicit `{}` have the same digest, while any explicitly present field, including an explicit default value, changes the digest. MrConfigV3 GPU launches also carry this digest as the optional `gpu_policy_hash` field; non-GPU launches omit it for compatibility. When the field is present, the guest compares it with the digest computed from app-compose; when it is absent, the guest skips this MrConfigV3 check. The MrConfigV3 document is bound by TDX `MR_CONFIG_ID` or SEV-SNP `HOST_DATA`, so the host cannot substitute a different GPU policy when this optional binding is present without changing the platform launch identity. The typed policy used for enforcement applies omitted-field defaults and rejects unknown fields. If GPU attestation is enabled and NVIDIA GPUs are present, dstack attests them and applies the basic settings and optional Rego policy before setting the GPU ready state. When no attestation claims are produced—because no GPU is attached or `gpu_policy.attest_gpu` is false—Rego is still evaluated with an empty array as `input` before any ready-state transition. This lets an application reject a launch whose attested GPU count is wrong. A false, undefined, malformed, or non-boolean Rego result stops boot before key provisioning. +Up to five init scripts may be configured. Their ordered SHA-256 digests are +measured as `init-script-hash` runtime events on platforms with a quoted +runtime register. MrConfigV3 also carries the ordered digest list; on SEV-SNP, +the signed report's `HOST_DATA` binds the exact canonical MrConfigV3 document. +An omitted `init_script_hashes` field disables this check, +while an explicit empty list requires app-compose to contain no init scripts. +When present, the guest compares the list with hashes computed from app-compose +before continuing boot. Current VMMs serialize the field explicitly for +manifest v3 launches, including `init_script_hashes: []` when the list is empty. + The policy digest is remotely verifiable on each supported platform, but through different carriers: - **TDX:** `gpu-policy-hash` contains the raw 32-byte digest and is measured into RTMR3. Replay the event log and compare the result with the quote's RTMR3, then compare the event payload with the expected digest. When an MrConfigV3 document includes `gpu_policy_hash`, TDX `MR_CONFIG_ID` additionally binds that field. @@ -114,6 +124,7 @@ For a successful TDX GPU launch, the GPU-relevant RTMR3 event order is: ```text compose-hash +init-script-hash (zero or more, in configured order) gpu-policy-hash gpu-attestation instance-id diff --git a/docs/tutorials/attestation-verification.md b/docs/tutorials/attestation-verification.md index b9661de2f..ea1cb1180 100644 --- a/docs/tutorials/attestation-verification.md +++ b/docs/tutorials/attestation-verification.md @@ -521,6 +521,7 @@ These are the standard events you'll see in the log: | `system-preparing` | System initialization marker | Always present | | `app-id` | Application identifier | Should match your app name | | `compose-hash` | SHA-256 of docker compose config | Should match `tcb_info.compose_hash` | +| `init-script-hash` | SHA-256 of one init script; repeated in configured order (maximum 5) | Should match the independently approved script bytes | | `gpu-policy-hash` | SHA-256 of the JCS-canonicalized GPU policy (default `{}`) | Should match the expected `requirements.gpu_policy` digest | | `gpu-attestation` | Verified GPU state and digest of the boot-time `nvattest` JSON | Required for an attested GPU launch; verify as described below | | `instance-id` | Unique instance identifier | Should match `instance_id` from response | @@ -531,8 +532,9 @@ These are the standard events you'll see in the log: | `storage-fs` | Storage filesystem type | Storage configuration | | `system-ready` | System ready marker | Always present at end | -For a successful GPU launch, the relevant order is `compose-hash`, -`gpu-policy-hash`, `gpu-attestation`, `instance-id`, and `boot-mr-done`. +For a successful GPU launch, the relevant order is `compose-hash`, any +`init-script-hash` events, `gpu-policy-hash`, `gpu-attestation`, `instance-id`, +and `boot-mr-done`. After replaying the log to the quote's RTMR3, decode the JSON payload of `gpu-attestation` and compare its `evidence_sha256` with the SHA-256 digest of the exact UTF-8 `GpuInfo.attestation` string. `GpuInfo` reads the result saved diff --git a/dstack/dstack-attest/src/attestation.rs b/dstack/dstack-attest/src/attestation.rs index 8c78d1867..111b6920d 100644 --- a/dstack/dstack-attest/src/attestation.rs +++ b/dstack/dstack-attest/src/attestation.rs @@ -390,6 +390,19 @@ fn find_event_payload(runtime_events: &[RuntimeEvent], event: &str) -> Result Vec> { + runtime_events + .iter() + .take_while(|event| event.event != "system-ready") + .filter(|event| event.event == name) + .map(|event| event.payload.clone()) + .collect() +} + fn decode_vm_config_with_fallback(config: &str, fallback_config: &str) -> Result { let config = if config.is_empty() { fallback_config @@ -890,6 +903,7 @@ impl AttestationV1 { key_provider_info, os_image_hash, compose_hash, + init_script_hashes: Some(find_event_payloads(runtime_events, "init-script-hash")), } }; @@ -1615,6 +1629,7 @@ fn decode_app_info_sev_snp( key_provider_info, os_image_hash, compose_hash: mr_config.compose_hash, + init_script_hashes: mr_config.init_script_hashes, }) } @@ -1937,6 +1952,10 @@ impl Attestation { key_provider_info, os_image_hash, compose_hash, + init_script_hashes: Some(find_event_payloads( + &self.runtime_events, + "init-script-hash", + )), } }; @@ -2027,6 +2046,12 @@ impl Attestation { self.find_event(event).map(|event| event.payload) } + /// SHA-256 payloads of all measured init scripts, in execution order. + /// Application-emitted events after `system-ready` are excluded. + pub fn decode_init_script_hashes(&self) -> Vec> { + find_event_payloads(&self.runtime_events, "init-script-hash") + } + fn find_event_hex_payload(&self, event: &str) -> Result { self.find_event(event) .map(|event| hex::encode(&event.payload)) @@ -2491,12 +2516,52 @@ pub struct AppInfo { /// Key provider info #[serde(with = "hex_bytes")] pub key_provider_info: Vec, + /// Optional SHA-256 pins for init scripts, in execution order. `None` + /// means the evidence did not bind this field. On SEV-SNP, `Some(vec![])` + /// explicitly binds an empty script list. On TDX and Nitro it only means + /// that no `init-script-hash` events were measured before `system-ready`; + /// pre-0.6.0 images emit no such events even when they run an init script. + #[serde(default, with = "dstack_types::init_script_hashes::option")] + pub init_script_hashes: Option>>, } #[cfg(test)] mod tests { use super::*; + #[test] + fn app_info_defaults_missing_init_script_hashes() { + let app_info: AppInfo = serde_json::from_value(serde_json::json!({ + "app_id": "", + "compose_hash": "", + "instance_id": "", + "device_id": "", + "mr_system": "0000000000000000000000000000000000000000000000000000000000000000", + "mr_aggregated": "0000000000000000000000000000000000000000000000000000000000000000", + "os_image_hash": "", + "key_provider_info": "" + })) + .unwrap(); + assert!(app_info.init_script_hashes.is_none()); + } + + #[test] + fn app_info_preserves_explicit_empty_init_script_hashes() { + let app_info: AppInfo = serde_json::from_value(serde_json::json!({ + "app_id": "", + "compose_hash": "", + "instance_id": "", + "device_id": "", + "mr_system": "0000000000000000000000000000000000000000000000000000000000000000", + "mr_aggregated": "0000000000000000000000000000000000000000000000000000000000000000", + "os_image_hash": "", + "key_provider_info": "", + "init_script_hashes": [] + })) + .unwrap(); + assert_eq!(app_info.init_script_hashes, Some(Vec::new())); + } + #[test] fn external_trust_anchor_requires_explicit_insecure_opt_in() { let config = AttestationVerifierConfig { @@ -2826,6 +2891,21 @@ mod tests { RuntimeEvent::new(event, payload, EventLogVersion::V1) } + #[test] + fn init_script_hashes_exclude_application_events_after_system_ready() { + let events = vec![ + v1_event("init-script-hash".into(), vec![0x11; 32]), + v1_event("init-script-hash".into(), vec![0x22; 32]), + v1_event("system-ready".into(), Vec::new()), + v1_event("init-script-hash".into(), vec![0xff; 32]), + ]; + + assert_eq!( + find_event_payloads(&events, "init-script-hash"), + vec![vec![0x11; 32], vec![0x22; 32]] + ); + } + #[test] fn nitro_pcrs_from_verified_extracts_0_1_2() { let mut map = std::collections::BTreeMap::new(); diff --git a/dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap b/dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap index cc08badca..d62ea8cc0 100644 --- a/dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap +++ b/dstack/dstack-attest/tests/snapshots/nitro_verify__app_info.snap @@ -1,6 +1,6 @@ --- source: dstack-attest/tests/nitro_verify.rs -assertion_line: 25 +assertion_line: 29 expression: app_info_str --- { @@ -11,5 +11,6 @@ expression: app_info_str "mr_system": "1894b0b29e94a9db16e88a2914f0923e52bb16c08bf3bb484df786a147e2eb79", "mr_aggregated": "1894b0b29e94a9db16e88a2914f0923e52bb16c08bf3bb484df786a147e2eb79", "os_image_hash": "1894b0b29e94a9db16e88a2914f0923e52bb16c08bf3bb484df786a147e2eb79", - "key_provider_info": "" + "key_provider_info": "", + "init_script_hashes": [] } diff --git a/dstack/dstack-types/src/lib.rs b/dstack/dstack-types/src/lib.rs index 141ed3bb9..55b81984f 100644 --- a/dstack/dstack-types/src/lib.rs +++ b/dstack/dstack-types/src/lib.rs @@ -10,6 +10,89 @@ use serde::{Deserialize, Serialize}; use serde_human_bytes as hex_bytes; use size_parser::human_size; +/// Bound event-log growth and MrConfigV3 size while supporting independent +/// infrastructure-provider initialization stages. +pub const MAX_INIT_SCRIPTS: usize = 5; + +pub mod init_script_hashes { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use serde_human_bytes::ByteBuf; + + pub fn serialize(values: &[Vec], serializer: S) -> Result + where + S: Serializer, + { + values + .iter() + .cloned() + .map(ByteBuf::from) + .collect::>() + .serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> + where + D: Deserializer<'de>, + { + let values = Vec::::deserialize(deserializer)?; + if values.len() > super::MAX_INIT_SCRIPTS { + return Err(serde::de::Error::custom(format!( + "init_script_hashes supports at most {} hashes", + super::MAX_INIT_SCRIPTS + ))); + } + if values.iter().any(|value| value.len() != 32) { + return Err(serde::de::Error::custom( + "each init_script_hash must be 32 bytes", + )); + } + Ok(values.into_iter().map(ByteBuf::into_vec).collect()) + } + + pub mod option { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use serde_human_bytes::ByteBuf; + + pub fn serialize(values: &Option>>, serializer: S) -> Result + where + S: Serializer, + { + values + .as_ref() + .map(|values| { + values + .iter() + .cloned() + .map(ByteBuf::from) + .collect::>() + }) + .serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result>>, D::Error> + where + D: Deserializer<'de>, + { + Option::>::deserialize(deserializer)? + .map(|values| { + if values.len() > super::super::MAX_INIT_SCRIPTS { + return Err(serde::de::Error::custom(format!( + "init_script_hashes supports at most {} hashes", + super::super::MAX_INIT_SCRIPTS + ))); + } + if values.iter().any(|value| value.len() != 32) { + return Err(serde::de::Error::custom( + "each init_script_hash must be 32 bytes", + )); + } + Ok(values.into_iter().map(ByteBuf::into_vec).collect()) + }) + .transpose() + } + } +} + /// Identifies which OVMF flavour the guest image was built with. /// /// Only the pre-202505 OVMF measurement layout is supported. @@ -131,6 +214,17 @@ pub struct AppCompose { pub snapshotter: Option, #[serde(default)] pub docker_compose_file: Option, + /// Bash scripts executed before the application runner starts. + /// + /// A single string is accepted for backward compatibility and is treated + /// as a one-element list. + #[serde( + default, + deserialize_with = "deserialize_init_scripts", + serialize_with = "serialize_init_scripts", + skip_serializing_if = "Vec::is_empty" + )] + pub init_script: Vec, #[serde(default)] pub public_logs: bool, #[serde(default)] @@ -139,7 +233,11 @@ pub struct AppCompose { pub public_tcbinfo: bool, #[serde(default)] pub kms_enabled: bool, - #[serde(deserialize_with = "deserialize_gateway_enabled", flatten)] + #[serde( + deserialize_with = "deserialize_gateway_enabled", + serialize_with = "serialize_gateway_enabled", + flatten + )] pub gateway_enabled: bool, #[serde(default)] pub local_key_provider_enabled: bool, @@ -178,6 +276,41 @@ pub struct AppCompose { pub verity_volumes: Vec, } +fn deserialize_init_scripts<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum InitScripts { + One(String), + Many(Vec), + } + + let scripts = match Option::::deserialize(deserializer)? { + None => Vec::new(), + Some(InitScripts::One(script)) => vec![script], + Some(InitScripts::Many(scripts)) => scripts, + }; + if scripts.len() > MAX_INIT_SCRIPTS { + return Err(serde::de::Error::custom(format!( + "init_script supports at most {MAX_INIT_SCRIPTS} scripts" + ))); + } + Ok(scripts) +} + +fn serialize_init_scripts(scripts: &[String], serializer: S) -> Result +where + S: serde::Serializer, +{ + if let [script] = scripts { + serializer.serialize_str(script) + } else { + scripts.serialize(serializer) + } +} + /// A pre-baked, read-only dm-verity volume attached to the CVM. #[derive(Deserialize, Serialize, Debug, Clone)] pub struct VerityVolume { @@ -490,6 +623,21 @@ where Ok(value.gateway_enabled || value.tproxy_enabled) } +fn serialize_gateway_enabled(enabled: &bool, serializer: S) -> Result +where + S: serde::Serializer, +{ + #[derive(Serialize)] + struct GatewayEnabled { + gateway_enabled: bool, + } + + GatewayEnabled { + gateway_enabled: *enabled, + } + .serialize(serializer) +} + #[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum KeyProviderKind { @@ -568,6 +716,71 @@ mod app_compose_tests { })) } + #[test] + fn init_script_accepts_string_array_and_null() { + assert!(parse_compose(serde_json::json!(2)) + .unwrap() + .init_script + .is_empty()); + + let single: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": "echo one" + })) + .unwrap(); + assert_eq!(single.init_script, ["echo one"]); + assert_eq!( + serde_json::to_value(&single).unwrap()["init_script"], + "echo one" + ); + + let multiple: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": ["echo one", "echo two"] + })) + .unwrap(); + assert_eq!(multiple.init_script, ["echo one", "echo two"]); + assert_eq!( + serde_json::to_value(&multiple).unwrap()["init_script"], + serde_json::json!(["echo one", "echo two"]) + ); + + let null: AppCompose = serde_json::from_value(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": null + })) + .unwrap(); + assert!(null.init_script.is_empty()); + assert!(serde_json::to_value(&null) + .unwrap() + .get("init_script") + .is_none()); + + assert!(serde_json::from_value::(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": ["echo one", 2] + })) + .is_err()); + + assert!(serde_json::from_value::(serde_json::json!({ + "manifest_version": 2, + "name": "test", + "runner": "docker-compose", + "init_script": ["1", "2", "3", "4", "5", "6"] + })) + .unwrap_err() + .to_string() + .contains("at most 5")); + } + #[test] fn manifest_version_accepts_string_versions() { let compose = parse_compose(serde_json::json!("3")).unwrap(); diff --git a/dstack/dstack-types/src/mr_config.rs b/dstack/dstack-types/src/mr_config.rs index 996a68695..ce38f4e7c 100644 --- a/dstack/dstack-types/src/mr_config.rs +++ b/dstack/dstack-types/src/mr_config.rs @@ -121,6 +121,10 @@ pub struct MrConfigV3 { pub key_provider_id: Option>, #[serde(default, with = "hex_bytes")] pub instance_id: Option>, + /// Optional SHA-256 pins for init scripts, in execution order. An omitted + /// field disables this check; an empty list requires no init scripts. + #[serde(default, with = "crate::init_script_hashes::option")] + pub init_script_hashes: Option>>, } impl MrConfigV3 { @@ -140,9 +144,15 @@ impl MrConfigV3 { key_provider, key_provider_id: (!key_provider_id.is_empty()).then_some(key_provider_id), instance_id: (!instance_id.is_empty()).then_some(instance_id), + init_script_hashes: None, } } + pub fn with_init_script_hashes(mut self, init_script_hashes: Vec>) -> Self { + self.init_script_hashes = Some(init_script_hashes); + self + } + pub fn to_snp_host_data(&self) -> [u8; 32] { Self::snp_host_data_from_document(&self.to_canonical_json()) } @@ -249,6 +259,29 @@ mod tests { Ok(()) } + #[test] + fn mr_config_v3_binds_ordered_init_script_hashes() -> Result<(), Box> { + let config = MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + None, + KeyProviderKind::None, + Vec::new(), + vec![0x44; 20], + ) + .with_init_script_hashes(vec![vec![0xaa; 32], vec![0xbb; 32]]); + let document = config.to_canonical_json(); + let decoded = MrConfigV3::from_document(&document)?; + + assert_eq!(decoded.init_script_hashes, config.init_script_hashes); + let reordered = MrConfigV3 { + init_script_hashes: Some(vec![vec![0xbb; 32], vec![0xaa; 32]]), + ..config.clone() + }; + assert_ne!(config.to_snp_host_data(), reordered.to_snp_host_data()); + Ok(()) + } + #[test] fn mr_config_v3_generates_jcs_but_hashes_document_bytes() -> Result<(), Box> { let config = MrConfigV3::new( diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 6f258d34b..8cd68b4b1 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -785,6 +785,11 @@ fn verify_manifest_feature_requirements(app_compose: &AppCompose) -> Result<()> "nerdctl-compose requires manifest_version >= {MANIFEST_VERSION_3}; use string manifest_version \"{MANIFEST_VERSION_3}\" so older guests fail closed" ); } + if app_compose.init_script.len() > 1 && manifest_version < MANIFEST_VERSION_3 { + bail!( + "multiple init scripts require manifest_version >= {MANIFEST_VERSION_3}; use string manifest_version \"{MANIFEST_VERSION_3}\" so older guests fail closed" + ); + } if app_compose.runner != "nerdctl-compose" && app_compose.snapshotter.is_some() { bail!("snapshotter is only supported by the nerdctl-compose runner"); } @@ -1969,6 +1974,7 @@ struct AppInfo { instance_info: InstanceInfo, compose_hash: [u8; 32], gpu_policy_hash: [u8; 32], + init_script_hashes: Vec>, } struct Stage0<'a> { @@ -2586,6 +2592,16 @@ impl<'a> Stage0<'a> { emit_runtime_event("system-preparing", &[])?; emit_runtime_event("app-id", &instance_info.app_id)?; emit_runtime_event("compose-hash", &compose_hash)?; + let init_script_hashes: Vec> = self + .shared + .app_compose + .init_script + .iter() + .map(|script| sha256(script.as_bytes()).to_vec()) + .collect(); + for script_hash in &init_script_hashes { + emit_runtime_event("init-script-hash", script_hash)?; + } let gpu_policy_hash = self .measure_gpu() .await @@ -2619,6 +2635,7 @@ impl<'a> Stage0<'a> { instance_info, compose_hash, gpu_policy_hash, + init_script_hashes, }) } @@ -2626,6 +2643,7 @@ impl<'a> Stage0<'a> { config_id_verifier::verify_mr_config_id( &app_info.compose_hash, &app_info.gpu_policy_hash, + &app_info.init_script_hashes, &app_info .instance_info .app_id @@ -3162,6 +3180,19 @@ fn test_nerdctl_compose_requires_v3_manifest() { verify_manifest_feature_requirements(&app_compose).unwrap(); } +#[test] +fn test_multiple_init_scripts_require_v3_manifest() { + let mut app_compose = test_app_compose(serde_json::json!(2), None, None); + app_compose.init_script = vec!["echo one".into(), "echo two".into()]; + let err = verify_manifest_feature_requirements(&app_compose).unwrap_err(); + assert!(err + .to_string() + .contains("multiple init scripts require manifest_version")); + + app_compose.manifest_version = "3".into(); + verify_manifest_feature_requirements(&app_compose).unwrap(); +} + #[test] fn test_snapshotter_is_rejected_for_other_runners() { let mut app_compose = test_app_compose(serde_json::json!("3"), None, None); diff --git a/dstack/dstack-util/src/system_setup/config_id_verifier.rs b/dstack/dstack-util/src/system_setup/config_id_verifier.rs index 02c905e57..89f888b8c 100644 --- a/dstack/dstack-util/src/system_setup/config_id_verifier.rs +++ b/dstack/dstack-util/src/system_setup/config_id_verifier.rs @@ -15,6 +15,7 @@ use tracing::info; struct LocalMrConfigValues<'a> { compose_hash: &'a [u8; 32], gpu_policy_hash: &'a [u8; 32], + init_script_hashes: &'a [Vec], app_id: &'a [u8; 20], instance_id: &'a [u8], key_provider: KeyProviderKind, @@ -67,6 +68,7 @@ fn read_snp_host_data() -> Result<[u8; 32]> { pub fn verify_mr_config_id( compose_hash: &[u8; 32], gpu_policy_hash: &[u8; 32], + init_script_hashes: &[Vec], app_id: &[u8; 20], instance_id: &[u8], key_provider: KeyProviderKind, @@ -76,6 +78,7 @@ pub fn verify_mr_config_id( let local = LocalMrConfigValues { compose_hash, gpu_policy_hash, + init_script_hashes, app_id, instance_id, key_provider, @@ -168,6 +171,11 @@ fn verify_mr_config_v3_document( bail!("Invalid mr_config gpu_policy_hash"); } } + if let Some(init_script_hashes) = mr_config.init_script_hashes.as_deref() { + if init_script_hashes != local.init_script_hashes { + bail!("Invalid mr_config init_script_hashes"); + } + } if let Some(app_id) = mr_config.app_id.as_deref() { if app_id != local.app_id { bail!("Invalid mr_config app_id"); @@ -221,6 +229,7 @@ mod tests { let local = LocalMrConfigValues { compose_hash: &compose_hash, gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &[], app_id: &app_id, instance_id: &instance_id, key_provider: KeyProviderKind::Kms, @@ -250,6 +259,7 @@ mod tests { let local = LocalMrConfigValues { compose_hash: &compose_hash, gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &[], app_id: &wrong_app_id, instance_id: &instance_id, key_provider: KeyProviderKind::Kms, @@ -281,6 +291,7 @@ mod tests { let local = LocalMrConfigValues { compose_hash: &compose_hash, gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &[], app_id: &app_id, instance_id: &instance_id, key_provider: KeyProviderKind::Kms, @@ -311,6 +322,7 @@ mod tests { let local = LocalMrConfigValues { compose_hash: &compose_hash, gpu_policy_hash: &wrong_gpu_policy_hash, + init_script_hashes: &[], app_id: &app_id, instance_id: &instance_id, key_provider: KeyProviderKind::Kms, @@ -344,6 +356,7 @@ mod tests { let local = LocalMrConfigValues { compose_hash: &compose_hash, gpu_policy_hash: &actual_gpu_policy_hash, + init_script_hashes: &[], app_id: &app_id, instance_id: &instance_id, key_provider: KeyProviderKind::Kms, @@ -352,4 +365,77 @@ mod tests { verify_tdx_mr_config_id_value(mr_config.to_tdx_mr_config_id(), Some(&document), local) } + + #[test] + fn mr_config_v3_document_rejects_mismatched_init_script_hashes() { + let compose_hash = [0x22u8; 32]; + let gpu_policy_hash = [0x55u8; 32]; + let app_id = [0x11u8; 20]; + let instance_id = [0x44u8; 20]; + let declared_hashes = vec![vec![0xaau8; 32]]; + let actual_hashes = vec![vec![0xbbu8; 32]]; + let document = MrConfigV3::new( + app_id.to_vec(), + compose_hash.to_vec(), + None, + KeyProviderKind::None, + Vec::new(), + instance_id.to_vec(), + ) + .with_init_script_hashes(declared_hashes) + .to_canonical_json(); + let local = LocalMrConfigValues { + compose_hash: &compose_hash, + gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &actual_hashes, + app_id: &app_id, + instance_id: &instance_id, + key_provider: KeyProviderKind::None, + key_provider_id: &[], + }; + + assert!(verify_mr_config_v3_document(&document, local) + .unwrap_err() + .to_string() + .contains("Invalid mr_config init_script_hashes")); + } + + #[test] + fn mr_config_v3_document_skips_init_script_check_when_field_is_missing() { + let compose_hash = [0x22u8; 32]; + let gpu_policy_hash = [0x55u8; 32]; + let app_id = [0x11u8; 20]; + let instance_id = [0x44u8; 20]; + let mut document = serde_json::to_value(MrConfigV3::new( + app_id.to_vec(), + compose_hash.to_vec(), + None, + KeyProviderKind::None, + Vec::new(), + instance_id.to_vec(), + )) + .unwrap(); + document + .as_object_mut() + .unwrap() + .remove("init_script_hashes"); + let local_without_scripts = LocalMrConfigValues { + compose_hash: &compose_hash, + gpu_policy_hash: &gpu_policy_hash, + init_script_hashes: &[], + app_id: &app_id, + instance_id: &instance_id, + key_provider: KeyProviderKind::None, + key_provider_id: &[], + }; + + verify_mr_config_v3_document(&document.to_string(), local_without_scripts).unwrap(); + + let actual_hashes = vec![vec![0xaau8; 32]]; + let local_with_script = LocalMrConfigValues { + init_script_hashes: &actual_hashes, + ..local_without_scripts + }; + verify_mr_config_v3_document(&document.to_string(), local_with_script).unwrap(); + } } diff --git a/dstack/guest-agent/src/rpc_service.rs b/dstack/guest-agent/src/rpc_service.rs index e2719786e..3a52ec1f6 100644 --- a/dstack/guest-agent/src/rpc_service.rs +++ b/dstack/guest-agent/src/rpc_service.rs @@ -739,6 +739,7 @@ mod tests { runner: String::new(), snapshotter: None, docker_compose_file: None, + init_script: Vec::new(), public_logs: false, public_sysinfo: false, public_tcbinfo: false, diff --git a/dstack/verifier/src/types.rs b/dstack/verifier/src/types.rs index 1d4851526..34665d27a 100644 --- a/dstack/verifier/src/types.rs +++ b/dstack/verifier/src/types.rs @@ -203,6 +203,7 @@ mod tests { mr_aggregated: [0x66; 32], os_image_hash: vec![0x77; 32], key_provider_info: br#"{"name":"tpm","id":"aws-test"}"#.to_vec(), + init_script_hashes: Some(Vec::new()), }; let boot_info = PolicyBootInfo::from_app_info( diff --git a/dstack/vmm/src/app/mr_config.rs b/dstack/vmm/src/app/mr_config.rs index d607255d0..e9aa83539 100644 --- a/dstack/vmm/src/app/mr_config.rs +++ b/dstack/vmm/src/app/mr_config.rs @@ -14,6 +14,18 @@ use sha2::{Digest, Sha256}; use super::{GpuConfig, Manifest, VmWorkDir}; use crate::config::{CvmConfig, CvmPlatform}; +fn bind_init_script_hashes(mut mr_config: MrConfigV3, app_compose: &AppCompose) -> MrConfigV3 { + if app_compose.manifest_version_u32().unwrap_or_default() >= 3 { + let hashes = app_compose + .init_script + .iter() + .map(|script| Sha256::digest(script.as_bytes()).to_vec()) + .collect(); + mr_config = mr_config.with_init_script_hashes(hashes); + } + mr_config +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum MrConfigVersion { V1, @@ -167,14 +179,61 @@ impl VmWorkDir { ) .context("failed to write instance info")?; - Ok(MrConfigV3::new( + // Manifest v3 is the fail-closed capability signal: older guests + // reject its string version instead of seeing an unknown MrConfigV3 + // field generated by a newer VMM. + let mr_config = MrConfigV3::new( app_id, compose_hash.to_vec(), gpu_policy_hash, app_compose.key_provider(), app_compose.key_provider_id.clone(), instance_id, + ); + Ok(bind_init_script_hashes(mr_config, app_compose).to_canonical_json()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use dstack_types::KeyProviderKind; + + fn app_compose(manifest_version: serde_json::Value) -> AppCompose { + serde_json::from_value(serde_json::json!({ + "manifest_version": manifest_version, + "name": "test", + "runner": "docker-compose" + })) + .unwrap() + } + + fn base_mr_config() -> MrConfigV3 { + MrConfigV3::new( + vec![0x11; 20], + vec![0x22; 32], + None, + KeyProviderKind::None, + Vec::new(), + vec![0x33; 20], ) - .to_canonical_json()) + } + + #[test] + fn manifest_v2_omits_init_script_hashes() { + let document = + bind_init_script_hashes(base_mr_config(), &app_compose(serde_json::json!(2))) + .to_canonical_json(); + let document: serde_json::Value = serde_json::from_str(&document).unwrap(); + assert!(document.get("init_script_hashes").is_none()); + } + + #[test] + fn manifest_v3_includes_empty_init_script_hashes() { + let document = + bind_init_script_hashes(base_mr_config(), &app_compose(serde_json::json!("3"))) + .to_canonical_json(); + let document: serde_json::Value = serde_json::from_str(&document).unwrap(); + assert_eq!(document["init_script_hashes"], serde_json::json!([])); } } diff --git a/dstack/vmm/ui/src/components/CreateVmDialog.ts b/dstack/vmm/ui/src/components/CreateVmDialog.ts index b484602f1..82653a62d 100644 --- a/dstack/vmm/ui/src/components/CreateVmDialog.ts +++ b/dstack/vmm/ui/src/components/CreateVmDialog.ts @@ -108,10 +108,15 @@ const CreateVmDialogComponent = {
-
diff --git a/dstack/vmm/ui/src/components/UpdateVmDialog.ts b/dstack/vmm/ui/src/components/UpdateVmDialog.ts index f994f0487..5dc9568b1 100644 --- a/dstack/vmm/ui/src/components/UpdateVmDialog.ts +++ b/dstack/vmm/ui/src/components/UpdateVmDialog.ts @@ -92,10 +92,15 @@ const UpdateVmDialogComponent = {
-