From c54960df25d3985b612a3a64597617b117cfbb26 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 29 Jul 2026 17:17:48 +0000 Subject: [PATCH 1/3] feat(kms): return configured historical root keys --- dstack/kms/kms.toml | 4 ++++ dstack/kms/src/config.rs | 10 ++++++++++ dstack/kms/src/main_service.rs | 35 +++++++++++++++++++++++++++++----- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/dstack/kms/kms.toml b/dstack/kms/kms.toml index 08e954662..25355c3bc 100644 --- a/dstack/kms/kms.toml +++ b/dstack/kms/kms.toml @@ -24,6 +24,10 @@ mandatory = false [core] cert_dir = "/etc/kms/certs" +# Previous root-key pairs for authorized handover, ordered newest to oldest: +# [[core.historical_keys]] +# ca_key = "/etc/kms/history/previous/root-ca.key" +# k256_key = "/etc/kms/history/previous/root-k256.key" subject_postfix = ".dstack" site_name = "" # Whether trusted RPCs require the KMS to first attest itself to its own diff --git a/dstack/kms/src/config.rs b/dstack/kms/src/config.rs index 4d95b0e17..11da915da 100644 --- a/dstack/kms/src/config.rs +++ b/dstack/kms/src/config.rs @@ -23,6 +23,12 @@ const RPC_DOMAIN: &str = "rpc-domain"; const K256_KEY: &str = "root-k256.key"; const BOOTSTRAP_INFO: &str = "bootstrap-info.json"; +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct HistoricalKeyConfig { + pub ca_key: PathBuf, + pub k256_key: PathBuf, +} + #[derive(Debug, Clone, Deserialize)] pub(crate) struct ImageConfig { pub verify: bool, @@ -35,6 +41,10 @@ pub(crate) struct ImageConfig { #[derive(Debug, Clone, Deserialize)] pub(crate) struct KmsConfig { pub cert_dir: PathBuf, + /// Previous root-key pairs returned after the current pair during an + /// authorized KMS handover. Configuration order is rotation order. + #[serde(default)] + pub historical_keys: Vec, #[serde(default)] pub attestation: AttestationVerifierConfig, pub auth_api: AuthApi, diff --git a/dstack/kms/src/main_service.rs b/dstack/kms/src/main_service.rs index 339fbd3bc..99043a3bf 100644 --- a/dstack/kms/src/main_service.rs +++ b/dstack/kms/src/main_service.rs @@ -32,7 +32,7 @@ use tracing::{info, warn}; use upgrade_authority::{build_boot_info, ensure_app_id_len, local_kms_boot_info, BootInfo}; use crate::{ - config::KmsConfig, + config::{HistoricalKeyConfig, KmsConfig}, crypto::{derive_k256_key, sign_message, sign_message_with_timestamp}, }; @@ -56,6 +56,7 @@ pub struct KmsStateInner { config: KmsConfig, root_ca: CaCert, k256_key: SigningKey, + historical_keys: Vec, temp_ca_cert: String, temp_ca_key: String, verifier: CvmVerifier, @@ -120,6 +121,24 @@ fn remove_cache(parent_dir: &Path, sub_dir: &str) -> Result<()> { Ok(()) } +fn load_historical_keys(configs: &[HistoricalKeyConfig]) -> Result> { + configs + .iter() + .enumerate() + .map(|(index, config)| { + let ca_key = fs::read_to_string(&config.ca_key) + .with_context(|| format!("Failed to read historical CA key {index}"))?; + ra_tls::rcgen::KeyPair::from_pem(&ca_key) + .with_context(|| format!("Failed to parse historical CA key {index}"))?; + let k256_key = fs::read(&config.k256_key) + .with_context(|| format!("Failed to read historical ECDSA key {index}"))?; + SigningKey::from_slice(&k256_key) + .with_context(|| format!("Failed to parse historical ECDSA key {index}"))?; + Ok(KmsKeys { ca_key, k256_key }) + }) + .collect() +} + impl KmsState { /// clear cached image and measurement material for the given hashes. Used by /// the admin `ClearImageCache` RPC; authorization is enforced by the admin @@ -139,6 +158,8 @@ impl KmsState { let key_bytes = fs::read(config.k256_key()).context("Failed to read ECDSA root key")?; let k256_key = SigningKey::from_slice(&key_bytes).context("Failed to load ECDSA root key")?; + let historical_keys = load_historical_keys(&config.historical_keys) + .context("Failed to load historical root keys")?; let temp_ca_key = fs::read_to_string(config.tmp_ca_key()).context("Faeild to read temp ca key")?; let temp_ca_cert = @@ -163,6 +184,7 @@ impl KmsState { config, root_ca, k256_key, + historical_keys, temp_ca_cert, temp_ca_key, verifier, @@ -485,12 +507,15 @@ impl KmsRpc for RpcHandler { self.state.config.sev_snp_key_release, self.state.config.aws_nitro_tpm_key_release, )?; + let mut keys = Vec::with_capacity(1 + self.state.inner.historical_keys.len()); + keys.push(KmsKeys { + ca_key: self.state.inner.root_ca.key.serialize_pem(), + k256_key: self.state.inner.k256_key.to_bytes().to_vec(), + }); + keys.extend(self.state.inner.historical_keys.iter().cloned()); Ok(KmsKeyResponse { temp_ca_key: self.state.inner.temp_ca_key.clone(), - keys: vec![KmsKeys { - ca_key: self.state.inner.root_ca.key.serialize_pem(), - k256_key: self.state.inner.k256_key.to_bytes().to_vec(), - }], + keys, }) } From cdaf03f2e8438cf963dbf8431358877d5941a190 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 29 Jul 2026 17:18:34 +0000 Subject: [PATCH 2/3] test(kms): cover historical root key inventory --- dstack/kms/src/main_service.rs | 62 ++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/dstack/kms/src/main_service.rs b/dstack/kms/src/main_service.rs index 99043a3bf..6ab1c3bc3 100644 --- a/dstack/kms/src/main_service.rs +++ b/dstack/kms/src/main_service.rs @@ -610,6 +610,68 @@ mod tests { use cc_eventlog::RuntimeEvent; use sha2::{Digest, Sha256, Sha384}; + fn historical_key_fixture( + directory: &Path, + name: &str, + scalar: u8, + ) -> (HistoricalKeyConfig, String, Vec) { + let key_dir = directory.join(name); + fs::create_dir_all(&key_dir).unwrap(); + let ca_key = ra_tls::rcgen::KeyPair::generate_for(&ra_tls::rcgen::PKCS_ECDSA_P256_SHA256) + .unwrap() + .serialize_pem(); + let k256_key = SigningKey::from_slice(&[scalar; 32]) + .unwrap() + .to_bytes() + .to_vec(); + let ca_path = key_dir.join("root-ca.key"); + let k256_path = key_dir.join("root-k256.key"); + fs::write(&ca_path, &ca_key).unwrap(); + fs::write(&k256_path, &k256_key).unwrap(); + ( + HistoricalKeyConfig { + ca_key: ca_path, + k256_key: k256_path, + }, + ca_key, + k256_key, + ) + } + + #[test] + fn historical_root_keys_preserve_configured_rotation_order() { + let directory = tempfile::tempdir().unwrap(); + let (newer, newer_ca, newer_k256) = historical_key_fixture(directory.path(), "newer", 0x21); + let (older, older_ca, older_k256) = historical_key_fixture(directory.path(), "older", 0x22); + let loaded = load_historical_keys(&[newer, older]).unwrap(); + + assert_eq!(loaded.len(), 2); + assert_eq!(loaded[0].ca_key, newer_ca); + assert_eq!(loaded[0].k256_key, newer_k256); + assert_eq!(loaded[1].ca_key, older_ca); + assert_eq!(loaded[1].k256_key, older_k256); + } + + #[test] + fn historical_root_keys_fail_closed_on_missing_or_malformed_material() { + let directory = tempfile::tempdir().unwrap(); + let missing = HistoricalKeyConfig { + ca_key: directory.path().join("missing-ca.key"), + k256_key: directory.path().join("missing-k256.key"), + }; + assert!(load_historical_keys(&[missing]).is_err()); + + let ca_path = directory.path().join("bad-ca.key"); + let k256_path = directory.path().join("bad-k256.key"); + fs::write(&ca_path, "not a PEM key").unwrap(); + fs::write(&k256_path, [0u8; 31]).unwrap(); + let malformed = HistoricalKeyConfig { + ca_key: ca_path, + k256_key: k256_path, + }; + assert!(load_historical_keys(&[malformed]).is_err()); + } + #[test] fn remove_cache_only_deletes_the_named_hex_entry() { let dir = tempfile::tempdir().unwrap(); From 7a87e219704ea41bed907793297b9e0771da56b6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 29 Jul 2026 17:30:17 +0000 Subject: [PATCH 3/3] docs(kms): define cold backup recovery procedure --- dstack/kms/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dstack/kms/README.md b/dstack/kms/README.md index 4ea96fbda..4d9241eb0 100644 --- a/dstack/kms/README.md +++ b/dstack/kms/README.md @@ -232,3 +232,21 @@ The `SignCert` RPC is used by the dstack app to sign a TLS certificate. In this - Verify the CSR signature - Query the smart contract to check if the app is authorized - If authorized, sign the CSR with the CA root key and return the certificate chain to the app + +## Cold backup and recovery + +KMS root material is backed up as a complete, offline copy of `core.cert_dir`. +There is no online backup RPC. Stop the KMS before taking or restoring a copy, +preserve file modes and ownership, and protect the backup with the same controls +as the live private keys. Do not copy only one key: the root CA, root K256 key, +temporary CA, RPC identity, domain, and public certificates form one identity +set. + +A supported recovery restores the complete directory while KMS is stopped, +verifies that every private-key file remains owner-only (`0600`), and then +starts KMS with the unchanged configuration. Compare `KMS.GetMeta` before the +backup and after recovery: `ca_cert` and `k256_pubkey` must match exactly. KMS +must fail closed when a required key is missing or malformed; never generate a +new identity to repair a partial restore. Orphaned `*.private-tmp` files from an +interrupted atomic write are not trust anchors and may be removed only after the +final key file has been verified.