From fa05044016194fdfc6da58fe15b50d10ed12d19d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 29 Jul 2026 08:09:07 +0000 Subject: [PATCH 1/3] fix(gateway): fail closed on corrupt ACME credentials --- dstack/gateway/src/distributed_certbot.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs index cbb316ea4..3d9f17758 100644 --- a/dstack/gateway/src/distributed_certbot.rs +++ b/dstack/gateway/src/distributed_certbot.rs @@ -318,7 +318,9 @@ impl DistributedCertBot { // Try to load global ACME credentials from KvStore if let Some(creds) = self.kv_store.get_acme_credentials() { - if acme_url_matches(&creds.acme_credentials, acme_url) { + if acme_url_matches(&creds.acme_credentials, acme_url) + .context("invalid ACME credentials in KvStore")? + { info!("loaded global ACME account credentials from KvStore"); return AcmeClient::load( dns01_client, @@ -495,15 +497,14 @@ fn get_cert_expiry(cert_pem: &str) -> Option { Some(cert.validity().not_after.timestamp() as u64) } -fn acme_url_matches(credentials_json: &str, expected_url: &str) -> bool { +fn acme_url_matches(credentials_json: &str, expected_url: &str) -> Result { #[derive(serde::Deserialize)] struct Creds { - #[serde(default)] acme_url: String, } - serde_json::from_str::(credentials_json) - .map(|c| c.acme_url == expected_url) - .unwrap_or(false) + let credentials = serde_json::from_str::(credentials_json) + .context("failed to decode ACME credentials")?; + Ok(credentials.acme_url == expected_url) } /// Extract account_id (URI) from ACME credentials JSON From 2e4c3541d8daeb340c5352ee6458db944051c27a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 29 Jul 2026 08:09:22 +0000 Subject: [PATCH 2/3] test(gateway): cover corrupt ACME credential handling --- dstack/gateway/src/distributed_certbot.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs index 3d9f17758..2adb8ed9f 100644 --- a/dstack/gateway/src/distributed_certbot.rs +++ b/dstack/gateway/src/distributed_certbot.rs @@ -519,3 +519,25 @@ fn extract_account_uri(credentials_json: &str) -> Option { .filter(|c| !c.account_id.is_empty()) .map(|c| c.account_id) } + +#[cfg(test)] +mod credential_tests { + use super::acme_url_matches; + + #[test] + fn corrupt_acme_credentials_fail_closed() { + assert!(acme_url_matches("not-json", "https://acme.test/directory").is_err()); + assert!(acme_url_matches("{}", "https://acme.test/directory").is_err()); + } + + #[test] + fn valid_acme_credentials_distinguish_directory() { + let credentials = r#"{"acme_url":"https://acme.test/directory"}"#; + assert!(acme_url_matches(credentials, "https://acme.test/directory") + .expect("valid credentials rejected")); + assert!( + !acme_url_matches(credentials, "https://other.test/directory") + .expect("valid credentials rejected") + ); + } +} From 4649a104b03c9380096e301dde6852c2c4f67d17 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 4 Aug 2026 19:29:16 -0700 Subject: [PATCH 3/3] feat(gateway): add safe ACME credential rotation --- dstack/gateway/rpc/proto/gateway_rpc.proto | 12 +++ dstack/gateway/src/admin_service.rs | 16 ++- dstack/gateway/src/distributed_certbot.rs | 116 ++++++++++++++++++--- dstack/gateway/src/main_service.rs | 4 + 4 files changed, 127 insertions(+), 21 deletions(-) diff --git a/dstack/gateway/rpc/proto/gateway_rpc.proto b/dstack/gateway/rpc/proto/gateway_rpc.proto index b7a58f9cd..c03d9f40d 100644 --- a/dstack/gateway/rpc/proto/gateway_rpc.proto +++ b/dstack/gateway/rpc/proto/gateway_rpc.proto @@ -164,6 +164,14 @@ message AcmeInfoResponse { string account_attestation = 5; } +// Result of replacing the shared ACME account credentials. +message RotateAcmeCredentialsResponse { + // URI of the newly-created ACME account. The private credentials are never returned. + string account_uri = 1; + // Number of ZT domains whose CAA records were updated for the new account. + uint32 domains_updated = 2; +} + // Get HostInfo for associated instance id. message GetInfoRequest { string id = 1; @@ -466,6 +474,10 @@ service Admin { rpc GetCertbotConfig(google.protobuf.Empty) returns (CertbotConfigResponse) {} // Set global certbot configuration (includes ACME URL) rpc SetCertbotConfig(SetCertbotConfigRequest) returns (google.protobuf.Empty) {} + // Create a new ACME account, update every ZT-domain CAA record, and then + // replace the shared credentials. Call only one gateway at a time because + // WaveKV does not provide compare-and-swap. + rpc RotateAcmeCredentials(google.protobuf.Empty) returns (RotateAcmeCredentialsResponse) {} // ==================== Per-Instance Port Policy Override ==================== // Set an admin override for an instance's port policy. Takes precedence diff --git a/dstack/gateway/src/admin_service.rs b/dstack/gateway/src/admin_service.rs index 60ba1f197..2042d78e0 100644 --- a/dstack/gateway/src/admin_service.rs +++ b/dstack/gateway/src/admin_service.rs @@ -18,10 +18,10 @@ use dstack_gateway_rpc::{ ListCertAttestationsResponse, ListDnsCredentialsResponse, ListZtDomainsResponse, NodeStatusEntry, PeerSyncStatus as ProtoPeerSyncStatus, PortAttrs as RpcPortAttrs, PortPolicy as RpcPortPolicy, RenewCertResponse, RenewZtDomainCertRequest, - RenewZtDomainCertResponse, SetCertbotConfigRequest, SetDefaultDnsCredentialRequest, - SetInstancePortPolicyRequest, SetNodeStatusRequest, SetNodeUrlRequest, StatusResponse, - StoreSyncStatus, UpdateDnsCredentialRequest, WaveKvStatusResponse, ZtDomainCertStatus, - ZtDomainConfig as ProtoZtDomainConfig, ZtDomainInfo, + RenewZtDomainCertResponse, RotateAcmeCredentialsResponse, SetCertbotConfigRequest, + SetDefaultDnsCredentialRequest, SetInstancePortPolicyRequest, SetNodeStatusRequest, + SetNodeUrlRequest, StatusResponse, StoreSyncStatus, UpdateDnsCredentialRequest, + WaveKvStatusResponse, ZtDomainCertStatus, ZtDomainConfig as ProtoZtDomainConfig, ZtDomainInfo, }; use ra_rpc::{CallContext, RpcCall}; use tracing::info; @@ -102,6 +102,14 @@ impl AdminRpc for AdminRpcHandler { self.state.reload_all_certs_from_kvstore() } + async fn rotate_acme_credentials(self) -> Result { + let (account_uri, domains_updated) = self.state.rotate_acme_credentials().await?; + Ok(RotateAcmeCredentialsResponse { + account_uri, + domains_updated: domains_updated.try_into().unwrap_or(u32::MAX), + }) + } + async fn status(self) -> Result { self.status().await } diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs index 2adb8ed9f..a10061b44 100644 --- a/dstack/gateway/src/distributed_certbot.rs +++ b/dstack/gateway/src/distributed_certbot.rs @@ -19,8 +19,8 @@ use tracing::{error, info, warn}; use crate::cert_store::CertResolver; use crate::kv::{ - AcmeAttestation, CertAttestation, CertCredentials, CertData, DnsProvider, KvStore, - ZtDomainConfig, + AcmeAttestation, CertAttestation, CertCredentials, CertData, DnsCredential, DnsProvider, + KvStore, ZtDomainConfig, }; /// Lock timeout for certificate renewal (10 minutes) @@ -33,6 +33,7 @@ const DEFAULT_ACME_URL: &str = "https://acme-v02.api.letsencrypt.org/directory"; pub struct DistributedCertBot { kv_store: Arc, cert_resolver: Arc, + rotation_lock: tokio::sync::Mutex<()>, } impl DistributedCertBot { @@ -40,9 +41,91 @@ impl DistributedCertBot { Self { kv_store, cert_resolver, + rotation_lock: tokio::sync::Mutex::new(()), } } + async fn dns_client(&self, domain: &str, config: &ZtDomainConfig) -> Result { + let dns_cred = if let Some(ref cred_id) = config.dns_cred_id { + self.kv_store + .get_dns_credential(cred_id) + .context("specified DNS credential not found")? + } else { + self.kv_store + .get_default_dns_credential() + .context("no default DNS credential configured")? + }; + + match &dns_cred.provider { + DnsProvider::Cloudflare { api_token, api_url } => { + Dns01Client::new_cloudflare(domain.to_string(), api_token.clone(), api_url.clone()) + .await + } + } + } + + /// Rotate the shared ACME account without interrupting certificate serving. + /// + /// CAA records are updated before the new credentials are published. WaveKV + /// has no CAS operation, so the lock only serializes calls handled by this + /// node; operators must not rotate through multiple nodes concurrently. + pub async fn rotate_acme_credentials(&self) -> Result<(String, usize)> { + let _guard = self.rotation_lock.lock().await; + let configs = self.kv_store.list_zt_domain_configs(); + let first = configs + .first() + .context("no ZT-Domain configured for ACME credential rotation")?; + let certbot_config = self.config(); + let acme_url = if certbot_config.acme_url.is_empty() { + DEFAULT_ACME_URL + } else { + &certbot_config.acme_url + }; + + let first_dns_cred = dns_credential_for(&self.kv_store, first)?; + let dns_client = self.dns_client(&first.domain, first).await?; + let client = AcmeClient::new_account( + acme_url, + dns_client, + first_dns_cred.max_dns_wait, + first_dns_cred.dns_txt_ttl, + ) + .await + .context("failed to create replacement ACME account")?; + let credentials = client + .dump_credentials() + .context("failed to encode replacement ACME credentials")?; + let account_uri = client.account_id().to_string(); + + for config in &configs { + let dns_cred = dns_credential_for(&self.kv_store, config)?; + let dns_client = self.dns_client(&config.domain, config).await?; + let client = AcmeClient::load( + dns_client, + &credentials, + dns_cred.max_dns_wait, + dns_cred.dns_txt_ttl, + ) + .await + .with_context(|| format!("failed to prepare ACME client for {}", config.domain))?; + client + .set_caa_records(&[format!("*.{}", config.domain)]) + .await + .with_context(|| format!("failed to update CAA for {}", config.domain))?; + } + + // Publish only after every CAA update succeeds. Readers create an ACME + // client per operation, so all nodes recover on their next retry after + // WaveKV propagates this value. + self.kv_store.save_acme_credentials(&CertCredentials { + acme_credentials: credentials, + })?; + if let Err(err) = self.generate_and_save_acme_attestation(&account_uri).await { + warn!("failed to attest rotated ACME account: {err:?}"); + } + Ok((account_uri, configs.len())) + } + /// Get the current certbot configuration from KV store fn config(&self) -> crate::kv::GlobalCertbotConfig { self.kv_store.get_certbot_config() @@ -290,23 +373,10 @@ impl DistributedCertBot { config: &ZtDomainConfig, ) -> Result { // Get DNS credential (from config or default) - let dns_cred = if let Some(ref cred_id) = config.dns_cred_id { - self.kv_store - .get_dns_credential(cred_id) - .context("specified DNS credential not found")? - } else { - self.kv_store - .get_default_dns_credential() - .context("no default DNS credential configured")? - }; + let dns_cred = dns_credential_for(&self.kv_store, config)?; // Create DNS client based on provider - let dns01_client = match &dns_cred.provider { - DnsProvider::Cloudflare { api_token, api_url } => { - Dns01Client::new_cloudflare(domain.to_string(), api_token.clone(), api_url.clone()) - .await? - } - }; + let dns01_client = self.dns_client(domain, config).await?; // Use ACME URL from certbot config, fall back to default if not set let config = self.config(); @@ -483,6 +553,18 @@ impl DistributedCertBot { } } +fn dns_credential_for(kv_store: &KvStore, config: &ZtDomainConfig) -> Result { + if let Some(ref cred_id) = config.dns_cred_id { + kv_store + .get_dns_credential(cred_id) + .context("specified DNS credential not found") + } else { + kv_store + .get_default_dns_credential() + .context("no default DNS credential configured") + } +} + fn now_secs() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/dstack/gateway/src/main_service.rs b/dstack/gateway/src/main_service.rs index df06f2cb5..d01329fc0 100644 --- a/dstack/gateway/src/main_service.rs +++ b/dstack/gateway/src/main_service.rs @@ -418,6 +418,10 @@ impl Proxy { } } + pub(crate) async fn rotate_acme_credentials(&self) -> Result<(String, usize)> { + self.certbot.rotate_acme_credentials().await + } + /// Get ACME info for all managed domains (or a specific domain) pub(crate) fn acme_info(&self, domain: Option<&str>) -> Result { let kv_store = self.kv_store.clone();