From 09830b9c492b85abeb711894a81651057c46a281 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Mon, 17 Aug 2026 05:29:46 +0800 Subject: [PATCH 1/2] feat(gateway): add VerifyDnsCredential admin RPC Read-only check of a DNS credential against the provider API: reports the Cloudflare token status (active/expired/disabled/invalid), the granted permission groups, whether issuance requirements (Zone Read, DNS Write) are covered, and zone resolution for every domain referencing the credential. Today an invalid or under-privileged token only surfaces as a log line during the next renewal attempt. certbot gains verify_cloudflare_token() (GET /user/tokens/verify followed by GET /user/tokens/{id}) and resolve_cloudflare_zone(); the mock CF API learns the matching token endpoints, with MOCK_TOKEN_STATUS and MOCK_TOKEN_PERMISSIONS env knobs for failure testing. --- dstack/certbot/src/dns01_client.rs | 22 +++ dstack/certbot/src/dns01_client/cloudflare.rs | 143 +++++++++++++++++ dstack/certbot/src/lib.rs | 4 +- dstack/gateway/rpc/proto/gateway_rpc.proto | 40 +++++ dstack/gateway/src/admin_service.rs | 150 +++++++++++++++++- tools/mock-cf-dns-api/app.py | 61 +++++++ 6 files changed, 412 insertions(+), 8 deletions(-) diff --git a/dstack/certbot/src/dns01_client.rs b/dstack/certbot/src/dns01_client.rs index 88fbf91a3..28b486a8f 100644 --- a/dstack/certbot/src/dns01_client.rs +++ b/dstack/certbot/src/dns01_client.rs @@ -10,6 +10,28 @@ use tracing::debug; mod cloudflare; +pub use cloudflare::CloudflareTokenInfo; + +/// Verify a Cloudflare API token and fetch its granted permission groups. +/// +/// Returns `Ok(None)` when Cloudflare rejects the token, `Err` when the check +/// itself failed (transport error, unexpected response). +pub async fn verify_cloudflare_token( + api_token: &str, + api_url: Option<&str>, +) -> Result> { + cloudflare::verify_token(api_token, api_url).await +} + +/// Resolve the ID of the Cloudflare zone covering `base_domain`. +pub async fn resolve_cloudflare_zone( + api_token: &str, + base_domain: &str, + api_url: Option<&str>, +) -> Result { + cloudflare::resolve_zone(api_token, base_domain, api_url).await +} + #[derive(Debug, Deserialize, Serialize)] /// Represents a DNS record pub(crate) struct Record { diff --git a/dstack/certbot/src/dns01_client/cloudflare.rs b/dstack/certbot/src/dns01_client/cloudflare.rs index 620defb0f..3aa8d3c5c 100644 --- a/dstack/certbot/src/dns01_client/cloudflare.rs +++ b/dstack/certbot/src/dns01_client/cloudflare.rs @@ -16,6 +16,149 @@ use super::Dns01Api; const DEFAULT_CLOUDFLARE_API_URL: &str = "https://api.cloudflare.com/client/v4"; +/// Token status and granted permissions reported by Cloudflare. +/// +/// Collected from `GET /user/tokens/verify` (validity + token id) and +/// `GET /user/tokens/{id}` (name, permission groups, usage timestamps). +#[derive(Debug, Clone)] +pub struct CloudflareTokenInfo { + /// Token status: "active", "expired" or "disabled". + pub status: String, + pub name: String, + /// RFC3339 timestamps as reported by Cloudflare. + pub not_before: Option, + pub expires_on: Option, + pub last_used_on: Option, + /// Names of the permission groups granted with effect "allow", + /// e.g. "Zone Read", "DNS Write". + pub permission_groups: Vec, +} + +#[derive(Deserialize)] +struct TokenVerifyResponse { + result: TokenVerifyResult, +} + +#[derive(Deserialize)] +struct TokenVerifyResult { + id: String, + not_before: Option, + expires_on: Option, +} + +#[derive(Deserialize)] +struct TokenDetailsResponse { + result: TokenDetails, +} + +#[derive(Deserialize)] +struct TokenDetails { + name: Option, + status: String, + not_before: Option, + expires_on: Option, + last_used_on: Option, + #[serde(default)] + policies: Vec, +} + +#[derive(Deserialize)] +struct TokenPolicy { + effect: Option, + #[serde(default)] + permission_groups: Vec, +} + +#[derive(Deserialize)] +struct PermissionGroup { + name: String, +} + +async fn authenticated_get( + api_url: &str, + path: &str, + api_token: &str, +) -> Result<(reqwest::StatusCode, String)> { + let client = Client::new(); + let url = format!("{api_url}{path}"); + debug!(url = %url, "cloudflare request"); + let response = client + .get(&url) + .header("Authorization", format!("Bearer {api_token}")) + .send() + .await + .with_context(|| format!("failed to send request to {url}"))?; + let status = response.status(); + let body = response + .text() + .await + .context("failed to read response body")?; + Ok((status, body)) +} + +/// Verify an API token and fetch its granted permission groups. +/// +/// Returns `Ok(None)` when Cloudflare rejects the token itself (401/403 on the +/// verify endpoint), `Err` when the check could not be performed. +pub(crate) async fn verify_token( + api_token: &str, + api_url: Option<&str>, +) -> Result> { + let api_url = api_url.unwrap_or(DEFAULT_CLOUDFLARE_API_URL); + + let (status, body) = authenticated_get(api_url, "/user/tokens/verify", api_token).await?; + if status == reqwest::StatusCode::UNAUTHORIZED || status == reqwest::StatusCode::FORBIDDEN { + return Ok(None); + } + if !status.is_success() { + bail!("failed to verify token: {body}"); + } + let verified: TokenVerifyResponse = + serde_json::from_str(&body).context("failed to parse token verify response")?; + + let (status, body) = authenticated_get( + api_url, + &format!("/user/tokens/{}", verified.result.id), + api_token, + ) + .await?; + if !status.is_success() { + bail!("failed to get token details: {body}"); + } + let details: TokenDetailsResponse = + serde_json::from_str(&body).context("failed to parse token details response")?; + + let token = details.result; + let permission_groups = token + .policies + .iter() + .filter(|p| p.effect.as_deref() != Some("deny")) + .flat_map(|p| p.permission_groups.iter().map(|g| g.name.clone())) + .collect(); + Ok(Some(CloudflareTokenInfo { + status: token.status, + name: token.name.unwrap_or_default(), + not_before: token.not_before.or(verified.result.not_before), + expires_on: token.expires_on.or(verified.result.expires_on), + last_used_on: token.last_used_on, + permission_groups, + })) +} + +/// Resolve the zone ID covering `base_domain` without constructing a client. +pub(crate) async fn resolve_zone( + api_token: &str, + base_domain: &str, + api_url: Option<&str>, +) -> Result { + CloudflareClient::resolve_zone_id( + api_token, + base_domain, + api_url.unwrap_or(DEFAULT_CLOUDFLARE_API_URL), + ) + .await +} + #[derive(Debug, Serialize, Deserialize)] pub struct CloudflareClient { zone_id: String, diff --git a/dstack/certbot/src/lib.rs b/dstack/certbot/src/lib.rs index df71b9935..f7cc80022 100644 --- a/dstack/certbot/src/lib.rs +++ b/dstack/certbot/src/lib.rs @@ -18,7 +18,9 @@ pub use acme_client::AcmeClient; pub use bot::{read_pubkey, CertBot, CertBotConfig}; -pub use dns01_client::Dns01Client; +pub use dns01_client::{ + resolve_cloudflare_zone, verify_cloudflare_token, CloudflareTokenInfo, Dns01Client, +}; pub use workdir::WorkDir; mod acme_client; diff --git a/dstack/gateway/rpc/proto/gateway_rpc.proto b/dstack/gateway/rpc/proto/gateway_rpc.proto index c20f8cd8c..d60397af2 100644 --- a/dstack/gateway/rpc/proto/gateway_rpc.proto +++ b/dstack/gateway/rpc/proto/gateway_rpc.proto @@ -461,6 +461,9 @@ service Admin { rpc GetDefaultDnsCredential(google.protobuf.Empty) returns (GetDefaultDnsCredentialResponse) {} // Set the default DNS credential ID rpc SetDefaultDnsCredential(SetDefaultDnsCredentialRequest) returns (google.protobuf.Empty) {} + // Verify a DNS credential's token status and granted permissions against + // the provider API. Read-only: it never creates or removes DNS records. + rpc VerifyDnsCredential(VerifyDnsCredentialRequest) returns (VerifyDnsCredentialResponse) {} // ==================== ZT-Domain Management ==================== // List all ZT-Domain configurations @@ -642,6 +645,43 @@ message SetDefaultDnsCredentialRequest { string id = 1; } +// Verify DNS credential request +message VerifyDnsCredentialRequest { + string id = 1; +} + +// Zone resolution result for one domain referencing the credential +message DomainDnsCheck { + string domain = 1; + // Whether the domain's zone resolved via the provider API + bool zone_ok = 2; + // Failure reason when zone_ok is false + string error = 3; +} + +// Verify DNS credential response: token status plus granted permissions +message VerifyDnsCredentialResponse { + // "active", "expired", "disabled", "invalid" (rejected by the provider), + // or "unknown" (the check itself failed; see error) + string token_status = 1; + string token_name = 2; + // RFC3339 timestamps reported by the provider; empty when absent + string expires_on = 3; + string not_before = 4; + string last_used_on = 5; + // Names of the permission groups granted to the token + repeated string permission_groups = 6; + // Whether the grants include what certificate issuance needs + bool zone_read_ok = 7; + bool dns_write_ok = 8; + // Zone resolution for each domain referencing this credential + repeated DomainDnsCheck domains = 9; + // Set when the verification itself failed (e.g. provider API unreachable) + string error = 10; + // When the check was performed (unix seconds) + uint64 checked_at = 11; +} + // ==================== ZT-Domain Messages ==================== // ZT-Domain configuration (shared by Add/Update/Info) diff --git a/dstack/gateway/src/admin_service.rs b/dstack/gateway/src/admin_service.rs index 0596e5b18..3a4f2daec 100644 --- a/dstack/gateway/src/admin_service.rs +++ b/dstack/gateway/src/admin_service.rs @@ -10,11 +10,11 @@ use dstack_gateway_rpc::{ admin_server::{AdminRpc, AdminServer}, CertAttestationInfo, CertbotConfigResponse, ClearInstancePortPolicyRequest, CreateDnsCredentialRequest, DeleteDnsCredentialRequest, DeleteZtDomainRequest, - DnsCredentialInfo, ExitRequest, ForceReleaseCertLockRequest, GetDefaultDnsCredentialResponse, - GetDnsCredentialRequest, GetInfoRequest, GetInfoResponse, GetInstanceHandshakesRequest, - GetInstanceHandshakesResponse, GetInstancePortPolicyRequest, GetInstancePortPolicyResponse, - GetMetaResponse, GetNodeStatusesResponse, GetZtDomainRequest, GlobalConnectionsStats, - HandshakeEntry, HostInfo, LastSeenEntry, ListCertAttestationsRequest, + DnsCredentialInfo, DomainDnsCheck, ExitRequest, ForceReleaseCertLockRequest, + GetDefaultDnsCredentialResponse, GetDnsCredentialRequest, GetInfoRequest, GetInfoResponse, + GetInstanceHandshakesRequest, GetInstanceHandshakesResponse, GetInstancePortPolicyRequest, + GetInstancePortPolicyResponse, GetMetaResponse, GetNodeStatusesResponse, GetZtDomainRequest, + GlobalConnectionsStats, HandshakeEntry, HostInfo, LastSeenEntry, ListCertAttestationsRequest, ListCertAttestationsResponse, ListDnsCredentialsResponse, ListRejectedInstancesResponse, ListZtDomainsResponse, NodeStatusEntry, PeerSyncStatus as ProtoPeerSyncStatus, PortAttrs as RpcPortAttrs, PortPolicy as RpcPortPolicy, RejectedInstanceInfo, RemoveCvmRequest, @@ -22,8 +22,8 @@ use dstack_gateway_rpc::{ RenewZtDomainCertRequest, RenewZtDomainCertResponse, RotateAcmeCredentialsResponse, SetCertbotConfigRequest, SetDefaultDnsCredentialRequest, SetInstancePortPolicyRequest, SetNodeStatusRequest, SetNodeUrlRequest, StatusResponse, StoreSyncStatus, - UpdateDnsCredentialRequest, WaveKvStatusResponse, ZtDomainCertStatus, - ZtDomainConfig as ProtoZtDomainConfig, ZtDomainInfo, + UpdateDnsCredentialRequest, VerifyDnsCredentialRequest, VerifyDnsCredentialResponse, + WaveKvStatusResponse, ZtDomainCertStatus, ZtDomainConfig as ProtoZtDomainConfig, ZtDomainInfo, }; use ra_rpc::{CallContext, RpcCall}; use tracing::{info, warn}; @@ -522,6 +522,81 @@ impl AdminRpc for AdminRpcHandler { Ok(()) } + async fn verify_dns_credential( + self, + request: VerifyDnsCredentialRequest, + ) -> Result { + let kv_store = self.state.kv_store(); + let cred = kv_store + .get_dns_credential(&request.id)? + .context("dns credential not found")?; + let (api_token, api_url) = match &cred.provider { + DnsProvider::Cloudflare { api_token, api_url } => (api_token.clone(), api_url.clone()), + }; + + let mut response = VerifyDnsCredentialResponse { + checked_at: now_secs(), + ..Default::default() + }; + + let info = match certbot::verify_cloudflare_token(&api_token, api_url.as_deref()).await { + Ok(Some(info)) => info, + Ok(None) => { + response.token_status = "invalid".into(); + response.error = "token rejected by cloudflare".into(); + return Ok(response); + } + Err(err) => { + response.token_status = "unknown".into(); + response.error = format!("{err:#}"); + return Ok(response); + } + }; + + response.token_status = info.status; + response.token_name = info.name; + response.expires_on = info.expires_on.unwrap_or_default(); + response.not_before = info.not_before.unwrap_or_default(); + response.last_used_on = info.last_used_on.unwrap_or_default(); + response.zone_read_ok = has_permission(&info.permission_groups, &["zone", "read"]); + response.dns_write_ok = has_permission(&info.permission_groups, &["dns", "write"]) + || has_permission(&info.permission_groups, &["dns", "edit"]); + response.permission_groups = info.permission_groups; + + // Resolve the zone of every domain referencing this credential, either + // directly or through the default-credential fallback. + let default_id = kv_store.get_default_dns_credential_id()?; + let domains = kv_store + .list_zt_domain_configs() + .into_iter() + .filter(|c| { + c.dns_cred_id.as_deref() == Some(cred.id.as_str()) + || (c.dns_cred_id.is_none() && default_id.as_deref() == Some(cred.id.as_str())) + }) + .map(|c| c.domain); + + for domain in domains { + let check = + match certbot::resolve_cloudflare_zone(&api_token, &domain, api_url.as_deref()) + .await + { + Ok(_) => DomainDnsCheck { + domain, + zone_ok: true, + error: String::new(), + }, + Err(err) => DomainDnsCheck { + domain, + zone_ok: false, + error: truncate_error(&format!("{err:#}")), + }, + }; + response.domains.push(check); + } + + Ok(response) + } + // ==================== ZT-Domain Management ==================== async fn list_zt_domains(self) -> Result { @@ -840,6 +915,39 @@ fn dns_cred_to_proto(cred: DnsCredential) -> DnsCredentialInfo { } } +/// Match a Cloudflare permission group by keywords ("Zone Read" covers +/// ["zone", "read"], "DNS Write" covers ["dns", "write"]). +fn has_permission(permission_groups: &[String], keywords: &[&str]) -> bool { + permission_groups.iter().any(|group| { + let name = group.to_lowercase(); + keywords.iter().all(|kw| name.contains(kw)) + }) +} + +/// Cap an error chain for storage and RPC responses: an upstream gateway +/// error can be a full HTML page. Keeps the head (outermost context) and the +/// tail (root cause) and marks the elided middle. +pub(crate) fn truncate_error(msg: &str) -> String { + const MAX_LEN: usize = 16 * 1024; + if msg.len() <= MAX_LEN { + return msg.to_string(); + } + let mut head_end = MAX_LEN / 2; + while !msg.is_char_boundary(head_end) { + head_end -= 1; + } + let mut tail_start = msg.len() - MAX_LEN / 2; + while !msg.is_char_boundary(tail_start) { + tail_start += 1; + } + let elided = tail_start - head_end; + format!( + "{}\n[... {elided} bytes truncated ...]\n{}", + &msg[..head_end], + &msg[tail_start..] + ) +} + fn redact_token(token: &str) -> String { let len = token.len(); if len <= 8 { @@ -1053,6 +1161,34 @@ mod certbot_config_tests { } } +#[cfg(test)] +mod observability_helper_tests { + use super::{has_permission, truncate_error}; + + #[test] + fn permission_matching_covers_issuance_requirements() { + let groups = vec!["Zone Read".to_string(), "DNS Write".to_string()]; + assert!(has_permission(&groups, &["zone", "read"])); + assert!(has_permission(&groups, &["dns", "write"])); + assert!(!has_permission(&groups, &["account", "read"])); + } + + #[test] + fn truncate_error_keeps_short_messages_intact() { + let msg = "context: root cause"; + assert_eq!(truncate_error(msg), msg); + } + + #[test] + fn truncate_error_keeps_head_and_tail() { + let msg = "x".repeat(32 * 1024); + let truncated = truncate_error(&msg); + assert!(truncated.len() < msg.len()); + assert!(truncated.contains("bytes truncated")); + assert!(truncated.starts_with('x') && truncated.ends_with('x')); + } +} + #[cfg(test)] mod zt_domain_tests { use super::validate_zt_domain; diff --git a/tools/mock-cf-dns-api/app.py b/tools/mock-cf-dns-api/app.py index daf97894f..39fa97ba1 100644 --- a/tools/mock-cf-dns-api/app.py +++ b/tools/mock-cf-dns-api/app.py @@ -12,6 +12,7 @@ - DELETE /client/v4/zones/{zone_id}/dns_records/{record_id} - Delete DNS record """ +import hashlib import json import os import uuid @@ -450,6 +451,66 @@ def get_zone(zone_id): return jsonify(resp), 200 +# ==================== Token Endpoints ==================== + +# Simulated token status / permission grants, configurable for failure testing. +# Example: MOCK_TOKEN_STATUS=expired MOCK_TOKEN_PERMISSIONS="Zone Read" +MOCK_TOKEN_STATUS = os.environ.get("MOCK_TOKEN_STATUS", "active") +MOCK_TOKEN_PERMISSIONS = os.environ.get("MOCK_TOKEN_PERMISSIONS", "Zone Read,DNS Write") + + +def token_id_for(token): + """Derive a deterministic Cloudflare-style token ID from the bearer token.""" + return hashlib.sha1(token.encode()).hexdigest()[:32] + + +@app.route("/client/v4/user/tokens/verify", methods=["GET"]) +@verify_auth +def verify_token(): + """Verify the API token (status only).""" + token = request.headers.get("Authorization", "")[7:] + result = { + "id": token_id_for(token), + "status": MOCK_TOKEN_STATUS, + "not_before": None, + "expires_on": None, + } + resp = cf_response(result) + log_request("*", "GET", "/user/tokens/verify", None, resp, 200) + return jsonify(resp), 200 + + +@app.route("/client/v4/user/tokens/", methods=["GET"]) +@verify_auth +def get_token_details(token_id): + """Get token details (name, status, permission groups).""" + groups = [g.strip() for g in MOCK_TOKEN_PERMISSIONS.split(",") if g.strip()] + result = { + "id": token_id, + "name": "mock-token", + "status": MOCK_TOKEN_STATUS, + "issued_on": "2024-01-01T00:00:00.000000Z", + "modified_on": get_current_time(), + "last_used_on": get_current_time(), + "not_before": None, + "expires_on": None, + "policies": [ + { + "id": "mock-policy-0", + "effect": "allow", + "resources": {"com.cloudflare.api.account.zone.*": "*"}, + "permission_groups": [ + {"id": f"mock-pg-{i}", "name": name} + for i, name in enumerate(groups) + ], + } + ], + } + resp = cf_response(result) + log_request("*", "GET", f"/user/tokens/{token_id}", None, resp, 200) + return jsonify(resp), 200 + + # ==================== Management UI ==================== MANAGEMENT_HTML = """ From 13c0599c18f6a10eacd92e6514081709e3959ad6 Mon Sep 17 00:00:00 2001 From: Leechael Yim Date: Mon, 17 Aug 2026 05:31:13 +0800 Subject: [PATCH 2/2] feat(gateway): record last cert attempt outcome per domain Certificate issuance and renewal run as a background task whose failures only reach the logs, so an expiring certificate is the first visible signal of a broken renewal. DistributedCertBot now keeps a node-local, in-memory record per domain: last attempt time and node, last success, consecutive failures, and the (16 KiB head+tail truncated) error chain of the last failed attempt. Both the periodic/manual renewal path (try_renew) and startup issuance (init_domain) record their outcome; nodes that skip because a peer holds the renew lock do not write. The record is exposed on the existing ZtDomainCertStatus, so GetZtDomain and ListZtDomains show it without a new RPC. In-memory storage targets the current single-node deployments; multi-node visibility can later move the record into WaveKV without changing the response shape. --- dstack/gateway/rpc/proto/gateway_rpc.proto | 12 ++++ dstack/gateway/src/admin_service.rs | 34 ++++++++-- dstack/gateway/src/distributed_certbot.rs | 78 +++++++++++++++++++++- 3 files changed, 119 insertions(+), 5 deletions(-) diff --git a/dstack/gateway/rpc/proto/gateway_rpc.proto b/dstack/gateway/rpc/proto/gateway_rpc.proto index d60397af2..ce29ae752 100644 --- a/dstack/gateway/rpc/proto/gateway_rpc.proto +++ b/dstack/gateway/rpc/proto/gateway_rpc.proto @@ -718,6 +718,18 @@ message ZtDomainCertStatus { uint64 issued_at = 4; // Whether the certificate is loaded in memory bool loaded_in_memory = 5; + // Status of the most recent issuance/renewal attempt. Node-local and + // in-memory only: zero/empty when this node has not attempted since restart. + uint64 last_attempt_at = 6; + // Node that performed the last attempt + uint32 attempted_by = 7; + // When the last successful attempt completed (0 = never succeeded) + uint64 last_success_at = 8; + // Failed attempts in a row since the last success + uint32 consecutive_failures = 9; + // Error chain of the last failed attempt (truncated to 16 KiB); empty when + // the last attempt succeeded + string last_error = 10; } // List ZT-Domains response diff --git a/dstack/gateway/src/admin_service.rs b/dstack/gateway/src/admin_service.rs index 3a4f2daec..178189e38 100644 --- a/dstack/gateway/src/admin_service.rs +++ b/dstack/gateway/src/admin_service.rs @@ -602,11 +602,12 @@ impl AdminRpc for AdminRpcHandler { async fn list_zt_domains(self) -> Result { let kv_store = self.state.kv_store(); let cert_resolver = &self.state.cert_resolver; + let certbot = &self.state.certbot; let domains = kv_store .list_zt_domain_configs() .into_iter() - .map(|config| zt_domain_to_proto(config, kv_store, cert_resolver)) + .map(|config| zt_domain_to_proto(config, kv_store, cert_resolver, certbot)) .collect(); Ok(ListZtDomainsResponse { domains }) @@ -621,7 +622,12 @@ impl AdminRpc for AdminRpcHandler { .get_zt_domain_config(&domain) .context("ZT-Domain config not found")?; - Ok(zt_domain_to_proto(config, kv_store, cert_resolver)) + Ok(zt_domain_to_proto( + config, + kv_store, + cert_resolver, + &self.state.certbot, + )) } async fn add_zt_domain(self, request: ProtoZtDomainConfig) -> Result { @@ -639,7 +645,12 @@ impl AdminRpc for AdminRpcHandler { kv_store.save_zt_domain_config(&config)?; info!("Added ZT-Domain config: {}", config.domain); - Ok(zt_domain_to_proto(config, kv_store, cert_resolver)) + Ok(zt_domain_to_proto( + config, + kv_store, + cert_resolver, + &self.state.certbot, + )) } async fn update_zt_domain(self, request: ProtoZtDomainConfig) -> Result { @@ -656,7 +667,12 @@ impl AdminRpc for AdminRpcHandler { kv_store.save_zt_domain_config(&config)?; info!("Updated ZT-Domain config: {}", config.domain); - Ok(zt_domain_to_proto(config, kv_store, cert_resolver)) + Ok(zt_domain_to_proto( + config, + kv_store, + cert_resolver, + &self.state.certbot, + )) } async fn delete_zt_domain(self, request: DeleteZtDomainRequest) -> Result<()> { @@ -1024,10 +1040,12 @@ fn zt_domain_to_proto( config: ZtDomainConfig, kv_store: &crate::kv::KvStore, cert_resolver: &crate::cert_store::CertResolver, + certbot: &crate::distributed_certbot::DistributedCertBot, ) -> ZtDomainInfo { // Get certificate data for status let cert_data = kv_store.get_cert_data(&config.domain); let loaded_in_memory = cert_resolver.has_cert(&config.domain); + let attempt = certbot.attempt_status(&config.domain); let cert_status = Some(ZtDomainCertStatus { has_cert: cert_data.is_some(), @@ -1035,6 +1053,14 @@ fn zt_domain_to_proto( issued_by: cert_data.as_ref().map(|d| d.issued_by).unwrap_or(0), issued_at: cert_data.as_ref().map(|d| d.issued_at).unwrap_or(0), loaded_in_memory, + last_attempt_at: attempt.as_ref().map(|a| a.last_attempt_at).unwrap_or(0), + attempted_by: attempt.as_ref().map(|a| a.attempted_by).unwrap_or(0), + last_success_at: attempt.as_ref().map(|a| a.last_success_at).unwrap_or(0), + consecutive_failures: attempt + .as_ref() + .map(|a| a.consecutive_failures) + .unwrap_or(0), + last_error: attempt.map(|a| a.last_error).unwrap_or_default(), }); ZtDomainInfo { diff --git a/dstack/gateway/src/distributed_certbot.rs b/dstack/gateway/src/distributed_certbot.rs index a651b36de..249efa5a1 100644 --- a/dstack/gateway/src/distributed_certbot.rs +++ b/dstack/gateway/src/distributed_certbot.rs @@ -7,6 +7,7 @@ //! This module provides distributed certificate management for multiple domains //! with dynamic DNS credential configuration and attestation storage. +use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; @@ -17,6 +18,7 @@ use ra_tls::attestation::QuoteContentType; use ra_tls::rcgen::KeyPair; use tokio::sync::Mutex; use tracing::{error, info, warn}; +use wavekv::types::NodeId; use crate::cert_store::CertResolver; use crate::kv::{ @@ -34,6 +36,25 @@ const ROTATION_LOCK_TIMEOUT_SECS: u64 = 600; /// Default ACME URL (Let's Encrypt production) const DEFAULT_ACME_URL: &str = "https://acme-v02.api.letsencrypt.org/directory"; +/// Node-local status of the most recent issuance/renewal attempt for a domain. +/// +/// In-memory only: the record resets on restart and reflects only this node's +/// view. Single-node deployment is the current target; move this into WaveKV +/// if multi-node visibility is ever needed. +#[derive(Debug, Clone, Default)] +pub struct CertAttemptStatus { + /// When the last attempt completed (unix seconds; 0 = no attempt since restart). + pub last_attempt_at: u64, + /// Node that performed the last attempt. + pub attempted_by: NodeId, + /// When the last successful attempt completed (0 = never succeeded). + pub last_success_at: u64, + /// Failed attempts in a row since the last success. + pub consecutive_failures: u32, + /// Error chain of the last failed attempt; empty after a success. + pub last_error: String, +} + /// Multi-domain certificate manager pub struct DistributedCertBot { kv_store: Arc, @@ -43,6 +64,8 @@ pub struct DistributedCertBot { /// Credential rotation is additionally guarded across nodes by a /// best-effort lock in WaveKV; see [`KvStore::try_acquire_rotation_lock`]. caa_lock: Mutex<()>, + /// Per-domain status of the most recent attempt. Never held across await. + statuses: std::sync::Mutex>, } impl DistributedCertBot { @@ -51,6 +74,32 @@ impl DistributedCertBot { kv_store, cert_resolver, caa_lock: Default::default(), + statuses: Default::default(), + } + } + + /// Status of the most recent issuance/renewal attempt for a domain. + pub fn attempt_status(&self, domain: &str) -> Option { + self.statuses.lock().unwrap().get(domain).cloned() + } + + /// Record the outcome of an issuance/renewal attempt. + fn record_attempt(&self, domain: &str, result: &Result) { + let now = now_secs(); + let mut statuses = self.statuses.lock().unwrap(); + let status = statuses.entry(domain.to_string()).or_default(); + status.last_attempt_at = now; + status.attempted_by = self.kv_store.my_node_id(); + match result { + Ok(_) => { + status.last_success_at = now; + status.consecutive_failures = 0; + status.last_error.clear(); + } + Err(err) => { + status.consecutive_failures = status.consecutive_failures.saturating_add(1); + status.last_error = crate::admin_service::truncate_error(&format!("{err:?}")); + } } } @@ -247,7 +296,9 @@ impl DistributedCertBot { // No valid cert, need to request new one info!(domain, "no valid certificate found, requesting from ACME"); - self.request_new_cert(domain).await + let result = self.request_new_cert(domain).await; + self.record_attempt(domain, &result); + result } /// Set CAA records for every configured ZT domain. @@ -371,6 +422,7 @@ impl DistributedCertBot { info!("no existing certificate, requesting new one"); self.do_request_new(domain, &config).await.map(|_| true) }; + self.record_attempt(domain, &result); // Release lock regardless of result if let Err(err) = self.kv_store.release_cert_lock(domain) { @@ -875,6 +927,30 @@ mod tests { assert!(certbot.kv_store.get_rotation_lock().is_none()); } + #[test] + fn attempt_status_tracks_failures_and_recovery() { + let data_dir = tempfile::tempdir().expect("failed to create temp dir"); + let certbot = test_certbot(data_dir.path()); + assert!(certbot.attempt_status("example.com").is_none()); + + let fail: Result = Err(anyhow::anyhow!("dns provider exploded")); + certbot.record_attempt("example.com", &fail); + certbot.record_attempt("example.com", &fail); + let status = certbot.attempt_status("example.com").unwrap(); + assert_eq!(status.consecutive_failures, 2); + assert_eq!(status.attempted_by, 1); + assert!(status.last_error.contains("dns provider exploded")); + assert_eq!(status.last_success_at, 0); + assert!(status.last_attempt_at > 0); + + let ok: Result = Ok(true); + certbot.record_attempt("example.com", &ok); + let status = certbot.attempt_status("example.com").unwrap(); + assert_eq!(status.consecutive_failures, 0); + assert!(status.last_error.is_empty()); + assert!(status.last_success_at > 0); + } + #[test] fn corrupt_acme_credentials_fail_closed() { assert!(acme_url_matches("not-json", "https://acme.test/directory").is_err());