feat(gateway): DNS credential verification and cert attempt observability

dstack/certbot/src/dns01_client.rs

@@ -10,6 +10,28 @@ use tracing::debug;
 
 mod cloudflare;
 
+pub use cloudflare::CloudflareTokenInfo;
+
+/// Verify a Cloudflare API token and fetch its granted permission groups.
+///
+/// Returns `Ok(None)` when Cloudflare rejects the token, `Err` when the check
+/// itself failed (transport error, unexpected response).
+pub async fn verify_cloudflare_token(
+    api_token: &str,
+    api_url: Option<&str>,
+) -> Result<Option<CloudflareTokenInfo>> {
+    cloudflare::verify_token(api_token, api_url).await
+}
+
+/// Resolve the ID of the Cloudflare zone covering `base_domain`.
+pub async fn resolve_cloudflare_zone(
+    api_token: &str,
+    base_domain: &str,
+    api_url: Option<&str>,
+) -> Result<String> {
+    cloudflare::resolve_zone(api_token, base_domain, api_url).await
+}
+
 #[derive(Debug, Deserialize, Serialize)]
 /// Represents a DNS record
 pub(crate) struct Record {

dstack/certbot/src/dns01_client/cloudflare.rs

@@ -16,6 +16,149 @@ use super::Dns01Api;
 
 const DEFAULT_CLOUDFLARE_API_URL: &str = "https://api.cloudflare.com/client/v4";
 
+/// Token status and granted permissions reported by Cloudflare.
+///
+/// Collected from `GET /user/tokens/verify` (validity + token id) and
+/// `GET /user/tokens/{id}` (name, permission groups, usage timestamps).
+#[derive(Debug, Clone)]
+pub struct CloudflareTokenInfo {
+    /// Token status: "active", "expired" or "disabled".
+    pub status: String,
+    pub name: String,
+    /// RFC3339 timestamps as reported by Cloudflare.
+    pub not_before: Option<String>,
+    pub expires_on: Option<String>,
+    pub last_used_on: Option<String>,
+    /// Names of the permission groups granted with effect "allow",
+    /// e.g. "Zone Read", "DNS Write".
+    pub permission_groups: Vec<String>,
+}
+
+#[derive(Deserialize)]
+struct TokenVerifyResponse {
+    result: TokenVerifyResult,
+}
+
+#[derive(Deserialize)]
+struct TokenVerifyResult {
+    id: String,
+    not_before: Option<String>,
+    expires_on: Option<String>,
+}
+
+#[derive(Deserialize)]
+struct TokenDetailsResponse {
+    result: TokenDetails,
+}
+
+#[derive(Deserialize)]
+struct TokenDetails {
+    name: Option<String>,
+    status: String,
+    not_before: Option<String>,
+    expires_on: Option<String>,
+    last_used_on: Option<String>,
+    #[serde(default)]
+    policies: Vec<TokenPolicy>,
+}
+
+#[derive(Deserialize)]
+struct TokenPolicy {
+    effect: Option<String>,
+    #[serde(default)]
+    permission_groups: Vec<PermissionGroup>,
+}
+
+#[derive(Deserialize)]
+struct PermissionGroup {
+    name: String,
+}
+
+async fn authenticated_get(
+    api_url: &str,
+    path: &str,
+    api_token: &str,
+) -> Result<(reqwest::StatusCode, String)> {
+    let client = Client::new();
+    let url = format!("{api_url}{path}");
+    debug!(url = %url, "cloudflare request");
+    let response = client
+        .get(&url)
+        .header("Authorization", format!("Bearer {api_token}"))
+        .send()
+        .await
+        .with_context(|| format!("failed to send request to {url}"))?;
+    let status = response.status();
+    let body = response
+        .text()
+        .await
+        .context("failed to read response body")?;
+    Ok((status, body))
+}
+
+/// Verify an API token and fetch its granted permission groups.
+///
+/// Returns `Ok(None)` when Cloudflare rejects the token itself (401/403 on the
+/// verify endpoint), `Err` when the check could not be performed.
+pub(crate) async fn verify_token(
+    api_token: &str,
+    api_url: Option<&str>,
+) -> Result<Option<CloudflareTokenInfo>> {
+    let api_url = api_url.unwrap_or(DEFAULT_CLOUDFLARE_API_URL);
+
+    let (status, body) = authenticated_get(api_url, "/user/tokens/verify", api_token).await?;
+    if status == reqwest::StatusCode::UNAUTHORIZED || status == reqwest::StatusCode::FORBIDDEN {
+        return Ok(None);
+    }
+    if !status.is_success() {
+        bail!("failed to verify token: {body}");
+    }
+    let verified: TokenVerifyResponse =
+        serde_json::from_str(&body).context("failed to parse token verify response")?;
+
+    let (status, body) = authenticated_get(
+        api_url,
+        &format!("/user/tokens/{}", verified.result.id),
+        api_token,
+    )
+    .await?;
+    if !status.is_success() {
+        bail!("failed to get token details: {body}");
+    }
+    let details: TokenDetailsResponse =
+        serde_json::from_str(&body).context("failed to parse token details response")?;
+
+    let token = details.result;
+    let permission_groups = token
+        .policies
+        .iter()
+        .filter(|p| p.effect.as_deref() != Some("deny"))
+        .flat_map(|p| p.permission_groups.iter().map(|g| g.name.clone()))
+        .collect();
+    Ok(Some(CloudflareTokenInfo {
+        status: token.status,
+        name: token.name.unwrap_or_default(),
+        not_before: token.not_before.or(verified.result.not_before),
+        expires_on: token.expires_on.or(verified.result.expires_on),
+        last_used_on: token.last_used_on,
+        permission_groups,
+    }))
+}
+
+/// Resolve the zone ID covering `base_domain` without constructing a client.
+pub(crate) async fn resolve_zone(
+    api_token: &str,
+    base_domain: &str,
+    api_url: Option<&str>,
+) -> Result<String> {
+    CloudflareClient::resolve_zone_id(
+        api_token,
+        base_domain,
+        api_url.unwrap_or(DEFAULT_CLOUDFLARE_API_URL),
+    )
+    .await
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub struct CloudflareClient {
     zone_id: String,

dstack/certbot/src/lib.rs

@@ -18,7 +18,9 @@
 
 pub use acme_client::AcmeClient;
 pub use bot::{read_pubkey, CertBot, CertBotConfig};
-pub use dns01_client::Dns01Client;
+pub use dns01_client::{
+    resolve_cloudflare_zone, verify_cloudflare_token, CloudflareTokenInfo, Dns01Client,
+};
 pub use workdir::WorkDir;
 
 mod acme_client;

dstack/gateway/rpc/proto/gateway_rpc.proto

@@ -461,6 +461,9 @@ service Admin {
   rpc GetDefaultDnsCredential(google.protobuf.Empty) returns (GetDefaultDnsCredentialResponse) {}
   // Set the default DNS credential ID
   rpc SetDefaultDnsCredential(SetDefaultDnsCredentialRequest) returns (google.protobuf.Empty) {}
+  // Verify a DNS credential's token status and granted permissions against
+  // the provider API. Read-only: it never creates or removes DNS records.
+  rpc VerifyDnsCredential(VerifyDnsCredentialRequest) returns (VerifyDnsCredentialResponse) {}
 
   // ==================== ZT-Domain Management ====================
   // List all ZT-Domain configurations
@@ -642,6 +645,43 @@ message SetDefaultDnsCredentialRequest {
   string id = 1;
 }
 
+// Verify DNS credential request
+message VerifyDnsCredentialRequest {
+  string id = 1;
+}
+
+// Zone resolution result for one domain referencing the credential
+message DomainDnsCheck {
+  string domain = 1;
+  // Whether the domain's zone resolved via the provider API
+  bool zone_ok = 2;
+  // Failure reason when zone_ok is false
+  string error = 3;
+}
+
+// Verify DNS credential response: token status plus granted permissions
+message VerifyDnsCredentialResponse {
+  // "active", "expired", "disabled", "invalid" (rejected by the provider),
+  // or "unknown" (the check itself failed; see error)
+  string token_status = 1;
+  string token_name = 2;
+  // RFC3339 timestamps reported by the provider; empty when absent
+  string expires_on = 3;
+  string not_before = 4;
+  string last_used_on = 5;
+  // Names of the permission groups granted to the token
+  repeated string permission_groups = 6;
+  // Whether the grants include what certificate issuance needs
+  bool zone_read_ok = 7;
+  bool dns_write_ok = 8;
+  // Zone resolution for each domain referencing this credential
+  repeated DomainDnsCheck domains = 9;
+  // Set when the verification itself failed (e.g. provider API unreachable)
+  string error = 10;
+  // When the check was performed (unix seconds)
+  uint64 checked_at = 11;
+}
+
 // ==================== ZT-Domain Messages ====================
 
 // ZT-Domain configuration (shared by Add/Update/Info)
@@ -678,6 +718,18 @@ message ZtDomainCertStatus {
   uint64 issued_at = 4;
   // Whether the certificate is loaded in memory
   bool loaded_in_memory = 5;
+  // Status of the most recent issuance/renewal attempt. Node-local and
+  // in-memory only: zero/empty when this node has not attempted since restart.
+  uint64 last_attempt_at = 6;
+  // Node that performed the last attempt
+  uint32 attempted_by = 7;
+  // When the last successful attempt completed (0 = never succeeded)
+  uint64 last_success_at = 8;
+  // Failed attempts in a row since the last success
+  uint32 consecutive_failures = 9;
+  // Error chain of the last failed attempt (truncated to 16 KiB); empty when
+  // the last attempt succeeded
+  string last_error = 10;
 }
 
 // List ZT-Domains response