diff --git a/.github/scripts/ci_changes.py b/.github/scripts/ci_changes.py index 45faf2bce4..734dc81d80 100644 --- a/.github/scripts/ci_changes.py +++ b/.github/scripts/ci_changes.py @@ -106,6 +106,30 @@ SCHEDULING_PACKAGES = frozenset(SHARDS["scheduling"]) STACK_CLIENT_PACKAGES = frozenset(SHARDS["stack-client"]) +# The runtime configuration conformance gate reads the sources of the runtimes +# it holds rows for, their generated runtime schemas, and the canonical shared +# configuration blocks schema. A product that joins the gate joins this set. +CONFIG_CONFORMANCE_PACKAGES = frozenset( + { + "registry-platform-config", + "registry-breg", + "registry-casework", + "registry-discovery", + "registry-evidence", + "registry-relay-v2", + "registry-relayctl", + "registry-render", + "registry-scheduling", + } +) +CONFIG_CONFORMANCE_INPUTS = ( + "products/platform/generated/*", + "products/platform/scripts/*config-conformance*", + "products/breg/generated/runtime/*", + "products/casework/generated/runtime/*", + "products/scheduling/generated/runtime/*", +) + # These are the cross-product semantic commitments implemented independently by # Base Registry Engine and Relay V2. A change must replay both real product routers, # while profile-only tooling and ordinary positive/negative fixtures remain on @@ -1002,6 +1026,11 @@ def classify( or path in {"clippy.toml", "deny.toml", "rustfmt.toml"} for path in paths ) + config_conformance = ( + complete + or any(matches(path, *CONFIG_CONFORMANCE_INPUTS) for path in paths) + or bool(affected & CONFIG_CONFORMANCE_PACKAGES) + ) release_tool = ( complete or "release_tool" in security_workflow_gates @@ -1175,6 +1204,7 @@ def classify( "platform_assurance": platform_assurance, "platform_coverage": platform_coverage, "platform_hygiene": platform_hygiene, + "config_conformance": config_conformance, "discovery_contracts": complete or bool(affected & DISCOVERY_PACKAGES) or any(matches(path, *DISCOVERY_PROVIDER_INPUTS) for path in paths) diff --git a/.github/scripts/test_ci_changes.py b/.github/scripts/test_ci_changes.py index f1dfe92630..e54bed0899 100644 --- a/.github/scripts/test_ci_changes.py +++ b/.github/scripts/test_ci_changes.py @@ -3,6 +3,7 @@ from __future__ import annotations import fnmatch +import importlib.util import json import os import re @@ -31,6 +32,7 @@ RELAY_CLIENT_PACKAGES, RELAY_TUTORIAL_INPUTS, STACK_CLIENT_PACKAGES, + CONFIG_CONFORMANCE_PACKAGES, SECURITY_WORKFLOW_GATES, SHARDS, LockChange, @@ -316,6 +318,9 @@ def test_deferred_ci_work_keeps_its_selector_and_explicit_status_guard( ), "docs-archives": "needs.changes.outputs.docs_archives == 'true'", "editor-extensions": "needs.changes.outputs.editors == 'true'", + "config-conformance": ( + "needs.changes.outputs.config_conformance == 'true'" + ), } deferred = { name @@ -400,6 +405,7 @@ def test_ci_scheduling_graph_retains_every_job_and_aggregate_dependency( "breg-contracts", "breg-wasm", "identifiers", + "config-conformance", "rust-result", "casework-postgres", "scheduling-contracts", @@ -439,6 +445,7 @@ def test_ci_scheduling_graph_retains_every_job_and_aggregate_dependency( "casework-postgres", "scheduling-postgres", "scheduling-contracts", + "config-conformance", ), "release-tool-required": ("changes", "release-tool"), "release-source-proof-required": ("changes", "release-source-proof"), @@ -462,6 +469,7 @@ def test_ci_scheduling_graph_retains_every_job_and_aggregate_dependency( "casework-postgres", "scheduling-postgres", "scheduling-contracts", + "config-conformance", "release-tool", "release-source-proof", "evidence-tutorials", @@ -515,7 +523,7 @@ def test_final_aggregate_flattens_rust_results_with_equivalent_outcomes( final_needs, previous_final_needs.difference({"rust-result"}).union(rust_needs), ) - self.assertEqual(29, len(final_needs)) + self.assertEqual(30, len(final_needs)) # Platform line coverage publishes from main and the nightly sweep; # it does not hold the merge queue. self.assertNotIn("platform-coverage", final_needs) @@ -615,6 +623,66 @@ def status(job: str, selected: str, result: str) -> int: with self.subTest(job=job, selected=selected, result=result): self.assertEqual(expected, status(job, selected, result)) + def test_config_conformance_inputs_select_the_conformance_gate(self) -> None: + for path in ( + "crates/registry-platform-config/src/blocks.rs", + "crates/registry-breg/src/runtime_config.rs", + "crates/registry-relay-v2/src/contract.rs", + "crates/registry-relayctl/schemas/authoring/runtime.schema.json", + "crates/registry-render/src/manifest.rs", + "crates/registry-discovery/src/startup.rs", + "crates/registry-evidence/src/config.rs", + "crates/registry-casework/src/config.rs", + "crates/registry-scheduling/src/config.rs", + "products/platform/generated/runtime-config-blocks.schema.json", + "products/platform/scripts/check-config-conformance.py", + "products/breg/generated/runtime/runtime.schema.json", + "products/casework/generated/runtime/runtime.schema.json", + "products/scheduling/generated/runtime/runtime.schema.json", + ): + with self.subTest(path=path): + self.assertTrue( + classify(self.workspace, (path,))["config_conformance"] + ) + self.assertFalse( + classify(self.workspace, ("docs/site/src/content/docs/index.mdx",))[ + "config_conformance" + ] + ) + + def test_every_config_conformance_row_is_routed(self) -> None: + script = Path("products/platform/scripts/check-config-conformance.py") + spec = importlib.util.spec_from_file_location("config_conformance", script) + assert spec is not None and spec.loader is not None + gate = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = gate + spec.loader.exec_module(gate) + packages = { + Path(source).parts[1] + for row in gate.ROWS + for source in row.loader_sources + } + self.assertLessEqual(packages, CONFIG_CONFORMANCE_PACKAGES) + self.assertIn("registry-platform-config", CONFIG_CONFORMANCE_PACKAGES) + schemas = { + entry.path for row in gate.ROWS for entry in row.hand_schemas + } | { + row.runtime_schema + for row in gate.ROWS + if isinstance(row.runtime_schema, str) + } + for path in sorted(schemas): + with self.subTest(path=path): + self.assertTrue( + classify(self.workspace, (path,))["config_conformance"] + ) + digest_tests = {row.digest_mismatch.path for row in gate.ROWS} + for path in sorted(digest_tests): + with self.subTest(path=path): + self.assertTrue( + classify(self.workspace, (path,))["config_conformance"] + ) + def test_shards_cover_every_workspace_package_once(self) -> None: assigned = [package for packages in SHARDS.values() for package in packages] self.assertCountEqual(assigned, self.workspace.package_names) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25773cba26..4c91f59207 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,7 @@ jobs: platform_assurance: ${{ steps.filter.outputs.platform_assurance }} platform_coverage: ${{ steps.filter.outputs.platform_coverage }} platform_hygiene: ${{ steps.filter.outputs.platform_hygiene }} + config_conformance: ${{ steps.filter.outputs.config_conformance }} discovery_contracts: ${{ steps.filter.outputs.discovery_contracts }} relay_v2_contracts: ${{ steps.filter.outputs.relay_v2_contracts }} relay_client_contracts: ${{ steps.filter.outputs.relay_client_contracts }} @@ -724,6 +725,18 @@ jobs: CASEWORKCTL_BIN: ${{ github.workspace }}/target/debug/caseworkctl BREGCTL_BIN: ${{ github.workspace }}/target/debug/bregctl run: products/casework/scripts/check-checkpoint.sh + - name: Verify the rewritten Evidence deployment loads through the Evidence loader + # The fixture machinery lives behind postgres-test, but this test needs + # neither a database nor an issuer. An inexact name filter that selects + # nothing still exits 0, so the step names the test exactly and fails + # unless it reports one pass. + shell: bash + run: | + set -euo pipefail + cargo test --locked -p registry-casework --features postgres-test --lib \ + -- --exact task_grants::native_exchange_tests::the_rewritten_evidence_deployment_loads_through_the_evidence_loader \ + | tee "${RUNNER_TEMP}/casework-evidence-fixture.log" + grep -q 'test result: ok\. 1 passed' "${RUNNER_TEMP}/casework-evidence-fixture.log" || { echo "::error::expected exactly one passing test for the_rewritten_evidence_deployment_loads_through_the_evidence_loader"; exit 1; } - name: Verify claims, reconciliation, and durable attempts env: CASEWORK_TEST_DATABASE_URL: postgresql://casework:casework_test@localhost:${{ job.services.postgres.ports['5432'] }}/casework @@ -1104,6 +1117,34 @@ jobs: - name: Check Registry Record profile artifacts run: products/registry-record/scripts/check.sh + config-conformance: + name: Runtime configuration conformance + needs: + - changes + - rust-policy + if: ${{ !cancelled() && needs.changes.result == 'success' && needs.changes.outputs.config_conformance == 'true' }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + submodules: false + + - name: Cache Cargo registry + uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 + with: + shared-key: workspace-registry + cache-targets: false + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Test runtime configuration conformance gate + run: python3 -m unittest products/platform/scripts/test_check_config_conformance.py + + - name: Check runtime configuration conformance + run: products/platform/scripts/check-config-conformance.py --check-generated + rust-result: name: Rust workspace if: always() @@ -1122,6 +1163,7 @@ jobs: - casework-postgres - scheduling-postgres - scheduling-contracts + - config-conformance runs-on: ubuntu-slim timeout-minutes: 5 env: @@ -2155,6 +2197,7 @@ jobs: - casework-postgres - scheduling-postgres - scheduling-contracts + - config-conformance - release-tool - release-source-proof - evidence-tutorials diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml index dcda12828f..5081756a8e 100644 --- a/.github/workflows/release-candidate.yml +++ b/.github/workflows/release-candidate.yml @@ -1764,7 +1764,7 @@ jobs: "${casework_install_dir}/caseworkctl" test "${casework_project}" "${casework_install_dir}/caseworkctl" package \ "${casework_project}" --output "${casework_package}" - test -f "${casework_package}/casework.package.json" + test -f "${casework_package}/SHA256SUMS" rm candidate/bundle-root/SHA256SUMS fi evidencectl_installer="evidencectl-${{ needs.validate.outputs.tag }}-install.sh" diff --git a/Cargo.lock b/Cargo.lock index 86fc728c66..a51c1abbd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4949,6 +4949,7 @@ dependencies = [ "anyhow", "axum", "base64 0.22.1", + "chrono", "clap", "getrandom 0.4.3", "hmac 0.13.0", @@ -5049,6 +5050,7 @@ dependencies = [ "base64 0.22.1", "registry-breg-client", "registry-casework-core", + "registry-platform-buildinfo", "registry-platform-config", "registry-platform-crypto", "registry-platform-hooks", @@ -5218,10 +5220,10 @@ dependencies = [ "registry-discovery-profile", "registry-platform-buildinfo", "registry-platform-canonical-json", + "registry-platform-config", "registry-platform-httpsec", "serde", "serde_json", - "serde_yaml_ng", "sha2 0.11.0", "tempfile", "thiserror 2.0.20", @@ -5316,6 +5318,7 @@ dependencies = [ "registry-discovery-profile", "registry-platform-buildinfo", "registry-platform-canonical-json", + "registry-platform-config", "registry-platform-httputil", "reqwest", "serde", @@ -5413,6 +5416,7 @@ dependencies = [ "p256", "registry-evidence", "registry-evidence-verifier", + "registry-platform-config", "registry-platform-crypto", "registry-platform-httpsec", "registry-platform-httputil", @@ -5557,6 +5561,7 @@ dependencies = [ "registry-language-server", "registry-platform-audit", "registry-platform-buildinfo", + "registry-platform-config", "registry-platform-crypto", "registry-thunderid-tooling", "rhai", @@ -5694,15 +5699,17 @@ dependencies = [ name = "registry-platform-config" version = "0.34.0" dependencies = [ - "base64 0.22.1", - "registry-platform-crypto", + "registry-platform-canonical-json", "rustix 1.1.5", + "schemars", "serde", "serde_json", + "serde_norway", + "serde_path_to_error", "sha2 0.11.0", "tempfile", "thiserror 2.0.20", - "time", + "url", "zeroize", ] diff --git a/crates/registry-breg-client-node/client.d.ts b/crates/registry-breg-client-node/client.d.ts index bcfe85b372..2327cf1b5c 100644 --- a/crates/registry-breg-client-node/client.d.ts +++ b/crates/registry-breg-client-node/client.d.ts @@ -608,6 +608,7 @@ export type BRegIngestionAttemptOutcome = | 'invalidItem' | 'refused' | 'bindingChanged' + | 'importAuthorityClosed' | 'chunkMismatch' | 'runNotOpen' | 'unavailable' diff --git a/crates/registry-breg-client/src/client.rs b/crates/registry-breg-client/src/client.rs index 4f6c074f09..cab1d4d390 100644 --- a/crates/registry-breg-client/src/client.rs +++ b/crates/registry-breg-client/src/client.rs @@ -27,6 +27,9 @@ const ANY_MEDIA_TYPE: &str = "*/*"; const PROBLEM_MEDIA_TYPE: &str = "application/problem+json"; const MAXIMUM_PROBLEM_BYTES: usize = 4 * 1024; const MAXIMUM_LOCATION_BYTES: usize = 2_048; +/// The response header naming the engine release that served a response. +const BREG_ENGINE_VERSION_HEADER: &str = "registry-engine-version"; +const MAXIMUM_ENGINE_VERSION_BYTES: usize = 64; const X_CONTENT_TYPE_OPTIONS: reqwest::header::HeaderName = reqwest::header::HeaderName::from_static("x-content-type-options"); @@ -110,7 +113,38 @@ impl BaseRegistryClient { &self, access_profile: Option<&str>, ) -> Result, BaseRegistryClientError> { - let raw = self.registry_metadata(access_profile).await?; + self.registry_contract_and_engine_version(access_profile) + .await + .1 + } + + /// Retrieve Registry Metadata v1 as [`Self::registry_contract`] does, and + /// also return the engine version the service reported beside it. + /// + /// The version is returned even when the document then fails to decode, + /// so a caller that runs in lock-step with one engine release can name a + /// release mismatch instead of reporting a shape failure. It is `None` + /// when no successful response arrived, or when the response carried no + /// well-formed `Registry-Engine-Version` header. + pub async fn registry_contract_and_engine_version( + &self, + access_profile: Option<&str>, + ) -> ( + Option, + Result, BaseRegistryClientError>, + ) { + let raw = match self.registry_metadata(access_profile).await { + Ok(raw) => raw, + Err(error) => return (None, Err(error)), + }; + let engine_version = raw.metadata.engine_version().map(ToOwned::to_owned); + (engine_version, self.bind_registry_contract(raw)) + } + + fn bind_registry_contract( + &self, + raw: BRegComplete, + ) -> Result, BaseRegistryClientError> { let value = BRegMetadata::from_slice(raw.value.as_bytes()) .map_err(|error| { BaseRegistryClientError::protocol_metadata( @@ -1632,13 +1666,14 @@ impl BaseRegistryClient { Some(trace_id), )); } + let engine_version = breg_engine_version(&headers); let body = self .transport .read(response, self.config.max_response_bytes.min(maximum_bytes)) .await?; Ok(BRegWire { body, - metadata: BRegResponseMetadata::new(trace_id, etag), + metadata: BRegResponseMetadata::new(trace_id, etag).with_engine_version(engine_version), media_type: expected_media.to_owned(), link, status: status.as_u16(), @@ -2357,6 +2392,24 @@ fn validate_no_store( ) } +/// Read the single, well-formed engine version a response reported. The value +/// only names the peer, so an absent, repeated, or malformed header is reported +/// as no version rather than failing a response every other caller can use. +fn breg_engine_version(headers: &reqwest::header::HeaderMap) -> Option { + let mut values = headers.get_all(BREG_ENGINE_VERSION_HEADER).iter(); + let value = values.next()?.to_str().ok()?; + if values.next().is_some() + || value.is_empty() + || value.len() > MAXIMUM_ENGINE_VERSION_BYTES + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b'+')) + { + return None; + } + Some(value.to_owned()) +} + fn validate_exact_header( status: StatusCode, headers: &reqwest::header::HeaderMap, diff --git a/crates/registry-breg-client/src/error.rs b/crates/registry-breg-client/src/error.rs index 89a4d86fb7..bd5332d97b 100644 --- a/crates/registry-breg-client/src/error.rs +++ b/crates/registry-breg-client/src/error.rs @@ -275,7 +275,7 @@ impl BRegProblemCode { "The selected access profile does not match the run's bound profile." } Self::IngestionReceiptErased => "The stored receipt of the chunk was erased.", - Self::IngestionRunBlocked => "The active package no longer matches the run binding.", + Self::IngestionRunBlocked => "The ingestion run is blocked and refuses further chunks.", Self::IngestionRunNotOpen => "The ingestion run is not open for this transition.", Self::LookupUnresolved => "The lookup did not resolve exactly one record.", Self::MutationConflict => "The mutation conflicts with current state.", diff --git a/crates/registry-breg-client/src/ingestion.rs b/crates/registry-breg-client/src/ingestion.rs index 78dae95e66..f1248ad1dd 100644 --- a/crates/registry-breg-client/src/ingestion.rs +++ b/crates/registry-breg-client/src/ingestion.rs @@ -133,10 +133,14 @@ impl Serialize for BRegIngestionRunStatus { } } -/// The one reason an open run refuses chunk submissions. +/// Why an open run refuses chunk submissions. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum BRegIngestionBlockedReason { + /// The active package no longer matches the run's binding. ActivePackageChanged, + /// The import authority an `import` run consumes is closed, expired, + /// superseded, exhausted, or has no room for the next chunk. + ImportAuthorityClosed, } impl BRegIngestionBlockedReason { @@ -144,12 +148,14 @@ impl BRegIngestionBlockedReason { pub const fn as_str(self) -> &'static str { match self { Self::ActivePackageChanged => "activePackageChanged", + Self::ImportAuthorityClosed => "importAuthorityClosed", } } pub(crate) fn parse(value: &str) -> Option { match value { "activePackageChanged" => Some(Self::ActivePackageChanged), + "importAuthorityClosed" => Some(Self::ImportAuthorityClosed), _ => None, } } @@ -172,6 +178,7 @@ pub enum BRegIngestionAttemptOutcome { InvalidItem, Refused, BindingChanged, + ImportAuthorityClosed, ChunkMismatch, RunNotOpen, Unavailable, @@ -179,12 +186,13 @@ pub enum BRegIngestionAttemptOutcome { impl BRegIngestionAttemptOutcome { /// Every attempt outcome the wire contract names. - pub const ALL: [Self; 8] = [ + pub const ALL: [Self; 9] = [ Self::Committed, Self::Replayed, Self::InvalidItem, Self::Refused, Self::BindingChanged, + Self::ImportAuthorityClosed, Self::ChunkMismatch, Self::RunNotOpen, Self::Unavailable, @@ -198,6 +206,7 @@ impl BRegIngestionAttemptOutcome { Self::InvalidItem => "invalidItem", Self::Refused => "refused", Self::BindingChanged => "bindingChanged", + Self::ImportAuthorityClosed => "importAuthorityClosed", Self::ChunkMismatch => "chunkMismatch", Self::RunNotOpen => "runNotOpen", Self::Unavailable => "unavailable", @@ -1758,6 +1767,14 @@ mod tests { ); assert_eq!(serde_json::to_value(&run).unwrap(), wire); + wire["blockedReason"] = json!("importAuthorityClosed"); + let run = decode_wire(&wire).expect("an authority-blocked run decodes"); + assert_eq!( + run.blocked_reason(), + Some(BRegIngestionBlockedReason::ImportAuthorityClosed) + ); + assert_eq!(serde_json::to_value(&run).unwrap(), wire); + let mut wire = run_wire(); wire["status"] = json!("complete"); wire["complete"] = json!(true); @@ -1774,6 +1791,7 @@ mod tests { |wire: &mut Value| wire["status"] = json!("archived"), |wire: &mut Value| wire["status"] = json!(7), |wire: &mut Value| wire["blockedReason"] = json!("active_package_changed"), + |wire: &mut Value| wire["blockedReason"] = json!("import_authority_closed"), |wire: &mut Value| wire["operation"] = json!("delete"), |wire: &mut Value| wire["lastAttempt"]["outcome"] = json!("crashed"), |wire: &mut Value| { diff --git a/crates/registry-breg-client/src/response.rs b/crates/registry-breg-client/src/response.rs index e63a674ed3..c315517d52 100644 --- a/crates/registry-breg-client/src/response.rs +++ b/crates/registry-breg-client/src/response.rs @@ -87,6 +87,10 @@ pub struct BRegResponseMetadata { etag: Option, #[serde(skip_serializing_if = "Option::is_none")] location: Option, + /// Kept for Rust callers that pin a peer release; language bindings do + /// not carry response headers. + #[serde(skip)] + engine_version: Option, } impl BRegResponseMetadata { @@ -95,9 +99,15 @@ impl BRegResponseMetadata { trace_id, etag, location: None, + engine_version: None, } } + pub(crate) fn with_engine_version(mut self, engine_version: Option) -> Self { + self.engine_version = engine_version; + self + } + pub(crate) fn with_location(mut self, location: String) -> Self { self.location = Some(location); self @@ -120,6 +130,14 @@ impl BRegResponseMetadata { pub fn location(&self) -> Option<&str> { self.location.as_deref() } + + /// The engine release the service reported in its + /// `Registry-Engine-Version` response header, when that header was present + /// and well formed. It describes the peer; it grants nothing. + #[must_use] + pub fn engine_version(&self) -> Option<&str> { + self.engine_version.as_deref() + } } impl fmt::Debug for BRegResponseMetadata { @@ -129,6 +147,7 @@ impl fmt::Debug for BRegResponseMetadata { .field("trace_id", &self.trace_id) .field("etag", &self.etag.is_some()) .field("location", &self.location.is_some()) + .field("engine_version", &self.engine_version) .finish() } } diff --git a/crates/registry-breg-client/tests/write_http_boundary.rs b/crates/registry-breg-client/tests/write_http_boundary.rs index 2b984f8ccc..20fe8352f9 100644 --- a/crates/registry-breg-client/tests/write_http_boundary.rs +++ b/crates/registry-breg-client/tests/write_http_boundary.rs @@ -568,6 +568,63 @@ async fn registry_contract_keeps_the_metadata_decode_reason_instead_of_discardin ); } +#[tokio::test] +async fn registry_contract_reports_the_engine_version_even_when_the_document_fails_to_decode() { + let mut malformed = metadata_fixture(); + malformed["entities"] = json!("not-an-array"); + let fixture = test_client(vec![ + metadata_response().with_header("registry-engine-version", "0.34.0"), + MockResponse::json(StatusCode::OK, malformed) + .with_header("registry-engine-version", "0.33.0"), + ]) + .await; + + let (version, contract) = fixture + .client + .registry_contract_and_engine_version(Some("company-writer")) + .await; + assert_eq!(version.as_deref(), Some("0.34.0")); + let contract = contract.expect("registry metadata decodes"); + assert_eq!(contract.metadata.engine_version(), Some("0.34.0")); + + let (version, contract) = fixture + .client + .registry_contract_and_engine_version(Some("company-writer")) + .await; + assert_eq!(version.as_deref(), Some("0.33.0")); + assert_eq!( + contract + .expect_err("malformed registry metadata is refused") + .metadata_error_kind(), + Some(BRegMetadataErrorKind::Shape) + ); +} + +#[tokio::test] +async fn registry_contract_reports_no_engine_version_for_an_absent_or_malformed_header() { + let fixture = test_client(vec![ + metadata_response(), + metadata_response().with_header("registry-engine-version", "0.34.0 (linux)"), + metadata_response().with_header("registry-engine-version", &"9".repeat(65)), + ]) + .await; + + for _ in 0..3 { + let (version, contract) = fixture + .client + .registry_contract_and_engine_version(Some("company-writer")) + .await; + assert_eq!(version, None); + assert_eq!( + contract + .expect("metadata decodes") + .metadata + .engine_version(), + None + ); + } +} + #[tokio::test] async fn metadata_selected_create_and_patch_use_the_exact_http_contract() { let fixture = test_client(vec![ @@ -1587,7 +1644,7 @@ fn problem_detail(code: BRegProblemCode) -> &'static str { "The selected access profile does not match the run's bound profile." } Code::IngestionReceiptErased => "The stored receipt of the chunk was erased.", - Code::IngestionRunBlocked => "The active package no longer matches the run binding.", + Code::IngestionRunBlocked => "The ingestion run is blocked and refuses further chunks.", Code::IngestionRunNotOpen => "The ingestion run is not open for this transition.", Code::LookupUnresolved => "The lookup did not resolve exactly one record.", Code::MutationConflict => "The mutation conflicts with current state.", diff --git a/crates/registry-breg/Cargo.toml b/crates/registry-breg/Cargo.toml index a50192d3f9..21ac9e1467 100644 --- a/crates/registry-breg/Cargo.toml +++ b/crates/registry-breg/Cargo.toml @@ -239,7 +239,6 @@ runtime = [ "dep:registry-platform-audit", "dep:registry-platform-authcommon", "dep:registry-platform-buildinfo", - "dep:registry-platform-config", "dep:registry-platform-crypto", "registry-platform-crypto/transit", "registry-platform-hooks/postgres", @@ -258,7 +257,12 @@ runtime = [ ] postgres-test = ["runtime", "dep:tower", "registry-platform-httputil/test-support"] postgres-tls-test = ["runtime"] -schema = ["dep:schemars", "registry-platform-audit?/schema", "registry-platform-hooks/schema"] +schema = [ + "dep:schemars", + "registry-platform-audit?/schema", + "registry-platform-config/schema", + "registry-platform-hooks/schema", +] tooling = ["dep:tempfile", "dep:tower"] wasm = ["registry-platform-script/wasm"] @@ -284,7 +288,7 @@ registry-manifest-core.workspace = true registry-platform-audit = { workspace = true, optional = true } registry-platform-authcommon = { workspace = true, optional = true } registry-platform-buildinfo = { workspace = true, optional = true } -registry-platform-config = { workspace = true, optional = true } +registry-platform-config.workspace = true registry-platform-crypto = { workspace = true, optional = true } registry-platform-hooks.workspace = true registry-platform-httpsec = { workspace = true, features = ["server"], optional = true } diff --git a/crates/registry-breg/src/access.rs b/crates/registry-breg/src/access.rs index adcc93b043..f47004db82 100644 --- a/crates/registry-breg/src/access.rs +++ b/crates/registry-breg/src/access.rs @@ -182,7 +182,7 @@ pub(crate) fn access_findings(entities: &BTreeMap) -> Vec< findings.push(Diagnostic::finding("access.profile.anonymous_collection", format!("{path}.operations"), "`list` is granted to unauthenticated callers, so every row this profile can read is world-readable and no claim can narrow it. Confirm the whole collection is meant to be public")); } - let write_operations = [Operation::Create, Operation::Patch] + let write_operations = [Operation::Create, Operation::Patch, Operation::Import] .into_iter() .filter(|operation| profile.operations.contains(operation)) .map(|operation| format!("`{}`", operation_id(operation))) @@ -195,7 +195,8 @@ pub(crate) fn access_findings(entities: &BTreeMap) -> Vec< // the caller's claim, so a permission that creates must keep the field // writable. The record id is never a writable field, so a boundary on it // is outside that advice. - let creates = profile.operations.contains(&Operation::Create); + let creates = profile.operations.contains(&Operation::Create) + || profile.operations.contains(&Operation::Import); let patches = profile.operations.contains(&Operation::Patch); let boundary_fields = profile .row_boundaries diff --git a/crates/registry-breg/src/action_evidence_maintenance.rs b/crates/registry-breg/src/action_evidence_maintenance.rs index 67efa7f8c1..dabc298700 100644 --- a/crates/registry-breg/src/action_evidence_maintenance.rs +++ b/crates/registry-breg/src/action_evidence_maintenance.rs @@ -40,15 +40,17 @@ impl ActionEvidenceRetentionOperatorService { return Err(MutationError::InvalidRequest); } let config = load_runtime_config(path).map_err(|_| MutationError::Unavailable)?; + let package = config + .load_active_package() + .map_err(|_| MutationError::Unavailable)?; let pool = config .runtime_database_connection_config() .map_err(|_| MutationError::Unavailable)? .build_pool() .map_err(|_| MutationError::Unavailable)?; let mut client = pool.get().await.map_err(|_| MutationError::Unavailable)?; - let startup = crate::startup::prepare_startup( - config.package().root(), - &config.package_load_context(), + let startup = crate::startup::prepare_loaded_startup( + package, &mut client, config.database().roles().migration(), config.database().roles().runtime(), diff --git a/crates/registry-breg/src/api/ingestion.rs b/crates/registry-breg/src/api/ingestion.rs index 46675d0c2f..d9443be75c 100644 --- a/crates/registry-breg/src/api/ingestion.rs +++ b/crates/registry-breg/src/api/ingestion.rs @@ -36,10 +36,11 @@ struct IngestionRoute { base: CompiledRoute, } -/// Bind the ingestion-run routes of every batch-driven entity. The compiled -/// batch route is resolved through the same lookup the client-side import plan -/// uses, so a run and a direct batch submission of the same bytes authorize -/// against the same operation. +/// Bind the ingestion-run routes of every batch-driven or import-driven +/// entity. The compiled bulk route is resolved through the same lookup the +/// client-side import plan uses, so a run and a direct batch submission of the +/// same bytes authorize against the same operation, and an `import` grant +/// authorizes against its own route, which nothing else mounts. pub(super) fn routes(service: &HttpService) -> Router> { let mut app = Router::new(); if service.mutations.is_none() { @@ -49,13 +50,14 @@ pub(super) fn routes(service: &HttpService) -> Router> { if entity.batch.is_none() { continue; } - // Every profile that grants the batch operation resolves the same - // compiled route, so the first hit is the route the runs drive. + // Every profile that grants the bulk operation resolves the same + // compiled route, so the first hit is the route the runs drive. The + // compiler refuses `batch` beside `import` on one entity. let Some(base) = entity .access_profiles .keys() .filter_map(|profile_id| { - crate::data::ingestion_batch_route(&service.registry, &entity.id, profile_id) + crate::data::ingestion_route(&service.registry, &entity.id, profile_id) }) .next() else { @@ -728,7 +730,8 @@ pub(super) fn batch_refusal_problem(refusal: crate::mutation::IngestionRefusal) crate::problem::ProblemCode::IngestionChunkMismatch, StatusCode::CONFLICT, ), - crate::mutation::IngestionRefusal::BindingChanged => catalogue_problem( + crate::mutation::IngestionRefusal::BindingChanged + | crate::mutation::IngestionRefusal::AuthorityClosed => catalogue_problem( crate::problem::ProblemCode::IngestionRunBlocked, StatusCode::CONFLICT, ), @@ -997,10 +1000,38 @@ fn parse_run_cursor(value: &str) -> Result { .ok_or(QueryParseError::Invalid) } +/// The authorized `import` surfaces of one caller. An `import` route is never +/// mounted as a route of its own, so it is absent from the visible surfaces; +/// the ingestion-run document is the only place it is advertised. +pub(super) fn import_surfaces<'a>( + service: &'a HttpService, + claims: &VerifiedRequestClaims, + options: &QueryOptions, +) -> Vec> { + if service.mutations.is_none() { + return Vec::new(); + } + service + .registry + .routes() + .routes + .iter() + .filter(|route| route.operation == Operation::Import) + .filter(|route| { + service + .registry + .entities() + .get(&route.entity_id) + .is_some_and(|entity| entity.batch.is_some()) + }) + .filter_map(|route| authorize_route(service, route, claims, options)) + .collect() +} + /// Advertise the ingestion routes of one selected profile. Like the attachment /// operations, they are appended to the compiled-route document only for a -/// caller that holds the batch operation under the selected profile and -/// answers with a principal, and they never appear in `/v1/registry`. +/// caller that holds the batch or import operation under the selected profile +/// and answers with a principal, and they never appear in `/v1/registry`. pub(super) fn append_openapi( service: &HttpService, surfaces: &[AuthorizedSurface<'_>], @@ -1012,13 +1043,35 @@ pub(super) fn append_openapi( } let mut advertised = false; for surface in surfaces.iter().filter(|surface| { - surface.route.operation == Operation::Batch - && surface.read_path.is_none() + matches!( + surface.route.operation, + Operation::Batch | Operation::Import + ) && surface.read_path.is_none() && surface.context.principal().is_some() }) { let Some(batch) = surface.entity.batch.as_ref() else { continue; }; + let profile = &surface.entity.access_profiles[surface.context.selected_profile()]; + let input_schema_id = + crate::artifacts::openapi_input_schema_id(&surface.entity.id, surface.route.operation); + // The batch input schema is collected from the visible routes; an + // import route is not one of them, so its input schema is added here. + if surface.route.operation == Operation::Import { + schemas.entry(input_schema_id.clone()).or_insert_with(|| { + crate::artifacts::openapi_entity_input_schema( + surface.entity, + Some(&profile.writable_fields), + ) + }); + } + let (allow_create, allow_patch) = match surface.route.operation { + Operation::Import => (true, false), + _ => ( + profile.operations.contains(&Operation::Create), + profile.operations.contains(&Operation::Patch), + ), + }; let root = format!("/v1/records/{}/ingestion-runs", surface.entity.route); let operation_id = |name: &str| format!("{}.ingestion.{name}", surface.entity.id); let methods = paths @@ -1141,7 +1194,12 @@ pub(super) fn append_openapi( "required": true, "content": { "application/json": { - "schema": submit_chunk_schema(surface.entity, batch) + "schema": submit_chunk_schema( + &input_schema_id, + batch, + allow_create, + allow_patch, + ) } } }, @@ -1378,24 +1436,27 @@ fn create_run_schema() -> Value { /// and the digests that bind the chunk to the announced input. The item /// count bound is the compiled batch maximum, the same ceiling the ordinary /// batch route enforces. -fn submit_chunk_schema(entity: &CompiledEntity, batch: &crate::contract::BatchSource) -> Value { +/// The chunk body: the batch item shapes the selected grant admits, each +/// create item's data drawn from the grant's input schema. +fn submit_chunk_schema( + input_schema_id: &str, + batch: &crate::contract::BatchSource, + allow_create: bool, + allow_patch: bool, +) -> Value { + let items = crate::artifacts::openapi_batch_items_schema( + json!({"$ref": format!("#/components/schemas/{input_schema_id}")}), + batch.maximum_items, + allow_create, + allow_patch, + ); json!({ "type": "object", "additionalProperties": false, "required": ["chunkIndex", "items", "digest", "prefixDigest"], "properties": { "chunkIndex": {"type": "integer", "minimum": 0}, - "items": { - "type": "array", - "minItems": 1, - "maxItems": batch.maximum_items, - "items": { - "$ref": format!( - "#/components/schemas/{}/properties/items/items", - crate::artifacts::openapi_input_schema_id(&entity.id, Operation::Batch) - ) - } - }, + "items": items, "digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, "prefixDigest": {"type": "string", "pattern": "^[0-9a-f]{64}$"} } @@ -1420,7 +1481,7 @@ fn ingestion_run_schema() -> Value { "blockedReason": { "oneOf": [ {"type": "null"}, - {"type": "string", "enum": ["activePackageChanged"]} + {"type": "string", "enum": ["activePackageChanged", "importAuthorityClosed"]} ] }, "entityId": {"type": "string"}, @@ -1450,8 +1511,8 @@ fn ingestion_run_schema() -> Value { "type": "string", "enum": [ "committed", "replayed", "invalidItem", "refused", - "bindingChanged", "chunkMismatch", "runNotOpen", - "unavailable" + "bindingChanged", "importAuthorityClosed", + "chunkMismatch", "runNotOpen", "unavailable" ] }, "chunkIndex": {"type": ["integer", "null"], "minimum": 0} diff --git a/crates/registry-breg/src/api/mod.rs b/crates/registry-breg/src/api/mod.rs index eed7c3ee9b..2203da4c11 100644 --- a/crates/registry-breg/src/api/mod.rs +++ b/crates/registry-breg/src/api/mod.rs @@ -350,7 +350,8 @@ async fn openapi( .unwrap_or_else(VerifiedRequestClaims::anonymous); let visible = visible_surfaces(&service, &claims, &options); let visible_actions = actions::visible_actions(&service, &claims, &options); - if visible.is_empty() && visible_actions.is_empty() { + let imports = ingestion::import_surfaces(&service, &claims, &options); + if visible.is_empty() && visible_actions.is_empty() && imports.is_empty() { return concealed(); } @@ -437,6 +438,7 @@ async fn openapi( crate::artifacts::append_review_completion_openapi(&mut paths, &mut schemas); } ingestion::append_openapi(&service, &visible, &mut paths, &mut schemas); + ingestion::append_openapi(&service, &imports, &mut paths, &mut schemas); Json(json!({ "openapi": "3.1.0", "info": {"title": service.registry.registry_id(), "version": service.registry.version()}, @@ -544,7 +546,17 @@ async fn registry_metadata( if !visible_actions.is_empty() { metadata["actions"] = actions::metadata(&visible_actions); } - Json(metadata).into_response() + // A peer that runs in lock-step with this engine release, such as a + // Casework source adapter, compares this header with its own release and + // names a mismatch instead of reporting an undecodable contract. + ( + [( + axum::http::HeaderName::from_static("registry-engine-version"), + registry_platform_buildinfo::DISPLAY_VERSION, + )], + Json(metadata), + ) + .into_response() } async fn entity_schema( @@ -2547,7 +2559,11 @@ fn authorize_direct_route_base<'a>( } if matches!( route.operation, - Operation::Create | Operation::Patch | Operation::Tombstone | Operation::Batch + Operation::Create + | Operation::Patch + | Operation::Tombstone + | Operation::Batch + | Operation::Import ) && profile.anonymous { return None; @@ -2732,7 +2748,9 @@ fn is_request_operation(operation: Operation) -> bool { fn served_operation(service: &HttpService, route: &CompiledRoute) -> bool { match route.operation { - Operation::Invoke => false, + // The ingestion-run surface serves an import grant; the route itself + // is never mounted and never listed as an operation. + Operation::Invoke | Operation::Import => false, Operation::Get | Operation::List => true, Operation::Lookup => true, Operation::Create => service.mutations.is_some(), @@ -4702,6 +4720,7 @@ fn operation_name(operation: Operation) -> &'static str { Operation::CancelRequest => "cancel_request", Operation::ApplyRequest => "apply_request", Operation::Snapshot => "snapshot", + Operation::Import => "import", } } diff --git a/crates/registry-breg/src/artifacts.rs b/crates/registry-breg/src/artifacts.rs index 08ab2ee60c..ca4a87241a 100644 --- a/crates/registry-breg/src/artifacts.rs +++ b/crates/registry-breg/src/artifacts.rs @@ -1888,6 +1888,11 @@ fn openapi_document( let mut paths = Map::new(); let mut input_schemas = Map::new(); for route in &routes.routes { + // An import grant is exercised only through the ingestion-run + // surface, which the static document does not describe. + if route.operation == Operation::Import { + continue; + } let entity = entities .get(&route.entity_id) .expect("compiled route refers to a compiled entity"); @@ -2314,6 +2319,9 @@ fn operation_response_shape(spec: OpenApiOperationSpec<'_>) -> &'static str { Operation::Revisions => "BRegRevisionCollectionV1", Operation::Batch => "BRegAtomicBatchMutationResponseV1", Operation::Invoke => "BRegImmediateActionResponseV1", + Operation::Import => { + unreachable!("import routes are served only by the ingestion-run surface") + } Operation::SubmitRequest | Operation::ReviseRequest | Operation::CancelRequest @@ -2382,6 +2390,9 @@ fn request_action_target_entities(spec: OpenApiOperationSpec<'_>) -> Vec | Operation::Revisions | Operation::Snapshot => {} Operation::Invoke => {} + Operation::Import => { + unreachable!("import routes are served only by the ingestion-run surface") + } } targets.into_iter().collect() } @@ -2523,6 +2534,9 @@ fn operation_parameters( } Operation::Revisions => {} Operation::Invoke => {} + Operation::Import => { + unreachable!("import routes are served only by the ingestion-run surface") + } Operation::SubmitRequest | Operation::ReviseRequest | Operation::CancelRequest @@ -2742,6 +2756,9 @@ fn operation_request_body(spec: OpenApiOperationSpec<'_>) -> Option { | Operation::Revisions | Operation::Snapshot => None, Operation::Invoke => None, + Operation::Import => { + unreachable!("import routes are served only by the ingestion-run surface") + } Operation::SubmitRequest | Operation::ReviseRequest | Operation::CancelRequest @@ -2800,6 +2817,30 @@ fn batch_input_schema( maximum_items: u16, allow_create: bool, allow_patch: bool, +) -> Value { + json!({ + "type": "object", + "additionalProperties": false, + "required": ["items"], + "properties": { + "changeContext": change_context_request_schema(), + "items": openapi_batch_items_schema( + create_data_schema, + maximum_items, + allow_create, + allow_patch, + ) + } + }) +} + +/// The bounded array of batch items one grant admits. The batch request body +/// and the ingestion chunk body carry the same item shapes. +pub(crate) fn openapi_batch_items_schema( + create_data_schema: Value, + maximum_items: u16, + allow_create: bool, + allow_patch: bool, ) -> Value { let mut item_schemas = Vec::new(); if allow_create { @@ -2827,18 +2868,10 @@ fn batch_input_schema( })); } json!({ - "type": "object", - "additionalProperties": false, - "required": ["items"], - "properties": { - "changeContext": change_context_request_schema(), - "items": { - "type": "array", - "minItems": 1, - "maxItems": maximum_items, - "items": {"oneOf": item_schemas} - } - } + "type": "array", + "minItems": 1, + "maxItems": maximum_items, + "items": {"oneOf": item_schemas} }) } @@ -2981,6 +3014,9 @@ fn operation_responses(spec: OpenApiOperationSpec<'_>) -> Value { revision_response_schema(spec, false), revision_response_schema(spec, true), ), + Operation::Import => { + unreachable!("import routes are served only by the ingestion-run surface") + } Operation::Invoke => success_response( "Action accepted", StatusResponseHeaders::ActionMutation, @@ -4523,6 +4559,7 @@ fn operation_name(operation: Operation) -> &'static str { Operation::CancelRequest => "cancel_request", Operation::ApplyRequest => "apply_request", Operation::Invoke => "invoke", + Operation::Import => "import", } } diff --git a/crates/registry-breg/src/change_request.rs b/crates/registry-breg/src/change_request.rs index a2e5edcd71..6db72eeac1 100644 --- a/crates/registry-breg/src/change_request.rs +++ b/crates/registry-breg/src/change_request.rs @@ -641,6 +641,7 @@ pub(crate) fn compile_change_requests( }) && !profile.anonymous && !profile.operations.contains(&Operation::Batch) + && !profile.operations.contains(&Operation::Import) }); if !valid { errors.push(Diagnostic::error( @@ -747,7 +748,7 @@ fn validate_change_controlled_direct_writes( errors.push(Diagnostic::error( "change_control.direct_write_grant", format!("{}.operations", profile_path(&entity.id, &profile.id)), - "a controlled mutation operation cannot remain directly granted", + "a controlled mutation operation cannot remain directly granted; to load new records under change control, grant `import` in place of `batch` or `create`", )); } } diff --git a/crates/registry-breg/src/cli.rs b/crates/registry-breg/src/cli.rs index d391f13865..0a1e7b4b37 100644 --- a/crates/registry-breg/src/cli.rs +++ b/crates/registry-breg/src/cli.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 //! Public process arguments, shared with the generated command reference. +use std::ffi::OsStr; use std::path::PathBuf; use clap::{CommandFactory, Parser}; @@ -13,8 +14,8 @@ use clap::{CommandFactory, Parser}; )] pub struct Arguments { /// Absolute path to the runtime configuration file. - #[arg(long, value_name = "ABSOLUTE_FILE")] - pub config: PathBuf, + #[arg(long = "runtime-config", value_name = "ABSOLUTE_FILE")] + pub runtime_config: PathBuf, } /// Return the public command tree for documentation and completion. @@ -22,6 +23,24 @@ pub fn command() -> clap::Command { Arguments::command() } +/// The refusal for a removed runtime-configuration flag, checked before +/// argument parsing so the operator reads the replacement instead of clap's +/// unknown-argument error. Arguments after `--` are not flags. +pub fn removed_config_flag(arguments: I) -> Option<&'static str> +where + I: IntoIterator, + S: AsRef, +{ + arguments + .into_iter() + .map_while(|argument| { + let argument = argument.as_ref().as_encoded_bytes().to_vec(); + (argument != b"--").then_some(argument) + }) + .any(|argument| argument == b"--config" || argument.starts_with(b"--config=")) + .then_some("--config is no longer accepted; pass --runtime-config FILE") +} + #[cfg(test)] mod tests { use super::*; @@ -29,9 +48,32 @@ mod tests { #[test] fn runtime_configuration_is_required() { assert!(Arguments::try_parse_from(["breg"]).is_err()); - let arguments = Arguments::try_parse_from(["breg", "--config", "/etc/breg/runtime.yaml"]) - .expect("the documented configuration argument parses"); - assert_eq!(arguments.config, PathBuf::from("/etc/breg/runtime.yaml")); + let arguments = + Arguments::try_parse_from(["breg", "--runtime-config", "/etc/breg/runtime.yaml"]) + .expect("the documented configuration argument parses"); + assert_eq!( + arguments.runtime_config, + PathBuf::from("/etc/breg/runtime.yaml") + ); command().debug_assert(); } + + #[test] + fn the_removed_config_flag_names_its_replacement() { + for arguments in [ + vec!["breg", "--config", "/etc/breg/runtime.yaml"], + vec!["breg", "--config=/etc/breg/runtime.yaml"], + ] { + assert_eq!( + removed_config_flag(&arguments), + Some("--config is no longer accepted; pass --runtime-config FILE") + ); + assert!(Arguments::try_parse_from(&arguments).is_err()); + } + assert_eq!( + removed_config_flag(["breg", "--runtime-config", "/etc/breg/runtime.yaml"]), + None + ); + assert_eq!(removed_config_flag(["breg", "--", "--config"]), None); + } } diff --git a/crates/registry-breg/src/compiler.rs b/crates/registry-breg/src/compiler.rs index 80ce85851d..75d30750fd 100644 --- a/crates/registry-breg/src/compiler.rs +++ b/crates/registry-breg/src/compiler.rs @@ -1892,7 +1892,10 @@ pub(crate) fn expand_project_access( .iter() .any(|operation| match operation { Operation::Create | Operation::Patch => !governed_request_draft, - Operation::Tombstone | Operation::Batch | Operation::Invoke => true, + Operation::Tombstone + | Operation::Batch + | Operation::Import + | Operation::Invoke => true, _ => false, }) }) { @@ -3356,6 +3359,47 @@ fn reaches<'a>( .any(|(_, next)| reaches(next, target, edges, visited)) } +/// An `import` grant creates records only through a durable ingestion run, +/// so it needs the entity's chunk bounds, a creator the run can be scoped to, +/// and no raw batch grant beside it on the entity: a batch grant would write +/// the same records outside any import authority, leaving the authority +/// bounding nothing. +fn validate_import_grant( + entity: &EntitySource, + access: &AccessProfileSource, + errors: &mut Vec, +) { + let path = format!( + "entities[id={}].accessProfiles[id={}].operations", + entity.id, access.id + ); + if entity.batch.is_none() { + errors.push(Diagnostic::error( + "import.batch_bounds.required", + path.clone(), + "an import grant loads records in chunks, so the entity must declare batch maximumItems and maximumBytes", + )); + } + if access.anonymous || access.principal_claim.as_deref().is_none_or(str::is_empty) { + errors.push(Diagnostic::error( + "import.principal.required", + path.clone(), + "an import run belongs to the principal that created it, so an import grant needs an authenticated profile with a principal claim", + )); + } + if entity + .access_profiles + .iter() + .any(|profile| profile.operations.contains(&Operation::Batch)) + { + errors.push(Diagnostic::error( + "import.batch.redundant", + path, + "a batch grant on the same entity writes records outside any import authority; remove batch and load through import", + )); + } +} + fn validate_profiles( entity: &EntitySource, entities: &BTreeMap, @@ -3404,6 +3448,7 @@ fn validate_profiles( && matches!(operation, Operation::Patch | Operation::Tombstone)) || (*operation == Operation::Tombstone && !entity.tombstone) || (is_request_operation(*operation) && entity.change_request.is_none()) + || (*operation == Operation::Import && entity.change_request.is_some()) || *operation == Operation::Invoke { errors.push(Diagnostic::error( @@ -3413,6 +3458,9 @@ fn validate_profiles( )); } } + if access.operations.contains(&Operation::Import) { + validate_import_grant(entity, access, errors); + } if access.operations.contains(&Operation::Batch) && !access .operations @@ -5549,7 +5597,7 @@ fn compile_routes_and_access( let mut errors = Vec::new(); for entity in entities.values() { for operation in routed_operations() { - if operation == Operation::Batch && entity.batch.is_none() { + if matches!(operation, Operation::Batch | Operation::Import) && entity.batch.is_none() { continue; } let profiles: Vec<&AccessProfileSource> = entity @@ -6606,6 +6654,9 @@ fn route_shape(entity: &CompiledEntity, operation: Operation) -> (HttpMethod, St Operation::Batch => (HttpMethod::Post, format!("{base}:batch")), Operation::Revisions => (HttpMethod::Get, format!("{base}/{{record_id}}/revisions")), Operation::Snapshot => (HttpMethod::Get, format!("{base}:snapshot")), + // The ingestion-run surface is the only place an import grant is + // exercised; the HTTP router never mounts this route directly. + Operation::Import => (HttpMethod::Post, format!("{base}/ingestion-runs")), Operation::SubmitRequest | Operation::ReviseRequest | Operation::CancelRequest @@ -6616,7 +6667,7 @@ fn route_shape(entity: &CompiledEntity, operation: Operation) -> (HttpMethod, St } } -fn routed_operations() -> [Operation; 9] { +fn routed_operations() -> [Operation; 10] { [ Operation::Create, Operation::Get, @@ -6627,10 +6678,11 @@ fn routed_operations() -> [Operation; 9] { Operation::Batch, Operation::Revisions, Operation::Snapshot, + Operation::Import, ] } -fn all_operations() -> [Operation; 14] { +fn all_operations() -> [Operation; 15] { [ Operation::Create, Operation::Get, @@ -6646,6 +6698,7 @@ fn all_operations() -> [Operation; 14] { Operation::CancelRequest, Operation::ApplyRequest, Operation::Invoke, + Operation::Import, ] } @@ -6675,6 +6728,7 @@ pub(crate) fn operation_id(operation: Operation) -> &'static str { Operation::CancelRequest => "cancel_request", Operation::ApplyRequest => "apply_request", Operation::Invoke => "invoke", + Operation::Import => "import", } } diff --git a/crates/registry-breg/src/consent.rs b/crates/registry-breg/src/consent.rs index 0f40312793..b4e24e7de9 100644 --- a/crates/registry-breg/src/consent.rs +++ b/crates/registry-breg/src/consent.rs @@ -406,6 +406,7 @@ fn validate_record( for profile in &entity.access_profiles { if profile.operations.contains(&Operation::Create) || profile.operations.contains(&Operation::Batch) + || profile.operations.contains(&Operation::Import) { errors.push(Diagnostic::error( "consent.record.direct_write", @@ -413,7 +414,7 @@ fn validate_record( "entities[id={}].accessProfiles[id={}].operations", entity.id, profile.id ), - "consent rows are created only by actions that declare consentIssuer; no profile grants create or batch", + "consent rows are created only by actions that declare consentIssuer; no profile grants create, batch, or import", )); } } diff --git a/crates/registry-breg/src/contract.rs b/crates/registry-breg/src/contract.rs index 920bf56b88..b94426c482 100644 --- a/crates/registry-breg/src/contract.rs +++ b/crates/registry-breg/src/contract.rs @@ -4,6 +4,9 @@ use std::collections::{BTreeMap, BTreeSet}; use jsonschema::{Draft, JSONSchema}; use registry_platform_canonical_json::{canonicalize_json, parse_json_strict}; +use registry_platform_config::{ + reject_environment_expressions_in_authored_yaml, RuntimeConfigErrorKind, +}; pub use registry_platform_hooks::{HookHandlerSource, HookPhase}; use serde::{ de::DeserializeOwned, de::Error as _, de::IntoDeserializer, Deserialize, Deserializer, @@ -2105,6 +2108,55 @@ pub enum FieldTypeSource { } impl FieldTypeSource { + /// Whether every value `previous` admitted is still a valid value of this + /// type with no change to its stored column type: the vocabulary gained + /// codes, a `text` length limit rose, or a `string` minimum length fell. + pub fn admits_every_value_of(&self, previous: &FieldTypeSource) -> bool { + self.keeps_vocabulary_codes_of(previous) || self.widens_length_limits_of(previous) + } + + /// Whether this type only relaxes a length bound `previous` enforced with + /// a column check: a `text` `maxLength` rose, or a `string` `minLength` + /// fell under the same `maxLength`. + pub fn widens_length_limits_of(&self, previous: &FieldTypeSource) -> bool { + self.widens_text_length_of(previous) || self.lowers_string_min_length_of(previous) + } + + /// Whether this is a `string` type with the same `maxLength` as + /// `previous` and a lower `minLength`. The minimum is a column check, so + /// the stored `varchar` column type is unchanged. + pub fn lowers_string_min_length_of(&self, previous: &FieldTypeSource) -> bool { + matches!( + (previous, self), + ( + FieldTypeSource::String { + min_length: previous_min, + max_length: previous_max, + }, + FieldTypeSource::String { + min_length, + max_length, + }, + ) if max_length == previous_max && min_length < previous_min + ) + } + + /// Whether this is a `text` type whose `maxLength` is higher than + /// `previous`'s. A `text` column bounds its length with a check, so the + /// stored column type stays `text`. A `string` field's `maxLength` is its + /// `varchar` column type, so raising it is a type change and not covered. + pub fn widens_text_length_of(&self, previous: &FieldTypeSource) -> bool { + matches!( + (previous, self), + ( + FieldTypeSource::Text { + max_length: previous_max, + }, + FieldTypeSource::Text { max_length }, + ) if max_length > previous_max + ) + } + /// Whether this type keeps `previous`'s vocabulary and every code it /// declared, possibly adding codes. A stored code of `previous` is then /// always a valid code of this type. @@ -2616,6 +2668,11 @@ pub enum Operation { CancelRequest, ApplyRequest, Invoke, + /// Create records through a durable ingestion run, and nothing else. The + /// grant is enabled only while an operator-opened import authority is open + /// for the entity and profile. It declares no item route and no raw batch + /// route, and change control does not count it as a direct write. + Import, } #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] @@ -3315,6 +3372,7 @@ fn parse_json(bytes: &[u8], root: &str) -> Result(bytes: &[u8], root: &str) -> Result { + reject_authored_environment_expression(bytes, root)?; let deserializer = serde_norway::Deserializer::from_slice(bytes); serde_path_to_error::deserialize(deserializer).map_err(|error| { CompileFailure::from_one(Diagnostic::error( @@ -3345,6 +3403,48 @@ fn deserialize_value( }) } +/// `${...}` substitution belongs to `runtime.yaml`; an authored project or +/// module is reviewed as written, so an environment expression in one is +/// refused with the member that holds it. A document the shared reader cannot +/// parse falls through to the ordinary parse, which reports its own diagnostic. +fn reject_authored_environment_expression(bytes: &[u8], root: &str) -> Result<(), CompileFailure> { + let Ok(text) = std::str::from_utf8(bytes) else { + return Ok(()); + }; + match reject_environment_expressions_in_authored_yaml(text) { + Err(error) if error.kind() == RuntimeConfigErrorKind::AuthoredExpression => { + Err(CompileFailure::from_one(Diagnostic::error( + "source.environment_expression", + authored_member_path(root, error.field()), + "the authored value holds an environment expression; ${...} substitution \ + applies to runtime.yaml only, so write the value in the authored file directly", + ))) + } + _ => Ok(()), + } +} + +/// Render the shared reader's dotted field, whose sequence indexes are numeric +/// segments, in the `root.member[index]` form the compiler's other +/// diagnostics use. +fn authored_member_path(root: &str, field: &str) -> String { + let mut path = root.to_owned(); + if field == "/" { + return path; + } + for segment in field.split('.') { + if !segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_digit()) { + path.push('['); + path.push_str(segment); + path.push(']'); + } else { + path.push('.'); + path.push_str(segment); + } + } + path +} + /// Join the document root with the member path `serde_path_to_error` recorded. pub(crate) fn document_path( root: &str, diff --git a/crates/registry-breg/src/data.rs b/crates/registry-breg/src/data.rs index dcb4e7ee13..77b952c321 100644 --- a/crates/registry-breg/src/data.rs +++ b/crates/registry-breg/src/data.rs @@ -23,10 +23,9 @@ use crate::contract::{ valid_crs84_point, valid_decimal_value, valid_structured_value, FieldTypeSource, MutationMode, Operation, }; -#[cfg(feature = "runtime")] -use crate::model::CompiledRoute; use crate::model::{ - CompiledEntity, CompiledQueryKind, CompiledRegistry, CompiledStoredField, HttpMethod, + CompiledEntity, CompiledQueryKind, CompiledRegistry, CompiledRoute, CompiledStoredField, + HttpMethod, }; const DATA_API_VERSION: &str = "registry.registrystack.org/v1alpha1"; @@ -364,6 +363,9 @@ pub struct DataImportPlan { maximum_bytes: u32, chunks: Vec, route_path: String, + /// The binding is an `import` grant: it executes only through a durable + /// ingestion run, never through a raw batch route. + through_import: bool, response_fields: BTreeMap, } @@ -389,7 +391,7 @@ impl DataImportPlan { profile_id: &str, input: &[u8], ) -> Result { - let (entity, maximum_items, maximum_bytes, route_path) = + let (entity, maximum_items, maximum_bytes, route) = resolve_import_binding(registry, entity_id, operation, profile_id)?; if input.is_empty() || input.len() > MAX_DATA_IMPORT_INPUT_BYTES { return Err(DataError::InvalidInput); @@ -423,7 +425,8 @@ impl DataImportPlan { maximum_items, maximum_bytes, chunks, - route_path, + route_path: route.path.clone(), + through_import: route.operation == Operation::Import, response_fields: entity.access_profiles[profile_id] .readable_fields .iter() @@ -474,14 +477,21 @@ impl DataImportPlan { pub fn chunks(&self) -> &[DataChunk] { &self.chunks } + + /// Whether the binding is an `import` grant, which a caller submits only + /// through a durable ingestion run. + pub fn through_import(&self) -> bool { + self.through_import + } } /// Whether the selected profile admits one item operation of a batch import, /// exactly as an admitted import binding requires it: the profile grants the /// operation, an access entry matches it, an item route serves the profile, -/// and a patch stays confined to mutable entities. The durable run creation -/// shares this decision so its announced operation is executable to the end -/// of every chunk. +/// and a patch stays confined to mutable entities. An `import` grant admits +/// create alone, needs no item route, and is admitted only for an +/// authenticated profile. The durable run creation shares this decision so +/// its announced operation is executable to the end of every chunk. pub(crate) fn ingestion_item_operation_admitted( registry: &CompiledRegistry, entity: &CompiledEntity, @@ -508,6 +518,11 @@ pub(crate) fn ingestion_item_operation_admitted( (Operation::Create, HttpMethod::Post) | (Operation::Patch, HttpMethod::Patch) ) }); + if profile.operations.contains(&Operation::Import) { + return !profile.anonymous + && compiled == Operation::Create + && access_matches(Operation::Import); + } !profile.anonymous && profile.operations.contains(&Operation::Batch) && profile.operations.contains(&compiled) @@ -522,7 +537,7 @@ pub(crate) fn resolve_import_binding<'a>( entity_id: &str, operation: DataImportOperation, profile_id: &str, -) -> Result<(&'a CompiledEntity, u16, u32, String), DataError> { +) -> Result<(&'a CompiledEntity, u16, u32, &'a CompiledRoute), DataError> { if !valid_binding(entity_id) || !valid_binding(profile_id) { return Err(DataError::InvalidBinding); } @@ -531,25 +546,15 @@ pub(crate) fn resolve_import_binding<'a>( .get(entity_id) .ok_or(DataError::InvalidBinding)?; let batch = entity.batch.as_ref().ok_or(DataError::InvalidBinding)?; - let batch_route = registry.routes().routes.iter().find(|route| { - route.entity_id == entity_id - && route.operation == Operation::Batch - && route.method == HttpMethod::Post - && route.access_profiles.iter().any(|id| id == profile_id) - }); + let route = ingestion_route(registry, entity_id, profile_id); if !ingestion_item_operation_admitted(registry, entity, profile_id, operation) - || batch_route.is_none() || batch.maximum_items == 0 || batch.maximum_bytes == 0 { return Err(DataError::InvalidBinding); } - Ok(( - entity, - batch.maximum_items, - batch.maximum_bytes, - batch_route.expect("checked batch route").path.clone(), - )) + let route = route.ok_or(DataError::InvalidBinding)?; + Ok((entity, batch.maximum_items, batch.maximum_bytes, route)) } fn validate_item( @@ -790,17 +795,18 @@ fn plan_chunks( #[cfg(feature = "runtime")] pub(crate) const RUN_CHUNK_ALGORITHM_VERSION: &str = CHUNK_ALGORITHM_VERSION; -/// The compiled batch route one ingestion run drives, under the same route -/// lookup resolve_import_binding uses to admit a client-side plan. -#[cfg(feature = "runtime")] -pub(crate) fn ingestion_batch_route<'a>( +/// The compiled bulk route one ingestion run drives: the `batch` route, or +/// the `import` route the ingestion-run surface alone serves. The compiler +/// refuses both on one entity, so at most one matches. resolve_import_binding +/// admits a client-side plan under the same lookup. +pub(crate) fn ingestion_route<'a>( registry: &'a CompiledRegistry, entity_id: &str, profile_id: &str, ) -> Option<&'a CompiledRoute> { registry.routes().routes.iter().find(|route| { route.entity_id == entity_id - && route.operation == Operation::Batch + && matches!(route.operation, Operation::Batch | Operation::Import) && route.method == HttpMethod::Post && route.access_profiles.iter().any(|id| id == profile_id) }) @@ -1099,6 +1105,11 @@ where Dispatch: FnMut(DataHttpRequest) -> DispatchFuture, DispatchFuture: Future>, { + // An `import` grant has no raw batch route; its chunks commit only + // through a durable ingestion run. + if plan.through_import { + return Err(DataError::InvalidBinding); + } checkpoint.validate_resume( plan, package_revision, diff --git a/crates/registry-breg/src/fixtures.rs b/crates/registry-breg/src/fixtures.rs index 0ec80c7c98..599c3e7f25 100644 --- a/crates/registry-breg/src/fixtures.rs +++ b/crates/registry-breg/src/fixtures.rs @@ -2312,7 +2312,7 @@ fn validate_expectation( | Operation::ReviseRequest | Operation::CancelRequest | Operation::ApplyRequest => 200, - Operation::Tombstone | Operation::Revisions => { + Operation::Tombstone | Operation::Revisions | Operation::Import => { return Err(FixtureError::LogicalReferenceRefused) } }; @@ -6092,9 +6092,11 @@ fn valid_stable_id(value: &str) -> bool { fn operation_method(operation: Operation) -> HttpMethod { match operation { - Operation::Create | Operation::Lookup | Operation::Batch | Operation::Invoke => { - HttpMethod::Post - } + Operation::Create + | Operation::Lookup + | Operation::Batch + | Operation::Invoke + | Operation::Import => HttpMethod::Post, Operation::Get | Operation::List | Operation::Revisions | Operation::Snapshot => { HttpMethod::Get } diff --git a/crates/registry-breg/src/generated_ddl.rs b/crates/registry-breg/src/generated_ddl.rs index b2b53b1e32..23d955b63b 100644 --- a/crates/registry-breg/src/generated_ddl.rs +++ b/crates/registry-breg/src/generated_ddl.rs @@ -865,27 +865,73 @@ pub(crate) fn replace_vocabulary_check_statement( entity: &CompiledEntity, names: &crate::physical_names::EntityPhysicalNames, field: &crate::model::CompiledField, +) -> Option { + replace_inline_field_check_statement(entity, names, field, "vocabulary") +} + +/// The statement that relaxes a stored column's length check to the candidate +/// field's bound, or `None` for an encrypted column, which stores envelopes +/// and carries no length check. A raised `text` `maxLength` or a lowered +/// nonzero `string` `minLength` replaces the inline check exactly as +/// [`replace_vocabulary_check_statement`] does, under the same lock and row +/// validation. A `string` `minLength` lowered to zero drops the check, since +/// a fresh install of the candidate declares none. +#[cfg(feature = "runtime")] +pub(crate) fn replace_length_check_statement( + entity: &CompiledEntity, + names: &crate::physical_names::EntityPhysicalNames, + field: &crate::model::CompiledField, +) -> Option { + replace_inline_field_check_statement(entity, names, field, "length") +} + +/// Replace the one inline `CHECK` over exactly this column with the candidate +/// field's check, keeping the name PostgreSQL chose. A field pattern is a +/// second single-column check under a compiler-assigned name, so that name is +/// excluded beside the entity's named constraints. The block's dollar-quote +/// tag is `breg_` followed by `purpose`, so the statement names the check it +/// replaces wherever an operator reads it. +#[cfg(feature = "runtime")] +fn replace_inline_field_check_statement( + entity: &CompiledEntity, + names: &crate::physical_names::EntityPhysicalNames, + field: &crate::model::CompiledField, + purpose: &str, ) -> Option { if field.encryption.is_some() { return None; } - let check = field_check("e_identifier(&field.physical_name), &field.field_type)?; let table = quote_literal(&entity.physical_table); + let alter = match field_check("e_identifier(&field.physical_name), &field.field_type) { + Some(check) => format!( + "EXECUTE format('ALTER TABLE registry_data.%I DROP CONSTRAINT %I, ADD CONSTRAINT %I CHECK (%s)', {table}, check_name, check_name, {check});", + check = quote_literal(&check), + ), + None => format!( + "EXECUTE format('ALTER TABLE registry_data.%I DROP CONSTRAINT %I', {table}, check_name);" + ), + }; let named_constraints = names .constraints .values() .cloned() .chain([temporal_order_constraint_name(&entity.id)]) + .chain( + field + .pattern + .is_some() + .then(|| field_pattern_constraint_name(&entity.id, &field.id)), + ) .collect::>() .iter() .map(|name| quote_literal(name)) .collect::>() .join(", "); Some(DdlStatement { - id: format!("entity.{}.field.{}.vocabulary", entity.id, field.id), + id: format!("entity.{}.field.{}.{purpose}", entity.id, field.id), kind: DdlStatementKind::Constraint, sql: format!( - "DO $breg_vocabulary$\n\ + "DO $breg_{purpose}$\n\ DECLARE\n\ \x20 check_name name;\n\ BEGIN\n\ @@ -897,11 +943,10 @@ pub(crate) fn replace_vocabulary_check_statement( \x20 WHERE n.nspname = 'registry_data' AND t.relname = {table}\n\ \x20 AND c.contype = 'c' AND c.conkey = ARRAY[a.attnum]\n\ \x20 AND c.conname <> ALL (ARRAY[{named_constraints}]::name[]);\n\ - \x20 EXECUTE format('ALTER TABLE registry_data.%I DROP CONSTRAINT %I, ADD CONSTRAINT %I CHECK (%s)', {table}, check_name, check_name, {check});\n\ + \x20 {alter}\n\ END\n\ - $breg_vocabulary$", + $breg_{purpose}$", column = quote_literal(&field.physical_name), - check = quote_literal(&check), ), }) } @@ -1058,7 +1103,7 @@ fn runtime_privileges( { privileges.insert(TablePrivilege::Select); } - if operations.contains(&Operation::Create) { + if profile_inserts(&operations) { privileges.insert(TablePrivilege::Insert); privileges.insert(TablePrivilege::Select); } @@ -1225,7 +1270,7 @@ fn policies( check_expression, }); } - if profile.operations.contains(&Operation::Create) + if profile_inserts(&profile.operations) && (!profile_supports_command(&profile.operations, PolicyCommand::Select) || profile.request_visibility.is_some()) { @@ -1489,6 +1534,13 @@ fn spatial_bbox_predicate( ) } +/// A profile inserts rows through `create`, or through `import` chunks of a +/// durable ingestion run. Neither grants a standing read: a create-only +/// profile reads back only the row it just created. +fn profile_inserts(operations: &BTreeSet) -> bool { + operations.contains(&Operation::Create) || operations.contains(&Operation::Import) +} + fn profile_supports_command(operations: &BTreeSet, command: PolicyCommand) -> bool { match command { PolicyCommand::Select => operations.iter().any(|operation| { @@ -1502,7 +1554,7 @@ fn profile_supports_command(operations: &BTreeSet, command: PolicyCom | Operation::Snapshot ) }), - PolicyCommand::Insert => operations.contains(&Operation::Create), + PolicyCommand::Insert => profile_inserts(operations), PolicyCommand::Update => operations .iter() .any(|operation| matches!(operation, Operation::Patch | Operation::Tombstone)), diff --git a/crates/registry-breg/src/history_maintenance.rs b/crates/registry-breg/src/history_maintenance.rs index 41db26f4a5..3a815d6b61 100644 --- a/crates/registry-breg/src/history_maintenance.rs +++ b/crates/registry-breg/src/history_maintenance.rs @@ -65,6 +65,7 @@ impl From for HistoryMaintenanceError { Self::InvalidInput } PostgresKernelError::Connection + | PostgresKernelError::Statement(_) | PostgresKernelError::Pool | PostgresKernelError::PoolBuild | PostgresKernelError::CatalogInvariant(_) diff --git a/crates/registry-breg/src/history_migration.rs b/crates/registry-breg/src/history_migration.rs index a5310a0144..8d1cf51619 100644 --- a/crates/registry-breg/src/history_migration.rs +++ b/crates/registry-breg/src/history_migration.rs @@ -58,8 +58,6 @@ impl SupportedHistoryMigrationStep { #[derive(Clone, Debug, Error, Eq, PartialEq)] pub(crate) enum HistoryMigrationError { - #[error("reviewed chunked backfills are not history-safe yet")] - ChunkedBackfillUnsupported, #[error("reviewed transactional SQL must declare affected-row bounds for history")] UnboundedTransactionalSql, #[error("reviewed transactional SQL must name at least one data object")] @@ -92,10 +90,10 @@ pub(crate) struct BoundedHistoryUpdateCapture { rows: BTreeMap, } -/// The page-scoped capture one field-encryption chunk journals: exactly the -/// rows the chunk selected, locked, sealed, and rewrote in this transaction. +/// The page-scoped capture one reviewed chunk journals: exactly the rows the +/// chunk selected, locked, and rewrote in this transaction. #[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct FieldEncryptionPageCapture { +pub(crate) struct ReviewedPageCapture { step: SupportedHistoryMigrationStep, rows: BTreeMap, } @@ -105,9 +103,26 @@ struct CapturedEntityRow { record_revision: i64, record_lifecycle: String, active_package_revision: String, + /// `created_at` and `updated_at` as text in the capturing session, so two + /// captures in one transaction compare exactly. + created_at: String, + updated_at: String, data: Map, } +impl CapturedEntityRow { + /// Whether `after` carries the same record metadata as this row. Only the + /// journal writes record metadata, so a reviewed step that changed any of + /// it is refused. + fn keeps_record_metadata_of(&self, after: &CapturedEntityRow) -> bool { + self.record_revision == after.record_revision + && self.record_lifecycle == after.record_lifecycle + && self.active_package_revision == after.active_package_revision + && self.created_at == after.created_at + && self.updated_at == after.updated_at + } +} + #[derive(Clone, Debug, Eq, PartialEq)] struct LatestRevisionBinding { record_reference: String, @@ -211,18 +226,19 @@ pub(crate) async fn finish_bounded_history_update( .await } -/// Capture the pre-change rows of one field-encryption chunk page. The step's -/// classified entity is the successor registry's entity, so encrypted fields -/// project their envelope column and the capture is already in journal shape. +/// Capture the pre-change rows of one chunk page of a reviewed chunked +/// backfill or a field-encryption backfill. The step's classified entity is +/// the successor registry's entity, so encrypted fields project their envelope +/// column and the capture is already in journal shape. #[cfg(feature = "runtime")] -pub(crate) async fn prepare_field_encryption_page_capture( +pub(crate) async fn prepare_reviewed_page_capture( transaction: &Transaction<'_>, registry: &CompiledRegistry, descriptor_path: &str, step: &ValidatedReviewedMigrationStep, page: &[Uuid], -) -> Result { - let supported = classify_reviewed_history_step(descriptor_path, step)?; +) -> Result { + let supported = check_reviewed_history_step(descriptor_path, step)?; let page_len = u64::try_from(page.len()).map_err(|_| HistoryMigrationError::InvalidAffectedRows)?; if page_len > supported.affected_rows.max { @@ -230,18 +246,18 @@ pub(crate) async fn prepare_field_encryption_page_capture( } let entity = entity_for_step(registry, &supported)?; let rows = capture_entity_rows_page(transaction, entity, page).await?; - Ok(FieldEncryptionPageCapture { + Ok(ReviewedPageCapture { step: supported, rows, }) } #[cfg(feature = "runtime")] -pub(crate) async fn finish_field_encryption_page_update( +pub(crate) async fn finish_reviewed_page_update( transaction: &Transaction<'_>, registry: &CompiledRegistry, package_revision: &str, - capture: FieldEncryptionPageCapture, + capture: ReviewedPageCapture, ) -> Result { if package_revision.is_empty() { return Err(HistoryMigrationError::RevisionUnavailable); @@ -284,10 +300,7 @@ async fn journal_captured_changes( let after = post_rows .get(record_id) .ok_or(HistoryMigrationError::UnexpectedRowShape)?; - if before.record_revision != after.record_revision - || before.record_lifecycle != after.record_lifecycle - || before.active_package_revision != after.active_package_revision - { + if !before.keeps_record_metadata_of(after) { return Err(HistoryMigrationError::UnexpectedRowShape); } if before.data == after.data { @@ -409,7 +422,8 @@ pub(crate) async fn verify_live_rows_match_journal_heads( ); for entity in entities.values() { let live_rows = capture_entity_rows(transaction, entity, true).await?; - let latest_revisions = load_latest_revision_snapshots(transaction, &entity.id).await?; + let latest_revisions = + load_latest_revision_snapshots(transaction, &entity.id, None).await?; if live_rows.keys().ne(latest_revisions.keys()) { return Err(HistoryMigrationError::UnexpectedRowShape); } @@ -417,16 +431,7 @@ pub(crate) async fn verify_live_rows_match_journal_heads( let latest = latest_revisions .get(&record_id) .ok_or(HistoryMigrationError::UnexpectedRowShape)?; - let snapshot = canonical_snapshot(&live.data) - .map_err(|_| HistoryMigrationError::RevisionUnavailable)?; - if live.record_revision != latest.record_revision - || live.record_lifecycle != latest.record_lifecycle - || required_package_revision - .is_some_and(|required| latest.package_revision != required) - || snapshot != latest.snapshot - { - return Err(HistoryMigrationError::UnexpectedRowShape); - } + verify_journal_head_reproduces_live_row(&live, latest, required_package_revision)?; members.push(BaselineMember { entity_id: entity.id.clone(), record_id, @@ -437,6 +442,152 @@ pub(crate) async fn verify_live_rows_match_journal_heads( Ok(members) } +/// The live rows one page of [`verify_every_live_row_matches_its_journal_head`] +/// reads, and the journal heads it loads beside them. +#[cfg(feature = "runtime")] +const LIVE_ROW_VERIFICATION_PAGE_ROWS: i64 = 1_000; + +/// Prove the retained journal head of every live row reproduces that row, and +/// that every retained journal head still has its live row, page by page. +/// Returns how many live rows were verified. +/// +/// Each entity table is locked against writes before its first page, so every +/// page reads one stable state for the rest of the caller's transaction. A +/// caller that has already lifted forced row security on the entity tables +/// holds them exclusively, reads included, and this lock adds nothing to that. +/// Every statement reads one page: at most [`LIVE_ROW_VERIFICATION_PAGE_ROWS`] +/// live rows, their journal heads, and the retained heads whose record +/// identifiers fall in the page's key range, so the number of live rows is not +/// bounded here. A statement's cost is bounded by its page's records and their +/// retained revisions, not by a fixed size: records with many revisions make a +/// larger page, and the final count per entity covers every head past its last +/// live row. Nothing is +/// collected for a commit: a caller that must index the live rows as members +/// uses [`verify_live_rows_match_journal_heads`], which the commit-member budget +/// bounds. +#[cfg(feature = "runtime")] +pub(crate) async fn verify_every_live_row_matches_its_journal_head( + transaction: &Transaction<'_>, + entities: &BTreeMap, +) -> Result { + let mut verified = 0_u64; + for entity in entities.values() { + let table_name = SqlIdentifier::parse(&entity.physical_table) + .map_err(|_| HistoryMigrationError::UnsupportedObject)?; + transaction + .batch_execute(&format!( + "LOCK TABLE registry_data.{} IN SHARE ROW EXCLUSIVE MODE", + table_name.quoted() + )) + .await + .map_err(|_| HistoryMigrationError::RevisionUnavailable)?; + let projection = history_returning_projection(entity); + let mut after: Option = None; + loop { + let rows = transaction + .query( + &format!( + "SELECT {projection} + FROM registry_data.{} + WHERE $1::uuid IS NULL OR record_id > $1::uuid + ORDER BY record_id + LIMIT $2", + table_name.quoted() + ), + &[&after, &LIVE_ROW_VERIFICATION_PAGE_ROWS], + ) + .await + .map_err(|_| HistoryMigrationError::RevisionUnavailable)?; + let fetched = rows.len(); + let live_rows = decode_captured_rows(rows, entity)?; + let Some(last) = live_rows.keys().next_back().copied() else { + break; + }; + let page = live_rows.keys().copied().collect::>(); + let latest_revisions = + load_latest_revision_snapshots(transaction, &entity.id, Some(&page)).await?; + if live_rows.keys().ne(latest_revisions.keys()) { + return Err(HistoryMigrationError::UnexpectedRowShape); + } + for (record_id, live) in &live_rows { + let latest = latest_revisions + .get(record_id) + .ok_or(HistoryMigrationError::UnexpectedRowShape)?; + verify_journal_head_reproduces_live_row(live, latest, None)?; + } + let page_rows = u64::try_from(live_rows.len()) + .map_err(|_| HistoryMigrationError::UnexpectedRowShape)?; + // Every live row of the page has a journal head, proved above, so + // an equal count of retained heads across the page's key range + // leaves no head there without its live row. + if count_retained_journal_heads_in_range(transaction, &entity.id, after, Some(last)) + .await? + != page_rows + { + return Err(HistoryMigrationError::UnexpectedRowShape); + } + verified = verified + .checked_add(page_rows) + .ok_or(HistoryMigrationError::UnexpectedRowShape)?; + after = Some(last); + if i64::try_from(fetched).map_or(true, |count| count < LIVE_ROW_VERIFICATION_PAGE_ROWS) + { + break; + } + } + // No live row lies past the last page, so no retained head may either. + if count_retained_journal_heads_in_range(transaction, &entity.id, after, None).await? != 0 { + return Err(HistoryMigrationError::UnexpectedRowShape); + } + } + Ok(verified) +} + +/// Count the distinct records holding a retained journal head for one entity +/// whose identifiers lie after `after` and up to `through`, either bound open +/// when absent. The primary key leads with the entity and record identifiers, +/// so the count reads only that key range. +#[cfg(feature = "runtime")] +async fn count_retained_journal_heads_in_range( + transaction: &Transaction<'_>, + entity_id: &str, + after: Option, + through: Option, +) -> Result { + let count = transaction + .query_one( + "SELECT count(DISTINCT record_id)::bigint + FROM registry_internal.registry_revisions + WHERE entity_id = $1 + AND ($2::uuid IS NULL OR record_id > $2::uuid) + AND ($3::uuid IS NULL OR record_id <= $3::uuid)", + &[&entity_id, &after, &through], + ) + .await + .map_err(|_| HistoryMigrationError::RevisionUnavailable)? + .try_get::<_, i64>(0) + .map_err(|_| HistoryMigrationError::RevisionUnavailable)?; + u64::try_from(count).map_err(|_| HistoryMigrationError::UnexpectedRowShape) +} + +#[cfg(feature = "runtime")] +fn verify_journal_head_reproduces_live_row( + live: &CapturedEntityRow, + latest: &LatestRevisionSnapshot, + required_package_revision: Option<&str>, +) -> Result<()> { + let snapshot = + canonical_snapshot(&live.data).map_err(|_| HistoryMigrationError::RevisionUnavailable)?; + if live.record_revision != latest.record_revision + || live.record_lifecycle != latest.record_lifecycle + || required_package_revision.is_some_and(|required| latest.package_revision != required) + || snapshot != latest.snapshot + { + return Err(HistoryMigrationError::UnexpectedRowShape); + } + Ok(()) +} + #[cfg(feature = "runtime")] async fn verify_revision_journal_uses_active_descriptor( transaction: &Transaction<'_>, @@ -512,6 +663,7 @@ async fn count_entity_rows(transaction: &Transaction<'_>, entity: &CompiledEntit async fn load_latest_revision_snapshots( transaction: &Transaction<'_>, entity_id: &str, + records: Option<&[Uuid]>, ) -> Result> { let rows = transaction .query( @@ -519,8 +671,9 @@ async fn load_latest_revision_snapshots( record_id, record_revision, record_lifecycle, package_revision, snapshot FROM registry_internal.registry_revisions WHERE entity_id = $1 + AND ($2::uuid[] IS NULL OR record_id = ANY($2::uuid[])) ORDER BY record_id, record_revision DESC", - &[&entity_id], + &[&entity_id, &records], ) .await .map_err(|_| HistoryMigrationError::RevisionUnavailable)?; @@ -649,8 +802,33 @@ fn classify_reviewed_history_step( affected_rows, }) } - ReviewedMigrationStepDescriptor::ChunkedBackfill { .. } => { - Err(HistoryMigrationError::ChunkedBackfillUnsupported) + ReviewedMigrationStepDescriptor::ChunkedBackfill { + id, + entity_id, + objects, + chunk_size, + .. + } => { + let chunk_size = u64::from(*chunk_size); + // One chunk's changed rows are one commit's member set, so the + // commit-member budget bounds the chunk size. + if chunk_size == 0 || chunk_size > MAX_HISTORY_MIGRATION_COMMIT_MEMBERS { + return Err(HistoryMigrationError::InvalidAffectedRows); + } + let (object_entity_id, physical_table) = classify_step_objects(objects)?; + if &object_entity_id != entity_id { + return Err(HistoryMigrationError::CrossEntityStep); + } + Ok(SupportedHistoryMigrationStep { + descriptor_path: descriptor_path.to_owned(), + step_id: id.clone(), + entity_id: object_entity_id, + physical_table, + affected_rows: AffectedRowBounds { + min: 0, + max: chunk_size, + }, + }) } ReviewedMigrationStepDescriptor::FieldEncryptionBackfill { id, @@ -716,32 +894,198 @@ fn classify_step_objects( Ok((entity_id, physical_table)) } +/// Classify a reviewed step apply journals and check its authored SQL has the +/// shape the journal accepts, without touching the database. Activation runs +/// this before a journaled step changes a row, and `bregctl test` runs it +/// during the rehearsal, so both refuse the same steps. +pub(crate) fn check_reviewed_history_step( + descriptor_path: &str, + step: &ValidatedReviewedMigrationStep, +) -> Result { + let supported = classify_reviewed_history_step(descriptor_path, step)?; + match &step.descriptor { + ReviewedMigrationStepDescriptor::TransactionalSql { .. } => { + validate_reviewed_update_sql(&step.sql)?; + } + ReviewedMigrationStepDescriptor::ChunkedBackfill { .. } => { + validate_reviewed_chunk_sql(&step.sql)?; + } + // The engine writes the field-encryption statement; no authored SQL. + ReviewedMigrationStepDescriptor::FieldEncryptionBackfill { .. } => {} + } + Ok(supported) +} + +/// Statements a reviewed update may not contain, and the record metadata only +/// the journal may write, each matched as a whole word. +const REFUSED_REVIEWED_UPDATE_WORDS: [&str; 13] = [ + "insert", + "delete", + "truncate", + "alter", + "drop", + "create", + "merge", + "record_revision", + "record_lifecycle", + "active_package_revision", + "created_at", + "updated_at", + "uescape", +]; + fn validate_reviewed_update_sql(sql: &str) -> Result<()> { - let normalized = sql.trim().trim_end_matches(';').trim(); - let lowercase = normalized.to_ascii_lowercase(); - if !lowercase.starts_with("update ") || lowercase.contains(';') { + let words = reviewed_update_words(sql)?; + if words.iter().any(|word| word == "record_id") { + return Err(HistoryMigrationError::UnsupportedSqlShape); + } + Ok(()) +} + +/// A chunked backfill binds its page of record identifiers as `$1`, so it may +/// name `record_id`; every other refusal of a reviewed update still applies. +fn validate_reviewed_chunk_sql(sql: &str) -> Result<()> { + reviewed_update_words(sql).map(|_| ()) +} + +/// The lowercased words of one reviewed update statement, refused unless the +/// statement starts with `UPDATE`, holds no second statement, and names no +/// refused word. This is a lexical first check only: activation parses the +/// statement and pins its shape, and the journal refuses a step that changes +/// record metadata. +fn reviewed_update_words(sql: &str) -> Result> { + // A Unicode-escape identifier or string (`U&"..."`, `U&'...'`) spells a + // name this scan cannot read, so it is refused anywhere in the text. + if sql + .as_bytes() + .windows(3) + .any(|window| matches!(window, [b'u' | b'U', b'&', b'"' | b'\''])) + { + return Err(HistoryMigrationError::UnsupportedSqlShape); + } + // A statement the scan cannot delimit with certainty is read as written, + // so a leading comment then refuses it and every word inside its literals + // counts. + let text = mask_comments_and_literals(sql).unwrap_or_else(|| sql.to_owned()); + let statement = text.trim().trim_end_matches(';').trim(); + if statement.contains(';') { + return Err(HistoryMigrationError::UnsupportedSqlShape); + } + let words = statement + // `$` splits words here, so a dollar-quoted body read as written + // still exposes the words inside it. + .split(|character: char| character == '$' || !is_sql_word_character(character)) + .filter(|word| !word.is_empty()) + .map(str::to_ascii_lowercase) + .collect::>(); + let starts_with_update = statement + .get(..6) + .is_some_and(|first| first.eq_ignore_ascii_case("update")) + && words.first().is_some_and(|word| word == "update"); + if !starts_with_update + || words + .iter() + .any(|word| REFUSED_REVIEWED_UPDATE_WORDS.contains(&word.as_str())) + { return Err(HistoryMigrationError::UnsupportedSqlShape); } - for refused in [ - " insert ", - " delete ", - " truncate ", - " alter ", - " drop ", - " create ", - " merge ", - " record_id", - " record_revision", - " record_lifecycle", - " active_package_revision", - " created_at", - " updated_at", - ] { - if lowercase.contains(refused) { - return Err(HistoryMigrationError::UnsupportedSqlShape); + Ok(words) +} + +fn is_sql_word_character(character: char) -> bool { + character.is_ascii_alphanumeric() + || character == '_' + || character == '$' + || !character.is_ascii() +} + +/// Replace comments and the contents of plain string literals with a space and +/// keep a quoted identifier's name as a word. Returns `None` when the statement +/// holds a construct whose extent depends on server settings or on a tag this +/// scan would have to trust: a dollar-quoted body, a backslash inside a quoted +/// literal, or an unterminated comment, literal, or identifier. +fn mask_comments_and_literals(sql: &str) -> Option { + let characters = sql.chars().collect::>(); + let mut masked = String::with_capacity(sql.len()); + let mut index = 0; + while let Some(&character) = characters.get(index) { + let next = characters.get(index + 1).copied(); + match character { + '-' if next == Some('-') => { + // PostgreSQL ends a line comment at either newline character. + while characters + .get(index) + .is_some_and(|&c| c != '\n' && c != '\r') + { + index += 1; + } + masked.push(' '); + } + '/' if next == Some('*') => { + let mut depth = 0_usize; + loop { + match (characters.get(index), characters.get(index + 1)) { + (Some('/'), Some('*')) => { + depth += 1; + index += 2; + } + (Some('*'), Some('/')) => { + depth -= 1; + index += 2; + if depth == 0 { + break; + } + } + (Some(_), _) => index += 1, + (None, _) => return None, + } + } + masked.push(' '); + } + '\'' | '"' => { + let mut name = String::new(); + index += 1; + loop { + match (characters.get(index), characters.get(index + 1)) { + (Some('\\'), _) => return None, + (Some(&c), Some(&following)) + if c == character && following == character => + { + name.push(c); + index += 2; + } + (Some(&c), _) if c == character => { + index += 1; + break; + } + (Some(&c), _) => { + name.push(c); + index += 1; + } + (None, _) => return None, + } + } + masked.push(' '); + if character == '"' { + masked.push_str(&name); + masked.push(' '); + } + } + '$' => { + let follows_word = index > 0 && is_sql_word_character(characters[index - 1]); + if !follows_word && !next.is_some_and(|c| c.is_ascii_digit()) { + return None; + } + masked.push(character); + index += 1; + } + _ => { + masked.push(character); + index += 1; + } } } - Ok(()) + Some(masked) } fn entity_for_step<'a>( @@ -792,8 +1136,8 @@ async fn capture_entity_rows( decode_captured_rows(rows, entity) } -/// Capture exactly the rows one field-encryption chunk selected and locked in -/// this transaction. The caller holds the page's row locks, so no table lock +/// Capture exactly the rows one reviewed chunk selected and locked in this +/// transaction. The caller holds the page's row locks, so no table lock /// or re-lock is needed here. #[cfg(feature = "runtime")] async fn capture_entity_rows_page( @@ -849,7 +1193,7 @@ fn decode_captured_rows( || record_revision <= 0 || !matches!(record_lifecycle.as_str(), "active" | "tombstoned") || active_package_revision.is_empty() - || row.len() != entity.fields.len() + 4 + || row.len() != entity.fields.len() + CAPTURED_METADATA_COLUMNS { return Err(HistoryMigrationError::UnexpectedRowShape); } @@ -870,6 +1214,12 @@ fn decode_captured_rows( }; data.insert(field.id.clone(), value); } + let timestamp = |index: usize| { + row.try_get::<_, String>(entity.fields.len() + index) + .map_err(|_| HistoryMigrationError::RevisionUnavailable) + }; + let created_at = timestamp(4)?; + let updated_at = timestamp(5)?; if captured .insert( record_uuid, @@ -877,6 +1227,8 @@ fn decode_captured_rows( record_revision, record_lifecycle, active_package_revision, + created_at, + updated_at, data, }, ) @@ -888,6 +1240,10 @@ fn decode_captured_rows( Ok(captured) } +/// The record metadata columns [`history_returning_projection`] reads beside +/// the declared fields: four before them, `created_at` and `updated_at` after. +const CAPTURED_METADATA_COLUMNS: usize = 6; + fn history_returning_projection(entity: &CompiledEntity) -> String { let mut expressions = vec![ "record_id::text".to_owned(), @@ -896,6 +1252,8 @@ fn history_returning_projection(entity: &CompiledEntity) -> String { "active_package_revision".to_owned(), ]; expressions.extend(entity.fields.values().map(field_json_projection)); + expressions.push("created_at::text".to_owned()); + expressions.push("updated_at::text".to_owned()); expressions.join(", ") } @@ -1077,26 +1435,216 @@ mod tests { assert_eq!(error, HistoryMigrationError::InvalidAffectedRows); } + fn chunked_step(entity_id: &str, chunk_size: u32) -> ValidatedReviewedMigrationStep { + let mut step = step(ReviewedMigrationStepDescriptor::ChunkedBackfill { + id: "backfill-household".to_owned(), + entity_id: entity_id.to_owned(), + sql_path: "migrations/backfill.sql".to_owned(), + objects: vec![object("household", "households")], + cursor: ChunkCursorProtocol::RecordIdUuidArray, + chunk_size, + max_total_rows: 10_000, + lock_timeout_ms: 1_000, + statement_timeout_ms: 10_000, + exact_affected_rows: true, + }); + step.sql = "UPDATE registry_data.households SET status = 'active' \ + WHERE record_id = ANY($1::pg_catalog.uuid[])" + .to_owned(); + step + } + #[test] - fn chunked_backfill_is_refused_for_history_migration() { - let error = classify_reviewed_history_step( + fn chunked_backfill_journals_each_chunk_as_one_commit() { + let classified = check_reviewed_history_step( "migrations/descriptor.json", - &step(ReviewedMigrationStepDescriptor::ChunkedBackfill { - id: "backfill-household".to_owned(), - entity_id: "household".to_owned(), - sql_path: "migrations/backfill.sql".to_owned(), - objects: vec![object("household", "households")], - cursor: ChunkCursorProtocol::RecordIdUuidArray, - chunk_size: 100, - max_total_rows: 1_000, - lock_timeout_ms: 1_000, - statement_timeout_ms: 10_000, - exact_affected_rows: true, - }), + &chunked_step("household", 100), ) - .expect_err("chunked backfills need a fuller history engine"); + .expect("a chunked backfill over one entity is journaled"); - assert_eq!(error, HistoryMigrationError::ChunkedBackfillUnsupported); + assert_eq!(classified.entity_id, "household"); + assert_eq!(classified.physical_table, "households"); + assert_eq!( + classified.affected_rows, + AffectedRowBounds { min: 0, max: 100 }, + "one chunk commits at most its chunk size, and may change nothing" + ); + } + + #[test] + fn chunked_backfill_above_the_commit_limit_is_refused() { + let chunk_size = u32::try_from(MAX_HISTORY_MIGRATION_COMMIT_MEMBERS + 1).unwrap(); + let error = check_reviewed_history_step( + "migrations/descriptor.json", + &chunked_step("household", chunk_size), + ) + .expect_err("one chunk is one commit, so it cannot exceed the commit-member cap"); + + assert_eq!(error, HistoryMigrationError::InvalidAffectedRows); + } + + #[test] + fn chunked_backfill_over_another_entity_than_its_objects_is_refused() { + let error = + check_reviewed_history_step("migrations/descriptor.json", &chunked_step("member", 100)) + .expect_err("the chunked entity and the step's objects must agree"); + + assert_eq!(error, HistoryMigrationError::CrossEntityStep); + } + + #[test] + fn chunked_backfill_may_bind_its_page_but_not_write_record_metadata() { + let mut forged = chunked_step("household", 100); + forged.sql = "UPDATE registry_data.households SET record_revision = 9 \ + WHERE record_id = ANY($1::pg_catalog.uuid[])" + .to_owned(); + assert_eq!( + check_reviewed_history_step("migrations/descriptor.json", &forged), + Err(HistoryMigrationError::UnsupportedSqlShape), + "record metadata belongs to the journal" + ); + + let mut transactional = step(ReviewedMigrationStepDescriptor::TransactionalSql { + id: "normalize-household".to_owned(), + sql_path: "migrations/normalize.sql".to_owned(), + objects: vec![object("household", "households")], + affected_rows: Some(AffectedRowBounds { min: 0, max: 10 }), + }); + transactional.sql = "UPDATE registry_data.households SET status = 'active' \ + WHERE record_id = ANY('{}'::pg_catalog.uuid[])" + .to_owned(); + assert_eq!( + check_reviewed_history_step("migrations/descriptor.json", &transactional), + Err(HistoryMigrationError::UnsupportedSqlShape), + "a transactional update binds no page, so it still may not name record_id" + ); + } + + fn chunked_sql_check(sql: &str) -> Result { + let mut step = chunked_step("household", 100); + step.sql = sql.to_owned(); + check_reviewed_history_step("migrations/descriptor.json", &step) + } + + #[test] + fn chunked_backfill_accepts_comments_line_breaks_and_words_inside_literals() { + for sql in [ + "-- SPDX-License-Identifier: Apache-2.0\n\ + /* Normalise the status, /* nested */ once. */\n\ + UPDATE registry_data.households SET status = 'active' \ + WHERE record_id = ANY($1::pg_catalog.uuid[]);", + "UPDATE\n registry_data.households\n SET status = 'active'\n\t WHERE \ + record_id = ANY($1::pg_catalog.uuid[])", + "UPDATE registry_data.households SET created_at_source = 'form', \ + updated_at_source = 'form' WHERE record_id = ANY($1::pg_catalog.uuid[])", + "UPDATE registry_data.households SET note = 'delete me; then insert it''s drop' \ + WHERE record_id = ANY($1::pg_catalog.uuid[]) -- create nothing else", + ] { + assert!( + chunked_sql_check(sql).is_ok(), + "a valid chunked update is accepted: {sql}" + ); + } + } + + #[test] + fn a_reviewed_step_that_changes_any_record_metadata_is_detected() { + let before = CapturedEntityRow { + record_revision: 3, + record_lifecycle: "active".to_owned(), + active_package_revision: "package-1".to_owned(), + created_at: "2026-01-01 00:00:00+00".to_owned(), + updated_at: "2026-02-01 00:00:00+00".to_owned(), + data: Map::new(), + }; + let mut data_only = before.clone(); + data_only + .data + .insert("status".to_owned(), Value::String("active".to_owned())); + assert!( + before.keeps_record_metadata_of(&data_only), + "a change to declared fields alone keeps the record metadata" + ); + type MetadataChange = (&'static str, fn(&mut CapturedEntityRow)); + let changes: [MetadataChange; 5] = [ + ("record_revision", |row| row.record_revision += 1), + ("record_lifecycle", |row| { + row.record_lifecycle = "tombstoned".to_owned(); + }), + ("active_package_revision", |row| { + row.active_package_revision = "package-2".to_owned(); + }), + ("created_at", |row| { + row.created_at = "2026-03-01 00:00:00+00".to_owned(); + }), + ("updated_at", |row| { + row.updated_at = "2026-03-01 00:00:00+00".to_owned(); + }), + ]; + for (column, change) in changes { + let mut after = before.clone(); + change(&mut after); + assert!( + !before.keeps_record_metadata_of(&after), + "a changed {column} is detected" + ); + } + } + + #[test] + fn chunked_backfill_refuses_the_statement_shapes_the_journal_cannot_hold() { + for sql in [ + // Record metadata, named plainly, quoted, in capitals, behind a + // comment PostgreSQL ends at a carriage return, or through a + // Unicode escape. + "UPDATE registry_data.households SET created_at = now() \ + WHERE record_id = ANY($1::pg_catalog.uuid[])", + "UPDATE registry_data.households SET status = 'a' --\r, record_revision = 9\n \ + WHERE record_id = ANY($1::pg_catalog.uuid[])", + "UPDATE registry_data.households SET U&\"created\\005Fat\" = now() \ + WHERE record_id = ANY($1::pg_catalog.uuid[])", + "UPDATE registry_data.households SET u&\"created!005Fat\" UESCAPE '!' = now() \ + WHERE record_id = ANY($1::pg_catalog.uuid[])", + "UPDATE registry_data.households AS h SET status = h.updated_at::text \ + WHERE record_id = ANY($1::pg_catalog.uuid[])", + "UPDATE registry_data.households SET \"record_revision\" = 9 \ + WHERE record_id = ANY($1::pg_catalog.uuid[])", + "UPDATE registry_data.households SET RECORD_LIFECYCLE = 'active' \ + WHERE record_id = ANY($1::pg_catalog.uuid[])", + // A second statement, or a statement that is not an update. + "UPDATE registry_data.households SET status = 'a' \ + WHERE record_id = ANY($1::pg_catalog.uuid[]); DELETE FROM registry_data.households", + "UPDATE registry_data.households SET status = 'a' \ + WHERE record_id = ANY($1::pg_catalog.uuid[]);\nSELECT 1", + "-- UPDATE registry_data.households\nSELECT 1", + "/* UPDATE */ DELETE FROM registry_data.households WHERE record_id = ANY($1)", + "WITH moved AS (DELETE FROM registry_data.households RETURNING record_id) \ + UPDATE registry_data.households SET status = 'a' WHERE record_id = ANY($1)", + "UPDATE registry_data.households SET status = (SELECT 1 FROM (INSERT INTO x VALUES (1)) i) \ + WHERE record_id = ANY($1::pg_catalog.uuid[])", + // Constructs whose extent the scan will not guess are read as + // written, so a refused word inside them still refuses. + "UPDATE registry_data.households SET status = $$delete$$ \ + WHERE record_id = ANY($1::pg_catalog.uuid[])", + "UPDATE registry_data.households SET status = E'\\' delete ' \ + WHERE record_id = ANY($1::pg_catalog.uuid[])", + "UPDATE registry_data.households SET status = 'unterminated delete \ + WHERE record_id = ANY($1::pg_catalog.uuid[])", + "$$ $$ UPDATE registry_data.households SET status = 'a' \ + WHERE record_id = ANY($1::pg_catalog.uuid[])", + "-- a comment the scan cannot end\n UPDATE registry_data.households \ + SET status = $x$a$x$ WHERE record_id = ANY($1::pg_catalog.uuid[])", + "/* unterminated UPDATE registry_data.households SET status = 'a' \ + WHERE record_id = ANY($1::pg_catalog.uuid[])", + "", + ";", + ] { + assert_eq!( + chunked_sql_check(sql), + Err(HistoryMigrationError::UnsupportedSqlShape), + "the statement is refused: {sql}" + ); + } } #[test] diff --git a/crates/registry-breg/src/history_rebaseline.rs b/crates/registry-breg/src/history_rebaseline.rs index d74fd404d5..97b69afd1e 100644 --- a/crates/registry-breg/src/history_rebaseline.rs +++ b/crates/registry-breg/src/history_rebaseline.rs @@ -27,7 +27,9 @@ use crate::history_maintenance::{ append_maintenance_entries, begin_maintenance_request, profile_is_keyed, set_local_timeouts, verify_ready_identity, HistoryMaintenanceError, }; -use crate::history_migration::{verify_live_rows_match_journal_heads, HistoryMigrationError}; +use crate::history_migration::{ + verify_every_live_row_matches_its_journal_head, HistoryMigrationError, +}; use crate::model::CompiledRegistry; use crate::postgres::{ set_force_row_security, verify_migration_role, ConnectionConfig, ExpectedRegistryIdentity, @@ -36,11 +38,6 @@ use crate::postgres::{ pub use crate::history_maintenance::HistoryMaintenanceTimeouts as HistoryRebaselineTimeouts; -/// The number of live rows one rebaseline verifies inside its single -/// transaction. Verification reads every live row and its journal head at once, -/// so the bound the existing-data migration enforces is the bound here too. -pub use crate::history_migration::MAX_HISTORY_MIGRATION_COMMIT_MEMBERS as MAX_REBASELINE_LIVE_ROWS; - const MAX_OPERATOR_REFERENCE_BYTES: usize = 512; const AUDIT_OPERATION_ID: &str = "history-rebaseline-maintenance"; const BASELINE_SYSTEM_ORIGIN: &str = "breg-coverage-rebaseline-v1"; @@ -81,8 +78,6 @@ pub enum HistoryRebaselineError { UnindexedRevisions, #[error("history rebaseline requires the retained journal head to reproduce every live row")] LiveHistoryMismatch, - #[error("history rebaseline exceeds the supported live-row budget")] - LiveRowBudgetExceeded, #[error("history rebaseline storage is unavailable")] Unavailable, } @@ -120,7 +115,6 @@ impl From for HistoryRebaselineError { fn from(error: HistoryMigrationError) -> Self { match error { HistoryMigrationError::UnexpectedRowShape => Self::LiveHistoryMismatch, - HistoryMigrationError::BaselineBudgetExceeded => Self::LiveRowBudgetExceeded, HistoryMigrationError::UnsupportedObject => Self::InvalidInput, _ => Self::Unavailable, } @@ -206,23 +200,29 @@ pub(crate) async fn rebaseline_history_coverage_in_transaction( return Err(HistoryRebaselineError::UnindexedRevisions); } - // Reuse the migration baseline check: it proves the retained journal head - // of every live row still reproduces that row, so the new baseline vouches - // only for state the journal already holds. A live registry's journal - // legitimately spans several package revisions, so no single revision is - // required of the heads. + // Prove the retained journal head of every live row still reproduces that + // row, so the new baseline vouches only for state the journal already + // holds. A live registry's journal legitimately spans several package + // revisions, so no single revision is required of the heads. The check + // reads the live rows page by page, so it bounds memory, not the number of + // live rows. // // Entity tables force row-level security on their owner, so the check reads // nothing until the migration authority lifts that force for this // transaction, exactly as an existing-data migration baseline does. Every // other role keeps its policies, and the force is restored before the - // transaction commits. + // transaction commits. Lifting it is an ALTER TABLE, so every entity table + // is held in ACCESS EXCLUSIVE mode from here until the transaction ends: + // reads wait as well as writes for the whole verification. The migration + // role holds no BYPASSRLS authority, which is what would let it read + // without that lock, and it keeps none. let tables = entity_tables(request.registry); set_force_row_security(transaction, &tables, false).await?; let verified = - verify_live_rows_match_journal_heads(transaction, request.registry.entities(), None).await; + verify_every_live_row_matches_its_journal_head(transaction, request.registry.entities()) + .await; set_force_row_security(transaction, &tables, true).await?; - let members = verified?; + let verified_record_count = verified?; // Every retained journal head is already indexed, which the refusal above // proved, so the baseline commit carries no member of its own. It is the @@ -255,8 +255,7 @@ pub(crate) async fn rebaseline_history_coverage_in_transaction( baseline_position: committed.position, verified_entity_count: u64::try_from(request.registry.entities().len()) .map_err(|_| HistoryRebaselineError::Unavailable)?, - verified_record_count: u64::try_from(members.len()) - .map_err(|_| HistoryRebaselineError::Unavailable)?, + verified_record_count, previous_coverage_baseline_position: head.coverage_baseline_position, previous_unavailable_after_position: head.unavailable_after_position, }; diff --git a/crates/registry-breg/src/history_schema.rs b/crates/registry-breg/src/history_schema.rs index 3c34d6ac1d..218b5d5143 100644 --- a/crates/registry-breg/src/history_schema.rs +++ b/crates/registry-breg/src/history_schema.rs @@ -461,14 +461,13 @@ impl HistoryFieldDescriptor { allow_retained_plaintext: bool, ) -> Result { self.validate()?; - // A revision recorded under fewer vocabulary codes stays readable: - // it is decoded against the codes it was recorded under, each of - // which the active field still declares. + // A revision recorded under fewer vocabulary codes, a lower text + // length limit, or a higher string minimum stays readable: it is + // decoded against the type it was recorded under, every value of + // which the active field still admits. if self.id != active.id || (self.field_type != *active.field_type - && !active - .field_type - .keeps_vocabulary_codes_of(&self.field_type)) + && !active.field_type.admits_every_value_of(&self.field_type)) { return Ok(false); } @@ -1131,6 +1130,99 @@ mod tests { entity } + fn with_note_field(field_type: FieldTypeSource) -> CompiledEntity { + let mut entity = membership_entity(); + let note = stored("note", "note", field_type, true, None); + entity.fields.insert( + "note".to_owned(), + crate::model::CompiledField { + pattern: None, + id: "note".to_owned(), + field_type: note.logical.field_type.clone(), + required: true, + classification: Classification::Restricted, + valid_time_role: None, + physical_name: "f_note".to_owned(), + encryption: None, + }, + ); + entity.stored_fields.push(note); + entity + } + + #[test] + fn a_raised_text_limit_keeps_recorded_values_readable() { + let old = with_note_field(FieldTypeSource::Text { max_length: 80 }); + let descriptor = descriptor_for(&old); + let widened = with_note_field(FieldTypeSource::Text { max_length: 200 }); + + let compatibility = descriptor + .compatibility_for_fields(&widened, &required(&["note"]), &required(&["note"])) + .expect("a raised text limit keeps every recorded value valid"); + assert_eq!( + compatibility.fields["note"].field_type, old.fields["note"].field_type, + "a revision is decoded against the limit it was recorded under" + ); + + for (active, reason) in [ + ( + with_note_field(FieldTypeSource::Text { max_length: 40 }), + "a lowered limit could surface a value the active field refuses", + ), + ( + with_note_field(FieldTypeSource::String { + min_length: 0, + max_length: 200, + }), + "a text field retyped as string is a type change", + ), + ] { + assert_eq!( + descriptor + .compatibility_for_fields(&active, &required(&["note"]), &required(&[])) + .expect_err(reason), + HistorySchemaError::IncompatibleField + ); + } + } + + #[test] + fn a_lowered_string_minimum_keeps_recorded_values_readable() { + let string = |min_length, max_length| FieldTypeSource::String { + min_length, + max_length, + }; + let old = with_note_field(string(5, 80)); + let descriptor = descriptor_for(&old); + let lowered = with_note_field(string(0, 80)); + + let compatibility = descriptor + .compatibility_for_fields(&lowered, &required(&["note"]), &required(&["note"])) + .expect("a lowered string minimum keeps every recorded value valid"); + assert_eq!( + compatibility.fields["note"].field_type, old.fields["note"].field_type, + "a revision is decoded against the minimum it was recorded under" + ); + + for (active, reason) in [ + ( + with_note_field(string(8, 80)), + "a raised minimum could surface a value the active field refuses", + ), + ( + with_note_field(string(0, 200)), + "a string maxLength is its column type", + ), + ] { + assert_eq!( + descriptor + .compatibility_for_fields(&active, &required(&["note"]), &required(&[])) + .expect_err(reason), + HistorySchemaError::IncompatibleField + ); + } + } + #[test] fn vocabulary_code_additions_keep_recorded_values_readable() { let old = with_status_field("status", &["open", "closed"]); diff --git a/crates/registry-breg/src/immediate_actions.rs b/crates/registry-breg/src/immediate_actions.rs index 4150c5f8a4..1bc4cfc6fd 100644 --- a/crates/registry-breg/src/immediate_actions.rs +++ b/crates/registry-breg/src/immediate_actions.rs @@ -2015,18 +2015,20 @@ fn entity_permission_fields_empty(grant: &crate::contract::AccessPermissionSourc } /// Whether `after` differs from `before` only by vocabulary codes added to its -/// inputs or to the fields of the entities it targets, so every request the +/// inputs, and by fields of the entities it targets whose candidate type +/// `target_field_admits` accepts over the previous one, so every request the /// previous contract accepted keeps the same meaning. An input may gain only /// codes new to its vocabulary in this revision: one that starts accepting a /// code its vocabulary already had, such as a withdrawal that starts giving, /// changes what the action does. Any other difference, including a fingerprint /// an earlier compiler derived differently, is not. #[cfg(feature = "runtime")] -pub(crate) fn contract_only_adds_vocabulary_codes( +pub(crate) fn contract_only_widens( (before, previous_vocabularies): (&CompiledAction, &BTreeMap>), previous_entities: &BTreeMap, (after, vocabularies): (&CompiledAction, &BTreeMap>), entities: &BTreeMap, + target_field_admits: fn(&FieldTypeSource, &FieldTypeSource) -> bool, ) -> bool { let inputs = after .inputs @@ -2053,10 +2055,7 @@ pub(crate) fn contract_only_adds_vocabulary_codes( if let Some(previous) = previous_entities.get(entity_id) { for (field_id, field) in &mut entity.fields { if let Some(previous_field) = previous.fields.get(field_id) { - if field - .field_type - .keeps_vocabulary_codes_of(&previous_field.field_type) - { + if target_field_admits(&field.field_type, &previous_field.field_type) { field.field_type = previous_field.field_type.clone(); } } diff --git a/crates/registry-breg/src/import_authority.rs b/crates/registry-breg/src/import_authority.rs new file mode 100644 index 0000000000..bb70493e2c --- /dev/null +++ b/crates/registry-breg/src/import_authority.rs @@ -0,0 +1,1173 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Operator-opened import authorities. +//! +//! An `import` grant loads new records only while an operator has opened a +//! bounded authority for its entity and profile: create only, under the +//! active package revision, before its expiry, and within its item volume. +//! Only the migration role opens or closes one. The runtime role reads it and +//! advances its three mutable columns inside the transaction that observes or +//! consumes it, so no API caller can open its own window. +//! +//! Every transition leaves one audit record: the transaction that makes it +//! collects the record, and the caller appends it through the process audit +//! writer once that transaction commits. Expiry and supersession by a successor package are recorded by +//! the first transaction that observes them: run creation, a chunk, or any +//! operator command. No transaction admits work under an authority whose +//! expiry has passed or whose package revision is no longer active. + +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use registry_platform_audit::{AuditEntry, AuditProfile}; +use serde::Serialize; +use serde_json::{json, Value}; +use tokio_postgres::Transaction; +use uuid::Uuid; + +use crate::contract::Operation; +use crate::model::CompiledRegistry; +use crate::postgres::{ + verify_catalog_identity_for_catalog, verify_migration_role, ConnectionConfig, + ExpectedManagedCatalog, ExpectedRegistryIdentity, RegistryLockKey, SqlIdentifier, +}; + +/// The window an authority stays open for when the operator names none. +pub const DEFAULT_IMPORT_AUTHORITY_WINDOW: Duration = Duration::from_secs(7 * 24 * 60 * 60); +/// The longest window one authority may hold. There is no extension: a +/// longer load opens a second authority, which leaves its own record. +pub const MAX_IMPORT_AUTHORITY_WINDOW: Duration = Duration::from_secs(30 * 24 * 60 * 60); +/// The most input digests one authority may pin. +pub const MAX_PINNED_INPUT_DIGESTS: usize = 16; +/// The largest volume one authority may admit, the bound ingestion runs use. +pub const MAX_IMPORT_AUTHORITY_ITEMS: i64 = 9_007_199_254_740_991; +/// The bounded page `list` answers with, newest first. +pub const MAX_LISTED_IMPORT_AUTHORITIES: i64 = 100; + +const MAX_IDENTIFIER_BYTES: usize = 256; +const MAX_OPERATOR_REFERENCE_BYTES: usize = 512; +const MAX_REASON_BYTES: usize = 1024; + +const AUDIT_SCHEMA: &str = "breg-import-authority-audit/v1"; +const AUDIT_OPERATION_ID: &str = "breg.import_authority"; +const OPERATOR_REFERENCE_DOMAIN: &str = "breg-import-authority-operator-v1"; +const REASON_REFERENCE_DOMAIN: &str = "breg-import-authority-reason-v1"; + +/// The product-owned authority table and the table privileges the runtime +/// role holds on it. Its column-level `UPDATE` grants are +/// [`RUNTIME_UPDATE_COLUMNS`]. Catalog closure consumes both lists. +pub(crate) const IMPORT_AUTHORITY_TABLES: &[(&str, &[&str])] = + &[("registry_import_authorities", &["SELECT"])]; + +/// The only columns the runtime role may change: the committed volume, the +/// terminal status, and the moment it was reached. +pub(crate) const RUNTIME_UPDATE_COLUMNS: &[&str] = &["committed_items", "status", "closed_at"]; + +/// Row security on the authority table, enabled but not forced, so the +/// owning migration role keeps its maintenance boundary while the runtime +/// role reads every row and advances only an open one. The advance policy +/// lets the runtime record `exhausted`, `expired`, and `superseded`; only +/// the migration role closes an authority, and the runtime role cannot +/// reopen one. +pub(crate) const READ_POLICY: &str = "import_authority_runtime_read"; +pub(crate) const ADVANCE_POLICY: &str = "import_authority_runtime_advance"; + +const AUTHORITY_COLUMNS: &str = "authority_id, entity_id, profile_id, operation, max_items, + committed_items, input_digests, activation_revision, opened_at, expires_at, status, + closed_at"; + +/// The lifecycle state of one authority. Only `open` admits work; every other +/// state is terminal. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ImportAuthorityStatus { + Open, + Exhausted, + Expired, + Closed, + Superseded, +} + +impl ImportAuthorityStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::Open => "open", + Self::Exhausted => "exhausted", + Self::Expired => "expired", + Self::Closed => "closed", + Self::Superseded => "superseded", + } + } + + fn parse(value: &str) -> Option { + match value { + "open" => Some(Self::Open), + "exhausted" => Some(Self::Exhausted), + "expired" => Some(Self::Expired), + "closed" => Some(Self::Closed), + "superseded" => Some(Self::Superseded), + _ => None, + } + } +} + +/// One authority as stored. It carries no operator reference or reason: the +/// row holds only their keyed hashes, and this view omits even those. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ImportAuthority { + pub authority_id: Uuid, + pub entity_id: String, + pub profile_id: String, + pub operation: String, + pub max_items: i64, + pub committed_items: i64, + pub input_digests: Vec, + pub activation_revision: String, + pub opened_at: DateTime, + pub expires_at: DateTime, + pub status: ImportAuthorityStatus, + pub closed_at: Option>, +} + +/// The closed refusal vocabulary of the authority store. It is value-free: +/// no operator reference, reason, or row value travels with it. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub enum ImportAuthorityError { + #[error("the import authority request is invalid")] + InvalidInput, + #[error("the entity and profile do not name an import grant of the active package")] + NotImportable, + #[error("an import authority is already open for this entity")] + AlreadyOpen, + #[error("no import authority has this identifier")] + NotFound, + #[error("the registry is not ready for import authority maintenance")] + NotReady, + #[error("the import authority store is unavailable")] + Unavailable, +} + +/// One operator request to open an authority. +pub struct ImportAuthorityOpenRequest<'a> { + pub entity_id: &'a str, + pub profile_id: &'a str, + pub max_items: i64, + pub expires_in: Duration, + pub input_digests: &'a [String], + pub operator_reference: &'a str, + pub reason: &'a str, +} + +/// One operator request to close an open authority. +pub struct ImportAuthorityCloseRequest<'a> { + pub authority_id: Uuid, + pub operator_reference: &'a str, + pub reason: &'a str, +} + +impl ImportAuthorityOpenRequest<'_> { + /// Check the request's bounds without opening any dependency, so a + /// caller can refuse a malformed request before it connects. + pub fn validate(&self) -> Result<(), ImportAuthorityError> { + validate_open(self) + } +} + +impl ImportAuthorityCloseRequest<'_> { + /// Check the request's bounds without opening any dependency. + pub fn validate(&self) -> Result<(), ImportAuthorityError> { + validate_close(self) + } +} + +/// KERNEL INTERNAL SCHEMA MIGRATION (import authorities): creates +/// `registry_internal.registry_import_authorities` and its runtime grants. +/// The ingestion-run half of the same migration lives in +/// `ingestion_store::install`. Both are additive and idempotent. +pub(crate) async fn install( + migration: &impl tokio_postgres::GenericClient, + runtime_role: &SqlIdentifier, +) -> Result<(), ImportAuthorityError> { + migration + .batch_execute(&format!( + "CREATE TABLE IF NOT EXISTS registry_internal.registry_import_authorities ( + authority_id uuid PRIMARY KEY, + entity_id text NOT NULL + CHECK (entity_id <> '' AND octet_length(entity_id) <= {MAX_IDENTIFIER_BYTES}), + profile_id text NOT NULL + CHECK (profile_id <> '' AND octet_length(profile_id) <= {MAX_IDENTIFIER_BYTES}), + operation text NOT NULL + CONSTRAINT registry_import_authorities_operation_values + CHECK (operation = 'create'), + max_items bigint NOT NULL + CHECK (max_items BETWEEN 1 AND {MAX_IMPORT_AUTHORITY_ITEMS}), + committed_items bigint NOT NULL DEFAULT 0, + input_digests text[] NOT NULL DEFAULT '{{}}', + activation_revision text NOT NULL CHECK (activation_revision <> ''), + opened_at timestamptz NOT NULL DEFAULT transaction_timestamp(), + expires_at timestamptz NOT NULL, + status text NOT NULL DEFAULT 'open' + CONSTRAINT registry_import_authorities_status_values + CHECK (status IN ('open', 'exhausted', 'expired', 'closed', 'superseded')), + closed_at timestamptz, + operator_reference text NOT NULL CHECK (operator_reference <> ''), + reason_reference text NOT NULL CHECK (reason_reference <> ''), + CONSTRAINT registry_import_authorities_volume CHECK ( + committed_items >= 0 AND committed_items <= max_items + AND (status <> 'exhausted' OR committed_items = max_items) + ), + CONSTRAINT registry_import_authorities_window CHECK ( + expires_at > opened_at + AND expires_at <= opened_at + interval '{window_days} days' + ), + CONSTRAINT registry_import_authorities_closed_shape CHECK ( + (status = 'open') = (closed_at IS NULL) + ), + CONSTRAINT registry_import_authorities_digests CHECK ( + cardinality(input_digests) <= {MAX_PINNED_INPUT_DIGESTS} + AND array_position(input_digests, NULL) IS NULL + AND (cardinality(input_digests) = 0 OR array_ndims(input_digests) = 1) + AND array_to_string(input_digests, ',') + ~ '^([0-9a-f]{{64}}(,[0-9a-f]{{64}})*)?$' + ) + ); + CREATE UNIQUE INDEX IF NOT EXISTS registry_import_authorities_one_open + ON registry_internal.registry_import_authorities (entity_id) + WHERE status = 'open'; + REVOKE ALL ON registry_internal.registry_import_authorities FROM PUBLIC; + REVOKE ALL ON registry_internal.registry_import_authorities FROM \"{role}\"; + GRANT SELECT ON registry_internal.registry_import_authorities TO \"{role}\"; + GRANT UPDATE ({columns}) ON registry_internal.registry_import_authorities + TO \"{role}\"; + ALTER TABLE registry_internal.registry_import_authorities + ENABLE ROW LEVEL SECURITY; + DROP POLICY IF EXISTS {READ_POLICY} + ON registry_internal.registry_import_authorities; + CREATE POLICY {READ_POLICY} ON registry_internal.registry_import_authorities + FOR SELECT TO \"{role}\" USING (true); + DROP POLICY IF EXISTS {ADVANCE_POLICY} + ON registry_internal.registry_import_authorities; + CREATE POLICY {ADVANCE_POLICY} ON registry_internal.registry_import_authorities + FOR UPDATE TO \"{role}\" + USING (status = 'open') + WITH CHECK (status IN ('open', 'exhausted', 'expired', 'superseded'));", + window_days = MAX_IMPORT_AUTHORITY_WINDOW.as_secs() / (24 * 60 * 60), + columns = RUNTIME_UPDATE_COLUMNS.join(", "), + role = runtime_role.as_str(), + )) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + Ok(()) +} + +fn parse_row(row: &tokio_postgres::Row) -> Result { + let status: String = row.get("status"); + Ok(ImportAuthority { + authority_id: row.get("authority_id"), + entity_id: row.get("entity_id"), + profile_id: row.get("profile_id"), + operation: row.get("operation"), + max_items: row.get("max_items"), + committed_items: row.get("committed_items"), + input_digests: row.get("input_digests"), + activation_revision: row.get("activation_revision"), + opened_at: row.get("opened_at"), + expires_at: row.get("expires_at"), + status: ImportAuthorityStatus::parse(&status).ok_or(ImportAuthorityError::Unavailable)?, + closed_at: row.get("closed_at"), + }) +} + +/// The operator references a transition carries in its audit record, as +/// keyed hashes. Runtime-observed transitions carry none. +struct OperatorReferences { + operator_reference: String, + reason_reference: String, +} + +fn operator_references( + profile: &AuditProfile, + package_revision: &str, + operator_reference: &str, + reason: &str, +) -> Result { + if !crate::audit::profile_is_keyed(profile) { + return Err(ImportAuthorityError::InvalidInput); + } + let hasher = profile.key_hasher(); + Ok(OperatorReferences { + operator_reference: hasher + .audit_reference_hash( + OPERATOR_REFERENCE_DOMAIN, + package_revision, + operator_reference, + ) + .map_err(|_| ImportAuthorityError::InvalidInput)?, + reason_reference: hasher + .audit_reference_hash(REASON_REFERENCE_DOMAIN, package_revision, reason) + .map_err(|_| ImportAuthorityError::InvalidInput)?, + }) +} + +fn audit_record( + transition: ImportAuthorityStatus, + authority: &ImportAuthority, + package_revision: &str, + references: Option<&OperatorReferences>, +) -> Value { + let mut record = json!({ + "transition": match transition { + ImportAuthorityStatus::Open => "opened", + other => other.as_str(), + }, + "operationId": AUDIT_OPERATION_ID, + "packageRevision": package_revision, + "authorityId": authority.authority_id.to_string(), + "entityId": authority.entity_id, + "profileId": authority.profile_id, + "operation": authority.operation, + "activationRevision": authority.activation_revision, + "maxItems": authority.max_items, + "committedItems": authority.committed_items, + "openedAt": authority.opened_at.to_rfc3339(), + "expiresAt": authority.expires_at.to_rfc3339(), + "inputDigests": authority.input_digests, + }); + if let Some(references) = references { + record["operatorReference"] = json!(references.operator_reference); + record["reasonReference"] = json!(references.reason_reference); + } + record +} + +/// Append the transition records a committed transaction collected, oldest +/// first, as `response` entries correlated by their authority id. A caller +/// appends only after the commit that made the transitions, and releases +/// nothing until every append is accepted. +pub(crate) async fn append_transitions( + audit: &crate::audit::RegistryAudit, + records: Vec, +) -> Result<(), ImportAuthorityError> { + for record in records { + let correlation = record + .get("authorityId") + .and_then(Value::as_str) + .ok_or(ImportAuthorityError::Unavailable)? + .to_owned(); + audit + .append(AuditEntry::response(AUDIT_SCHEMA, correlation, record)) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + } + Ok(()) +} + +/// Move one open authority to a terminal status and collect its transition +/// record into `pending`. A row that is no longer open is returned unchanged +/// with no record. +async fn transition( + transaction: &Transaction<'_>, + pending: &mut Vec, + authority_id: Uuid, + to: ImportAuthorityStatus, + package_revision: &str, + references: Option<&OperatorReferences>, +) -> Result, ImportAuthorityError> { + let row = transaction + .query_opt( + &format!( + "UPDATE registry_internal.registry_import_authorities + SET status = $2, closed_at = transaction_timestamp() + WHERE authority_id = $1 AND status = 'open' + RETURNING {AUTHORITY_COLUMNS}" + ), + &[&authority_id, &to.as_str()], + ) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let Some(row) = row else { + return Ok(None); + }; + let authority = parse_row(&row)?; + pending.push(audit_record(to, &authority, package_revision, references)); + Ok(Some(authority)) +} + +/// The terminal status a still-open authority has already reached, if any: +/// a successor package supersedes it, and a passed expiry expires it. +/// Supersession wins, because a model change retires the grant it named. +fn due_transition( + authority: &ImportAuthority, + package_revision: &str, + now: DateTime, +) -> Option { + if authority.status != ImportAuthorityStatus::Open { + return None; + } + if authority.activation_revision != package_revision { + return Some(ImportAuthorityStatus::Superseded); + } + if authority.expires_at <= now { + return Some(ImportAuthorityStatus::Expired); + } + None +} + +async fn transaction_now( + transaction: &Transaction<'_>, +) -> Result, ImportAuthorityError> { + Ok(transaction + .query_one("SELECT transaction_timestamp()", &[]) + .await + .map_err(|_| ImportAuthorityError::Unavailable)? + .get(0)) +} + +/// Lock the open authority rows the filter selects, collect every transition +/// already due, and answer the rows that stay open. +async fn settle_open( + transaction: &Transaction<'_>, + pending: &mut Vec, + package_revision: &str, + entity_id: Option<&str>, +) -> Result<(Vec, Vec), ImportAuthorityError> { + let rows = transaction + .query( + &format!( + "SELECT {AUTHORITY_COLUMNS} + FROM registry_internal.registry_import_authorities + WHERE status = 'open' AND ($1::text IS NULL OR entity_id = $1) + ORDER BY authority_id + FOR UPDATE" + ), + &[&entity_id], + ) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let now = transaction_now(transaction).await?; + let mut open = Vec::new(); + let mut transitioned = Vec::new(); + for row in &rows { + let authority = parse_row(row)?; + match due_transition(&authority, package_revision, now) { + None => open.push(authority), + Some(to) => { + if let Some(done) = transition( + transaction, + pending, + authority.authority_id, + to, + package_revision, + None, + ) + .await? + { + transitioned.push(done); + } + } + } + } + Ok((open, transitioned)) +} + +/// Supersede every open authority inside the caller's transaction, which +/// holds the registry lock, collecting one transition record for each into +/// `pending` for the caller to append once it commits, and answer the +/// authorities it moved. Adopting a restored copy calls this, so +/// an authority the copy carries from its backup, even one an operator +/// closed after the backup was taken, admits no work until an operator +/// opens a new one. +#[cfg(feature = "tooling")] +pub(crate) async fn supersede_every_open( + transaction: &Transaction<'_>, + pending: &mut Vec, + package_revision: &str, +) -> Result, ImportAuthorityError> { + let rows = transaction + .query( + "SELECT authority_id + FROM registry_internal.registry_import_authorities + WHERE status = 'open' + ORDER BY authority_id + FOR UPDATE", + &[], + ) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let mut superseded = Vec::with_capacity(rows.len()); + for row in &rows { + let authority_id: Uuid = row.get(0); + transition( + transaction, + pending, + authority_id, + ImportAuthorityStatus::Superseded, + package_revision, + None, + ) + .await? + .ok_or(ImportAuthorityError::Unavailable)?; + superseded.push(authority_id); + } + Ok(superseded) +} + +/// Decide whether one import run may be created, inside the run-creation +/// transaction. Every transition already due is collected into `pending` +/// first, so the caller must commit this transaction and append those +/// records even when it refuses the run. +/// Answers the admitting authority, or `None` when no open authority names +/// this entity and profile, admits the run's whole volume, and pins its input. +pub(crate) async fn admit_run( + transaction: &Transaction<'_>, + pending: &mut Vec, + package_revision: &str, + entity_id: &str, + profile_id: &str, + item_count: i64, + input_digest: &str, +) -> Result, ImportAuthorityError> { + let (open, _) = settle_open(transaction, pending, package_revision, Some(entity_id)).await?; + Ok(open + .into_iter() + .find(|authority| { + authority.profile_id == profile_id + && authority.operation == "create" + && item_count <= authority.max_items - authority.committed_items + && (authority.input_digests.is_empty() + || authority + .input_digests + .iter() + .any(|digest| digest == input_digest)) + }) + .map(|authority| authority.authority_id)) +} + +/// Decide whether one chunk of an import run may commit, inside the chunk +/// transaction and under the run row lock. The authority row is locked +/// `FOR UPDATE` so a close waits for an in-flight chunk and the next chunk +/// sees it. A transition already due is collected into `pending` first, so +/// the caller must commit this transaction and append that record even when +/// it refuses the chunk. Answers whether +/// the authority is still open and has room for `chunk_items`. +/// +/// The runtime role's row security admits only open rows to a locking read, +/// so a terminal authority reads as absent, which refuses the chunk exactly +/// as a closed one does. +pub(crate) async fn admit_chunk( + transaction: &Transaction<'_>, + pending: &mut Vec, + package_revision: &str, + authority_id: Uuid, + chunk_items: i64, +) -> Result { + let Some(row) = transaction + .query_opt( + &format!( + "SELECT {AUTHORITY_COLUMNS} + FROM registry_internal.registry_import_authorities + WHERE authority_id = $1 AND status = 'open' + FOR UPDATE" + ), + &[&authority_id], + ) + .await + .map_err(|_| ImportAuthorityError::Unavailable)? + else { + return Ok(false); + }; + let authority = parse_row(&row)?; + let now = transaction_now(transaction).await?; + if let Some(to) = due_transition(&authority, package_revision, now) { + transition( + transaction, + pending, + authority_id, + to, + package_revision, + None, + ) + .await?; + return Ok(false); + } + Ok(chunk_items <= authority.max_items - authority.committed_items) +} + +/// Count one committed chunk against its authority, in the chunk commit +/// transaction. The authority reaching its volume moves to `exhausted` and +/// collects that transition record into `pending`. The caller has +/// already admitted the chunk under the same row lock, so an authority that +/// no longer counts it is corruption and refuses the commit. +pub(crate) async fn consume( + transaction: &Transaction<'_>, + pending: &mut Vec, + package_revision: &str, + authority_id: Uuid, + chunk_items: i64, +) -> Result<(), ImportAuthorityError> { + let row = transaction + .query_opt( + &format!( + "UPDATE registry_internal.registry_import_authorities + SET committed_items = committed_items + $2, + status = CASE WHEN committed_items + $2 = max_items + THEN 'exhausted' ELSE 'open' END, + closed_at = CASE WHEN committed_items + $2 = max_items + THEN transaction_timestamp() END + WHERE authority_id = $1 AND status = 'open' + AND committed_items + $2 <= max_items + RETURNING {AUTHORITY_COLUMNS}" + ), + &[&authority_id, &chunk_items], + ) + .await + .map_err(|_| ImportAuthorityError::Unavailable)? + .ok_or(ImportAuthorityError::Unavailable)?; + let authority = parse_row(&row)?; + if authority.status == ImportAuthorityStatus::Exhausted { + pending.push(audit_record( + ImportAuthorityStatus::Exhausted, + &authority, + package_revision, + None, + )); + } + Ok(()) +} + +fn is_digest(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn bounded_text(value: &str, maximum: usize) -> bool { + !value.is_empty() && value.len() <= maximum && !value.chars().any(char::is_control) +} + +fn validate_open(request: &ImportAuthorityOpenRequest<'_>) -> Result<(), ImportAuthorityError> { + let mut digests = request.input_digests.to_vec(); + digests.sort(); + digests.dedup(); + if !bounded_text(request.entity_id, MAX_IDENTIFIER_BYTES) + || !bounded_text(request.profile_id, MAX_IDENTIFIER_BYTES) + || !(1..=MAX_IMPORT_AUTHORITY_ITEMS).contains(&request.max_items) + || request.expires_in < Duration::from_secs(1) + || request.expires_in > MAX_IMPORT_AUTHORITY_WINDOW + || request.input_digests.len() > MAX_PINNED_INPUT_DIGESTS + || digests.len() != request.input_digests.len() + || !request.input_digests.iter().all(|digest| is_digest(digest)) + || !bounded_text(request.operator_reference, MAX_OPERATOR_REFERENCE_BYTES) + || !bounded_text(request.reason, MAX_REASON_BYTES) + { + return Err(ImportAuthorityError::InvalidInput); + } + Ok(()) +} + +fn validate_close(request: &ImportAuthorityCloseRequest<'_>) -> Result<(), ImportAuthorityError> { + if !bounded_text(request.operator_reference, MAX_OPERATOR_REFERENCE_BYTES) + || !bounded_text(request.reason, MAX_REASON_BYTES) + { + return Err(ImportAuthorityError::InvalidInput); + } + Ok(()) +} + +/// Whether the entity and profile name an `import` grant of this package. +fn names_import_grant(registry: &CompiledRegistry, entity_id: &str, profile_id: &str) -> bool { + crate::data::ingestion_route(registry, entity_id, profile_id) + .is_some_and(|route| route.operation == Operation::Import) +} + +/// The migration-role service that opens, closes, settles, and lists import +/// authorities. Each command verifies the migration identity, the active +/// package binding, and the managed catalog in the same transaction that +/// changes an authority, under the registry's exclusive interlock, so a +/// close serializes with every in-flight chunk. +pub struct ImportAuthorityOperatorService { + expected: ExpectedRegistryIdentity, + expected_catalog: ExpectedManagedCatalog, + lock_key: RegistryLockKey, + migration_connection: ConnectionConfig, + migration_role: SqlIdentifier, + runtime_role: SqlIdentifier, + lock_timeout: Duration, + statement_timeout: Duration, + audit: crate::audit::RegistryAudit, + registry: Arc, +} + +impl ImportAuthorityOperatorService { + pub async fn from_runtime_config(path: &Path) -> Result { + if !path.is_absolute() { + return Err(ImportAuthorityError::InvalidInput); + } + let config = crate::runtime_config::load_runtime_config(path) + .map_err(|_| ImportAuthorityError::Unavailable)?; + let package = config + .load_active_package() + .map_err(|_| ImportAuthorityError::Unavailable)?; + let audit = crate::audit::RegistryAudit::open_companion(&config) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + if !crate::audit::profile_is_keyed(audit.profile()) { + return Err(ImportAuthorityError::Unavailable); + } + let pool = config + .runtime_database_connection_config() + .map_err(|_| ImportAuthorityError::Unavailable)? + .build_pool() + .map_err(|_| ImportAuthorityError::Unavailable)?; + let mut client = pool + .get() + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let startup = crate::startup::prepare_loaded_startup( + package, + &mut client, + config.database().roles().migration(), + config.database().roles().runtime(), + ) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + Ok(Self { + expected: startup.expected_identity().clone(), + expected_catalog: startup.expected_catalog().clone(), + lock_key: startup.lock_key(), + migration_connection: config + .migration_database_connection_config() + .map_err(|_| ImportAuthorityError::Unavailable)?, + migration_role: config.database().roles().migration().clone(), + runtime_role: config.database().roles().runtime().clone(), + lock_timeout: config.operational_timeouts().migration_lock, + statement_timeout: config.operational_timeouts().migration_statement, + audit, + registry: Arc::new(startup.package().registry().clone()), + }) + } + + #[cfg(feature = "postgres-test")] + #[doc(hidden)] + #[allow(clippy::too_many_arguments)] + pub fn new_for_test( + expected: ExpectedRegistryIdentity, + expected_catalog: ExpectedManagedCatalog, + lock_key: RegistryLockKey, + migration_connection: ConnectionConfig, + migration_role: SqlIdentifier, + runtime_role: SqlIdentifier, + audit: crate::audit::RegistryAudit, + registry: Arc, + ) -> Self { + Self { + expected, + expected_catalog, + lock_key, + migration_connection, + migration_role, + runtime_role, + lock_timeout: Duration::from_secs(5), + statement_timeout: Duration::from_secs(10), + audit, + registry, + } + } + + /// Open one authority for an `import` grant of the active package. Any + /// transition already due on the entity's open authority is recorded + /// first; a still-open one refuses the request by name. + pub async fn open( + &self, + request: ImportAuthorityOpenRequest<'_>, + ) -> Result { + validate_open(&request)?; + if !names_import_grant(&self.registry, request.entity_id, request.profile_id) { + return Err(ImportAuthorityError::NotImportable); + } + let package_revision = &self.expected.package_revision; + let references = operator_references( + self.audit.profile(), + package_revision, + request.operator_reference, + request.reason, + )?; + let mut digests = request.input_digests.to_vec(); + digests.sort(); + let expires_in = i64::try_from(request.expires_in.as_secs()) + .map_err(|_| ImportAuthorityError::InvalidInput)?; + let pool = self.pool()?; + let mut client = pool + .get() + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let transaction = self.begin(&mut client).await?; + let mut pending = Vec::new(); + let (open, _) = settle_open( + &transaction, + &mut pending, + package_revision, + Some(request.entity_id), + ) + .await?; + if !open.is_empty() { + // At most one authority is open per entity, so nothing settled + // here: the refusal rolls back an empty transaction. + return Err(ImportAuthorityError::AlreadyOpen); + } + let row = transaction + .query_one( + &format!( + "INSERT INTO registry_internal.registry_import_authorities + (authority_id, entity_id, profile_id, operation, max_items, + input_digests, activation_revision, expires_at, + operator_reference, reason_reference) + VALUES ($1, $2, $3, 'create', $4, $5, $6, + transaction_timestamp() + make_interval(secs => $7::bigint), + $8, $9) + RETURNING {AUTHORITY_COLUMNS}" + ), + &[ + &Uuid::new_v4(), + &request.entity_id, + &request.profile_id, + &request.max_items, + &digests, + package_revision, + &expires_in, + &references.operator_reference, + &references.reason_reference, + ], + ) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let authority = parse_row(&row)?; + pending.push(audit_record( + ImportAuthorityStatus::Open, + &authority, + package_revision, + Some(&references), + )); + self.commit(transaction, pending, authority).await + } + + /// Close one authority. An authority that already reached a terminal + /// status is answered as it stands with no second record. One whose + /// expiry or supersession is already due records that transition, not a + /// close, because it stopped admitting work before this command ran. + pub async fn close( + &self, + request: ImportAuthorityCloseRequest<'_>, + ) -> Result { + validate_close(&request)?; + let package_revision = &self.expected.package_revision; + let references = operator_references( + self.audit.profile(), + package_revision, + request.operator_reference, + request.reason, + )?; + let pool = self.pool()?; + let mut client = pool + .get() + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let transaction = self.begin(&mut client).await?; + let row = transaction + .query_opt( + &format!( + "SELECT {AUTHORITY_COLUMNS} + FROM registry_internal.registry_import_authorities + WHERE authority_id = $1 + FOR UPDATE" + ), + &[&request.authority_id], + ) + .await + .map_err(|_| ImportAuthorityError::Unavailable)? + .ok_or(ImportAuthorityError::NotFound)?; + let authority = parse_row(&row)?; + if authority.status != ImportAuthorityStatus::Open { + return Ok(authority); + } + let now = transaction_now(&transaction).await?; + let to = due_transition(&authority, package_revision, now) + .unwrap_or(ImportAuthorityStatus::Closed); + let recorded = (to == ImportAuthorityStatus::Closed).then_some(&references); + let mut pending = Vec::new(); + let closed = transition( + &transaction, + &mut pending, + authority.authority_id, + to, + package_revision, + recorded, + ) + .await? + .ok_or(ImportAuthorityError::Unavailable)?; + self.commit(transaction, pending, closed).await + } + + /// Record every expiry and supersession already due, and answer the + /// authorities this command moved. + pub async fn close_expired(&self) -> Result, ImportAuthorityError> { + let pool = self.pool()?; + let mut client = pool + .get() + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let transaction = self.begin(&mut client).await?; + let mut pending = Vec::new(); + let (_, transitioned) = settle_open( + &transaction, + &mut pending, + &self.expected.package_revision, + None, + ) + .await?; + self.commit(transaction, pending, transitioned).await + } + + /// Answer the newest authorities, bounded, in a read-only transaction. + /// It takes no registry lock and needs no ready maintenance state, so it + /// neither waits for a write nor holds one back, and it still answers + /// while an apply is interrupted. It records nothing: an open authority + /// whose expiry has passed, or whose package revision is no longer + /// active, is listed with the status it has reached, and the next run, + /// chunk, or `close_expired` records that transition. + pub async fn list(&self) -> Result, ImportAuthorityError> { + let pool = self.pool()?; + let mut pooled = pool + .get() + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let client: &mut tokio_postgres::Client = &mut pooled; + verify_migration_role(client, &self.migration_role) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let transaction = client + .build_transaction() + .read_only(true) + .start() + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + self.set_timeouts(&transaction).await?; + verify_catalog_identity_for_catalog( + &transaction, + &self.expected, + &self.expected_catalog, + &self.migration_role, + &self.runtime_role, + ) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let rows = transaction + .query( + &format!( + "SELECT {AUTHORITY_COLUMNS} + FROM registry_internal.registry_import_authorities + ORDER BY opened_at DESC, authority_id + LIMIT {MAX_LISTED_IMPORT_AUTHORITIES}" + ), + &[], + ) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let now = transaction_now(&transaction).await?; + let listed = rows + .iter() + .map(|row| { + let mut authority = parse_row(row)?; + if let Some(reached) = + due_transition(&authority, &self.expected.package_revision, now) + { + authority.status = reached; + } + Ok(authority) + }) + .collect::, _>>()?; + self.commit(transaction, Vec::new(), listed).await + } + + /// Open one verified maintenance transaction on a fresh migration + /// connection: the migration role, bounded timeouts, the exclusive + /// registry interlock, the active package and catalog, and a ready + /// maintenance state. The caller does its work and commits. + async fn begin<'c>( + &self, + client: &'c mut tokio_postgres::Client, + ) -> Result, ImportAuthorityError> { + verify_migration_role(client, &self.migration_role) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let transaction = client + .transaction() + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + self.set_timeouts(&transaction).await?; + transaction + .execute( + "SELECT pg_catalog.pg_advisory_xact_lock($1)", + &[&self.lock_key.get()], + ) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + verify_catalog_identity_for_catalog( + &transaction, + &self.expected, + &self.expected_catalog, + &self.migration_role, + &self.runtime_role, + ) + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + let ready: bool = transaction + .query_one( + "SELECT maintenance_status = 'ready' + FROM registry_internal.registry_state WHERE singleton", + &[], + ) + .await + .map_err(|_| ImportAuthorityError::Unavailable)? + .get(0); + if !ready { + return Err(ImportAuthorityError::NotReady); + } + Ok(transaction) + } + + async fn set_timeouts( + &self, + transaction: &Transaction<'_>, + ) -> Result<(), ImportAuthorityError> { + transaction + .query_one( + "SELECT set_config('lock_timeout', $1, true), + set_config('statement_timeout', $2, true)", + &[ + &format!("{}ms", self.lock_timeout.as_millis()), + &format!("{}ms", self.statement_timeout.as_millis()), + ], + ) + .await + .map(drop) + .map_err(|_| ImportAuthorityError::Unavailable) + } + + fn pool(&self) -> Result { + self.migration_connection + .build_pool() + .map_err(|_| ImportAuthorityError::Unavailable) + } + + /// Commit the command's transaction, then append the transition records + /// it collected before answering. A refused append answers the command + /// unavailable although its transitions stand. + async fn commit( + &self, + transaction: Transaction<'_>, + pending: Vec, + value: T, + ) -> Result { + transaction + .commit() + .await + .map_err(|_| ImportAuthorityError::Unavailable)?; + append_transitions(&self.audit, pending).await?; + Ok(value) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn open_request<'a>(digests: &'a [String]) -> ImportAuthorityOpenRequest<'a> { + ImportAuthorityOpenRequest { + entity_id: "enrollment", + profile_id: "loader", + max_items: 10, + expires_in: DEFAULT_IMPORT_AUTHORITY_WINDOW, + input_digests: digests, + operator_reference: "operator-a", + reason: "initial enrollment load", + } + } + + #[test] + fn open_requests_are_bounded_before_any_database_work() { + let none: Vec = Vec::new(); + assert!(validate_open(&open_request(&none)).is_ok()); + let pinned = vec!["a".repeat(64)]; + assert!(validate_open(&open_request(&pinned)).is_ok()); + + let mut request = open_request(&none); + request.max_items = 0; + assert_eq!( + validate_open(&request), + Err(ImportAuthorityError::InvalidInput) + ); + request.max_items = MAX_IMPORT_AUTHORITY_ITEMS + 1; + assert!(validate_open(&request).is_err()); + + let mut request = open_request(&none); + request.expires_in = MAX_IMPORT_AUTHORITY_WINDOW + Duration::from_secs(1); + assert!(validate_open(&request).is_err(), "no window above 30 days"); + request.expires_in = Duration::ZERO; + assert!(validate_open(&request).is_err()); + + let too_many = vec!["b".repeat(64); MAX_PINNED_INPUT_DIGESTS + 1]; + assert!(validate_open(&open_request(&too_many)).is_err()); + let duplicated = vec!["c".repeat(64), "c".repeat(64)]; + assert!(validate_open(&open_request(&duplicated)).is_err()); + let uppercase = vec!["C".repeat(64)]; + assert!(validate_open(&open_request(&uppercase)).is_err()); + let short = vec!["c".repeat(63)]; + assert!(validate_open(&open_request(&short)).is_err()); + + let mut request = open_request(&none); + request.reason = "line\nbreak"; + assert!(validate_open(&request).is_err()); + request.reason = ""; + assert!(validate_open(&request).is_err()); + let mut request = open_request(&none); + let long = "o".repeat(MAX_OPERATOR_REFERENCE_BYTES + 1); + request.operator_reference = &long; + assert!(validate_open(&request).is_err()); + } + + #[test] + fn supersession_wins_over_expiry_and_only_open_rows_move() { + let now = Utc::now(); + let mut authority = ImportAuthority { + authority_id: Uuid::nil(), + entity_id: "enrollment".to_owned(), + profile_id: "loader".to_owned(), + operation: "create".to_owned(), + max_items: 1, + committed_items: 0, + input_digests: Vec::new(), + activation_revision: "revision-1".to_owned(), + opened_at: now - chrono::Duration::days(2), + expires_at: now - chrono::Duration::days(1), + status: ImportAuthorityStatus::Open, + closed_at: None, + }; + assert_eq!( + due_transition(&authority, "revision-2", now), + Some(ImportAuthorityStatus::Superseded) + ); + assert_eq!( + due_transition(&authority, "revision-1", now), + Some(ImportAuthorityStatus::Expired) + ); + authority.expires_at = now + chrono::Duration::days(1); + assert_eq!(due_transition(&authority, "revision-1", now), None); + authority.status = ImportAuthorityStatus::Closed; + assert_eq!(due_transition(&authority, "revision-2", now), None); + } +} diff --git a/crates/registry-breg/src/ingestion_store.rs b/crates/registry-breg/src/ingestion_store.rs index 523b7eddae..b9706876c7 100644 --- a/crates/registry-breg/src/ingestion_store.rs +++ b/crates/registry-breg/src/ingestion_store.rs @@ -86,12 +86,16 @@ impl IngestionRunStatus { #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] pub(crate) enum IngestionBlockedReason { ActivePackageChanged, + /// The import authority an `import` run consumes is no longer open or + /// has no room for the next chunk. + ImportAuthorityClosed, } impl IngestionBlockedReason { pub(crate) fn as_str(self) -> &'static str { match self { Self::ActivePackageChanged => "active_package_changed", + Self::ImportAuthorityClosed => "import_authority_closed", } } @@ -99,12 +103,14 @@ impl IngestionBlockedReason { pub(crate) fn wire_str(self) -> &'static str { match self { Self::ActivePackageChanged => "activePackageChanged", + Self::ImportAuthorityClosed => "importAuthorityClosed", } } fn parse(value: &str) -> Option { match value { "active_package_changed" => Some(Self::ActivePackageChanged), + "import_authority_closed" => Some(Self::ImportAuthorityClosed), _ => None, } } @@ -118,6 +124,9 @@ pub(crate) enum IngestionAttemptOutcome { InvalidItem, Refused, BindingChanged, + /// The import authority the run consumes admitted no further chunk, so + /// the run blocked. + ImportAuthorityClosed, ChunkMismatch, RunNotOpen, Unavailable, @@ -131,6 +140,7 @@ impl IngestionAttemptOutcome { Self::InvalidItem => "invalid_item", Self::Refused => "refused", Self::BindingChanged => "binding_changed", + Self::ImportAuthorityClosed => "import_authority_closed", Self::ChunkMismatch => "chunk_mismatch", Self::RunNotOpen => "run_not_open", Self::Unavailable => "unavailable", @@ -145,6 +155,7 @@ impl IngestionAttemptOutcome { Self::InvalidItem => "invalidItem", Self::Refused => "refused", Self::BindingChanged => "bindingChanged", + Self::ImportAuthorityClosed => "importAuthorityClosed", Self::ChunkMismatch => "chunkMismatch", Self::RunNotOpen => "runNotOpen", Self::Unavailable => "unavailable", @@ -158,6 +169,7 @@ impl IngestionAttemptOutcome { "invalid_item" => Some(Self::InvalidItem), "refused" => Some(Self::Refused), "binding_changed" => Some(Self::BindingChanged), + "import_authority_closed" => Some(Self::ImportAuthorityClosed), "chunk_mismatch" => Some(Self::ChunkMismatch), "run_not_open" => Some(Self::RunNotOpen), "unavailable" => Some(Self::Unavailable), @@ -187,6 +199,9 @@ pub(crate) struct IngestionRunRecord { pub(crate) chunk_algorithm_version: String, pub(crate) maximum_items: i64, pub(crate) maximum_bytes: i64, + /// The import authority an `import` run consumes, absent for a `batch` + /// run. Every chunk of the run is admitted and counted against it. + pub(crate) import_authority_id: Option, pub(crate) status: IngestionRunStatus, pub(crate) blocked_reason: Option, pub(crate) next_chunk_index: i64, @@ -316,6 +331,7 @@ pub(crate) struct NewIngestionRun { pub(crate) chunk_algorithm_version: String, pub(crate) maximum_items: i64, pub(crate) maximum_bytes: i64, + pub(crate) import_authority_id: Option, } /// One committed chunk and the receipt that proves it. @@ -421,6 +437,8 @@ pub enum IngestionRefusal { ChunkMismatch, #[error("active package no longer matches the run binding")] BindingChanged, + #[error("the import authority the run consumes admits no further chunk")] + AuthorityClosed, #[error("the stored receipt of the committed chunk was erased")] ReceiptErased, } @@ -449,13 +467,15 @@ pub(crate) async fn install( chunk_algorithm_version text NOT NULL CHECK (chunk_algorithm_version <> ''), maximum_items int NOT NULL CHECK (maximum_items > 0), maximum_bytes bigint NOT NULL CHECK (maximum_bytes > 0), + import_authority_id uuid + REFERENCES registry_internal.registry_import_authorities(authority_id), status text NOT NULL CONSTRAINT registry_ingestion_runs_status_values CHECK (status IN ('open', 'complete', 'cancelled', 'blocked')), blocked_reason text CONSTRAINT registry_ingestion_runs_blocked_reason_values CHECK (blocked_reason IS NULL OR - blocked_reason IN ('active_package_changed')), + blocked_reason IN ('active_package_changed', 'import_authority_closed')), next_chunk_index bigint NOT NULL CHECK (next_chunk_index >= 0), committed_items bigint NOT NULL CHECK (committed_items >= 0), committed_prefix_digest text NOT NULL @@ -464,7 +484,8 @@ pub(crate) async fn install( CONSTRAINT registry_ingestion_runs_attempt_values CHECK (last_attempt_outcome IS NULL OR last_attempt_outcome IN ('committed', 'replayed', 'invalid_item', 'refused', - 'binding_changed', 'chunk_mismatch', 'run_not_open', 'unavailable')), + 'binding_changed', 'import_authority_closed', 'chunk_mismatch', + 'run_not_open', 'unavailable')), last_attempt_chunk_index bigint CHECK (last_attempt_chunk_index IS NULL OR last_attempt_chunk_index >= 0), created_at timestamptz NOT NULL DEFAULT transaction_timestamp(), @@ -516,6 +537,45 @@ pub(crate) async fn install( CREATE INDEX IF NOT EXISTS registry_ingestion_run_chunk_records_erased_record ON registry_internal.registry_ingestion_run_chunk_records (record_id, record_revision); + -- KERNEL INTERNAL SCHEMA MIGRATION (import authorities): a run + -- table created before import authorities gains the authority + -- reference and the `import_authority_closed` blocked reason and + -- attempt outcome. + ALTER TABLE registry_internal.registry_ingestion_runs + ADD COLUMN IF NOT EXISTS import_authority_id uuid + REFERENCES registry_internal.registry_import_authorities(authority_id); + DO $registry_ingestion_import_authority_upgrade$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_constraint + WHERE conrelid = 'registry_internal.registry_ingestion_runs'::regclass + AND conname = 'registry_ingestion_runs_blocked_reason_values' + AND pg_catalog.pg_get_constraintdef(oid) + LIKE '%import_authority_closed%' + ) THEN + ALTER TABLE registry_internal.registry_ingestion_runs + DROP CONSTRAINT IF EXISTS registry_ingestion_runs_blocked_reason_values, + ADD CONSTRAINT registry_ingestion_runs_blocked_reason_values + CHECK (blocked_reason IS NULL OR blocked_reason IN + ('active_package_changed', 'import_authority_closed')); + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_constraint + WHERE conrelid = 'registry_internal.registry_ingestion_runs'::regclass + AND conname = 'registry_ingestion_runs_attempt_values' + AND pg_catalog.pg_get_constraintdef(oid) + LIKE '%import_authority_closed%' + ) THEN + ALTER TABLE registry_internal.registry_ingestion_runs + DROP CONSTRAINT IF EXISTS registry_ingestion_runs_attempt_values, + ADD CONSTRAINT registry_ingestion_runs_attempt_values + CHECK (last_attempt_outcome IS NULL OR last_attempt_outcome IN + ('committed', 'replayed', 'invalid_item', 'refused', + 'binding_changed', 'import_authority_closed', 'chunk_mismatch', + 'run_not_open', 'unavailable')); + END IF; + END + $registry_ingestion_import_authority_upgrade$; REVOKE ALL ON registry_internal.registry_ingestion_runs, registry_internal.registry_ingestion_run_chunks, registry_internal.registry_ingestion_run_chunk_records FROM PUBLIC; @@ -550,6 +610,7 @@ fn parse_run_row(row: &tokio_postgres::Row) -> Option { chunk_algorithm_version: row.get("chunk_algorithm_version"), maximum_items: i64::from(row.get::<_, i32>("maximum_items")), maximum_bytes: row.get("maximum_bytes"), + import_authority_id: row.get("import_authority_id"), status: IngestionRunStatus::parse(&status)?, blocked_reason: row .try_get::<_, Option>("blocked_reason") @@ -578,8 +639,8 @@ fn parse_run_row(row: &tokio_postgres::Row) -> Option { const RUN_COLUMNS: &str = "run_id, created_principal_reference, package_revision, schema_fingerprint, entity_id, operation, profile_id, bound_context_reference, input_digest, input_length, - item_count, chunk_count, chunk_algorithm_version, maximum_items, maximum_bytes, status, - blocked_reason, next_chunk_index, committed_items, committed_prefix_digest, + item_count, chunk_count, chunk_algorithm_version, maximum_items, maximum_bytes, + import_authority_id, status, blocked_reason, next_chunk_index, committed_items, committed_prefix_digest, last_attempt_outcome, last_attempt_chunk_index, created_at, updated_at"; pub(crate) fn validate_new_run(run: &NewIngestionRun) -> Result<(), IngestionStoreError> { @@ -635,11 +696,11 @@ pub(crate) async fn insert_run( (run_id, created_principal_reference, package_revision, schema_fingerprint, entity_id, operation, profile_id, bound_context_reference, input_digest, input_length, item_count, chunk_count, chunk_algorithm_version, - maximum_items, maximum_bytes, status, blocked_reason, next_chunk_index, - committed_items, committed_prefix_digest, last_attempt_outcome, - last_attempt_chunk_index) + maximum_items, maximum_bytes, import_authority_id, status, blocked_reason, + next_chunk_index, committed_items, committed_prefix_digest, + last_attempt_outcome, last_attempt_chunk_index) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, - 'open', NULL, 0, 0, $16, NULL, NULL) + $17, 'open', NULL, 0, 0, $16, NULL, NULL) RETURNING {RUN_COLUMNS}", ), &[ @@ -659,6 +720,7 @@ pub(crate) async fn insert_run( &maximum_items, &run.maximum_bytes, &empty_digest_hex(), + &run.import_authority_id, ], ) .await @@ -1326,6 +1388,12 @@ pub(crate) fn run_audit_record( "nextChunkIndex": run.next_chunk_index, "status": run.status.as_str(), }); + if let Some(authority_id) = run.import_authority_id { + record["importAuthorityId"] = json!(authority_id.to_string()); + } + if let Some(reason) = run.blocked_reason { + record["blockedReason"] = json!(reason.as_str()); + } if let Some(correlation) = correlation { record["correlation"] = json!(correlation); } @@ -1388,6 +1456,7 @@ mod tests { chunk_algorithm_version: "greedy-canonical-http-batch-v1".to_owned(), maximum_items: 4, maximum_bytes: 262_144, + import_authority_id: None, } } @@ -1479,6 +1548,7 @@ mod tests { chunk_algorithm_version: "greedy-canonical-http-batch-v1".to_owned(), maximum_items: 4, maximum_bytes: 262_144, + import_authority_id: None, status: IngestionRunStatus::Open, blocked_reason: None, next_chunk_index: 1, diff --git a/crates/registry-breg/src/instance_claim.rs b/crates/registry-breg/src/instance_claim.rs new file mode 100644 index 0000000000..5e2251ac46 --- /dev/null +++ b/crates/registry-breg/src/instance_claim.rs @@ -0,0 +1,672 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The instance claim: which physical database a Registry serves from. +//! +//! A logical restore (`pg_dump` and `pg_restore`, or a copy into another +//! database or cluster) carries every row into a database with another +//! physical identity. Were the copy to serve beside the original, the two +//! would be divergent writers of one Registry: each would accept writes and +//! commit revisions from the same history, each would admit imports under the +//! import authorities the backup carried open, and each would deliver the +//! same outbox work. +//! +//! The claim is one row naming the physical identity the Registry serves +//! from: the cluster's system identifier and the database oid. The first +//! apply into a fresh database, one with no committed revision and no commit +//! head, records the database it runs in. An apply into a database that +//! already holds committed history, a Registry installed before the claim +//! existed or a copy restored from a backup taken before it, records no +//! claim, so that database refuses to serve until an operator adopts it. The +//! serving runtime compares the claim with the database it is connected to at +//! startup and on every readiness probe, and refuses by name when they differ. +//! An operator makes a copy the Registry's database with an explicit adoption, +//! which moves the claim, raises its epoch, supersedes every open import +//! authority, and, once that commits, appends one audit entry naming the claim +//! it replaced. +//! +//! The system identifier comes from `pg_control_system()`, which PostgreSQL +//! grants to every role by default but a managed service may withhold. When +//! the role reading it holds no `EXECUTE` privilege on it, the identifier is +//! absent, a claim is recorded without it, and a claim or a connection +//! without it compares the database oid alone. The oid still tells another +//! database in the same cluster apart, but a copy restored into a fresh +//! cluster can reach the same oid, so the oid alone is the weaker check. +//! +//! The runtime role reads the claim and cannot change it. Operator tooling is +//! not refused on a copy, so a copy can be inspected, verified, and adopted. +//! +//! A physical copy (a base backup, point-in-time recovery, a storage snapshot, +//! or a promoted replica) keeps the system identifier and the database oid, +//! so the claim cannot tell it from its original. Fencing the original before +//! a physical copy serves remains the operator's work. + +use serde::Serialize; +use tokio_postgres::GenericClient; + +use crate::postgres::SqlIdentifier; + +/// The product-owned claim table and the table privileges the runtime role +/// holds on it. Catalog closure consumes this list. +pub(crate) const INSTANCE_CLAIM_TABLES: &[(&str, &[&str])] = + &[("registry_instance_claim", &["SELECT"])]; + +const DATABASE_OID: &str = "SELECT oid + FROM pg_catalog.pg_database + WHERE datname = pg_catalog.current_database()"; + +/// Whether the connected role may call `pg_control_system()`. A function +/// privilege is checked when a statement starts, before any branch of it +/// runs, so the identifier is read by a second statement only when this one +/// answers true. +const SYSTEM_IDENTIFIER_READABLE: &str = "SELECT COALESCE( + pg_catalog.has_function_privilege( + pg_catalog.to_regprocedure('pg_catalog.pg_control_system()'), + 'EXECUTE'), + false)"; + +const SYSTEM_IDENTIFIER: &str = "SELECT system_identifier FROM pg_catalog.pg_control_system()"; + +/// The physical identity of one PostgreSQL database. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct InstanceIdentity { + /// The cluster's system identifier, in decimal, or `None` when the role + /// that read it may not call `pg_control_system()`. It is a 64-bit value, + /// so it is written as a string to survive readers that hold numbers as + /// doubles. + pub system_identifier: Option, + pub database_oid: u32, +} + +impl InstanceIdentity { + /// Whether a recorded claim names this database: the oids agree, and so + /// do the system identifiers when both sides carry one. + fn named_by(&self, claim: &InstanceIdentity) -> bool { + self.database_oid == claim.database_oid + && match (&self.system_identifier, &claim.system_identifier) { + (Some(live), Some(claimed)) => live == claimed, + _ => true, + } + } +} + +/// Whether the claim names the database a connection reached. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ClaimCheck { + Current, + Mismatch, + Unavailable, +} + +/// KERNEL INTERNAL SCHEMA MIGRATION (instance claim): creates +/// `registry_internal.registry_instance_claim` and records the database the +/// installing transaction runs in, only when no claim is present and the +/// database is fresh: no committed revision and no commit head. A restored +/// copy therefore keeps the claim of its original, and a database that +/// already holds committed history gains no claim until an operator adopts +/// it. The revision and commit head tables must already exist. Additive and +/// idempotent. +pub(crate) async fn install( + migration: &impl GenericClient, + runtime_role: &SqlIdentifier, +) -> Result<(), tokio_postgres::Error> { + migration + .batch_execute(&format!( + "CREATE TABLE IF NOT EXISTS registry_internal.registry_instance_claim ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + system_identifier bigint, + database_oid oid NOT NULL, + epoch bigint NOT NULL DEFAULT 1 CHECK (epoch >= 1), + claimed_at timestamptz NOT NULL DEFAULT transaction_timestamp() + ); + REVOKE ALL ON registry_internal.registry_instance_claim FROM PUBLIC; + REVOKE ALL ON registry_internal.registry_instance_claim FROM \"{role}\"; + GRANT SELECT ON registry_internal.registry_instance_claim TO \"{role}\";", + role = runtime_role.as_str(), + )) + .await?; + let live = live_identity(migration).await?; + migration + .execute( + "INSERT INTO registry_internal.registry_instance_claim + (singleton, system_identifier, database_oid) + SELECT true, $1::text::bigint, $2 + WHERE NOT EXISTS (SELECT 1 FROM registry_internal.registry_revisions) + AND NOT EXISTS (SELECT 1 FROM registry_internal.registry_commit_head) + ON CONFLICT (singleton) DO NOTHING", + &[&live.system_identifier, &live.database_oid], + ) + .await?; + Ok(()) +} + +/// Compare the claim with the database the connection reached. A missing +/// claim row is a mismatch: no claim names this database. +pub(crate) async fn check(client: &impl GenericClient) -> ClaimCheck { + let Ok(live) = live_identity(client).await else { + return ClaimCheck::Unavailable; + }; + let row = match client + .query_opt( + "SELECT system_identifier, database_oid + FROM registry_internal.registry_instance_claim + WHERE singleton", + &[], + ) + .await + { + Ok(row) => row, + Err(_) => return ClaimCheck::Unavailable, + }; + let Some(row) = row else { + return ClaimCheck::Mismatch; + }; + match identity_from(&row) { + Ok(claim) if live.named_by(&claim) => ClaimCheck::Current, + Ok(_) => ClaimCheck::Mismatch, + Err(_) => ClaimCheck::Unavailable, + } +} + +async fn live_identity( + client: &impl GenericClient, +) -> Result { + let database_oid = client.query_one(DATABASE_OID, &[]).await?.try_get(0)?; + let readable: bool = client + .query_one(SYSTEM_IDENTIFIER_READABLE, &[]) + .await? + .try_get(0)?; + let system_identifier = if readable { + Some( + client + .query_one(SYSTEM_IDENTIFIER, &[]) + .await? + .try_get::<_, i64>(0)? + .to_string(), + ) + } else { + None + }; + Ok(InstanceIdentity { + system_identifier, + database_oid, + }) +} + +/// The identity a claim row records, from its first two columns. +fn identity_from(row: &tokio_postgres::Row) -> Result { + Ok(InstanceIdentity { + system_identifier: row + .try_get::<_, Option>(0)? + .map(|value| value.to_string()), + database_oid: row.try_get(1)?, + }) +} + +#[cfg(feature = "tooling")] +pub use operator::{ + InstanceClaim, InstanceClaimAdoption, InstanceClaimError, InstanceClaimService, + InstanceClaimStatus, +}; + +#[cfg(feature = "tooling")] +mod operator { + use std::path::Path; + use std::time::Duration; + + use chrono::{DateTime, Utc}; + use registry_platform_audit::AuditEntry; + use serde::Serialize; + use serde_json::{json, Value}; + use tokio_postgres::{GenericClient, Transaction}; + use uuid::Uuid; + + use super::{identity_from, live_identity, InstanceIdentity}; + use crate::audit::RegistryAudit; + use crate::postgres::{ + verify_catalog_identity_for_catalog, verify_migration_role, ConnectionConfig, + ExpectedManagedCatalog, ExpectedRegistryIdentity, RegistryLockKey, SqlIdentifier, + }; + + const AUDIT_SCHEMA: &str = "breg-instance-claim-audit/v1"; + const AUDIT_OPERATION_ID: &str = "breg.instance_claim.adopt"; + + /// Value-free refusal of an instance claim operation. + #[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] + pub enum InstanceClaimError { + #[error("the instance claim already names this database")] + AlreadyCurrent, + #[error("the instance claim is unavailable")] + Unavailable, + } + + /// The recorded claim. + #[derive(Clone, Debug, Eq, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct InstanceClaim { + #[serde(flatten)] + pub identity: InstanceIdentity, + pub epoch: i64, + pub claimed_at: DateTime, + } + + /// The recorded claim beside the database the connection reached. + #[derive(Clone, Debug, Eq, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct InstanceClaimStatus { + pub live: InstanceIdentity, + pub claim: Option, + pub matches: bool, + } + + /// The claim an adoption replaced, the one it recorded, and the import + /// authorities it superseded because the copy carried them open. + #[derive(Clone, Debug, Eq, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct InstanceClaimAdoption { + pub previous: Option, + pub current: InstanceClaim, + pub superseded_import_authorities: Vec, + } + + /// Package-bound operator boundary used by `bregctl instance-claim`. + /// + /// The same runtime configuration, package, database identity, catalog, + /// roles, and Registry lock close before a claim is read or moved, and an + /// adoption appends to the operator companion of the configured audit + /// destination. + pub struct InstanceClaimService { + expected: ExpectedRegistryIdentity, + expected_catalog: ExpectedManagedCatalog, + lock_key: RegistryLockKey, + migration_connection: ConnectionConfig, + runtime_connection: ConnectionConfig, + migration_role: SqlIdentifier, + runtime_role: SqlIdentifier, + lock_timeout: Duration, + statement_timeout: Duration, + audit: RegistryAudit, + } + + impl InstanceClaimService { + pub async fn from_runtime_config(path: &Path) -> Result { + if !path.is_absolute() { + return Err(InstanceClaimError::Unavailable); + } + let config = crate::runtime_config::load_runtime_config(path) + .map_err(|_| InstanceClaimError::Unavailable)?; + let audit = RegistryAudit::open_companion(&config) + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + let runtime_connection = config + .runtime_database_connection_config() + .map_err(|_| InstanceClaimError::Unavailable)?; + let pool = runtime_connection + .build_pool() + .map_err(|_| InstanceClaimError::Unavailable)?; + let mut client = pool + .get() + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + let startup = crate::startup::prepare_startup( + config.package().root(), + &config.package_load_context(), + &mut client, + config.database().roles().migration(), + config.database().roles().runtime(), + ) + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + Ok(Self { + expected: startup.expected_identity().clone(), + expected_catalog: startup.expected_catalog().clone(), + lock_key: startup.lock_key(), + migration_connection: config + .migration_database_connection_config() + .map_err(|_| InstanceClaimError::Unavailable)?, + runtime_connection, + migration_role: config.database().roles().migration().clone(), + runtime_role: config.database().roles().runtime().clone(), + lock_timeout: config.operational_timeouts().migration_lock, + statement_timeout: config.operational_timeouts().migration_statement, + audit, + }) + } + + #[cfg(feature = "postgres-test")] + #[doc(hidden)] + #[allow(clippy::too_many_arguments)] + #[must_use] + pub fn new_for_test( + expected: ExpectedRegistryIdentity, + expected_catalog: ExpectedManagedCatalog, + lock_key: RegistryLockKey, + migration_connection: ConnectionConfig, + runtime_connection: ConnectionConfig, + migration_role: SqlIdentifier, + runtime_role: SqlIdentifier, + audit: RegistryAudit, + ) -> Self { + Self { + expected, + expected_catalog, + lock_key, + migration_connection, + runtime_connection, + migration_role, + runtime_role, + lock_timeout: Duration::from_secs(5), + statement_timeout: Duration::from_secs(10), + audit, + } + } + + /// Read the claim and the identity of the database the runtime role + /// reaches, in a read-only transaction that first closes the catalog + /// identity. + pub async fn status(&self) -> Result { + let pool = self + .runtime_connection + .build_pool() + .map_err(|_| InstanceClaimError::Unavailable)?; + let mut client = pool + .get() + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + let pg_client: &mut tokio_postgres::Client = &mut client; + let transaction = pg_client + .build_transaction() + .read_only(true) + .start() + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + self.set_local_timeouts(&transaction).await?; + verify_catalog_identity_for_catalog( + &transaction, + &self.expected, + &self.expected_catalog, + &self.migration_role, + &self.runtime_role, + ) + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + let status = read_status(&transaction, false).await?; + transaction + .commit() + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + Ok(status) + } + + /// Make the connected database the one the claim names. + /// + /// A request entry is accepted before any database work, so an audit + /// outage moves nothing. Under the Registry lock, the claim then names + /// this database and its epoch rises by one, and every open import + /// authority is superseded, in one transaction. Once it commits, each + /// supersession's transition record and the response naming the claim + /// replaced and the authorities superseded are appended. An adoption + /// that ends without a response, as after a commit error, writes the + /// unfinished outcome. + pub async fn adopt(&self) -> Result { + if !crate::audit::profile_is_keyed(self.audit.profile()) { + return Err(InstanceClaimError::Unavailable); + } + let correlation = Uuid::new_v4().to_string(); + let request = json!({ + "phase": "attempt", + "operationId": AUDIT_OPERATION_ID, + "packageRevision": self.expected.package_revision, + }); + let mut attempt = self + .audit + .begin( + AuditEntry::request(AUDIT_SCHEMA, correlation, request.clone()), + outcome_record(&request, "unfinished"), + ) + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + let pool = self + .migration_connection + .build_pool() + .map_err(|_| InstanceClaimError::Unavailable)?; + let mut client = match pool.get().await { + Ok(client) => client, + Err(_) => { + respond_refused(&mut attempt, &request, "failed").await; + return Err(InstanceClaimError::Unavailable); + } + }; + let mut pending = Vec::new(); + let adopted = match self.adopt_in_transaction(&mut client, &mut pending).await { + Ok(adopted) => adopted, + Err(error) => { + let outcome = if error == InstanceClaimError::AlreadyCurrent { + "refused" + } else { + "failed" + }; + respond_refused(&mut attempt, &request, outcome).await; + return Err(error); + } + }; + // A commit error leaves the attempt unanswered, so it records the + // unfinished outcome when dropped. + let (adoption, record) = adopted.commit().await?; + crate::import_authority::append_transitions(&self.audit, pending) + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + attempt + .respond(record) + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + Ok(adoption) + } + + async fn adopt_in_transaction<'c>( + &self, + client: &'c mut tokio_postgres::Client, + pending: &mut Vec, + ) -> Result, InstanceClaimError> { + verify_migration_role(client, &self.migration_role) + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + let transaction = client + .transaction() + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + self.set_local_timeouts(&transaction).await?; + transaction + .execute( + "SELECT pg_catalog.pg_advisory_xact_lock($1)", + &[&self.lock_key.get()], + ) + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + verify_catalog_identity_for_catalog( + &transaction, + &self.expected, + &self.expected_catalog, + &self.migration_role, + &self.runtime_role, + ) + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + let (adoption, record) = + adopt_in(&transaction, pending, &self.expected.package_revision).await?; + Ok(Adopted { + transaction, + adoption, + record, + }) + } + + async fn set_local_timeouts( + &self, + transaction: &Transaction<'_>, + ) -> Result<(), InstanceClaimError> { + transaction + .query_one( + "SELECT set_config('lock_timeout', $1, true), + set_config('statement_timeout', $2, true)", + &[ + &format!("{}ms", self.lock_timeout.as_millis()), + &format!("{}ms", self.statement_timeout.as_millis()), + ], + ) + .await + .map(|_| ()) + .map_err(|_| InstanceClaimError::Unavailable) + } + } + + /// An adoption written and not yet committed. + struct Adopted<'c> { + transaction: Transaction<'c>, + adoption: InstanceClaimAdoption, + record: Value, + } + + impl Adopted<'_> { + async fn commit(self) -> Result<(InstanceClaimAdoption, Value), InstanceClaimError> { + self.transaction + .commit() + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + Ok((self.adoption, self.record)) + } + } + + /// The response of an adoption that committed nothing: the request's + /// fields with `outcome`. + fn outcome_record(request: &Value, outcome: &str) -> Value { + let mut record = request.clone(); + if let Some(fields) = record.as_object_mut() { + fields.insert("phase".to_owned(), json!("terminal")); + fields.insert("outcome".to_owned(), json!(outcome)); + } + record + } + + async fn respond_refused( + attempt: &mut registry_platform_audit::AuditRequest, + request: &Value, + outcome: &str, + ) { + if attempt + .respond(outcome_record(request, outcome)) + .await + .is_err() + { + tracing::error!("the instance claim adoption's response audit entry was not recorded"); + } + } + + pub(crate) async fn read_status( + client: &impl GenericClient, + lock: bool, + ) -> Result { + let live = live_identity(client) + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + let row = client + .query_opt( + if lock { + "SELECT system_identifier, database_oid, epoch, claimed_at + FROM registry_internal.registry_instance_claim + WHERE singleton + FOR UPDATE" + } else { + "SELECT system_identifier, database_oid, epoch, claimed_at + FROM registry_internal.registry_instance_claim + WHERE singleton" + }, + &[], + ) + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + let claim = row.map(|row| claim_from(&row)).transpose()?; + let matches = claim + .as_ref() + .is_some_and(|claim| live.named_by(&claim.identity)); + Ok(InstanceClaimStatus { + live, + claim, + matches, + }) + } + + /// Move the claim to the connected database inside the caller's + /// transaction, which already holds the Registry lock, and supersede + /// every open import authority, collecting each transition record into + /// `pending`. Returns the response record to append once the transaction + /// commits. + pub(crate) async fn adopt_in( + transaction: &Transaction<'_>, + pending: &mut Vec, + package_revision: &str, + ) -> Result<(InstanceClaimAdoption, Value), InstanceClaimError> { + let status = read_status(transaction, true).await?; + if status.matches { + return Err(InstanceClaimError::AlreadyCurrent); + } + let row = transaction + .query_one( + "INSERT INTO registry_internal.registry_instance_claim + (singleton, system_identifier, database_oid, epoch, claimed_at) + VALUES (true, $1::text::bigint, $2, 1, transaction_timestamp()) + ON CONFLICT (singleton) DO UPDATE + SET system_identifier = EXCLUDED.system_identifier, + database_oid = EXCLUDED.database_oid, + epoch = registry_instance_claim.epoch + 1, + claimed_at = EXCLUDED.claimed_at + RETURNING system_identifier, database_oid, epoch, claimed_at", + &[&status.live.system_identifier, &status.live.database_oid], + ) + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + let current = claim_from(&row)?; + let superseded_import_authorities = + crate::import_authority::supersede_every_open(transaction, pending, package_revision) + .await + .map_err(|_| InstanceClaimError::Unavailable)?; + let record = json!({ + "phase": "terminal", + "outcome": "committed", + "event": "adopted", + "operationId": AUDIT_OPERATION_ID, + "packageRevision": package_revision, + "previous": status.claim.as_ref().map(audit_claim), + "current": audit_claim(¤t), + "supersededImportAuthorities": superseded_import_authorities + .iter() + .map(ToString::to_string) + .collect::>(), + }); + Ok(( + InstanceClaimAdoption { + previous: status.claim, + current, + superseded_import_authorities, + }, + record, + )) + } + + fn audit_claim(claim: &InstanceClaim) -> Value { + json!({ + "systemIdentifier": claim.identity.system_identifier, + "databaseOid": claim.identity.database_oid, + "epoch": claim.epoch, + }) + } + + fn claim_from(row: &tokio_postgres::Row) -> Result { + let unavailable = |_| InstanceClaimError::Unavailable; + Ok(InstanceClaim { + identity: identity_from(row).map_err(unavailable)?, + epoch: row.try_get(2).map_err(unavailable)?, + claimed_at: row.try_get(3).map_err(unavailable)?, + }) + } +} diff --git a/crates/registry-breg/src/lib.rs b/crates/registry-breg/src/lib.rs index 6dfaf333bf..4f2318893d 100644 --- a/crates/registry-breg/src/lib.rs +++ b/crates/registry-breg/src/lib.rs @@ -82,7 +82,11 @@ pub mod hook_handler; pub mod idempotency; pub mod immediate_actions; #[cfg(feature = "runtime")] +pub mod import_authority; +#[cfg(feature = "runtime")] mod ingestion_store; +#[cfg(feature = "runtime")] +pub mod instance_claim; pub mod lifecycle; pub mod logical_names; pub mod manifest_adapter; @@ -121,6 +125,8 @@ mod request_store; pub mod request_workflow; pub mod review_integration; #[cfg(feature = "runtime")] +pub mod review_recovery; +#[cfg(feature = "runtime")] pub mod review_store; #[cfg(feature = "runtime")] pub mod revision; diff --git a/crates/registry-breg/src/main.rs b/crates/registry-breg/src/main.rs index d47a30764b..51e074ac33 100644 --- a/crates/registry-breg/src/main.rs +++ b/crates/registry-breg/src/main.rs @@ -2,7 +2,7 @@ //! Base Registry Engine process entry point. use clap::Parser; -use registry_breg::cli::Arguments; +use registry_breg::cli::{removed_config_flag, Arguments}; use registry_breg::startup::{ operational_log_level, prepare, serve, OperationalEvent, OperationalLogLevel, }; @@ -11,6 +11,10 @@ use tracing_subscriber::prelude::*; #[tokio::main] async fn main() { + if let Some(refusal) = removed_config_flag(std::env::args_os().skip(1)) { + eprintln!("breg: {refusal}"); + std::process::exit(2); + } let arguments = Arguments::parse(); let level = match operational_log_level(std::env::var("BREG_LOG").ok().as_deref()) { Ok(level) => level, @@ -22,12 +26,12 @@ async fn main() { }; initialize_logging_filter(level); - if !arguments.config.is_absolute() { + if !arguments.runtime_config.is_absolute() { OperationalEvent::Stopped.emit(); std::process::exit(2); } OperationalEvent::StartupBegan.emit(); - let prepared = match prepare(&arguments.config).await { + let prepared = match prepare(&arguments.runtime_config).await { Ok(prepared) => prepared, Err(error) => { OperationalEvent::StoppedWithError(error).emit(); diff --git a/crates/registry-breg/src/migration.rs b/crates/registry-breg/src/migration.rs index a827afb7cb..e709a2d8e7 100644 --- a/crates/registry-breg/src/migration.rs +++ b/crates/registry-breg/src/migration.rs @@ -25,16 +25,17 @@ use crate::package::{ use crate::postgres::{ statement_checksum, ConnectionConfig, ExpectedManagedCatalog, ExpectedRegistryIdentity, MaintenanceTransition, MigrationArtifactBinding, MigrationLedgerEntry, MigrationLedgerStep, - MigrationLedgerStepKind, MigrationPlanKind, PackageDdlStatement, RegistryLockKey, - ReviewedExecutionOutcome, ReviewedFieldEncryptionContext, ReviewedPackageExecutionRequest, - SqlIdentifier, VerifiedPackageApplyConnection, + MigrationLedgerStepKind, MigrationPlanKind, PackageDdlStatement, PostgresFailure, + RegistryLockKey, ReviewedExecutionOutcome, ReviewedFieldEncryptionContext, + ReviewedPackageExecutionRequest, SqlIdentifier, VerifiedPackageApplyConnection, }; const MAX_LOCK_TIMEOUT: Duration = Duration::from_secs(300); const MAX_STATEMENT_TIMEOUT: Duration = Duration::from_secs(60 * 60); -/// Value-free apply failures. Only authored identifiers cross this boundary; -/// SQL, stored values, and physical database names do not. +/// Value-free apply failures. Authored identifiers cross this boundary, and +/// a refused statement adds its SQLSTATE and the object names PostgreSQL +/// reported; SQL, PostgreSQL messages, and stored values do not. #[derive(Debug, Error, Clone, Eq, PartialEq)] pub enum MigrationError { #[error("the verified package is not a valid activation successor")] @@ -43,6 +44,10 @@ pub enum MigrationError { EmptyPlan, #[error("the Registry package apply failed")] ApplyFailed, + /// PostgreSQL refused a statement after maintenance began. The target + /// stays pinned in maintenance, as for [`Self::ApplyFailed`]. + #[error("PostgreSQL refused an apply statement: {0}")] + StatementFailed(PostgresFailure), /// The database, read under the exclusive apply lock, does not record the /// presented package as its active package with maintenance ready. #[error("the database does not record this package as its active, ready package")] @@ -912,6 +917,9 @@ async fn fail_with_error_and_release( crate::postgres::PostgresKernelError::RetiredAuditRowsPresent => { MigrationError::RetiredAuditRowsPresent } + crate::postgres::PostgresKernelError::Statement(failure) => { + MigrationError::StatementFailed(failure) + } _ => MigrationError::ApplyFailed, }) } diff --git a/crates/registry-breg/src/migration_plan.rs b/crates/registry-breg/src/migration_plan.rs index d23e193047..7ffb29be93 100644 --- a/crates/registry-breg/src/migration_plan.rs +++ b/crates/registry-breg/src/migration_plan.rs @@ -52,12 +52,10 @@ const MAX_LOCK_TIMEOUT_MS: u64 = 300_000; #[cfg(feature = "tooling")] const MAX_STATEMENT_TIMEOUT_MS: u64 = 3_600_000; #[cfg(feature = "tooling")] -const MAX_CHUNK_SIZE: u32 = 10_000; -#[cfg(feature = "tooling")] -/// Every field-encryption backfill chunk journals one commit whose member -/// budget the history machinery caps, so its chunk size shares that cap. -const MAX_FIELD_ENCRYPTION_CHUNK_SIZE: u32 = - crate::history_migration::MAX_HISTORY_MIGRATION_COMMIT_MEMBERS as u32; +/// Every chunk of a chunked or field-encryption backfill journals one commit +/// whose member budget the history machinery caps, so the chunk size shares +/// that cap. +const MAX_CHUNK_SIZE: u32 = crate::history_migration::MAX_HISTORY_MIGRATION_COMMIT_MEMBERS as u32; #[cfg(feature = "tooling")] const MAX_TOTAL_ROWS: u64 = 100_000_000; #[cfg(feature = "tooling")] @@ -874,7 +872,7 @@ fn validate_descriptor_shape( .. } if !valid_id(entity_id) || *chunk_size == 0 - || *chunk_size > MAX_FIELD_ENCRYPTION_CHUNK_SIZE + || *chunk_size > MAX_CHUNK_SIZE || *max_total_rows == 0 || *max_total_rows > MAX_TOTAL_ROWS || !valid_timeout(*lock_timeout_ms, descriptor.lock_timeout_ms) @@ -1995,6 +1993,7 @@ fn covers_are_metadata_only(covers: &[ReviewedChangeCover]) -> bool { | CompiledRegistryChangeCode::ActionRemoved | CompiledRegistryChangeCode::ActionChanged | CompiledRegistryChangeCode::ActionVocabularyCodesAdded + | CompiledRegistryChangeCode::ActionTargetFieldsWidened | CompiledRegistryChangeCode::RecipientOrganizationAdded | CompiledRegistryChangeCode::RecipientOrganizationRemoved | CompiledRegistryChangeCode::RecipientOrganizationChanged diff --git a/crates/registry-breg/src/migration_reconcile.rs b/crates/registry-breg/src/migration_reconcile.rs index 80ffb3a432..1d251a0e66 100644 --- a/crates/registry-breg/src/migration_reconcile.rs +++ b/crates/registry-breg/src/migration_reconcile.rs @@ -184,6 +184,7 @@ impl From for ReconcileError { Self::InvalidInput } PostgresKernelError::Connection + | PostgresKernelError::Statement(_) | PostgresKernelError::Pool | PostgresKernelError::PoolBuild | PostgresKernelError::CatalogInvariant(_) @@ -199,6 +200,7 @@ impl From for ReconcileError { match error { MigrationError::PackageBinding | MigrationError::EmptyPlan => Self::PackageBinding, MigrationError::ApplyFailed + | MigrationError::StatementFailed(_) | MigrationError::ActivePackageMismatch | MigrationError::HistoryCoverage | MigrationError::DatabaseUnavailable diff --git a/crates/registry-breg/src/mutation.rs b/crates/registry-breg/src/mutation.rs index 9233861adc..456a110679 100644 --- a/crates/registry-breg/src/mutation.rs +++ b/crates/registry-breg/src/mutation.rs @@ -438,12 +438,21 @@ pub async fn install_mutation_schema( .await .map_err(|_| MutationError::Unavailable)?; crate::request_store::install(migration, runtime_role).await?; + // The run table references the authority table, so authorities install first. + crate::import_authority::install(migration, runtime_role) + .await + .map_err(|_| MutationError::Unavailable)?; crate::ingestion_store::install(migration, runtime_role) .await .map_err(|_| MutationError::Unavailable)?; install_history_commit_schema(migration, runtime_role) .await .map_err(MutationError::from)?; + // The claim reads the revision and commit head tables to decide whether + // this database is fresh, so it installs after them. + crate::instance_claim::install(migration, runtime_role) + .await + .map_err(|_| MutationError::Unavailable)?; Ok(()) } @@ -455,6 +464,9 @@ pub struct MutationPlan { submitter_target_entities: BTreeMap, event_deliveries: Vec, temporal_exclusion_constraints: Vec, + /// A create item of an `import` route. Change control does not count it + /// as a direct write: an operator-opened import authority bounds it. + imported_item: bool, } impl MutationPlan { @@ -476,7 +488,8 @@ impl MutationPlan { (Operation::Create, HttpMethod::Post) | (Operation::Patch, HttpMethod::Patch) | (Operation::Tombstone, HttpMethod::Delete) - | (Operation::Batch, HttpMethod::Post) => {} + | (Operation::Batch, HttpMethod::Post) + | (Operation::Import, HttpMethod::Post) => {} _ => return Err(MutationError::InvalidRequest), } if matches!(route.operation, Operation::Patch | Operation::Tombstone) @@ -487,7 +500,8 @@ impl MutationPlan { if route.operation == Operation::Tombstone && !entity.tombstone { return Err(MutationError::InvalidRequest); } - if route.operation == Operation::Batch && entity.batch.is_none() { + if matches!(route.operation, Operation::Batch | Operation::Import) && entity.batch.is_none() + { return Err(MutationError::InvalidRequest); } let inventory = registry @@ -526,6 +540,7 @@ impl MutationPlan { .collect::>()?, event_deliveries, temporal_exclusion_constraints, + imported_item: false, }) } @@ -561,9 +576,14 @@ impl MutationPlan { } fn batch_item(&self, operation: Operation, profile_id: &str) -> Result { - if self.route.operation != Operation::Batch - || !matches!(operation, Operation::Create | Operation::Patch) - { + // An `import` route drives create items alone; it never reaches an + // existing record. + let admitted = match self.route.operation { + Operation::Batch => matches!(operation, Operation::Create | Operation::Patch), + Operation::Import => operation == Operation::Create, + _ => false, + }; + if !admitted { return Err(MutationError::InvalidRequest); } let (method, path) = match operation { @@ -603,6 +623,7 @@ impl MutationPlan { submitter_target_entities: self.submitter_target_entities.clone(), event_deliveries: self.event_deliveries.clone(), temporal_exclusion_constraints: self.temporal_exclusion_constraints.clone(), + imported_item: self.route.operation == Operation::Import, }) } } @@ -1651,6 +1672,63 @@ impl MutationCoordinator { }) } + /// Move an open ingestion run to `blocked` for `reason`, recording the + /// attempt that names the same cause in the caller's transaction, and + /// answer the refusal the caller returns with the blocked audit record it + /// appends once that transaction commits. + /// + /// The blocking transition verifies it changed the row: the run row lock + /// serializes this against every other transition, so a zero-row update + /// means the run was no longer stored open and belongs to the terminal + /// answer, not a blocked audit record this request never earned. + async fn block_ingestion_run( + &self, + transaction: &tokio_postgres::Transaction<'_>, + run: &crate::ingestion_store::IngestionRunRecord, + chunk_index: i64, + reason: crate::ingestion_store::IngestionBlockedReason, + correlation: &RequestCorrelation, + ) -> Result<(IngestionRefusal, Option), MutationError> { + let (outcome, refusal) = match reason { + crate::ingestion_store::IngestionBlockedReason::ActivePackageChanged => ( + IngestionAttemptOutcome::BindingChanged, + IngestionRefusal::BindingChanged, + ), + crate::ingestion_store::IngestionBlockedReason::ImportAuthorityClosed => ( + IngestionAttemptOutcome::ImportAuthorityClosed, + IngestionRefusal::AuthorityClosed, + ), + }; + let changed = crate::ingestion_store::mark_blocked(transaction, run.run_id, reason) + .await + .map_err(|_| MutationError::Unavailable)?; + if changed == 0 { + record_attempt( + transaction, + run.run_id, + IngestionAttemptOutcome::RunNotOpen, + chunk_index, + ) + .await + .map_err(|_| MutationError::Unavailable)?; + return Ok((IngestionRefusal::RunNotOpen, None)); + } + record_attempt(transaction, run.run_id, outcome, chunk_index) + .await + .map_err(|_| MutationError::Unavailable)?; + let mut audited_run = run.clone(); + audited_run.status = IngestionRunStatus::Blocked; + audited_run.blocked_reason = Some(reason); + let blocked_record = crate::ingestion_store::run_audit_record( + "blocked", + &audited_run, + &self.expected.package_revision, + &run.created_principal_reference, + Some(&correlation.request_id().to_string()), + ); + Ok((refusal, Some(blocked_record))) + } + async fn execute_batch_after_attempt( &self, client: &mut Client, @@ -1795,67 +1873,79 @@ impl MutationCoordinator { &self.expected.package_revision, &self.expected.schema_fingerprint, ) { - // The blocking transition verifies it changed the row: the run - // row lock serializes this arm against every other - // transition, so a zero-row update means the run was no - // longer stored open and belongs to the terminal answer, not - // a blocked audit record this request never earned. - let changed = crate::ingestion_store::mark_blocked( - transaction.transaction(), - chunk_binding.run_id, - crate::ingestion_store::IngestionBlockedReason::ActivePackageChanged, - ) - .await - .map_err(|_| MutationError::Unavailable)?; - if changed == 0 { - record_attempt( + let (refusal, blocked_record) = self + .block_ingestion_run( transaction.transaction(), - chunk_binding.run_id, - IngestionAttemptOutcome::RunNotOpen, + &run, chunk_binding.chunk_index, + crate::ingestion_store::IngestionBlockedReason::ActivePackageChanged, + &request.correlation, ) + .await?; + // The blocked marking must outlive the refusal, so the refusal + // returns only after an explicit commit and the blocked audit + // entry it earned. + transaction + .commit() .await .map_err(|_| MutationError::Unavailable)?; - transaction - .commit() + if let Some(record) = blocked_record { + crate::ingestion_store::append_run_audit(&self.audit, record) .await .map_err(|_| MutationError::Unavailable)?; - return Err(MutationError::IngestionRefusal( - IngestionRefusal::RunNotOpen, - )); } - record_attempt( + return Err(MutationError::IngestionRefusal(refusal)); + } + // An `import` run commits a chunk only while the authority it + // consumes is open, unexpired, opened under the active package, + // and has room for the chunk. The authority row lock taken here + // is held to the chunk commit, which counts the chunk against it, + // so a concurrent close waits for this chunk and stops the next. + // A run no longer stored open is answered by the window check + // below and consumes nothing. + if let Some(authority_id) = run + .import_authority_id + .filter(|_| run.status == IngestionRunStatus::Open) + { + let chunk_items = + i64::try_from(request.items.len()).map_err(|_| MutationError::Unavailable)?; + let mut authority_records = Vec::new(); + let admitted = crate::import_authority::admit_chunk( transaction.transaction(), - chunk_binding.run_id, - IngestionAttemptOutcome::BindingChanged, - chunk_binding.chunk_index, + &mut authority_records, + &self.expected.package_revision, + authority_id, + chunk_items, ) .await .map_err(|_| MutationError::Unavailable)?; - let mut audited_run = run.clone(); - audited_run.status = IngestionRunStatus::Blocked; - audited_run.blocked_reason = - Some(crate::ingestion_store::IngestionBlockedReason::ActivePackageChanged); - let blocked_record = crate::ingestion_store::run_audit_record( - "blocked", - &audited_run, - &self.expected.package_revision, - &run.created_principal_reference, - Some(&request.correlation.request_id().to_string()), - ); - // The blocked marking and its audit must outlive the refusal, - // so the refusal returns only after an explicit commit and - // the accepted audit append that follows it. - transaction - .commit() - .await - .map_err(|_| MutationError::Unavailable)?; - crate::ingestion_store::append_run_audit(&self.audit, blocked_record) - .await - .map_err(|_| MutationError::Unavailable)?; - return Err(MutationError::IngestionRefusal( - IngestionRefusal::BindingChanged, - )); + if !admitted { + let (refusal, blocked_record) = self + .block_ingestion_run( + transaction.transaction(), + &run, + chunk_binding.chunk_index, + crate::ingestion_store::IngestionBlockedReason::ImportAuthorityClosed, + &request.correlation, + ) + .await?; + // The authority transition and the blocked marking must + // outlive the refusal, and their audit entries are + // appended once that commit holds. + transaction + .commit() + .await + .map_err(|_| MutationError::Unavailable)?; + crate::import_authority::append_transitions(&self.audit, authority_records) + .await + .map_err(|_| MutationError::Unavailable)?; + if let Some(record) = blocked_record { + crate::ingestion_store::append_run_audit(&self.audit, record) + .await + .map_err(|_| MutationError::Unavailable)?; + } + return Err(MutationError::IngestionRefusal(refusal)); + } } let announced_items = i64::try_from(request.items.len()).map_err(|_| MutationError::Unavailable)?; @@ -2189,6 +2279,7 @@ impl MutationCoordinator { .await?; } let mut run_record = None; + let mut authority_records = Vec::new(); if let (Some(chunk_binding), Some(run)) = (request.ingestion, ingestion_run.as_ref()) { // The chunk receipt, its record links, and the checkpoint advance // join the mutation transaction itself, so the committed prefix @@ -2213,6 +2304,17 @@ impl MutationCoordinator { ) .await .map_err(|_| MutationError::Unavailable)?; + if let Some(authority_id) = run.import_authority_id { + crate::import_authority::consume( + transaction.transaction(), + &mut authority_records, + &self.expected.package_revision, + authority_id, + chunk_binding.item_count, + ) + .await + .map_err(|_| MutationError::Unavailable)?; + } let mut audited_run = run.clone(); audited_run.committed_items += chunk_binding.item_count; audited_run.next_chunk_index = chunk_binding.chunk_index + 1; @@ -2236,6 +2338,9 @@ impl MutationCoordinator { .append(entry) .await .map_err(|_| MutationError::CommitUnresolved)?; + crate::import_authority::append_transitions(&self.audit, authority_records) + .await + .map_err(|_| MutationError::CommitUnresolved)?; if let Some(record) = run_record { crate::ingestion_store::append_run_audit(&self.audit, record) .await @@ -2658,7 +2763,11 @@ async fn apply_current_row( .entity .change_control .as_ref() - .is_some_and(|control| control.required_for.contains(&request.plan.route.operation)) + .is_some_and(|control| { + control.required_for.contains(&request.plan.route.operation) + && !(request.plan.imported_item + && request.plan.route.operation == Operation::Create) + }) || (request.plan.entity.change_request.is_some() && request.plan.route.operation == Operation::Tombstone) { @@ -3804,12 +3913,19 @@ fn validate_batch_request( .access_profiles .get(request.claims.access_profile()) .ok_or(MutationError::InvalidRequest)?; - if request.plan.route.operation != Operation::Batch + // An `import` route executes only as a chunk of a durable ingestion run, + // whose binding the transaction checks under the run lock. + let bulk_route = match request.plan.route.operation { + Operation::Batch => true, + Operation::Import => request.ingestion.is_some(), + _ => false, + }; + if !bulk_route || request.plan.route.method != HttpMethod::Post || request.claims.entity_id() != request.plan.entity.id || request.claims.principal().is_none() || profile.anonymous - || !profile.operations.contains(&Operation::Batch) + || !profile.operations.contains(&request.plan.route.operation) || !request .plan .route @@ -3827,7 +3943,11 @@ fn validate_batch_request( } for item in &request.items { - if !profile.operations.contains(&item.operation()) + let item_granted = match request.plan.route.operation { + Operation::Import => item.operation() == Operation::Create, + _ => profile.operations.contains(&item.operation()), + }; + if !item_granted || item.operation() == Operation::Patch && request.plan.entity.mutation_mode != MutationMode::Mutable { diff --git a/crates/registry-breg/src/mutation/request.rs b/crates/registry-breg/src/mutation/request.rs index 5f057c4808..01126cf625 100644 --- a/crates/registry-breg/src/mutation/request.rs +++ b/crates/registry-breg/src/mutation/request.rs @@ -2503,6 +2503,7 @@ impl MutationCoordinator { temporal_exclusion_constraints: temporal_exclusion_constraints( registry, entity, inventory, )?, + imported_item: false, }; let id = target.record_id.to_string(); let request = MutationRequest { diff --git a/crates/registry-breg/src/package.rs b/crates/registry-breg/src/package.rs index f1d4ae5aa1..992970949b 100644 --- a/crates/registry-breg/src/package.rs +++ b/crates/registry-breg/src/package.rs @@ -11,6 +11,10 @@ use std::io::{Read, Write}; use std::path::{Component, Path, PathBuf}; use registry_platform_canonical_json::{canonicalize_json, parse_json_strict}; +use registry_platform_config::package::{ + write_sum_file, PackageLimits as SharedPackageLimits, VerifiedPackage as SharedVerifiedPackage, + REVISION_FILE, SUM_FILE, +}; use registry_platform_crypto::{verify, PublicJwk}; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -27,9 +31,10 @@ use crate::derived_sql::MAX_DERIVED_SQL_BYTES; use crate::generated_ddl::{ add_blind_index_column_statement, add_column_statement, drop_spatial_bbox_function_statement, drop_spatial_candidate_view_statement, generate_ddl_with_actions, quote_identifier, - replace_vocabulary_check_statement, set_column_not_null_statement, - spatial_bbox_function_statement, spatial_projection_fields, spatial_projection_statements, - DdlInventory, DdlPolicy, DdlPolicyRole, DdlStatement, DdlStatementKind, DdlTable, + replace_length_check_statement, replace_vocabulary_check_statement, + set_column_not_null_statement, spatial_bbox_function_statement, spatial_projection_fields, + spatial_projection_statements, DdlInventory, DdlPolicy, DdlPolicyRole, DdlStatement, + DdlStatementKind, DdlTable, }; use crate::history_schema::{ serialize_descriptor, HistoryEntityDescriptor, HistoryLifecycleDescriptor, @@ -286,6 +291,7 @@ pub enum CompiledRegistryChangeCode { FieldRemoved, FieldTypeChanged, FieldVocabularyCodesAdded, + FieldLengthWidened, FieldPhysicalNameChanged, FieldRequirednessChanged, FieldPatternAdded, @@ -319,6 +325,7 @@ pub enum CompiledRegistryChangeCode { ActionRemoved, ActionChanged, ActionVocabularyCodesAdded, + ActionTargetFieldsWidened, ConsentRecordChanged, RecipientOrganizationAdded, RecipientOrganizationRemoved, @@ -839,6 +846,8 @@ pub enum PackageError { Closure, #[error("the package integrity check failed")] Integrity, + #[error("the shared package envelope is invalid")] + Envelope, #[error("the package deployment binding is invalid")] Binding, /// One deployment binding differs from the runtime configuration. Only @@ -992,6 +1001,13 @@ impl PreparedPackage { &self.files } + /// The reviewed migration plan this candidate carries, validated again + /// from its captured files exactly as package loading validates it. + #[cfg(feature = "tooling")] + pub fn reviewed_migration_plan(&self) -> Result> { + rederive_reviewed_migration_plan(&self.manifest, &self.files, &self.registry) + } + pub fn envelope(&self, signatures: Vec) -> Result { validate_publication_signatures(&self.manifest, &signatures)?; Ok(PackageEnvelope { @@ -1008,6 +1024,19 @@ impl PreparedPackage { destination: &Path, signatures: Vec, ) -> Result<()> { + self.publish_to_directory_with_revision(destination, signatures, None) + .map(|_| ()) + } + + /// Publish the existing signed BReg package inside the shared Registry + /// Stack package envelope. BReg signatures and deployment bindings remain + /// authoritative until the package-ledger work removes them. + pub fn publish_to_directory_with_revision( + &self, + destination: &Path, + signatures: Vec, + revision: Option<&str>, + ) -> Result { reject_symlink_components(destination)?; if destination.exists() { return Err(PackageError::Closure); @@ -1042,7 +1071,21 @@ impl PreparedPackage { &destination.join(MANIFEST_PATH), &manifest_bytes, self.manifest.environment != "local", + )?; + let package = write_sum_file( + destination, + revision, + &shared_package_limits(), + "bregctl package", ) + .map_err(|_| PackageError::Closure)?; + if self.manifest.environment != "local" { + set_safe_file_permissions(&destination.join(SUM_FILE))?; + if revision.is_some() { + set_safe_file_permissions(&destination.join(REVISION_FILE))?; + } + } + Ok(package) })(); if publish.is_err() { let _ = remove_created_package_dir(destination); @@ -1051,6 +1094,16 @@ impl PreparedPackage { } } +pub(crate) fn shared_package_limits() -> SharedPackageLimits { + SharedPackageLimits { + max_files: MAX_PACKAGE_FILES + 2, + max_file_bytes: MAX_FILE_BYTES, + max_total_bytes: MAX_PACKAGE_BYTES, + max_depth: MAX_PATH_COMPONENTS, + max_path_bytes: MAX_PATH_BYTES, + } +} + /// Compare two compiled Registries by stable logical identifiers and return a /// value-free change set. The embedded migration plan is present only when /// every change is compiler-owned and can be applied without reviewed SQL. @@ -1560,6 +1613,22 @@ fn compare_fields( Some(field_id.as_str()), ), ); + } else if candidate_field + .field_type + .widens_length_limits_of(&previous_field.field_type) + { + // Every stored value is within the relaxed limit, so the change + // only replaces or drops the column's length check. + push_change( + changes, + CompiledRegistryChangeClass::CompatibleAdditive, + CompiledRegistryChangeCode::FieldLengthWidened, + target( + CompiledRegistryChangeTargetKind::Field, + Some(entity_id), + Some(field_id.as_str()), + ), + ); } else if previous_field.field_type != candidate_field.field_type { let code = match (&previous_field.field_type, &candidate_field.field_type) { ( @@ -1761,11 +1830,12 @@ fn compare_actions( // Every request the previous contract accepted keeps its meaning; // the action only accepts codes new to its vocabularies. Some(after) - if crate::immediate_actions::contract_only_adds_vocabulary_codes( + if crate::immediate_actions::contract_only_widens( (before, &previous.actions.input_vocabularies), &previous.entities, (after, &candidate.actions.input_vocabularies), &candidate.entities, + FieldTypeSource::keeps_vocabulary_codes_of, ) => { ( @@ -1773,6 +1843,22 @@ fn compare_actions( CompiledRegistryChangeCode::ActionVocabularyCodesAdded, ) } + // A target field also raised its `text` length limit, which every + // stored and requested value already meets. + Some(after) + if crate::immediate_actions::contract_only_widens( + (before, &previous.actions.input_vocabularies), + &previous.entities, + (after, &candidate.actions.input_vocabularies), + &candidate.entities, + FieldTypeSource::admits_every_value_of, + ) => + { + ( + CompiledRegistryChangeClass::CompatibleAdditive, + CompiledRegistryChangeCode::ActionTargetFieldsWidened, + ) + } Some(_) => ( CompiledRegistryChangeClass::AccessOrDisclosureChange, CompiledRegistryChangeCode::ActionChanged, @@ -2264,6 +2350,18 @@ fn additive_migration_plan( ), ); } + if field + .field_type + .widens_length_limits_of(&previous_field.field_type) + { + widened_checks.entry(entity_id.clone()).or_default().extend( + replace_length_check_statement( + candidate_entity, + &candidate.physical_names().entities[entity_id], + field, + ), + ); + } // Turning encryption on swaps the field's storage: the envelope // and blind-index columns arrive nullable, a unique lookup // index lands empty ahead of the reviewed backfill that fills @@ -3752,14 +3850,26 @@ pub fn derive_package_revision(manifest: &PackageManifest) -> Result { /// no network resolution and must complete before a database mutation or /// listener construction is attempted. pub fn load_package(root: &Path, context: &PackageLoadContext<'_>) -> Result { + let shared = verify_shared_package(root)?; + load_package_with_verified_envelope(root, context, &shared) +} + +/// Load the BReg package whose complete shared envelope was just verified. +pub fn load_package_with_verified_envelope( + root: &Path, + context: &PackageLoadContext<'_>, + shared: &SharedVerifiedPackage, +) -> Result { validate_root(root)?; let production = context.database_initialization_environment != "local"; if production { ensure_safe_permissions(root)?; } + bind_shared_envelope_files(root, shared, production)?; let manifest_path = root.join(MANIFEST_PATH); let manifest_bytes = read_bounded_regular(&manifest_path, MAX_MANIFEST_BYTES, production)?; + bind_shared_file(shared, MANIFEST_PATH, &manifest_bytes)?; let envelope: PackageEnvelope = parse_canonical(&manifest_bytes)?; if envelope.api_version != PACKAGE_API_VERSION || envelope.signed.files.is_empty() @@ -3789,6 +3899,7 @@ pub fn load_package(root: &Path, context: &PackageLoadContext<'_>) -> Result) -> Result Result { + registry_platform_config::package::verify_package( + root, + &shared_package_limits(), + "bregctl package", + ) + .map_err(|error| match error.kind() { + // A symbolic-link, missing, or special package root or entry is a + // path refusal, so operators get the path fix rather than a rebuild. + registry_platform_config::package::PackageErrorKind::RootInvalid { .. } + | registry_platform_config::package::PackageErrorKind::UnsafeEntry { .. } => { + PackageError::UnsafePath + } + _ => PackageError::Envelope, + }) +} + +fn bind_shared_file(shared: &SharedVerifiedPackage, relative: &str, bytes: &[u8]) -> Result<()> { + if shared.file_digest(relative).as_deref() != Some(digest(bytes).as_str()) { + return Err(PackageError::Envelope); + } + Ok(()) +} + +fn bind_shared_envelope_files( + root: &Path, + shared: &SharedVerifiedPackage, + production: bool, +) -> Result<()> { + let sums = read_bounded_regular(&root.join(SUM_FILE), MAX_MANIFEST_BYTES, production)?; + if digest(&sums) != shared.digest() { + return Err(PackageError::Envelope); + } + if shared.file_digest(REVISION_FILE).is_some() { + let revision = read_bounded_regular(&root.join(REVISION_FILE), 257, production)?; + bind_shared_file(shared, REVISION_FILE, &revision)?; + } + Ok(()) +} + /// Rederive a closed package for integrity-only comparison. /// /// Signatures are checked for structural consistency but are not treated as a @@ -3807,7 +3958,8 @@ pub fn load_package(root: &Path, context: &PackageLoadContext<'_>) -> Result Result { - inspect_package(root, None) + let shared = verify_shared_package(root)?; + inspect_package(root, None, &shared) } /// Verify the active predecessor package for read-only successor planning. @@ -3821,14 +3973,61 @@ pub fn load_predecessor_package( root: &Path, context: &PredecessorPackageContext<'_>, ) -> Result { + let shared = verify_shared_package(root)?; + load_predecessor_package_with_verified_envelope(root, context, &shared) +} + +/// Load an active predecessor from the same shared envelope verification used +/// to select it. +pub fn load_predecessor_package_with_verified_envelope( + root: &Path, + context: &PredecessorPackageContext<'_>, + shared: &SharedVerifiedPackage, +) -> Result { + load_predecessor_closure(root, context, shared).map(|(package, _)| package) +} + +/// Verify the active predecessor package exactly as +/// [`load_predecessor_package`] does, then compile its signed sources with the +/// current compiler so a successor can be rehearsed over the predecessor's +/// schema. The historical generated artifacts are still not compared: the +/// rehearsal instead holds the installed schema to the signed predecessor +/// fingerprint, and refuses when the current compiler cannot reproduce it. +#[cfg(feature = "tooling")] +pub fn load_predecessor_rehearsal_baseline( + root: &Path, + context: &PredecessorPackageContext<'_>, +) -> Result<(VerifiedPredecessorPackage, CompiledRegistry)> { + let shared = verify_shared_package(root)?; + load_predecessor_rehearsal_baseline_with_verified_envelope(root, context, &shared) +} + +#[cfg(feature = "tooling")] +pub fn load_predecessor_rehearsal_baseline_with_verified_envelope( + root: &Path, + context: &PredecessorPackageContext<'_>, + shared: &SharedVerifiedPackage, +) -> Result<(VerifiedPredecessorPackage, CompiledRegistry)> { + let (package, loaded) = load_predecessor_closure(root, context, shared)?; + let registry = compile_signed_sources(&package.manifest, &loaded)?; + Ok((package, registry)) +} + +fn load_predecessor_closure( + root: &Path, + context: &PredecessorPackageContext<'_>, + shared: &SharedVerifiedPackage, +) -> Result<(VerifiedPredecessorPackage, BTreeMap>)> { validate_root(root)?; let production = context.database_initialization_environment != "local"; if production { ensure_safe_permissions(root)?; } + bind_shared_envelope_files(root, shared, production)?; let manifest_path = root.join(MANIFEST_PATH); let manifest_bytes = read_bounded_regular(&manifest_path, MAX_MANIFEST_BYTES, production)?; + bind_shared_file(shared, MANIFEST_PATH, &manifest_bytes)?; let envelope: PackageEnvelope = parse_canonical(&manifest_bytes)?; if envelope.api_version != PACKAGE_API_VERSION || envelope.signed.files.is_empty() @@ -3860,6 +4059,7 @@ pub fn load_predecessor_package( &envelope.signed.files, manifest_bytes.len(), production, + shared, )?; validate_source_inventory(&envelope.signed)?; let governed = signed_predecessor_governed_model(&envelope.signed, &loaded)?; @@ -3869,11 +4069,14 @@ pub fn load_predecessor_package( let history_schema_descriptor = governed.history_schema_descriptor(&envelope.signed.package_revision)?; - Ok(VerifiedPredecessorPackage { - manifest: envelope.signed, - migration_baseline, - history_schema_descriptor, - }) + Ok(( + VerifiedPredecessorPackage { + manifest: envelope.signed, + migration_baseline, + history_schema_descriptor, + }, + loaded, + )) } /// Rederive a closed package and verify its configured deployment bindings and @@ -3882,18 +4085,32 @@ pub fn inspect_package_with_context( root: &Path, context: &PackageInspectionContext<'_>, ) -> Result { - inspect_package(root, Some(context)) + let shared = verify_shared_package(root)?; + inspect_package_with_context_and_verified_envelope(root, context, &shared) +} + +/// Inspect a runtime-selected BReg package through the retained shared +/// envelope verification that selected it. +pub fn inspect_package_with_context_and_verified_envelope( + root: &Path, + context: &PackageInspectionContext<'_>, + shared: &SharedVerifiedPackage, +) -> Result { + inspect_package(root, Some(context), shared) } fn inspect_package( root: &Path, context: Option<&PackageInspectionContext<'_>>, + shared: &SharedVerifiedPackage, ) -> Result { validate_root(root)?; ensure_safe_permissions(root)?; + bind_shared_envelope_files(root, shared, true)?; let manifest_path = root.join(MANIFEST_PATH); let manifest_bytes = read_bounded_regular(&manifest_path, MAX_MANIFEST_BYTES, true)?; + bind_shared_file(shared, MANIFEST_PATH, &manifest_bytes)?; let envelope: PackageEnvelope = parse_canonical(&manifest_bytes)?; if envelope.api_version != PACKAGE_API_VERSION || envelope.signed.files.is_empty() @@ -3915,7 +4132,13 @@ fn inspect_package( } None => validate_publication_signatures(&envelope.signed, &envelope.signatures)?, } - let loaded = load_closure(root, &envelope.signed.files, manifest_bytes.len(), true)?; + let loaded = load_closure( + root, + &envelope.signed.files, + manifest_bytes.len(), + true, + shared, + )?; let (registry, _reviewed_migration_plan) = rederive(&envelope.signed, &loaded)?; #[cfg(feature = "tooling")] let migration = @@ -4781,6 +5004,7 @@ fn load_closure( entries: &[PackageFile], manifest_size: usize, production: bool, + shared: &SharedVerifiedPackage, ) -> Result>> { let mut listed = BTreeSet::new(); let mut loaded = BTreeMap::new(); @@ -4800,6 +5024,7 @@ fn load_closure( reject_relative_symlinks(root, relative)?; let path = root.join(relative); let bytes = read_bounded_regular(&path, MAX_FILE_BYTES, production)?; + bind_shared_file(shared, &entry.path, &bytes)?; if bytes.len() as u64 != entry.size || digest(&bytes) != entry.sha256 { return Err(PackageError::Integrity); } @@ -4815,16 +5040,26 @@ fn load_closure( .map(str::to_owned) .collect::>(); expected.insert(MANIFEST_PATH.to_owned()); + let shared_files = shared.files().map(str::to_owned).collect::>(); + if shared_files.contains(REVISION_FILE) { + expected.insert(REVISION_FILE.to_owned()); + } + if shared_files != expected { + return Err(PackageError::Envelope); + } + expected.insert(SUM_FILE.to_owned()); if actual != expected { return Err(PackageError::Closure); } Ok(loaded) } -fn rederive( +/// Compile the signed sources of one verified package closure. The caller +/// decides whether generated artifacts must also match byte for byte. +fn compile_signed_sources( manifest: &PackageManifest, loaded: &BTreeMap>, -) -> Result<(CompiledRegistry, Option)> { +) -> Result { validate_source_inventory(manifest)?; let fixture_journeys = loaded .get(&manifest.sources.fixture_journeys) @@ -4887,7 +5122,14 @@ fn rederive( if compiled.registry_id() != manifest.package_id { return Err(PackageError::Derivation); } + Ok(compiled) +} +fn rederive( + manifest: &PackageManifest, + loaded: &BTreeMap>, +) -> Result<(CompiledRegistry, Option)> { + let compiled = compile_signed_sources(manifest, loaded)?; let expected_artifacts = expected_artifact_bytes(manifest, &compiled)?; let packaged_artifacts = manifest .files diff --git a/crates/registry-breg/src/postgres/catalog.rs b/crates/registry-breg/src/postgres/catalog.rs index f722bdee67..679d98c2df 100644 --- a/crates/registry-breg/src/postgres/catalog.rs +++ b/crates/registry-breg/src/postgres/catalog.rs @@ -258,6 +258,39 @@ impl ExpectedManagedCatalog { Some((false, false)), ); } + for (table, privileges) in crate::instance_claim::INSTANCE_CLAIM_TABLES { + catalog.table( + &format!("registry_internal.{table}"), + privileges.iter().copied(), + std::iter::empty::<&str>(), + Some((false, false)), + ); + } + for (table, privileges) in crate::import_authority::IMPORT_AUTHORITY_TABLES { + let name = format!("registry_internal.{table}"); + catalog.table( + &name, + privileges.iter().copied(), + std::iter::empty::<&str>(), + Some((true, false)), + ); + for column in crate::import_authority::RUNTIME_UPDATE_COLUMNS { + catalog.column_privilege(&name, column, "runtime", "UPDATE"); + } + for (policy, command, has_check) in [ + (crate::import_authority::READ_POLICY, "r", false), + (crate::import_authority::ADVANCE_POLICY, "w", true), + ] { + catalog.policies.insert(ManagedPolicy { + table: name.clone(), + name: policy.to_owned(), + command: command.to_owned(), + role: ManagedPolicyRole::Runtime, + has_using: true, + has_check, + }); + } + } for (table, privileges) in crate::attachment_store::ATTACHMENT_TABLES { catalog.table( &format!("registry_internal.{table}"), diff --git a/crates/registry-breg/src/postgres/failure.rs b/crates/registry-breg/src/postgres/failure.rs new file mode 100644 index 0000000000..44c7b982c7 --- /dev/null +++ b/crates/registry-breg/src/postgres/failure.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! The value-free part of a PostgreSQL error, shared by the migration +//! rehearsal and `apply`. + +use std::fmt; + +use tokio_postgres::error::DbError; + +/// The value-free part of one PostgreSQL error: its SQLSTATE and the schema +/// objects the server named. The message, detail, hint, and statement text +/// can carry row values and are never retained. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct PostgresFailure { + pub sqlstate: Option, + pub table: Option, + pub column: Option, + pub constraint: Option, +} + +impl PostgresFailure { + /// Retain only the SQLSTATE and object names of one database error. + #[must_use] + pub fn from_error(error: &tokio_postgres::Error) -> Self { + error + .as_db_error() + .map(Self::from_db_error) + .unwrap_or_default() + } + + fn from_db_error(error: &DbError) -> Self { + Self { + sqlstate: Some(error.code().code().to_owned()), + table: error.table().map(str::to_owned), + column: error.column().map(str::to_owned), + constraint: error.constraint().map(str::to_owned), + } + } + + /// The SQLSTATE class name, from the first two characters of the code. + #[must_use] + pub fn class_name(&self) -> Option<&'static str> { + self.sqlstate + .as_deref() + .and_then(|code| code.get(..2)) + .map(sqlstate_class_name) + } +} + +impl fmt::Display for PostgresFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let Some(sqlstate) = &self.sqlstate else { + return formatter.write_str("the database refused the statement"); + }; + write!(formatter, "SQLSTATE {sqlstate}")?; + if let Some(class) = self.class_name() { + write!(formatter, " ({class})")?; + } + for (label, value) in [ + ("table", &self.table), + ("column", &self.column), + ("constraint", &self.constraint), + ] { + if let Some(value) = value { + write!(formatter, ", {label} {value}")?; + } + } + Ok(()) + } +} + +/// PostgreSQL's documented SQLSTATE class names (Appendix A). +fn sqlstate_class_name(class: &str) -> &'static str { + match class { + "08" => "connection exception", + "0A" => "feature not supported", + "21" => "cardinality violation", + "22" => "data exception", + "23" => "integrity constraint violation", + "25" => "invalid transaction state", + "28" => "invalid authorization specification", + "2B" => "dependent privilege descriptors still exist", + "40" => "transaction rollback", + "42" => "syntax error or access rule violation", + "44" => "WITH CHECK OPTION violation", + "53" => "insufficient resources", + "54" => "program limit exceeded", + "55" => "object not in prerequisite state", + "57" => "operator intervention", + "58" => "system error", + "P0" => "PL/pgSQL error", + "XX" => "internal error", + _ => "other PostgreSQL error class", + } +} diff --git a/crates/registry-breg/src/postgres/interlock.rs b/crates/registry-breg/src/postgres/interlock.rs index 53d4b8712b..d2aa207715 100644 --- a/crates/registry-breg/src/postgres/interlock.rs +++ b/crates/registry-breg/src/postgres/interlock.rs @@ -17,8 +17,8 @@ use crate::generated_ddl::DdlStatementKind; use crate::history_commit::{install_empty_history_baseline, install_history_commit_schema}; use crate::history_migration::{ ensure_successor_history_ready as ensure_successor_history_ready_state, - finish_bounded_history_update, finish_field_encryption_page_update, - prepare_bounded_history_update, prepare_field_encryption_page_capture, + finish_bounded_history_update, finish_reviewed_page_update, prepare_bounded_history_update, + prepare_reviewed_page_capture, }; use crate::history_schema::HistorySchemaDescriptor; use crate::history_store::{install_history_schema_store, retain_descriptor}; @@ -175,6 +175,9 @@ struct FieldEncryptionChunkRequest<'a> { } struct ReviewedChunkExecutionRequest<'a> { + registry: &'a CompiledRegistry, + target_package_revision: &'a str, + descriptor_path: &'a str, step: &'a ValidatedReviewedMigrationStep, ledger: &'a MigrationLedgerEntry, ledger_step: &'a MigrationLedgerStep, @@ -326,9 +329,9 @@ impl DedicatedApplyConnection { ) .await?; for statement in statements { - if transaction.batch_execute(statement).await.is_err() { + if let Err(error) = transaction.batch_execute(statement).await { transaction.rollback().await?; - return Err(PostgresKernelError::Connection); + return Err(PostgresKernelError::from_statement_error(&error)); } } transaction.commit().await?; @@ -472,6 +475,9 @@ impl DedicatedApplyConnection { loop { let advanced = self .execute_reviewed_chunk(ReviewedChunkExecutionRequest { + registry, + target_package_revision, + descriptor_path: &migration.descriptor_path, step, ledger, ledger_step, @@ -704,42 +710,9 @@ impl DedicatedApplyConnection { lock_timeout: Duration, statement_timeout: Duration, ) -> Result<()> { - // These two schemas are a closed compiler-owned boundary. Reviewed - // column changes may require their dependent views to be removed - // first; exact package DDL recreates every candidate view afterward. let transaction = self.client.transaction().await?; set_local_duration_timeouts(&transaction, lock_timeout, statement_timeout).await?; - let rows = transaction - .query( - "SELECT schemaname, viewname - FROM pg_catalog.pg_views - WHERE schemaname IN ('registry_derived', 'registry_source') - ORDER BY CASE schemaname WHEN 'registry_derived' THEN 0 ELSE 1 END, - viewname", - &[], - ) - .await?; - for row in rows { - let schema = row - .try_get::<_, String>(0) - .map_err(|_| PostgresKernelError::RegistryUnavailable)?; - let view = row - .try_get::<_, String>(1) - .map_err(|_| PostgresKernelError::RegistryUnavailable)?; - if !matches!(schema.as_str(), "registry_derived" | "registry_source") { - return Err(PostgresKernelError::RegistryUnavailable); - } - let schema = SqlIdentifier::parse(&schema)?; - let view = SqlIdentifier::parse(&view)?; - transaction - .batch_execute(&format!( - "DROP VIEW {}.{} RESTRICT", - schema.quoted(), - view.quoted() - )) - .await - .map_err(|_| PostgresKernelError::Connection)?; - } + drop_managed_read_view_set(&transaction).await?; transaction.commit().await?; Ok(()) } @@ -835,6 +808,9 @@ impl DedicatedApplyConnection { request: ReviewedChunkExecutionRequest<'_>, ) -> Result { let ReviewedChunkExecutionRequest { + registry, + target_package_revision, + descriptor_path, step, ledger, ledger_step, @@ -896,14 +872,24 @@ impl DedicatedApplyConnection { .checked_add(selected) .filter(|total| *total <= max_total_rows) .ok_or(PostgresKernelError::RegistryUnavailable)?; + // Each chunk journals the rows it changed as one history commit in + // the chunk's own transaction, so a resumed backfill never journals a + // committed chunk twice. + let capture = + prepare_reviewed_page_capture(&transaction, registry, descriptor_path, step, &ids) + .await + .map_err(|_| PostgresKernelError::RegistryUnavailable)?; let affected = transaction .execute(&step.sql, &[&ids]) .await - .map_err(|_| PostgresKernelError::Connection)?; - set_force_row_security(&transaction, &[table.as_str().to_owned()], true).await?; + .map_err(|error| map_reviewed_pattern_error(error, registry, step))?; if affected != selected { return Err(PostgresKernelError::RegistryUnavailable); } + finish_reviewed_page_update(&transaction, registry, target_package_revision, capture) + .await + .map_err(|_| PostgresKernelError::RegistryUnavailable)?; + set_force_row_security(&transaction, &[table.as_str().to_owned()], true).await?; let checkpoint = ids .last() .copied() @@ -1275,15 +1261,10 @@ impl DedicatedApplyConnection { .filter(|total| *total <= max_total_rows) .ok_or(PostgresKernelError::RegistryUnavailable)?; - let capture = prepare_field_encryption_page_capture( - &transaction, - registry, - descriptor_path, - step, - &ids, - ) - .await - .map_err(|_| PostgresKernelError::RegistryUnavailable)?; + let capture = + prepare_reviewed_page_capture(&transaction, registry, descriptor_path, step, &ids) + .await + .map_err(|_| PostgresKernelError::RegistryUnavailable)?; let update_sql = field_encryption_update_statement(&table, covered); for (row_index, record_id) in ids.iter().enumerate() { @@ -1337,14 +1318,9 @@ impl DedicatedApplyConnection { } } - finish_field_encryption_page_update( - &transaction, - registry, - target_package_revision, - capture, - ) - .await - .map_err(|_| PostgresKernelError::RegistryUnavailable)?; + finish_reviewed_page_update(&transaction, registry, target_package_revision, capture) + .await + .map_err(|_| PostgresKernelError::RegistryUnavailable)?; set_force_row_security(&transaction, &[table.as_str().to_owned()], true).await?; let checkpoint = ids .last() @@ -2368,7 +2344,48 @@ async fn verify_retained_webhook_delivery_bindings( Ok(()) } -fn compiler_statement_runs_after_reviewed_steps(statement: &PackageDdlStatement<'_>) -> bool { +/// Drop every view in the two compiler-owned read schemas. +pub(super) async fn drop_managed_read_view_set(client: &impl GenericClient) -> Result<()> { + // These two schemas are a closed compiler-owned boundary. Reviewed + // column changes may require their dependent views to be removed + // first; exact package DDL recreates every candidate view afterward. + let rows = client + .query( + "SELECT schemaname, viewname + FROM pg_catalog.pg_views + WHERE schemaname IN ('registry_derived', 'registry_source') + ORDER BY CASE schemaname WHEN 'registry_derived' THEN 0 ELSE 1 END, + viewname", + &[], + ) + .await?; + for row in rows { + let schema = row + .try_get::<_, String>(0) + .map_err(|_| PostgresKernelError::RegistryUnavailable)?; + let view = row + .try_get::<_, String>(1) + .map_err(|_| PostgresKernelError::RegistryUnavailable)?; + if !matches!(schema.as_str(), "registry_derived" | "registry_source") { + return Err(PostgresKernelError::RegistryUnavailable); + } + let schema = SqlIdentifier::parse(&schema)?; + let view = SqlIdentifier::parse(&view)?; + client + .batch_execute(&format!( + "DROP VIEW {}.{} RESTRICT", + schema.quoted(), + view.quoted() + )) + .await + .map_err(|_| PostgresKernelError::Connection)?; + } + Ok(()) +} + +pub(super) fn compiler_statement_runs_after_reviewed_steps( + statement: &PackageDdlStatement<'_>, +) -> bool { if is_spatial_candidate_view_drop_sql(statement.sql) { return false; } @@ -3678,7 +3695,14 @@ mod tests { Duration::from_millis(20), ) .await; - assert!(matches!(timed_out, Err(PostgresKernelError::Connection))); + assert!( + matches!( + &timed_out, + Err(PostgresKernelError::Statement(failure)) + if failure.sqlstate.as_deref() == Some("57014") + ), + "a statement timeout is a refused statement, as on the package DDL path: {timed_out:?}" + ); assert_eq!(database.state_snapshot().await, initial_state); apply .resume_failed(¤t, target_revision) diff --git a/crates/registry-breg/src/postgres/mod.rs b/crates/registry-breg/src/postgres/mod.rs index ada20e9389..3b265717d8 100644 --- a/crates/registry-breg/src/postgres/mod.rs +++ b/crates/registry-breg/src/postgres/mod.rs @@ -6,11 +6,14 @@ mod baseline; mod catalog; mod config; mod context; +mod failure; mod history_read; mod interlock; mod migration_ledger; mod mutation; mod read; +#[cfg(all(feature = "runtime", feature = "tooling"))] +mod rehearsal; mod revision_read; mod roles; mod schema; @@ -42,6 +45,7 @@ pub use context::{ RowBoundaryOperator, }; pub(crate) use context::{install_spatial_bbox_context, validate_field_value, SpatialBboxContext}; +pub use failure::PostgresFailure; pub use history_read::PostgresSnapshotReadService; #[cfg(feature = "postgres-test")] pub use history_read::SnapshotReadFaultPoint; @@ -67,6 +71,11 @@ pub use mutation::{ pub use read::PostgresRecordReadService; #[cfg(feature = "postgres-test")] pub use read::ReadFaultPoint; +#[cfg(all(feature = "runtime", feature = "tooling"))] +pub use rehearsal::{ + rehearse_successor_migration, MigrationRehearsalError, RehearsalAssertionPhase, + SuccessorMigrationRehearsal, +}; pub use revision_read::PostgresRevisionReadService; #[cfg(feature = "postgres-test")] pub use revision_read::RevisionReadFaultPoint; @@ -216,6 +225,37 @@ pub enum PostgresKernelError { /// caller has not acknowledged discarding. #[error("a retired audit table still carries unacknowledged rows")] RetiredAuditRowsPresent, + /// PostgreSQL refused a migration statement. Only the SQLSTATE and the + /// object names the server reported are retained. + #[error("PostgreSQL refused a migration statement: {0}")] + Statement(PostgresFailure), +} + +impl PostgresKernelError { + /// Classifies the error of one migration statement. A server refusal + /// keeps its SQLSTATE and object names; a lost connection, whether the + /// client lost it or the server ended the session, stays + /// [`Self::Connection`]. + pub(crate) fn from_statement_error(error: &tokio_postgres::Error) -> Self { + match error.as_db_error() { + Some(db_error) if !sqlstate_ends_the_session(db_error.code().code()) => { + Self::Statement(PostgresFailure::from_error(error)) + } + _ => Self::Connection, + } + } +} + +/// Whether a server-reported SQLSTATE ends the session rather than refusing +/// one statement: a connection exception (class `08`), an administrator or +/// crash shutdown, a server not accepting connections, a dropped database, +/// or an idle-session or idle-in-transaction timeout. +fn sqlstate_ends_the_session(code: &str) -> bool { + code.starts_with("08") + || matches!( + code, + "57P01" | "57P02" | "57P03" | "57P04" | "57P05" | "25P03" + ) } impl From for PostgresKernelError { @@ -226,3 +266,25 @@ impl From for PostgresKernelError { /// Result returned by PostgreSQL kernel operations. pub type Result = std::result::Result; + +#[cfg(test)] +mod tests { + use super::sqlstate_ends_the_session; + + #[test] + fn a_server_ended_session_is_a_connection_failure_not_a_refused_statement() { + for code in [ + "08000", "08003", "08006", "57P01", "57P02", "57P03", "57P04", "57P05", "25P03", + ] { + assert!(sqlstate_ends_the_session(code), "{code} ends the session"); + } + for code in [ + "57014", "55P03", "22P02", "23502", "42703", "25P02", "40001", + ] { + assert!( + !sqlstate_ends_the_session(code), + "{code} refuses one statement" + ); + } + } +} diff --git a/crates/registry-breg/src/postgres/mutation.rs b/crates/registry-breg/src/postgres/mutation.rs index bf2903924f..71cc62f6e0 100644 --- a/crates/registry-breg/src/postgres/mutation.rs +++ b/crates/registry-breg/src/postgres/mutation.rs @@ -1208,11 +1208,12 @@ impl PostgresRecordMutationService { if input.chunk_algorithm_version != crate::data::RUN_CHUNK_ALGORITHM_VERSION { return Err(IngestionServiceError::RequestInvalid); } - if crate::data::ingestion_batch_route(&self.registry, &input.entity_id, &input.profile_id) - .is_none() - { + let Some(route) = + crate::data::ingestion_route(&self.registry, &input.entity_id, &input.profile_id) + else { return Err(IngestionServiceError::RequestInvalid); - } + }; + let through_import = route.operation == crate::contract::Operation::Import; let entity = self .registry .entities() @@ -1254,6 +1255,7 @@ impl PostgresRecordMutationService { chunk_algorithm_version: input.chunk_algorithm_version, maximum_items: i64::from(batch.maximum_items), maximum_bytes: i64::from(batch.maximum_bytes), + import_authority_id: None, }; ingestion_store::validate_new_run(&run) .map_err(|_| IngestionServiceError::RequestInvalid)?; @@ -1293,6 +1295,39 @@ impl PostgresRecordMutationService { .await .map_err(|_| IngestionServiceError::Unavailable)?; let tx: &tokio_postgres::Transaction<'_> = transaction.transaction(); + let mut run = run; + let mut authority_records = Vec::new(); + if through_import { + // An `import` run needs an open authority for its entity and + // profile that admits its whole volume and pins its input. The + // check collects any expiry or supersession it observes, so the + // refusal commits those transitions and appends their records + // before it answers. + run.import_authority_id = crate::import_authority::admit_run( + tx, + &mut authority_records, + &self.expected.package_revision, + &run.entity_id, + &run.profile_id, + run.item_count, + &run.input_digest, + ) + .await + .map_err(|_| IngestionServiceError::Unavailable)?; + if run.import_authority_id.is_none() { + // The run is refused whether or not the observed transitions + // commit: no run exists either way. Their records are + // appended only once they did commit; a transition whose + // commit failed is observed and recorded again by the next + // transaction that reads the authority. + if transaction.commit().await.is_ok() { + crate::import_authority::append_transitions(&self.audit, authority_records) + .await + .map_err(|_| IngestionServiceError::Unavailable)?; + } + return Err(IngestionServiceError::PreconditionFailed); + } + } let record = ingestion_store::insert_run(tx, &run) .await .map_err(|_| IngestionServiceError::Unavailable)?; @@ -1311,6 +1346,9 @@ impl PostgresRecordMutationService { // The run exists once the transaction commits; its answer leaves only // after the audit entry is accepted. self.fail_before_run_response()?; + crate::import_authority::append_transitions(&self.audit, authority_records) + .await + .map_err(|_| IngestionServiceError::Unavailable)?; ingestion_store::append_run_audit(&self.audit, audit_record) .await .map_err(|_| IngestionServiceError::Unavailable)?; @@ -1935,9 +1973,8 @@ impl PostgresRecordMutationService { &input.digest, ) .map_err(|_| IngestionServiceError::RequestInvalid)?; - let route = - crate::data::ingestion_batch_route(&self.registry, &run.entity_id, &run.profile_id) - .ok_or(IngestionServiceError::Unavailable)?; + let route = crate::data::ingestion_route(&self.registry, &run.entity_id, &run.profile_id) + .ok_or(IngestionServiceError::Unavailable)?; let plan = MutationPlan::from_compiled(&self.registry, &route.id) .map_err(|_| IngestionServiceError::RequestInvalid)?; let response_fields = plan_readable_fields(&self.registry, &run.entity_id, &run.profile_id) @@ -2094,6 +2131,10 @@ impl PostgresRecordMutationService { IngestionAttemptOutcome::BindingChanged, Some(IngestionServiceError::RunBlocked), ), + crate::mutation::IngestionRefusal::AuthorityClosed => ( + IngestionAttemptOutcome::ImportAuthorityClosed, + Some(IngestionServiceError::RunBlocked), + ), crate::mutation::IngestionRefusal::ReceiptErased => ( IngestionAttemptOutcome::Replayed, Some(IngestionServiceError::ReceiptErased), diff --git a/crates/registry-breg/src/postgres/rehearsal.rs b/crates/registry-breg/src/postgres/rehearsal.rs new file mode 100644 index 0000000000..ae1014664b --- /dev/null +++ b/crates/registry-breg/src/postgres/rehearsal.rs @@ -0,0 +1,485 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Rolled-back rehearsal of a successor package's migration. +//! +//! `bregctl test` installs the verified predecessor schema into an empty +//! schema-test database, runs the candidate's compiler DDL and reviewed steps +//! in the order activation runs them, and measures the result against the +//! candidate's fresh-install fingerprint. Everything happens inside one +//! transaction that is always rolled back, so the database stays clean for the +//! schema test that follows. The predecessor tables are empty, so the +//! rehearsal proves statement validity, ordering, and the final catalog; it +//! does not prove the data-dependent behavior of a step over live rows. + +use std::fmt; + +use tokio_postgres::{types::Type, GenericClient}; +use uuid::Uuid; + +use crate::generated_ddl::DdlStatementKind; +use crate::history_migration::check_reviewed_history_step; +use crate::migration_plan::{ReviewedMigrationStepDescriptor, ValidatedReviewedMigrationPlan}; +use crate::model::CompiledRegistry; +use crate::mutation::install_mutation_schema; +use crate::package::PreparedPackage; + +use super::{ + catalog::{managed_schema_fingerprint, ExpectedManagedCatalog}, + failure::PostgresFailure, + interlock::{ + compiler_statement_runs_after_reviewed_steps, drop_managed_read_view_set, + set_force_row_security, PackageDdlStatement, + }, + migration_ledger::statement_checksum, + schema::{ + compiled_pattern_field, connect_schema_test, execute_compiled_ddl_statement, + install_compiled_schema, is_spatial_candidate_view_sql, pattern_field_for_constraint, + reconcile_compiled_runtime_acl, refuse_existing_managed_objects, + }, + verify_migration_role, ConnectionConfig, SqlIdentifier, +}; + +/// The verified predecessor and the prepared candidate one rehearsal binds. +pub struct SuccessorMigrationRehearsal<'a> { + /// The predecessor registry compiled from its signed sources. + pub predecessor: &'a CompiledRegistry, + /// The schema fingerprint the signed predecessor manifest binds. + pub predecessor_schema_fingerprint: &'a str, + /// The prepared successor candidate, before any signature. + pub candidate: &'a PreparedPackage, +} + +/// Which assertion set of a reviewed migration a refusal names. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RehearsalAssertionPhase { + Pre, + Post, +} + +impl fmt::Display for RehearsalAssertionPhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Pre => "pre", + Self::Post => "post", + }) + } +} + +/// Why a rehearsal refused a successor. Every variant is value-free: it names +/// reviewed identifiers, compiler statement identifiers, SQLSTATE codes, and +/// schema objects only. +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] +pub enum MigrationRehearsalError { + #[error( + "the schema-test database is unavailable, not clean, or not owned by the migration role" + )] + Database, + #[error("the candidate package does not carry a successor migration plan")] + NotSuccessor, + #[error("the candidate package's reviewed migration plan could not be rederived")] + ReviewedPlan, + #[error("the current compiler does not reproduce the verified predecessor schema fingerprint")] + BaselineNotReproducible, + #[error("compiler statement {statement_id} failed: {failure}")] + CompilerStatement { + statement_id: String, + failure: PostgresFailure, + }, + #[error( + "reviewed migration {migration_id} {phase}-assertion {assertion_id} failed: {failure}" + )] + Assertion { + migration_id: String, + phase: RehearsalAssertionPhase, + assertion_id: String, + failure: PostgresFailure, + }, + #[error( + "reviewed migration {migration_id} {phase}-assertion {assertion_id} does not return exactly one boolean column" + )] + AssertionShape { + migration_id: String, + phase: RehearsalAssertionPhase, + assertion_id: String, + }, + #[error("reviewed migration {migration_id} step {step_id} failed: {failure}")] + Step { + migration_id: String, + step_id: String, + failure: PostgresFailure, + }, + #[error("reviewed migration {migration_id} step {step_id} cannot be journaled: {reason}")] + HistoryStep { + migration_id: String, + step_id: String, + reason: String, + }, + #[error( + "the rehearsed migration does not reach the candidate schema fingerprint; activation would refuse it" + )] + FinalSchemaMismatch, +} + +type RehearsalResult = std::result::Result; + +/// Rehearse one successor candidate over an empty reproduction of its verified +/// predecessor schema. The transaction is rolled back on every path. +pub async fn rehearse_successor_migration( + migration_connection: &ConnectionConfig, + migration_role: &SqlIdentifier, + runtime_role: &SqlIdentifier, + rehearsal: SuccessorMigrationRehearsal<'_>, +) -> RehearsalResult<()> { + let candidate = rehearsal.candidate; + let manifest = candidate.manifest(); + if manifest.prior_revision.is_none() || manifest.migration_plan.prior_baseline.is_none() { + return Err(MigrationRehearsalError::NotSuccessor); + } + let plan = candidate + .reviewed_migration_plan() + .map_err(|_| MigrationRehearsalError::ReviewedPlan)?; + let checksums = manifest + .migration_plan + .statements + .iter() + .map(|statement| statement_checksum(&statement.sql)) + .collect::>(); + let statements = manifest + .migration_plan + .statements + .iter() + .zip(&checksums) + .enumerate() + .map(|(ordinal, (statement, checksum))| { + Some(RehearsedStatement { + id: &statement.id, + ddl: PackageDdlStatement { + sql: &statement.sql, + checksum, + kind: statement.kind, + pattern_field: compiled_pattern_field(candidate.registry(), &statement.id), + ordinal: i32::try_from(ordinal).ok()?, + }, + }) + }) + .collect::>>() + .ok_or(MigrationRehearsalError::NotSuccessor)?; + + let (mut client, task) = connect_schema_test(migration_connection) + .await + .map_err(|_| MigrationRehearsalError::Database)?; + let result = async { + verify_migration_role(&client, migration_role) + .await + .map_err(|_| MigrationRehearsalError::Database)?; + let transaction = client + .transaction() + .await + .map_err(|_| MigrationRehearsalError::Database)?; + let outcome = rehearse_in_transaction( + &transaction, + runtime_role, + &rehearsal, + plan.as_ref(), + &statements, + ) + .await; + // The rehearsal never commits; a failed statement has already aborted + // the transaction, and rollback discards it either way. + let _ = transaction.rollback().await; + outcome + } + .await; + task.abort(); + result +} + +struct RehearsedStatement<'a> { + id: &'a str, + ddl: PackageDdlStatement<'a>, +} + +async fn rehearse_in_transaction( + transaction: &impl GenericClient, + runtime_role: &SqlIdentifier, + rehearsal: &SuccessorMigrationRehearsal<'_>, + plan: Option<&ValidatedReviewedMigrationPlan>, + statements: &[RehearsedStatement<'_>], +) -> RehearsalResult<()> { + transaction + .batch_execute("SET LOCAL lock_timeout = '5s'; SET LOCAL statement_timeout = '300s'") + .await + .map_err(|_| MigrationRehearsalError::Database)?; + refuse_existing_managed_objects(transaction) + .await + .map_err(|_| MigrationRehearsalError::Database)?; + install_compiled_schema(transaction, rehearsal.predecessor, runtime_role) + .await + .map_err(|_| MigrationRehearsalError::BaselineNotReproducible)?; + let predecessor_fingerprint = managed_schema_fingerprint( + transaction, + runtime_role, + &ExpectedManagedCatalog::compiled(rehearsal.predecessor), + ) + .await + .map_err(|_| MigrationRehearsalError::BaselineNotReproducible)?; + if predecessor_fingerprint != rehearsal.predecessor_schema_fingerprint { + return Err(MigrationRehearsalError::BaselineNotReproducible); + } + + let candidate = rehearsal.candidate.registry(); + match plan { + Some(plan) => { + let prior_tables = entity_tables(rehearsal.predecessor); + let candidate_tables = entity_tables(candidate); + rehearse_assertions( + transaction, + plan, + &prior_tables, + RehearsalAssertionPhase::Pre, + ) + .await?; + for statement in statements + .iter() + .filter(|statement| !compiler_statement_runs_after_reviewed_steps(&statement.ddl)) + { + rehearse_compiler_statement(transaction, statement, runtime_role).await?; + } + if statements.iter().any(|statement| { + statement.ddl.kind == DdlStatementKind::View + && !is_spatial_candidate_view_sql(statement.ddl.sql) + }) { + drop_managed_read_view_set(transaction) + .await + .map_err(|_| MigrationRehearsalError::Database)?; + } + rehearse_reviewed_steps(transaction, candidate, plan).await?; + for statement in statements + .iter() + .filter(|statement| compiler_statement_runs_after_reviewed_steps(&statement.ddl)) + { + rehearse_compiler_statement(transaction, statement, runtime_role).await?; + } + rehearse_assertions( + transaction, + plan, + &candidate_tables, + RehearsalAssertionPhase::Post, + ) + .await?; + } + None => { + for statement in statements { + rehearse_compiler_statement(transaction, statement, runtime_role).await?; + } + } + } + + install_mutation_schema( + transaction, + runtime_role, + // The rehearsal database is disposable and was installed from the + // prior package by this build, so a retired pre-simplification audit + // table can never be present here to discard. + false, + ) + .await + .map_err(|_| MigrationRehearsalError::Database)?; + reconcile_compiled_runtime_acl(transaction, candidate, runtime_role) + .await + .map_err(|_| MigrationRehearsalError::Database)?; + let measured = managed_schema_fingerprint( + transaction, + runtime_role, + &ExpectedManagedCatalog::compiled(candidate), + ) + .await + .map_err(|_| MigrationRehearsalError::FinalSchemaMismatch)?; + if measured != rehearsal.candidate.manifest().schema_fingerprint { + return Err(MigrationRehearsalError::FinalSchemaMismatch); + } + Ok(()) +} + +fn entity_tables(registry: &CompiledRegistry) -> Vec { + registry + .entities() + .values() + .map(|entity| entity.physical_table.clone()) + .collect() +} + +async fn rehearse_compiler_statement( + transaction: &impl GenericClient, + statement: &RehearsedStatement<'_>, + runtime_role: &SqlIdentifier, +) -> RehearsalResult<()> { + let failed = |failure| MigrationRehearsalError::CompilerStatement { + statement_id: statement.id.to_owned(), + failure, + }; + if statement.ddl.kind == DdlStatementKind::View + && is_spatial_candidate_view_sql(statement.ddl.sql) + { + return execute_compiled_ddl_statement( + transaction, + statement.ddl.sql, + statement.ddl.kind, + None, + runtime_role, + ) + .await + .map_err(|_| failed(PostgresFailure::default())); + } + transaction + .batch_execute(statement.ddl.sql) + .await + .map_err(|error| failed(PostgresFailure::from_error(&error))) +} + +async fn rehearse_assertions( + transaction: &impl GenericClient, + plan: &ValidatedReviewedMigrationPlan, + tables: &[String], + phase: RehearsalAssertionPhase, +) -> RehearsalResult<()> { + set_force_row_security(transaction, tables, false) + .await + .map_err(|_| MigrationRehearsalError::Database)?; + for migration in plan.migrations() { + let assertions = match phase { + RehearsalAssertionPhase::Pre => &migration.pre_assertions, + RehearsalAssertionPhase::Post => &migration.post_assertions, + }; + for assertion in assertions { + let named = |failure: Option| match failure { + Some(failure) => MigrationRehearsalError::Assertion { + migration_id: migration.descriptor.id.clone(), + phase, + assertion_id: assertion.descriptor.id.clone(), + failure, + }, + None => MigrationRehearsalError::AssertionShape { + migration_id: migration.descriptor.id.clone(), + phase, + assertion_id: assertion.descriptor.id.clone(), + }, + }; + // Activation also requires the value to be true. Over the empty + // predecessor tables a truthful assertion about live rows may be + // false, so the rehearsal holds only the shape and executability. + let prepared = transaction + .prepare(&assertion.sql) + .await + .map_err(|error| named(Some(PostgresFailure::from_error(&error))))?; + if prepared.columns().len() != 1 || prepared.columns()[0].type_() != &Type::BOOL { + return Err(named(None)); + } + transaction + .query(&prepared, &[]) + .await + .map_err(|error| named(Some(PostgresFailure::from_error(&error))))?; + } + } + set_force_row_security(transaction, tables, true) + .await + .map_err(|_| MigrationRehearsalError::Database) +} + +async fn rehearse_reviewed_steps( + transaction: &impl GenericClient, + candidate: &CompiledRegistry, + plan: &ValidatedReviewedMigrationPlan, +) -> RehearsalResult<()> { + for migration in plan.migrations() { + for step in &migration.steps { + let (step_id, objects) = match &step.descriptor { + ReviewedMigrationStepDescriptor::TransactionalSql { id, objects, .. } + | ReviewedMigrationStepDescriptor::ChunkedBackfill { id, objects, .. } + | ReviewedMigrationStepDescriptor::FieldEncryptionBackfill { + id, objects, .. + } => (id, objects), + }; + let failed = |error: tokio_postgres::Error| MigrationRehearsalError::Step { + migration_id: migration.descriptor.id.clone(), + step_id: step_id.clone(), + failure: PostgresFailure::from_error(&error), + }; + // Activation journals every row a bounded or chunked update + // changes, and refuses before the step runs when the journal + // cannot record it, so the rehearsal applies the same check. + let journaled = match &step.descriptor { + ReviewedMigrationStepDescriptor::TransactionalSql { affected_rows, .. } => { + affected_rows.is_some() + } + ReviewedMigrationStepDescriptor::ChunkedBackfill { .. } => true, + ReviewedMigrationStepDescriptor::FieldEncryptionBackfill { .. } => false, + }; + if journaled { + check_reviewed_history_step(&migration.descriptor_path, step).map_err(|error| { + MigrationRehearsalError::HistoryStep { + migration_id: migration.descriptor.id.clone(), + step_id: step_id.clone(), + reason: error.to_string(), + } + })?; + } + let tables = objects + .iter() + .map(|object| object.table.clone()) + .collect::>() + .into_iter() + .collect::>(); + match &step.descriptor { + ReviewedMigrationStepDescriptor::TransactionalSql { affected_rows, .. } => { + for object in objects { + let Some((entity_id, field_id)) = + pattern_field_for_constraint(candidate, &object.physical_name) + else { + continue; + }; + if let Some(pattern) = + &candidate.entities()[entity_id].fields[field_id].pattern + { + transaction + .query_one("SELECT '' ~ $1::text", &[pattern]) + .await + .map_err(failed)?; + } + } + if affected_rows.is_some() { + set_force_row_security(transaction, &tables, false) + .await + .map_err(|_| MigrationRehearsalError::Database)?; + transaction.execute(&step.sql, &[]).await.map_err(failed)?; + set_force_row_security(transaction, &tables, true) + .await + .map_err(|_| MigrationRehearsalError::Database)?; + } else { + transaction.batch_execute(&step.sql).await.map_err(failed)?; + } + } + ReviewedMigrationStepDescriptor::ChunkedBackfill { .. } => { + // No predecessor row exists, so activation would never run + // this statement. Binding an empty chunk still makes + // PostgreSQL parse, plan, and type-check it. + set_force_row_security(transaction, &tables, false) + .await + .map_err(|_| MigrationRehearsalError::Database)?; + let chunk: Vec = Vec::new(); + transaction + .execute(&step.sql, &[&chunk]) + .await + .map_err(failed)?; + set_force_row_security(transaction, &tables, true) + .await + .map_err(|_| MigrationRehearsalError::Database)?; + } + // The engine seals rows it reads; with no predecessor rows the + // step has nothing to seal and carries no authored SQL. + ReviewedMigrationStepDescriptor::FieldEncryptionBackfill { .. } => {} + } + } + } + Ok(()) +} diff --git a/crates/registry-breg/src/postgres/request_read.rs b/crates/registry-breg/src/postgres/request_read.rs index 8d105e1c0f..be01e1f86a 100644 --- a/crates/registry-breg/src/postgres/request_read.rs +++ b/crates/registry-breg/src/postgres/request_read.rs @@ -1517,6 +1517,7 @@ fn operation_name(operation: Operation) -> &'static str { Operation::ApplyRequest => "apply_request", Operation::Invoke => "invoke", Operation::Snapshot => "snapshot", + Operation::Import => "import", } } diff --git a/crates/registry-breg/src/postgres/schema.rs b/crates/registry-breg/src/postgres/schema.rs index 18c3517505..4bb1eb89f6 100644 --- a/crates/registry-breg/src/postgres/schema.rs +++ b/crates/registry-breg/src/postgres/schema.rs @@ -392,8 +392,10 @@ pub(crate) fn pattern_field_for_constraint<'a>( }) } -/// Only the compiler-resolved field address survives the PostgreSQL boundary. -/// Expressions, row values, and physical names never enter the returned error. +/// A pattern failure keeps only the compiler-resolved field address. Any other +/// server refusal keeps only its SQLSTATE and the object names the server +/// reported. Expressions, messages, and row values never enter the returned +/// error. pub(crate) fn map_pattern_database_error( error: tokio_postgres::Error, pattern_field: Option<(&str, &str)>, @@ -412,7 +414,7 @@ pub(crate) fn map_pattern_database_error( field_id: field.to_owned(), } } - _ => PostgresKernelError::Connection, + _ => PostgresKernelError::from_statement_error(&error), } } @@ -819,7 +821,7 @@ pub(crate) async fn rehearse_schema_fingerprint_with_connection( } #[cfg(all(feature = "runtime", feature = "tooling"))] -async fn connect_schema_test( +pub(super) async fn connect_schema_test( config: &ConnectionConfig, ) -> Result<(Client, tokio::task::JoinHandle<()>)> { match config.tls_connector() { @@ -842,7 +844,7 @@ async fn connect_schema_test( } #[cfg(all(feature = "runtime", feature = "tooling"))] -async fn refuse_existing_managed_objects(client: &impl GenericClient) -> Result<()> { +pub(super) async fn refuse_existing_managed_objects(client: &impl GenericClient) -> Result<()> { client .batch_execute("SAVEPOINT registry_empty_schema_probe") .await?; diff --git a/crates/registry-breg/src/problem.rs b/crates/registry-breg/src/problem.rs index a5f264d8de..a816abc5e6 100644 --- a/crates/registry-breg/src/problem.rs +++ b/crates/registry-breg/src/problem.rs @@ -209,7 +209,7 @@ impl ProblemCode { "The selected access profile does not match the run's bound profile." } Self::IngestionReceiptErased => "The stored receipt of the chunk was erased.", - Self::IngestionRunBlocked => "The active package no longer matches the run binding.", + Self::IngestionRunBlocked => "The ingestion run is blocked and refuses further chunks.", Self::IngestionRunNotOpen => "The ingestion run is not open for this transition.", Self::LookupUnresolved => "The lookup did not resolve exactly one record.", Self::MutationConflict => "The mutation conflicts with current state.", diff --git a/crates/registry-breg/src/request_retention.rs b/crates/registry-breg/src/request_retention.rs index f5b8fd2cce..bfd9f90c37 100644 --- a/crates/registry-breg/src/request_retention.rs +++ b/crates/registry-breg/src/request_retention.rs @@ -208,7 +208,9 @@ impl RequestRetentionOperatorService { return Err(RequestRetentionError::Unavailable); } let config = load_runtime_config(path).map_err(|_| RequestRetentionError::Unavailable)?; - let package_root = config.package().root().to_path_buf(); + let package = config + .load_active_package() + .map_err(|_| RequestRetentionError::Unavailable)?; let runtime_connection = config .runtime_database_connection_config() .map_err(|_| RequestRetentionError::Unavailable)?; @@ -219,10 +221,8 @@ impl RequestRetentionOperatorService { .get() .await .map_err(|_| RequestRetentionError::Unavailable)?; - let context = config.package_load_context(); - let startup = crate::startup::prepare_startup( - &package_root, - &context, + let startup = crate::startup::prepare_loaded_startup( + package, &mut client, config.database().roles().migration(), config.database().roles().runtime(), @@ -784,7 +784,25 @@ impl RequestRetentionOperatorService { Ok(()) } - fn request_plan(&self, request_entity_id: &str) -> Result<()> { + /// Opens a migration-role connection for one operator invocation. + pub(crate) async fn migration_client(&self) -> Result { + self.migration_connection + .build_pool() + .map_err(|_| RequestRetentionError::Unavailable)? + .get() + .await + .map_err(|_| RequestRetentionError::Unavailable) + } + + pub(crate) fn audit(&self) -> &RegistryAudit { + &self.audit + } + + pub(crate) fn package_revision(&self) -> &str { + &self.expected.package_revision + } + + pub(crate) fn request_plan(&self, request_entity_id: &str) -> Result<()> { self.registry .entities() .get(request_entity_id) @@ -793,7 +811,7 @@ impl RequestRetentionOperatorService { .ok_or(RequestRetentionError::Unavailable) } - async fn begin_verified_transaction<'a>( + pub(crate) async fn begin_verified_transaction<'a>( &self, client: &'a mut deadpool_postgres::Client, ) -> Result> { diff --git a/crates/registry-breg/src/review_recovery.rs b/crates/registry-breg/src/review_recovery.rs new file mode 100644 index 0000000000..9b12fd5454 --- /dev/null +++ b/crates/registry-breg/src/review_recovery.rs @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Operator recovery for a change-request review its authority will not answer. +//! +//! A review environment that was restored from an older backup, or replaced +//! by a fresh one, no longer holds the reviews BReg submitted to it. The +//! result poller names such a review `result-unknown-to-authority`, and the +//! poll budget eventually fails it. These operations give the operator two +//! supported answers: resubmit the exact retained review request, or close a +//! review that will never be answered. Each runs in one verified migration +//! transaction under the Registry lock, with an audit `request` entry accepted +//! before the transaction opens and its `response` written after the commit. + +use std::path::Path; + +use registry_platform_audit::AuditEntry; +use serde::Serialize; +use serde_json::Value; +use uuid::Uuid; + +use crate::request_retention::{RequestRetentionError, RequestRetentionOperatorService}; + +/// Codes a failed review may carry and still be resubmitted: each one means +/// BReg gave up waiting, never that the authority refused or decided. +const RESUBMITTABLE_FAILURE_CODES: [&str; 3] = [ + "result-poll-attempts-exhausted", + "submission-recovery-expired", + "operator-closed", +]; +const UNKNOWN_TO_AUTHORITY: &str = "result-unknown-to-authority"; +const OPERATOR_CLOSED: &str = "operator-closed"; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ReviewRecoveryError { + /// Configuration, package, database identity, or lock verification failed. + Unavailable, + /// No review submission exists for this exact request proposal version. + NotFound, + /// The submission exists but this operation does not apply to it. + Ineligible { + reason: ReviewRecoveryRefusal, + state: String, + code: Option, + }, + /// The recovery committed, but the audit destination refused its + /// `response` entry. + RecoveryUnaudited, +} + +/// Why a review submission refuses an operator recovery. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ReviewRecoveryRefusal { + /// The proposal was withdrawn, so its review is being cancelled. + Withdrawn, + /// A review result is already recorded for the proposal. + ResultRecorded, + /// Retention erased the review request, so there is nothing to resubmit. + RequestErased, + /// The request no longer awaits review for this proposal version. + ProposalNotSubmitted, + /// The submission's state and code do not call for this operation. + SubmissionState, +} + +impl ReviewRecoveryRefusal { + pub fn as_str(self) -> &'static str { + match self { + Self::Withdrawn => "withdrawn", + Self::ResultRecorded => "result-recorded", + Self::RequestErased => "request-erased", + Self::ProposalNotSubmitted => "proposal-not-submitted", + Self::SubmissionState => "submission-state", + } + } +} + +pub type Result = std::result::Result; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ReviewRecoveryScope<'a> { + pub request_entity_id: &'a str, + pub request_id: Uuid, + pub proposal_version: i64, +} + +/// The submission before and after one operator recovery. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ReviewRecovery { + pub request_entity_id: String, + pub request_id: String, + pub proposal_version: i64, + pub authority: String, + pub previous_state: String, + pub previous_code: Option, + pub state: &'static str, + pub code: Option<&'static str>, +} + +/// Where a recovery that did not finish stopped. +enum Stopped { + /// Before the commit, so nothing committed. + BeforeCommit(ReviewRecoveryError), + /// At a commit that returned an error, which may still have committed. + AtCommit, +} + +impl From for Stopped { + fn from(error: ReviewRecoveryError) -> Self { + Self::BeforeCommit(error) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Operation { + Resubmit, + Close, +} + +impl Operation { + fn as_str(self) -> &'static str { + match self { + Self::Resubmit => "resubmit", + Self::Close => "close", + } + } +} + +/// Package-bound operator boundary used by `bregctl review-recovery`. +/// +/// Construction verifies the same runtime configuration, package, database +/// identity, managed catalog, and migration role as request retention. +pub struct ReviewRecoveryOperatorService { + verified: RequestRetentionOperatorService, +} + +impl ReviewRecoveryOperatorService { + pub async fn from_runtime_config(path: &Path) -> Result { + RequestRetentionOperatorService::from_runtime_config(path) + .await + .map(|verified| Self { verified }) + .map_err(|_| ReviewRecoveryError::Unavailable) + } + + #[cfg(feature = "postgres-test")] + #[doc(hidden)] + pub fn over_retention_service_for_test(verified: RequestRetentionOperatorService) -> Self { + Self { verified } + } + + /// Returns the review to `pending` so the submission worker sends the + /// exact retained request again under its original idempotency key. + /// + /// Accepted only for a review the authority answered as unknown, or one + /// BReg failed for waiting too long, while its proposal still awaits + /// review. The authority that still holds the request replays its + /// original acceptance; one that lost it accepts the request anew. + pub async fn resubmit(&self, scope: ReviewRecoveryScope<'_>) -> Result { + self.recover(scope, Operation::Resubmit).await + } + + /// Fails an accepted review that will never be answered, keeping its + /// binding so an operator can find the review at the authority. A result + /// the authority delivers for it afterwards is refused, not recorded. + /// BReg sends nothing to the authority. + pub async fn close(&self, scope: ReviewRecoveryScope<'_>) -> Result { + self.recover(scope, Operation::Close).await + } + + async fn recover( + &self, + scope: ReviewRecoveryScope<'_>, + operation: Operation, + ) -> Result { + if scope.proposal_version <= 0 { + return Err(ReviewRecoveryError::NotFound); + } + self.verified + .request_plan(scope.request_entity_id) + .map_err(|_| ReviewRecoveryError::NotFound)?; + // The request entry is accepted before the recovery transaction + // opens, so an audit outage changes no review; the response shares + // its correlation. A recovery that ends without one writes the + // unfinished outcome when the held request is dropped. + let audit = self.verified.audit(); + let package_revision = self.verified.package_revision(); + let record_reference = audit + .profile() + .key_hasher() + .audit_reference_hash( + "breg-record-v1", + package_revision, + &scope.request_id.to_string(), + ) + .map_err(|_| ReviewRecoveryError::Unavailable)?; + let correlation = Uuid::new_v4().to_string(); + let request = serde_json::json!({ + "kind":"reviewRecovery", "operation":operation.as_str(), + "packageRevision":package_revision, + "actor":"breg:review-recovery-operator", + "correlation":correlation, + "entityId":scope.request_entity_id, "recordReference":record_reference, + "proposalVersion":scope.proposal_version, + }); + let mut attempt = audit + .begin( + AuditEntry::request( + crate::audit::AUDIT_SCHEMA, + correlation.clone(), + request.clone(), + ), + with_outcome(&request, "unfinished", Value::Null), + ) + .await + .map_err(|_| ReviewRecoveryError::Unavailable)?; + match self.recover_in_transaction(scope, operation).await { + Ok(recovery) => { + let committed = with_outcome( + &request, + "committed", + serde_json::json!({ + "authority":recovery.authority, + "previousState":recovery.previous_state, + "previousCode":recovery.previous_code, + "state":recovery.state, "code":recovery.code, + }), + ); + attempt + .respond(committed) + .await + .map_err(|_| ReviewRecoveryError::RecoveryUnaudited)?; + Ok(recovery) + } + Err(Stopped::BeforeCommit(error)) => { + let outcome = if error == ReviewRecoveryError::Unavailable { + "failed" + } else { + "refused" + }; + if attempt + .respond(with_outcome(&request, outcome, Value::Null)) + .await + .is_err() + { + tracing::error!( + "the stopped review recovery's response audit entry was not recorded" + ); + } + Err(error) + } + Err(Stopped::AtCommit) => { + if attempt + .respond(with_outcome(&request, "unfinished", Value::Null)) + .await + .is_err() + { + tracing::error!( + "the unacknowledged review recovery's response audit entry was not recorded" + ); + } + Err(ReviewRecoveryError::Unavailable) + } + } + } + + async fn recover_in_transaction( + &self, + scope: ReviewRecoveryScope<'_>, + operation: Operation, + ) -> std::result::Result { + let mut client = self + .verified + .migration_client() + .await + .map_err(unavailable)?; + let transaction = self + .verified + .begin_verified_transaction(&mut client) + .await + .map_err(unavailable)?; + let row = transaction + .query_opt( + "SELECT s.state,s.last_error_code,s.withdrawn,s.authority, + s.create_request <> '{}'::jsonb, + EXISTS ( + SELECT 1 FROM registry_internal.registry_request_review_results r + WHERE (r.request_entity_id,r.request_id,r.proposal_version) + =(s.request_entity_id,s.request_id,s.proposal_version)), + w.state='submitted' AND w.proposal_version=s.proposal_version + FROM registry_internal.registry_request_review_submissions s + LEFT JOIN registry_internal.registry_request_state w + ON (w.request_entity_id,w.request_id)=(s.request_entity_id,s.request_id) + WHERE s.request_entity_id=$1 AND s.request_id=$2 AND s.proposal_version=$3 + FOR UPDATE OF s", + &[ + &scope.request_entity_id, + &scope.request_id, + &scope.proposal_version, + ], + ) + .await + .map_err(|_| ReviewRecoveryError::Unavailable)? + .ok_or(ReviewRecoveryError::NotFound)?; + let state: String = row.get(0); + let code: Option = row.get(1); + let withdrawn: bool = row.get(2); + let authority: String = row.get(3); + let request_retained: bool = row.get(4); + let result_recorded: bool = row.get(5); + let proposal_submitted = row.get::<_, Option>(6) == Some(true); + let refusal = eligibility( + operation, + &state, + code.as_deref(), + withdrawn, + result_recorded, + request_retained, + proposal_submitted, + ); + if let Some(reason) = refusal { + return Err(ReviewRecoveryError::Ineligible { + reason, + state, + code, + } + .into()); + } + let (next_state, next_code) = match operation { + Operation::Resubmit => { + // The binding and every budget return to a fresh submission; + // the idempotency key and request stay exactly as retained, and + // the recovery window restarts at its configured length. + transaction + .execute( + "UPDATE registry_internal.registry_request_review_submissions + SET state='pending',accepted_binding=NULL,attempt_count=0, + result_poll_attempts=0,lease_until=NULL,last_error_code=NULL, + next_attempt_at=transaction_timestamp(), + next_result_poll_at=transaction_timestamp(), + recovery_deadline=transaction_timestamp() + +(recovery_deadline-created_at), + updated_at=transaction_timestamp() + WHERE request_entity_id=$1 AND request_id=$2 AND proposal_version=$3", + &[ + &scope.request_entity_id, + &scope.request_id, + &scope.proposal_version, + ], + ) + .await + .map_err(|_| ReviewRecoveryError::Unavailable)?; + ("pending", None) + } + Operation::Close => { + transaction + .execute( + "UPDATE registry_internal.registry_request_review_submissions + SET state='failed',lease_until=NULL,last_error_code=$4, + updated_at=transaction_timestamp() + WHERE request_entity_id=$1 AND request_id=$2 AND proposal_version=$3", + &[ + &scope.request_entity_id, + &scope.request_id, + &scope.proposal_version, + &OPERATOR_CLOSED, + ], + ) + .await + .map_err(|_| ReviewRecoveryError::Unavailable)?; + ("failed", Some(OPERATOR_CLOSED)) + } + }; + transaction.commit().await.map_err(|_| Stopped::AtCommit)?; + Ok(ReviewRecovery { + request_entity_id: scope.request_entity_id.to_owned(), + request_id: scope.request_id.to_string(), + proposal_version: scope.proposal_version, + authority, + previous_state: state, + previous_code: code, + state: next_state, + code: next_code, + }) + } +} + +/// One `response` record: the request's fields with `outcome` and, for a +/// committed recovery, the submission before and after it. +fn with_outcome(request: &Value, outcome: &str, fields: Value) -> Value { + let mut record = request.clone(); + if let Some(record) = record.as_object_mut() { + record.insert("outcome".to_owned(), Value::from(outcome)); + if let Value::Object(fields) = fields { + record.extend(fields); + } + } + record +} + +fn eligibility( + operation: Operation, + state: &str, + code: Option<&str>, + withdrawn: bool, + result_recorded: bool, + request_retained: bool, + proposal_submitted: bool, +) -> Option { + if withdrawn { + return Some(ReviewRecoveryRefusal::Withdrawn); + } + if result_recorded { + return Some(ReviewRecoveryRefusal::ResultRecorded); + } + match operation { + Operation::Close => (state != "accepted").then_some(ReviewRecoveryRefusal::SubmissionState), + Operation::Resubmit => { + let lost = match (state, code) { + ("accepted", Some(code)) => code == UNKNOWN_TO_AUTHORITY, + ("failed", Some(code)) => RESUBMITTABLE_FAILURE_CODES.contains(&code), + _ => false, + }; + if !lost { + Some(ReviewRecoveryRefusal::SubmissionState) + } else if !request_retained { + Some(ReviewRecoveryRefusal::RequestErased) + } else if !proposal_submitted { + Some(ReviewRecoveryRefusal::ProposalNotSubmitted) + } else { + None + } + } + } +} + +fn unavailable(_error: RequestRetentionError) -> ReviewRecoveryError { + ReviewRecoveryError::Unavailable +} + +#[cfg(test)] +mod tests { + use super::*; + + fn resubmit(state: &str, code: Option<&str>) -> Option { + eligibility(Operation::Resubmit, state, code, false, false, true, true) + } + + #[test] + fn resubmit_accepts_only_reviews_the_authority_lost_or_breg_stopped_waiting_for() { + assert_eq!(resubmit("accepted", Some(UNKNOWN_TO_AUTHORITY)), None); + for code in RESUBMITTABLE_FAILURE_CODES { + assert_eq!(resubmit("failed", Some(code)), None, "{code}"); + } + for (state, code) in [ + ("accepted", None), + ("accepted", Some("result-lookup-uncertain")), + ("failed", Some("remote-refused")), + ("failed", Some("result-expired")), + ("pending", None), + ("uncertain", Some("remote-uncertain")), + ("cancelling", None), + ("cancelled", None), + ] { + assert_eq!( + resubmit(state, code), + Some(ReviewRecoveryRefusal::SubmissionState), + "{state} {code:?}" + ); + } + let lost = ("accepted", Some(UNKNOWN_TO_AUTHORITY)); + for (withdrawn, result, retained, submitted, refusal) in [ + (true, false, true, true, ReviewRecoveryRefusal::Withdrawn), + ( + false, + true, + true, + true, + ReviewRecoveryRefusal::ResultRecorded, + ), + ( + false, + false, + false, + true, + ReviewRecoveryRefusal::RequestErased, + ), + ( + false, + false, + true, + false, + ReviewRecoveryRefusal::ProposalNotSubmitted, + ), + ] { + assert_eq!( + eligibility( + Operation::Resubmit, + lost.0, + lost.1, + withdrawn, + result, + retained, + submitted + ), + Some(refusal) + ); + } + } + + #[test] + fn close_accepts_only_an_accepted_review_without_a_result() { + let close = |state, withdrawn, result| { + eligibility(Operation::Close, state, None, withdrawn, result, true, true) + }; + assert_eq!(close("accepted", false, false), None); + assert_eq!( + close("accepted", true, false), + Some(ReviewRecoveryRefusal::Withdrawn) + ); + assert_eq!( + close("accepted", false, true), + Some(ReviewRecoveryRefusal::ResultRecorded) + ); + for state in ["pending", "submitting", "uncertain", "cancelling", "failed"] { + assert_eq!( + close(state, false, false), + Some(ReviewRecoveryRefusal::SubmissionState), + "{state}" + ); + } + } +} diff --git a/crates/registry-breg/src/review_store.rs b/crates/registry-breg/src/review_store.rs index 69c9f9261d..c84d865301 100644 --- a/crates/registry-breg/src/review_store.rs +++ b/crates/registry-breg/src/review_store.rs @@ -2036,6 +2036,20 @@ pub async fn receive_completion( if persisted != 1 { return Err(MutationError::PreconditionFailed); } + if state == "pending" { + // The authority announced a result BReg does not hold yet, so the + // review's lookup is due now instead of after its backoff. + transaction + .execute( + "UPDATE registry_internal.registry_request_review_submissions + SET next_result_poll_at=LEAST(next_result_poll_at,transaction_timestamp()) + WHERE authority=$1 AND accepted_binding->>'requestId'=$2 + AND state='accepted'", + &[&authority, &completion.request_id.to_string()], + ) + .await + .map_err(|_| MutationError::Unavailable)?; + } Ok(()) } @@ -2167,7 +2181,11 @@ pub async fn poll_one_result( lease_seconds: i64, ) -> Result { // The lookup is claimed with a lease before the outbound exchange so two - // instances cannot both act on one submission's result at once. + // instances cannot both act on one submission's result at once. Among due + // reviews, one a webhook completion already announced goes first, and one + // the authority last answered as unknown goes last, so reviews an + // authority lost (a restored or replaced review environment) cannot hold + // back live ones. Every due review is still reached in turn. let Some(row) = client .query_opt( "UPDATE registry_internal.registry_request_review_submissions s @@ -2184,7 +2202,15 @@ pub async fn poll_one_result( WHERE r.request_entity_id=c.request_entity_id AND r.request_id=c.request_id AND r.proposal_version=c.proposal_version) - ORDER BY c.updated_at FOR UPDATE SKIP LOCKED LIMIT 1) + ORDER BY EXISTS ( + SELECT 1 + FROM registry_internal.registry_request_review_completions k + WHERE k.authority=c.authority AND k.state='pending' + AND k.review_request_id::text=c.accepted_binding->>'requestId') + DESC, + c.last_error_code IS NOT DISTINCT FROM 'result-unknown-to-authority', + c.updated_at + FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING s.request_entity_id,s.request_id,s.proposal_version, s.authority,s.accepted_binding,s.lease_until", &[&authority_id, &lease_seconds], @@ -2269,13 +2295,15 @@ pub async fn poll_one_result( // one, so it spends the give-up budget like a failed lookup. The // lease fence keeps a worker that lost its claim (expired lease, // reclaimed row) from republishing a backoff over the new - // holder's schedule. The lookup itself answered, so it clears the - // lookup error a previous failure or credential outage recorded. + // holder's schedule. The lookup itself answered, so its own code + // replaces the lookup error a previous failure or credential + // outage recorded: the authority reached BReg and does not know + // the review, which a slow reviewer never produces. client .execute( "UPDATE registry_internal.registry_request_review_submissions SET result_poll_attempts=LEAST(result_poll_attempts+1,1000), - last_error_code=NULL, + last_error_code='result-unknown-to-authority', next_result_poll_at=transaction_timestamp()+ (LEAST(60,5*LEAST(result_poll_attempts+1,1000)) * interval '1 second'), updated_at=transaction_timestamp() diff --git a/crates/registry-breg/src/runtime_config.rs b/crates/registry-breg/src/runtime_config.rs index b5061981d7..fe1185df59 100644 --- a/crates/registry-breg/src/runtime_config.rs +++ b/crates/registry-breg/src/runtime_config.rs @@ -16,7 +16,9 @@ use base64::Engine as _; use jsonwebtoken::jwk::JwkSet; use registry_platform_audit::{AuditDestination, AuditDestinationKind, AuditProfile}; use registry_platform_config::{ - expand_config_env_vars_with, SecretError, SecretProvider, SecretReference, SecretResolver, + AuditKeyConfig, JwksSource, ListenerBind, OidcIssuerConfig, + PackageConfig as SharedPackageConfig, RuntimeConfigErrorKind, RuntimeConfigLoader, + RuntimeEnvelope, SecretError, SecretReference, SecretResolver, }; use registry_platform_crypto::{parse_json_strict, PublicJwk, SigningAlgorithm}; #[cfg(feature = "schema")] @@ -43,7 +45,11 @@ use crate::{ ActivatedEventDestinationRegistry, EventDestinationConfigs, RawEventDestinationConfigs, }, model::CompiledRegistry, - package::{PackageIntent, PackageLoadContext}, + package::{ + load_package_with_verified_envelope, load_predecessor_package_with_verified_envelope, + PackageError, PackageIntent, PackageLoadContext, PredecessorPackageContext, + VerifiedPackage, VerifiedPredecessorPackage, + }, postgres::{ConnectionConfig, PoolBounds, SqlIdentifier}, }; @@ -129,6 +135,8 @@ pub enum RuntimeConfigError { Bounds, #[error("runtime configuration environment expansion was refused")] EnvExpansion, + #[error("runtime configuration substitutes into a secret reference or secret provider")] + SubstitutionInReference, #[error("the runtime configuration document is invalid")] Document, #[error("runtime configuration uses an unsupported apiVersion")] @@ -219,6 +227,7 @@ impl RuntimeConfigError { Self::UnsafeFile => "runtime_config.unsafe_file", Self::Bounds => "runtime_config.bounds", Self::EnvExpansion => "runtime_config.env_expansion", + Self::SubstitutionInReference => "runtime_config.substitution_in_reference", Self::Document => "runtime_config.document", Self::InvalidApiVersion => "runtime_config.invalid_api_version", Self::InvalidKind => "runtime_config.invalid_kind", @@ -258,6 +267,7 @@ impl RuntimeConfigError { | Self::UnsafeFile | Self::Bounds | Self::EnvExpansion + | Self::SubstitutionInReference | Self::Document | Self::GovernedMember | Self::InvalidBinding @@ -330,31 +340,84 @@ pub fn parse_runtime_config_with_env( { return Err(RuntimeConfigError::Bounds); } - let expanded = - expand_config_env_vars_with(raw, lookup).map_err(|_| RuntimeConfigError::EnvExpansion)?; - if expanded.len() > usize::try_from(MAX_RUNTIME_CONFIG_BYTES).unwrap_or(usize::MAX) { + let substituted = RuntimeConfigLoader::new(RuntimeEnvelope { + api_version: RUNTIME_CONFIG_API_VERSION, + kind: RUNTIME_CONFIG_KIND, + }) + .max_bytes(MAX_RUNTIME_CONFIG_BYTES) + .parse_str::(raw, lookup) + .map_err(|error| runtime_config_error_from_loader(&error))? + .config; + // A substituted value may be longer than the expression it replaced, so + // the substituted document is held to the same bound as the file. + let substituted_len = serde_json::to_string(&substituted) + .map_err(|_| RuntimeConfigError::Document)? + .len(); + if substituted_len > usize::try_from(MAX_RUNTIME_CONFIG_BYTES).unwrap_or(usize::MAX) { return Err(RuntimeConfigError::Bounds); } - parse_expanded_runtime_config(&expanded) -} - -fn parse_expanded_runtime_config(expanded: &str) -> Result { - reject_governed_members(expanded)?; + if contains_governed_member(&substituted) { + return Err(RuntimeConfigError::GovernedMember); + } + reject_invalid_binding_text(&substituted)?; let raw: RawRuntimeConfig = - serde_norway::from_str(expanded).map_err(|_| RuntimeConfigError::Document)?; + serde_json::from_value(substituted).map_err(|_| RuntimeConfigError::Document)?; RuntimeConfig::from_raw(raw) } -fn reject_governed_members(raw: &str) -> Result<()> { - let value: serde_norway::Value = - serde_norway::from_str(raw).map_err(|_| RuntimeConfigError::Document)?; - if contains_governed_member(&value) { - return Err(RuntimeConfigError::GovernedMember); +/// The shared loader parses the document, checks its envelope, and substitutes +/// `${VAR}` inside string values; each of its refusals keeps the code this +/// runtime reported for the same cause. +fn runtime_config_error_from_loader( + error: ®istry_platform_config::RuntimeConfigError, +) -> RuntimeConfigError { + match error.kind() { + RuntimeConfigErrorKind::Envelope if error.field() == "apiVersion" => { + RuntimeConfigError::InvalidApiVersion + } + RuntimeConfigErrorKind::Envelope => RuntimeConfigError::InvalidKind, + RuntimeConfigErrorKind::Substitution => RuntimeConfigError::EnvExpansion, + RuntimeConfigErrorKind::SubstitutionInReference => { + RuntimeConfigError::SubstitutionInReference + } + RuntimeConfigErrorKind::Bounds => RuntimeConfigError::Bounds, + RuntimeConfigErrorKind::Path | RuntimeConfigErrorKind::UnsafeFile => { + RuntimeConfigError::UnsafeFile + } + RuntimeConfigErrorKind::Unavailable => RuntimeConfigError::Unavailable, + _ => RuntimeConfigError::Document, + } +} + +/// A listener address or an audit key reference that does not parse is +/// refused as that binding, the code it had when the binding parsed its own +/// text, before the typed document reports every other shape problem as a +/// document error. +fn reject_invalid_binding_text(document: &Value) -> Result<()> { + let refuses = |pointer: &str, parses: fn(&Value) -> bool| { + document + .pointer(pointer) + .is_some_and(|value| value.is_string() && !parses(value)) + }; + if refuses("/listener/bind", |value| { + ListenerBind::deserialize(value).is_ok() + }) { + return Err(RuntimeConfigError::InvalidListener); + } + if refuses("/metricsListener/bind", |value| { + ListenerBind::deserialize(value).is_ok() + }) { + return Err(RuntimeConfigError::InvalidMetricsListener); + } + if refuses("/audit/hashKeyRef", |value| { + SecretReference::deserialize(value).is_ok() + }) { + return Err(RuntimeConfigError::InvalidAudit); } Ok(()) } -fn contains_governed_member(value: &serde_norway::Value) -> bool { +fn contains_governed_member(value: &Value) -> bool { const GOVERNED: &[&str] = &[ "entities", "fields", @@ -379,22 +442,20 @@ fn contains_governed_member(value: &serde_norway::Value) -> bool { "cors", ]; match value { - serde_norway::Value::Mapping(mapping) => mapping.iter().any(|(key, value)| { - key.as_str().is_some_and(|key| GOVERNED.contains(&key)) + Value::Object(mapping) => mapping.iter().any(|(key, value)| { + GOVERNED.contains(&key.as_str()) // Binding-map keys are compiler-issued logical ids. Do not // reinterpret an id such as `hooks` as a governed field; the // strict binding value types still reject undeployed members. - || (key.as_str().is_none_or(|key| { - !matches!( - key, - "eventDestinations" - | "evidenceProviders" - | "reviewAuthorities" - | "reviewExecutors" - ) - }) && contains_governed_member(value)) + || (!matches!( + key.as_str(), + "eventDestinations" + | "evidenceProviders" + | "reviewAuthorities" + | "reviewExecutors" + ) && contains_governed_member(value)) }), - serde_norway::Value::Sequence(values) => values.iter().any(contains_governed_member), + Value::Array(values) => values.iter().any(contains_governed_member), _ => false, } } @@ -817,6 +878,58 @@ impl RuntimeConfig { &self.package } + /// Verify the shared package envelope and optional package digest pin + /// before any product-specific signature or deployment-binding work. + pub fn verify_package_envelope( + &self, + ) -> std::result::Result< + registry_platform_config::package::VerifiedPackage, + registry_platform_config::package::PackageError, + > { + self.package + .shared + .verify_package(&crate::package::shared_package_limits(), "bregctl package") + .map_err(|error| error.naming_root_as("package.root")) + } + + /// Verify the configured package pin and consume that same shared + /// envelope through BReg's signature, binding, and derivation checks. + pub fn load_active_package(&self) -> std::result::Result { + let shared = self + .verify_package_envelope() + .map_err(|_| PackageError::Envelope)?; + load_package_with_verified_envelope( + self.package().root(), + &self.package_load_context(), + &shared, + ) + } + + /// Verify and retain the configured shared package while loading the + /// database-active predecessor representation used for successor planning. + pub fn load_active_predecessor_package( + &self, + ) -> std::result::Result { + let shared = self + .verify_package_envelope() + .map_err(|_| PackageError::Envelope)?; + load_predecessor_package_with_verified_envelope( + self.package().root(), + &PredecessorPackageContext { + environment: self.identity().environment(), + instance_id: self.identity().instance_id(), + database_id: self.identity().database_id(), + database_initialization_environment: self + .identity() + .database_initialization_environment(), + trust_anchor: self.package_trust_anchor(), + expected_package_revision: self.package().active_revision(), + expected_sequence: self.package().active_sequence(), + }, + &shared, + ) + } + pub fn authentication(&self) -> &AuthenticationConfig { &self.authentication } @@ -957,7 +1070,7 @@ impl RuntimeConfig { fn validate_loaded_paths(&self) -> Result<()> { validate_existing_directory( - &self.package.root, + self.package.root(), RuntimeConfigError::UnsafePackageRoot, RuntimeConfigError::PackageRootUnavailable, )?; @@ -1011,10 +1124,7 @@ pub struct ListenerConfig { impl ListenerConfig { fn from_raw(raw: RawListenerConfig) -> Result { - let bind = raw - .bind - .parse::() - .map_err(|_| RuntimeConfigError::InvalidListener)?; + let bind = raw.bind.socket_addr(); Ok(Self { bind, public_origin: raw @@ -1058,10 +1168,7 @@ pub struct MetricsListenerConfig { impl MetricsListenerConfig { fn from_raw(raw: RawMetricsListenerConfig) -> Result { - let bind = raw - .bind - .parse::() - .map_err(|_| RuntimeConfigError::InvalidMetricsListener)?; + let bind = raw.bind.socket_addr(); if bind.port() == 0 { return Err(RuntimeConfigError::InvalidMetricsListener); } @@ -1249,44 +1356,29 @@ impl fmt::Debug for DeploymentIdentity { } } +/// The shared secret-provider block, checked once at load. Its `Debug` keeps +/// the file root out of logs. #[derive(Clone)] pub struct SecretProvidersConfig { - environment: bool, - file: Option, + block: registry_platform_config::SecretProvidersConfig, } impl SecretProvidersConfig { - fn from_raw(raw: RawSecretProvidersConfig) -> Result { - if raw.environment.is_none() && raw.file.is_none() { - return Err(RuntimeConfigError::InvalidSecretProvider); + fn from_raw(raw: registry_platform_config::SecretProvidersConfig) -> Result { + raw.check() + .map_err(|_| RuntimeConfigError::InvalidSecretProvider)?; + if let Some(file) = &raw.file { + validate_absolute_lexical_path(&file.root, RuntimeConfigError::InvalidSecretProvider)?; } - let file = raw - .file - .map(FileSecretProviderConfig::from_raw) - .transpose()?; - Ok(Self { - environment: raw.environment.is_some(), - file, - }) + Ok(Self { block: raw }) } fn resolver(&self) -> Result { - let mut providers = Vec::new(); - if self.environment { - providers.push(SecretProvider::Environment); - } - if self.file.is_some() { - providers.push(SecretProvider::File); - } - let root = self - .file - .as_ref() - .map_or_else(PathBuf::new, |file| file.root.clone()); - SecretResolver::new(providers, root).map_err(Into::into) + self.block.resolver().map_err(Into::into) } fn file_root(&self) -> Option<&Path> { - self.file.as_ref().map(|file| file.root.as_path()) + self.block.file.as_ref().map(|file| file.root.as_path()) } } @@ -1294,24 +1386,12 @@ impl fmt::Debug for SecretProvidersConfig { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter .debug_struct("SecretProvidersConfig") - .field("environment", &self.environment) - .field("file", &self.file.as_ref().map(|_| "")) + .field("environment", &self.block.environment.is_some()) + .field("file", &self.block.file.as_ref().map(|_| "")) .finish() } } -#[derive(Clone)] -pub struct FileSecretProviderConfig { - root: PathBuf, -} - -impl FileSecretProviderConfig { - fn from_raw(raw: RawFileSecretProviderConfig) -> Result { - validate_absolute_lexical_path(&raw.root, RuntimeConfigError::InvalidSecretProvider)?; - Ok(Self { root: raw.root }) - } -} - #[derive(Clone)] pub struct DatabaseConfig { runtime_url_ref: SecretReference, @@ -1365,7 +1445,7 @@ impl fmt::Debug for DatabaseConfig { #[derive(Clone)] pub struct PackageConfig { - root: PathBuf, + shared: SharedPackageConfig, trust_anchor_path: PathBuf, compiler_source_revision: String, active_revision: String, @@ -1374,7 +1454,9 @@ pub struct PackageConfig { impl PackageConfig { fn from_raw(raw: RawPackageConfig) -> Result { - validate_absolute_lexical_path(&raw.root, RuntimeConfigError::InvalidPackage)?; + raw.shared + .check() + .map_err(|_| RuntimeConfigError::InvalidPackage)?; validate_absolute_lexical_path(&raw.trust_anchor_path, RuntimeConfigError::InvalidPackage)?; validate_deployment_value(&raw.compiler_source_revision)?; validate_deployment_value(&raw.active_revision)?; @@ -1382,7 +1464,7 @@ impl PackageConfig { return Err(RuntimeConfigError::InvalidPackage); } Ok(Self { - root: raw.root, + shared: raw.shared, trust_anchor_path: raw.trust_anchor_path, compiler_source_revision: raw.compiler_source_revision, active_revision: raw.active_revision, @@ -1391,7 +1473,11 @@ impl PackageConfig { } pub fn root(&self) -> &Path { - &self.root + &self.shared.root + } + + pub fn shared(&self) -> &SharedPackageConfig { + &self.shared } pub fn trust_anchor_path(&self) -> &Path { @@ -1415,7 +1501,7 @@ impl fmt::Debug for PackageConfig { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter .debug_struct("PackageConfig") - .field("root", &"") + .field("shared", &"") .field("trust_anchor_path", &"") .field("compiler_source_revision", &"") .field("active_revision", &"") @@ -1476,8 +1562,14 @@ pub struct OidcVerifierConfig { impl OidcVerifierConfig { fn from_raw(raw: RawOidcVerifierConfig) -> Result { - validate_oidc_value(&raw.issuer)?; - validate_oidc_value(&raw.audience)?; + validate_oidc_value(&raw.provider.issuer)?; + validate_oidc_value(&raw.provider.audience)?; + // A loopback `http` issuer or key URI stays accepted for the local + // development and test deployments that run their issuer beside the + // registry; every other issuer and key URI is `https`. + raw.provider + .check("authentication.oidc", true) + .map_err(|_| RuntimeConfigError::InvalidOidc)?; validate_oidc_value(&raw.access_token_type)?; validate_claim_name(&raw.scope_claim)?; if raw.scope_separator.is_control() || raw.scope_separator.is_alphanumeric() { @@ -1494,9 +1586,9 @@ impl OidcVerifierConfig { return Err(RuntimeConfigError::InvalidOidc); } // Duplicate assertion-issuer client keys are already refused before this - // point: `reject_governed_members` parses the whole document into a - // generic value first, and that parse rejects any duplicate YAML mapping - // key anywhere in the document, including here. + // point: the shared loader parses the whole document into a generic + // value first, and that parse rejects any duplicate YAML mapping key + // anywhere in the document, including here. let assertion_issuer_clients = raw.assertion_issuers.keys().cloned().collect::>(); validate_bounded_list(&assertion_issuer_clients)?; for issuers in raw.assertion_issuers.values() { @@ -1508,9 +1600,10 @@ impl OidcVerifierConfig { } let max_token_lifetime = seconds_bounded(raw.max_token_lifetime_seconds, 1, 7200)?; let leeway = oidc_leeway(raw.leeway_milliseconds)?; + let jwks_source = OidcJwksSource::from_block(raw.provider.jwks_source)?; Ok(Self { - issuer: raw.issuer, - audience: raw.audience, + issuer: raw.provider.issuer, + audience: raw.provider.audience, allowed_algorithm: raw.allowed_algorithm, access_token_type: raw.access_token_type, scope_claim: raw.scope_claim, @@ -1521,11 +1614,7 @@ impl OidcVerifierConfig { max_token_lifetime, leeway, jwks_cache: JwksCacheConfig::from_raw(raw.jwks_cache)?, - jwks_source: raw - .jwks_source - .map(OidcJwksSource::from_raw) - .transpose()? - .unwrap_or(OidcJwksSource::Discovery), + jwks_source, }) } @@ -1576,6 +1665,10 @@ impl OidcVerifierConfig { self.jwks_fetcher_config(), ))) } + OidcJwksSource::Uri { uri } => Ok(Arc::new(JwksFetcher::new( + uri.clone(), + self.jwks_fetcher_config(), + ))), OidcJwksSource::Static { document_ref } => { let document = resolver .resolve_reference(document_ref) @@ -1663,14 +1756,16 @@ impl OidcAlgorithm { #[derive(Clone)] enum OidcJwksSource { Discovery, + Uri { uri: String }, Static { document_ref: SecretReference }, } impl OidcJwksSource { - fn from_raw(raw: RawOidcJwksSource) -> Result { - match raw { - RawOidcJwksSource::Discovery {} => Ok(Self::Discovery), - RawOidcJwksSource::Static { document_ref } => Ok(Self::Static { + fn from_block(block: JwksSource) -> Result { + match block { + JwksSource::Discovery {} => Ok(Self::Discovery), + JwksSource::Uri { uri } => Ok(Self::Uri { uri }), + JwksSource::Static { document_ref } => Ok(Self::Static { document_ref: parse_secret_reference( document_ref, RuntimeConfigError::InvalidOidc, @@ -1682,6 +1777,7 @@ impl OidcJwksSource { const fn kind(&self) -> OidcJwksSourceKind { match self { Self::Discovery => OidcJwksSourceKind::Discovery, + Self::Uri { .. } => OidcJwksSourceKind::Uri, Self::Static { .. } => OidcJwksSourceKind::Static, } } @@ -1690,6 +1786,7 @@ impl OidcJwksSource { #[derive(Clone, Copy, Debug)] enum OidcJwksSourceKind { Discovery, + Uri, Static, } @@ -2013,8 +2110,7 @@ pub struct AuditConfig { impl AuditConfig { fn from_raw(raw: RawAuditConfig) -> Result { - let hash_key_ref = - parse_secret_reference(raw.hash_key_ref, RuntimeConfigError::InvalidAudit)?; + let hash_key_ref = raw.key.hash_key_ref; let destination = AuditDestination::from_settings( raw.destination, raw.path.map(PathBuf::from), @@ -2441,7 +2537,7 @@ struct RawRuntimeConfig { kind: String, listener: RawListenerConfig, identity: RawDeploymentIdentity, - secret_providers: RawSecretProvidersConfig, + secret_providers: registry_platform_config::SecretProvidersConfig, database: RawDatabaseConfig, /// Defaults to PostgreSQL; S3 requires a bucket with versioning never enabled. #[serde(default)] @@ -2488,7 +2584,7 @@ struct RawRuntimeConfig { #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct RawListenerConfig { - bind: String, + bind: ListenerBind, /// Canonical HTTPS origin (loopback HTTP for local development) for QGIS /// discovery and pagination. Required when the registry exposes GIS collections. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -2501,7 +2597,7 @@ struct RawListenerConfig { struct RawMetricsListenerConfig { /// Operator-private loopback or private numeric address and named port, /// for example `127.0.0.1:9100`. - bind: String, + bind: ListenerBind, } #[cfg_attr(feature = "schema", derive(serde::Serialize, schemars::JsonSchema))] @@ -2514,26 +2610,6 @@ struct RawDeploymentIdentity { database_initialization_environment: String, } -#[cfg_attr(feature = "schema", derive(serde::Serialize, schemars::JsonSchema))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct RawSecretProvidersConfig { - environment: Option, - file: Option, -} - -#[cfg_attr(feature = "schema", derive(serde::Serialize, schemars::JsonSchema))] -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct RawEnvironmentSecretProviderConfig {} - -#[cfg_attr(feature = "schema", derive(serde::Serialize, schemars::JsonSchema))] -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct RawFileSecretProviderConfig { - root: PathBuf, -} - #[cfg_attr(feature = "schema", derive(serde::Serialize, schemars::JsonSchema))] #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] @@ -2581,7 +2657,8 @@ struct RawSqlRoles { #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct RawPackageConfig { - root: PathBuf, + #[serde(flatten)] + shared: SharedPackageConfig, trust_anchor_path: PathBuf, compiler_source_revision: String, active_revision: String, @@ -2600,8 +2677,8 @@ struct RawAuthenticationConfig { #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct RawOidcVerifierConfig { - issuer: String, - audience: String, + #[serde(flatten)] + provider: OidcIssuerConfig, allowed_algorithm: OidcAlgorithm, /// The one admitted access-token `typ` semantics. Configuring the /// RFC 9068 access-token media type as `at+jwt` or @@ -2622,21 +2699,6 @@ struct RawOidcVerifierConfig { /// Optional JWKS fetch and cache tuning. Defaults to bounded cache behavior. #[serde(default)] jwks_cache: RawJwksCacheConfig, - #[serde(default)] - jwks_source: Option, -} - -#[cfg_attr(feature = "schema", derive(serde::Serialize, schemars::JsonSchema))] -#[derive(Deserialize)] -#[serde( - rename_all = "camelCase", - rename_all_fields = "camelCase", - deny_unknown_fields, - tag = "kind" -)] -enum RawOidcJwksSource { - Discovery {}, - Static { document_ref: String }, } #[cfg_attr(feature = "schema", derive(serde::Serialize, schemars::JsonSchema))] @@ -2711,7 +2773,8 @@ impl From for ClaimNames { #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct RawAuditConfig { - hash_key_ref: String, + #[serde(flatten)] + key: AuditKeyConfig, /// `file` (the default) writes a durable, rotated JSON Lines file at /// `path`; `stdout` writes one JSON line per entry to standard output. #[serde(default)] @@ -3178,12 +3241,6 @@ fn install_schema_constraints(schema: &mut Value) { MAX_DEPLOYMENT_VALUE_BYTES, VALUE_NO_EDGE_WHITESPACE_SCHEMA_PATTERN, ), - ( - "/$defs/RawFileSecretProviderConfig/properties/root", - 1, - MAX_PATH_BYTES, - "", - ), ( "/$defs/RawDatabaseConfig/properties/runtimeUrlRef", 1, @@ -3236,12 +3293,12 @@ fn install_schema_constraints(schema: &mut Value) { "/$defs/RawOidcVerifierConfig/properties/issuer", 1, MAX_OIDC_VALUE_BYTES, - VALUE_NO_EDGE_WHITESPACE_SCHEMA_PATTERN, + "", ), ( "/$defs/RawOidcVerifierConfig/properties/audience", 1, - MAX_OIDC_VALUE_BYTES, + registry_platform_config::MAX_OIDC_AUDIENCE_CHARACTERS, VALUE_NO_EDGE_WHITESPACE_SCHEMA_PATTERN, ), ( @@ -3262,12 +3319,6 @@ fn install_schema_constraints(schema: &mut Value) { 1, SCOPE_SEPARATOR_SCHEMA_PATTERN, ), - ( - "/$defs/RawOidcJwksSource/oneOf/1/properties/documentRef", - 1, - MAX_SECRET_REFERENCE_SCHEMA_LENGTH, - SECRET_REFERENCE_SCHEMA_PATTERN, - ), ( "/$defs/RawAuthorityClaimsConfig/properties/principal", 1, @@ -3280,12 +3331,6 @@ fn install_schema_constraints(schema: &mut Value) { 128, CLAIM_NAME_SCHEMA_PATTERN, ), - ( - "/$defs/RawAuditConfig/properties/hashKeyRef", - 1, - MAX_SECRET_REFERENCE_SCHEMA_LENGTH, - SECRET_REFERENCE_SCHEMA_PATTERN, - ), ( "/$defs/RawCursorConfig/properties/secretRef", 1, diff --git a/crates/registry-breg/src/startup.rs b/crates/registry-breg/src/startup.rs index 41111e1968..b5ba7158ff 100644 --- a/crates/registry-breg/src/startup.rs +++ b/crates/registry-breg/src/startup.rs @@ -31,7 +31,8 @@ use crate::metrics::{self, Metrics}; #[cfg(all(feature = "runtime", feature = "tooling"))] use crate::model::CompiledRegistry; use crate::package::{ - load_package, PackageError, PackageIntent, PackageLoadContext, VerifiedPackage, + load_package_with_verified_envelope, PackageError, PackageIntent, PackageLoadContext, + VerifiedPackage, }; use crate::postgres::{ inspect_baseline, verify_catalog_identity_for_catalog, AdvisorySeverity, BaselineAdvisory, @@ -53,10 +54,16 @@ pub enum StartupError { RuntimeConfig(RuntimeConfigError), #[error("the Registry package was refused")] PackageRefused(PackageError), + #[error("{0}")] + PackageEnvelopeRefused(String), #[error("the Registry database connection was refused")] DatabaseConnection, #[error("the Registry database is not ready for this package")] DatabaseUnready, + /// The database is not the physical instance the Registry's instance + /// claim names, as a restored copy is until an operator adopts it. + #[error("the Registry database is not the instance its claim names")] + InstanceClaimMismatch, /// Authored field address only, never the expression or database diagnostic. #[error("a persisted field pattern has invalid PostgreSQL syntax")] FieldPatternSyntax { entity_id: String, field_id: String }, @@ -342,8 +349,12 @@ impl StartupError { match self { Self::RuntimeConfig(_) => "the Registry runtime configuration was refused", Self::PackageRefused(_) => "the Registry package was refused", + Self::PackageEnvelopeRefused(_) => "the Registry package was refused", Self::DatabaseConnection => "the Registry database connection was refused", Self::DatabaseUnready => "the Registry database is not ready for this package", + Self::InstanceClaimMismatch => { + "the Registry database is not the instance its claim names; adopt a restored copy with bregctl instance-claim adopt" + } Self::FieldPatternSyntax { .. } => { "a persisted field pattern has invalid PostgreSQL syntax" } @@ -512,10 +523,14 @@ impl PreparedServer { /// database connection, OIDC discovery, audit profile, or listener bind. pub async fn prepare(config_path: &Path) -> Result { let config = load_runtime_config(config_path).map_err(map_runtime_config_error)?; + let shared = config + .verify_package_envelope() + .map_err(|error| StartupError::PackageEnvelopeRefused(error.to_string()))?; let package_root = config.package().root().to_path_buf(); let package = { let package_context = config.package_load_context(); - load_package(&package_root, &package_context).map_err(StartupError::PackageRefused)? + load_package_with_verified_envelope(&package_root, &package_context, &shared) + .map_err(StartupError::PackageRefused)? }; let connection = config .runtime_database_connection_config() @@ -533,10 +548,14 @@ pub async fn prepare(config_path: &Path) -> Result { /// can append. pub async fn check(config_path: &Path) -> Result> { let config = load_runtime_config(config_path).map_err(map_runtime_config_error)?; + let shared = config + .verify_package_envelope() + .map_err(|error| StartupError::PackageEnvelopeRefused(error.to_string()))?; let package_root = config.package().root().to_path_buf(); let package = { let package_context = config.package_load_context(); - load_package(&package_root, &package_context).map_err(StartupError::PackageRefused)? + load_package_with_verified_envelope(&package_root, &package_context, &shared) + .map_err(StartupError::PackageRefused)? }; let connection = config .runtime_database_connection_config() @@ -581,6 +600,29 @@ pub async fn rehearse_schema_fingerprint( rehearse_schema_fingerprint_with_connection_config(config, registry, &migration).await } +/// Rehearse a successor candidate's migration over an empty reproduction of +/// its verified predecessor schema, using the configured migration role +/// against the clean schema-test database. The outer error is a deployment +/// binding or configuration refusal; the inner one is the rehearsal's own +/// value-free refusal. The rehearsal always rolls back. +#[cfg(all(feature = "runtime", feature = "tooling"))] +pub async fn rehearse_successor_migration( + config: &RuntimeConfig, + rehearsal: crate::postgres::SuccessorMigrationRehearsal<'_>, +) -> Result> { + validate_schema_test_candidate_binding(config, rehearsal.candidate)?; + let migration = config + .migration_database_connection_config() + .map_err(map_runtime_config_error)?; + Ok(crate::postgres::rehearse_successor_migration( + &migration, + config.database().roles().migration(), + config.database().roles().runtime(), + rehearsal, + ) + .await) +} + #[cfg(all(feature = "runtime", feature = "tooling", feature = "postgres-test"))] #[doc(hidden)] pub async fn rehearse_schema_fingerprint_with_connection_config_for_test( @@ -702,10 +744,14 @@ pub async fn prepare_with_connection_config_for_test( connection: crate::postgres::ConnectionConfig, ) -> Result { let config = load_runtime_config(config_path).map_err(map_runtime_config_error)?; + let shared = config + .verify_package_envelope() + .map_err(|error| StartupError::PackageEnvelopeRefused(error.to_string()))?; let package_root = config.package().root().to_path_buf(); let package = { let package_context = config.package_load_context(); - load_package(&package_root, &package_context).map_err(StartupError::PackageRefused)? + load_package_with_verified_envelope(&package_root, &package_context, &shared) + .map_err(StartupError::PackageRefused)? }; prepare_verified_package_with_connection(config, package, connection, AuditOpening::Serve).await } @@ -717,10 +763,14 @@ pub async fn check_with_connection_config_for_test( connection: crate::postgres::ConnectionConfig, ) -> Result<()> { let config = load_runtime_config(config_path).map_err(map_runtime_config_error)?; + let shared = config + .verify_package_envelope() + .map_err(|error| StartupError::PackageEnvelopeRefused(error.to_string()))?; let package_root = config.package().root().to_path_buf(); let package = { let package_context = config.package_load_context(); - load_package(&package_root, &package_context).map_err(StartupError::PackageRefused)? + load_package_with_verified_envelope(&package_root, &package_context, &shared) + .map_err(StartupError::PackageRefused)? }; prepare_verified_package_with_connection(config, package, connection, AuditOpening::CheckOnly) .await @@ -735,10 +785,14 @@ pub async fn prepare_with_connection_and_key_source_for_test( key_source: Arc, ) -> Result { let config = load_runtime_config(config_path).map_err(map_runtime_config_error)?; + let shared = config + .verify_package_envelope() + .map_err(|error| StartupError::PackageEnvelopeRefused(error.to_string()))?; let package_root = config.package().root().to_path_buf(); let package = { let package_context = config.package_load_context(); - load_package(&package_root, &package_context).map_err(StartupError::PackageRefused)? + load_package_with_verified_envelope(&package_root, &package_context, &shared) + .map_err(StartupError::PackageRefused)? }; prepare_verified_package_with_key_source(config, package, connection, key_source).await } @@ -903,6 +957,11 @@ async fn prepare_database_startup( .await .map_err(|_| StartupError::DatabaseConnection)?; let startup = verify_opened_startup(package, &mut client, migration_role, runtime_role).await?; + // Only the serving runtime checks the claim. Operator tooling opens the + // same verified startup and must keep working on a copy, so the copy can + // be inspected, verified, and adopted. + let pg_client: &Client = &client; + verify_instance_claim(pg_client).await?; let advisories = inspect_baseline(&client, pool.status().max_size).await; drop(client); Ok((pool, startup, advisories)) @@ -1589,9 +1648,27 @@ pub async fn prepare_startup( if !matches!(context.intent, PackageIntent::Startup { .. }) { return Err(StartupError::PackageRefused(PackageError::Binding)); } + let shared = registry_platform_config::package::verify_package( + package_root, + &crate::package::shared_package_limits(), + "bregctl package", + ) + .map_err(|error| StartupError::PackageEnvelopeRefused(error.to_string()))?; // Ordering is security-relevant: no database call precedes package closure, // signature, binding, and compiler-derivation verification. - let package = load_package(package_root, context).map_err(StartupError::PackageRefused)?; + let package = load_package_with_verified_envelope(package_root, context, &shared) + .map_err(StartupError::PackageRefused)?; + prepare_loaded_startup(package, client, migration_role, runtime_role).await +} + +/// Verify database readiness for a package already loaded through the runtime +/// configuration's retained shared envelope. +pub(crate) async fn prepare_loaded_startup( + package: VerifiedPackage, + client: &mut Client, + migration_role: &SqlIdentifier, + runtime_role: &SqlIdentifier, +) -> Result { verify_opened_startup(package, client, migration_role, runtime_role).await } @@ -1752,6 +1829,7 @@ impl DynamicRuntimeReadiness { ) .await .map_err(|_| StartupError::DatabaseUnready)?; + verify_instance_claim(&*transaction).await?; transaction .commit() .await @@ -1769,6 +1847,15 @@ impl DynamicRuntimeReadiness { } } +/// Refuse a database the instance claim does not name, by name. +async fn verify_instance_claim(client: &impl GenericClient) -> Result<()> { + match crate::instance_claim::check(client).await { + crate::instance_claim::ClaimCheck::Current => Ok(()), + crate::instance_claim::ClaimCheck::Mismatch => Err(StartupError::InstanceClaimMismatch), + crate::instance_claim::ClaimCheck::Unavailable => Err(StartupError::DatabaseUnready), + } +} + impl ReadinessProbe for DynamicRuntimeReadiness { fn is_ready(&self) -> ServiceFuture<'_, bool> { Box::pin(async move { self.check().await.is_ok() }) diff --git a/crates/registry-breg/src/tooling.rs b/crates/registry-breg/src/tooling.rs index 5047a078cd..450a9fe076 100644 --- a/crates/registry-breg/src/tooling.rs +++ b/crates/registry-breg/src/tooling.rs @@ -101,7 +101,7 @@ fn classify_change( // The migration stays additive, but it replaces the column check under // an exclusive table lock and validates every row against it. An // encrypted column stores envelopes and carries no check to replace. - Code::FieldVocabularyCodesAdded => { + Code::FieldVocabularyCodesAdded | Code::FieldLengthWidened => { let encrypted = change .target .entity_id @@ -192,9 +192,10 @@ fn access_change_details( | Code::RecipientGroupAdded | Code::RecipientGroupRemoved | Code::RecipientGroupChanged => return recipient_details(baseline, candidate, change), - Code::ActionAdded | Code::ActionChanged | Code::ActionVocabularyCodesAdded => { - return action_details(baseline, candidate, change) - } + Code::ActionAdded + | Code::ActionChanged + | Code::ActionVocabularyCodesAdded + | Code::ActionTargetFieldsWidened => return action_details(baseline, candidate, change), _ => {} } let Some(entity) = change.target.entity_id.as_deref() else { @@ -505,7 +506,10 @@ fn action_details( reason: Some(STEWARD_ISSUER.to_owned()), }); } - if change.code == Code::ActionVocabularyCodesAdded { + if matches!( + change.code, + Code::ActionVocabularyCodesAdded | Code::ActionTargetFieldsWidened + ) { let Some(before) = before else { return details; }; @@ -528,6 +532,11 @@ fn action_details( widened_after.insert(input.id.clone(), codes(input)); } } + // A contract widened only through its target fields accepts no new + // input code, so there is nothing for this detail to show. + if change.code == Code::ActionTargetFieldsWidened && widened_after.is_empty() { + return details; + } details.push(AccessChangeDetail { field: "inputCodes".into(), direction: AccessChangeDirection::ReviewRequired, @@ -778,6 +787,27 @@ mod tests { DiffClassification::LockOrRewriteRisk, ); + let short_note = compiled( + "1", + "internal", + r#",{"id":"note","type":"text","maxLength":80,"classification":"internal"}"#, + "", + "principal", + ); + let long_note = compiled( + "1", + "internal", + r#",{"id":"note","type":"text","maxLength":200,"classification":"internal"}"#, + "", + "principal", + ); + assert_class( + &short_note, + &long_note, + CompiledRegistryChangeCode::FieldLengthWidened, + DiffClassification::LockOrRewriteRisk, + ); + let access = compiled("1", "internal", "", "", "subject"); assert_class( &baseline, diff --git a/crates/registry-breg/src/webhook.rs b/crates/registry-breg/src/webhook.rs index 8745a55290..8215c7f74b 100644 --- a/crates/registry-breg/src/webhook.rs +++ b/crates/registry-breg/src/webhook.rs @@ -50,7 +50,6 @@ use crate::field_encryption::FieldEncryptionService; use crate::hook_handler::{BregHookHandler, HookHandlerRegistry}; use crate::model::CompiledRegistry; use crate::mutation::{HookProposalApplication, HookProposalOutcome, MutationCoordinator}; -use crate::package::load_package; use crate::postgres::{ExpectedRegistryIdentity, RegistryLockKey, RuntimePool}; use crate::runtime_config::load_runtime_config; use crate::startup::{OperationalEvent, WebhookStateTransitionCode}; @@ -94,11 +93,9 @@ pub struct WebhookOperatorService { impl WebhookOperatorService { pub async fn from_runtime_config(path: &Path) -> Result { let config = load_runtime_config(path).map_err(|_| WebhookOperatorError::Unavailable)?; - let package_root = config.package().root().to_path_buf(); - { - let context = config.package_load_context(); - load_package(&package_root, &context).map_err(|_| WebhookOperatorError::Unavailable)?; - } + let package = config + .load_active_package() + .map_err(|_| WebhookOperatorError::Unavailable)?; let connection = config .runtime_database_connection_config() .map_err(|_| WebhookOperatorError::Unavailable)?; @@ -109,10 +106,8 @@ impl WebhookOperatorService { .get() .await .map_err(|_| WebhookOperatorError::Unavailable)?; - let context = config.package_load_context(); - let startup = crate::startup::prepare_startup( - &package_root, - &context, + let startup = crate::startup::prepare_loaded_startup( + package, &mut client, config.database().roles().migration(), config.database().roles().runtime(), diff --git a/crates/registry-breg/tests/action_vocabulary_codes.rs b/crates/registry-breg/tests/action_vocabulary_codes.rs index 40712c23d4..91a386d0d9 100644 --- a/crates/registry-breg/tests/action_vocabulary_codes.rs +++ b/crates/registry-breg/tests/action_vocabulary_codes.rs @@ -253,3 +253,138 @@ fn a_baseline_without_input_vocabularies_keeps_every_widening_under_review() { ); assert!(change_set_to_applicable_migration_plan(&changes).is_err()); } + +/// A raised `text` limit on a field of the entity the actions target, which +/// no action input sets, keeps every request they accepted valid, so each +/// keeps its contract; a lowered limit does not. +#[test] +fn a_raised_text_limit_on_a_targeted_entity_keeps_the_action_contracts() { + const SOURCE_REFERENCE: &str = + "{id: source-reference, type: string, maxLength: 255, classification: internal}"; + let with_limit = |max_length: u32| { + replace_once( + CONSENT_MODULE, + SOURCE_REFERENCE, + &format!( + "{SOURCE_REFERENCE}\n - {{id: review-note, type: text, maxLength: {max_length}, classification: internal}}" + ), + ) + }; + let before = consent(CONSENT_PROJECT, &with_limit(255)); + + let widened = compiled_registry_change_set( + &before, + &consent(CONSENT_PROJECT, &with_limit(1000)), + "prior-package", + ); + let changes = widened + .changes + .iter() + .map(|change| (change.code, change.target.member_id.as_deref())) + .collect::>(); + assert_eq!( + changes[0], + ( + CompiledRegistryChangeCode::FieldLengthWidened, + Some("review-note") + ) + ); + for action in [ + "give-person-consent", + "import-person-consent", + "invalidate-person-consent", + "record-person-consent-assisted", + "refuse-person-consent", + "withdraw-person-consent", + ] { + assert_eq!( + action_change(&widened, action), + ( + CompiledRegistryChangeCode::ActionTargetFieldsWidened, + CompiledRegistryChangeClass::CompatibleAdditive + ) + ); + } + assert_eq!(changes.len(), 7, "{changes:?}"); + assert!(change_set_to_applicable_migration_plan(&widened).is_ok()); + + let narrowed = compiled_registry_change_set( + &before, + &consent(CONSENT_PROJECT, &with_limit(100)), + "prior-package", + ); + assert!( + narrowed + .changes + .iter() + .any(|change| change.code == CompiledRegistryChangeCode::ActionChanged), + "a lowered limit changes the requests an action accepts: {:#?}", + narrowed.changes + ); +} + +#[test] +fn a_lowered_string_minimum_on_a_targeted_entity_keeps_the_action_contracts() { + const SOURCE_REFERENCE: &str = + "{id: source-reference, type: string, maxLength: 255, classification: internal}"; + let with_minimum = |min_length: u32| { + replace_once( + CONSENT_MODULE, + SOURCE_REFERENCE, + &format!( + "{SOURCE_REFERENCE}\n - {{id: review-code, type: string, minLength: {min_length}, maxLength: 64, classification: internal}}" + ), + ) + }; + let before = consent(CONSENT_PROJECT, &with_minimum(8)); + + let lowered = compiled_registry_change_set( + &before, + &consent(CONSENT_PROJECT, &with_minimum(2)), + "prior-package", + ); + let changes = lowered + .changes + .iter() + .map(|change| (change.code, change.target.member_id.as_deref())) + .collect::>(); + assert_eq!( + changes[0], + ( + CompiledRegistryChangeCode::FieldLengthWidened, + Some("review-code") + ) + ); + for action in [ + "give-person-consent", + "import-person-consent", + "invalidate-person-consent", + "record-person-consent-assisted", + "refuse-person-consent", + "withdraw-person-consent", + ] { + assert_eq!( + action_change(&lowered, action), + ( + CompiledRegistryChangeCode::ActionTargetFieldsWidened, + CompiledRegistryChangeClass::CompatibleAdditive + ) + ); + } + assert_eq!(changes.len(), 7, "{changes:?}"); + assert!(change_set_to_applicable_migration_plan(&lowered).is_ok()); + + let raised = compiled_registry_change_set( + &before, + &consent(CONSENT_PROJECT, &with_minimum(16)), + "prior-package", + ); + assert!( + raised + .changes + .iter() + .any(|change| change.code == CompiledRegistryChangeCode::ActionChanged), + "a raised minimum changes the requests an action accepts: {:#?}", + raised.changes + ); +} diff --git a/crates/registry-breg/tests/compiler_contract.rs b/crates/registry-breg/tests/compiler_contract.rs index 498706d626..7008e00b51 100644 --- a/crates/registry-breg/tests/compiler_contract.rs +++ b/crates/registry-breg/tests/compiler_contract.rs @@ -3410,6 +3410,46 @@ registry: assert!(failure.diagnostics()[0].message.contains("kind")); } +#[test] +fn an_authored_project_carrying_an_environment_expression_is_refused() { + let failure = parse_project_yaml( + br#" +apiVersion: registry.registrystack.org/v1alpha1 +kind: RegistryProject +registry: + id: neutral + version: "1" + defaultLanguage: en + canonicalBaseIri: ${AUTHORED_BASE_IRI} +entities: + - id: case + title: "Case ${AUTHORED_TITLE:-default}" +"#, + ) + .expect_err("an environment expression in an authored project is refused"); + let diagnostic = &failure.diagnostics()[0]; + assert_eq!(diagnostic.code, "source.environment_expression"); + assert_eq!(diagnostic.path, "project.registry.canonicalBaseIri"); + assert!(diagnostic.message.contains("runtime.yaml")); + assert!(!diagnostic.message.contains("AUTHORED_BASE_IRI")); + + let module = parse_module_yaml( + br#" +apiVersion: registry.registrystack.org/v1alpha1 +kind: RegistryModule +entities: + - id: case + fields: + - id: label + description: ${AUTHORED_DESCRIPTION} +"#, + ) + .expect_err("an environment expression in an authored module is refused"); + let diagnostic = &module.diagnostics()[0]; + assert_eq!(diagnostic.code, "source.environment_expression"); + assert_eq!(diagnostic.path, "module.entities[0].fields[0].description"); +} + #[test] fn source_parse_diagnostics_name_the_member_the_alternatives_and_the_location() { let unknown_yaml_member = parse_project_yaml( diff --git a/crates/registry-breg/tests/documented_access.rs b/crates/registry-breg/tests/documented_access.rs index 514f5de144..0d124e2545 100644 --- a/crates/registry-breg/tests/documented_access.rs +++ b/crates/registry-breg/tests/documented_access.rs @@ -153,19 +153,24 @@ fn task_profile_scenarios_match_documented_admission_without_claiming_row_access #[cfg(feature = "runtime")] #[test] -fn documented_runtime_config_parses_after_filling_the_package_revision() { +fn documented_runtime_config_parses_after_filling_the_package_revision_and_digest() { let page = include_str!("../../../docs/site/src/content/docs/operate/breg.mdx"); let mut source = fragment( page, "apiVersion: registry.registrystack.org/breg-runtime/v1alpha1", ); - // The operator substitutes the package digest; retain every authored key, + // The operator substitutes the package revision and digest; retain every authored key, // kind, token-verifier setting, and secret reference from the example. assert_eq!( source["package"]["activeRevision"], "sha256:" ); + assert_eq!( + source["package"]["expectedDigest"], + "sha256:" + ); source["package"]["activeRevision"] = json!(format!("sha256:{}", "a".repeat(64))); + source["package"]["expectedDigest"] = json!(format!("sha256:{}", "b".repeat(64))); registry_breg::runtime_config::parse_runtime_config_with_env( &serde_norway::to_string(&source).unwrap(), |_| None, diff --git a/crates/registry-breg/tests/frozen_rhai_package.rs b/crates/registry-breg/tests/frozen_rhai_package.rs index afa35ff8d6..14d5447346 100644 --- a/crates/registry-breg/tests/frozen_rhai_package.rs +++ b/crates/registry-breg/tests/frozen_rhai_package.rs @@ -12,6 +12,11 @@ //! candidate-package inspection, still accepts it for predecessor inspection, //! and still runs its Rhai action handler. //! +//! The frozen directory predates the shared package envelope and carries no +//! `SHA256SUMS`. The loading tests publish an envelope over a temporary copy +//! first, the step an operator takes for such a package; whether an +//! envelope-less signed predecessor should load as is stays open in #1633. +//! //! To deliberately re-freeze the fixture after an approved package-format //! change, delete the directory and run the ignored writer test: //! @@ -34,6 +39,7 @@ use registry_breg::package::{ }; use registry_breg::CompiledRegistry; use registry_platform_canonical_json::canonicalize_json; +use registry_platform_config::package::{write_sum_file, PackageLimits as SharedPackageLimits}; use serde_json::{Map, Value}; use sha2::{Digest, Sha256}; @@ -55,6 +61,45 @@ fn frozen_root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/person-registration-rhai-package") } +/// A temporary copy of the frozen package with the shared envelope published +/// over it, leaving the frozen bytes untouched. +fn enveloped_frozen_copy() -> tempfile::TempDir { + let copy = tempfile::Builder::new() + .prefix("registry-frozen-package-") + .tempdir_in( + std::env::temp_dir() + .canonicalize() + .expect("canonical temporary root"), + ) + .expect("temporary package directory"); + copy_tree(&frozen_root(), copy.path()); + write_sum_file( + copy.path(), + None, + &SharedPackageLimits::default(), + "bregctl package", + ) + .expect("the shared envelope publishes over the frozen package copy"); + copy +} + +fn copy_tree(source: &Path, destination: &Path) { + for entry in fs::read_dir(source).expect("frozen package directory reads") { + let entry = entry.expect("frozen package entry reads"); + let target = destination.join(entry.file_name()); + if entry + .file_type() + .expect("frozen package entry type") + .is_dir() + { + fs::create_dir(&target).expect("package copy directory creates"); + copy_tree(&entry.path(), &target); + } else { + fs::copy(entry.path(), &target).expect("package file copies"); + } + } +} + /// The acceptance project bound to the unsigned local deployment identity the /// frozen package carries. fn local_project() -> registry_breg::contract::RegistryProject { @@ -233,8 +278,9 @@ fn frozen_package_bytes_match_the_current_compiler() { #[test] fn frozen_package_loads_and_runs_its_rhai_handler() { + let package = enveloped_frozen_copy(); let inspected = - inspect_package_integrity(&frozen_root()).expect("the frozen package still verifies"); + inspect_package_integrity(package.path()).expect("the frozen package still verifies"); let action = register_person(inspected.registry()); let handler = action .handler @@ -282,7 +328,8 @@ fn frozen_package_loads_and_runs_its_rhai_handler() { #[test] fn frozen_package_remains_a_readable_predecessor() { - let inspected = inspect_package_integrity(&frozen_root()) + let package = enveloped_frozen_copy(); + let inspected = inspect_package_integrity(package.path()) .expect("the frozen package revision is derived before predecessor binding"); let package_revision = inspected.package_revision().to_owned(); let context = PredecessorPackageContext { @@ -294,7 +341,7 @@ fn frozen_package_remains_a_readable_predecessor() { expected_package_revision: &package_revision, expected_sequence: 1, }; - let predecessor = load_predecessor_package(&frozen_root(), &context) + let predecessor = load_predecessor_package(package.path(), &context) .expect("the frozen package is still accepted for predecessor inspection"); let baseline = predecessor.migration_baseline(); assert_eq!(baseline.package_revision, package_revision); diff --git a/crates/registry-breg/tests/http_read_only.rs b/crates/registry-breg/tests/http_read_only.rs index ec1ae1629e..1a843bf09d 100644 --- a/crates/registry-breg/tests/http_read_only.rs +++ b/crates/registry-breg/tests/http_read_only.rs @@ -3391,6 +3391,7 @@ fn operation_name(operation: Operation) -> &'static str { Operation::ApplyRequest => "apply_request", Operation::Invoke => "invoke", Operation::Snapshot => "snapshot", + Operation::Import => "import", } } diff --git a/crates/registry-breg/tests/import_grant_admission.rs b/crates/registry-breg/tests/import_grant_admission.rs new file mode 100644 index 0000000000..7ce99788be --- /dev/null +++ b/crates/registry-breg/tests/import_grant_admission.rs @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Proves an `import` grant is admitted for create through the durable +//! ingestion-run path only: a client-side plan binds it for create and refuses +//! patch, the legacy direct batch path refuses the plan, and the generated +//! database grants insert the rows without a standing read. + +use std::future::Future; +use std::pin::pin; +use std::task::{Context, Poll, Waker}; + +use registry_breg::compiler::{compile_project, CompileProfile}; +use registry_breg::contract::parse_project_json; +use registry_breg::data::{ + execute_import_chunk, DataError, DataHttpResponse, DataImportCheckpoint, DataImportOperation, + DataImportPlan, +}; +use registry_breg::generated_ddl::{PolicyCommand, TablePrivilege}; +use registry_breg::CompiledRegistry; +use serde_json::{json, Value}; + +fn project(loader_operations: &[&str]) -> Value { + json!({ + "apiVersion":"registry.registrystack.org/v1alpha1", + "kind":"RegistryProject", + "registry":{"id":"import-admission","version":"1","defaultLanguage":"en","canonicalBaseIri":"https://authoring.example.test"}, + "entities":[{ + "id":"enrollment","primaryDataset":"test-dataset","route":"enrollments","mutationMode":"mutable", + "batch":{"maximumItems":2,"maximumBytes":4096}, + "changeControl":{"requiredFor":["patch"]}, + "fields":[{"id":"label","type":"string","maxLength":32,"required":true,"classification":"internal"}] + },{ + "id":"enrollment-change","primaryDataset":"test-dataset","route":"enrollment-changes","mutationMode":"mutable", + "fields":[ + {"id":"enrollment","type":"reference","target":"enrollment","required":true,"classification":"internal"}, + {"id":"label","type":"string","maxLength":32,"required":true,"classification":"internal"} + ], + "changeRequest":{"effects":[{"id":"apply-label","target":{"fromField":"enrollment"},"operation":"patch","set":{"label":{"fromField":"label"}}}], + "review":{"authority":"casework-main","policyId":"request-review"},"onApproved":{"mode":"manual"}} + }], + "accessProfiles":[{ + "id":"loader","principalClaim":"principal","permissions":[{ + "entity":"enrollment","operations":loader_operations,"readableFields":["label"],"writableFields":["label"], + "rowBoundaries": [] + }] + },{ + "id":"reviewer","default":true,"principalClaim":"principal","permissions":[{ + "entity":"enrollment-change","operations":["get","submit_request","apply_request"],"readableFields":["enrollment","label"], + "applyTargets":[{"entity":"enrollment", "rowBoundaries": []}], + "rowBoundaries": [] + }] + }] + }) +} + +fn compile(project: &Value) -> CompiledRegistry { + let source = serde_json::to_vec(project).expect("project serializes"); + let project = parse_project_json(&source).expect("source shape parses"); + compile_project(&project, &[], CompileProfile::Authoring).expect("project compiles") +} + +const INPUT: &[u8] = b"{\"operation\":\"create\",\"data\":{\"label\":\"north\"}}\n"; + +#[test] +fn an_import_grant_plans_a_create_load_and_refuses_patch() { + let registry = compile(&project(&["import"])); + let plan = DataImportPlan::from_jsonl( + ®istry, + "enrollment", + DataImportOperation::Create, + "loader", + INPUT, + ) + .expect("an import grant admits a create plan"); + assert!(plan.through_import()); + assert_eq!(plan.item_count(), 1); + + let patch = DataImportPlan::from_jsonl( + ®istry, + "enrollment", + DataImportOperation::Patch, + "loader", + b"{\"operation\":\"patch\",\"recordId\":\"00000000-0000-4000-8000-000000000001\",\"ifMatch\":\"\\\"breg-1\\\"\",\"patch\":[{\"op\":\"replace\",\"path\":\"/label\",\"value\":\"south\"}]}\n", + ); + assert_eq!( + patch.expect_err("import is create only"), + DataError::InvalidBinding + ); +} + +#[test] +fn an_import_plan_is_refused_on_the_direct_batch_path() { + let registry = compile(&project(&["import"])); + let plan = DataImportPlan::from_jsonl( + ®istry, + "enrollment", + DataImportOperation::Create, + "loader", + INPUT, + ) + .expect("an import grant admits a create plan"); + let mut checkpoint = + DataImportCheckpoint::start(&plan, "package-revision", "schema-fingerprint") + .expect("checkpoint binds"); + let import_id = checkpoint.import_id().to_owned(); + let mut dispatched = false; + let outcome = { + let future = execute_import_chunk( + &plan, + &mut checkpoint, + "package-revision", + "schema-fingerprint", + &import_id, + |_request| { + dispatched = true; + async { Err::(()) } + }, + ); + let mut future = pin!(future); + let Poll::Ready(outcome) = future + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + else { + panic!("the refusal is decided before any request is dispatched"); + }; + outcome + }; + assert_eq!( + outcome.expect_err("an import binding has no raw batch route"), + DataError::InvalidBinding + ); + assert!(!dispatched); +} + +#[test] +fn a_batch_plan_is_not_an_import_plan() { + let mut source = project(&["create", "batch"]); + source["entities"][0] + .as_object_mut() + .expect("entity") + .remove("changeControl"); + source["entities"] + .as_array_mut() + .expect("entities") + .truncate(1); + source["accessProfiles"] + .as_array_mut() + .expect("profiles") + .truncate(1); + source["accessProfiles"][0]["default"] = json!(true); + let registry = compile(&source); + let plan = DataImportPlan::from_jsonl( + ®istry, + "enrollment", + DataImportOperation::Create, + "loader", + INPUT, + ) + .expect("a batch grant admits a create plan"); + assert!(!plan.through_import()); +} + +#[test] +fn an_import_grant_inserts_without_a_standing_read() { + let registry = compile(&project(&["import"])); + let table = registry + .ddl() + .tables + .iter() + .find(|table| table.entity_id == "enrollment") + .expect("enrollment table"); + assert!(table.runtime_privileges.contains(&TablePrivilege::Insert)); + let loader = table + .policies + .iter() + .filter(|policy| policy.access_profile == "loader") + .collect::>(); + assert_eq!( + loader + .iter() + .filter(|policy| policy.command == PolicyCommand::Insert) + .count(), + 1, + "the import profile inserts under its own row policy" + ); + let reads = loader + .iter() + .filter(|policy| policy.command == PolicyCommand::Select) + .collect::>(); + assert_eq!(reads.len(), 1, "only the create-returning read: {reads:?}"); + assert!(reads[0] + .using_expression + .as_deref() + .is_some_and(|expression| expression.contains("registry.created_record_id"))); + assert!(loader + .iter() + .all(|policy| policy.command != PolicyCommand::Update)); +} diff --git a/crates/registry-breg/tests/import_grant_compiler.rs b/crates/registry-breg/tests/import_grant_compiler.rs new file mode 100644 index 0000000000..e38e91b058 --- /dev/null +++ b/crates/registry-breg/tests/import_grant_compiler.rs @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Proves the `import` operation compiles as a create-only, ingestion-run +//! capability that change control does not count as a direct write, and that +//! every `import.*` refusal names the grant it concerns. + +use registry_breg::compiler::{compile_project, CompileProfile}; +use registry_breg::contract::{parse_project_json, Operation}; +use registry_breg::diagnostics::CompileFailure; +use registry_breg::CompiledRegistry; +use serde_json::{json, Value}; + +fn compile(project: &Value) -> Result { + let source = serde_json::to_vec(project).expect("project serializes"); + let project = parse_project_json(&source).expect("source shape parses"); + compile_project(&project, &[], CompileProfile::Authoring) +} + +fn codes(failure: &CompileFailure) -> Vec { + failure + .diagnostics() + .iter() + .map(|diagnostic| diagnostic.code.clone()) + .collect() +} + +fn diagnostic<'a>( + failure: &'a CompileFailure, + code: &str, +) -> &'a registry_breg::diagnostics::Diagnostic { + failure + .diagnostics() + .iter() + .find(|diagnostic| diagnostic.code == code) + .unwrap_or_else(|| panic!("expected {code:?}, got {:?}", codes(failure))) +} + +/// A governed `enrollment` entity loaded by an `import` profile, with a +/// change-request entity whose effect patches enrollments. +fn governed_project(required_for: &[&str], loader_operations: &[&str]) -> Value { + let mut enrollment = json!({ + "id":"enrollment","primaryDataset":"test-dataset","route":"enrollments","mutationMode":"mutable", + "batch":{"maximumItems":10,"maximumBytes":65536}, + "fields":[{"id":"label","type":"string","maxLength":32,"required":true,"classification":"internal"}] + }); + if !required_for.is_empty() { + enrollment["changeControl"] = json!({"requiredFor": required_for}); + } + json!({ + "apiVersion":"registry.registrystack.org/v1alpha1", + "kind":"RegistryProject", + "registry":{"id":"import-grants","version":"1","defaultLanguage":"en","canonicalBaseIri":"https://authoring.example.test"}, + "entities":[enrollment,{ + "id":"enrollment-change","primaryDataset":"test-dataset","route":"enrollment-changes","mutationMode":"mutable", + "fields":[ + {"id":"enrollment","type":"reference","target":"enrollment","required":true,"classification":"internal"}, + {"id":"label","type":"string","maxLength":32,"required":true,"classification":"internal"} + ], + "changeRequest":{"effects":[{"id":"apply-label","target":{"fromField":"enrollment"},"operation":"patch","set":{"label":{"fromField":"label"}}}], + "review":{"authority":"casework-main","policyId":"request-review"},"onApproved":{"mode":"manual"}} + }], + "accessProfiles":[{ + "id":"loader","principalClaim":"principal","permissions":[{ + "entity":"enrollment","operations":loader_operations,"readableFields":["label"],"writableFields":["label"], + "rowBoundaries": [] + }] + },{ + "id":"reviewer","default":true,"principalClaim":"principal","permissions":[{ + "entity":"enrollment-change","operations":["get","submit_request","apply_request"],"readableFields":["enrollment","label"], + "applyTargets":[{"entity":"enrollment", "rowBoundaries": []}], + "rowBoundaries": [] + }] + }] + }) +} + +fn route_ids(registry: &CompiledRegistry) -> Vec { + registry + .routes() + .routes + .iter() + .map(|route| route.id.clone()) + .collect() +} + +#[test] +fn import_is_accepted_on_an_entity_controlled_for_create_and_patch() { + let registry = compile(&governed_project(&["create", "patch"], &["import"])) + .expect("import is not a direct write, so change control accepts it"); + let route = registry + .routes() + .routes + .iter() + .find(|route| route.id == "records.enrollment.import") + .expect("an import grant compiles an import route"); + assert_eq!(route.operation, Operation::Import); + assert_eq!(route.path, "/v1/records/enrollments/ingestion-runs"); + assert_eq!(route.access_profiles, vec!["loader".to_owned()]); + let routes = route_ids(®istry); + assert!( + !routes.contains(&"records.enrollment.create".to_owned()) + && !routes.contains(&"records.enrollment.batch".to_owned()), + "import exposes no item route and no raw batch route: {routes:?}" + ); +} + +#[test] +fn batch_is_still_refused_on_a_controlled_entity_and_the_message_suggests_import() { + let failure = compile(&governed_project( + &["create", "patch"], + &["create", "batch"], + )) + .expect_err("batch remains a direct write"); + let found = diagnostic(&failure, "change_control.direct_write_grant"); + assert_eq!( + found.path, + "entities[id=enrollment].accessProfiles[id=loader].operations" + ); + assert!( + found.message.contains("`import`"), + "the refusal names the governed bulk-load grant: {}", + found.message + ); +} + +#[test] +fn adding_change_control_to_an_imported_entity_removes_no_route() { + let ungoverned = compile(&ungoverned_project()) + .expect("import is allowed on an entity without change control"); + let governed = compile(&governed_project(&["create", "patch"], &["import"])) + .expect("a successor that adds change control keeps the import grant"); + assert_eq!(enrollment_routes(&ungoverned), enrollment_routes(&governed)); + assert_eq!( + enrollment_routes(&governed), + vec!["records.enrollment.import".to_owned()] + ); +} + +/// The first package of the adoption journey: enrollments are loaded through +/// `import` before any change request governs them. +fn ungoverned_project() -> Value { + let mut project = governed_project(&[], &["import"]); + project["entities"] + .as_array_mut() + .expect("entities") + .truncate(1); + project["accessProfiles"] + .as_array_mut() + .expect("profiles") + .truncate(1); + project["accessProfiles"][0]["default"] = json!(true); + project +} + +fn enrollment_routes(registry: &CompiledRegistry) -> Vec { + route_ids(registry) + .into_iter() + .filter(|id| id.starts_with("records.enrollment.")) + .collect() +} + +#[test] +fn import_requires_entity_batch_bounds() { + let mut project = governed_project(&["patch"], &["import"]); + project["entities"][0] + .as_object_mut() + .expect("entity object") + .remove("batch"); + let failure = compile(&project).expect_err("an import run needs chunk bounds"); + assert_eq!( + diagnostic(&failure, "import.batch_bounds.required").path, + "entities[id=enrollment].accessProfiles[id=loader].operations" + ); +} + +#[test] +fn import_refuses_an_anonymous_profile() { + let mut project = ungoverned_project(); + let loader = project["accessProfiles"][0] + .as_object_mut() + .expect("profile object"); + loader.remove("principalClaim"); + loader.insert("anonymous".to_owned(), json!(true)); + let failure = compile(&project).expect_err("runs are creator-scoped"); + assert_eq!( + diagnostic(&failure, "import.principal.required").path, + "entities[id=enrollment].accessProfiles[id=loader].operations" + ); +} + +#[test] +fn import_beside_batch_on_one_entity_is_refused_as_redundant() { + let mut project = ungoverned_project(); + project["accessProfiles"] + .as_array_mut() + .expect("profiles") + .push(json!({ + "id":"bulk-writer","principalClaim":"principal","permissions":[{ + "entity":"enrollment","operations":["create","batch"],"readableFields":["label"],"writableFields":["label"], + "rowBoundaries": [] + }] + })); + let failure = + compile(&project).expect_err("batch beside import leaves the authority bounding nothing"); + assert_eq!( + diagnostic(&failure, "import.batch.redundant").path, + "entities[id=enrollment].accessProfiles[id=loader].operations" + ); + + let mut same = ungoverned_project(); + same["accessProfiles"][0]["permissions"][0]["operations"] = + json!(["create", "batch", "import"]); + let same_profile = + compile(&same).expect_err("one profile holding batch and import is refused too"); + assert!(codes(&same_profile).contains(&"import.batch.redundant".to_owned())); +} + +#[test] +fn import_is_unavailable_on_a_change_request_entity() { + let mut project = governed_project(&["patch"], &["import"]); + project["accessProfiles"][1]["permissions"][0]["operations"] = + json!(["get", "submit_request", "apply_request", "import"]); + project["entities"][1]["batch"] = json!({"maximumItems":10,"maximumBytes":65536}); + let failure = compile(&project).expect_err("request drafts are authored, never imported"); + assert!( + codes(&failure).contains(&"access_profile.operation.unavailable".to_owned()), + "{:?}", + codes(&failure) + ); +} + +#[test] +fn import_is_a_forbidden_direct_mutation_for_a_task_grant_profile() { + let mut project = ungoverned_project(); + project["accessProfiles"][0]["taskGrant"] = json!({"sourceIssuer":"https://casework.example"}); + let failure = compile(&project).expect_err("a task grant cannot load records"); + assert!( + codes(&failure).contains(&"access_profile.task_grant.direct_mutation_forbidden".to_owned()), + "{:?}", + codes(&failure) + ); +} diff --git a/crates/registry-breg/tests/migration_plan.rs b/crates/registry-breg/tests/migration_plan.rs index 6872c4a864..0d49d6ef43 100644 --- a/crates/registry-breg/tests/migration_plan.rs +++ b/crates/registry-breg/tests/migration_plan.rs @@ -23,6 +23,9 @@ use registry_breg::package::{ }; use registry_breg::CompiledRegistry; use registry_platform_canonical_json::canonicalize_json; +use registry_platform_config::package::{ + write_sum_file, PackageLimits as SharedPackageLimits, REVISION_FILE, SUM_FILE, +}; use serde::Serialize; use sha2::{Digest, Sha256}; @@ -476,6 +479,7 @@ fn reviewed_migration_plan_rejects_uncovered_changes_forbidden_sql_and_unbound_e b"SELECT 'source-path-record-sql-canary'", ) .expect("tamper reviewed SQL"); + refresh_shared_package_envelope(&package); assert_eq!( inspect_package_integrity(&package).err(), Some(PackageError::Integrity), @@ -630,6 +634,43 @@ fn reviewed_encryption_flip_refuses_a_chunk_size_beyond_the_commit_budget() { ); } +#[test] +fn reviewed_chunked_backfill_refuses_a_chunk_size_beyond_the_commit_budget() { + let previous = compile_variant(Variant::Base, 1); + let candidate = compile_variant(Variant::RequiredField, 2); + let mut artifacts = backfill_artifacts("required-field", &previous, &candidate); + // Every chunk journals the rows it changed as one history commit, so a + // chunked backfill shares the commit-member budget too. + let ReviewedMigrationStepDescriptor::ChunkedBackfill { chunk_size, .. } = + &mut artifacts.descriptor.steps[0] + else { + panic!("the backfill step is chunked"); + }; + *chunk_size = 1_000; + artifacts.rebind(); + prepare_reviewed_package( + Variant::RequiredField, + previous.clone(), + vec![artifacts.source()], + ) + .expect("a chunk size at the commit-member budget prepares"); + + let ReviewedMigrationStepDescriptor::ChunkedBackfill { chunk_size, .. } = + &mut artifacts.descriptor.steps[0] + else { + panic!("the backfill step is chunked"); + }; + *chunk_size = 1_001; + artifacts.rebind(); + assert_refused( + Variant::RequiredField, + previous, + vec![artifacts.source()], + ReviewedMigrationError::Descriptor, + "a chunked backfill chunk size beyond the commit-member budget", + ); +} + #[test] fn reviewed_encryption_flip_refuses_multiple_backfill_steps_for_one_entity() { let previous = compile_variant(Variant::EncryptedBase, 1); @@ -1266,3 +1307,28 @@ fn digest(bytes: &[u8]) -> String { } result } + +/// Republish the shared package envelope over a test-authored change, so the +/// refusal under test comes from the registry package check rather than from +/// the checksum file. +fn refresh_shared_package_envelope(root: &std::path::Path) { + let revision_path = root.join(REVISION_FILE); + let revision = revision_path.exists().then(|| { + fs::read_to_string(&revision_path) + .expect("shared package revision reads") + .strip_suffix('\n') + .expect("shared package revision has one trailing newline") + .to_owned() + }); + fs::remove_file(root.join(SUM_FILE)).expect("old shared package checksum file removes"); + if revision.is_some() { + fs::remove_file(revision_path).expect("old shared package revision file removes"); + } + write_sum_file( + root, + revision.as_deref(), + &SharedPackageLimits::default(), + "bregctl package", + ) + .expect("test-authored shared package envelope republishes"); +} diff --git a/crates/registry-breg/tests/package_change_plan.rs b/crates/registry-breg/tests/package_change_plan.rs index 2875be7d53..60dde9022a 100644 --- a/crates/registry-breg/tests/package_change_plan.rs +++ b/crates/registry-breg/tests/package_change_plan.rs @@ -34,6 +34,10 @@ use registry_breg::package::{ use registry_breg::CompiledRegistry; use registry_platform_canonical_json::canonicalize_json; #[cfg(feature = "tooling")] +use registry_platform_config::{ + write_sum_file, PackageLimits as SharedPackageLimits, REVISION_FILE, SUM_FILE, +}; +#[cfg(feature = "tooling")] use serde::Serialize; #[cfg(feature = "tooling")] use serde_json::json; @@ -181,6 +185,7 @@ fn project_rhai_planner_package_is_deterministic_and_rederives_exact_source() { b"fn plan(ctx) { #{ disposition: \"apply\", effects: [] } }\n", ) .unwrap(); + refresh_shared_package_envelope(&tampered_package); assert_eq!( inspect_package_integrity(&tampered_package) .err() @@ -208,6 +213,7 @@ fn project_rhai_planner_package_is_deterministic_and_rederives_exact_source() { .role = PackageFileRole::SourceModulePlannerScript; envelope.signed.package_revision = derive_package_revision(&envelope.signed).unwrap(); fs::write(&manifest_path, canonical(&envelope)).unwrap(); + refresh_shared_package_envelope(&role_swapped_package); assert_eq!( inspect_package_integrity(&role_swapped_package) .err() @@ -807,6 +813,10 @@ fn vocabulary_code_additions_are_additive_and_replace_the_column_check() { sql.contains("'archived'") && sql.contains("DROP CONSTRAINT"), "the statement replaces the column check with the candidate codes: {sql}" ); + assert!( + sql.starts_with("DO $breg_vocabulary$\n") && sql.ends_with("$breg_vocabulary$"), + "the vocabulary statement keeps its quoting tag: {sql}" + ); for (candidate, reason) in [ ( @@ -835,6 +845,137 @@ fn vocabulary_code_additions_are_additive_and_replace_the_column_check() { } } +#[test] +fn text_length_widening_is_additive_and_replaces_the_length_check() { + for pattern in [None, Some("^[a-z ]*$")] { + let previous = length_registry("text", 80, pattern); + let candidate = length_registry("text", 200, pattern); + let change_set = compiled_registry_change_set(&previous, &candidate, PRIOR_REVISION); + + assert_eq!(change_set.changes.len(), 1, "{:#?}", change_set.changes); + assert_change( + &change_set, + CompiledRegistryChangeClass::CompatibleAdditive, + CompiledRegistryChangeCode::FieldLengthWidened, + ); + let plan = change_set_to_applicable_migration_plan(&change_set) + .expect("a raised text length limit is applicable"); + let ids = plan + .statements + .iter() + .map(|statement| statement.id.as_str()) + .collect::>(); + assert_eq!(ids, vec!["entity.entry.field.note.length"]); + let sql = &plan.statements[0].sql; + assert!( + sql.contains("<= 200") && sql.contains("DROP CONSTRAINT"), + "the statement replaces the length check with the candidate limit: {sql}" + ); + assert!( + sql.starts_with("DO $breg_length$\n") && sql.ends_with("$breg_length$"), + "the statement's quoting tag names the check it replaces: {sql}" + ); + assert_eq!( + sql.contains("breg_pattern_"), + pattern.is_some(), + "a field pattern's check is excluded from the replaced check: {sql}" + ); + } + + for (previous, candidate, reason) in [ + ( + length_registry("text", 80, None), + length_registry("text", 40, None), + "a lowered text limit can strand stored values", + ), + ( + length_registry("string", 80, None), + length_registry("string", 200, None), + "a string maxLength is its varchar column type", + ), + ] { + let change_set = compiled_registry_change_set(&previous, &candidate, PRIOR_REVISION); + assert!( + change_set.changes.iter().any(|change| { + change.class == CompiledRegistryChangeClass::DestructiveOrIrreversible + && change.code == CompiledRegistryChangeCode::FieldTypeChanged + }), + "{reason}: {:#?}", + change_set.changes + ); + assert_eq!(change_set.migration_plan, None, "{reason}"); + } +} + +#[test] +fn string_minimum_lowering_is_additive_and_replaces_or_drops_the_length_check() { + for pattern in [None, Some("^[a-z ]*$")] { + for (lowered, expected, drops) in [(2, ">= 2", false), (0, "DROP CONSTRAINT %I'", true)] { + let previous = string_length_registry(5, 80, pattern); + let candidate = string_length_registry(lowered, 80, pattern); + let change_set = compiled_registry_change_set(&previous, &candidate, PRIOR_REVISION); + + assert_eq!(change_set.changes.len(), 1, "{:#?}", change_set.changes); + assert_change( + &change_set, + CompiledRegistryChangeClass::CompatibleAdditive, + CompiledRegistryChangeCode::FieldLengthWidened, + ); + let plan = change_set_to_applicable_migration_plan(&change_set) + .expect("a lowered string minimum is applicable"); + let ids = plan + .statements + .iter() + .map(|statement| statement.id.as_str()) + .collect::>(); + assert_eq!(ids, vec!["entity.entry.field.note.length"]); + let sql = &plan.statements[0].sql; + assert!( + sql.contains(expected), + "the statement lowers the minimum to {lowered}: {sql}" + ); + assert_eq!( + sql.contains("ADD CONSTRAINT"), + !drops, + "a zero minimum drops the check a fresh install never creates: {sql}" + ); + assert!( + sql.starts_with("DO $breg_length$\n") && sql.ends_with("$breg_length$"), + "the statement's quoting tag names the check it replaces: {sql}" + ); + assert_eq!( + sql.contains("breg_pattern_"), + pattern.is_some(), + "a field pattern's check is excluded from the replaced check: {sql}" + ); + } + } + + for (previous, candidate, reason) in [ + ( + string_length_registry(2, 80, None), + string_length_registry(5, 80, None), + "a raised string minimum can strand stored values", + ), + ( + string_length_registry(5, 80, None), + string_length_registry(2, 200, None), + "a string maxLength is its varchar column type", + ), + ] { + let change_set = compiled_registry_change_set(&previous, &candidate, PRIOR_REVISION); + assert!( + change_set.changes.iter().any(|change| { + change.class == CompiledRegistryChangeClass::DestructiveOrIrreversible + && change.code == CompiledRegistryChangeCode::FieldTypeChanged + }), + "{reason}: {:#?}", + change_set.changes + ); + assert_eq!(change_set.migration_plan, None, "{reason}"); + } +} + #[test] fn plaintext_to_encrypted_type_change_is_unsupported() { let previous = compile_variant(Variant::PlaintextSecret, 1); @@ -1438,6 +1579,75 @@ fn vocabulary_registry(vocabulary: &str, values: &[&str]) -> CompiledRegistry { compile_project(&project, &[], CompileProfile::Authoring).expect("vocabulary fixture compiles") } +fn length_registry(field_type: &str, max_length: u32, pattern: Option<&str>) -> CompiledRegistry { + let mut field = serde_json::json!({ + "id": "note", + "type": field_type, + "maxLength": max_length, + "required": true, + "classification": "internal" + }); + if let Some(pattern) = pattern { + field["pattern"] = serde_json::json!(pattern); + } + field_registry(field) +} + +/// A `string` field with the given `minLength` and `maxLength`. +fn string_length_registry( + min_length: u32, + max_length: u32, + pattern: Option<&str>, +) -> CompiledRegistry { + let mut field = serde_json::json!({ + "id": "note", + "type": "string", + "minLength": min_length, + "maxLength": max_length, + "required": true, + "classification": "internal" + }); + if let Some(pattern) = pattern { + field["pattern"] = serde_json::json!(pattern); + } + field_registry(field) +} + +fn field_registry(field: serde_json::Value) -> CompiledRegistry { + let project = serde_json::json!({ + "apiVersion": "registry.registrystack.org/v1alpha1", + "kind": "RegistryProject", + "registry": { + "id": "length-catalog", + "version": "1", + "defaultLanguage": "en", + "canonicalBaseIri": "https://authoring.example.test" + }, + "entities": [{ + "id": "entry", + "primaryDataset": "test-dataset", + "route": "entries", + "mutationMode": "mutable", + "fields": [field] + }], + "accessProfiles": [{ + "id": "writer", + "default": true, + "principalClaim": "registry_principal", + "permissions": [{ + "entity": "entry", + "operations": ["create", "get", "list", "patch"], + "readableFields": ["note"], + "writableFields": ["note"], + "rowBoundaries": [] + }] + }] + }); + let bytes = serde_json::to_vec(&project).expect("fixture serializes"); + let project = parse_project_json(&bytes).expect("length fixture parses"); + compile_project(&project, &[], CompileProfile::Authoring).expect("length fixture compiles") +} + fn assert_change( change_set: ®istry_breg::package::CompiledRegistryChangeSet, class: CompiledRegistryChangeClass, @@ -2494,6 +2704,32 @@ fn inspect_prepared( inspect_package_integrity(&package).expect("package inspects") } +/// Republish the shared package envelope over a test-authored change, so the +/// refusal under test comes from the registry package check rather than from +/// the checksum file. +#[cfg(feature = "tooling")] +fn refresh_shared_package_envelope(root: &std::path::Path) { + let revision_path = root.join(REVISION_FILE); + let revision = revision_path.exists().then(|| { + fs::read_to_string(&revision_path) + .expect("shared package revision reads") + .strip_suffix('\n') + .expect("shared package revision has one trailing newline") + .to_owned() + }); + fs::remove_file(root.join(SUM_FILE)).expect("old shared package checksum file removes"); + if revision.is_some() { + fs::remove_file(revision_path).expect("old shared package revision file removes"); + } + write_sum_file( + root, + revision.as_deref(), + &SharedPackageLimits::default(), + "bregctl package", + ) + .expect("test-authored shared package envelope republishes"); +} + #[cfg(feature = "tooling")] fn canonical(value: &impl Serialize) -> Vec { canonicalize_json(&serde_json::to_value(value).expect("value serializes")) diff --git a/crates/registry-breg/tests/postgres_change_requests.rs b/crates/registry-breg/tests/postgres_change_requests.rs index 44ddb25a6f..637b540aaa 100644 --- a/crates/registry-breg/tests/postgres_change_requests.rs +++ b/crates/registry-breg/tests/postgres_change_requests.rs @@ -345,6 +345,238 @@ async fn review_submissions_bind_the_subject_to_the_registrys_request_entity() { database.cleanup().await; } +#[tokio::test] +async fn an_operator_resubmits_or_closes_a_review_its_authority_lost() { + use registry_breg::review_recovery::{ + ReviewRecoveryError, ReviewRecoveryOperatorService, ReviewRecoveryRefusal, + ReviewRecoveryScope, + }; + let database = TestDatabase::create(8).await; + let mut source = serde_json::to_value(two_stage_project()).unwrap(); + source["entities"][2]["changeRequest"]["review"] = + json!({"authority":"casework-a","policyId":"correction-review"}); + let registry = Arc::new( + compile_project( + &parse_project_json(&serde_json::to_vec(&source).unwrap()).unwrap(), + &[], + CompileProfile::Authoring, + ) + .unwrap(), + ); + let identity = install_registry(&database, ®istry, "review-recovery", false).await; + let (service, _) = change_request_service_with_evidence_options( + &database, + registry.clone(), + identity.clone(), + "review-recovery", + None, + None, + registry_breg::attachment_storage::AttachmentStorage::Database, + None, + registry_breg::attachment_verification::AttachmentVerification::Disabled, + None, + Some(review_authority_registry( + "http://127.0.0.1:9/" + .parse() + .expect("unroutable authority URL"), + )), + ); + let app = router(service); + let (request, _digest) = submit_two_stage_correction(&app).await; + let request_id = Uuid::parse_str(&request.id).expect("request id"); + let recovery = ReviewRecoveryOperatorService::over_retention_service_for_test( + registry_breg::request_retention::RequestRetentionOperatorService::new_for_test( + registry.as_ref().clone(), + identity, + registry_breg::postgres::ExpectedManagedCatalog::compiled(®istry), + RegistryLockKey::derive("review-recovery").unwrap(), + database.migration_config.clone(), + database.migration_role.clone(), + database.runtime_role.clone(), + database + .audit(AuditProfile::production_from_secret_bytes(vec![0x9b; 32].into()).unwrap()), + ), + ); + let scope = || ReviewRecoveryScope { + request_entity_id: "correction-request", + request_id, + proposal_version: 1, + }; + let lost_binding = json!({"requestId": Uuid::from_u128(0xf1)}); + let submission = |columns: &'static str| { + let admin = &database.admin; + async move { + admin + .query_one( + &format!( + "SELECT {columns} + FROM registry_internal.registry_request_review_submissions + WHERE request_entity_id='correction-request' AND request_id=$1" + ), + &[&request_id], + ) + .await + .expect("review submission") + } + }; + let key_before: String = submission("idempotency_key").await.get(0); + + // A pending submission was never accepted, so neither operation applies. + for refused in [ + recovery.resubmit(scope()).await, + recovery.close(scope()).await, + ] { + assert_eq!( + refused, + Err(ReviewRecoveryError::Ineligible { + reason: ReviewRecoveryRefusal::SubmissionState, + state: "pending".to_owned(), + code: None, + }) + ); + } + + // The restored review environment answers the accepted review as unknown. + let lose = || async { + database + .admin + .execute( + "UPDATE registry_internal.registry_request_review_submissions + SET state='accepted',accepted_binding=$2,result_poll_attempts=40, + attempt_count=3,last_error_code='result-unknown-to-authority', + created_at=transaction_timestamp()-interval '40 days', + recovery_deadline=transaction_timestamp()-interval '10 days' + WHERE request_entity_id='correction-request' AND request_id=$1", + &[&request_id, &lost_binding], + ) + .await + .expect("lose the accepted review") + }; + lose().await; + let resubmitted = recovery.resubmit(scope()).await.expect("resubmit"); + assert_eq!(resubmitted.previous_state, "accepted"); + assert_eq!( + resubmitted.previous_code.as_deref(), + Some("result-unknown-to-authority") + ); + assert_eq!(resubmitted.state, "pending"); + let row = submission( + "state,accepted_binding IS NULL,attempt_count,result_poll_attempts,last_error_code, + idempotency_key,recovery_deadline > transaction_timestamp()+interval '29 days', + next_attempt_at <= transaction_timestamp()", + ) + .await; + assert_eq!(row.get::<_, String>(0), "pending"); + assert!(row.get::<_, bool>(1), "the lost binding is released"); + assert_eq!(row.get::<_, i32>(2), 0); + assert_eq!(row.get::<_, i32>(3), 0); + assert_eq!(row.get::<_, Option>(4), None); + assert_eq!( + row.get::<_, String>(5), + key_before, + "the exact retained request replays under its original idempotency key" + ); + assert!( + row.get::<_, bool>(6), + "the configured recovery window restarts" + ); + assert!(row.get::<_, bool>(7), "the submission is due at once"); + + // Closing records an operator decision and keeps the binding. + lose().await; + let closed = recovery.close(scope()).await.expect("close"); + assert_eq!( + (closed.state, closed.code), + ("failed", Some("operator-closed")) + ); + let row = submission("state,last_error_code,accepted_binding").await; + assert_eq!(row.get::<_, String>(0), "failed"); + assert_eq!( + row.get::<_, Option>(1).as_deref(), + Some("operator-closed") + ); + assert_eq!(row.get::<_, Value>(2), lost_binding); + assert_eq!( + recovery.close(scope()).await, + Err(ReviewRecoveryError::Ineligible { + reason: ReviewRecoveryRefusal::SubmissionState, + state: "failed".to_owned(), + code: Some("operator-closed".to_owned()), + }) + ); + // A closed review may still be resubmitted if the operator changes course. + assert_eq!( + recovery + .resubmit(scope()) + .await + .expect("resubmit closed") + .state, + "pending" + ); + + // A result already recorded settles the review for good. + lose().await; + database + .admin + .execute( + "INSERT INTO registry_internal.registry_request_review_results + (request_entity_id,request_id,proposal_version,authority,result_id,result,status, + completed_at,available_until) + VALUES ('correction-request',$1,1,'casework-a',$2,'{}'::jsonb,'approved', + now(),now()+interval '1 day')", + &[&request_id, &Uuid::from_u128(0xf2)], + ) + .await + .expect("recorded result"); + assert!(matches!( + recovery.resubmit(scope()).await, + Err(ReviewRecoveryError::Ineligible { + reason: ReviewRecoveryRefusal::ResultRecorded, + .. + }) + )); + assert_eq!( + recovery + .close(ReviewRecoveryScope { + request_id: Uuid::from_u128(0xf3), + ..scope() + }) + .await, + Err(ReviewRecoveryError::NotFound) + ); + + // Every recovery writes a request entry before its transaction and one + // response after it; a committed one names the request only by its keyed + // reference. + database.assert_every_audit_request_answered_once(); + let recoveries: Vec = database + .audit_entries() + .into_iter() + .filter(|entry| entry["record"]["kind"] == "reviewRecovery") + .collect(); + let mut committed: Vec<&Value> = recoveries + .iter() + .filter(|entry| entry["phase"] == "response" && entry["record"]["outcome"] == "committed") + .map(|entry| &entry["record"]) + .collect(); + committed.sort_by_key(|record| record["operation"].to_string()); + let operations: Vec = committed + .iter() + .map(|record| record["operation"].to_string()) + .collect(); + assert_eq!(operations, [r#""close""#, r#""resubmit""#, r#""resubmit""#]); + for record in committed { + assert_eq!(record["entityId"], "correction-request"); + assert_eq!(record["authority"], "casework-a"); + } + for entry in &recoveries { + assert_eq!(entry["schema"], "breg-audit/v2"); + assert!(!entry.to_string().contains(&request.id)); + } + + database.cleanup().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn expired_online_review_blocks_final_automatic_attempt_until_authorized_recovery() { let (endpoint, authority_state, authority_server) = serve_review_result_authority().await; diff --git a/crates/registry-breg/tests/postgres_compiled_schema.rs b/crates/registry-breg/tests/postgres_compiled_schema.rs index 1ffe6ded2a..f595bbf742 100644 --- a/crates/registry-breg/tests/postgres_compiled_schema.rs +++ b/crates/registry-breg/tests/postgres_compiled_schema.rs @@ -420,6 +420,271 @@ async fn optional_field_additive_upgrade_matches_fresh_catalog_fingerprint_despi fresh.cleanup().await; } +/// A raised `text` limit replaces the inline length check alone: stored +/// rows stay, the higher limit holds, and the field pattern's own check stays +/// in force, so the upgraded catalog matches a fresh install. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn text_length_widening_replaces_the_length_check_and_keeps_existing_rows() { + let previous = length_catalog_registry(8); + let candidate = length_catalog_registry(16); + let change_set = compiled_registry_change_set(&previous, &candidate, PRIOR_REVISION); + assert_eq!(change_set.changes.len(), 1, "{:?}", change_set.changes); + assert_change( + &change_set, + CompiledRegistryChangeClass::CompatibleAdditive, + CompiledRegistryChangeCode::FieldLengthWidened, + ); + let plan = change_set_to_applicable_migration_plan(&change_set) + .expect("a raised text length limit is compiler-applicable"); + + let entity = &candidate.entities()["entry"]; + let table = quote_identifier(&entity.physical_table); + let note = quote_identifier(&entity.fields["note"].physical_name); + let insert = format!( + "INSERT INTO registry_data.{table} + (record_id, active_package_revision, {note}) + VALUES ($1::text::uuid, 'length-package-1', $2)" + ); + + let upgraded = TestDatabase::create(1).await; + let (upgraded_migration, upgraded_task) = upgraded.connect_migration().await; + install_compiled_schema(&upgraded_migration, &previous, &upgraded.runtime_role) + .await + .expect("previous schema installs"); + // The administrator writes below the row policies: this test observes + // the column constraints alone. + upgraded + .admin + .execute(&insert, &[&RECORD_ALPHA, &"eightchr"]) + .await + .expect("a value at the previous limit is stored"); + assert!( + upgraded + .admin + .execute(&insert, &[&RECORD_BETA, &"twelve chars"]) + .await + .is_err(), + "the previous check refuses a value above its limit" + ); + for statement in &plan.statements { + upgraded_migration + .batch_execute(&statement.sql) + .await + .expect("compiler-produced length statement applies"); + } + upgraded + .admin + .execute(&insert, &[&RECORD_BETA, &"twelve chars"]) + .await + .expect("the replaced check accepts a value under the higher limit"); + for (record, value, reason) in [ + ( + "00000000-0000-0000-0000-000000000203", + "seventeen chars x", + "the replaced check still refuses a value above the higher limit", + ), + ( + "00000000-0000-0000-0000-000000000204", + "UPPER", + "the field pattern's check stays in force", + ), + ] { + assert!( + upgraded + .admin + .execute(&insert, &[&record, &value]) + .await + .is_err(), + "{reason}" + ); + } + let kept: Vec<(String, String)> = upgraded + .admin + .query( + &format!( + "SELECT record_id::text, {note} FROM registry_data.{table} ORDER BY record_id" + ), + &[], + ) + .await + .expect("stored rows are readable") + .into_iter() + .map(|row| (row.get(0), row.get(1))) + .collect(); + assert_eq!( + kept, + [ + (RECORD_ALPHA.to_owned(), "eightchr".to_owned()), + (RECORD_BETA.to_owned(), "twelve chars".to_owned()), + ] + ); + let candidate_catalog = ExpectedManagedCatalog::compiled(&candidate); + let upgraded_fingerprint = managed_schema_fingerprint( + &upgraded_migration, + &upgraded.runtime_role, + &candidate_catalog, + ) + .await + .expect("upgraded candidate catalog is fingerprinted"); + upgraded_task.abort(); + + let fresh = TestDatabase::create(1).await; + let (fresh_migration, fresh_task) = fresh.connect_migration().await; + install_compiled_schema(&fresh_migration, &candidate, &fresh.runtime_role) + .await + .expect("candidate schema installs cleanly"); + let fresh_fingerprint = + managed_schema_fingerprint(&fresh_migration, &fresh.runtime_role, &candidate_catalog) + .await + .expect("fresh candidate catalog is fingerprinted"); + fresh_task.abort(); + assert_eq!( + upgraded_fingerprint, fresh_fingerprint, + "the replaced check keeps the name and definition a fresh install gives it" + ); + + upgraded.cleanup().await; + fresh.cleanup().await; +} + +/// A lowered `string` minimum replaces the inline length check, or drops it +/// when the minimum falls to zero: stored rows stay, the `varchar` bound and +/// the field pattern's own check stay in force, and the upgraded catalog +/// matches a fresh install. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn string_minimum_lowering_replaces_or_drops_the_length_check_and_keeps_existing_rows() { + for (lowered, short) in [(2, "ab"), (0, "a")] { + let previous = string_length_catalog_registry(5); + let candidate = string_length_catalog_registry(lowered); + let change_set = compiled_registry_change_set(&previous, &candidate, PRIOR_REVISION); + assert_eq!(change_set.changes.len(), 1, "{:?}", change_set.changes); + assert_change( + &change_set, + CompiledRegistryChangeClass::CompatibleAdditive, + CompiledRegistryChangeCode::FieldLengthWidened, + ); + let plan = change_set_to_applicable_migration_plan(&change_set) + .expect("a lowered string minimum is compiler-applicable"); + + let entity = &candidate.entities()["entry"]; + let table = quote_identifier(&entity.physical_table); + let note = quote_identifier(&entity.fields["note"].physical_name); + let insert = format!( + "INSERT INTO registry_data.{table} + (record_id, active_package_revision, {note}) + VALUES ($1::text::uuid, 'length-package-1', $2)" + ); + + let upgraded = TestDatabase::create(1).await; + let (upgraded_migration, upgraded_task) = upgraded.connect_migration().await; + install_compiled_schema(&upgraded_migration, &previous, &upgraded.runtime_role) + .await + .expect("previous schema installs"); + // The administrator writes below the row policies: this test observes + // the column constraints alone. + upgraded + .admin + .execute(&insert, &[&RECORD_ALPHA, &"abcde"]) + .await + .expect("a value at the previous minimum is stored"); + assert!( + upgraded + .admin + .execute(&insert, &[&RECORD_BETA, &short]) + .await + .is_err(), + "the previous check refuses a value below its minimum" + ); + for statement in &plan.statements { + upgraded_migration + .batch_execute(&statement.sql) + .await + .expect("compiler-produced length statement applies"); + } + upgraded + .admin + .execute(&insert, &[&RECORD_BETA, &short]) + .await + .expect("the relaxed check accepts a value at the lower minimum"); + let mut refused = vec![ + ( + "00000000-0000-0000-0000-000000000203", + "seventeen chars x", + "the varchar bound still refuses a value above maxLength", + ), + ( + "00000000-0000-0000-0000-000000000204", + "UPPER", + "the field pattern's check stays in force", + ), + ]; + if lowered > 0 { + refused.push(( + "00000000-0000-0000-0000-000000000205", + "a", + "the replaced check still refuses a value below the lower minimum", + )); + } + for (record, value, reason) in refused { + assert!( + upgraded + .admin + .execute(&insert, &[&record, &value]) + .await + .is_err(), + "{reason}" + ); + } + let kept: Vec<(String, String)> = upgraded + .admin + .query( + &format!( + "SELECT record_id::text, {note} FROM registry_data.{table} ORDER BY record_id" + ), + &[], + ) + .await + .expect("stored rows are readable") + .into_iter() + .map(|row| (row.get(0), row.get(1))) + .collect(); + assert_eq!( + kept, + [ + (RECORD_ALPHA.to_owned(), "abcde".to_owned()), + (RECORD_BETA.to_owned(), short.to_owned()), + ] + ); + let candidate_catalog = ExpectedManagedCatalog::compiled(&candidate); + let upgraded_fingerprint = managed_schema_fingerprint( + &upgraded_migration, + &upgraded.runtime_role, + &candidate_catalog, + ) + .await + .expect("upgraded candidate catalog is fingerprinted"); + upgraded_task.abort(); + + let fresh = TestDatabase::create(1).await; + let (fresh_migration, fresh_task) = fresh.connect_migration().await; + install_compiled_schema(&fresh_migration, &candidate, &fresh.runtime_role) + .await + .expect("candidate schema installs cleanly"); + let fresh_fingerprint = + managed_schema_fingerprint(&fresh_migration, &fresh.runtime_role, &candidate_catalog) + .await + .expect("fresh candidate catalog is fingerprinted"); + fresh_task.abort(); + assert_eq!( + upgraded_fingerprint, fresh_fingerprint, + "a lowered minimum of {lowered} leaves the catalog a fresh install gives" + ); + + upgraded.cleanup().await; + fresh.cleanup().await; + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn vocabulary_code_addition_replaces_the_check_and_keeps_existing_rows() { let previous = vocabulary_catalog_registry(&["open", "closed"]); @@ -1449,6 +1714,83 @@ fn additive_catalog_registry(variant: AdditiveCatalogVariant) -> registry_breg:: /// A plain registry with two vocabulary-code fields. Only the `status` /// vocabulary varies, so a migration that touched `kind` would show. +/// One required `text` field with a pattern, bounded by `max_length`. +fn length_catalog_registry(max_length: u32) -> registry_breg::CompiledRegistry { + let project = serde_json::json!({ + "apiVersion": "registry.registrystack.org/v1alpha1", + "kind": "RegistryProject", + "registry": { + "id": "length-catalog", + "version": "1", + "defaultLanguage": "en", "canonicalBaseIri": "https://authoring.example.test" + }, + "entities": [{ + "id": "entry", + "primaryDataset": "test-dataset", + "route": "entries", + "mutationMode": "mutable", + "fields": [ + {"id": "note", "type": "text", "maxLength": max_length, "pattern": "^[a-z ]*$", + "required": true, "classification": "internal"} + ] + }], + "accessProfiles": [{ + "id": "writer", + "default": true, + "principalClaim": "registry_principal", + "permissions": [{ + "entity": "entry", + "operations": ["create", "get", "list", "patch"], + "readableFields": ["note"], + "writableFields": ["note"], + "rowBoundaries": [] + }] + }] + }); + let project_bytes = serde_json::to_vec(&project).expect("fixture serializes"); + let project = parse_project_json(&project_bytes).expect("length catalog fixture parses"); + compile_project(&project, &[], CompileProfile::Authoring) + .expect("length catalog fixture compiles") +} + +fn string_length_catalog_registry(min_length: u32) -> registry_breg::CompiledRegistry { + let project = serde_json::json!({ + "apiVersion": "registry.registrystack.org/v1alpha1", + "kind": "RegistryProject", + "registry": { + "id": "length-catalog", + "version": "1", + "defaultLanguage": "en", "canonicalBaseIri": "https://authoring.example.test" + }, + "entities": [{ + "id": "entry", + "primaryDataset": "test-dataset", + "route": "entries", + "mutationMode": "mutable", + "fields": [ + {"id": "note", "type": "string", "minLength": min_length, "maxLength": 16, + "pattern": "^[a-z ]*$", "required": true, "classification": "internal"} + ] + }], + "accessProfiles": [{ + "id": "writer", + "default": true, + "principalClaim": "registry_principal", + "permissions": [{ + "entity": "entry", + "operations": ["create", "get", "list", "patch"], + "readableFields": ["note"], + "writableFields": ["note"], + "rowBoundaries": [] + }] + }] + }); + let project_bytes = serde_json::to_vec(&project).expect("fixture serializes"); + let project = parse_project_json(&project_bytes).expect("string length fixture parses"); + compile_project(&project, &[], CompileProfile::Authoring) + .expect("string length fixture compiles") +} + fn vocabulary_catalog_registry(status_values: &[&str]) -> registry_breg::CompiledRegistry { vocabulary_catalog_registry_with_constraints(status_values, serde_json::json!([])) } diff --git a/crates/registry-breg/tests/postgres_history_rebaseline.rs b/crates/registry-breg/tests/postgres_history_rebaseline.rs index 127428c32b..d876217415 100644 --- a/crates/registry-breg/tests/postgres_history_rebaseline.rs +++ b/crates/registry-breg/tests/postgres_history_rebaseline.rs @@ -547,6 +547,335 @@ async fn rebaseline_restores_coverage_when_a_migration_baseline_indexed_only_jou database.cleanup().await; } +/// Live rows beyond what one history commit may index, so the rebaseline +/// verifies them page by page. +const MANY_RECORDS: usize = 1_201; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rebaseline_verifies_more_live_rows_than_one_commit_indexes() { + let database = TestDatabase::create(4).await; + let (mut migration, migration_task) = database.connect_migration().await; + let registry = compiled_registry(); + let expected = install_ready_history_registry(&database, &mut migration, ®istry).await; + let lock_key = RegistryLockKey::derive(&expected.package_id).expect("lock key derives"); + let audit_profile = AuditProfile::production_from_secret_bytes(vec![0x86; 32].into()) + .expect("test owns a keyed audit profile"); + let many = seed_many_records(&database.admin, &mut migration, ®istry).await; + erase_one_superseded_revision( + &database, + &mut migration, + ®istry, + &expected, + lock_key, + &audit_profile, + ) + .await; + + let outcome = rebaseline( + &mut migration, + &database, + &expected, + lock_key, + &audit_profile, + ®istry, + ) + .await + .expect("a registry larger than one commit is rebaselined"); + assert_eq!( + outcome.verified_record_count, + u64::try_from(many.len() + 2).unwrap() + ); + let transaction = migration.transaction().await.expect("transaction begins"); + let restored = capture_latest_snapshot_reference(&transaction) + .await + .expect("a fresh reference resolves after the rebaseline"); + assert_eq!( + reconstruct_records(&transaction, restored.position) + .await + .len(), + many.len() + 2 + ); + transaction.commit().await.expect("read commits"); + + migration_task.abort(); + database.cleanup().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rebaseline_refuses_a_mismatch_on_a_later_page() { + let database = TestDatabase::create(4).await; + let (mut migration, migration_task) = database.connect_migration().await; + let registry = compiled_registry(); + let expected = install_ready_history_registry(&database, &mut migration, ®istry).await; + let lock_key = RegistryLockKey::derive(&expected.package_id).expect("lock key derives"); + let audit_profile = AuditProfile::production_from_secret_bytes(vec![0x87; 32].into()) + .expect("test owns a keyed audit profile"); + let many = seed_many_records(&database.admin, &mut migration, ®istry).await; + erase_one_superseded_revision( + &database, + &mut migration, + ®istry, + &expected, + lock_key, + &audit_profile, + ) + .await; + // The greatest record identifier is verified on the last page. + let last = *many.iter().max().expect("records were seeded"); + let entity = ®istry.entities()[ENTITY]; + database + .admin + .execute( + &format!( + "UPDATE registry_data.{} SET {} = 'changed-outside-history' WHERE record_id = $1", + quote(&entity.physical_table), + quote(&entity.fields["household"].physical_name) + ), + &[&last], + ) + .await + .expect("fixture diverges one live row from its journal head"); + + assert_eq!( + rebaseline( + &mut migration, + &database, + &expected, + lock_key, + &audit_profile, + ®istry + ) + .await + .err(), + Some(HistoryRebaselineError::LiveHistoryMismatch), + "a live row on a later page that its journal head does not reproduce is refused" + ); + + migration_task.abort(); + database.cleanup().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rebaseline_refuses_a_journal_head_with_no_live_row() { + assert_a_journal_head_with_no_live_row_is_refused(0x88, Uuid::new_v4()).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rebaseline_refuses_a_journal_head_before_the_first_live_row() { + assert_a_journal_head_with_no_live_row_is_refused( + 0x89, + Uuid::parse_str("00000000-0000-4000-8000-000000000001").unwrap(), + ) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rebaseline_refuses_a_journal_head_past_the_last_live_row() { + assert_a_journal_head_with_no_live_row_is_refused( + 0x8a, + Uuid::parse_str("ffffffff-ffff-4fff-bfff-ffffffffffff").unwrap(), + ) + .await; +} + +/// Seed more live rows than one verification page, add an indexed journal head +/// for `orphan` with no live row, and require the rebaseline to refuse it. +async fn assert_a_journal_head_with_no_live_row_is_refused(audit_byte: u8, orphan: Uuid) { + let database = TestDatabase::create(4).await; + let (mut migration, migration_task) = database.connect_migration().await; + let registry = compiled_registry(); + let expected = install_ready_history_registry(&database, &mut migration, ®istry).await; + let lock_key = RegistryLockKey::derive(&expected.package_id).expect("lock key derives"); + let audit_profile = AuditProfile::production_from_secret_bytes(vec![audit_byte; 32].into()) + .expect("test owns a keyed audit profile"); + seed_many_records(&database.admin, &mut migration, ®istry).await; + erase_one_superseded_revision( + &database, + &mut migration, + ®istry, + &expected, + lock_key, + &audit_profile, + ) + .await; + let transaction = migration.transaction().await.expect("transaction begins"); + insert_revision(&transaction, orphan, 1, CURRENT_PACKAGE, "create").await; + allocate_revision_commit( + &transaction, + CommitAllocation { + package_revision: CURRENT_PACKAGE, + origin: CommitOrigin::Mutation { + actor_reference: "actor:hash", + request_reference: "request:hash", + }, + change_context: None, + members: &[RevisionCommitMember { + entity_id: ENTITY, + record_id: orphan, + record_revision: 1, + }], + }, + ) + .await + .expect("the orphan journal head is indexed"); + transaction.commit().await.expect("fixture commits"); + + assert_eq!( + rebaseline( + &mut migration, + &database, + &expected, + lock_key, + &audit_profile, + ®istry + ) + .await + .err(), + Some(HistoryRebaselineError::LiveHistoryMismatch), + "a retained journal head whose live row is gone cannot be vouched for" + ); + + migration_task.abort(); + database.cleanup().await; +} + +async fn rebaseline( + migration: &mut tokio_postgres::Client, + database: &TestDatabase, + expected: &ExpectedRegistryIdentity, + lock_key: RegistryLockKey, + audit_profile: &AuditProfile, + registry: ®istry_breg::CompiledRegistry, +) -> Result { + rebaseline_history_coverage( + migration, + HistoryRebaselineRequest { + expected, + migration_role: &database.migration_role, + lock_key, + timeouts: HistoryRebaselineTimeouts::new( + Duration::from_secs(5), + Duration::from_secs(30), + ) + .unwrap(), + audit: &database.audit(audit_profile.clone()), + operator_reference: OPERATOR_CANARY, + registry, + }, + ) + .await +} + +/// Seed the kept and erased records, then erase the erased record's +/// superseded first revision so coverage needs a rebaseline. +async fn erase_one_superseded_revision( + database: &TestDatabase, + migration: &mut tokio_postgres::Client, + registry: ®istry_breg::CompiledRegistry, + expected: &ExpectedRegistryIdentity, + lock_key: RegistryLockKey, + audit_profile: &AuditProfile, +) { + let kept = Uuid::parse_str(KEPT_RECORD).unwrap(); + let erased = Uuid::parse_str(ERASED_RECORD).unwrap(); + seed_two_records(&database.admin, migration, registry, kept, erased).await; + erase_record_history( + migration, + HistoryErasureRequest { + expected, + migration_role: &database.migration_role, + lock_key, + timeouts: HistoryErasureTimeouts::new(Duration::from_secs(5), Duration::from_secs(5)) + .unwrap(), + audit: &database.audit(audit_profile.clone()), + operator_reference: "operator-run-1", + reason: "approved retention request", + target: RecordHistoryErasureTarget::new(ENTITY, erased, 1), + }, + ) + .await + .expect("targeted erasure succeeds"); +} + +/// Seed `MANY_RECORDS` live rows whose first revision is their journal head, +/// indexed by as many commits as the per-commit member cap requires. +async fn seed_many_records( + admin: &tokio_postgres::Client, + migration: &mut tokio_postgres::Client, + registry: ®istry_breg::CompiledRegistry, +) -> Vec { + let ids = (0..MANY_RECORDS) + .map(|_| Uuid::new_v4()) + .collect::>(); + let snapshot = canonicalize_json(&snapshot_json(1)).expect("snapshot canonicalizes"); + let references = ids + .iter() + .map(|id| format!("{ENTITY}:{id}")) + .collect::>(); + let transaction = migration.transaction().await.expect("transaction begins"); + transaction + .execute( + "INSERT INTO registry_internal.registry_revisions + (entity_id, record_id, record_reference, record_revision, + predecessor_revision, record_lifecycle, package_revision, operation_id, + mutation_kind, principal_reference, request_reference, snapshot) + SELECT $1, seeded.record_id, seeded.record_reference, 1, NULL, 'active', $2, + 'op-1', 'create', 'actor:hash', 'request:hash', $3 + FROM unnest($4::uuid[], $5::text[]) AS seeded(record_id, record_reference)", + &[&ENTITY, &OLD_PACKAGE, &snapshot, &ids, &references], + ) + .await + .expect("seeded revisions insert"); + for page in ids.chunks(1_000) { + let members = page + .iter() + .map(|record_id| RevisionCommitMember { + entity_id: ENTITY, + record_id: *record_id, + record_revision: 1, + }) + .collect::>(); + allocate_revision_commit( + &transaction, + CommitAllocation { + package_revision: OLD_PACKAGE, + origin: CommitOrigin::Mutation { + actor_reference: "actor:hash", + request_reference: "request:hash", + }, + change_context: None, + members: &members, + }, + ) + .await + .expect("seeded revisions are indexed"); + } + transaction.commit().await.expect("fixture commits"); + let entity = ®istry.entities()[ENTITY]; + admin + .execute( + &format!( + "INSERT INTO registry_data.{} + (record_id, record_revision, record_lifecycle, active_package_revision, + {}, {}, {}) + SELECT seeded, 1, 'active', $2, $3, 'household-1', DATE '2026-06-01' + FROM unnest($1::uuid[]) AS seeded", + quote(&entity.physical_table), + quote(&entity.fields["person"].physical_name), + quote(&entity.fields["household"].physical_name), + quote(&entity.fields["valid-from"].physical_name), + ), + &[ + &ids, + &OLD_PACKAGE, + &Uuid::parse_str("00000000-0000-4000-8000-000000000010").unwrap(), + ], + ) + .await + .expect("seeded live rows insert"); + ids +} + async fn seed_two_records( admin: &tokio_postgres::Client, migration: &mut tokio_postgres::Client, diff --git a/crates/registry-breg/tests/postgres_import_authority.rs b/crates/registry-breg/tests/postgres_import_authority.rs new file mode 100644 index 0000000000..651d691769 --- /dev/null +++ b/crates/registry-breg/tests/postgres_import_authority.rs @@ -0,0 +1,1456 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Proves an `import` grant loads records only inside an operator-opened +//! import authority: run creation and every chunk are admitted against an +//! open authority for the same entity and profile, under the active package, +//! before its expiry, within its volume, and for a pinned input when inputs +//! are pinned. Every authority transition leaves one audit record. + +#![cfg(all(feature = "postgres-test", feature = "tooling"))] + +#[path = "support/postgres_harness.rs"] +#[allow(dead_code)] +mod postgres_harness; + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; +use std::time::Duration; + +use axum::body::{to_bytes, Body}; +use axum::http::{Method, Request, StatusCode}; +use postgres_harness::TestDatabase; +use registry_breg::api::{ + router, HttpService, ReadRuntimeIdentity, ReadinessProbe, ServiceFuture, VerifiedClaimValue, + VerifiedRequestClaims, +}; +use registry_breg::compiler::{compile_project, CompileProfile}; +use registry_breg::contract::parse_project_json; +use registry_breg::cursor::CursorCodec; +use registry_breg::import_authority::{ + ImportAuthority, ImportAuthorityCloseRequest, ImportAuthorityError, ImportAuthorityOpenRequest, + ImportAuthorityOperatorService, ImportAuthorityStatus, DEFAULT_IMPORT_AUTHORITY_WINDOW, +}; +use registry_breg::instance_claim::{InstanceClaimError, InstanceClaimService}; +use registry_breg::mutation::install_mutation_schema; +use registry_breg::postgres::{ + initialize_compiled_registry_state_for_test, install_compiled_schema, ExpectedManagedCatalog, + ExpectedRegistryIdentity, PostgresRecordMutationService, PostgresRecordReadService, + RegistryLockKey, RegistryStateTestIdentity, +}; +use registry_breg::CompiledRegistry; +use registry_platform_audit::AuditProfile; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use tower::Service as _; +use uuid::Uuid; +use zeroize::Zeroizing; + +const PRINCIPAL: &str = "import-authority-principal"; +const PACKAGE_ID: &str = "import-authority-registry"; +const PACKAGE_REVISION: &str = "package-import-1"; +const SUCCESSOR_REVISION: &str = "package-import-2"; + +const FIXTURE: &str = r#"{ + "apiVersion":"registry.registrystack.org/v1alpha1", + "kind":"RegistryProject", + "registry":{"id":"import-authority-registry","version":"1","defaultLanguage":"en","canonicalBaseIri":"https://authoring.example.test"}, + "entities":[{ + "id":"widget","primaryDataset":"test-dataset","route":"widgets","mutationMode":"mutable","classification":"public", + "batch":{"maximumItems":3,"maximumBytes":8192}, + "constraints":[{"kind":"unique","fields":["label"]}], + "fields":[ + {"id":"jurisdiction","type":"string","maxLength":32,"required":true,"classification":"public"}, + {"id":"label","type":"string","maxLength":128,"required":true,"classification":"public"} + ] + },{ + "id":"gadget","primaryDataset":"test-dataset","route":"gadgets","mutationMode":"mutable","classification":"public", + "batch":{"maximumItems":3,"maximumBytes":8192}, + "fields":[ + {"id":"jurisdiction","type":"string","maxLength":32,"required":true,"classification":"public"}, + {"id":"label","type":"string","maxLength":128,"required":true,"classification":"public"} + ] + },{ + "id":"ledger","primaryDataset":"test-dataset","route":"ledgers","mutationMode":"mutable","classification":"public", + "batch":{"maximumItems":3,"maximumBytes":8192}, + "fields":[ + {"id":"jurisdiction","type":"string","maxLength":32,"required":true,"classification":"public"}, + {"id":"label","type":"string","maxLength":128,"required":true,"classification":"public"} + ] + }], + "accessProfiles":[{ + "id":"loader","default":true,"principalClaim":"registry_principal", + "permissions":[{ + "entity":"widget","operations":["import"], + "readableFields":["jurisdiction","label"],"writableFields":["jurisdiction","label"], + "rowBoundaries":[{"field":"jurisdiction","claim":"jurisdiction","operator":"equals"}] + },{ + "entity":"gadget","operations":["import"], + "readableFields":["jurisdiction","label"],"writableFields":["jurisdiction","label"], + "rowBoundaries":[{"field":"jurisdiction","claim":"jurisdiction","operator":"equals"}] + },{ + "entity":"ledger","operations":["create","batch"], + "readableFields":["jurisdiction","label"],"writableFields":["jurisdiction","label"], + "rowBoundaries":[{"field":"jurisdiction","claim":"jurisdiction","operator":"equals"}] + }] + },{ + "id":"second-loader","principalClaim":"registry_principal", + "permissions":[{ + "entity":"widget","operations":["import"], + "readableFields":["jurisdiction","label"],"writableFields":["jurisdiction","label"], + "rowBoundaries":[{"field":"jurisdiction","claim":"jurisdiction","operator":"equals"}] + }] + }] +}"#; + +fn compiled_registry() -> CompiledRegistry { + let project = parse_project_json(FIXTURE.as_bytes()).expect("fixture parses"); + compile_project(&project, &[], CompileProfile::Authoring).expect("fixture compiles") +} + +struct Harness { + database: TestDatabase, + registry: Arc, + identity: ExpectedRegistryIdentity, + lock_key: RegistryLockKey, + audit_profile: AuditProfile, + app: axum::Router, +} + +impl Harness { + async fn create() -> Self { + let registry = Arc::new(compiled_registry()); + let database = TestDatabase::create(8).await; + let (migration, migration_task) = database.connect_migration().await; + install_compiled_schema(&migration, ®istry, &database.runtime_role) + .await + .expect("migration installs the compiler-owned schema"); + let identity = initialize_compiled_registry_state_for_test( + &migration, + &database.runtime_role, + ®istry, + RegistryStateTestIdentity { + package_id: PACKAGE_ID, + environment: "local", + instance_id: "import-authority-instance", + database_id: "import-authority-database", + package_revision: PACKAGE_REVISION, + package_sequence: 1, + }, + ) + .await + .expect("active package identity is initialized"); + migration_task.abort(); + let lock_key = RegistryLockKey::derive(PACKAGE_ID).expect("lock key derives"); + let audit_profile = AuditProfile::production_from_secret_bytes(vec![0x7c; 32].into()) + .expect("test owns a keyed audit profile"); + let app = build_router( + &database, + registry.clone(), + identity.clone(), + lock_key, + audit_profile.clone(), + ); + Self { + database, + registry, + identity, + lock_key, + audit_profile, + app, + } + } + + fn operator(&self) -> ImportAuthorityOperatorService { + self.operator_under(&self.identity) + } + + fn operator_under( + &self, + identity: &ExpectedRegistryIdentity, + ) -> ImportAuthorityOperatorService { + ImportAuthorityOperatorService::new_for_test( + identity.clone(), + ExpectedManagedCatalog::compiled(&self.registry), + self.lock_key, + self.database.migration_config.clone(), + self.database.migration_role.clone(), + self.database.runtime_role.clone(), + self.database.audit(self.audit_profile.clone()), + self.registry.clone(), + ) + } + + fn claims(&self) -> InstanceClaimService { + InstanceClaimService::new_for_test( + self.identity.clone(), + ExpectedManagedCatalog::compiled(&self.registry), + self.lock_key, + self.database.migration_config.clone(), + self.database.runtime_config.clone(), + self.database.migration_role.clone(), + self.database.runtime_role.clone(), + self.database.audit(self.audit_profile.clone()), + ) + } + + async fn simulate_restored_copy(&self) { + self.database + .admin + .execute( + "UPDATE registry_internal.registry_instance_claim + SET database_oid = 1 + WHERE singleton", + &[], + ) + .await + .expect("test simulates a restored copy"); + } + + async fn open( + &self, + entity_id: &str, + profile_id: &str, + max_items: i64, + input_digests: &[String], + ) -> ImportAuthority { + self.operator() + .open(open_request( + entity_id, + profile_id, + max_items, + input_digests, + )) + .await + .expect("the operator opens an import authority") + } + + async fn close(&self, authority_id: Uuid) -> ImportAuthority { + self.operator() + .close(ImportAuthorityCloseRequest { + authority_id, + operator_reference: "operator-b", + reason: "load finished", + }) + .await + .expect("the operator closes the authority") + } + + /// Activate a successor revision of the same package, as `bregctl apply` + /// would, and answer the HTTP surface a restarted process serves under it. + async fn activate_successor(&self) -> (axum::Router, ExpectedRegistryIdentity) { + let successor = ExpectedRegistryIdentity { + package_revision: SUCCESSOR_REVISION.to_owned(), + package_sequence: 2, + ..self.identity.clone() + }; + let changed = self + .database + .admin + .execute( + "UPDATE registry_internal.registry_state + SET active_package_revision = $1, package_sequence = $2 + WHERE singleton", + &[&successor.package_revision, &successor.package_sequence], + ) + .await + .expect("successor revision activates"); + assert_eq!(changed, 1); + let app = build_router( + &self.database, + self.registry.clone(), + successor.clone(), + self.lock_key, + self.audit_profile.clone(), + ); + (app, successor) + } + + /// Move one authority's whole window into the past, as the passage of + /// time would, without touching its status. + async fn age_past_expiry(&self, authority_id: Uuid) { + let changed = self + .database + .admin + .execute( + "UPDATE registry_internal.registry_import_authorities + SET opened_at = now() - interval '2 days', + expires_at = now() - interval '1 day' + WHERE authority_id = $1", + &[&authority_id], + ) + .await + .expect("administrator ages the authority"); + assert_eq!(changed, 1); + } + + async fn authority(&self, authority_id: Uuid) -> (String, i64) { + let row = self + .database + .admin + .query_one( + "SELECT status, committed_items + FROM registry_internal.registry_import_authorities + WHERE authority_id = $1", + &[&authority_id], + ) + .await + .expect("administrator reads the authority"); + (row.get(0), row.get(1)) + } + + async fn run_authority(&self, run_id: &str) -> Option { + self.database + .admin + .query_one( + "SELECT import_authority_id FROM registry_internal.registry_ingestion_runs + WHERE run_id = $1", + &[&Uuid::parse_str(run_id).expect("run id")], + ) + .await + .expect("administrator reads the run") + .get(0) + } + + async fn run_count(&self) -> i64 { + self.database + .admin + .query_one( + "SELECT count(*) FROM registry_internal.registry_ingestion_runs", + &[], + ) + .await + .expect("administrator counts runs") + .get(0) + } + + /// The authority audit records, oldest first. Each is one `response` + /// entry correlated by its authority id. + async fn authority_records(&self, authority_id: Uuid) -> Vec { + self.database + .audit_entries() + .into_iter() + .filter(|entry| { + entry["schema"] == "breg-import-authority-audit/v1" + && entry["correlation"] == authority_id.to_string() + }) + .map(|mut entry| { + assert_eq!(entry["phase"], "response"); + entry["record"].take() + }) + .collect() + } + + async fn create_run( + &self, + app: &axum::Router, + entity_route: &str, + profile_id: &str, + plan: &Plan, + package_revision: &str, + ) -> axum::response::Response { + send( + app, + Method::POST, + &format!("/v1/records/{entity_route}/ingestion-runs?accessProfile={profile_id}"), + json!({ + "operation": "create", + "profileId": profile_id, + "packageRevision": package_revision, + "schemaFingerprint": self.identity.schema_fingerprint, + "inputDigest": plan.input_digest, + "inputLength": plan.input_length, + "itemCount": plan.items.len(), + "chunkCount": plan.chunks.len(), + "chunkAlgorithmVersion": "greedy-canonical-http-batch-v1", + }), + ) + .await + } + + async fn created_run(&self, entity_route: &str, profile_id: &str, plan: &Plan) -> String { + let response = self + .create_run(&self.app, entity_route, profile_id, plan, PACKAGE_REVISION) + .await; + assert_eq!(response.status(), StatusCode::CREATED); + body_json(response).await["run"]["runId"] + .as_str() + .expect("run id") + .to_owned() + } + + async fn submit( + &self, + entity_route: &str, + profile_id: &str, + run_id: &str, + plan: &Plan, + index: usize, + ) -> axum::response::Response { + send( + &self.app, + Method::POST, + &format!( + "/v1/records/{entity_route}/ingestion-runs/{run_id}/chunks?accessProfile={profile_id}" + ), + plan.chunk_body(index), + ) + .await + } + + async fn committed_chunk(&self, run_id: &str, plan: &Plan, index: usize) { + let response = self.submit("widgets", "loader", run_id, plan, index).await; + assert_eq!(response.status(), StatusCode::OK); + } + + /// Submit the chunk and prove the run blocked on its authority: the + /// refusal is `ingestion.run_blocked`, the run and its last attempt both + /// report `importAuthorityClosed`, and no widget was written by the chunk. + async fn blocked_chunk(&self, run_id: &str, plan: &Plan, index: usize) { + let before = self.widget_count().await; + let response = self.submit("widgets", "loader", run_id, plan, index).await; + assert_eq!(response.status(), StatusCode::CONFLICT); + let problem = body_json(response).await; + assert_eq!(problem["code"], "ingestion.run_blocked"); + // The run names why it blocked; the refusal names no single cause. + assert_eq!( + problem["detail"], + "The ingestion run is blocked and refuses further chunks." + ); + assert_eq!(self.widget_count().await, before); + let run = send( + &self.app, + Method::GET, + &format!("/v1/records/widgets/ingestion-runs/{run_id}?accessProfile=loader"), + Value::Null, + ) + .await; + assert_eq!(run.status(), StatusCode::OK); + let run = body_json(run).await; + let run = if run.get("run").is_some() { + run["run"].clone() + } else { + run + }; + assert_eq!(run["status"], "blocked", "{run}"); + assert_eq!(run["blockedReason"], "importAuthorityClosed"); + assert_eq!( + run["lastAttempt"]["outcome"], "importAuthorityClosed", + "{run}" + ); + assert_eq!(run["lastAttempt"]["chunkIndex"], index); + } + + async fn widget_count(&self) -> i64 { + let table = &self.registry.entities()["widget"].physical_table; + self.database + .admin + .query_one( + &format!("SELECT count(*) FROM registry_data.\"{table}\""), + &[], + ) + .await + .expect("administrator counts widgets") + .get(0) + } + + /// The run audit records of one run, oldest first. + async fn run_records(&self, run_id: &str) -> Vec { + self.database + .audit_records() + .into_iter() + .filter(|record| record["kind"] == "ingestionRun" && record["runId"] == run_id) + .collect() + } + + async fn refused_run(&self, entity_route: &str, profile_id: &str, plan: &Plan) { + let before = self.run_count().await; + let response = self + .create_run(&self.app, entity_route, profile_id, plan, PACKAGE_REVISION) + .await; + assert_eq!(response.status(), StatusCode::PRECONDITION_FAILED); + assert_eq!(body_json(response).await["code"], "precondition.failed"); + assert_eq!(self.run_count().await, before, "a refused run never exists"); + } +} + +fn open_request<'a>( + entity_id: &'a str, + profile_id: &'a str, + max_items: i64, + input_digests: &'a [String], +) -> ImportAuthorityOpenRequest<'a> { + ImportAuthorityOpenRequest { + entity_id, + profile_id, + max_items, + expires_in: DEFAULT_IMPORT_AUTHORITY_WINDOW, + input_digests, + operator_reference: "operator-a", + reason: "initial load of reviewed records", + } +} + +fn build_router( + database: &TestDatabase, + registry: Arc, + identity: ExpectedRegistryIdentity, + lock_key: RegistryLockKey, + profile: AuditProfile, +) -> axum::Router { + let audit = database.audit(profile); + let pool = database + .runtime_config + .build_pool() + .expect("bounded runtime pool builds"); + let cursors = Arc::new( + CursorCodec::new(Zeroizing::new(vec![0x53; 32]), Duration::from_secs(300)) + .expect("cursor key is valid"), + ); + let records = Arc::new(PostgresRecordReadService::new( + pool.clone(), + registry.clone(), + identity.clone(), + lock_key, + Duration::from_secs(2), + audit.clone(), + cursors.clone(), + )); + let mutations = PostgresRecordMutationService::new( + pool, + registry.clone(), + identity.clone(), + lock_key, + Duration::from_secs(2), + audit, + ); + router(Arc::new( + HttpService::new( + registry, + ReadRuntimeIdentity { + package_revision: identity.package_revision, + schema_fingerprint: identity.schema_fingerprint, + }, + records, + Arc::new(AlwaysReady), + cursors, + ) + .with_postgres_mutations(Arc::new(mutations)), + )) +} + +struct AlwaysReady; + +impl ReadinessProbe for AlwaysReady { + fn is_ready(&self) -> ServiceFuture<'_, bool> { + Box::pin(async { true }) + } +} + +fn claims() -> VerifiedRequestClaims { + VerifiedRequestClaims::authenticated( + "registry_principal", + PRINCIPAL, + BTreeSet::new(), + None, + BTreeMap::from([( + "jurisdiction".to_owned(), + VerifiedClaimValue::direct_string("zone-a").expect("direct claim"), + )]), + ) + .expect("verified claims are bounded") +} + +async fn send( + app: &axum::Router, + method: Method, + uri: &str, + body: Value, +) -> axum::response::Response { + let mut request = Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body).expect("request JSON"))) + .expect("request"); + request.extensions_mut().insert(claims()); + let mut app = app.clone(); + app.call(request).await.expect("response") +} + +async fn body_json(response: axum::response::Response) -> Value { + let bytes = to_bytes(response.into_body(), 2 * 1024 * 1024) + .await + .expect("response body"); + serde_json::from_slice(&bytes).expect("JSON response") +} + +fn hex_digest(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +/// One import input: its items, the greedy chunks the run announces, and the +/// digests the run and every chunk bind. +struct Plan { + items: Vec, + chunks: Vec<(usize, usize)>, + input_digest: String, + input_length: i64, + chunk_digests: Vec, + prefix_digests: Vec, +} + +impl Plan { + fn chunk_body(&self, index: usize) -> Value { + let (start, end) = self.chunks[index]; + json!({ + "chunkIndex": index, + "items": self.items[start..end], + "digest": self.chunk_digests[index], + "prefixDigest": self.prefix_digests[index], + }) + } +} + +fn plan(label_prefix: &str, count: usize) -> Plan { + let items: Vec = (0..count) + .map(|index| { + json!({"operation":"create","data":{ + "jurisdiction":"zone-a","label":format!("{label_prefix}-{index}") + }}) + }) + .collect(); + let lines: Vec> = items + .iter() + .map(|item| { + let mut line = + registry_platform_canonical_json::canonicalize_json(item).expect("canonical JSON"); + line.push(b'\n'); + line + }) + .collect(); + let mut chunks = Vec::new(); + let mut start = 0; + while start < items.len() { + let end = (start + 3).min(items.len()); + chunks.push((start, end)); + start = end; + } + let chunk_digests = chunks + .iter() + .map(|&(start, end)| { + hex_digest( + ®istry_platform_canonical_json::canonicalize_json( + &json!({"items": items[start..end]}), + ) + .expect("canonical JSON"), + ) + }) + .collect(); + let prefix_digests = chunks + .iter() + .map(|&(_, end)| hex_digest(&lines[..end].concat())) + .collect(); + let input = lines.concat(); + Plan { + items, + chunks, + input_digest: hex_digest(&input), + input_length: input.len() as i64, + chunk_digests, + prefix_digests, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_import_run_is_refused_without_an_open_authority() { + let harness = Harness::create().await; + harness + .refused_run("widgets", "loader", &plan("none", 2)) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_open_authority_admits_a_run_that_names_it() { + let harness = Harness::create().await; + let authority = harness.open("widget", "loader", 10, &[]).await; + assert_eq!(authority.status, ImportAuthorityStatus::Open); + assert_eq!(authority.activation_revision, PACKAGE_REVISION); + let run_id = harness + .created_run("widgets", "loader", &plan("admitted", 4)) + .await; + assert_eq!( + harness.run_authority(&run_id).await, + Some(authority.authority_id) + ); + let opened = harness.authority_records(authority.authority_id).await; + assert_eq!(opened.len(), 1, "{opened:?}"); + assert_eq!(opened[0]["transition"], "opened"); + assert_eq!(opened[0]["operationId"], "breg.import_authority"); + assert_eq!(opened[0]["maxItems"], 10); + for clear in ["operator-a", "initial load of reviewed records"] { + assert!( + !opened[0].to_string().contains(clear), + "the operator reference and reason appear only as keyed hashes" + ); + } + assert!(opened[0]["operatorReference"].is_string()); + assert!(opened[0]["reasonReference"].is_string()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_batch_run_needs_no_authority() { + let harness = Harness::create().await; + let run_id = harness + .created_run("ledgers", "loader", &plan("ledger", 2)) + .await; + assert_eq!(harness.run_authority(&run_id).await, None); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_authority_for_another_profile_does_not_admit_the_run() { + let harness = Harness::create().await; + harness.open("widget", "second-loader", 10, &[]).await; + harness + .refused_run("widgets", "loader", &plan("wrong-profile", 2)) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_authority_for_another_entity_does_not_admit_the_run() { + let harness = Harness::create().await; + harness.open("gadget", "loader", 10, &[]).await; + harness + .refused_run("widgets", "loader", &plan("wrong-entity", 2)) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_run_larger_than_the_remaining_volume_never_starts() { + let harness = Harness::create().await; + harness.open("widget", "loader", 3, &[]).await; + harness + .refused_run("widgets", "loader", &plan("too-many", 4)) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_pinned_authority_admits_only_the_pinned_input() { + let harness = Harness::create().await; + let reviewed = plan("reviewed", 2); + let unreviewed = plan("unreviewed", 2); + harness + .open( + "widget", + "loader", + 10, + std::slice::from_ref(&reviewed.input_digest), + ) + .await; + harness.refused_run("widgets", "loader", &unreviewed).await; + harness.created_run("widgets", "loader", &reviewed).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_closed_authority_admits_no_run_and_records_its_close() { + let harness = Harness::create().await; + let authority = harness.open("widget", "loader", 10, &[]).await; + let closed = harness.close(authority.authority_id).await; + assert_eq!(closed.status, ImportAuthorityStatus::Closed); + assert!(closed.closed_at.is_some()); + harness + .refused_run("widgets", "loader", &plan("revoked", 2)) + .await; + let again = harness.close(authority.authority_id).await; + assert_eq!(again, closed, "closing a closed authority changes nothing"); + let records = harness.authority_records(authority.authority_id).await; + let transitions: Vec<&str> = records + .iter() + .map(|record| record["transition"].as_str().expect("transition")) + .collect(); + assert_eq!(transitions, ["opened", "closed"]); + assert!(records[1]["operatorReference"].is_string()); + assert_ne!( + records[1]["operatorReference"], records[0]["operatorReference"], + "the close names its own operator" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_expired_authority_admits_no_run_and_records_one_expiry() { + let harness = Harness::create().await; + let authority = harness.open("widget", "loader", 10, &[]).await; + harness.age_past_expiry(authority.authority_id).await; + harness + .refused_run("widgets", "loader", &plan("expired", 2)) + .await; + harness + .refused_run("widgets", "loader", &plan("expired-again", 2)) + .await; + assert_eq!(harness.authority(authority.authority_id).await.0, "expired"); + let records = harness.authority_records(authority.authority_id).await; + let transitions: Vec<&str> = records + .iter() + .map(|record| record["transition"].as_str().expect("transition")) + .collect(); + assert_eq!(transitions, ["opened", "expired"]); + assert!( + records[1].get("operatorReference").is_none(), + "an observed expiry names no operator" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_successor_package_supersedes_an_open_authority() { + let harness = Harness::create().await; + let authority = harness.open("widget", "loader", 10, &[]).await; + let (successor_app, successor) = harness.activate_successor().await; + let before = harness.run_count().await; + let response = harness + .create_run( + &successor_app, + "widgets", + "loader", + &plan("superseded", 2), + SUCCESSOR_REVISION, + ) + .await; + assert_eq!(response.status(), StatusCode::PRECONDITION_FAILED); + assert_eq!(harness.run_count().await, before); + assert_eq!( + harness.authority(authority.authority_id).await.0, + "superseded" + ); + let records = harness.authority_records(authority.authority_id).await; + assert_eq!(records.len(), 2, "{records:?}"); + assert_eq!(records[1]["transition"], "superseded"); + assert_eq!(records[1]["packageRevision"], SUCCESSOR_REVISION); + + // The operator re-opens deliberately under the successor. + let reopened = harness + .operator_under(&successor) + .open(open_request("widget", "loader", 10, &[])) + .await + .expect("a new authority opens under the successor"); + assert_eq!(reopened.activation_revision, SUCCESSOR_REVISION); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn one_entity_holds_at_most_one_open_authority() { + let harness = Harness::create().await; + harness.open("widget", "loader", 10, &[]).await; + assert_eq!( + harness + .operator() + .open(open_request("widget", "second-loader", 10, &[])) + .await + .err(), + Some(ImportAuthorityError::AlreadyOpen) + ); + harness.open("gadget", "loader", 10, &[]).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_authority_opens_only_over_an_import_grant() { + let harness = Harness::create().await; + for (entity, profile) in [ + ("ledger", "loader"), + ("gadget", "second-loader"), + ("missing", "loader"), + ] { + assert_eq!( + harness + .operator() + .open(open_request(entity, profile, 10, &[])) + .await + .err(), + Some(ImportAuthorityError::NotImportable), + "{entity}/{profile}" + ); + } + assert_eq!( + harness + .operator() + .close(ImportAuthorityCloseRequest { + authority_id: Uuid::new_v4(), + operator_reference: "operator-b", + reason: "no such authority", + }) + .await + .err(), + Some(ImportAuthorityError::NotFound) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn close_expired_records_every_due_transition_once() { + let harness = Harness::create().await; + let widget = harness.open("widget", "loader", 10, &[]).await; + let gadget = harness.open("gadget", "loader", 10, &[]).await; + harness.age_past_expiry(widget.authority_id).await; + let moved = harness + .operator() + .close_expired() + .await + .expect("close-expired settles due transitions"); + assert_eq!(moved.len(), 1); + assert_eq!(moved[0].authority_id, widget.authority_id); + assert_eq!(moved[0].status, ImportAuthorityStatus::Expired); + assert!(harness + .operator() + .close_expired() + .await + .expect("a second sweep") + .is_empty()); + let listed = harness.operator().list().await.expect("list answers"); + assert_eq!(listed.len(), 2); + let status = |id: Uuid| { + listed + .iter() + .find(|authority| authority.authority_id == id) + .expect("listed") + .status + }; + assert_eq!(status(widget.authority_id), ImportAuthorityStatus::Expired); + assert_eq!(status(gadget.authority_id), ImportAuthorityStatus::Open); + assert_eq!( + harness.authority_records(widget.authority_id).await.len(), + 2 + ); +} + +/// Listing is a read. It answers while a write holds the shared registry +/// lock and while maintenance is not ready, records no transition, and names +/// the status an authority has already reached. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn listing_takes_no_registry_lock_and_records_nothing() { + let harness = Harness::create().await; + let widget = harness.open("widget", "loader", 10, &[]).await; + harness.age_past_expiry(widget.authority_id).await; + + // Hold the shared registry lock, as an in-flight write does, and leave + // maintenance short of ready, as an interrupted apply does. + let (holder, holder_task) = harness.database.connect_admin().await; + holder + .batch_execute( + "UPDATE registry_internal.registry_state + SET maintenance_status = 'failed', + maintenance_target_revision = 'successor' + WHERE singleton; + BEGIN", + ) + .await + .expect("the holder opens a transaction"); + holder + .execute( + "SELECT pg_catalog.pg_advisory_xact_lock_shared($1)", + &[&harness.lock_key.get()], + ) + .await + .expect("the holder takes the shared registry lock"); + let listed = tokio::time::timeout(Duration::from_secs(5), harness.operator().list()) + .await + .expect("listing does not wait for the registry lock") + .expect("listing answers"); + holder + .batch_execute( + "COMMIT; + UPDATE registry_internal.registry_state + SET maintenance_status = 'ready', + maintenance_target_revision = NULL + WHERE singleton", + ) + .await + .expect("the holder releases the lock"); + holder_task.abort(); + + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].status, ImportAuthorityStatus::Expired); + assert_eq!( + harness.authority(widget.authority_id).await.0, + "open", + "listing records no transition" + ); + assert_eq!( + harness.authority_records(widget.authority_id).await.len(), + 1 + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_runtime_role_cannot_open_close_or_reopen_an_authority() { + let harness = Harness::create().await; + let authority = harness.open("widget", "loader", 10, &[]).await; + let (runtime, runtime_task) = harness.database.connect_admin().await; + runtime + .batch_execute(&format!( + "SET ROLE \"{}\"", + harness.database.runtime_role.as_str() + )) + .await + .expect("the session takes the runtime role"); + let insert = runtime + .execute( + "INSERT INTO registry_internal.registry_import_authorities + (authority_id, entity_id, profile_id, operation, max_items, + activation_revision, expires_at, operator_reference, reason_reference) + VALUES ($1, 'gadget', 'loader', 'create', 5, 'package-import-1', + now() + interval '1 day', 'r', 'r')", + &[&Uuid::new_v4()], + ) + .await; + assert!(insert.is_err(), "the runtime role cannot open an authority"); + let widen = runtime + .execute( + "UPDATE registry_internal.registry_import_authorities + SET max_items = 1000 WHERE authority_id = $1", + &[&authority.authority_id], + ) + .await; + assert!(widen.is_err(), "the runtime role cannot widen the volume"); + let close = runtime + .execute( + "UPDATE registry_internal.registry_import_authorities + SET status = 'closed', closed_at = now() WHERE authority_id = $1", + &[&authority.authority_id], + ) + .await; + assert!(close.is_err(), "only the operator closes an authority"); + + harness.close(authority.authority_id).await; + let reopened = runtime + .execute( + "UPDATE registry_internal.registry_import_authorities + SET status = 'open', closed_at = NULL WHERE authority_id = $1", + &[&authority.authority_id], + ) + .await + .expect("row security hides the terminal row from the runtime update"); + assert_eq!(reopened, 0, "the runtime role cannot reopen an authority"); + assert_eq!(harness.authority(authority.authority_id).await.0, "closed"); + runtime_task.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_open_authority_opens_no_direct_write_route() { + let harness = Harness::create().await; + harness.open("widget", "loader", 10, &[]).await; + let item = json!({"jurisdiction":"zone-a","label":"direct"}); + for (method, uri, body) in [ + ( + Method::POST, + "/v1/records/widgets?accessProfile=loader", + item.clone(), + ), + ( + Method::POST, + "/v1/records/widgets:batch?accessProfile=loader", + json!({"items":[{"operation":"create","data":item}]}), + ), + ] { + let response = send(&harness.app, method, uri, body).await; + assert!( + response.status() == StatusCode::NOT_FOUND + || response.status() == StatusCode::METHOD_NOT_ALLOWED, + "{uri} answered {}", + response.status() + ); + } + assert_eq!( + harness.widget_count().await, + 0, + "no route but the ingestion run writes a widget" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn every_chunk_counts_against_the_authority_until_it_is_exhausted() { + let harness = Harness::create().await; + let authority = harness.open("widget", "loader", 5, &[]).await; + let load = plan("counted", 5); + let run_id = harness.created_run("widgets", "loader", &load).await; + harness.committed_chunk(&run_id, &load, 0).await; + assert_eq!( + harness.authority(authority.authority_id).await, + ("open".to_owned(), 3) + ); + harness.committed_chunk(&run_id, &load, 1).await; + assert_eq!( + harness.authority(authority.authority_id).await, + ("exhausted".to_owned(), 5) + ); + assert_eq!(harness.widget_count().await, 5); + + let transitions: Vec = harness + .authority_records(authority.authority_id) + .await + .iter() + .map(|record| record["transition"].clone()) + .collect(); + assert_eq!(transitions, [json!("opened"), json!("exhausted")]); + let runs = harness.run_records(&run_id).await; + let committed = runs + .iter() + .filter(|record| record["outcome"] == "committed") + .collect::>(); + assert_eq!(committed.len(), 2); + assert!(committed + .iter() + .all(|record| record["importAuthorityId"] == authority.authority_id.to_string())); + harness + .refused_run("widgets", "loader", &plan("after-exhaustion", 1)) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn closing_the_authority_blocks_the_next_chunk_and_keeps_committed_ones() { + let harness = Harness::create().await; + let authority = harness.open("widget", "loader", 10, &[]).await; + let load = plan("revoked-mid-run", 6); + let run_id = harness.created_run("widgets", "loader", &load).await; + harness.committed_chunk(&run_id, &load, 0).await; + harness.close(authority.authority_id).await; + harness.blocked_chunk(&run_id, &load, 1).await; + assert_eq!( + harness.widget_count().await, + 3, + "revoking never undoes data" + ); + let blocked: Vec = harness + .run_records(&run_id) + .await + .into_iter() + .filter(|record| record["outcome"] == "blocked") + .collect(); + assert_eq!(blocked.len(), 1, "{blocked:?}"); + assert_eq!(blocked[0]["blockedReason"], "import_authority_closed"); + assert_eq!( + blocked[0]["importAuthorityId"], + authority.authority_id.to_string() + ); + // A blocked run stays blocked; a later chunk is refused without a second + // blocked record. + let again = harness.submit("widgets", "loader", &run_id, &load, 1).await; + assert_eq!(again.status(), StatusCode::CONFLICT); + assert_eq!( + harness + .run_records(&run_id) + .await + .iter() + .filter(|record| record["outcome"] == "blocked") + .count(), + 1 + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn expiry_between_two_chunks_blocks_the_run_and_records_the_expiry() { + let harness = Harness::create().await; + let authority = harness.open("widget", "loader", 10, &[]).await; + let load = plan("expires-mid-run", 6); + let run_id = harness.created_run("widgets", "loader", &load).await; + harness.committed_chunk(&run_id, &load, 0).await; + harness.age_past_expiry(authority.authority_id).await; + harness.blocked_chunk(&run_id, &load, 1).await; + assert_eq!( + harness.authority(authority.authority_id).await, + ("expired".to_owned(), 3) + ); + let transitions: Vec = harness + .authority_records(authority.authority_id) + .await + .iter() + .map(|record| record["transition"].clone()) + .collect(); + assert_eq!(transitions, [json!("opened"), json!("expired")]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_second_run_cannot_spend_volume_the_first_already_committed() { + let harness = Harness::create().await; + let authority = harness.open("widget", "loader", 6, &[]).await; + let first = plan("first", 3); + let second = plan("second", 6); + let first_run = harness.created_run("widgets", "loader", &first).await; + // Both runs are admitted while the whole volume is still free. + let second_run = harness.created_run("widgets", "loader", &second).await; + harness.committed_chunk(&first_run, &first, 0).await; + harness.committed_chunk(&second_run, &second, 0).await; + assert_eq!( + harness.authority(authority.authority_id).await, + ("exhausted".to_owned(), 6) + ); + harness.blocked_chunk(&second_run, &second, 1).await; + assert_eq!(harness.widget_count().await, 6); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_close_racing_a_chunk_waits_for_it_and_stops_the_next() { + let harness = Harness::create().await; + let authority = harness.open("widget", "loader", 10, &[]).await; + let load = plan("raced", 6); + let run_id = harness.created_run("widgets", "loader", &load).await; + + // Hold the authority row lock from a second session, as an in-flight + // chunk transaction would, and start the close behind it. + let (holder, holder_task) = harness.database.connect_admin().await; + holder + .batch_execute("BEGIN") + .await + .expect("the holder opens a transaction"); + holder + .execute( + "SELECT 1 FROM registry_internal.registry_import_authorities + WHERE authority_id = $1 FOR UPDATE", + &[&authority.authority_id], + ) + .await + .expect("the holder locks the authority"); + let operator = harness.operator(); + let close = tokio::spawn(async move { + operator + .close(ImportAuthorityCloseRequest { + authority_id: authority.authority_id, + operator_reference: "operator-b", + reason: "stop the load", + }) + .await + }); + tokio::time::sleep(Duration::from_millis(300)).await; + assert!(!close.is_finished(), "the close waits for the row lock"); + assert_eq!(harness.authority(authority.authority_id).await.0, "open"); + holder + .batch_execute("COMMIT") + .await + .expect("the holder releases the lock"); + holder_task.abort(); + let closed = close + .await + .expect("the close task joins") + .expect("the close commits"); + assert_eq!(closed.status, ImportAuthorityStatus::Closed); + harness.blocked_chunk(&run_id, &load, 0).await; +} + +/// A restored copy carries every authority that was open when its backup was +/// taken, including one an operator closed afterwards. Adopting the copy +/// supersedes each open authority in the adopting transaction, so the copy +/// admits no import until an operator opens a new authority. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn adopting_a_restored_copy_supersedes_every_open_authority() { + let harness = Harness::create().await; + let widget = harness.open("widget", "loader", 10, &[]).await; + let gadget = harness.open("gadget", "loader", 10, &[]).await; + harness.close(gadget.authority_id).await; + harness.simulate_restored_copy().await; + + let adoption = harness + .claims() + .adopt() + .await + .expect("the operator adopts the copy"); + assert_eq!( + adoption.superseded_import_authorities, + [widget.authority_id], + "the adoption names the authorities it superseded" + ); + assert_eq!(harness.authority(widget.authority_id).await.0, "superseded"); + assert_eq!(harness.authority(gadget.authority_id).await.0, "closed"); + let transitions: Vec = harness + .authority_records(widget.authority_id) + .await + .iter() + .map(|record| record["transition"].clone()) + .collect(); + assert_eq!(transitions, [json!("opened"), json!("superseded")]); + assert_eq!( + harness.authority_records(gadget.authority_id).await.len(), + 2, + "an authority already closed gains no record" + ); + let adoptions: Vec = harness + .database + .audit_entries() + .into_iter() + .filter(|entry| entry["schema"] == "breg-instance-claim-audit/v1") + .collect(); + assert_eq!(adoptions.len(), 2, "one request and one response"); + assert_eq!(adoptions[0]["phase"], "request"); + assert_eq!(adoptions[1]["phase"], "response"); + assert_eq!(adoptions[0]["correlation"], adoptions[1]["correlation"]); + assert_eq!( + adoptions[1]["record"]["supersededImportAuthorities"], + json!([widget.authority_id.to_string()]) + ); + harness + .refused_run("widgets", "loader", &plan("after-restore", 1)) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_missing_claim_is_reported_and_adopting_records_a_fresh_one() { + let harness = Harness::create().await; + harness + .database + .admin + .execute("DELETE FROM registry_internal.registry_instance_claim", &[]) + .await + .expect("the owning role can remove the claim"); + let claims = harness.claims(); + + let missing = claims + .status() + .await + .expect("a missing claim still reports"); + assert_eq!(missing.claim, None); + assert!(!missing.matches, "no claim names this database"); + + let adoption = claims + .adopt() + .await + .expect("the operator claims the database"); + assert_eq!(adoption.previous, None); + assert_eq!(adoption.current.epoch, 1); + assert_eq!(adoption.current.identity, missing.live); + assert!(claims.status().await.expect("the claim reads").matches); + assert_eq!( + claims.adopt().await.err(), + Some(InstanceClaimError::AlreadyCurrent), + "the database the claim names has nothing to adopt" + ); + let adoptions: Vec<(Value, Value)> = harness + .database + .audit_entries() + .into_iter() + .filter(|entry| entry["schema"] == "breg-instance-claim-audit/v1") + .map(|entry| (entry["phase"].clone(), entry["record"]["outcome"].clone())) + .collect(); + assert_eq!( + adoptions, + [ + (json!("request"), Value::Null), + (json!("response"), json!("committed")), + (json!("request"), Value::Null), + (json!("response"), json!("refused")), + ], + "each adoption answers its request once" + ); +} + +/// A database that already holds committed history is either a Registry +/// installed before the claim existed or a copy restored from a backup taken +/// before it. Installing the claim there records none, so the database waits +/// for an operator to adopt it instead of claiming itself. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn installing_the_claim_beside_committed_history_leaves_the_database_to_adopt() { + let harness = Harness::create().await; + harness.open("widget", "loader", 2, &[]).await; + let load = plan("history", 2); + let run_id = harness.created_run("widgets", "loader", &load).await; + harness.committed_chunk(&run_id, &load, 0).await; + let (migration, migration_task) = harness.database.connect_migration().await; + migration + .batch_execute("DROP TABLE registry_internal.registry_instance_claim") + .await + .expect("the owning role can drop the claim table"); + install_mutation_schema(&migration, &harness.database.runtime_role, false) + .await + .expect("the mutation schema installs again"); + let claims = harness.claims(); + + let unclaimed = claims.status().await.expect("the claim table reads"); + assert_eq!(unclaimed.claim, None, "no claim is recorded beside history"); + assert!(!unclaimed.matches); + + let adoption = claims + .adopt() + .await + .expect("the operator claims the database"); + assert_eq!(adoption.previous, None); + assert_eq!(adoption.current.epoch, 1); + install_mutation_schema(&migration, &harness.database.runtime_role, false) + .await + .expect("the mutation schema installs again"); + assert_eq!( + claims + .status() + .await + .expect("the claim reads") + .claim + .map(|claim| claim.epoch), + Some(1), + "a later install leaves the adopted claim in place" + ); + drop(migration); + migration_task.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_runtime_role_cannot_rewrite_or_remove_the_instance_claim() { + let harness = Harness::create().await; + let (runtime, runtime_task) = harness.database.connect_admin().await; + runtime + .batch_execute(&format!( + "SET ROLE \"{}\"", + harness.database.runtime_role.as_str() + )) + .await + .expect("the session takes the runtime role"); + let claimed: i64 = runtime + .query_one( + "SELECT epoch FROM registry_internal.registry_instance_claim WHERE singleton", + &[], + ) + .await + .expect("the runtime role reads the claim") + .get(0); + assert_eq!(claimed, 1); + for statement in [ + "UPDATE registry_internal.registry_instance_claim + SET database_oid = (SELECT oid FROM pg_database WHERE datname = current_database())", + "UPDATE registry_internal.registry_instance_claim SET epoch = epoch + 1", + "DELETE FROM registry_internal.registry_instance_claim", + "INSERT INTO registry_internal.registry_instance_claim + (singleton, system_identifier, database_oid) + VALUES (true, 1, 1)", + ] { + assert!( + runtime.execute(statement, &[]).await.is_err(), + "the runtime role cannot move its own claim: {statement}" + ); + } + drop(runtime); + runtime_task.abort(); +} + +/// A run table created before the authority attempt outcome existed accepts +/// it after the next schema install, so a run blocked by its authority on an +/// upgraded registry records the outcome instead of failing the chunk. +#[tokio::test] +async fn an_upgraded_run_table_records_the_authority_attempt_outcome() { + let harness = Harness::create().await; + let (migration, migration_task) = harness.database.connect_migration().await; + migration + .batch_execute( + "ALTER TABLE registry_internal.registry_ingestion_runs + DROP CONSTRAINT registry_ingestion_runs_attempt_values, + ADD CONSTRAINT registry_ingestion_runs_attempt_values + CHECK (last_attempt_outcome IS NULL OR last_attempt_outcome IN + ('committed', 'replayed', 'invalid_item', 'refused', + 'binding_changed', 'chunk_mismatch', 'run_not_open', 'unavailable'))", + ) + .await + .expect("the test restores the earlier attempt vocabulary"); + install_mutation_schema(&migration, &harness.database.runtime_role, false) + .await + .expect("the install upgrades the attempt vocabulary"); + migration_task.abort(); + + let authority = harness.open("widget", "loader", 10, &[]).await; + let load = plan("upgraded-attempt-vocabulary", 6); + let run_id = harness.created_run("widgets", "loader", &load).await; + harness.committed_chunk(&run_id, &load, 0).await; + harness.close(authority.authority_id).await; + harness.blocked_chunk(&run_id, &load, 1).await; +} diff --git a/crates/registry-breg/tests/postgres_migration.rs b/crates/registry-breg/tests/postgres_migration.rs index 8bf3ef033d..4006a27ebd 100644 --- a/crates/registry-breg/tests/postgres_migration.rs +++ b/crates/registry-breg/tests/postgres_migration.rs @@ -36,11 +36,12 @@ use registry_breg::package::{ compiled_registry_change_set, load_package, prepare_package, CompiledRegistryChangeClass, CompiledRegistryChangeCode, CompiledRegistryMigrationBaseline, PackageBuildRequest, PackageIntent, PackageLoadContext, PackageMigrationPlanInput, PackageModuleSource, - PackageSourceFile, SignaturePolicy, VerifiedPackage, + PackageSourceFile, PreparedPackage, SignaturePolicy, VerifiedPackage, }; use registry_breg::postgres::{ - install_compiled_schema, managed_schema_fingerprint, ExpectedManagedCatalog, - ExpectedRegistryIdentity, + install_compiled_schema, managed_schema_fingerprint, rehearse_successor_migration, + ExpectedManagedCatalog, ExpectedRegistryIdentity, MigrationRehearsalError, PostgresFailure, + SuccessorMigrationRehearsal, }; use registry_breg::CompiledRegistry; use registry_platform_audit::AuditProfile; @@ -122,6 +123,11 @@ async fn real_postgres_backfill_and_destructive_recovery_are_bounded_resumable_a assert_eq!(first_checkpoint.0, "applying"); assert_eq!(first_checkpoint.2, 2); assert!(first_checkpoint.1.is_some()); + assert_eq!( + reviewed_migration_history(&database).await, + (2, vec![2]), + "the committed chunk appended one revision per row it changed in one commit" + ); assert_non_ready_target(&database, &active, &required_package, "applying").await; let wrong_source = backfill_source(BackfillSourceRequest { @@ -166,6 +172,12 @@ async fn real_postgres_backfill_and_destructive_recovery_are_bounded_resumable_a assert_eq!(completed.0, "completed"); assert_eq!(completed.2, 5); assert_all_ranks(&database, &required, 1).await; + assert_eq!( + reviewed_migration_history(&database).await, + (5, vec![2, 2, 1]), + "every chunk commit journals its own rows once, and the resumed run does not \ + journal the committed chunk again" + ); assert_ready_target(&database, &required_active).await; let ledger_before_destructive = ledger_snapshot(&database).await; @@ -418,7 +430,15 @@ async fn real_postgres_backfill_and_destructive_recovery_are_bounded_resumable_a ) .await .expect_err("the second reviewed drop deterministically faults after the first committed drop"); - assert_value_free(Some(destructive_fault), MigrationError::ApplyFailed); + // The second drop names the column the first already dropped, so the + // refusal carries PostgreSQL's undefined-column SQLSTATE. + assert_value_free( + Some(destructive_fault), + MigrationError::StatementFailed(PostgresFailure { + sqlstate: Some("42703".to_owned()), + ..PostgresFailure::default() + }), + ); assert_non_ready_target(&database, &required_active, &destructive_package, "failed").await; assert_eq!( step_snapshot(&database, &destructive_package, "drop-legacy") @@ -508,6 +528,7 @@ async fn real_postgres_backfill_and_destructive_recovery_are_bounded_resumable_a false_assertion_refusals_are_closed().await; row_count_mismatch_is_closed().await; lock_timeout_is_bounded().await; + refused_step_reports_its_sqlstate().await; } /// Re-presenting the active package is a no-op only when the database, read @@ -1363,6 +1384,216 @@ async fn real_postgres_added_required_field_backfills_before_the_column_is_const database.cleanup().await; } +/// `bregctl test` rehearses a successor over an empty reproduction of the +/// verified predecessor schema before it measures the candidate. The rehearsal +/// accepts the plan activation accepts, refuses the plans activation would +/// refuse with a value-free PostgreSQL class and object, and rolls back so the +/// schema-test database is still clean afterward. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn real_postgres_rehearsal_refuses_a_reviewed_plan_activation_would_refuse() { + let database = TestDatabase::create(1).await; + let _unused_harness_configs = (&database.runtime_config, &database.tls_runtime_config); + database + .admin + .batch_execute("CREATE EXTENSION btree_gist") + .await + .expect("administrator installs the required extension"); + + let base = compile_variant(Variant::Base, 1); + let base_fingerprint = initial_fingerprint(&database, &base).await; + let initial = prepare_and_load_initial(&base, &base_fingerprint); + let active = target_identity(&initial); + let candidate = compile_variant(Variant::RankRequired, 2); + // A package's fingerprint is the fresh-install fingerprint of its registry, + // and activation holds the migrated catalog to it. + let target_fingerprint = initial_fingerprint(&database, &candidate).await; + let rehearsal_request = |id: &'static str| BackfillSourceRequest { + id, + current: &active, + prior: &base, + candidate: &candidate, + final_fingerprint: &target_fingerprint, + pre: AssertionMode::True, + post: AssertionMode::True, + rehearsed_rows: 0, + }; + + let accepted = prepare_reviewed_candidate( + &active, + &base, + &target_fingerprint, + backfill_source(rehearsal_request("rank-reviewed")), + ); + rehearse(&database, &base, &base_fingerprint, &accepted) + .await + .expect("the reviewed plan activation accepts also rehearses"); + assert_rehearsal_database_clean(&database).await; + + // Comments, a line break after UPDATE, and statement words inside a + // comment or a plain literal leave a chunk the journal can record. + let entity = &candidate.entities()["asset"]; + let rank = &entity.fields["rank"].physical_name; + let commented = prepare_reviewed_candidate( + &active, + &base, + &target_fingerprint, + backfill_source_with_steps( + rehearsal_request("rank-commented"), + Some(format!( + "-- SPDX-License-Identifier: Apache-2.0\nUPDATE\n registry_data.{}\n SET {rank} = CASE WHEN 'never drop a row' = 'x' THEN 1 ELSE 1 END /* never drop a row */\n WHERE record_id = ANY($1::pg_catalog.uuid[]);\n", + entity.physical_table + )), + true, + ), + ); + rehearse(&database, &base, &base_fingerprint, &commented) + .await + .expect("a commented chunk activation accepts also rehearses"); + assert_rehearsal_database_clean(&database).await; + + let unconstrained = prepare_reviewed_candidate( + &active, + &base, + &target_fingerprint, + backfill_source_with_steps(rehearsal_request("rank-unconstrained"), None, false), + ); + let refused = rehearse(&database, &base, &base_fingerprint, &unconstrained) + .await + .expect_err("a plan that leaves the column nullable misses the candidate schema"); + assert_eq!(refused, MigrationRehearsalError::FinalSchemaMismatch); + assert_rehearsal_database_clean(&database).await; + + let canary = "rehearsal-canary-value"; + let uncastable = prepare_reviewed_candidate( + &active, + &base, + &target_fingerprint, + backfill_source_with_steps( + rehearsal_request("rank-uncastable"), + Some(format!( + "UPDATE registry_data.{} SET {rank} = '{canary}' WHERE record_id = ANY($1::pg_catalog.uuid[])", + entity.physical_table + )), + true, + ), + ); + let refused = rehearse(&database, &base, &base_fingerprint, &uncastable) + .await + .expect_err("PostgreSQL refuses the reviewed step's constant"); + let MigrationRehearsalError::Step { + migration_id, + step_id, + failure, + } = &refused + else { + panic!("the refusal names the reviewed step: {refused:?}"); + }; + assert_eq!(migration_id, "rank-uncastable"); + assert_eq!(step_id, "backfill-rank"); + assert_eq!(failure.sqlstate.as_deref(), Some("22P02")); + let rendered = format!("{refused} {refused:?}"); + assert!( + rendered.contains("data exception"), + "the refusal names the PostgreSQL error class: {rendered}" + ); + assert!( + !rendered.contains(canary), + "the refusal carries no value from the statement: {rendered}" + ); + assert_rehearsal_database_clean(&database).await; + + // The package accepts a chunk whose parsed statement is a plain UPDATE, + // but activation also refuses a reviewed update whose text names a + // refused statement word inside a dollar-quoted body, which the lexical + // check reads as written, before the chunk runs. + let metadata_writer = prepare_reviewed_candidate( + &active, + &base, + &target_fingerprint, + backfill_source_with_steps( + rehearsal_request("rank-dollar-quoted"), + Some(format!( + "UPDATE registry_data.{} SET {rank} = CASE WHEN $$never drop a row$$ = $$x$$ THEN 1 ELSE 1 END WHERE record_id = ANY($1::pg_catalog.uuid[])", + entity.physical_table + )), + true, + ), + ); + let refused = rehearse(&database, &base, &base_fingerprint, &metadata_writer) + .await + .expect_err("a chunk activation would refuse before it runs is refused"); + assert!( + matches!( + &refused, + MigrationRehearsalError::HistoryStep { migration_id, step_id, .. } + if migration_id == "rank-dollar-quoted" && step_id == "backfill-rank" + ), + "the refusal names the reviewed step the journal refuses: {refused:?}" + ); + assert_rehearsal_database_clean(&database).await; + + let wrong_baseline = rehearse(&database, &base, &target_fingerprint, &accepted) + .await + .expect_err("a baseline that does not reproduce is refused before any step"); + assert_eq!( + wrong_baseline, + MigrationRehearsalError::BaselineNotReproducible + ); + assert_rehearsal_database_clean(&database).await; + + database.cleanup().await; +} + +/// A reviewed plan that also carries compiler DDL rehearses it in activation +/// order: the added column arrives nullable, the managed read views are +/// rebuilt, and the deferred `SET NOT NULL` runs after the reviewed backfill. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn real_postgres_rehearsal_runs_compiler_ddl_around_the_reviewed_steps() { + let database = TestDatabase::create(1).await; + let _unused_harness_configs = (&database.runtime_config, &database.tls_runtime_config); + database + .admin + .batch_execute("CREATE EXTENSION btree_gist") + .await + .expect("administrator installs the required extension"); + + let base = compile_variant(Variant::Base, 1); + let base_fingerprint = initial_fingerprint(&database, &base).await; + let active = target_identity(&prepare_and_load_initial(&base, &base_fingerprint)); + let candidate = compile_variant(Variant::BatchAddedRequired, 2); + let target_fingerprint = initial_fingerprint(&database, &candidate).await; + let prepared = prepare_package(build_request( + Variant::BatchAddedRequired, + 2, + Some(&active.package_revision), + &target_fingerprint, + PackageMigrationPlanInput::ReviewedSuccessor { + prior_registry: Box::new(base.clone()), + prior_schema_fingerprint: active.schema_fingerprint.clone(), + migrations: vec![added_required_source( + "add-required-rehearsal", + &active, + &base, + &candidate, + &target_fingerprint, + 0, + )], + }, + DATABASE, + )) + .expect("reviewed candidate prepares"); + assert!( + !prepared.manifest().migration_plan.statements.is_empty(), + "the candidate carries compiler DDL around its reviewed step" + ); + rehearse(&database, &base, &base_fingerprint, &prepared) + .await + .expect("compiler DDL and the reviewed backfill rehearse in activation order"); + assert_rehearsal_database_clean(&database).await; + + database.cleanup().await; +} + /// A reviewed field-encryption flip over rows an active registry already /// holds: the engine seals each keyset chunk under lock in one transaction /// with its journal capture and durable cursor, an injected fault after one @@ -2179,6 +2410,76 @@ async fn lock_timeout_is_bounded() { database.cleanup().await; } +/// A reviewed step PostgreSQL refuses fails `apply` with the SQLSTATE and the +/// object names the server reported, never its message or a statement value. +async fn refused_step_reports_its_sqlstate() { + let database = TestDatabase::create(1).await; + database + .admin + .batch_execute("CREATE EXTENSION btree_gist") + .await + .expect("administrator installs extension"); + let base = compile_variant(Variant::Base, 1); + let fingerprint = initial_fingerprint(&database, &base).await; + let initial = prepare_and_load_initial(&base, &fingerprint); + let active = apply(&database, &initial, ApplyPrecondition::InitialActivation) + .await + .expect("refused-step scenario initial package activates"); + seed_backfill_rows(&database, &base, 1).await; + let required = compile_variant(Variant::RankRequired, 2); + let target_fingerprint = required_target_fingerprint(&database, &required).await; + let entity = &required.entities()["asset"]; + let canary = "apply-statement-canary"; + let source = backfill_source_with_steps( + BackfillSourceRequest { + id: "rank-uncastable", + current: &active, + prior: &base, + candidate: &required, + final_fingerprint: &target_fingerprint, + pre: AssertionMode::True, + post: AssertionMode::True, + rehearsed_rows: 1, + }, + Some(format!( + "UPDATE registry_data.{} SET {} = '{canary}' WHERE record_id = ANY($1::pg_catalog.uuid[])", + entity.physical_table, entity.fields["rank"].physical_name + )), + true, + ); + let package = prepare_and_load_reviewed( + 2, + &active, + &base, + Variant::RankRequired, + &target_fingerprint, + source, + ); + let refused = apply( + &database, + &package, + ApplyPrecondition::Successor { current: &active }, + ) + .await + .expect_err("PostgreSQL refuses the reviewed step's constant"); + let MigrationError::StatementFailed(failure) = &refused else { + panic!("the refusal carries the PostgreSQL failure: {refused:?}"); + }; + assert_eq!(failure.sqlstate.as_deref(), Some("22P02")); + let rendered = format!("{refused} {refused:?}"); + assert!( + rendered.contains("SQLSTATE 22P02 (data exception)"), + "the refusal names the SQLSTATE and its class: {rendered}" + ); + assert!( + !rendered.contains(canary), + "the refusal carries no value from the statement: {rendered}" + ); + assert_value_free(Some(refused.clone()), refused); + assert_non_ready_target(&database, &active, &package, "failed").await; + database.cleanup().await; +} + fn compile_variant(variant: Variant, sequence: u64) -> CompiledRegistry { let module_bytes = module_bytes(variant); let module = parse_module_yaml(&module_bytes).expect("test module parses"); @@ -2402,6 +2703,17 @@ struct BackfillSourceRequest<'a> { } fn backfill_source(request: BackfillSourceRequest<'_>) -> ReviewedMigrationSource { + backfill_source_with_steps(request, None, true) +} + +/// The rank backfill with its update statement optionally replaced and its +/// `SET NOT NULL` step optionally left out, so a rehearsal can be handed a +/// plan PostgreSQL or activation would refuse. +fn backfill_source_with_steps( + request: BackfillSourceRequest<'_>, + update_sql_override: Option, + constrain_rank: bool, +) -> ReviewedMigrationSource { let BackfillSourceRequest { id, current, @@ -2424,10 +2736,12 @@ fn backfill_source(request: BackfillSourceRequest<'_>) -> ReviewedMigrationSourc let alter_path = format!("{base}/steps/set-rank-not-null.sql"); let pre_path = format!("{base}/assertions/pre.sql"); let post_path = format!("{base}/assertions/post.sql"); - let update_sql = format!( - "UPDATE registry_data.{} SET {} = 1 WHERE record_id = ANY($1::pg_catalog.uuid[])", - entity.physical_table, field.physical_name - ); + let update_sql = update_sql_override.unwrap_or_else(|| { + format!( + "UPDATE registry_data.{} SET {} = 1 WHERE record_id = ANY($1::pg_catalog.uuid[])", + entity.physical_table, field.physical_name + ) + }); let alter_sql = format!( "ALTER TABLE registry_data.{} ALTER COLUMN {} SET NOT NULL", entity.physical_table, field.physical_name @@ -2463,26 +2777,27 @@ fn backfill_source(request: BackfillSourceRequest<'_>) -> ReviewedMigrationSourc recovery: ReviewedMigrationRecovery::ExactTargetResume, lock_timeout_ms: 50, statement_timeout_ms: 5_000, - steps: vec![ - ReviewedMigrationStepDescriptor::ChunkedBackfill { - id: "backfill-rank".to_owned(), - entity_id: "asset".to_owned(), - sql_path: update_path.clone(), - objects: vec![object.clone()], - cursor: ChunkCursorProtocol::RecordIdUuidArray, - chunk_size: 2, - max_total_rows: 10, - lock_timeout_ms: 50, - statement_timeout_ms: 5_000, - exact_affected_rows: true, - }, - ReviewedMigrationStepDescriptor::TransactionalSql { + steps: std::iter::once(ReviewedMigrationStepDescriptor::ChunkedBackfill { + id: "backfill-rank".to_owned(), + entity_id: "asset".to_owned(), + sql_path: update_path.clone(), + objects: vec![object.clone()], + cursor: ChunkCursorProtocol::RecordIdUuidArray, + chunk_size: 2, + max_total_rows: 10, + lock_timeout_ms: 50, + statement_timeout_ms: 5_000, + exact_affected_rows: true, + }) + .chain( + constrain_rank.then(|| ReviewedMigrationStepDescriptor::TransactionalSql { id: "set-rank-not-null".to_owned(), sql_path: alter_path.clone(), objects: vec![object], affected_rows: None, - }, - ], + }), + ) + .collect(), pre_assertions: vec![ReviewedMigrationAssertionDescriptor { id: "pre".to_owned(), sql_path: pre_path.clone(), @@ -2499,7 +2814,9 @@ fn backfill_source(request: BackfillSourceRequest<'_>) -> ReviewedMigrationSourc descriptor, current, final_fingerprint, - steps: vec![(update_path, update_sql), (alter_path, alter_sql)], + steps: std::iter::once((update_path, update_sql)) + .chain(constrain_rank.then_some((alter_path, alter_sql))) + .collect(), pre: (pre_path, pre_sql), post: (post_path, post_sql), backup: None, @@ -3420,6 +3737,63 @@ fn reviewed_source(request: ReviewedSourceRequest<'_>) -> ReviewedMigrationSourc } } +fn prepare_reviewed_candidate( + current: &ExpectedRegistryIdentity, + prior: &CompiledRegistry, + fingerprint: &str, + source: ReviewedMigrationSource, +) -> PreparedPackage { + prepare_package(build_request( + Variant::RankRequired, + 2, + Some(¤t.package_revision), + fingerprint, + PackageMigrationPlanInput::ReviewedSuccessor { + prior_registry: Box::new(prior.clone()), + prior_schema_fingerprint: current.schema_fingerprint.clone(), + migrations: vec![source], + }, + DATABASE, + )) + .expect("reviewed candidate prepares") +} + +async fn rehearse( + database: &TestDatabase, + predecessor: &CompiledRegistry, + predecessor_schema_fingerprint: &str, + candidate: &PreparedPackage, +) -> Result<(), MigrationRehearsalError> { + rehearse_successor_migration( + &database.migration_config, + &database.migration_role, + &database.runtime_role, + SuccessorMigrationRehearsal { + predecessor, + predecessor_schema_fingerprint, + candidate, + }, + ) + .await +} + +async fn assert_rehearsal_database_clean(database: &TestDatabase) { + let managed = database + .admin + .query_one( + "SELECT pg_catalog.count(*) + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname IN ('registry_internal', 'registry_data', 'registry_source', + 'registry_derived', 'registry_context')", + &[], + ) + .await + .expect("managed schemas are counted") + .get::<_, i64>(0); + assert_eq!(managed, 0, "the rehearsal rolls back every managed object"); +} + async fn initial_fingerprint(database: &TestDatabase, registry: &CompiledRegistry) -> String { let (mut migration, task) = database.connect_migration().await; let transaction = migration @@ -3666,6 +4040,42 @@ async fn seed_backfill_rows(database: &TestDatabase, registry: &CompiledRegistry .await .expect("administrator seeds a backfill row"); } + // Every live row carries its journal head, as rows written through the + // runtime do, so a reviewed chunk can append the next revision. + let (mut migration, migration_task) = database.connect_migration().await; + let transaction = migration + .transaction() + .await + .expect("backfill seed journal transaction starts"); + for index in 0..count { + let record_id = Uuid::from_u128(index as u128 + 1); + let snapshot = canonical(&serde_json::json!({ + "code": format!("c{index}"), + "legacy": format!("legacy-{index}"), + })); + transaction + .execute( + "INSERT INTO registry_internal.registry_revisions + (entity_id, record_id, record_reference, record_revision, + predecessor_revision, record_lifecycle, package_revision, operation_id, + mutation_kind, principal_reference, request_reference, snapshot) + VALUES ('asset', $1, $2, 1, NULL, 'active', $3, 'op-1', + 'create', 'actor:hash', 'request:hash', $4)", + &[ + &record_id, + &format!("asset:{record_id}"), + &active_revision, + &snapshot, + ], + ) + .await + .expect("backfill seed revision inserts"); + } + transaction + .commit() + .await + .expect("backfill seed revisions commit"); + migration_task.abort(); } fn synthetic_backup_sql( @@ -3977,6 +4387,43 @@ async fn assert_added_required_column( assert!(rows.iter().all(|row| row.get::<_, String>(0) == expected)); } +/// The reviewed-migration history a backfill appended: how many records reached +/// revision 2 and the member count of each reviewed-migration commit, in +/// commit order. +async fn reviewed_migration_history(database: &TestDatabase) -> (i64, Vec) { + let revised: i64 = database + .admin + .query_one( + "SELECT count(*) + FROM registry_internal.registry_revisions + WHERE entity_id = 'asset' + AND record_revision = 2 + AND predecessor_revision = 1", + &[], + ) + .await + .expect("migration revisions count") + .get(0); + let commits = database + .admin + .query( + "SELECT count(member.record_id) + FROM registry_internal.registry_revision_commits AS commit + JOIN registry_internal.registry_revision_commit_members AS member + ON member.commit_position = commit.commit_position + WHERE commit.system_origin = 'breg-reviewed-migration-v1' + GROUP BY commit.commit_position + ORDER BY commit.commit_position", + &[], + ) + .await + .expect("migration commits read") + .iter() + .map(|row| row.get::<_, i64>(0)) + .collect(); + (revised, commits) +} + async fn assert_all_ranks(database: &TestDatabase, registry: &CompiledRegistry, expected: i64) { let entity = ®istry.entities()["asset"]; let rows = database diff --git a/crates/registry-breg/tests/postgres_package.rs b/crates/registry-breg/tests/postgres_package.rs index 843420282f..45bb9a08d7 100644 --- a/crates/registry-breg/tests/postgres_package.rs +++ b/crates/registry-breg/tests/postgres_package.rs @@ -35,12 +35,13 @@ use registry_breg::migration_plan::{ }; use registry_breg::package::{ change_set_to_applicable_migration_plan, compiled_registry_change_set, derive_package_revision, - load_package, load_predecessor_package, prepare_package, CompiledRegistryChangeClass, - CompiledRegistryChangeCode, PackageBindingField, PackageBuildRequest, PackageEnvelope, - PackageError, PackageFile, PackageFileRole, PackageIntent, PackageLoadContext, PackageManifest, - PackageMigrationPlanInput, PackageModuleSource, PackageSignature, PackageSourceFile, - PackageTrustAnchor, PredecessorPackageContext, SignaturePolicy, TrustAnchorKey, - MAX_PACKAGE_SOURCE_FILE_BYTES, TRUST_ANCHOR_API_VERSION, + load_package, load_package_with_verified_envelope, load_predecessor_package, prepare_package, + CompiledRegistryChangeClass, CompiledRegistryChangeCode, PackageBindingField, + PackageBuildRequest, PackageEnvelope, PackageError, PackageFile, PackageFileRole, + PackageIntent, PackageLoadContext, PackageManifest, PackageMigrationPlanInput, + PackageModuleSource, PackageSignature, PackageSourceFile, PackageTrustAnchor, + PredecessorPackageContext, SignaturePolicy, TrustAnchorKey, MAX_PACKAGE_SOURCE_FILE_BYTES, + TRUST_ANCHOR_API_VERSION, }; use registry_breg::postgres::{ begin_record_transaction, install_compiled_schema, managed_schema_fingerprint, ClaimContext, @@ -49,6 +50,10 @@ use registry_breg::postgres::{ use registry_breg::runtime_config::parse_runtime_config; use registry_breg::startup::{prepare_startup, StartupError}; use registry_platform_canonical_json::canonicalize_json; +use registry_platform_config::package::{ + verify_package as verify_shared_package, write_sum_file, PackageLimits as SharedPackageLimits, + REVISION_FILE, SUM_FILE, +}; use registry_platform_crypto::{generate_private_jwk, sign, GeneratedKeyAlgorithm, PrivateJwk}; use serde::Serialize; use serde_json::{json, Value}; @@ -101,9 +106,37 @@ fn package_builder_is_deterministic_and_local_publication_loads() { assert_eq!(first.registry(), second.registry()); let root = TempRoot::create(); - first - .publish_to_directory(root.path(), Vec::new()) + let first_shared = first + .publish_to_directory_with_revision(root.path(), Vec::new(), Some("source-1")) .expect("local package publishes"); + let repeated_root = TempRoot::create(); + let repeated_shared = second + .publish_to_directory_with_revision(repeated_root.path(), Vec::new(), Some("source-1")) + .expect("the same local package publishes again"); + assert_eq!(first_shared.digest(), repeated_shared.digest()); + assert_eq!(first_shared.revision(), Some("source-1")); + assert_eq!( + verify_shared_package( + root.path(), + &SharedPackageLimits::default(), + "bregctl package" + ) + .expect("shared package verifies") + .digest(), + first_shared.digest() + ); + let replacement_root = TempRoot::create(); + second + .publish_to_directory_with_revision(replacement_root.path(), Vec::new(), Some("source-2")) + .expect("same signed package publishes with different operator revision"); + assert!(matches!( + load_package_with_verified_envelope( + replacement_root.path(), + &local_context(PackageIntent::InitialActivation), + &first_shared, + ), + Err(PackageError::Envelope) + )); load_package( root.path(), &local_context(PackageIntent::InitialActivation), @@ -473,14 +506,13 @@ fn derived_sql_asset_tampering_is_refused_before_activation() { b"SELECT r.id AS id, (r.code) AS summary FROM registry_source.neutral_record r", ) .expect("asset tamper writes"); + refresh_shared_package_envelope(root.path()); assert_eq!(load_error(root.path(), &context), PackageError::Integrity); fs::write(&asset_path, original).expect("asset restores"); + refresh_shared_package_envelope(root.path()); fs::remove_file(&asset_path).expect("asset removes"); - assert!(matches!( - load_error(root.path(), &context), - PackageError::Read | PackageError::Closure - )); + assert_eq!(load_error(root.path(), &context), PackageError::Envelope); } #[test] @@ -500,9 +532,11 @@ fn derived_sql_asset_extra_path_swap_and_size_are_refused() { b"SELECT r.id AS id, r.code AS summary FROM registry_source.neutral_record r", ) .expect("extra asset writes"); - assert_eq!(load_error(root.path(), &context), PackageError::Closure); + refresh_shared_package_envelope(root.path()); + assert_eq!(load_error(root.path(), &context), PackageError::Envelope); fs::remove_file(root.path().join("source/modules/core/sql/unlisted.sql")) .expect("extra asset removes"); + refresh_shared_package_envelope(root.path()); let original_path = root.path().join("source/modules/core/sql/summary.sql"); let swapped_path = root.path().join("source/modules/core/sql/swapped.sql"); @@ -562,7 +596,7 @@ fn signed_package_refuses_missing_or_rehashed_substituted_fixture_journeys() { missing.root.path(), &missing.context(PackageIntent::InitialActivation), ), - PackageError::Read + PackageError::Envelope ); let substituted = PackageFixture::build( @@ -708,11 +742,13 @@ fn local_unsigned_package_rederives_every_artifact_and_refuses_filesystem_tamper let artifact_path = first_generated_path(fixture.root.path()); let original = fs::read(&artifact_path).expect("artifact reads"); fs::write(&artifact_path, b"tampered artifact").expect("artifact tamper writes"); + refresh_shared_package_envelope(fixture.root.path()); assert_eq!( load_error(fixture.root.path(), &context), PackageError::Integrity ); fs::write(&artifact_path, original).expect("artifact restores"); + refresh_shared_package_envelope(fixture.root.path()); let manifest_projection_path = manifest_projection_path(fixture.root.path()); let original_manifest_projection_bytes = @@ -760,33 +796,39 @@ fn local_unsigned_package_rederives_every_artifact_and_refuses_filesystem_tamper let source = fixture.root.path().join("source/registry.yaml"); let original = fs::read(&source).expect("source reads"); fs::write(&source, b"tampered source").expect("source tamper writes"); + refresh_shared_package_envelope(fixture.root.path()); assert_eq!( load_error(fixture.root.path(), &context), PackageError::Integrity ); fs::write(&source, original).expect("source restores"); + refresh_shared_package_envelope(fixture.root.path()); let module = fixture.root.path().join("source/modules/core/module.yaml"); let original = fs::read(&module).expect("module reads"); fs::write(&module, b"tampered module").expect("module tamper writes"); + refresh_shared_package_envelope(fixture.root.path()); assert_eq!( load_error(fixture.root.path(), &context), PackageError::Integrity ); fs::write(&module, original).expect("module restores"); + refresh_shared_package_envelope(fixture.root.path()); fs::write(fixture.root.path().join("unlisted"), b"unlisted").expect("unlisted file writes"); + refresh_shared_package_envelope(fixture.root.path()); assert_eq!( load_error(fixture.root.path(), &context), - PackageError::Closure + PackageError::Envelope ); fs::remove_file(fixture.root.path().join("unlisted")).expect("unlisted file removes"); + refresh_shared_package_envelope(fixture.root.path()); fs::remove_file(&artifact_path).expect("listed artifact removes"); - assert!(matches!( + assert_eq!( load_error(fixture.root.path(), &context), - PackageError::Read | PackageError::Closure - )); + PackageError::Envelope + ); } #[test] @@ -861,7 +903,7 @@ fn package_manifest_refuses_ddl_checksum_path_and_canonical_json_tampering() { fs::create_dir(&path).expect("non-regular replacement creates"); assert_eq!( load_error(nonregular.root.path(), &context), - PackageError::Closure + PackageError::Envelope ); let noncanonical = @@ -870,6 +912,7 @@ fn package_manifest_refuses_ddl_checksum_path_and_canonical_json_tampering() { let mut bytes = fs::read(&path).expect("manifest reads"); bytes.push(b'\n'); fs::write(&path, bytes).expect("noncanonical manifest writes"); + refresh_shared_package_envelope(noncanonical.root.path()); assert_eq!( load_error(noncanonical.root.path(), &context), PackageError::CanonicalJson @@ -1255,11 +1298,13 @@ fn predecessor_package_refuses_altered_or_forged_closure_bytes() { let original = fs::read(&model_path).expect("governed model reads"); fs::write(&model_path, b"tampered governed model").expect("governed model tamper writes"); + refresh_shared_package_envelope(altered.root.path()); assert_eq!( predecessor_load_error(altered.root.path(), &context), PackageError::Integrity ); fs::write(&model_path, original).expect("governed model restores"); + refresh_shared_package_envelope(altered.root.path()); let forged = PackageFixture::build("local", 1, None, fingerprint(1), PlanChoice::Schema, None); let active_revision = read_envelope(forged.root.path()).signed.package_revision; @@ -2294,7 +2339,7 @@ async fn real_postgres_package_startup_apply_failure_and_old_process_are_closed( .await; assert!(matches!( no_listener_gate.err(), - Some(StartupError::PackageRefused(_)) + Some(StartupError::PackageEnvelopeRefused(_)) )); drop(runtime); @@ -4364,6 +4409,7 @@ fn write_signed_files(root: &Path, files: [(&str, Vec); N]) derive_package_revision(&envelope.signed).expect("mutated revision derives"); envelope.signatures.clear(); write_json(&root.join("package.json"), &envelope); + refresh_shared_package_envelope(root); } fn rewrite_unsigned(root: &Path, mutate: impl FnOnce(&mut PackageManifest)) { @@ -4391,12 +4437,36 @@ fn rewrite_unsigned(root: &Path, mutate: impl FnOnce(&mut PackageManifest)) { derive_package_revision(&envelope.signed).expect("mutated revision derives"); envelope.signatures.clear(); write_json(&root.join("package.json"), &envelope); + refresh_shared_package_envelope(root); } fn rewrite_envelope(root: &Path, mutate: impl FnOnce(&mut PackageEnvelope)) { let mut envelope = read_envelope(root); mutate(&mut envelope); write_json(&root.join("package.json"), &envelope); + refresh_shared_package_envelope(root); +} + +fn refresh_shared_package_envelope(root: &Path) { + let revision_path = root.join(REVISION_FILE); + let revision = revision_path.exists().then(|| { + fs::read_to_string(&revision_path) + .expect("shared package revision reads") + .strip_suffix('\n') + .expect("shared package revision has one trailing newline") + .to_owned() + }); + fs::remove_file(root.join(SUM_FILE)).expect("old shared package checksum file removes"); + if revision.is_some() { + fs::remove_file(revision_path).expect("old shared package revision file removes"); + } + write_sum_file( + root, + revision.as_deref(), + &SharedPackageLimits::default(), + "bregctl package", + ) + .expect("test-authored shared package envelope republishes"); } fn read_envelope(root: &Path) -> PackageEnvelope { diff --git a/crates/registry-breg/tests/postgres_review_executor.rs b/crates/registry-breg/tests/postgres_review_executor.rs index 87b76ee8aa..3f8207b4c4 100644 --- a/crates/registry-breg/tests/postgres_review_executor.rs +++ b/crates/registry-breg/tests/postgres_review_executor.rs @@ -4371,13 +4371,165 @@ async fn a_404_result_lookup_clears_a_stale_token_outage_code() { attempts_before + 1, "a 404 still spends the give-up budget" ); - assert_eq!(row.get::<_, Option>(1), None); + assert_eq!( + row.get::<_, Option>(1).as_deref(), + Some("result-unknown-to-authority"), + "a 404 replaces the stale outage code with its own, so an operator can tell a review the authority does not know from a slow one" + ); + + // The authority answers pending again, so the review is live after all + // and no longer asks for attention. + _script.answer(ScriptedLookup::Pending); + make_result_poll_due(&database.admin, request_id).await; + assert!(poll_scripted_result(&mut database, &endpoint) + .await + .expect("pending result lookup completes")); + let code: Option = database + .admin + .query_one( + "SELECT last_error_code + FROM registry_internal.registry_request_review_submissions + WHERE request_id=$1", + &[&request_id], + ) + .await + .expect("row after pending lookup") + .get(0); + assert_eq!(code, None); drop(pool); server.abort(); database.cleanup().await; } +async fn result_poll_attempts(client: &tokio_postgres::Client, request_id: Uuid) -> i32 { + client + .query_one( + "SELECT result_poll_attempts + FROM registry_internal.registry_request_review_submissions + WHERE request_id=$1", + &[&request_id], + ) + .await + .expect("poll attempts") + .get(0) +} + +#[tokio::test] +async fn a_live_review_is_polled_before_one_its_authority_does_not_know() { + let mut database = prepare_review_database().await; + let unknown = Uuid::from_u128(0xd1); + let live = Uuid::from_u128(0xd3); + seed_accepted_submission(&database, unknown, Uuid::from_u128(0xd2)).await; + let live_accepted = seed_accepted_submission(&database, live, Uuid::from_u128(0xd4)).await; + // The unknown review has waited longest, so plain oldest-first ordering + // would claim it before the live one. + database + .admin + .execute( + "UPDATE registry_internal.registry_request_review_submissions + SET lease_until=NULL,next_result_poll_at=transaction_timestamp(), + last_error_code=CASE WHEN request_id=$1 + THEN 'result-unknown-to-authority' END, + updated_at=transaction_timestamp()-CASE WHEN request_id=$1 + THEN interval '1 hour' ELSE interval '0' END", + &[&unknown], + ) + .await + .expect("make both result polls due"); + let before = result_poll_attempts(&database.admin, unknown).await; + + // The scripted authority only answers for the live review; claiming the + // unknown one first would fail the lookup. + let (endpoint, _script, server) = + serve_scripted_result_authority(live_accepted, ScriptedLookup::Pending).await; + assert!(poll_scripted_result(&mut database, &endpoint) + .await + .expect("the live review is claimed first")); + assert_eq!(result_poll_attempts(&database.admin, live).await, 1); + assert_eq!(result_poll_attempts(&database.admin, unknown).await, before); + + server.abort(); + database.cleanup().await; +} + +#[tokio::test] +async fn a_webhook_completion_makes_its_review_due_and_first_in_the_poll_queue() { + let mut database = prepare_review_database().await; + let waiting = Uuid::from_u128(0xe1); + let completed = Uuid::from_u128(0xe3); + let completed_review = Uuid::from_u128(0xe4); + seed_accepted_submission(&database, waiting, Uuid::from_u128(0xe2)).await; + let completed_accepted = seed_accepted_submission(&database, completed, completed_review).await; + // The completed review is backed off for an hour, and the other review is + // due and older, so without the completion it would wait its turn. + database + .admin + .execute( + "UPDATE registry_internal.registry_request_review_submissions + SET lease_until=NULL, + next_result_poll_at=transaction_timestamp()+CASE WHEN request_id=$1 + THEN interval '1 hour' ELSE interval '0' END, + updated_at=transaction_timestamp()-CASE WHEN request_id=$1 + THEN interval '0' ELSE interval '1 hour' END", + &[&completed], + ) + .await + .expect("back off the completed review"); + + let completion = ReviewCompletion { + event_type: ReviewCompletionType::ReviewCompleted, + event_id: Uuid::from_u128(0xe5), + request_id: completed_review, + result_id: Uuid::from_u128(0xb3), + completed_at: chrono::Utc::now(), + }; + let transaction = database.admin.transaction().await.unwrap(); + receive_completion( + &transaction, + "casework-a", + &completion, + chrono::Utc::now() + chrono::Duration::days(7), + ) + .await + .expect("completion is stored"); + transaction.commit().await.unwrap(); + let due: bool = database + .admin + .query_one( + "SELECT next_result_poll_at <= transaction_timestamp() + FROM registry_internal.registry_request_review_submissions + WHERE request_id=$1", + &[&completed], + ) + .await + .expect("completed review schedule") + .get(0); + assert!(due, "a matched completion makes its review due at once"); + + // The scripted authority only answers for the completed review. + let (endpoint, _script, server) = + serve_scripted_result_authority(completed_accepted, ScriptedLookup::Approved).await; + assert!(poll_scripted_result(&mut database, &endpoint) + .await + .expect("the completed review is claimed first")); + let state: String = database + .admin + .query_one( + "SELECT state FROM registry_internal.registry_request_review_completions + WHERE authority='casework-a' AND event_id=$1", + &[&completion.event_id], + ) + .await + .expect("completion state") + .get(0); + assert_eq!(state, "correlated"); + assert_eq!(result_poll_attempts(&database.admin, waiting).await, 0); + + server.abort(); + database.cleanup().await; +} + #[tokio::test] async fn a_concealed_or_unknown_result_stream_still_exhausts_the_poll_budget() { let mut database = prepare_review_database().await; diff --git a/crates/registry-breg/tests/postgres_spatial_migration.rs b/crates/registry-breg/tests/postgres_spatial_migration.rs index 18750c7cb2..d2c5bd8dde 100644 --- a/crates/registry-breg/tests/postgres_spatial_migration.rs +++ b/crates/registry-breg/tests/postgres_spatial_migration.rs @@ -31,7 +31,8 @@ use registry_breg::package::{ use registry_breg::postgres::{ begin_record_transaction, install_compiled_schema, managed_schema_fingerprint, provision_postgis_prerequisites, verify_catalog_identity_for_catalog, verify_postgis, - ClaimContext, ExpectedManagedCatalog, ExpectedRegistryIdentity, RegistryLockKey, SqlIdentifier, + ClaimContext, ExpectedManagedCatalog, ExpectedRegistryIdentity, PostgresFailure, + RegistryLockKey, SqlIdentifier, }; use registry_breg::startup::{prepare_startup, StartupError}; use registry_breg::CompiledRegistry; @@ -254,7 +255,15 @@ async fn enabling_bbox_on_existing_point_registry_preserves_data_and_recovers_sa &backup_evidence, ) .await; - assert_value_free(destructive_failure.err(), MigrationError::ApplyFailed); + // The recovery fault drops the column the first step already dropped, so + // the refusal carries PostgreSQL's undefined-column SQLSTATE. + assert_value_free( + destructive_failure.err(), + MigrationError::StatementFailed(PostgresFailure { + sqlstate: Some("42703".to_owned()), + ..PostgresFailure::default() + }), + ); assert_non_ready_target( &database, &spatial_active, diff --git a/crates/registry-breg/tests/postgres_startup.rs b/crates/registry-breg/tests/postgres_startup.rs index 5bc1567446..1d779d52e7 100644 --- a/crates/registry-breg/tests/postgres_startup.rs +++ b/crates/registry-breg/tests/postgres_startup.rs @@ -752,6 +752,236 @@ async fn prepared_server_sessions_are_named_bounded_and_pg_stat_statements_stays database.cleanup().await; } +/// A logical restore carries the original instance claim into a database with +/// another physical identity. The copy refuses to serve by name, and readiness +/// fails on a running server, until an operator adopts it. +#[cfg(feature = "tooling")] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_restored_copy_refuses_to_serve_until_adopted() { + restored_copy_journey(false).await; +} + +/// A managed PostgreSQL service may withhold `pg_control_system()` from +/// ordinary roles. The claim then compares the database oid alone, so the +/// Registry still installs, serves, refuses a copy, and adopts one. +#[cfg(feature = "tooling")] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_database_that_withholds_its_system_identifier_still_serves_and_refuses_a_copy() { + restored_copy_journey(true).await; +} + +#[cfg(feature = "tooling")] +async fn restored_copy_journey(withhold_system_identifier: bool) { + use registry_breg::instance_claim::{InstanceClaimError, InstanceClaimService}; + use registry_breg::postgres::RegistryLockKey; + use registry_platform_audit::AuditProfile; + + const AUDIT_KEY: &str = "0123456789abcdef0123456789abcdef"; + let _runtime_guard = WASM_RUNTIME_TEST_LOCK.lock().await; + let database = TestDatabase::create(4).await; + if withhold_system_identifier { + // The function catalog belongs to this disposable database, so the + // revocation reaches no other database in the cluster. + database + .admin + .batch_execute("REVOKE EXECUTE ON FUNCTION pg_catalog.pg_control_system() FROM PUBLIC") + .await + .expect("test withholds the system identifier"); + for role in [&database.migration_role, &database.runtime_role] { + let readable: bool = database + .admin + .query_one( + "SELECT pg_catalog.has_function_privilege( + $1, 'pg_catalog.pg_control_system()', 'EXECUTE')", + &[&role.as_str()], + ) + .await + .expect("function privilege reads") + .get(0); + assert!( + !readable, + "the fixture roles cannot read the system identifier" + ); + } + } + let (migration, migration_task) = database.connect_migration().await; + let fixture = StartupFixture::new(); + let signing = + generate_private_jwk(GeneratedKeyAlgorithm::Es384).expect("fixture signing key generates"); + let provisional = PackageFixture::build(&fixture.root, fingerprint(1), &signing); + let provisional_context = provisional.context(PackageIntent::InitialActivation); + let verified_provisional = load_package(&provisional.root, &provisional_context) + .expect("provisional package loads enough to install schema"); + install_compiled_schema( + &migration, + verified_provisional.registry(), + &database.runtime_role, + ) + .await + .expect("compiled schema installs"); + let expected_catalog = ExpectedManagedCatalog::compiled(verified_provisional.registry()); + let schema_fingerprint = + managed_schema_fingerprint(&migration, &database.runtime_role, &expected_catalog) + .await + .expect("compiled schema fingerprints"); + drop(provisional); + + let package = PackageFixture::build(&fixture.root, schema_fingerprint.clone(), &signing); + let context = package.context(PackageIntent::InitialActivation); + let verified = load_package(&package.root, &context).expect("final package verifies"); + let manifest = verified.manifest(); + let package_sequence = i64::try_from(manifest.sequence).expect("fixture sequence fits"); + initialize_registry_state_for_catalog_test( + &migration, + &database.runtime_role, + &ExpectedManagedCatalog::compiled(verified.registry()), + RegistryStateTestIdentity { + package_id: &manifest.package_id, + environment: &manifest.environment, + instance_id: &manifest.instance_id, + database_id: &manifest.database_id, + package_revision: &manifest.package_revision, + package_sequence, + }, + ) + .await + .expect("Registry state initializes"); + migration_task.abort(); + + let idp = MockIdp::start().await; + let config_path = fixture.write_static_jwks_config( + &package, + &database.migration_role, + &database.runtime_role, + &idp, + Some(AUDIT_KEY), + ); + let prepared = + prepare_with_connection_config_for_test(&config_path, database.runtime_config.clone()) + .await + .expect("the database that recorded the claim serves"); + assert_ready(&prepared, StatusCode::OK).await; + + let claims = InstanceClaimService::new_for_test( + ExpectedRegistryIdentity { + package_id: manifest.package_id.clone(), + environment: manifest.environment.clone(), + instance_id: manifest.instance_id.clone(), + database_id: manifest.database_id.clone(), + package_revision: manifest.package_revision.clone(), + schema_fingerprint: manifest.schema_fingerprint.clone(), + package_sequence, + }, + ExpectedManagedCatalog::compiled(verified.registry()), + RegistryLockKey::derive(&manifest.package_id).expect("lock key derives"), + database.migration_config.clone(), + database.runtime_config.clone(), + database.migration_role.clone(), + database.runtime_role.clone(), + database.audit( + AuditProfile::production_from_secret_bytes(AUDIT_KEY.as_bytes().to_vec().into()) + .expect("test audit profile is keyed"), + ), + ); + let original = claims.status().await.expect("the claim reads"); + assert!( + original.matches, + "the installing database holds its own claim" + ); + assert_eq!( + original.live.system_identifier.is_none(), + withhold_system_identifier, + "the status names a system identifier exactly when it is readable" + ); + assert_eq!( + original + .claim + .as_ref() + .map(|claim| claim.identity.system_identifier.is_none()), + Some(withhold_system_identifier), + "the claim records a system identifier exactly when it was readable" + ); + assert_eq!(original.claim.map(|claim| claim.epoch), Some(1)); + assert_eq!( + claims.adopt().await.err(), + Some(InstanceClaimError::AlreadyCurrent), + "the database the claim names has nothing to adopt" + ); + + // A logical restore keeps every row, so the copy holds the claim the + // original recorded while the database it lands in has another oid. + database + .admin + .execute( + "UPDATE registry_internal.registry_instance_claim + SET database_oid = 1 + WHERE singleton", + &[], + ) + .await + .expect("test simulates a restored copy"); + assert_ready(&prepared, StatusCode::SERVICE_UNAVAILABLE).await; + assert_eq!( + prepare_with_connection_config_for_test(&config_path, database.runtime_config.clone()) + .await + .err(), + Some(StartupError::InstanceClaimMismatch), + "a copy the claim does not name refuses to serve by name" + ); + let copied = claims.status().await.expect("the operator reads the claim"); + assert!(!copied.matches); + assert_eq!(copied.claim.map(|claim| claim.epoch), Some(1)); + + let adoption = claims.adopt().await.expect("the operator adopts the copy"); + assert_eq!(adoption.previous.map(|claim| claim.epoch), Some(1)); + assert_eq!(adoption.current.epoch, 2); + let adopted = claims.status().await.expect("the adopted claim reads"); + assert!(adopted.matches); + assert_eq!(adopted.claim.map(|claim| claim.epoch), Some(2)); + assert_ready(&prepared, StatusCode::OK).await; + // One runtime holds the script engine at a time, so the running server + // stops before the adopted copy starts afresh. + drop(prepared); + prepare_with_connection_config_for_test(&config_path, database.runtime_config.clone()) + .await + .expect("the adopted copy serves"); + + let adoptions: Vec = database + .audit_entries() + .into_iter() + .filter(|entry| { + entry["schema"] == "breg-instance-claim-audit/v1" && entry["phase"] == "response" + }) + .map(|entry| entry["record"].clone()) + .filter(|record| record["outcome"] == "committed") + .collect(); + assert_eq!(adoptions.len(), 1, "one adoption leaves one audit record"); + assert_eq!(adoptions[0]["event"], "adopted"); + assert_eq!(adoptions[0]["previous"]["epoch"], 1); + assert_eq!(adoptions[0]["previous"]["databaseOid"], 1); + assert_eq!(adoptions[0]["current"]["epoch"], 2); + if withhold_system_identifier { + // A claim recorded without the system identifier still names the + // database once the identifier becomes readable. + database + .admin + .batch_execute("GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO PUBLIC") + .await + .expect("test restores the default function privilege"); + let readable = claims.status().await.expect("the claim reads"); + assert!(readable.live.system_identifier.is_some()); + assert!( + readable.matches, + "an unrecorded identifier compares the oid" + ); + prepare_with_connection_config_for_test(&config_path, database.runtime_config.clone()) + .await + .expect("the claim without an identifier keeps serving"); + } + idp.stop().await; + database.cleanup().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 6)] async fn live_old_server_drains_apply_and_exact_successor_restart_becomes_ready() { let _runtime_guard = WASM_RUNTIME_TEST_LOCK.lock().await; diff --git a/crates/registry-breg/tests/runtime_config.rs b/crates/registry-breg/tests/runtime_config.rs index 888898d130..8328c84c75 100644 --- a/crates/registry-breg/tests/runtime_config.rs +++ b/crates/registry-breg/tests/runtime_config.rs @@ -20,6 +20,7 @@ use registry_breg::runtime_config::{ parse_runtime_config_with_env, RuntimeConfigError, RUNTIME_CONFIG_API_VERSION, RUNTIME_CONFIG_KIND, }; +use registry_platform_config::package::{write_sum_file, PackageLimits}; use registry_platform_crypto::PrivateJwk; use registry_platform_httputil::destination::{ DestinationDnsFamily, DestinationSendError, EventDeliveryHeaders, @@ -176,6 +177,82 @@ fn runtime_with_event_destinations(fixture: &RuntimeFixture, bindings: &str) -> ) } +#[test] +fn shared_package_envelope_and_pin_are_checked_before_startup() { + let fixture = RuntimeFixture::new(); + let governed = fixture.package_root.join("package.json"); + fs::write(&governed, b"governed-package\n").expect("governed package placeholder writes"); + let written = write_sum_file( + &fixture.package_root, + Some("source-1"), + &PackageLimits::default(), + "bregctl package", + ) + .expect("shared package envelope writes"); + let raw = valid_runtime( + &fixture.secret_root, + &fixture.package_root, + &fixture.trust_anchor, + ) + .replace( + &format!(" root: {}\n", fixture.package_root.display()), + &format!( + " root: {}\n expectedDigest: {}\n", + fixture.package_root.display(), + written.digest() + ), + ); + let config = parse_runtime_config(&raw).expect("runtime with matching pin parses"); + assert_eq!( + config + .verify_package_envelope() + .expect("matching package pin verifies") + .digest(), + written.digest() + ); + + fs::write(&governed, b"changed\n").expect("governed package changes"); + let changed = config + .verify_package_envelope() + .expect_err("changed governed bytes are refused") + .to_string(); + assert!(changed.contains("package.json"), "{changed}"); + + fs::write(&governed, b"governed-package\n").expect("governed package restores"); + fs::write(fixture.package_root.join("extra.txt"), b"extra\n") + .expect("extra package file writes"); + let extra = config + .verify_package_envelope() + .expect_err("extra package file is refused") + .to_string(); + assert!(extra.contains("extra.txt"), "{extra}"); + fs::remove_file(fixture.package_root.join("extra.txt")).expect("extra package file removes"); + + fs::remove_file(&governed).expect("governed package removes"); + let missing = config + .verify_package_envelope() + .expect_err("missing governed file is refused") + .to_string(); + assert!(missing.contains("package.json"), "{missing}"); + + fs::write(&governed, b"governed-package\n").expect("governed package restores again"); + let wrong = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + let wrong_pin = raw.replace(written.digest(), wrong); + let mismatch = parse_runtime_config(&wrong_pin) + .expect("runtime with a well-formed wrong pin parses") + .verify_package_envelope() + .expect_err("wrong package pin is refused") + .to_string(); + assert_eq!( + mismatch, + format!( + "package.expectedDigest is {wrong} but the package at package.root is {}; \ + deploy the pinned package or update package.expectedDigest", + written.digest() + ) + ); +} + fn event_destination_binding( logical_id: &str, origin: &str, @@ -377,16 +454,16 @@ fn runtime_document_identity_is_required_and_exact() { &base.replace(&format!("apiVersion: {RUNTIME_CONFIG_API_VERSION}\n"), ""), env_lookup ) - .expect_err("missing apiVersion refused by strict document shape"), - RuntimeConfigError::Document + .expect_err("missing apiVersion refused by the envelope"), + RuntimeConfigError::InvalidApiVersion ); assert_eq!( parse_runtime_config_with_env( &base.replace(&format!("kind: {RUNTIME_CONFIG_KIND}\n"), ""), env_lookup ) - .expect_err("missing kind refused by strict document shape"), - RuntimeConfigError::Document + .expect_err("missing kind refused by the envelope"), + RuntimeConfigError::InvalidKind ); } @@ -481,6 +558,21 @@ fn audit_destination_defaults_to_a_rotated_file_and_refuses_incomplete_settings( .expect_err("the destination set is closed"), RuntimeConfigError::Document ); + for retired in [ + "minimumRetentionDays: 400", + "format: keyed-jsonl", + "failClosed: true", + ] { + assert_eq!( + parse_runtime_config_with_env( + &base.replace(&path_line, &format!("{path_line} {retired}\n")), + env_lookup + ) + .expect_err("the audit block beside the flattened key is closed"), + RuntimeConfigError::Document, + "{retired}" + ); + } } #[test] @@ -625,7 +717,8 @@ fn operational_defaults_materialize_without_defaulting_authority() { assert_eq!( parse_runtime_config_with_env(&raw.replace(line, ""), env_lookup) .expect_err("authority-bearing runtime member is never defaulted"), - expected + expected, + "removing {line:?}" ); } } @@ -843,7 +936,7 @@ fn wasm_execution_backend_refuses_unknown_values() { &fixture.package_root, &fixture.trust_anchor, ); - for backend in ["warp", "Pulley", ""] { + for backend in ["warp", "Pulley", "\"\""] { let configured = format!("{base}wasmExecution:\n backend: {backend}\n"); let metadata = parse_runtime_config(&configured) .expect_err("an unknown WASM execution backend is refused") @@ -1931,7 +2024,7 @@ fn debug_and_errors_do_not_render_secret_or_expanded_canaries() { } #[test] -fn unsafe_embedded_env_expansion_is_refused_without_echoing_value() { +fn a_substituted_value_cannot_inject_document_structure() { let fixture = RuntimeFixture::new(); let raw = valid_runtime( &fixture.secret_root, @@ -1943,27 +2036,171 @@ fn unsafe_embedded_env_expansion_is_refused_without_echoing_value() { "OIDC_HOST" => Some("issuer.example\nentities: []".to_owned()), _ => env_lookup(name), }) - .expect_err("unsafe embedded expansion refused"); - assert_eq!(error, RuntimeConfigError::EnvExpansion); + .expect_err("a substituted newline stays inside the issuer string"); + assert_eq!(error, RuntimeConfigError::InvalidOidc); let rendered = format!("{error:?} {error}"); assert!(!rendered.contains("issuer.example")); assert!(!rendered.contains("entities")); } #[test] -fn expanded_runtime_document_is_bounded_before_yaml_parsing() { - let raw = "listener: ${OVERSIZED_RUNTIME_VALUE}\n: malformed\n"; - let error = parse_runtime_config_with_env(raw, |name| match name { +fn substituted_runtime_document_stays_within_the_size_bound() { + let fixture = RuntimeFixture::new(); + let raw = valid_runtime( + &fixture.secret_root, + &fixture.package_root, + &fixture.trust_anchor, + ) + .replace( + "instanceId: registry-primary", + "instanceId: ${OVERSIZED_RUNTIME_VALUE}", + ); + let error = parse_runtime_config_with_env(&raw, |name| match name { "OVERSIZED_RUNTIME_VALUE" => Some("runtime-bound-canary".repeat(4096)), _ => env_lookup(name), }) - .expect_err("oversized expansion refused before parsing"); + .expect_err("oversized substitution refused"); assert_eq!(error, RuntimeConfigError::Bounds); let rendered = format!("{error:?} {error}"); assert!(!rendered.contains("runtime-bound-canary")); } +#[test] +fn a_substitution_inside_a_secret_reference_is_refused() { + let fixture = RuntimeFixture::new(); + let base = valid_runtime( + &fixture.secret_root, + &fixture.package_root, + &fixture.trust_anchor, + ); + let lookup = |name: &str| match name { + "AUDIT_KEY_REF" => Some("secret:file/audit-key".to_owned()), + "SECRET_ROOT" => Some(fixture.secret_root.display().to_string()), + _ => env_lookup(name), + }; + for raw in [ + base.replace( + "hashKeyRef: secret:file/audit-key", + "hashKeyRef: ${AUDIT_KEY_REF}", + ), + base.replace( + "hashKeyRef: secret:file/audit-key", + "hashKeyRef: secret:file/${AUDIT_KEY_REF}", + ), + base.replace( + &format!("root: {}", fixture.secret_root.display()), + "root: ${SECRET_ROOT}", + ), + ] { + let error = parse_runtime_config_with_env(&raw, lookup) + .expect_err("a secret reference or provider takes no substitution"); + assert_eq!(error, RuntimeConfigError::SubstitutionInReference); + assert_eq!(error.code(), "runtime_config.substitution_in_reference"); + assert_eq!(error.path(), "/"); + } +} + +#[test] +fn substituted_values_stay_strings() { + let fixture = RuntimeFixture::new(); + let base = valid_runtime( + &fixture.secret_root, + &fixture.package_root, + &fixture.trust_anchor, + ); + let lookup = |name: &str| match name { + "INSTANCE_ID" => Some("12345".to_owned()), + "POOL_SIZE" => Some("4".to_owned()), + _ => env_lookup(name), + }; + let config = parse_runtime_config_with_env( + &base.replace("instanceId: registry-primary", "instanceId: ${INSTANCE_ID}"), + lookup, + ) + .expect("a numeric-looking substitution is the string it was written as"); + assert_eq!(config.package_load_context().instance_id, "12345"); + + assert_eq!( + parse_runtime_config_with_env(&base.replace("maxSize: 4", "maxSize: ${POOL_SIZE}"), lookup) + .expect_err("a substitution never becomes a number"), + RuntimeConfigError::Document + ); +} + +#[test] +fn oidc_issuer_and_audience_follow_the_shared_issuer_block() { + let fixture = RuntimeFixture::new(); + let base = valid_runtime( + &fixture.secret_root, + &fixture.package_root, + &fixture.trust_anchor, + ); + parse_runtime_config_with_env( + &base.replace("https://issuer.example", "http://127.0.0.1:8095"), + env_lookup, + ) + .expect("a loopback http issuer serves local development"); + for issuer in [ + "urn:breg:issuer", + "http://issuer.example", + "https://user:secret@issuer.example", + "https://issuer.example#fragment", + ] { + assert_eq!( + parse_runtime_config_with_env( + &base.replace("https://issuer.example", issuer), + env_lookup + ) + .expect_err("issuer outside the shared block refused"), + RuntimeConfigError::InvalidOidc, + "{issuer}" + ); + } + let long_audience = "a".repeat(513); + assert_eq!( + parse_runtime_config_with_env(&base.replace("urn:breg:test", &long_audience), env_lookup) + .expect_err("audience above 512 characters refused"), + RuntimeConfigError::InvalidOidc + ); +} + +#[test] +fn jwks_source_uri_kind_skips_discovery_under_the_shared_rules() { + let fixture = RuntimeFixture::new(); + let with_uri = |uri: &str| { + valid_runtime( + &fixture.secret_root, + &fixture.package_root, + &fixture.trust_anchor, + ) + .replace( + " jwksCache:\n", + &format!(" jwksSource:\n kind: uri\n uri: {uri}\n jwksCache:\n"), + ) + }; + let config = + parse_runtime_config_with_env(&with_uri("https://issuer.example/jwks"), env_lookup) + .expect("uri source parses"); + let debug = format!("{config:?}"); + assert!(debug.contains("Uri")); + assert!(!debug.contains("https://issuer.example/jwks")); + parse_runtime_config_with_env(&with_uri("http://127.0.0.1:8095/jwks"), env_lookup) + .expect("loopback http uri source parses"); + for uri in [ + "http://issuer.example/jwks", + "https://user:secret@issuer.example/jwks", + "file:///etc/jwks.json", + ] { + assert_eq!( + parse_runtime_config_with_env(&with_uri(uri), env_lookup) + .expect_err("uri outside the shared rules refused"), + RuntimeConfigError::InvalidOidc, + "{uri}" + ); + } +} + #[cfg(unix)] #[test] fn runtime_config_file_must_not_be_a_symlink() { @@ -3278,3 +3515,24 @@ fn field_encryption_is_absent_by_default_and_validates_operator_binding() { RuntimeConfigError::Document ); } + +#[test] +fn an_audit_key_that_is_not_a_secret_reference_is_an_invalid_audit_binding() { + let fixture = RuntimeFixture::new(); + let base = valid_runtime( + &fixture.secret_root, + &fixture.package_root, + &fixture.trust_anchor, + ); + for reference in ["audit-key", "secret:vault/audit-key", "\"\""] { + let configured = base.replace( + " hashKeyRef: secret:file/audit-key\n", + &format!(" hashKeyRef: {reference}\n"), + ); + assert_eq!( + parse_runtime_config(&configured).err(), + Some(RuntimeConfigError::InvalidAudit), + "{reference} is refused as the audit key" + ); + } +} diff --git a/crates/registry-breg/tests/startup_http.rs b/crates/registry-breg/tests/startup_http.rs index 78107a6cb6..ca75d36509 100644 --- a/crates/registry-breg/tests/startup_http.rs +++ b/crates/registry-breg/tests/startup_http.rs @@ -485,7 +485,7 @@ async fn request_operational_log_has_only_closed_value_free_fields() { /// A description the audit writer gives for a torn audit file. const AUDIT_DESTINATION_REASON: &str = "the audit file could not be opened: audit file has an incomplete final entry; archive it and restart with a fresh path"; -fn startup_errors() -> [StartupError; 18] { +fn startup_errors() -> [StartupError; 19] { [ // The wrapped cause never changes the rendered operational message: it // only lets `bregctl doctor` name it. Any `RuntimeConfigError` variant @@ -494,6 +494,7 @@ fn startup_errors() -> [StartupError; 18] { StartupError::PackageRefused(PackageError::Integrity), StartupError::DatabaseConnection, StartupError::DatabaseUnready, + StartupError::InstanceClaimMismatch, StartupError::FieldPatternSyntax { entity_id: "pattern-private-entity".to_owned(), field_id: "pattern-private-field".to_owned(), @@ -632,8 +633,12 @@ fn expected_startup_error(error: StartupError) -> &'static str { match error { StartupError::RuntimeConfig(_) => "the Registry runtime configuration was refused", StartupError::PackageRefused(_) => "the Registry package was refused", + StartupError::PackageEnvelopeRefused(_) => "the Registry package was refused", StartupError::DatabaseConnection => "the Registry database connection was refused", StartupError::DatabaseUnready => "the Registry database is not ready for this package", + StartupError::InstanceClaimMismatch => { + "the Registry database is not the instance its claim names; adopt a restored copy with bregctl instance-claim adopt" + } StartupError::FieldPatternSyntax { .. } => { "a persisted field pattern has invalid PostgreSQL syntax" } @@ -854,6 +859,14 @@ async fn provenance_operational_logs_metrics_and_traces_are_separate_closed_and_ .await .expect("provenance response"); assert_eq!(provenance.status(), StatusCode::OK); + assert_eq!( + provenance + .headers() + .get("registry-engine-version") + .and_then(|value| value.to_str().ok()), + Some(registry_platform_buildinfo::DISPLAY_VERSION), + "the contract document names the engine release a peer must match" + ); let provenance: Value = serde_json::from_slice( &to_bytes(provenance.into_body(), 1024 * 1024) .await @@ -959,7 +972,7 @@ async fn provenance_operational_logs_metrics_and_traces_are_separate_closed_and_ fs::write(&config_path, canary_runtime_document()).expect("canary runtime config writes"); let output = Command::new(env!("CARGO_BIN_EXE_breg")) .args([ - "--config", + "--runtime-config", config_path.to_str().expect("config path is UTF-8"), ]) .env("BREG_LOG", "info") @@ -1000,11 +1013,25 @@ async fn provenance_operational_logs_metrics_and_traces_are_separate_closed_and_ } } - let invalid_level = Command::new(env!("CARGO_BIN_EXE_breg")) + let removed_flag = Command::new(env!("CARGO_BIN_EXE_breg")) .args([ "--config", config_path.to_str().expect("config path is UTF-8"), ]) + .output() + .expect("breg process runs"); + assert_eq!(removed_flag.status.code(), Some(2)); + assert!(removed_flag.stdout.is_empty()); + assert_eq!( + std::str::from_utf8(&removed_flag.stderr).expect("refusal is UTF-8"), + "breg: --config is no longer accepted; pass --runtime-config FILE\n" + ); + + let invalid_level = Command::new(env!("CARGO_BIN_EXE_breg")) + .args([ + "--runtime-config", + config_path.to_str().expect("config path is UTF-8"), + ]) .env("BREG_LOG", "debug") .output() .expect("invalid log level is rendered through the production logger"); diff --git a/crates/registry-breg/tests/startup_ordering.rs b/crates/registry-breg/tests/startup_ordering.rs index 4c3b792e73..caec2a09fe 100644 --- a/crates/registry-breg/tests/startup_ordering.rs +++ b/crates/registry-breg/tests/startup_ordering.rs @@ -22,6 +22,9 @@ use serde::Serialize; const INSTANCE: &str = "instance-under-test"; const DATABASE: &str = "database-under-test"; const SOURCE_REVISION: &str = "compiler-source-revision"; +const SHARED_PACKAGE_TAMPER: &str = "the package at package.root does not match its SHA256SUMS; \ +changed: openapi/openapi.json; rebuild the package with `bregctl package` and deploy the whole \ +directory"; const FIXTURE_JOURNEYS: &[u8] = br#"apiVersion: registry.registrystack.org/breg-journeys/v1 journeys: - id: neutral-record-list @@ -51,7 +54,10 @@ async fn tampered_package_refuses_before_database_audit_oidc_or_listener_access( Err(error) => error, }; - assert_eq!(error, StartupError::PackageRefused(PackageError::Integrity)); + assert_eq!( + error, + StartupError::PackageEnvelopeRefused(SHARED_PACKAGE_TAMPER.to_owned()) + ); } #[tokio::test] @@ -81,7 +87,7 @@ async fn a_refused_package_keeps_the_cause_that_refused_it() { assert_eq!( tampered, - StartupError::PackageRefused(PackageError::Integrity) + StartupError::PackageEnvelopeRefused(SHARED_PACKAGE_TAMPER.to_owned()) ); assert_eq!( mismatched, diff --git a/crates/registry-bregctl/Cargo.toml b/crates/registry-bregctl/Cargo.toml index 3bb649f0d4..8df30c3b03 100644 --- a/crates/registry-bregctl/Cargo.toml +++ b/crates/registry-bregctl/Cargo.toml @@ -63,6 +63,7 @@ registry-platform-crypto.workspace = true url.workspace = true [dev-dependencies] +chrono.workspace = true registry-breg.workspace = true registry-platform-testing = { workspace = true, features = ["test-utils"] } rustls.workspace = true diff --git a/crates/registry-bregctl/src/apply_lifecycle.rs b/crates/registry-bregctl/src/apply_lifecycle.rs index 6b9df8a415..ac6ff48a1e 100644 --- a/crates/registry-bregctl/src/apply_lifecycle.rs +++ b/crates/registry-bregctl/src/apply_lifecycle.rs @@ -10,8 +10,7 @@ use registry_breg::migration::{ DestructiveBackupEvidence, MigrationError, }; use registry_breg::package::{ - load_package, load_predecessor_package, PackageError, PackageIntent, PackageLoadContext, - PredecessorPackageContext, VerifiedPredecessorPackage, + load_package, PackageError, PackageIntent, PackageLoadContext, VerifiedPredecessorPackage, }; use registry_breg::postgres::ExpectedRegistryIdentity; use registry_breg::runtime_config::{load_runtime_config, RuntimeConfig, RuntimeConfigError}; @@ -73,21 +72,9 @@ pub(crate) fn run( None } else { Some( - load_predecessor_package( - config.package().root(), - &PredecessorPackageContext { - environment: config.identity().environment(), - instance_id: config.identity().instance_id(), - database_id: config.identity().database_id(), - database_initialization_environment: config - .identity() - .database_initialization_environment(), - trust_anchor: config.package_trust_anchor(), - expected_package_revision: config.package().active_revision(), - expected_sequence: config.package().active_sequence(), - }, - ) - .map_err(ApplyLifecycleError::CurrentPackage)?, + config + .load_active_predecessor_package() + .map_err(ApplyLifecycleError::CurrentPackage)?, ) }; let current_identity = current_package diff --git a/crates/registry-bregctl/src/data_lifecycle.rs b/crates/registry-bregctl/src/data_lifecycle.rs index d4402f5f5e..a33e54c80b 100644 --- a/crates/registry-bregctl/src/data_lifecycle.rs +++ b/crates/registry-bregctl/src/data_lifecycle.rs @@ -22,10 +22,10 @@ use registry_breg::data::{ }; use registry_breg::package::{inspect_package_integrity, PackageEnvelope, PackageError}; use registry_breg_client::{ - ingestion_prefix_digest, BRegBatchOperation, BRegIngestionChunk, BRegIngestionChunkReceipt, - BRegIngestionError, BRegIngestionRun, BRegIngestionRunRequest, BRegIngestionRunStatus, - BRegProblemCode, BaseRegistryClient, BaseRegistryClientConfig, BaseRegistryClientError, - BearerToken, BREG_INGESTION_CHUNK_ALGORITHM_VERSION, + ingestion_prefix_digest, BRegBatchOperation, BRegIngestionBlockedReason, BRegIngestionChunk, + BRegIngestionChunkReceipt, BRegIngestionError, BRegIngestionRun, BRegIngestionRunRequest, + BRegIngestionRunStatus, BRegProblemCode, BaseRegistryClient, BaseRegistryClientConfig, + BaseRegistryClientError, BearerToken, BREG_INGESTION_CHUNK_ALGORITHM_VERSION, }; use registry_platform_canonical_json::{canonicalize_json, parse_json_strict}; use registry_platform_httputil::client::{ @@ -69,10 +69,18 @@ pub(crate) enum DataLifecycleError { /// Resuming its committed items under a new run id would duplicate /// mutations, so it is refused rather than upgraded. LegacyImportCheckpoint, - /// The ingestion run is blocked because the active package changed. - ImportRunBlocked, + /// The ingestion run is blocked and refuses further chunks. The reason is + /// the one the run state names, absent only when the run could not be + /// re-read after the refusal. + ImportRunBlocked(Option), /// The ingestion run was cancelled and refuses new chunks. ImportRunCancelled, + /// The service refused to create the run on a failed precondition. For a + /// plan through an `import` grant that means no open import authority + /// admits it; `through_import` says which hint the operator needs. + IngestionRunPrecondition { + through_import: bool, + }, BRegUrl, Token, Runtime, @@ -136,6 +144,8 @@ pub(crate) struct DataValidateOutcome { pub profile_id: String, pub operation: DataImportOperation, pub input_length: u64, + /// SHA-256 of the raw input bytes: the digest an import authority pins. + pub input_digest: String, pub item_count: u64, pub chunk_count: usize, pub maximum_items: u16, @@ -206,6 +216,7 @@ pub(crate) fn validate_import( profile_id: plan.profile_id().to_owned(), operation: plan.operation(), input_length: plan.input_length(), + input_digest: plan.input_digest().to_owned(), item_count: plan.item_count(), chunk_count: plan.chunks().len(), maximum_items: plan.maximum_items(), @@ -269,7 +280,15 @@ impl IngestionDrive<'_> { let created = self .runtime .block_on(self.client.create_ingestion_run(self.entity_route, request)) - .map_err(map_ingestion_client_error)?; + .map_err(|error| match error { + BaseRegistryClientError::Problem { + code: BRegProblemCode::PreconditionFailed, + .. + } => DataLifecycleError::IngestionRunPrecondition { + through_import: self.plan.through_import(), + }, + error => map_ingestion_client_error(error), + })?; Ok(created.value) } @@ -645,7 +664,11 @@ fn submit_ingestion_chunks( while !started.run.complete() && submitted < max_chunks { match started.run.status() { BRegIngestionRunStatus::Open => {} - BRegIngestionRunStatus::Blocked => return Err(DataLifecycleError::ImportRunBlocked), + BRegIngestionRunStatus::Blocked => { + return Err(DataLifecycleError::ImportRunBlocked( + started.run.blocked_reason(), + )) + } BRegIngestionRunStatus::Cancelled => { return Err(DataLifecycleError::ImportRunCancelled) } @@ -704,6 +727,15 @@ fn recover_lost_submission( .. } ); + if matches!( + error, + BaseRegistryClientError::Problem { + code: BRegProblemCode::IngestionRunBlocked, + .. + } + ) { + return Err(blocked_run_reason(drive, run_id)); + } if !run_may_have_committed { return Err(map_ingestion_client_error(error)); } @@ -716,7 +748,9 @@ fn recover_lost_submission( return Ok(RecoveredSubmission::Answered(replayed.run().clone())); } match run.status() { - BRegIngestionRunStatus::Blocked => Err(DataLifecycleError::ImportRunBlocked), + BRegIngestionRunStatus::Blocked => { + Err(DataLifecycleError::ImportRunBlocked(run.blocked_reason())) + } BRegIngestionRunStatus::Cancelled => Err(DataLifecycleError::ImportRunCancelled), BRegIngestionRunStatus::Open | BRegIngestionRunStatus::Complete if run.next_chunk_index() > chunk_index => @@ -727,6 +761,18 @@ fn recover_lost_submission( } } +/// A chunk refused because its run is blocked carries no cause, so the run is +/// read once to name it. The refusal stands whatever the read returns; a +/// failed read leaves the reason unnamed rather than replacing the refusal. +fn blocked_run_reason(drive: &IngestionDrive<'_>, run_id: Uuid) -> DataLifecycleError { + let reason = drive + .read_run(run_id) + .ok() + .filter(|run| run.status() == BRegIngestionRunStatus::Blocked) + .and_then(|run| run.blocked_reason()); + DataLifecycleError::ImportRunBlocked(reason) +} + /// The receipt must name the chunk that was sent: index and digest both. fn validate_ingestion_receipt( receipt: &BRegIngestionChunkReceipt, @@ -810,7 +856,7 @@ fn map_ingestion_client_error(error: BaseRegistryClientError) -> DataLifecycleEr BaseRegistryClientError::Problem { code: BRegProblemCode::IngestionRunBlocked, .. - } => DataLifecycleError::ImportRunBlocked, + } => DataLifecycleError::ImportRunBlocked(None), BaseRegistryClientError::Problem { code: BRegProblemCode::AuthenticationRefused, .. @@ -1647,6 +1693,10 @@ mod tests { } fn compiled() -> registry_breg::CompiledRegistry { + compiled_with(&["create", "batch", "list"]) + } + + fn compiled_with(operations: &[&str]) -> registry_breg::CompiledRegistry { let source = json!({ "apiVersion": "registry.registrystack.org/v1alpha1", "kind": "RegistryProject", @@ -1668,7 +1718,7 @@ mod tests { "principalClaim": "principal", "permissions": [{ "entity": ENTITY, - "operations": ["create", "batch", "list"], + "operations": operations, "readableFields": ["code"], "writableFields": ["code"], "allowDataExport": true, @@ -1681,7 +1731,18 @@ mod tests { } fn import_plan_and_inspected(input: &[u8]) -> (DataImportPlan, InspectedDataPackage) { - let registry = compiled(); + plan_and_inspected(compiled(), input) + } + + /// A plan through an `import` grant, which drives the same run routes. + fn import_grant_plan_and_inspected(input: &[u8]) -> (DataImportPlan, InspectedDataPackage) { + plan_and_inspected(compiled_with(&["import", "list"]), input) + } + + fn plan_and_inspected( + registry: registry_breg::CompiledRegistry, + input: &[u8], + ) -> (DataImportPlan, InspectedDataPackage) { let plan = DataImportPlan::from_jsonl( ®istry, ENTITY, @@ -1912,6 +1973,42 @@ mod tests { ingestion_http_response(200, "OK", &body) } + /// One run document blocked for the named wire reason. + fn blocked_run_response(plan: &DataImportPlan, input: &[u8], reason: &str) -> Vec { + let mut run = ingestion_run_value(plan, input, 0, "blocked"); + run["blockedReason"] = json!(reason); + let body = canonicalize_json(&json!({ "run": run })).unwrap(); + ingestion_http_response(200, "OK", &body) + } + + /// One problem answer carrying the published type, title, and detail the + /// maintained client checks for `code`. + fn problem_response(code: BRegProblemCode, reason: &str) -> Vec { + let body = serde_json::to_vec(&json!({ + "type": format!( + "https://id.registrystack.org/problems/registry-breg/{}", + code.code().replace('.', "/") + ), + "title": reason, + "status": code.status(), + "detail": code.detail(), + "code": code.code(), + "traceId": INGESTION_TRACE_ID + })) + .unwrap(); + let head = format!( + "HTTP/1.1 {} {reason}\r\nContent-Type: application/problem+json\r\n\ + Cache-Control: no-store\r\nVary: authorization, accept\r\n\ + traceparent: 00-{INGESTION_TRACE_ID}-{INGESTION_SPAN_ID}-01\r\n\ + Connection: close\r\nContent-Length: {}\r\n\r\n", + code.status(), + body.len() + ); + let mut response = head.into_bytes(); + response.extend_from_slice(&body); + response + } + /// Stage the sidecar pair a rerun resumes from: the v2 state naming the /// run, and optionally the checkpoint at the boundary the local file /// happens to hold. @@ -2469,7 +2566,12 @@ mod tests { // server, and no chunk was sent to a run that refuses it. assert_eq!(requests.len(), 1); let surfaced = if blocked { - matches!(error, DataLifecycleError::ImportRunBlocked) + matches!( + error, + DataLifecycleError::ImportRunBlocked(Some( + BRegIngestionBlockedReason::ActivePackageChanged + )) + ) } else { matches!(error, DataLifecycleError::ImportRunCancelled) }; @@ -2481,6 +2583,152 @@ mod tests { fs::remove_dir_all(directory).unwrap(); } + #[test] + fn an_import_grant_drives_the_same_run_routes() { + let input = import_input(); + let (plan, inspected) = import_grant_plan_and_inspected(&input); + assert!(plan.through_import()); + let directory = test_directory("ingestion-import-grant"); + let checkpoint_path = directory.join("import.checkpoint.json"); + let (address, handle) = spawn_scripted_server(vec![ + ScriptedExchange::Respond(ingestion_run_response( + 201, "Created", &plan, &input, 0, "open", + )), + ScriptedExchange::Respond(ingestion_submission_response( + &plan, &input, 0, false, "open", + )), + ScriptedExchange::Respond(ingestion_submission_response( + &plan, &input, 1, false, "complete", + )), + ]); + let base = parse_breg_url(&format!("http://{address}")).unwrap(); + let client = ingestion_client(&base, "TEST-TOKEN").unwrap(); + let drive = ingestion_drive(&plan, &inspected, &input, &client); + + let destinations = ImportDestinations::resolve(&checkpoint_path).unwrap(); + let mut started = load_or_start_ingestion(&drive, destinations).unwrap(); + let outcome = submit_ingestion_chunks(&drive, &mut started, None).unwrap(); + let requests = handle.join().unwrap(); + + assert_eq!(requests.len(), 3); + assert!(outcome.complete); + assert_eq!(outcome.committed_items, 3); + let (request_line, _, _) = request_parts(&requests[0]); + assert!(request_line.starts_with("POST /v1/records/records/ingestion-runs")); + + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn a_run_blocked_by_its_import_authority_names_that_reason() { + let input = import_input(); + let (plan, inspected) = import_grant_plan_and_inspected(&input); + let directory = test_directory("ingestion-authority-blocked"); + let checkpoint_path = directory.join("import.checkpoint.json"); + staged_import(&plan, &inspected, &checkpoint_path, None); + let (address, handle) = spawn_scripted_server(vec![ScriptedExchange::Respond( + blocked_run_response(&plan, &input, "importAuthorityClosed"), + )]); + let base = parse_breg_url(&format!("http://{address}")).unwrap(); + let client = ingestion_client(&base, "TEST-TOKEN").unwrap(); + let drive = ingestion_drive(&plan, &inspected, &input, &client); + + let destinations = ImportDestinations::resolve(&checkpoint_path).unwrap(); + let mut started = load_or_start_ingestion(&drive, destinations).unwrap(); + let error = submit_ingestion_chunks(&drive, &mut started, None).unwrap_err(); + assert_eq!(handle.join().unwrap().len(), 1); + assert!( + matches!( + error, + DataLifecycleError::ImportRunBlocked(Some( + BRegIngestionBlockedReason::ImportAuthorityClosed + )) + ), + "{error:?}" + ); + + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn a_blocked_chunk_answer_reads_the_run_to_name_its_reason() { + let input = import_input(); + let (plan, inspected) = import_grant_plan_and_inspected(&input); + let directory = test_directory("ingestion-blocked-answer"); + let checkpoint_path = directory.join("import.checkpoint.json"); + // The refusal carries a code and no cause; the run state names the + // cause, so the drive reads it once and sends nothing further. + let (address, handle) = spawn_scripted_server(vec![ + ScriptedExchange::Respond(ingestion_run_response( + 201, "Created", &plan, &input, 0, "open", + )), + ScriptedExchange::Respond(problem_response( + BRegProblemCode::IngestionRunBlocked, + "Conflict", + )), + ScriptedExchange::Respond(blocked_run_response(&plan, &input, "importAuthorityClosed")), + ]); + let base = parse_breg_url(&format!("http://{address}")).unwrap(); + let client = ingestion_client(&base, "TEST-TOKEN").unwrap(); + let drive = ingestion_drive(&plan, &inspected, &input, &client); + + let destinations = ImportDestinations::resolve(&checkpoint_path).unwrap(); + let mut started = load_or_start_ingestion(&drive, destinations).unwrap(); + let error = submit_ingestion_chunks(&drive, &mut started, None).unwrap_err(); + let requests = handle.join().unwrap(); + assert_eq!(requests.len(), 3); + let (request_line, _, _) = request_parts(&requests[2]); + assert!(request_line.starts_with("GET "), "{request_line}"); + assert!( + matches!( + error, + DataLifecycleError::ImportRunBlocked(Some( + BRegIngestionBlockedReason::ImportAuthorityClosed + )) + ), + "{error:?}" + ); + + fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn a_refused_run_creation_says_whether_an_import_authority_is_in_play() { + let input = import_input(); + for (through_import, (plan, inspected)) in [ + (true, import_grant_plan_and_inspected(&input)), + (false, import_plan_and_inspected(&input)), + ] { + let directory = test_directory("ingestion-precondition"); + let checkpoint_path = directory.join("import.checkpoint.json"); + let (address, handle) = spawn_scripted_server(vec![ScriptedExchange::Respond( + problem_response(BRegProblemCode::PreconditionFailed, "Precondition Failed"), + )]); + let base = parse_breg_url(&format!("http://{address}")).unwrap(); + let client = ingestion_client(&base, "TEST-TOKEN").unwrap(); + let drive = ingestion_drive(&plan, &inspected, &input, &client); + + let destinations = ImportDestinations::resolve(&checkpoint_path).unwrap(); + let error = match load_or_start_ingestion(&drive, destinations) { + Ok(_) => panic!("a refused run creation starts no import"), + Err(error) => error, + }; + assert_eq!(handle.join().unwrap().len(), 1); + assert!( + matches!( + error, + DataLifecycleError::IngestionRunPrecondition { through_import: found } + if found == through_import + ), + "{error:?}" + ); + // No run exists, so no sidecar names one. + assert!(!import_state_path(&checkpoint_path).exists()); + assert!(!checkpoint_path.exists()); + fs::remove_dir_all(directory).unwrap(); + } + } + #[cfg(unix)] #[test] fn write_atomic_skips_temp_symlink_collision_and_refuses_final_symlink() { diff --git a/crates/registry-bregctl/src/dev/mod.rs b/crates/registry-bregctl/src/dev/mod.rs index f1bd8943eb..d00dcc9b8f 100644 --- a/crates/registry-bregctl/src/dev/mod.rs +++ b/crates/registry-bregctl/src/dev/mod.rs @@ -1489,7 +1489,7 @@ pub fn run_supervisor(args: SupervisorArgs) -> Result<()> { command(&mut verify, &root, "verify", None)?; children.breg = Some(service( &args.breg_bin, - &["--config"], + &["--runtime-config"], &root.join("runtime.yaml"), &root, "breg", diff --git a/crates/registry-bregctl/src/doctor.rs b/crates/registry-bregctl/src/doctor.rs index c17410eb0a..838bcfddcc 100644 --- a/crates/registry-bregctl/src/doctor.rs +++ b/crates/registry-bregctl/src/doctor.rs @@ -104,6 +104,9 @@ fn startup_diagnostic(error: StartupError) -> Diagnostic { &format!("the runtime package was refused: {cause}"), ); } + if let StartupError::PackageEnvelopeRefused(cause) = error { + return diagnostic("startup.package.refused", "package", &cause); + } // The writer's own refusal names the rule and the recovery step, and // never a path or a secret. if let StartupError::AuditDestination(reason) = error { @@ -116,6 +119,7 @@ fn startup_diagnostic(error: StartupError) -> Diagnostic { let (code, path, message) = match error { StartupError::RuntimeConfig(_) | StartupError::PackageRefused(_) + | StartupError::PackageEnvelopeRefused(_) | StartupError::AuditDestination(_) => { unreachable!("handled above") } @@ -131,6 +135,11 @@ fn startup_diagnostic(error: StartupError) -> Diagnostic { "database", "the database is not ready for the runtime package", ), + StartupError::InstanceClaimMismatch => ( + "startup.instance_claim.mismatch", + "database", + "the database is not the instance the Registry's claim names, as a restored copy is: once the original is retired, run bregctl instance-claim adopt", + ), StartupError::Audit => ( "startup.audit.refused", "audit", @@ -254,6 +263,7 @@ mod tests { StartupError::PackageRefused(PackageError::Integrity), StartupError::DatabaseConnection, StartupError::DatabaseUnready, + StartupError::InstanceClaimMismatch, StartupError::Audit, StartupError::Cursor, StartupError::Oidc, @@ -358,6 +368,11 @@ mod tests { "startup.database.unready", "database", ), + ( + StartupError::InstanceClaimMismatch, + "startup.instance_claim.mismatch", + "database", + ), (StartupError::Audit, "startup.audit.refused", "audit"), ( StartupError::AuditDestination("the audit file could not be opened".to_owned()), @@ -485,6 +500,11 @@ mod tests { "startup.runtime_config.env_expansion", "/", ), + ( + RuntimeConfigError::SubstitutionInReference, + "startup.runtime_config.substitution_in_reference", + "/", + ), ( RuntimeConfigError::Document, "startup.runtime_config.document", diff --git a/crates/registry-bregctl/src/field_encryption_lifecycle.rs b/crates/registry-bregctl/src/field_encryption_lifecycle.rs index 17148919cc..880764b373 100644 --- a/crates/registry-bregctl/src/field_encryption_lifecycle.rs +++ b/crates/registry-bregctl/src/field_encryption_lifecycle.rs @@ -27,8 +27,7 @@ use registry_breg::field_encryption_backfill::{ }; use registry_breg::migration_plan::ReviewedMigrationStepDescriptor; use registry_breg::package::{ - load_package, load_predecessor_package, PackageError, PackageIntent, PackageLoadContext, - PredecessorPackageContext, VerifiedPredecessorPackage, + load_package, PackageError, PackageIntent, PackageLoadContext, VerifiedPredecessorPackage, }; use registry_breg::postgres::{ExpectedRegistryIdentity, RegistryLockKey}; use registry_breg::runtime_config::{load_runtime_config, RuntimeConfigError}; @@ -80,21 +79,9 @@ pub(crate) fn run_preflight( // The counts are only meaningful against the state the apply would start // from, so bind the same predecessor an apply binds and let the engine pin // the database to it. - let predecessor = load_predecessor_package( - config.package().root(), - &PredecessorPackageContext { - environment: config.identity().environment(), - instance_id: config.identity().instance_id(), - database_id: config.identity().database_id(), - database_initialization_environment: config - .identity() - .database_initialization_environment(), - trust_anchor: config.package_trust_anchor(), - expected_package_revision: config.package().active_revision(), - expected_sequence: config.package().active_sequence(), - }, - ) - .map_err(FieldEncryptionPreflightLifecycleError::PredecessorPackage)?; + let predecessor = config + .load_active_predecessor_package() + .map_err(FieldEncryptionPreflightLifecycleError::PredecessorPackage)?; let expected = predecessor_identity(&predecessor)?; let target_intent = PackageIntent::Activation { active_revision: &expected.package_revision, @@ -200,7 +187,8 @@ pub(crate) fn run_erase_history( let erase = load_erase_request(request.request_file)?; let config = load_runtime_config(request.runtime_config) .map_err(FieldEncryptionEraseHistoryLifecycleError::RuntimeConfig)?; - let package = load_package(config.package().root(), &config.package_load_context()) + let package = config + .load_active_package() .map_err(FieldEncryptionEraseHistoryLifecycleError::ActivePackage)?; let manifest = package.manifest(); let package_sequence = i64::try_from(manifest.sequence).map_err(|_| { diff --git a/crates/registry-bregctl/src/history_erasure_lifecycle.rs b/crates/registry-bregctl/src/history_erasure_lifecycle.rs index b9bcb4820a..ac24942283 100644 --- a/crates/registry-bregctl/src/history_erasure_lifecycle.rs +++ b/crates/registry-bregctl/src/history_erasure_lifecycle.rs @@ -18,7 +18,7 @@ use registry_breg::history_erasure::{ erase_record_history_with_connection, HistoryErasureError, HistoryErasureOutcome, HistoryErasureRequest, HistoryErasureTimeouts, RecordHistoryErasureTarget, }; -use registry_breg::package::{load_package, PackageError}; +use registry_breg::package::PackageError; use registry_breg::postgres::{ExpectedRegistryIdentity, RegistryLockKey}; use registry_breg::runtime_config::{load_runtime_config, RuntimeConfigError}; use registry_platform_canonical_json::parse_json_strict; @@ -79,7 +79,8 @@ pub(crate) fn run( Uuid::parse_str(&erasure.record_id).map_err(|_| HistoryErasureLifecycleError::Target)?; let config = load_runtime_config(request.runtime_config) .map_err(HistoryErasureLifecycleError::RuntimeConfig)?; - let package = load_package(config.package().root(), &config.package_load_context()) + let package = config + .load_active_package() .map_err(HistoryErasureLifecycleError::Package)?; let manifest = package.manifest(); let package_sequence = i64::try_from(manifest.sequence) diff --git a/crates/registry-bregctl/src/history_rebaseline_lifecycle.rs b/crates/registry-bregctl/src/history_rebaseline_lifecycle.rs index 8a2da1fd8f..e4ebf0676d 100644 --- a/crates/registry-bregctl/src/history_rebaseline_lifecycle.rs +++ b/crates/registry-bregctl/src/history_rebaseline_lifecycle.rs @@ -18,7 +18,7 @@ use registry_breg::history_rebaseline::{ rebaseline_history_coverage_with_connection, HistoryRebaselineError, HistoryRebaselineOutcome, HistoryRebaselineRequest, HistoryRebaselineTimeouts, }; -use registry_breg::package::{load_package, PackageError}; +use registry_breg::package::PackageError; use registry_breg::postgres::{ExpectedRegistryIdentity, RegistryLockKey}; use registry_breg::runtime_config::{load_runtime_config, RuntimeConfigError}; use registry_platform_canonical_json::parse_json_strict; @@ -68,7 +68,8 @@ pub(crate) fn run( let rebaseline = load_rebaseline_request(request.request_file)?; let config = load_runtime_config(request.runtime_config) .map_err(HistoryRebaselineLifecycleError::RuntimeConfig)?; - let package = load_package(config.package().root(), &config.package_load_context()) + let package = config + .load_active_package() .map_err(HistoryRebaselineLifecycleError::Package)?; let manifest = package.manifest(); let package_sequence = i64::try_from(manifest.sequence) diff --git a/crates/registry-bregctl/src/import_authority_lifecycle.rs b/crates/registry-bregctl/src/import_authority_lifecycle.rs new file mode 100644 index 0000000000..5a40e49b5a --- /dev/null +++ b/crates/registry-bregctl/src/import_authority_lifecycle.rs @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Operator lifecycle for the import authorities that bound `import` runs. +//! +//! The command opens only the configured migration connection, verifies the +//! active package binding from the runtime configuration, and delegates every +//! database change to `registry_breg::import_authority`. The operator +//! reference and reason reach the database only as keyed hashes, and no +//! refusal repeats them. + +use std::path::Path; +use std::time::Duration; + +use registry_breg::import_authority::{ + ImportAuthority, ImportAuthorityCloseRequest, ImportAuthorityError, ImportAuthorityOpenRequest, + ImportAuthorityOperatorService, MAX_IMPORT_AUTHORITY_WINDOW, +}; +use uuid::Uuid; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ImportAuthorityCliError { + RuntimeConfigPath, + ExpiresIn, + AuthorityId, + Authority(ImportAuthorityError), +} + +pub(crate) struct OpenArguments<'a> { + pub runtime_config: &'a Path, + pub entity: &'a str, + pub profile: &'a str, + pub max_items: i64, + pub expires_in: &'a str, + pub input_sha256: &'a [String], + pub operator_reference: &'a str, + pub reason: &'a str, +} + +pub(crate) struct CloseArguments<'a> { + pub runtime_config: &'a Path, + pub authority_id: &'a str, + pub operator_reference: &'a str, + pub reason: &'a str, +} + +pub(crate) fn open( + arguments: &OpenArguments<'_>, +) -> Result { + runtime_config_path(arguments.runtime_config)?; + let expires_in = + parse_expires_in(arguments.expires_in).ok_or(ImportAuthorityCliError::ExpiresIn)?; + let request = ImportAuthorityOpenRequest { + entity_id: arguments.entity, + profile_id: arguments.profile, + max_items: arguments.max_items, + expires_in, + input_digests: arguments.input_sha256, + operator_reference: arguments.operator_reference, + reason: arguments.reason, + }; + request + .validate() + .map_err(ImportAuthorityCliError::Authority)?; + block_on(async { + service(arguments.runtime_config) + .await? + .open(request) + .await + .map_err(ImportAuthorityCliError::Authority) + }) +} + +pub(crate) fn close( + arguments: &CloseArguments<'_>, +) -> Result { + runtime_config_path(arguments.runtime_config)?; + let authority_id = Uuid::parse_str(arguments.authority_id) + .map_err(|_| ImportAuthorityCliError::AuthorityId)?; + let request = ImportAuthorityCloseRequest { + authority_id, + operator_reference: arguments.operator_reference, + reason: arguments.reason, + }; + request + .validate() + .map_err(ImportAuthorityCliError::Authority)?; + block_on(async { + service(arguments.runtime_config) + .await? + .close(request) + .await + .map_err(ImportAuthorityCliError::Authority) + }) +} + +pub(crate) fn close_expired( + runtime_config: &Path, +) -> Result, ImportAuthorityCliError> { + runtime_config_path(runtime_config)?; + block_on(async { + service(runtime_config) + .await? + .close_expired() + .await + .map_err(ImportAuthorityCliError::Authority) + }) +} + +pub(crate) fn list(runtime_config: &Path) -> Result, ImportAuthorityCliError> { + runtime_config_path(runtime_config)?; + block_on(async { + service(runtime_config) + .await? + .list() + .await + .map_err(ImportAuthorityCliError::Authority) + }) +} + +fn runtime_config_path(path: &Path) -> Result<(), ImportAuthorityCliError> { + if path.is_absolute() { + Ok(()) + } else { + Err(ImportAuthorityCliError::RuntimeConfigPath) + } +} + +async fn service( + runtime_config: &Path, +) -> Result { + ImportAuthorityOperatorService::from_runtime_config(runtime_config) + .await + .map_err(ImportAuthorityCliError::Authority) +} + +fn block_on( + future: impl std::future::Future>, +) -> Result { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|_| ImportAuthorityCliError::Authority(ImportAuthorityError::Unavailable))? + .block_on(future) +} + +/// Parse an authority window written as a whole number of minutes, hours, +/// or days (`90m`, `12h`, `7d`), between one minute and the 30-day maximum. +pub(crate) fn parse_expires_in(text: &str) -> Option { + let unit = match text.as_bytes().last()? { + b'm' => 60, + b'h' => 60 * 60, + b'd' => 24 * 60 * 60, + _ => return None, + }; + let count = &text[..text.len() - 1]; + if count.is_empty() || count.len() > 6 || !count.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let seconds = count.parse::().ok()?.checked_mul(unit)?; + let window = Duration::from_secs(seconds); + (seconds > 0 && window <= MAX_IMPORT_AUTHORITY_WINDOW).then_some(window) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn expiry_accepts_minutes_hours_and_days_within_the_window() { + assert_eq!(parse_expires_in("1m"), Some(Duration::from_secs(60))); + assert_eq!( + parse_expires_in("12h"), + Some(Duration::from_secs(12 * 3600)) + ); + assert_eq!( + parse_expires_in("7d"), + Some(Duration::from_secs(7 * 86_400)) + ); + assert_eq!(parse_expires_in("30d"), Some(MAX_IMPORT_AUTHORITY_WINDOW)); + assert_eq!( + parse_expires_in("43200m"), + Some(MAX_IMPORT_AUTHORITY_WINDOW) + ); + } + + #[test] + fn expiry_refuses_every_other_shape() { + for text in [ + "", "d", "0d", "0m", "31d", "721h", "43201m", "7", "7x", "7D", "+7d", "-7d", "1.5d", + " 7d", "7d ", "9999999d", + ] { + assert_eq!(parse_expires_in(text), None, "{text:?}"); + } + } +} diff --git a/crates/registry-bregctl/src/instance_claim_lifecycle.rs b/crates/registry-bregctl/src/instance_claim_lifecycle.rs new file mode 100644 index 0000000000..49ab072e3a --- /dev/null +++ b/crates/registry-bregctl/src/instance_claim_lifecycle.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Operator lifecycle for the instance claim a restored copy must adopt. +//! +//! The command verifies the active package binding from the runtime +//! configuration and delegates every read and write to +//! `registry_breg::instance_claim`. Adoption moves the claim under the +//! migration authority in one transaction. + +use std::path::Path; + +use registry_breg::instance_claim::{ + InstanceClaimAdoption, InstanceClaimError, InstanceClaimService, InstanceClaimStatus, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum InstanceClaimCliError { + RuntimeConfigPath, + Claim(InstanceClaimError), +} + +pub(crate) fn status(runtime_config: &Path) -> Result { + runtime_config_path(runtime_config)?; + block_on(async { + service(runtime_config) + .await? + .status() + .await + .map_err(InstanceClaimCliError::Claim) + }) +} + +pub(crate) fn adopt(runtime_config: &Path) -> Result { + runtime_config_path(runtime_config)?; + block_on(async { + service(runtime_config) + .await? + .adopt() + .await + .map_err(InstanceClaimCliError::Claim) + }) +} + +fn runtime_config_path(path: &Path) -> Result<(), InstanceClaimCliError> { + if path.is_absolute() { + Ok(()) + } else { + Err(InstanceClaimCliError::RuntimeConfigPath) + } +} + +async fn service(runtime_config: &Path) -> Result { + InstanceClaimService::from_runtime_config(runtime_config) + .await + .map_err(InstanceClaimCliError::Claim) +} + +fn block_on( + future: impl std::future::Future>, +) -> Result { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|_| InstanceClaimCliError::Claim(InstanceClaimError::Unavailable))? + .block_on(future) +} diff --git a/crates/registry-bregctl/src/lib.rs b/crates/registry-bregctl/src/lib.rs index c8e2b900aa..452a5b2de0 100644 --- a/crates/registry-bregctl/src/lib.rs +++ b/crates/registry-bregctl/src/lib.rs @@ -26,6 +26,7 @@ use registry_breg::package::{ FIXTURE_JOURNEYS_PATH, MAX_PACKAGE_SOURCE_FILE_BYTES, MAX_RHAI_PLANNER_PATH_BYTES, MAX_RHAI_PLANNER_SOURCE_BYTES, }; +use registry_breg::postgres::MigrationRehearsalError; use registry_breg::runtime_config::RuntimeConfigError; use registry_breg::tooling::{classify_registry_diff, CompiledRegistryDiff, DiffClassification}; use registry_breg::{ @@ -48,13 +49,16 @@ mod field_encryption; mod field_encryption_lifecycle; mod history_erasure_lifecycle; mod history_rebaseline_lifecycle; +mod import_authority_lifecycle; mod init_from_model; +mod instance_claim_lifecycle; mod package_inspection; mod package_lifecycle; mod project_migration; mod reconcile_lifecycle; mod report; mod request_retention; +mod review_recovery; mod reviewed_migrations; mod safe_path; mod starters; @@ -77,8 +81,11 @@ use history_rebaseline_lifecycle::{ HistoryRebaselineLifecycleError, HistoryRebaselineLifecycleOutcome, HistoryRebaselineLifecycleRequest, }; +use import_authority_lifecycle::ImportAuthorityCliError; +use instance_claim_lifecycle::InstanceClaimCliError; use package_inspection::{ - inspect_runtime_package, inspect_runtime_predecessor_package, RuntimePackageInspectionError, + inspect_runtime_package, inspect_runtime_predecessor_package, + inspect_runtime_predecessor_rehearsal_baseline, RuntimePackageInspectionError, }; use package_lifecycle::{PackageLifecycleError, PackageLifecycleState}; use reconcile_lifecycle::{ @@ -86,10 +93,12 @@ use reconcile_lifecycle::{ }; use registry_breg::data::DataError; use registry_breg::migration_reconcile::{ReconcileError, ReconcileOutcome}; +use registry_breg_client::BRegIngestionBlockedReason; use request_retention::{ RequestRetentionCliError, RequestRetentionDryRunOutcome, RequestRetentionEraseOutcome, RequestRetentionListOutcome, }; +use review_recovery::{ReviewRecoveryCliError, ReviewRecoveryOperation, ReviewRecoveryOutcome}; use safe_path::{EntryStat, SafeDir, SafeEntry, SafePathError, MAX_REMOVE_TREE_DEPTH}; use test_lifecycle::{remove_exact_file, TestLifecycleError, TestLifecycleRequest}; use webhook_lifecycle::{ @@ -168,8 +177,14 @@ enum Command { Webhook(WebhookArgs), /// Inspect and erase eligible change-request retention detail. RequestRetention(RequestRetentionArgs), + /// Resubmit or close a change-request review its authority will not answer. + ReviewRecovery(ReviewRecoveryArgs), /// Erase expired protected action Evidence using configured migration authority. EvidenceRetention(EvidenceRetentionArgs), + /// Open, close, and list the authorities that bound `import` runs. + ImportAuthority(ImportAuthorityArgs), + /// Inspect and adopt the claim naming the database the Registry serves from. + InstanceClaim(InstanceClaimArgs), /// Maintain field-encryption key material. FieldEncryption(FieldEncryptionArgs), } @@ -524,6 +539,10 @@ struct PackageArgs { /// New build directory containing signing-input.json and, once approved, package/. #[arg(long, value_name = "DIRECTORY")] output: PathBuf, + + /// One printable line recorded in the published package as REVISION. + #[arg(long, value_name = "TEXT")] + revision: Option, } #[derive(Debug, Args)] @@ -632,7 +651,7 @@ enum RequestRetentionCommand { #[derive(Debug, Args)] struct AttachmentCleanupArgs { /// Absolute Base Registry Engine runtime configuration file. - #[arg(long, visible_alias = "config", value_name = "ABSOLUTE_FILE")] + #[arg(long, value_name = "ABSOLUTE_FILE")] runtime_config: PathBuf, } @@ -674,6 +693,159 @@ struct RequestRetentionExactArgs { proposal_version: i64, } +#[derive(Debug, Args)] +struct ReviewRecoveryArgs { + #[command(subcommand)] + command: ReviewRecoveryCommand, +} + +#[derive(Debug, Subcommand)] +enum ReviewRecoveryCommand { + /// Submit the exact retained review request again under its original idempotency key. + Resubmit(ReviewRecoveryExactArgs), + /// Close an accepted review without a result so it stops waiting on its authority. + Close(ReviewRecoveryExactArgs), +} + +#[derive(Debug, Args)] +struct ReviewRecoveryExactArgs { + /// Absolute Base Registry Engine runtime configuration file. + #[arg(long, value_name = "ABSOLUTE_FILE")] + runtime_config: PathBuf, + + /// Compiled change-request entity identifier. + #[arg(long, value_name = "ENTITY")] + request_entity: String, + + /// Exact request record UUID. + #[arg(long, value_name = "UUID")] + request_id: String, + + /// Exact proposal version whose review submission to recover. + #[arg(long, value_name = "VERSION")] + proposal_version: i64, +} + +#[derive(Debug, Args)] +struct ImportAuthorityArgs { + #[command(subcommand)] + command: ImportAuthorityCommand, +} + +#[derive(Debug, Subcommand)] +enum ImportAuthorityCommand { + /// Open one bounded authority for an `import` grant of the active package. + Open(ImportAuthorityOpenArgs), + /// Close one authority; the next chunk of every run under it is blocked. + Close(ImportAuthorityCloseArgs), + /// Record every expiry and supersession already due. + CloseExpired(ImportAuthorityRuntimeArgs), + /// List the newest authorities, read only, with the status each has reached. + List(ImportAuthorityRuntimeArgs), +} + +#[derive(Debug, Args)] +struct InstanceClaimArgs { + #[command(subcommand)] + command: InstanceClaimCommand, +} + +#[derive(Debug, Subcommand)] +enum InstanceClaimCommand { + /// Report the claimed database beside the one the runtime role reaches. + Status(InstanceClaimStatusArgs), + /// Make the connected database, such as a restored copy, the one the claim names. + Adopt(InstanceClaimAdoptArgs), +} + +#[derive(Debug, Args)] +struct InstanceClaimStatusArgs { + /// Absolute Base Registry Engine runtime configuration file. + #[arg(long, value_name = "ABSOLUTE_FILE")] + runtime_config: PathBuf, +} + +#[derive(Debug, Args)] +struct InstanceClaimAdoptArgs { + /// Absolute Base Registry Engine runtime configuration file. + #[arg(long, value_name = "ABSOLUTE_FILE")] + runtime_config: PathBuf, + + /// Acknowledge that the database the claim names today no longer serves + /// and never will again. + /// + /// Two databases serving one Registry become divergent writers of its + /// records, import authorities, and outbox work. Without this + /// flag adoption is refused before any database connection is opened. + #[arg(long)] + acknowledge_original_retired: bool, +} + +#[derive(Debug, Args)] +struct ImportAuthorityOpenArgs { + /// Absolute Base Registry Engine runtime configuration file. + #[arg(long, value_name = "ABSOLUTE_FILE")] + runtime_config: PathBuf, + + /// Entity the authority admits creates for. + #[arg(long, value_name = "ENTITY")] + entity: String, + + /// Access profile holding the entity's `import` grant. + #[arg(long, value_name = "PROFILE")] + profile: String, + + /// Most records every run under the authority may create, together. + #[arg(long, value_name = "COUNT")] + max_items: i64, + + /// How long the authority stays open: whole minutes, hours, or days + /// (`90m`, `12h`, `7d`), at most 30 days. There is no extension. + #[arg(long, value_name = "DURATION", default_value = "7d")] + expires_in: String, + + /// SHA-256 input digest a run must announce, as `bregctl data validate` + /// reports it. The client computes it and the run records it; the server + /// does not recompute it over the written items. Repeat to allow several; + /// omit to admit any. + #[arg(long = "input-sha256", value_name = "SHA256")] + input_sha256: Vec, + + /// Operator change reference recorded as a keyed hash. + #[arg(long, value_name = "REFERENCE")] + operator_reference: String, + + /// Reason recorded as a keyed hash, never in clear. + #[arg(long, value_name = "TEXT")] + reason: String, +} + +#[derive(Debug, Args)] +struct ImportAuthorityCloseArgs { + /// Absolute Base Registry Engine runtime configuration file. + #[arg(long, value_name = "ABSOLUTE_FILE")] + runtime_config: PathBuf, + + /// Identifier `import-authority open` reported. + #[arg(long, value_name = "UUID")] + authority_id: String, + + /// Operator change reference recorded as a keyed hash. + #[arg(long, value_name = "REFERENCE")] + operator_reference: String, + + /// Reason recorded as a keyed hash, never in clear. + #[arg(long, value_name = "TEXT")] + reason: String, +} + +#[derive(Debug, Args)] +struct ImportAuthorityRuntimeArgs { + /// Absolute Base Registry Engine runtime configuration file. + #[arg(long, value_name = "ABSOLUTE_FILE")] + runtime_config: PathBuf, +} + #[derive(Debug, Args)] struct WebhookSampleArgs { /// Base Registry Engine authoring project directory. @@ -1183,7 +1355,10 @@ enum DiagnosticArtifact { WebhookSample, WebhookOperations, RequestRetentionOperation, + ReviewRecoveryOperation, EvidenceRetentionOperation, + ImportAuthority, + InstanceClaim, HistoryErasure, HistoryRebaseline, FieldEncryption, @@ -1232,7 +1407,11 @@ enum SuggestedAction { SelectWebhookEvent, VerifyWebhookOperation, VerifyRequestRetentionOperation, + VerifyReviewRecoveryOperation, VerifyEvidenceRetentionOperation, + CorrectImportAuthorityRequest, + VerifyImportAuthority, + VerifyInstanceClaim, PrepareHistoryErasureRequest, PrepareHistoryRebaselineRequest, ReviewRetainedHistory, @@ -1354,6 +1533,10 @@ struct PackageSuccessReport { signature_threshold: u16, provided_signatures: usize, package_files: usize, + #[serde(skip_serializing_if = "Option::is_none")] + package_digest: Option, + #[serde(skip_serializing_if = "Option::is_none")] + revision: Option, signing_input: ArtifactReport, } @@ -1399,6 +1582,7 @@ struct DataValidateSuccessReport { profile_id: String, operation: DataOperationArg, input_length: u64, + input_digest: String, item_count: u64, chunk_count: usize, maximum_items: u16, @@ -1493,6 +1677,15 @@ struct RequestRetentionEraseSuccessReport { outcome: RequestRetentionEraseOutcome, } +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct ReviewRecoverySuccessReport { + ok: bool, + command: &'static str, + #[serde(flatten)] + outcome: ReviewRecoveryOutcome, +} + #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct AttachmentCleanupSuccessReport { @@ -1603,6 +1796,16 @@ struct CapturedPackageCandidate { fixture_journeys: PackageSourceFile, migration_plan: PackageMigrationPlanInput, prevalidation_schema_fingerprint: Option, + rehearsal_baseline: Option, +} + +/// The verified predecessor a successor candidate is rehearsed over: its +/// registry compiled from the signed sources and the schema fingerprint its +/// signed manifest binds. +#[derive(Clone, Debug)] +struct RehearsalBaseline { + registry: CompiledRegistry, + schema_fingerprint: String, } impl CapturedPackageCandidate { @@ -2006,6 +2209,44 @@ where DiagnosticArtifact::EvidenceRetentionOperation, SuggestedAction::VerifyEvidenceRetentionOperation), format, stdout, stderr), }; } + Command::ReviewRecovery(args) => { + let (command, operation, args) = match args.command { + ReviewRecoveryCommand::Resubmit(args) => ( + "review-recovery resubmit", + ReviewRecoveryOperation::Resubmit, + args, + ), + ReviewRecoveryCommand::Close(args) => ( + "review-recovery close", + ReviewRecoveryOperation::Close, + args, + ), + }; + return match review_recovery::recover( + operation, + &args.runtime_config, + &args.request_entity, + &args.request_id, + args.proposal_version, + ) { + Ok(outcome) => write_review_recovery_success( + &ReviewRecoverySuccessReport { + ok: true, + command, + outcome, + }, + format, + stdout, + stderr, + ), + Err(error) => write_failure( + &review_recovery_failure(command, error), + format, + stdout, + stderr, + ), + }; + } Command::RequestRetention(args) => { return match args.command { RequestRetentionCommand::List(args) => match request_retention_list(&args) { @@ -2051,6 +2292,91 @@ where } }; } + Command::ImportAuthority(args) => { + let (command, outcome) = match args.command { + ImportAuthorityCommand::Open(args) => ( + "import-authority open", + import_authority_lifecycle::open(&import_authority_lifecycle::OpenArguments { + runtime_config: &args.runtime_config, + entity: &args.entity, + profile: &args.profile, + max_items: args.max_items, + expires_in: &args.expires_in, + input_sha256: &args.input_sha256, + operator_reference: &args.operator_reference, + reason: &args.reason, + }) + .map(|authority| vec![authority]), + ), + ImportAuthorityCommand::Close(args) => ( + "import-authority close", + import_authority_lifecycle::close( + &import_authority_lifecycle::CloseArguments { + runtime_config: &args.runtime_config, + authority_id: &args.authority_id, + operator_reference: &args.operator_reference, + reason: &args.reason, + }, + ) + .map(|authority| vec![authority]), + ), + ImportAuthorityCommand::CloseExpired(args) => ( + "import-authority close-expired", + import_authority_lifecycle::close_expired(&args.runtime_config), + ), + ImportAuthorityCommand::List(args) => ( + "import-authority list", + import_authority_lifecycle::list(&args.runtime_config), + ), + }; + return match outcome { + Ok(authorities) => { + write_import_authority_success(command, &authorities, format, stdout, stderr) + } + Err(error) => write_failure( + &import_authority_failure(command, error), + format, + stdout, + stderr, + ), + }; + } + Command::InstanceClaim(args) => { + return match args.command { + InstanceClaimCommand::Status(args) => { + match instance_claim_lifecycle::status(&args.runtime_config) { + Ok(status) => write_instance_claim_status(&status, format, stdout, stderr), + Err(error) => write_failure( + &instance_claim_failure("instance-claim status", error), + format, + stdout, + stderr, + ), + } + } + InstanceClaimCommand::Adopt(args) => { + if !args.acknowledge_original_retired { + return write_failure( + &instance_claim_acknowledgement_required(), + format, + stdout, + stderr, + ); + } + match instance_claim_lifecycle::adopt(&args.runtime_config) { + Ok(adoption) => { + write_instance_claim_adoption(&adoption, format, stdout, stderr) + } + Err(error) => write_failure( + &instance_claim_failure("instance-claim adopt", error), + format, + stdout, + stderr, + ), + } + } + }; + } Command::FieldEncryption(args) => { return match args.command { FieldEncryptionCommand::Keygen(args) => { @@ -2169,15 +2495,362 @@ fn request_retention_failure( "restore the original attachment storage binding and verification policy before retrying; the registry pin, retained content, or deletion tombstones still require them", ), }; - FailureReport { - ok: false, - command, - diagnostics: vec![tool_diagnostic( - diagnostic(code, "requestRetention", message), - DiagnosticArtifact::RequestRetentionOperation, - SuggestedAction::VerifyRequestRetentionOperation, - )], + FailureReport { + ok: false, + command, + diagnostics: vec![tool_diagnostic( + diagnostic(code, "requestRetention", message), + DiagnosticArtifact::RequestRetentionOperation, + SuggestedAction::VerifyRequestRetentionOperation, + )], + } +} + +fn review_recovery_failure(command: &'static str, error: ReviewRecoveryCliError) -> FailureReport { + let (code, message) = match error { + ReviewRecoveryCliError::Operator => ( + "review_recovery.operation.refused", + "the review recovery operation was refused; verify the absolute runtime configuration, migration authority, request UUID and positive proposal version".to_owned(), + ), + ReviewRecoveryCliError::NotFound => ( + "review_recovery.submission.not_found", + "no review submission exists for this exact request proposal version".to_owned(), + ), + ReviewRecoveryCliError::Ineligible { + reason, + state, + code, + } => ( + "review_recovery.submission.ineligible", + format!( + "the review submission does not accept this operation: reason {reason}, state {state}, code {}", + code.as_deref().unwrap_or("none") + ), + ), + ReviewRecoveryCliError::RecoveryUnaudited => ( + "review_recovery.recovery.unaudited", + "the review recovery committed but its audit entry was not recorded; restore the audit destination, then read the submission's state from the database before retrying".to_owned(), + ), + }; + FailureReport { + ok: false, + command, + diagnostics: vec![tool_diagnostic( + diagnostic(code, "reviewRecovery", &message), + DiagnosticArtifact::ReviewRecoveryOperation, + SuggestedAction::VerifyReviewRecoveryOperation, + )], + } +} + +fn import_authority_failure( + command: &'static str, + error: ImportAuthorityCliError, +) -> FailureReport { + use registry_breg::import_authority::ImportAuthorityError; + let (failure_diagnostic, action) = match error { + ImportAuthorityCliError::RuntimeConfigPath => ( + diagnostic( + "import_authority.runtime_config.invalid", + "runtimeConfig", + "the runtime configuration must be an absolute path", + ), + SuggestedAction::CorrectCommandUsage, + ), + ImportAuthorityCliError::ExpiresIn => ( + diagnostic( + "import_authority.expires_in.invalid", + "expiresIn", + "the authority window must be a whole number of minutes, hours, or days (for example 90m, 12h, or 7d), from one minute to at most 30 days", + ), + SuggestedAction::CorrectImportAuthorityRequest, + ), + ImportAuthorityCliError::AuthorityId => ( + diagnostic( + "import_authority.authority_id.invalid", + "authorityId", + "the authority identifier must be the UUID `import-authority open` or `import-authority list` reported", + ), + SuggestedAction::CorrectImportAuthorityRequest, + ), + ImportAuthorityCliError::Authority(ImportAuthorityError::InvalidInput) => ( + diagnostic( + "import_authority.request.invalid", + "importAuthority", + "the request is out of bounds: the entity, profile, operator reference, and reason must be present and free of control characters, the volume at least one, and each pinned input digest 64 lowercase hexadecimal characters, named once, at most 16", + ), + SuggestedAction::CorrectImportAuthorityRequest, + ), + ImportAuthorityCliError::Authority(ImportAuthorityError::NotImportable) => ( + diagnostic( + "import_authority.grant.not_importable", + "entity", + "the entity and profile do not name an `import` grant of the active package; check them with `bregctl explain access`", + ), + SuggestedAction::CorrectImportAuthorityRequest, + ), + ImportAuthorityCliError::Authority(ImportAuthorityError::AlreadyOpen) => ( + diagnostic( + "import_authority.already_open", + "entity", + "an import authority is already open for this entity; close it with `bregctl import-authority close` before opening another", + ), + SuggestedAction::VerifyImportAuthority, + ), + ImportAuthorityCliError::Authority(ImportAuthorityError::NotFound) => ( + diagnostic( + "import_authority.not_found", + "authorityId", + "no import authority has this identifier; `bregctl import-authority list` names the recorded ones", + ), + SuggestedAction::VerifyImportAuthority, + ), + ImportAuthorityCliError::Authority(ImportAuthorityError::NotReady) => ( + diagnostic( + "import_authority.not_ready", + "importAuthority", + "the registry is not ready for import authority maintenance; apply the configured package first", + ), + SuggestedAction::VerifyImportAuthority, + ), + ImportAuthorityCliError::Authority(ImportAuthorityError::Unavailable) => ( + diagnostic( + "import_authority.unavailable", + "importAuthority", + "the import authority store is unavailable; verify the runtime configuration, the migration authority, the active package binding, and a keyed audit profile", + ), + SuggestedAction::VerifyImportAuthority, + ), + }; + FailureReport { + ok: false, + command, + diagnostics: vec![tool_diagnostic( + failure_diagnostic, + DiagnosticArtifact::ImportAuthority, + action, + )], + } +} + +fn instance_claim_acknowledgement_required() -> FailureReport { + FailureReport { + ok: false, + command: "instance-claim adopt", + diagnostics: vec![tool_diagnostic( + diagnostic( + "instance_claim.acknowledgement.required", + "acknowledgeOriginalRetired", + "adopting moves the Registry to this database for good, and two databases serving one Registry become divergent writers of it: stop and retire the database the claim names, then pass --acknowledge-original-retired", + ), + DiagnosticArtifact::CommandArguments, + SuggestedAction::CorrectCommandUsage, + )], + } +} + +fn instance_claim_failure(command: &'static str, error: InstanceClaimCliError) -> FailureReport { + use registry_breg::instance_claim::InstanceClaimError; + let (failure_diagnostic, artifact, action) = match error { + InstanceClaimCliError::RuntimeConfigPath => ( + diagnostic( + "instance_claim.runtime_config.invalid", + "runtimeConfig", + "the runtime configuration must be an absolute path", + ), + DiagnosticArtifact::CommandArguments, + SuggestedAction::CorrectCommandUsage, + ), + InstanceClaimCliError::Claim(InstanceClaimError::AlreadyCurrent) => ( + diagnostic( + "instance_claim.already_current", + "instanceClaim", + "the instance claim already names this database; there is nothing to adopt", + ), + DiagnosticArtifact::InstanceClaim, + SuggestedAction::VerifyInstanceClaim, + ), + InstanceClaimCliError::Claim(InstanceClaimError::Unavailable) => ( + diagnostic( + "instance_claim.unavailable", + "instanceClaim", + "the instance claim is unavailable; verify the runtime configuration, both database roles, the active package binding, and a keyed audit profile, and apply the package if the claim table is not yet installed", + ), + DiagnosticArtifact::InstanceClaim, + SuggestedAction::VerifyInstanceClaim, + ), + }; + FailureReport { + ok: false, + command, + diagnostics: vec![tool_diagnostic(failure_diagnostic, artifact, action)], + } +} + +/// A system identifier the role could not read is named as such, so the +/// operator sees that the claim compares the database oid alone. +fn system_identifier_text(value: Option<&str>) -> String { + value.map_or_else( + || "not readable (the claim compares the database oid alone)".to_owned(), + str::to_owned, + ) +} + +fn instance_claim_pairs( + claim: ®istry_breg::instance_claim::InstanceClaim, +) -> Vec<(&'static str, String)> { + vec![ + ( + "system identifier", + system_identifier_text(claim.identity.system_identifier.as_deref()), + ), + ("database oid", claim.identity.database_oid.to_string()), + ("epoch", claim.epoch.to_string()), + ("claimed at", claim.claimed_at.to_rfc3339()), + ] +} + +fn write_instance_claim_status( + status: ®istry_breg::instance_claim::InstanceClaimStatus, + format: OutputFormat, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> ExitCode { + let result = if format == OutputFormat::Json { + let body = json!({"ok": true, "command": "instance-claim status", "status": status}); + serde_json::to_writer_pretty(&mut *stdout, &body) + .map_err(io::Error::other) + .and_then(|()| writeln!(stdout)) + } else { + let lead = match (&status.claim, status.matches) { + (_, true) => "The instance claim names this database.", + (Some(_), false) => { + "The instance claim names another database. Once that database is retired, adopt this one with bregctl instance-claim adopt." + } + (None, false) => { + "No instance claim is recorded. Claim this database with bregctl instance-claim adopt." + } + }; + let mut lines = report::Lines::new(); + lines.lead(lead); + lines.blank(); + lines.pairs(&[ + ( + "this database system identifier", + system_identifier_text(status.live.system_identifier.as_deref()), + ), + ("this database oid", status.live.database_oid.to_string()), + ]); + if let Some(claim) = &status.claim { + lines.blank(); + lines.pairs(&instance_claim_pairs(claim)); + } + stdout.write_all(lines.finish().as_bytes()) + }; + write_result(result, stderr) +} + +fn write_instance_claim_adoption( + adoption: ®istry_breg::instance_claim::InstanceClaimAdoption, + format: OutputFormat, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> ExitCode { + let result = if format == OutputFormat::Json { + let body = json!({"ok": true, "command": "instance-claim adopt", "adoption": adoption}); + serde_json::to_writer_pretty(&mut *stdout, &body) + .map_err(io::Error::other) + .and_then(|()| writeln!(stdout)) + } else { + let mut lines = report::Lines::new(); + lines.lead(&format!( + "Adopted this database. The instance claim is at epoch {}.", + adoption.current.epoch + )); + lines.blank(); + lines.pairs(&instance_claim_pairs(&adoption.current)); + if !adoption.superseded_import_authorities.is_empty() { + lines.heading( + "Superseded import authorities the copy carried open; open a new one before importing again", + ); + for authority_id in &adoption.superseded_import_authorities { + lines.bullet(&authority_id.to_string()); + } + } + stdout.write_all(lines.finish().as_bytes()) + }; + write_result(result, stderr) +} + +fn write_import_authority_success( + command: &'static str, + authorities: &[registry_breg::import_authority::ImportAuthority], + format: OutputFormat, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> ExitCode { + let result = if format == OutputFormat::Json { + let body = match command { + "import-authority open" | "import-authority close" => { + json!({"ok": true, "command": command, "authority": authorities.first()}) + } + _ => json!({"ok": true, "command": command, "authorities": authorities}), + }; + serde_json::to_writer_pretty(&mut *stdout, &body) + .map_err(io::Error::other) + .and_then(|()| writeln!(stdout)) + } else { + let lead = match command { + "import-authority open" => "Opened the import authority.".to_owned(), + "import-authority close" => "Recorded the import authority's final state.".to_owned(), + "import-authority close-expired" => format!( + "Recorded the due import authority transitions. {}.", + report::counted(authorities.len(), "authority") + ), + _ => format!( + "Listed the newest import authorities. {}.", + report::counted(authorities.len(), "authority") + ), + }; + let mut lines = report::Lines::new(); + lines.lead(&lead); + for authority in authorities { + lines.blank(); + lines.pairs(&import_authority_pairs(authority)); + } + stdout.write_all(lines.finish().as_bytes()) + }; + write_result(result, stderr) +} + +fn import_authority_pairs( + authority: ®istry_breg::import_authority::ImportAuthority, +) -> Vec<(&'static str, String)> { + let mut pairs = vec![ + ("authority id", authority.authority_id.to_string()), + ("status", authority.status.as_str().to_owned()), + ("entity", authority.entity_id.clone()), + ("profile", authority.profile_id.clone()), + ( + "committed items", + format!("{} of {}", authority.committed_items, authority.max_items), + ), + ("activation revision", authority.activation_revision.clone()), + ("opened at", authority.opened_at.to_rfc3339()), + ("expires at", authority.expires_at.to_rfc3339()), + ]; + if let Some(closed_at) = authority.closed_at { + pairs.push(("closed at", closed_at.to_rfc3339())); } + pairs.push(( + "announced input digests allowed", + if authority.input_digests.is_empty() { + "none (any input)".to_owned() + } else { + authority.input_digests.join(", ") + }, + )); + pairs } fn history_erase(args: &HistoryEraseArgs) -> Result { @@ -2445,14 +3118,6 @@ fn history_rebaseline_lifecycle_failure(error: HistoryRebaselineLifecycleError) DiagnosticArtifact::HistoryRebaseline, SuggestedAction::ReviewRetainedHistory, ), - registry_breg::history_rebaseline::HistoryRebaselineError::LiveRowBudgetExceeded => ( - "history.rebaseline.live_rows.budget_exceeded", - "history", - "history rebaseline verifies at most 1000 live rows in one transaction and this \ - registry holds more, so retrying cannot restore snapshot coverage", - DiagnosticArtifact::HistoryRebaseline, - SuggestedAction::ReviewRetainedHistory, - ), registry_breg::history_rebaseline::HistoryRebaselineError::HistoryNotReady | registry_breg::history_rebaseline::HistoryRebaselineError::Unavailable => ( "history.rebaseline.unavailable", @@ -2855,14 +3520,6 @@ fn field_encryption_erase_history_failure( DiagnosticArtifact::FieldEncryption, SuggestedAction::ReviewRetainedHistory, ), - registry_breg::history_rebaseline::HistoryRebaselineError::LiveRowBudgetExceeded => ( - "field_encryption.erase_history.rebaseline.live_rows_budget_exceeded", - "history", - "the closing rebaseline verifies at most 1000 live rows in one transaction and this \ - registry holds more, so retrying cannot restore snapshot coverage", - DiagnosticArtifact::FieldEncryption, - SuggestedAction::ReviewRetainedHistory, - ), registry_breg::history_rebaseline::HistoryRebaselineError::HistoryNotReady | registry_breg::history_rebaseline::HistoryRebaselineError::Unavailable => ( "field_encryption.erase_history.rebaseline.unavailable", @@ -3015,6 +3672,7 @@ fn data_validate(args: &DataValidateArgs) -> Result ( + DataLifecycleError::ImportRunBlocked(Some(BRegIngestionBlockedReason::ActivePackageChanged)) => ( format!("{prefix}.ingestion_run.blocked"), "ingestionRun", "the ingestion run is blocked because the active package changed; the run stays inspectable, and a new import under the active package needs a fresh checkpoint path", DiagnosticArtifact::DataOperation, SuggestedAction::VerifyDataCheckpoint, ), + DataLifecycleError::ImportRunBlocked(Some(BRegIngestionBlockedReason::ImportAuthorityClosed)) => ( + format!("{prefix}.ingestion_run.import_authority_closed"), + "ingestionRun", + "the ingestion run is blocked because its import authority closed, expired, or has too little volume left for the next chunk; the committed chunks stay, and the remaining items need a new authority (bregctl import-authority list, then open) and a fresh checkpoint path", + DiagnosticArtifact::DataOperation, + SuggestedAction::VerifyDataCheckpoint, + ), + DataLifecycleError::ImportRunBlocked(None) => ( + format!("{prefix}.ingestion_run.blocked"), + "ingestionRun", + "the ingestion run is blocked and refuses further chunks; read the run to see its blockedReason, and continue in a new import with a fresh checkpoint path", + DiagnosticArtifact::DataOperation, + SuggestedAction::VerifyDataCheckpoint, + ), + DataLifecycleError::IngestionRunPrecondition { through_import: true } => ( + format!("{prefix}.ingestion_run.import_authority_required"), + "ingestionRun", + "the ingestion run was refused because no open import authority admits it: none is open for the entity, it names another profile, it expired, it has too little volume left, or it lists other input digests than the one the run announces; check bregctl import-authority list and open one that covers this input", + DiagnosticArtifact::DataOperation, + SuggestedAction::CorrectDataBinding, + ), + DataLifecycleError::IngestionRunPrecondition { through_import: false } => ( + format!("{prefix}.ingestion_run.precondition_failed"), + "ingestionRun", + "the ingestion run was refused on a failed precondition", + DiagnosticArtifact::DataOperation, + SuggestedAction::CorrectDataBinding, + ), DataLifecycleError::ImportRunCancelled => ( format!("{prefix}.ingestion_run.cancelled"), "ingestionRun", @@ -3265,6 +3951,9 @@ fn diff(args: &DiffArgs) -> Result { runtime_config_diff_failure(error) } RuntimePackageInspectionError::Package(error) => package_diff_failure(error), + RuntimePackageInspectionError::SharedPackage(message) => { + diff_failure("diff.package.integrity_refused", "package", &message) + } })?; (inspected, BaselineAssurance::RuntimeBound) } @@ -3278,12 +3967,15 @@ fn diff(args: &DiffArgs) -> Result { classify_registry_diff(baseline.registry(), &candidate, baseline.package_revision()); let mut compiler_findings = candidate.findings().to_vec(); compiler_findings.extend(unsupported_diff_findings(&compiled_diff)); + compiler_findings.extend(removed_value_findings(&compiled_diff)); compiler_findings.sort(); compiler_findings.dedup(); let findings = compiler_findings .into_iter() .map(|diagnostic| { - let (artifact, action) = if diagnostic.code == "diff.classification.unsupported" { + let (artifact, action) = if diagnostic.code == "diff.classification.unsupported" + || diagnostic.code == REMOVED_VALUES_RETAINED_CODE + { ( DiagnosticArtifact::CompiledDiff, SuggestedAction::ReviewCompiledDiff, @@ -3322,9 +4014,14 @@ fn package(args: &PackageArgs) -> Result { args.schema_fingerprint.as_deref(), ) .map_err(package_lifecycle_failure)?; - let outcome = - package_lifecycle::run(prepared, receipt, &args.output, args.signatures.as_deref()) - .map_err(package_lifecycle_failure)?; + let outcome = package_lifecycle::run( + prepared, + receipt, + &args.output, + args.signatures.as_deref(), + args.revision.as_deref(), + ) + .map_err(package_lifecycle_failure)?; Ok(PackageSuccessReport { ok: true, command: "package", @@ -3337,6 +4034,8 @@ fn package(args: &PackageArgs) -> Result { signature_threshold: outcome.signature_threshold, provided_signatures: outcome.provided_signatures, package_files: outcome.package_files, + package_digest: outcome.package_digest, + revision: outcome.revision, signing_input: ArtifactReport { path: "signing-input.json".to_owned(), media_type: "application/json".to_owned(), @@ -3348,7 +4047,7 @@ fn package(args: &PackageArgs) -> Result { fn test(args: &TestArgs) -> Result { let output = test_lifecycle::preflight_output(&args.output).map_err(test_lifecycle_failure)?; - let candidate = capture_candidate(&args.candidate, "test")?; + let candidate = capture_candidate(&args.candidate, "test", true)?; let outcome = test_lifecycle::run(TestLifecycleRequest { candidate, runtime_config: &args.runtime_config, @@ -3373,12 +4072,41 @@ fn test(args: &TestArgs) -> Result { }) } +/// Compile the verified predecessor so `test` can rehearse the successor +/// migration over its schema. The predecessor was verified a moment earlier; +/// this second read is bound to the same signed revision. +fn capture_rehearsal_baseline( + command: &'static str, + runtime_config: &std::path::Path, + package_revision: &str, +) -> Result { + let unavailable = || { + candidate_failure( + command, + "migration.rehearsal.baseline_unavailable", + "baselineRuntimeConfig", + "the current compiler cannot rebuild the verified predecessor registry from its signed sources, so the successor migration cannot be rehearsed; run test with a bregctl release that compiles the predecessor sources", + DiagnosticArtifact::VerifiedPackage, + SuggestedAction::VerifyPackageIntegrity, + ) + }; + let (predecessor, registry) = inspect_runtime_predecessor_rehearsal_baseline(runtime_config) + .map_err(|_| unavailable())?; + if predecessor.package_revision() != package_revision { + return Err(unavailable()); + } + Ok(RehearsalBaseline { + registry, + schema_fingerprint: predecessor.schema_fingerprint().to_owned(), + }) +} + fn prepare_candidate( args: &PackageCandidateArgs, schema_fingerprint: String, command: &'static str, ) -> Result { - capture_candidate(args, command)? + capture_candidate(args, command, false)? .prepare(schema_fingerprint) .map_err(|error| candidate_package_error(command, error)) } @@ -3386,6 +4114,7 @@ fn prepare_candidate( fn capture_candidate( args: &PackageCandidateArgs, command: &'static str, + rehearse_successor: bool, ) -> Result { let source = capture_project_source(&args.project).map_err(|diagnostic| { source_failure( @@ -3452,6 +4181,7 @@ fn capture_candidate( )], })?; let mut prevalidation_schema_fingerprint = None; + let mut rehearsal_baseline = None; let mut reviewed_changes = String::new(); let (prior_revision, migration_plan) = match args.baseline_runtime_config.as_deref() { Some(runtime_config) => { @@ -3470,6 +4200,13 @@ fn capture_candidate( SuggestedAction::CorrectRuntimeConfiguration, )); } + if rehearse_successor { + rehearsal_baseline = Some(capture_rehearsal_baseline( + command, + runtime_config, + baseline.package_revision(), + )?); + } let changes = registry_breg::package::compiled_registry_change_set_from_baseline( baseline.migration_baseline(), &compiled, @@ -3568,6 +4305,7 @@ fn capture_candidate( }, migration_plan, prevalidation_schema_fingerprint, + rehearsal_baseline, }; if args.reviewed_migrations.is_some() { candidate.prevalidate().map_err(|error| { @@ -3864,6 +4602,7 @@ fn test_lifecycle_failure(error: TestLifecycleError) -> FailureReport { )], }; } + TestLifecycleError::Rehearsal(error) => return migration_rehearsal_failure(*error), TestLifecycleError::Credentials { path, message } => { return FailureReport { ok: false, @@ -3892,6 +4631,7 @@ fn test_lifecycle_failure(error: TestLifecycleError) -> FailureReport { TestLifecycleError::Credentials { .. } => unreachable!("handled before match"), TestLifecycleError::JourneyStep { .. } => unreachable!("handled before match"), TestLifecycleError::CandidateBinding { .. } => unreachable!("handled before match"), + TestLifecycleError::Rehearsal(_) => unreachable!("handled before match"), TestLifecycleError::Candidate => ( "test.candidate.refused", "candidate", @@ -3954,6 +4694,89 @@ fn test_lifecycle_failure(error: TestLifecycleError) -> FailureReport { } } +/// Report a refused successor-migration rehearsal. Every message is built from +/// authored identifiers and PostgreSQL's value-free error fields, never from a +/// server message or detail. +fn migration_rehearsal_failure(error: MigrationRehearsalError) -> FailureReport { + let (code, path, action) = match &error { + MigrationRehearsalError::Database => ( + "test.database.unavailable", + "database".to_owned(), + SuggestedAction::RecreateDisposableDatabase, + ), + MigrationRehearsalError::NotSuccessor | MigrationRehearsalError::ReviewedPlan => ( + "test.candidate.refused", + "candidate".to_owned(), + SuggestedAction::CorrectPackageBuild, + ), + MigrationRehearsalError::BaselineNotReproducible => ( + "migration.rehearsal.baseline_not_reproducible", + "baselineRuntimeConfig".to_owned(), + SuggestedAction::VerifyPackageIntegrity, + ), + MigrationRehearsalError::CompilerStatement { statement_id, .. } => ( + "migration.rehearsal.compiler_statement_failed", + format!("migrationPlan.statements[{statement_id}]"), + SuggestedAction::CorrectPackageBuild, + ), + MigrationRehearsalError::Assertion { + migration_id, + phase, + assertion_id, + .. + } + | MigrationRehearsalError::AssertionShape { + migration_id, + phase, + assertion_id, + } => ( + "migration.rehearsal.assertion_failed", + format!("reviewedMigrations[{migration_id}].{phase}Assertions[{assertion_id}]"), + SuggestedAction::CorrectPackageBuild, + ), + MigrationRehearsalError::Step { + migration_id, + step_id, + .. + } => ( + "migration.rehearsal.step_failed", + format!("reviewedMigrations[{migration_id}].steps[{step_id}]"), + SuggestedAction::CorrectPackageBuild, + ), + MigrationRehearsalError::HistoryStep { + migration_id, + step_id, + .. + } => ( + "migration.rehearsal.history_step_refused", + format!("reviewedMigrations[{migration_id}].steps[{step_id}]"), + SuggestedAction::CorrectPackageBuild, + ), + MigrationRehearsalError::FinalSchemaMismatch => ( + "migration.rehearsal.schema_mismatch", + "reviewedMigrations".to_owned(), + SuggestedAction::CorrectPackageBuild, + ), + }; + let artifact = if matches!(error, MigrationRehearsalError::Database) { + DiagnosticArtifact::SchemaTestDatabase + } else { + DiagnosticArtifact::DatabaseMigration + }; + let message = format!( + "the successor migration was rehearsed over an empty copy of the verified predecessor schema and refused, so apply would refuse it too: {error}" + ); + FailureReport { + ok: false, + command: "test", + diagnostics: vec![tool_diagnostic( + diagnostic(code, &path, &message), + artifact, + action, + )], + } +} + fn apply_lifecycle_failure(error: ApplyLifecycleError) -> FailureReport { let error = match error { ApplyLifecycleError::RuntimeConfig(error) => { @@ -4177,6 +5000,20 @@ fn apply_lifecycle_failure(error: ApplyLifecycleError) -> FailureReport { DiagnosticArtifact::DatabaseMigration, SuggestedAction::VerifyMigrationAuthority, ), + registry_breg::migration::MigrationError::StatementFailed(failure) => { + return source_failure( + "apply", + diagnostic( + "apply.migration.statement_failed", + "database", + &format!( + "PostgreSQL refused an apply statement with {failure}. The exact target remains pinned in maintenance: fix the cause the SQLSTATE and the named objects point at and retry the same target, or assess the pinned target with migration reconcile" + ), + ), + DiagnosticArtifact::DatabaseMigration, + SuggestedAction::ReconcileFailedMigration, + ); + } registry_breg::migration::MigrationError::ApplyFailed => ( "apply.migration.failed", "database", @@ -4511,9 +5348,27 @@ fn inspection_failure( prefix: &'static str, error: RuntimePackageInspectionError, ) -> FailureReport { - if let RuntimePackageInspectionError::RuntimeConfig(error) = error { - return runtime_config_failure(command, prefix, error); - } + let error = match error { + RuntimePackageInspectionError::RuntimeConfig(error) => { + return runtime_config_failure(command, prefix, error); + } + RuntimePackageInspectionError::SharedPackage(message) => { + return FailureReport { + ok: false, + command, + diagnostics: vec![tool_diagnostic( + diagnostic( + &format!("{prefix}.package.integrity_refused"), + "package", + &message, + ), + DiagnosticArtifact::VerifiedPackage, + SuggestedAction::VerifyPackageIntegrity, + )], + }; + } + other => other, + }; let (code, path, message, artifact, action) = match error { RuntimePackageInspectionError::RuntimeConfigPath => ( format!("{prefix}.runtime_config.path_invalid"), @@ -4523,6 +5378,7 @@ fn inspection_failure( SuggestedAction::CorrectRuntimeConfiguration, ), RuntimePackageInspectionError::RuntimeConfig(_) => unreachable!("handled before match"), + RuntimePackageInspectionError::SharedPackage(_) => unreachable!("handled before match"), RuntimePackageInspectionError::Package(error) => { let (suffix, action) = match error { PackageError::UnsafePath => ("path_refused", SuggestedAction::VerifyPackagePath), @@ -4543,7 +5399,8 @@ fn inspection_failure( "anchor_not_canonical", SuggestedAction::VerifyPackageIntegrity, ), - PackageError::Closure + PackageError::Envelope + | PackageError::Closure | PackageError::Integrity | PackageError::CanonicalJson | PackageError::Derivation @@ -4637,7 +5494,8 @@ fn package_diff_failure(error: PackageError) -> FailureReport { "diff.baseline.anchor_not_canonical", SuggestedAction::VerifyPackageIntegrity, ), - PackageError::Closure + PackageError::Envelope + | PackageError::Closure | PackageError::Integrity | PackageError::CanonicalJson | PackageError::Derivation @@ -4701,6 +5559,34 @@ fn unsupported_diff_findings(diff: &CompiledRegistryDiff) -> Vec { .collect() } +const REMOVED_VALUES_RETAINED_CODE: &str = "diff.history.removed_values_retained"; + +/// Removing a field or an entity drops its live column or table, but every +/// revision snapshot recorded before the change still holds the values in the +/// retained history. Each removal says so, so an operator does not mistake a +/// package change for erasure. +fn removed_value_findings(diff: &CompiledRegistryDiff) -> Vec { + diff.changes + .iter() + .filter(|change| { + matches!( + change.change.code, + registry_breg::package::CompiledRegistryChangeCode::FieldRemoved + | registry_breg::package::CompiledRegistryChangeCode::EntityRemoved + ) + }) + .map(|change| Diagnostic { + severity: DiagnosticSeverity::Finding, + code: REMOVED_VALUES_RETAINED_CODE.to_owned(), + path: diff_change_path(&change.change), + message: "removing this from the package is not erasure: the values stay in every \ + revision snapshot recorded before the change; only `bregctl history erase` \ + removes them, one record's revisions at a time" + .to_owned(), + }) + .collect() +} + /// List the selected changes as `code at target`, with the sentence a code /// carries beyond its name, so a refusal names what an adopter has to act on. fn rendered_changes( @@ -8696,6 +9582,7 @@ fn operation_wire_name(operation: registry_breg::contract::Operation) -> &'stati registry_breg::contract::Operation::ApplyRequest => "apply_request", registry_breg::contract::Operation::Invoke => "invoke", registry_breg::contract::Operation::Snapshot => "snapshot", + registry_breg::contract::Operation::Import => "import", } } @@ -11037,6 +11924,7 @@ fn write_data_validate_success( data_operation_name(report.operation).to_owned(), ), ("input bytes", report.input_length.to_string()), + ("input sha256", report.input_digest.clone()), ("items", report.item_count.to_string()), ("chunks", report.chunk_count.to_string()), ("maximum items", report.maximum_items.to_string()), @@ -11371,6 +12259,46 @@ fn write_request_retention_erase_success( write_result(result, stderr) } +fn write_review_recovery_success( + report: &ReviewRecoverySuccessReport, + format: OutputFormat, + stdout: &mut dyn Write, + stderr: &mut dyn Write, +) -> ExitCode { + let result = if format == OutputFormat::Json { + serde_json::to_writer_pretty(&mut *stdout, report) + .map_err(io::Error::other) + .and_then(|()| writeln!(stdout)) + } else { + let recovery = &report.outcome.recovery; + render_report( + if recovery.state == "pending" { + "Queued the review for resubmission." + } else { + "Closed the review." + }, + &[ + ("request entity", recovery.request_entity_id.clone()), + ("request id", recovery.request_id.clone()), + ("proposal version", recovery.proposal_version.to_string()), + ("authority", recovery.authority.clone()), + ("previous state", recovery.previous_state.clone()), + ( + "previous code", + recovery + .previous_code + .clone() + .unwrap_or_else(|| "none".to_owned()), + ), + ("state", recovery.state.to_owned()), + ("code", recovery.code.unwrap_or("none").to_owned()), + ], + stdout, + ) + }; + write_result(result, stderr) +} + fn data_operation_name(operation: DataOperationArg) -> &'static str { match operation { DataOperationArg::Create => "create", @@ -11756,22 +12684,29 @@ mod tests { #[test] fn request_retention_cleanup_accepts_config_without_request_scope() { - for flag in ["--runtime-config", "--config"] { - let parsed = Cli::try_parse_from([ - "bregctl", - "request-retention", - "cleanup-attachments", - flag, - "/tmp/runtime.yaml", - ]) - .unwrap(); - assert!(matches!( - parsed.command, - Command::RequestRetention(RequestRetentionArgs { - command: RequestRetentionCommand::CleanupAttachments(_) - }) - )); - } + let parsed = Cli::try_parse_from([ + "bregctl", + "request-retention", + "cleanup-attachments", + "--runtime-config", + "/tmp/runtime.yaml", + ]) + .unwrap(); + assert!(matches!( + parsed.command, + Command::RequestRetention(RequestRetentionArgs { + command: RequestRetentionCommand::CleanupAttachments(_) + }) + )); + let removed = Cli::try_parse_from([ + "bregctl", + "request-retention", + "cleanup-attachments", + "--config", + "/tmp/runtime.yaml", + ]) + .expect_err("the removed --config alias is refused"); + assert_eq!(removed.kind(), clap::error::ErrorKind::UnknownArgument); let report = AttachmentCleanupSuccessReport { ok: true, command: "request-retention cleanup-attachments", @@ -11789,6 +12724,22 @@ mod tests { ); } + #[test] + fn review_recovery_ineligible_diagnostic_names_reason_state_and_code() { + let report = review_recovery_failure( + "review-recovery resubmit", + ReviewRecoveryCliError::Ineligible { + reason: "request-erased", + state: "failed".to_owned(), + code: Some("result-poll-attempts-exhausted".to_owned()), + }, + ); + let encoded = serde_json::to_string(&report).expect("report encodes"); + assert!(encoded.contains("review_recovery.submission.ineligible")); + assert!(encoded + .contains("reason request-erased, state failed, code result-poll-attempts-exhausted")); + } + #[test] fn request_retention_binding_diagnostic_identifies_recovery() { let report = request_retention_failure( @@ -12622,6 +13573,111 @@ mod tests { .contains("without the output")); } + #[test] + fn a_blocked_or_refused_import_run_names_its_cause_and_next_step() { + for (error, code, hint) in [ + ( + DataLifecycleError::ImportRunBlocked(Some( + BRegIngestionBlockedReason::ActivePackageChanged, + )), + "data.import.ingestion_run.blocked", + "active package changed", + ), + ( + DataLifecycleError::ImportRunBlocked(Some( + BRegIngestionBlockedReason::ImportAuthorityClosed, + )), + "data.import.ingestion_run.import_authority_closed", + "bregctl import-authority list", + ), + ( + DataLifecycleError::ImportRunBlocked(None), + "data.import.ingestion_run.blocked", + "blockedReason", + ), + ( + DataLifecycleError::IngestionRunPrecondition { + through_import: true, + }, + "data.import.ingestion_run.import_authority_required", + "bregctl import-authority list", + ), + ( + DataLifecycleError::IngestionRunPrecondition { + through_import: false, + }, + "data.import.ingestion_run.precondition_failed", + "precondition", + ), + ] { + let report = + serde_json::to_value(data_lifecycle_failure("data import", "data.import", error)) + .expect("the failure report serializes"); + assert_eq!(report["diagnostics"][0]["code"], code); + assert_eq!(report["diagnostics"][0]["path"], "ingestionRun"); + let message = report["diagnostics"][0]["message"] + .as_str() + .expect("the message renders"); + assert!(message.contains(hint), "{code}: {message}"); + } + } + + #[test] + fn import_authority_reports_name_the_bounds_in_both_formats() { + use registry_breg::import_authority::{ImportAuthority, ImportAuthorityStatus}; + let opened_at = chrono::DateTime::parse_from_rfc3339("2026-09-25T10:00:00Z") + .expect("instant parses") + .with_timezone(&chrono::Utc); + let authority = ImportAuthority { + authority_id: uuid::Uuid::nil(), + entity_id: "widget".to_owned(), + profile_id: "loader".to_owned(), + operation: "create".to_owned(), + max_items: 10, + committed_items: 4, + input_digests: Vec::new(), + activation_revision: "package-1".to_owned(), + opened_at, + expires_at: opened_at + chrono::Duration::days(7), + status: ImportAuthorityStatus::Open, + closed_at: None, + }; + let mut json_out = Vec::new(); + let mut stderr = Vec::new(); + assert_eq!( + write_import_authority_success( + "import-authority open", + std::slice::from_ref(&authority), + OutputFormat::Json, + &mut json_out, + &mut stderr, + ), + ExitCode::SUCCESS + ); + let report: Value = serde_json::from_slice(&json_out).expect("report is JSON"); + assert_eq!(report["command"], "import-authority open"); + assert_eq!(report["authority"]["status"], "open"); + assert_eq!(report["authority"]["committedItems"], 4); + assert_eq!(report["authority"]["maxItems"], 10); + + let mut listed = Vec::new(); + assert_eq!( + write_import_authority_success( + "import-authority list", + &[authority], + OutputFormat::Human, + &mut listed, + &mut stderr, + ), + ExitCode::SUCCESS + ); + let listed = String::from_utf8(listed).expect("report is UTF-8"); + assert!(listed.contains("Listed the newest import authorities. 1 authority.")); + assert!(listed.contains("4 of 10")); + assert!(listed.contains("none (any input)")); + assert!(stderr.is_empty()); + } + #[test] fn public_command_surface_is_explicit() { let command = command(); @@ -12652,7 +13708,10 @@ mod tests { "data", "webhook", "request-retention", + "review-recovery", "evidence-retention", + "import-authority", + "instance-claim", "field-encryption" ] ); @@ -13265,7 +14324,7 @@ accessProfiles: #[test] fn rebaseline_history_diagnostics_point_at_the_retained_history() { - use registry_breg::history_rebaseline::{HistoryRebaselineError, MAX_REBASELINE_LIVE_ROWS}; + use registry_breg::history_rebaseline::HistoryRebaselineError; for (error, code) in [ ( @@ -13276,10 +14335,6 @@ accessProfiles: HistoryRebaselineError::LiveHistoryMismatch, "history.rebaseline.live_rows.unverified", ), - ( - HistoryRebaselineError::LiveRowBudgetExceeded, - "history.rebaseline.live_rows.budget_exceeded", - ), ] { let report = history_rebaseline_lifecycle_failure( HistoryRebaselineLifecycleError::Rebaseline(error), @@ -13294,20 +14349,6 @@ accessProfiles: ); } - let budget = - history_rebaseline_lifecycle_failure(HistoryRebaselineLifecycleError::Rebaseline( - HistoryRebaselineError::LiveRowBudgetExceeded, - )); - assert_eq!( - budget.diagnostics[0].message, - format!( - "history rebaseline verifies at most {MAX_REBASELINE_LIVE_ROWS} live rows in one \ - transaction and this registry holds more, so retrying cannot restore snapshot \ - coverage" - ), - "the budget diagnostic states the limit it enforces and what retrying cannot do" - ); - let mismatch = history_rebaseline_lifecycle_failure(HistoryRebaselineLifecycleError::Rebaseline( HistoryRebaselineError::LiveHistoryMismatch, @@ -13851,6 +14892,41 @@ fn apply_refuses_an_already_active_package_the_database_does_not_run() { } } +#[cfg(test)] +#[test] +fn apply_reports_a_refused_statement_with_its_sqlstate_and_objects() { + let report = apply_lifecycle_failure(ApplyLifecycleError::Apply( + registry_breg::migration::MigrationError::StatementFailed( + registry_breg::postgres::PostgresFailure { + sqlstate: Some("23502".to_owned()), + table: Some("asset_table".to_owned()), + column: Some("rank_column".to_owned()), + constraint: None, + }, + ), + )); + let diagnostic = &report.diagnostics[0]; + assert_eq!(diagnostic.code, "apply.migration.statement_failed"); + assert_eq!(diagnostic.path, "database"); + assert_eq!(diagnostic.artifact, DiagnosticArtifact::DatabaseMigration); + assert_eq!( + diagnostic.suggested_action, + SuggestedAction::ReconcileFailedMigration + ); + for fragment in [ + "SQLSTATE 23502 (integrity constraint violation), table asset_table, column rank_column", + "pinned in maintenance", + "retry the same target", + "migration reconcile", + ] { + assert!( + diagnostic.message.contains(fragment), + "{fragment}: {}", + diagnostic.message + ); + } +} + #[cfg(test)] #[test] fn apply_reports_an_unavailable_database_before_maintenance_as_retryable() { diff --git a/crates/registry-bregctl/src/package_inspection.rs b/crates/registry-bregctl/src/package_inspection.rs index 8ca5084fc7..d9b4f90460 100644 --- a/crates/registry-bregctl/src/package_inspection.rs +++ b/crates/registry-bregctl/src/package_inspection.rs @@ -5,16 +5,20 @@ use std::path::Path; use registry_breg::package::{ - inspect_package_with_context, load_predecessor_package, IntegrityInspectedPackage, + inspect_package_with_context_and_verified_envelope, + load_predecessor_package_with_verified_envelope, + load_predecessor_rehearsal_baseline_with_verified_envelope, IntegrityInspectedPackage, PackageError, PackageInspectionContext, PredecessorPackageContext, VerifiedPredecessorPackage, }; -use registry_breg::runtime_config::{load_runtime_config, RuntimeConfigError}; +use registry_breg::runtime_config::{load_runtime_config, RuntimeConfig, RuntimeConfigError}; +use registry_breg::CompiledRegistry; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub(crate) enum RuntimePackageInspectionError { RuntimeConfigPath, RuntimeConfig(RuntimeConfigError), Package(PackageError), + SharedPackage(String), } /// Inspect exactly the package selected and bound by one strict runtime @@ -27,6 +31,9 @@ pub(crate) fn inspect_runtime_package( } let config = load_runtime_config(runtime_config) .map_err(RuntimePackageInspectionError::RuntimeConfig)?; + let shared = config + .verify_package_envelope() + .map_err(|error| RuntimePackageInspectionError::SharedPackage(error.to_string()))?; let context = PackageInspectionContext { environment: config.identity().environment(), instance_id: config.identity().instance_id(), @@ -39,7 +46,7 @@ pub(crate) fn inspect_runtime_package( expected_package_revision: config.package().active_revision(), expected_sequence: config.package().active_sequence(), }; - inspect_package_with_context(config.package().root(), &context) + inspect_package_with_context_and_verified_envelope(config.package().root(), &context, &shared) .map_err(RuntimePackageInspectionError::Package) } @@ -50,12 +57,49 @@ pub(crate) fn inspect_runtime_package( pub(crate) fn inspect_runtime_predecessor_package( runtime_config: &Path, ) -> Result { + let config = load_predecessor_runtime_config(runtime_config)?; + let shared = config + .verify_package_envelope() + .map_err(|error| RuntimePackageInspectionError::SharedPackage(error.to_string()))?; + load_predecessor_package_with_verified_envelope( + config.package().root(), + &predecessor_context(&config), + &shared, + ) + .map_err(RuntimePackageInspectionError::Package) +} + +/// Verify the same predecessor package as [`inspect_runtime_predecessor_package`] +/// and compile its signed sources with the current compiler, so a successor +/// can be rehearsed over the predecessor schema. +pub(crate) fn inspect_runtime_predecessor_rehearsal_baseline( + runtime_config: &Path, +) -> Result<(VerifiedPredecessorPackage, CompiledRegistry), RuntimePackageInspectionError> { + let config = load_predecessor_runtime_config(runtime_config)?; + let shared = config + .verify_package_envelope() + .map_err(|error| RuntimePackageInspectionError::SharedPackage(error.to_string()))?; + load_predecessor_rehearsal_baseline_with_verified_envelope( + config.package().root(), + &predecessor_context(&config), + &shared, + ) + .map_err(RuntimePackageInspectionError::Package) +} + +fn load_predecessor_runtime_config( + runtime_config: &Path, +) -> Result { if !runtime_config.is_absolute() { return Err(RuntimePackageInspectionError::RuntimeConfigPath); } let config = load_runtime_config(runtime_config) .map_err(RuntimePackageInspectionError::RuntimeConfig)?; - let context = PredecessorPackageContext { + Ok(config) +} + +fn predecessor_context(config: &RuntimeConfig) -> PredecessorPackageContext<'_> { + PredecessorPackageContext { environment: config.identity().environment(), instance_id: config.identity().instance_id(), database_id: config.identity().database_id(), @@ -65,7 +109,5 @@ pub(crate) fn inspect_runtime_predecessor_package( trust_anchor: config.package_trust_anchor(), expected_package_revision: config.package().active_revision(), expected_sequence: config.package().active_sequence(), - }; - load_predecessor_package(config.package().root(), &context) - .map_err(RuntimePackageInspectionError::Package) + } } diff --git a/crates/registry-bregctl/src/package_lifecycle.rs b/crates/registry-bregctl/src/package_lifecycle.rs index 19c423b6b0..50d9edd87f 100644 --- a/crates/registry-bregctl/src/package_lifecycle.rs +++ b/crates/registry-bregctl/src/package_lifecycle.rs @@ -40,6 +40,8 @@ pub(crate) struct PackageLifecycleOutcome { pub signature_threshold: u16, pub provided_signatures: usize, pub package_files: usize, + pub package_digest: Option, + pub revision: Option, } /// Canonical receipt bytes that have been rederived against one exact @@ -112,6 +114,7 @@ pub(crate) fn run( test_receipt: ValidatedTestReceipt, build_directory: &Path, signature_document: Option<&Path>, + revision: Option<&str>, ) -> Result { let signed_bytes = prepared.canonical_signed_bytes(); ensure_reviewer_evidence(build_directory, signed_bytes, &test_receipt.bytes)?; @@ -130,8 +133,12 @@ pub(crate) fn run( )); } - prepared - .publish_to_directory(&build_directory.join(PACKAGE_DIRECTORY), signatures.clone()) + let shared = prepared + .publish_to_directory_with_revision( + &build_directory.join(PACKAGE_DIRECTORY), + signatures.clone(), + revision, + ) .map_err(PackageLifecycleError::Package)?; Ok(PackageLifecycleOutcome { state: PackageLifecycleState::Published, @@ -140,7 +147,9 @@ pub(crate) fn run( signing_input_bytes: signed_bytes.len(), signature_threshold: threshold, provided_signatures: signatures.len(), - package_files: prepared.file_bytes().len() + 1, + package_files: shared.files().count() + 1, + package_digest: Some(shared.digest().to_owned()), + revision: shared.revision().map(str::to_owned), }) } @@ -291,6 +300,8 @@ fn outcome( signature_threshold: prepared.manifest().signature_policy.threshold, provided_signatures, package_files: prepared.file_bytes().len() + 1, + package_digest: None, + revision: None, } } diff --git a/crates/registry-bregctl/src/reconcile_lifecycle.rs b/crates/registry-bregctl/src/reconcile_lifecycle.rs index 80c304d70a..e2aba05ecc 100644 --- a/crates/registry-bregctl/src/reconcile_lifecycle.rs +++ b/crates/registry-bregctl/src/reconcile_lifecycle.rs @@ -81,7 +81,8 @@ pub(crate) fn run( // The active package carries the compiled registry whose expected catalog // an abandoned target has to leave behind, so it is verified in full under // the same startup binding the server itself requires. - let active = load_package(config.package().root(), &config.package_load_context()) + let active = config + .load_active_package() .map_err(ReconcileLifecycleError::ActivePackage)?; let current = active_identity(&active)?; let target = load_package( diff --git a/crates/registry-bregctl/src/review_recovery.rs b/crates/registry-bregctl/src/review_recovery.rs new file mode 100644 index 0000000000..c60da5fb75 --- /dev/null +++ b/crates/registry-bregctl/src/review_recovery.rs @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Operator recovery for a change-request review its authority will not answer. +//! +//! The CLI owns argument validation and rendering only. The recovery itself is +//! delegated to Base Registry Engine so package, catalog, lock, audit, and SQL +//! boundaries stay in the product runtime. + +use std::path::Path; + +use registry_breg::review_recovery::{ + ReviewRecovery, ReviewRecoveryError, ReviewRecoveryOperatorService, ReviewRecoveryScope, +}; +use serde::Serialize; +use uuid::Uuid; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ReviewRecoveryCliError { + Operator, + NotFound, + Ineligible { + reason: &'static str, + state: String, + code: Option, + }, + RecoveryUnaudited, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReviewRecoveryOperation { + Resubmit, + Close, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ReviewRecoveryOutcome { + #[serde(flatten)] + pub recovery: ReviewRecovery, +} + +pub(crate) fn recover( + operation: ReviewRecoveryOperation, + runtime_config: &Path, + request_entity: &str, + request_id: &str, + proposal_version: i64, +) -> Result { + if !runtime_config.is_absolute() || request_entity.is_empty() || proposal_version <= 0 { + return Err(ReviewRecoveryCliError::Operator); + } + let request_id = Uuid::parse_str(request_id).map_err(|_| ReviewRecoveryCliError::Operator)?; + let scope = ReviewRecoveryScope { + request_entity_id: request_entity, + request_id, + proposal_version, + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|_| ReviewRecoveryCliError::Operator)?; + let recovery = runtime.block_on(async { + let service = ReviewRecoveryOperatorService::from_runtime_config(runtime_config) + .await + .map_err(map_error)?; + match operation { + ReviewRecoveryOperation::Resubmit => service.resubmit(scope).await, + ReviewRecoveryOperation::Close => service.close(scope).await, + } + .map_err(map_error) + })?; + Ok(ReviewRecoveryOutcome { recovery }) +} + +fn map_error(error: ReviewRecoveryError) -> ReviewRecoveryCliError { + match error { + ReviewRecoveryError::Unavailable => ReviewRecoveryCliError::Operator, + ReviewRecoveryError::NotFound => ReviewRecoveryCliError::NotFound, + ReviewRecoveryError::Ineligible { + reason, + state, + code, + } => ReviewRecoveryCliError::Ineligible { + reason: reason.as_str(), + state, + code, + }, + ReviewRecoveryError::RecoveryUnaudited => ReviewRecoveryCliError::RecoveryUnaudited, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use registry_breg::review_recovery::ReviewRecoveryRefusal; + + #[test] + fn ineligible_refusal_keeps_its_reason_state_and_code() { + assert_eq!( + map_error(ReviewRecoveryError::Ineligible { + reason: ReviewRecoveryRefusal::RequestErased, + state: "failed".to_owned(), + code: Some("result-poll-attempts-exhausted".to_owned()), + }), + ReviewRecoveryCliError::Ineligible { + reason: "request-erased", + state: "failed".to_owned(), + code: Some("result-poll-attempts-exhausted".to_owned()), + } + ); + } + + #[test] + fn an_unaudited_recovery_stays_distinct_from_a_refused_one() { + assert_eq!( + map_error(ReviewRecoveryError::RecoveryUnaudited), + ReviewRecoveryCliError::RecoveryUnaudited + ); + } + + #[test] + fn a_relative_configuration_or_malformed_scope_is_refused_before_any_connection() { + for (config, entity, id, version) in [ + ( + "runtime.yaml", + "e", + "00000000-0000-0000-0000-000000000001", + 1, + ), + ( + "/runtime.yaml", + "", + "00000000-0000-0000-0000-000000000001", + 1, + ), + ("/runtime.yaml", "e", "not-a-uuid", 1), + ( + "/runtime.yaml", + "e", + "00000000-0000-0000-0000-000000000001", + 0, + ), + ] { + assert_eq!( + recover( + ReviewRecoveryOperation::Close, + Path::new(config), + entity, + id, + version + ), + Err(ReviewRecoveryCliError::Operator) + ); + } + } +} diff --git a/crates/registry-bregctl/src/test_lifecycle.rs b/crates/registry-bregctl/src/test_lifecycle.rs index fa733c6e5d..fe30b698cd 100644 --- a/crates/registry-bregctl/src/test_lifecycle.rs +++ b/crates/registry-bregctl/src/test_lifecycle.rs @@ -11,6 +11,7 @@ use registry_breg::fixtures::{ execute_schema_test, validate_fixture_journeys, FixtureError, SchemaTestCredentialBinding, SchemaTestCredentialBindings, SchemaTestRuntimeSetupError, }; +use registry_breg::postgres::{MigrationRehearsalError, SuccessorMigrationRehearsal}; use registry_breg::runtime_config::{load_runtime_config, RuntimeConfig, RuntimeConfigError}; use registry_breg::startup; use serde::Deserialize; @@ -61,6 +62,7 @@ pub(crate) enum TestLifecycleError { Candidate, CandidateBinding { path: &'static str }, ReviewFingerprint, + Rehearsal(Box), Journeys { message: String }, JourneySyntax { path: String, message: &'static str }, JourneyStep { path: String, message: String }, @@ -192,10 +194,24 @@ pub(crate) fn run( { return Err(TestLifecycleError::ReviewFingerprint); } + let rehearsal_baseline = request.candidate.rehearsal_baseline.clone(); let prepared = request .candidate .prepare(schema_fingerprint.clone()) .map_err(|_| TestLifecycleError::Candidate)?; + if let Some(baseline) = &rehearsal_baseline { + runtime + .block_on(startup::rehearse_successor_migration( + &config, + SuccessorMigrationRehearsal { + predecessor: &baseline.registry, + predecessor_schema_fingerprint: &baseline.schema_fingerprint, + candidate: &prepared, + }, + )) + .map_err(schema_preparation_error)? + .map_err(|error| TestLifecycleError::Rehearsal(Box::new(error)))?; + } let signing_input_sha256 = sha256(prepared.canonical_signed_bytes()); let package_revision = prepared.package_revision().to_owned(); let receipt = runtime.block_on(async { @@ -856,6 +872,76 @@ mod tests { assert!(!report.to_string().contains("recreate")); } + #[test] + fn a_refused_rehearsal_step_names_the_step_and_postgres_class_only() { + use registry_breg::postgres::PostgresFailure; + + let error = TestLifecycleError::Rehearsal(Box::new(MigrationRehearsalError::Step { + migration_id: "rank-backfill".into(), + step_id: "backfill-rank".into(), + failure: PostgresFailure { + sqlstate: Some("23502".into()), + table: Some("entity_record".into()), + column: Some("rank".into()), + constraint: None, + }, + })); + let report = serde_json::to_value(crate::test_lifecycle_failure(error)) + .expect("rehearsal failure report serializes"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], "migration.rehearsal.step_failed"); + assert_eq!( + diagnostic["path"], + "reviewedMigrations[rank-backfill].steps[backfill-rank]" + ); + let message = diagnostic["message"].as_str().unwrap(); + assert!(message.contains("SQLSTATE 23502"), "{message}"); + assert!( + message.contains("integrity constraint violation"), + "{message}" + ); + assert!(message.contains("table entity_record"), "{message}"); + assert!(message.contains("column rank"), "{message}"); + assert!(message.contains("apply would refuse"), "{message}"); + } + + #[test] + fn a_rehearsal_that_misses_the_candidate_schema_is_a_migration_refusal() { + let report = serde_json::to_value(crate::test_lifecycle_failure( + TestLifecycleError::Rehearsal(Box::new(MigrationRehearsalError::FinalSchemaMismatch)), + )) + .expect("rehearsal failure report serializes"); + assert_eq!( + report["diagnostics"][0]["code"], + "migration.rehearsal.schema_mismatch" + ); + assert_eq!(report["diagnostics"][0]["path"], "reviewedMigrations"); + } + + #[test] + fn a_step_the_history_journal_refuses_names_the_step_and_the_reason() { + let report = serde_json::to_value(crate::test_lifecycle_failure( + TestLifecycleError::Rehearsal(Box::new(MigrationRehearsalError::HistoryStep { + migration_id: "rank-backfill".into(), + step_id: "backfill-rank".into(), + reason: "history migration supports only direct reviewed UPDATE statements".into(), + })), + )) + .expect("rehearsal failure report serializes"); + let diagnostic = &report["diagnostics"][0]; + assert_eq!( + diagnostic["code"], + "migration.rehearsal.history_step_refused" + ); + assert_eq!( + diagnostic["path"], + "reviewedMigrations[rank-backfill].steps[backfill-rank]" + ); + let message = diagnostic["message"].as_str().unwrap(); + assert!(message.contains("cannot be journaled"), "{message}"); + assert!(message.contains("apply would refuse"), "{message}"); + } + #[test] fn a_refused_logical_reference_reports_which_reference_and_why() { use registry_breg::fixtures::LogicalReferenceRefusal; diff --git a/crates/registry-bregctl/tests/cli.rs b/crates/registry-bregctl/tests/cli.rs index df95075435..2ceffd03f8 100644 --- a/crates/registry-bregctl/tests/cli.rs +++ b/crates/registry-bregctl/tests/cli.rs @@ -598,6 +598,9 @@ const PACKAGE_INSTANCE: &str = "verify-instance"; const PACKAGE_DATABASE: &str = "verify-database"; const PACKAGE_SOURCE_REVISION: &str = "verify-compiler-source"; const PACKAGE_VALUE_CANARY: &str = "verify-path-trust-secret-sql-canary"; +const VERIFY_RUNTIME_DATABASE_SECRET_CANARY: &str = "VERIFY_RUNTIME_DATABASE_SECRET_IS_NOT_OPENED"; +const VERIFY_MIGRATION_DATABASE_SECRET_CANARY: &str = + "VERIFY_MIGRATION_DATABASE_SECRET_IS_NOT_OPENED"; const SCHEMA_TEST_AUTHORED_SOURCE_CEILING_BYTES: usize = 1024 * 1024; const PACKAGE_FIXTURE_JOURNEYS: &[u8] = br#"apiVersion: registry.registrystack.org/breg-journeys/v1 journeys: @@ -5226,7 +5229,8 @@ fn apply_verifies_package_intent_before_database_authority_and_stays_value_free( fixture.package_revision.as_str(), other.package_revision.as_str(), PACKAGE_VALUE_CANARY, - "VERIFY_DATABASE_SECRET_IS_NOT_OPENED", + VERIFY_RUNTIME_DATABASE_SECRET_CANARY, + VERIFY_MIGRATION_DATABASE_SECRET_CANARY, ] { assert!(!rendered.contains(forbidden)); } @@ -5248,8 +5252,9 @@ fn apply_verifies_package_intent_before_database_authority_and_stays_value_free( json_stdout(&already_active)["diagnostics"][0]["code"], "apply.database_configuration.refused" ); - assert!(!String::from_utf8_lossy(&already_active.stdout) - .contains("VERIFY_DATABASE_SECRET_IS_NOT_OPENED")); + let rendered = String::from_utf8_lossy(&already_active.stdout); + assert!(!rendered.contains(VERIFY_RUNTIME_DATABASE_SECRET_CANARY)); + assert!(!rendered.contains(VERIFY_MIGRATION_DATABASE_SECRET_CANARY)); let database_refusal = bregctl(&[ "--format", @@ -5270,8 +5275,34 @@ fn apply_verifies_package_intent_before_database_authority_and_stays_value_free( json_stdout(&database_refusal)["diagnostics"][0]["code"], "apply.database_configuration.refused" ); - assert!(!String::from_utf8_lossy(&database_refusal.stdout) - .contains("VERIFY_DATABASE_SECRET_IS_NOT_OPENED")); + let rendered = String::from_utf8_lossy(&database_refusal.stdout); + assert!(!rendered.contains(VERIFY_RUNTIME_DATABASE_SECRET_CANARY)); + assert!(!rendered.contains(VERIFY_MIGRATION_DATABASE_SECRET_CANARY)); +} + +#[test] +fn apply_refuses_a_stale_shared_envelope_before_database_authority() { + let fixture = RuntimePackageFixture::production("127.0.0.1:1".parse().unwrap()); + fs::write(fixture.package.join("SHA256SUMS"), b"stale\n").expect("shared sum file is replaced"); + + let refused = bregctl(&[ + "--format", + "json", + "apply", + "--runtime-config", + path(&fixture.runtime_config), + "--package", + path(&fixture.package), + "--initial", + ]); + assert_eq!(refused.status.code(), Some(1), "{refused:?}"); + assert_eq!( + json_stdout(&refused)["diagnostics"][0]["code"], + "apply.package.refused" + ); + let rendered = String::from_utf8(refused.stdout).expect("diagnostic is UTF-8"); + assert!(!rendered.contains(VERIFY_RUNTIME_DATABASE_SECRET_CANARY)); + assert!(!rendered.contains(VERIFY_MIGRATION_DATABASE_SECRET_CANARY)); } #[test] @@ -5428,7 +5459,8 @@ fn migration_reconcile_verifies_intent_before_database_authority_and_stays_value path(&fixture.package), path(&fixture.anchor), PACKAGE_VALUE_CANARY, - "VERIFY_DATABASE_SECRET_IS_NOT_OPENED", + VERIFY_RUNTIME_DATABASE_SECRET_CANARY, + VERIFY_MIGRATION_DATABASE_SECRET_CANARY, ] { assert!(!rendered.contains(forbidden)); } @@ -5489,7 +5521,8 @@ fn verify_is_runtime_bound_deterministic_and_listener_free() { path(&fixture.package), path(&fixture.anchor), "oidc-is-not-opened.invalid", - "VERIFY_DATABASE_SECRET_IS_NOT_OPENED", + VERIFY_RUNTIME_DATABASE_SECRET_CANARY, + VERIFY_MIGRATION_DATABASE_SECRET_CANARY, ] { assert!(!rendered.contains(forbidden)); } @@ -5944,11 +5977,8 @@ fn data_validate_uses_a_closed_package_plan_and_value_free_usage() { let (directory, package) = data_package_fixture(); let input = directory.path().join("input.jsonl"); - fs::write( - &input, - r#"{"operation":"create","data":{"code":"AA"}}"#.to_owned() + "\n", - ) - .expect("data input writes"); + let input_bytes = r#"{"operation":"create","data":{"code":"AA"}}"#.to_owned() + "\n"; + fs::write(&input, &input_bytes).expect("data input writes"); let output = bregctl(&[ "--format", @@ -5978,6 +6008,11 @@ fn data_validate_uses_a_closed_package_plan_and_value_free_usage() { assert_eq!(report["operation"], "create"); assert_eq!(report["itemCount"], 1); assert_eq!(report["chunkCount"], 1); + // The digest an operator pins with `import-authority open --input-sha256`. + assert_eq!( + report["inputDigest"], + hex(Sha256::digest(input_bytes.as_bytes()).as_slice()) + ); let refused = bregctl(&[ "--format", @@ -6210,8 +6245,8 @@ secretProviders: file: root: {secret_root} database: - runtimeUrlRef: secret:env/VERIFY_RUNTIME_DATABASE_SECRET_IS_NOT_OPENED - migrationUrlRef: secret:env/VERIFY_MIGRATION_DATABASE_SECRET_IS_NOT_OPENED + runtimeUrlRef: secret:env/{VERIFY_RUNTIME_DATABASE_SECRET_CANARY} + migrationUrlRef: secret:env/{VERIFY_MIGRATION_DATABASE_SECRET_CANARY} pool: maxSize: 1 waitTimeoutMilliseconds: 1000 diff --git a/crates/registry-bregctl/tests/diff.rs b/crates/registry-bregctl/tests/diff.rs index db5bba010c..31a6f32a98 100644 --- a/crates/registry-bregctl/tests/diff.rs +++ b/crates/registry-bregctl/tests/diff.rs @@ -165,6 +165,67 @@ fn diff_inventory_is_deterministic_and_classification_direction_is_exact() { .any(|finding| finding["code"] == "diff.classification.unsupported")); } +#[test] +fn a_removed_field_is_reported_as_retained_in_history_not_erased() { + let directory = TestDirectory::create(); + let baseline = publish_package(&directory.path, "baseline", "local", "internal", None); + let module = String::from_utf8(module_bytes("internal")) + .expect("module is UTF-8") + .replacen(r#""id":"code""#, r#""id":"label""#, 1) + .replacen( + r#""readableFields":["code"]"#, + r#""readableFields":["label"]"#, + 1, + ); + let removal = + write_project_with_module(&directory.path, "removal", "local", module.into_bytes()); + + let output = run(&[ + "--format", + "json", + "diff", + path(&removal), + "--package", + path(&baseline.package), + ]); + assert!(output.status.success(), "{output:?}"); + let report = json_stdout(&output); + assert!(report["changes"] + .as_array() + .expect("changes array") + .iter() + .any(|change| change["change"]["code"] == "field_removed")); + let retained: Vec<&Value> = report["findings"] + .as_array() + .expect("findings array") + .iter() + .filter(|finding| finding["code"] == "diff.history.removed_values_retained") + .collect(); + assert_eq!(retained.len(), 1, "{report}"); + assert_eq!(retained[0]["path"], "changes.record.code"); + assert_eq!(retained[0]["artifact"], "compiled_diff"); + assert_eq!(retained[0]["suggestedAction"], "review_compiled_diff"); + let message = retained[0]["message"].as_str().expect("message is text"); + assert!(message.contains("not erasure"), "{message}"); + assert!(message.contains("bregctl history erase"), "{message}"); + + let unchanged = write_project(&directory.path, "unchanged", "local", "internal"); + let quiet = run(&[ + "--format", + "json", + "diff", + path(&unchanged), + "--package", + path(&baseline.package), + ]); + assert!(quiet.status.success(), "{quiet:?}"); + assert!(!json_stdout(&quiet)["findings"] + .as_array() + .expect("findings array") + .iter() + .any(|finding| finding["code"] == "diff.history.removed_values_retained")); +} + #[test] fn package_closure_and_path_disclosure_threats_are_enforced_by_value_free_negatives() { let directory = TestDirectory::create(); @@ -415,7 +476,9 @@ fn diff_help_and_selector_usage_preserve_the_closed_command_inventory_and_exit_c let malformed_runtime = directory.path.join(format!("{VALUE_CANARY}.yaml")); fs::write( &malformed_runtime, - format!("unexpectedSetting: {VALUE_CANARY}\n"), + format!( + "apiVersion: registry.registrystack.org/breg-runtime/v1alpha1\nkind: BRegRuntimeConfig\nunexpectedSetting: {VALUE_CANARY}\n" + ), ) .expect("malformed runtime configuration is written"); let refused_runtime = run(&[ @@ -526,8 +589,16 @@ fn publish_package( } fn write_project(parent: &Path, name: &str, environment: &str, classification: &str) -> PathBuf { + write_project_with_module(parent, name, environment, module_bytes(classification)) +} + +fn write_project_with_module( + parent: &Path, + name: &str, + environment: &str, + module: Vec, +) -> PathBuf { let root = parent.join(name); - let module = module_bytes(classification); let parsed = parse_module_json(&module).expect("candidate module parses"); fs::create_dir_all(root.join("modules/core")).expect("candidate directories create"); fs::write( diff --git a/crates/registry-bregctl/tests/doctor.rs b/crates/registry-bregctl/tests/doctor.rs index f7e1bc8c45..17087b0bde 100644 --- a/crates/registry-bregctl/tests/doctor.rs +++ b/crates/registry-bregctl/tests/doctor.rs @@ -111,7 +111,9 @@ fn startup_value_disclosure_and_listener_activation_threats_are_enforced_by_prep drop(probe); fs::write( &runtime_config, - format!("listener:\n bind: {address}\nunexpected: {CONFIG_VALUE_CANARY}\n"), + format!( + "apiVersion: registry.registrystack.org/breg-runtime/v1alpha1\nkind: BRegRuntimeConfig\nlistener:\n bind: {address}\nunexpected: {CONFIG_VALUE_CANARY}\n" + ), ) .expect("invalid runtime configuration is written"); diff --git a/crates/registry-bregctl/tests/import_authority.rs b/crates/registry-bregctl/tests/import_authority.rs new file mode 100644 index 0000000000..563a0de7fa --- /dev/null +++ b/crates/registry-bregctl/tests/import_authority.rs @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Offline contract tests for `bregctl import-authority`. They prove the +//! refusals an operator meets before any database connection is opened, and +//! that no refusal repeats an operator reference, a reason, or a path. + +use serde_json::Value; + +const MISSING_RUNTIME: &str = "/registry/import-authority-runtime-that-does-not-exist.yaml"; +const OPERATOR_CANARY: &str = "import-authority-operator-canary"; +const REASON_CANARY: &str = "import-authority-reason-canary"; +const PATH_CANARY: &str = "import-authority-path-canary"; +const DIGEST: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const AUTHORITY_ID: &str = "00000000-0000-4000-8000-000000000001"; + +fn run(arguments: &[&str]) -> (u8, String, String) { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let status = registry_bregctl::run_from(arguments.iter().copied(), &mut stdout, &mut stderr); + ( + if status == std::process::ExitCode::SUCCESS { + 0 + } else { + 1 + }, + String::from_utf8(stdout).expect("stdout is UTF-8"), + String::from_utf8(stderr).expect("stderr is UTF-8"), + ) +} + +fn open(runtime_config: &str, extra: &[&str]) -> Vec { + let mut arguments = vec![ + "bregctl", + "--format", + "json", + "import-authority", + "open", + "--runtime-config", + runtime_config, + "--entity", + "widget", + "--profile", + "loader", + "--max-items", + "10", + "--operator-reference", + OPERATOR_CANARY, + "--reason", + REASON_CANARY, + ]; + arguments.extend_from_slice(extra); + arguments.into_iter().map(str::to_owned).collect() +} + +fn refusal(arguments: &[String], code: &str, path: &str) -> Value { + let arguments = arguments.iter().map(String::as_str).collect::>(); + let (status, stdout, stderr) = run(&arguments); + assert_eq!(status, 1, "{stdout}"); + assert!(stderr.is_empty(), "{stderr}"); + for canary in [OPERATOR_CANARY, REASON_CANARY, PATH_CANARY, MISSING_RUNTIME] { + assert!(!stdout.contains(canary), "{canary} leaked: {stdout}"); + } + let report: Value = serde_json::from_str(&stdout).expect("failure is JSON"); + assert_eq!(report["ok"], false); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], code, "{stdout}"); + assert_eq!(diagnostic["path"], path, "{stdout}"); + assert_eq!(diagnostic["artifact"], "import_authority"); + report +} + +#[test] +fn a_relative_runtime_configuration_is_refused_for_every_subcommand() { + let cases = [ + open(PATH_CANARY, &[]), + [ + "bregctl", + "--format", + "json", + "import-authority", + "close", + "--runtime-config", + PATH_CANARY, + "--authority-id", + AUTHORITY_ID, + "--operator-reference", + OPERATOR_CANARY, + "--reason", + REASON_CANARY, + ] + .map(str::to_owned) + .to_vec(), + [ + "bregctl", + "--format", + "json", + "import-authority", + "close-expired", + "--runtime-config", + PATH_CANARY, + ] + .map(str::to_owned) + .to_vec(), + [ + "bregctl", + "--format", + "json", + "import-authority", + "list", + "--runtime-config", + PATH_CANARY, + ] + .map(str::to_owned) + .to_vec(), + ]; + for arguments in cases { + refusal( + &arguments, + "import_authority.runtime_config.invalid", + "runtimeConfig", + ); + } +} + +#[test] +fn an_expiry_outside_one_minute_to_thirty_days_is_refused_before_any_dependency() { + for expires_in in ["31d", "0d", "0m", "721h", "7", "7x", "d", "1.5d", "+1d", ""] { + refusal( + &open(MISSING_RUNTIME, &["--expires-in", expires_in]), + "import_authority.expires_in.invalid", + "expiresIn", + ); + } +} + +#[test] +fn the_window_bounds_themselves_are_accepted() { + // Each bound passes the offline checks and reaches the runtime + // configuration, which does not exist, so the refusal names it instead. + for expires_in in ["30d", "720h", "1m", "7d"] { + refusal( + &open(MISSING_RUNTIME, &["--expires-in", expires_in]), + "import_authority.unavailable", + "importAuthority", + ); + } +} + +#[test] +fn a_malformed_or_repeated_pinned_digest_is_refused_before_any_dependency() { + let upper = DIGEST.to_uppercase(); + let short = &DIGEST[..63]; + for digests in [ + vec!["--input-sha256", upper.as_str()], + vec!["--input-sha256", short], + vec!["--input-sha256", DIGEST, "--input-sha256", DIGEST], + ] { + refusal( + &open(MISSING_RUNTIME, &digests), + "import_authority.request.invalid", + "importAuthority", + ); + } +} + +#[test] +fn an_empty_reason_or_a_volume_below_one_is_refused_before_any_dependency() { + let mut empty_reason = open(MISSING_RUNTIME, &[]); + let at = empty_reason + .iter() + .position(|argument| argument == REASON_CANARY) + .expect("reason argument"); + empty_reason[at] = String::new(); + refusal( + &empty_reason, + "import_authority.request.invalid", + "importAuthority", + ); + + let mut zero = open(MISSING_RUNTIME, &[]); + let at = zero + .iter() + .position(|argument| argument == "10") + .expect("max-items argument"); + zero[at] = "0".to_owned(); + refusal(&zero, "import_authority.request.invalid", "importAuthority"); +} + +#[test] +fn close_refuses_an_authority_identifier_that_is_not_a_uuid() { + let arguments = [ + "bregctl", + "--format", + "json", + "import-authority", + "close", + "--runtime-config", + MISSING_RUNTIME, + "--authority-id", + PATH_CANARY, + "--operator-reference", + OPERATOR_CANARY, + "--reason", + REASON_CANARY, + ] + .map(str::to_owned); + refusal( + &arguments, + "import_authority.authority_id.invalid", + "authorityId", + ); +} + +#[test] +fn an_unavailable_runtime_configuration_is_one_value_free_refusal() { + refusal( + &open(MISSING_RUNTIME, &["--input-sha256", DIGEST]), + "import_authority.unavailable", + "importAuthority", + ); + for subcommand in ["close-expired", "list"] { + let arguments = [ + "bregctl", + "--format", + "json", + "import-authority", + subcommand, + "--runtime-config", + MISSING_RUNTIME, + ] + .map(str::to_owned); + refusal( + &arguments, + "import_authority.unavailable", + "importAuthority", + ); + } +} + +#[test] +fn import_authority_help_describes_the_operator_contract() { + let (status, stdout, stderr) = run(&["bregctl", "import-authority", "--help"]); + assert_eq!(status, 0); + assert!(stderr.is_empty()); + for subcommand in ["open", "close", "close-expired", "list"] { + assert!( + stdout + .lines() + .any(|line| line.trim_start().starts_with(subcommand)), + "{subcommand}: {stdout}" + ); + } + + let (status, stdout, stderr) = run(&["bregctl", "import-authority", "open", "--help"]); + assert_eq!(status, 0); + assert!(stderr.is_empty()); + for flag in [ + "--runtime-config ", + "--entity ", + "--profile ", + "--max-items ", + "--expires-in ", + "[default: 7d]", + "--input-sha256 ", + "--operator-reference ", + "--reason ", + ] { + assert!(stdout.contains(flag), "{flag}: {stdout}"); + } + + let (status, stdout, stderr) = run(&["bregctl", "import-authority", "close", "--help"]); + assert_eq!(status, 0); + assert!(stderr.is_empty()); + for flag in [ + "--runtime-config ", + "--authority-id ", + "--operator-reference ", + "--reason ", + ] { + assert!(stdout.contains(flag), "{flag}: {stdout}"); + } +} diff --git a/crates/registry-bregctl/tests/instance_claim.rs b/crates/registry-bregctl/tests/instance_claim.rs new file mode 100644 index 0000000000..6ed1e2add6 --- /dev/null +++ b/crates/registry-bregctl/tests/instance_claim.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Offline contract tests for `bregctl instance-claim`. They prove the +//! refusals an operator meets before any database connection is opened. + +use serde_json::Value; + +const MISSING_RUNTIME: &str = "/registry/instance-claim-runtime-that-does-not-exist.yaml"; + +fn run(arguments: &[&str]) -> (u8, String, String) { + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let status = registry_bregctl::run_from(arguments.iter().copied(), &mut stdout, &mut stderr); + ( + if status == std::process::ExitCode::SUCCESS { + 0 + } else { + 1 + }, + String::from_utf8(stdout).expect("stdout is UTF-8"), + String::from_utf8(stderr).expect("stderr is UTF-8"), + ) +} + +fn refusal(arguments: &[&str], code: &str, path: &str, artifact: &str) -> Value { + let (status, stdout, stderr) = run(arguments); + assert_eq!(status, 1, "{stdout}"); + assert!(stderr.is_empty(), "{stderr}"); + let report: Value = serde_json::from_str(&stdout).expect("failure is JSON"); + assert_eq!(report["ok"], false); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], code, "{stdout}"); + assert_eq!(diagnostic["path"], path, "{stdout}"); + assert_eq!(diagnostic["artifact"], artifact, "{stdout}"); + report +} + +#[test] +fn adopting_without_acknowledging_the_retired_original_is_refused_before_any_connection() { + let report = refusal( + &[ + "bregctl", + "--format", + "json", + "instance-claim", + "adopt", + "--runtime-config", + MISSING_RUNTIME, + ], + "instance_claim.acknowledgement.required", + "acknowledgeOriginalRetired", + "command_arguments", + ); + assert_eq!(report["command"], "instance-claim adopt"); +} + +#[test] +fn a_relative_runtime_configuration_is_refused_for_every_subcommand() { + for arguments in [ + vec![ + "bregctl", + "--format", + "json", + "instance-claim", + "status", + "--runtime-config", + "runtime.yaml", + ], + vec![ + "bregctl", + "--format", + "json", + "instance-claim", + "adopt", + "--runtime-config", + "runtime.yaml", + "--acknowledge-original-retired", + ], + ] { + refusal( + &arguments, + "instance_claim.runtime_config.invalid", + "runtimeConfig", + "command_arguments", + ); + } +} + +#[test] +fn an_unreadable_runtime_configuration_reports_the_claim_unavailable() { + for arguments in [ + vec![ + "bregctl", + "--format", + "json", + "instance-claim", + "status", + "--runtime-config", + MISSING_RUNTIME, + ], + vec![ + "bregctl", + "--format", + "json", + "instance-claim", + "adopt", + "--runtime-config", + MISSING_RUNTIME, + "--acknowledge-original-retired", + ], + ] { + let report = refusal( + &arguments, + "instance_claim.unavailable", + "instanceClaim", + "instance_claim", + ); + let text = report.to_string(); + assert!(!text.contains(MISSING_RUNTIME), "path leaked: {text}"); + } +} diff --git a/crates/registry-bregctl/tests/review_recovery.rs b/crates/registry-bregctl/tests/review_recovery.rs new file mode 100644 index 0000000000..97c58c9299 --- /dev/null +++ b/crates/registry-bregctl/tests/review_recovery.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::ffi::OsStr; +use std::path::Path; + +use serde_json::Value; + +const PATH_VALUE_CANARY: &str = "review-recovery-runtime-path-value-canary"; +const REQUEST_VALUE_CANARY: &str = "review-recovery-request-value-canary"; + +fn run(arguments: I) -> (u8, String, String) +where + I: IntoIterator, + T: Into + Clone, +{ + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let status = registry_bregctl::run_from(arguments, &mut stdout, &mut stderr); + ( + if status == std::process::ExitCode::SUCCESS { + 0 + } else { + 1 + }, + String::from_utf8(stdout).expect("stdout is UTF-8"), + String::from_utf8(stderr).expect("stderr is UTF-8"), + ) +} + +#[test] +fn review_recovery_commands_share_one_value_free_refusal() { + let relative = Path::new(PATH_VALUE_CANARY); + for operation in ["resubmit", "close"] { + let (status, stdout, stderr) = run([ + OsStr::new("bregctl"), + OsStr::new("--format"), + OsStr::new("json"), + OsStr::new("review-recovery"), + OsStr::new(operation), + OsStr::new("--runtime-config"), + relative.as_os_str(), + OsStr::new("--request-entity"), + OsStr::new(REQUEST_VALUE_CANARY), + OsStr::new("--request-id"), + OsStr::new("00000000-0000-4000-8000-000000000001"), + OsStr::new("--proposal-version"), + OsStr::new("1"), + ]); + assert_eq!(status, 1); + assert!(stderr.is_empty()); + assert!(!stdout.contains(PATH_VALUE_CANARY)); + assert!(!stdout.contains(REQUEST_VALUE_CANARY)); + let report: Value = serde_json::from_str(&stdout).expect("failure is JSON"); + assert_eq!(report["command"], format!("review-recovery {operation}")); + assert_eq!( + report["diagnostics"][0]["code"], + "review_recovery.operation.refused" + ); + assert_eq!(report["diagnostics"][0]["path"], "reviewRecovery"); + assert_eq!( + report["diagnostics"][0]["artifact"], + "review_recovery_operation" + ); + assert_eq!( + report["diagnostics"][0]["suggestedAction"], + "verify_review_recovery_operation" + ); + } +} + +#[test] +fn review_recovery_help_describes_the_exact_operator_contract() { + for operation in ["resubmit", "close"] { + let (status, stdout, stderr) = run(["bregctl", "review-recovery", operation, "--help"]); + + assert_eq!(status, 0); + assert!(stderr.is_empty()); + assert!(stdout.contains("--runtime-config ")); + assert!(stdout.contains("--request-entity ")); + assert!(stdout.contains("--request-id ")); + assert!(stdout.contains("--proposal-version ")); + } +} diff --git a/crates/registry-casework-breg/Cargo.toml b/crates/registry-casework-breg/Cargo.toml index f84eab8da4..fe6b968559 100644 --- a/crates/registry-casework-breg/Cargo.toml +++ b/crates/registry-casework-breg/Cargo.toml @@ -19,6 +19,7 @@ workspace = true async-trait.workspace = true registry-casework-core.workspace = true registry-breg-client.workspace = true +registry-platform-buildinfo.workspace = true registry-platform-config.workspace = true registry-platform-crypto.workspace = true registry-platform-hooks.workspace = true diff --git a/crates/registry-casework-breg/src/config.rs b/crates/registry-casework-breg/src/config.rs index f5b69af17e..ce1058f25e 100644 --- a/crates/registry-casework-breg/src/config.rs +++ b/crates/registry-casework-breg/src/config.rs @@ -112,6 +112,17 @@ impl BregBinding { ) -> Result { build_adapter(self, source, project_root, secrets) } + + /// Build from source-description bytes captured by the runtime's verified + /// package load. + pub fn build_adapter_from_description( + &self, + source: &SourcePolicy, + description: &[u8], + secrets: &SecretResolver, + ) -> Result { + build_adapter_from_description(self, source, description, secrets) + } } /// Validate one authored description and construct its pooled BReg client. @@ -120,6 +131,17 @@ pub fn build_adapter( source: &SourcePolicy, project_root: &Path, secrets: &SecretResolver, +) -> Result { + let description = read_description(project_root, &source.description)?; + build_adapter_from_description(binding, source, &description, secrets) +} + +/// Validate captured source-description bytes and construct its pooled BReg client. +pub fn build_adapter_from_description( + binding: &BregBinding, + source: &SourcePolicy, + description: &[u8], + secrets: &SecretResolver, ) -> Result { validate_binding(binding)?; if source.adapter != "breg" @@ -128,8 +150,10 @@ pub fn build_adapter( { return Err(SourceAdapterError::Invalid); } - let description_bytes = read_description(project_root, &source.description)?; - let (requests, expected_registry_revision) = validate_description(source, &description_bytes)?; + if description.is_empty() || description.len() > MAXIMUM_DESCRIPTION_BYTES { + return Err(SourceAdapterError::Invalid); + } + let (requests, expected_registry_revision) = validate_description(source, description)?; let client_id_secret = resolve_secret(secrets, &binding.client_id_ref)?; let client_id = std::str::from_utf8(client_id_secret.expose_secret()) @@ -147,7 +171,7 @@ pub fn build_adapter( .as_deref() .map(|reference| resolve_secret(secrets, reference)) .transpose()?; - let generation = binding_generation(binding, source, &description_bytes)?; + let generation = binding_generation(binding, source, description)?; let request_timeout = Duration::from_millis(binding.request_timeout_milliseconds); let connect_timeout = Duration::from_millis(binding.connect_timeout_milliseconds); diff --git a/crates/registry-casework-breg/src/lib.rs b/crates/registry-casework-breg/src/lib.rs index e6a4cc2b36..bd1a5e60c0 100644 --- a/crates/registry-casework-breg/src/lib.rs +++ b/crates/registry-casework-breg/src/lib.rs @@ -75,11 +75,86 @@ pub struct BregAdapter { /// The source reader failure cause last logged, so a persistent failure is /// reported once per change instead of once per subject read. reader_failure: Mutex>, + /// The caller wire-contract failure last logged, kept apart so a caller + /// read neither reports nor clears a source reader failure. + caller_failure: Mutex>, + /// The peer engine version last logged, so it is reported when first read + /// and whenever it changes. + peer_version: Mutex>, + /// The release mismatch that refused the latest contract read, if any. + peer_mismatch: Mutex>, +} + +/// Whether a BReg engine reporting `peer` runs the release `own` names. +/// +/// A build without the release marker reports its package version followed +/// by `-dev`, so one trailing `-dev` is set aside on each side and a release +/// build matches a development build of the same version. Every other part +/// of the version, a prerelease tag included, must match exactly: `0.34.0` +/// matches `0.34.0-dev`, and `0.34.0-rc.1` matches only `0.34.0-rc.1` and +/// `0.34.0-rc.1-dev`. +fn same_release(peer: &str, own: &str) -> bool { + fn version(text: &str) -> &str { + text.strip_suffix("-dev").unwrap_or(text) + } + version(peer) == version(own) +} + +/// A BReg source whose engine release is not this Casework's release. +/// +/// Casework and BReg run in lock-step: the adapter reads the engine version +/// BReg reports beside its registry contract and refuses every other release +/// by name, because a contract from another release may omit a member such as +/// a change request's effects, and an absent member must never read as an +/// empty one. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PeerVersionMismatch { + source_id: String, + peer_version: Option, +} + +impl PeerVersionMismatch { + /// The Casework source whose BReg peer was refused. + #[must_use] + pub fn source_id(&self) -> &str { + &self.source_id + } + + /// The engine version the peer reported, or `None` when it reported none. + #[must_use] + pub fn peer_version(&self) -> Option<&str> { + self.peer_version.as_deref() + } + + /// The release this Casework adapter was built from. + #[must_use] + pub fn casework_version(&self) -> &'static str { + registry_platform_buildinfo::DISPLAY_VERSION + } +} + +impl std::fmt::Display for PeerVersionMismatch { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let own = self.casework_version(); + match &self.peer_version { + Some(peer) => write!( + formatter, + "BReg source {} runs engine version {peer} and this Casework runs {own}. Casework and BReg run in lock-step, so upgrade both to the same release", + self.source_id + ), + None => write!( + formatter, + "BReg source {} does not report its engine version and this Casework runs {own}. The engine predates this release, or a proxy removes its Registry-Engine-Version header. Casework and BReg run in lock-step, so upgrade both to the same release", + self.source_id + ), + } + } } /// The credential behind a BReg read. A source reader failure stops Casework /// learning about source changes, so its cause is logged. A caller's failure -/// is that caller's own refusal and is only returned. +/// is that caller's own refusal and is only returned, unless the registry +/// contract itself does not decode or comes from another engine release. #[derive(Clone, Copy)] enum ReadClient<'a> { SourceReader, @@ -127,6 +202,9 @@ impl BregAdapter { reader, webhook_key: Zeroizing::new(webhook_key), reader_failure: Mutex::new(None), + caller_failure: Mutex::new(None), + peer_version: Mutex::new(None), + peer_mismatch: Mutex::new(None), }) } @@ -283,60 +361,183 @@ impl BregAdapter { } } + /// The release mismatch that refused this source's latest contract read, + /// so an operator tool can name both versions instead of a generic + /// source failure. A contract read from the matching release clears it. + #[must_use] + pub fn peer_version_mismatch(&self) -> Option { + self.peer_mismatch + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + fn failure_slot(&self, client: ReadClient<'_>) -> &Mutex> { + match client { + ReadClient::SourceReader => &self.reader_failure, + ReadClient::Caller(_) => &self.caller_failure, + } + } + + /// Run `log` when `key` differs from the failure last logged for this + /// credential, so a persistent failure is reported once per change. + fn report_failure(&self, client: ReadClient<'_>, key: String, log: impl FnOnce()) { + let mut reported = self + .failure_slot(client) + .lock() + .unwrap_or_else(PoisonError::into_inner); + if reported.as_deref() != Some(key.as_str()) { + log(); + *reported = Some(key); + } + } + + fn report_success(&self, client: ReadClient<'_>) { + let recovered = self + .failure_slot(client) + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + .is_some(); + if recovered && matches!(client, ReadClient::SourceReader) { + tracing::info!( + source_id = %self.config.source_id, + "Casework source reader requests to BReg succeed again" + ); + } + } + /// Map a BReg read result. A source reader failure is logged when its - /// cause first appears or changes, and the next success logs recovery. + /// cause first appears or changes, and the next success logs recovery. A + /// caller's refusal, or an outage it meets, is that caller's own answer + /// and is only returned; a registry contract that does not decode is a + /// deployment fault whichever credential met it, so it is logged too. fn read_result( &self, client: ReadClient<'_>, result: Result, ) -> Result { - if let ReadClient::Caller(_) = client { - return result.map_err(read_error); - } - let mut reported = self - .reader_failure - .lock() - .unwrap_or_else(PoisonError::into_inner); - match result { + let error = match result { Ok(value) => { - if reported.take().is_some() { - tracing::info!( - source_id = %self.config.source_id, - "Casework source reader requests to BReg succeed again" - ); + // A caller only reports contract failures, so only a + // contract read that succeeds clears one. + if matches!(client, ReadClient::SourceReader) { + self.report_success(client); } - Ok(value) + return Ok(value); } - Err(error) => { - let cause = error.to_string(); - // Metadata decode failures of different kinds render the same - // message, so the kind is part of what counts as a change. - let key = match error.metadata_error_kind() { - Some(kind) => format!("{cause} ({kind:?})"), - None => cause.clone(), - }; - if reported.as_deref() != Some(key.as_str()) { - // A runtime metadata decode failure only ever comes from - // the GET /v1/registry contract read, so the route is - // named here rather than threaded through every caller. - match error.metadata_error_kind() { - Some(kind) => tracing::warn!( - source_id = %self.config.source_id, - route = "GET /v1/registry", - metadata_error_kind = ?kind, - error = %cause, - "Casework source reader request to BReg failed" - ), - None => tracing::warn!( - source_id = %self.config.source_id, - error = %cause, - "Casework source reader request to BReg failed" - ), - } - *reported = Some(key); - } - Err(read_error(error)) + Err(error) => error, + }; + let cause = error.to_string(); + // Metadata decode failures of different kinds render the same + // message, so the kind is part of what counts as a change. + let key = match error.metadata_error_kind() { + Some(kind) => format!("{cause} ({kind:?})"), + None => cause.clone(), + }; + // A runtime metadata decode failure only ever comes from the + // GET /v1/registry contract read, so the route is named here rather + // than threaded through every caller. + match (client, error.metadata_error_kind()) { + (ReadClient::Caller(_), None) => {} + (ReadClient::Caller(_), Some(kind)) => self.report_failure(client, key, || { + tracing::warn!( + source_id = %self.config.source_id, + credential = "caller", + route = "GET /v1/registry", + metadata_error_kind = ?kind, + error = %cause, + "Casework caller request to BReg failed" + ); + }), + (ReadClient::SourceReader, Some(kind)) => self.report_failure(client, key, || { + tracing::warn!( + source_id = %self.config.source_id, + route = "GET /v1/registry", + metadata_error_kind = ?kind, + error = %cause, + "Casework source reader request to BReg failed" + ); + }), + (ReadClient::SourceReader, None) => self.report_failure(client, key, || { + tracing::warn!( + source_id = %self.config.source_id, + error = %cause, + "Casework source reader request to BReg failed" + ); + }), + } + Err(read_error(error)) + } + + /// Refuse a contract read from another engine release by name. + fn refuse_peer_version( + &self, + client: ReadClient<'_>, + peer_version: Option, + metadata_error_kind: Option, + ) -> SourceAdapterError { + let mismatch = PeerVersionMismatch { + source_id: self.config.source_id.clone(), + peer_version, + }; + let credential = match client { + ReadClient::SourceReader => "source reader", + ReadClient::Caller(_) => "caller", + }; + let kind = metadata_error_kind.map(|kind| format!("{kind:?}")); + self.report_failure( + client, + format!("peer engine version {:?}", mismatch.peer_version), + || { + tracing::warn!( + source_id = %self.config.source_id, + credential, + route = "GET /v1/registry", + peer_engine_version = mismatch.peer_version(), + casework_version = mismatch.casework_version(), + metadata_error_kind = kind.as_deref(), + "{mismatch}" + ); + }, + ); + *self + .peer_mismatch + .lock() + .unwrap_or_else(PoisonError::into_inner) = Some(mismatch); + SourceAdapterError::Unavailable + } + + /// Log the peer engine version when it is first read or changes. + fn note_peer_version(&self, peer_version: &str) { + self.peer_mismatch + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + let mut logged = self + .peer_version + .lock() + .unwrap_or_else(PoisonError::into_inner); + if logged.as_deref() != Some(peer_version) { + let own = registry_platform_buildinfo::DISPLAY_VERSION; + if peer_version == own { + tracing::info!( + source_id = %self.config.source_id, + peer_engine_version = peer_version, + casework_version = own, + "Casework reads a BReg source on its own release" + ); + } else { + tracing::warn!( + source_id = %self.config.source_id, + peer_engine_version = peer_version, + casework_version = own, + "Casework reads a BReg source on its own release, one side built without \ + the release marker; a development build is matched by its version only, \ + so run release builds of both in production" + ); } + *logged = Some(peer_version.to_owned()); } } @@ -345,8 +546,31 @@ impl BregAdapter { client: ReadClient<'_>, profile: &str, ) -> Result { - let contract = self.client(client).registry_contract(Some(profile)).await; + let (peer_version, contract) = self + .client(client) + .registry_contract_and_engine_version(Some(profile)) + .await; + let own = registry_platform_buildinfo::DISPLAY_VERSION; + // A contract read from another release is refused whether or not it + // decoded; one that reported no version is refused once it decoded, + // since an outage or a refusal carries no version to compare. + let unreported = peer_version.is_none() && contract.is_ok(); + if unreported + || peer_version + .as_deref() + .is_some_and(|peer| !same_release(peer, own)) + { + let kind = contract + .as_ref() + .err() + .and_then(BaseRegistryClientError::metadata_error_kind); + return Err(self.refuse_peer_version(client, peer_version, kind)); + } let metadata = self.read_result(client, contract)?.value; + if matches!(client, ReadClient::Caller(_)) { + self.report_success(client); + } + self.note_peer_version(peer_version.as_deref().unwrap_or(own)); if metadata.registry_revision() != self.config.expected_registry_revision { return Err(SourceAdapterError::BindingMoved); } @@ -734,6 +958,16 @@ impl SourceAdapter for BregAdapter { .map(|entry| &entry.routing_metadata) } + fn caller_disclosure_fields(&self, entity: &str) -> Option> { + self.request_entry(entity).ok().map(|entry| { + entry + .context_projection + .iter() + .map(|field| field.api_name.clone()) + .collect() + }) + } + async fn verify_transition( &self, request: EventRequest, @@ -1256,6 +1490,29 @@ mod tests { } } + #[test] + fn a_development_build_matches_its_release_and_nothing_else() { + for (peer, own) in [ + ("0.34.0", "0.34.0"), + ("0.34.0-dev", "0.34.0"), + ("0.34.0", "0.34.0-dev"), + ("0.34.0-dev", "0.34.0-dev"), + ("0.34.0-rc.1", "0.34.0-rc.1-dev"), + ] { + assert!(super::same_release(peer, own), "{peer} and {own}"); + } + for (peer, own) in [ + ("0.34.1", "0.34.0"), + ("0.34.0-rc.1", "0.34.0"), + ("0.34.0-rc.1", "0.34.0-dev"), + ("0.34.0-dev-dev", "0.34.0"), + ("0.34", "0.34.0"), + ("", "0.34.0"), + ] { + assert!(!super::same_release(peer, own), "{peer} and {own}"); + } + } + #[test] fn saved_attempt_current_version_round_trips_with_additive_fields() { let expected = saved_attempt(CURRENT_SAVED_ATTEMPT_VERSION); diff --git a/crates/registry-casework-breg/tests/prepared_recovery.rs b/crates/registry-casework-breg/tests/prepared_recovery.rs index a484e8af69..6ceffc1e0f 100644 --- a/crates/registry-casework-breg/tests/prepared_recovery.rs +++ b/crates/registry-casework-breg/tests/prepared_recovery.rs @@ -135,6 +135,10 @@ fn json_response(value: Value) -> ResponseTemplate { "traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", ) + .insert_header( + "registry-engine-version", + registry_platform_buildinfo::DISPLAY_VERSION, + ) } async fn prepare_for_recovery( diff --git a/crates/registry-casework-breg/tests/signed_event_intake.rs b/crates/registry-casework-breg/tests/signed_event_intake.rs index 5d8daf304f..c5fb494bc5 100644 --- a/crates/registry-casework-breg/tests/signed_event_intake.rs +++ b/crates/registry-casework-breg/tests/signed_event_intake.rs @@ -134,7 +134,11 @@ async fn mount_metadata(server: &MockServer, revision: &str) { .respond_with( ResponseTemplate::new(200) .set_body_json(support::lifecycle_metadata(revision)) - .insert_header("traceparent", TRACEPARENT), + .insert_header("traceparent", TRACEPARENT) + .insert_header( + "registry-engine-version", + registry_platform_buildinfo::DISPLAY_VERSION, + ), ) .expect(1) .mount(server) @@ -651,7 +655,11 @@ async fn mount_metadata_times(server: &MockServer, times: u64) { .respond_with( ResponseTemplate::new(200) .set_body_json(support::lifecycle_metadata(REGISTRY_REVISION)) - .insert_header("traceparent", TRACEPARENT), + .insert_header("traceparent", TRACEPARENT) + .insert_header( + "registry-engine-version", + registry_platform_buildinfo::DISPLAY_VERSION, + ), ) .expect(times) .mount(server) diff --git a/crates/registry-casework-breg/tests/source_boundary.rs b/crates/registry-casework-breg/tests/source_boundary.rs index c68c6467cb..8a366a8e8f 100644 --- a/crates/registry-casework-breg/tests/source_boundary.rs +++ b/crates/registry-casework-breg/tests/source_boundary.rs @@ -17,6 +17,7 @@ use wiremock::{ const ID: &str = "00000000-0000-4000-8000-000000000001"; const TRACE: &str = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; const DIGEST: &str = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const ENGINE: &str = registry_platform_buildinfo::DISPLAY_VERSION; fn adapter(base: &str) -> BregAdapter { adapter_with_reference_config(base, None) } @@ -193,6 +194,7 @@ async fn mount_metadata_revision(server: &MockServer, token: &str, profile: &str .and(query_param("accessProfile", profile)) .respond_with(ResponseTemplate::new(200) .insert_header("traceparent", TRACE) + .insert_header("registry-engine-version", ENGINE) .set_body_json(json!({"id":"test","version":"1.0.0","revision":revision,"metadataVersion":"1","entities":[],"operations":[]}))) .expect(1) .mount(server).await; @@ -299,6 +301,7 @@ async fn mount_reader_diagnostic( .respond_with( ResponseTemplate::new(200) .insert_header("traceparent", TRACE) + .insert_header("registry-engine-version", ENGINE) .set_body_json(metadata), ) .expect(1) @@ -490,6 +493,7 @@ async fn source_reader_logs_a_changed_metadata_failure_kind_behind_the_same_mess .respond_with( ResponseTemplate::new(200) .insert_header("traceparent", TRACE) + .insert_header("registry-engine-version", ENGINE) .set_body_json(metadata), ) .expect(1) @@ -750,6 +754,15 @@ async fn live_registry_change_refuses_projection_under_the_imported_source_contr SourceAdapterError::BindingMoved ); } +#[test] +fn caller_disclosure_fields_are_the_imported_api_names_of_the_context_projection() { + let adapter = adapter_with_context_projection("http://127.0.0.1:9"); + assert_eq!( + adapter.caller_disclosure_fields("correction"), + Some(vec!["summary".to_owned(), "attachmentMetadata".to_owned()]) + ); + assert_eq!(adapter.caller_disclosure_fields("licence"), None); +} #[tokio::test] async fn caller_context_projection_is_value_bounded_and_never_widens() { let server = MockServer::start().await; @@ -896,12 +909,16 @@ async fn captured_logs(work: impl std::future::Future) -> String { String::from_utf8(raw).unwrap() } -/// Assert the source reader entries, in order, as a level and an optional -/// cause the logged error must contain. +/// Assert the source reader failure and recovery entries, in order, as a +/// level and an optional cause the logged error must contain. The peer +/// engine version reported on a first matching read has its own test. fn assert_reader_log(raw: &str, expected: &[(&str, Option<&str>)]) { let entries = raw .lines() .map(|line| serde_json::from_str::(line).expect("structured tracing entry")) + .filter(|entry| { + entry["level"] != "INFO" || entry["fields"]["peer_engine_version"].is_null() + }) .collect::>(); assert_eq!(entries.len(), expected.len(), "{raw}"); for (entry, (level, cause)) in entries.iter().zip(expected) { @@ -1201,3 +1218,284 @@ async fn stale_source_generation_is_refused_before_any_read_or_write() { assert_eq!(result.unwrap_err(), SourceAdapterError::BindingMoved); assert!(server.received_requests().await.unwrap().is_empty()); } + +/// Mount the registry contract for one bearer token, reporting `engine` as +/// the peer's engine version, or no version at all. +async fn mount_contract(server: &MockServer, token: &str, body: Value, engine: Option<&str>) { + let mut response = ResponseTemplate::new(200) + .insert_header("traceparent", TRACE) + .set_body_json(body); + if let Some(engine) = engine { + response = response.insert_header("registry-engine-version", engine); + } + Mock::given(method("GET")) + .and(path("/v1/registry")) + .and(header("authorization", format!("Bearer {token}"))) + .respond_with(response) + .mount(server) + .await; +} + +async fn mount_caller_record(server: &MockServer, token: &str) { + Mock::given(method("GET")) + .and(path(format!("/v1/records/correction/{ID}"))) + .and(header("authorization", format!("Bearer {token}"))) + .respond_with(response(record("submitted", None))) + .mount(server) + .await; +} + +fn empty_contract() -> Value { + json!({"id":"test","version":"1.0.0","revision":DIGEST,"metadataVersion":"1","entities":[],"operations":[]}) +} + +fn log_entries(raw: &str) -> Vec { + raw.lines() + .map(|line| serde_json::from_str::(line).expect("structured tracing entry")) + .collect() +} + +#[tokio::test] +async fn a_breg_engine_from_another_release_is_refused_naming_both_versions() { + let server = MockServer::start().await; + let adapter = adapter(&server.uri()); + mount_contract(&server, "reader-token", empty_contract(), Some("0.0.1")).await; + let logs = captured_logs(async { + for _ in 0..2 { + assert_eq!( + adapter.discover_active(None, 100).await.unwrap_err(), + SourceAdapterError::Unavailable + ); + } + }) + .await; + + let mismatch = adapter + .peer_version_mismatch() + .expect("the mismatch is kept for doctor"); + assert_eq!(mismatch.peer_version(), Some("0.0.1")); + let text = mismatch.to_string(); + for expected in ["source", "0.0.1", ENGINE, "same release"] { + assert!(text.contains(expected), "{text} names {expected}"); + } + let entries = log_entries(&logs); + assert_eq!( + entries.len(), + 1, + "a repeated mismatch is logged once: {logs}" + ); + assert_eq!(entries[0]["level"], "WARN"); + assert_eq!(entries[0]["fields"]["source_id"], "source"); + assert_eq!(entries[0]["fields"]["route"], "GET /v1/registry"); + assert_eq!(entries[0]["fields"]["peer_engine_version"], "0.0.1"); + assert_eq!(entries[0]["fields"]["casework_version"], ENGINE); +} + +#[tokio::test] +async fn a_development_build_of_the_same_release_is_accepted_and_a_prerelease_is_not() { + // A build without the release marker reports `-dev`; its + // counterpart is the same version built as a release, or the reverse. + let release = ENGINE.strip_suffix("-dev").unwrap_or(ENGINE); + let counterpart = if ENGINE == release { + format!("{release}-dev") + } else { + release.to_owned() + }; + + let server = MockServer::start().await; + let development = adapter(&server.uri()); + mount_contract( + &server, + "reader-token", + empty_contract(), + Some(&counterpart), + ) + .await; + let logs = captured_logs(async { + let _ = development.discover_active(None, 100).await; + }) + .await; + assert_eq!(development.peer_version_mismatch(), None, "{logs}"); + let entries = log_entries(&logs); + let noted = entries + .iter() + .find(|entry| entry["fields"]["peer_engine_version"] == counterpart.as_str()) + .unwrap_or_else(|| panic!("the development build is logged: {logs}")); + assert_eq!(noted["level"], "WARN", "{logs}"); + assert_eq!(noted["fields"]["casework_version"], ENGINE, "{logs}"); + + let server = MockServer::start().await; + let prerelease_adapter = adapter(&server.uri()); + let prerelease = format!("{release}-rc.1"); + mount_contract(&server, "reader-token", empty_contract(), Some(&prerelease)).await; + assert_eq!( + prerelease_adapter + .discover_active(None, 100) + .await + .unwrap_err(), + SourceAdapterError::Unavailable + ); + assert_eq!( + prerelease_adapter + .peer_version_mismatch() + .expect("a prerelease of the same version is another release") + .peer_version(), + Some(prerelease.as_str()) + ); +} + +#[tokio::test] +async fn a_peer_version_mismatch_explains_a_contract_that_does_not_decode() { + let server = MockServer::start().await; + let adapter = adapter(&server.uri()); + let mut contract = empty_contract(); + contract["entities"] = json!("not-an-array"); + mount_contract(&server, "reader-token", contract, Some("0.0.1")).await; + let logs = captured_logs(async { + assert_eq!( + adapter.discover_active(None, 100).await.unwrap_err(), + SourceAdapterError::Unavailable + ); + }) + .await; + + assert_eq!( + adapter + .peer_version_mismatch() + .expect("the mismatch names the cause") + .peer_version(), + Some("0.0.1") + ); + let entries = log_entries(&logs); + assert_eq!(entries.len(), 1, "{logs}"); + assert_eq!(entries[0]["fields"]["peer_engine_version"], "0.0.1"); + assert_eq!(entries[0]["fields"]["metadata_error_kind"], "Shape"); +} + +#[tokio::test] +async fn a_breg_engine_that_reports_no_version_is_refused_by_name() { + let server = MockServer::start().await; + let adapter = adapter(&server.uri()); + mount_contract(&server, "reader-token", empty_contract(), None).await; + assert_eq!( + adapter.discover_active(None, 100).await.unwrap_err(), + SourceAdapterError::Unavailable + ); + + let mismatch = adapter + .peer_version_mismatch() + .expect("an unreported version is refused"); + assert_eq!(mismatch.peer_version(), None); + let text = mismatch.to_string(); + assert!( + text.contains("does not report its engine version"), + "{text}" + ); + assert!(text.contains(ENGINE), "{text}"); +} + +#[tokio::test] +async fn the_breg_engine_version_is_logged_when_first_read_and_a_mismatch_clears_on_recovery() { + let server = MockServer::start().await; + let adapter = adapter(&server.uri()); + let logs = captured_logs(async { + let other = Mock::given(method("GET")) + .and(path("/v1/registry")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("traceparent", TRACE) + .insert_header("registry-engine-version", "0.0.1") + .set_body_json(empty_contract()), + ) + .mount_as_scoped(&server) + .await; + assert!(adapter.discover_active(None, 100).await.is_err()); + drop(other); + + mount_reader_diagnostic( + &server, + diagnostic_metadata(&["get", "list"], &[("record", "record")]), + 200, + true, + ) + .await; + adapter.verify_reader_readiness().await.unwrap(); + }) + .await; + + assert_eq!(adapter.peer_version_mismatch(), None); + let entries = log_entries(&logs); + let levels = entries + .iter() + .map(|entry| entry["level"].as_str().unwrap().to_owned()) + .collect::>(); + assert_eq!(levels, ["WARN", "INFO", "INFO"], "{logs}"); + assert_eq!( + entries[2]["fields"]["peer_engine_version"], ENGINE, + "{logs}" + ); +} + +#[tokio::test] +async fn a_caller_contract_failure_names_the_route_and_metadata_kind() { + let server = MockServer::start().await; + let adapter = adapter(&server.uri()); + let mut contract = empty_contract(); + contract["metadataVersion"] = json!("2"); + mount_contract(&server, "alice-token", contract, Some(ENGINE)).await; + mount_caller_record(&server, "alice-token").await; + let logs = captured_logs(async { + for _ in 0..2 { + assert_eq!( + adapter + .read_for_caller( + &subject(), + "reviewer", + EphemeralCredential::new("alice-token") + ) + .await + .unwrap_err(), + SourceAdapterError::Unavailable + ); + } + }) + .await; + + assert!(!logs.contains("alice-token"), "{logs}"); + let entries = log_entries(&logs); + assert_eq!( + entries.len(), + 1, + "a repeated caller failure is logged once: {logs}" + ); + assert_eq!(entries[0]["level"], "WARN"); + assert_eq!(entries[0]["fields"]["route"], "GET /v1/registry"); + assert_eq!(entries[0]["fields"]["metadata_error_kind"], "Version"); + assert_eq!(entries[0]["fields"]["credential"], "caller"); +} + +#[tokio::test] +async fn a_caller_read_is_refused_when_the_engine_is_from_another_release() { + let server = MockServer::start().await; + let adapter = adapter(&server.uri()); + mount_contract(&server, "alice-token", empty_contract(), Some("0.0.1")).await; + mount_caller_record(&server, "alice-token").await; + assert_eq!( + adapter + .read_for_caller( + &subject(), + "reviewer", + EphemeralCredential::new("alice-token") + ) + .await + .unwrap_err(), + SourceAdapterError::Unavailable + ); + assert_eq!( + adapter + .peer_version_mismatch() + .expect("a caller read also names the mismatch") + .peer_version(), + Some("0.0.1") + ); +} diff --git a/crates/registry-casework-core/src/adapter.rs b/crates/registry-casework-core/src/adapter.rs index ffce94e203..f0b384fd20 100644 --- a/crates/registry-casework-core/src/adapter.rs +++ b/crates/registry-casework-core/src/adapter.rs @@ -224,6 +224,14 @@ pub trait SourceAdapter: Send + Sync { None } + /// The keys a caller read of one request entity may place in + /// `CallerSubjectView::disclosed`. Activation compares them with the + /// display schemas pinned by in-flight reviews. `None` means the adapter + /// does not declare them, and that comparison is skipped for the entity. + fn caller_disclosure_fields(&self, _entity: &str) -> Option> { + None + } + async fn verify_transition( &self, request: EventRequest, diff --git a/crates/registry-casework-core/src/config.rs b/crates/registry-casework-core/src/config.rs index 96e9447f2a..f380e2073e 100644 --- a/crates/registry-casework-core/src/config.rs +++ b/crates/registry-casework-core/src/config.rs @@ -84,7 +84,12 @@ pub struct CaseworkProject { impl CaseworkProject { pub fn load(path: impl AsRef) -> Result { let bytes = std::fs::read(path).map_err(ConfigLoadError::Read)?; - let deserializer = serde_norway::Deserializer::from_slice(&bytes); + Self::from_slice(&bytes) + } + + /// Decode and validate one captured Casework project document. + pub fn from_slice(bytes: &[u8]) -> Result { + let deserializer = serde_norway::Deserializer::from_slice(bytes); let project: Self = serde_path_to_error::deserialize(deserializer).map_err(|error| { let path = error.path().to_string(); ConfigLoadError::Parse { diff --git a/crates/registry-casework/Cargo.toml b/crates/registry-casework/Cargo.toml index 59c9fe4347..ef65fe3c78 100644 --- a/crates/registry-casework/Cargo.toml +++ b/crates/registry-casework/Cargo.toml @@ -15,7 +15,12 @@ path = "src/main.rs" [features] default = [] postgres-test = ["dep:registry-breg", "registry-breg/postgres-test"] -schema = ["dep:schemars", "registry-casework-breg/schema", "registry-platform-audit/schema"] +schema = [ + "dep:schemars", + "registry-casework-breg/schema", + "registry-platform-audit/schema", + "registry-platform-config/schema", +] [lints] workspace = true diff --git a/crates/registry-casework/migrations/0018_source_reconciliation_health.sql b/crates/registry-casework/migrations/0018_source_reconciliation_health.sql new file mode 100644 index 0000000000..f0b75aab06 --- /dev/null +++ b/crates/registry-casework/migrations/0018_source_reconciliation_health.sql @@ -0,0 +1,9 @@ +ALTER TABLE casework_source_status + ADD COLUMN IF NOT EXISTS consecutive_failures integer NOT NULL DEFAULT 0 + CHECK (consecutive_failures >= 0), + ADD COLUMN IF NOT EXISTS last_succeeded_at timestamptz, + ADD COLUMN IF NOT EXISTS last_failed_at timestamptz, + ADD COLUMN IF NOT EXISTS last_failure text CHECK ( + last_failure IS NULL + OR last_failure IN ('source-unavailable', 'source-refused', 'store', 'configuration') + ); diff --git a/crates/registry-casework/src/config.rs b/crates/registry-casework/src/config.rs index eb1cc0a4a4..ea77706526 100644 --- a/crates/registry-casework/src/config.rs +++ b/crates/registry-casework/src/config.rs @@ -3,78 +3,62 @@ use std::net::{IpAddr, Ipv6Addr, SocketAddr}; use std::path::{Path, PathBuf}; use std::time::Duration; -use jsonwebtoken::jwk::{AlgorithmParameters, JwkSet}; use jsonwebtoken::Algorithm; use registry_casework_core::{check_routing_policy, CaseworkProject}; use registry_platform_audit::{AuditDestination, AuditDestinationError, AuditDestinationKind}; +pub(crate) use registry_platform_config::describe_secret_failure; +use registry_platform_config::package::is_envelope_file; use registry_platform_config::{ - SecretError, SecretProvider, SecretReference, SecretResolver, MAX_SECRET_BYTES, + is_sha256_label, reject_environment_expressions_in_authored_yaml, sha256_uri, ConfigBlockError, + PackageConfig, PackageDigestMismatch, PackageError, PackageErrorKind, PackageLimits, + RemovedKey, RuntimeConfigErrorKind, RuntimeConfigLoader, RuntimeEnvelope, SecretResolver, + VerifiedPackage, REMOVED_OIDC_JWKS_URI, +}; +pub use registry_platform_config::{ + AuditKeyConfig, DatabaseConfig, EnvironmentSecretProviderConfig, FileSecretProviderConfig, + JwksSource, ListenerNetworkExposure, OidcClientsConfig, OidcIssuerConfig, + PrivateListenerConfig as ListenerConfig, SecretProvidersConfig, TlsTermination, }; use registry_platform_oidc::{ - access_token_typ_set, fetch_discovery, JwksFetcher, JwksFetcherConfig, OidcDiscoveryConfig, - TokenVerifierConfig, + access_token_typ_set, fetch_discovery, parse_static_jwks, JwksFetcher, JwksFetcherConfig, + OidcDiscoveryConfig, TokenVerifierConfig, }; -use serde::{Deserialize, Serialize}; -use sha2::{Digest as _, Sha256}; +use serde::Deserialize; use thiserror::Error; -/// Explain one refused secret reference without disclosing what it protects. -/// -/// A startup refusal reaches an operator as a single line, and the resolver -/// reports only which rule broke. A valid reference is safe and useful to name, -/// but invalid operator-authored text might itself be a literal credential, so -/// only its field is named. The resolved bytes and opened path never appear. -pub(crate) fn describe_secret_failure( - field: &'static str, - reference: &str, - error: &SecretError, -) -> String { - let reason = match error { - SecretError::InvalidReference => { - "it is not an exact secret:env/NAME or secret:file/name reference".to_owned() - } - SecretError::ProviderDisabled => "its provider is not enabled for this runtime".to_owned(), - SecretError::InvalidProviderConfiguration => { - "the secret provider configuration is invalid".to_owned() - } - SecretError::Unavailable => { - "no readable secret of that name exists under the configured provider".to_owned() - } - SecretError::UnsafeFile => concat!( - "the secret file must be a regular file owned by the runtime user, ", - "with mode 0400 or 0600, and exactly one hard link" - ) - .to_owned(), - SecretError::Read => "the secret could not be read".to_owned(), - SecretError::InvalidValue => format!( - "the secret value must be non-empty text of at most {MAX_SECRET_BYTES} bytes \ - without NUL bytes" - ), - }; - if error == &SecretError::InvalidReference { - format!("the secret reference configured at {field} could not be resolved: {reason}") - } else { - format!("the secret reference {reference} could not be resolved: {reason}") - } -} - -pub const POLICY_PACKAGE_API_VERSION: &str = - "registry.registrystack.org/casework-policy-package/v1alpha1"; -pub const POLICY_PACKAGE_KIND: &str = "CaseworkPolicyPackage"; -pub const POLICY_PACKAGE_MANIFEST_FILE: &str = "casework.package.json"; +/// The command that builds a Casework package, named in every package refusal. +pub const PACKAGE_COMMAND: &str = "caseworkctl package"; +/// The manifest earlier Casework packages carried. A package root holding it +/// is refused, naming the command that writes the current package. +pub const RETIRED_PACKAGE_MANIFEST_FILE: &str = "casework.package.json"; pub const RUNTIME_CONFIG_API_VERSION: &str = "registry.registrystack.org/casework-runtime/v1alpha1"; pub const RUNTIME_CONFIG_KIND: &str = "CaseworkRuntimeConfig"; pub const POLICY_FILE: &str = "casework.yaml"; + +/// The envelope every Casework runtime configuration carries. +pub const CASEWORK_RUNTIME_ENVELOPE: RuntimeEnvelope = RuntimeEnvelope { + api_version: RUNTIME_CONFIG_API_VERSION, + kind: RUNTIME_CONFIG_KIND, +}; + +/// Keys an earlier Casework runtime configuration accepted, each refused with +/// the key that replaced it. +pub const CASEWORK_REMOVED_KEYS: &[RemovedKey] = &[ + REMOVED_OIDC_JWKS_URI, + RemovedKey { + path: "authentication.oidc.principalClaim", + replacement: "declare accessProfiles[].principalClaim in casework.yaml", + }, + RemovedKey { + path: "package.expectedPolicyDigest", + replacement: "package.expectedDigest, the digest `caseworkctl package` reports", + }, +]; /// The RFC 9068 access-token media type this runtime verifies. The pair of /// spellings it admits is derived from this one value, never authored, so no /// deployment can widen it to an ordinary JWT. const CASEWORK_ACCESS_TOKEN_TYPE: &str = "at+jwt"; -const MAXIMUM_POLICY_PACKAGE_FILE_BYTES: usize = 1024 * 1024; -const MAXIMUM_POLICY_PACKAGE_MANIFEST_BYTES: usize = 1024 * 1024; -pub(crate) const MAXIMUM_ASSERTION_ISSUER_CLIENTS: usize = 64; -pub(crate) const MAXIMUM_ASSERTION_ISSUER_CLIENT_BYTES: usize = 128; -pub(crate) const MAXIMUM_ASSERTION_ISSUERS_PER_CLIENT: usize = 16; -pub(crate) const MAXIMUM_ASSERTION_ISSUER_BYTES: usize = 512; +const MAXIMUM_PACKAGE_FILE_BYTES: usize = 1024 * 1024; pub(crate) const MINIMUM_REVIEW_COMPLETION_TIMEOUT_MILLISECONDS: u64 = 100; pub(crate) const MAXIMUM_REVIEW_COMPLETION_TIMEOUT_MILLISECONDS: u64 = 30_000; pub(crate) const MINIMUM_REVIEW_COMPLETION_ATTEMPTS: u32 = 1; @@ -82,163 +66,100 @@ pub(crate) const MAXIMUM_REVIEW_COMPLETION_ATTEMPTS: u32 = 100; pub(crate) const MINIMUM_REVIEW_COMPLETION_RETRY_SECONDS: u64 = 1; pub(crate) const MAXIMUM_REVIEW_COMPLETION_RETRY_SECONDS: u64 = 86_400; -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct PolicyPackageManifest { - pub api_version: String, - pub kind: String, - pub policy_digest: String, - pub files: Vec, -} - -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct PolicyPackageFile { - pub path: String, - pub sha256: String, - pub bytes: u64, +/// The bounds a Casework package holds to: each file at most one MiB. +#[must_use] +pub fn package_limits() -> PackageLimits { + PackageLimits { + max_file_bytes: MAXIMUM_PACKAGE_FILE_BYTES as u64, + ..PackageLimits::default() + } } -impl PolicyPackageManifest { - /// Build the immutable identity for already validated policy inputs. - pub fn build( - files: impl IntoIterator)>, - ) -> Result { - let mut files = files - .into_iter() - .map(|(path, bytes)| { - if normalized_relative_path(&path).is_none() - || bytes.len() > MAXIMUM_POLICY_PACKAGE_FILE_BYTES - { - return Err(PolicyPackageError::Invalid); - } - Ok(PolicyPackageFile { - path, - sha256: sha256_bytes(&bytes), - bytes: u64::try_from(bytes.len()).map_err(|_| PolicyPackageError::Invalid)?, - }) - }) - .collect::, _>>()?; - files.sort_by(|left, right| left.path.cmp(&right.path)); - if !files.iter().any(|file| file.path == POLICY_FILE) - || files.windows(2).any(|pair| pair[0].path == pair[1].path) +/// The files a Casework package holds besides `SHA256SUMS` and `REVISION`: +/// `casework.yaml` and every source description the project names. +pub fn package_inputs(project: &CaseworkProject) -> Result, RuntimeConfigError> { + let mut inputs = BTreeSet::from([POLICY_FILE.to_owned()]); + for source in &project.sources { + if normalized_relative_path(&source.description).is_none() + || !inputs.insert(source.description.clone()) { - return Err(PolicyPackageError::Invalid); + return Err(RuntimeConfigError::SourceDescription); } - let policy_digest = - package_digest(POLICY_PACKAGE_API_VERSION, POLICY_PACKAGE_KIND, &files)?; - Ok(Self { - api_version: POLICY_PACKAGE_API_VERSION.to_owned(), - kind: POLICY_PACKAGE_KIND.to_owned(), - policy_digest, - files, - }) } + Ok(inputs) +} - pub fn verify(&self, root: &Path, project: &CaseworkProject) -> Result<(), PolicyPackageError> { - if self.api_version != POLICY_PACKAGE_API_VERSION - || self.kind != POLICY_PACKAGE_KIND - || self.files.is_empty() - || self - .files - .windows(2) - .any(|pair| pair[0].path >= pair[1].path) - || self.policy_digest != package_digest(&self.api_version, &self.kind, &self.files)? - { - return Err(PolicyPackageError::Invalid); - } - - let mut expected = BTreeSet::from([POLICY_FILE.to_owned()]); - for source in &project.sources { - if normalized_relative_path(&source.description).is_none() - || !expected.insert(source.description.clone()) - { - return Err(PolicyPackageError::Invalid); - } - } - let declared = self - .files - .iter() - .map(|file| file.path.clone()) - .collect::>(); - if declared != expected { - return Err(PolicyPackageError::Invalid); - } +/// Verify the Casework package at `package.root` and its pin. Every listener +/// mode serves a package: a directory without `SHA256SUMS` is refused with the +/// packaging command, whether or not a digest is pinned. +pub fn verify_casework_package( + package: &RuntimePackageConfig, + project: &CaseworkProject, +) -> Result { + let verified = verify_casework_envelope(package)?; + verify_casework_contents(project, &verified)?; + Ok(verified) +} - for file in &self.files { - let relative = - normalized_relative_path(&file.path).ok_or(PolicyPackageError::Invalid)?; - let bytes = read_bounded_file(&root.join(relative), MAXIMUM_POLICY_PACKAGE_FILE_BYTES)?; - if file.bytes != u64::try_from(bytes.len()).map_err(|_| PolicyPackageError::Invalid)? - || file.sha256 != sha256_bytes(&bytes) - { - return Err(PolicyPackageError::Invalid); +fn verify_casework_envelope( + package: &RuntimePackageConfig, +) -> Result { + if std::fs::symlink_metadata(package.root.join(RETIRED_PACKAGE_MANIFEST_FILE)).is_ok() { + return Err(RuntimeConfigError::RetiredPackageManifest); + } + let verified = package + .shared() + .verify_package(&package_limits(), PACKAGE_COMMAND) + .map_err(|error| match error.kind() { + PackageErrorKind::DigestMismatch(mismatch) => { + RuntimeConfigError::PackageDigest(mismatch.clone()) } - } - - let mut on_disk = package_files_on_disk(root)?; - on_disk.remove(POLICY_PACKAGE_MANIFEST_FILE); - if on_disk != expected { - return Err(PolicyPackageError::Invalid); - } - Ok(()) - } + _ => RuntimeConfigError::Package(error), + })?; + Ok(verified) } -/// Verify a package next to `casework.yaml`, returning its immutable identity. -/// An absent manifest is distinguished so local authored development remains usable. -pub fn verify_policy_package( - project_path: &Path, +fn verify_casework_contents( project: &CaseworkProject, -) -> Result, PolicyPackageError> { - if project_path.file_name().and_then(|name| name.to_str()) != Some(POLICY_FILE) { - return Err(PolicyPackageError::Invalid); - } - let root = project_path.parent().ok_or(PolicyPackageError::Invalid)?; - let manifest_path = root.join(POLICY_PACKAGE_MANIFEST_FILE); - if !manifest_path.exists() { - return Ok(None); - } - let bytes = read_bounded_file(&manifest_path, MAXIMUM_POLICY_PACKAGE_MANIFEST_BYTES)?; - let manifest: PolicyPackageManifest = - serde_json::from_slice(&bytes).map_err(|_| PolicyPackageError::Invalid)?; - manifest.verify(root, project)?; - Ok(Some(manifest.policy_digest)) + verified: &VerifiedPackage, +) -> Result<(), RuntimeConfigError> { + let expected = package_inputs(project)?; + let found = verified + .files() + .filter(|path| !is_envelope_file(path)) + .map(ToOwned::to_owned) + .collect::>(); + if found != expected { + return Err(RuntimeConfigError::PackageContents { + missing: expected.difference(&found).cloned().collect(), + extra: found.difference(&expected).cloned().collect(), + }); + } + Ok(()) } -fn package_digest( - api_version: &str, - kind: &str, - files: &[PolicyPackageFile], -) -> Result { - let identity = serde_json::json!({ - "apiVersion": api_version, - "kind": kind, - "files": files, - }); - let canonical = registry_platform_canonical_json::canonicalize_json(&identity) - .map_err(|_| PolicyPackageError::Invalid)?; - Ok(sha256_bytes(&canonical)) +#[derive(Debug)] +pub struct LoadedCaseworkPackage { + digest: String, + project: CaseworkProject, + source_descriptions: BTreeMap>, } -fn valid_policy_digest(value: &str) -> bool { - value.strip_prefix("sha256:").is_some_and(|hex| { - hex.len() == 64 - && hex - .bytes() - .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) - }) -} +impl LoadedCaseworkPackage { + #[must_use] + pub fn digest(&self) -> &str { + &self.digest + } -fn sha256_bytes(bytes: &[u8]) -> String { - let digest = Sha256::digest(bytes); - format!( - "sha256:{}", - digest - .iter() - .map(|byte| format!("{byte:02x}")) - .collect::() - ) + #[must_use] + pub const fn project(&self) -> &CaseworkProject { + &self.project + } + + #[must_use] + pub fn source_description(&self, path: &str) -> Option<&[u8]> { + self.source_descriptions.get(path).map(Vec::as_slice) + } } fn normalized_relative_path(value: &str) -> Option { @@ -256,55 +177,32 @@ fn normalized_relative_path(value: &str) -> Option { (normalized.to_str() == Some(value)).then_some(normalized) } -fn read_bounded_file(path: &Path, maximum: usize) -> Result, PolicyPackageError> { - let metadata = std::fs::symlink_metadata(path).map_err(PolicyPackageError::Read)?; - if !metadata.file_type().is_file() - || metadata.file_type().is_symlink() - || metadata.len() > maximum as u64 - { - return Err(PolicyPackageError::Invalid); +/// Read a regular file of at most `maximum` bytes, refusing a link. +fn read_bounded_file(path: &Path, maximum: usize) -> Option> { + let metadata = std::fs::symlink_metadata(path).ok()?; + if !metadata.file_type().is_file() || metadata.len() > maximum as u64 { + return None; } - std::fs::read(path).map_err(PolicyPackageError::Read) + std::fs::read(path).ok() } -fn package_files_on_disk(root: &Path) -> Result, PolicyPackageError> { - let mut pending = vec![root.to_path_buf()]; - let mut files = BTreeSet::new(); - while let Some(directory) = pending.pop() { - for entry in std::fs::read_dir(directory).map_err(PolicyPackageError::Read)? { - let entry = entry.map_err(PolicyPackageError::Read)?; - let metadata = - std::fs::symlink_metadata(entry.path()).map_err(PolicyPackageError::Read)?; - if metadata.file_type().is_symlink() { - return Err(PolicyPackageError::Invalid); +fn capture_verified_file( + root: &Path, + verified: &VerifiedPackage, + relative: &str, +) -> Result, RuntimeConfigError> { + let bytes = + read_bounded_file(&root.join(relative), MAXIMUM_PACKAGE_FILE_BYTES).ok_or_else(|| { + RuntimeConfigError::PackageFileChanged { + path: relative.to_owned(), } - if metadata.is_dir() { - pending.push(entry.path()); - } else if metadata.is_file() { - let relative = entry - .path() - .strip_prefix(root) - .map_err(|_| PolicyPackageError::Invalid)? - .to_str() - .ok_or(PolicyPackageError::Invalid)? - .to_owned(); - if normalized_relative_path(&relative).is_none() || !files.insert(relative) { - return Err(PolicyPackageError::Invalid); - } - } else { - return Err(PolicyPackageError::Invalid); - } - } + })?; + if verified.file_digest(relative).as_deref() != Some(sha256_uri(&bytes).as_str()) { + return Err(RuntimeConfigError::PackageFileChanged { + path: relative.to_owned(), + }); } - Ok(files) -} - -#[derive(Debug, Error)] -pub enum PolicyPackageError { - #[error("the Casework policy package could not be read")] - Read(#[source] std::io::Error), - #[error("the Casework policy package is invalid or does not match its exact inputs")] - Invalid, + Ok(bytes) } /// Validate one imported BReg description through the adapter's owning strict @@ -328,6 +226,8 @@ pub struct RuntimeConfig { pub kind: String, pub package: RuntimePackageConfig, pub listener: ListenerConfig, + #[serde(default)] + pub metrics_listener: Option, pub secret_providers: SecretProvidersConfig, pub database: DatabaseConfig, pub authentication: AuthenticationConfig, @@ -533,74 +433,70 @@ pub struct TaskAuthorityConfig { #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RuntimePackageConfig { + /// Absolute path of the package directory. pub root: PathBuf, - /// The `policyDigest` of the one reviewed package this runtime may load. - /// When set, a package whose manifest names any other digest, or a - /// directory with no manifest, is refused before the runtime starts. + /// `sha256:` label of the package digest, the digest of the package's + /// `SHA256SUMS` file. When set, the runtime refuses to start on any other + /// package, and on an authored project that has no `SHA256SUMS`. #[serde(default)] - pub expected_policy_digest: Option, -} - -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct ListenerConfig { - #[serde(default = "default_listener_bind")] - #[cfg_attr(feature = "schema", schemars(with = "String"))] - pub bind: SocketAddr, - pub tls_termination: TlsTermination, + pub expected_digest: Option, + /// The package digest of a package the operator has accepted will strand + /// in-flight work pinned under an earlier package. Startup and `doctor` + /// refuse such a package unless this names its exact digest, so an + /// acknowledgement never carries over to a later package. #[serde(default)] - pub network_exposure: ListenerNetworkExposure, + pub acknowledge_stranded_work: Option, } -fn default_listener_bind() -> SocketAddr { - "127.0.0.1:8100" - .parse() - .expect("valid Casework listener default") +impl RuntimePackageConfig { + /// The shared `package.root` and `package.expectedDigest` block. + #[must_use] + pub fn shared(&self) -> PackageConfig { + PackageConfig { + root: self.root.clone(), + expected_digest: self.expected_digest.clone(), + } + } } -/// Declares the trusted transport boundary for the runtime's plaintext HTTP listener. +/// Operator-private listener for `/metrics` and `/version`. /// -/// Production listeners require operator-controlled upstream TLS termination. -/// Direct plaintext is limited to the explicit loopback-only development mode. -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] -#[serde(rename_all = "kebab-case")] -pub enum TlsTermination { - OperatorControlledUpstream, - DevelopmentLoopback, -} - -/// The operator-declared private network placement of the HTTP listener. -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] -#[serde(rename_all = "kebab-case")] -pub enum ListenerNetworkExposure { - #[default] - PrivateAddress, - ContainerPrivate, -} - +/// It is separate from the API listener so the counters and the active +/// package digest never appear on the surface the public contract describes. #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct SecretProvidersConfig { - #[serde(default)] - pub file: Option, - #[serde(default)] - pub environment: Option, +pub struct MetricsListenerConfig { + #[cfg_attr(feature = "schema", schemars(with = "String"))] + pub bind: SocketAddr, } -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EnvironmentSecretProviderConfig {} - -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct FileSecretProviderConfig { - pub root: PathBuf, +impl MetricsListenerConfig { + fn validate(&self, listener: &ListenerConfig) -> Result<(), RuntimeConfigError> { + let address = self.bind.ip(); + let private = match address { + IpAddr::V4(address) => address.is_loopback() || address.is_private(), + IpAddr::V6(address) => address.is_loopback() || is_unique_local(address), + }; + if self.bind.port() == 0 || !private || address.is_multicast() { + return Err(RuntimeConfigError::InvalidMetricsListener); + } + // An IPv6 wildcard socket is commonly dual-stack, so it is treated as + // covering both families on every host; the IPv4 wildcard covers + // only IPv4. + let listener_address = listener.bind.ip(); + let listener_port = listener.bind.socket_addr().port(); + let wildcard_covers_metrics = match listener_address { + IpAddr::V4(listener_address) => listener_address.is_unspecified() && address.is_ipv4(), + IpAddr::V6(listener_address) => listener_address.is_unspecified(), + }; + if (address == listener_address || wildcard_covers_metrics) + && self.bind.port() == listener_port + { + return Err(RuntimeConfigError::InvalidMetricsListener); + } + Ok(()) + } } #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] @@ -610,53 +506,22 @@ pub struct AuthenticationConfig { pub oidc: OidcConfig, } -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct DatabaseConfig { - pub runtime_url_ref: String, - pub migration_url_ref: String, - #[serde(default)] - pub trusted_root_certificate_ref: Option, - #[serde(default)] - pub test_only_plaintext: bool, -} - -impl std::fmt::Debug for DatabaseConfig { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("DatabaseConfig") - .field("runtime_url_ref", &"") - .field("migration_url_ref", &"") - .field( - "trusted_root_certificate_ref", - &self - .trusted_root_certificate_ref - .as_ref() - .map(|_| ""), - ) - .field("test_only_plaintext", &self.test_only_plaintext) - .finish() - } -} - +/// The access tokens this runtime accepts: the issuer and its keys, the +/// clients admitted, the scope claim, and the claim that marks a human actor. #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct OidcConfig { - #[serde(default)] - pub allowed_clients: Vec, - /// Assertion authorities each client may exchange a subject token from, - /// keyed by client identifier. An empty map applies no rule; see + /// The exact issuer, the one audience every token carries, and where the + /// issuer's signing keys come from. + #[serde(flatten)] + pub provider: OidcIssuerConfig, + /// The clients admitted and the assertion authorities each may exchange + /// a subject token from, keyed by client identifier. An empty + /// `assertionIssuers` map applies no rule; see /// [`registry_platform_oidc::TokenVerifierConfig::assertion_issuers`]. - #[serde(default)] - pub assertion_issuers: BTreeMap>, - pub issuer: String, - pub audience: String, - #[serde(default)] - pub jwks_uri: Option, - #[serde(default)] - pub jwks_source: OidcJwksSource, + #[serde(flatten)] + pub clients: OidcClientsConfig, #[serde(default = "default_scope_claim")] pub scope_claim: String, #[serde(default)] @@ -682,18 +547,6 @@ impl Default for HumanIdentityConfig { } } -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Debug, Default, Deserialize)] -#[serde(tag = "kind", rename_all = "camelCase", deny_unknown_fields)] -pub enum OidcJwksSource { - #[default] - Discovery, - Static { - #[serde(rename = "documentRef")] - document_ref: String, - }, -} - fn default_scope_claim() -> String { "registry_scopes".to_owned() } @@ -704,11 +557,14 @@ fn default_human_identity_value() -> String { "human".to_owned() } +/// The audit journal: where it is written and the key its hashes use. #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct AuditConfig { - pub hash_key_ref: String, + /// The secret keying the audit journal's hashes, `hashKeyRef`. + #[serde(flatten)] + pub key: AuditKeyConfig, /// Where audit entries go: a rotated `file` (the default) or `stdout`. #[serde(default)] pub destination: AuditDestinationKind, @@ -742,38 +598,19 @@ impl AuditConfig { } impl RuntimeConfig { + /// Load and validate the operator document, reading the authored project + /// and its package beside it. pub fn load(path: impl AsRef) -> Result { - if !path.as_ref().is_absolute() { - return Err(RuntimeConfigError::RelativeRuntimePath); - } - let bytes = std::fs::read(path.as_ref()).map_err(RuntimeConfigError::Read)?; - let value: serde_norway::Value = - serde_norway::from_slice(&bytes).map_err(|source| RuntimeConfigError::Parse { - path: "/".to_owned(), - source, - })?; - if value - .get("authentication") - .and_then(|value| value.get("oidc")) - .and_then(|value| value.get("principalClaim")) - .is_some() - { - return Err(RuntimeConfigError::RemovedPrincipalClaim); - } - let deserializer = serde_norway::Deserializer::from_slice(&bytes); - let config: Self = serde_path_to_error::deserialize(deserializer).map_err(|error| { - // `serde_path_to_error` renders a refusal it never attributed to a - // member as `.`, which names nothing an operator can look up, so a - // document refused whole is reported at the root every other - // whole-document refusal here already names. - let path = error.path().to_string(); - RuntimeConfigError::Parse { - path: if path == "." { "/".to_owned() } else { path }, - source: error.into_inner(), - } - })?; - config.check()?; - Ok(config) + let loaded = Self::loader().load::(path.as_ref())?; + loaded.config.check()?; + Ok(loaded.config) + } + + /// The shared runtime configuration loader under Casework's envelope and + /// removed keys. + #[must_use] + pub const fn loader() -> RuntimeConfigLoader { + RuntimeConfigLoader::new(CASEWORK_RUNTIME_ENVELOPE).removed_keys(CASEWORK_REMOVED_KEYS) } #[must_use] @@ -788,89 +625,167 @@ impl RuntimeConfig { if self.kind != RUNTIME_CONFIG_KIND { return Err(RuntimeConfigError::InvalidKind); } - if !self.package.root.is_absolute() { - return Err(RuntimeConfigError::RelativeOperatedPath("package.root")); - } + self.package.shared().check()?; + self.secret_providers.check()?; + self.audit.destination()?; + self.validate_secret_references()?; if self - .secret_providers - .file - .as_ref() - .is_some_and(|file| !file.root.is_absolute()) + .package + .acknowledge_stranded_work + .as_deref() + .is_some_and(|acknowledged| !is_sha256_label(acknowledged)) { - return Err(RuntimeConfigError::RelativeOperatedPath( - "secretProviders.file.root", - )); + return Err(RuntimeConfigError::InvalidStrandedWorkAcknowledgement); } - self.audit.destination()?; - if self.secret_providers.file.is_none() && self.secret_providers.environment.is_none() { - return Err(RuntimeConfigError::InvalidSecretProviders); + let package = self.capture_package_after_verification(|| {})?; + let project = package.project(); + if !self.listener.is_valid() { + return Err(RuntimeConfigError::InvalidListener); } - self.validate_secret_references()?; - let policy_path = self.policy_path(); - let project = CaseworkProject::load(&policy_path).map_err(RuntimeConfigError::Project)?; - let package_digest = verify_policy_package(&policy_path, &project) - .map_err(RuntimeConfigError::PolicyPackage)?; + if let Some(metrics_listener) = &self.metrics_listener { + metrics_listener.validate(&self.listener)?; + } + self.authentication.oidc.provider.check( + "authentication.oidc", + self.listener.tls_termination == TlsTermination::DevelopmentLoopback, + )?; + self.authentication + .oidc + .clients + .check("authentication.oidc")?; + // An empty client list admits every client the issuer verifies, so a + // deployment that simply forgot the field would accept a token minted + // for an unrelated application in the same realm. Development loopback + // keeps that convenience; a deployment behind an operator-controlled + // terminator must name the clients it admits. if self.listener.tls_termination == TlsTermination::OperatorControlledUpstream - && package_digest.is_none() + && self.authentication.oidc.clients.allowed_clients.is_empty() { - return Err(RuntimeConfigError::ProductionPolicyPackageRequired); - } - if let Some(expected) = &self.package.expected_policy_digest { - if !valid_policy_digest(expected) { - return Err(RuntimeConfigError::InvalidExpectedPolicyDigest); - } - if package_digest.as_deref() != Some(expected.as_str()) { - return Err(RuntimeConfigError::PolicyDigestMismatch { - expected: expected.clone(), - actual: package_digest, - }); - } - } - validate_project_source_inputs(&policy_path, &project)?; - let declared_sources = project - .sources - .iter() - .map(|source| source.id.as_str()) - .collect::>(); - let configured_sources = self - .sources - .keys() - .map(String::as_str) - .collect::>(); - if !valid_listener( - self.listener.bind.ip(), - self.listener.network_exposure, - self.listener.tls_termination, - ) { - return Err(RuntimeConfigError::InvalidListener); + return Err(RuntimeConfigError::AllowedClientsRequired); } - if self.authentication.oidc.issuer.is_empty() - || self.authentication.oidc.audience.is_empty() - || self.authentication.oidc.scope_claim.is_empty() + if self.authentication.oidc.scope_claim.is_empty() || self.authentication.oidc.human_identity.claim.is_empty() || self.authentication.oidc.human_identity.value.is_empty() || self.authentication.oidc.human_identity.claim == self.authentication.oidc.scope_claim - || project.access_profiles.iter().any(|profile| { - profile.principal_claim == self.authentication.oidc.human_identity.claim + { + return Err(RuntimeConfigError::InvalidOidc); + } + self.validate_project_bindings(project)?; + if self.database.runtime_url_ref.is_empty() || self.database.migration_url_ref.is_empty() { + return Err(RuntimeConfigError::InvalidDatabaseReference); + } + if self + .review_completion_destinations + .iter() + .any(|(id, destination)| { + id.is_empty() + || !valid_review_completion_url(&destination.url) + || !(MINIMUM_REVIEW_COMPLETION_TIMEOUT_MILLISECONDS + ..=MAXIMUM_REVIEW_COMPLETION_TIMEOUT_MILLISECONDS) + .contains(&destination.timeout_milliseconds) + || !(MINIMUM_REVIEW_COMPLETION_ATTEMPTS..=MAXIMUM_REVIEW_COMPLETION_ATTEMPTS) + .contains(&destination.maximum_attempts) + || !(MINIMUM_REVIEW_COMPLETION_RETRY_SECONDS + ..=MAXIMUM_REVIEW_COMPLETION_RETRY_SECONDS) + .contains(&destination.retry_seconds) }) + { + return Err(RuntimeConfigError::InvalidSourceBindings); + } + self.validate_source_bindings()?; + #[cfg(not(feature = "postgres-test"))] + if self.database.test_only_plaintext { + return Err(RuntimeConfigError::PlaintextDatabase); + } + Ok(()) + } + + /// Verify the package at `package.root` again and return its digest. + pub fn package_digest(&self) -> Result { + Ok(self.load_package()?.digest) + } + + /// Verify, validate, and capture the exact package bytes a consumer will use. + pub fn load_package(&self) -> Result { + self.load_package_after_verification(|| {}) + } + + fn load_package_after_verification( + &self, + after_verification: impl FnOnce(), + ) -> Result { + let package = self.capture_package_after_verification(after_verification)?; + self.validate_project_bindings(package.project())?; + Ok(package) + } + + fn capture_package_after_verification( + &self, + after_verification: impl FnOnce(), + ) -> Result { + let verified = verify_casework_envelope(&self.package)?; + after_verification(); + + let policy_bytes = capture_verified_file(&self.package.root, &verified, POLICY_FILE)?; + // Preserve the typed project diagnostic when malformed YAML also + // resembles an environment expression. Both checks use the same + // bytes captured from the verified package. + if let Err(error) = reject_authored_environment_expressions_bytes(&policy_bytes) { + if matches!(error, RuntimeConfigError::Load(_)) { + CaseworkProject::from_slice(&policy_bytes).map_err(RuntimeConfigError::Project)?; + } + return Err(error); + } + let project = + CaseworkProject::from_slice(&policy_bytes).map_err(RuntimeConfigError::Project)?; + verify_casework_contents(&project, &verified)?; + + let mut source_descriptions = BTreeMap::new(); + for source in &project.sources { + let bytes = + capture_verified_file(&self.package.root, &verified, source.description.as_str())?; + source_descriptions.insert(source.description.clone(), bytes); + } + validate_project_source_description_bytes(&project, &source_descriptions)?; + + Ok(LoadedCaseworkPackage { + digest: verified.digest().to_owned(), + project, + source_descriptions, + }) + } + + fn validate_project_bindings( + &self, + project: &CaseworkProject, + ) -> Result<(), RuntimeConfigError> { + if project + .access_profiles + .iter() + .any(|profile| profile.principal_claim == self.authentication.oidc.human_identity.claim) { return Err(RuntimeConfigError::InvalidOidc); } - self.validate_assertion_issuers()?; if let Some(authority) = &self.task_authority { if !registry_platform_httputil::valid_resource_uri(&authority.issuer) || !registry_platform_httputil::valid_resource_uri(&authority.exchange_audience) - || self.authentication.oidc.allowed_clients.is_empty() + || self.authentication.oidc.clients.allowed_clients.is_empty() || authority.status_clients.len() > 64 || authority.status_clients.iter().any(|(client, resource)| { - !self.authentication.oidc.allowed_clients.contains(client) + !self + .authentication + .oidc + .clients + .allowed_clients + .contains(client) || !registry_platform_httputil::valid_resource_uri(resource) }) || project.task_templates.iter().any(|template| { - template.agent.issuer != self.authentication.oidc.issuer + template.agent.issuer != self.authentication.oidc.provider.issuer || !self .authentication .oidc + .clients .allowed_clients .contains(&template.client) || !registry_platform_httputil::valid_resource_uri(&template.resource) @@ -881,12 +796,17 @@ impl RuntimeConfig { } else if !project.task_templates.is_empty() { return Err(RuntimeConfigError::InvalidOidc); } - if self.database.runtime_url_ref.is_empty() || self.database.migration_url_ref.is_empty() { - return Err(RuntimeConfigError::InvalidDatabaseReference); - } - if self.audit.hash_key_ref.is_empty() { - return Err(RuntimeConfigError::InvalidAuditReference); - } + + let declared_sources = project + .sources + .iter() + .map(|source| source.id.as_str()) + .collect::>(); + let configured_sources = self + .sources + .keys() + .map(String::as_str) + .collect::>(); if self.sources.keys().any(String::is_empty) || configured_sources != declared_sources { return Err(RuntimeConfigError::InvalidSourceBindings); } @@ -910,6 +830,7 @@ impl RuntimeConfig { }) { return Err(RuntimeConfigError::InactiveReviewSourceNamespace); } + let declared_destinations = project .review_producers .iter() @@ -921,58 +842,24 @@ impl RuntimeConfig { .keys() .map(String::as_str) .collect::>(); - if !declared_destinations.is_subset(&configured_destinations) - || self - .review_completion_destinations - .iter() - .any(|(id, destination)| { - id.is_empty() - || !valid_review_completion_url(&destination.url) - || !(MINIMUM_REVIEW_COMPLETION_TIMEOUT_MILLISECONDS - ..=MAXIMUM_REVIEW_COMPLETION_TIMEOUT_MILLISECONDS) - .contains(&destination.timeout_milliseconds) - || !(MINIMUM_REVIEW_COMPLETION_ATTEMPTS - ..=MAXIMUM_REVIEW_COMPLETION_ATTEMPTS) - .contains(&destination.maximum_attempts) - || !(MINIMUM_REVIEW_COMPLETION_RETRY_SECONDS - ..=MAXIMUM_REVIEW_COMPLETION_RETRY_SECONDS) - .contains(&destination.retry_seconds) - }) - { + if !declared_destinations.is_subset(&configured_destinations) { return Err(RuntimeConfigError::InvalidSourceBindings); } - self.validate_source_bindings()?; - #[cfg(not(feature = "postgres-test"))] - if self.database.test_only_plaintext { - return Err(RuntimeConfigError::PlaintextDatabase); - } Ok(()) } - /// Return the verified deployment policy identity, if this is a packaged - /// local-development configuration. Production configurations always have one. - pub fn policy_package_digest(&self) -> Result, RuntimeConfigError> { - let policy_path = self.policy_path(); - let project = CaseworkProject::load(&policy_path).map_err(RuntimeConfigError::Project)?; - verify_policy_package(&policy_path, &project).map_err(RuntimeConfigError::PolicyPackage) - } - fn validate_secret_references(&self) -> Result<(), RuntimeConfigError> { - let mut references = vec![ - ( - "database.runtimeUrlRef".to_owned(), - &self.database.runtime_url_ref, - ), - ( - "database.migrationUrlRef".to_owned(), - &self.database.migration_url_ref, - ), - ("audit.hashKeyRef".to_owned(), &self.audit.hash_key_ref), - ]; - if let Some(reference) = &self.database.trusted_root_certificate_ref { - references.push(("database.trustedRootCertificateRef".to_owned(), reference)); - } - if let OidcJwksSource::Static { document_ref } = &self.authentication.oidc.jwks_source { + let mut references: Vec<(String, &str)> = self + .database + .references() + .into_iter() + .map(|(field, reference)| (field.to_owned(), reference)) + .collect(); + references.push(( + "audit.hashKeyRef".to_owned(), + self.audit.key.hash_key_ref.as_str(), + )); + if let Some(document_ref) = self.authentication.oidc.provider.jwks_source.document_ref() { references.push(( "authentication.oidc.jwksSource.documentRef".to_owned(), document_ref, @@ -1018,46 +905,8 @@ impl RuntimeConfig { } } } - for (path, raw) in references { - let reference = SecretReference::parse(raw.clone()) - .map_err(|_| RuntimeConfigError::InvalidSecretReference { path: path.clone() })?; - let enabled = match reference.provider() { - SecretProvider::File => self.secret_providers.file.is_some(), - SecretProvider::Environment => self.secret_providers.environment.is_some(), - }; - if !enabled { - return Err(RuntimeConfigError::SecretProviderRequired { path }); - } - } - Ok(()) - } - - /// Refuse an assertion-issuer map with too many clients, an oversized - /// client key or issuer string, too many issuers listed for one client, or - /// a repeated issuer within one client's list. This runs at configuration - /// load, before any verifier is built, so an operator sees the refusal - /// without the runtime ever starting. - fn validate_assertion_issuers(&self) -> Result<(), RuntimeConfigError> { - let assertion_issuers = &self.authentication.oidc.assertion_issuers; - if assertion_issuers.len() > MAXIMUM_ASSERTION_ISSUER_CLIENTS { - return Err(RuntimeConfigError::InvalidOidc); - } - for (client, issuers) in assertion_issuers { - if client.is_empty() - || client.len() > MAXIMUM_ASSERTION_ISSUER_CLIENT_BYTES - || issuers.len() > MAXIMUM_ASSERTION_ISSUERS_PER_CLIENT - { - return Err(RuntimeConfigError::InvalidOidc); - } - let mut seen = BTreeSet::new(); - for issuer in issuers { - if issuer.is_empty() - || issuer.len() > MAXIMUM_ASSERTION_ISSUER_BYTES - || !seen.insert(issuer) - { - return Err(RuntimeConfigError::InvalidOidc); - } - } + for (field, raw) in references { + self.secret_providers.check_reference(&field, raw)?; } Ok(()) } @@ -1094,20 +943,21 @@ impl RuntimeConfig { &self, secrets: &SecretResolver, ) -> Result<(TokenVerifierConfig, std::sync::Arc), RuntimeConfigError> { - let discovery_config = OidcDiscoveryConfig { - issuer: self.authentication.oidc.issuer.clone(), - jwks_uri_override: self.authentication.oidc.jwks_uri.clone(), - discovery_timeout: Duration::from_secs(5), - max_doc_bytes: 1024 * 1024, - }; - let fetcher = match &self.authentication.oidc.jwks_source { - OidcJwksSource::Discovery => { + let fetcher = match &self.authentication.oidc.provider.jwks_source { + JwksSource::Uri { uri } => JwksFetcher::new(uri.clone(), JwksFetcherConfig::defaults()), + JwksSource::Discovery {} => { + let discovery_config = OidcDiscoveryConfig { + issuer: self.authentication.oidc.provider.issuer.clone(), + jwks_uri_override: None, + discovery_timeout: Duration::from_secs(5), + max_doc_bytes: 1024 * 1024, + }; let discovery = fetch_discovery(&discovery_config) .await .map_err(|_| RuntimeConfigError::Oidc)?; JwksFetcher::new(discovery.jwks_uri, JwksFetcherConfig::defaults()) } - OidcJwksSource::Static { document_ref } => { + JwksSource::Static { document_ref } => { let document = secrets.resolve(document_ref).map_err(|error| { RuntimeConfigError::OidcJwksSecret(describe_secret_failure( "authentication.oidc.jwksSource.documentRef", @@ -1115,7 +965,8 @@ impl RuntimeConfig { &error, )) })?; - let jwks = parse_static_jwks(document.expose_secret())?; + let jwks = parse_static_jwks(document.expose_secret()) + .map_err(|_| RuntimeConfigError::Oidc)?; JwksFetcher::new_static(jwks, JwksFetcherConfig::defaults()) } }; @@ -1134,14 +985,14 @@ impl RuntimeConfig { /// another purpose claim, draft, and act on casework. pub(crate) fn verifier_profile(&self) -> TokenVerifierConfig { TokenVerifierConfig::access_token_profile( - self.authentication.oidc.issuer.clone(), - vec![self.authentication.oidc.audience.clone()], + self.authentication.oidc.provider.issuer.clone(), + vec![self.authentication.oidc.provider.audience.clone()], vec![Algorithm::RS256, Algorithm::ES256], access_token_typ_set(CASEWORK_ACCESS_TOKEN_TYPE), ) .with_scope_claim(self.authentication.oidc.scope_claim.clone()) - .with_allowed_clients(self.authentication.oidc.allowed_clients.clone()) - .with_assertion_issuers(self.authentication.oidc.assertion_issuers.clone()) + .with_allowed_clients(self.authentication.oidc.clients.allowed_clients.clone()) + .with_assertion_issuers(self.authentication.oidc.clients.assertion_issuers.clone()) } } @@ -1163,11 +1014,10 @@ fn valid_review_completion_url(raw: &str) -> bool { } } -fn validate_project_source_inputs( - project_path: &Path, +fn validate_project_source_description_bytes( project: &CaseworkProject, + descriptions: &BTreeMap>, ) -> Result<(), RuntimeConfigError> { - let root = project_path.parent().ok_or(RuntimeConfigError::Invalid)?; let queues = project .queues .iter() @@ -1180,11 +1030,10 @@ fn validate_project_source_inputs( { return Err(RuntimeConfigError::SourceDescription); } - let relative = normalized_relative_path(&source.description) + let bytes = descriptions + .get(&source.description) .ok_or(RuntimeConfigError::SourceDescription)?; - let bytes = read_bounded_file(&root.join(relative), MAXIMUM_POLICY_PACKAGE_FILE_BYTES) - .map_err(|_| RuntimeConfigError::SourceDescription)?; - let metadata = validate_breg_source_description(source, &bytes) + let metadata = validate_breg_source_description(source, bytes) .map_err(|_| RuntimeConfigError::SourceDescription)?; for request in &source.requests { check_routing_policy( @@ -1204,60 +1053,30 @@ fn validate_project_source_inputs( Ok(()) } -fn valid_listener( - address: IpAddr, - exposure: ListenerNetworkExposure, - tls_termination: TlsTermination, -) -> bool { - if address.is_multicast() { - return false; - } - if tls_termination == TlsTermination::DevelopmentLoopback { - return exposure == ListenerNetworkExposure::PrivateAddress && address.is_loopback(); - } - match (address, exposure) { - (IpAddr::V4(address), ListenerNetworkExposure::PrivateAddress) => { - address.is_loopback() || address.is_private() - } - (IpAddr::V6(address), ListenerNetworkExposure::PrivateAddress) => { - address.is_loopback() || is_unique_local(address) - } - (IpAddr::V4(address), ListenerNetworkExposure::ContainerPrivate) => { - address.is_unspecified() || address.is_loopback() || address.is_private() - } - (IpAddr::V6(address), ListenerNetworkExposure::ContainerPrivate) => { - address.is_unspecified() || address.is_loopback() || is_unique_local(address) +fn reject_authored_environment_expressions_bytes(bytes: &[u8]) -> Result<(), RuntimeConfigError> { + let text = String::from_utf8_lossy(bytes); + reject_environment_expressions_in_authored_yaml(&text).map_err(|error| { + if error.kind() == RuntimeConfigErrorKind::AuthoredSyntax { + RuntimeConfigError::Load(error) + } else { + RuntimeConfigError::PolicyEnvironmentExpression { + field: error.field().to_owned(), + } } - } + }) } fn is_unique_local(address: Ipv6Addr) -> bool { address.octets()[0] & 0xfe == 0xfc } -fn parse_static_jwks(bytes: &[u8]) -> Result { - let jwks: JwkSet = serde_json::from_slice(bytes).map_err(|_| RuntimeConfigError::Oidc)?; - let mut kids = BTreeSet::new(); - if jwks.keys.is_empty() - || jwks.keys.iter().any(|key| { - !matches!( - key.algorithm, - AlgorithmParameters::RSA(_) | AlgorithmParameters::EllipticCurve(_) - ) || key - .common - .key_id - .as_ref() - .is_none_or(|kid| kid.is_empty() || !kids.insert(kid.clone())) - }) - { - return Err(RuntimeConfigError::Oidc); - } - Ok(jwks) -} - #[cfg(test)] mod tests { use super::*; + use registry_platform_config::{ + MAX_ASSERTION_ISSUERS_PER_CLIENT, MAX_ASSERTION_ISSUER_BYTES, MAX_ASSERTION_ISSUER_CLIENTS, + MAX_ASSERTION_ISSUER_CLIENT_BYTES, SUM_FILE, + }; use registry_platform_oidc::is_access_token_typ_pair; #[test] @@ -1351,25 +1170,25 @@ reviewProducers: } "#; - fn write_package_with_project(root: &Path, project: &str) -> PolicyPackageManifest { + fn write_package_with_project(root: &Path, project: &str) -> VerifiedPackage { std::fs::create_dir_all(root.join("sources")).unwrap(); std::fs::write(root.join("casework.yaml"), project).unwrap(); std::fs::write(root.join("sources/professional.json"), SOURCE_DESCRIPTION).unwrap(); - let manifest = PolicyPackageManifest::build([ - ("casework.yaml".to_owned(), project.as_bytes().to_vec()), - ( - "sources/professional.json".to_owned(), - SOURCE_DESCRIPTION.as_bytes().to_vec(), - ), - ]) - .unwrap(); - let mut bytes = serde_json::to_vec_pretty(&manifest).unwrap(); - bytes.push(b'\n'); - std::fs::write(root.join(POLICY_PACKAGE_MANIFEST_FILE), bytes).unwrap(); - manifest + registry_platform_config::write_sum_file(root, None, &package_limits(), PACKAGE_COMMAND) + .unwrap() + } + + fn canonical_tempdir() -> tempfile::TempDir { + tempfile::tempdir_in(std::fs::canonicalize(std::env::temp_dir()).unwrap()).unwrap() } - fn write_package(root: &Path) -> PolicyPackageManifest { + fn write_operator(root: &Path, document: &serde_json::Value) -> PathBuf { + let operator = root.join("operator.yaml"); + std::fs::write(&operator, serde_norway::to_string(document).unwrap()).unwrap(); + operator + } + + fn write_package(root: &Path) -> VerifiedPackage { write_package_with_project(root, SOURCE_PROJECT) } @@ -1377,9 +1196,11 @@ reviewProducers: serde_norway::to_string(&operator_value(package, tls)).unwrap() } + /// A production fixture names the one client it admits, as a production + /// deployment must; a loopback fixture leaves the list empty. fn operator_value(package: &Path, tls: &str) -> serde_json::Value { let root = package.parent().expect("package parent"); - serde_json::json!({ + let mut document = serde_json::json!({ "apiVersion": RUNTIME_CONFIG_API_VERSION, "kind": RUNTIME_CONFIG_KIND, "package": {"root": package}, @@ -1403,103 +1224,348 @@ reviewProducers: "webhookSecretRef": "secret:file/webhook", "eventSource": "urn:registrystack:registry:professional:instance:pilot" }} - }) + }); + if tls == "operator-controlled-upstream" { + document["authentication"]["oidc"]["allowedClients"] = + serde_json::json!(["casework-console"]); + } + document + } + + /// The commented alternative in the operator example must be loadable + /// exactly as written, and its refusal must name the reference an operator + /// has to go and fix. + #[cfg(unix)] + #[tokio::test] + async fn a_static_jwks_source_loads_and_names_its_reference_when_refused() { + use std::os::unix::fs::PermissionsExt as _; + + let root = canonical_tempdir(); + let package = root.path().join("package"); + std::fs::create_dir(&package).unwrap(); + write_package(&package); + let secrets_root = root.path().join("secrets"); + std::fs::create_dir(&secrets_root).unwrap(); + std::fs::write( + secrets_root.join("jwks.json"), + br#"{"keys":[{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"}]}"#, + ) + .unwrap(); + std::fs::set_permissions( + secrets_root.join("jwks.json"), + std::fs::Permissions::from_mode(0o644), + ) + .unwrap(); + + let mut document = operator_value(&package, "operator-controlled-upstream"); + document["authentication"]["oidc"]["jwksSource"] = + serde_json::json!({"kind": "static", "documentRef": "secret:file/jwks.json"}); + let operator = root.path().join("operator.yaml"); + std::fs::write(&operator, serde_norway::to_string(&document).unwrap()).unwrap(); + + let mut config = RuntimeConfig::load(&operator).expect("static JWKS source is accepted"); + assert!(matches!( + &config.authentication.oidc.provider.jwks_source, + JwksSource::Static { document_ref } if document_ref == "secret:file/jwks.json" + )); + + let secrets = SecretResolver::new( + [registry_platform_config::SecretProvider::File], + &secrets_root, + ) + .unwrap(); + let message = config + .oidc_verifier(&secrets) + .await + .map(|_| ()) + .expect_err("a group-readable JWKS document is refused") + .to_string(); + assert!( + message.contains("secret:file/jwks.json") && message.contains("0400 or 0600"), + "the failure does not name the reference and the mode rule: {message}" + ); + + let literal_secret = "literal-jwks-credential-canary"; + let JwksSource::Static { document_ref } = + &mut config.authentication.oidc.provider.jwks_source + else { + panic!("configured static JWKS source changed kind") + }; + *document_ref = literal_secret.to_owned(); + let message = config + .oidc_verifier(&secrets) + .await + .map(|_| ()) + .expect_err("a literal credential is not a secret reference") + .to_string(); + assert!( + message.contains("authentication.oidc.jwksSource.documentRef") + && message.contains("secret:env/NAME or secret:file/name"), + "the failure does not name the field and reference grammar: {message}" + ); + assert!( + !message.contains(literal_secret), + "the failure renders the literal credential: {message}" + ); + } + + /// The runtime's static JWKS arm hands the resolved document to the + /// shared parser and refuses a key set it cannot trust before any + /// verifier exists: a symmetric key, an unnamed key, two keys sharing a + /// name, or no key at all. + #[cfg(unix)] + #[tokio::test] + async fn static_jwks_requires_unique_named_asymmetric_keys() { + use std::os::unix::fs::PermissionsExt as _; + + let root = canonical_tempdir(); + let package = root.path().join("package"); + std::fs::create_dir(&package).unwrap(); + write_package(&package); + let secrets_root = root.path().join("secrets"); + std::fs::create_dir(&secrets_root).unwrap(); + let mut document = operator_value(&package, "development-loopback"); + document["authentication"]["oidc"]["jwksSource"] = + serde_json::json!({"kind": "static", "documentRef": "secret:file/jwks.json"}); + let config = RuntimeConfig::load(write_operator(root.path(), &document)).unwrap(); + let secrets = config.secret_providers.resolver().unwrap(); + let jwks = secrets_root.join("jwks.json"); + + let write_jwks = |bytes: &[u8]| { + std::fs::write(&jwks, bytes).unwrap(); + std::fs::set_permissions(&jwks, std::fs::Permissions::from_mode(0o600)).unwrap(); + }; + + write_jwks(br#"{"keys":[{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"}]}"#); + config + .oidc_verifier(&secrets) + .await + .expect("one named asymmetric key is accepted"); + + for (case, invalid) in [ + ("symmetric", br#"{"keys":[{"kty":"oct","kid":"one","k":"AA"}]}"#.as_slice()), + ("unnamed", br#"{"keys":[{"kty":"RSA","n":"AQAB","e":"AQAB"}]}"#), + ( + "duplicate name", + br#"{"keys":[{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"},{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"}]}"#, + ), + ("empty", br#"{"keys":[]}"#), + ] { + write_jwks(invalid); + let error = config + .oidc_verifier(&secrets) + .await + .map(|_| ()) + .expect_err(case); + assert!( + matches!(error, RuntimeConfigError::Oidc), + "{case}: {error}" + ); + assert_eq!(error.path(), "authentication.oidc", "{case}"); + } + } + + #[test] + fn the_runtime_configuration_is_read_through_the_shared_loader() { + let error = RuntimeConfig::load("runtime.yaml").unwrap_err(); + assert!(error.to_string().contains("absolute"), "{error}"); + + let root = canonical_tempdir(); + let package = root.path().join("package"); + std::fs::create_dir(&package).unwrap(); + write_package(&package); + let mut document = operator_value(&package, "development-loopback"); + document["listener"].as_object_mut().unwrap().remove("bind"); + let error = RuntimeConfig::load(write_operator(root.path(), &document)).unwrap_err(); + assert_eq!(error.path(), "listener"); + assert!(error.to_string().contains("bind"), "{error}"); + } + + #[test] + fn a_removed_jwks_uri_names_its_replacement() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + std::fs::create_dir(&package).unwrap(); + write_package(&package); + let mut document = operator_value(&package, "development-loopback"); + document["authentication"]["oidc"]["jwksUri"] = + serde_json::json!("https://identity.example.test/jwks"); + let error = RuntimeConfig::load(write_operator(root.path(), &document)).unwrap_err(); + assert_eq!(error.path(), "authentication.oidc.jwksUri"); + assert!( + error.to_string().contains("authentication.oidc.jwksSource"), + "{error}" + ); + + let mut document = operator_value(&package, "development-loopback"); + document["authentication"]["oidc"]["jwksSource"] = + serde_json::json!({"kind": "uri", "uri": "https://identity.example.test/jwks"}); + let config = RuntimeConfig::load(write_operator(root.path(), &document)).unwrap(); + assert_eq!( + config.authentication.oidc.provider.jwks_source.uri(), + Some("https://identity.example.test/jwks") + ); + } + + #[test] + fn the_oidc_issuer_and_clients_are_the_shared_blocks() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + std::fs::create_dir(&package).unwrap(); + write_package(&package); + + let mut document = operator_value(&package, "operator-controlled-upstream"); + document["authentication"]["oidc"]["issuer"] = + serde_json::json!("http://identity.example.test"); + let error = RuntimeConfig::load(write_operator(root.path(), &document)).unwrap_err(); + assert_eq!(error.path(), "authentication.oidc.issuer"); + + let mut document = operator_value(&package, "operator-controlled-upstream"); + document["authentication"]["oidc"]["issuerr"] = + serde_json::json!("https://identity.example.test"); + let error = RuntimeConfig::load(write_operator(root.path(), &document)).unwrap_err(); + assert!(error.to_string().contains("issuerr"), "{error}"); + } + + #[test] + fn environment_expressions_substitute_values_but_never_secret_references() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + std::fs::create_dir(&package).unwrap(); + write_package(&package); + let mut document = operator_value(&package, "development-loopback"); + document["authentication"]["oidc"]["audience"] = + serde_json::json!("${CASEWORK_TEST_AUDIENCE:-urn:example:substituted}"); + let config = RuntimeConfig::load(write_operator(root.path(), &document)).unwrap(); + assert_eq!( + config.authentication.oidc.provider.audience, + "urn:example:substituted" + ); + + for (pointer, field) in [ + ("/database/runtimeUrlRef", "database.runtimeUrlRef"), + ("/audit/hashKeyRef", "audit.hashKeyRef"), + ] { + let mut document = operator_value(&package, "development-loopback"); + *document.pointer_mut(pointer).unwrap() = + serde_json::json!("${CASEWORK_TEST_REFERENCE:-secret:env/RUNTIME}"); + let error = RuntimeConfig::load(write_operator(root.path(), &document)).unwrap_err(); + assert!( + matches!( + &error, + RuntimeConfigError::Load(load) + if load.code() == "runtime_config.substitution_in_reference" + ), + "{error}" + ); + assert_eq!(error.path(), field); + } + } + + #[test] + fn an_authored_project_carrying_an_environment_expression_is_refused() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + std::fs::create_dir(&package).unwrap(); + write_package_with_project( + &package, + &SOURCE_PROJECT.replace("label: Review", "label: \"${QUEUE_LABEL}\""), + ); + let operator = write_operator( + root.path(), + &operator_value(&package, "development-loopback"), + ); + let error = RuntimeConfig::load(&operator).unwrap_err(); + assert!( + matches!( + &error, + RuntimeConfigError::PolicyEnvironmentExpression { field } + if field == "queues.0.label" + ), + "{error}" + ); + assert_eq!(error.path(), "package.root/casework.yaml"); + assert!(error.to_string().contains("runtime.yaml only"), "{error}"); } + /// A typed field holding an expression would otherwise fail the typed + /// project read first, with a type error that quotes the expression. #[test] - fn static_jwks_requires_unique_named_asymmetric_keys() { - assert!(parse_static_jwks( - br#"{"keys":[{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"}]}"# - ) - .is_ok()); - for invalid in [br#"{}"#.as_slice(),br#"{"keys":[]}"#,br#"{"keys":[{"kty":"oct","kid":"one","k":"AA"}]}"#,br#"{"keys":[{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"},{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"}]}"#,b"not-json"] { - assert!(parse_static_jwks(invalid).is_err()); - } + fn an_environment_expression_in_a_non_string_field_is_the_authored_refusal() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + std::fs::create_dir(&package).unwrap(); + let project = SOURCE_CONTEXT_REVIEW_PROJECT + .replace("recoveryDays: 7", "recoveryDays: ${RECOVERY_DAYS}"); + assert_ne!(project, SOURCE_CONTEXT_REVIEW_PROJECT); + write_package_with_project(&package, &project); + let operator = write_operator( + root.path(), + &operator_value(&package, "development-loopback"), + ); + let error = RuntimeConfig::load(&operator).unwrap_err(); + assert!( + matches!( + &error, + RuntimeConfigError::PolicyEnvironmentExpression { field } + if field == "reviewProducers.0.recoveryDays" + ), + "{error}" + ); + assert!(!error.to_string().contains("RECOVERY_DAYS"), "{error}"); } - /// The commented alternative in the operator example must be loadable - /// exactly as written, and its refusal must name the reference an operator - /// has to go and fix. - #[cfg(unix)] - #[tokio::test] - async fn a_static_jwks_source_loads_and_names_its_reference_when_refused() { - use std::os::unix::fs::PermissionsExt as _; - - let root = tempfile::tempdir().unwrap(); + #[test] + fn production_names_its_allowed_clients_while_loopback_may_leave_them_empty() { + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); - let secrets_root = root.path().join("secrets"); - std::fs::create_dir(&secrets_root).unwrap(); - std::fs::write( - secrets_root.join("jwks.json"), - br#"{"keys":[{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"}]}"#, - ) - .unwrap(); - std::fs::set_permissions( - secrets_root.join("jwks.json"), - std::fs::Permissions::from_mode(0o644), - ) - .unwrap(); let mut document = operator_value(&package, "operator-controlled-upstream"); - document["authentication"]["oidc"]["jwksSource"] = - serde_json::json!({"kind": "static", "documentRef": "secret:file/jwks.json"}); - let operator = root.path().join("operator.yaml"); - std::fs::write(&operator, serde_norway::to_string(&document).unwrap()).unwrap(); - - let mut config = RuntimeConfig::load(&operator).expect("static JWKS source is accepted"); - assert!(matches!( - &config.authentication.oidc.jwks_source, - OidcJwksSource::Static { document_ref } if document_ref == "secret:file/jwks.json" - )); - - let secrets = SecretResolver::new( - [registry_platform_config::SecretProvider::File], - &secrets_root, - ) - .unwrap(); - let message = config - .oidc_verifier(&secrets) - .await - .map(|_| ()) - .expect_err("a group-readable JWKS document is refused") - .to_string(); + document["authentication"]["oidc"]["allowedClients"] = serde_json::json!([]); + let error = RuntimeConfig::load(write_operator(root.path(), &document)).unwrap_err(); assert!( - message.contains("secret:file/jwks.json") && message.contains("0400 or 0600"), - "the failure does not name the reference and the mode rule: {message}" + matches!(error, RuntimeConfigError::AllowedClientsRequired), + "{error}" ); - - let literal_secret = "literal-jwks-credential-canary"; - let OidcJwksSource::Static { document_ref } = &mut config.authentication.oidc.jwks_source - else { - panic!("configured static JWKS source changed kind") - }; - *document_ref = literal_secret.to_owned(); - let message = config - .oidc_verifier(&secrets) - .await - .map(|_| ()) - .expect_err("a literal credential is not a secret reference") - .to_string(); + assert_eq!(error.path(), "authentication.oidc.allowedClients"); assert!( - message.contains("authentication.oidc.jwksSource.documentRef") - && message.contains("secret:env/NAME or secret:file/name"), - "the failure does not name the field and reference grammar: {message}" + error.to_string().contains("operator-controlled-upstream"), + "{error}" ); - assert!( - !message.contains(literal_secret), - "the failure renders the literal credential: {message}" + + document["authentication"]["oidc"]["allowedClients"] = + serde_json::json!(["casework-console"]); + let config = RuntimeConfig::load(write_operator(root.path(), &document)) + .expect("a named client list is accepted"); + assert_eq!( + config.authentication.oidc.clients.allowed_clients, + ["casework-console"] ); + assert!(config.task_authority.is_none()); + + let config = RuntimeConfig::load(write_operator( + root.path(), + &operator_value(&package, "development-loopback"), + )) + .expect("development loopback keeps an empty client list"); + assert!(config + .authentication + .oidc + .clients + .allowed_clients + .is_empty()); + assert!(config.task_authority.is_none()); } #[test] fn static_source_uses_document_ref_camel_case() { - let source: OidcJwksSource = + let source: JwksSource = serde_json::from_str(r#"{"kind":"static","documentRef":"secret:file/keys.json"}"#) .expect("static source"); assert!( - matches!(source,OidcJwksSource::Static{document_ref} if document_ref=="secret:file/keys.json") + matches!(source,JwksSource::Static{document_ref} if document_ref=="secret:file/keys.json") ); } @@ -1542,7 +1608,7 @@ reviewProducers: async fn built_verifier(assertion_issuers: Option) -> TokenVerifierConfig { use std::os::unix::fs::PermissionsExt as _; - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); @@ -1605,7 +1671,7 @@ reviewProducers: #[test] fn a_duplicate_issuer_within_one_clients_assertion_issuer_list_is_refused() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); @@ -1620,13 +1686,13 @@ reviewProducers: std::fs::write(&operator, serde_norway::to_string(&document).unwrap()).unwrap(); assert!(matches!( RuntimeConfig::load(&operator), - Err(RuntimeConfigError::InvalidOidc) + Err(RuntimeConfigError::Block(error)) if error.field() == "authentication.oidc.assertionIssuers" )); } #[test] fn an_empty_assertion_issuer_string_is_refused() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); @@ -1638,13 +1704,13 @@ reviewProducers: std::fs::write(&operator, serde_norway::to_string(&document).unwrap()).unwrap(); assert!(matches!( RuntimeConfig::load(&operator), - Err(RuntimeConfigError::InvalidOidc) + Err(RuntimeConfigError::Block(error)) if error.field() == "authentication.oidc.assertionIssuers" )); } #[test] fn an_empty_assertion_issuer_client_key_is_refused() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); @@ -1656,32 +1722,32 @@ reviewProducers: std::fs::write(&operator, serde_norway::to_string(&document).unwrap()).unwrap(); assert!(matches!( RuntimeConfig::load(&operator), - Err(RuntimeConfigError::InvalidOidc) + Err(RuntimeConfigError::Block(error)) if error.field() == "authentication.oidc.assertionIssuers" )); } #[test] fn an_over_long_assertion_issuer_client_key_is_refused() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); let operator = root.path().join("runtime.yaml"); let mut document = operator_value(&package, "development-loopback"); - let client = "a".repeat(MAXIMUM_ASSERTION_ISSUER_CLIENT_BYTES + 1); + let client = "a".repeat(MAX_ASSERTION_ISSUER_CLIENT_BYTES + 1); document["authentication"]["oidc"]["assertionIssuers"] = serde_json::json!({ client: ["https://exchange.example.test"] }); std::fs::write(&operator, serde_norway::to_string(&document).unwrap()).unwrap(); assert!(matches!( RuntimeConfig::load(&operator), - Err(RuntimeConfigError::InvalidOidc) + Err(RuntimeConfigError::Block(error)) if error.field() == "authentication.oidc.assertionIssuers" )); } #[test] fn an_over_long_assertion_issuer_value_is_refused() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); @@ -1689,7 +1755,7 @@ reviewProducers: let mut document = operator_value(&package, "development-loopback"); let issuer = format!( "https://{}.example.test", - "a".repeat(MAXIMUM_ASSERTION_ISSUER_BYTES) + "a".repeat(MAX_ASSERTION_ISSUER_BYTES) ); document["authentication"]["oidc"]["assertionIssuers"] = serde_json::json!({ "task-agent": [issuer] @@ -1697,20 +1763,20 @@ reviewProducers: std::fs::write(&operator, serde_norway::to_string(&document).unwrap()).unwrap(); assert!(matches!( RuntimeConfig::load(&operator), - Err(RuntimeConfigError::InvalidOidc) + Err(RuntimeConfigError::Block(error)) if error.field() == "authentication.oidc.assertionIssuers" )); } #[test] fn too_many_assertion_issuer_clients_is_refused() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); let operator = root.path().join("runtime.yaml"); let mut document = operator_value(&package, "development-loopback"); let mut assertion_issuers = serde_json::Map::new(); - for index in 0..=MAXIMUM_ASSERTION_ISSUER_CLIENTS { + for index in 0..=MAX_ASSERTION_ISSUER_CLIENTS { assertion_issuers.insert( format!("task-agent-{index}"), serde_json::json!(["https://exchange.example.test"]), @@ -1721,19 +1787,19 @@ reviewProducers: std::fs::write(&operator, serde_norway::to_string(&document).unwrap()).unwrap(); assert!(matches!( RuntimeConfig::load(&operator), - Err(RuntimeConfigError::InvalidOidc) + Err(RuntimeConfigError::Block(error)) if error.field() == "authentication.oidc.assertionIssuers" )); } #[test] fn too_many_issuers_for_one_assertion_issuer_client_is_refused() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); let operator = root.path().join("runtime.yaml"); let mut document = operator_value(&package, "development-loopback"); - let issuers: Vec = (0..=MAXIMUM_ASSERTION_ISSUERS_PER_CLIENT) + let issuers: Vec = (0..=MAX_ASSERTION_ISSUERS_PER_CLIENT) .map(|index| format!("https://exchange-{index}.example.test")) .collect(); document["authentication"]["oidc"]["assertionIssuers"] = serde_json::json!({ @@ -1742,73 +1808,231 @@ reviewProducers: std::fs::write(&operator, serde_norway::to_string(&document).unwrap()).unwrap(); assert!(matches!( RuntimeConfig::load(&operator), - Err(RuntimeConfigError::InvalidOidc) + Err(RuntimeConfigError::Block(error)) if error.field() == "authentication.oidc.assertionIssuers" )); } + fn packaged_operator(root: &Path, tls: &str) -> (PathBuf, PathBuf, VerifiedPackage) { + let package = root.join("package"); + std::fs::create_dir(&package).unwrap(); + let verified = write_package(&package); + let operator = write_operator(root, &operator_value(&package, tls)); + (package, operator, verified) + } + #[test] - fn package_identity_covers_exact_policy_and_imported_inputs() { - let package = tempfile::tempdir().unwrap(); - let manifest = write_package(package.path()); - let project = CaseworkProject::load(package.path().join("casework.yaml")).unwrap(); - assert_eq!( - verify_policy_package(&package.path().join("casework.yaml"), &project).unwrap(), - Some(manifest.policy_digest) + fn a_packaged_project_is_verified_against_its_sum_file() { + let root = canonical_tempdir(); + let (package, operator, verified) = + packaged_operator(root.path(), "operator-controlled-upstream"); + let config = RuntimeConfig::load(&operator).unwrap(); + assert_eq!(config.package_digest().unwrap(), verified.digest()); + + std::fs::write(package.join("sources/professional.json"), b"changed").unwrap(); + let error = RuntimeConfig::load(&operator).expect_err("a changed file is refused"); + assert_eq!(error.path(), "package.root"); + let message = error.to_string(); + assert!( + message.contains("changed: sources/professional.json"), + "{message}" ); + assert!(message.contains(PACKAGE_COMMAND), "{message}"); + } + + #[test] + fn final_package_load_binds_captured_policy_and_source_bytes_to_the_verified_digest() { + for (changed_path, replacement) in [ + ("casework.yaml", b"changed policy\n".as_slice()), + ( + "sources/professional.json", + b"{\"changed\":true}\n".as_slice(), + ), + ] { + let root = canonical_tempdir(); + let (package, operator, _) = + packaged_operator(root.path(), "operator-controlled-upstream"); + let config = RuntimeConfig::load(&operator).expect("the original package loads"); + + let error = config + .load_package_after_verification(|| { + std::fs::write(package.join(changed_path), replacement).unwrap(); + }) + .expect_err("bytes replaced after verification are refused"); + assert!( + matches!( + error, + RuntimeConfigError::PackageFileChanged { ref path } if path == changed_path + ), + "{changed_path}: {error:?}" + ); + } + } + + #[test] + fn final_unpinned_package_load_revalidates_project_bindings_after_replacement() { + let root = canonical_tempdir(); + let (package, operator, _) = packaged_operator(root.path(), "operator-controlled-upstream"); + let config = RuntimeConfig::load(&operator).expect("package A is valid"); + + std::fs::remove_file(package.join(SUM_FILE)).unwrap(); + let replacement = + SOURCE_PROJECT.replace("principalClaim: sub", "principalClaim: registry_actor_kind"); + std::fs::write(package.join(POLICY_FILE), replacement).unwrap(); + registry_platform_config::write_sum_file( + &package, + None, + &package_limits(), + PACKAGE_COMMAND, + ) + .unwrap(); + + let error = config + .load_package() + .expect_err("package B conflicts with the configured human identity claim"); + assert!(matches!(error, RuntimeConfigError::InvalidOidc)); + } + + #[test] + fn a_missing_or_extra_package_file_is_refused_by_name() { + let root = canonical_tempdir(); + let (package, operator, _) = packaged_operator(root.path(), "operator-controlled-upstream"); + std::fs::write(package.join("sources/stale.json"), b"stale").unwrap(); + let message = RuntimeConfig::load(&operator) + .expect_err("an extra file is refused") + .to_string(); + assert!(message.contains("extra: sources/stale.json"), "{message}"); + std::fs::remove_file(package.join("sources/stale.json")).unwrap(); - std::fs::write(package.path().join("sources/stale.json"), b"stale").unwrap(); - assert!(verify_policy_package(&package.path().join("casework.yaml"), &project).is_err()); - std::fs::remove_file(package.path().join("sources/stale.json")).unwrap(); - std::fs::write(package.path().join("sources/professional.json"), b"changed").unwrap(); - assert!(verify_policy_package(&package.path().join("casework.yaml"), &project).is_err()); + std::fs::remove_file(package.join("sources/professional.json")).unwrap(); + let message = RuntimeConfig::load(&operator) + .expect_err("a missing file is refused") + .to_string(); + assert!( + message.contains("missing: sources/professional.json"), + "{message}" + ); } #[test] - fn production_requires_a_verified_package_while_loopback_accepts_authoring() { - let root = tempfile::tempdir().unwrap(); + fn a_package_holds_exactly_the_policy_and_its_source_descriptions() { + let root = canonical_tempdir(); let package = root.path().join("package"); - std::fs::create_dir(&package).unwrap(); - write_package(&package); - let operator = root.path().join("operator.yaml"); + std::fs::create_dir_all(package.join("sources")).unwrap(); + std::fs::write(package.join("casework.yaml"), SOURCE_PROJECT).unwrap(); std::fs::write( - &operator, - operator_document(&package, "operator-controlled-upstream"), + package.join("sources/professional.json"), + SOURCE_DESCRIPTION, ) .unwrap(); - let config = RuntimeConfig::load(&operator).unwrap(); - assert!(config.policy_package_digest().unwrap().is_some()); - - std::fs::remove_file(package.join(POLICY_PACKAGE_MANIFEST_FILE)).unwrap(); + std::fs::write(package.join("notes.txt"), b"not an input").unwrap(); + registry_platform_config::write_sum_file( + &package, + None, + &package_limits(), + PACKAGE_COMMAND, + ) + .unwrap(); + let operator = write_operator( + root.path(), + &operator_value(&package, "operator-controlled-upstream"), + ); + let error = RuntimeConfig::load(&operator).expect_err("a stray input is refused"); assert!(matches!( - RuntimeConfig::load(&operator), - Err(RuntimeConfigError::ProductionPolicyPackageRequired) + &error, + RuntimeConfigError::PackageContents { missing, extra } + if missing.is_empty() && extra == &["notes.txt".to_owned()] )); + assert!(error.to_string().contains("extra: notes.txt"), "{error}"); + } + + #[test] + fn a_retired_package_manifest_is_refused_with_its_replacement() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + std::fs::create_dir(&package).unwrap(); + std::fs::create_dir_all(package.join("sources")).unwrap(); + std::fs::write(package.join("casework.yaml"), SOURCE_PROJECT).unwrap(); std::fs::write( - &operator, - operator_document(&package, "development-loopback"), + package.join("sources/professional.json"), + SOURCE_DESCRIPTION, ) .unwrap(); - assert!(RuntimeConfig::load(&operator).is_ok()); + std::fs::write(package.join(RETIRED_PACKAGE_MANIFEST_FILE), b"{}").unwrap(); + let operator = write_operator( + root.path(), + &operator_value(&package, "development-loopback"), + ); + let error = RuntimeConfig::load(&operator).expect_err("the retired manifest is refused"); + assert!(matches!(error, RuntimeConfigError::RetiredPackageManifest)); + assert!(error.to_string().contains(PACKAGE_COMMAND), "{error}"); } #[test] - fn an_expected_policy_digest_admits_only_the_package_it_names() { - let root = tempfile::tempdir().unwrap(); - let package = root.path().join("package"); - std::fs::create_dir(&package).unwrap(); - let manifest = write_package(&package); - let operator = root.path().join("operator.yaml"); + fn every_listener_mode_verifies_the_package_without_a_pin() { + for tls in ["operator-controlled-upstream", "development-loopback"] { + let root = canonical_tempdir(); + let (package, operator, verified) = packaged_operator(root.path(), tls); + let config = RuntimeConfig::load(&operator).expect("the package is admitted"); + assert!(config.package.expected_digest.is_none()); + assert_eq!(config.package_digest().unwrap(), verified.digest()); + + let refusal = |expected: &str| { + let error = RuntimeConfig::load(&operator).expect_err(expected); + assert_eq!(error.path(), "package.root", "{tls}: {error}"); + let message = error.to_string(); + assert!(message.contains(expected), "{tls}: {message}"); + assert!(message.contains(PACKAGE_COMMAND), "{tls}: {message}"); + }; + + std::fs::write( + package.join("casework.yaml"), + format!("{SOURCE_PROJECT}\n# changed\n"), + ) + .unwrap(); + refusal("changed: casework.yaml"); + std::fs::write(package.join("casework.yaml"), SOURCE_PROJECT).unwrap(); + + std::fs::write(package.join("sources/stale.json"), b"stale").unwrap(); + refusal("extra: sources/stale.json"); + std::fs::remove_file(package.join("sources/stale.json")).unwrap(); + + std::fs::remove_file(package.join("sources/professional.json")).unwrap(); + refusal("missing: sources/professional.json"); + std::fs::write( + package.join("sources/professional.json"), + SOURCE_DESCRIPTION, + ) + .unwrap(); + assert!(RuntimeConfig::load(&operator).is_ok(), "{tls}"); + + std::fs::remove_file(package.join(SUM_FILE)).unwrap(); + let error = RuntimeConfig::load(&operator).expect_err("an authored project is refused"); + assert!(matches!( + &error, + RuntimeConfigError::Package(error) + if matches!(error.kind(), PackageErrorKind::SumFileMissing) + )); + assert!(error.to_string().contains("has no SHA256SUMS"), "{error}"); + assert!(error.to_string().contains(PACKAGE_COMMAND), "{error}"); + } + } + + #[test] + fn a_package_digest_mismatch_is_refused_in_the_shared_shape() { + let root = canonical_tempdir(); + let (package, operator, verified) = + packaged_operator(root.path(), "operator-controlled-upstream"); let write = |expected: &str| { let mut document = operator_value(&package, "operator-controlled-upstream"); - document["package"]["expectedPolicyDigest"] = serde_json::json!(expected); - std::fs::write(&operator, serde_norway::to_string(&document).unwrap()).unwrap(); + document["package"]["expectedDigest"] = serde_json::json!(expected); + write_operator(root.path(), &document); }; - write(&manifest.policy_digest); + write(verified.digest()); let config = RuntimeConfig::load(&operator).expect("the pinned package is admitted"); assert_eq!( - config.package.expected_policy_digest.as_deref(), - Some(manifest.policy_digest.as_str()) + config.package.expected_digest.as_deref(), + Some(verified.digest()) ); let pinned = format!("sha256:{}", "0".repeat(64)); @@ -1816,69 +2040,115 @@ reviewProducers: let error = RuntimeConfig::load(&operator).expect_err("a different package is refused"); assert!(matches!( &error, - RuntimeConfigError::PolicyDigestMismatch { expected, actual } - if expected == &pinned && actual.as_deref() == Some(manifest.policy_digest.as_str()) + RuntimeConfigError::PackageDigest(PackageDigestMismatch { expected, found }) + if expected == &pinned && found == verified.digest() )); - assert_eq!(error.path(), "package.expectedPolicyDigest"); - let message = error.to_string(); - assert!(message.contains(&pinned), "{message}"); - assert!(message.contains(&manifest.policy_digest), "{message}"); + assert_eq!(error.path(), "package.expectedDigest"); + assert_eq!( + error.to_string(), + PackageDigestMismatch { + expected: pinned, + found: verified.digest().to_owned(), + } + .to_string() + ); } #[test] - fn an_expected_policy_digest_refuses_an_unpackaged_project() { - let root = tempfile::tempdir().unwrap(); - let package = root.path().join("package"); - std::fs::create_dir(&package).unwrap(); - let manifest = write_package(&package); - std::fs::remove_file(package.join(POLICY_PACKAGE_MANIFEST_FILE)).unwrap(); - let operator = root.path().join("operator.yaml"); + fn an_expected_digest_refuses_an_unpackaged_project() { + let root = canonical_tempdir(); + let (package, _, verified) = packaged_operator(root.path(), "development-loopback"); + std::fs::remove_file(package.join(SUM_FILE)).unwrap(); let mut document = operator_value(&package, "development-loopback"); - document["package"]["expectedPolicyDigest"] = serde_json::json!(manifest.policy_digest); - std::fs::write(&operator, serde_norway::to_string(&document).unwrap()).unwrap(); + document["package"]["expectedDigest"] = serde_json::json!(verified.digest()); + let operator = write_operator(root.path(), &document); let error = RuntimeConfig::load(&operator).expect_err("an unpackaged project is refused"); assert!(matches!( &error, - RuntimeConfigError::PolicyDigestMismatch { expected, actual: None } - if expected == &manifest.policy_digest + RuntimeConfigError::Package(error) + if matches!(error.kind(), PackageErrorKind::SumFileMissing) )); - assert_eq!(error.path(), "package.expectedPolicyDigest"); - assert!(error.to_string().contains(&manifest.policy_digest)); + assert_eq!(error.path(), "package.root"); } #[test] - fn a_malformed_expected_policy_digest_is_refused() { - let root = tempfile::tempdir().unwrap(); - let package = root.path().join("package"); - std::fs::create_dir(&package).unwrap(); - let manifest = write_package(&package); - let operator = root.path().join("operator.yaml"); + fn a_malformed_expected_digest_is_refused() { + let root = canonical_tempdir(); + let (package, _, verified) = packaged_operator(root.path(), "operator-controlled-upstream"); for malformed in [ String::new(), - manifest.policy_digest.to_uppercase(), - manifest - .policy_digest - .trim_start_matches("sha256:") - .to_owned(), - format!("{}0", manifest.policy_digest), + verified.digest().to_uppercase(), + verified.digest().trim_start_matches("sha256:").to_owned(), + format!("{}0", verified.digest()), format!("sha512:{}", "0".repeat(64)), ] { let mut document = operator_value(&package, "operator-controlled-upstream"); - document["package"]["expectedPolicyDigest"] = serde_json::json!(malformed); + document["package"]["expectedDigest"] = serde_json::json!(malformed); + let operator = write_operator(root.path(), &document); + let error = RuntimeConfig::load(&operator).expect_err("a malformed digest is refused"); + assert!( + matches!(error, RuntimeConfigError::Block(_)), + "{malformed}: {error:?}" + ); + assert_eq!(error.path(), "package.expectedDigest"); + } + } + + #[test] + fn the_retired_expected_policy_digest_key_names_its_replacement() { + let root = canonical_tempdir(); + let (package, _, verified) = packaged_operator(root.path(), "operator-controlled-upstream"); + let mut document = operator_value(&package, "operator-controlled-upstream"); + document["package"]["expectedPolicyDigest"] = serde_json::json!(verified.digest()); + let operator = write_operator(root.path(), &document); + let message = RuntimeConfig::load(&operator) + .expect_err("the retired key is refused") + .to_string(); + assert!( + message.contains("package.expectedPolicyDigest"), + "{message}" + ); + assert!(message.contains("package.expectedDigest"), "{message}"); + } + + #[test] + fn a_stranded_work_acknowledgement_names_one_package_digest() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + std::fs::create_dir(&package).unwrap(); + let manifest = write_package(&package); + let operator = root.path().join("operator.yaml"); + let write = |acknowledged: &str| { + let mut document = operator_value(&package, "operator-controlled-upstream"); + document["package"]["acknowledgeStrandedWork"] = serde_json::json!(acknowledged); std::fs::write(&operator, serde_norway::to_string(&document).unwrap()).unwrap(); + }; + + write(manifest.digest()); + let config = RuntimeConfig::load(&operator).expect("a digest acknowledgement loads"); + assert_eq!( + config.package.acknowledge_stranded_work.as_deref(), + Some(manifest.digest()) + ); + + for malformed in ["yes", "sha256:ABC"] { + write(malformed); let error = RuntimeConfig::load(&operator).expect_err("a malformed digest is refused"); assert!( - matches!(error, RuntimeConfigError::InvalidExpectedPolicyDigest), + matches!( + error, + RuntimeConfigError::InvalidStrandedWorkAcknowledgement + ), "{malformed}: {error:?}" ); - assert_eq!(error.path(), "package.expectedPolicyDigest"); + assert_eq!(error.path(), "package.acknowledgeStrandedWork"); } } #[test] fn source_context_review_namespaces_require_activated_adapters() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("source-context"); std::fs::create_dir(&package).unwrap(); write_package_with_project(&package, SOURCE_CONTEXT_REVIEW_PROJECT); @@ -1926,7 +2196,7 @@ reviewProducers: #[test] fn retained_completion_destinations_may_remain_configured_after_policy_removal() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("source-context"); std::fs::create_dir(&package).unwrap(); write_package_with_project(&package, SOURCE_CONTEXT_REVIEW_PROJECT); @@ -1945,7 +2215,7 @@ reviewProducers: #[test] fn a_completion_destination_names_a_safe_header_for_its_secret() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("source-context"); std::fs::create_dir(&package).unwrap(); write_package_with_project(&package, SOURCE_CONTEXT_REVIEW_PROJECT); @@ -2058,27 +2328,97 @@ reviewProducers: "auth": {"header": "x-api-key", "secretRef": "secret:file/completion-key", "scheme": "Basic"} })) .expect_err("closed auth object"); - assert!(matches!(unknown, RuntimeConfigError::Parse { .. })); + assert!(matches!(unknown, RuntimeConfigError::Load(_))); + } + + #[test] + fn the_optional_metrics_listener_is_absent_by_default_and_stays_operator_private() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + std::fs::create_dir(&package).unwrap(); + write_package(&package); + let operator = root.path().join("runtime.yaml"); + let load = |metrics: Option, listener: &str| { + let mut document = operator_value(&package, "operator-controlled-upstream"); + document["listener"]["bind"] = serde_json::json!(listener); + document["listener"]["networkExposure"] = serde_json::json!("container-private"); + if let Some(metrics) = metrics { + document["metricsListener"] = metrics; + } + std::fs::write(&operator, serde_norway::to_string(&document).unwrap()).unwrap(); + RuntimeConfig::load(&operator) + }; + + assert!(load(None, "127.0.0.1:8100") + .unwrap() + .metrics_listener + .is_none()); + for accepted in [ + "127.0.0.1:9100", + "10.0.0.5:9100", + "[fd00::1]:9100", + "[::1]:9100", + ] { + let config = load( + Some(serde_json::json!({"bind": accepted})), + "127.0.0.1:8100", + ) + .unwrap_or_else(|error| panic!("{accepted}: {error}")); + assert_eq!( + config.metrics_listener.expect("metrics listener").bind, + accepted.parse().unwrap() + ); + } + for (refused, listener) in [ + ("0.0.0.0:9100", "127.0.0.1:8100"), + ("[::]:9100", "127.0.0.1:8100"), + ("203.0.113.9:9100", "127.0.0.1:8100"), + ("127.0.0.1:0", "127.0.0.1:8100"), + ("127.0.0.1:8100", "127.0.0.1:8100"), + ("10.0.0.5:8100", "0.0.0.0:8100"), + ("[fd00::1]:8100", "[::]:8100"), + ("10.0.0.5:8100", "[::]:8100"), + ] { + let error = + load(Some(serde_json::json!({"bind": refused})), listener).expect_err(refused); + assert!( + matches!(error, RuntimeConfigError::InvalidMetricsListener), + "{refused} beside {listener}: {error}" + ); + assert_eq!(error.path(), "metricsListener"); + } + assert!(load( + Some(serde_json::json!({"bind": "10.0.0.5:8100"})), + "[fd00::1]:8100" + ) + .is_ok()); + assert!(matches!( + load( + Some(serde_json::json!({"bind": "127.0.0.1:9100", "path": "/metrics"})), + "127.0.0.1:8100" + ), + Err(RuntimeConfigError::Load(_)) + )); } #[test] fn runtime_envelope_listener_and_operated_paths_are_strict() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); let operator = root.path().join("runtime.yaml"); let valid = operator_document(&package, "operator-controlled-upstream"); - let defaulted_bind = valid.replace(" bind: 127.0.0.1:8100\n", ""); - std::fs::write(&operator, &defaulted_bind).unwrap(); + let omitted_bind = valid.replace(" bind: 127.0.0.1:8100\n", ""); + std::fs::write(&operator, &omitted_bind).unwrap(); assert_eq!( - RuntimeConfig::load(&operator).unwrap().listener.bind.port(), - 8100 + RuntimeConfig::load(&operator).unwrap_err().path(), + "listener" ); assert!(matches!( RuntimeConfig::load("runtime.yaml"), - Err(RuntimeConfigError::RelativeRuntimePath) + Err(RuntimeConfigError::Load(load)) if load.kind() == RuntimeConfigErrorKind::Path )); std::fs::write( @@ -2088,7 +2428,7 @@ reviewProducers: .unwrap(); assert!(matches!( RuntimeConfig::load(&operator), - Err(RuntimeConfigError::InvalidApiVersion) + Err(RuntimeConfigError::Load(load)) if load.kind() == RuntimeConfigErrorKind::Envelope )); std::fs::write( @@ -2098,13 +2438,13 @@ reviewProducers: .unwrap(); assert!(matches!( RuntimeConfig::load(&operator), - Err(RuntimeConfigError::Parse { .. }) + Err(RuntimeConfigError::Load(_)) )); } #[test] fn removed_runtime_principal_claim_names_the_authored_replacement() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); @@ -2116,7 +2456,11 @@ reviewProducers: ); std::fs::write(&operator, document).unwrap(); let error = RuntimeConfig::load(&operator).unwrap_err(); - assert!(matches!(&error, RuntimeConfigError::RemovedPrincipalClaim)); + assert!(matches!( + &error, + RuntimeConfigError::Load(load) if load.kind() == RuntimeConfigErrorKind::RemovedKey + )); + assert_eq!(error.path(), "authentication.oidc.principalClaim"); assert!(error .to_string() .contains("accessProfiles[].principalClaim")); @@ -2124,7 +2468,7 @@ reviewProducers: #[test] fn an_out_of_range_source_binding_interval_is_refused_at_load() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); @@ -2144,7 +2488,7 @@ reviewProducers: #[test] fn a_request_timeout_shorter_than_the_connect_timeout_names_that_rule() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); @@ -2168,7 +2512,7 @@ reviewProducers: #[test] fn environment_secret_references_require_the_explicit_provider() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); @@ -2179,8 +2523,8 @@ reviewProducers: let error = RuntimeConfig::load(&operator).unwrap_err(); assert!(matches!( &error, - RuntimeConfigError::SecretProviderRequired { path } - if path == "database.runtimeUrlRef" + RuntimeConfigError::Block(block) + if block.kind() == registry_platform_config::ConfigBlockErrorKind::SecretProviderDisabled )); assert_eq!(error.path(), "database.runtimeUrlRef"); } @@ -2193,7 +2537,7 @@ reviewProducers: /// access token and act with the scopes it carries. #[test] fn the_verifier_admits_only_the_rfc_9068_access_token_type() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); @@ -2217,17 +2561,17 @@ reviewProducers: /// refusal here already names. #[test] fn a_document_refused_whole_is_reported_at_its_root() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let runtime = root.path().join("runtime.yaml"); std::fs::write(&runtime, "a scalar, not a runtime configuration\n").unwrap(); let error = RuntimeConfig::load(&runtime).unwrap_err(); - assert!(matches!(error, RuntimeConfigError::Parse { .. })); + assert!(matches!(error, RuntimeConfigError::Load(_))); assert_eq!(error.path(), "/"); } #[test] fn typed_parse_path_does_not_echo_the_rejected_value() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); std::fs::create_dir(&package).unwrap(); write_package(&package); @@ -2245,30 +2589,16 @@ reviewProducers: #[test] fn startup_redecodes_packaged_source_metadata_instead_of_trusting_its_hash() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); - std::fs::create_dir(&package).unwrap(); - write_package(&package); - let invalid_description = b"{}\n"; - std::fs::write( - package.join("sources/professional.json"), - invalid_description, - ) - .unwrap(); - let manifest = PolicyPackageManifest::build([ - ( - "casework.yaml".to_owned(), - SOURCE_PROJECT.as_bytes().to_vec(), - ), - ( - "sources/professional.json".to_owned(), - invalid_description.to_vec(), - ), - ]) - .unwrap(); - std::fs::write( - package.join(POLICY_PACKAGE_MANIFEST_FILE), - serde_json::to_vec_pretty(&manifest).unwrap(), + std::fs::create_dir_all(package.join("sources")).unwrap(); + std::fs::write(package.join("casework.yaml"), SOURCE_PROJECT).unwrap(); + std::fs::write(package.join("sources/professional.json"), b"{}\n").unwrap(); + registry_platform_config::write_sum_file( + &package, + None, + &package_limits(), + PACKAGE_COMMAND, ) .unwrap(); let operator = root.path().join("operator.yaml"); @@ -2285,112 +2615,97 @@ reviewProducers: #[test] fn listener_requires_a_private_address_or_explicit_container_network() { - use std::net::{Ipv4Addr, Ipv6Addr}; - - assert!(valid_listener( - IpAddr::V4(Ipv4Addr::LOCALHOST), - ListenerNetworkExposure::PrivateAddress, - TlsTermination::OperatorControlledUpstream, - )); - assert!(valid_listener( - "10.20.30.40".parse().expect("private IPv4"), - ListenerNetworkExposure::PrivateAddress, - TlsTermination::OperatorControlledUpstream, - )); - assert!(!valid_listener( - IpAddr::V4(Ipv4Addr::UNSPECIFIED), - ListenerNetworkExposure::PrivateAddress, - TlsTermination::OperatorControlledUpstream, - )); - assert!(valid_listener( - IpAddr::V6(Ipv6Addr::UNSPECIFIED), - ListenerNetworkExposure::ContainerPrivate, - TlsTermination::OperatorControlledUpstream, - )); - assert!(!valid_listener( - "203.0.113.10".parse().expect("public IPv4"), - ListenerNetworkExposure::ContainerPrivate, - TlsTermination::OperatorControlledUpstream, - )); - assert!(valid_listener( - IpAddr::V4(Ipv4Addr::LOCALHOST), - ListenerNetworkExposure::PrivateAddress, - TlsTermination::DevelopmentLoopback, - )); - assert!(!valid_listener( - "10.20.30.40".parse().expect("private IPv4"), - ListenerNetworkExposure::PrivateAddress, - TlsTermination::DevelopmentLoopback, - )); - assert!(!valid_listener( - IpAddr::V4(Ipv4Addr::UNSPECIFIED), - ListenerNetworkExposure::ContainerPrivate, - TlsTermination::DevelopmentLoopback, - )); + let valid = |bind: &str, exposure, tls_termination| { + ListenerConfig { + bind: bind.parse().expect("listener address"), + tls_termination, + network_exposure: exposure, + } + .is_valid() + }; + let upstream = TlsTermination::OperatorControlledUpstream; + let loopback = TlsTermination::DevelopmentLoopback; + let private = ListenerNetworkExposure::PrivateAddress; + let container = ListenerNetworkExposure::ContainerPrivate; + + assert!(valid("127.0.0.1:8100", private, upstream)); + assert!(valid("10.20.30.40:8100", private, upstream)); + assert!(!valid("0.0.0.0:8100", private, upstream)); + assert!(valid("[::]:8100", container, upstream)); + assert!(!valid("203.0.113.10:8100", container, upstream)); + assert!(valid("127.0.0.1:8100", private, loopback)); + assert!(!valid("10.20.30.40:8100", private, loopback)); + assert!(!valid("0.0.0.0:8100", container, loopback)); } } #[derive(Debug, Error)] pub enum RuntimeConfigError { - #[error("the Casework runtime configuration could not be read")] - Read(#[source] std::io::Error), - #[error("the Casework runtime configuration is not valid YAML at {path}")] - Parse { - path: String, - #[source] - source: serde_norway::Error, - }, + #[error(transparent)] + Load(#[from] registry_platform_config::RuntimeConfigError), + #[error(transparent)] + Block(#[from] ConfigBlockError), #[error("unsupported Casework runtime apiVersion; expected registry.registrystack.org/casework-runtime/v1alpha1")] InvalidApiVersion, #[error("unsupported Casework runtime kind; expected CaseworkRuntimeConfig")] InvalidKind, #[error("the operated runtime path {0} must be absolute")] RelativeOperatedPath(&'static str), - #[error("the selected Casework runtime configuration path must be absolute")] - RelativeRuntimePath, - #[error("secretProviders must explicitly enable file, environment, or both")] - InvalidSecretProviders, - #[error("{path} is not a valid secret reference")] - InvalidSecretReference { path: String }, - #[error("{path} uses a secret provider that is not explicitly enabled")] - SecretProviderRequired { path: String }, - #[error("authentication.oidc.principalClaim has been removed; configure accessProfiles[].principalClaim in casework.yaml")] - RemovedPrincipalClaim, #[error("the Casework project is invalid")] Project(#[source] registry_casework_core::ConfigLoadError), - #[error("the Casework policy package is invalid")] - PolicyPackage(#[source] PolicyPackageError), - #[error("operator-controlled production requires a verified Casework policy package")] - ProductionPolicyPackageRequired, #[error( - "package.expectedPolicyDigest must be sha256: followed by 64 lowercase hexadecimal digits" + "{field} in the authored Casework project holds an environment expression; ${{...}} substitution applies to runtime.yaml only, so write the value in casework.yaml directly" )] - InvalidExpectedPolicyDigest, + PolicyEnvironmentExpression { field: String }, #[error( - "package.expectedPolicyDigest is {expected}, but package.root holds {}", - actual.as_deref().map_or_else( - || "no casework.package.json".to_owned(), - |actual| format!("the package with policy digest {actual}"), - ) + "package.root/casework.yaml must be a regular file of at most one MiB; rebuild the package with `caseworkctl package`" + )] + PolicyUnreadable, + #[error(transparent)] + Package(PackageError), + #[error(transparent)] + PackageDigest(PackageDigestMismatch), + #[error( + "the package at package.root must hold exactly casework.yaml and the source descriptions it names{}{}; rebuild it with `caseworkctl package`", + if missing.is_empty() { String::new() } else { format!("; missing: {}", missing.join(", ")) }, + if extra.is_empty() { String::new() } else { format!("; extra: {}", extra.join(", ")) }, )] - PolicyDigestMismatch { - expected: String, - actual: Option, + PackageContents { + missing: Vec, + extra: Vec, }, + #[error( + "the packaged file {path} changed after package verification; rebuild it with `caseworkctl package`" + )] + PackageFileChanged { path: String }, + #[error( + "package.root holds casework.package.json, which Casework no longer reads; rebuild the package with `caseworkctl package`, which writes SHA256SUMS" + )] + RetiredPackageManifest, + #[error( + "package.acknowledgeStrandedWork must be sha256: followed by 64 lowercase hexadecimal digits" + )] + InvalidStrandedWorkAcknowledgement, #[error("an imported source description does not match the exact configured source policy")] SourceDescription, #[error("the Casework runtime configuration is invalid")] Invalid, #[error("listener is not valid for its declared TLS termination and network exposure")] InvalidListener, + #[error( + "metricsListener.bind must be a loopback or private address with a nonzero port that the API listener does not also occupy" + )] + InvalidMetricsListener, #[error("authentication.oidc is invalid or conflicts with accessProfiles[].principalClaim")] InvalidOidc, + #[error( + "authentication.oidc.allowedClients must name every client the deployment admits under operator-controlled-upstream; an empty list admits every client the issuer verifies" + )] + AllowedClientsRequired, #[error( "database.runtimeUrlRef and database.migrationUrlRef must be non-empty secret references" )] InvalidDatabaseReference, - #[error("audit.hashKeyRef must be a non-empty secret reference")] - InvalidAuditReference, #[error("{0}")] InvalidAuditDestination(#[source] AuditDestinationError), #[error("sources must exactly match the source ids declared by package.root/casework.yaml")] @@ -2417,22 +2732,19 @@ impl RuntimeConfigError { #[must_use] pub fn path(&self) -> &str { match self { + Self::Load(error) => error.field(), + Self::Block(error) => error.field(), Self::InvalidApiVersion => "apiVersion", Self::InvalidKind => "kind", Self::RelativeOperatedPath(path) => path, - Self::RelativeRuntimePath | Self::Read(_) => "/", - Self::Parse { path, .. } => path, - Self::InvalidSecretProviders => "secretProviders", - Self::InvalidSecretReference { path } - | Self::SecretProviderRequired { path } - | Self::InvalidReviewCompletionAuth { path } => path, + Self::InvalidReviewCompletionAuth { path } => path, Self::InvalidSourceBinding { path, .. } => path, - Self::RemovedPrincipalClaim => "authentication.oidc.principalClaim", Self::InvalidOidc | Self::Oidc => "authentication.oidc", + Self::AllowedClientsRequired => "authentication.oidc.allowedClients", Self::OidcJwksSecret(_) => "authentication.oidc.jwksSource.documentRef", Self::InvalidListener => "listener", + Self::InvalidMetricsListener => "metricsListener", Self::InvalidDatabaseReference | Self::PlaintextDatabase => "database", - Self::InvalidAuditReference => "audit.hashKeyRef", Self::InvalidAuditDestination(error) => match error { AuditDestinationError::MissingPath | AuditDestinationError::RelativePath => { "audit.path" @@ -2449,12 +2761,17 @@ impl RuntimeConfigError { _ => "audit", }, Self::InvalidSourceBindings | Self::SourceDescription => "sources", - Self::InactiveReviewSourceNamespace => "package.root/casework.yaml", - Self::Project(_) => "package.root/casework.yaml", - Self::PolicyPackage(_) | Self::ProductionPolicyPackageRequired => "package.root", - Self::InvalidExpectedPolicyDigest | Self::PolicyDigestMismatch { .. } => { - "package.expectedPolicyDigest" + Self::InactiveReviewSourceNamespace | Self::PolicyEnvironmentExpression { .. } => { + "package.root/casework.yaml" } + Self::Project(_) => "package.root/casework.yaml", + Self::PolicyUnreadable => "package.root/casework.yaml", + Self::Package(_) + | Self::PackageContents { .. } + | Self::PackageFileChanged { .. } + | Self::RetiredPackageManifest => "package.root", + Self::PackageDigest(_) => "package.expectedDigest", + Self::InvalidStrandedWorkAcknowledgement => "package.acknowledgeStrandedWork", Self::Invalid => "/", } } diff --git a/crates/registry-casework/src/http.rs b/crates/registry-casework/src/http.rs index 70acb15162..5ba6308118 100644 --- a/crates/registry-casework/src/http.rs +++ b/crates/registry-casework/src/http.rs @@ -1747,9 +1747,10 @@ impl From for HttpError { StoreError::AttemptPending => Self::RecoveryPending(None), StoreError::Invalid => Self::Invalid, StoreError::ReviewValidation(validation) => Self::Validation(validation), - StoreError::Unavailable | StoreError::AuditUnavailable | StoreError::Postgres(_) => { - Self::ServiceUnavailable - } + StoreError::Unavailable + | StoreError::AuditUnavailable + | StoreError::Postgres(_) + | StoreError::SchemaNotCurrent { .. } => Self::ServiceUnavailable, StoreError::Configuration | StoreError::SecretConfiguration(_) | StoreError::Corrupt @@ -1765,6 +1766,7 @@ impl From for HttpError { fn from(error: ServiceError) -> Self { match error { ServiceError::Configuration => Self::Internal, + ServiceError::ReconciliationFailing { .. } => Self::ServiceUnavailable, ServiceError::Source => Self::SourceNotFound, ServiceError::SourceProtocol => Self::SourceBadGateway, ServiceError::Store(error) => error.into(), diff --git a/crates/registry-casework/src/lib.rs b/crates/registry-casework/src/lib.rs index 7a3e6dc864..c371b1ada8 100644 --- a/crates/registry-casework/src/lib.rs +++ b/crates/registry-casework/src/lib.rs @@ -6,6 +6,8 @@ mod auth; mod clocks; mod config; mod http; +mod metrics; +mod pinned_work; pub mod problem; mod review; mod runtime; @@ -24,6 +26,7 @@ pub use audit::{CaseworkAudit, CASEWORK_AUDIT_SCHEMA}; pub use auth::*; pub use config::*; pub use http::*; +pub use pinned_work::*; pub use review::*; pub use runtime::*; pub use service::*; diff --git a/crates/registry-casework/src/metrics.rs b/crates/registry-casework/src/metrics.rs new file mode 100644 index 0000000000..f4abc6822a --- /dev/null +++ b/crates/registry-casework/src/metrics.rs @@ -0,0 +1,464 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Opt-in operator telemetry for Registry Casework. +//! +//! The router below only ever serves the operator-private listener started by +//! an explicit `metricsListener` runtime configuration member. It is a +//! separate binding rather than a route on the Casework listener, so the +//! public Casework contract on that listener is unchanged and reaching the +//! telemetry requires reaching a different socket. +//! +//! Every value is read at scrape time: the build and the verified policy +//! package this process serves and, from the Casework database, each +//! configured source's reconciliation health. The database readings are +//! shared by every scrape inside a five-second window and bounded by a read +//! timeout, so the scrape rate never sets the database load. The only label +//! values are the configured source identifiers, the build version, and the +//! package digest, so no caller, record, or request value can become a series. + +use std::fmt::Write as _; +use std::sync::Arc; + +use async_trait::async_trait; +use axum::extract::State; +use axum::http::header::CONTENT_TYPE; +use axum::http::HeaderValue; +use axum::response::{IntoResponse, Response}; +use axum::routing::get; +use axum::{Json, Router}; +use chrono::{DateTime, Utc}; +use serde_json::json; + +use crate::store::{PostgresStore, SourceReconciliationHealth, StoreError}; + +const METRICS_MEDIA_TYPE: &str = "text/plain; version=0.0.4"; +/// How long one scrape waits for the database readings before it reports +/// the database down. +const READINGS_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); +/// How long one set of database readings answers later scrapes. Scrapes are +/// unauthenticated on the operator-private listener, so this bounds the +/// database work any scrape rate can cause to one reading per window, on one +/// pool connection at a time. +const READINGS_REUSE: std::time::Duration = std::time::Duration::from_secs(5); + +/// The database readings one scrape reports, or `None` when they could not +/// be read. +type Readings = Option>; + +/// What the telemetry listener reports about this process. +#[derive(Clone)] +pub(crate) struct MetricsState { + inner: Arc, +} + +/// The database readings one scrape reports. +#[async_trait] +pub(crate) trait MetricsReadings: Send + Sync { + async fn reconciliation_health( + &self, + source_ids: &[String], + ) -> Result, StoreError>; +} + +#[async_trait] +impl MetricsReadings for PostgresStore { + async fn reconciliation_health( + &self, + source_ids: &[String], + ) -> Result, StoreError> { + PostgresStore::reconciliation_health(self, source_ids).await + } +} + +struct MetricsInner { + store: Arc, + source_ids: Vec, + package_digest: String, + read_timeout: std::time::Duration, + /// The last readings and when they were taken. Held across a reading, so + /// concurrent scrapes wait for the one reading in flight. + last_readings: tokio::sync::Mutex>, +} + +impl MetricsState { + pub(crate) fn new( + store: Arc, + source_ids: Vec, + package_digest: String, + ) -> Self { + Self::with_read_timeout(store, source_ids, package_digest, READINGS_TIMEOUT) + } + + fn with_read_timeout( + store: Arc, + source_ids: Vec, + package_digest: String, + read_timeout: std::time::Duration, + ) -> Self { + Self { + inner: Arc::new(MetricsInner { + store, + source_ids, + package_digest, + read_timeout, + last_readings: tokio::sync::Mutex::new(None), + }), + } + } +} + +impl MetricsInner { + /// The database readings for one scrape: the last readings while they + /// are inside the reuse window, otherwise one fresh reading bounded by + /// the read timeout. + async fn readings(&self) -> Readings { + let mut last = self.last_readings.lock().await; + if let Some((taken, readings)) = last.as_ref() { + if taken.elapsed() < READINGS_REUSE { + return readings.clone(); + } + } + let readings = match tokio::time::timeout(self.read_timeout, self.read_database()).await { + Ok(readings) => readings, + Err(_) => { + tracing::warn!( + timeout_seconds = self.read_timeout.as_secs_f64(), + "Casework metrics timed out reading the database" + ); + None + } + }; + *last = Some((tokio::time::Instant::now(), readings.clone())); + readings + } + + async fn read_database(&self) -> Readings { + match self.store.reconciliation_health(&self.source_ids).await { + Ok(health) => Some(health), + Err(error) => { + tracing::warn!(error = %error, "Casework metrics could not read reconciliation health"); + None + } + } + } +} + +/// The operator-private telemetry routes: `/metrics` in the Prometheus text +/// format and `/version` as JSON. Every other path is `404`. +pub(crate) fn metrics_router(state: MetricsState) -> Router { + Router::new() + .route("/metrics", get(metrics)) + .route("/version", get(version)) + .with_state(state) +} + +async fn version(State(state): State) -> Response { + Json(json!({ + "version": registry_platform_buildinfo::DISPLAY_VERSION, + "packageDigest": state.inner.package_digest, + })) + .into_response() +} + +async fn metrics(State(state): State) -> Response { + let inner = &state.inner; + let database = inner.readings().await; + let body = render( + registry_platform_buildinfo::DISPLAY_VERSION, + &inner.package_digest, + database.as_deref(), + Utc::now(), + ); + let mut response = body.into_response(); + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static(METRICS_MEDIA_TYPE)); + response +} + +/// Render one scrape. `database` is `None` when the scrape could not read the +/// Casework database, which `casework_database_up` reports as `0`. +fn render( + version: &str, + package_digest: &str, + database: Option<&[SourceReconciliationHealth]>, + now: DateTime, +) -> String { + let mut out = String::new(); + gauge( + &mut out, + "casework_build_info", + "The Casework build and the verified policy package digest this process serves.", + ); + let _ = writeln!( + out, + "casework_build_info{{version=\"{}\",package_digest=\"{}\"}} 1", + escape(version), + escape(package_digest) + ); + gauge( + &mut out, + "casework_database_up", + "Whether this scrape could read the Casework database.", + ); + let _ = writeln!(out, "casework_database_up {}", u8::from(database.is_some())); + let Some(health) = database else { + return out; + }; + gauge( + &mut out, + "casework_source_reconciliation_consecutive_failures", + "Reconciliation passes that failed in a row for each configured source.", + ); + for source in health { + let _ = writeln!( + out, + "casework_source_reconciliation_consecutive_failures{{source_id=\"{}\"}} {}", + escape(&source.source_id), + source.consecutive_failures + ); + } + gauge( + &mut out, + "casework_source_reconciliation_last_success_age_seconds", + "Seconds since each configured source last reconciled successfully; absent until the first success.", + ); + for source in health { + if let Some(succeeded) = source.last_succeeded_at { + let age = (now - succeeded).num_milliseconds().max(0) as f64 / 1000.0; + let _ = writeln!( + out, + "casework_source_reconciliation_last_success_age_seconds{{source_id=\"{}\"}} {age}", + escape(&source.source_id) + ); + } + } + out +} + +fn gauge(out: &mut String, name: &str, help: &str) { + let _ = writeln!(out, "# HELP {name} {help}"); + let _ = writeln!(out, "# TYPE {name} gauge"); +} + +/// Escape a label value as the Prometheus text format requires. +fn escape(value: &str) -> String { + value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n") +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone as _; + + fn source( + id: &str, + failures: i32, + succeeded: Option>, + ) -> SourceReconciliationHealth { + SourceReconciliationHealth { + source_id: id.to_owned(), + consecutive_failures: failures, + last_succeeded_at: succeeded, + last_failed_at: None, + last_failure: None, + } + } + + #[test] + fn a_scrape_reports_build_and_reconciliation_lag() { + let now = Utc.with_ymd_and_hms(2026, 9, 25, 12, 0, 0).unwrap(); + let health = [ + source("permits", 0, Some(now - chrono::Duration::seconds(90))), + source("licences", 4, None), + ]; + let rendered = render("1.2.3", "sha256:abc", Some(&health), now); + for line in [ + "casework_build_info{version=\"1.2.3\",package_digest=\"sha256:abc\"} 1", + "casework_database_up 1", + "casework_source_reconciliation_consecutive_failures{source_id=\"permits\"} 0", + "casework_source_reconciliation_consecutive_failures{source_id=\"licences\"} 4", + "casework_source_reconciliation_last_success_age_seconds{source_id=\"permits\"} 90", + "# TYPE casework_source_reconciliation_consecutive_failures gauge", + ] { + assert!( + rendered.lines().any(|rendered| rendered == line), + "missing {line:?} in:\n{rendered}" + ); + } + assert!( + !rendered.contains("last_success_age_seconds{source_id=\"licences\"}"), + "a source that never succeeded has no age:\n{rendered}" + ); + } + + #[test] + fn a_scrape_without_the_database_reports_it_down_and_omits_database_series() { + let rendered = render( + "1.2.3", + "sha256:abc", + None, + Utc.with_ymd_and_hms(2026, 9, 25, 12, 0, 0).unwrap(), + ); + assert!(rendered.contains("casework_database_up 0\n")); + assert!(rendered.contains("package_digest=\"sha256:abc\"} 1\n")); + assert!(!rendered.contains("casework_audit_")); + assert!(!rendered.contains("casework_source_reconciliation")); + } + + #[derive(Default)] + struct FakeReadings { + available: bool, + stall: bool, + reads: std::sync::atomic::AtomicUsize, + } + + #[async_trait] + impl MetricsReadings for FakeReadings { + async fn reconciliation_health( + &self, + source_ids: &[String], + ) -> Result, StoreError> { + self.reads.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if self.stall { + std::future::pending::<()>().await; + } + if !self.available { + return Err(StoreError::Configuration); + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + Ok(source_ids.iter().map(|id| source(id, 2, None)).collect()) + } + } + + fn fake_state(readings: Arc, read_timeout: std::time::Duration) -> MetricsState { + MetricsState::with_read_timeout( + readings, + vec!["permits".to_owned()], + "sha256:abc".to_owned(), + read_timeout, + ) + } + + async fn get(available: bool, path: &str) -> (axum::http::StatusCode, Option, String) { + let readings = Arc::new(FakeReadings { + available, + ..FakeReadings::default() + }); + get_from(metrics_router(fake_state(readings, READINGS_TIMEOUT)), path).await + } + + async fn get_from( + router: Router, + path: &str, + ) -> (axum::http::StatusCode, Option, String) { + use tower::ServiceExt as _; + let response = router + .oneshot( + axum::http::Request::get(path) + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let media = response + .headers() + .get(CONTENT_TYPE) + .map(|value| value.to_str().unwrap().to_owned()); + let body = axum::body::to_bytes(response.into_body(), 1 << 20) + .await + .unwrap(); + (status, media, String::from_utf8(body.to_vec()).unwrap()) + } + + #[tokio::test] + async fn the_router_serves_metrics_version_and_nothing_else() { + let (status, media, body) = get(true, "/metrics").await; + assert_eq!(status, axum::http::StatusCode::OK); + assert_eq!(media.as_deref(), Some(METRICS_MEDIA_TYPE)); + assert!(!body.contains("casework_audit_"), "{body}"); + assert!(body.contains( + "casework_source_reconciliation_consecutive_failures{source_id=\"permits\"} 2\n" + )); + + let (status, _, body) = get(false, "/metrics").await; + assert_eq!(status, axum::http::StatusCode::OK); + assert!(body.contains("casework_database_up 0\n"), "{body}"); + + let (status, _, body) = get(true, "/version").await; + assert_eq!(status, axum::http::StatusCode::OK); + let version: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!( + version, + json!({ + "version": registry_platform_buildinfo::DISPLAY_VERSION, + "packageDigest": "sha256:abc", + }) + ); + + for path in ["/", "/healthz", "/readyz", "/v1/inbox"] { + assert_eq!( + get(true, path).await.0, + axum::http::StatusCode::NOT_FOUND, + "{path}" + ); + } + } + + #[tokio::test] + async fn a_scrape_burst_reads_the_database_once() { + let readings = Arc::new(FakeReadings { + available: true, + ..FakeReadings::default() + }); + let router = metrics_router(fake_state(readings.clone(), READINGS_TIMEOUT)); + let mut scrapes = tokio::task::JoinSet::new(); + for _ in 0..16 { + scrapes.spawn(get_from(router.clone(), "/metrics")); + } + while let Some(scrape) = scrapes.join_next().await { + let (status, _, body) = scrape.expect("scrape task"); + assert_eq!(status, axum::http::StatusCode::OK); + assert!( + body.contains( + "casework_source_reconciliation_consecutive_failures{source_id=\"permits\"} 2\n" + ), + "{body}" + ); + } + let (_, _, body) = get_from(router, "/metrics").await; + assert!(body.contains("casework_database_up 1\n"), "{body}"); + assert_eq!( + readings.reads.load(std::sync::atomic::Ordering::SeqCst), + 1, + "scrapes inside the reuse window share one database reading" + ); + } + + #[tokio::test] + async fn a_stalled_database_reading_reports_the_database_down() { + let readings = Arc::new(FakeReadings { + available: true, + stall: true, + ..FakeReadings::default() + }); + let router = metrics_router(fake_state(readings, std::time::Duration::from_millis(50))); + let (status, _, body) = tokio::time::timeout( + std::time::Duration::from_secs(10), + get_from(router, "/metrics"), + ) + .await + .expect("a stalled reading ends the scrape at the read timeout"); + assert_eq!(status, axum::http::StatusCode::OK); + assert!(body.contains("casework_database_up 0\n"), "{body}"); + } + + #[test] + fn label_values_are_escaped() { + assert_eq!(escape("a\"b\\c\nd"), "a\\\"b\\\\c\\nd"); + } +} diff --git a/crates/registry-casework/src/pinned_work.rs b/crates/registry-casework/src/pinned_work.rs new file mode 100644 index 0000000000..dd5fe91d9e --- /dev/null +++ b/crates/registry-casework/src/pinned_work.rs @@ -0,0 +1,833 @@ +//! Activation preflight for work pinned under an earlier policy package. +//! +//! A review request pins its kind's policy snapshot when it is admitted, and a +//! work item keeps the queue it was routed to. A later package can remove the +//! queue or access profile that pinned work still names, change a kind's +//! content under an unchanged version, or change the fields a source read +//! discloses so that the pinned display schema refuses them. Each of those +//! hides or orphans in-flight work without any error, so the runtime compares +//! the package it is about to activate with the work already retained. + +use std::collections::{BTreeMap, BTreeSet}; + +use registry_casework_core::{ + CaseworkProject, ReviewContextStrategy, ReviewKindPolicySnapshot, SourceAdapter, +}; +use serde::Serialize; +use serde_json::Value; + +use crate::{PostgresStore, StoreError}; + +/// Pinned in-flight work a candidate package would hide or orphan, with the +/// number of retained records affected. The counts carry no subject data. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde( + tag = "reason", + rename_all = "kebab-case", + rename_all_fields = "camelCase" +)] +pub enum StrandedWork { + /// In-flight reviews or open work items name a queue the package no + /// longer declares, so no team can be configured to serve them. + QueueRemoved { + queue: String, + reviews: u64, + work_items: u64, + }, + /// In-flight reviews have a remaining stage decided only through an + /// access profile the package no longer declares. + ProfileRemoved { profile: String, reviews: u64 }, + /// The package declares the pinned review kind version with different + /// content, so one kind version would name two policies. + ReviewKindChanged { + review_kind: String, + version: String, + reviews: u64, + }, + /// In-flight source-context reviews or open work items name a source the + /// runtime no longer binds, so their context can no longer be read and no + /// action on them can reach the source. + SourceRemoved { + source: String, + reviews: u64, + work_items: u64, + }, + /// A source read now discloses a field the pinned display schema does not + /// declare, so the pinned schema refuses every reviewer's context. + ContextFieldNotDisplayed { + source: String, + entity: String, + review_kind: String, + version: String, + field: String, + reviews: u64, + }, + /// The pinned display schema requires a field the source read no longer + /// discloses, so the pinned schema refuses every reviewer's context. + DisplayedFieldNotProjected { + source: String, + entity: String, + review_kind: String, + version: String, + field: String, + reviews: u64, + }, +} + +impl StrandedWork { + /// One value-free sentence naming the conflict and its counts. + pub fn describe(&self) -> String { + match self { + Self::QueueRemoved { + queue, + reviews, + work_items, + } => format!( + "{} and {} are in queue {queue}, which the package no longer declares", + count(*reviews, "in-flight review"), + count(*work_items, "open work item"), + ), + Self::ProfileRemoved { profile, reviews } => format!( + "{} can be decided only through access profile {profile}, which the package no longer declares", + count(*reviews, "in-flight review"), + ), + Self::ReviewKindChanged { + review_kind, + version, + reviews, + } => format!( + "{} pin review kind {review_kind} version {version}, which the package declares with different content; give the changed kind a new version", + count(*reviews, "in-flight review"), + ), + Self::SourceRemoved { + source, + reviews, + work_items, + } => format!( + "{} and {} depend on source {source}, which the package no longer declares", + count(*reviews, "in-flight review"), + count(*work_items, "open work item"), + ), + Self::ContextFieldNotDisplayed { + source, + entity, + review_kind, + version, + field, + reviews, + } => format!( + "{} of review kind {review_kind} version {version} would receive field {field} from source {source} entity {entity}, which their pinned displaySchema does not declare", + count(*reviews, "in-flight review"), + ), + Self::DisplayedFieldNotProjected { + source, + entity, + review_kind, + version, + field, + reviews, + } => format!( + "{} of review kind {review_kind} version {version} require field {field}, which source {source} entity {entity} no longer projects", + count(*reviews, "in-flight review"), + ), + } + } +} + +fn count(value: u64, noun: &str) -> String { + if value == 1 { + format!("1 {noun}") + } else { + format!("{value} {noun}s") + } +} + +/// Join conflicts into one operator sentence. +pub fn describe_stranded_work(conflicts: &[StrandedWork]) -> String { + conflicts + .iter() + .map(StrandedWork::describe) + .collect::>() + .join("; ") +} + +/// Whether a runtime may activate a package given the pinned work it strands. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PinnedWorkVerdict { + /// The package strands no pinned work. + Clear, + /// The package strands pinned work and `package.acknowledgeStrandedWork` + /// names its exact digest. + Acknowledged, + /// The package strands pinned work the operator has not acknowledged. + Refused, +} + +/// Decide activation of the package with `package_digest`. +pub fn pinned_work_verdict( + conflicts: &[StrandedWork], + package_digest: &str, + acknowledged: Option<&str>, +) -> PinnedWorkVerdict { + if conflicts.is_empty() { + PinnedWorkVerdict::Clear + } else if acknowledged == Some(package_digest) { + PinnedWorkVerdict::Acknowledged + } else { + PinnedWorkVerdict::Refused + } +} + +/// The operator sentence for a refused activation: what is stranded, and +/// the two ways forward. +pub fn stranded_work_refusal(conflicts: &[StrandedWork], package_digest: &str) -> String { + format!( + "the policy package would strand work pinned under an earlier package: {}. Let that \ + work finish under the earlier package, or set package.acknowledgeStrandedWork to \ + {package_digest} to activate this package anyway", + describe_stranded_work(conflicts) + ) +} + +/// In-flight reviews that share one pinned policy, subject source and type, +/// and active stage. +#[derive(Clone, Debug)] +pub(crate) struct PinnedReviewGroup { + pub(crate) policy: ReviewKindPolicySnapshot, + pub(crate) subject_source: String, + pub(crate) subject_type: String, + pub(crate) active_stage_index: usize, + pub(crate) reviews: u64, +} + +/// Retained in-flight work, grouped without subject data. +#[derive(Clone, Debug, Default)] +pub(crate) struct PinnedWorkInventory { + pub(crate) reviews: Vec, + /// Open work items per queue. + pub(crate) work_items: BTreeMap, + /// Open work items per source. + pub(crate) work_items_by_source: BTreeMap, +} + +/// Compare `project` and the fields its source adapters disclose with the +/// in-flight work retained in `store`. An empty result means activation +/// strands nothing. +pub async fn stranded_pinned_work( + store: &PostgresStore, + project: &CaseworkProject, + adapters: &[&dyn SourceAdapter], +) -> Result, StoreError> { + let inventory = store.pinned_work_inventory().await?; + Ok(compare_pinned_work(project, adapters, &inventory)) +} + +pub(crate) fn compare_pinned_work( + project: &CaseworkProject, + adapters: &[&dyn SourceAdapter], + inventory: &PinnedWorkInventory, +) -> Vec { + let queues: BTreeSet<&str> = project + .queues + .iter() + .map(|queue| queue.id.as_str()) + .collect(); + let profiles: BTreeSet<&str> = project + .access_profiles + .iter() + .map(|profile| profile.id.as_str()) + .collect(); + let kinds: BTreeMap<(&str, &str), _> = project + .review_kinds + .iter() + .map(|kind| ((kind.id.as_str(), kind.version.as_str()), kind)) + .collect(); + + let mut queue_reviews = BTreeMap::::new(); + let mut profile_reviews = BTreeMap::::new(); + let mut changed_kinds = BTreeMap::<(String, String), u64>::new(); + let mut removed_sources = BTreeMap::::new(); + let mut context_fields = BTreeMap::::new(); + + for group in &inventory.reviews { + let identity = &group.policy.identity; + let remaining = group + .policy + .stages + .iter() + .skip(group.active_stage_index) + .collect::>(); + for queue in remaining + .iter() + .map(|stage| stage.queue.as_str()) + .collect::>() + { + if !queues.contains(queue) { + *queue_reviews.entry(queue.to_owned()).or_default() += group.reviews; + } + } + for profile in remaining + .iter() + .flat_map(|stage| stage.deciding_profiles.iter().map(String::as_str)) + .collect::>() + { + if !profiles.contains(profile) { + *profile_reviews.entry(profile.to_owned()).or_default() += group.reviews; + } + } + if let Some(current) = kinds.get(&(identity.id.as_str(), identity.version.as_str())) { + // A current kind that does not check is refused by project + // validation before this comparison runs. + if current + .policy_digest() + .is_ok_and(|digest| digest != identity.digest) + { + *changed_kinds + .entry((identity.id.clone(), identity.version.clone())) + .or_default() += group.reviews; + } + } + if group.policy.context_strategy != ReviewContextStrategy::Source { + continue; + } + let Some(adapter) = adapters + .iter() + .find(|adapter| adapter.source_id() == group.subject_source) + else { + *removed_sources + .entry(group.subject_source.clone()) + .or_default() += group.reviews; + continue; + }; + let Some(disclosed) = adapter.caller_disclosure_fields(&group.subject_type) else { + continue; + }; + let (declared, required) = display_fields(&group.policy.display_schema); + let key = |field: &str, reason: ContextReason| ContextKey { + source: group.subject_source.clone(), + entity: group.subject_type.clone(), + review_kind: identity.id.clone(), + version: identity.version.clone(), + field: field.to_owned(), + reason, + }; + for field in &disclosed { + if !declared.contains(field.as_str()) { + *context_fields + .entry(key(field, ContextReason::NotDisplayed)) + .or_default() += group.reviews; + } + } + for field in required { + if !disclosed.iter().any(|disclosed| disclosed == field) { + *context_fields + .entry(key(field, ContextReason::NotProjected)) + .or_default() += group.reviews; + } + } + } + + let mut conflicts = Vec::new(); + let stranded_queues = queue_reviews + .keys() + .chain(inventory.work_items.keys()) + .filter(|queue| !queues.contains(queue.as_str())) + .cloned() + .collect::>(); + for queue in stranded_queues { + conflicts.push(StrandedWork::QueueRemoved { + reviews: queue_reviews.get(&queue).copied().unwrap_or_default(), + work_items: inventory + .work_items + .get(&queue) + .copied() + .unwrap_or_default(), + queue, + }); + } + conflicts.extend( + profile_reviews + .into_iter() + .map(|(profile, reviews)| StrandedWork::ProfileRemoved { profile, reviews }), + ); + conflicts.extend( + changed_kinds + .into_iter() + .map( + |((review_kind, version), reviews)| StrandedWork::ReviewKindChanged { + review_kind, + version, + reviews, + }, + ), + ); + let bound = |source: &str| adapters.iter().any(|adapter| adapter.source_id() == source); + let stranded_sources = removed_sources + .keys() + .chain(inventory.work_items_by_source.keys()) + .filter(|source| !bound(source.as_str())) + .cloned() + .collect::>(); + for source in stranded_sources { + conflicts.push(StrandedWork::SourceRemoved { + reviews: removed_sources.get(&source).copied().unwrap_or_default(), + work_items: inventory + .work_items_by_source + .get(&source) + .copied() + .unwrap_or_default(), + source, + }); + } + conflicts.extend(context_fields.into_iter().map(|(key, reviews)| { + let ContextKey { + source, + entity, + review_kind, + version, + field, + reason, + } = key; + match reason { + ContextReason::NotDisplayed => StrandedWork::ContextFieldNotDisplayed { + source, + entity, + review_kind, + version, + field, + reviews, + }, + ContextReason::NotProjected => StrandedWork::DisplayedFieldNotProjected { + source, + entity, + review_kind, + version, + field, + reviews, + }, + } + })); + conflicts +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +enum ContextReason { + NotDisplayed, + NotProjected, +} + +#[derive(Debug, Eq, Ord, PartialEq, PartialOrd)] +struct ContextKey { + source: String, + entity: String, + review_kind: String, + version: String, + field: String, + reason: ContextReason, +} + +/// The property names a closed display schema declares and the ones it +/// requires. Review kinds admit only closed object schemas. +fn display_fields(schema: &Value) -> (BTreeSet<&str>, Vec<&str>) { + let declared = schema + .get("properties") + .and_then(Value::as_object) + .map(|properties| properties.keys().map(String::as_str).collect()) + .unwrap_or_default(); + let required = schema + .get("required") + .and_then(Value::as_array) + .map(|required| required.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default(); + (declared, required) +} + +#[cfg(test)] +mod tests { + use async_trait::async_trait; + use registry_casework_core::{ + AccessProfile, ActiveSubjectsPage, AuthoritativeObservation, CallerSubjectView, + CaseworkIdentity, CaseworkRole, DiscoveryCursor, EphemeralCredential, EventRequest, + ExecutePreparedRequest, PrepareActionRequest, PreparedSourceAttempt, QueuePolicy, + ReviewKindPolicy, ReviewKindPurpose, ReviewRetentionPolicy, ReviewStagePolicy, + SourceAdapterError, SourceReceipt, SubjectRef, TransitionHint, + }; + use serde_json::json; + + use super::*; + + fn stage(id: &str, queue: &str, profile: &str) -> ReviewStagePolicy { + ReviewStagePolicy { + id: id.to_owned(), + queue: queue.to_owned(), + deciding_profiles: vec![profile.to_owned()], + required_approvals: 1, + exclude_initiator: false, + exclude_previous_stage_reviewers: false, + } + } + + fn kind(strategy: ReviewContextStrategy) -> ReviewKindPolicy { + ReviewKindPolicy { + id: "correction".to_owned(), + version: "1".to_owned(), + purpose: ReviewKindPurpose::Approval, + context_strategy: strategy, + stages: vec![ + stage("first", "intake", "clerk"), + stage("second", "senior", "senior-officer"), + ], + clocks: Vec::new(), + retention: ReviewRetentionPolicy { + terminal_days: 30, + accountability_days: 365, + }, + display_schema: json!({ + "type": "object", + "additionalProperties": false, + "required": ["summary"], + "properties": {"summary": {"type": "string"}} + }), + result_schema: None, + outcomes: Vec::new(), + } + } + + fn profile(id: &str) -> AccessProfile { + AccessProfile { + id: id.to_owned(), + principal_claim: "sub".to_owned(), + required_scopes: Vec::new(), + role: CaseworkRole::Staff, + } + } + + fn project(kind: ReviewKindPolicy) -> CaseworkProject { + CaseworkProject { + api_version: registry_casework_core::CASEWORK_API_VERSION.to_owned(), + kind: registry_casework_core::CASEWORK_KIND.to_owned(), + casework: CaseworkIdentity { + id: "pinned".to_owned(), + version: "1".to_owned(), + }, + access_profiles: vec![profile("clerk"), profile("senior-officer")], + queues: ["intake", "senior"] + .into_iter() + .map(|id| QueuePolicy { + id: id.to_owned(), + label: id.to_owned(), + }) + .collect(), + sources: Vec::new(), + review_kinds: vec![kind], + review_producers: Vec::new(), + calendars: Vec::new(), + clocks: Vec::new(), + inbox: Default::default(), + task_templates: Vec::new(), + } + } + + fn group( + kind: &ReviewKindPolicy, + active_stage_index: usize, + reviews: u64, + ) -> PinnedReviewGroup { + PinnedReviewGroup { + policy: kind.snapshot().expect("pinned snapshot"), + subject_source: "registry".to_owned(), + subject_type: "licence".to_owned(), + active_stage_index, + reviews, + } + } + + struct Disclosing(Option>); + + #[async_trait] + impl SourceAdapter for Disclosing { + fn source_id(&self) -> &str { + "registry" + } + + fn binding_generation(&self) -> &str { + "generation" + } + + fn caller_disclosure_fields(&self, entity: &str) -> Option> { + assert_eq!(entity, "licence"); + self.0.clone() + } + + async fn verify_transition( + &self, + _request: EventRequest, + ) -> Result { + Err(SourceAdapterError::Invalid) + } + + async fn read_authoritative( + &self, + _subject: &SubjectRef, + ) -> Result { + Err(SourceAdapterError::Invalid) + } + + async fn discover_active( + &self, + _cursor: Option<&DiscoveryCursor>, + _limit: usize, + ) -> Result { + Err(SourceAdapterError::Invalid) + } + + async fn read_for_caller( + &self, + _subject: &SubjectRef, + _source_profile_id: &str, + _credential: EphemeralCredential<'_>, + ) -> Result { + Err(SourceAdapterError::Invalid) + } + + async fn prepare_action( + &self, + _request: PrepareActionRequest<'_>, + ) -> Result { + Err(SourceAdapterError::Invalid) + } + + async fn execute_prepared( + &self, + _request: ExecutePreparedRequest<'_>, + ) -> Result { + Err(SourceAdapterError::Invalid) + } + } + + #[test] + fn the_package_that_admitted_the_work_strands_nothing() { + let pinned = kind(ReviewContextStrategy::Submitted); + let inventory = PinnedWorkInventory { + reviews: vec![group(&pinned, 0, 3)], + work_items: BTreeMap::from([("intake".to_owned(), 2)]), + work_items_by_source: BTreeMap::new(), + }; + assert!(compare_pinned_work(&project(pinned), &[], &inventory).is_empty()); + } + + #[test] + fn a_removed_queue_counts_remaining_stages_and_open_work_items() { + let pinned = kind(ReviewContextStrategy::Submitted); + let inventory = PinnedWorkInventory { + // Two reviews still at the first stage and one already at the + // second: only the first two still need the intake queue. + reviews: vec![group(&pinned, 0, 2), group(&pinned, 1, 1)], + work_items: BTreeMap::from([("intake".to_owned(), 4), ("retired".to_owned(), 1)]), + work_items_by_source: BTreeMap::new(), + }; + let mut renamed = pinned.clone(); + renamed.version = "2".to_owned(); + renamed.stages[0].queue = "front-desk".to_owned(); + let mut candidate = project(renamed); + candidate.queues[0].id = "front-desk".to_owned(); + + let conflicts = compare_pinned_work(&candidate, &[], &inventory); + assert_eq!( + conflicts, + vec![ + StrandedWork::QueueRemoved { + queue: "intake".to_owned(), + reviews: 2, + work_items: 4, + }, + StrandedWork::QueueRemoved { + queue: "retired".to_owned(), + reviews: 0, + work_items: 1, + }, + ] + ); + assert_eq!( + describe_stranded_work(&conflicts), + "2 in-flight reviews and 4 open work items are in queue intake, which the package \ + no longer declares; 0 in-flight reviews and 1 open work item are in queue retired, \ + which the package no longer declares" + ); + } + + #[test] + fn a_removed_profile_and_changed_content_under_one_version_are_named() { + let pinned = kind(ReviewContextStrategy::Submitted); + let inventory = PinnedWorkInventory { + reviews: vec![group(&pinned, 0, 1), group(&pinned, 1, 2)], + work_items: BTreeMap::new(), + work_items_by_source: BTreeMap::new(), + }; + let mut renamed = pinned.clone(); + renamed.stages[1].deciding_profiles = vec!["licence-officer".to_owned()]; + let mut candidate = project(renamed); + candidate.access_profiles[1].id = "licence-officer".to_owned(); + + assert_eq!( + compare_pinned_work(&candidate, &[], &inventory), + vec![ + StrandedWork::ProfileRemoved { + profile: "senior-officer".to_owned(), + reviews: 3, + }, + StrandedWork::ReviewKindChanged { + review_kind: "correction".to_owned(), + version: "1".to_owned(), + reviews: 3, + }, + ] + ); + } + + #[test] + fn a_source_read_the_pinned_display_schema_refuses_is_named_by_field() { + let pinned = kind(ReviewContextStrategy::Source); + let inventory = PinnedWorkInventory { + reviews: vec![group(&pinned, 0, 5)], + work_items: BTreeMap::new(), + work_items_by_source: BTreeMap::new(), + }; + let candidate = project(pinned); + + let matching = Disclosing(Some(vec!["summary".to_owned()])); + assert!(compare_pinned_work(&candidate, &[&matching], &inventory).is_empty()); + + let undeclared = Disclosing(None); + assert!(compare_pinned_work(&candidate, &[&undeclared], &inventory).is_empty()); + + let widened = Disclosing(Some(vec!["summary".to_owned(), "reason".to_owned()])); + assert_eq!( + compare_pinned_work(&candidate, &[&widened], &inventory), + vec![StrandedWork::ContextFieldNotDisplayed { + source: "registry".to_owned(), + entity: "licence".to_owned(), + review_kind: "correction".to_owned(), + version: "1".to_owned(), + field: "reason".to_owned(), + reviews: 5, + }] + ); + + let narrowed = Disclosing(Some(Vec::new())); + assert_eq!( + compare_pinned_work(&candidate, &[&narrowed], &inventory), + vec![StrandedWork::DisplayedFieldNotProjected { + source: "registry".to_owned(), + entity: "licence".to_owned(), + review_kind: "correction".to_owned(), + version: "1".to_owned(), + field: "summary".to_owned(), + reviews: 5, + }] + ); + + assert_eq!( + compare_pinned_work(&candidate, &[], &inventory), + vec![StrandedWork::SourceRemoved { + source: "registry".to_owned(), + reviews: 5, + work_items: 0, + }] + ); + } + + #[test] + fn open_work_items_from_a_source_the_runtime_no_longer_binds_are_named() { + let pinned = kind(ReviewContextStrategy::Source); + let inventory = PinnedWorkInventory { + reviews: vec![group(&pinned, 0, 1)], + work_items: BTreeMap::from([("intake".to_owned(), 3)]), + work_items_by_source: BTreeMap::from([ + ("registry".to_owned(), 2), + ("retired-registry".to_owned(), 1), + ]), + }; + let candidate = project(pinned); + + let bound = Disclosing(Some(vec!["summary".to_owned()])); + let conflicts = compare_pinned_work(&candidate, &[&bound], &inventory); + assert_eq!( + conflicts, + vec![StrandedWork::SourceRemoved { + source: "retired-registry".to_owned(), + reviews: 0, + work_items: 1, + }] + ); + assert_eq!( + describe_stranded_work(&conflicts), + "0 in-flight reviews and 1 open work item depend on source retired-registry, which \ + the package no longer declares" + ); + + assert_eq!( + compare_pinned_work(&candidate, &[], &inventory), + vec![ + StrandedWork::SourceRemoved { + source: "registry".to_owned(), + reviews: 1, + work_items: 2, + }, + StrandedWork::SourceRemoved { + source: "retired-registry".to_owned(), + reviews: 0, + work_items: 1, + }, + ] + ); + } + + #[test] + fn only_an_acknowledgement_of_the_exact_package_digest_admits_stranded_work() { + let digest = format!("sha256:{}", "a".repeat(64)); + let other = format!("sha256:{}", "b".repeat(64)); + let conflicts = [StrandedWork::ProfileRemoved { + profile: "clerk".to_owned(), + reviews: 1, + }]; + assert_eq!( + pinned_work_verdict(&[], &digest, None), + PinnedWorkVerdict::Clear + ); + assert_eq!( + pinned_work_verdict(&conflicts, &digest, None), + PinnedWorkVerdict::Refused + ); + assert_eq!( + pinned_work_verdict(&conflicts, &digest, Some(&other)), + PinnedWorkVerdict::Refused + ); + assert_eq!( + pinned_work_verdict(&conflicts, &digest, Some(&digest)), + PinnedWorkVerdict::Acknowledged + ); + assert_eq!( + stranded_work_refusal(&conflicts, &digest), + format!( + "the policy package would strand work pinned under an earlier package: 1 \ + in-flight review can be decided only through access profile clerk, which the \ + package no longer declares. Let that work finish under the earlier package, or \ + set package.acknowledgeStrandedWork to {digest} to activate this package anyway" + ) + ); + } + + #[test] + fn conflicts_serialize_with_a_closed_reason_and_camel_case_counts() { + let conflict = StrandedWork::QueueRemoved { + queue: "intake".to_owned(), + reviews: 1, + work_items: 2, + }; + assert_eq!( + serde_json::to_value(&conflict).unwrap(), + json!({"reason": "queue-removed", "queue": "intake", "reviews": 1, "workItems": 2}) + ); + } +} diff --git a/crates/registry-casework/src/runtime.rs b/crates/registry-casework/src/runtime.rs index 61e3c0c526..2f6e63f092 100644 --- a/crates/registry-casework/src/runtime.rs +++ b/crates/registry-casework/src/runtime.rs @@ -8,7 +8,7 @@ use clap::{Arg, Command}; use registry_casework_breg::BregBinding; use registry_casework_core::{CaseworkProject, SourceAdapter}; use registry_platform_audit::{AuditProfile, AuditWriter}; -use registry_platform_config::{SecretProvider, SecretResolver}; +use registry_platform_config::SecretResolver; use registry_platform_httputil::{read_bounded, BearerToken, OutboundClientBuilder}; use thiserror::Error; use tokio::sync::mpsc; @@ -400,17 +400,41 @@ pub async fn migrate_from_path(path: impl AsRef) -> Result<(), RuntimeErro Ok(()) } +/// Refuse to activate a package that strands in-flight work pinned under an +/// earlier package, unless the operator acknowledged that exact package. +async fn check_pinned_work( + store: &PostgresStore, + project: &CaseworkProject, + adapters: &[Arc], + package_digest: &str, + acknowledged: Option<&str>, +) -> Result<(), RuntimeError> { + let adapters = adapters + .iter() + .map(|adapter| adapter.as_ref()) + .collect::>(); + let conflicts = crate::stranded_pinned_work(store, project, &adapters).await?; + match crate::pinned_work_verdict(&conflicts, package_digest, acknowledged) { + crate::PinnedWorkVerdict::Clear => Ok(()), + crate::PinnedWorkVerdict::Acknowledged => { + tracing::warn!( + stranded = %crate::describe_stranded_work(&conflicts), + "activating an acknowledged Casework policy package that strands pinned work" + ); + Ok(()) + } + crate::PinnedWorkVerdict::Refused => Err(RuntimeError::StrandedPinnedWork( + crate::stranded_work_refusal(&conflicts, package_digest), + )), + } +} + pub async fn serve_from_path(path: impl AsRef) -> Result<(), RuntimeError> { let config = RuntimeConfig::load(path)?; - match config.policy_package_digest()? { - Some(digest) => tracing::info!( - policy_package_digest = %digest, - "verified Casework policy package" - ), - None => tracing::info!("loading authored Casework policy for loopback development"), - } - let project_path = config.policy_path(); - let project = CaseworkProject::load(&project_path)?; + let package = config.load_package()?; + let package_digest = package.digest().to_owned(); + tracing::info!(package_digest = %package_digest, "verified Casework package"); + let project = package.project(); let secrets = secret_resolver(&config)?; let audit = open_audit(&config, &secrets, None).await?; let store = @@ -419,7 +443,6 @@ pub async fn serve_from_path(path: impl AsRef) -> Result<(), RuntimeError> validate_retained_completion_destinations(&store, &config.review_completion_destinations) .await?; - let project_root = config.package.root.as_path(); let mut adapters: Vec> = Vec::new(); for source in &project.sources { let binding = config @@ -427,11 +450,14 @@ pub async fn serve_from_path(path: impl AsRef) -> Result<(), RuntimeError> .get(&source.id) .ok_or_else(|| RuntimeError::SourceConfiguration(source.id.clone()))?; let adapter = binding - .build_adapter(source, project_root, &secrets) + .build_adapter_from_description( + source, + package + .source_description(&source.description) + .ok_or_else(|| RuntimeError::SourceConfiguration(source.id.clone()))?, + &secrets, + ) .map_err(|_| RuntimeError::SourceConfiguration(source.id.clone()))?; - store - .register_source_generation(&source.id, adapter.binding_generation()) - .await?; adapters.push(Arc::new(adapter)); } if config.sources.len() != adapters.len() { @@ -443,10 +469,24 @@ pub async fn serve_from_path(path: impl AsRef) -> Result<(), RuntimeError> .unwrap_or_default(); return Err(RuntimeError::SourceConfiguration(unmatched)); } + // Refuse before any activation step writes to the database. + check_pinned_work( + &store, + project, + &adapters, + &package_digest, + config.package.acknowledge_stranded_work.as_deref(), + ) + .await?; + for adapter in &adapters { + store + .register_source_generation(adapter.source_id(), adapter.binding_generation()) + .await?; + } let (verifier, keys) = config.oidc_verifier(&secrets).await?; let authenticator = Arc::new(CaseworkAuthenticator::new( - &project, + project, verifier, keys, config.authentication.oidc.human_identity.clone(), @@ -557,15 +597,31 @@ pub async fn serve_from_path(path: impl AsRef) -> Result<(), RuntimeError> )); } drop(worker_stopped); + let metrics_state = crate::metrics::MetricsState::new( + Arc::new(store), + config.sources.keys().cloned().collect(), + package_digest.clone(), + ); let app = router(HttpState { service, authenticator, - project: Arc::new(project), + project: Arc::new(project.clone()), }); - let listener = tokio::net::TcpListener::bind(config.listener.bind) + // Both sockets bind before either serves, so a metrics address already in + // use refuses startup instead of leaving an API without its telemetry. + let metrics = match &config.metrics_listener { + Some(metrics_listener) => Some(( + tokio::net::TcpListener::bind(metrics_listener.bind) + .await + .map_err(RuntimeError::MetricsListen)?, + crate::metrics::metrics_router(metrics_state), + )), + None => None, + }; + let listener = tokio::net::TcpListener::bind(config.listener.bind.socket_addr()) .await .map_err(RuntimeError::Listen)?; - let served = serve_until_worker_stops(listener, app, worker_stops).await; + let served = serve_until_worker_stops(listener, app, metrics, worker_stops).await; for worker in workers { worker.abort(); } @@ -586,16 +642,40 @@ fn reconciliation_timer(period: Duration) -> Interval { /// Serve until a supervised background loop stops. The listener never stops on /// its own, so a clean return means a worker stopped, and the process reports -/// that as a failure for whatever supervises it to restart. +/// that as a failure for whatever supervises it to restart. The optional +/// operator-private metrics listener serves beside it and stops with it. async fn serve_until_worker_stops( listener: tokio::net::TcpListener, app: axum::Router, + metrics: Option<(tokio::net::TcpListener, axum::Router)>, stops: mpsc::Receiver<&'static str>, ) -> Result<(), RuntimeError> { - axum::serve(listener, app) + let (api_stopped, mut metrics_stop) = tokio::sync::watch::channel(()); + let metrics = metrics.map(|(metrics_listener, metrics_app)| { + tokio::spawn(async move { + axum::serve(metrics_listener, metrics_app) + .with_graceful_shutdown(async move { + let _ = metrics_stop.changed().await; + }) + .await + }) + }); + let served = axum::serve(listener, app) .with_graceful_shutdown(worker_stop(stops)) - .await - .map_err(RuntimeError::Listen)?; + .await; + drop(api_stopped); + if let Some(metrics) = metrics { + match metrics.await { + Ok(Ok(())) => {} + Ok(Err(error)) => { + tracing::error!(error = %error, "the Casework metrics listener failed"); + } + Err(error) => { + tracing::error!(error = %error, "the Casework metrics listener stopped unexpectedly"); + } + } + } + served.map_err(RuntimeError::Listen)?; Err(RuntimeError::WorkerStopped) } @@ -650,7 +730,7 @@ pub async fn open_audit( secrets: &SecretResolver, process: Option<&str>, ) -> Result { - let audit_secret = resolve_audit_secret(secrets, &config.audit.hash_key_ref)?; + let audit_secret = resolve_audit_secret(secrets, config.audit.key.hash_key_ref.as_str())?; let audit_profile = AuditProfile::production_from_secret_bytes(zeroize::Zeroizing::new( audit_secret.expose_secret().to_vec(), )) @@ -692,27 +772,16 @@ fn resolve_audit_secret( } pub fn secret_resolver(config: &RuntimeConfig) -> Result { - let mut providers = Vec::new(); - if config.secret_providers.file.is_some() { - providers.push(SecretProvider::File); - } - if config.secret_providers.environment.is_some() { - providers.push(SecretProvider::Environment); - } - SecretResolver::new( - providers, - config - .secret_providers - .file - .as_ref() - .map_or_else(|| Path::new(""), |file| file.root.as_path()), - ) - .map_err(|_| RuntimeError::SecretConfiguration) + config + .secret_providers + .resolver() + .map_err(|_| RuntimeError::SecretConfiguration) } #[cfg(test)] mod tests { use chrono::{TimeZone as _, Utc}; + use registry_platform_config::SecretProvider; use uuid::Uuid; use wiremock::matchers::{body_json, header, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -1006,13 +1075,65 @@ mod tests { .expect("report a stopped worker"); let served = tokio::time::timeout( Duration::from_secs(5), - serve_until_worker_stops(listener, axum::Router::new(), stops), + serve_until_worker_stops(listener, axum::Router::new(), None, stops), ) .await .expect("the listener stops after a background worker stops"); assert!(matches!(served, Err(RuntimeError::WorkerStopped))); } + #[tokio::test] + async fn the_metrics_listener_serves_beside_the_api_and_stops_with_it() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind a local listener"); + let metrics = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind a local metrics listener"); + let metrics_address = metrics.local_addr().expect("metrics address"); + let metrics_app = + axum::Router::new().route("/version", axum::routing::get(|| async { "ok" })); + let (stopped, stops) = mpsc::channel(1); + let served = tokio::spawn(serve_until_worker_stops( + listener, + axum::Router::new(), + Some((metrics, metrics_app)), + stops, + )); + + let mut stream = tokio::net::TcpStream::connect(metrics_address) + .await + .expect("reach the metrics listener"); + stream + .write_all(b"GET /version HTTP/1.1\r\nhost: metrics\r\nconnection: close\r\n\r\n") + .await + .expect("send a request"); + let mut response = String::new(); + stream + .read_to_string(&mut response) + .await + .expect("read the response"); + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + assert!(response.ends_with("ok"), "{response}"); + + stopped + .send("clock") + .await + .expect("report a stopped worker"); + let served = tokio::time::timeout(Duration::from_secs(5), served) + .await + .expect("the listeners stop after a background worker stops") + .expect("the serving task completes"); + assert!(matches!(served, Err(RuntimeError::WorkerStopped))); + assert!( + tokio::net::TcpStream::connect(metrics_address) + .await + .is_err(), + "the metrics listener closes with the API listener" + ); + } + fn breg_binding(reconciliation_interval_milliseconds: u64) -> BregBinding { BregBinding { base_url: "https://registry.example.test".into(), @@ -1091,6 +1212,8 @@ pub enum RuntimeError { SecretConfiguration, #[error("the Casework source binding for source {0} is invalid")] SourceConfiguration(String), + #[error("{0}")] + StrandedPinnedWork(String), #[error("the Casework audit destination could not be initialized")] Audit, #[error("the Casework audit destination could not be initialized: {0}")] @@ -1107,6 +1230,8 @@ pub enum RuntimeError { Service(#[from] crate::ServiceError), #[error("the Casework listener failed")] Listen(#[source] std::io::Error), + #[error("the Casework metrics listener at metricsListener.bind failed")] + MetricsListen(#[source] std::io::Error), #[error("a Casework background worker stopped")] WorkerStopped, } diff --git a/crates/registry-casework/src/schema.rs b/crates/registry-casework/src/schema.rs index 9efb2f7049..143f9b434e 100644 --- a/crates/registry-casework/src/schema.rs +++ b/crates/registry-casework/src/schema.rs @@ -5,14 +5,14 @@ use std::collections::BTreeMap; use serde_json::{Map, Value}; +use registry_platform_config::blocks::SECRET_PROVIDER_PATTERN; + use crate::{RuntimeConfig, RUNTIME_CONFIG_API_VERSION, RUNTIME_CONFIG_KIND}; pub const RUNTIME_CONFIG_SCHEMA_FILE: &str = "runtime.schema.json"; pub const RUNTIME_CONFIG_SCHEMA_ID: &str = "https://id.registrystack.org/schemas/casework/runtime/runtime.v1alpha1.schema.json"; -const POLICY_DIGEST_SCHEMA_PATTERN: &str = "^sha256:[0-9a-f]{64}$"; -const SECRET_REFERENCE_SCHEMA_PATTERN: &str = - "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$"; +const PACKAGE_DIGEST_SCHEMA_PATTERN: &str = "^sha256:[0-9a-f]{64}$"; pub fn runtime_documents() -> Result, serde_json::Error> { let mut derived = serde_json::to_value(schemars::schema_for!(RuntimeConfig))?; @@ -40,11 +40,13 @@ pub fn runtime_documents() -> Result, serde_json: Ok([(RUNTIME_CONFIG_SCHEMA_FILE, rendered)].into()) } +/// State in the schema the bounds `RuntimeConfig::check` and +/// `validate_secret_references` enforce at load beyond the shared blocks, +/// which carry their own: operated paths are absolute, every Casework secret +/// field is a secret reference, and a static JWKS document names an enabled +/// provider. fn install_runtime_constraints(schema: &mut Value) { - for (definition, property) in [ - ("RuntimePackageConfig", "root"), - ("FileSecretProviderConfig", "root"), - ] { + for (definition, property) in [("RuntimePackageConfig", "root")] { set_definition_property( schema, definition, @@ -62,10 +64,6 @@ fn install_runtime_constraints(schema: &mut Value) { Value::String(registry_platform_audit::ABSOLUTE_AUDIT_PATH_PATTERN.to_owned()), ); for (definition, property) in [ - ("DatabaseConfig", "runtimeUrlRef"), - ("DatabaseConfig", "migrationUrlRef"), - ("DatabaseConfig", "trustedRootCertificateRef"), - ("AuditConfig", "hashKeyRef"), ("TaskAuthorityConfig", "signingKeyRef"), ("BregBinding", "clientIdRef"), ("BregBinding", "clientAssertionKeyRef"), @@ -79,30 +77,17 @@ fn install_runtime_constraints(schema: &mut Value) { definition, property, "pattern", - Value::String("^secret:(?:env|file)/".to_owned()), + Value::String(SECRET_PROVIDER_PATTERN.to_owned()), ); } set_definition_property( schema, "RuntimePackageConfig", - "expectedPolicyDigest", + "expectedDigest", "pattern", - Value::String(POLICY_DIGEST_SCHEMA_PATTERN.to_owned()), + Value::String(PACKAGE_DIGEST_SCHEMA_PATTERN.to_owned()), ); - set_jwks_document_reference_constraints(schema); - if let Some(providers) = schema - .get_mut("$defs") - .and_then(|definitions| definitions.get_mut("SecretProvidersConfig")) - .and_then(Value::as_object_mut) - { - providers.insert( - "anyOf".to_owned(), - serde_json::json!([ - {"required": ["file"], "properties": {"file": {"$ref": "#/$defs/FileSecretProviderConfig"}}}, - {"required": ["environment"], "properties": {"environment": {"$ref": "#/$defs/EnvironmentSecretProviderConfig"}}} - ]), - ); - } + set_jwks_document_provider_requirement(schema); if let Some(sources) = schema .get_mut("properties") .and_then(|properties| properties.get_mut("sources")) @@ -113,7 +98,6 @@ fn install_runtime_constraints(schema: &mut Value) { serde_json::json!({"minLength": 1}), ); } - set_assertion_issuer_constraints(schema); set_review_completion_auth_constraints(schema); set_audit_destination_constraints(schema); } @@ -216,114 +200,17 @@ fn set_review_completion_auth_constraints(schema: &mut Value) { ); } -/// State the bounds `RuntimeConfig::validate_assertion_issuers` applies, so a -/// document the published schema accepts is one the runtime starts on rather -/// than one it refuses after the operator has already written it. -fn set_assertion_issuer_constraints(schema: &mut Value) { - let Some(member) = schema - .pointer_mut("/$defs/OidcConfig/properties/assertionIssuers") - .and_then(Value::as_object_mut) - else { - return; - }; - member.insert( - "maxProperties".to_owned(), - Value::from(crate::config::MAXIMUM_ASSERTION_ISSUER_CLIENTS), - ); - member.insert( - "propertyNames".to_owned(), - serde_json::json!({ - "type": "string", - "minLength": 1, - "maxLength": crate::config::MAXIMUM_ASSERTION_ISSUER_CLIENT_BYTES, - }), - ); - member.insert( - "additionalProperties".to_owned(), - serde_json::json!({ - "type": "array", - "maxItems": crate::config::MAXIMUM_ASSERTION_ISSUERS_PER_CLIENT, - "uniqueItems": true, - "items": { - "type": "string", - "minLength": 1, - "maxLength": crate::config::MAXIMUM_ASSERTION_ISSUER_BYTES, - }, - }), - ); -} - -fn set_jwks_document_reference_constraints(schema: &mut Value) { - if let Some(variants) = schema - .pointer_mut("/$defs/OidcJwksSource/oneOf") - .and_then(Value::as_array_mut) - { - for variant in variants { - if let Some(document_reference) = variant - .pointer_mut("/properties/documentRef") - .and_then(Value::as_object_mut) - { - document_reference.insert( - "pattern".to_owned(), - Value::String(SECRET_REFERENCE_SCHEMA_PATTERN.to_owned()), - ); - } - } - } - +/// A static JWKS document reference names its provider by its prefix, so a +/// configuration carrying one must enable that provider. +fn set_jwks_document_provider_requirement(schema: &mut Value) { if let Some(root) = schema.as_object_mut() { root.insert( "allOf".to_owned(), - serde_json::json!([ - secret_provider_requirement("^secret:env/", "environment"), - secret_provider_requirement("^secret:file/", "file") - ]), + registry_platform_config::schema::jwks_document_provider_requirements(), ); } } -fn secret_provider_requirement(reference_pattern: &str, provider: &str) -> Value { - serde_json::json!({ - "if": { - "properties": { - "authentication": { - "properties": { - "oidc": { - "properties": { - "jwksSource": { - "properties": { - "documentRef": {"pattern": reference_pattern} - }, - "required": ["documentRef"] - } - }, - "required": ["jwksSource"] - } - }, - "required": ["oidc"] - } - }, - "required": ["authentication"] - }, - "then": { - "properties": { - "secretProviders": { - "properties": { - provider: { - "$ref": format!("#/$defs/{}SecretProviderConfig", match provider { - "environment" => "Environment", - "file" => "File", - _ => unreachable!("closed secret provider schema"), - }) - } - }, - "required": [provider] - } - } - } - }) -} - fn set_definition_property( schema: &mut Value, definition: &str, @@ -357,6 +244,11 @@ mod tests { use super::*; use crate::RuntimeConfigError; use jsonschema::{Draft, JSONSchema}; + use registry_platform_config::blocks::SECRET_REFERENCE_PATTERN; + use registry_platform_config::{ + ConfigBlockErrorKind, MAX_ASSERTION_ISSUERS_PER_CLIENT, MAX_ASSERTION_ISSUER_BYTES, + MAX_ASSERTION_ISSUER_CLIENTS, MAX_ASSERTION_ISSUER_CLIENT_BYTES, + }; fn runtime_schema() -> JSONSchema { let documents = runtime_documents().expect("the runtime schema generates"); @@ -387,7 +279,7 @@ mod tests { "apiVersion": RUNTIME_CONFIG_API_VERSION, "kind": RUNTIME_CONFIG_KIND, "package": {"root": "/var/lib/casework/package"}, - "listener": {"tlsTermination": "development-loopback"}, + "listener": {"bind": "127.0.0.1:8100", "tlsTermination": "development-loopback"}, "secretProviders": secret_providers, "database": { "runtimeUrlRef": supporting_reference, @@ -430,8 +322,17 @@ mod tests { ]) ); assert_eq!( - document["$defs"]["OidcJwksSource"]["oneOf"][1]["properties"]["documentRef"]["pattern"], - SECRET_REFERENCE_SCHEMA_PATTERN + document["$defs"]["JwksSource"]["oneOf"][2]["properties"]["documentRef"]["pattern"], + SECRET_REFERENCE_PATTERN + ); + // The shared blocks carry their own bounds into this schema. + assert_eq!( + document["$defs"]["AuditConfig"]["properties"]["hashKeyRef"]["$ref"], + "#/$defs/SecretReference" + ); + assert_eq!( + document["$defs"]["OidcConfig"]["properties"]["issuer"]["pattern"], + "^https?://" ); } @@ -439,15 +340,15 @@ mod tests { fn assertion_issuer_schema_states_the_bounds_the_runtime_enforces() { // The published schema is what an operator's editor reads before the // runtime ever sees the document, so it refuses the same maps - // `RuntimeConfig::validate_assertion_issuers` refuses at load, whose own - // coverage lives beside it in `config`. + // the shared `OidcClientsConfig::check` refuses at load, whose own + // coverage lives beside it in `registry-platform-config`. let schema = runtime_schema(); let with = |issuers: Value| { let mut instance = runtime_instance("secret:file/jwks.json", "file"); instance["authentication"]["oidc"]["assertionIssuers"] = issuers; instance }; - let many_clients = (0..=crate::config::MAXIMUM_ASSERTION_ISSUER_CLIENTS) + let many_clients = (0..=MAX_ASSERTION_ISSUER_CLIENTS) .map(|index| { ( format!("task-agent-{index}"), @@ -455,12 +356,12 @@ mod tests { ) }) .collect::>(); - let many_issuers = (0..=crate::config::MAXIMUM_ASSERTION_ISSUERS_PER_CLIENT) + let many_issuers = (0..=MAX_ASSERTION_ISSUERS_PER_CLIENT) .map(|index| Value::String(format!("https://exchange-{index}.example.test"))) .collect::>(); let mut long_client = Map::new(); long_client.insert( - "a".repeat(crate::config::MAXIMUM_ASSERTION_ISSUER_CLIENT_BYTES + 1), + "a".repeat(MAX_ASSERTION_ISSUER_CLIENT_BYTES + 1), serde_json::json!(["https://exchange.example.test"]), ); for (label, issuers) in [ @@ -479,7 +380,7 @@ mod tests { "over-long issuer", serde_json::json!({"task-agent": [format!( "https://{}.example.test", - "a".repeat(crate::config::MAXIMUM_ASSERTION_ISSUER_BYTES) + "a".repeat(MAX_ASSERTION_ISSUER_BYTES) )]}), ), ( @@ -654,8 +555,9 @@ mod tests { serde_json::from_value(instance.clone()).expect("the runtime shape parses"); assert!(matches!( config.check(), - Err(RuntimeConfigError::InvalidSecretReference { path }) - if path == "authentication.oidc.jwksSource.documentRef" + Err(RuntimeConfigError::Block(error)) + if error.kind() == ConfigBlockErrorKind::InvalidSecretReference + && error.field() == "authentication.oidc.jwksSource.documentRef" )); assert!( !schema.is_valid(&instance), @@ -676,8 +578,9 @@ mod tests { serde_json::from_value(instance.clone()).expect("the runtime shape parses"); assert!(matches!( config.check(), - Err(RuntimeConfigError::SecretProviderRequired { path }) - if path == "authentication.oidc.jwksSource.documentRef" + Err(RuntimeConfigError::Block(error)) + if error.kind() == ConfigBlockErrorKind::SecretProviderDisabled + && error.field() == "authentication.oidc.jwksSource.documentRef" )); assert!( !schema.is_valid(&instance), @@ -691,11 +594,11 @@ mod tests { } #[test] - fn expected_policy_digest_schema_matches_runtime_validation() { + fn expected_digest_schema_matches_runtime_validation() { let schema = runtime_schema(); let with = |digest: &str| { let mut instance = runtime_instance("secret:env/CASEWORK_JWKS", "environment"); - instance["package"]["expectedPolicyDigest"] = Value::String(digest.to_owned()); + instance["package"]["expectedDigest"] = Value::String(digest.to_owned()); instance }; assert!(schema.is_valid(&with(&format!("sha256:{}", "0a".repeat(32))))); diff --git a/crates/registry-casework/src/service.rs b/crates/registry-casework/src/service.rs index 95521b9277..20302faea9 100644 --- a/crates/registry-casework/src/service.rs +++ b/crates/registry-casework/src/service.rs @@ -15,7 +15,25 @@ use sha2::{Digest, Sha256}; use thiserror::Error; use uuid::Uuid; -use crate::{PostgresStore, StoreError}; +use crate::{PostgresStore, ReconciliationFailure, StoreError}; + +/// Readiness fails once a source's reconciliation has failed this many +/// consecutive passes, and recovers on the next pass that succeeds. +pub const RECONCILIATION_FAILURE_THRESHOLD: i32 = 5; + +/// The closed failure class recorded for a failed reconciliation pass. +fn reconciliation_failure(error: &ServiceError) -> ReconciliationFailure { + match error { + ServiceError::Adapter(SourceAdapterError::Unavailable | SourceAdapterError::Concealed) => { + ReconciliationFailure::SourceUnavailable + } + ServiceError::Adapter(_) | ServiceError::SourceProtocol | ServiceError::BindingMoved => { + ReconciliationFailure::SourceRefused + } + ServiceError::Store(_) => ReconciliationFailure::Store, + _ => ReconciliationFailure::Configuration, + } +} #[derive(serde::Serialize)] #[serde(rename_all = "camelCase")] @@ -124,6 +142,19 @@ impl CaseworkService { return Err(StoreError::AuditUnavailable.into()); } self.store.ready().await?; + let source_ids: Vec = self.adapters.keys().cloned().collect(); + if let Some(failing) = self + .store + .reconciliation_health(&source_ids) + .await? + .into_iter() + .find(|health| health.consecutive_failures >= RECONCILIATION_FAILURE_THRESHOLD) + { + return Err(ServiceError::ReconciliationFailing { + source_id: failing.source_id, + consecutive_failures: failing.consecutive_failures, + }); + } Ok(()) } @@ -182,33 +213,63 @@ impl CaseworkService { pub async fn synchronize_pending(&self, maximum: i64) -> Result { let subjects = self.store.claim_sync_batch(maximum.min(100), 30).await?; let mut applied = 0; + let mut failed = 0_usize; + let mut first_failure = None; + // A subject that fails keeps its claim lease and is read again later; + // the subjects behind it are still applied in this batch. for subject in subjects { - let adapter = self.adapter(&subject.source_id)?; - match adapter.read_authoritative(&subject).await { - Ok(observation) => { - let (routing, target) = self.routing_policy_for(&observation)?; - let routing_policy_digest = - routing_policy_digest(self.request_policy_for(&observation.subject)?)?; - let clock = self.clock_policy_for(&subject)?; - self.store - .apply_observation_with_policy_context( - &observation, - &routing.queue, - target, - Some(&routing), - Some(&routing_policy_digest), - clock.as_ref(), - ) - .await?; - applied += 1; + let outcome = match self.adapter(&subject.source_id) { + Ok(adapter) => self.synchronize_subject(adapter.as_ref(), &subject).await, + Err(error) => Err(error), + }; + match outcome { + Ok(()) => applied += 1, + Err(ServiceError::Adapter( + SourceAdapterError::Unavailable | SourceAdapterError::Concealed, + )) => {} + Err(error) => { + failed += 1; + first_failure.get_or_insert(error); } - Err(SourceAdapterError::Unavailable | SourceAdapterError::Concealed) => {} - Err(error) => return Err(error.into()), } } + if let Some(error) = first_failure { + tracing::warn!( + applied, + failed, + error = %error, + "Casework synchronization could not apply every claimed subject" + ); + return Err(error); + } Ok(applied) } + /// Read and apply one claimed subject. The caller decides whether a + /// failure stops its batch. + async fn synchronize_subject( + &self, + adapter: &dyn SourceAdapter, + subject: &SubjectRef, + ) -> Result<(), ServiceError> { + let observation = adapter.read_authoritative(subject).await?; + let (routing, target) = self.routing_policy_for(&observation)?; + let routing_policy_digest = + routing_policy_digest(self.request_policy_for(&observation.subject)?)?; + let clock = self.clock_policy_for(subject)?; + self.store + .apply_observation_with_policy_context( + &observation, + &routing.queue, + target, + Some(&routing), + Some(&routing_policy_digest), + clock.as_ref(), + ) + .await?; + Ok(()) + } + async fn synchronize_source_pending( &self, source_id: &str, @@ -226,37 +287,75 @@ impl CaseworkService { .await?; let mut applied = 0; let mut unavailable = false; + let mut first_failure = None; + let mut failed = 0_usize; + // A subject that fails keeps its claim lease and is read again on a + // later pass; the subjects behind it are still applied in this one. for subject in subjects { - match adapter.read_authoritative(&subject).await { - Ok(observation) => { - let (routing, target) = self.routing_policy_for(&observation)?; - let routing_policy_digest = - routing_policy_digest(self.request_policy_for(&observation.subject)?)?; - let clock = self.clock_policy_for(&subject)?; - self.store - .apply_observation_with_policy_context( - &observation, - &routing.queue, - target, - Some(&routing), - Some(&routing_policy_digest), - clock.as_ref(), - ) - .await?; - applied += 1; - } - Err(SourceAdapterError::Unavailable | SourceAdapterError::Concealed) => { - unavailable = true + match self.synchronize_subject(adapter.as_ref(), &subject).await { + Ok(()) => applied += 1, + Err(ServiceError::Adapter( + SourceAdapterError::Unavailable | SourceAdapterError::Concealed, + )) => unavailable = true, + Err(error) => { + failed += 1; + first_failure.get_or_insert(error); } - Err(error) => return Err(error.into()), } } + if let Some(error) = first_failure { + tracing::warn!( + source_id, + applied, + failed, + error = %error, + "Casework reconciliation could not apply every claimed subject" + ); + return Err(error); + } Ok((applied, unavailable)) } /// Run the two independent repair passes: remote-active discovery, then - /// authoritative reads of every locally active subject. + /// authoritative reads of every locally active subject. The outcome is + /// recorded per source, so readiness and `caseworkctl doctor` report a + /// reconciliation that keeps failing. pub async fn reconcile_source(&self, source_id: &str) -> Result { + let adapter = self.adapter(source_id)?; + let outcome = self.reconcile_source_pass(source_id).await; + let failure = outcome.as_ref().err().map(reconciliation_failure); + match self + .store + .record_reconciliation_outcome(source_id, adapter.binding_generation(), failure) + .await + { + Ok(consecutive_failures) => { + if let Some(failure) = failure { + if consecutive_failures == RECONCILIATION_FAILURE_THRESHOLD { + tracing::error!( + source_id, + consecutive_failures, + failure = failure.as_str(), + "Casework reconciliation keeps failing; readiness fails until a pass succeeds" + ); + } + } + } + Err(error) => { + tracing::warn!( + source_id, + error = %error, + "Casework reconciliation outcome could not be recorded" + ); + if outcome.is_ok() { + return Err(error.into()); + } + } + } + outcome + } + + async fn reconcile_source_pass(&self, source_id: &str) -> Result { let adapter = self.adapter(source_id)?; self.store .begin_reconciliation_cycle(source_id, adapter.binding_generation()) @@ -2080,6 +2179,13 @@ fn local_actions( pub enum ServiceError { #[error("the Casework service configuration is invalid")] Configuration, + #[error( + "reconciliation of source {source_id} failed {consecutive_failures} consecutive passes" + )] + ReconciliationFailing { + source_id: String, + consecutive_failures: i32, + }, #[error("the source is not registered")] Source, #[error("the source response does not match its registration")] diff --git a/crates/registry-casework/src/store.rs b/crates/registry-casework/src/store.rs index 7478c59123..45bdcecf7f 100644 --- a/crates/registry-casework/src/store.rs +++ b/crates/registry-casework/src/store.rs @@ -45,9 +45,11 @@ const UNIFIED_REVIEWS_MIGRATION: &str = include_str!("../migrations/0015_unified const OCCURRENCE_IDENTITY_MIGRATION: &str = include_str!("../migrations/0016_occurrence_identity_excludes_superseded.sql"); const AUDIT_WRITER_MIGRATION: &str = include_str!("../migrations/0017_audit_writer.sql"); +const SOURCE_RECONCILIATION_HEALTH_MIGRATION: &str = + include_str!("../migrations/0018_source_reconciliation_health.sql"); /// Every schema version in ledger order. -const MIGRATIONS: [(i64, &str); 17] = [ +const MIGRATIONS: [(i64, &str); 18] = [ (1, MIGRATION), (2, HOSTED_MIGRATION), (3, ASSIGNMENT_MIGRATION), @@ -65,6 +67,7 @@ const MIGRATIONS: [(i64, &str); 17] = [ (15, UNIFIED_REVIEWS_MIGRATION), (16, OCCURRENCE_IDENTITY_MIGRATION), (17, AUDIT_WRITER_MIGRATION), + (18, SOURCE_RECONCILIATION_HEALTH_MIGRATION), ]; /// The newest schema version this binary knows how to run against. @@ -83,6 +86,13 @@ fn refuse_newer_schema(newest_applied: Option) -> Result<(), StoreError> { } } +fn applied_schema(applied: Option) -> String { + match applied { + Some(version) => format!("its newest applied migration is version {version}"), + None => "no migration has been applied".to_owned(), + } +} + /// The schema version that replaces the hosted work tables with unified /// reviews, and the tables it drops, in its drop order. const HOSTED_WORK_DROP_VERSION: i64 = 15; @@ -181,6 +191,33 @@ async fn refuse_to_drop_unpublished_audit( } } +/// Bound how long a database connection takes to notice a server that +/// stopped answering, for every setting the database URL leaves unset. +/// +/// A connection that hangs on a dead server must fail within seconds instead +/// of the operating system's hours, so a request, a worker pass, or a +/// readiness probe waiting on it fails visibly. Keepalive probes find a silent peer on an +/// idle connection; the TCP user timeout, which Linux honours, fails a +/// connection whose sent data stays unacknowledged. A URL that sets any of +/// these, or turns keepalives off, keeps its own choice. +fn bound_database_connection(postgres: &mut PgConfig) { + if postgres.get_connect_timeout().is_none() { + postgres.connect_timeout(Duration::from_secs(5)); + } + if postgres.get_keepalives_idle() == PgConfig::new().get_keepalives_idle() { + postgres.keepalives_idle(Duration::from_secs(15)); + } + if postgres.get_keepalives_interval().is_none() { + postgres.keepalives_interval(Duration::from_secs(5)); + } + if postgres.get_keepalives_retries().is_none() { + postgres.keepalives_retries(3); + } + if postgres.get_tcp_user_timeout().is_none() { + postgres.tcp_user_timeout(Duration::from_secs(30)); + } +} + /// Serializes operator-run migrations on one session lock. A second migrator /// waits here instead of racing the ledger primary key. The key spells the /// ASCII bytes of "casework". @@ -261,6 +298,7 @@ impl PostgresStore { if postgres.get_user().is_none() || postgres.get_dbname().is_none() { return Err(StoreError::Configuration); } + bound_database_connection(&mut postgres); let manager_config = ManagerConfig { recycling_method: RecyclingMethod::Verified, }; @@ -431,6 +469,19 @@ impl PostgresStore { pub async fn ready(&self) -> Result<(), StoreError> { let client = self.client().await?; + let ledger_exists: bool = client + .query_one( + "SELECT to_regclass('casework_schema_migrations') IS NOT NULL", + &[], + ) + .await? + .get(0); + if !ledger_exists { + return Err(StoreError::SchemaNotCurrent { + applied: None, + required: SUPPORTED_SCHEMA_VERSION, + }); + } let applied = client .query( "SELECT version FROM casework_schema_migrations ORDER BY version", @@ -454,7 +505,13 @@ impl PostgresStore { if schema_is_current { Ok(()) } else { - Err(StoreError::Corrupt) + Err(StoreError::SchemaNotCurrent { + applied: applied + .last() + .map(|row| row.try_get::<_, i64>(0)) + .transpose()?, + required: SUPPORTED_SCHEMA_VERSION, + }) } } @@ -3277,6 +3334,126 @@ impl PostgresStore { Ok(()) } + /// Record the outcome of one reconciliation pass for a source and return + /// the number of consecutive failed passes after it. A success resets the + /// count and keeps the last failure for the operator to read. + pub async fn record_reconciliation_outcome( + &self, + source_id: &str, + generation: &str, + failure: Option, + ) -> Result { + let client = self.client().await?; + let row = match failure { + None => client.query_one( + "INSERT INTO casework_source_status(source_id,binding_generation,remote_complete,unavailable,checked_at,consecutive_failures,last_succeeded_at) VALUES($1,$2,false,false,now(),0,now()) ON CONFLICT(source_id) DO UPDATE SET consecutive_failures=0,last_succeeded_at=EXCLUDED.last_succeeded_at RETURNING consecutive_failures", + &[&source_id, &generation], + ).await?, + Some(failure) => client.query_one( + "INSERT INTO casework_source_status(source_id,binding_generation,remote_complete,unavailable,checked_at,consecutive_failures,last_failed_at,last_failure) VALUES($1,$2,false,false,now(),1,now(),$3) ON CONFLICT(source_id) DO UPDATE SET consecutive_failures=LEAST(casework_source_status.consecutive_failures,2147483646)+1,last_failed_at=EXCLUDED.last_failed_at,last_failure=EXCLUDED.last_failure RETURNING consecutive_failures", + &[&source_id, &generation, &failure.as_str()], + ).await?, + }; + Ok(row.get(0)) + } + + /// The recorded reconciliation health of each named source, in the order + /// given. A source with no recorded pass reports no pass and no failure. + pub async fn reconciliation_health( + &self, + source_ids: &[String], + ) -> Result, StoreError> { + let client = self.client().await?; + let rows = client + .query( + "SELECT source_id,consecutive_failures,last_succeeded_at,last_failed_at,last_failure FROM casework_source_status WHERE source_id=ANY($1)", + &[&source_ids], + ) + .await?; + let mut recorded = std::collections::BTreeMap::new(); + for row in rows { + let failure: Option = row.try_get(4)?; + let last_failure = failure + .as_deref() + .map(ReconciliationFailure::parse) + .transpose()?; + recorded.insert( + row.try_get::<_, String>(0)?, + SourceReconciliationHealth { + source_id: String::new(), + consecutive_failures: row.try_get(1)?, + last_succeeded_at: row.try_get(2)?, + last_failed_at: row.try_get(3)?, + last_failure, + }, + ); + } + Ok(source_ids + .iter() + .map(|source_id| { + let mut health = recorded.remove(source_id).unwrap_or_default(); + health.source_id.clone_from(source_id); + health + }) + .collect()) + } + + /// In-flight reviews grouped by pinned policy, subject source and type, + /// and active stage, plus open work items per queue and per source. The inventory holds + /// counts and pinned policy only, never subject data. + pub(crate) async fn pinned_work_inventory( + &self, + ) -> Result { + let client = self.client().await?; + let mut inventory = crate::pinned_work::PinnedWorkInventory::default(); + for row in client + .query( + "SELECT policy_snapshot,subject_source,subject_type,active_stage_index,count(*) FROM casework_review_requests WHERE lifecycle='reviewing' GROUP BY policy_snapshot,subject_source,subject_type,active_stage_index", + &[], + ) + .await? + { + let active_stage_index: i32 = row.try_get(3)?; + let reviews: i64 = row.try_get(4)?; + inventory.reviews.push(crate::pinned_work::PinnedReviewGroup { + policy: serde_json::from_value(row.try_get(0)?) + .map_err(|_| StoreError::Corrupt)?, + subject_source: row.try_get(1)?, + subject_type: row.try_get(2)?, + active_stage_index: usize::try_from(active_stage_index) + .map_err(|_| StoreError::Corrupt)?, + reviews: u64::try_from(reviews).map_err(|_| StoreError::Corrupt)?, + }); + } + for row in client + .query( + "SELECT queue_id,count(*) FROM casework_items WHERE erased_at IS NULL AND state NOT IN ('completed','superseded','cancelled') GROUP BY queue_id", + &[], + ) + .await? + { + let items: i64 = row.try_get(1)?; + inventory.work_items.insert( + row.try_get(0)?, + u64::try_from(items).map_err(|_| StoreError::Corrupt)?, + ); + } + for row in client + .query( + "SELECT source_id,count(*) FROM casework_items WHERE erased_at IS NULL AND state NOT IN ('completed','superseded','cancelled') GROUP BY source_id", + &[], + ) + .await? + { + let items: i64 = row.try_get(1)?; + inventory.work_items_by_source.insert( + row.try_get(0)?, + u64::try_from(items).map_err(|_| StoreError::Corrupt)?, + ); + } + Ok(inventory) + } + pub async fn source_status( &self, source_id: &str, @@ -3801,6 +3978,91 @@ fn history_from_row(row: Row) -> Result { mod tests { use super::*; + #[test] + fn migrations_are_contiguous_and_the_outbox_drop_is_the_audit_writer_migration() { + let mut files: Vec<(i64, String)> = + std::fs::read_dir(concat!(env!("CARGO_MANIFEST_DIR"), "/migrations")) + .expect("read the migrations directory") + .map(|entry| { + let name = entry + .expect("read a migrations entry") + .file_name() + .into_string() + .expect("a UTF-8 migration file name"); + let version = name + .split_once('_') + .and_then(|(prefix, _)| prefix.parse().ok()) + .unwrap_or_else(|| panic!("{name} starts with a version number")); + (version, name) + }) + .collect(); + files.sort(); + let versions: Vec = MIGRATIONS.iter().map(|(version, _)| *version).collect(); + let expected: Vec = (1..=i64::try_from(MIGRATIONS.len()).unwrap()).collect(); + assert_eq!( + versions, expected, + "the ledger versions run 1..=N without a gap" + ); + assert_eq!( + files + .iter() + .map(|(version, _)| *version) + .collect::>(), + expected, + "every migration file has its own version and the ledger names each one" + ); + let (_, drop_file) = &files[usize::try_from(AUDIT_OUTBOX_DROP_VERSION - 1).unwrap()]; + assert_eq!(drop_file, "0017_audit_writer.sql"); + assert_eq!( + MIGRATIONS[usize::try_from(AUDIT_OUTBOX_DROP_VERSION - 1).unwrap()].1, + AUDIT_WRITER_MIGRATION, + "the outbox drop version is the migration that installs the audit writer" + ); + } + + #[test] + fn database_connections_detect_a_dead_server_unless_the_url_says_otherwise() { + let mut bounded = + PgConfig::from_str("postgresql://casework@db.example/casework").expect("database url"); + bound_database_connection(&mut bounded); + assert_eq!(bounded.get_connect_timeout(), Some(&Duration::from_secs(5))); + assert!(bounded.get_keepalives()); + assert_eq!(bounded.get_keepalives_idle(), Duration::from_secs(15)); + assert_eq!( + bounded.get_keepalives_interval(), + Some(Duration::from_secs(5)) + ); + assert_eq!(bounded.get_keepalives_retries(), Some(3)); + assert_eq!( + bounded.get_tcp_user_timeout(), + Some(&Duration::from_secs(30)) + ); + + let mut chosen = PgConfig::from_str( + "postgresql://casework@db.example/casework?connect_timeout=3&keepalives_idle=60\ + &keepalives_interval=7&keepalives_retries=9&tcp_user_timeout=90", + ) + .expect("database url"); + bound_database_connection(&mut chosen); + assert_eq!(chosen.get_connect_timeout(), Some(&Duration::from_secs(3))); + assert_eq!(chosen.get_keepalives_idle(), Duration::from_secs(60)); + assert_eq!( + chosen.get_keepalives_interval(), + Some(Duration::from_secs(7)) + ); + assert_eq!(chosen.get_keepalives_retries(), Some(9)); + assert_eq!( + chosen.get_tcp_user_timeout(), + Some(&Duration::from_secs(90)) + ); + + let mut disabled = + PgConfig::from_str("postgresql://casework@db.example/casework?keepalives=0") + .expect("database url"); + bound_database_connection(&mut disabled); + assert!(!disabled.get_keepalives()); + } + #[test] fn settlement_text_is_bounded_non_empty_and_free_of_control_characters() { let at_bound = "x".repeat(256); @@ -4058,6 +4320,54 @@ impl From for AttemptSettlementError { } } +/// Why a reconciliation pass failed, as a closed vocabulary an operator can +/// read without the source's or the database's own error text. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum ReconciliationFailure { + /// The source could not be reached or did not answer the reader. + SourceUnavailable, + /// The source answered with something the adapter refused. + SourceRefused, + /// The Casework database refused or failed an operation. + Store, + /// The served package or runtime configuration could not handle the work. + Configuration, +} + +impl ReconciliationFailure { + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::SourceUnavailable => "source-unavailable", + Self::SourceRefused => "source-refused", + Self::Store => "store", + Self::Configuration => "configuration", + } + } + + fn parse(value: &str) -> Result { + match value { + "source-unavailable" => Ok(Self::SourceUnavailable), + "source-refused" => Ok(Self::SourceRefused), + "store" => Ok(Self::Store), + "configuration" => Ok(Self::Configuration), + _ => Err(StoreError::Corrupt), + } + } +} + +/// The recorded outcome of a source's reconciliation passes. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SourceReconciliationHealth { + pub source_id: String, + pub consecutive_failures: i32, + pub last_succeeded_at: Option>, + pub last_failed_at: Option>, + pub last_failure: Option, +} + #[derive(Debug, Error)] pub enum StoreError { #[error("the Casework database configuration is invalid")] @@ -4102,6 +4412,11 @@ pub enum StoreError { "the Casework database schema version {found} is newer than this binary supports ({supported}); run a casework release that supports it" )] SchemaNewer { found: i64, supported: i64 }, + #[error( + "the Casework database schema is not current: {}, and this binary requires version {required}; apply the migrations with `casework migrate` or `caseworkctl db migrate`", + applied_schema(*.applied) + )] + SchemaNotCurrent { applied: Option, required: i64 }, #[error( "the Casework database holds hosted work that schema migration {version} would drop: {}; nothing was changed. This release does not carry hosted work forward: keep this database with the release that wrote it until the work it holds is exported, then migrate a fresh Casework database for this release", hosted_row_counts(.tables) diff --git a/crates/registry-casework/src/task_grants/local_session_tests.rs b/crates/registry-casework/src/task_grants/local_session_tests.rs index e114d1005b..61d3a54bdc 100644 --- a/crates/registry-casework/src/task_grants/local_session_tests.rs +++ b/crates/registry-casework/src/task_grants/local_session_tests.rs @@ -181,7 +181,9 @@ async fn create_review(url: &str, token: &str, body: Value) -> (StatusCode, Valu #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "requires Docker, built casework/caseworkctl and disposable BREG_TEST_DATABASE_URL"] async fn source_backed_dev_approves_exchanges_and_revokes_on_stock_issuer() { - let workspace = tempfile::tempdir().unwrap(); + // The runtime configuration loader refuses a path through a symbolic + // link, and the system temporary root is one on some hosts. + let workspace = tempfile::tempdir_in(fs::canonicalize(std::env::temp_dir()).unwrap()).unwrap(); let held = (0..4) .map(|_| std::net::TcpListener::bind("127.0.0.1:0").unwrap()) .collect::>(); diff --git a/crates/registry-casework/src/task_grants/native_exchange_tests.rs b/crates/registry-casework/src/task_grants/native_exchange_tests.rs index 681027e3ea..bd89e22fa7 100644 --- a/crates/registry-casework/src/task_grants/native_exchange_tests.rs +++ b/crates/registry-casework/src/task_grants/native_exchange_tests.rs @@ -189,14 +189,25 @@ struct EvidenceDeployment { base_url: String, _runtime: Arc, server: tokio::task::JoinHandle>, - bundle_root: PathBuf, - runtime_path: PathBuf, + _prepared: PreparedEvidence, _source: MockServer, } impl Drop for EvidenceDeployment { fn drop(&mut self) { self.server.abort(); + } +} + +/// A sealed Evidence bundle and its runtime file, unsealed again on drop so +/// the temporary root can be removed. +struct PreparedEvidence { + bundle_root: PathBuf, + runtime_path: PathBuf, +} + +impl Drop for PreparedEvidence { + fn drop(&mut self) { let _ = fs::set_permissions(&self.runtime_path, fs::Permissions::from_mode(0o644)); unseal(&self.bundle_root); } @@ -216,6 +227,51 @@ async fn start_evidence(root: &Path, issuer: &Issuer) -> EvidenceDeployment { let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); drop(listener); + let prepared = prepare_evidence(root, &source.uri(), &issuer.url(), port); + let runtime = Arc::new( + EvidenceRuntime::initialize(&prepared.runtime_path) + .await + .unwrap(), + ); + let served = Arc::clone(&runtime); + let server = + tokio::spawn( + async move { evidence_server::serve(served, std::future::pending::<()>()).await }, + ); + let base_url = format!("http://127.0.0.1:{port}"); + let probe = reqwest::Client::builder().no_proxy().build().unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + if probe + .get(format!("{base_url}/ready")) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + EvidenceDeployment { + base_url, + _runtime: runtime, + server, + _prepared: prepared, + _source: source, + } +} + +/// Copy the adult-status acceptance bundle, point it at the local source and +/// issuer, write its runtime file and secrets, and seal the bundle. +fn prepare_evidence( + root: &Path, + source_origin: &str, + issuer_origin: &str, + port: u16, +) -> PreparedEvidence { let evidence_root = root.join("evidence-deployment"); let bundle_root = evidence_root.join("bundle"); let secret_root = evidence_root.join("secrets"); @@ -231,9 +287,9 @@ async fn start_evidence(root: &Path, issuer: &Issuer) -> EvidenceDeployment { ); rewrite_evidence_fixture( &bundle_root, - &source.uri(), - &issuer.url(), - &format!("{}/oauth2/jwks", issuer.url()), + source_origin, + issuer_origin, + &format!("{issuer_origin}/oauth2/jwks"), &format!("http://127.0.0.1:{port}"), ); write_secret( @@ -251,11 +307,12 @@ async fn start_evidence(root: &Path, issuer: &Issuer) -> EvidenceDeployment { fs::write( &runtime_path, format!( - r#"version: 1 -bundleDirectory: {bundle} + r#"apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: {bundle} listener: - bindHost: 127.0.0.1 - port: {port} + bind: 127.0.0.1:{port} tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 @@ -281,40 +338,35 @@ outboundTls: ) .unwrap(); fs::set_permissions(&runtime_path, fs::Permissions::from_mode(0o444)).unwrap(); + refresh_package_envelope(&bundle_root); seal(&bundle_root); - let runtime = Arc::new(EvidenceRuntime::initialize(&runtime_path).await.unwrap()); - let served = Arc::clone(&runtime); - let server = - tokio::spawn( - async move { evidence_server::serve(served, std::future::pending::<()>()).await }, - ); - let base_url = format!("http://127.0.0.1:{port}"); - let probe = reqwest::Client::builder().no_proxy().build().unwrap(); - tokio::time::timeout(std::time::Duration::from_secs(10), async { - loop { - if probe - .get(format!("{base_url}/ready")) - .send() - .await - .is_ok_and(|response| response.status().is_success()) - { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - EvidenceDeployment { - base_url, - _runtime: runtime, - server, + PreparedEvidence { bundle_root, runtime_path, - _source: source, } } +// The cross-product journey above needs Docker and two databases; this proves +// without either that the rewritten Evidence fixture and runtime file still +// load through Evidence's own deployment loader. +#[test] +fn the_rewritten_evidence_deployment_loads_through_the_evidence_loader() { + let root = canonical_tempdir(); + let prepared = prepare_evidence( + root.path(), + "http://127.0.0.1:9", + "http://127.0.0.1:10", + 8080, + ); + registry_evidence::bundle::DeploymentInputs::load(&prepared.runtime_path).unwrap(); +} + +/// The runtime configuration loaders refuse a path through a symbolic link, +/// and the system temporary root is one on some hosts. +fn canonical_tempdir() -> tempfile::TempDir { + tempfile::tempdir_in(fs::canonicalize(std::env::temp_dir()).unwrap()).unwrap() +} + fn rewrite_evidence_fixture( bundle_root: &Path, source_origin: &str, @@ -339,17 +391,17 @@ fn rewrite_evidence_fixture( &format!("issuer: {issuer_origin}"), ), ( - "audiences: [evidence-fixture]", - &format!("audiences: [{EVIDENCE_RESOURCE}]"), + "audience: evidence-fixture", + &format!("audience: {EVIDENCE_RESOURCE}"), ), ( - "jwksUri: https://identity.invalid/.well-known/jwks.json", - &format!("jwksUri: {issuer_jwks_uri}"), + "uri: https://identity.invalid/.well-known/jwks.json", + &format!("uri: {issuer_jwks_uri}"), ), ("algorithms: [ES256]", "algorithms: [RS256]"), ( - " principalClaim: sub\n requesterTagsClaim: evidence_tags", - " principalClaim: sub\n allowedClients: [evidence-task-agent]\n requesterTagsClaim: evidence_tags", + " principalClaim: sub\n requesterTagsClaim: evidence_tags", + " principalClaim: sub\n allowedClients: [evidence-task-agent]\n requesterTagsClaim: evidence_tags", ), ( " statutory-caseworker-v1:\n kind: statutory", @@ -391,6 +443,27 @@ fn copy_tree(source: &Path, target: &Path) { } } +/// Republish the staged Evidence package after its intended authored changes. +fn refresh_package_envelope(root: &Path) { + let sum_file = root.join(registry_platform_config::SUM_FILE); + if sum_file.exists() { + fs::remove_file(&sum_file).unwrap(); + } + registry_platform_config::write_sum_file( + root, + None, + ®istry_platform_config::PackageLimits { + max_files: registry_evidence::bundle::MAX_BUNDLE_FILES, + max_file_bytes: registry_evidence::bundle::MAX_ARTIFACT_BYTES, + max_total_bytes: registry_evidence::bundle::MAX_BUNDLE_BYTES, + max_depth: 3, + max_path_bytes: 128, + }, + "evidencectl package", + ) + .unwrap(); +} + fn seal(root: &Path) { for entry in fs::read_dir(root).unwrap() { let entry = entry.unwrap(); @@ -839,7 +912,7 @@ async fn request( #[ignore = "requires Docker, disposable CASEWORK_ASSIGNMENT_TEST_DATABASE_URL and BREG_TEST_DATABASE_URL, and absolute SCHEDULING_AUTH_PROBE_BIN"] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn approved_casework_tasks_reach_evidence_breg_and_scheduling_through_stock_thunderid() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let listener = tokio::net::TcpListener::bind("0.0.0.0:0").await.unwrap(); let casework_port = listener.local_addr().unwrap().port(); let (human_key, agent_key, evidence_agent_key, status_key, seed_key) = ( diff --git a/crates/registry-casework/tests/postgres_transactions.rs b/crates/registry-casework/tests/postgres_transactions.rs index 2e6c607184..746a4392d7 100644 --- a/crates/registry-casework/tests/postgres_transactions.rs +++ b/crates/registry-casework/tests/postgres_transactions.rs @@ -994,7 +994,7 @@ async fn repeated_migration_is_a_ledger_no_op_and_never_drops_the_occurrence_ind let (store, client, schema) = isolated_schema("migrate").await; store.migrate().await.expect("first migration"); let applied = applied_versions(&client).await; - assert_eq!(applied, (1..=17).collect::>()); + assert_eq!(applied, (1..=18).collect::>()); let index = occurrence_index(&client, &schema).await; assert!(index.1, "the occurrence identity index is unique"); @@ -1073,6 +1073,53 @@ async fn returning_to_an_earlier_binding_generation_opens_a_fresh_occurrence() { ); } +fn queue_project(queue: &str) -> registry_casework_core::CaseworkProject { + serde_json::from_value(serde_json::json!({ + "apiVersion": registry_casework_core::CASEWORK_API_VERSION, + "kind": registry_casework_core::CASEWORK_KIND, + "casework": {"id": "pinned", "version": "1"}, + "accessProfiles": [{"id": "staff", "principalClaim": "sub", "requiredScopes": [], "role": "staff"}], + "queues": [{"id": queue, "label": "Queue"}] + })) + .expect("queue project") +} + +#[tokio::test] +async fn activation_preflight_counts_open_work_items_in_a_removed_queue() { + let (store, _client, _schema) = isolated_schema("pinned_queue").await; + store.migrate().await.expect("migrate"); + // The first item is superseded by the second, so only one stays open. + observe_open_in_generation(&store, "binding-a").await; + observe_open_in_generation(&store, "binding-b").await; + // No adapter binds the item's source here, so it is stranded by source + // as well as, once the queue goes, by queue. + let source_removed = registry_casework::StrandedWork::SourceRemoved { + source: "source-a".to_owned(), + reviews: 0, + work_items: 1, + }; + + let kept = registry_casework::stranded_pinned_work(&store, &queue_project("default"), &[]) + .await + .expect("preflight with the queue kept"); + assert_eq!(kept, vec![source_removed.clone()]); + + let removed = registry_casework::stranded_pinned_work(&store, &queue_project("triage"), &[]) + .await + .expect("preflight with the queue removed"); + assert_eq!( + removed, + vec![ + registry_casework::StrandedWork::QueueRemoved { + queue: "default".to_owned(), + reviews: 0, + work_items: 1, + }, + source_removed, + ] + ); +} + async fn item_reference_revision_and_history( client: &tokio_postgres::Client, item_id: uuid::Uuid, @@ -1192,7 +1239,7 @@ async fn migration_16_releases_superseded_identities_in_a_database_that_holds_th assert_eq!( applied_versions(&client).await, - (1..=17).collect::>() + (1..=18).collect::>() ); let second_a = observe_open_in_generation(&store, "binding-a").await; let states: Vec<(uuid::Uuid, String)> = items_by_state(&client) @@ -1343,7 +1390,7 @@ async fn migration_refuses_to_drop_unpublished_audit_and_drops_a_drained_outbox( "CREATE TABLE casework_audit_outbox (event_id uuid PRIMARY KEY, audit_record jsonb NOT NULL, published_at timestamptz); \ INSERT INTO casework_audit_outbox(event_id,audit_record) \ VALUES('00000000-0000-4000-8000-0000000000c1','{}'); \ - DELETE FROM casework_schema_migrations WHERE version=17;", + DELETE FROM casework_schema_migrations WHERE version>=17;", ) .await .expect("simulate the schema before migration 17"); @@ -1375,7 +1422,7 @@ async fn migration_refuses_to_drop_unpublished_audit_and_drops_a_drained_outbox( store.migrate().await.expect("a drained outbox is dropped"); assert_eq!( applied_versions(&client).await, - (1..=17).collect::>() + (1..=18).collect::>() ); let dropped: bool = client .query_one("SELECT to_regclass('casework_audit_outbox') IS NULL", &[]) @@ -1397,7 +1444,7 @@ async fn migration_replaces_empty_hosted_tables_through_the_ledger_head() { assert_eq!( applied_versions(&client).await, - (1..=17).collect::>() + (1..=18).collect::>() ); let hosted_tables_remaining: bool = client .query_one( @@ -1464,7 +1511,7 @@ async fn migration_locks_hosted_work_before_counting_it_for_the_drop() { .expect("migration completes once the blocker releases the table"); assert_eq!( applied_versions(&client).await, - (1..=17).collect::>() + (1..=18).collect::>() ); } @@ -1485,7 +1532,7 @@ async fn migration_13_adds_sync_claim_indexes_to_an_existing_schema() { assert_eq!( applied_versions(&client).await, - (1..=17).collect::>() + (1..=18).collect::>() ); let indexes: Vec = client .query( @@ -1525,9 +1572,19 @@ async fn readiness_rejects_an_unmigrated_schema() { let (store, _client, _schema) = isolated_schema("ready_unmigrated").await; assert!( - matches!(store.ready().await, Err(StoreError::Postgres(_))), + matches!( + store.ready().await, + Err(StoreError::SchemaNotCurrent { + applied: None, + required: 18 + }) + ), "a schema without the migration ledger must fail readiness" ); + assert_eq!( + store.ready().await.unwrap_err().to_string(), + "the Casework database schema is not current: no migration has been applied, and this binary requires version 18; apply the migrations with `casework migrate` or `caseworkctl db migrate`" + ); } #[tokio::test] @@ -1546,7 +1603,13 @@ async fn readiness_rejects_a_partial_schema_missing_review_tables() { .expect("simulate a partial schema without the unified review migration"); assert!( - matches!(store.ready().await, Err(StoreError::Corrupt)), + matches!( + store.ready().await, + Err(StoreError::SchemaNotCurrent { + applied: Some(18), + required: 18 + }) + ), "a partial migration ledger must fail readiness" ); } @@ -1575,15 +1638,15 @@ async fn readiness_rejects_an_unsupported_migration_version() { matches!( refusal, StoreError::SchemaNewer { - found: 18, - supported: 17 + found: 19, + supported: 18 } ), "a newer schema is not reported as corrupt data: {refusal:?}" ); assert_eq!( refusal.to_string(), - "the Casework database schema version 18 is newer than this binary supports (17); run a casework release that supports it" + "the Casework database schema version 19 is newer than this binary supports (18); run a casework release that supports it" ); } @@ -1596,7 +1659,7 @@ async fn migration_refuses_a_schema_newer_than_this_binary_and_writes_nothing() .expect("migrate to the current schema"); client .execute( - "INSERT INTO casework_schema_migrations(version,applied_at) VALUES(18,now())", + "INSERT INTO casework_schema_migrations(version,applied_at) VALUES(19,now())", &[], ) .await @@ -1611,8 +1674,8 @@ async fn migration_refuses_a_schema_newer_than_this_binary_and_writes_nothing() matches!( refusal, StoreError::SchemaNewer { - found: 18, - supported: 17 + found: 19, + supported: 18 } ), "{refusal:?}" diff --git a/crates/registry-casework/tests/review_postgres.rs b/crates/registry-casework/tests/review_postgres.rs index cf7d059eab..e13bd47f39 100644 --- a/crates/registry-casework/tests/review_postgres.rs +++ b/crates/registry-casework/tests/review_postgres.rs @@ -1522,6 +1522,98 @@ async fn recovery_pins_policy_and_terminal_settlement_emits_atomically() { ); } +#[tokio::test] +async fn activation_preflight_counts_in_flight_reviews_a_package_would_strand() { + let fixture = fixture().await; + for index in 0..3 { + fixture + .service_v1 + .create_review_request( + &fixture.producer, + request( + &format!("record-pinned-{index}"), + &format!("producer-ref-pinned-{index}"), + ), + &format!("create-pinned-{index}"), + ) + .await + .expect("create pinned review"); + } + let cancelled = request("record-pinned-cancelled", "producer-ref-pinned-cancelled"); + let created = fixture + .service_v1 + .create_review_request( + &fixture.producer, + cancelled.clone(), + "create-pinned-cancelled", + ) + .await + .expect("create cancelled review"); + fixture + .service_v1 + .cancel_review_request( + &fixture.producer, + created.accepted.request_id, + ReviewCancelRequest { + subject: cancelled.subject, + reason: "no longer needed".to_owned(), + }, + "cancel-pinned", + ) + .await + .expect("cancel the fourth review"); + + let stranded = |candidate: CaseworkProject| { + let store = fixture.store.clone(); + async move { + candidate.check().expect("candidate project"); + registry_casework::stranded_pinned_work(&store, &candidate, &[]) + .await + .expect("preflight") + } + }; + + assert!(stranded(project("1")).await.is_empty()); + + let mut moved_queue = project("2"); + moved_queue.queues[0].id = "triage".to_owned(); + for stage in &mut moved_queue.review_kinds[0].stages { + stage.queue = "triage".to_owned(); + } + assert_eq!( + stranded(moved_queue).await, + vec![registry_casework::StrandedWork::QueueRemoved { + queue: "review".to_owned(), + reviews: 3, + work_items: 0, + }] + ); + + let mut renamed_profile = project("2"); + renamed_profile.access_profiles[0].id = "clerk".to_owned(); + for stage in &mut renamed_profile.review_kinds[0].stages { + stage.deciding_profiles = vec!["clerk".to_owned()]; + } + assert_eq!( + stranded(renamed_profile).await, + vec![registry_casework::StrandedWork::ProfileRemoved { + profile: "staff".to_owned(), + reviews: 3, + }] + ); + + let mut edited_in_place = project("1"); + edited_in_place.review_kinds[0].stages[1].deciding_profiles = vec!["supervisor".to_owned()]; + assert_eq!( + stranded(edited_in_place).await, + vec![registry_casework::StrandedWork::ReviewKindChanged { + review_kind: "registry-correction".to_owned(), + version: "1".to_owned(), + reviews: 3, + }] + ); +} + #[tokio::test] async fn retained_reviews_remain_bound_to_the_admitted_producer_identity() { let fixture = fixture().await; diff --git a/crates/registry-casework/tests/service_visibility.rs b/crates/registry-casework/tests/service_visibility.rs index 0d0ec9024a..970f2860b2 100644 --- a/crates/registry-casework/tests/service_visibility.rs +++ b/crates/registry-casework/tests/service_visibility.rs @@ -13,7 +13,8 @@ use jsonwebtoken::jwk::JwkSet; use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; use registry_casework::{ router, CaseworkAuthenticator, CaseworkService, DatabaseConfig, HttpState, HumanIdentityConfig, - PostgresStore, ServiceError, StoreError, + PostgresStore, ReconciliationFailure, ServiceError, StoreError, + RECONCILIATION_FAILURE_THRESHOLD, }; use registry_casework_core::{ AccessProfile, ActiveSubjectsPage, ActorContext, AttemptState, AuthoritativeObservation, @@ -3963,3 +3964,160 @@ fn authenticated_request( async fn response_body(response: axum::response::Response) -> serde_json::Value { serde_json::from_slice(&to_bytes(response.into_body(), 1024 * 1024).await.unwrap()).unwrap() } + +#[tokio::test] +async fn one_failing_subject_does_not_stall_the_rest_of_a_reconciliation_pass() { + let _database = DATABASE.lock().await; + let (mut source, _) = MockSource::with_large_discovery(3); + let failing = Uuid::from_u128(1).to_string(); + source.open_reads.remove(&failing); + let fixture = fixture_with_source(source, policy(10, 1_000)).await; + + assert!(matches!( + fixture.service.reconcile_source(SOURCE_ID).await, + Err(ServiceError::Adapter(SourceAdapterError::Invalid)) + )); + + let applied: Vec = fixture + .database + .query( + "SELECT subject_id FROM casework_subjects WHERE source_id=$1 AND applied_revision > 0 ORDER BY subject_id", + &[&SOURCE_ID], + ) + .await + .expect("read applied subjects") + .into_iter() + .map(|row| row.get(0)) + .collect(); + assert_eq!( + applied, + [ + Uuid::from_u128(2).to_string(), + Uuid::from_u128(3).to_string() + ], + "the subjects behind a failing one are applied in the same pass" + ); + let health = fixture + .service + .store() + .reconciliation_health(&[SOURCE_ID.to_owned()]) + .await + .expect("read reconciliation health"); + assert_eq!(health.len(), 1); + assert_eq!(health[0].consecutive_failures, 1); + assert_eq!( + health[0].last_failure, + Some(ReconciliationFailure::SourceRefused) + ); + assert!(health[0].last_succeeded_at.is_none()); + assert!(health[0].last_failed_at.is_some()); +} + +#[tokio::test] +async fn readiness_fails_after_consecutive_failed_reconciliation_passes_and_recovers() { + let _database = DATABASE.lock().await; + let (source, unavailable) = MockSource::with_discovery_control(); + unavailable.store(true, Ordering::SeqCst); + let fixture = fixture_with_source(source, policy(10, 1_000)).await; + fixture + .service + .ready() + .await + .expect("ready before any pass"); + + for pass in 1..RECONCILIATION_FAILURE_THRESHOLD { + assert!(fixture.service.reconcile_source(SOURCE_ID).await.is_err()); + fixture + .service + .ready() + .await + .unwrap_or_else(|error| panic!("pass {pass} stays below the threshold: {error}")); + } + assert!(fixture.service.reconcile_source(SOURCE_ID).await.is_err()); + let refusal = fixture + .service + .ready() + .await + .expect_err("a wedged reconciliation fails readiness"); + assert!( + matches!( + &refusal, + ServiceError::ReconciliationFailing { source_id, consecutive_failures } + if source_id == SOURCE_ID && *consecutive_failures == RECONCILIATION_FAILURE_THRESHOLD + ), + "{refusal:?}" + ); + let health = fixture + .service + .store() + .reconciliation_health(&[SOURCE_ID.to_owned()]) + .await + .expect("read reconciliation health"); + assert_eq!( + health[0].last_failure, + Some(ReconciliationFailure::SourceUnavailable) + ); + + unavailable.store(false, Ordering::SeqCst); + fixture + .service + .reconcile_source(SOURCE_ID) + .await + .expect("the source answers again"); + fixture + .service + .ready() + .await + .expect("one successful pass restores readiness"); + let health = fixture + .service + .store() + .reconciliation_health(&[SOURCE_ID.to_owned()]) + .await + .expect("read reconciliation health"); + assert_eq!(health[0].consecutive_failures, 0); + assert!(health[0].last_succeeded_at.is_some()); + assert_eq!( + health[0].last_failure, + Some(ReconciliationFailure::SourceUnavailable), + "the last failure stays recorded for the operator after recovery" + ); +} + +#[tokio::test] +async fn one_failing_subject_does_not_stall_the_rest_of_an_event_synchronization() { + let _database = DATABASE.lock().await; + let (mut source, _) = MockSource::with_large_discovery(3); + source.open_reads.remove(&Uuid::from_u128(1).to_string()); + let fixture = fixture_with_source(source, policy(10, 1_000)).await; + for value in 1..=3 { + assert!(fixture + .service + .store() + .ingest_transition( + GENERATION, + &TransitionHint { + subject: subject(Uuid::from_u128(value)), + deduplication_key: format!("event-{value}"), + ordered_revision: 1, + }, + ) + .await + .unwrap()); + } + + assert!(matches!( + fixture.service.synchronize_pending(10).await, + Err(ServiceError::Adapter(SourceAdapterError::Invalid)) + )); + let applied: i64 = fixture + .database + .query_one( + "SELECT count(*) FROM casework_subjects WHERE source_id=$1 AND applied_revision > 0", + &[&SOURCE_ID], + ) + .await + .expect("count applied subjects") + .get(0); + assert_eq!(applied, 2, "the subjects behind a failing one are applied"); +} diff --git a/crates/registry-caseworkctl/src/cli_contract_tests.rs b/crates/registry-caseworkctl/src/cli_contract_tests.rs index 77d9dec074..2b09f4f0ab 100644 --- a/crates/registry-caseworkctl/src/cli_contract_tests.rs +++ b/crates/registry-caseworkctl/src/cli_contract_tests.rs @@ -84,7 +84,7 @@ fn generated_schemas_are_current() { #[test] fn every_public_json_report_matches_its_schema() { - let root = tempfile::tempdir().expect("temporary contract fixture root"); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); let missing = root.path().join("missing"); let mut reports = Vec::new(); @@ -291,7 +291,7 @@ fn db_migrate_reports_a_schema_newer_than_this_binary_with_its_own_refusal() { let secret = format!("CASEWORKCTL_TEST_{}", Uuid::new_v4().simple()).to_ascii_uppercase(); std::env::set_var(&secret, &scoped); - let root = tempfile::tempdir().expect("temporary project root"); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); let (exit, _) = invoke(vec![ OsString::from("init"), @@ -300,6 +300,7 @@ fn db_migrate_reports_a_schema_newer_than_this_binary_with_its_own_refusal() { OsString::from("standalone-decision"), ]); assert_eq!(exit, ExitCode::SUCCESS); + package_locally(&project); std::fs::create_dir(project.join("secrets")).expect("secret root"); let mut runtime: Value = serde_norway::from_slice( &std::fs::read(project.join("runtime.example.yaml")).expect("runtime example reads"), @@ -327,10 +328,10 @@ fn db_migrate_reports_a_schema_newer_than_this_binary_with_its_own_refusal() { assert_eq!(migrated["status"], "migrated"); execute_in_test_database( &scoped, - "INSERT INTO casework_schema_migrations(version,applied_at) VALUES(18,now())", + "INSERT INTO casework_schema_migrations(version,applied_at) VALUES(19,now())", ); - let refusal = "the Casework database schema version 18 is newer than this binary supports (17); run a casework release that supports it"; + let refusal = "the Casework database schema version 19 is newer than this binary supports (18); run a casework release that supports it"; let (exit, report) = invoke(migrate.clone()); assert_eq!(exit, ExitCode::from(DOMAIN_REFUSAL_EXIT), "{report:#?}"); assert_eq!(report["ok"], false); @@ -359,3 +360,18 @@ fn db_migrate_reports_a_schema_newer_than_this_binary_with_its_own_refusal() { std::env::remove_var(&secret); execute_in_test_database(&base, &format!("DROP SCHEMA {schema} CASCADE")); } + +/// Package an initialized project where its generated runtime example +/// selects the package. +#[cfg(feature = "postgres-test")] +fn package_locally(project: &Path) { + let (exit, report) = invoke(vec![ + OsString::from("package"), + project.as_os_str().to_owned(), + OsString::from("--output"), + project + .join(crate::project::LOCAL_PACKAGE_DIRECTORY) + .into_os_string(), + ]); + assert_eq!(exit, ExitCode::SUCCESS, "{report:#?}"); +} diff --git a/crates/registry-caseworkctl/src/dev/config.rs b/crates/registry-caseworkctl/src/dev/config.rs index 16add4dd28..d997327076 100644 --- a/crates/registry-caseworkctl/src/dev/config.rs +++ b/crates/registry-caseworkctl/src/dev/config.rs @@ -605,9 +605,10 @@ pub(super) fn prepare(root: &Path, state: &State, clients: &Clients) -> Result<( /// The operator configuration the supervised `casework` children read. /// -/// It binds the authored `casework.yaml` the reader edits, not a copy, so -/// `caseworkctl doctor --runtime-config ` reports on the same -/// policy the reader's `caseworkctl check` reads. +/// It binds the package each start builds from the authored project, so the +/// runtime verifies its package exactly as a deployed one does, and +/// `caseworkctl doctor --runtime-config ` reports on the package +/// the session serves. pub(super) fn operator(state: &State) -> Value { let root = state.root(); let sources = state @@ -632,7 +633,7 @@ pub(super) fn operator(state: &State) -> Value { json!({ "apiVersion": registry_casework::RUNTIME_CONFIG_API_VERSION, "kind": registry_casework::RUNTIME_CONFIG_KIND, - "package": {"root": state.project}, + "package": {"root": root.join(super::SESSION_PACKAGE)}, "listener": { "bind": format!("127.0.0.1:{}", state.casework_port), "tlsTermination": "development-loopback", diff --git a/crates/registry-caseworkctl/src/dev/integrations.rs b/crates/registry-caseworkctl/src/dev/integrations.rs index 53687c928e..c95e95864a 100644 --- a/crates/registry-caseworkctl/src/dev/integrations.rs +++ b/crates/registry-caseworkctl/src/dev/integrations.rs @@ -522,10 +522,14 @@ impl Integrations { } } -/// Check the existing adapter contract entirely offline. The staging resolver -/// reads generated secrets before the session is atomically installed. +/// Check the existing adapter contract entirely offline against the authored +/// project. The staging resolver reads generated secrets before the session is +/// atomically installed, and before its start builds the package the runtime +/// verifies, so the operator document is read without that verification. pub(super) fn validate_bindings(root: &Path, project: &Path) -> Result<()> { - let mut config = registry_casework::RuntimeConfig::load(root.join("operator.yaml"))?; + let mut config = registry_casework::RuntimeConfig::loader() + .load::(&root.join("operator.yaml"))? + .config; config .secret_providers .file diff --git a/crates/registry-caseworkctl/src/dev/mod.rs b/crates/registry-caseworkctl/src/dev/mod.rs index 6e10b018ff..e04ba8c2b7 100644 --- a/crates/registry-caseworkctl/src/dev/mod.rs +++ b/crates/registry-caseworkctl/src/dev/mod.rs @@ -1451,6 +1451,7 @@ fn start(args: StartArgs) -> Result { } } probe(state.database_port)?; + package_session(&root, &project)?; remove_socket(&root)?; state.status = Status::Starting; state.failure = None; @@ -1524,6 +1525,36 @@ fn reap_failed_supervisor(supervisor: &mut Child) -> Result<()> { } } +/// The session directory holding the package the supervised runtime serves. +pub(super) const SESSION_PACKAGE: &str = "package"; + +/// Build the session package from the authored project, replacing the one the +/// previous start served. The runtime verifies it like any deployed package; +/// the reader keeps editing the authored project, and the next start packages +/// those edits. +fn package_session(root: &Path, project: &Path) -> Result<()> { + let staged = root.join(".package-staged"); + let retired = root.join(".package-retired"); + for owned in [&staged, &retired] { + if fs::symlink_metadata(owned).is_ok() { + fs::remove_dir_all(owned) + .with_context(|| format!("removing the owned {}", owned.display()))?; + } + } + crate::project::package(project, &staged, None) + .context("packaging the authored project for the local session")?; + let current = root.join(SESSION_PACKAGE); + if fs::symlink_metadata(¤t).is_ok() { + fs::rename(¤t, &retired)?; + } + fs::rename(&staged, ¤t)?; + if fs::symlink_metadata(&retired).is_ok() { + fs::remove_dir_all(&retired) + .with_context(|| format!("removing the owned {}", retired.display()))?; + } + Ok(()) +} + /// Stage the whole session beside its final path and rename it into place, so /// an interrupted first start never leaves a half-generated session behind. fn initialize(root: &Path, original: &State, clients: &Clients) -> Result<()> { diff --git a/crates/registry-caseworkctl/src/dev/tests.rs b/crates/registry-caseworkctl/src/dev/tests.rs index 8f902b6ef0..a11565528e 100644 --- a/crates/registry-caseworkctl/src/dev/tests.rs +++ b/crates/registry-caseworkctl/src/dev/tests.rs @@ -80,7 +80,7 @@ fn announced_pid(path: &Path) -> Option { #[test] fn init_clients_bind_the_standalone_template() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let policy = crate::project::load_and_check_policy(&project).unwrap(); let clients = config::clients(STANDALONE_DEV_CLIENTS.as_bytes()).unwrap(); @@ -287,7 +287,7 @@ fn clients_file_refuses_two_teams_assigned_to_one_queue() { #[test] fn binding_refuses_a_requester_with_a_human_claim() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let policy = crate::project::load_and_check_policy(&project).unwrap(); let text = STANDALONE_DEV_CLIENTS.replace( @@ -303,7 +303,7 @@ fn binding_refuses_a_requester_with_a_human_claim() { #[test] fn binding_accepts_a_client_with_every_required_profile_scope() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); fs::write( project.join("casework.yaml"), @@ -325,7 +325,7 @@ fn binding_accepts_a_client_with_every_required_profile_scope() { #[test] fn binding_refuses_a_client_missing_one_required_profile_scope() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); fs::write( project.join("casework.yaml"), @@ -348,7 +348,7 @@ fn binding_refuses_a_client_missing_one_required_profile_scope() { #[test] fn binding_accepts_directory_members_with_matching_roles() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let policy = crate::project::load_and_check_policy(&project).unwrap(); let clients = config::clients(STANDALONE_DEV_CLIENTS.as_bytes()).unwrap(); @@ -370,7 +370,7 @@ fn binding_accepts_directory_members_with_matching_roles() { #[test] fn binding_refuses_repeated_resolved_principals_within_each_membership_kind() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); for (role, existing_id, second_id, membership_kind) in [ @@ -422,7 +422,7 @@ fn binding_refuses_repeated_resolved_principals_within_each_membership_kind() { #[test] fn binding_accepts_one_resolved_principal_in_each_membership_kind() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let mut policy = crate::project::load_and_check_policy(&project).unwrap(); for profile in &mut policy.access_profiles { @@ -444,7 +444,7 @@ fn binding_accepts_one_resolved_principal_in_each_membership_kind() { #[test] fn binding_refuses_directory_members_with_mismatched_roles() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let policy = crate::project::load_and_check_policy(&project).unwrap(); @@ -467,7 +467,7 @@ fn binding_refuses_directory_members_with_mismatched_roles() { #[test] fn binding_refuses_an_unserved_queue_and_a_missing_administrator() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let policy = crate::project::load_and_check_policy(&project).unwrap(); @@ -505,9 +505,73 @@ fn generated_secrets_are_nul_free_lowercase_hexadecimal() { } } +#[test] +fn each_start_packages_the_authored_project_the_runtime_verifies() { + let root = crate::canonical_tempdir(); + let project = standalone(root.path()); + let state = session(&project); + let session_root = state.root(); + fs::create_dir_all(&session_root).unwrap(); + fs::set_permissions(project.join(".casework"), fs::Permissions::from_mode(0o700)).unwrap(); + fs::set_permissions(&session_root, fs::Permissions::from_mode(0o700)).unwrap(); + let path = session_root.join("operator.yaml"); + config::write_yaml(&path, &config::operator(&state)).unwrap(); + + // The session never serves the authored project directly. + let error = RuntimeConfig::load(&path).expect_err("an unpackaged session is refused"); + assert!(error.to_string().contains("caseworkctl package"), "{error}"); + + // A staging directory an interrupted start left behind is replaced. + fs::create_dir(session_root.join(".package-staged")).unwrap(); + package_session(&session_root, &project).unwrap(); + assert!(!session_root.join(".package-staged").exists()); + assert!(!session_root.join(".package-retired").exists()); + let first = RuntimeConfig::load(&path) + .unwrap() + .package_digest() + .unwrap(); + let package = session_root.join(SESSION_PACKAGE); + assert_eq!( + fs::read(package.join("casework.yaml")).unwrap(), + STANDALONE_YAML.as_bytes() + ); + + // An edit to the authored project reaches the runtime only through the + // next start's package, and a package changed in place is refused. + fs::write( + project.join("casework.yaml"), + format!("{STANDALONE_YAML}\n# edited\n"), + ) + .unwrap(); + assert_eq!( + RuntimeConfig::load(&path) + .unwrap() + .package_digest() + .unwrap(), + first + ); + fs::write( + package.join("casework.yaml"), + format!("{STANDALONE_YAML}\n# edited\n"), + ) + .unwrap(); + let error = RuntimeConfig::load(&path).expect_err("a changed package is refused"); + assert!( + error.to_string().contains("changed: casework.yaml"), + "{error}" + ); + + package_session(&session_root, &project).unwrap(); + let second = RuntimeConfig::load(&path) + .unwrap() + .package_digest() + .unwrap(); + assert_ne!(first, second); +} + #[test] fn generated_operator_config_loads_through_the_runtime_contract() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let state = session(&project); let session_root = state.root(); @@ -516,6 +580,7 @@ fn generated_operator_config_loads_through_the_runtime_contract() { fs::set_permissions(&session_root, fs::Permissions::from_mode(0o700)).unwrap(); let path = session_root.join("operator.yaml"); config::write_yaml(&path, &config::operator(&state)).unwrap(); + package_session(&session_root, &project).unwrap(); let config = RuntimeConfig::load(&path).unwrap(); assert_eq!( @@ -523,13 +588,13 @@ fn generated_operator_config_loads_through_the_runtime_contract() { registry_casework::RUNTIME_CONFIG_API_VERSION ); assert_eq!(config.kind, registry_casework::RUNTIME_CONFIG_KIND); - assert_eq!(config.package.root, project); + assert_eq!(config.package.root, session_root.join(SESSION_PACKAGE)); assert_eq!(config.listener.bind, "127.0.0.1:8092".parse().unwrap()); // The local issuer emits one space-delimited `scope` claim. assert_eq!(config.authentication.oidc.scope_claim, "scope"); assert!(matches!( - config.authentication.oidc.jwks_source, - registry_casework::OidcJwksSource::Static { ref document_ref } + config.authentication.oidc.provider.jwks_source, + registry_casework::JwksSource::Static { ref document_ref } if document_ref == "secret:file/issuer-jwks" )); assert_eq!( @@ -549,7 +614,10 @@ fn generated_operator_config_loads_through_the_runtime_contract() { Some(session_root.join("secrets").as_path()) ); assert!(config.secret_providers.environment.is_none()); - assert_eq!(config.audit.hash_key_ref, "secret:file/casework-audit-key"); + assert_eq!( + config.audit.key.hash_key_ref.as_str(), + "secret:file/casework-audit-key" + ); assert_eq!( config.database.runtime_url_ref, "secret:file/runtime-database-url" @@ -563,13 +631,19 @@ fn generated_operator_config_loads_through_the_runtime_contract() { Some("secret:file/database-root.pem") ); assert!(config.sources.is_empty()); - assert_eq!(config.authentication.oidc.issuer, state.issuer_origin()); - assert_eq!(config.authentication.oidc.audience, state.audience()); + assert_eq!( + config.authentication.oidc.provider.issuer, + state.issuer_origin() + ); + assert_eq!( + config.authentication.oidc.provider.audience, + state.audience() + ); } #[test] fn a_project_declaring_sources_is_refused_before_anything_starts() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("project"); crate::project::init(&project, "professional-review").unwrap(); let clients = fs::read(project.join("dev-clients.yaml")).unwrap(); @@ -579,7 +653,7 @@ fn a_project_declaring_sources_is_refused_before_anything_starts() { #[test] fn borrowed_source_mode_is_explicit_pinned_and_refuses_session_qualified_subjects() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("project"); crate::project::init(&project, "professional-review").unwrap(); let policy = crate::project::load_and_check_policy(&project).unwrap(); @@ -625,7 +699,7 @@ fn task_templates_cannot_use_the_borrowed_source_issuer() { #[test] fn the_source_digest_pins_the_project_and_its_clients() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let clients = fs::read(project.join("dev-clients.yaml")).unwrap(); let first = capture(&project, &clients).unwrap().digest; @@ -773,7 +847,7 @@ fn bare_dev_alias_ports_also_fall_back_to_the_environment() { #[test] fn events_reports_only_the_bounded_journal_tail() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let logs = project.join(".casework/dev/logs"); fs::create_dir_all(&logs).unwrap(); @@ -825,7 +899,7 @@ fn events_reports_only_the_bounded_journal_tail() { #[test] fn stopping_a_project_that_never_started_is_refused() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let refusal = format!("{:#}", stop(&project, false, None).unwrap_err()); assert!(refusal.contains("nothing was stopped"), "{refusal}"); @@ -835,7 +909,7 @@ fn stopping_a_project_that_never_started_is_refused() { #[test] fn a_first_start_without_a_clients_file_names_the_flag() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("project"); fs::create_dir(&project).unwrap(); let refusal = format!("{:#}", clients_file(None, None, &project).unwrap_err()); @@ -845,7 +919,7 @@ fn a_first_start_without_a_clients_file_names_the_flag() { #[test] fn a_stopped_session_retains_an_explicit_equivalent_clients_file() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = fs::canonicalize(standalone(root.path())).unwrap(); let original_clients = fs::read(project.join("dev-clients.yaml")).unwrap(); let replacement = project.join("replacement-clients.yaml"); @@ -899,7 +973,7 @@ fn a_stopped_session_retains_an_explicit_equivalent_clients_file() { #[test] fn an_active_session_refuses_an_equivalent_clients_file_at_a_new_path() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = fs::canonicalize(standalone(root.path())).unwrap(); let original_clients = fs::read(project.join("dev-clients.yaml")).unwrap(); let replacement = project.join("replacement-clients.yaml"); @@ -957,7 +1031,7 @@ fn an_active_session_refuses_an_equivalent_clients_file_at_a_new_path() { #[test] fn the_report_names_every_local_credential_without_a_secret() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let mut state = session(&project); state.status = Status::Ready; @@ -1018,7 +1092,7 @@ struct DockerInventory { impl DockerInventory { fn new(state: &State) -> Self { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let executable = root.path().join("docker"); let container = root.path().join("container-active"); let volume = root.path().join("volume-active"); @@ -1113,7 +1187,7 @@ fn persisted_session(project: &Path) -> State { #[test] fn config_change_keeps_the_owner_after_container_creation_fails() { - let workspace = tempfile::tempdir().unwrap(); + let workspace = crate::canonical_tempdir(); let project = standalone(workspace.path()); let mut state = persisted_session(&project); let docker = DockerInventory::new(&state); @@ -1147,7 +1221,7 @@ fn config_change_keeps_the_owner_after_container_creation_fails() { #[test] fn config_change_keeps_the_owner_after_created_container_cannot_be_saved() { - let workspace = tempfile::tempdir().unwrap(); + let workspace = crate::canonical_tempdir(); let project = standalone(workspace.path()); let mut state = persisted_session(&project); let docker = DockerInventory::new(&state); @@ -1180,7 +1254,7 @@ fn config_change_keeps_the_owner_after_created_container_cannot_be_saved() { #[test] fn volume_removal_requires_the_retained_owner_label() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let state = session(&project); let mut wrong_labels = serde_json::Map::new(); @@ -1238,7 +1312,7 @@ fn legacy_database_container(state: &State, volume_name: &str, destination: &str #[test] fn legacy_unlabeled_volume_requires_the_exact_retained_container_and_mount() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let mut state = session(&project); state.container_id = Some("retained-container-id".to_owned()); @@ -1307,7 +1381,7 @@ fn legacy_unlabeled_volume_requires_the_exact_retained_container_and_mount() { #[test] fn foreground_interruption_terminates_and_reaps_its_owned_supervisor() { - let workspace = tempfile::tempdir().unwrap(); + let workspace = crate::canonical_tempdir(); let project = standalone(workspace.path()); let mut state = session(&project); state.status = Status::Starting; @@ -1340,7 +1414,7 @@ fn foreground_interruption_terminates_and_reaps_its_owned_supervisor() { #[test] fn failed_start_waits_for_the_supervisor_lock_to_be_released() { - let workspace = tempfile::tempdir().unwrap(); + let workspace = crate::canonical_tempdir(); let project = standalone(workspace.path()); let mut state = session(&project); state.status = Status::Failed; @@ -1377,7 +1451,7 @@ fn failed_start_waits_for_the_supervisor_lock_to_be_released() { #[test] fn migration_failures_use_the_bounded_native_diagnostic_stream() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); private::directory(&root.path().join("logs")).unwrap(); let refusal = format!( "{:#}", @@ -1401,7 +1475,7 @@ fn migration_failures_use_the_bounded_native_diagnostic_stream() { #[test] fn database_readiness_commands_stop_at_the_aggregate_deadline() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); private::directory(&root.path().join("logs")).unwrap(); let started = Instant::now(); let deadline = started + Duration::from_millis(75); @@ -1428,7 +1502,7 @@ fn database_readiness_commands_stop_at_the_aggregate_deadline() { #[test] fn interrupted_native_prerequisite_is_killed_and_reaped() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); private::directory(&root.path().join("logs")).unwrap(); let marker = root.path().join("prerequisite.pid"); let terminate = Arc::new(AtomicBool::new(false)); @@ -1472,7 +1546,7 @@ fn interrupted_native_prerequisite_is_killed_and_reaped() { #[test] fn native_pump_setup_failures_reap_the_child_and_join_started_pumps() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); private::directory(&root.path().join("logs")).unwrap(); for fail_on in [1, 2] { // The child outlasts every wait below, so only cleanup can end it. @@ -1527,7 +1601,7 @@ fn native_pump_setup_failures_reap_the_child_and_join_started_pumps() { #[test] fn failed_native_stdin_write_reaps_the_child_and_joins_pumps() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); private::directory(&root.path().join("logs")).unwrap(); let child = Command::new("/bin/sh") .args(["-c", "exec 0<&-; while :; do :; done"]) @@ -1674,7 +1748,7 @@ fn service_http_readiness_keeps_the_normal_request_timeout() { #[test] fn prerequisite_logs_are_bounded_and_keep_the_latest_diagnostics() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let logs = root.path().join("logs"); private::directory(&logs).unwrap(); let latest = format!("diagnostic-{}", MAX_PREREQUISITE_LOGS + 7); @@ -1697,7 +1771,7 @@ fn prerequisite_logs_are_bounded_and_keep_the_latest_diagnostics() { assert_eq!(metadata.nlink(), 1); } - let unsafe_root = tempfile::tempdir().unwrap(); + let unsafe_root = crate::canonical_tempdir(); let unsafe_logs = unsafe_root.path().join("logs"); private::directory(&unsafe_logs).unwrap(); let unsafe_path = unsafe_logs.join(format!("probe-{}.log", uuid::Uuid::new_v4())); @@ -1710,14 +1784,14 @@ fn prerequisite_logs_are_bounded_and_keep_the_latest_diagnostics() { #[test] fn prerequisite_log_rotation_stays_in_the_opened_directory_after_a_path_swap() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let logs = root.path().join("logs"); private::directory(&logs).unwrap(); for index in 0..MAX_PREREQUISITE_LOGS { let mut log = log_file(root.path(), "probe").unwrap(); writeln!(log, "diagnostic-{index}").unwrap(); } - let redirected = tempfile::tempdir().unwrap(); + let redirected = crate::canonical_tempdir(); let names = fs::read_dir(&logs) .unwrap() .map(|entry| entry.unwrap().file_name()) @@ -1747,7 +1821,7 @@ fn prerequisite_log_rotation_stays_in_the_opened_directory_after_a_path_swap() { #[test] fn retained_service_journal_stays_bounded_and_keeps_latest_diagnostics() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let logs = root.path().join("logs"); private::directory(&logs).unwrap(); let path = logs.join("casework.log"); @@ -1793,7 +1867,7 @@ fn retained_service_journal_stays_bounded_and_keeps_latest_diagnostics() { #[test] fn supervisor_log_stays_bounded_and_resists_path_swaps_and_links() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let logs = root.path().join("logs"); private::directory(&logs).unwrap(); let path = logs.join("supervisor.log"); @@ -1827,7 +1901,7 @@ fn supervisor_log_stays_bounded_and_resists_path_swaps_and_links() { assert_eq!(metadata.permissions().mode() & 0o077, 0); assert_eq!(metadata.nlink(), 1); - let swap_root = tempfile::tempdir().unwrap(); + let swap_root = crate::canonical_tempdir(); let swap_logs = swap_root.path().join("logs"); let moved_logs = swap_root.path().join("original-logs"); let replacement_logs = swap_root.path().join("replacement-logs"); @@ -1859,7 +1933,7 @@ fn supervisor_log_stays_bounded_and_resists_path_swaps_and_links() { b"original recent failure\ncurrent failure\n" ); - let hardlink_root = tempfile::tempdir().unwrap(); + let hardlink_root = crate::canonical_tempdir(); let hardlink_logs = hardlink_root.path().join("logs"); private::directory(&hardlink_logs).unwrap(); let target = hardlink_logs.join("target.log"); @@ -1870,7 +1944,7 @@ fn supervisor_log_stays_bounded_and_resists_path_swaps_and_links() { assert!(refusal.contains("single-link"), "{refusal}"); assert_eq!(fs::read(&target).unwrap(), b"preserve me"); - let symlink_root = tempfile::tempdir().unwrap(); + let symlink_root = crate::canonical_tempdir(); let symlink_logs = symlink_root.path().join("logs"); private::directory(&symlink_logs).unwrap(); let target = symlink_logs.join("target.log"); @@ -1884,7 +1958,7 @@ fn supervisor_log_stays_bounded_and_resists_path_swaps_and_links() { #[test] fn invalid_service_journal_is_refused_before_the_child_starts() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let logs = root.path().join("logs"); private::directory(&logs).unwrap(); let journal = logs.join("casework.log"); @@ -1978,7 +2052,7 @@ fn nonzero_outer_guard_helper() { #[test] fn guarded_service_stops_after_its_supervisor_is_killed() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let binary = root.path().join("service.sh"); fs::write( &binary, @@ -2021,7 +2095,7 @@ fn guarded_service_stops_after_its_supervisor_is_killed() { #[test] fn service_guard_owns_a_stubborn_child_during_startup_interruption() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let binary = root.path().join("stubborn.sh"); fs::write( &binary, @@ -2087,7 +2161,7 @@ fn service_guard_does_not_force_kill_after_a_fast_term_exit() { #[test] fn established_service_keeps_its_graceful_shutdown_window() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); private::directory(&root.path().join("logs")).unwrap(); let binary = root.path().join("service.sh"); let graceful = root.path().join("graceful"); @@ -2124,7 +2198,7 @@ fn established_service_keeps_its_graceful_shutdown_window() { #[test] fn established_stubborn_service_reports_forced_shutdown() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); private::directory(&root.path().join("logs")).unwrap(); let binary = root.path().join("stubborn.sh"); fs::write( @@ -2149,7 +2223,7 @@ fn established_stubborn_service_reports_forced_shutdown() { #[test] fn killed_guard_leaves_the_supervisor_to_clean_its_exact_service_group() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); private::directory(&root.path().join("logs")).unwrap(); let binary = root.path().join("stubborn.sh"); fs::write( @@ -2191,7 +2265,7 @@ fn killed_guard_leaves_the_supervisor_to_clean_its_exact_service_group() { #[test] fn nonzero_guard_exit_is_detected_without_waiting_for_pump_eof() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); private::directory(&root.path().join("logs")).unwrap(); let service_pid_file = root.path().join("service.pid"); let service_binary = root.path().join("service.sh"); @@ -2240,7 +2314,7 @@ fn nonzero_guard_exit_is_detected_without_waiting_for_pump_eof() { #[test] fn live_guard_timeout_kills_the_pinned_group_before_reaping() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); private::directory(&root.path().join("logs")).unwrap(); let guard_binary = root.path().join("guard.sh"); let service_pid_file = root.path().join("service.pid"); @@ -2341,7 +2415,7 @@ fn failed_group_kill_never_enters_a_blocking_guard_wait() { #[test] fn service_pump_setup_failures_reap_the_child_and_join_started_pumps() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let logs = root.path().join("logs"); private::directory(&logs).unwrap(); for fail_on in [1, 2] { @@ -2388,7 +2462,7 @@ fn service_pump_setup_failures_reap_the_child_and_join_started_pumps() { #[test] fn guardian_pump_setup_failure_reaps_a_stubborn_owned_service() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let logs = root.path().join("logs"); private::directory(&logs).unwrap(); let binary = root.path().join("stubborn.sh"); @@ -2430,7 +2504,7 @@ fn guardian_pump_setup_failure_reaps_a_stubborn_owned_service() { #[test] fn seeding_administrator_token_is_issued_after_every_other_client() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let mut state = session(&project); let clients = Clients { @@ -2488,7 +2562,7 @@ fn seeding_administrator_token_is_issued_after_every_other_client() { #[test] fn token_issuance_stops_between_clients_when_interrupted() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let mut state = session(&project); let clients = Clients { @@ -2568,7 +2642,7 @@ fn service_cleanup_joins_every_log_pump() { #[test] fn legacy_issuer_state_and_unsafe_token_clients_are_refused_without_effects() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = standalone(root.path()); let state = session(&project); private::directory(&project.join(".casework")).unwrap(); @@ -2626,7 +2700,7 @@ fn a_borrowed_session_admits_only_the_clients_its_project_declares() { // and an omitted list admits all of them, so it is stated whenever the // project declares integrations rather than only when it adds clients of // its own beyond the ones it borrows. - let workspace = tempfile::tempdir().unwrap(); + let workspace = crate::canonical_tempdir(); let project = standalone(workspace.path()); let mut policy = crate::project::load_and_check_policy(&project).unwrap(); policy.sources.push(serde_json::from_value(json!({"id":"source","adapter":"breg","description":"source.json","requests":[{"entity":"correction","queue":"decisions"}]})).unwrap()); @@ -2663,7 +2737,7 @@ fn a_borrowed_session_admits_only_the_clients_its_project_declares() { #[test] fn explicit_local_integrations_render_only_governed_authority_and_bind_the_source() { - let workspace = tempfile::tempdir().unwrap(); + let workspace = crate::canonical_tempdir(); let project = standalone(workspace.path()); let mut policy = crate::project::load_and_check_policy(&project).unwrap(); policy.sources.push(serde_json::from_value(json!({"id":"source","adapter":"breg","description":"source.json","requests":[{"entity":"correction","queue":"decisions"}]})).unwrap()); @@ -2828,10 +2902,10 @@ fn explicit_local_integrations_render_only_governed_authority_and_bind_the_sourc #[test] fn borrowed_casework_client_requires_exact_owner_claims_scopes_and_resource() { - let project_temp = tempfile::tempdir().unwrap(); + let project_temp = crate::canonical_tempdir(); let project = fs::canonicalize(project_temp.path()).unwrap(); fs::set_permissions(&project, fs::Permissions::from_mode(0o700)).unwrap(); - let owner_temp = tempfile::tempdir().unwrap(); + let owner_temp = crate::canonical_tempdir(); let owner_project = fs::canonicalize(owner_temp.path()).unwrap(); fs::set_permissions(&owner_project, fs::Permissions::from_mode(0o700)).unwrap(); let owner_root = owner_project.join(".breg/dev"); @@ -2894,10 +2968,10 @@ fn a_borrowed_client_the_owner_registered_for_exchange_must_declare_it() { // registered for exchange without this project declaring it would be // admitted with no pairing to refuse the other authorities with, so the // two declarations must agree exactly. - let project_temp = tempfile::tempdir().unwrap(); + let project_temp = crate::canonical_tempdir(); let project = fs::canonicalize(project_temp.path()).unwrap(); fs::set_permissions(&project, fs::Permissions::from_mode(0o700)).unwrap(); - let owner_temp = tempfile::tempdir().unwrap(); + let owner_temp = crate::canonical_tempdir(); let owner_project = fs::canonicalize(owner_temp.path()).unwrap(); fs::set_permissions(&owner_project, fs::Permissions::from_mode(0o700)).unwrap(); let owner_root = owner_project.join(".breg/dev"); @@ -2963,7 +3037,7 @@ fn a_borrowed_client_the_owner_registered_for_exchange_must_declare_it() { #[test] fn task_template_subject_follows_the_actual_local_issuer_owner() { - let workspace = tempfile::tempdir().unwrap(); + let workspace = crate::canonical_tempdir(); let project = standalone(workspace.path()); let mut policy = crate::project::load_and_check_policy(&project).unwrap(); let client = "task-agent"; @@ -2997,7 +3071,7 @@ fn task_template_subject_follows_the_actual_local_issuer_owner() { no_exchange.service_clients[0].task_exchange = false; assert!(no_exchange.validate(&clients, &policy).is_err()); - let owner_temp = tempfile::tempdir().unwrap(); + let owner_temp = crate::canonical_tempdir(); let owner_project = fs::canonicalize(owner_temp.path()).unwrap(); fs::set_permissions(&owner_project, fs::Permissions::from_mode(0o700)).unwrap(); let owner_root = owner_project.join(".breg/dev"); @@ -3065,13 +3139,13 @@ fn task_template_subject_follows_the_actual_local_issuer_owner() { #[test] fn borrowed_browser_admission_requires_exact_owner_resource() { - let workspace = tempfile::tempdir().unwrap(); + let workspace = crate::canonical_tempdir(); let project = standalone(workspace.path()); let mut policy = crate::project::load_and_check_policy(&project).unwrap(); for profile in &mut policy.access_profiles { profile.principal_claim = "registry_principal".to_owned(); } - let owner_temp = tempfile::tempdir().unwrap(); + let owner_temp = crate::canonical_tempdir(); let owner_project = fs::canonicalize(owner_temp.path()).unwrap(); fs::set_permissions(&owner_project, fs::Permissions::from_mode(0o700)).unwrap(); let owner_root = owner_project.join(".breg/dev"); @@ -3133,13 +3207,13 @@ fn a_borrowed_task_authority_connection_pairs_the_task_exchange_clients() { // with one of its other connections would reach the owner's resource // servers as that authority's, so the pairing is read here and not only // the connection itself. - let workspace = tempfile::tempdir().unwrap(); + let workspace = crate::canonical_tempdir(); let project = standalone(workspace.path()); let mut policy = crate::project::load_and_check_policy(&project).unwrap(); for profile in &mut policy.access_profiles { profile.principal_claim = "registry_principal".to_owned(); } - let owner_temp = tempfile::tempdir().unwrap(); + let owner_temp = crate::canonical_tempdir(); let owner_project = fs::canonicalize(owner_temp.path()).unwrap(); fs::set_permissions(&owner_project, fs::Permissions::from_mode(0o700)).unwrap(); let owner_root = owner_project.join(".breg/dev"); @@ -3236,7 +3310,7 @@ impl RegistrySession { } fn new() -> Self { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = Self::create_project(root.path(), "registry"); let executable = root.path().join("bregctl"); fs::write( @@ -3356,7 +3430,7 @@ fn retained_credential_canaries(state: &State, clients: &Clients) -> BTreeMap, - /// Report the same policyDigest and files a package would produce, without writing one. + /// Report the same packageDigest and files a package would produce, without writing one. #[arg(long, conflicts_with = "output", required_unless_present = "output")] dry_run: bool, + /// Free-text revision recorded in the package's REVISION file and covered by its digest. + #[arg(long, value_name = "TEXT")] + revision: Option, } #[derive(Debug, Args)] @@ -307,7 +311,7 @@ enum OutputFormat { const DOMAIN_REFUSAL_EXIT: u8 = 1; const OPERATIONAL_FAILURE_EXIT: u8 = 3; -const CLI_API_VERSION: &str = "registry.registrystack.org/caseworkctl/v1alpha1"; +const CLI_API_VERSION: &str = "registry.registrystack.org/caseworkctl/v1alpha2"; pub fn main_entry() -> ExitCode { main_entry_from( @@ -504,22 +508,45 @@ fn classify_failure(kind: CommandKind, error: &anyhow::Error) -> (u8, Value) { if let Some(diagnostic) = operator_refusal(kind, error) { return (DOMAIN_REFUSAL_EXIT, diagnostic); } - let io_failure = error.chain().any(|cause| cause.is::()); + if let Some(check) = error + .chain() + .find_map(|cause| cause.downcast_ref::()) + { + return ( + OPERATIONAL_FAILURE_EXIT, + json!({ + "severity":"error", "code":"casework.doctor.check-failed", + "artifact":"runtime_dependency", "path":format!("doctor:/checks/{}", check.check), + "message":check.message, "suggestedAction":check.action + }), + ); + } let runtime_error = error .chain() .find_map(|cause| cause.downcast_ref::()); + let io_failure = error.chain().any(|cause| cause.is::()) + || matches!( + runtime_error, + Some(RuntimeConfigError::Load(load)) + if load.kind() == RuntimeConfigErrorKind::Unavailable + ); let project_error = error .chain() .find_map(|cause| cause.downcast_ref::()); let semantic_error = error .chain() .find_map(|cause| cause.downcast_ref::()); + let authored_expression = error + .chain() + .filter_map(|cause| cause.downcast_ref::()) + .find(|authored| authored.kind() == RuntimeConfigErrorKind::AuthoredExpression); let runtime_project_error = matches!(runtime_error, Some(RuntimeConfigError::Project(_))) && project_error.is_some(); let runtime_dependency_unavailable = matches!(runtime_error, Some(RuntimeConfigError::Oidc)); let domain = !io_failure && !runtime_dependency_unavailable && (runtime_error.is_some() + || authored_expression.is_some() || project_error.is_some() || semantic_error.is_some() || matches!(kind, CommandKind::Authoring)); @@ -535,6 +562,12 @@ fn classify_failure(kind: CommandKind, error: &anyhow::Error) -> (u8, Value) { "runtime.yaml:/authentication/oidc".to_owned(), "Restore access to the configured OIDC issuer or mounted JWKS, then retry.", ) + } else if let Some(authored) = authored_expression { + ( + "casework_project", + format!("casework.yaml:/{}", authored.field().replace('.', "/")), + "Write the value in casework.yaml directly; environment substitution applies to runtime.yaml only.", + ) } else if runtime_project_error { project_diagnostic_location(project_error.expect("runtime project error has source")) } else if let Some(runtime) = runtime_error { @@ -560,7 +593,11 @@ fn classify_failure(kind: CommandKind, error: &anyhow::Error) -> (u8, Value) { "caseworkctl.io-failure" } else if runtime_dependency_unavailable { "casework.runtime-dependency.unavailable" - } else if runtime_project_error || project_error.is_some() || semantic_error.is_some() { + } else if runtime_project_error + || authored_expression.is_some() + || project_error.is_some() + || semantic_error.is_some() + { "casework.project.invalid" } else if runtime_error.is_some() { "casework.runtime-configuration.invalid" @@ -573,6 +610,8 @@ fn classify_failure(kind: CommandKind, error: &anyhow::Error) -> (u8, Value) { "A required filesystem operation failed.".to_owned() } else if runtime_dependency_unavailable { "The configured OIDC runtime dependency is unavailable.".to_owned() + } else if let Some(authored) = authored_expression { + authored.to_string() } else if runtime_project_error { project_error .expect("runtime project error has source") @@ -719,17 +758,34 @@ fn operator_refusal(kind: CommandKind, error: &anyhow::Error) -> Option { } fn runtime_diagnostic_location(error: &RuntimeConfigError) -> (&'static str, String, &'static str) { - let path = if matches!(error, RuntimeConfigError::RemovedPrincipalClaim) { - "runtime.yaml:/authentication/oidc/principalClaim".to_owned() + let removed_key = match error { + RuntimeConfigError::Load(load) if load.kind() == RuntimeConfigErrorKind::RemovedKey => { + Some(load.field()) + } + _ => None, + }; + let path = if let Some(field) = removed_key { + format!("runtime.yaml:/{}", field.replace('.', "/")) } else if error.path() == "/" { "runtime.yaml".to_owned() } else { format!("runtime.yaml:/{}", error.path().trim_start_matches('/')) }; - let action = if matches!(error, RuntimeConfigError::RemovedPrincipalClaim) { - "Remove authentication.oidc.principalClaim and configure accessProfiles[].principalClaim in casework.yaml." - } else { - "Correct the named runtime configuration field, then retry." + if matches!(error, RuntimeConfigError::AllowedClientsRequired) { + return ( + "runtime_configuration", + path, + "List every client identifier this deployment admits in authentication.oidc.allowedClients, then retry.", + ); + } + let action = match removed_key { + Some("authentication.oidc.principalClaim") => { + "Remove authentication.oidc.principalClaim and configure accessProfiles[].principalClaim in casework.yaml." + } + Some("authentication.oidc.jwksUri") => { + "Replace authentication.oidc.jwksUri with authentication.oidc.jwksSource, kind: uri, and the same https URL as uri." + } + _ => "Correct the named runtime configuration field, then retry.", }; ("runtime_configuration", path, action) } @@ -942,8 +998,8 @@ fn run(cli: Cli) -> Result { Command::Explain(args) => project::explain(&args.project), Command::Lifecycle => lifecycle::lifecycle(), Command::Package(args) => match args.output { - Some(output) => project::package(&args.project, &output), - None => project::package_dry_run(&args.project), + Some(output) => project::package(&args.project, &output, args.revision.as_deref()), + None => project::package_dry_run(&args.project, args.revision.as_deref()), }, Command::Simulate(args) => project::simulate(&args.project, &args.fixture), Command::Test(args) => project::test(&args.project), @@ -996,6 +1052,8 @@ fn run(cli: Cli) -> Result { #[cfg(test)] mod tests { use super::*; + use registry_casework::RuntimeConfig; + use std::fs; #[test] fn internal_service_guard_preserves_hyphenated_service_arguments() { @@ -1195,7 +1253,7 @@ mod tests { #[test] fn authoring_refusal_preserves_its_message_and_names_authored_input() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("casework"); project::init(&project, "standalone-decision").unwrap(); for entry in std::fs::read_dir(project.join("fixtures")).unwrap() { @@ -1247,7 +1305,7 @@ mod tests { #[test] fn check_reports_the_exact_broken_review_policy_binding() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("casework"); let example = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../products/casework/examples/multi-stage-routing-clocks"); @@ -1295,7 +1353,7 @@ mod tests { #[test] fn denied_authoring_findings_use_exit_one_and_the_same_diagnostics() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("casework"); project::init(&project, "professional-review").unwrap(); let mut stdout = Vec::new(); @@ -1345,9 +1403,47 @@ mod tests { .starts_with("finding[casework.source-description.missing]")); } + #[test] + fn check_refuses_an_environment_expression_in_the_authored_project() { + let root = crate::canonical_tempdir(); + let project = root.path().join("standalone"); + project::init(&project, "standalone-decision").unwrap(); + let policy_path = project.join("casework.yaml"); + let mut policy: Value = + serde_norway::from_slice(&std::fs::read(&policy_path).unwrap()).unwrap(); + assert!(policy["queues"][0]["label"].is_string()); + policy["queues"][0]["label"] = json!("${QUEUE_LABEL}"); + std::fs::write(&policy_path, serde_norway::to_string(&policy).unwrap()).unwrap(); + + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let exit = main_entry_from( + [ + OsString::from("caseworkctl"), + OsString::from("--format=json"), + OsString::from("check"), + project.into_os_string(), + ], + &mut stdout, + &mut stderr, + ); + assert_eq!(exit, ExitCode::from(1)); + assert!(stderr.is_empty()); + let report: Value = serde_json::from_slice(&stdout).unwrap(); + let diagnostic = &report["diagnostics"][0]; + assert_eq!(diagnostic["code"], "casework.project.invalid"); + assert_eq!(diagnostic["path"], "casework.yaml:/queues/0/label"); + assert!(diagnostic["message"] + .as_str() + .is_some_and(|message| message.contains("runtime.yaml only"))); + assert!(diagnostic["suggestedAction"] + .as_str() + .is_some_and(|action| action.contains("casework.yaml"))); + } + #[test] fn review_connection_retention_diagnostic_names_the_incompatible_pair() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); project::init(&project, "standalone-decision").unwrap(); let policy_path = project.join("casework.yaml"); @@ -1388,7 +1484,7 @@ mod tests { #[test] fn operational_failures_follow_the_selected_output_channel() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let missing = root.path().join("missing-runtime.yaml"); let mut stdout = Vec::new(); let mut stderr = Vec::new(); @@ -1437,7 +1533,7 @@ mod tests { #[test] fn runtime_and_project_parse_diagnostics_never_echo_rejected_values() { const REJECTED_VALUE: &str = "value-that-must-not-be-echoed"; - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let runtime = root.path().join("runtime.yaml"); std::fs::write( &runtime, @@ -1489,18 +1585,63 @@ mod tests { } #[test] - fn removed_principal_claim_diagnostic_names_the_replacement_path_and_action() { - let error = anyhow::Error::new(RuntimeConfigError::RemovedPrincipalClaim); + fn removed_runtime_keys_name_the_replacement_path_and_action() { + let root = crate::canonical_tempdir(); + let runtime_config = root.path().join("runtime.yaml"); + for (removed, path, replacement) in [ + ( + "principalClaim: sub", + "runtime.yaml:/authentication/oidc/principalClaim", + "accessProfiles[].principalClaim", + ), + ( + "jwksUri: https://issuer.example/jwks", + "runtime.yaml:/authentication/oidc/jwksUri", + "authentication.oidc.jwksSource", + ), + ] { + fs::write( + &runtime_config, + format!( + "apiVersion: {}\nkind: {}\nauthentication:\n oidc:\n {removed}\n", + registry_casework::RUNTIME_CONFIG_API_VERSION, + registry_casework::RUNTIME_CONFIG_KIND, + ), + ) + .unwrap(); + let error = anyhow::Error::new(RuntimeConfig::load(&runtime_config).unwrap_err()); + let (exit, diagnostic) = classify_failure(CommandKind::Operational, &error); + assert_eq!(exit, DOMAIN_REFUSAL_EXIT); + assert_eq!(diagnostic["code"], "casework.runtime-configuration.invalid"); + assert_eq!(diagnostic["path"], path); + assert!( + diagnostic["suggestedAction"] + .as_str() + .unwrap() + .contains(replacement), + "{diagnostic}" + ); + } + } + + #[test] + fn a_production_runtime_without_allowed_clients_names_the_field_to_fill() { + let error = anyhow::Error::new(RuntimeConfigError::AllowedClientsRequired); let (exit, diagnostic) = classify_failure(CommandKind::Operational, &error); + assert_eq!(exit, DOMAIN_REFUSAL_EXIT); + assert_eq!(diagnostic["code"], "casework.runtime-configuration.invalid"); assert_eq!( diagnostic["path"], - "runtime.yaml:/authentication/oidc/principalClaim" + "runtime.yaml:/authentication.oidc.allowedClients" + ); + assert!( + diagnostic["suggestedAction"] + .as_str() + .unwrap() + .contains("List every client identifier"), + "{diagnostic}" ); - assert!(diagnostic["suggestedAction"] - .as_str() - .unwrap() - .contains("accessProfiles[].principalClaim")); } #[test] @@ -1653,6 +1794,93 @@ mod tests { } } + #[test] + fn doctor_names_the_check_that_failed_instead_of_a_generic_dependency_failure() { + let doctor = command_kind( + &Cli::try_parse_from(["caseworkctl", "doctor", "--runtime-config", "runtime.yaml"]) + .unwrap() + .command, + ); + let unmigrated = project::doctor_dependency_failure( + "database", + "the Casework runtime database is not ready".to_owned(), + "Apply the migrations with casework migrate or caseworkctl db migrate, then retry.", + anyhow::Error::new(StoreError::SchemaNotCurrent { + applied: None, + required: 17, + }) + .context("checking the database"), + ); + let (exit, diagnostic) = classify_failure(doctor, &unmigrated); + assert_eq!(exit, OPERATIONAL_FAILURE_EXIT); + assert_eq!(diagnostic["code"], "casework.doctor.check-failed"); + assert_eq!(diagnostic["artifact"], "runtime_dependency"); + assert_eq!(diagnostic["path"], "doctor:/checks/database"); + assert_eq!( + diagnostic["message"], + "the Casework runtime database is not ready: the Casework database schema is not current: no migration has been applied, and this binary requires version 17; apply the migrations with `casework migrate` or `caseworkctl db migrate`" + ); + assert_eq!( + diagnostic["suggestedAction"], + "Apply the migrations with casework migrate or caseworkctl db migrate, then retry." + ); + + // A cause outside the closed set of Casework errors is never echoed: + // it may carry a connection string or a response body. + let opaque = project::doctor_dependency_failure( + "sourceConnections", + "source registry did not answer the reader readiness check".to_owned(), + "Check the source binding, then retry.", + anyhow::anyhow!("postgresql://user:do-not-echo@db.example.test/casework"), + ); + let (_, diagnostic) = classify_failure(doctor, &opaque); + assert_eq!(diagnostic["path"], "doctor:/checks/sourceConnections"); + assert_eq!( + diagnostic["message"], + "source registry did not answer the reader readiness check" + ); + + // A BReg engine from another release is named with the lock-step + // action instead of the generic connection advice. + let mismatch = project::source_readiness_failure( + "registry", + Some("BReg source registry runs engine version 0.0.1 and this Casework runs 0.0.2. Casework and BReg run in lock-step, so upgrade both to the same release".to_owned()), + anyhow::Error::new(registry_casework_core::SourceAdapterError::Unavailable), + ); + let (exit, diagnostic) = classify_failure(doctor, &mismatch); + assert_eq!(exit, OPERATIONAL_FAILURE_EXIT); + assert_eq!(diagnostic["path"], "doctor:/checks/sourceConnections"); + assert!(diagnostic["message"] + .as_str() + .is_some_and(|message| message.contains("0.0.1") && message.contains("0.0.2"))); + assert!(diagnostic["suggestedAction"] + .as_str() + .is_some_and(|action| action.contains("same release"))); + let unmatched = project::source_readiness_failure( + "registry", + None, + anyhow::Error::new(registry_casework_core::SourceAdapterError::Unavailable), + ); + let (_, diagnostic) = classify_failure(doctor, &unmatched); + assert_eq!( + diagnostic["message"], + "source registry did not pass the reader readiness check: the source is temporarily unavailable" + ); + + // A typed configuration refusal keeps its precise location. + let typed = project::doctor_dependency_failure( + "configuration", + "the Casework runtime configuration is invalid".to_owned(), + "Correct the configuration, then retry.", + anyhow::Error::new(RuntimeConfigError::Oidc), + ); + let (_, diagnostic) = classify_failure(doctor, &typed); + assert_eq!( + diagnostic["code"], + "casework.runtime-dependency.unavailable" + ); + } + #[test] fn a_pending_attempt_settlement_refusal_names_the_recovery_step() { let pending = anyhow::Error::new(AttemptSettlementError::NotUncertain("pending")) @@ -1985,3 +2213,12 @@ mod tests { #[cfg(test)] mod cli_contract_tests; + +/// A temporary directory under the canonical system temporary root. The +/// runtime configuration loader refuses a path through a symbolic link, and +/// the system temporary root is one on some hosts. +#[cfg(test)] +pub(crate) fn canonical_tempdir() -> tempfile::TempDir { + let root = std::fs::canonicalize(std::env::temp_dir()).expect("canonical temporary root"); + tempfile::tempdir_in(root).expect("temporary directory") +} diff --git a/crates/registry-caseworkctl/src/project.rs b/crates/registry-caseworkctl/src/project.rs index 450346da0b..81565f3fe1 100644 --- a/crates/registry-caseworkctl/src/project.rs +++ b/crates/registry-caseworkctl/src/project.rs @@ -2,8 +2,8 @@ use anyhow::{bail, Context, Result}; use registry_casework::{ - open_audit, secret_resolver, validate_breg_source_description, verify_policy_package, - PolicyPackageManifest, PostgresStore, RuntimeConfig, POLICY_PACKAGE_MANIFEST_FILE, + open_audit, package_limits, secret_resolver, validate_breg_source_description, PostgresStore, + RuntimeConfig, SourceReconciliationHealth, PACKAGE_COMMAND, RECONCILIATION_FAILURE_THRESHOLD, }; use registry_casework_breg::MAXIMUM_REQUEST_ENTITIES; use registry_casework_core::{ @@ -11,8 +11,12 @@ use registry_casework_core::{ AttemptUncertainMarkingReport, CaseworkProject, ReviewContextStrategy, ReviewKindPurpose, SourcePolicy, SourceRequestPolicy, SourceRetentionReport, SourceRetentionSelector, }; -use registry_platform_config::{SecretError, SecretProvider, SecretReference, SecretResolver}; +use registry_platform_config::{ + plan_package, sha256_uri, write_package, SecretError, SecretProvider, SecretReference, + SecretResolver, +}; use serde_json::{json, Value}; +use std::collections::BTreeMap; use std::fs; use std::os::unix::fs::DirBuilderExt as _; use std::path::{Path, PathBuf}; @@ -22,8 +26,12 @@ const CASEWORK_YAML: &str = include_str!("../templates/professional-review/casew const RUNTIME_SCHEMA: &str = include_str!("../../../products/casework/generated/runtime/runtime.schema.json"); +/// Where the generated runtime example expects the package +/// `caseworkctl package . --output .casework/package` writes. +pub(crate) const LOCAL_PACKAGE_DIRECTORY: &str = ".casework/package"; + fn runtime_example(project: &Path, include_source: bool) -> Result { - let package_root = if project.is_absolute() { + let project_root = if project.is_absolute() { project.to_path_buf() } else { std::env::current_dir()?.join(project) @@ -31,9 +39,9 @@ fn runtime_example(project: &Path, include_source: bool) -> Result { let mut document = json!({ "apiVersion": "registry.registrystack.org/casework-runtime/v1alpha1", "kind": "CaseworkRuntimeConfig", - "package": {"root": &package_root}, + "package": {"root": project_root.join(LOCAL_PACKAGE_DIRECTORY)}, "listener": {"bind": "127.0.0.1:8100", "tlsTermination": "development-loopback", "networkExposure": "private-address"}, - "secretProviders": {"file": {"root": package_root.join("secrets")}}, + "secretProviders": {"file": {"root": project_root.join("secrets")}}, "database": {"runtimeUrlRef": "secret:file/runtime-database-url", "migrationUrlRef": "secret:file/migration-database-url"}, "authentication": {"oidc": { "issuer": "https://identity.example.test/realms/registry", @@ -41,7 +49,7 @@ fn runtime_example(project: &Path, include_source: bool) -> Result { "scopeClaim": "scope", "humanIdentity": {"claim": "registry_actor_kind", "value": "human"} }}, - "audit": {"path": package_root.join("state/audit.ndjson"), "hashKeyRef": "secret:file/casework-audit-key"}, + "audit": {"path": project_root.join("state/audit.ndjson"), "hashKeyRef": "secret:file/casework-audit-key"}, "sources": {} }); if include_source { @@ -508,8 +516,11 @@ fn load_yaml(path: &Path, label: &str) -> Result { serde_norway::from_slice(&bytes).with_context(|| format!("parsing {label}")) } pub(super) fn load_and_check_policy(project: &Path) -> Result { - let policy = CaseworkProject::load(project.join("casework.yaml")) - .context("loading and checking casework.yaml")?; + let policy_path = project.join("casework.yaml"); + let policy = + CaseworkProject::load(&policy_path).context("loading and checking casework.yaml")?; + let authored = fs::read_to_string(&policy_path).context("reading casework.yaml")?; + registry_platform_config::reject_environment_expressions_in_authored_yaml(&authored)?; if policy.sources.is_empty() { if policy.review_kinds.is_empty() || policy.review_producers.is_empty() { bail!("declare a review kind and producer or connect a source before checking the project"); @@ -544,49 +555,61 @@ pub(super) fn simulate(project: &Path, fixture: &Path) -> Result { /// `package_dry_run` share before any filesystem write. struct PackageContents { project: PathBuf, - manifest: PolicyPackageManifest, - inputs: Vec<(String, Vec)>, + inputs: BTreeMap>, } /// Canonicalize the project, run every package validation, and assemble the -/// exact inputs and identity a package would carry. Performs no writes, so -/// both `package` and `package_dry_run` can share it. +/// exact inputs a package would carry. Performs no writes, so both `package` +/// and `package_dry_run` can share it. fn compute_package(project: &Path) -> Result { let project = fs::canonicalize(project).context("resolving the Casework authoring project")?; let policy = load_and_check_policy(&project)?; check_source_descriptions(&project)?; crate::policy::check(&project, &policy)?; - let mut inputs = vec![( + let mut inputs = BTreeMap::from([( "casework.yaml".to_owned(), read_package_input(&project.join("casework.yaml"))?, - )]; + )]); for source in &policy.sources { let path = project_input_path(&project, &source.description)?; - inputs.push((source.description.clone(), read_package_input(&path)?)); - } - let manifest = PolicyPackageManifest::build(inputs.clone()) - .context("building the Casework policy package identity")?; - Ok(PackageContents { - project, - manifest, - inputs, - }) + inputs.insert(source.description.clone(), read_package_input(&path)?); + } + Ok(PackageContents { project, inputs }) +} + +fn package_files(inputs: &BTreeMap>) -> Vec { + inputs + .iter() + .map(|(path, bytes)| { + json!({ + "path": path, + "sha256": sha256_uri(bytes), + "bytes": bytes.len(), + }) + }) + .collect() } -/// Report the exact `policyDigest` and `files` a package of this project +/// Report the exact `packageDigest` and `files` a package of this project /// would carry, without writing anything. -pub(super) fn package_dry_run(project: &Path) -> Result { - let PackageContents { - project, manifest, .. - } = compute_package(project)?; +pub(super) fn package_dry_run(project: &Path, revision: Option<&str>) -> Result { + let PackageContents { project, inputs } = compute_package(project)?; + let digest = plan_package( + &project, + &inputs, + revision, + &package_limits(), + PACKAGE_COMMAND, + )?; Ok(json!({ "ok": true, "command": "package", "project": project, "dryRun": true, - "policyDigest": manifest.policy_digest, - "files": manifest.files, + "packageDigest": digest, + "revision": revision, + "files": package_files(&inputs), "runtimeConfigurationIncluded": false, "secretsIncluded": false, "networkAccess": false, @@ -594,57 +617,24 @@ pub(super) fn package_dry_run(project: &Path) -> Result { })) } -pub(super) fn package(project: &Path, output: &Path) -> Result { - let PackageContents { - project, - manifest, - inputs, - } = compute_package(project)?; - - if output.exists() { - bail!("policy package output already exists"); - } - if let Some(parent) = output - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - { - fs::create_dir_all(parent).context("creating the policy package parent directory")?; - } - fs::create_dir(output).context("creating the new policy package directory")?; - let published = (|| -> Result<()> { - for (relative, bytes) in &inputs { - let target = output.join(relative); - if let Some(parent) = target.parent() { - fs::create_dir_all(parent).context("creating policy package directories")?; - } - fs::write(&target, bytes).context("writing a policy package input")?; - } - let mut manifest_bytes = serde_json::to_vec_pretty(&manifest)?; - manifest_bytes.push(b'\n'); - fs::write(output.join(POLICY_PACKAGE_MANIFEST_FILE), manifest_bytes) - .context("writing the policy package manifest")?; - let loaded = CaseworkProject::load(output.join("casework.yaml")) - .context("loading the staged Casework policy")?; - let verified = verify_policy_package(&output.join("casework.yaml"), &loaded) - .context("verifying the staged Casework policy package")?; - if verified.as_deref() != Some(manifest.policy_digest.as_str()) { - bail!("staged Casework policy package identity changed"); - } - Ok(()) - })(); - if let Err(error) = published { - let _ = fs::remove_dir_all(output); - return Err(error); - } - +pub(super) fn package(project: &Path, output: &Path, revision: Option<&str>) -> Result { + let PackageContents { project, inputs } = compute_package(project)?; + let written = write_package( + output, + &inputs, + revision, + &package_limits(), + PACKAGE_COMMAND, + )?; Ok(json!({ "ok": true, "command": "package", "project": project, "output": output, "dryRun": false, - "policyDigest": manifest.policy_digest, - "files": manifest.files, + "packageDigest": written.digest(), + "revision": written.revision(), + "files": package_files(&inputs), "runtimeConfigurationIncluded": false, "secretsIncluded": false, "networkAccess": false, @@ -778,8 +768,8 @@ fn secret_references(config: &RuntimeConfig) -> Vec<(String, &str)> { if let Some(reference) = config.database.trusted_root_certificate_ref.as_deref() { references.push(("database.trustedRootCertificateRef".to_owned(), reference)); } - if let registry_casework::OidcJwksSource::Static { document_ref } = - &config.authentication.oidc.jwks_source + if let registry_casework::JwksSource::Static { document_ref } = + &config.authentication.oidc.provider.jwks_source { references.push(( "authentication.oidc.jwksSource.documentRef".to_owned(), @@ -788,7 +778,7 @@ fn secret_references(config: &RuntimeConfig) -> Vec<(String, &str)> { } references.push(( "audit.hashKeyRef".to_owned(), - config.audit.hash_key_ref.as_str(), + config.audit.key.hash_key_ref.as_str(), )); for (id, binding) in &config.sources { for (setting, reference) in [ @@ -906,6 +896,132 @@ fn secret_file_checks(config: &RuntimeConfig, resolver: &SecretResolver) -> Resu Ok(checks) } +/// A `doctor` check that failed, named so the operator knows which +/// dependency to repair. +/// +/// Its message is built only from the check's own sentence and from errors +/// whose text never carries a connection string, a response body, or a +/// secret value. +#[derive(Debug)] +pub(crate) struct DoctorCheckFailure { + pub(crate) check: &'static str, + pub(crate) message: String, + pub(crate) action: &'static str, +} + +impl std::fmt::Display for DoctorCheckFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for DoctorCheckFailure {} + +/// Name the doctor check a runtime dependency failed. +/// +/// A typed configuration error is returned unchanged, because it already +/// carries its exact location. Any other cause is summarized by the first +/// Casework error in its chain whose text is value-free; an opaque cause is +/// not echoed. +pub(crate) fn doctor_dependency_failure( + check: &'static str, + message: String, + action: &'static str, + error: anyhow::Error, +) -> anyhow::Error { + if carries_typed_configuration_error(&error) { + return error; + } + let cause = error.chain().find_map(|cause| { + if let Some(store) = cause.downcast_ref::() { + Some(store.to_string()) + } else if let Some(source) = + cause.downcast_ref::() + { + Some(source.to_string()) + } else { + cause.downcast_ref::().map(ToString::to_string) + } + }); + let message = match cause { + Some(cause) => format!("{message}: {cause}"), + None => message, + }; + anyhow::Error::new(DoctorCheckFailure { + check, + message, + action, + }) +} + +/// Name a source that failed the reader readiness check. A BReg engine from +/// another release is named with both versions and the lock-step action, since +/// no connection or grant change would repair it. +pub(crate) fn source_readiness_failure( + source_id: &str, + peer_version_mismatch: Option, + error: anyhow::Error, +) -> anyhow::Error { + match peer_version_mismatch { + Some(message) => anyhow::Error::new(DoctorCheckFailure { + check: "sourceConnections", + message, + action: DOCTOR_PEER_VERSION_ACTION, + }), + None => doctor_dependency_failure( + "sourceConnections", + format!("source {source_id} did not pass the reader readiness check"), + DOCTOR_SOURCE_ACTION, + error, + ), + } +} + +/// Name the doctor check an authored-input check failed. Its refusal text is +/// written by caseworkctl from the operator's own files, so it is kept whole. +fn doctor_check_failure( + check: &'static str, + action: &'static str, + error: anyhow::Error, +) -> anyhow::Error { + if carries_typed_configuration_error(&error) + || error.chain().any(|cause| cause.is::()) + { + return error; + } + let message = format!("{error:#}"); + anyhow::Error::new(DoctorCheckFailure { + check, + message, + action, + }) +} + +fn carries_typed_configuration_error(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause.is::() + || cause.is::() + || cause.is::() + }) +} + +const DOCTOR_CONFIGURATION_ACTION: &str = + "Correct the operator source bindings named by the refusal so they match casework.yaml, then retry."; +const DOCTOR_SOURCE_ACTION: &str = "Check that the source runtime is reachable and ready, and that the configured reader profile holds exact get and list access to every declared request projection, then retry."; +const DOCTOR_PEER_VERSION_ACTION: &str = "Run Casework and every BReg source it reads from the same release: upgrade the one that is behind, then retry. Casework resumes source reads once the versions match, without a restart."; +const DOCTOR_DATABASE_ACTION: &str = "Restore the Casework database named by database.runtimeUrlRef; if its schema is not current, apply the migrations with casework migrate or caseworkctl db migrate, then retry."; +const DOCTOR_DIRECTORY_ACTION: &str = + "Authenticate as an Administrator and give every declared queue a serving team, then retry."; +const DOCTOR_RECONCILIATION_ACTION: &str = "Restore the source named by the refusal and read the casework runtime log for the failing pass; readiness recovers after the next pass that succeeds."; +const DOCTOR_PINNED_WORK_ACTION: &str = "Keep the earlier package active until the named work finishes, or, once you accept that it stays hidden or orphaned, set package.acknowledgeStrandedWork to the digest the refusal names, then retry."; +const DOCTOR_AUDIT_ACTION: &str = "Give audit.path an owner-only directory this user can write, and archive an audit file that ends in an incomplete entry before starting on a fresh path, then retry."; + +fn load_doctor_package(config: &RuntimeConfig) -> Result { + config + .load_package() + .context("verifying the configured Casework package") +} + pub(super) fn doctor(runtime_config: &Path) -> Result { let config = RuntimeConfig::load(runtime_config).context("loading Casework runtime configuration")?; @@ -913,65 +1029,170 @@ pub(super) fn doctor(runtime_config: &Path) -> Result { fs::canonicalize(runtime_config).context("resolving Casework runtime configuration")?; let package_root = fs::canonicalize(&config.package.root) .context("resolving the configured Casework package root")?; - check_source_descriptions(&package_root)?; + let package = load_doctor_package(&config)?; + let package_digest = package.digest(); + let policy = package.project(); let resolver = secret_resolver(&config).context("configuring Casework secret providers")?; - let secret_files = secret_file_checks(&config, &resolver)?; + let secret_files = secret_file_checks(&config, &resolver).map_err(|error| { + doctor_check_failure( + "secretFiles", + "Correct the secret files named by the refusal, then retry.", + error, + ) + })?; let runtime = async_runtime()?; // Resolve the audit key as a readiness check without retaining or reporting // its bytes. Database references are resolved inside PostgresStore. resolver - .resolve(&config.audit.hash_key_ref) - .context("the audit secret is unavailable")?; + .resolve(config.audit.key.hash_key_ref.as_str()) + .map_err(|error| { + doctor_dependency_failure( + "secretFiles", + "the audit secret named by audit.hashKeyRef is unavailable".to_owned(), + "Provide the audit secret named by audit.hashKeyRef, then retry.", + error.into(), + ) + })?; + let audit_failure = |message: &str, error: anyhow::Error| { + doctor_dependency_failure("audit", message.to_owned(), DOCTOR_AUDIT_ACTION, error) + }; config .audit .destination() - .context("the Casework audit destination is invalid")? + .map_err(|error| audit_failure("the Casework audit destination is invalid", error.into()))? .check_writable() - .context("the Casework audit destination is not writable by this user")?; - check_operator_audit_companion(&config, &resolver, &runtime)?; - let policy = load_and_check_policy(&package_root)?; + .map_err(|error| { + audit_failure( + "the Casework audit destination is not writable by this user", + error.into(), + ) + })?; + check_operator_audit_companion(&config, &resolver, &runtime).map_err(|error| { + audit_failure( + "the Casework operator audit destination is not writable, \ + or another caseworkctl invocation already holds its lock", + error, + ) + })?; if config.sources.len() != policy.sources.len() { - bail!("operator source bindings do not exactly match the authored Casework sources"); + return Err(anyhow::Error::new(DoctorCheckFailure { + check: "configuration", + message: "operator source bindings do not exactly match the authored Casework sources" + .to_owned(), + action: DOCTOR_CONFIGURATION_ACTION, + })); } - let mut source_checks = Vec::with_capacity(policy.sources.len()); + let mut adapters = Vec::with_capacity(policy.sources.len()); for source in &policy.sources { - let binding = config - .sources - .get(&source.id) - .with_context(|| format!("operator source binding {} is missing", source.id))?; + let Some(binding) = config.sources.get(&source.id) else { + return Err(anyhow::Error::new(DoctorCheckFailure { + check: "configuration", + message: format!("operator source binding {} is missing", source.id), + action: DOCTOR_CONFIGURATION_ACTION, + })); + }; let adapter = binding - .build_adapter(source, &package_root, &resolver) - .with_context(|| format!("source binding {} is invalid", source.id))?; + .build_adapter_from_description( + source, + package + .source_description(&source.description) + .ok_or_else(|| { + anyhow::anyhow!("verified package omitted {}", source.description) + })?, + &resolver, + ) + .map_err(|error| { + doctor_dependency_failure( + "configuration", + format!("source binding {} is invalid", source.id), + DOCTOR_CONFIGURATION_ACTION, + error.into(), + ) + })?; + adapters.push((source.id.clone(), adapter)); + } + for (source_id, adapter) in &adapters { runtime .block_on(adapter.verify_reader_readiness()) - .with_context(|| { - format!( - "source {} is unavailable, unready, or its configured reader lacks exact get/list access to the declared request projection or a readableRequestFields grant naming review_state", - source.id + .map_err(|error| { + source_readiness_failure( + source_id, + adapter + .peer_version_mismatch() + .map(|mismatch| mismatch.to_string()), + error.into(), ) })?; - source_checks.push(doctor_source_check(&source.id)); } + let database_failure = |error: anyhow::Error| { + doctor_dependency_failure( + "database", + "the Casework runtime database is not ready".to_owned(), + DOCTOR_DATABASE_ACTION, + error, + ) + }; let store = PostgresStore::connect_runtime(&config.database, &resolver) - .context("the Casework runtime database configuration is invalid")?; + .map_err(|error| database_failure(error.into()))?; runtime .block_on(store.ready()) - .context("the Casework runtime database is unavailable")?; + .map_err(|error| database_failure(error.into()))?; + let dyn_adapters = adapters + .iter() + .map(|(_, adapter)| adapter as &dyn registry_casework_core::SourceAdapter) + .collect::>(); + let conflicts = runtime + .block_on(registry_casework::stranded_pinned_work( + &store, + policy, + &dyn_adapters, + )) + .map_err(|error| database_failure(error.into()))?; + let pinned_work = doctor_pinned_work( + conflicts, + package_digest, + config.package.acknowledge_stranded_work.as_deref(), + )?; runtime .block_on(config.oidc_verifier(&resolver)) .context("the configured OIDC issuer is unavailable or incompatible")?; let expected_queue_ids: Vec<_> = policy.queues.iter().map(|queue| queue.id.clone()).collect(); if !runtime .block_on(store.directory_ready(&expected_queue_ids)) - .context("checking Casework directory readiness")? + .map_err(|error| database_failure(error.into()))? { - bail!("the Casework directory does not have a team serving every declared queue; authenticate as an Administrator and complete the queue assignments before retrying doctor"); + return Err(anyhow::Error::new(DoctorCheckFailure { + check: "directory", + message: "the Casework directory does not have a team serving every declared queue" + .to_owned(), + action: DOCTOR_DIRECTORY_ACTION, + })); } + let source_ids: Vec<_> = policy + .sources + .iter() + .map(|source| source.id.clone()) + .collect(); + let reconciliation = runtime + .block_on(store.reconciliation_health(&source_ids)) + .map_err(|error| database_failure(error.into()))?; + if let Some(failing) = reconciliation + .iter() + .find(|health| health.consecutive_failures >= RECONCILIATION_FAILURE_THRESHOLD) + { + return Err(anyhow::Error::new(DoctorCheckFailure { + check: "reconciliation", + message: reconciliation_failure_message(failing), + action: DOCTOR_RECONCILIATION_ACTION, + })); + } + let source_checks: Vec<_> = reconciliation.iter().map(doctor_source_check).collect(); Ok(json!({ "ok": true, "command": "doctor", "runtimeConfig": runtime_config, "packageRoot": package_root, + "packageDigest": package_digest, "checks": { "configuration": "ready", "secretFiles": "ready", @@ -979,22 +1200,75 @@ pub(super) fn doctor(runtime_config: &Path) -> Result { "sourceConnections": "ready", "audit": "ready", "database": "ready", + "pinnedWork": "ready", "oidcIssuer": "ready", - "directory": "ready" + "directory": "ready", + "reconciliation": "ready" }, "secretFileChecks": secret_files, "sourceChecks": source_checks, + "pinnedWork": pinned_work, "eventWiringGuidance": EVENT_WIRING_GUIDANCE })) } -fn doctor_source_check(source_id: &str) -> Value { +/// The doctor view of pinned work a package would strand, or the named +/// failure when the operator has not acknowledged that exact package. +fn doctor_pinned_work( + conflicts: Vec, + package_digest: &str, + acknowledged: Option<&str>, +) -> Result { + use registry_casework::PinnedWorkVerdict; + + let verdict = + match registry_casework::pinned_work_verdict(&conflicts, package_digest, acknowledged) { + PinnedWorkVerdict::Clear => "clear", + PinnedWorkVerdict::Acknowledged => "acknowledged", + PinnedWorkVerdict::Refused => { + return Err(anyhow::Error::new(DoctorCheckFailure { + check: "pinnedWork", + message: registry_casework::stranded_work_refusal(&conflicts, package_digest), + action: DOCTOR_PINNED_WORK_ACTION, + })); + } + }; + Ok(json!({"verdict": verdict, "conflicts": conflicts})) +} + +fn reconciliation_failure_message(health: &SourceReconciliationHealth) -> String { + let cause = health + .last_failure + .map(|failure| format!("; the last failure was {}", failure.as_str())) + .unwrap_or_default(); + let since = health + .last_succeeded_at + .map(|at| { + format!( + "; the last pass that succeeded finished at {}", + at.to_rfc3339() + ) + }) + .unwrap_or_else(|| "; no pass has succeeded".to_owned()); + format!( + "reconciliation of source {} failed {} consecutive passes{cause}{since}", + health.source_id, health.consecutive_failures + ) +} + +fn doctor_source_check(health: &SourceReconciliationHealth) -> Value { json!({ - "sourceId": source_id, + "sourceId": health.source_id, "runtime": "ready", "readerProfile": "ready", "requiredGrants": "ready", - "eventWiring": "unknown" + "eventWiring": "unknown", + "reconciliation": { + "consecutiveFailures": health.consecutive_failures, + "lastSucceededAt": health.last_succeeded_at, + "lastFailedAt": health.last_failed_at, + "lastFailure": health.last_failure + } }) } @@ -1164,7 +1438,6 @@ fn load_runtime(project: &Path, requested: Option<&Path>) -> Result Result { #[cfg(test)] mod tests { use super::*; + use registry_platform_config::{verify_package, ConfigBlockErrorKind, SUM_FILE}; + + /// Package the authored project where the generated runtime example + /// selects it, as `caseworkctl package . --output .casework/package` does. + fn package_locally(project: &Path) { + fs::create_dir_all(project.join(".casework")).unwrap(); + package(project, &project.join(LOCAL_PACKAGE_DIRECTORY), None).unwrap(); + } #[test] fn template_has_only_checkpoint_capabilities() { @@ -1465,7 +1746,10 @@ mod tests { #[test] fn doctor_never_reports_unattested_event_wiring_as_ready() { - let check = doctor_source_check("professional-register"); + let check = doctor_source_check(&SourceReconciliationHealth { + source_id: "professional-register".into(), + ..SourceReconciliationHealth::default() + }); assert_eq!(check["sourceId"], "professional-register"); assert_eq!(check["runtime"], "ready"); assert_eq!(check["readerProfile"], "ready"); @@ -1476,6 +1760,79 @@ mod tests { assert!(EVENT_WIRING_GUIDANCE.contains("confirm one lifecycle delivery")); } + #[test] + fn doctor_reports_reconciliation_health_and_names_a_failing_source() { + use chrono::TimeZone as _; + + let succeeded = chrono::Utc.with_ymd_and_hms(2026, 9, 24, 8, 0, 0).unwrap(); + let failed = chrono::Utc.with_ymd_and_hms(2026, 9, 24, 8, 5, 0).unwrap(); + let health = SourceReconciliationHealth { + source_id: "professional-register".into(), + consecutive_failures: RECONCILIATION_FAILURE_THRESHOLD, + last_succeeded_at: Some(succeeded), + last_failed_at: Some(failed), + last_failure: Some(registry_casework::ReconciliationFailure::SourceUnavailable), + }; + + let check = doctor_source_check(&health); + assert_eq!( + check["reconciliation"], + json!({ + "consecutiveFailures": RECONCILIATION_FAILURE_THRESHOLD, + "lastSucceededAt": "2026-09-24T08:00:00Z", + "lastFailedAt": "2026-09-24T08:05:00Z", + "lastFailure": "source-unavailable" + }) + ); + assert_eq!( + reconciliation_failure_message(&health), + format!( + "reconciliation of source professional-register failed {RECONCILIATION_FAILURE_THRESHOLD} consecutive passes; the last failure was source-unavailable; the last pass that succeeded finished at 2026-09-24T08:00:00+00:00" + ) + ); + assert_eq!( + reconciliation_failure_message(&SourceReconciliationHealth { + source_id: "professional-register".into(), + consecutive_failures: 7, + ..SourceReconciliationHealth::default() + }), + "reconciliation of source professional-register failed 7 consecutive passes; no pass has succeeded" + ); + } + + #[test] + fn doctor_refuses_unacknowledged_stranded_work_by_name_and_reports_the_rest() { + let digest = format!("sha256:{}", "a".repeat(64)); + let conflicts = vec![registry_casework::StrandedWork::QueueRemoved { + queue: "intake".into(), + reviews: 2, + work_items: 1, + }]; + + assert_eq!( + doctor_pinned_work(Vec::new(), &digest, None).unwrap(), + json!({"verdict": "clear", "conflicts": []}) + ); + assert_eq!( + doctor_pinned_work(conflicts.clone(), &digest, Some(&digest)).unwrap(), + json!({ + "verdict": "acknowledged", + "conflicts": [{"reason": "queue-removed", "queue": "intake", "reviews": 2, "workItems": 1}] + }) + ); + + let error = doctor_pinned_work(conflicts, &digest, None).unwrap_err(); + let failure = error.downcast_ref::().unwrap(); + assert_eq!(failure.check, "pinnedWork"); + assert_eq!( + failure.message, + format!( + "the policy package would strand work pinned under an earlier package: 2 in-flight reviews and 1 open work item are in queue intake, which the package no longer declares. Let that work finish under the earlier package, or set package.acknowledgeStrandedWork to {digest} to activate this package anyway" + ) + ); + assert_eq!(failure.action, DOCTOR_PINNED_WORK_ACTION); + } + #[test] fn source_retention_output_is_count_only() { let output = source_retention_output( @@ -1516,7 +1873,7 @@ mod tests { fn attempt_mark_uncertain_without_the_migration_credential_refuses_before_the_database() { use std::os::unix::fs::PermissionsExt as _; const RUNTIME_URL: &str = "postgresql://runtime-only@127.0.0.1:1/casework"; - let directory = tempfile::tempdir().unwrap(); + let directory = crate::canonical_tempdir(); fs::write(directory.path().join("casework.yaml"), CASEWORK_YAML).unwrap(); fs::create_dir(directory.path().join("sources")).unwrap(); fs::write( @@ -1529,6 +1886,7 @@ mod tests { runtime_example(directory.path(), true).unwrap(), ) .unwrap(); + package_locally(directory.path()); let secrets = directory.path().join("secrets"); fs::create_dir(&secrets).unwrap(); fs::set_permissions(&secrets, fs::Permissions::from_mode(0o700)).unwrap(); @@ -1681,7 +2039,7 @@ mod tests { ], ), ] { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join(template); let report = init(&project, template).unwrap(); assert!(report["created"] @@ -1725,7 +2083,7 @@ mod tests { .collect::>() ); } - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); init(&project, "standalone-decision").unwrap(); assert!(init(&project, "standalone-decision").is_err()); @@ -1735,7 +2093,7 @@ mod tests { fn the_secret_file_preflight_names_every_unusable_reference() { use std::os::unix::fs::PermissionsExt as _; - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let secrets = root.path().to_path_buf(); let write = |name: &str, bytes: &[u8], mode: u32| { let path = secrets.join(name); @@ -1790,9 +2148,10 @@ mod tests { fn the_secret_file_preflight_reports_one_result_per_reference() { use std::os::unix::fs::PermissionsExt as _; - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); init(&project, "standalone-decision").unwrap(); + package_locally(&project); let secrets = project.join("secrets"); fs::create_dir(&secrets).unwrap(); let audit = secrets.join("casework-audit-key"); @@ -1840,9 +2199,10 @@ mod tests { fn the_secret_file_preflight_checks_completion_destination_secrets() { use std::os::unix::fs::PermissionsExt as _; - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); init(&project, "standalone-decision").unwrap(); + package_locally(&project); let secrets = project.join("secrets"); fs::create_dir(&secrets).unwrap(); for name in ["casework-audit-key", "receiver-token", "notifier-key"] { @@ -1906,9 +2266,10 @@ mod tests { fn doctor_refuses_a_symlinked_file_root_for_a_migration_only_file_reference() { use std::os::unix::fs::PermissionsExt as _; - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); init(&project, "standalone-decision").unwrap(); + package_locally(&project); let actual_secrets = root.path().join("actual-secrets"); fs::create_dir(&actual_secrets).unwrap(); let migration = actual_secrets.join("migration-database-url"); @@ -1941,7 +2302,7 @@ mod tests { fn doctor_refuses_malformed_secret_references_before_live_checks() { use std::os::unix::fs::PermissionsExt as _; - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); init(&project, "standalone-decision").unwrap(); let secrets = project.join("secrets"); @@ -1979,14 +2340,15 @@ mod tests { assert!( matches!( runtime_error, - Some(registry_casework::RuntimeConfigError::InvalidSecretReference { path }) - if path == setting + Some(registry_casework::RuntimeConfigError::Block(block)) + if block.kind() == ConfigBlockErrorKind::InvalidSecretReference + && block.field() == setting ), "{error:#}" ); let refusal = format!("{error:#}"); assert!( - refusal.contains(&format!("{setting} is not a valid secret reference")), + refusal.contains(&format!("{setting} must be an exact secret")), "{refusal}" ); assert!(!refusal.contains(malformed), "{refusal}"); @@ -1995,7 +2357,7 @@ mod tests { #[test] fn doctor_refuses_a_completion_secret_in_a_reserved_header() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); init(&project, "standalone-decision").unwrap(); let runtime_config = project.join("runtime.example.yaml"); @@ -2030,6 +2392,7 @@ mod tests { use std::os::unix::fs::PermissionsExt as _; init(project, "standalone-decision").unwrap(); + package_locally(project); let secrets = project.join("secrets"); fs::create_dir(&secrets).unwrap(); let audit_key = secrets.join("casework-audit-key"); @@ -2048,7 +2411,7 @@ mod tests { #[test] fn doctor_refuses_a_locked_operator_audit_companion() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); let runtime_config = runtime_config_for_audit_companion_tests(&project); let config = RuntimeConfig::load(&runtime_config).unwrap(); @@ -2069,7 +2432,7 @@ mod tests { #[test] fn doctor_accepts_an_available_operator_audit_companion() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); let runtime_config = runtime_config_for_audit_companion_tests(&project); let config = RuntimeConfig::load(&runtime_config).unwrap(); @@ -2084,7 +2447,7 @@ mod tests { fn standalone_starter_checks_real_review_display_schema_without_a_source() { use std::os::unix::fs::PermissionsExt as _; - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); init(&project, "standalone-decision").unwrap(); assert_eq!( @@ -2114,6 +2477,7 @@ mod tests { value["review"]["display"]["undeclared"] = json!("synthetic"); fs::write(&fixture, serde_norway::to_string(&value).unwrap()).unwrap(); assert!(test(&project).is_err()); + package_locally(&project); let runtime = RuntimeConfig::load(project.join("runtime.example.yaml")).unwrap(); assert!(runtime.sources.is_empty()); assert_eq!( @@ -2131,33 +2495,38 @@ mod tests { #[test] fn package_stages_only_verified_policy_inputs_and_refuses_replacement() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); let output = root.path().join("deployment/policy"); init(&project, "standalone-decision").unwrap(); - let report = package(&project, &output).unwrap(); + let report = package(&project, &output, None).unwrap(); assert_eq!(report["command"], "package"); - assert!(report["policyDigest"] - .as_str() - .unwrap() - .starts_with("sha256:")); + let digest = report["packageDigest"].as_str().unwrap(); + assert!(digest.starts_with("sha256:")); + assert_eq!(report["revision"], Value::Null); assert_eq!(report["runtimeConfigurationIncluded"], false); assert_eq!(report["secretsIncluded"], false); assert!(output.join("casework.yaml").is_file()); - assert!(output.join(POLICY_PACKAGE_MANIFEST_FILE).is_file()); + assert!(output.join(SUM_FILE).is_file()); + assert!(!output.join("casework.package.json").exists()); assert!(!output.join("runtime.example.yaml").exists()); assert!(!output.join("fixtures").exists()); - assert!(package(&project, &output).is_err()); + let verified = verify_package(&output, &package_limits(), PACKAGE_COMMAND).unwrap(); + assert_eq!(verified.digest(), digest); + assert!(package(&project, &output, None).is_err()); fs::write(output.join("undeclared-input.json"), "{}\n").unwrap(); - let policy = CaseworkProject::load(output.join("casework.yaml")).unwrap(); - assert!(verify_policy_package(&output.join("casework.yaml"), &policy).is_err()); + let error = verify_package(&output, &package_limits(), PACKAGE_COMMAND).unwrap_err(); + assert!( + error.to_string().contains("undeclared-input.json"), + "{error}" + ); } #[test] fn package_pins_the_exact_strictly_decoded_source_description() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("authored"); let output = root.path().join("package"); fs::create_dir_all(project.join("sources")).unwrap(); @@ -2168,16 +2537,43 @@ mod tests { ) .unwrap(); - package(&project, &output).unwrap(); + package(&project, &output, None).unwrap(); assert!(output.join("sources/professional-licences.json").is_file()); - let policy = CaseworkProject::load(output.join("casework.yaml")).unwrap(); + verify_package(&output, &package_limits(), PACKAGE_COMMAND).unwrap(); + fs::write(output.join("sources/professional-licences.json"), "{}\n").unwrap(); + let error = verify_package(&output, &package_limits(), PACKAGE_COMMAND).unwrap_err(); assert!( - verify_policy_package(&output.join("casework.yaml"), &policy) - .unwrap() - .is_some() + error + .to_string() + .contains("sources/professional-licences.json"), + "{error}" ); - fs::write(output.join("sources/professional-licences.json"), "{}\n").unwrap(); - assert!(verify_policy_package(&output.join("casework.yaml"), &policy).is_err()); + } + + #[test] + fn packaging_twice_gives_the_same_digest_and_a_revision_changes_it() { + let root = crate::canonical_tempdir(); + let project = root.path().join("standalone"); + init(&project, "standalone-decision").unwrap(); + + let first = package(&project, &root.path().join("first"), None).unwrap(); + let second = package(&project, &root.path().join("second"), None).unwrap(); + assert_eq!(first["packageDigest"], second["packageDigest"]); + + let revised = package( + &project, + &root.path().join("revised"), + Some("2026-09 intake"), + ) + .unwrap(); + assert_eq!(revised["revision"], "2026-09 intake"); + assert_ne!(revised["packageDigest"], first["packageDigest"]); + assert_eq!( + fs::read_to_string(root.path().join("revised/REVISION")).unwrap(), + "2026-09 intake\n" + ); + assert!(package(&project, &root.path().join("bad"), Some("two\nlines")).is_err()); + assert!(!root.path().join("bad").exists()); } #[test] @@ -2195,29 +2591,32 @@ mod tests { count } - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); init(&project, "standalone-decision").unwrap(); let files_before = count_files(&project); let output = root.path().join("package"); - let dry = package_dry_run(&project).unwrap(); + let dry = package_dry_run(&project, None).unwrap(); assert_eq!(dry["command"], "package"); assert_eq!(dry["dryRun"], true); - assert!(dry["policyDigest"].as_str().unwrap().starts_with("sha256:")); + assert!(dry["packageDigest"] + .as_str() + .unwrap() + .starts_with("sha256:")); assert!(dry.get("output").is_none()); assert!(!output.exists()); assert_eq!(count_files(&project), files_before); - let real = package(&project, &output).unwrap(); + let real = package(&project, &output, None).unwrap(); assert_eq!(real["dryRun"], false); - assert_eq!(real["policyDigest"], dry["policyDigest"]); + assert_eq!(real["packageDigest"], dry["packageDigest"]); assert_eq!(real["files"], dry["files"]); } #[test] fn package_dry_run_still_refuses_an_invalid_project() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("standalone"); init(&project, "standalone-decision").unwrap(); let actual_policy = root.path().join("actual-casework.yaml"); @@ -2225,8 +2624,8 @@ mod tests { std::os::unix::fs::symlink(&actual_policy, project.join("casework.yaml")).unwrap(); let output = root.path().join("package"); - let real_error = format!("{:#}", package(&project, &output).unwrap_err()); - let dry_run_error = format!("{:#}", package_dry_run(&project).unwrap_err()); + let real_error = format!("{:#}", package(&project, &output, None).unwrap_err()); + let dry_run_error = format!("{:#}", package_dry_run(&project, None).unwrap_err()); assert_eq!(real_error, dry_run_error); assert!(dry_run_error.contains("regular file"), "{dry_run_error}"); assert!(!output.exists()); @@ -2234,14 +2633,14 @@ mod tests { #[test] fn source_description_paths_cannot_leave_the_project() { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); assert!(project_input_path(root.path(), "../source.json").is_err()); assert!(project_input_path(root.path(), "/tmp/source.json").is_err()); } #[test] fn generated_runtime_config_loads_through_runtime_contract() { - let directory = tempfile::tempdir().unwrap(); + let directory = crate::canonical_tempdir(); fs::write(directory.path().join("casework.yaml"), CASEWORK_YAML).unwrap(); fs::create_dir(directory.path().join("sources")).unwrap(); fs::write( @@ -2251,11 +2650,58 @@ mod tests { .unwrap(); let runtime = directory.path().join("runtime.yaml"); fs::write(&runtime, runtime_example(directory.path(), true).unwrap()).unwrap(); + let unpackaged = RuntimeConfig::load(&runtime).expect_err("the runtime serves a package"); + assert!( + unpackaged.to_string().contains("caseworkctl package"), + "{unpackaged}" + ); + package_locally(directory.path()); let config = RuntimeConfig::load(runtime).unwrap(); assert_eq!(config.listener.bind, "127.0.0.1:8100".parse().unwrap()); assert!(config.sources.contains_key("professional-licences")); } + #[test] + fn doctor_package_helper_returns_one_validated_replacement_package() { + let directory = crate::canonical_tempdir(); + fs::write(directory.path().join("casework.yaml"), CASEWORK_YAML).unwrap(); + fs::create_dir(directory.path().join("sources")).unwrap(); + fs::write( + directory.path().join("sources/professional-licences.json"), + BREG_SOURCE_DESCRIPTION, + ) + .unwrap(); + let runtime = directory.path().join("runtime.yaml"); + fs::write(&runtime, runtime_example(directory.path(), true).unwrap()).unwrap(); + package_locally(directory.path()); + let config = RuntimeConfig::load(&runtime).expect("package A is valid"); + let first_digest = config.load_package().unwrap().digest().to_owned(); + + fs::rename( + directory.path().join(LOCAL_PACKAGE_DIRECTORY), + directory.path().join(".casework/package-a"), + ) + .unwrap(); + fs::write( + directory.path().join("casework.yaml"), + CASEWORK_YAML.replace( + "label: Licence corrections", + "label: Replacement corrections", + ), + ) + .unwrap(); + package_locally(directory.path()); + + let package = load_doctor_package(&config).expect("doctor validates package B"); + assert_ne!(package.digest(), first_digest); + assert_eq!(package.project().queues[0].label, "Replacement corrections"); + let source = &package.project().sources[0]; + assert_eq!( + package.source_description(&source.description).unwrap(), + BREG_SOURCE_DESCRIPTION.as_bytes() + ); + } + // registrystack/registry-stack#1256: a pinned BReg source description // may name a review.policyId that resolves to no declared reviewKinds // entry (or to one that cannot actually accept the submission). Nothing @@ -2267,7 +2713,7 @@ mod tests { casework_yaml: &str, description: &str, ) -> (tempfile::TempDir, PathBuf) { - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("authored"); fs::create_dir_all(project.join("sources")).unwrap(); fs::write(project.join("casework.yaml"), casework_yaml).unwrap(); @@ -2953,7 +3399,7 @@ mod tests { description["sourceId"] = json!("response-register"); description["request"]["requestEntity"] = json!("response-correction"); - let root = tempfile::tempdir().unwrap(); + let root = crate::canonical_tempdir(); let project = root.path().join("authored"); fs::create_dir_all(project.join("sources")).unwrap(); fs::write( diff --git a/crates/registry-caseworkctl/tests/dev_lifecycle.rs b/crates/registry-caseworkctl/tests/dev_lifecycle.rs index 0d27c65db5..2851adf40e 100644 --- a/crates/registry-caseworkctl/tests/dev_lifecycle.rs +++ b/crates/registry-caseworkctl/tests/dev_lifecycle.rs @@ -617,6 +617,26 @@ fn dev_serves_a_tutorial_project_through_candidate_facades_and_retains_its_recor ]); assert_eq!(doctor["checks"]["directory"], "ready", "{doctor:#}"); assert_eq!(doctor["checks"]["secretFiles"], "ready", "{doctor:#}"); + assert_eq!(doctor["checks"]["reconciliation"], "ready", "{doctor:#}"); + assert_eq!(doctor["checks"]["audit"], "ready", "{doctor:#}"); + assert_eq!(doctor["checks"]["pinnedWork"], "ready", "{doctor:#}"); + assert_eq!(doctor["pinnedWork"]["conflicts"], json!([]), "{doctor:#}"); + + // The runtime serves the package the start built from the authored + // project, verified at startup exactly as a deployed package is. + let package = fs::canonicalize(session.project.join(".casework/dev/package")) + .expect("the start built a session package"); + assert_eq!(doctor["packageRoot"], json!(package), "{doctor:#}"); + let sums = fs::read(package.join("SHA256SUMS")).expect("the session package has SHA256SUMS"); + assert_eq!( + doctor["packageDigest"], + registry_platform_config::sha256_uri(&sums), + "{doctor:#}" + ); + assert_eq!( + fs::read(package.join("casework.yaml")).expect("the packaged policy"), + fs::read(&policy_path).expect("the authored policy"), + ); let accepted = create_review_request(&session, &first, "synthetic-batch-0042"); let request = accepted["requestId"] diff --git a/crates/registry-cli-docs/src/lib.rs b/crates/registry-cli-docs/src/lib.rs index 58952fecbe..f0899ed5ad 100644 --- a/crates/registry-cli-docs/src/lib.rs +++ b/crates/registry-cli-docs/src/lib.rs @@ -106,7 +106,7 @@ mod tests { "registry-render init", "registry-render check", "registry-render validate", - "registry-render seal", + "registry-render package", "registry-render compile", "registry-render serve", "registry-render healthcheck", @@ -120,7 +120,7 @@ mod tests { let catalog = catalog(); let breg = find_command(&catalog.binaries, "breg"); assert!(breg.options.iter().any(|option| { - option.display == "--config " && option.always_required + option.display == "--runtime-config " && option.always_required })); let generate = find_command(&catalog.binaries, "bregctl generate"); assert!(generate.arguments.iter().any(|argument| { @@ -702,16 +702,17 @@ mod tests { argument.display == "--attribute-column " && argument.repeatable })); - for (invocation, option) in [ - ("evidence-oid4vci check", "--config "), - ("relay serve", "--runtime "), - ] { - assert!(find_command(&catalog.binaries, invocation) - .options - .iter() - .any(|argument| argument.display == option - && argument.environment.is_some() - && !argument.always_required)); - } + assert!(find_command(&catalog.binaries, "evidence-oid4vci check") + .options + .iter() + .any(|argument| argument.display == "--config " + && argument.environment.is_some() + && !argument.always_required)); + assert!(find_command(&catalog.binaries, "relay serve") + .options + .iter() + .any(|argument| argument.display == "--runtime-config " + && argument.environment.is_none() + && argument.always_required)); } } diff --git a/crates/registry-discovery-client/tests/native_journey.rs b/crates/registry-discovery-client/tests/native_journey.rs index bf4a3ff7f1..ae5c2019aa 100644 --- a/crates/registry-discovery-client/tests/native_journey.rs +++ b/crates/registry-discovery-client/tests/native_journey.rs @@ -18,7 +18,7 @@ use chrono::{SecondsFormat, TimeDelta, Utc}; use registry_discovery::{ load_index, mapping_revision, router as discovery_router, CompiledEvidenceMapping, Directory, DiscoveryService, EvidenceTypeAlternative, EvidenceTypeResolveRequest, ServiceFilters, - ServiceKind, + ServiceKind, INDEX_FILE, }; use registry_discovery_client::{ accept_service_selection, validate_service_selection_structure, DiscoveryClient, @@ -26,7 +26,7 @@ use registry_discovery_client::{ EvidenceTypeResolveSelectionExt, MatchedCapability, RelayCapabilityMatch, RelaySelectionRequest, RelayServiceQuery, ServiceSearchSelectionExt, ServiceSelection, }; -use registry_discoveryctl::{build_project_at, BuildError}; +use registry_discoveryctl::{package_project_at, BuildError}; use registry_evidence::config::EvidenceConfig; use registry_evidence_client::{ AssuranceProfile, EvidenceClient, EvidenceClientConfig, EvidenceRequestSpec, @@ -546,7 +546,8 @@ fn relay_description(provider_base: &str) -> Vec { let package = project.path().join("package-output"); let report = package_project(&PackageOptions { project_root: project.path().to_path_buf(), - output_dir: package.clone(), + output_dir: Some(package.clone()), + revision: None, }) .expect("the maintained Relay package operation runs"); assert!(report.is_success(), "Relay packaging refused: {report:?}"); @@ -721,23 +722,26 @@ async fn complete_evidence_and_relay_journeys_build_select_trust_and_invoke_nati let provider = start_provider().await; let project = TempDir::new().expect("the Discovery authoring project creates"); write_authoring_project(project.path(), &provider, false); - let index_path = project.path().join("discovery-index.json"); - build_project_at( + let package_root = project.path().join("discovery-package"); + package_project_at( project.path(), - &index_path, + &package_root, true, + None, OffsetDateTime::UNIX_EPOCH, ) .await - .expect("discoveryctl builds every approved local origin"); + .expect("discoveryctl packages every approved local origin"); assert_eq!(provider.origin_requests.load(Ordering::SeqCst), 3); + let index_path = package_root.join(INDEX_FILE); let valid_index = fs::read(&index_path).expect("the valid immutable index reads"); write_authoring_project(project.path(), &provider, true); - let invalid = build_project_at( + let invalid = package_project_at( project.path(), - &index_path, + &package_root, true, + None, OffsetDateTime::UNIX_EPOCH, ) .await; diff --git a/crates/registry-discovery/Cargo.toml b/crates/registry-discovery/Cargo.toml index 9914aa2630..1145fb4ec5 100644 --- a/crates/registry-discovery/Cargo.toml +++ b/crates/registry-discovery/Cargo.toml @@ -23,8 +23,8 @@ server = [ "dep:clap", "dep:http", "dep:registry-platform-buildinfo", + "dep:registry-platform-config", "dep:registry-platform-httpsec", - "dep:serde_yaml_ng", "dep:tokio", "dep:tower-http", "dep:tracing", @@ -38,10 +38,10 @@ http = { workspace = true, optional = true } registry-discovery-profile.workspace = true registry-platform-buildinfo = { workspace = true, optional = true } registry-platform-canonical-json.workspace = true +registry-platform-config = { workspace = true, optional = true } registry-platform-httpsec = { workspace = true, features = ["server"], optional = true } serde.workspace = true serde_json.workspace = true -serde_yaml_ng = { workspace = true, optional = true } sha2.workspace = true thiserror.workspace = true time.workspace = true diff --git a/crates/registry-discovery/src/lib.rs b/crates/registry-discovery/src/lib.rs index aedebb48f8..bfde753b4a 100644 --- a/crates/registry-discovery/src/lib.rs +++ b/crates/registry-discovery/src/lib.rs @@ -1,6 +1,17 @@ // SPDX-License-Identifier: Apache-2.0 //! Closed immutable index and read-only Registry Discovery service. +/// The canonical index inside every Discovery package. +pub const INDEX_FILE: &str = "discovery-index.json"; +/// The command that creates a Discovery package, named in package refusals. +pub const PACKAGE_COMMAND: &str = "discoveryctl package"; +/// The index plus the optional shared `REVISION` envelope file. +pub const MAXIMUM_PACKAGE_FILES: usize = 2; +/// The index plus a maximum-length shared `REVISION` line. +pub const MAXIMUM_PACKAGE_BYTES: u64 = MAXIMUM_INDEX_BYTES + 257; +/// Discovery package files live directly under the package root. +pub const MAXIMUM_PACKAGE_DEPTH: usize = 1; + pub mod model; pub mod openapi; #[cfg(feature = "server")] @@ -16,4 +27,8 @@ pub use query::{parse_service_filters, Directory, QueryError}; #[cfg(feature = "server")] pub use server::{router, DiscoveryService, ServiceConfigError}; #[cfg(feature = "server")] -pub use startup::{load_index, load_runtime, prepare, serve, PreparedDiscovery, StartupError}; +pub use startup::{ + load_index, load_runtime, load_verified_index, package_limits, prepare, serve, LogLevel, + PreparedDiscovery, RuntimeConfig, RuntimeLimits, StartupError, + MAXIMUM_LISTENER_BIND_CHARACTERS, RUNTIME_API_VERSION, RUNTIME_KIND, +}; diff --git a/crates/registry-discovery/src/main.rs b/crates/registry-discovery/src/main.rs index 4367c34c33..78e83e3d12 100644 --- a/crates/registry-discovery/src/main.rs +++ b/crates/registry-discovery/src/main.rs @@ -15,14 +15,15 @@ use tracing_subscriber::prelude::*; version = registry_platform_buildinfo::DISPLAY_VERSION )] struct Arguments { - #[arg(long, value_name = "FILE")] - runtime: PathBuf, + /// Absolute path of the runtime file naming the listener, index, and limits + #[arg(long = "runtime-config", value_name = "FILE")] + runtime_config: PathBuf, } #[tokio::main] async fn main() { let arguments = Arguments::parse(); - let level = load_runtime(&arguments.runtime) + let level = load_runtime(&arguments.runtime_config) .map(|(_, runtime)| match runtime.log_level { LogLevel::Error => LevelFilter::ERROR, LogLevel::Warn => LevelFilter::WARN, @@ -40,7 +41,7 @@ async fn main() { .with_span_list(false), ) .init(); - if let Err(error) = serve(&arguments.runtime).await { + if let Err(error) = serve(&arguments.runtime_config).await { tracing::error!(target: "registry_discovery::startup", error = %error, "Discovery stopped"); std::process::exit(1); } diff --git a/crates/registry-discovery/src/model.rs b/crates/registry-discovery/src/model.rs index 13182d6f2f..a72fa377d1 100644 --- a/crates/registry-discovery/src/model.rs +++ b/crates/registry-discovery/src/model.rs @@ -13,7 +13,6 @@ use url::Url; pub use registry_discovery_profile::ServiceKind; pub const INDEX_SCHEMA: &str = "registry-discovery/index/v1alpha1"; -pub const RUNTIME_SCHEMA: &str = "registry-discovery/runtime/v1alpha1"; pub const MAXIMUM_INDEX_BYTES: u64 = 64 * 1024 * 1024; pub const MAXIMUM_ORIGINS: usize = 1_024; pub const MAXIMUM_SERVICES: usize = 100_000; @@ -23,7 +22,6 @@ pub const MAXIMUM_EVIDENCE_TYPES_PER_ALTERNATIVE: usize = 128; pub const MAXIMUM_VALUES_PER_FIELD: usize = 256; pub const MAXIMUM_IDENTIFIER_CHARACTERS: usize = 4_096; pub const MAXIMUM_TEXT_CHARACTERS: usize = 16 * 1024; -pub const MAXIMUM_LISTENER_ADDRESS_CHARACTERS: usize = 128; pub const MAXIMUM_FILTER_VALUES: usize = 100; pub const MAXIMUM_QUERY_BYTES: usize = 64 * 1024; const MAXIMUM_QUERY_PARAMETERS: usize = (7 * MAXIMUM_FILTER_VALUES) + 2; @@ -167,41 +165,6 @@ pub struct ServiceSearchResponse { pub items: Vec, } -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct ListenerConfig { - pub address: String, -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct RuntimeLimits { - pub maximum_request_bytes: usize, - pub maximum_response_bytes: usize, - pub maximum_result_records: usize, - pub maximum_result_alternatives: usize, - pub request_timeout_seconds: u64, - pub shutdown_timeout_seconds: u64, -} - -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct RuntimeConfig { - pub schema_version: String, - pub listener: ListenerConfig, - pub index_path: String, - pub limits: RuntimeLimits, - pub log_level: LogLevel, -} - -#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "lowercase")] -pub enum LogLevel { - Error, - Warn, - Info, -} - #[derive(Debug, Error, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub enum IndexError { diff --git a/crates/registry-discovery/src/startup.rs b/crates/registry-discovery/src/startup.rs index ac5d2cba22..f06fd19b34 100644 --- a/crates/registry-discovery/src/startup.rs +++ b/crates/registry-discovery/src/startup.rs @@ -1,39 +1,123 @@ // SPDX-License-Identifier: Apache-2.0 //! Strict startup-only runtime and immutable-index activation. +use std::collections::BTreeSet; use std::fs; #[cfg(unix)] use std::future::Future; +use std::io::Read as _; use std::net::SocketAddr; -use std::path::{Component, Path, PathBuf}; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; use axum::Router; +use registry_platform_config::package::is_envelope_file; +use registry_platform_config::{ + sha256_uri, ListenerConfig, PackageConfig, PackageError, PackageLimits, RemovedKey, + RuntimeConfigLoader, RuntimeEnvelope, VerifiedPackage, +}; +use serde::Deserialize; use thiserror::Error; use tokio::net::TcpListener; use tokio::sync::oneshot; use crate::model::{ - parse_index, DiscoveryIndex, RuntimeConfig, MAXIMUM_HTTP_BODY_BYTES, - MAXIMUM_IDENTIFIER_CHARACTERS, MAXIMUM_INDEX_BYTES, MAXIMUM_LISTENER_ADDRESS_CHARACTERS, + parse_index, DiscoveryIndex, MAXIMUM_HTTP_BODY_BYTES, MAXIMUM_INDEX_BYTES, MAXIMUM_RESULT_ALTERNATIVES, MAXIMUM_RESULT_RECORDS, MINIMUM_HTTP_RESPONSE_BYTES, - RUNTIME_SCHEMA, }; use crate::query::Directory; use crate::server::{router, DiscoveryService}; +use crate::{ + INDEX_FILE, MAXIMUM_PACKAGE_BYTES, MAXIMUM_PACKAGE_DEPTH, MAXIMUM_PACKAGE_FILES, + PACKAGE_COMMAND, +}; + +pub use registry_platform_config::MAX_LISTENER_BIND_CHARACTERS as MAXIMUM_LISTENER_BIND_CHARACTERS; + +pub const RUNTIME_API_VERSION: &str = "registry.registrystack.org/discovery-runtime/v1alpha1"; +pub const RUNTIME_KIND: &str = "DiscoveryRuntimeConfig"; + +const RUNTIME_ENVELOPE: RuntimeEnvelope = RuntimeEnvelope { + api_version: RUNTIME_API_VERSION, + kind: RUNTIME_KIND, +}; + +const REMOVED_RUNTIME_KEYS: &[RemovedKey] = &[ + RemovedKey { + path: "schemaVersion", + replacement: "declare apiVersion registry.registrystack.org/discovery-runtime/v1alpha1 \ + and kind DiscoveryRuntimeConfig instead", + }, + RemovedKey { + path: "listener.address", + replacement: "declare listener.bind instead", + }, + RemovedKey { + path: "indexPath", + replacement: "declare package.root instead and build it with `discoveryctl package`", + }, +]; -const MAXIMUM_RUNTIME_BYTES: u64 = 1024 * 1024; const MAXIMUM_REQUEST_TIMEOUT_SECONDS: u64 = 300; const MAXIMUM_SHUTDOWN_TIMEOUT_SECONDS: u64 = 300; -#[derive(Debug, Error, Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct RuntimeLimits { + pub maximum_request_bytes: usize, + pub maximum_response_bytes: usize, + pub maximum_result_records: usize, + pub maximum_result_alternatives: usize, + pub request_timeout_seconds: u64, + pub shutdown_timeout_seconds: u64, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct RuntimeConfig { + pub api_version: String, + pub kind: String, + pub listener: ListenerConfig, + pub package: PackageConfig, + pub limits: RuntimeLimits, + pub log_level: LogLevel, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum LogLevel { + Error, + Warn, + Info, +} + +#[derive(Debug, Error, Clone, PartialEq, Eq)] #[non_exhaustive] pub enum StartupError { - #[error("the Discovery runtime configuration could not be loaded")] - RuntimeLoad, + /// The shared loader refused the runtime file; the message names the + /// file and the field and never carries a configured value. + #[error("the Discovery runtime configuration was refused: {0}")] + RuntimeRefused(String), #[error("the Discovery runtime configuration is invalid")] RuntimeInvalid, + #[error("{0}")] + Package(#[from] PackageError), + #[error( + "the Discovery package at package.root has invalid contents{details}; rebuild it with \ + `discoveryctl package`" + )] + PackageContents { details: String }, + #[error( + "the Discovery package file {path} changed after package verification; redeploy the whole \ + directory built by `discoveryctl package`" + )] + PackageFileChanged { path: String }, + #[error( + "the Discovery package file discovery-index.json is invalid; rebuild the package with \ + `discoveryctl package`" + )] + PackageIndexInvalid, #[error("the Discovery index could not be loaded")] IndexLoad, #[error("the Discovery index is invalid")] @@ -62,9 +146,17 @@ impl PreparedDiscovery { } pub fn prepare(runtime_path: &Path) -> Result { - let (root, runtime) = load_runtime(runtime_path)?; - let index_path = safe_existing_file(&root, &runtime.index_path)?; - let index = load_index(&index_path)?; + let (_, runtime) = load_runtime(runtime_path)?; + let verified = runtime + .package + .verify_package(&package_limits(), PACKAGE_COMMAND) + .map_err(|error| error.naming_root_as("package.root"))?; + let index = load_verified_index(&runtime.package.root, &verified)?; + tracing::info!( + target: "registry_discovery::startup", + package_digest = verified.digest(), + "Discovery package accepted" + ); let directory = Directory::new( index, runtime.limits.maximum_result_records, @@ -81,13 +173,8 @@ pub fn prepare(runtime_path: &Path) -> Result { Duration::from_secs(runtime.limits.request_timeout_seconds), ) .map_err(|_| StartupError::RuntimeInvalid)?; - let bind = runtime - .listener - .address - .parse() - .map_err(|_| StartupError::RuntimeInvalid)?; Ok(PreparedDiscovery { - bind, + bind: runtime.listener.bind.socket_addr(), app, shutdown_timeout: Duration::from_secs(runtime.limits.shutdown_timeout_seconds), }) @@ -167,46 +254,150 @@ fn map_server_result( } } +/// Load the runtime file through the shared runtime configuration loader. +/// Returns the directory holding the file and the validated configuration. pub fn load_runtime(path: &Path) -> Result<(PathBuf, RuntimeConfig), StartupError> { - let bytes = bounded_regular_file(path, MAXIMUM_RUNTIME_BYTES, StartupError::RuntimeLoad)?; - let runtime: RuntimeConfig = - serde_yaml_ng::from_slice(&bytes).map_err(|_| StartupError::RuntimeInvalid)?; + let runtime = RuntimeConfigLoader::new(RUNTIME_ENVELOPE) + .removed_keys(REMOVED_RUNTIME_KEYS) + .load::(path) + .map_err(|error| StartupError::RuntimeRefused(error.to_string()))? + .config; validate_runtime(&runtime)?; - let root = effective_parent(path) - .canonicalize() - .map_err(|_| StartupError::RuntimeLoad)?; + let root = path + .parent() + .ok_or(StartupError::RuntimeInvalid)? + .to_path_buf(); Ok((root, runtime)) } -fn effective_parent(path: &Path) -> &Path { - path.parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")) -} - pub fn load_index(path: &Path) -> Result { - let bytes = bounded_regular_file(path, MAXIMUM_INDEX_BYTES, StartupError::IndexLoad)?; + let bytes = bounded_regular_file(path, MAXIMUM_INDEX_BYTES)?; parse_index(&bytes).map_err(|_| StartupError::IndexInvalid) } -fn bounded_regular_file( - path: &Path, +/// The bounds for the one-index Discovery package and optional revision. +#[must_use] +pub fn package_limits() -> PackageLimits { + PackageLimits { + max_files: MAXIMUM_PACKAGE_FILES, + max_file_bytes: MAXIMUM_INDEX_BYTES, + max_total_bytes: MAXIMUM_PACKAGE_BYTES, + max_depth: MAXIMUM_PACKAGE_DEPTH, + ..PackageLimits::default() + } +} + +/// Capture and parse the exact index bytes named by an already verified +/// package. The second digest check binds the bytes consumed by the directory +/// to the `SHA256SUMS` entry retained in `verified`. +pub fn load_verified_index( + root: &Path, + verified: &VerifiedPackage, +) -> Result { + let expected = BTreeSet::from([INDEX_FILE.to_owned()]); + let found = verified + .files() + .filter(|path| !is_envelope_file(path)) + .map(str::to_owned) + .collect::>(); + if found != expected { + let mut details = String::new(); + let missing = expected.difference(&found).cloned().collect::>(); + let extra = found.difference(&expected).cloned().collect::>(); + if !missing.is_empty() { + details.push_str(&format!("; missing: {}", missing.join(", "))); + } + if !extra.is_empty() { + details.push_str(&format!("; extra: {}", extra.join(", "))); + } + return Err(StartupError::PackageContents { details }); + } + let path = root.join(INDEX_FILE); + let bytes = bounded_regular_file(&path, MAXIMUM_INDEX_BYTES).map_err(|_| { + StartupError::PackageFileChanged { + path: INDEX_FILE.to_owned(), + } + })?; + if verified.file_digest(INDEX_FILE).as_deref() != Some(sha256_uri(&bytes).as_str()) { + return Err(StartupError::PackageFileChanged { + path: INDEX_FILE.to_owned(), + }); + } + parse_index(&bytes).map_err(|_| StartupError::PackageIndexInvalid) +} + +fn bounded_regular_file(path: &Path, maximum: u64) -> Result, StartupError> { + let scanned = fs::symlink_metadata(path).map_err(|_| StartupError::IndexLoad)?; + if scanned.file_type().is_symlink() + || !scanned.is_file() + || scanned.len() == 0 + || scanned.len() > maximum + { + return Err(StartupError::IndexLoad); + } + let file = fs::File::open(path).map_err(|_| StartupError::IndexLoad)?; + let opened = file.metadata().map_err(|_| StartupError::IndexLoad)?; + let current = fs::symlink_metadata(path).map_err(|_| StartupError::IndexLoad)?; + if current.file_type().is_symlink() + || !current.is_file() + || !same_file(&scanned, &opened) + || !same_file(&opened, ¤t) + { + return Err(StartupError::IndexLoad); + } + read_opened_regular_file(file, &opened, maximum) +} + +fn read_opened_regular_file( + file: fs::File, + opened: &fs::Metadata, maximum: u64, - load_error: StartupError, ) -> Result, StartupError> { - let metadata = fs::symlink_metadata(path).map_err(|_| load_error)?; - if !metadata.file_type().is_file() || metadata.len() == 0 || metadata.len() > maximum { - return Err(load_error); + let capacity = usize::try_from(opened.len()).map_err(|_| StartupError::IndexLoad)?; + let mut bytes = Vec::new(); + bytes + .try_reserve(capacity) + .map_err(|_| StartupError::IndexLoad)?; + let mut reader = file.take(maximum.saturating_add(1)); + reader + .read_to_end(&mut bytes) + .map_err(|_| StartupError::IndexLoad)?; + let after = reader + .get_ref() + .metadata() + .map_err(|_| StartupError::IndexLoad)?; + if bytes.is_empty() + || u64::try_from(bytes.len()).map_or(true, |length| length > maximum) + || !same_file(opened, &after) + || u64::try_from(bytes.len()).ok() != Some(after.len()) + { + return Err(StartupError::IndexLoad); } - fs::read(path).map_err(|_| load_error) + Ok(bytes) +} + +#[cfg(unix)] +fn same_file(left: &fs::Metadata, right: &fs::Metadata) -> bool { + use std::os::unix::fs::MetadataExt as _; + + left.dev() == right.dev() + && left.ino() == right.ino() + && left.len() == right.len() + && left.mtime() == right.mtime() + && left.mtime_nsec() == right.mtime_nsec() +} + +#[cfg(not(unix))] +fn same_file(left: &fs::Metadata, right: &fs::Metadata) -> bool { + left.len() == right.len() && left.modified().ok() == right.modified().ok() } fn validate_runtime(runtime: &RuntimeConfig) -> Result<(), StartupError> { - if runtime.schema_version != RUNTIME_SCHEMA - || runtime.listener.address.chars().count() > MAXIMUM_LISTENER_ADDRESS_CHARACTERS - || runtime.index_path.is_empty() - || runtime.index_path.chars().count() > MAXIMUM_IDENTIFIER_CHARACTERS - || runtime.limits.maximum_request_bytes == 0 + runtime + .package + .check() + .map_err(|error| StartupError::RuntimeRefused(error.to_string()))?; + if runtime.limits.maximum_request_bytes == 0 || runtime.limits.maximum_request_bytes > MAXIMUM_HTTP_BODY_BYTES || runtime.limits.maximum_response_bytes < MINIMUM_HTTP_RESPONSE_BYTES || runtime.limits.maximum_response_bytes > MAXIMUM_HTTP_BODY_BYTES @@ -221,49 +412,16 @@ fn validate_runtime(runtime: &RuntimeConfig) -> Result<(), StartupError> { { return Err(StartupError::RuntimeInvalid); } - runtime - .listener - .address - .parse::() - .map_err(|_| StartupError::RuntimeInvalid)?; Ok(()) } -fn safe_existing_file(root: &Path, value: &str) -> Result { - let path = Path::new(value); - if value.is_empty() - || path.is_absolute() - || path - .components() - .any(|component| !matches!(component, Component::Normal(_))) - { - return Err(StartupError::RuntimeInvalid); - } - let mut resolved = root.to_path_buf(); - for component in path.components() { - let Component::Normal(component) = component else { - return Err(StartupError::RuntimeInvalid); - }; - resolved.push(component); - let metadata = fs::symlink_metadata(&resolved).map_err(|_| StartupError::IndexLoad)?; - if metadata.file_type().is_symlink() { - return Err(StartupError::RuntimeInvalid); - } - } - if !fs::symlink_metadata(&resolved) - .map_err(|_| StartupError::IndexLoad)? - .file_type() - .is_file() - { - return Err(StartupError::IndexInvalid); - } - Ok(resolved) -} - #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use super::*; use crate::model::{canonical_index_bytes, tests::example_index}; + use registry_platform_config::write_package; #[test] fn startup_loads_one_canonical_index_and_rejects_noncanonical_input() { @@ -284,12 +442,12 @@ mod tests { assert_eq!(load_index(&index_path), Err(StartupError::IndexInvalid)); } - #[test] - fn runtime_is_closed_and_contains_no_origin_mapping_trust_or_fetch_configuration() { - let raw = br#" -schemaVersion: registry-discovery/runtime/v1alpha1 -listener: { address: 127.0.0.1:8080 } -indexPath: discovery-index.json + const RUNTIME: &str = "\ +apiVersion: registry.registrystack.org/discovery-runtime/v1alpha1 +kind: DiscoveryRuntimeConfig +listener: { bind: 127.0.0.1:8080 } +package: + root: /tmp/registry-discovery-package limits: maximumRequestBytes: 65536 maximumResponseBytes: 1048576 @@ -298,50 +456,299 @@ limits: requestTimeoutSeconds: 10 shutdownTimeoutSeconds: 10 logLevel: info -origins: [{ catalogUrl: https://attacker.invalid/catalog.jsonld }] -"#; - assert!(serde_yaml_ng::from_slice::(raw).is_err()); +"; + + fn canonical_tempdir() -> tempfile::TempDir { + tempfile::tempdir_in(std::env::temp_dir().canonicalize().unwrap()).unwrap() + } + + fn write_runtime(directory: &Path, text: &str) -> PathBuf { + let path = directory.join("runtime.yaml"); + fs::write(&path, text).unwrap(); + path + } + + fn prepare_error(path: &Path) -> StartupError { + match prepare(path) { + Ok(_) => panic!("expected Discovery startup to fail"), + Err(error) => error, + } + } + + fn refusal(text: &str) -> String { + let temporary = canonical_tempdir(); + match load_runtime(&write_runtime(temporary.path(), text)) { + Err(StartupError::RuntimeRefused(message)) => message, + other => panic!("expected a loader refusal, got {other:?}"), + } + } + + #[test] + fn runtime_is_closed_and_contains_no_origin_mapping_trust_or_fetch_configuration() { + let message = refusal(&format!( + "{RUNTIME}origins: [{{ catalogUrl: https://attacker.invalid/catalog.jsonld }}]\n" + )); + assert!(message.contains("origins"), "{message}"); + } + + #[test] + fn the_runtime_file_is_read_through_the_shared_loader() { + let temporary = canonical_tempdir(); + let path = write_runtime(temporary.path(), RUNTIME); + let (root, runtime) = load_runtime(&path).expect("runtime loads"); + assert_eq!(root, temporary.path()); + assert_eq!(runtime.listener.bind.socket_addr().port(), 8080); + + let message = refusal(&RUNTIME.replace("kind: DiscoveryRuntimeConfig", "kind: Other")); + assert!( + message.contains("kind must be exactly DiscoveryRuntimeConfig"), + "{message}" + ); + + let relative = Path::new("runtime.yaml"); + assert!(matches!( + load_runtime(relative), + Err(StartupError::RuntimeRefused(_)) + )); + + #[cfg(unix)] + { + let linked = temporary.path().join("linked.yaml"); + std::os::unix::fs::symlink(&path, &linked).unwrap(); + assert!(matches!( + load_runtime(&linked), + Err(StartupError::RuntimeRefused(_)) + )); + } + } + + #[test] + fn removed_runtime_keys_name_their_replacements() { + let message = refusal(&RUNTIME.replace( + "apiVersion: registry.registrystack.org/discovery-runtime/v1alpha1\nkind: DiscoveryRuntimeConfig", + "schemaVersion: registry-discovery/runtime/v1alpha1", + )); + assert!( + message.contains("schemaVersion is no longer accepted; declare apiVersion"), + "{message}" + ); + let message = refusal(&RUNTIME.replace("bind:", "address:")); + assert!( + message + .contains("listener.address is no longer accepted; declare listener.bind instead"), + "{message}" + ); + let message = refusal(&RUNTIME.replace( + "package:\n root: /tmp/registry-discovery-package", + "indexPath: discovery-index.json", + )); + assert!( + message.contains("indexPath is no longer accepted; declare package.root instead"), + "{message}" + ); + } + + #[test] + fn the_listener_bind_is_required_and_substitutes_from_the_environment() { + let message = refusal(&RUNTIME.replace("listener: { bind: 127.0.0.1:8080 }\n", "")); + assert!(message.contains("listener"), "{message}"); + + let temporary = canonical_tempdir(); + let path = write_runtime( + temporary.path(), + &RUNTIME.replace( + "127.0.0.1:8080", + "\"${DISCOVERY_TEST_BIND:-127.0.0.1:9090}\"", + ), + ); + let (_, runtime) = load_runtime(&path).expect("default substitutes"); + assert_eq!(runtime.listener.bind.socket_addr().port(), 9090); } #[test] fn shipped_runtime_fixture_matches_the_closed_runtime_contract() { let path = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../products/discovery/fixtures/project/runtime.yaml"); + .join("../../products/discovery/fixtures/project/runtime.yaml") + .canonicalize() + .unwrap(); let (_, runtime) = load_runtime(&path).expect("shipped runtime fixture validates"); - assert_eq!(runtime.schema_version, RUNTIME_SCHEMA); - assert_eq!(runtime.index_path, "discovery-index.json"); + assert_eq!(runtime.api_version, RUNTIME_API_VERSION); + assert_eq!(runtime.kind, RUNTIME_KIND); + assert!(runtime.package.root.is_absolute()); } #[test] - fn runtime_paths_cannot_escape_or_follow_symlinks() { - let temporary = tempfile::tempdir().unwrap(); - let root = temporary.path(); - fs::write(root.join("index.json"), b"index").unwrap(); - for value in ["", "/etc/passwd", "../index.json", "nested/../index.json"] { - assert!(safe_existing_file(root, value).is_err(), "{value}"); + fn package_root_must_be_absolute_and_normalized() { + for value in ["package", "../package", "/tmp/../package"] { + let message = refusal(&RUNTIME.replace("/tmp/registry-discovery-package", value)); + assert!(message.contains("package.root"), "{value}: {message}"); } - assert_eq!( - safe_existing_file(root, "index.json").unwrap(), - root.join("index.json") + } + + fn package_files(index: &DiscoveryIndex) -> BTreeMap> { + BTreeMap::from([(INDEX_FILE.to_owned(), canonical_index_bytes(index).unwrap())]) + } + + fn write_index_package(root: &Path) -> registry_platform_config::VerifiedPackage { + write_package( + root, + &package_files(&example_index()), + None, + &package_limits(), + PACKAGE_COMMAND, + ) + .unwrap() + } + + fn runtime_for(package_root: &Path, expected_digest: Option<&str>) -> String { + let pin = expected_digest + .map(|digest| format!("\n expectedDigest: {digest}")) + .unwrap_or_default(); + RUNTIME.replace( + "/tmp/registry-discovery-package", + &format!("{}{pin}", package_root.display()), + ) + } + + #[test] + fn startup_verifies_package_and_refuses_expected_digest_mismatch_with_common_shape() { + let temporary = canonical_tempdir(); + let package_root = temporary.path().join("package"); + let package = write_index_package(&package_root); + let runtime_path = write_runtime( + temporary.path(), + &runtime_for(&package_root, Some(package.digest())), ); + prepare(&runtime_path).expect("matching pinned package starts"); - #[cfg(unix)] - { - use std::os::unix::fs::symlink; - symlink(root.join("index.json"), root.join("linked.json")).unwrap(); - assert!(safe_existing_file(root, "linked.json").is_err()); + let expected = format!("sha256:{}", "0".repeat(64)); + let runtime_path = write_runtime( + temporary.path(), + &runtime_for(&package_root, Some(&expected)), + ); + let message = prepare_error(&runtime_path).to_string(); + assert!( + message.contains(&format!( + "package.expectedDigest is {expected} but the package at package.root is {}", + package.digest() + )), + "{message}" + ); + assert!( + message.contains("deploy the pinned package or update package.expectedDigest"), + "{message}" + ); + } + + #[test] + fn startup_refuses_changed_missing_and_extra_package_files_by_name() { + for (case, expected) in [ + ("changed", "changed: discovery-index.json"), + ("extra", "extra: note.txt"), + ] { + let temporary = canonical_tempdir(); + let package_root = temporary.path().join("package"); + write_index_package(&package_root); + let target = if case == "changed" { + package_root.join(INDEX_FILE) + } else { + package_root.join("note.txt") + }; + fs::write(&target, b"replacement").unwrap(); + let runtime_path = write_runtime(temporary.path(), &runtime_for(&package_root, None)); + let message = prepare_error(&runtime_path).to_string(); + assert!(message.contains(expected), "{case}: {message}"); } + + let temporary = canonical_tempdir(); + let package_root = temporary.path().join("package"); + write_index_package(&package_root); + fs::remove_file(package_root.join(INDEX_FILE)).unwrap(); + let runtime_path = write_runtime(temporary.path(), &runtime_for(&package_root, None)); + let message = prepare_error(&runtime_path).to_string(); + assert!( + message.contains("missing: discovery-index.json"), + "{message}" + ); + } + + #[test] + fn startup_refuses_a_hash_covered_non_index_file_by_name() { + let temporary = canonical_tempdir(); + let package_root = temporary.path().join("package"); + let mut files = package_files(&example_index()); + files.insert("note.txt".to_owned(), b"not part of Discovery".to_vec()); + write_package( + &package_root, + &files, + None, + &package_limits(), + PACKAGE_COMMAND, + ) + .unwrap(); + let runtime_path = write_runtime(temporary.path(), &runtime_for(&package_root, None)); + let message = prepare_error(&runtime_path).to_string(); + assert!(message.contains("extra: note.txt"), "{message}"); } #[test] - fn a_bare_runtime_filename_uses_the_current_directory() { - assert_eq!(effective_parent(Path::new("runtime.yaml")), Path::new(".")); + fn exact_consumed_index_bytes_remain_bound_to_the_verified_package() { + let temporary = canonical_tempdir(); + let package_root = temporary.path().join("package"); + let package = write_index_package(&package_root); + let mut replacement = example_index(); + replacement.built_at = "2026-09-26T00:00:00Z".to_owned(); + fs::write( + package_root.join(INDEX_FILE), + canonical_index_bytes(&replacement).unwrap(), + ) + .unwrap(); + + assert_eq!( + load_verified_index(&package_root, &package), + Err(StartupError::PackageFileChanged { + path: INDEX_FILE.to_owned(), + }) + ); + } + + #[test] + fn index_capture_refuses_growth_beyond_the_read_limit() { + use std::io::Write as _; + + let temporary = canonical_tempdir(); + let path = temporary.path().join(INDEX_FILE); + fs::write(&path, b"12").unwrap(); + let file = fs::File::open(&path).unwrap(); + let opened = file.metadata().unwrap(); + fs::OpenOptions::new() + .append(true) + .open(&path) + .unwrap() + .write_all(b"3") + .unwrap(); + assert_eq!( - effective_parent(Path::new("config/runtime.yaml")), - Path::new("config") + read_opened_regular_file(file, &opened, 2), + Err(StartupError::IndexLoad) ); } + #[test] + fn package_root_symlink_is_refused_before_index_consumption() { + #[cfg(unix)] + { + let temporary = canonical_tempdir(); + let package_root = temporary.path().join("package"); + write_index_package(&package_root); + let linked = temporary.path().join("linked-package"); + std::os::unix::fs::symlink(&package_root, &linked).unwrap(); + let runtime_path = write_runtime(temporary.path(), &runtime_for(&linked, None)); + let message = prepare_error(&runtime_path).to_string(); + assert!(message.contains("symbolic link"), "{message}"); + } + } + #[cfg(unix)] #[tokio::test] async fn injected_ctrl_c_enters_the_shared_graceful_shutdown_path() { diff --git a/crates/registry-discovery/tests/http_journey.rs b/crates/registry-discovery/tests/http_journey.rs index c76bb76674..cdaecfe9b7 100644 --- a/crates/registry-discovery/tests/http_journey.rs +++ b/crates/registry-discovery/tests/http_journey.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 //! Product-owned acceptance entry point through the real Discovery router. +use std::collections::BTreeMap; +use std::fs; use std::sync::Arc; use std::time::Duration; @@ -8,10 +10,12 @@ use axum::body::{to_bytes, Body}; use axum::http::header::CONTENT_TYPE; use axum::http::{Request, StatusCode}; use registry_discovery::{ - catalog_revision, mapping_revision, router, CompiledEvidenceMapping, Directory, DiscoveryIndex, - DiscoveryService, EvidenceTypeAlternative, EvidenceTypeResolveResponse, OriginSummary, - ServiceKind, ServiceRecord, ServiceSearchResponse, INDEX_SCHEMA, + canonical_index_bytes, catalog_revision, mapping_revision, package_limits, prepare, router, + CompiledEvidenceMapping, Directory, DiscoveryIndex, DiscoveryService, EvidenceTypeAlternative, + EvidenceTypeResolveResponse, OriginSummary, ServiceKind, ServiceRecord, ServiceSearchResponse, + INDEX_FILE, INDEX_SCHEMA, PACKAGE_COMMAND, }; +use registry_platform_config::write_package; use tower::ServiceExt as _; fn index() -> DiscoveryIndex { @@ -71,6 +75,66 @@ fn app() -> axum::Router { router(service, 64 * 1024, Duration::from_secs(5)).unwrap() } +#[tokio::test] +async fn verified_package_startup_serves_the_packaged_index_through_the_real_router() { + let temporary = tempfile::tempdir_in(std::env::temp_dir().canonicalize().unwrap()).unwrap(); + let package_root = temporary.path().join("package"); + let files = BTreeMap::from([( + INDEX_FILE.to_owned(), + canonical_index_bytes(&index()).unwrap(), + )]); + let package = write_package( + &package_root, + &files, + Some("http-journey"), + &package_limits(), + PACKAGE_COMMAND, + ) + .unwrap(); + let runtime = temporary.path().join("runtime.yaml"); + fs::write( + &runtime, + format!( + r#"apiVersion: registry.registrystack.org/discovery-runtime/v1alpha1 +kind: DiscoveryRuntimeConfig +listener: {{ bind: 127.0.0.1:0 }} +package: + root: {} + expectedDigest: {} +limits: + maximumRequestBytes: 65536 + maximumResponseBytes: 1048576 + maximumResultRecords: 100 + maximumResultAlternatives: 100 + requestTimeoutSeconds: 10 + shutdownTimeoutSeconds: 10 +logLevel: info +"#, + package_root.display(), + package.digest() + ), + ) + .unwrap(); + + let response = prepare(&runtime) + .unwrap() + .app() + .oneshot( + Request::get( + "/v1/services?serviceKind=evidence&evidenceType=urn%3Aexample%3Aevidence-type", + ) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), 1024 * 1024).await.unwrap(); + let search: ServiceSearchResponse = serde_json::from_slice(&body).unwrap(); + assert_eq!(search.items.len(), 1); + assert_eq!(search.items[0].record_id, "acceptance-record"); +} + #[tokio::test] async fn immutable_index_supports_resolution_and_exact_service_search() { for route in ["/health", "/ready", "/openapi.json"] { diff --git a/crates/registry-discoveryctl/Cargo.toml b/crates/registry-discoveryctl/Cargo.toml index 57a3a28ee2..a138a2a2c9 100644 --- a/crates/registry-discoveryctl/Cargo.toml +++ b/crates/registry-discoveryctl/Cargo.toml @@ -4,7 +4,7 @@ version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true -description = "Offline Registry Discovery catalog authoring and immutable index builds." +description = "Offline Registry Discovery catalog authoring and immutable index packaging." repository.workspace = true publish = false @@ -22,13 +22,13 @@ registry-discovery.workspace = true registry-discovery-profile.workspace = true registry-platform-buildinfo.workspace = true registry-platform-canonical-json.workspace = true +registry-platform-config.workspace = true registry-platform-httputil.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true serde_yaml_ng.workspace = true sha2.workspace = true -tempfile.workspace = true thiserror.workspace = true time.workspace = true tokio.workspace = true diff --git a/crates/registry-discoveryctl/src/build.rs b/crates/registry-discoveryctl/src/build.rs index bf66a220a6..298e7bd77c 100644 --- a/crates/registry-discoveryctl/src/build.rs +++ b/crates/registry-discoveryctl/src/build.rs @@ -1,19 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 -use std::collections::BTreeSet; -use std::io::Write as _; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use std::time::Duration; use registry_discovery::{ canonical_index_bytes, catalog_revision, mapping_revision, validate_index, CompiledEvidenceMapping, DiscoveryIndex, EvidenceTypeAlternative, OriginSummary, ServiceRecord, - INDEX_SCHEMA, MAXIMUM_INDEX_BYTES, + INDEX_FILE, INDEX_SCHEMA, MAXIMUM_INDEX_BYTES, MAXIMUM_PACKAGE_BYTES, MAXIMUM_PACKAGE_DEPTH, + MAXIMUM_PACKAGE_FILES, PACKAGE_COMMAND, }; +use registry_platform_config::{write_package, PackageError, PackageLimits, VerifiedPackage}; use registry_platform_httputil::{read_bounded, validate_response_headers, FetchUrlPolicy}; use reqwest::header::{ACCEPT, CONTENT_ENCODING, CONTENT_TYPE}; use sha2::{Digest as _, Sha256}; -use tempfile::NamedTempFile; use thiserror::Error; use time::{format_description::well_known::Rfc3339, OffsetDateTime}; use url::Url; @@ -22,8 +22,16 @@ use crate::project::{check_project, AuthoredEvidenceMapping, CheckedProject, Pro const DNS_TIMEOUT: Duration = Duration::from_secs(5); const FETCH_TIMEOUT: Duration = Duration::from_secs(20); -#[cfg(unix)] -const OUTPUT_FILE_MODE: u32 = 0o644; + +fn package_limits() -> PackageLimits { + PackageLimits { + max_files: MAXIMUM_PACKAGE_FILES, + max_file_bytes: MAXIMUM_INDEX_BYTES, + max_total_bytes: MAXIMUM_PACKAGE_BYTES, + max_depth: MAXIMUM_PACKAGE_DEPTH, + ..PackageLimits::default() + } +} #[derive(Debug, Error)] pub enum BuildError { @@ -35,19 +43,28 @@ pub enum BuildError { Description, #[error("the Discovery index could not be compiled")] Compile, - #[error("the Discovery index could not be written atomically")] - Write, + #[error("{0}")] + Package(#[from] PackageError), } -pub async fn build_project( +/// The index and shared package identity produced from one collection. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PackagedDiscovery { + pub index: DiscoveryIndex, + pub package_digest: String, +} + +pub async fn package_project( project_root: &Path, output: &Path, allow_loopback: bool, -) -> Result { - build_project_with_timeouts( + revision: Option<&str>, +) -> Result { + package_project_with_timeouts( project_root, output, allow_loopback, + revision, None, DNS_TIMEOUT, FETCH_TIMEOUT, @@ -55,16 +72,18 @@ pub async fn build_project( .await } -pub async fn build_project_at( +pub async fn package_project_at( project_root: &Path, output: &Path, allow_loopback: bool, + revision: Option<&str>, built_at: OffsetDateTime, -) -> Result { - build_project_with_timeouts( +) -> Result { + package_project_with_timeouts( project_root, output, allow_loopback, + revision, Some(built_at), DNS_TIMEOUT, FETCH_TIMEOUT, @@ -72,14 +91,15 @@ pub async fn build_project_at( .await } -async fn build_project_with_timeouts( +async fn package_project_with_timeouts( project_root: &Path, output: &Path, allow_loopback: bool, + revision: Option<&str>, fixed_built_at: Option, dns_timeout: Duration, fetch_timeout: Duration, -) -> Result { +) -> Result { let checked = check_project(project_root, allow_loopback)?; let (mut origins, mut services) = fetch_origins(&checked, allow_loopback, dns_timeout, fetch_timeout).await?; @@ -108,21 +128,27 @@ async fn build_project_with_timeouts( services, mappings, }; - compile_and_activate(&index, output, MAXIMUM_INDEX_BYTES)?; - Ok(index) + let package = write_index_package(&index, output, revision, MAXIMUM_INDEX_BYTES)?; + Ok(PackagedDiscovery { + index, + package_digest: package.digest().to_owned(), + }) } -fn compile_and_activate( +fn write_index_package( index: &DiscoveryIndex, output: &Path, + revision: Option<&str>, maximum_index_bytes: u64, -) -> Result<(), BuildError> { +) -> Result { validate_index(index).map_err(|_| BuildError::Compile)?; let bytes = canonical_index_bytes(index).map_err(|_| BuildError::Compile)?; if u64::try_from(bytes.len()).map_or(true, |length| length > maximum_index_bytes) { return Err(BuildError::Compile); } - atomic_replace(output, &bytes) + let files = BTreeMap::from([(INDEX_FILE.to_owned(), bytes)]); + write_package(output, &files, revision, &package_limits(), PACKAGE_COMMAND) + .map_err(BuildError::from) } async fn fetch_origins( @@ -264,54 +290,6 @@ fn sha256_digest(bytes: &[u8]) -> String { format!("sha256:{}", hex::encode(Sha256::digest(bytes))) } -fn atomic_replace(output: &Path, bytes: &[u8]) -> Result<(), BuildError> { - let parent = effective_parent(output); - if !parent.is_dir() { - return Err(BuildError::Write); - } - let mut temporary = NamedTempFile::new_in(parent).map_err(|_| BuildError::Write)?; - temporary.write_all(bytes).map_err(|_| BuildError::Write)?; - set_output_permissions(temporary.as_file())?; - temporary - .as_file() - .sync_all() - .map_err(|_| BuildError::Write)?; - temporary.persist(output).map_err(|_| BuildError::Write)?; - sync_parent_directory(parent)?; - Ok(()) -} - -fn effective_parent(path: &Path) -> &Path { - path.parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")) -} - -#[cfg(unix)] -fn set_output_permissions(file: &std::fs::File) -> Result<(), BuildError> { - use std::os::unix::fs::PermissionsExt as _; - - file.set_permissions(std::fs::Permissions::from_mode(OUTPUT_FILE_MODE)) - .map_err(|_| BuildError::Write) -} - -#[cfg(not(unix))] -fn set_output_permissions(_file: &std::fs::File) -> Result<(), BuildError> { - Ok(()) -} - -#[cfg(unix)] -fn sync_parent_directory(parent: &Path) -> Result<(), BuildError> { - std::fs::File::open(parent) - .and_then(|directory| directory.sync_all()) - .map_err(|_| BuildError::Write) -} - -#[cfg(not(unix))] -fn sync_parent_directory(_parent: &Path) -> Result<(), BuildError> { - Ok(()) -} - #[cfg(test)] mod tests { use std::fs; @@ -332,15 +310,6 @@ mod tests { use super::*; - #[test] - fn a_bare_output_filename_uses_the_current_directory() { - assert_eq!(effective_parent(Path::new("index.json")), Path::new(".")); - assert_eq!( - effective_parent(Path::new("output/index.json")), - Path::new("output") - ); - } - #[test] fn compiled_index_byte_overflow_preserves_the_previous_output() { let services = Vec::new(); @@ -359,7 +328,7 @@ mod tests { fs::write(&output, b"previous-index-canary").expect("previous index"); assert!(matches!( - compile_and_activate(&index, &output, 1), + write_index_package(&index, &output, None, 1), Err(BuildError::Compile) )); assert_eq!( @@ -368,6 +337,49 @@ mod tests { ); } + #[test] + fn packaging_the_same_compiled_index_twice_is_repeatable() { + let services = Vec::new(); + let mappings = Vec::new(); + let index = DiscoveryIndex { + schema_version: INDEX_SCHEMA.into(), + catalog_revision: catalog_revision(&services).expect("catalog revision"), + mapping_revision: mapping_revision(&mappings).expect("mapping revision"), + built_at: "2026-08-14T00:00:00Z".into(), + origins: Vec::new(), + services, + mappings, + }; + let directory = TempDir::new().expect("temporary directory"); + let first = directory.path().join("first-package"); + let second = directory.path().join("second-package"); + + let first_package = write_index_package( + &index, + &first, + Some("example-revision"), + MAXIMUM_INDEX_BYTES, + ) + .expect("first package"); + let second_package = write_index_package( + &index, + &second, + Some("example-revision"), + MAXIMUM_INDEX_BYTES, + ) + .expect("second package"); + + assert_eq!(first_package.digest(), second_package.digest()); + assert_eq!( + fs::read(first.join("SHA256SUMS")).unwrap(), + fs::read(second.join("SHA256SUMS")).unwrap() + ); + assert_eq!( + fs::read(first.join(INDEX_FILE)).unwrap(), + fs::read(second.join(INDEX_FILE)).unwrap() + ); + } + #[tokio::test] async fn origin_fetch_timeout_leaves_the_previous_output_untouched() { let service = ServiceDescription::new( @@ -428,16 +440,17 @@ mod tests { ) .expect("origins"); fs::create_dir(project.path().join("mappings")).expect("mappings"); - let output = project.path().join("index.json"); + let output = project.path().join("package"); fs::write(&output, b"previous-index-canary").expect("previous index"); let project_path = project.path().to_path_buf(); - let build_output = output.clone(); - let mut build = tokio::spawn(async move { - build_project_with_timeouts( + let package_output = output.clone(); + let mut package = tokio::spawn(async move { + package_project_with_timeouts( &project_path, - &build_output, + &package_output, true, + None, Some(OffsetDateTime::UNIX_EPOCH), DNS_TIMEOUT, Duration::from_millis(500), @@ -446,13 +459,13 @@ mod tests { }); tokio::select! { () = entered.notified() => {} - result = &mut build => panic!("fetch ended before the provider received it: {result:?}"), + result = &mut package => panic!("fetch ended before the provider received it: {result:?}"), () = tokio::time::sleep(Duration::from_secs(5)) => panic!("provider was not reached"), } - let result = tokio::time::timeout(Duration::from_secs(5), build) + let result = tokio::time::timeout(Duration::from_secs(5), package) .await .expect("fetch timeout elapsed") - .expect("build task"); + .expect("package task"); assert!(matches!(result, Err(BuildError::Fetch))); assert_eq!(counter.load(Ordering::SeqCst), 1); diff --git a/crates/registry-discoveryctl/src/lib.rs b/crates/registry-discoveryctl/src/lib.rs index 7603705d9e..509c197049 100644 --- a/crates/registry-discoveryctl/src/lib.rs +++ b/crates/registry-discoveryctl/src/lib.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 -//! Finite Registry Discovery authoring and immutable index builds. +//! Finite Registry Discovery authoring and immutable index packages. +use std::ffi::OsString; use std::path::PathBuf; use std::process::ExitCode; @@ -9,7 +10,7 @@ use clap::{Parser, Subcommand}; mod build; mod project; -pub use build::{build_project, build_project_at, BuildError}; +pub use build::{package_project, package_project_at, BuildError, PackagedDiscovery}; pub use project::{ check_project, ApprovedOrigin, AuthoredEvidenceMapping, AuthoredEvidenceTypeAlternative, CheckedProject, OriginsFile, ProjectError, MAPPING_SCHEMA, MAX_MAPPING_FILE_BYTES, @@ -19,7 +20,7 @@ pub use project::{ #[derive(Debug, Parser)] #[command( name = "discoveryctl", - about = "Check and build one immutable Registry Discovery index", + about = "Check and package one immutable Registry Discovery index", version = registry_platform_buildinfo::DISPLAY_VERSION )] struct Arguments { @@ -36,21 +37,43 @@ enum Command { #[arg(long)] allow_loopback: bool, }, - /// Fetch every enabled approved origin once and atomically build one index. - Build { + /// Fetch every enabled approved origin once and write one immutable package. + Package { #[arg(long)] project: PathBuf, #[arg(long)] output: PathBuf, #[arg(long)] allow_loopback: bool, + /// Optional operator revision recorded in the package envelope. + #[arg(long)] + revision: Option, + }, + /// Removed. Use `discoveryctl package`. + #[command(hide = true, trailing_var_arg = true)] + Build { + #[arg(allow_hyphen_values = true)] + _legacy_arguments: Vec, }, } +const RETIRED_BUILD_ERROR: &str = + "discoveryctl build was removed; use discoveryctl package with a new output directory"; + #[must_use] pub fn main_entry() -> ExitCode { let arguments = Arguments::parse(); - let result = match arguments.command { + match run(arguments) { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("{error}"); + ExitCode::from(1) + } + } +} + +fn run(arguments: Arguments) -> Result<(), String> { + match arguments.command { Command::Check { project, allow_loopback, @@ -63,31 +86,63 @@ pub fn main_entry() -> ExitCode { ); }) .map_err(|error| error.to_string()), - Command::Build { + Command::Package { project, output, allow_loopback, + revision, } => tokio::runtime::Builder::new_current_thread() .enable_all() .build() - .map_err(|_| "the Discovery build runtime could not start".to_owned()) + .map_err(|_| "the Discovery package runtime could not start".to_owned()) .and_then(|runtime| { runtime - .block_on(build_project(&project, &output, allow_loopback)) - .map(|index| { + .block_on(package_project( + &project, + &output, + allow_loopback, + revision.as_deref(), + )) + .map(|package| { println!( - "built catalogRevision={} mappingRevision={}", - index.catalog_revision, index.mapping_revision + "packaged packageDigest={} catalogRevision={} mappingRevision={}", + package.package_digest, + package.index.catalog_revision, + package.index.mapping_revision ); }) .map_err(|error| error.to_string()) }), - }; - match result { - Ok(()) => ExitCode::SUCCESS, - Err(error) => { - eprintln!("{error}"); - ExitCode::from(1) - } + Command::Build { .. } => Err(RETIRED_BUILD_ERROR.to_owned()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retired_build_refusal_names_discoveryctl_package() { + let arguments = Arguments::try_parse_from([ + "discoveryctl", + "build", + "--project", + "project", + "--output", + "discovery-index.json", + ]) + .expect("the retired command reaches its migration refusal"); + let message = run(arguments).unwrap_err(); + assert!(message.contains("discoveryctl package"), "{message}"); + + assert!(Arguments::try_parse_from([ + "discoveryctl", + "package", + "--project", + "project", + "--output", + "package", + ]) + .is_ok()); } } diff --git a/crates/registry-discoveryctl/tests/build.rs b/crates/registry-discoveryctl/tests/build.rs index aafca818b0..abd04a269d 100644 --- a/crates/registry-discoveryctl/tests/build.rs +++ b/crates/registry-discoveryctl/tests/build.rs @@ -9,15 +9,36 @@ use axum::http::header::CONTENT_TYPE; use axum::http::{Response, StatusCode}; use axum::routing::get; use axum::Router; -use registry_discovery::parse_index; +use registry_discovery::{parse_index, INDEX_FILE}; use registry_discovery_profile::{ render_description, DiscoveryDescription, ServiceDescription, ServiceKind, ServiceRoles, MEDIA_TYPE, }; -use registry_discoveryctl::{build_project, build_project_at, BuildError}; +use registry_discoveryctl::{package_project, package_project_at, BuildError}; use tempfile::TempDir; use time::{macros::datetime, OffsetDateTime}; +async fn build_project( + project: &std::path::Path, + output: &std::path::Path, + allow_loopback: bool, +) -> Result { + package_project(project, output, allow_loopback, None) + .await + .map(|package| package.index) +} + +async fn build_project_at( + project: &std::path::Path, + output: &std::path::Path, + allow_loopback: bool, + built_at: OffsetDateTime, +) -> Result { + package_project_at(project, output, allow_loopback, None, built_at) + .await + .map(|package| package.index) +} + fn description() -> Vec { let service = ServiceDescription::new( "urn:example:service:evidence".into(), @@ -213,11 +234,11 @@ fn authoring_project(catalog_url: &str) -> TempDir { } #[tokio::test] -async fn build_fetches_each_origin_once_and_preserves_semantic_revisions() { +async fn package_fetches_each_origin_once_and_preserves_semantic_revisions() { let counter = Arc::new(AtomicUsize::new(0)); let (catalog_url, task) = provider(description(), StatusCode::OK, Arc::clone(&counter)).await; let project = authoring_project(&catalog_url); - let first_output = project.path().join("first.json"); + let first_output = project.path().join("first-package"); let first = build_project_at( project.path(), &first_output, @@ -236,11 +257,11 @@ async fn build_fetches_each_origin_once_and_preserves_semantic_revisions() { first.services[0].origin_fetched_at ); assert_eq!( - parse_index(&fs::read(&first_output).expect("index")).expect("valid index"), + parse_index(&fs::read(first_output.join(INDEX_FILE)).expect("index")).expect("valid index"), first ); - let second_output = project.path().join("second.json"); + let second_output = project.path().join("second-package"); let second = build_project_at( project.path(), &second_output, @@ -257,11 +278,11 @@ async fn build_fetches_each_origin_once_and_preserves_semantic_revisions() { } #[tokio::test] -async fn production_build_time_is_captured_after_origin_collection() { +async fn production_package_time_is_captured_after_origin_collection() { let counter = Arc::new(AtomicUsize::new(0)); let (catalog_url, task) = provider(description(), StatusCode::OK, Arc::clone(&counter)).await; let project = authoring_project(&catalog_url); - let output = project.path().join("index.json"); + let output = project.path().join("package"); let index = build_project(project.path(), &output, true) .await @@ -284,13 +305,13 @@ async fn production_build_time_is_captured_after_origin_collection() { #[cfg(unix)] #[tokio::test] -async fn successful_builds_publish_a_stable_runtime_readable_file_mode() { +async fn successful_packages_publish_a_stable_runtime_readable_file_mode() { use std::os::unix::fs::PermissionsExt as _; let counter = Arc::new(AtomicUsize::new(0)); let (catalog_url, task) = provider(description(), StatusCode::OK, Arc::clone(&counter)).await; let project = authoring_project(&catalog_url); - let output = project.path().join("index.json"); + let output = project.path().join("first-package"); build_project_at( project.path(), @@ -301,7 +322,7 @@ async fn successful_builds_publish_a_stable_runtime_readable_file_mode() { .await .expect("initial build"); assert_eq!( - fs::metadata(&output) + fs::metadata(output.join(INDEX_FILE)) .expect("metadata") .permissions() .mode() @@ -309,8 +330,7 @@ async fn successful_builds_publish_a_stable_runtime_readable_file_mode() { 0o644 ); - fs::set_permissions(&output, fs::Permissions::from_mode(0o600)) - .expect("restrict previous output"); + let output = project.path().join("second-package"); build_project_at( project.path(), &output, @@ -320,7 +340,7 @@ async fn successful_builds_publish_a_stable_runtime_readable_file_mode() { .await .expect("replacement build"); assert_eq!( - fs::metadata(&output) + fs::metadata(output.join(INDEX_FILE)) .expect("metadata") .permissions() .mode() @@ -596,7 +616,7 @@ async fn compiled_service_bound_leaves_the_previous_output_untouched() { } #[tokio::test] -async fn write_failure_preserves_previous_output_and_leaves_no_visible_temporary_file() { +async fn package_refuses_an_existing_output_without_changing_it() { let counter = Arc::new(AtomicUsize::new(0)); let (catalog_url, task) = provider(description(), StatusCode::OK, Arc::clone(&counter)).await; let project = authoring_project(&catalog_url); @@ -609,7 +629,9 @@ async fn write_failure_preserves_previous_output_and_leaves_no_visible_temporary let result = build_project_at(project.path(), &output, true, OffsetDateTime::UNIX_EPOCH).await; - assert!(matches!(result, Err(BuildError::Write))); + let message = result.unwrap_err().to_string(); + assert!(message.contains("already exists"), "{message}"); + assert!(message.contains("discoveryctl package"), "{message}"); assert_eq!(counter.load(Ordering::SeqCst), 1); assert_eq!( fs::read(&canary).expect("previous target canary"), diff --git a/crates/registry-discoveryctl/tests/schema_contract.rs b/crates/registry-discoveryctl/tests/schema_contract.rs index 87788fc42f..a37be92c41 100644 --- a/crates/registry-discoveryctl/tests/schema_contract.rs +++ b/crates/registry-discoveryctl/tests/schema_contract.rs @@ -5,12 +5,13 @@ use std::fs; use std::path::{Path, PathBuf}; use jsonschema::{Draft, JSONSchema}; -use registry_discovery::{parse_index, prepare}; +use registry_discovery::{package_limits, parse_index, prepare, INDEX_FILE, PACKAGE_COMMAND}; use registry_discovery_profile::{ parse_description, render_description, DiscoveryDescription, ServiceDescription, }; use registry_discoveryctl::{check_project, MAX_MAPPING_FILE_BYTES}; use registry_platform_canonical_json::canonicalize_json; +use registry_platform_config::write_package; use serde_json::Value; const PRODUCT_ROOT: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../products/discovery"); @@ -177,19 +178,42 @@ fn accepted_by_rust(contract: &str, document: &Value) -> bool { check_project(project.path(), false).is_ok() } "runtime" => { - let temporary = tempfile::tempdir().expect("temporary runtime"); + let temporary = tempfile::tempdir_in( + std::env::temp_dir() + .canonicalize() + .expect("temporary root resolves"), + ) + .expect("temporary runtime"); + let mut runtime = document.clone(); + let package_root = temporary.path().join("package"); + if runtime.pointer("/package/root") + == positive_document("runtime").pointer("/package/root") + { + let index = positive_document("index"); + let files = BTreeMap::from([( + INDEX_FILE.to_owned(), + canonicalize_json(&index).expect("positive index canonicalizes"), + )]); + write_package( + &package_root, + &files, + None, + &package_limits(), + PACKAGE_COMMAND, + ) + .expect("positive package writes"); + set_pointer( + &mut runtime, + "/package/root", + Value::String(package_root.display().to_string()), + ); + } let path = temporary.path().join("runtime.yaml"); fs::write( &path, - serde_yaml_ng::to_string(document).expect("runtime serializes"), + serde_yaml_ng::to_string(&runtime).expect("runtime serializes"), ) .expect("runtime write"); - let index = positive_document("index"); - fs::write( - temporary.path().join("discovery-index.json"), - canonicalize_json(&index).expect("positive index canonicalizes"), - ) - .expect("index write"); prepare(&path).is_ok() } "index" => canonicalize_json(document).is_ok_and(|bytes| parse_index(&bytes).is_ok()), @@ -358,9 +382,9 @@ fn runtime_response_and_listener_boundaries_match_the_closed_rust_parser() { "the public schema must carry the stable Rust response-size minimum" ); assert_eq!( - schema["properties"]["listener"]["properties"]["address"]["maxLength"].as_u64(), - u64::try_from(registry_discovery::MAXIMUM_LISTENER_ADDRESS_CHARACTERS).ok(), - "the public schema must carry the Rust listener-address bound" + schema["properties"]["listener"]["properties"]["bind"]["maxLength"].as_u64(), + u64::try_from(registry_discovery::MAXIMUM_LISTENER_BIND_CHARACTERS).ok(), + "the public schema must carry the Rust listener-bind bound" ); assert!( registry_discovery::openapi::OPENAPI_BYTES.len() <= minimum_response_bytes, @@ -414,7 +438,7 @@ fn runtime_response_and_listener_boundaries_match_the_closed_rust_parser() { let mut document = positive_document("runtime"); set_pointer( &mut document, - "/listener/address", + "/listener/bind", Value::String(address.to_owned()), ); assert!( @@ -439,7 +463,7 @@ fn runtime_response_and_listener_boundaries_match_the_closed_rust_parser() { let mut document = positive_document("runtime"); set_pointer( &mut document, - "/listener/address", + "/listener/bind", Value::String(address.to_owned()), ); assert!( @@ -454,12 +478,12 @@ fn runtime_response_and_listener_boundaries_match_the_closed_rust_parser() { let overlong_address = format!( "127.0.0.1:{}80", - "0".repeat(registry_discovery::MAXIMUM_LISTENER_ADDRESS_CHARACTERS) + "0".repeat(registry_discovery::MAXIMUM_LISTENER_BIND_CHARACTERS) ); let mut document = positive_document("runtime"); set_pointer( &mut document, - "/listener/address", + "/listener/bind", Value::String(overlong_address), ); assert!(!validator.is_valid(&document)); diff --git a/crates/registry-evidence-authoring/Cargo.toml b/crates/registry-evidence-authoring/Cargo.toml index b7177873f8..e065f636d7 100644 --- a/crates/registry-evidence-authoring/Cargo.toml +++ b/crates/registry-evidence-authoring/Cargo.toml @@ -29,7 +29,9 @@ testing = [] [dependencies] anyhow.workspace = true -rhai.workspace = true +# `internals` exposes the parsed program, so a check can read what a +# derivation reads without running it. It adds no engine capability. +rhai = { workspace = true, features = ["internals"] } schemars = { workspace = true, optional = true } serde.workspace = true serde_json.workspace = true diff --git a/crates/registry-evidence-authoring/src/derivation.rs b/crates/registry-evidence-authoring/src/derivation.rs index 6d5ee270ac..25a30e7129 100644 --- a/crates/registry-evidence-authoring/src/derivation.rs +++ b/crates/registry-evidence-authoring/src/derivation.rs @@ -6,7 +6,7 @@ //! and no way to reach a standard stream, so an authored `import` names a path //! that nothing here will open and an authored `print` has nowhere to land. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use rhai::module_resolvers::DummyModuleResolver; @@ -90,3 +90,153 @@ pub fn validate_authored_answer(source: &str) -> Vec { } Vec::new() } + +/// Name every fact the `answer` function reads that `declared` does not hold. +/// +/// A fact read is an index read with a string literal key (`facts["status"]`) +/// or a property read (`facts.status`, `facts?.status`) on the first parameter +/// of `answer`, however that parameter is named. Only the first key of a chain +/// is a fact name: `facts.address.region` reads the fact `address`. +/// +/// The operands of one `??` fallback are read together: when any of them reads +/// a declared fact, none of its reads is named, so `facts.new ?? facts.old` +/// stays valid while a source moves from one name to the other. +/// +/// The check is deliberately partial, and never refuses a program it cannot +/// read. A computed key (`facts[key]`) is not a literal read, so it is the +/// way to read a fact the check should not see, and like a read inside another +/// function the facts are passed to, it is left to the fixtures. So is an +/// `answer` that rebinds or writes its first parameter anywhere, through +/// `let`, `const`, an assignment to it or to one of its keys, a `for` loop +/// variable, or a `catch` variable. A program the other derivation checks +/// refuse reports nothing here, because those checks already name what is +/// wrong with it. +/// +/// Each undeclared fact is named once, in name order. +#[must_use] +pub fn validate_answer_fact_reads(source: &str, declared: &BTreeSet) -> Vec { + let Ok(ast) = parser().compile(source) else { + return Vec::new(); + }; + let mut answers = ast + .iter_functions() + .filter(|function| function.name == "answer" && function.params.len() == 3); + let (Some(answer), None) = (answers.next(), answers.next()) else { + return Vec::new(); + }; + let facts = answer.params[0]; + // `retain_functions` rather than `clone_functions_only_filtered`: in the + // pinned rhai the latter copies every function when its target module is + // empty, which a fresh clone always is. + let mut body = ast.clone_functions_only(); + body.retain_functions(|_, _, name, params| name == "answer" && params == 3); + + let mut rebound = false; + let mut read = BTreeSet::new(); + // The reads under each outermost `??`, keyed by that node's address, which + // is stable for the walk. + let mut fallbacks: BTreeMap> = BTreeMap::new(); + body.walk(&mut |path| { + match path.last() { + Some(rhai::ASTNode::Stmt(statement)) if rebinds(statement, facts) => { + rebound = true; + return false; + } + Some(rhai::ASTNode::Expr( + rhai::Expr::Index(chain, ..) | rhai::Expr::Dot(chain, ..), + )) if names_variable(&chain.lhs, facts) => { + if let Some(name) = first_key(path.last(), &chain.rhs) { + let fallback = path.iter().find_map(|node| match node { + rhai::ASTNode::Expr(expression @ rhai::Expr::Coalesce(..)) => { + Some(std::ptr::from_ref(*expression) as usize) + } + _ => None, + }); + match fallback { + Some(fallback) => { + fallbacks.entry(fallback).or_default().insert(name); + } + None => { + read.insert(name); + } + } + } + } + _ => {} + } + true + }); + if rebound { + return Vec::new(); + } + for names in fallbacks.into_values() { + if !names.iter().any(|name| declared.contains(name)) { + read.extend(names); + } + } + read.into_iter() + .filter(|name| !declared.contains(name)) + .map(|name| { + Finding::new( + FieldPath::root(), + "derivation-fact-undeclared", + format!( + "authored derivation reads fact {name:?}, which the question's source does not declare" + ), + ) + }) + .collect() +} + +/// Whether a statement binds or writes `name`: a `let` or `const`, an +/// assignment to it or to one of its keys, a `for` loop variable, or a `catch` +/// variable. +fn rebinds(statement: &rhai::Stmt, name: &str) -> bool { + match statement { + rhai::Stmt::Var(binding, ..) => binding.0.name == name, + rhai::Stmt::Assignment(assignment) => { + let mut target = &assignment.1.lhs; + while let rhai::Expr::Index(chain, ..) | rhai::Expr::Dot(chain, ..) = target { + target = &chain.lhs; + } + names_variable(target, name) + } + rhai::Stmt::For(binding, ..) => { + binding.0.name == name + || binding + .1 + .as_ref() + .is_some_and(|counter| counter.name == name) + } + rhai::Stmt::TryCatch(flow, ..) => names_variable(&flow.expr, name), + _ => false, + } +} + +/// Whether an expression is the plain, unqualified variable `name`. +fn names_variable(expression: &rhai::Expr, name: &str) -> bool { + matches!( + expression, + rhai::Expr::Variable(variable, ..) if variable.1 == name && variable.2.is_empty() + ) +} + +/// The key a chain reads first, when it is written literally: a string index +/// under `[` or a property under `.`. A method call, a computed index, and any +/// other link name no key this check can read. +fn first_key(link: Option<&rhai::ASTNode>, rhs: &rhai::Expr) -> Option { + let mut key = rhs; + while let rhai::Expr::Index(chain, ..) | rhai::Expr::Dot(chain, ..) = key { + key = &chain.lhs; + } + match (link, key) { + ( + Some(rhai::ASTNode::Expr(rhai::Expr::Index(..))), + rhai::Expr::StringConstant(name, ..), + ) => Some(name.to_string()), + (Some(rhai::ASTNode::Expr(rhai::Expr::Dot(..))), rhai::Expr::Property(property, ..)) => { + Some(property.2.to_string()) + } + _ => None, + } +} diff --git a/crates/registry-evidence-authoring/src/lib.rs b/crates/registry-evidence-authoring/src/lib.rs index c028ea2e7c..af3822af05 100644 --- a/crates/registry-evidence-authoring/src/lib.rs +++ b/crates/registry-evidence-authoring/src/lib.rs @@ -32,7 +32,7 @@ pub mod schema; pub mod testing; pub mod validate; -pub use derivation::validate_authored_answer; +pub use derivation::{validate_answer_fact_reads, validate_authored_answer}; pub use finding::{FieldPath, FieldStep, Finding}; pub use marker::{ default_project_marker_document, parse_project_marker, ProjectKind, ProjectMarker, diff --git a/crates/registry-evidence-authoring/tests/derivation_fact_reads.rs b/crates/registry-evidence-authoring/tests/derivation_fact_reads.rs new file mode 100644 index 0000000000..7975cecafd --- /dev/null +++ b/crates/registry-evidence-authoring/tests/derivation_fact_reads.rs @@ -0,0 +1,161 @@ +//! A derivation may read only the facts its question's source declares. +//! +//! The check is static: it reads the `answer` function's own index and +//! property reads on its first parameter, and names each literal key the +//! declared set does not hold. A key it cannot read statically, and a read +//! inside another function, are left to the fixtures. + +use std::collections::BTreeSet; + +use registry_evidence_authoring::{validate_answer_fact_reads, Finding}; + +fn declared(names: &[&str]) -> BTreeSet { + names.iter().map(|name| (*name).to_owned()).collect() +} + +fn messages(findings: &[Finding]) -> Vec { + findings + .iter() + .map(|finding| format!("{}: {}", finding.code, finding.message)) + .collect() +} + +#[test] +fn an_index_read_of_an_undeclared_fact_is_named() { + let findings = validate_answer_fact_reads( + "fn answer(facts, selectors, context) {\n\ + let status = required(facts[\"status\"], \"status_missing\");\n\ + #{ active: status == \"active\" }\n\ + }", + &declared(&["code", "lifecycle_status"]), + ); + assert_eq!( + messages(&findings), + [ + "derivation-fact-undeclared: authored derivation reads fact \"status\", \ + which the question's source does not declare" + ] + ); +} + +#[test] +fn a_property_read_and_a_chained_read_are_named() { + let findings = validate_answer_fact_reads( + "fn answer(record, selectors, context) {\n\ + #{ a: record.status == \"active\", b: record[\"address\"].region, c: record?.missing }\n\ + }", + &declared(&["status"]), + ); + assert_eq!( + messages(&findings), + [ + "derivation-fact-undeclared: authored derivation reads fact \"address\", \ + which the question's source does not declare", + "derivation-fact-undeclared: authored derivation reads fact \"missing\", \ + which the question's source does not declare", + ] + ); +} + +#[test] +fn declared_reads_and_reads_the_check_cannot_resolve_report_nothing() { + let source = "fn helper(facts) { facts[\"elsewhere\"] }\n\ + fn answer(facts, selectors, context) {\n\ + let key = \"computed\";\n\ + let chosen = facts[key];\n\ + let marker = facts[\"marker\"];\n\ + let nested = facts.marker.inner;\n\ + let size = facts.len();\n\ + let other = selectors[\"not_a_fact\"];\n\ + #{ marked: marker == true }\n\ + }"; + assert_eq!( + validate_answer_fact_reads(source, &declared(&["marker"])), + Vec::new() + ); +} + +#[test] +fn a_repeated_undeclared_read_is_named_once() { + let findings = validate_answer_fact_reads( + "fn answer(facts, selectors, context) { #{ a: facts.gone, b: facts[\"gone\"] } }", + &declared(&[]), + ); + assert_eq!(findings.len(), 1, "findings were: {findings:?}"); +} + +#[test] +fn a_shadowed_facts_parameter_is_not_read_as_the_source_facts() { + let source = "fn answer(facts, selectors, context) {\n\ + let facts = #{ local: 1 };\n\ + #{ marked: facts.local == 1 }\n\ + }"; + assert_eq!( + validate_answer_fact_reads(source, &declared(&[])), + Vec::new() + ); +} + +#[test] +fn an_answer_that_assigns_or_rebinds_its_first_parameter_reports_nothing() { + for body in [ + "facts = #{ local: 1 };\n#{ marked: facts.local == 1 }", + "facts.local = 1;\n#{ marked: facts.local == 1 }", + "facts[\"local\"] += 1;\n#{ marked: facts.local == 2 }", + "let total = 0;\nfor facts in [#{ local: 1 }] { total += facts.local; }\n#{ total: total }", + "let total = 0;\nfor (row, facts) in [1] { total += row; }\n#{ total: total + facts.local }", + "try { throw 1; } catch (facts) { return #{ caught: facts.local }; }\n#{}", + ] { + let source = format!("fn answer(facts, selectors, context) {{\n{body}\n}}"); + assert_eq!( + validate_answer_fact_reads(&source, &declared(&[])), + Vec::new(), + "{source}" + ); + } +} + +#[test] +fn a_fallback_between_names_is_satisfied_by_any_declared_name() { + let source = "fn answer(facts, selectors, context) {\n\ + #{ status: facts.status_code ?? facts[\"status\"] ?? \"unknown\" }\n\ + }"; + assert_eq!( + validate_answer_fact_reads(source, &declared(&["status"])), + Vec::new() + ); + assert_eq!( + validate_answer_fact_reads(source, &declared(&["status_code"])), + Vec::new() + ); +} + +#[test] +fn a_fallback_with_no_declared_name_names_each_of_them() { + let findings = validate_answer_fact_reads( + "fn answer(facts, selectors, context) { #{ s: facts.old ?? facts.older ?? \"none\" } }", + &declared(&["status"]), + ); + assert_eq!( + messages(&findings), + [ + "derivation-fact-undeclared: authored derivation reads fact \"old\", \ + which the question's source does not declare", + "derivation-fact-undeclared: authored derivation reads fact \"older\", \ + which the question's source does not declare", + ] + ); +} + +#[test] +fn a_program_the_other_checks_refuse_reports_nothing_here() { + for source in [ + "fn answer(facts, selectors, context) {", + "fn helper() { 1 }", + ] { + assert_eq!( + validate_answer_fact_reads(source, &declared(&[])), + Vec::new() + ); + } +} diff --git a/crates/registry-evidence-client/Cargo.toml b/crates/registry-evidence-client/Cargo.toml index 3c8a92c1be..38ccd6670b 100644 --- a/crates/registry-evidence-client/Cargo.toml +++ b/crates/registry-evidence-client/Cargo.toml @@ -35,6 +35,7 @@ ed25519-dalek.workspace = true p256.workspace = true registry-evidence.workspace = true registry-thunderid-tooling.workspace = true +registry-platform-config.workspace = true registry-platform-sdjwt.workspace = true tempfile.workspace = true wiremock.workspace = true diff --git a/crates/registry-evidence-client/tests/against_a_real_deployment.rs b/crates/registry-evidence-client/tests/against_a_real_deployment.rs index 0b335d1e3a..13594cb5ea 100644 --- a/crates/registry-evidence-client/tests/against_a_real_deployment.rs +++ b/crates/registry-evidence-client/tests/against_a_real_deployment.rs @@ -1593,6 +1593,7 @@ async fn start_trusting_with_request_burst_jwks_and_task_authority( .expect("write the runtime configuration"); fs::set_permissions(&runtime_path, fs::Permissions::from_mode(0o444)) .expect("the runtime configuration is immutable"); + refresh_package_envelope(&bundle_root); seal(&bundle_root); let runtime = Arc::new( @@ -2986,14 +2987,14 @@ fn rewrite_for_local_profile( ); replace_exact( &mut document, - "audiences: [evidence-fixture]", - &format!("audiences: [{TOKEN_AUDIENCE}]"), + "audience: evidence-fixture", + &format!("audience: {TOKEN_AUDIENCE}"), 1, ); replace_exact( &mut document, - "jwksUri: https://identity.invalid/.well-known/jwks.json", - &format!("jwksUri: {issuer_jwks_uri}"), + "uri: https://identity.invalid/.well-known/jwks.json", + &format!("uri: {issuer_jwks_uri}"), 1, ); replace_exact( @@ -3094,11 +3095,12 @@ fn runtime_document( audit_path: &Path, ) -> String { format!( - r#"version: 1 -bundleDirectory: {bundle} + r#"apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: {bundle} listener: - bindHost: 127.0.0.1 - port: {port} + bind: 127.0.0.1:{port} tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 @@ -3226,6 +3228,27 @@ fn copy_tree(source: &Path, target: &Path) { } /// Make the staged bundle immutable, as the runtime requires. +/// Republish the staged package after its intended authored changes. +fn refresh_package_envelope(root: &Path) { + let sum_file = root.join(registry_platform_config::SUM_FILE); + if sum_file.exists() { + fs::remove_file(&sum_file).expect("remove the stale package sum file"); + } + registry_platform_config::write_sum_file( + root, + None, + ®istry_platform_config::PackageLimits { + max_files: registry_evidence::bundle::MAX_BUNDLE_FILES, + max_file_bytes: registry_evidence::bundle::MAX_ARTIFACT_BYTES, + max_total_bytes: registry_evidence::bundle::MAX_BUNDLE_BYTES, + max_depth: 3, + max_path_bytes: 128, + }, + "evidencectl package", + ) + .expect("publish the staged test package"); +} + fn seal(root: &Path) { for entry in fs::read_dir(root).expect("the staged bundle is readable") { let entry = entry.expect("the staged entry is readable"); diff --git a/crates/registry-evidence/README.md b/crates/registry-evidence/README.md index 3caf20d37e..dba27c713c 100644 --- a/crates/registry-evidence/README.md +++ b/crates/registry-evidence/README.md @@ -11,9 +11,9 @@ and serves those items at its own paths. The `evidence` binary takes a runtime file and one subcommand: ```text -evidence --runtime check -evidence --runtime evaluate --fixture -evidence --runtime serve +evidence check --runtime-config +evidence evaluate --runtime-config --fixture +evidence serve --runtime-config evidence verify --jws --jwks --policy [--at ] ``` diff --git a/crates/registry-evidence/src/audit.rs b/crates/registry-evidence/src/audit.rs index 45ffecc874..fb16b2e038 100644 --- a/crates/registry-evidence/src/audit.rs +++ b/crates/registry-evidence/src/audit.rs @@ -908,6 +908,24 @@ impl EvidenceAuditLog { }) } + /// Prove everything [`Self::initialize`] proves except the writer lock, + /// for a candidate staged beside the running writer that holds it. + /// + /// The hash key and key version are checked as startup checks them, and + /// the destination is checked with [`AuditDestination::check_writable`]: + /// the directory and file modes, write access, and a complete final entry + /// in the active file. Nothing is appended and the running writer's lock + /// is never taken, so a second writer is not detected here. + pub fn preflight( + destination: &AuditDestination, + master_secret: Vec, + key_version: u32, + ) -> Result<(), EvidenceAuditError> { + identifier_key_hasher(master_secret, key_version)?; + destination.check_writable()?; + Ok(()) + } + pub fn pseudonym( &self, class: &str, diff --git a/crates/registry-evidence/src/auth.rs b/crates/registry-evidence/src/auth.rs index bc0cb76752..4bf4b08239 100644 --- a/crates/registry-evidence/src/auth.rs +++ b/crates/registry-evidence/src/auth.rs @@ -19,6 +19,7 @@ use thiserror::Error; use crate::config::{ AccessTokenAlgorithm, AccessTokenType, AssuranceProfile, AuthenticationConfig, + OidcAuthenticationConfig, }; const MAX_PRINCIPAL_BYTES: usize = 512; @@ -255,7 +256,17 @@ const CONFIRMATION_CLAIM: &str = "cnf"; impl Authenticator { /// Build the one strict resource-server profile from the loaded bundle. - pub fn from_config(config: &AuthenticationConfig, assurance_profile: AssuranceProfile) -> Self { + /// + /// `issuer_roots` are the private certificate authorities the runtime + /// file binds to the bundle's `tlsTrustProfile`. They are trusted beside + /// the system roots for the `jwksSource.uri` connection alone, and are empty + /// when the bundle names no profile. + pub fn from_config( + config: &AuthenticationConfig, + assurance_profile: AssuranceProfile, + issuer_roots: Vec, + ) -> Self { + let config = &config.oidc; let algorithms = config .algorithms .iter() @@ -274,8 +285,8 @@ impl Authenticator { }) .collect(); let verifier_config = TokenVerifierConfig::access_token_profile( - config.issuer.clone(), - config.audiences.clone(), + config.issuer().to_owned(), + vec![config.audience().to_owned()], algorithms, token_types, ) @@ -285,10 +296,11 @@ impl Authenticator { ))) .with_allowed_clients(config.allowed_clients.clone().unwrap_or_default()) .with_assertion_issuers(config.assertion_issuers.clone().unwrap_or_default()); - let fetcher = Arc::new(JwksFetcher::new_with_fetch_url_policy( - config.jwks_uri.clone(), + let fetcher = Arc::new(JwksFetcher::new_trusting_additional_roots( + config.jwks_uri().to_owned(), JwksFetcherConfig::defaults(), jwks_fetch_policy(config, assurance_profile), + issuer_roots, )); let verifier = Arc::new(TokenVerifier::new(verifier_config, fetcher)); let claims = AuthenticationClaimsConfig { @@ -300,7 +312,7 @@ impl Authenticator { }; Self::new(verifier, claims) .with_required_scopes(config.required_scopes.clone().unwrap_or_default()) - .with_resources(config.audiences.clone()) + .with_resources(vec![config.audience().to_owned()]) } pub fn new(verifier: Arc, claims: AuthenticationClaimsConfig) -> Self { @@ -409,7 +421,7 @@ impl Authenticator { /// Attempt the issuer's key set once at startup, and name it if it cannot /// be had. /// - /// A misspelled or unreachable `jwksUri` is otherwise discovered one + /// A misspelled or unreachable `jwksSource.uri` is otherwise discovered one /// rejected request at a time, and the rejection an operator sees is the /// same closed `401` a bad token gets. Startup is where an operator is /// looking, so startup is where it should be said. @@ -624,7 +636,7 @@ fn exactly_matched_resource<'a>( } fn jwks_fetch_policy( - config: &AuthenticationConfig, + config: &OidcAuthenticationConfig, assurance_profile: AssuranceProfile, ) -> FetchUrlPolicy { if config.uses_local_issuer_http(assurance_profile) { @@ -791,19 +803,30 @@ fn valid_claim_path_segment(segment: &str) -> bool { mod tests { use super::*; - fn authentication_config() -> AuthenticationConfig { + fn authentication_config() -> OidcAuthenticationConfig { crate::config::EvidenceConfig::parse_yaml(include_bytes!( "../../../products/evidence/fixtures/acceptance/adult-status/evidence.yaml" )) .expect("acceptance configuration parses") .authentication + .oidc + } + + fn set_issuer(config: &mut OidcAuthenticationConfig, issuer: &str, jwks_uri: &str) { + config.provider.issuer = issuer.to_owned(); + config.provider.jwks_source = registry_platform_config::JwksSource::Uri { + uri: jwks_uri.to_owned(), + }; } #[test] fn jwks_fetch_policy_opens_http_only_for_the_supervised_local_issuer() { let mut exact = authentication_config(); - exact.issuer = "http://127.0.0.1:8081".to_owned(); - exact.jwks_uri = "http://127.0.0.1:8081/.well-known/jwks.json".to_owned(); + set_issuer( + &mut exact, + "http://127.0.0.1:8081", + "http://127.0.0.1:8081/.well-known/jwks.json", + ); let local = jwks_fetch_policy(&exact, AssuranceProfile::Local); assert_eq!(local.allowed_schemes, ["http"]); assert!(local.allow_localhost); @@ -812,7 +835,11 @@ mod tests { // The JWKS path is the issuer's to choose; what stays fixed is the // exact same numeric loopback origin. let mut other_path = exact.clone(); - other_path.jwks_uri = "http://127.0.0.1:8081/oauth2/jwks".to_owned(); + set_issuer( + &mut other_path, + "http://127.0.0.1:8081", + "http://127.0.0.1:8081/oauth2/jwks", + ); assert_eq!( jwks_fetch_policy(&other_path, AssuranceProfile::Local).allowed_schemes, ["http"] @@ -846,8 +873,7 @@ mod tests { ), ] { let mut candidate = authentication_config(); - candidate.issuer = issuer.to_owned(); - candidate.jwks_uri = jwks_uri.to_owned(); + set_issuer(&mut candidate, issuer, jwks_uri); let policy = jwks_fetch_policy(&candidate, profile); assert_eq!( policy.allowed_schemes, diff --git a/crates/registry-evidence/src/bundle.rs b/crates/registry-evidence/src/bundle.rs index 85850749a3..fe7000942c 100644 --- a/crates/registry-evidence/src/bundle.rs +++ b/crates/registry-evidence/src/bundle.rs @@ -8,6 +8,10 @@ use std::path::{Path, PathBuf}; use base64::Engine as _; use jsonschema::{Draft, JSONSchema}; +use registry_platform_config::{ + package::is_envelope_file, sha256_uri, verify_package, PackageError, PackageLimits, + VerifiedPackage, SUM_FILE, +}; use registry_platform_crypto::{ canonicalize_json, PublicJwk, SigningAlgorithm as ProviderSigningAlgorithm, }; @@ -35,8 +39,6 @@ pub const MAX_PUBLIC_JWK_BYTES: u64 = 64 * 1024; const CONFIG_FILE: &str = "evidence.yaml"; pub const DISCOVERY_DESCRIPTION_FILE: &str = "catalog.jsonld"; const RUNTIME_FILE: &str = "runtime.yaml"; -const REVISION_DOMAIN: &[u8] = b"registry.evidence.bundle-revision/v1\0"; -const RUNTIME_REVISION_DOMAIN: &[u8] = b"registry.evidence.runtime-revision/v1\0"; const REQUIREMENT_REVISION_DOMAIN: &[u8] = b"registry.evidence.requirement-revision/v1\0"; /// Path the canonical configuration projection takes inside a requirement's /// closure. An artifact path can hold no `#`, so this can never collide with a @@ -47,6 +49,7 @@ const ACQUISITION_CAPABILITIES: &str = "acquisitionCapabilities"; const PROVIDER_PUBLICATION: &str = "publication"; const SIGNING: &str = "signing"; const AUTHENTICATION: &str = "authentication"; +const OIDC: &str = "oidc"; const ACTIVE_PUBLIC_JWK_FILE: &str = "activePublicJwkFile"; const PUBLISHED_PUBLIC_JWK_FILES: &str = "publishedPublicJwkFiles"; const REVOKED_KEY_IDS: &str = "revokedKeyIds"; @@ -80,6 +83,8 @@ pub enum BundleError { UnknownFile(ArtifactFault), #[error("the Evidence deployment bundle exceeds a Version 1 size bound")] TooLarge, + #[error("{0}")] + Package(PackageError), #[error("the Evidence deployment configuration is invalid: {0}")] Config(ArtifactFault), #[error("an Evidence bundle artifact is invalid: {0}")] @@ -100,6 +105,7 @@ impl BundleError { | Self::InvalidScript(fault) | Self::NotImmutable(fault) | Self::UnknownFile(fault) => Some(fault), + Self::Package(_) => None, _ => None, } } @@ -114,6 +120,7 @@ impl BundleError { Self::InvalidArtifact(fault) => Self::InvalidArtifact(fault.bind(artifact)), Self::InvalidScript(fault) => Self::InvalidScript(fault.bind(artifact)), Self::NotImmutable(fault) => Self::NotImmutable(fault.bind(artifact)), + Self::Package(error) => Self::Package(error), other => other, } } @@ -277,16 +284,16 @@ impl Codelist { } } -/// One fully captured, validated bundle revision. +/// One fully captured, validated Evidence package. /// /// Runtime consumers use these captured bytes and compiled artifacts. They do -/// not reopen the deployment directory, which prevents a later filesystem -/// change from partially replacing the revision used by a serving process. +/// not reopen the package directory, which prevents a later filesystem change +/// from partially replacing the package used by a serving process. #[derive(Debug, Clone)] pub struct Bundle { root: PathBuf, pub config: EvidenceConfig, - revision: String, + package_digest: String, requirement_revisions: BTreeMap, files: BTreeMap>, discovery_description: Option>, @@ -301,13 +308,12 @@ pub struct Bundle { /// One captured operator runtime configuration and its bound trust anchors. /// /// Secret values and audit contents are deliberately not captured. The -/// runtime digest covers the reviewed runtime YAML, the private-CA bytes, and -/// the digest of every process-local extract the document binds. +/// reviewed runtime YAML and private-CA bytes are captured, while every +/// process-local extract is bound by a digest and stable file identity. #[derive(Debug, Clone)] pub struct RuntimeDocument { path: PathBuf, pub config: RuntimeConfig, - revision: String, bytes: Vec, pub ca_bundles: BTreeMap>, pub source_extracts: BTreeMap, @@ -368,7 +374,7 @@ impl SourceExtract { /// symlink refusal, not the regular-file refusal, and not the writability /// refusal that is what makes `immutable=1` a checked fact. Checking here /// keeps a refresh landing mid-startup a startup failure rather than a - /// deployment answering from bytes its runtime revision does not name. + /// deployment answering from extract bytes that were never validated. /// /// Narrowing rather than proof: a path renamed away and back inside the /// window still passes, and anyone who can write the containing directory @@ -409,10 +415,16 @@ impl RuntimeDocument { writable_runtime, ) .map_err(|error| error.in_artifact(RUNTIME_FILE))?; - let config = RuntimeConfig::parse_yaml(&bytes).map_err(|error| { - BundleError::Config(ArtifactFault::new(RUNTIME_FILE, error.fault())) - })?; - validate_secret_root(Path::new(&config.secret_providers.file.root))?; + // `${NAME}` in a string value reads the process environment; the + // loader refuses it in a secret reference and under secretProviders. + let loaded = RuntimeConfig::parse_yaml_with(&bytes, |name| std::env::var(name).ok()) + .map_err(|error| { + BundleError::Config(ArtifactFault::new(RUNTIME_FILE, error.fault())) + })?; + let config = loaded.config; + if let Some(file) = &config.secret_providers.file { + validate_secret_root(&file.root)?; + } let mut ca_bundles = BTreeMap::new(); for (profile, binding) in config.outbound_tls.trust_profiles.iter() { @@ -442,11 +454,9 @@ impl RuntimeDocument { .map_err(|error| error.in_artifact(&source_extract_artifact(profile)))?; source_extracts.insert(profile.to_owned(), extract); } - let revision = compute_runtime_revision(&bytes, &ca_bundles, &source_extracts)?; Ok(Self { path: path.to_path_buf(), config, - revision, bytes, ca_bundles, source_extracts, @@ -457,10 +467,6 @@ impl RuntimeDocument { &self.path } - pub fn revision(&self) -> &str { - &self.revision - } - pub fn bytes(&self) -> &[u8] { &self.bytes } @@ -489,7 +495,12 @@ pub struct DeploymentInputs { impl DeploymentInputs { pub fn load(runtime_path: impl AsRef) -> Result { let runtime = RuntimeDocument::load(runtime_path)?; - let bundle = Bundle::load(&runtime.config.bundle_directory)?; + let verified = runtime + .config + .package + .verify_package(&evidence_package_limits(), "evidencectl package") + .map_err(package_error)?; + let bundle = Bundle::load_verified(&runtime.config.package.root, &verified)?; validate_runtime_bindings(&bundle.config, &runtime.config)?; Ok(Self { bundle, runtime }) } @@ -513,7 +524,13 @@ pub type EvidenceBundle = Bundle; impl Bundle { pub fn load(root: impl AsRef) -> Result { let root = root.as_ref(); - let files = capture_bundle_files(root)?; + let verified = verify_package(root, &evidence_package_limits(), "evidencectl package") + .map_err(package_error)?; + Self::load_verified(root, &verified) + } + + fn load_verified(root: &Path, verified: &VerifiedPackage) -> Result { + let files = capture_bundle_files(root, verified)?; let config_bytes = files.get(CONFIG_FILE).ok_or(BundleError::Unavailable)?; let config = EvidenceConfig::parse_yaml(config_bytes) .map_err(|error| BundleError::Config(ArtifactFault::new(CONFIG_FILE, error.fault())))?; @@ -548,13 +565,12 @@ impl Bundle { validate_codelist_references(&config, &codelists)?; let fixtures = load_fixtures(&config, &files)?; let (active_public_jwk, published_public_jwks) = load_public_jwks(&config, &files)?; - let revision = compute_revision(&files)?; let requirement_revisions = compute_requirement_revisions(&config, &files)?; Ok(Self { root: root.to_path_buf(), config, - revision, + package_digest: verified.digest().to_owned(), requirement_revisions, files, discovery_description, @@ -584,13 +600,13 @@ impl Bundle { .map(String::as_str) } - /// The digest of every file in the deployment bundle. + /// The digest of the package's `SHA256SUMS` file. /// /// This is the deployment's own identity, for audit, status, and operator /// diagnostics. It is not what an assertion carries: see /// [`Bundle::configuration_revision`]. - pub fn revision(&self) -> &str { - &self.revision + pub fn package_digest(&self) -> &str { + &self.package_digest } pub fn artifact(&self, path: &str) -> Option<&[u8]> { @@ -623,7 +639,24 @@ pub fn load_bundle(root: impl AsRef) -> Result { Bundle::load(root) } -fn capture_bundle_files(root: &Path) -> Result>, BundleError> { +pub(crate) fn evidence_package_limits() -> PackageLimits { + PackageLimits { + max_files: MAX_BUNDLE_FILES, + max_file_bytes: MAX_ARTIFACT_BYTES, + max_total_bytes: MAX_BUNDLE_BYTES, + max_depth: 3, + max_path_bytes: 128, + } +} + +fn package_error(error: PackageError) -> BundleError { + BundleError::Package(error.naming_root_as("package.root")) +} + +fn capture_bundle_files( + root: &Path, + verified: &VerifiedPackage, +) -> Result>, BundleError> { let root_metadata = fs::symlink_metadata(root).map_err(|_| BundleError::Unavailable)?; if root_metadata.file_type().is_symlink() || !root_metadata.is_dir() { return Err(BundleError::InvalidPath); @@ -649,6 +682,7 @@ fn capture_bundle_files(root: &Path) -> Result>, Bundle } let mut files = BTreeMap::new(); + let mut uncaptured = verified.files().map(str::to_owned).collect::>(); let mut total = 0_u64; for (relative, path, scanned_metadata) in paths { let cap = file_size_cap(&relative); @@ -666,7 +700,32 @@ fn capture_bundle_files(root: &Path) -> Result>, Bundle if total > MAX_BUNDLE_BYTES { return Err(BundleError::TooLarge); } - files.insert(relative, bytes); + if relative == SUM_FILE { + if sha256_uri(&bytes) != verified.digest() { + return Err( + not_immutable("SHA256SUMS was replaced after package verification") + .in_artifact(SUM_FILE), + ); + } + continue; + } + let digest = sha256_uri(&bytes); + if verified.file_digest(&relative).as_deref() != Some(digest.as_str()) { + return Err(not_immutable( + "the package artifact was replaced after package verification", + ) + .in_artifact(&relative)); + } + uncaptured.remove(&relative); + if !is_envelope_file(&relative) { + files.insert(relative, bytes); + } + } + if let Some(missing) = uncaptured.into_iter().next() { + return Err( + not_immutable("the package artifact disappeared after package verification") + .in_artifact(&missing), + ); } if !files.contains_key(CONFIG_FILE) { return Err(BundleError::Unavailable); @@ -715,7 +774,9 @@ fn collect_paths( if !ALLOWED_DIRECTORIES.contains(&top) { return Err(BundleError::InvalidPath); } - } else if !matches!(relative.as_str(), CONFIG_FILE | DISCOVERY_DESCRIPTION_FILE) { + } else if !matches!(relative.as_str(), CONFIG_FILE | DISCOVERY_DESCRIPTION_FILE) + && !is_envelope_file(&relative) + { return Err(unknown_file( &relative, "bundle root contains a file other than the configuration", @@ -2503,7 +2564,36 @@ fn validate_runtime_bindings( "runtime signer kind does not match the bundle assurance profile", )); } - let audit_ref = &bundle.audit.hash_key_ref; + // A governed reference resolves only through a provider the operator + // enabled, so the bundle cannot reach the process environment unless the + // runtime file opts in to it. + let governed_refs = bundle + .sources + .iter() + .filter_map(|(_, source)| source.authentication()) + .chain( + bundle + .source_connections + .iter() + .map(|(_, connection)| connection.authentication.as_ref()), + ) + .flat_map(|authentication| authentication.secret_refs()) + .chain([ + &bundle.audit.key.hash_key_ref, + &bundle.subject_binding.secret_ref, + ]); + for reference in governed_refs { + if runtime + .secret_providers + .check_reference("secretProviders", reference.as_str()) + .is_err() + { + return Err(invalid_artifact( + "a bundle secret reference names a provider the runtime secretProviders does not enable", + )); + } + } + let audit_ref = &bundle.audit.key.hash_key_ref; let subject_ref = &bundle.subject_binding.secret_ref; if let Some(signing_ref) = runtime.signer.private_key_ref() { if signing_ref == audit_ref || signing_ref == subject_ref { @@ -2512,7 +2602,11 @@ fn validate_runtime_bindings( )); } } - let secret_root = Path::new(&runtime.secret_providers.file.root); + let secret_root = runtime + .secret_providers + .file + .as_ref() + .map(|file| file.root.as_path()); if let Some(audit_path) = runtime.audit.path.as_deref().map(Path::new) { let configured_secret_paths = [ Some(audit_ref), @@ -2522,7 +2616,7 @@ fn validate_runtime_bindings( .into_iter() .flatten() .filter_map(|reference| reference.as_str().strip_prefix("secret:file/")) - .map(|name| secret_root.join(name)); + .filter_map(|name| secret_root.map(|root| root.join(name))); if configured_secret_paths .into_iter() .any(|path| path == audit_path) @@ -2535,13 +2629,15 @@ fn validate_runtime_bindings( // The binding is exact in both directions, but the two directions are // different repairs in different files: a profile the bundle names and the // runtime does not bind is missing trust material the deployment must add, - // while a profile the runtime binds that no source names is trust the - // deployment grants nobody asked for. One cause covering both leaves the - // operator to diff the two files by hand to learn which way round it went. + // while a profile the runtime binds that neither a source nor the issuer + // names is trust the deployment grants nobody asked for. One cause + // covering both leaves the operator to diff the two files by hand to learn + // which way round it went. let required = bundle .sources .iter() .filter_map(|(_, source)| source.tls_trust_profile()) + .chain(bundle.authentication.oidc.tls_trust_profile.as_deref()) .collect::>(); let configured = runtime .outbound_tls @@ -2551,13 +2647,13 @@ fn validate_runtime_bindings( if let Some(unbound) = required.difference(&configured).next() { return Err(trust_profile_fault( unbound, - "the runtime configuration does not bind a TLS trust profile a bundle source names", + "the runtime configuration does not bind a TLS trust profile the bundle names", )); } if let Some(unused) = configured.difference(&required).next() { return Err(trust_profile_fault( unused, - "the runtime configuration binds a TLS trust profile no bundle source names", + "the runtime configuration binds a TLS trust profile the bundle does not name", )); } // Extracts bind exactly, in both directions, and each direction says which @@ -2689,31 +2785,6 @@ fn validate_ca_bundle(bytes: &[u8]) -> Result<(), BundleError> { Ok(()) } -fn compute_runtime_revision( - runtime_bytes: &[u8], - ca_bundles: &BTreeMap>, - source_extracts: &BTreeMap, -) -> Result { - let mut files = BTreeMap::from([("runtime.yaml".to_owned(), runtime_bytes.to_vec())]); - for (profile, bytes) in ca_bundles { - files.insert(format!("trust-profile/{profile}.pem"), bytes.clone()); - } - // An extract enters the revision as its digest rather than its bytes. The - // revision still covers every byte that was read, because a digest changes - // whenever the file it was taken over changes. - for (profile, extract) in source_extracts { - files.insert( - format!("source-extract/{profile}"), - extract.digest().as_bytes().to_vec(), - ); - } - compute_named_revision(RUNTIME_REVISION_DOMAIN, &files) -} - -fn compute_revision(files: &BTreeMap>) -> Result { - compute_named_revision(REVISION_DOMAIN, files) -} - /// One configuration revision per configured requirement. /// /// Each digest covers exactly what can change that requirement's assertions: @@ -2759,7 +2830,7 @@ fn compute_requirement_revisions( /// bounded acquisition names, and the codelists of the selector profiles its subject roles and /// grants use. The active and published public signing keys are not in any /// requirement's closure: they are trust a relying party takes from the JWKS, -/// and the bundle revision still covers them. +/// and the package digest still covers them. fn requirement_artifact_paths( config: &EvidenceConfig, requirement: &RequirementConfig, @@ -2941,13 +3012,14 @@ fn remove_signing_key_trust(members: &mut JsonMap) -> Result< fn remove_caller_token_revocations( members: &mut JsonMap, ) -> Result<(), BundleError> { - let authentication = members + let oidc = members .get_mut(AUTHENTICATION) + .and_then(|authentication| authentication.get_mut(OIDC)) .and_then(JsonValue::as_object_mut) .ok_or_else(|| { invalid_artifact("the authentication configuration does not project as a mapping") })?; - authentication.remove(REVOKED_KEY_IDS); + oidc.remove(REVOKED_KEY_IDS); Ok(()) } @@ -3199,7 +3271,7 @@ mod tests { } #[test] - fn revision_binds_paths_and_exact_bytes_deterministically() { + fn package_digest_binds_paths_and_exact_bytes_deterministically() { let first = BTreeMap::from([ ("evidence.yaml".to_owned(), b"version: 1\n".to_vec()), ("schemas/facts.yaml".to_owned(), b"type: object\n".to_vec()), @@ -3208,13 +3280,124 @@ mod tests { ("schemas/facts.yaml".to_owned(), b"type: object\n".to_vec()), ("evidence.yaml".to_owned(), b"version: 1\n".to_vec()), ]); - assert_eq!(compute_revision(&first), compute_revision(&same)); + let digest = |files: &BTreeMap>| { + registry_platform_config::plan_package( + Path::new("evidence-package"), + files, + None, + &evidence_package_limits(), + "evidencectl package", + ) + .expect("the package digest plans") + }; + assert_eq!(digest(&first), digest(&same)); let renamed = BTreeMap::from([ ("evidence.yaml".to_owned(), b"version: 1\n".to_vec()), ("schemas/other.yaml".to_owned(), b"type: object\n".to_vec()), ]); - assert_ne!(compute_revision(&first), compute_revision(&renamed)); + assert_ne!(digest(&first), digest(&renamed)); + } + + #[cfg(unix)] + fn refresh_package_envelope(root: &Path) { + for reserved in [SUM_FILE, registry_platform_config::REVISION_FILE] { + let path = root.join(reserved); + if path.exists() { + fs::remove_file(path).expect("remove the prior package envelope"); + } + } + registry_platform_config::write_sum_file( + root, + None, + &evidence_package_limits(), + "evidencectl package", + ) + .expect("refresh the package envelope"); + } + + #[cfg(unix)] + #[test] + fn package_verification_names_changed_missing_and_extra_files() { + type MutationCase = (&'static str, fn(&Path), &'static str); + let cases: [MutationCase; 3] = [ + ( + "changed", + |root: &Path| { + fs::write( + root.join("derivations/adult-status.rhai"), + "fn derive(_) { no_match() }\n", + ) + .expect("change a listed artifact"); + }, + "changed: derivations/adult-status.rhai", + ), + ( + "missing", + |root: &Path| { + fs::remove_file(root.join("derivations/adult-status.rhai")) + .expect("remove a listed artifact"); + }, + "missing: derivations/adult-status.rhai", + ), + ( + "extra", + |root: &Path| { + fs::write(root.join("schemas/unlisted.yaml"), "type: object\n") + .expect("add an unlisted artifact"); + }, + "extra: schemas/unlisted.yaml", + ), + ]; + for (case, mutate, expected) in cases { + let directory = tempfile::tempdir().expect("temporary package"); + copy_acceptance_bundle("adult-status", directory.path()); + mutate(directory.path()); + set_tree_mode(directory.path(), 0o555, 0o444); + let error = Bundle::load(directory.path()).expect_err(case); + assert!(error.to_string().contains(expected), "{error}"); + set_tree_mode(directory.path(), 0o755, 0o644); + } + } + + #[cfg(unix)] + #[test] + fn package_digest_is_repeatable_and_unpinned_packages_are_verified() { + let directory = tempfile::tempdir().expect("temporary package"); + copy_acceptance_bundle("adult-status", directory.path()); + set_tree_mode(directory.path(), 0o555, 0o444); + let first = Bundle::load(directory.path()).expect("the unpinned package loads"); + let again = Bundle::load(directory.path()).expect("the same package reloads"); + assert_eq!(first.package_digest(), again.package_digest()); + assert!(first.package_digest().starts_with("sha256:")); + } + + #[cfg(unix)] + #[test] + fn consumer_capture_refuses_bytes_replaced_after_package_verification() { + let directory = tempfile::tempdir().expect("temporary package"); + copy_acceptance_bundle("adult-status", directory.path()); + let verified = verify_package( + directory.path(), + &evidence_package_limits(), + "evidencectl package", + ) + .expect("the package verifies"); + fs::write( + directory.path().join("derivations/adult-status.rhai"), + "fn derive(_) { no_match() }\n", + ) + .expect("replace a verified artifact"); + set_tree_mode(directory.path(), 0o555, 0o444); + let error = capture_bundle_files(directory.path(), &verified) + .expect_err("consumer capture refuses replaced bytes"); + let fault = error.artifact_fault().expect("the artifact is named"); + assert_eq!(fault.artifact(), "derivations/adult-status.rhai"); + assert_eq!( + fault.fault().cause(), + "the package artifact was replaced after package verification" + ); + set_tree_mode(directory.path(), 0o755, 0o644); } /// The revision every configured requirement carries, keyed by requirement. @@ -3245,11 +3428,13 @@ mod tests { ) -> (BTreeMap, BTreeMap) { let directory = tempfile::tempdir().expect("temporary bundle"); copy_acceptance_bundle("all-definitions", directory.path()); + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let before = requirement_revisions(directory.path()); set_tree_mode(directory.path(), 0o755, 0o644); edit(directory.path()); + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let after = requirement_revisions(directory.path()); (before, after) @@ -3374,10 +3559,11 @@ mod tests { ) .expect("batch response schema writes"); + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let before = Bundle::load(directory.path()).expect("batch bundle loads"); let requirement_id = before.config.requirements[0].id.clone(); - let before_bundle = before.revision().to_owned(); + let before_package = before.package_digest().to_owned(); let before_requirement = before .configuration_revision(&requirement_id) .expect("requirement revision exists") @@ -3391,9 +3577,10 @@ mod tests { format!("{source}\n// reviewed batch edit\n"), ) .expect("batch preparation changes"); + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let after = Bundle::load(directory.path()).expect("edited batch bundle loads"); - assert_ne!(before_bundle, after.revision()); + assert_ne!(before_package, after.package_digest()); assert_ne!( before_requirement, after @@ -3502,7 +3689,7 @@ mod tests { /// relying party takes from the JWKS, not through the revision it pins. /// Every step of a planned rotation (publish the next key, activate it, /// retire the previous one) therefore leaves each requirement revision - /// alone, while the bundle revision still records every step. + /// alone, while the package digest still records every step. #[cfg(unix)] #[test] fn a_planned_signing_key_rotation_leaves_every_requirement_revision_alone() { @@ -3521,11 +3708,12 @@ mod tests { fs::write(&path, text.replace(from, to)).expect("the configuration writes"); }; let observe = || { + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let bundle = Bundle::load(directory.path()).expect("the rotated bundle loads"); let revisions = requirement_revisions(directory.path()); set_tree_mode(directory.path(), 0o755, 0o644); - (bundle.revision().to_owned(), revisions) + (bundle.package_digest().to_owned(), revisions) }; let (initial_bundle, initial) = observe(); @@ -3558,23 +3746,23 @@ mod tests { assert_eq!(published, initial, "publishing a key keeps every revision"); assert_eq!(activated, initial, "activating a key keeps every revision"); assert_eq!(retired, initial, "retiring a key keeps every revision"); - let bundle_revisions = BTreeSet::from([ + let package_digests = BTreeSet::from([ initial_bundle, published_bundle, activated_bundle, retired_bundle, ]); assert_eq!( - bundle_revisions.len(), + package_digests.len(), 4, - "the bundle revision still records every rotation step" + "the package digest still records every rotation step" ); } /// An emergency revocation denies a key through the JWKS and the denylist a /// relying party's policy carries, not through the revision it pins, so - /// revoking a key leaves each requirement revision alone while the bundle - /// revision still records it. + /// revoking a key leaves each requirement revision alone while the package + /// digest still records it. #[cfg(unix)] #[test] fn revoking_a_signing_key_leaves_every_requirement_revision_alone() { @@ -3584,11 +3772,12 @@ mod tests { let directory = tempfile::tempdir().expect("temporary bundle"); copy_acceptance_bundle("all-definitions", directory.path()); let observe = || { + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let bundle = Bundle::load(directory.path()).expect("the bundle loads"); let revisions = requirement_revisions(directory.path()); set_tree_mode(directory.path(), 0o755, 0o644); - (bundle.revision().to_owned(), revisions) + (bundle.package_digest().to_owned(), revisions) }; let (initial_bundle, initial) = observe(); @@ -3606,28 +3795,29 @@ mod tests { assert_eq!(revoked, initial, "revoking a key keeps every revision"); assert_ne!( revoked_bundle, initial_bundle, - "the bundle revision still records the revocation" + "the package digest still records the revocation" ); } /// Revoking an identity-provider key changes which caller tokens Evidence /// accepts, not what any assertion means, and a relying party never /// verifies those tokens. The revocation therefore leaves each requirement - /// revision alone while the bundle revision still records it. + /// revision alone while the package digest still records it. #[cfg(unix)] #[test] fn revoking_a_caller_token_key_leaves_every_requirement_revision_alone() { - const UNREVOKED: &str = "maximumTokenLifetimeSeconds: 300\n revokedKeyIds: []"; - const REVOKED: &str = "maximumTokenLifetimeSeconds: 300\n revokedKeyIds: [-RNgdUjduVCNV-y15KSAVZnF2gNjGb_02KQ2-MoMu4U]"; + const UNREVOKED: &str = "maximumTokenLifetimeSeconds: 300\n revokedKeyIds: []"; + const REVOKED: &str = "maximumTokenLifetimeSeconds: 300\n revokedKeyIds: [-RNgdUjduVCNV-y15KSAVZnF2gNjGb_02KQ2-MoMu4U]"; let directory = tempfile::tempdir().expect("temporary bundle"); copy_acceptance_bundle("all-definitions", directory.path()); let observe = || { + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let bundle = Bundle::load(directory.path()).expect("the bundle loads"); let revisions = requirement_revisions(directory.path()); set_tree_mode(directory.path(), 0o755, 0o644); - (bundle.revision().to_owned(), revisions) + (bundle.package_digest().to_owned(), revisions) }; let (initial_bundle, initial) = observe(); @@ -3648,7 +3838,7 @@ mod tests { ); assert_ne!( revoked_bundle, initial_bundle, - "the bundle revision still records the revocation" + "the package digest still records the revocation" ); } @@ -3712,10 +3902,10 @@ mod tests { assert!(!projected_signing.contains_key(REVOKED_KEY_IDS)); // The authentication member loses only its revoked caller token keys; // the issuer, JWKS location, and claims authorization reads stay. - let configured_authentication = configured[AUTHENTICATION] + let configured_authentication = configured[AUTHENTICATION][OIDC] .as_object() .expect("the authentication configuration is a mapping"); - let projected_authentication = projected[AUTHENTICATION] + let projected_authentication = projected[AUTHENTICATION][OIDC] .as_object() .expect("the projected authentication configuration is a mapping"); assert!(configured_authentication.contains_key(REVOKED_KEY_IDS)); @@ -3807,6 +3997,9 @@ mod tests { fs::write(&config_path, local).expect("local configuration writes"); fs::remove_file(directory.path().join("fixtures/cases.yaml")) .expect("unreferenced fixture is removed"); + fs::remove_dir(directory.path().join("fixtures")) + .expect("empty fixture directory is removed"); + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let bundle = Bundle::load(directory.path()).expect("local bundle loads without fixtures"); @@ -3822,6 +4015,7 @@ mod tests { &format!("assuranceProfile: {profile}"), ); fs::write(&config_path, candidate).expect("strict configuration writes"); + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); assert!( Bundle::load(directory.path()).is_err(), @@ -3835,6 +4029,7 @@ mod tests { "assuranceProfile: local", ); fs::write(&config_path, reset).expect("local configuration restores"); + refresh_package_envelope(directory.path()); } } @@ -3862,6 +4057,7 @@ mod tests { .collect::>() .join("\n"); fs::write(&fixtures_path, fixtures).expect("partial fixtures write"); + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let error = Bundle::load(directory.path()).expect_err(&format!( @@ -4001,6 +4197,7 @@ mod tests { "type: object\nadditionalProperties: false\nrequired: []\nproperties: {}\n", ) .unwrap(); + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); Bundle::load(directory.path()).expect("a closed empty startup contract loads"); set_tree_mode(directory.path(), 0o755, 0o644); @@ -4032,6 +4229,7 @@ mod tests { ); fs::write(directory.path().join("schemas/structured.yaml"), schema) .expect("write reviewed schema"); + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let bundle = Bundle::load(directory.path()).expect("reviewed schema resolves"); assert!(bundle.fact_schemas.contains_key("schemas/structured.yaml")); @@ -4053,6 +4251,7 @@ mod tests { ), ) .expect("write invalid reviewed schema"); + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); assert!( matches!( @@ -4067,6 +4266,7 @@ mod tests { fs::write(&schema_path, schema).expect("restore reviewed schema"); fs::write(directory.path().join("schemas/duplicate.yaml"), schema) .expect("write duplicate schema"); + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let ambiguous = Bundle::load(directory.path()).expect_err("duplicate schema is rejected"); assert!(matches!(ambiguous, BundleError::InvalidArtifact(_))); @@ -4543,6 +4743,7 @@ mod tests { b"synthetic_only: true\n", ) .expect("write unknown artifact"); + refresh_package_envelope(unknown.path()); set_tree_mode(unknown.path(), 0o555, 0o444); let unreferenced = Bundle::load(unknown.path()).expect_err("unknown artifact is rejected"); assert!(matches!(unreferenced, BundleError::UnknownFile(_))); @@ -4558,6 +4759,9 @@ mod tests { copy_acceptance_bundle("adult-status", missing.path()); fs::remove_file(missing.path().join("derivations/adult-status.rhai")) .expect("remove referenced derivation"); + fs::remove_dir(missing.path().join("derivations")) + .expect("empty derivation directory is removed"); + refresh_package_envelope(missing.path()); set_tree_mode(missing.path(), 0o555, 0o444); let absent = Bundle::load(missing.path()).expect_err("missing artifact is rejected"); let fault = absent.artifact_fault().expect("closure names a file"); @@ -4576,6 +4780,7 @@ mod tests { copy_acceptance_bundle("adult-status", missing.path()); fs::remove_file(missing.path().join(DISCOVERY_DESCRIPTION_FILE)) .expect("remove packaged description"); + refresh_package_envelope(missing.path()); set_tree_mode(missing.path(), 0o555, 0o444); let error = Bundle::load(missing.path()).expect_err("missing description fails closed"); assert_eq!( @@ -4588,6 +4793,7 @@ mod tests { copy_acceptance_bundle("adult-status", drifted.path()); fs::write(drifted.path().join(DISCOVERY_DESCRIPTION_FILE), b"{}\n") .expect("drift packaged description"); + refresh_package_envelope(drifted.path()); set_tree_mode(drifted.path(), 0o555, 0o444); let error = Bundle::load(drifted.path()).expect_err("drifted description fails closed"); let fault = error.artifact_fault().expect("artifact fault"); @@ -4609,6 +4815,7 @@ mod tests { .join("\n") + "\n"; fs::write(&config_path, without_publication).expect("remove publication declaration"); + refresh_package_envelope(unconfigured.path()); set_tree_mode(unconfigured.path(), 0o555, 0o444); let error = Bundle::load(unconfigured.path()) .expect_err("an unconfigured packaged description fails closed"); @@ -4633,16 +4840,17 @@ mod tests { fs::remove_file(&adapter).expect("remove copied adapter"); symlink(outside.path(), adapter).expect("create symlink"); set_tree_mode(directory.path(), 0o555, 0o444); - assert!(matches!( - Bundle::load(directory.path()), - Err(BundleError::InvalidPath) - )); + let error = Bundle::load(directory.path()).expect_err("a symlink is refused"); + assert!( + error.to_string().contains("adapters/source-a.rhai"), + "{error}" + ); set_tree_mode(directory.path(), 0o755, 0o444); } #[cfg(unix)] #[test] - fn runtime_and_ca_bytes_are_captured_under_an_independent_read_only_revision() { + fn runtime_and_ca_bytes_are_captured_from_independent_read_only_inputs() { use std::os::unix::fs::PermissionsExt as _; let directory = tempfile::tempdir().expect("temporary runtime root"); @@ -4661,7 +4869,7 @@ mod tests { fs::write( &runtime_path, format!( - "version: 1\nbundleDirectory: /etc/registry-evidence/bundle\nlistener:\n bindHost: 127.0.0.1\n port: 8080\n tlsTermination: operator-controlled-upstream\n trustProxyIdentityHeaders: false\n maximumRequestBytes: 65536\n maximumConcurrentRequests: 64\n requestTimeoutMilliseconds: 10000\n shutdownGraceMilliseconds: 30000\nsecretProviders:\n file: {{root: {}}}\nsigner:\n kind: transit\n unixSocketPath: /run/registry-evidence/transit-proxy.sock\n mount: transit\n keyName: evidence-signing\n keyVersion: 7\n timeoutMilliseconds: 2000\naudit:\n path: /var/lib/registry-evidence/audit/evidence.jsonl\noutboundTls:\n systemRoots: true\n trustProfiles:\n internal-pki: {{caBundleFile: {}}}\n", + "apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1\nkind: EvidenceRuntimeConfig\npackage:\n root: /etc/registry-evidence/bundle\nlistener:\n bind: 127.0.0.1:8080\n tlsTermination: operator-controlled-upstream\n trustProxyIdentityHeaders: false\n maximumRequestBytes: 65536\n maximumConcurrentRequests: 64\n requestTimeoutMilliseconds: 10000\n shutdownGraceMilliseconds: 30000\nsecretProviders:\n file: {{root: {}}}\nsigner:\n kind: transit\n unixSocketPath: /run/registry-evidence/transit-proxy.sock\n mount: transit\n keyName: evidence-signing\n keyVersion: 7\n timeoutMilliseconds: 2000\naudit:\n path: /var/lib/registry-evidence/audit/evidence.jsonl\noutboundTls:\n systemRoots: true\n trustProfiles:\n internal-pki: {{caBundleFile: {}}}\n", secret_root.display(), ca_path.display() ), @@ -4677,8 +4885,6 @@ mod tests { fs::set_permissions(&runtime_path, fs::Permissions::from_mode(0o444)) .expect("lock runtime document"); let runtime = RuntimeDocument::load(&runtime_path).expect("runtime loads"); - assert!(runtime.revision().starts_with("sha256:")); - assert_eq!(runtime.revision().len(), 71); assert_eq!(runtime.ca_bundles.len(), 1); assert_eq!( runtime.bytes(), @@ -4704,11 +4910,13 @@ mod tests { /// One operator runtime document, written the way a deployment that /// predates the acquisition gate is written: it says nothing about /// acquisition capabilities, because there was nothing to say. - const OPERATOR_RUNTIME_DOCUMENT: &str = "version: 1 -bundleDirectory: /etc/registry-evidence/bundle + const OPERATOR_RUNTIME_DOCUMENT: &str = + "apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /etc/registry-evidence/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 @@ -4832,7 +5040,7 @@ outboundTls: assert_eq!(fault.artifact(), "trustProfiles/internal-pki"); assert_eq!( fault.fault().cause(), - "the runtime configuration does not bind a TLS trust profile a bundle source names" + "the runtime configuration does not bind a TLS trust profile the bundle names" ); let binding = RuntimeConfig::parse_yaml( @@ -4857,30 +5065,121 @@ outboundTls: assert_eq!(fault.artifact(), "trustProfiles/internal-pki"); assert_eq!( fault.fault().cause(), - "the runtime configuration binds a TLS trust profile no bundle source names" + "the runtime configuration binds a TLS trust profile the bundle does not name" ); } - /// New operator surface must not move the revision of a deployment that did - /// not ask for it. The runtime revision digests the exact runtime.yaml - /// bytes, so a file written before the acquisition gate existed keeps the - /// revision it already published; the pinned digest is what proves the - /// digest is still taken over those bytes and not over a serialization that - /// grew a member. The absent list also projects to nothing, so the same - /// deployment would keep its revision either way. + /// The issuer's trust profile joins the same exact binding as the + /// sources': the runtime must bind the profile `authentication` names, and + /// a profile only the issuer names is not unused trust. #[test] - fn an_absent_acquisition_capability_list_leaves_the_runtime_revision_byte_identical() { - let revision = compute_runtime_revision( - OPERATOR_RUNTIME_DOCUMENT.as_bytes(), - &BTreeMap::new(), - &BTreeMap::new(), + fn the_issuer_trust_profile_is_bound_exactly_like_a_source_profile() { + const ACCEPTANCE: &str = include_str!( + "../../../products/evidence/fixtures/acceptance/all-definitions/evidence.yaml" + ); + let naming = EvidenceConfig::parse_yaml( + ACCEPTANCE + .replace( + " uri: https://identity.invalid/.well-known/jwks.json\n", + " uri: https://identity.invalid/.well-known/jwks.json\n tlsTrustProfile: issuer-pki\n", + ) + .as_bytes(), ) - .expect("the runtime revision computes"); + .expect("a bundle naming an issuer trust profile validates"); + let silent = RuntimeConfig::parse_yaml(OPERATOR_RUNTIME_DOCUMENT.as_bytes()) + .expect("the operator runtime document parses"); + let missing = validate_runtime_bindings(&naming, &silent) + .expect_err("an issuer profile the runtime does not bind is refused"); + let fault = missing + .artifact_fault() + .expect("the refusal names the profile"); + assert_eq!(fault.artifact(), "trustProfiles/issuer-pki"); assert_eq!( - revision, "sha256:b83a1f816b4201e5d12c14c9c8677354d3adab3827735eb22688e2bf54be48fc", - "an operator who adopted nothing must keep the revision they published" + fault.fault().cause(), + "the runtime configuration does not bind a TLS trust profile the bundle names" ); + let binding = RuntimeConfig::parse_yaml( + OPERATOR_RUNTIME_DOCUMENT + .replace( + " trustProfiles: {}\n", + " trustProfiles: {issuer-pki: {caBundleFile: /etc/registry-evidence/issuer-pki.pem}}\n", + ) + .as_bytes(), + ) + .expect("a runtime binding the issuer profile parses"); + validate_runtime_bindings(&naming, &binding) + .expect("a profile only the issuer names is bound, not unused"); + } + + /// A governed reference to the process environment binds only where the + /// runtime file enables the environment provider, so a bundle author + /// cannot reach an operator's environment on their own. + #[test] + fn a_bundle_environment_reference_binds_only_where_the_runtime_enables_it() { + const ACCEPTANCE: &str = include_str!( + "../../../products/evidence/fixtures/acceptance/all-definitions/evidence.yaml" + ); + let environment = ACCEPTANCE.replace( + "hashKeyRef: secret:file/audit-hash-key", + "hashKeyRef: secret:env/EVIDENCE_AUDIT_HASH_KEY", + ); + assert_ne!( + environment, ACCEPTANCE, + "fixture mutation must remain effective" + ); + let bundle = EvidenceConfig::parse_yaml(environment.as_bytes()) + .expect("a bundle naming an environment secret validates on its own"); + let file_only = RuntimeConfig::parse_yaml(OPERATOR_RUNTIME_DOCUMENT.as_bytes()) + .expect("the operator runtime document parses"); + let refused = validate_runtime_bindings(&bundle, &file_only) + .expect_err("the environment provider is not enabled"); + assert_eq!( + refused.artifact_fault().map(|fault| fault.fault().cause()), + Some( + "a bundle secret reference names a provider the runtime secretProviders does not enable" + ) + ); + + let enabled = RuntimeConfig::parse_yaml( + OPERATOR_RUNTIME_DOCUMENT + .replace( + " file: {root: /run/secrets/registry-evidence}\n", + " file: {root: /run/secrets/registry-evidence}\n environment: {}\n", + ) + .as_bytes(), + ) + .expect("a runtime enabling the environment provider parses"); + validate_runtime_bindings(&bundle, &enabled) + .expect("an enabled environment provider binds the reference"); + } + + /// Only the operator's runtime file enables a secret provider. A bundle + /// that declares one itself is refused before any reference is bound, so + /// the authority to read the process environment never travels with the + /// governed package. + #[test] + fn a_bundle_cannot_enable_a_secret_provider_itself() { + const ACCEPTANCE: &str = include_str!( + "../../../products/evidence/fixtures/acceptance/all-definitions/evidence.yaml" + ); + EvidenceConfig::parse_yaml(ACCEPTANCE.as_bytes()).expect("the fixture validates"); + let self_enabling = format!("{ACCEPTANCE}secretProviders:\n environment: {{}}\n"); + let error = EvidenceConfig::parse_yaml(self_enabling.as_bytes()) + .expect_err("a bundle may not declare a secret provider"); + let appended_line = ACCEPTANCE.lines().count() + 1; + assert!( + error + .to_string() + .contains(&format!("unknown field (line {appended_line} column 1)")), + "{error}" + ); + } + + /// A deployment that does not opt into a capability must not acquire one + /// through serde defaults or the serialized runtime projection. + #[test] + fn an_absent_acquisition_capability_list_projects_to_nothing() { let config = RuntimeConfig::parse_yaml(OPERATOR_RUNTIME_DOCUMENT.as_bytes()) .expect("the operator runtime document parses"); assert!(config.acquisition_capabilities.is_empty()); @@ -4890,26 +5189,6 @@ outboundTls: .contains("acquisitionCapabilities"), "an absent capability list must serialize to nothing at all" ); - - // Recording the operator's decision is an edit to the file the digest - // covers, so the deployment that adopted the kind says so in its - // revision. - let adopted = format!( - "{OPERATOR_RUNTIME_DOCUMENT}acquisitionCapabilities: [search-then-fetch-set]\n" - ); - assert_ne!( - compute_runtime_revision(adopted.as_bytes(), &BTreeMap::new(), &BTreeMap::new()) - .expect("the runtime revision computes"), - revision - ); - let source_batch = - format!("{OPERATOR_RUNTIME_DOCUMENT}acquisitionCapabilities: [source-batch]\n"); - assert_ne!( - compute_runtime_revision(source_batch.as_bytes(), &BTreeMap::new(), &BTreeMap::new()) - .expect("the source-batch runtime revision computes"), - revision, - "the operator's source-batch authorization must affect runtime identity" - ); } /// The acceptance bundle's one source, restated on the statement transport. @@ -4981,6 +5260,7 @@ outboundTls: .expect("remove the displaced adapter-parameter schema"); fs::create_dir(destination.join("queries")).expect("create the statement directory"); fs::write(destination.join(STATEMENT_PATH), statement).expect("the statement writes"); + refresh_package_envelope(destination); } /// A statement is reviewed, bounded, executable text, so it is bounded like @@ -5060,6 +5340,7 @@ outboundTls: b"SELECT 1;\n", ) .expect("write an unreferenced statement"); + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let unreferenced = Bundle::load(directory.path()).expect_err("an unreferenced statement is refused"); @@ -5097,6 +5378,7 @@ outboundTls: b"SELECT total, date_of_birth FROM residents WHERE id = :record_reference LIMIT 1;\n", ) .expect("the statement rewrites"); + refresh_package_envelope(directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let after = Bundle::load(directory.path()) .expect("the edited statement bundle loads") @@ -5169,12 +5451,11 @@ outboundTls: } /// An extract is bound by digest, never by its bytes: a register-sized file - /// does not belong in a serving process's memory. The digest still has to - /// reach the runtime revision, or a deployment could answer from different - /// data under a revision it already published. + /// does not belong in a serving process's memory. The captured digest and + /// file identity still have to change when its bytes change. #[cfg(unix)] #[test] - fn an_extract_reaches_the_runtime_revision_as_a_digest_of_its_bytes() { + fn an_extract_is_bound_as_a_digest_of_its_bytes() { let directory = tempfile::tempdir().expect("temporary runtime root"); let extract = directory.path().join("residence-register.sqlite"); locked_extract(&extract, b"extract-content-one"); @@ -5193,9 +5474,17 @@ outboundTls: sha256_label(hasher).expect("the expected digest computes"), "the digest is taken over the extract's bytes and nothing else" ); + let first_digest = bound.digest().to_owned(); let again = RuntimeDocument::load(&runtime_path).expect("the runtime document reloads"); - assert_eq!(first.revision(), again.revision()); + assert_eq!( + again + .source_extracts + .get(EXTRACT_PROFILE) + .expect("the reloaded extract is captured") + .digest(), + first_digest + ); assert_eq!( first.bytes(), again.bytes(), @@ -5205,11 +5494,18 @@ outboundTls: locked_extract(&extract, b"extract-content-two"); let replaced = RuntimeDocument::load(&runtime_path).expect("the replaced extract still loads"); - assert_ne!(replaced.revision(), first.revision()); + assert_ne!( + replaced + .source_extracts + .get(EXTRACT_PROFILE) + .expect("the replaced extract is captured") + .digest(), + first_digest + ); assert_eq!( replaced.bytes(), first.bytes(), - "only the extract changed, so only its digest can have moved the revision" + "the runtime document itself did not change" ); } @@ -5345,8 +5641,8 @@ outboundTls: /// The capture and the SQLite open are not the same moment: the bundle is /// read, the kernel is compiled, and the audit log is initialized in /// between. A publisher who refreshes the bound path inside that window - /// gets a startup failure rather than a deployment serving bytes its - /// runtime revision does not name. + /// gets a startup failure rather than a deployment serving extract bytes + /// that were never validated. /// /// The replacement is a different length for the reason the test above /// gives. diff --git a/crates/registry-evidence/src/cli.rs b/crates/registry-evidence/src/cli.rs index eef2cf860c..d626776863 100644 --- a/crates/registry-evidence/src/cli.rs +++ b/crates/registry-evidence/src/cli.rs @@ -1,10 +1,15 @@ //! Evidence runtime command-line contract. +use std::ffi::OsStr; use std::path::PathBuf; use clap::{ArgGroup, CommandFactory, Parser, Subcommand, ValueEnum}; -const DEFAULT_RUNTIME_PATH: &str = "/etc/registry-evidence/runtime.yaml"; +/// The environment variable that used to name the runtime file. Evidence +/// refuses to start while it is set, so a deployment that still sets it learns +/// the file is now named on the command line instead of silently reading a +/// different one. +pub const REMOVED_RUNTIME_ENVIRONMENT_VARIABLE: &str = "REGISTRY_EVIDENCE_RUNTIME"; #[derive(Debug, Parser)] #[command( @@ -13,24 +18,52 @@ const DEFAULT_RUNTIME_PATH: &str = "/etc/registry-evidence/runtime.yaml"; about = "Evidence Gateway Version 1" )] pub struct Cli { - /// One closed operator runtime file that binds the governed bundle. - #[arg( - long, - global = true, - env = "REGISTRY_EVIDENCE_RUNTIME", - default_value = DEFAULT_RUNTIME_PATH - )] - pub runtime: PathBuf, - #[command(subcommand)] pub command: Command, } +/// The removed runtime-file input an invocation still uses, as the refusal to +/// report, or `None`. +/// +/// `arguments` are the command-line arguments after the program name. They are +/// read before parsing, so an invocation written for the removed flag is +/// refused with its replacement named rather than with a missing-argument +/// error. `runtime_environment_set` is whether +/// [`REMOVED_RUNTIME_ENVIRONMENT_VARIABLE`] is set in the process environment. +pub fn removed_runtime_input( + arguments: I, + runtime_environment_set: bool, +) -> Option<&'static str> +where + I: IntoIterator, + S: AsRef, +{ + let uses_removed_flag = arguments + .into_iter() + .map_while(|argument| { + let argument = argument.as_ref().as_encoded_bytes().to_vec(); + (argument != b"--").then_some(argument) + }) + .any(|argument| argument == b"--runtime" || argument.starts_with(b"--runtime=")); + if uses_removed_flag { + return Some("--runtime is no longer accepted; pass --runtime-config FILE"); + } + if runtime_environment_set { + return Some( + "REGISTRY_EVIDENCE_RUNTIME is no longer read; unset it and pass --runtime-config FILE", + ); + } + None +} + #[derive(Debug, Subcommand)] pub enum Command { /// Validate and compile the complete immutable bundle, and validate the /// mounted secret material exactly as startup does. Check { + /// The closed operator runtime file that binds the governed bundle. + #[arg(long = "runtime-config", value_name = "FILE")] + runtime_config: PathBuf, /// Also prove audit writability, signer readiness, source credentials, /// and access-token JWKS reachability in the target runtime context. #[arg(long)] @@ -49,9 +82,21 @@ pub enum Command { requires = "require_runtime_dependencies" )] require_audit_under: Option, + /// Prove the audit destination without taking its single-writer lock, + /// for a candidate staged beside the running instance that holds it. + /// + /// Modes, write access, and a complete final entry in the active file + /// are still proved, and every other dependency is proved as without + /// this flag. A second writer is not detected, so `serve` still + /// refuses to start while one holds the lock. + #[arg(long, requires = "require_runtime_dependencies")] + without_audit_lock: bool, }, /// Evaluate one bundle-owned fixture without source or credential access. Evaluate { + /// The closed operator runtime file that binds the governed bundle. + #[arg(long = "runtime-config", value_name = "FILE")] + runtime_config: PathBuf, /// Bundle-relative fixture path referenced by exactly one requirement. #[arg(long)] fixture: PathBuf, @@ -108,7 +153,11 @@ pub enum Command { config: PathBuf, }, /// Start the native Evidence Gateway HTTP service. - Serve, + Serve { + /// The closed operator runtime file that binds the governed bundle. + #[arg(long = "runtime-config", value_name = "FILE")] + runtime_config: PathBuf, + }, /// Re-verify one stored signed response offline against a pinned key set. /// /// Exactly one stored response is named, and its format is named with it. @@ -164,13 +213,20 @@ pub enum Command { /// Internal local-adopter seam for bearer-free relying-procedure closure. #[command(hide = true)] PrepareLocalRelyingProcedure { + /// The closed operator runtime file that binds the governed bundle. + #[arg(long = "runtime-config", value_name = "FILE")] + runtime_config: PathBuf, /// Owner-only JSON draft containing the request shape and audience. #[arg(long)] input: PathBuf, }, /// Internal stopped-service audit inspection seam. #[command(hide = true)] - LocalAuditLastOperation, + LocalAuditLastOperation { + /// The closed operator runtime file that binds the governed bundle. + #[arg(long = "runtime-config", value_name = "FILE")] + runtime_config: PathBuf, + }, } /// Who the `--explain` trace is rendered for. @@ -199,19 +255,24 @@ mod tests { let parsed = Cli::try_parse_from([ "evidence", "check", + "--runtime-config", + "/etc/registry-evidence/runtime.yaml", "--require-runtime-dependencies", "--require-audit-under", "/var/lib/registry-evidence", ]) .expect("the audit root pairs with the dependency proof"); let Command::Check { + runtime_config: _, require_runtime_dependencies, require_audit_under, + without_audit_lock, } = parsed.command else { panic!("check parsed as another command"); }; assert!(require_runtime_dependencies); + assert!(!without_audit_lock); assert_eq!( Some(PathBuf::from("/var/lib/registry-evidence")), require_audit_under @@ -222,6 +283,8 @@ mod tests { assert!(Cli::try_parse_from([ "evidence", "check", + "--runtime-config", + "/etc/registry-evidence/runtime.yaml", "--require-audit-under", "/var/lib/registry-evidence", ]) @@ -246,4 +309,81 @@ mod tests { ); } } + + const RUNTIME_COMMANDS: [&str; 5] = [ + "check", + "evaluate", + "serve", + "prepare-local-relying-procedure", + "local-audit-last-operation", + ]; + + #[test] + fn every_runtime_command_names_its_runtime_configuration_explicitly() { + let command = command(); + for subcommand in RUNTIME_COMMANDS { + let found = command + .find_subcommand(subcommand) + .expect("subcommand exists"); + let runtime = found + .get_arguments() + .find(|argument| argument.get_id() == "runtime_config") + .expect("runtime configuration argument exists"); + assert_eq!(runtime.get_long(), Some("runtime-config"), "{subcommand}"); + assert_eq!(runtime.get_env(), None, "{subcommand}"); + assert!(runtime.get_default_values().is_empty(), "{subcommand}"); + assert!(runtime.is_required_set(), "{subcommand}"); + } + assert!(Cli::try_parse_from(["evidence", "serve"]).is_err()); + } + + #[test] + fn the_removed_runtime_flag_is_refused_with_its_replacement_named() { + const REFUSAL: Option<&str> = + Some("--runtime is no longer accepted; pass --runtime-config FILE"); + for arguments in [ + &["--runtime", "/etc/registry-evidence/runtime.yaml", "serve"][..], + &["serve", "--runtime", "/etc/registry-evidence/runtime.yaml"], + &["check", "--runtime=/etc/registry-evidence/runtime.yaml"], + ] { + assert_eq!( + removed_runtime_input(arguments, false), + REFUSAL, + "{arguments:?}" + ); + } + assert_eq!( + removed_runtime_input( + [ + "serve", + "--runtime-config", + "/etc/registry-evidence/runtime.yaml" + ], + false + ), + None + ); + // After the terminator an argument is a value, never a flag. + assert_eq!(removed_runtime_input(["--", "--runtime"], false), None); + assert!( + Cli::try_parse_from(["evidence", "--runtime", "/etc/runtime.yaml", "serve"]).is_err(), + "the removed flag is not an alias" + ); + } + + #[test] + fn the_removed_runtime_environment_variable_is_refused_with_its_replacement_named() { + let arguments = [ + "serve", + "--runtime-config", + "/etc/registry-evidence/runtime.yaml", + ]; + assert_eq!(removed_runtime_input(arguments, false), None); + assert_eq!( + removed_runtime_input(arguments, true), + Some( + "REGISTRY_EVIDENCE_RUNTIME is no longer read; unset it and pass --runtime-config FILE" + ) + ); + } } diff --git a/crates/registry-evidence/src/config.rs b/crates/registry-evidence/src/config.rs index 8b658a43b0..6d457f8a42 100644 --- a/crates/registry-evidence/src/config.rs +++ b/crates/registry-evidence/src/config.rs @@ -1,8 +1,8 @@ //! Typed Evidence Version 1 deployment configuration. //! //! Configuration is trusted deployment data, but it is still parsed as a -//! closed contract. Secret-bearing fields contain only [`SecretRef`] values; -//! this module never resolves them. +//! closed contract. Secret-bearing fields contain only [`SecretReference`] +//! values; this module never resolves them. use std::collections::{BTreeMap, BTreeSet}; use std::fmt; @@ -12,6 +12,12 @@ use std::str::FromStr; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use registry_platform_audit::{AuditDestination, AuditDestinationError, AuditDestinationKind}; +pub use registry_platform_config::SecretReference; +use registry_platform_config::{ + reject_environment_expressions_in_authored_yaml, AuditKeyConfig, JwksSource, ListenerBind, + LoadedRuntimeConfig, OidcIssuerConfig, PackageConfig, RemovedKey, RuntimeConfigError, + RuntimeConfigErrorKind, RuntimeConfigLoader, RuntimeEnvelope, SecretProvidersConfig, +}; use schemars::JsonSchema; use serde::de::{self, MapAccess, Visitor}; use serde::ser::SerializeMap; @@ -88,10 +94,38 @@ impl fmt::Display for TextLocation { /// for the field it bound, which is structure, not content. #[derive(Debug, Clone, Eq, PartialEq)] pub struct SchemaFault { - location: Option, - path: Option, + location: Option, + path: Option>, cause: &'static str, field: Option<&'static str>, + remedy: Option<&'static str>, +} + +/// A text position held in 32-bit fields, keeping `SchemaFault` small enough +/// to travel inside every error that wraps it. A configuration document is +/// bounded far below `u32::MAX` lines or columns; a larger value saturates. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct CompactLocation { + line: u32, + column: u32, +} + +impl From for CompactLocation { + fn from(location: TextLocation) -> Self { + Self { + line: u32::try_from(location.line).unwrap_or(u32::MAX), + column: u32::try_from(location.column).unwrap_or(u32::MAX), + } + } +} + +impl From for TextLocation { + fn from(location: CompactLocation) -> Self { + Self { + line: location.line as usize, + column: location.column as usize, + } + } } /// The longest schema path a diagnostic will carry. @@ -141,6 +175,7 @@ impl SchemaFault { path: None, cause, field: None, + remedy: None, } } @@ -161,7 +196,7 @@ impl SchemaFault { /// A line and a column are structure, so they are safe to keep. The text /// standing at that position is content, and this type never sees it. pub fn at(mut self, location: TextLocation) -> Self { - self.location = Some(location); + self.location = Some(location.into()); self } @@ -180,7 +215,27 @@ impl SchemaFault { } pub fn location(&self) -> Option { - self.location + self.location.map(TextLocation::from) + } + + /// The fixed sentence telling an operator what to write instead, when the + /// refusal has one. + pub fn remedy(&self) -> Option<&'static str> { + self.remedy + } + + /// The same fault, pointing at a schema path. The path is kept only when + /// it matches the structural path grammar. + fn at_path(mut self, path: &str) -> Self { + if is_safe_schema_path(path) { + self.path = Some(path.into()); + } + self + } + + fn with_remedy(mut self, remedy: &'static str) -> Self { + self.remedy = Some(remedy); + self } /// Reduce a decoder error to a location, a safe schema path, and a cause. @@ -188,13 +243,17 @@ impl SchemaFault { let rendered = error.to_string(); let (path, message) = split_schema_path(&rendered); Self { - location: error.location().map(|location| TextLocation { - line: location.line(), - column: location.column(), + location: error.location().map(|location| { + TextLocation { + line: location.line(), + column: location.column(), + } + .into() }), - path, + path: path.map(String::into_boxed_str), cause: classify_decode_cause(message, fallback), field: None, + remedy: None, } } } @@ -209,7 +268,10 @@ impl fmt::Display for SchemaFault { write!(formatter, " at {path}")?; } if let Some(location) = self.location { - write!(formatter, " ({location})")?; + write!(formatter, " ({})", TextLocation::from(location))?; + } + if let Some(remedy) = self.remedy { + write!(formatter, "; {remedy}")?; } Ok(()) } @@ -285,6 +347,220 @@ fn decode_yaml(text: &str) -> Result", + ), + ( + "authentication.tokenTypes", + "declare authentication.oidc.tokenTypes instead", + ), + ( + "authentication.algorithms", + "declare authentication.oidc.algorithms instead", + ), + ( + "authentication.principalClaim", + "declare authentication.oidc.principalClaim instead", + ), + ( + "authentication.requesterTagsClaim", + "declare authentication.oidc.requesterTagsClaim instead", + ), + ( + "authentication.evidenceAudienceClaim", + "declare authentication.oidc.evidenceAudienceClaim instead", + ), + ( + "authentication.claims", + "declare authentication.oidc.claims instead", + ), + ( + "authentication.maximumTokenLifetimeSeconds", + "declare authentication.oidc.maximumTokenLifetimeSeconds instead", + ), + ( + "authentication.revokedKeyIds", + "declare authentication.oidc.revokedKeyIds instead", + ), + ( + "authentication.allowedClients", + "declare authentication.oidc.allowedClients instead", + ), + ( + "authentication.assertionIssuers", + "declare authentication.oidc.assertionIssuers instead", + ), + ( + "authentication.requiredScopes", + "declare authentication.oidc.requiredScopes instead", + ), + ( + "authentication.actorClaim", + "declare authentication.oidc.actorClaim instead", + ), + ( + "authentication.tlsTrustProfile", + "declare authentication.oidc.tlsTrustProfile instead", + ), + ( + "authentication.oidc.kind", + "Evidence accepts OIDC access tokens only; remove kind", + ), + ( + "authentication.oidc.audiences", + "declare the one accepted audience as authentication.oidc.audience", + ), + ( + "authentication.oidc.jwksUri", + "declare authentication.oidc.jwksSource with kind: uri and uri: ", + ), + ("audit.hashSecretRef", "declare audit.hashKeyRef instead"), +]; + +/// Refuse a bundle key an earlier grammar accepted, naming its replacement. +/// +/// A document that is not a YAML mapping is left to the closed decoder, which +/// reports it with a location. +fn reject_removed_bundle_keys(text: &str) -> Result<(), ConfigError> { + let Ok(YamlValue::Mapping(document)) = serde_norway::from_str::(text) else { + return Ok(()); + }; + for (path, replacement) in REMOVED_BUNDLE_KEYS { + let mut node = Some(&document); + let mut segments = path.split('.').peekable(); + while let (Some(mapping), Some(segment)) = (node, segments.next()) { + let Some(value) = mapping.get(segment) else { + break; + }; + if segments.peek().is_none() { + return Err(ConfigError::InvalidYaml( + SchemaFault::because("key is no longer accepted") + .at_path(path) + .with_remedy(replacement), + )); + } + node = value.as_mapping(); + } + } + Ok(()) +} + +/// Refuse a `${...}` expression anywhere in the governed bundle. +/// +/// Substitution applies to `runtime.yaml` only, so the reviewed bundle is the +/// one that runs. The fault names the field and never the expression. A +/// document that is not YAML is left to the closed decoder. +fn reject_bundle_environment_expressions(text: &str) -> Result<(), ConfigError> { + match reject_environment_expressions_in_authored_yaml(text) { + Err(error) if error.kind() == RuntimeConfigErrorKind::AuthoredExpression => { + let fault = SchemaFault::because( + "environment expressions are not accepted in the governed bundle", + ) + .with_remedy("write the value in the bundle directly"); + Err(ConfigError::InvalidYaml(if error.field() == "/" { + fault + } else { + fault.at_path(error.field()) + })) + } + _ => Ok(()), + } +} + +/// Reduce a shared-loader refusal of the runtime file to a value-free fault. +/// +/// The loader's message is read only for its fixed leading text and then +/// discarded, like a decoder message: the cause is static, the path is kept +/// only when it matches the structural path grammar, and a removed key or a +/// wrong envelope carries the fixed sentence naming what to write instead. +fn runtime_fault(error: &RuntimeConfigError) -> SchemaFault { + let field = error.field(); + let fault = match error.kind() { + RuntimeConfigErrorKind::RemovedKey => { + let remedy = EVIDENCE_RUNTIME_REMOVED_KEYS + .iter() + .find(|removed| removed.path == field) + .map_or(EVIDENCE_RUNTIME_ENVELOPE_REMEDY, |removed| { + removed.replacement + }); + SchemaFault::because("key is no longer accepted").with_remedy(remedy) + } + RuntimeConfigErrorKind::Envelope => { + SchemaFault::because("document does not declare the Evidence runtime envelope") + .with_remedy(EVIDENCE_RUNTIME_ENVELOPE_REMEDY) + } + RuntimeConfigErrorKind::Syntax => { + let message = error.message(); + let cause = match message.split_once("is not valid YAML: ") { + Some((_, decoder)) => { + classify_decode_cause(decoder, "document is not well-formed YAML") + } + None if message.ends_with("must be a YAML mapping") => { + "document is not a YAML mapping" + } + None if message.contains("key that is not a string") => { + "mapping key is not a string" + } + None if message.contains("YAML tag") => "document carries a YAML tag", + None => "document is not well-formed YAML", + }; + SchemaFault::because(cause) + } + RuntimeConfigErrorKind::Substitution => { + SchemaFault::because("environment expression cannot be substituted") + } + RuntimeConfigErrorKind::SubstitutionInReference => SchemaFault::because( + "environment expressions are not accepted in secret references or secretProviders", + ), + RuntimeConfigErrorKind::InvalidValue => { + let reason = error + .message() + .split_once(" is invalid: ") + .map_or("", |(_, reason)| reason); + let cause = RUNTIME_VALUE_CAUSES + .iter() + .find(|(prefix, _)| reason.starts_with(prefix)) + .map_or_else( + || classify_decode_cause(reason, "document does not match the closed schema"), + |(_, cause)| cause, + ); + SchemaFault::because(cause) + } + RuntimeConfigErrorKind::Bounds => { + SchemaFault::because("document exceeds the Version 1 size limit") + } + RuntimeConfigErrorKind::Encoding => SchemaFault::because("document is not UTF-8"), + RuntimeConfigErrorKind::Path + | RuntimeConfigErrorKind::UnsafeFile + | RuntimeConfigErrorKind::Unavailable + | RuntimeConfigErrorKind::AuthoredExpression + | RuntimeConfigErrorKind::AuthoredSyntax => { + SchemaFault::because("document could not be read as a runtime configuration") + } + }; + if field == "/" { + fault + } else { + fault.at_path(field) + } +} + /// A mapping that rejects duplicate keys and preserves declaration order. /// /// Selector declaration order is part of canonical selector encoding, so a @@ -516,6 +792,8 @@ impl EvidenceConfig { } let text = std::str::from_utf8(bytes) .map_err(|_| ConfigError::InvalidYaml(SchemaFault::because("document is not UTF-8")))?; + reject_removed_bundle_keys(text)?; + reject_bundle_environment_expressions(text)?; let config: Self = decode_yaml(text)?; config.validate()?; Ok(config) @@ -535,7 +813,7 @@ impl EvidenceConfig { self.authentication.validate(self.assurance_profile)?; self.audit.validate()?; self.subject_binding.validate()?; - if self.audit.hash_key_ref == self.subject_binding.secret_ref { + if self.audit.key.hash_key_ref == self.subject_binding.secret_ref { return invalid("audit and subject-binding secret references must be distinct"); } self.rate_limits.validate()?; @@ -586,6 +864,7 @@ impl EvidenceConfig { if !task_grant_profiles.is_empty() { let allowed_clients = self.authentication + .oidc .allowed_clients .as_ref() .ok_or(ConfigError::Invalid( @@ -1240,17 +1519,78 @@ fn validate_publication_identifier(value: &str) -> Result<(), ConfigError> { Ok(()) } +/// The `apiVersion` every Evidence runtime file declares. +pub const EVIDENCE_RUNTIME_API_VERSION: &str = + "registry.registrystack.org/evidence-runtime/v1alpha1"; +/// The `kind` every Evidence runtime file declares. +pub const EVIDENCE_RUNTIME_KIND: &str = "EvidenceRuntimeConfig"; + +/// The envelope every Evidence runtime file carries. +pub const EVIDENCE_RUNTIME_ENVELOPE: RuntimeEnvelope = RuntimeEnvelope { + api_version: EVIDENCE_RUNTIME_API_VERSION, + kind: EVIDENCE_RUNTIME_KIND, +}; + +/// What an operator writes when the envelope is missing or wrong. +const EVIDENCE_RUNTIME_ENVELOPE_REMEDY: &str = "declare apiVersion: \ + registry.registrystack.org/evidence-runtime/v1alpha1 and kind: EvidenceRuntimeConfig"; + +/// Keys an earlier Evidence runtime file accepted, each refused with the key +/// that replaced it. +pub const EVIDENCE_RUNTIME_REMOVED_KEYS: &[RemovedKey] = &[ + RemovedKey { + path: "version", + replacement: "declare apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 \ + and kind: EvidenceRuntimeConfig", + }, + RemovedKey { + path: "bundleDirectory", + replacement: "declare package.root as the absolute path of the package directory", + }, + RemovedKey { + path: "listener.bindHost", + replacement: "declare listener.bind as host:port, such as 127.0.0.1:8080", + }, + RemovedKey { + path: "listener.port", + replacement: "declare listener.bind as host:port, such as 127.0.0.1:8080", + }, + RemovedKey { + path: "metricsListener.bindHost", + replacement: "declare metricsListener.bind as host:port, such as 127.0.0.1:9090", + }, + RemovedKey { + path: "metricsListener.port", + replacement: "declare metricsListener.bind as host:port, such as 127.0.0.1:9090", + }, +]; + +/// Runtime refusals the shared loader reports in its own words, each mapped to +/// one value-free cause. Only the fixed leading text is read. +const RUNTIME_VALUE_CAUSES: [(&str, &str); 2] = [ + ( + "listener.bind must be host:port", + "listener bind must be host:port with an IP address host", + ), + ( + "expected an exact secret:env/NAME or secret:file/name reference", + "secret reference does not use an exact permitted grammar", + ), +]; + #[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct RuntimeConfig { - pub version: u8, - pub bundle_directory: String, + pub api_version: String, + pub kind: String, + /// The one governed package directory this process verifies and loads at startup. + pub package: PackageConfig, pub listener: ListenerConfig, /// Optional operator-only metrics listener. Absent means the deployment /// serves no metrics endpoint at all, which is the default posture. #[serde(default, skip_serializing_if = "Option::is_none")] pub metrics_listener: Option, - pub secret_providers: RuntimeSecretProviders, + pub secret_providers: SecretProvidersConfig, /// Process-local binding to the signer that controls the governed active /// public key. This cannot change the governed key set or algorithm. pub signer: RuntimeSignerConfig, @@ -1258,43 +1598,78 @@ pub struct RuntimeConfig { pub audit: RuntimeAuditConfig, pub outbound_tls: OutboundTlsConfig, /// Process-local files bound to the logical extract names the bundle's - /// statement sources read. Absent binds none, which is what every runtime - /// file written before an extract source existed says. + /// statement sources read. Absent binds none, which is what a runtime file + /// for a bundle with no extract source says. #[serde(default, skip_serializing_if = "OrderedMap::is_empty")] pub source_extracts: OrderedMap, /// Acquisition kinds this deployment enables beyond the frozen Version 1 - /// forms. Absent enables none of them, which is what every runtime file - /// written before a gated form existed says, so adopting a form is a - /// deliberate operator decision rather than a consequence of the bundle - /// that arrived. A bundle requiring a kind absent here is refused before - /// the deployment serves anything. + /// forms. Absent enables none of them, so adopting a form is a deliberate + /// operator decision rather than a consequence of the bundle that arrived. + /// A bundle requiring a kind absent here is refused before the deployment + /// serves anything. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub acquisition_capabilities: Vec, } impl RuntimeConfig { + /// The shared loader configured with the Evidence envelope and the keys + /// it no longer accepts. + pub fn loader() -> RuntimeConfigLoader { + RuntimeConfigLoader::new(EVIDENCE_RUNTIME_ENVELOPE) + .removed_keys(EVIDENCE_RUNTIME_REMOVED_KEYS) + .max_bytes(MAX_CONFIG_BYTES as u64) + } + + /// Parse and validate one runtime document with no environment: an + /// environment expression resolves only through its own default. pub fn parse_yaml(bytes: &[u8]) -> Result { + Self::parse_yaml_with(bytes, |_| None).map(|loaded| loaded.config) + } + + /// Parse and validate one runtime document, substituting environment + /// expressions in string values from `lookup`. The returned digest covers + /// the document after substitution. + pub fn parse_yaml_with( + bytes: &[u8], + lookup: impl Fn(&str) -> Option, + ) -> Result, ConfigError> { if bytes.len() > MAX_CONFIG_BYTES { return Err(ConfigError::TooLarge); } let text = std::str::from_utf8(bytes) .map_err(|_| ConfigError::InvalidYaml(SchemaFault::because("document is not UTF-8")))?; - let config: Self = decode_yaml(text)?; - config.validate()?; - Ok(config) + let loaded = Self::loader() + .parse_str::(text, lookup) + .map_err(|error| ConfigError::InvalidYaml(runtime_fault(&error)))?; + loaded.config.validate()?; + Ok(loaded) } pub fn validate(&self) -> Result<(), ConfigError> { - if self.version != 1 { - return invalid("runtime version must equal 1"); - } - validate_absolute_path(&self.bundle_directory)?; + self.package.check().map_err(|error| match error.kind() { + registry_platform_config::ConfigBlockErrorKind::InvalidDigest => { + ConfigError::InvalidField( + "package expectedDigest must be sha256: followed by 64 lowercase hex digits", + "package.expectedDigest", + ) + } + _ => ConfigError::InvalidField("package root must be an absolute path", "package.root"), + })?; + validate_absolute_path(&self.package.root.to_string_lossy())?; self.listener.validate()?; if let Some(metrics) = &self.metrics_listener { metrics.validate(&self.listener)?; } - self.secret_providers.validate()?; - self.signer.validate()?; + self.secret_providers.check().map_err(|_| { + ConfigError::InvalidField( + "secretProviders must enable file, environment, or both, and a file root must be absolute", + "secretProviders", + ) + })?; + if let Some(file) = &self.secret_providers.file { + validate_absolute_path(&file.root.to_string_lossy())?; + } + self.signer.validate(&self.secret_providers)?; self.audit.validate()?; self.outbound_tls.validate()?; validate_named_map(&self.source_extracts, 0, 64, SourceExtractBinding::validate)?; @@ -1324,7 +1699,7 @@ impl RuntimeConfig { pub enum RuntimeSignerConfig { LocalJwk { #[serde(rename = "privateKeyRef")] - private_key_ref: SecretRef, + private_key_ref: SecretReference, }, Transit { #[serde(rename = "unixSocketPath")] @@ -1340,9 +1715,16 @@ pub enum RuntimeSignerConfig { } impl RuntimeSignerConfig { - fn validate(&self) -> Result<(), ConfigError> { + fn validate(&self, secret_providers: &SecretProvidersConfig) -> Result<(), ConfigError> { match self { - Self::LocalJwk { .. } => Ok(()), + Self::LocalJwk { private_key_ref } => secret_providers + .check_reference("signer.privateKeyRef", private_key_ref.as_str()) + .map_err(|_| { + ConfigError::InvalidField( + "the secret reference names a provider secretProviders does not enable", + "signer.privateKeyRef", + ) + }), Self::Transit { unix_socket_path, mount, @@ -1375,7 +1757,7 @@ impl RuntimeSignerConfig { matches!(self, Self::Transit { .. }) } - pub fn private_key_ref(&self) -> Option<&SecretRef> { + pub fn private_key_ref(&self) -> Option<&SecretReference> { match self { Self::LocalJwk { private_key_ref } => Some(private_key_ref), Self::Transit { .. } => None, @@ -1383,30 +1765,6 @@ impl RuntimeSignerConfig { } } -#[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct RuntimeSecretProviders { - pub file: FileSecretProvider, -} - -impl RuntimeSecretProviders { - fn validate(&self) -> Result<(), ConfigError> { - self.file.validate() - } -} - -#[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -pub struct FileSecretProvider { - pub root: String, -} - -impl FileSecretProvider { - fn validate(&self) -> Result<(), ConfigError> { - validate_absolute_path(&self.root) - } -} - /// The process-local audit destination: an append-only JSON Lines file this /// process alone writes, or standard output for a collector that owns /// durability. @@ -1509,10 +1867,10 @@ impl SourceExtractBinding { #[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct ListenerConfig { - pub bind_host: String, + /// Numeric `host:port` the evidence API binds. + pub bind: ListenerBind, #[serde(default)] pub network_exposure: ListenerNetworkExposure, - pub port: u16, pub tls_termination: TlsTermination, pub trust_proxy_identity_headers: bool, pub maximum_request_bytes: u64, @@ -1525,13 +1883,13 @@ impl ListenerConfig { fn validate(&self) -> Result<(), ConfigError> { match self.network_exposure { ListenerNetworkExposure::PrivateAddress => { - validate_private_bind_host(&self.bind_host)?; + validate_private_bind_host(self.bind.ip())?; } ListenerNetworkExposure::ContainerPrivate => { - validate_container_private_bind_host(&self.bind_host)?; + validate_container_private_bind_host(self.bind.ip())?; } } - validate_listener_port(self.port)?; + validate_listener_port(self.bind.socket_addr().port())?; if self.trust_proxy_identity_headers { return invalid("proxy identity headers must not be trusted"); } @@ -1584,25 +1942,19 @@ pub enum ListenerNetworkExposure { #[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct MetricsListenerConfig { - pub bind_host: String, - pub port: u16, + /// Numeric `host:port` the metrics endpoint binds. + pub bind: ListenerBind, } impl MetricsListenerConfig { fn validate(&self, evidence_listener: &ListenerConfig) -> Result<(), ConfigError> { - validate_private_bind_host(&self.bind_host)?; - validate_listener_port(self.port)?; + validate_private_bind_host(self.bind.ip())?; + validate_listener_port(self.bind.socket_addr().port())?; // Sharing the evidence binding would publish the counters on the // listener the public contract describes, which is the separation this // block exists to enforce. - let metrics_ip: IpAddr = self - .bind_host - .parse() - .expect("validated metrics listener address is numeric"); - let evidence_ip: IpAddr = evidence_listener - .bind_host - .parse() - .expect("validated evidence listener address is numeric"); + let metrics_ip = self.bind.ip(); + let evidence_ip = evidence_listener.bind.ip(); // Linux commonly creates an IPv6 wildcard socket as dual-stack, so // `[::]:port` can also occupy the corresponding IPv4 port. Evidence // does not force IPV6_V6ONLY, and configuration validation must be @@ -1614,7 +1966,9 @@ impl MetricsListenerConfig { IpAddr::V6(ip) => ip.is_unspecified(), }; let binding_overlaps = metrics_ip == evidence_ip || wildcard_covers_metrics; - if binding_overlaps && self.port == evidence_listener.port { + if binding_overlaps + && self.bind.socket_addr().port() == evidence_listener.bind.socket_addr().port() + { return invalid("metricsListener must not share the evidence listener binding"); } Ok(()) @@ -1636,39 +1990,27 @@ fn validate_listener_port(port: u16) -> Result<(), ConfigError> { /// Accept only numeric loopback, RFC 1918 private IPv4, and RFC 4193 /// unique-local IPv6 bindings. Every listener this service opens is an /// operator-network listener; TLS and exposure are upstream concerns. -fn validate_private_bind_host(bind_host: &str) -> Result<(), ConfigError> { - if bind_host.len() < 2 || bind_host.len() > 64 { - return invalid("listener bindHost length is invalid"); - } - let ip: IpAddr = bind_host - .parse() - .map_err(|_| ConfigError::Invalid("listener bindHost must be a private numeric IP"))?; +fn validate_private_bind_host(ip: IpAddr) -> Result<(), ConfigError> { let private = match ip { IpAddr::V4(ip) => ip.is_loopback() || ip.is_private(), IpAddr::V6(ip) => ip.is_loopback() || is_unique_local(ip), }; if !private || ip.is_unspecified() || ip.is_multicast() { - return invalid("listener bindHost must be loopback or private"); + return invalid("listener bind address must be loopback or private"); } Ok(()) } /// Accept a wildcard or private numeric address only when the operator has /// explicitly placed the listener on a container-private network. -fn validate_container_private_bind_host(bind_host: &str) -> Result<(), ConfigError> { - if bind_host.len() < 2 || bind_host.len() > 64 { - return invalid("listener bindHost length is invalid"); - } - let ip: IpAddr = bind_host - .parse() - .map_err(|_| ConfigError::Invalid("listener bindHost must be a numeric IP"))?; +fn validate_container_private_bind_host(ip: IpAddr) -> Result<(), ConfigError> { let private_or_unspecified = match ip { IpAddr::V4(ip) => ip.is_unspecified() || ip.is_loopback() || ip.is_private(), IpAddr::V6(ip) => ip.is_unspecified() || ip.is_loopback() || is_unique_local(ip), }; if !private_or_unspecified || ip.is_multicast() { return invalid( - "container-private listener bindHost must be unspecified, loopback, or private", + "container-private listener bind address must be unspecified, loopback, or private", ); } Ok(()) @@ -1684,15 +2026,29 @@ pub enum TlsTermination { OperatorControlledUpstream, } +/// The governed access-token rules. Evidence accepts OIDC access tokens only, +/// so the block holds exactly one member. #[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct AuthenticationConfig { - pub kind: AuthenticationKind, - pub issuer: String, - pub audiences: Vec, + pub oidc: OidcAuthenticationConfig, +} + +impl AuthenticationConfig { + fn validate(&self, assurance_profile: AssuranceProfile) -> Result<(), ConfigError> { + self.oidc.validate(assurance_profile) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct OidcAuthenticationConfig { + /// The exact issuer, the one audience every token carries, and the JWKS + /// source. Evidence reads keys only from an explicit `uri` source. + #[serde(flatten)] + pub provider: OidcIssuerConfig, pub token_types: Vec, pub algorithms: Vec, - pub jwks_uri: String, pub principal_claim: String, pub requester_tags_claim: String, pub evidence_audience_claim: String, @@ -1738,13 +2094,41 @@ pub struct AuthenticationConfig { pub required_scopes: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub actor_claim: Option, + /// Logical name of the private certificate authority the runtime file + /// binds for the JWKS connection, trusted beside the system roots + /// for that connection alone. Absent trusts the system roots only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tls_trust_profile: Option, } -impl AuthenticationConfig { +impl OidcAuthenticationConfig { + /// The exact issuer accepted in `iss`. + pub fn issuer(&self) -> &str { + &self.provider.issuer + } + + /// The one audience every accepted token carries in `aud`. + pub fn audience(&self) -> &str { + &self.provider.audience + } + + /// The JWKS URI. Validation refuses every source other than `uri`, so a + /// validated configuration always has one; an unvalidated one answers + /// with an empty string, which no URL parser accepts. + pub fn jwks_uri(&self) -> &str { + self.provider.jwks_source.uri().unwrap_or("") + } + fn validate(&self, assurance_profile: AssuranceProfile) -> Result<(), ConfigError> { - let issuer = Url::parse(&self.issuer) + let JwksSource::Uri { .. } = &self.provider.jwks_source else { + return Err(ConfigError::InvalidField( + "Evidence reads access-token keys only from jwksSource kind uri", + "authentication.oidc.jwksSource", + )); + }; + let issuer = Url::parse(self.issuer()) .map_err(|_| ConfigError::Invalid("authentication issuer is invalid"))?; - let jwks_uri = Url::parse(&self.jwks_uri) + let jwks_uri = Url::parse(self.jwks_uri()) .map_err(|_| ConfigError::Invalid("authentication JWKS URI is invalid"))?; if issuer.scheme() == "http" || jwks_uri.scheme() == "http" { if assurance_profile != AssuranceProfile::Local { @@ -1755,7 +2139,7 @@ impl AuthenticationConfig { // JWKS served from that exact origin. It is not a permission for // private-network HTTP, and a JWKS origin or port other than the // issuer's is not supervised by the issuer that vouches for it. - let origin = validate_local_issuer_origin(&self.issuer)?; + let origin = validate_local_issuer_origin(self.issuer())?; if jwks_uri.scheme() != "http" || jwks_uri.host_str() != Some("127.0.0.1") || jwks_uri.port() != issuer.port() @@ -1764,17 +2148,39 @@ impl AuthenticationConfig { || jwks_uri.fragment().is_some() || !jwks_uri.username().is_empty() || jwks_uri.password().is_some() - || self.jwks_uri != format!("{origin}{}", jwks_uri.path()) + || self.jwks_uri() != format!("{origin}{}", jwks_uri.path()) { return invalid( "local authentication JWKS URI must use the exact issuer origin, an absolute path, and no query or fragment", ); } + if self.tls_trust_profile.is_some() { + return invalid( + "a local HTTP authentication JWKS URI cannot use a TLS trust profile", + ); + } } else { - validate_https_issuer(&self.issuer)?; - validate_https_url(&self.jwks_uri, false)?; + validate_https_issuer(self.issuer())?; + validate_https_url(self.jwks_uri(), false)?; + } + self.provider + .check( + "authentication.oidc", + assurance_profile == AssuranceProfile::Local, + ) + .map_err(|_| { + ConfigError::InvalidField( + "the OIDC issuer, audience, or JWKS source is invalid", + "authentication.oidc", + ) + })?; + if self + .tls_trust_profile + .as_deref() + .is_some_and(|profile| !valid_local_id(profile)) + { + return invalid("authentication TLS trust profile identifier is invalid"); } - validate_unique_strings(&self.audiences, 1, 16, 1, 512, "authentication audiences")?; validate_unique(&self.token_types, 1, 4, "authentication tokenTypes")?; validate_unique(&self.algorithms, 1, 3, "authentication algorithms")?; validate_range( @@ -1885,15 +2291,15 @@ impl AuthenticationConfig { if assurance_profile != AssuranceProfile::Local { return false; } - let Ok(issuer) = Url::parse(&self.issuer) else { + let Ok(issuer) = Url::parse(self.issuer()) else { return false; }; - let Ok(jwks_uri) = Url::parse(&self.jwks_uri) else { + let Ok(jwks_uri) = Url::parse(self.jwks_uri()) else { return false; }; issuer.scheme() == "http" && jwks_uri.scheme() == "http" - && validate_local_issuer_origin(&self.issuer).is_ok() + && validate_local_issuer_origin(self.issuer()).is_ok() && jwks_uri.host_str() == Some("127.0.0.1") && jwks_uri.port() == issuer.port() && jwks_uri.path().starts_with('/') @@ -1936,12 +2342,6 @@ fn validate_local_issuer_origin(value: &str) -> Result<&str, ConfigError> { Ok(value) } -#[derive(Debug, Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case")] -pub enum AuthenticationKind { - OidcAccessToken, -} - #[derive(Debug, Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Deserialize, Serialize)] pub enum AccessTokenType { #[serde(rename = "at+jwt")] @@ -1957,73 +2357,14 @@ pub enum AccessTokenAlgorithm { RS256, } -#[derive(Debug, Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Deserialize, Serialize)] -#[serde(rename_all = "lowercase")] -pub enum SecretProvider { - Environment, - File, -} - -#[derive(Clone, Eq, Ord, PartialEq, PartialOrd, Hash)] -pub struct SecretRef(String); - -impl SecretRef { - pub fn parse(value: &str) -> Result { - if let Some(name) = value.strip_prefix("secret:file/") { - if valid_file_secret_name(name) { - return Ok(Self(value.to_owned())); - } - } - invalid("secret reference does not use an exact permitted grammar") - } - - pub fn as_str(&self) -> &str { - &self.0 - } - - pub fn provider(&self) -> SecretProvider { - SecretProvider::File - } -} - -impl fmt::Debug for SecretRef { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.debug_tuple("SecretRef").field(&self.0).finish() - } -} - -impl<'de> Deserialize<'de> for SecretRef { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let value = String::deserialize(deserializer)?; - Self::parse(&value).map_err(de::Error::custom) - } -} - -impl Serialize for SecretRef { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.0) - } -} - -fn valid_file_secret_name(name: &str) -> bool { - let bytes = name.as_bytes(); - matches!(bytes.first(), Some(b'a'..=b'z')) - && bytes.len() <= 128 - && bytes[1..].iter().all(|byte| { - byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'.' | b'_' | b'-') - }) -} - #[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct AuditConfig { - pub hash_key_ref: SecretRef, + /// The secret keying audit pseudonyms. + #[serde(flatten)] + pub key: AuditKeyConfig, + /// Labels the pseudonym key generation, so a rotated key is told apart + /// from the one it replaced. pub hash_key_version: u32, } @@ -2039,7 +2380,7 @@ impl AuditConfig { #[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct SubjectBindingConfig { - pub secret_ref: SecretRef, + pub secret_ref: SecretReference, pub key_version: u32, } @@ -2639,6 +2980,15 @@ impl SourceConfig { } } + /// The credentials an inline HTTP source presents, where the transport + /// opens a connection of its own. + pub fn authentication(&self) -> Option<&SourceAuthentication> { + match self { + Self::HttpJson { authentication, .. } => Some(authentication), + Self::SqliteExtract { .. } => None, + } + } + /// The private trust profile an outbound connection uses, where the /// transport opens one. pub fn tls_trust_profile(&self) -> Option<&str> { @@ -2922,13 +3272,13 @@ pub enum SourceAuthentication { None {}, Basic { #[serde(rename = "usernameRef")] - username_ref: SecretRef, + username_ref: SecretReference, #[serde(rename = "passwordRef")] - password_ref: SecretRef, + password_ref: SecretReference, }, StaticAuthorization { #[serde(rename = "tokenRef")] - token_ref: SecretRef, + token_ref: SecretReference, /// Authentication scheme the resolved token is presented under. /// /// RFC 9110 section 11.1 lets the origin choose the scheme, and @@ -2942,13 +3292,13 @@ pub enum SourceAuthentication { #[serde(rename = "headerName")] header_name: String, #[serde(rename = "valueRef")] - value_ref: SecretRef, + value_ref: SecretReference, }, Oauth2ClientCredentials { #[serde(rename = "tokenEndpoint")] token_endpoint: String, #[serde(rename = "clientIdRef")] - client_id_ref: SecretRef, + client_id_ref: SecretReference, /// Shared client secret, for the RFC 6749 section 2.3.1 form. /// /// Present with `credentialPlacement` and without @@ -2958,7 +3308,7 @@ pub enum SourceAuthentication { default, skip_serializing_if = "Option::is_none" )] - client_secret_ref: Option, + client_secret_ref: Option, /// Private JWK the client assertion is signed with, for the RFC 7523 /// section 2.2 form. /// @@ -2969,7 +3319,7 @@ pub enum SourceAuthentication { default, skip_serializing_if = "Option::is_none" )] - client_assertion_key_ref: Option, + client_assertion_key_ref: Option, /// Audience claim of the signed assertion; set only with /// `clientAssertionKeyRef`, and defaulting to `tokenEndpoint`. /// @@ -3127,7 +3477,7 @@ impl SourceAuthentication { } } - pub fn secret_refs(&self) -> Vec<&SecretRef> { + pub fn secret_refs(&self) -> Vec<&SecretReference> { match self { Self::None {} => Vec::new(), Self::Basic { @@ -5860,7 +6210,8 @@ mod tests { match field { "authentication" => { **authentication = SourceAuthentication::StaticAuthorization { - token_ref: SecretRef::parse("secret:file/different-token").unwrap(), + token_ref: SecretReference::parse("secret:file/different-token") + .unwrap(), scheme: None, } } @@ -6086,13 +6437,19 @@ mod tests { )) .expect("strict fixture validates"); config.assurance_profile = AssuranceProfile::Local; - config.authentication.issuer = "http://127.0.0.1:8081".to_owned(); - config.authentication.jwks_uri = "http://127.0.0.1:8081/.well-known/jwks.json".to_owned(); + config.authentication.oidc.provider.issuer = "http://127.0.0.1:8081".to_owned(); + config.authentication.oidc.provider.jwks_source = + registry_platform_config::JwksSource::Uri { + uri: "http://127.0.0.1:8081/.well-known/jwks.json".to_owned(), + }; config .validate() .expect("local profile accepts the supervised loopback identity"); // The JWKS path is the configured issuer's to choose. - config.authentication.jwks_uri = "http://127.0.0.1:8081/oauth2/jwks".to_owned(); + config.authentication.oidc.provider.jwks_source = + registry_platform_config::JwksSource::Uri { + uri: "http://127.0.0.1:8081/oauth2/jwks".to_owned(), + }; config .validate() .expect("local profile accepts any same-origin absolute JWKS path"); @@ -6108,7 +6465,7 @@ mod tests { "http://127.0.0.1:8081/", ] { let mut candidate = config.clone(); - candidate.authentication.issuer = invalid.to_owned(); + candidate.authentication.oidc.provider.issuer = invalid.to_owned(); assert!( candidate.validate().is_err(), "local assurance accepted issuer {invalid}" @@ -6128,7 +6485,10 @@ mod tests { "http://user@127.0.0.1:8081/oauth2/jwks", ] { let mut candidate = config.clone(); - candidate.authentication.jwks_uri = invalid.to_owned(); + candidate.authentication.oidc.provider.jwks_source = + registry_platform_config::JwksSource::Uri { + uri: invalid.to_owned(), + }; assert!( candidate.validate().is_err(), "local assurance accepted JWKS URI {invalid}" @@ -6148,6 +6508,49 @@ mod tests { } } + /// The issuer's trust profile is a logical id the runtime file binds, and + /// it has nothing to trust on the supervised local HTTP issuer, where no + /// certificate is presented. + #[test] + fn an_issuer_trust_profile_is_a_local_id_for_an_https_key_set_only() { + let mut config = EvidenceConfig::parse_yaml(include_bytes!( + "../../../products/evidence/fixtures/acceptance/adult-status/evidence.yaml" + )) + .expect("strict fixture validates"); + config.authentication.oidc.tls_trust_profile = Some("issuer-pki".to_owned()); + config + .validate() + .expect("an HTTPS key set accepts a named trust profile"); + + for invalid in ["", "Issuer-PKI", "issuer pki", "../issuer"] { + let mut candidate = config.clone(); + candidate.authentication.oidc.tls_trust_profile = Some(invalid.to_owned()); + assert!( + matches!( + candidate.validate(), + Err(ConfigError::Invalid( + "authentication TLS trust profile identifier is invalid" + )) + ), + "accepted trust profile {invalid:?}" + ); + } + + let mut local = config.clone(); + local.assurance_profile = AssuranceProfile::Local; + local.authentication.oidc.provider.issuer = "http://127.0.0.1:8081".to_owned(); + local.authentication.oidc.provider.jwks_source = + registry_platform_config::JwksSource::Uri { + uri: "http://127.0.0.1:8081/.well-known/jwks.json".to_owned(), + }; + assert!(matches!( + local.validate(), + Err(ConfigError::Invalid( + "a local HTTP authentication JWKS URI cannot use a TLS trust profile" + )) + )); + } + /// The admission fields are optional, but a present list must be a /// nonempty, bounded, unique list — and required scopes must be scope /// tokens, because they are compared against verified token scopes. @@ -6161,8 +6564,8 @@ mod tests { .validate() .expect("absent admission fields keep the existing behavior"); - config.authentication.allowed_clients = Some(vec!["records-reader".to_owned()]); - config.authentication.required_scopes = Some(vec!["evidence:invoke".to_owned()]); + config.authentication.oidc.allowed_clients = Some(vec!["records-reader".to_owned()]); + config.authentication.oidc.required_scopes = Some(vec!["evidence:invoke".to_owned()]); config.validate().expect("stated admission validates"); for clients in [ @@ -6173,7 +6576,7 @@ mod tests { vec!["reader".to_owned(), "reader".to_owned()], ] { let mut candidate = config.clone(); - candidate.authentication.allowed_clients = Some(clients.clone()); + candidate.authentication.oidc.allowed_clients = Some(clients.clone()); assert!( candidate.validate().is_err(), "accepted allowedClients {clients:?}" @@ -6188,7 +6591,7 @@ mod tests { vec!["evidence:invoke".to_owned(), "evidence:invoke".to_owned()], ] { let mut candidate = config.clone(); - candidate.authentication.required_scopes = Some(scopes.clone()); + candidate.authentication.oidc.required_scopes = Some(scopes.clone()); assert!( candidate.validate().is_err(), "accepted requiredScopes {scopes:?}" @@ -6209,7 +6612,7 @@ mod tests { .validate() .expect("absent assertionIssuers keeps the existing behavior"); - config.authentication.assertion_issuers = Some(BTreeMap::from([ + config.authentication.oidc.assertion_issuers = Some(BTreeMap::from([ ( "evidence-task-agent".to_owned(), vec!["https://identity.invalid".to_owned()], @@ -6240,7 +6643,7 @@ mod tests { .collect(), ] { let mut candidate = config.clone(); - candidate.authentication.assertion_issuers = Some(clients.clone()); + candidate.authentication.oidc.assertion_issuers = Some(clients.clone()); assert!( candidate.validate().is_err(), "accepted assertionIssuers {clients:?}" @@ -6257,7 +6660,7 @@ mod tests { ], ] { let mut candidate = config.clone(); - candidate.authentication.assertion_issuers = Some(BTreeMap::from([( + candidate.authentication.oidc.assertion_issuers = Some(BTreeMap::from([( "evidence-task-agent".to_owned(), issuers.clone(), )])); @@ -6518,9 +6921,10 @@ mod tests { let mut duplicate = config.clone(); duplicate .authentication + .oidc .claims .grant_id - .clone_from(&config.authentication.claims.grant_source_issuer); + .clone_from(&config.authentication.oidc.claims.grant_source_issuer); assert_eq!( duplicate.validate(), invalid("contextual authorization claim names are invalid"), @@ -6528,8 +6932,8 @@ mod tests { ); let mut duplicate_actor = config.clone(); - duplicate_actor.authentication.actor_claim = - Some(config.authentication.requester_tags_claim.clone()); + duplicate_actor.authentication.oidc.actor_claim = + Some(config.authentication.oidc.requester_tags_claim.clone()); assert_eq!( duplicate_actor.validate(), invalid("authority claim names must be distinct"), @@ -6537,8 +6941,8 @@ mod tests { ); let mut shared_shadows_product = config.clone(); - shared_shadows_product.authentication.claims.purpose = - config.authentication.requester_tags_claim.clone(); + shared_shadows_product.authentication.oidc.claims.purpose = + config.authentication.oidc.requester_tags_claim.clone(); assert_eq!( shared_shadows_product.validate(), invalid("authority claim names must be distinct"), @@ -6554,7 +6958,7 @@ mod tests { // to explain why. for reserved in ["iss", "aud", "exp", "iat", "nbf", "jti", "client_id", "cnf"] { let mut candidate = config.clone(); - candidate.authentication.claims.grant_source_issuer = reserved.to_owned(); + candidate.authentication.oidc.claims.grant_source_issuer = reserved.to_owned(); assert_eq!( candidate.validate(), invalid("contextual authorization claim names are invalid"), @@ -6565,16 +6969,18 @@ mod tests { // `sub` carries the principal, so the principal claim may name it and // the fixture does. Any other claim naming it would read the principal. let mut principal_is_subject = config.clone(); - principal_is_subject.authentication.principal_claim = "sub".to_owned(); + principal_is_subject.authentication.oidc.principal_claim = "sub".to_owned(); principal_is_subject .validate() .expect("the principal may be read from sub"); // Moved off `sub` first, so this proves the shadowing rule rather than // colliding with the principal and tripping distinctness instead. let mut source_issuer_is_subject = config.clone(); - source_issuer_is_subject.authentication.principal_claim = "evidence_principal".to_owned(); + source_issuer_is_subject.authentication.oidc.principal_claim = + "evidence_principal".to_owned(); source_issuer_is_subject .authentication + .oidc .claims .grant_source_issuer = "sub".to_owned(); assert_eq!( @@ -6584,7 +6990,7 @@ mod tests { ); let mut distinct = config.clone(); - distinct.authentication.actor_claim = Some("evidence_actor".to_owned()); + distinct.authentication.oidc.actor_claim = Some("evidence_actor".to_owned()); distinct .validate() .expect("distinct, unreserved claim names load"); @@ -6612,14 +7018,15 @@ mod tests { ); let mut no_global_admission = config.clone(); - no_global_admission.authentication.allowed_clients = None; + no_global_admission.authentication.oidc.allowed_clients = None; assert_eq!( no_global_admission.validate(), invalid("task-grant authority profiles require authentication allowedClients") ); let mut client_not_admitted = config; - client_not_admitted.authentication.allowed_clients = Some(vec!["other-client".to_owned()]); + client_not_admitted.authentication.oidc.allowed_clients = + Some(vec!["other-client".to_owned()]); assert_eq!( client_not_admitted.validate(), invalid( @@ -6639,7 +7046,8 @@ mod tests { ); let mut requester_clients = config.clone(); - requester_clients.authentication.allowed_clients = Some(vec!["evidence-cli".to_owned()]); + requester_clients.authentication.oidc.allowed_clients = + Some(vec!["evidence-cli".to_owned()]); requester_clients.authority_profiles.0[0] .1 .requester_clients = vec!["evidence-cli".to_owned()]; @@ -7625,12 +8033,15 @@ mod tests { #[test] fn exact_secret_reference_grammars_are_closed() { - for valid in ["secret:file/a", "secret:file/source-token_v2.json"] { - assert!(SecretRef::parse(valid).is_ok(), "{valid}"); + for valid in [ + "secret:file/a", + "secret:file/source-token_v2.json", + "secret:env/SOURCE_2_PASSWORD", + ] { + assert!(SecretReference::parse(valid).is_ok(), "{valid}"); } for invalid in [ "secret:env/", - "secret:env/SOURCE_2_PASSWORD", "secret:env/lower", "secret:env/A-B", "secret:file/Upper", @@ -7638,7 +8049,7 @@ mod tests { "secret:file/.token", "plain-value", ] { - assert!(SecretRef::parse(invalid).is_err(), "{invalid}"); + assert!(SecretReference::parse(invalid).is_err(), "{invalid}"); } } @@ -8354,9 +8765,10 @@ mod tests { EvidenceConfig::parse_yaml(valid.as_bytes()).expect("fixture validates"); *http_authentication(&mut oauth) = SourceAuthentication::Oauth2ClientCredentials { token_endpoint: format!("https://source.invalid/token{query}"), - client_id_ref: SecretRef::parse("secret:file/oauth-client-id").expect("secret ref"), + client_id_ref: SecretReference::parse("secret:file/oauth-client-id") + .expect("secret ref"), client_secret_ref: Some( - SecretRef::parse("secret:file/oauth-client-secret").expect("secret ref"), + SecretReference::parse("secret:file/oauth-client-secret").expect("secret ref"), ), client_assertion_key_ref: None, client_assertion_audience: None, @@ -8441,9 +8853,10 @@ mod tests { EvidenceConfig::parse_yaml(valid.as_bytes()).expect("fixture validates"); *http_authentication(&mut oauth) = SourceAuthentication::Oauth2ClientCredentials { token_endpoint: "https://source.invalid/token".to_owned(), - client_id_ref: SecretRef::parse("secret:file/oauth-client-id").expect("secret ref"), + client_id_ref: SecretReference::parse("secret:file/oauth-client-id") + .expect("secret ref"), client_secret_ref: Some( - SecretRef::parse("secret:file/oauth-client-secret").expect("secret ref"), + SecretReference::parse("secret:file/oauth-client-secret").expect("secret ref"), ), client_assertion_key_ref: None, client_assertion_audience: None, @@ -8882,14 +9295,24 @@ mod tests { ); } + /// A `listener.bind` value for `host` and `port`, bracketing an IPv6 host. + fn bind_of(host: &str, port: u16) -> String { + if host.contains(':') { + format!("'[{host}]:{port}'") + } else { + format!("{host}:{port}") + } + } + #[test] fn runtime_document_is_closed_and_contains_no_governed_override_surface() { let valid = br#" -version: 1 -bundleDirectory: /etc/registry-evidence/bundle +apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /etc/registry-evidence/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 @@ -8930,10 +9353,13 @@ outboundTls: for rejected_host in ["evidence.internal", "0.0.0.0", "8.8.8.8", "ff02::1"] { let candidate = String::from_utf8(valid.to_vec()) .expect("runtime fixture is UTF-8") - .replace("bindHost: 127.0.0.1", &format!("bindHost: {rejected_host}")); + .replace( + "bind: 127.0.0.1:8080", + &format!("bind: {}", bind_of(rejected_host, 8080)), + ); assert!( RuntimeConfig::parse_yaml(candidate.as_bytes()).is_err(), - "runtime accepted prohibited bindHost {rejected_host}" + "runtime accepted prohibited bind host {rejected_host}" ); } @@ -8941,8 +9367,11 @@ outboundTls: let candidate = String::from_utf8(valid.to_vec()) .expect("runtime fixture is UTF-8") .replace( - "bindHost: 127.0.0.1", - &format!("bindHost: '{wildcard}'\n networkExposure: container-private"), + "bind: 127.0.0.1:8080", + &format!( + "bind: {}\n networkExposure: container-private", + bind_of(wildcard, 8080) + ), ); let parsed = RuntimeConfig::parse_yaml(candidate.as_bytes()) .expect("explicit container-private wildcard parses"); @@ -8956,12 +9385,15 @@ outboundTls: let candidate = String::from_utf8(valid.to_vec()) .expect("runtime fixture is UTF-8") .replace( - "bindHost: 127.0.0.1", - &format!("bindHost: '{rejected_host}'\n networkExposure: container-private"), + "bind: 127.0.0.1:8080", + &format!( + "bind: {}\n networkExposure: container-private", + bind_of(rejected_host, 8080) + ), ); assert!( RuntimeConfig::parse_yaml(candidate.as_bytes()).is_err(), - "container-private mode accepted prohibited bindHost {rejected_host}" + "container-private mode accepted prohibited bind host {rejected_host}" ); } for governed_key in [ @@ -8988,9 +9420,10 @@ outboundTls: "unknown field", "governed bundle key {governed_key} was rejected for the wrong reason" ); - assert!( - fault.location().is_some(), - "governed bundle key {governed_key} was rejected without a location" + assert_eq!( + fault.path(), + Some(governed_key), + "governed bundle key {governed_key} was rejected without naming it" ); assert!( !validator.is_valid(&bundle_contract_instance(&candidate)), @@ -9166,11 +9599,12 @@ outboundTls: #[test] fn the_optional_metrics_listener_is_absent_by_default_and_stays_operator_private() { let base = r#" -version: 1 -bundleDirectory: /etc/registry-evidence/bundle +apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /etc/registry-evidence/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 @@ -9199,28 +9633,29 @@ outboundTls: "a deployment that asked for no metrics listener must not get one" ); - let configured = format!("{base}metricsListener:\n bindHost: 127.0.0.1\n port: 9090\n"); + let configured = format!("{base}metricsListener:\n bind: 127.0.0.1:9090\n"); assert!(validator.is_valid(&bundle_contract_instance(configured.as_bytes()))); let parsed = RuntimeConfig::parse_yaml(configured.as_bytes()).expect("metrics listener parses"); let metrics = parsed .metrics_listener .expect("the configured metrics listener is retained"); - assert_eq!(metrics.bind_host, "127.0.0.1"); - assert_eq!(metrics.port, 9090); + assert_eq!(metrics.bind.socket_addr().to_string(), "127.0.0.1:9090"); for rejected_host in ["evidence.internal", "0.0.0.0", "8.8.8.8", "ff02::1"] { - let candidate = - format!("{base}metricsListener:\n bindHost: {rejected_host}\n port: 9090\n"); + let candidate = format!( + "{base}metricsListener:\n bind: {}\n", + bind_of(rejected_host, 9090) + ); assert!( RuntimeConfig::parse_yaml(candidate.as_bytes()).is_err(), - "metrics listener accepted prohibited bindHost {rejected_host}" + "metrics listener accepted prohibited bind host {rejected_host}" ); } // Reusing the evidence binding would put the counters on the listener // the public contract describes. - let shared = format!("{base}metricsListener:\n bindHost: 127.0.0.1\n port: 8080\n"); + let shared = format!("{base}metricsListener:\n bind: 127.0.0.1:8080\n"); assert!(matches!( RuntimeConfig::parse_yaml(shared.as_bytes()), Err(ConfigError::Invalid( @@ -9230,11 +9665,16 @@ outboundTls: for (wildcard, private) in [("0.0.0.0", "10.0.0.10"), ("::", "fd00::10")] { let wildcard_base = base.replace( - "bindHost: 127.0.0.1", - &format!("bindHost: '{wildcard}'\n networkExposure: container-private"), + "bind: 127.0.0.1:8080", + &format!( + "bind: {}\n networkExposure: container-private", + bind_of(wildcard, 8080) + ), + ); + let shared = format!( + "{wildcard_base}metricsListener:\n bind: {}\n", + bind_of(private, 8080) ); - let shared = - format!("{wildcard_base}metricsListener:\n bindHost: {private}\n port: 8080\n"); assert!(matches!( RuntimeConfig::parse_yaml(shared.as_bytes()), Err(ConfigError::Invalid( @@ -9243,11 +9683,11 @@ outboundTls: )); } let dual_stack_base = base.replace( - "bindHost: 127.0.0.1", - "bindHost: '::'\n networkExposure: container-private", + "bind: 127.0.0.1:8080", + "bind: '[::]:8080'\n networkExposure: container-private", ); let dual_stack_collision = - format!("{dual_stack_base}metricsListener:\n bindHost: 127.0.0.1\n port: 8080\n"); + format!("{dual_stack_base}metricsListener:\n bind: 127.0.0.1:8080\n"); assert!(matches!( RuntimeConfig::parse_yaml(dual_stack_collision.as_bytes()), Err(ConfigError::Invalid( @@ -9256,9 +9696,8 @@ outboundTls: )); // The block is closed like every other level of the document. - let unknown = format!( - "{base}metricsListener:\n bindHost: 127.0.0.1\n port: 9090\n path: /telemetry\n" - ); + let unknown = + format!("{base}metricsListener:\n bind: 127.0.0.1:9090\n path: /telemetry\n"); assert!(RuntimeConfig::parse_yaml(unknown.as_bytes()).is_err()); assert!(!validator.is_valid(&bundle_contract_instance(unknown.as_bytes()))); } @@ -9271,11 +9710,12 @@ outboundTls: #[test] fn the_optional_operator_acquisition_capabilities_enable_nothing_by_default() { let base = r#" -version: 1 -bundleDirectory: /etc/registry-evidence/bundle +apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /etc/registry-evidence/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 @@ -9606,11 +10046,12 @@ outboundTls: #[test] fn a_listener_port_of_zero_is_refused_on_both_listeners() { let base = r#" -version: 1 -bundleDirectory: /etc/registry-evidence/bundle +apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /etc/registry-evidence/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 @@ -9635,7 +10076,7 @@ outboundTls: let validator = runtime_contract_validator(); RuntimeConfig::parse_yaml(base.as_bytes()).expect("the configured ports load"); - let ephemeral_evidence = base.replace("port: 8080", "port: 0"); + let ephemeral_evidence = base.replace("bind: 127.0.0.1:8080", "bind: 127.0.0.1:0"); assert!( RuntimeConfig::parse_yaml(ephemeral_evidence.as_bytes()).is_err(), "the evidence listener accepted an ephemeral port" @@ -9645,8 +10086,7 @@ outboundTls: "the published schema must already refuse this, so Rust is matching it" ); - let ephemeral_metrics = - format!("{base}metricsListener:\n bindHost: 127.0.0.1\n port: 0\n"); + let ephemeral_metrics = format!("{base}metricsListener:\n bind: 127.0.0.1:0\n"); assert!( RuntimeConfig::parse_yaml(ephemeral_metrics.as_bytes()).is_err(), "the metrics listener accepted an ephemeral port" @@ -9656,8 +10096,8 @@ outboundTls: // Both at zero would compare equal and trip the collision rule instead, // so the port rule has to be the one that fires. let both = format!( - "{}metricsListener:\n bindHost: 127.0.0.1\n port: 0\n", - base.replace("port: 8080", "port: 0") + "{}metricsListener:\n bind: 127.0.0.1:0\n", + base.replace("bind: 127.0.0.1:8080", "bind: 127.0.0.1:0") ); assert!(RuntimeConfig::parse_yaml(both.as_bytes()).is_err()); } @@ -9711,17 +10151,352 @@ outboundTls: "../../../products/evidence/fixtures/acceptance/adult-status/evidence.yaml" )) .expect("fixture validates"); - config.authentication.issuer = "https://identity.example.test/realms/registry".to_owned(); + config.authentication.oidc.provider.issuer = + "https://identity.example.test/realms/registry".to_owned(); assert!(config.validate().is_ok()); - config.authentication.issuer.push_str("?tenant=wrong"); + config + .authentication + .oidc + .provider + .issuer + .push_str("?tenant=wrong"); assert!(config.validate().is_err()); } + /// A runtime document every refusal test below edits one member of. + const LOADER_RUNTIME_DOCUMENT: &str = + "apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /etc/registry-evidence/bundle +listener: + bind: 127.0.0.1:8080 + tlsTermination: operator-controlled-upstream + trustProxyIdentityHeaders: false + maximumRequestBytes: 65536 + maximumConcurrentRequests: 64 + requestTimeoutMilliseconds: 10000 + shutdownGraceMilliseconds: 30000 +secretProviders: + file: {root: /run/secrets/registry-evidence} +signer: + kind: local-jwk + privateKeyRef: secret:file/signing-key +audit: + path: /var/lib/registry-evidence/audit/evidence.jsonl +outboundTls: + systemRoots: true + trustProfiles: {} +"; + + fn runtime_refusal(text: &str) -> SchemaFault { + match RuntimeConfig::parse_yaml(text.as_bytes()) { + Err(ConfigError::InvalidYaml(fault)) => fault, + other => panic!("the runtime document was not refused as a schema fault: {other:?}"), + } + } + + #[test] + fn the_runtime_document_declares_the_evidence_envelope() { + RuntimeConfig::parse_yaml(LOADER_RUNTIME_DOCUMENT.as_bytes()) + .expect("the enveloped document loads"); + + let unenveloped = LOADER_RUNTIME_DOCUMENT + .replace( + "apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1\n", + "", + ) + .replace("kind: EvidenceRuntimeConfig\n", ""); + let fault = runtime_refusal(&unenveloped); + assert_eq!( + fault.cause(), + "document does not declare the Evidence runtime envelope" + ); + assert_eq!(fault.remedy(), Some(EVIDENCE_RUNTIME_ENVELOPE_REMEDY)); + + let other_kind = LOADER_RUNTIME_DOCUMENT + .replace("kind: EvidenceRuntimeConfig", "kind: RelayRuntimeConfig"); + assert_eq!( + runtime_refusal(&other_kind).cause(), + "document does not declare the Evidence runtime envelope" + ); + } + + #[test] + fn every_removed_runtime_key_is_refused_with_its_replacement_named() { + let cases = [ + ("version: 1\n", "version"), + ( + "bundleDirectory: /etc/registry-evidence/bundle\n", + "bundleDirectory", + ), + ]; + for (member, path) in cases { + let fault = runtime_refusal(&format!("{LOADER_RUNTIME_DOCUMENT}{member}")); + assert_eq!(fault.cause(), "key is no longer accepted", "{path}"); + assert_eq!(fault.path(), Some(path)); + let expected = EVIDENCE_RUNTIME_REMOVED_KEYS + .iter() + .find(|removed| removed.path == path) + .expect("the key is listed") + .replacement; + assert_eq!(fault.remedy(), Some(expected), "{path}"); + } + for (listener, path) in [ + ("listener", "listener.bindHost"), + ("listener", "listener.port"), + ] { + let member = path.rsplit('.').next().expect("a leaf"); + let value = if member == "port" { + "8080" + } else { + "127.0.0.1" + }; + let text = LOADER_RUNTIME_DOCUMENT.replace( + &format!("{listener}:\n bind: 127.0.0.1:8080\n"), + &format!("{listener}:\n bind: 127.0.0.1:8080\n {member}: {value}\n"), + ); + let fault = runtime_refusal(&text); + assert_eq!(fault.path(), Some(path)); + assert!( + fault + .remedy() + .is_some_and(|remedy| remedy.contains("listener.bind")), + "{path}" + ); + } + let metrics = format!( + "{LOADER_RUNTIME_DOCUMENT}metricsListener:\n bindHost: 127.0.0.1\n port: 9090\n" + ); + let fault = runtime_refusal(&metrics); + assert!(fault + .remedy() + .is_some_and(|remedy| remedy.contains("metricsListener.bind"))); + } + + #[test] + fn a_duplicated_runtime_key_is_refused() { + let duplicated = format!("{LOADER_RUNTIME_DOCUMENT}kind: EvidenceRuntimeConfig\n"); + assert!(RuntimeConfig::parse_yaml(duplicated.as_bytes()).is_err()); + } + + /// Substitution fills operator values from the environment. It never + /// reaches a secret reference or the provider configuration, because a + /// reference that the environment can rename is no longer the reference + /// the file declares. + #[test] + fn environment_substitution_fills_values_and_never_a_secret_reference() { + let templated = LOADER_RUNTIME_DOCUMENT.replace( + "path: /var/lib/registry-evidence/audit/evidence.jsonl", + "path: ${EVIDENCE_AUDIT_PATH}", + ); + let loaded = RuntimeConfig::parse_yaml_with(templated.as_bytes(), |name| { + (name == "EVIDENCE_AUDIT_PATH").then(|| "/srv/audit/evidence.jsonl".to_owned()) + }) + .expect("the substituted document loads"); + assert_eq!( + loaded.config.audit.path.as_deref(), + Some("/srv/audit/evidence.jsonl") + ); + let plain = RuntimeConfig::parse_yaml_with(LOADER_RUNTIME_DOCUMENT.as_bytes(), |_| None) + .expect("the plain document loads"); + assert_ne!( + loaded.effective_digest, plain.effective_digest, + "the effective digest covers the substituted value" + ); + + let unset = RuntimeConfig::parse_yaml_with(templated.as_bytes(), |_| None); + assert!( + unset.is_err(), + "an unset variable without a default is refused" + ); + + for (from, to) in [ + ( + "privateKeyRef: secret:file/signing-key", + "privateKeyRef: ${SIGNING_KEY_REF}", + ), + ( + "file: {root: /run/secrets/registry-evidence}", + "file: {root: \"${SECRET_ROOT}\"}", + ), + ] { + let text = LOADER_RUNTIME_DOCUMENT.replace(from, to); + let refused = RuntimeConfig::parse_yaml_with(text.as_bytes(), |_| { + Some("secret:file/other".to_owned()) + }); + let Err(ConfigError::InvalidYaml(fault)) = refused else { + panic!("substitution into {from} was accepted"); + }; + assert_eq!( + fault.cause(), + "environment expressions are not accepted in secret references or secretProviders" + ); + } + } + + #[test] + fn the_environment_secret_provider_is_enabled_only_by_declaration() { + let enabled = LOADER_RUNTIME_DOCUMENT.replace( + "file: {root: /run/secrets/registry-evidence}", + "file: {root: /run/secrets/registry-evidence}\n environment: {}", + ); + let config = RuntimeConfig::parse_yaml(enabled.as_bytes()).expect("both providers load"); + assert!(config.secret_providers.environment.is_some()); + + let only_environment = LOADER_RUNTIME_DOCUMENT + .replace( + "file: {root: /run/secrets/registry-evidence}", + "environment: {}", + ) + .replace( + "privateKeyRef: secret:file/signing-key", + "privateKeyRef: secret:env/EVIDENCE_SIGNING_KEY", + ); + RuntimeConfig::parse_yaml(only_environment.as_bytes()) + .expect("an environment-only deployment loads"); + + let undeclared = LOADER_RUNTIME_DOCUMENT.replace( + "privateKeyRef: secret:file/signing-key", + "privateKeyRef: secret:env/EVIDENCE_SIGNING_KEY", + ); + assert!( + RuntimeConfig::parse_yaml(undeclared.as_bytes()).is_err(), + "a runtime reference to a provider the file does not enable is refused" + ); + + let none = LOADER_RUNTIME_DOCUMENT.replace( + "secretProviders:\n file: {root: /run/secrets/registry-evidence}\n", + "secretProviders: {}\n", + ); + assert!(RuntimeConfig::parse_yaml(none.as_bytes()).is_err()); + } + + #[test] + fn an_expected_package_digest_must_be_a_sha256_label() { + let pinned = LOADER_RUNTIME_DOCUMENT.replace( + " root: /etc/registry-evidence/bundle\n", + " root: /etc/registry-evidence/bundle\n expectedDigest: sha256:0000000000000000000000000000000000000000000000000000000000000000\n", + ); + RuntimeConfig::parse_yaml(pinned.as_bytes()).expect("a pinned package loads"); + let malformed = LOADER_RUNTIME_DOCUMENT.replace( + " root: /etc/registry-evidence/bundle\n", + " root: /etc/registry-evidence/bundle\n expectedDigest: md5:00\n", + ); + assert!(RuntimeConfig::parse_yaml(malformed.as_bytes()).is_err()); + } + + #[test] + fn every_removed_bundle_authentication_and_audit_key_is_refused_with_its_replacement_named() { + let valid = String::from_utf8( + include_bytes!( + "../../../products/evidence/fixtures/acceptance/adult-status/evidence.yaml" + ) + .to_vec(), + ) + .expect("fixture is UTF-8"); + EvidenceConfig::parse_yaml(valid.as_bytes()).expect("fixture validates"); + for (path, replacement) in REMOVED_BUNDLE_KEYS { + let mut segments = path.split('.').collect::>(); + let leaf = segments.pop().expect("a leaf"); + let mut value = + serde_norway::from_str::(&valid).expect("fixture parses"); + let mut parent = &mut value; + for segment in &segments { + parent = parent + .as_mapping_mut() + .expect("a mapping") + .entry(serde_norway::Value::from(*segment)) + .or_insert_with(|| serde_norway::Value::Mapping(Default::default())); + } + parent.as_mapping_mut().expect("a mapping").insert( + serde_norway::Value::from(leaf), + serde_norway::Value::from("x"), + ); + let text = serde_norway::to_string(&value).expect("the candidate serializes"); + let Err(ConfigError::InvalidYaml(fault)) = EvidenceConfig::parse_yaml(text.as_bytes()) + else { + panic!("{path} was not refused as a removed key"); + }; + assert_eq!(fault.cause(), "key is no longer accepted", "{path}"); + assert_eq!(fault.path(), Some(*path)); + assert_eq!(fault.remedy(), Some(*replacement), "{path}"); + } + } + + #[test] + fn an_authored_bundle_carrying_an_environment_expression_is_refused() { + let valid = String::from_utf8( + include_bytes!( + "../../../products/evidence/fixtures/acceptance/adult-status/evidence.yaml" + ) + .to_vec(), + ) + .expect("fixture is UTF-8"); + let candidate = valid.replace( + "publicOrigin: https://evidence.invalid", + "publicOrigin: 'https://${EVIDENCE_HOST}'", + ); + assert_ne!(candidate, valid, "the fixture carries the public origin"); + let Err(ConfigError::InvalidYaml(fault)) = EvidenceConfig::parse_yaml(candidate.as_bytes()) + else { + panic!("an environment expression in the bundle was not refused"); + }; + assert_eq!( + fault.cause(), + "environment expressions are not accepted in the governed bundle" + ); + assert_eq!(fault.path(), Some("service.publicOrigin")); + assert!(!fault.to_string().contains("EVIDENCE_HOST")); + } + + #[test] + fn access_token_keys_come_only_from_a_fixed_jwks_uri() { + let valid = String::from_utf8( + include_bytes!( + "../../../products/evidence/fixtures/acceptance/adult-status/evidence.yaml" + ) + .to_vec(), + ) + .expect("fixture is UTF-8"); + let config = EvidenceConfig::parse_yaml(valid.as_bytes()).expect("fixture validates"); + let JwksSource::Uri { uri } = &config.authentication.oidc.provider.jwks_source else { + panic!("the fixture names a fixed JWKS URI"); + }; + for refused in [ + JwksSource::Discovery {}, + JwksSource::Static { + document_ref: "secret:file/jwks".to_owned(), + }, + ] { + let mut candidate = config.clone(); + candidate.authentication.oidc.provider.jwks_source = refused; + assert_eq!( + candidate.validate(), + Err(ConfigError::InvalidField( + "Evidence reads access-token keys only from jwksSource kind uri", + "authentication.oidc.jwksSource", + )) + ); + } + let omitted = valid.replace( + &format!(" jwksSource:\n kind: uri\n uri: {uri}\n"), + "", + ); + assert_ne!(omitted, valid, "fixture mutation must remain effective"); + assert!( + EvidenceConfig::parse_yaml(omitted.as_bytes()).is_err(), + "an omitted key source means discovery, which Evidence refuses" + ); + } + #[test] - fn file_secret_references_are_the_only_governed_secret_form() { - assert!(SecretRef::parse("secret:file/source-token").is_ok()); - assert!(SecretRef::parse("secret:env/SOURCE_TOKEN").is_err()); - assert!(SecretRef::parse("literal-token").is_err()); + fn governed_secret_references_name_a_provider_and_never_a_value() { + assert!(SecretReference::parse("secret:file/source-token").is_ok()); + // Whether the runtime enables the environment provider is checked when + // the bundle is bound to its runtime configuration. + assert!(SecretReference::parse("secret:env/SOURCE_TOKEN").is_ok()); + assert!(SecretReference::parse("literal-token").is_err()); } #[test] diff --git a/crates/registry-evidence/src/kernel.rs b/crates/registry-evidence/src/kernel.rs index a1ae7985e5..57ea953ab6 100644 --- a/crates/registry-evidence/src/kernel.rs +++ b/crates/registry-evidence/src/kernel.rs @@ -1,7 +1,7 @@ //! Generic offline Evidence evaluation and core-owned output projection. //! //! This module deliberately knows nothing about an acceptance case or source -//! product. It joins one captured bundle revision to the hardened Rhai runtime, +//! product. It joins one captured verified package to the hardened Rhai runtime, //! validates the complete declared Supported Value set, and constructs the //! unsigned Evidence payload that the production release path later signs. @@ -476,7 +476,7 @@ impl std::fmt::Debug for OfflineKernel { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter .debug_struct("OfflineKernel") - .field("bundle_revision", &self.bundle.revision()) + .field("package_digest", &self.bundle.package_digest()) .field("source_count", &self.extractions.len()) .field("requirement_count", &self.derivations.len()) .finish() @@ -2235,6 +2235,7 @@ fn extract_batch(response, context) { "$schema: https://json-schema.org/draft/2020-12/schema\ntype: object\nadditionalProperties: false\nrequired: [kind]\nproperties:\n kind:\n type: string\n enum: [wrong-outer, wrong-member, extra-key, invalid-facts, missing, duplicate, extra, out-of-range, negative]\n", ) .expect("batch response schema writes"); + refresh_package_envelope(temporary.path()); make_read_only(temporary.path()); let bundle = Arc::new(Bundle::load(temporary.path()).expect("batch bundle loads")); OfflineKernel::compile(bundle).expect("batch kernel compiles") @@ -3743,6 +3744,7 @@ fn extract_batch(response, context) { let mut text = fs::read_to_string(&path).expect("reads copied artifact"); text.push_str(appended); fs::write(&path, text).expect("writes copied artifact"); + refresh_package_envelope(temporary.path()); make_read_only(temporary.path()); temporary } @@ -3754,6 +3756,7 @@ fn extract_batch(response, context) { copy_tree(&source, temporary.path()); fs::write(temporary.path().join("adapters/source-a.rhai"), extraction) .expect("replacement extraction writes"); + refresh_package_envelope(temporary.path()); make_read_only(temporary.path()); let bundle = Arc::new(Bundle::load(temporary.path()).expect("bundle loads")); OfflineKernel::compile(bundle).expect("kernel compiles") @@ -3782,6 +3785,20 @@ fn extract_batch(response, context) { } } + /// Republish a copied test package after its intended authored inputs change. + /// Tests that deliberately corrupt a package do not use this helper. + fn refresh_package_envelope(root: &Path) { + fs::remove_file(root.join(registry_platform_config::SUM_FILE)) + .expect("prior package sum file is removed"); + registry_platform_config::write_sum_file( + root, + None, + &crate::bundle::evidence_package_limits(), + "evidencectl package", + ) + .expect("mutated test package is republished"); + } + #[cfg(unix)] fn make_read_only(path: &Path) { use std::os::unix::fs::PermissionsExt as _; diff --git a/crates/registry-evidence/src/local_verification.rs b/crates/registry-evidence/src/local_verification.rs index d01e0aa0d8..2ee81e33c0 100644 --- a/crates/registry-evidence/src/local_verification.rs +++ b/crates/registry-evidence/src/local_verification.rs @@ -13,7 +13,6 @@ use crate::{ config::{AssuranceProfile, ConceptForm, ResponseFormat, SubjectBindingMode}, model::{JwksDocument, RequestedSubject}, runtime::{validate_verification_material, ValidatedVerificationMaterial}, - secrets::{SecretProvider, SecretResolver}, selector::{resolve_request_origin_subjects, ResolvedSubject}, verifier::{ ExpectedFormDocument, ExpectedOutputDocument, ExpectedScalarFormDocument, @@ -123,11 +122,12 @@ pub async fn prepare_local_relying_procedure( resolve_request_origin_subjects(bundle, requirement, &input.purpose, &input.subjects) .map_err(|_| LocalProcedureError)?; - let secrets = SecretResolver::new( - [SecretProvider::File], - &deployment.runtime().config.secret_providers.file.root, - ) - .map_err(|_| LocalProcedureError)?; + let secrets = deployment + .runtime() + .config + .secret_providers + .resolver() + .map_err(|_| LocalProcedureError)?; let ValidatedVerificationMaterial { subject_binding_secret, signer: _, diff --git a/crates/registry-evidence/src/main.rs b/crates/registry-evidence/src/main.rs index 8eb76ac5e4..8f501d2a7a 100644 --- a/crates/registry-evidence/src/main.rs +++ b/crates/registry-evidence/src/main.rs @@ -40,7 +40,7 @@ use registry_evidence::{ rhai_runtime::{DerivedConceptValue, DerivedValue}, runtime::{ source_failure_problem, validate_secret_material, AuditInitializationFault, - EvidenceRuntime, RuntimeInitializationError, + EvidenceRuntime, RuntimeInitializationError, SigningInitializationFault, }, secrets::{SecretProvider, SecretResolver}, selector::{ @@ -66,6 +66,7 @@ use registry_evidence::{ }, }; use registry_platform_audit::{require_audit_under, PersistentRootFault}; +use registry_platform_config::SecretProvidersConfig; use registry_platform_crypto::{canonicalize_json, parse_json_strict, LocalJwkSigner, PrivateJwk}; use serde_json::{Map as JsonMap, Value}; use zeroize::Zeroizing; @@ -100,6 +101,7 @@ impl std::error::Error for CliError {} enum CommandError { Cli(CliError), Deployment(&'static str, ArtifactFault), + Package(registry_platform_config::PackageError), /// One or more governed extract sources cannot answer at check time. /// /// Source identifiers come from the reviewed bundle. Publisher metadata @@ -107,10 +109,15 @@ enum CommandError { StaleExtracts(Vec), /// The audit boundary refused, with the value-free cause it reported. /// - /// It is the one startup boundary that separates its causes, because an - /// out-of-range destination, a permission bit, and a second writer holding - /// the destination lock have nothing in common but the moment they fail. + /// It separates its causes because an out-of-range destination, a + /// permission bit, and a second writer holding the destination lock have + /// nothing in common but the moment they fail. Audit(&'static str, AuditInitializationFault), + /// The signing boundary refused, with the value-free cause it reported. + /// + /// A missing Transit socket, a retired key version, and a key that is not + /// the governed one each need a different fix, so each names itself. + Signing(SigningInitializationFault), /// The configured audit destination was not proven persistent. /// /// The fault names the side that failed and nothing else. Neither the @@ -125,6 +132,7 @@ impl fmt::Display for CommandError { match self { Self::Cli(error) => fmt::Display::fmt(error, formatter), Self::Deployment(message, fault) => write!(formatter, "{message}: {fault}"), + Self::Package(error) => fmt::Display::fmt(error, formatter), Self::StaleExtracts(sources) => write!( formatter, "bound extract is stale for source{} {}", @@ -132,6 +140,9 @@ impl fmt::Display for CommandError { sources.join(", ") ), Self::Audit(message, fault) => write!(formatter, "{message}: {fault}"), + Self::Signing(fault) => { + write!(formatter, "runtime signing initialization failed: {fault}") + } Self::AuditRoot(fault) => write!(formatter, "audit destination check failed: {fault}"), Self::Service(reason) => write!(formatter, "service failed: {reason}"), } @@ -153,7 +164,17 @@ struct FixtureSummary { #[tokio::main] async fn main() -> ExitCode { - match run(Cli::parse()).await { + let runtime_environment_set = + std::env::var_os(registry_evidence::cli::REMOVED_RUNTIME_ENVIRONMENT_VARIABLE).is_some(); + if let Some(refusal) = registry_evidence::cli::removed_runtime_input( + std::env::args_os().skip(1), + runtime_environment_set, + ) { + eprintln!("evidence: {refusal}"); + return ExitCode::FAILURE; + } + let cli = Cli::parse(); + match run(cli).await { Ok(code) => code, Err(error) => { eprintln!("evidence: {error}"); @@ -165,13 +186,16 @@ async fn main() -> ExitCode { async fn run(cli: Cli) -> Result { match cli.command { Command::Check { + runtime_config, require_runtime_dependencies, require_audit_under: audit_root, + without_audit_lock, } => { // The inputs are captured once. Every proof below, and the runtime // the dependency check initializes, reads this capture rather than // the pathname again, so what passed is what gets opened. - let deployment = DeploymentInputs::load(&cli.runtime).map_err(deployment_load_error)?; + let deployment = + DeploymentInputs::load(&runtime_config).map_err(deployment_load_error)?; let runtime = deployment.runtime().clone(); let bundle = Arc::new(deployment.bundle().clone()); OfflineKernel::compile(Arc::clone(&bundle)) @@ -190,11 +214,10 @@ async fn run(cli: Cli) -> Result { // deployment the server would refuse fails check instead of first // start. // Source credentials stay unresolved: readiness owns them. - let secrets = SecretResolver::new( - [SecretProvider::File], - &runtime.config.secret_providers.file.root, - ) - .map_err(|_| runtime_initialization_error(RuntimeInitializationError::Secrets))?; + let secrets = + runtime.config.secret_providers.resolver().map_err(|_| { + runtime_initialization_error(RuntimeInitializationError::Secrets) + })?; validate_secret_material(&bundle, &runtime.config, &secrets) .await .map_err(runtime_initialization_error)?; @@ -212,28 +235,38 @@ async fn run(cli: Cli) -> Result { require_audit_under(Path::new(audit_path), root) .map_err(CommandError::AuditRoot)?; } - let serving = EvidenceRuntime::initialize_from(deployment) - .await - .map_err(runtime_initialization_error)?; - if !serving.key_source_ready().await || !serving.ready().await { + let available = if without_audit_lock { + // The candidate shares the running writer's audit path, so + // the lock is that writer's. Everything else is proved. + EvidenceRuntime::check_dependencies_without_audit_lock(deployment) + .await + .map_err(runtime_initialization_error)? + } else { + let serving = EvidenceRuntime::initialize_from(deployment) + .await + .map_err(runtime_initialization_error)?; + serving.key_source_ready().await && serving.ready().await + }; + if !available { return Err(CliError("a required runtime dependency is unavailable").into()); } } println!( - "Evidence deployment {} / {} passed check ({} requirements)", - bundle.revision(), - runtime.revision(), + "Evidence package {} passed check ({} requirements)", + bundle.package_digest(), bundle.config.requirements.len() ); Ok(ExitCode::SUCCESS) } Command::Evaluate { + runtime_config, fixture, case, explain, explain_format, } => { - let deployment = DeploymentInputs::load(&cli.runtime).map_err(deployment_load_error)?; + let deployment = + DeploymentInputs::load(&runtime_config).map_err(deployment_load_error)?; let (bundle, runtime) = deployment.into_parts(); let bundle = Arc::new(bundle); let kernel = OfflineKernel::compile(Arc::clone(&bundle)).map_err(|error| { @@ -299,11 +332,11 @@ async fn run(cli: Cli) -> Result { .map_err(|error| kernel_compile_error("bundle compilation failed", error))?; let _source_plans = compile_bundle_source_plans(&bundle)?; if json { - println!("{}", bundle_revision_report(&bundle)?); + println!("{}", package_digest_report(&bundle)?); } else { println!( - "Evidence bundle {} passed check ({} requirements)", - bundle.revision(), + "Evidence package {} passed check ({} requirements)", + bundle.package_digest(), bundle.config.requirements.len() ); } @@ -380,10 +413,10 @@ async fn run(cli: Cli) -> Result { } Ok(ExitCode::SUCCESS) } - Command::Serve => { + Command::Serve { runtime_config } => { install_operational_logging(); let runtime = Arc::new( - EvidenceRuntime::initialize(&cli.runtime) + EvidenceRuntime::initialize(&runtime_config) .await .map_err(runtime_initialization_error)?, ); @@ -427,10 +460,13 @@ async fn run(cli: Cli) -> Result { &policy, at.as_deref(), )?), - Command::PrepareLocalRelyingProcedure { input } => { - prepare_local_relying_procedure_command(&cli.runtime, &input).await + Command::PrepareLocalRelyingProcedure { + runtime_config, + input, + } => prepare_local_relying_procedure_command(&runtime_config, &input).await, + Command::LocalAuditLastOperation { runtime_config } => { + local_audit_last_operation_command(&runtime_config) } - Command::LocalAuditLastOperation => local_audit_last_operation_command(&cli.runtime), } } @@ -479,6 +515,9 @@ fn discovery_config_invalid(error: ConfigError) -> CommandError { /// `evidence check` names a file, a schema path, and a text location instead /// of only a class. Public HTTP problems are unaffected and stay generic. fn deployment_load_error(error: BundleError) -> CommandError { + if let BundleError::Package(error) = error { + return CommandError::Package(error); + } let message = match &error { BundleError::Unavailable => "deployment input is unavailable", BundleError::NotImmutable(_) => "deployment input is not immutable", @@ -489,6 +528,7 @@ fn deployment_load_error(error: BundleError) -> CommandError { BundleError::Config(_) => "deployment configuration is invalid", BundleError::InvalidArtifact(_) => "deployment artifact is invalid", BundleError::InvalidScript(_) => "deployment script is invalid", + BundleError::Package(_) => unreachable!("package errors returned above"), }; match error.artifact_fault() { Some(fault) => CommandError::Deployment(message, fault.clone()), @@ -521,15 +561,17 @@ fn runtime_initialization_error(error: RuntimeInitializationError) -> CommandErr RuntimeInitializationError::Audit(fault) => { CommandError::Audit("runtime audit initialization failed", fault) } - RuntimeInitializationError::Signing => { - CliError("runtime signing initialization failed").into() - } + RuntimeInitializationError::Signing(fault) => CommandError::Signing(fault), RuntimeInitializationError::Source => { CliError("runtime source initialization failed").into() } RuntimeInitializationError::RateLimit => { CliError("runtime rate-limit initialization failed").into() } + RuntimeInitializationError::IssuerTrust => CliError( + "runtime authentication initialization failed: the access-token issuer TLS trust profile does not resolve to a bound certificate bundle trusted beside the system roots", + ) + .into(), } } @@ -573,7 +615,7 @@ fn compile_source_plans( compile_source_plans_with_runtime( &bundle.config, &source_statements(bundle, Some(&runtime.source_extracts))?, - &runtime.config.secret_providers.file.root, + &runtime.config.secret_providers, &runtime.config.outbound_tls, &runtime.ca_bundles, ) @@ -582,12 +624,13 @@ fn compile_source_plans( fn compile_source_plans_with_runtime( config: &EvidenceConfig, statements: &BTreeMap>, - secret_root: &str, + secret_providers: &SecretProvidersConfig, outbound_tls: &OutboundTlsConfig, ca_bundles: &BTreeMap>, ) -> Result, CommandError> { let secrets = Arc::new( - SecretResolver::new([SecretProvider::File], secret_root) + secret_providers + .resolver() .map_err(|_| CliError("source plan compilation failed"))?, ); let connection_pool = registry_evidence::source::SourceConnectionPool::new( @@ -615,7 +658,7 @@ fn compile_source_plans_with_runtime( Ok(plans) } -fn bundle_revision_report(bundle: &Bundle) -> Result { +fn package_digest_report(bundle: &Bundle) -> Result { let requirements = bundle .config .requirements @@ -629,7 +672,7 @@ fn bundle_revision_report(bundle: &Bundle) -> Result { Ok(serde_json::json!({"id": requirement.id, "configurationRevision": revision})) }) .collect::, CliError>>()?; - Ok(serde_json::json!({"bundleRevision": bundle.revision(), "requirements": requirements})) + Ok(serde_json::json!({"packageDigest": bundle.package_digest(), "requirements": requirements})) } fn compile_bundle_source_plans( @@ -5038,6 +5081,31 @@ mod tests { use registry_evidence::verifier::{ExpectedListItemForm, ExpectedValueForm}; use std::fs; + fn file_secret_providers() -> SecretProvidersConfig { + SecretProvidersConfig { + file: Some(registry_platform_config::FileSecretProviderConfig { + root: "/run/secrets/evidence".into(), + }), + environment: None, + } + } + + /// The issuer trust refusal covers every way a named profile fails to + /// resolve, not only a file that holds no certificate. + #[test] + fn an_issuer_trust_refusal_names_a_profile_that_does_not_resolve() { + let CommandError::Cli(CliError(message)) = + runtime_initialization_error(RuntimeInitializationError::IssuerTrust) + else { + panic!("an issuer trust refusal is a plain CLI error"); + }; + assert_eq!( + message, + "runtime authentication initialization failed: the access-token issuer TLS trust \ + profile does not resolve to a bound certificate bundle trusted beside the system roots" + ); + } + /// Every command that compiles a kernel renders the same two things: the /// failure class it owns, and the artifact diagnostic the kernel produced. #[test] @@ -5076,7 +5144,6 @@ mod tests { fn render_discovery_description_cli(config: &Path) -> Cli { Cli { - runtime: PathBuf::from("/nonexistent/registry-evidence/runtime.yaml"), command: Command::RenderDiscoveryDescription { config: config.to_path_buf(), }, @@ -5234,15 +5301,15 @@ mod tests { #[cfg(unix)] #[test] - fn bundle_check_json_uses_exact_runtime_requirement_revisions() { + fn bundle_check_json_uses_the_package_digest_and_exact_requirement_revisions() { let directory = tempfile::tempdir().expect("temporary bundle"); let source = Path::new(env!("CARGO_MANIFEST_DIR")) .join("../../products/evidence/fixtures/acceptance/all-definitions"); copy_tree(&source, directory.path()); set_tree_mode(directory.path(), 0o555, 0o444); let bundle = Bundle::load(directory.path()).expect("fixture bundle loads"); - let report = bundle_revision_report(&bundle).expect("revision report"); - assert_eq!(report["bundleRevision"], bundle.revision()); + let report = package_digest_report(&bundle).expect("package report"); + assert_eq!(report["packageDigest"], bundle.package_digest()); let requirements = report["requirements"].as_array().unwrap(); assert_eq!(requirements.len(), bundle.config.requirements.len()); for (entry, requirement) in requirements.iter().zip(&bundle.config.requirements) { @@ -5769,7 +5836,7 @@ mod tests { let source_plans = compile_source_plans_with_runtime( &bundle.config, &source_statements(&bundle, None).expect("source statements bind"), - "/run/secrets/evidence", + &file_secret_providers(), &OutboundTlsConfig { system_roots: true, trust_profiles: Default::default(), @@ -6304,7 +6371,7 @@ mod tests { compile_source_plans_with_runtime( &valid_config, &BTreeMap::new(), - "/run/secrets/evidence", + &file_secret_providers(), &outbound_tls, &Default::default(), ) @@ -6320,7 +6387,7 @@ mod tests { compile_source_plans_with_runtime( &invalid_config, &BTreeMap::new(), - "/run/secrets/evidence", + &file_secret_providers(), &outbound_tls, &Default::default(), ) @@ -6350,7 +6417,7 @@ mod tests { let source_plans = compile_source_plans_with_runtime( &bundle.config, &source_statements(&bundle, None).expect("statement sources bind"), - "/run/secrets/evidence", + &file_secret_providers(), &OutboundTlsConfig { system_roots: true, trust_profiles: Default::default(), @@ -6511,7 +6578,7 @@ mod tests { let source_plans = compile_source_plans_with_runtime( &bundle.config, &source_statements(&bundle, None).expect("statement sources bind"), - "/run/secrets/evidence", + &file_secret_providers(), &OutboundTlsConfig { system_roots: true, trust_profiles: Default::default(), @@ -6566,7 +6633,7 @@ mod tests { let source_plans = compile_source_plans_with_runtime( &bundle.config, &source_statements(&bundle, None).expect("statement sources bind"), - "/run/secrets/evidence", + &file_secret_providers(), &OutboundTlsConfig { system_roots: true, trust_profiles: Default::default(), @@ -6642,7 +6709,7 @@ mod tests { let source_plans = compile_source_plans_with_runtime( &bundle.config, &source_statements(&bundle, None).expect("statement sources bind"), - "/run/secrets/evidence", + &file_secret_providers(), &OutboundTlsConfig { system_roots: true, trust_profiles: Default::default(), @@ -6697,7 +6764,7 @@ mod tests { let source_plans = compile_source_plans_with_runtime( &bundle.config, &source_statements(&bundle, None).expect("statement sources bind"), - "/run/secrets/evidence", + &file_secret_providers(), &OutboundTlsConfig { system_roots: true, trust_profiles: Default::default(), @@ -6790,7 +6857,7 @@ mod tests { let source_plans = compile_source_plans_with_runtime( &bundle.config, &source_statements(&bundle, None).expect("statement sources bind"), - "/run/secrets/evidence", + &file_secret_providers(), &outbound_tls, &ca_bundles, ) diff --git a/crates/registry-evidence/src/runtime.rs b/crates/registry-evidence/src/runtime.rs index 53118fd8d0..bacb050292 100644 --- a/crates/registry-evidence/src/runtime.rs +++ b/crates/registry-evidence/src/runtime.rs @@ -11,9 +11,10 @@ use std::{ use async_trait::async_trait; use chrono::{SubsecRound as _, Utc}; -use registry_platform_audit::{AuditError, AuditProfile}; +use registry_platform_audit::{AuditDestination, AuditError, AuditProfile}; use registry_platform_crypto::{ - LocalJwkSigner, PrivateJwk, SigningProvider, TransitSigner, TransitSignerConfig, + LocalJwkSigner, PrivateJwk, SigningProvider, TransitInitializationError, TransitSigner, + TransitSignerConfig, }; use registry_platform_oidc::ActorKind; use serde_json::{Map as JsonMap, Value}; @@ -29,7 +30,7 @@ use crate::{ EvidenceRequestBatchAuditOutcomeKind, EvidenceRequestBatchAuditPhase, ResponseProtection, }, auth::{AuthenticatedContext, AuthenticationError, Authenticator}, - bundle::{Bundle, DeploymentInputs}, + bundle::{Bundle, DeploymentInputs, RuntimeDocument}, config::{ subject_binding_permits_response_format, AcquisitionConfig, AssuranceProfile, AuthorityKind, ConceptForm, RequirementKind, ResponseFormat, RuntimeConfig, @@ -51,13 +52,13 @@ use crate::{ problem::ProblemCode, rate_limit::{EvidenceRateLimiter, RateLimitConfig, RateLimitError}, sdjwt_vc, - secrets::{ProtectedSecret, SecretProvider, SecretResolver}, + secrets::{ProtectedSecret, SecretResolver}, selector::{ match_entitlement, resolve_selectors, validate_entitlement_context, validate_subject_binding_key, AuthorizationError, MatchedEntitlement, ResolvedAuthorization, ResolvedSelectorValue, ResolvedSubjectScope, }, - signing::EvidenceSigner, + signing::{EvidenceSigner, EvidenceSigningError}, source::{ statement_inputs, ResolvedSourceSelector, SourceError, SourceExecutor, SourceResponse, }, @@ -83,12 +84,14 @@ pub enum RuntimeInitializationError { Secrets, #[error("the Evidence audit boundary could not initialize: {0}")] Audit(AuditInitializationFault), - #[error("the Evidence signing boundary could not initialize")] - Signing, + #[error("the Evidence signing boundary could not initialize: {0}")] + Signing(SigningInitializationFault), #[error("an Evidence source plan could not initialize")] Source, #[error("the Evidence rate limiter could not initialize")] RateLimit, + #[error("the Evidence access-token issuer TLS trust profile could not initialize")] + IssuerTrust, } /// Why the audit boundary refused to initialize. @@ -145,6 +148,95 @@ impl From<&EvidenceAuditError> for AuditInitializationFault { } } +/// Why the signing boundary refused to initialize. +/// +/// A socket that is not there, a key version Transit has retired, and a key +/// that is not the one the bundle governs are unrelated faults with unrelated +/// remedies: a proxy to restart, a `keyVersion` to change, or a bundle to +/// publish. They are reported separately so the operator does not have to +/// guess. No cause carries a path, a provider response, or key material. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SigningInitializationFault { + /// The `local-jwk` signing secret is missing, unreadable, or not a usable + /// private JWK. + LocalKey, + /// The runtime `transit` signer binding is out of range. + TransitConfiguration, + /// The Transit provider refused, for the reason it named. + Transit(TransitInitializationError), + /// The signing key is not the bundle's governed active public JWK. + GovernedKey, + /// The signing key did not produce a verifying signature. + SelfTest, + /// The published key set could not be assembled from the bundle's keys. + KeySet, +} + +impl SigningInitializationFault { + /// The value-free cause, for the operator message this fault appears in. + pub fn cause(self) -> &'static str { + match self { + Self::LocalKey => { + "the local-jwk signing secret is missing, unreadable, or not a private JWK" + } + Self::TransitConfiguration => "the transit signer configuration is invalid", + Self::Transit(fault) => match fault { + TransitInitializationError::Client => { + "the Transit client could not be built for unixSocketPath" + } + TransitInitializationError::Unavailable => { + "the Transit provider did not answer on the configured Unix socket (missing socket, refused connection, or timeout)" + } + TransitInitializationError::Refused => { + "the Transit provider refused the key metadata read (check the proxy token policy, mount, and keyName)" + } + TransitInitializationError::ProviderFailed => { + "the Transit provider failed the key metadata read with a server error (for example, sealed or without an active backend)" + } + TransitInitializationError::InvalidResponse => { + "the Transit provider response is malformed or too large" + } + TransitInitializationError::Custody => { + "the Transit key is not a non-derived, non-exportable ecdsa-p256 signing key without plaintext backup" + } + TransitInitializationError::KeyVersionNotCreated => { + "the runtime keyVersion is above the Transit key's latest_version" + } + TransitInitializationError::KeyVersionRetired => { + "the runtime keyVersion is below the Transit key's min_encryption_version and can no longer sign" + } + TransitInitializationError::PublicKeyMismatch => { + "the Transit public key for the runtime keyVersion is missing or is not the bundle's governed active public JWK" + } + TransitInitializationError::SelfTest => { + "the Transit sign-and-verify self-test failed" + } + _ => "the Transit provider refused initialization", + }, + Self::GovernedKey => "the signing key is not the bundle's governed active public JWK", + Self::SelfTest => "the signing key failed its sign-and-verify self-test", + Self::KeySet => { + "the published signing key set could not be assembled from the bundle's keys" + } + } + } +} + +impl fmt::Display for SigningInitializationFault { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.cause()) + } +} + +impl From<&EvidenceSigningError> for SigningInitializationFault { + fn from(error: &EvidenceSigningError) -> Self { + match error { + EvidenceSigningError::Provider(_) | EvidenceSigningError::SelfTest => Self::SelfTest, + _ => Self::GovernedKey, + } + } +} + /// Deployment secret material, resolved and validated exactly as service /// startup validates it. pub struct ValidatedSecretMaterial { @@ -176,7 +268,7 @@ pub async fn validate_secret_material( secrets: &SecretResolver, ) -> Result { let audit_secret = secrets - .resolve(bundle.config.audit.hash_key_ref.as_str()) + .resolve(bundle.config.audit.key.hash_key_ref.as_str()) .map_err(|_| RuntimeInitializationError::Audit(AuditInitializationFault::Secret))?; AuditProfile::production_from_secret_bytes(Zeroizing::new( audit_secret.expose_secret().to_vec(), @@ -216,17 +308,15 @@ pub async fn validate_verification_material( let provider: Arc = match signer_config { RuntimeSignerConfig::LocalJwk { private_key_ref } => { + let local_key = + || RuntimeInitializationError::Signing(SigningInitializationFault::LocalKey); let signing_secret = secrets .resolve(private_key_ref.as_str()) - .map_err(|_| RuntimeInitializationError::Signing)?; - let signing_json = str::from_utf8(signing_secret.expose_secret()) - .map_err(|_| RuntimeInitializationError::Signing)?; - let private_jwk = - PrivateJwk::parse(signing_json).map_err(|_| RuntimeInitializationError::Signing)?; - Arc::new( - LocalJwkSigner::new(private_jwk) - .map_err(|_| RuntimeInitializationError::Signing)?, - ) + .map_err(|_| local_key())?; + let signing_json = + str::from_utf8(signing_secret.expose_secret()).map_err(|_| local_key())?; + let private_jwk = PrivateJwk::parse(signing_json).map_err(|_| local_key())?; + Arc::new(LocalJwkSigner::new(private_jwk).map_err(|_| local_key())?) } RuntimeSignerConfig::Transit { unix_socket_path, @@ -243,23 +333,25 @@ pub async fn validate_verification_material( bundle.active_public_jwk.clone(), Duration::from_millis(*timeout_milliseconds), ) - .map_err(|_| RuntimeInitializationError::Signing)?; - Arc::new( - TransitSigner::initialize(config) - .await - .map_err(|_| RuntimeInitializationError::Signing)?, - ) + .map_err(|_| { + RuntimeInitializationError::Signing( + SigningInitializationFault::TransitConfiguration, + ) + })?; + Arc::new(TransitSigner::initialize(config).await.map_err(|fault| { + RuntimeInitializationError::Signing(SigningInitializationFault::Transit(fault)) + })?) } }; let signer = EvidenceSigner::initialize_governed(provider, &bundle.active_public_jwk) .await - .map_err(|_| RuntimeInitializationError::Signing)?; + .map_err(|error| RuntimeInitializationError::Signing((&error).into()))?; let jwks = crate::signing::jwks_document_with_revocations( signer.public_jwk(), bundle.published_public_jwks.values().cloned(), bundle.config.signing.revoked_key_ids.clone(), ) - .map_err(|_| RuntimeInitializationError::Signing)?; + .map_err(|_| RuntimeInitializationError::Signing(SigningInitializationFault::KeySet))?; Ok(ValidatedVerificationMaterial { subject_binding_secret, @@ -341,11 +433,10 @@ impl std::fmt::Debug for ReleasedEvidence { } } -/// All runtime state is derived from one captured immutable bundle revision. +/// All runtime state is derived from one captured immutable package. pub struct EvidenceRuntime { kernel: OfflineKernel, runtime_config: RuntimeConfig, - runtime_revision: String, authenticator: Arc, sources: BTreeMap, audit: Arc, @@ -395,13 +486,197 @@ impl std::fmt::Debug for EvidenceRuntime { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter .debug_struct("EvidenceRuntime") - .field("bundle_revision", &self.kernel.bundle().revision()) + .field("package_digest", &self.kernel.bundle().package_digest()) .field("source_count", &self.sources.len()) .field("signing_key_id", &self.signer.key_id()) .finish_non_exhaustive() } } +/// Compile every governed source against the runtime document that binds it. +fn build_sources( + bundle: &Bundle, + runtime_document: &RuntimeDocument, + secrets: &Arc, +) -> Result, RuntimeInitializationError> { + let runtime_config = &runtime_document.config; + let connection_pool = crate::source::SourceConnectionPool::new( + &bundle.config, + &runtime_config.outbound_tls, + &runtime_document.ca_bundles, + Arc::clone(secrets), + ) + .map_err(|_| RuntimeInitializationError::Source)?; + let mut sources = BTreeMap::new(); + for (source_id, source) in bundle.config.sources.iter() { + let allowed_selector_sets = bundle.config.source_selector_sets(source_id); + // A serving deployment has a runtime document, so a statement + // source is compiled against the file it will actually read. The + // statement's strong check runs here, at startup, rather than on + // the first request that needs it. + let statement = statement_inputs(source, bundle, Some(&runtime_document.source_extracts)) + .map_err(|_| RuntimeInitializationError::Source)?; + let executor = SourceExecutor::new_with_selector_sets_and_connection_pool( + source, + &allowed_selector_sets, + &runtime_config.outbound_tls, + &runtime_document.ca_bundles, + statement, + Arc::clone(secrets), + &connection_pool, + ) + .map_err(|_| RuntimeInitializationError::Source)?; + sources.insert(source_id.to_owned(), executor); + } + Ok(sources) +} + +/// The private certificate authorities the runtime binds to the access-token +/// issuer's `tlsTrustProfile`, empty when the bundle names none. A named +/// profile that does not resolve, whether unbound, bound to no captured +/// certificate bundle, or bound where system roots are off, refuses, so the +/// issuer is never silently left on the system roots alone. +fn issuer_trust_roots( + bundle: &Bundle, + runtime_document: &RuntimeDocument, +) -> Result, RuntimeInitializationError> { + let Some(profile_name) = bundle + .config + .authentication + .oidc + .tls_trust_profile + .as_deref() + else { + return Ok(Vec::new()); + }; + crate::source::trust_profile_roots( + profile_name, + &runtime_document.config.outbound_tls, + &runtime_document.ca_bundles, + ) + .ok_or(RuntimeInitializationError::IssuerTrust) +} + +/// Build the per-principal rate limiter the bundle declares. +fn rate_limiter(bundle: &Bundle) -> Result { + let configured_limits = &bundle.config.rate_limits; + EvidenceRateLimiter::new(RateLimitConfig { + requests_per_principal_per_minute: u32::try_from( + configured_limits.requests_per_principal_per_minute, + ) + .map_err(|_| RuntimeInitializationError::RateLimit)?, + burst_per_principal: u32::try_from(configured_limits.burst_per_principal) + .map_err(|_| RuntimeInitializationError::RateLimit)?, + failed_selector_attempts_per_principal_authority_per_minute: u32::try_from( + configured_limits.failed_selector_attempts_per_principal_authority_per_minute, + ) + .map_err(|_| RuntimeInitializationError::RateLimit)?, + }) + .map_err(|_| RuntimeInitializationError::RateLimit) +} + +/// Everything startup builds from one captured deployment, with whatever its +/// audit step produced. +struct Startup { + kernel: OfflineKernel, + runtime_config: RuntimeConfig, + authenticator: Arc, + sources: BTreeMap, + audit: A, + material: ValidatedSecretMaterial, + rate_limiter: EvidenceRateLimiter, +} + +/// The one startup sequence, shared by `serve` and by the dependency check +/// beside a running writer, so the two cannot drift apart. They differ only in +/// `audit_step`: `serve` opens the destination as its single writer, the lock-free +/// check proves it without the writer lock. The step receives the configured +/// destination, validated hash key, and hash key version. +async fn assemble( + deployment: DeploymentInputs, + authenticator_override: Option>, + audit_step: impl AsyncFnOnce( + AuditDestination, + Vec, + u32, + ) -> Result, +) -> Result, RuntimeInitializationError> { + let (bundle, runtime_document) = deployment.into_parts(); + let runtime_config = runtime_document.config.clone(); + let bundle = Arc::new(bundle); + let kernel = OfflineKernel::compile(Arc::clone(&bundle)) + .map_err(|_| RuntimeInitializationError::Bundle)?; + + let secrets = Arc::new( + runtime_config + .secret_providers + .resolver() + .map_err(|_| RuntimeInitializationError::Secrets)?, + ); + + let material = validate_secret_material(&bundle, &runtime_config, &secrets).await?; + let destination = runtime_config + .audit + .destination() + .map_err(|_| RuntimeInitializationError::Audit(AuditInitializationFault::Configuration))?; + let audit = audit_step( + destination, + material.audit_secret.expose_secret().to_vec(), + bundle.config.audit.hash_key_version, + ) + .await?; + + let sources = build_sources(&bundle, &runtime_document, &secrets)?; + + let rate_limiter = rate_limiter(&bundle)?; + + let authenticator = match authenticator_override { + Some(authenticator) => authenticator, + None => Arc::new(Authenticator::from_config( + &bundle.config.authentication, + bundle.config.assurance_profile, + issuer_trust_roots(&bundle, &runtime_document)?, + )), + }; + + Ok(Startup { + kernel, + runtime_config, + authenticator, + sources, + audit, + material, + rate_limiter, + }) +} + +/// The readiness proof both readiness and the lock-free dependency check make +/// beyond the audit sink: the subject-binding key, the signer, and every +/// source's credentials. +async fn dependencies_ready( + bundle: &Bundle, + subject_binding_secret: &ProtectedSecret, + signer: &EvidenceSigner, + sources: &BTreeMap, +) -> bool { + if validate_subject_binding_key( + subject_binding_secret.expose_secret(), + bundle.config.subject_binding.key_version, + &bundle.config.service.trust_domain, + ) + .is_err() + || !signer.ensure_ready().await + { + return false; + } + for source in sources.values() { + if source.credentials_ready().await.is_err() { + return false; + } + } + true +} + impl EvidenceRuntime { /// Capture and initialize the complete Version 1 deployment at one revision. pub async fn initialize(runtime_path: &Path) -> Result { @@ -443,95 +718,26 @@ impl EvidenceRuntime { deployment: DeploymentInputs, authenticator_override: Option>, ) -> Result { - let (bundle, runtime_document) = deployment.into_parts(); - let runtime_config = runtime_document.config.clone(); - let runtime_revision = runtime_document.revision().to_owned(); - let bundle = Arc::new(bundle); - let kernel = OfflineKernel::compile(Arc::clone(&bundle)) - .map_err(|_| RuntimeInitializationError::Bundle)?; - - let secrets = Arc::new( - SecretResolver::new( - [SecretProvider::File], - &runtime_config.secret_providers.file.root, - ) - .map_err(|_| RuntimeInitializationError::Secrets)?, - ); - - let material = validate_secret_material(&bundle, &runtime_config, &secrets).await?; - let destination = runtime_config.audit.destination().map_err(|_| { - RuntimeInitializationError::Audit(AuditInitializationFault::Configuration) - })?; - let audit = EvidenceAuditLog::initialize( - destination, - material.audit_secret.expose_secret().to_vec(), - bundle.config.audit.hash_key_version, - ) - .await - .map_err(|error| RuntimeInitializationError::Audit((&error).into()))?; - - let connection_pool = crate::source::SourceConnectionPool::new( - &bundle.config, - &runtime_config.outbound_tls, - &runtime_document.ca_bundles, - Arc::clone(&secrets), + let startup = assemble( + deployment, + authenticator_override, + async |destination, secret, key_version| { + EvidenceAuditLog::initialize(destination, secret, key_version) + .await + .map_err(|error| RuntimeInitializationError::Audit((&error).into())) + }, ) - .map_err(|_| RuntimeInitializationError::Source)?; - let mut sources = BTreeMap::new(); - for (source_id, source) in bundle.config.sources.iter() { - let allowed_selector_sets = bundle.config.source_selector_sets(source_id); - // A serving deployment has a runtime document, so a statement - // source is compiled against the file it will actually read. The - // statement's strong check runs here, at startup, rather than on - // the first request that needs it. - let statement = - statement_inputs(source, &bundle, Some(&runtime_document.source_extracts)) - .map_err(|_| RuntimeInitializationError::Source)?; - let executor = SourceExecutor::new_with_selector_sets_and_connection_pool( - source, - &allowed_selector_sets, - &runtime_config.outbound_tls, - &runtime_document.ca_bundles, - statement, - Arc::clone(&secrets), - &connection_pool, - ) - .map_err(|_| RuntimeInitializationError::Source)?; - sources.insert(source_id.to_owned(), executor); - } - - let configured_limits = &bundle.config.rate_limits; - let rate_limiter = EvidenceRateLimiter::new(RateLimitConfig { - requests_per_principal_per_minute: u32::try_from( - configured_limits.requests_per_principal_per_minute, - ) - .map_err(|_| RuntimeInitializationError::RateLimit)?, - burst_per_principal: u32::try_from(configured_limits.burst_per_principal) - .map_err(|_| RuntimeInitializationError::RateLimit)?, - failed_selector_attempts_per_principal_authority_per_minute: u32::try_from( - configured_limits.failed_selector_attempts_per_principal_authority_per_minute, - ) - .map_err(|_| RuntimeInitializationError::RateLimit)?, - }) - .map_err(|_| RuntimeInitializationError::RateLimit)?; - let rate_limiter = Arc::new(rate_limiter); - + .await?; Ok(Self { - kernel, - runtime_config, - runtime_revision, - authenticator: authenticator_override.unwrap_or_else(|| { - Arc::new(Authenticator::from_config( - &bundle.config.authentication, - bundle.config.assurance_profile, - )) - }), - sources, - audit: Arc::new(audit), - signer: material.signer, - jwks: material.jwks, - subject_binding_secret: material.subject_binding_secret, - rate_limiter, + kernel: startup.kernel, + runtime_config: startup.runtime_config, + authenticator: startup.authenticator, + sources: startup.sources, + audit: Arc::new(startup.audit), + signer: startup.material.signer, + jwks: startup.material.jwks, + subject_binding_secret: startup.material.subject_binding_secret, + rate_limiter: Arc::new(startup.rate_limiter), }) } @@ -543,10 +749,6 @@ impl EvidenceRuntime { &self.runtime_config } - pub fn runtime_revision(&self) -> &str { - &self.runtime_revision - } - pub fn jwks(&self) -> &JwksDocument { &self.jwks } @@ -577,23 +779,45 @@ impl EvidenceRuntime { /// failure, not a diagnosis. pub async fn ready(&self) -> bool { self.authenticator.probe_key_source().await; - if validate_subject_binding_key( - self.subject_binding_secret.expose_secret(), - self.bundle().config.subject_binding.key_version, - &self.bundle().config.service.trust_domain, + self.audit.ready().await + && dependencies_ready( + self.bundle(), + &self.subject_binding_secret, + &self.signer, + &self.sources, + ) + .await + } + + /// Prove the runtime dependencies of a candidate staged beside the running + /// instance it will replace, without taking the audit writer lock that + /// instance holds. + /// + /// Everything [`Self::initialize_from`] followed by [`Self::key_source_ready`] + /// and [`Self::ready`] proves is proved here, in the same order, except + /// that the audit destination is checked by [`EvidenceAuditLog::preflight`] + /// rather than opened. `Ok(false)` names a dependency that is unavailable, as + /// readiness does. No listener is bound and no audit event is appended. + pub async fn check_dependencies_without_audit_lock( + deployment: DeploymentInputs, + ) -> Result { + let startup = assemble( + deployment, + None, + async |destination, secret, key_version| { + EvidenceAuditLog::preflight(&destination, secret, key_version) + .map_err(|error| RuntimeInitializationError::Audit((&error).into())) + }, ) - .is_err() - || !self.signer.ensure_ready().await - || !self.audit.ready().await - { - return false; - } - for source in self.sources.values() { - if source.credentials_ready().await.is_err() { - return false; - } - } - true + .await?; + Ok(startup.authenticator.key_source_ready().await + && dependencies_ready( + startup.kernel.bundle(), + &startup.material.subject_binding_secret, + &startup.material.signer, + &startup.sources, + ) + .await) } /// Fail-closed key-source check used only by an explicit deployment @@ -602,7 +826,7 @@ impl EvidenceRuntime { self.authenticator.key_source_ready().await } - /// Attempt the access-token issuer's key set once, so a `jwksUri` this + /// Attempt the access-token issuer's key set once, so a `jwksSource.uri` this /// deployment cannot use is named at startup rather than discovered one /// rejected request at a time. It reports; it does not refuse to start. pub async fn announce_key_source(&self) { @@ -611,7 +835,7 @@ impl EvidenceRuntime { /// The sources whose mounted extract is already older than they allow. /// - /// Startup names these for the same reason it names an unusable `jwksUri`, + /// Startup names these for the same reason it names an unusable `jwksSource.uri`, /// and with the same standing: it reports, it does not refuse to start, and /// it does not decide readiness. Age is an evaluation-time question, asked /// against the instant each evaluation carries, so this is a startup @@ -1183,7 +1407,7 @@ impl EvidenceRuntime { let audit_material = RequestBatchAuditMaterial { assurance_profile: self.bundle().config.assurance_profile, requirement: batch.requirement.clone(), - bundle_revision: self.bundle().revision().to_owned(), + bundle_revision: self.bundle().package_digest().to_owned(), purpose: batch.purpose.clone(), requester_pseudonym, actor_pseudonym, @@ -3216,7 +3440,7 @@ impl EvidenceRuntime { Ok(AuditMaterial { assurance_profile: self.bundle().config.assurance_profile, requirement: resolved.requirement.clone(), - bundle_revision: self.bundle().revision().to_owned(), + bundle_revision: self.bundle().package_digest().to_owned(), purpose: resolved.purpose.clone(), requester_pseudonym, actor_pseudonym, @@ -3265,7 +3489,7 @@ impl EvidenceRuntime { let mut event = EvidenceAuthorizationRefusalAuditEvent::new( self.bundle().config.assurance_profile, operation.to_owned(), - self.bundle().revision().to_owned(), + self.bundle().package_digest().to_owned(), requester_pseudonym, elapsed_millis(started), ); diff --git a/crates/registry-evidence/src/runtime_tests.rs b/crates/registry-evidence/src/runtime_tests.rs index bddb1b4a5e..a14fecf8df 100644 --- a/crates/registry-evidence/src/runtime_tests.rs +++ b/crates/registry-evidence/src/runtime_tests.rs @@ -529,7 +529,7 @@ async fn real_router_serves_all_definitions_concurrently_without_crossing_bounda protected_resource.json::(), json!({ "resource": fixture.runtime.bundle().config.service.public_origin, - "authorization_servers": [fixture.runtime.bundle().config.authentication.issuer], + "authorization_servers": [fixture.runtime.bundle().config.authentication.oidc.issuer()], "jwks_uri": format!( "{}{}", fixture.runtime.bundle().config.service.public_origin, @@ -987,12 +987,12 @@ async fn a_configured_metrics_listener_serves_beside_the_evidence_listener() { fs::read_to_string(&prepared.runtime_path).expect("runtime configuration is readable"); replace_exact( &mut document, - "port: 8080", - &format!("port: {evidence_port}"), + "bind: 127.0.0.1:8080", + &format!("bind: 127.0.0.1:{evidence_port}"), 1, ); document.push_str(&format!( - "metricsListener:\n bindHost: 127.0.0.1\n port: {metrics_port}\n" + "metricsListener:\n bind: 127.0.0.1:{metrics_port}\n" )); // The prepared document is already read-only, as deployment requires, so // this variant is written before the runtime captures it. @@ -1200,13 +1200,13 @@ async fn openapi_route_serves_the_generated_contract_without_authentication_or_s assert_eq!(document.text(), generated[crate::contracts::OPENAPI_FILE]); // The document is static public material: it names no definition, reveals - // no deployment revision, and reaches no source. + // no package digest, and reaches no source. let served = document.json::(); assert_eq!(served["openapi"], json!("3.1.0")); assert!(served["paths"]["/openapi.json"]["get"].is_object()); assert!(!document .text() - .contains(&fixture.runtime.bundle().revision().to_string())); + .contains(&fixture.runtime.bundle().package_digest().to_string())); assert!(fixture .server .received_requests() @@ -1373,6 +1373,7 @@ async fn discovery_omits_an_authority_shape_that_the_runtime_would_deny_as_ambig 1, ); fs::write(&configuration_path, configuration).expect("test configuration is rewritten"); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); let runtime = Arc::new( EvidenceRuntime::initialize_with_authenticator(&prepared.runtime_path, authenticator()) @@ -1439,6 +1440,7 @@ async fn task_grant_router_matches_local_profile_by_issuer_and_refuses_ambiguity 1, ); fs::write(&configuration_path, configuration).expect("local profile id is written"); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); let runtime = Arc::new( EvidenceRuntime::initialize_with_authenticator(&prepared.runtime_path, authenticator()) @@ -1482,6 +1484,7 @@ async fn task_grant_router_matches_local_profile_by_issuer_and_refuses_ambiguity ); configuration.insert_str(requirements_start, &duplicate); fs::write(&configuration_path, configuration).expect("duplicate local profile is written"); + refresh_package_envelope(&ambiguous.bundle_root); make_read_only(&ambiguous.bundle_root); let runtime = Arc::new( EvidenceRuntime::initialize_with_authenticator(&ambiguous.runtime_path, authenticator()) @@ -1543,6 +1546,7 @@ async fn discovery_refuses_duplicate_handles_visible_to_one_requester() { 1, ); fs::write(&configuration_path, configuration).expect("test configuration is rewritten"); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); let runtime = Arc::new( EvidenceRuntime::initialize_with_authenticator(&prepared.runtime_path, authenticator()) @@ -1585,6 +1589,7 @@ async fn discovery_uses_the_bounded_per_principal_request_budget() { 1, ); fs::write(&configuration_path, configuration).expect("test configuration is rewritten"); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); let runtime = Arc::new( EvidenceRuntime::initialize_with_authenticator(&prepared.runtime_path, authenticator()) @@ -1678,10 +1683,9 @@ async fn discovery_declares_the_holder_bound_mode_for_a_holder_bound_requirement } #[tokio::test] -async fn serving_runtime_never_reloads_merges_or_falls_back_after_bundle_capture() { +async fn serving_runtime_never_reloads_merges_or_falls_back_after_package_capture() { let fixture = acceptance_runtime().await; - let captured_revision = fixture.runtime.bundle().revision().to_owned(); - let captured_runtime_revision = fixture.runtime.runtime_revision().to_owned(); + let captured_digest = fixture.runtime.bundle().package_digest().to_owned(); let adult_requirement = adult_request().requirement; let captured_requirement_revision = fixture .runtime @@ -1722,7 +1726,7 @@ async fn serving_runtime_never_reloads_merges_or_falls_back_after_bundle_capture ) .expect("add an unreferenced fallback-like artifact"); - assert_eq!(fixture.runtime.bundle().revision(), captured_revision); + assert_eq!(fixture.runtime.bundle().package_digest(), captured_digest); assert_eq!( fixture .runtime @@ -1730,10 +1734,6 @@ async fn serving_runtime_never_reloads_merges_or_falls_back_after_bundle_capture .configuration_revision(&adult_requirement), Some(captured_requirement_revision.as_str()) ); - assert_eq!( - fixture.runtime.runtime_revision(), - captured_runtime_revision - ); assert_eq!( fixture.runtime.bundle().artifact("evidence.yaml"), Some(captured_config.as_slice()) @@ -1809,16 +1809,14 @@ async fn local_runtime_prepares_a_bearer_free_procedure_and_keeps_the_real_secur ); replace_exact( &mut local, - "jwksUri: https://identity.invalid/.well-known/jwks.json", - &format!("jwksUri: {local_issuer}/.well-known/jwks.json"), + "uri: https://identity.invalid/.well-known/jwks.json", + &format!("uri: {local_issuer}/.well-known/jwks.json"), 1, ); fs::write(&configuration_path, local).expect("local configuration is written"); - let fixture_directory = prepared.bundle_root.join("fixtures"); - for entry in fs::read_dir(&fixture_directory).expect("fixture directory reads") { - fs::remove_file(entry.expect("fixture entry reads").path()) - .expect("unreferenced fixture is removed"); - } + fs::remove_dir_all(prepared.bundle_root.join("fixtures")) + .expect("unreferenced fixture directory is removed"); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); // Close independent expectations before the source exists and before a @@ -4446,6 +4444,7 @@ async fn unsigned_output_requires_both_bundle_and_grant_permission() { ); fs::write(&configuration_path, &configuration).expect("test configuration is rewritten"); regenerate_discovery_description(&prepared.bundle_root); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); let runtime = Arc::new( EvidenceRuntime::initialize_with_authenticator(&prepared.runtime_path, authenticator()) @@ -4495,6 +4494,7 @@ async fn unsigned_output_requires_both_bundle_and_grant_permission() { 1, ); fs::write(&configuration_path, &configuration).expect("test configuration is rewritten"); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); let runtime = Arc::new( EvidenceRuntime::initialize_with_authenticator(&prepared.runtime_path, authenticator()) @@ -4814,6 +4814,7 @@ async fn sd_jwt_vc_acceptance(grant_permits: bool) -> PreparedAcceptance { } fs::write(&configuration_path, &configuration).expect("test configuration is rewritten"); regenerate_discovery_description(&prepared.bundle_root); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); prepared } @@ -4848,6 +4849,7 @@ async fn holder_bound_acceptance() -> PreparedAcceptance { ); fs::write(&configuration_path, &configuration).expect("test configuration is rewritten"); regenerate_discovery_description(&prepared.bundle_root); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); prepared } @@ -5859,6 +5861,7 @@ async fn holder_bound_batch_acceptance(ceiling: u16) -> PreparedAcceptance { ); fs::write(&configuration_path, &configuration).expect("test configuration is rewritten"); regenerate_discovery_description(&prepared.bundle_root); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); prepared } @@ -6821,6 +6824,7 @@ async fn reordered_grant_subjects_resolve_by_role_and_emit_declaration_order() { 1, ); fs::write(&configuration_path, configuration).expect("test configuration is rewritten"); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); let runtime = Arc::new( EvidenceRuntime::initialize_with_authenticator(&prepared.runtime_path, authenticator()) @@ -7073,6 +7077,7 @@ async fn failed_selector_budget_is_enforced_by_the_runtime_and_scoped_to_authori 1, ); fs::write(&configuration_path, configuration).expect("test configuration is rewritten"); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); let runtime = EvidenceRuntime::initialize_with_authenticator(&prepared.runtime_path, authenticator()) @@ -7505,6 +7510,7 @@ async fn runtime_output_gate_rejects_every_fixture_injected_derivation_without_r make_writable(&prepared.bundle_root); fs::write(prepared.bundle_root.join(script_path), script) .expect("test derivation replacement succeeds"); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); let runtime = EvidenceRuntime::initialize_with_authenticator(&prepared.runtime_path, authenticator()) @@ -7663,6 +7669,7 @@ async fn runtime_rejects_an_extra_extracted_fact_before_derivation_or_release() 1, ); fs::write(adapter_path, adapter).expect("test adapter replacement succeeds"); + refresh_package_envelope(&prepared.bundle_root); make_read_only(&prepared.bundle_root); let runtime = EvidenceRuntime::initialize_with_authenticator(&prepared.runtime_path, authenticator()) @@ -9438,6 +9445,48 @@ async fn search_then_fetch_release_still_omits_source_arrays() { assert!(events[2]["record"].get("adapterIds").is_none()); } +/// An operator who pins the package digest starts only on that package. +#[tokio::test] +async fn an_expected_package_digest_admits_only_the_bundle_it_names() { + let prepared = prepare_acceptance("subject-binding-secret-canary-32-bytes-minimum").await; + let digest = DeploymentInputs::load(&prepared.runtime_path) + .expect("the immutable deployment loads") + .bundle() + .package_digest() + .to_owned(); + let original = + fs::read_to_string(&prepared.runtime_path).expect("runtime configuration is readable"); + let root_line = format!(" root: {}\n", prepared.bundle_root.display()); + + let pin = |digest: &str| { + let mut pinned = original.clone(); + replace_exact( + &mut pinned, + &root_line, + &format!("{root_line} expectedDigest: {digest}\n"), + 1, + ); + make_file_writable(&prepared.runtime_path); + fs::write(&prepared.runtime_path, pinned).expect("runtime configuration is rewritten"); + make_file_read_only(&prepared.runtime_path); + }; + + pin(&digest); + DeploymentInputs::load(&prepared.runtime_path).expect("the pinned package loads"); + + let other = format!("sha256:{}", "0".repeat(64)); + assert_ne!(other, digest); + pin(&other); + let refused = DeploymentInputs::load(&prepared.runtime_path) + .expect_err("a different package digest is refused"); + assert!( + refused.to_string().contains(&format!( + "package.expectedDigest is {other} but the package at package.root is {digest}" + )), + "{refused}" + ); +} + #[tokio::test] async fn initialize_from_opens_the_deployment_it_was_handed_not_the_runtime_pathname() { let prepared = prepare_acceptance("subject-binding-secret-canary-32-bytes-minimum").await; @@ -9496,27 +9545,21 @@ async fn initialize_from_opens_the_deployment_it_was_handed_not_the_runtime_path /// wanted to hand initialization a different bundle or runtime document has no /// way to write it. That is a compile-time property, and this test pins the /// runtime half of it: what the accessors report before initialization is what -/// the initialized runtime reports afterwards, on both captures, so a future -/// change that let the pair be rebuilt between the two would show up here as a -/// revision that no longer matches. +/// the initialized runtime reports afterwards, on both captures. #[tokio::test] -async fn initialize_from_serves_the_revisions_the_captured_inputs_carry() { +async fn initialize_from_serves_the_captured_package_and_runtime_config() { let prepared = prepare_acceptance("subject-binding-secret-canary-32-bytes-minimum").await; let deployment = DeploymentInputs::load(&prepared.runtime_path).expect("the immutable deployment loads"); - let captured_bundle = deployment.bundle().revision().to_owned(); - let captured_runtime = deployment.runtime().revision().to_owned(); - assert_ne!( - captured_bundle, captured_runtime, - "the two captures carry independent revisions" - ); + let captured_package = deployment.bundle().package_digest().to_owned(); + let captured_runtime = deployment.runtime().config.clone(); let runtime = EvidenceRuntime::initialize_from(deployment) .await .expect("the captured deployment initializes"); - assert_eq!(runtime.bundle().revision(), captured_bundle); - assert_eq!(runtime.runtime_revision(), captured_runtime); + assert_eq!(runtime.bundle().package_digest(), captured_package); + assert_eq!(runtime.runtime_config(), &captured_runtime); } async fn prepare_acceptance(binding_secret: &str) -> PreparedAcceptance { @@ -9674,6 +9717,7 @@ fn prepare_fixture_root_with_mutation( &audit_path, ceilings, ); + refresh_package_envelope(&bundle_root); make_file_read_only(&runtime_path); make_read_only(&bundle_root); @@ -9685,6 +9729,25 @@ fn prepare_fixture_root_with_mutation( } } +fn refresh_package_envelope(root: &Path) { + for reserved in [ + registry_platform_config::SUM_FILE, + registry_platform_config::REVISION_FILE, + ] { + let path = root.join(reserved); + if path.exists() { + fs::remove_file(path).expect("prior package envelope is removed"); + } + } + registry_platform_config::write_sum_file( + root, + None, + &crate::bundle::evidence_package_limits(), + "evidencectl package", + ) + .expect("package envelope is refreshed"); +} + fn authenticator() -> Authenticator { authenticator_with_optional_actor_claim(None) } @@ -11261,11 +11324,12 @@ fn write_runtime_config( ceilings: &FixtureCeilings, ) { let document = format!( - r#"version: 1 -bundleDirectory: {} + r#"apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: {} listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 @@ -12330,3 +12394,60 @@ async fn sustained_load_holds_one_thousand_requests_per_second() { "one disclosure-release record per released assertion" ); } + +/// Each Transit startup refusal reaches the operator as its own sentence, and +/// the two version faults name the Transit field the operator has to compare +/// the runtime `keyVersion` against. None carries a path or provider response. +#[test] +fn signing_initialization_faults_name_distinct_causes() { + use crate::runtime::SigningInitializationFault; + use registry_platform_crypto::TransitInitializationError; + + let faults = [ + SigningInitializationFault::LocalKey, + SigningInitializationFault::TransitConfiguration, + SigningInitializationFault::GovernedKey, + SigningInitializationFault::SelfTest, + SigningInitializationFault::KeySet, + SigningInitializationFault::Transit(TransitInitializationError::Client), + SigningInitializationFault::Transit(TransitInitializationError::Unavailable), + SigningInitializationFault::Transit(TransitInitializationError::Refused), + SigningInitializationFault::Transit(TransitInitializationError::ProviderFailed), + SigningInitializationFault::Transit(TransitInitializationError::InvalidResponse), + SigningInitializationFault::Transit(TransitInitializationError::Custody), + SigningInitializationFault::Transit(TransitInitializationError::KeyVersionNotCreated), + SigningInitializationFault::Transit(TransitInitializationError::KeyVersionRetired), + SigningInitializationFault::Transit(TransitInitializationError::PublicKeyMismatch), + SigningInitializationFault::Transit(TransitInitializationError::SelfTest), + ]; + let causes = faults + .iter() + .map(|fault| fault.cause()) + .collect::>(); + assert_eq!(causes.len(), faults.len(), "two faults share one cause"); + // A provider-side failure is not a policy problem, so its cause must not + // send the operator to the token policy. + assert!( + !SigningInitializationFault::Transit(TransitInitializationError::ProviderFailed) + .cause() + .contains("policy") + ); + assert!( + SigningInitializationFault::Transit(TransitInitializationError::KeyVersionRetired) + .cause() + .contains("min_encryption_version") + ); + assert!( + SigningInitializationFault::Transit(TransitInitializationError::KeyVersionNotCreated) + .cause() + .contains("latest_version") + ); + assert_eq!( + RuntimeInitializationError::Signing(SigningInitializationFault::Transit( + TransitInitializationError::Unavailable + )) + .to_string(), + "the Evidence signing boundary could not initialize: the Transit provider did not \ + answer on the configured Unix socket (missing socket, refused connection, or timeout)" + ); +} diff --git a/crates/registry-evidence/src/selector.rs b/crates/registry-evidence/src/selector.rs index 4e6a7c71c2..b416bfa36a 100644 --- a/crates/registry-evidence/src/selector.rs +++ b/crates/registry-evidence/src/selector.rs @@ -868,13 +868,8 @@ pub fn resolve_offline_fixture_authorization( .grant_source_issuer .as_deref() .ok_or(AuthorizationError::Unauthorized)?; - let resource = bundle - .config - .authentication - .audiences - .first() - .ok_or(AuthorizationError::Unauthorized)?; - let names = &bundle.config.authentication.claims; + let resource = bundle.config.authentication.oidc.audience(); + let names = &bundle.config.authentication.oidc.claims; let now = chrono::Utc::now().timestamp(); let mut grant_claims = serde_json::json!({ "sub": "offline-fixture-principal", diff --git a/crates/registry-evidence/src/server.rs b/crates/registry-evidence/src/server.rs index eba98bd578..57ad8db594 100644 --- a/crates/registry-evidence/src/server.rs +++ b/crates/registry-evidence/src/server.rs @@ -10,7 +10,7 @@ use std::{ collections::BTreeSet, future::{Future, IntoFuture}, io, - net::{IpAddr, SocketAddr}, + net::SocketAddr, sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -281,9 +281,8 @@ where { let listener_config = runtime.runtime_config().listener.clone(); let metrics_config = runtime.runtime_config().metrics_listener.clone(); - let bundle_revision = runtime.bundle().revision().to_owned(); - let runtime_revision = runtime.runtime_revision().to_owned(); - let listener = bind(&listener_config.bind_host, listener_config.port).await?; + let package_digest = runtime.bundle().package_digest().to_owned(); + let listener = bind(listener_config.bind.socket_addr()).await?; let startup_runtime = Arc::clone(&runtime); let (app, evaluations, metrics) = build_app_with_tracker(runtime); @@ -291,7 +290,7 @@ where // metrics binding fails startup instead of leaving a service that reports // healthy while publishing no telemetry. let metrics_listener = match &metrics_config { - Some(config) => Some(bind(&config.bind_host, config.port).await?), + Some(config) => Some(bind(config.bind.socket_addr()).await?), None => None, }; @@ -302,10 +301,8 @@ where // that won the port. tracing::info!( target: "registry_evidence::startup", - bundle_revision, - runtime_revision, - bind_host = listener_config.bind_host, - port = listener_config.port, + package_digest, + bind = %listener_config.bind.socket_addr(), metrics = metrics_config.is_some(), "evidence service listening" ); @@ -360,18 +357,10 @@ where /// controls neither exclusively. Without the address, the two most common /// causes read alike and neither points at its fix: a port already taken by /// another process, and an address this host does not own. -async fn bind(bind_host: &str, port: u16) -> io::Result { - let ip = bind_host.parse::().map_err(|error| { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("{bind_host} is not an address: {error}"), - ) - })?; - TcpListener::bind(SocketAddr::new(ip, port)) +async fn bind(address: SocketAddr) -> io::Result { + TcpListener::bind(address) .await - .map_err(|error| { - io::Error::new(error.kind(), format!("could not bind {ip}:{port}: {error}")) - }) + .map_err(|error| io::Error::new(error.kind(), format!("could not bind {address}: {error}"))) } /// Serve a pre-bound listener and drain client-bound handlers on shutdown. @@ -862,7 +851,7 @@ async fn protected_resource_metadata( let public_origin = &state.runtime.bundle().config.service.public_origin; let metadata = ProtectedResourceMetadata { resource: public_origin, - authorization_servers: [&state.runtime.bundle().config.authentication.issuer], + authorization_servers: [state.runtime.bundle().config.authentication.oidc.issuer()], jwks_uri: format!( "{}{}", public_origin, @@ -1130,10 +1119,12 @@ mod tests { /// the configuration is wrong or a previous instance is still running. #[tokio::test] async fn a_taken_port_is_refused_by_address() { - let held = bind("127.0.0.1", 0).await.expect("an ephemeral port binds"); + let held = bind(SocketAddr::from(([127, 0, 0, 1], 0))) + .await + .expect("an ephemeral port binds"); let taken = held.local_addr().expect("the held listener has an address"); - let refused = bind("127.0.0.1", taken.port()) + let refused = bind(taken) .await .expect_err("a held port cannot be bound twice"); assert_eq!(refused.kind(), io::ErrorKind::AddrInUse); @@ -1143,17 +1134,6 @@ mod tests { .starts_with(&format!("could not bind 127.0.0.1:{}: ", taken.port())), "the refusal names the address it failed on: {refused}" ); - - let malformed = bind("localhost", 0) - .await - .expect_err("a host name is not an address"); - assert_eq!(malformed.kind(), io::ErrorKind::InvalidInput); - assert!( - malformed - .to_string() - .starts_with("localhost is not an address: "), - "the refusal names the value it could not parse: {malformed}" - ); } #[test] diff --git a/crates/registry-evidence/src/source.rs b/crates/registry-evidence/src/source.rs index 44718f05bb..ab6a0dd5b7 100644 --- a/crates/registry-evidence/src/source.rs +++ b/crates/registry-evidence/src/source.rs @@ -29,7 +29,7 @@ use crate::config::{ is_http_token_byte, is_uri_byte, validate_local_unauthenticated_source_origin, validate_oauth_resource, AcquisitionPosture, CredentialPlacement, DeclaredUnresolvedProblem, EvidenceConfig, FixedRequest, HttpMethod, OutboundTlsConfig, PathBindingConfig, - PreparationChannelPolicy, SchemaFault, SecretRef, SelectorInput, SourceAuthentication, + PreparationChannelPolicy, SchemaFault, SecretReference, SelectorInput, SourceAuthentication, SourceConfig, SourceConnectionConfig, SourceSelectorSet, SqliteParameterBinding, SqliteRequest, RESERVED_SQL_PARAMETER, }; @@ -625,16 +625,16 @@ enum PathBindingPlan { enum AuthenticationPlan { None, Basic { - username_ref: SecretRef, - password_ref: SecretRef, + username_ref: SecretReference, + password_ref: SecretReference, }, StaticAuthorization { - token_ref: SecretRef, + token_ref: SecretReference, scheme: String, }, StaticApiKey { header_name: HeaderName, - value_ref: SecretRef, + value_ref: SecretReference, }, Oauth2(Box), } @@ -645,11 +645,11 @@ enum AuthenticationPlan { /// is where that alternation becomes a choice the runtime cannot get wrong. enum OauthClientAuthentication { ClientSecret { - secret_ref: SecretRef, + secret_ref: SecretReference, placement: CredentialPlacement, }, PrivateKeyJwt { - key_ref: SecretRef, + key_ref: SecretReference, /// Resolved at compile time to the configured audience or, when the /// bundle names none, the token endpoint. audience: String, @@ -658,7 +658,7 @@ enum OauthClientAuthentication { struct OauthPlan { token_endpoint: Url, - client_id_ref: SecretRef, + client_id_ref: SecretReference, client_authentication: OauthClientAuthentication, scope: Option, audience: Option, @@ -1544,27 +1544,8 @@ fn build_client( return builder.build().map_err(|_| SourceError::InvalidPlan); } let (tls, captured_ca_bundles) = outbound_tls.ok_or(SourceError::InvalidPlan)?; - if !tls.system_roots { - return Err(SourceError::InvalidPlan); - } - let binding = tls - .trust_profiles - .get(profile_name) - .ok_or(SourceError::InvalidPlan)?; - if binding.ca_bundle_file.is_empty() { - return Err(SourceError::InvalidPlan); - } - let pem = captured_ca_bundles - .get(profile_name) + let certificates = trust_profile_roots(profile_name, tls, captured_ca_bundles) .ok_or(SourceError::InvalidPlan)?; - if pem.is_empty() || pem.len() as u64 > PRIVATE_CA_MAXIMUM_BYTES { - return Err(SourceError::InvalidPlan); - } - let certificates = - reqwest::Certificate::from_pem_bundle(pem).map_err(|_| SourceError::InvalidPlan)?; - if certificates.is_empty() { - return Err(SourceError::InvalidPlan); - } for certificate in certificates { builder = builder.add_root_certificate(certificate); } @@ -1572,6 +1553,32 @@ fn build_client( builder.build().map_err(|_| SourceError::InvalidPlan) } +/// The private certificate authorities a runtime-bound TLS trust profile +/// names, to be trusted beside the system roots, or `None` when the profile is +/// unbound, empty, oversized, or not a PEM certificate bundle. +pub(crate) fn trust_profile_roots( + profile_name: &str, + tls: &OutboundTlsConfig, + captured_ca_bundles: &BTreeMap>, +) -> Option> { + if !tls.system_roots { + return None; + } + let binding = tls.trust_profiles.get(profile_name)?; + if binding.ca_bundle_file.is_empty() { + return None; + } + let pem = captured_ca_bundles.get(profile_name)?; + if pem.is_empty() || pem.len() as u64 > PRIVATE_CA_MAXIMUM_BYTES { + return None; + } + let certificates = reqwest::Certificate::from_pem_bundle(pem).ok()?; + if certificates.is_empty() { + return None; + } + Some(certificates) +} + fn conservative_selector_sets( source: &SourceConfig, ) -> Result, SourceError> { @@ -2274,7 +2281,7 @@ impl OauthPlan { fn client_assertion( &self, secrets: &SecretResolver, - key_ref: &SecretRef, + key_ref: &SecretReference, audience: &str, client_id: &str, ) -> Result, SourceError> { @@ -2430,7 +2437,7 @@ fn project_value(value: &JsonValue, node: &ProjectionNode) -> Result Result { resolver .resolve(reference.as_str()) @@ -3691,10 +3698,10 @@ mod tests { let plan = OauthPlan { token_endpoint: Url::parse("http://127.0.0.1:1/token") .expect("synthetic endpoint parses"), - client_id_ref: SecretRef::parse("secret:file/missing-client-id") + client_id_ref: SecretReference::parse("secret:file/missing-client-id") .expect("secret reference parses"), client_authentication: OauthClientAuthentication::ClientSecret { - secret_ref: SecretRef::parse("secret:file/missing-client-secret") + secret_ref: SecretReference::parse("secret:file/missing-client-secret") .expect("secret reference parses"), placement: CredentialPlacement::FormBody, }, @@ -3771,10 +3778,10 @@ mod tests { let plan = OauthPlan { token_endpoint: Url::parse(&format!("{}/token", server.uri())) .expect("token endpoint parses"), - client_id_ref: SecretRef::parse("secret:file/oauth-client-id") + client_id_ref: SecretReference::parse("secret:file/oauth-client-id") .expect("secret reference parses"), client_authentication: OauthClientAuthentication::ClientSecret { - secret_ref: SecretRef::parse("secret:file/oauth-client-secret") + secret_ref: SecretReference::parse("secret:file/oauth-client-secret") .expect("secret reference parses"), placement: CredentialPlacement::FormBody, }, diff --git a/crates/registry-evidence/src/source_sqlite.rs b/crates/registry-evidence/src/source_sqlite.rs index aadf315d40..134554cd44 100644 --- a/crates/registry-evidence/src/source_sqlite.rs +++ b/crates/registry-evidence/src/source_sqlite.rs @@ -307,7 +307,7 @@ impl SqliteExtractSource { /// Open the exact snapshot already validated and digest-bound by the /// runtime document. Re-capturing by path here could accept a replacement - /// under the old runtime revision. + /// after startup validation. pub(crate) fn open_captured( source: &SourceConfig, statement_sql: &str, diff --git a/crates/registry-evidence/tests/cli.rs b/crates/registry-evidence/tests/cli.rs index 0de2893799..40bbb86de1 100644 --- a/crates/registry-evidence/tests/cli.rs +++ b/crates/registry-evidence/tests/cli.rs @@ -3,7 +3,7 @@ use std::{ collections::BTreeMap, fs, - io::{Read as _, Write as _}, + io::Write as _, net::{TcpListener, TcpStream}, os::unix::fs::{MetadataExt as _, PermissionsExt as _}, path::{Path, PathBuf}, @@ -35,6 +35,39 @@ struct JwksServer { impl JwksServer { fn start() -> Self { + Self::start_with(None) + } + + /// The same endpoint served over HTTPS, under a certificate a private + /// certificate authority signed. Returns the authority's PEM certificate, + /// which no platform root store holds. + fn start_under_private_ca() -> (Self, String) { + use tokio_rustls::rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; + + let mut authority = + rcgen::CertificateParams::new(Vec::::new()).expect("private CA parameters"); + authority.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let authority_key = rcgen::KeyPair::generate().expect("private CA key"); + let authority = authority + .self_signed(&authority_key) + .expect("private CA certificate"); + let server_key = rcgen::KeyPair::generate().expect("server key"); + let server = rcgen::CertificateParams::new(vec!["127.0.0.1".to_owned()]) + .expect("server parameters") + .signed_by(&server_key, &authority, &authority_key) + .expect("the private CA signs the server certificate"); + let config = tokio_rustls::rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert( + vec![server.der().clone()], + PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(server_key.serialize_der())), + ) + .expect("TLS server configuration"); + let server = Self::start_with(Some(Arc::new(config))); + (server, pem_certificate(authority.der())) + } + + fn start_with(tls: Option>) -> Self { let listener = TcpListener::bind("127.0.0.1:0").expect("JWKS server binds"); listener .set_nonblocking(true) @@ -57,33 +90,30 @@ impl JwksServer { ); let stop = Arc::new(AtomicBool::new(false)); let worker_stop = Arc::clone(&stop); + let tls_scheme = tls.is_some(); let worker = std::thread::spawn(move || { while !worker_stop.load(Ordering::Acquire) { match listener.accept() { - Ok((mut stream, _)) => { + Ok((stream, _)) => { + // A BSD socket inherits the listener's nonblocking + // mode, which would read an empty request whenever + // the client's bytes had not yet arrived. + let _ = stream.set_nonblocking(false); let _ = stream.set_read_timeout(Some(Duration::from_secs(2))); - let mut request = Vec::with_capacity(1_024); - while request.len() < 8_192 - && !request.windows(4).any(|window| window == b"\r\n\r\n") - { - let mut chunk = [0_u8; 512]; - let Ok(read) = stream.read(&mut chunk) else { - break; - }; - if read == 0 { - break; + match &tls { + None => answer_jwks_request(stream, &response), + Some(config) => { + let Ok(connection) = + tokio_rustls::rustls::ServerConnection::new(Arc::clone(config)) + else { + continue; + }; + answer_jwks_request( + tokio_rustls::rustls::StreamOwned::new(connection, stream), + &response, + ); } - request.extend_from_slice(&chunk[..read]); } - let valid_request = - request.starts_with(b"GET /.well-known/jwks.json HTTP/1.1\r\n"); - let rejected = b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; - let _ = stream.write_all(if valid_request { - response.as_ref() - } else { - rejected - }); - let _ = stream.flush(); } Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { std::thread::sleep(Duration::from_millis(10)); @@ -92,8 +122,9 @@ impl JwksServer { } } }); + let scheme = if tls_scheme { "https" } else { "http" }; Self { - origin: format!("http://{address}"), + origin: format!("{scheme}://{address}"), stop, worker: Some(worker), } @@ -104,6 +135,40 @@ impl JwksServer { } } +/// Read one request and answer it with the key set, or with 404 for any other +/// request line. +fn answer_jwks_request(mut stream: impl std::io::Read + std::io::Write, response: &[u8]) { + let mut request = Vec::with_capacity(1_024); + while request.len() < 8_192 && !request.windows(4).any(|window| window == b"\r\n\r\n") { + let mut chunk = [0_u8; 512]; + let Ok(read) = stream.read(&mut chunk) else { + break; + }; + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + let valid_request = request.starts_with(b"GET /.well-known/jwks.json HTTP/1.1\r\n"); + let rejected = b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let _ = stream.write_all(if valid_request { response } else { rejected }); + let _ = stream.flush(); +} + +/// One DER certificate as a PEM block, the form a `caBundleFile` holds. +fn pem_certificate(der: &[u8]) -> String { + use base64::Engine as _; + + let encoded = base64::engine::general_purpose::STANDARD.encode(der); + let body = encoded + .as_bytes() + .chunks(64) + .map(|line| std::str::from_utf8(line).expect("base64 is ASCII")) + .collect::>() + .join("\n"); + format!("-----BEGIN CERTIFICATE-----\n{body}\n-----END CERTIFICATE-----\n") +} + impl Drop for JwksServer { fn drop(&mut self) { self.stop.store(true, Ordering::Release); @@ -189,6 +254,7 @@ impl ReferenceProject { fn sealed(&self, run: impl FnOnce(&Path) -> T) -> T { let bundle = self.root.path().join("bundle"); let runtime = self.root.path().join("runtime.yaml"); + refresh_package_envelope(&bundle); set_tree_mode(&bundle, 0o555, 0o444); fs::set_permissions(&runtime, fs::Permissions::from_mode(0o444)) .expect("set immutable runtime mode"); @@ -215,7 +281,7 @@ fn actual_binary_checks_and_evaluates_an_immutable_project() { assert_success( &check, - "Evidence deployment ", + "Evidence package ", " passed check (3 requirements)\n", ); assert_success( @@ -234,7 +300,7 @@ fn dependency_check_proves_the_real_runtime_boundaries() { assert_success( &deployment.check_with_runtime_dependencies(), - "Evidence deployment ", + "Evidence package ", " passed check (4 requirements)\n", ); } @@ -248,7 +314,7 @@ fn dependency_check_accepts_an_audit_sink_inside_the_required_root() { assert_success( &deployment.check_with_audit_under(deployment.root.path()), - "Evidence deployment ", + "Evidence package ", " passed check (4 requirements)\n", ); } @@ -308,7 +374,7 @@ fn dependency_check_accepts_a_stdout_audit_destination() { assert_success( &deployment.check_with_runtime_dependencies(), - "Evidence deployment ", + "Evidence package ", " passed check (4 requirements)\n", ); assert!(!deployment.path("audit.jsonl").exists()); @@ -375,6 +441,83 @@ fn dependency_check_fails_closed_when_the_jwks_endpoint_is_unavailable() { assert!(!String::from_utf8_lossy(&output.stderr).contains(&origin)); } +/// An issuer whose key set is served under a private certificate authority is +/// reachable only through the trust profile the bundle names for it: the +/// system roots alone refuse its certificate, and naming the profile adds that +/// one authority for that one connection. +#[test] +fn dependency_check_trusts_a_private_ca_issuer_only_through_its_named_profile() { + let (key_server, authority) = JwksServer::start_under_private_ca(); + let deployment = Deployment::stage("all-definitions"); + deployment.stage_acceptance_secrets(); + deployment.point_authentication_to(key_server.origin()); + + let refused = deployment.check_with_runtime_dependencies(); + assert!( + !refused.status.success(), + "an issuer under an untrusted private CA passed check" + ); + assert_eq!( + std::str::from_utf8(&refused.stderr).expect("diagnostic is UTF-8"), + "evidence: a required runtime dependency is unavailable\n" + ); + + deployment.trust_issuer_through("issuer-pki", &authority); + let output = deployment.check_with_runtime_dependencies(); + + assert_success( + &output, + "Evidence package ", + " passed check (4 requirements)\n", + ); +} + +/// A trust profile trusts the authority it names and nothing else: an issuer +/// whose certificate another private authority signed is still refused. +#[test] +fn dependency_check_refuses_an_issuer_the_named_profile_does_not_vouch_for() { + let (key_server, _authority) = JwksServer::start_under_private_ca(); + let (_other_server, other_authority) = JwksServer::start_under_private_ca(); + let deployment = Deployment::stage("all-definitions"); + deployment.stage_acceptance_secrets(); + deployment.point_authentication_to(key_server.origin()); + deployment.trust_issuer_through("issuer-pki", &other_authority); + + let output = deployment.check_with_runtime_dependencies(); + + assert!( + !output.status.success(), + "a profile naming another authority admitted the issuer" + ); + assert_eq!( + std::str::from_utf8(&output.stderr).expect("diagnostic is UTF-8"), + "evidence: a required runtime dependency is unavailable\n" + ); +} + +/// A profile bound to a file that holds no certificate refuses when the +/// runtime file loads, rather than leaving the issuer on the system roots +/// alone. +#[test] +fn dependency_check_refuses_an_issuer_profile_that_is_not_a_certificate_bundle() { + let (key_server, _authority) = JwksServer::start_under_private_ca(); + let deployment = Deployment::stage("all-definitions"); + deployment.stage_acceptance_secrets(); + deployment.point_authentication_to(key_server.origin()); + deployment.trust_issuer_through("issuer-pki", "not a certificate\n"); + + let output = deployment.check_with_runtime_dependencies(); + + assert!( + !output.status.success(), + "a profile holding no certificate passed check" + ); + assert_eq!( + std::str::from_utf8(&output.stderr).expect("diagnostic is UTF-8"), + "evidence: deployment artifact is invalid: artifact runtime.yaml: TLS CA bundle contains non-certificate PEM data\n" + ); +} + #[tokio::test] async fn dependency_check_fails_when_an_audit_writer_already_holds_the_destination() { use registry_evidence::audit::EvidenceAuditLog; @@ -410,6 +553,186 @@ async fn dependency_check_fails_when_an_audit_writer_already_holds_the_destinati drop(writer); } +/// A candidate staged beside the instance it will replace shares that +/// instance's audit path, so the writer lock is held by design. The lock-free +/// form proves everything else the candidate's start needs and leaves the +/// running writer untouched. +#[tokio::test] +async fn dependency_check_without_the_audit_lock_passes_beside_a_running_writer() { + use registry_evidence::audit::EvidenceAuditLog; + use registry_platform_audit::{AuditDestination, FileDestination}; + + let key_server = JwksServer::start(); + let deployment = Deployment::stage("all-definitions"); + deployment.stage_acceptance_secrets(); + deployment.point_authentication_to(key_server.origin()); + let writer = EvidenceAuditLog::initialize( + AuditDestination::File( + FileDestination::new(deployment.path("audit.jsonl")) + .expect("the audit destination is valid"), + ), + b"audit-hash-secret-32-bytes-minimum-value".to_vec(), + 1, + ) + .await + .expect("first audit writer initializes"); + + let output = deployment.check_without_audit_lock(); + + assert_success( + &output, + "Evidence package ", + " passed check (4 requirements)\n", + ); + assert!( + writer.ready().await, + "the lock-free check disturbed the running writer" + ); + drop(writer); +} + +/// One audit boundary the lock-free check must still refuse, with the exact +/// operator text, and whether a writer holds the destination while it runs. +struct LockFreeAuditCase { + label: &'static str, + running_writer: bool, + break_audit: fn(&Deployment), + expected: &'static str, +} + +/// Leaving the lock to the running writer does not leave the rest of the audit +/// boundary unchecked: a mode, a file the candidate could not write, and an +/// active file ending in an incomplete entry each refuse as they would at +/// `serve`. +#[tokio::test] +async fn dependency_check_without_the_audit_lock_still_refuses_an_unusable_audit_boundary() { + use registry_evidence::audit::EvidenceAuditLog; + use registry_platform_audit::{AuditDestination, FileDestination}; + + const OWNER_ONLY: &str = "evidence: runtime audit initialization failed: the audit file could \ + not be opened: audit files must be owner-only, singly linked \ + regular files; chmod them 0600 and remove any extra hard link\n"; + let cases = [ + LockFreeAuditCase { + label: "an audit file readable beyond its owner", + running_writer: false, + break_audit: |deployment| { + deployment.stage_audit_file(""); + set_mode(&deployment.path("audit.jsonl"), 0o644); + }, + expected: OWNER_ONLY, + }, + LockFreeAuditCase { + label: "an audit file the service owner cannot write", + running_writer: false, + break_audit: |deployment| { + deployment.stage_audit_file(""); + set_mode(&deployment.path("audit.jsonl"), 0o400); + }, + expected: "evidence: runtime audit initialization failed: the audit file could not \ + be opened: audit file is not readable and writable\n", + }, + LockFreeAuditCase { + label: "a running writer's audit file made readable beyond its owner", + running_writer: true, + break_audit: |deployment| set_mode(&deployment.path("audit.jsonl"), 0o644), + expected: OWNER_ONLY, + }, + LockFreeAuditCase { + label: "an audit file whose last entry was torn by an interrupted write", + running_writer: false, + break_audit: |deployment| { + deployment.stage_audit_file("{\"written\":\"before the interruption\""); + }, + expected: "evidence: runtime audit initialization failed: the audit file could not \ + be opened: audit file has an incomplete final entry; archive it and \ + restart with a fresh path\n", + }, + ]; + + for case in cases { + let key_server = JwksServer::start(); + let deployment = Deployment::stage("all-definitions"); + deployment.stage_acceptance_secrets(); + deployment.point_authentication_to(key_server.origin()); + let writer = if case.running_writer { + Some( + EvidenceAuditLog::initialize( + AuditDestination::File( + FileDestination::new(deployment.path("audit.jsonl")) + .expect("the audit destination is valid"), + ), + b"audit-hash-secret-32-bytes-minimum-value".to_vec(), + 1, + ) + .await + .expect("first audit writer initializes"), + ) + } else { + None + }; + (case.break_audit)(&deployment); + let output = deployment.check_without_audit_lock(); + drop(writer); + + assert!( + !output.status.success(), + "{}: the lock-free check accepted an unusable audit boundary", + case.label + ); + assert!( + output.stdout.is_empty(), + "{}: a failed dependency check wrote output", + case.label + ); + assert_eq!( + std::str::from_utf8(&output.stderr).expect("diagnostic is UTF-8"), + case.expected, + "{}: unexpected diagnostic", + case.label + ); + } +} + +#[test] +fn dependency_check_refuses_an_audit_directory_the_candidate_could_not_write() { + let key_server = JwksServer::start(); + let deployment = Deployment::stage("all-definitions"); + deployment.stage_acceptance_secrets(); + deployment.point_authentication_to(key_server.origin()); + set_mode(deployment.root.path(), 0o500); + let output = deployment.check_without_audit_lock(); + set_mode(deployment.root.path(), 0o700); + + assert!( + !output.status.success(), + "the lock-free check accepted an audit directory it could not write" + ); + assert_eq!( + std::str::from_utf8(&output.stderr).expect("diagnostic is UTF-8"), + "evidence: runtime audit initialization failed: the audit file could not be opened: \ + audit directory is not readable, writable, and searchable\n" + ); +} + +#[test] +fn check_accepts_the_lock_free_form_only_with_the_dependency_check() { + let deployment = Deployment::stage("all-definitions"); + deployment.stage_acceptance_secrets(); + deployment.seal(); + let output = invoke( + &deployment.path("runtime.yaml"), + &["check", "--without-audit-lock"], + ); + deployment.unseal(); + + assert!( + !output.status.success(), + "the lock-free form ran without the dependency check it qualifies" + ); + assert!(String::from_utf8_lossy(&output.stderr).contains("--require-runtime-dependencies")); +} + #[test] fn check_refuses_an_already_stale_bound_extract_with_only_the_governed_source() { let root = tempfile::tempdir().expect("temporary deployment"); @@ -485,6 +808,7 @@ fn check_refuses_an_already_stale_bound_extract_with_only_the_governed_source() ); let runtime_path = root.path().join("runtime.yaml"); fs::write(&runtime_path, runtime).expect("stage runtime"); + refresh_package_envelope(&bundle); set_tree_mode(&bundle, 0o555, 0o444); fs::set_permissions(&runtime_path, fs::Permissions::from_mode(0o444)).expect("seal runtime"); @@ -1516,11 +1840,11 @@ fn failure_cases() -> Vec { break_deployment: |deployment| { deployment.replace( "bundle/evidence.yaml", - " principalClaim: sub\n", - &format!(" principalClaim: sub\n unknownField: {CANARY}\n"), + " principalClaim: sub\n", + &format!(" principalClaim: sub\n unknownField: {CANARY}\n"), ); }, - prefix: "evidence: deployment configuration is invalid: artifact evidence.yaml: unknown field at authentication (line ", + prefix: "evidence: deployment configuration is invalid: artifact evidence.yaml: unknown field at authentication.oidc (line ", suffix: ")\n", needs_runtime: false, }, @@ -1544,11 +1868,11 @@ fn failure_cases() -> Vec { break_deployment: |deployment| { deployment.replace( "bundle/evidence.yaml", - " kind: oidc-access-token\n", - &format!(" kind: {CANARY}\n"), + " format: flattened-jws-json\n", + &format!(" format: {CANARY}\n"), ); }, - prefix: "evidence: deployment configuration is invalid: artifact evidence.yaml: field value is not one of the accepted variants at authentication.kind (line ", + prefix: "evidence: deployment configuration is invalid: artifact evidence.yaml: field value is not one of the accepted variants at signing.format (line ", suffix: ")\n", needs_runtime: false, }, @@ -1674,8 +1998,8 @@ fn failure_cases() -> Vec { break_deployment: |deployment| { deployment.append("runtime.yaml", &format!("unknownField: {CANARY}\n")); }, - prefix: "evidence: deployment configuration is invalid: artifact runtime.yaml: unknown field (line ", - suffix: ")\n", + prefix: "evidence: deployment configuration is invalid: artifact runtime.yaml: unknown field at unknownField\n", + suffix: "", needs_runtime: true, }, FailureCase { @@ -1684,12 +2008,12 @@ fn failure_cases() -> Vec { break_deployment: |deployment| { deployment.replace( "runtime.yaml", - " port: 8080\n", - &format!(" port: \"{CANARY}\"\n"), + " maximumRequestBytes: 65536\n", + &format!(" maximumRequestBytes: \"{CANARY}\"\n"), ); }, - prefix: "evidence: deployment configuration is invalid: artifact runtime.yaml: field has the wrong type at listener.port (line ", - suffix: ")\n", + prefix: "evidence: deployment configuration is invalid: artifact runtime.yaml: field has the wrong type at listener.maximumRequestBytes\n", + suffix: "", needs_runtime: true, }, FailureCase { @@ -1698,14 +2022,40 @@ fn failure_cases() -> Vec { break_deployment: |deployment| { deployment.replace_line( "runtime.yaml", - "bundleDirectory: ", - &format!("bundleDirectory: relative/{CANARY}\n"), + " root: ", + &format!(" root: relative/{CANARY}\n"), ); }, - prefix: "evidence: deployment configuration is invalid: artifact runtime.yaml: absolute operator path is invalid\n", + prefix: "evidence: deployment configuration is invalid: artifact runtime.yaml: package root must be an absolute path (package.root)\n", suffix: "", needs_runtime: true, }, + // A key an earlier runtime grammar accepted is refused with the key + // that replaced it, and the value it carried is never echoed. + FailureCase { + label: "removed runtime key", + bundle: "all-definitions", + break_deployment: |deployment| { + deployment.append("runtime.yaml", &format!("bundleDirectory: /{CANARY}\n")); + }, + prefix: "evidence: deployment configuration is invalid: artifact runtime.yaml: key is no longer accepted at bundleDirectory; declare package.root as the absolute path of the package directory\n", + suffix: "", + needs_runtime: true, + }, + FailureCase { + label: "removed bundle key", + bundle: "all-definitions", + break_deployment: |deployment| { + deployment.replace( + "bundle/evidence.yaml", + " principalClaim: sub\n", + &format!(" principalClaim: sub\n audiences: [{CANARY}]\n"), + ); + }, + prefix: "evidence: deployment configuration is invalid: artifact evidence.yaml: key is no longer accepted at authentication.oidc.audiences; declare the one accepted audience as authentication.oidc.audience\n", + suffix: "", + needs_runtime: false, + }, // Nothing is broken here: an operator runtime file that says nothing // about acquisition capabilities enables nothing beyond the frozen // Version 1 acquisition forms, so a bundle that requires a gated kind @@ -1831,6 +2181,7 @@ fn bundle_check_names_a_safe_artifact_and_a_value_free_cause_for_a_broken_statem ) .expect("write broken statement"); + refresh_package_envelope(&bundle); set_tree_mode(&bundle, 0o555, 0o444); let output = invoke_bundle_check(&bundle); set_tree_mode(&bundle, 0o755, 0o644); @@ -1874,7 +2225,7 @@ fn check_accepts_a_gated_acquisition_kind_the_operator_enabled() { assert_success( &deployment.check(), - "Evidence deployment ", + "Evidence package ", " passed check (1 requirements)\n", ); } @@ -1894,7 +2245,33 @@ fn check_rejects_secret_material_the_server_would_refuse_at_startup() { SecretFailureCase { label: "signing key differs from the governed active public JWK", break_secrets: |deployment| deployment.write_mismatched_signing_key(), - expected: "evidence: runtime signing initialization failed\n", + expected: "evidence: runtime signing initialization failed: the signing key is not \ + the bundle's governed active public JWK\n", + }, + SecretFailureCase { + label: "Transit signer socket is missing", + break_secrets: |deployment| { + // Transit custody is what the evidence-grade profile demands, + // so the staged bundle returns to the profile it ships with. + deployment.replace( + "bundle/evidence.yaml", + "assuranceProfile: local", + "assuranceProfile: evidence-grade", + ); + let socket = deployment.path("transit.sock"); + deployment.replace( + "runtime.yaml", + "signer:\n kind: local-jwk\n privateKeyRef: secret:file/signing-key\n", + &format!( + "signer:\n kind: transit\n unixSocketPath: {}\n mount: transit\n \ + keyName: evidence-signing\n keyVersion: 7\n timeoutMilliseconds: 2000\n", + socket.display() + ), + ); + }, + expected: "evidence: runtime signing initialization failed: the Transit provider did \ + not answer on the configured Unix socket (missing socket, refused \ + connection, or timeout)\n", }, SecretFailureCase { label: "audit hash key below the minimum length", @@ -3092,11 +3469,12 @@ impl Deployment { fn runtime_document(&self) -> String { format!( - "version: 1 -bundleDirectory: {bundle} + "apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: {bundle} listener: - bindHost: 127.0.0.1 - port: {port} + bind: 127.0.0.1:{port} tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 @@ -3195,8 +3573,37 @@ outboundTls: ); self.replace( "bundle/evidence.yaml", - " jwksUri: https://identity.invalid/.well-known/jwks.json\n", - &format!(" jwksUri: {origin}/.well-known/jwks.json\n"), + " uri: https://identity.invalid/.well-known/jwks.json\n", + &format!(" uri: {origin}/.well-known/jwks.json\n"), + ); + } + + /// Name `profile` as the issuer's TLS trust profile in the bundle and bind + /// it in the runtime file to a read-only file holding `authority_pem`. + fn trust_issuer_through(&self, profile: &str, authority_pem: &str) { + let bundle_path = self.path(&format!("{profile}.pem")); + fs::write(&bundle_path, authority_pem).expect("stage the issuer CA bundle"); + fs::set_permissions(&bundle_path, fs::Permissions::from_mode(0o444)) + .expect("seal the issuer CA bundle"); + let text = + fs::read_to_string(self.path("bundle/evidence.yaml")).expect("read staged bundle"); + let jwks_line = text + .lines() + .find(|line| line.starts_with(" uri: ")) + .expect("the bundle names a JWKS URI") + .to_owned(); + self.replace( + "bundle/evidence.yaml", + &format!("{jwks_line}\n"), + &format!("{jwks_line}\n tlsTrustProfile: {profile}\n"), + ); + self.replace( + "runtime.yaml", + " trustProfiles: {}\n", + &format!( + " trustProfiles:\n {profile}: {{caBundleFile: {}}}\n", + bundle_path.display() + ), ); } @@ -3226,9 +3633,9 @@ outboundTls: /// Start `serve` against the sealed deployment. fn serve(&self) -> Child { Command::new(env!("CARGO_BIN_EXE_evidence")) - .arg("--runtime") - .arg(self.path("runtime.yaml")) .arg("serve") + .arg("--runtime-config") + .arg(self.path("runtime.yaml")) .env_remove("REGISTRY_EVIDENCE_RUNTIME") .stdout(Stdio::null()) .stderr(Stdio::null()) @@ -3265,6 +3672,22 @@ outboundTls: output } + /// Run the dependency proof without taking the audit writer lock, the form + /// that checks a candidate beside the instance holding that lock. + fn check_without_audit_lock(&self) -> Output { + self.seal(); + let output = invoke( + &self.path("runtime.yaml"), + &[ + "check", + "--require-runtime-dependencies", + "--without-audit-lock", + ], + ); + self.unseal(); + output + } + /// Run the dependency proof and additionally require the configured audit /// file to resolve inside `root`, the path an operator declares persistent. fn check_with_audit_under(&self, root: &Path) -> Output { @@ -3293,6 +3716,7 @@ outboundTls: } fn seal(&self) { + refresh_package_envelope(&self.path("bundle")); set_tree_mode(&self.path("bundle"), 0o555, 0o444); fs::set_permissions(self.path("runtime.yaml"), fs::Permissions::from_mode(0o444)) .expect("seal runtime"); @@ -3305,6 +3729,31 @@ outboundTls: } } +fn refresh_package_envelope(root: &Path) { + for reserved in [ + registry_platform_config::SUM_FILE, + registry_platform_config::REVISION_FILE, + ] { + let path = root.join(reserved); + if path.exists() { + fs::remove_file(path).expect("prior package envelope is removed"); + } + } + registry_platform_config::write_sum_file( + root, + None, + ®istry_platform_config::PackageLimits { + max_files: 1_024, + max_file_bytes: 1024 * 1024, + max_total_bytes: 16 * 1024 * 1024, + max_depth: 3, + max_path_bytes: 128, + }, + "evidencectl package", + ) + .expect("package envelope is refreshed"); +} + impl Drop for Deployment { fn drop(&mut self) { if self.path("bundle").is_dir() { @@ -3325,9 +3774,9 @@ fn write_private_json(path: &Path, value: &Value) { fn invoke_local_relying_procedure(runtime: &Path, input: &Path, stdin: &[u8]) -> Output { let mut child = Command::new(env!("CARGO_BIN_EXE_evidence")) - .arg("--runtime") - .arg(runtime) .arg("prepare-local-relying-procedure") + .arg("--runtime-config") + .arg(runtime) .arg("--input") .arg(input) .env_remove("REGISTRY_EVIDENCE_RUNTIME") @@ -3344,19 +3793,57 @@ fn invoke_local_relying_procedure(runtime: &Path, input: &Path, stdin: &[u8]) -> child.wait_with_output().expect("evidence command exits") } +#[test] +fn the_removed_runtime_inputs_are_refused_with_their_replacement_named() { + let flag = Command::new(env!("CARGO_BIN_EXE_evidence")) + .args(["check", "--runtime", "/etc/registry-evidence/runtime.yaml"]) + .env_remove("REGISTRY_EVIDENCE_RUNTIME") + .output() + .expect("evidence binary starts"); + assert!(!flag.status.success(), "the removed flag was accepted"); + assert_eq!( + std::str::from_utf8(&flag.stderr).expect("stderr is UTF-8"), + "evidence: --runtime is no longer accepted; pass --runtime-config FILE\n" + ); + assert!(flag.stdout.is_empty(), "the refusal wrote to stdout"); + + let environment = Command::new(env!("CARGO_BIN_EXE_evidence")) + .args([ + "check", + "--runtime-config", + "/etc/registry-evidence/runtime.yaml", + ]) + .env( + "REGISTRY_EVIDENCE_RUNTIME", + "/etc/registry-evidence/runtime.yaml", + ) + .output() + .expect("evidence binary starts"); + assert!( + !environment.status.success(), + "the removed environment variable was ignored" + ); + assert_eq!( + std::str::from_utf8(&environment.stderr).expect("stderr is UTF-8"), + "evidence: REGISTRY_EVIDENCE_RUNTIME is no longer read; unset it and pass --runtime-config FILE\n" + ); +} + +/// Run one runtime subcommand, `arguments[0]`, against `runtime`. fn invoke(runtime: &Path, arguments: &[&str]) -> Output { + let (command, rest) = arguments.split_first().expect("a subcommand is named"); Command::new(env!("CARGO_BIN_EXE_evidence")) - .arg("--runtime") + .arg(command) + .arg("--runtime-config") .arg(runtime) - .args(arguments) + .args(rest) .env_remove("REGISTRY_EVIDENCE_RUNTIME") .output() .expect("evidence binary starts") } /// Run `bundle-check` against a bundle directory, with no runtime document at -/// all: the global `--runtime` argument has a default value this subcommand -/// never reads. +/// all: the subcommand takes no runtime configuration. fn invoke_bundle_check(bundle: &Path) -> Output { Command::new(env!("CARGO_BIN_EXE_evidence")) .arg("bundle-check") diff --git a/crates/registry-evidence/tests/selector_conformance.rs b/crates/registry-evidence/tests/selector_conformance.rs index a62c60d119..5fec29fb52 100644 --- a/crates/registry-evidence/tests/selector_conformance.rs +++ b/crates/registry-evidence/tests/selector_conformance.rs @@ -113,7 +113,7 @@ impl PreparedService { operation.to_owned(), AuditPhase::AccessAttempt, request.requirement.clone(), - self.bundle.revision().to_owned(), + self.bundle.package_digest().to_owned(), request.purpose.clone(), requester.clone(), authority.clone(), @@ -237,7 +237,7 @@ impl PreparedService { operation.to_owned(), AuditPhase::DisclosureRelease, request.requirement.clone(), - self.bundle.revision().to_owned(), + self.bundle.package_digest().to_owned(), request.purpose.clone(), requester, authority, @@ -1139,6 +1139,7 @@ async fn prepare_service_for_actor( write_secret(&secret_root, "source-token", SOURCE_TOKEN.as_bytes()); } write_runtime(&runtime_path, &bundle_root, &secret_root, &audit_path); + refresh_package_envelope(&bundle_root); make_read_only(&bundle_root); #[cfg(unix)] fs::set_permissions(&runtime_path, fs::Permissions::from_mode(0o444)) @@ -1146,11 +1147,7 @@ async fn prepare_service_for_actor( let deployment = DeploymentInputs::load(&runtime_path).expect("closed selector deployment inputs load"); - assert_ne!( - deployment.bundle().revision(), - deployment.runtime().revision(), - "governed bundle and runtime have independent revisions" - ); + assert!(deployment.bundle().package_digest().starts_with("sha256:")); let bundle = Arc::new(deployment.into_parts().0); let kernel = OfflineKernel::compile(Arc::clone(&bundle)).expect("selector kernel compiles"); let secrets = Arc::new( @@ -1685,6 +1682,7 @@ fn assert_invalid_bundle(mutate: impl FnOnce(&mut String)) { let mut config = fs::read_to_string(&config_path).expect("bundle config is readable"); mutate(&mut config); fs::write(config_path, config).expect("invalid config mutation writes"); + refresh_package_envelope(&bundle_root); make_read_only(&bundle_root); assert!(matches!( Bundle::load(&bundle_root), @@ -1719,11 +1717,12 @@ fn rewrite_source_origin(bundle_root: &Path, source_origin: &str) { fn write_runtime(runtime_path: &Path, bundle_root: &Path, secret_root: &Path, audit_path: &Path) { let runtime = format!( concat!( - "version: 1\n", - "bundleDirectory: {}\n", + "apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1\n", + "kind: EvidenceRuntimeConfig\n", + "package:\n", + " root: {}\n", "listener:\n", - " bindHost: 127.0.0.1\n", - " port: 8080\n", + " bind: 127.0.0.1:8080\n", " tlsTermination: operator-controlled-upstream\n", " trustProxyIdentityHeaders: false\n", " maximumRequestBytes: 65536\n", @@ -1783,6 +1782,31 @@ fn copy_tree(source: &Path, target: &Path) { } } +fn refresh_package_envelope(root: &Path) { + for reserved in [ + registry_platform_config::SUM_FILE, + registry_platform_config::REVISION_FILE, + ] { + let path = root.join(reserved); + if path.exists() { + fs::remove_file(path).expect("prior package envelope is removed"); + } + } + registry_platform_config::write_sum_file( + root, + None, + ®istry_platform_config::PackageLimits { + max_files: 1_024, + max_file_bytes: 1024 * 1024, + max_total_bytes: 16 * 1024 * 1024, + max_depth: 3, + max_path_bytes: 128, + }, + "evidencectl package", + ) + .expect("package envelope is refreshed"); +} + #[cfg(unix)] fn make_read_only(path: &Path) { for entry in fs::read_dir(path).expect("copied selector bundle is readable") { diff --git a/crates/registry-evidence/tests/source_contracts.rs b/crates/registry-evidence/tests/source_contracts.rs index c200d1a913..f632d40ad0 100644 --- a/crates/registry-evidence/tests/source_contracts.rs +++ b/crates/registry-evidence/tests/source_contracts.rs @@ -3158,7 +3158,7 @@ fn runtime_ca_capture_rejects_symlink_malformed_and_mutable_files() { fs::write( &runtime_path, format!( - "version: 1\nbundleDirectory: /etc/registry-evidence/bundle\nlistener:\n bindHost: 127.0.0.1\n port: 8080\n tlsTermination: operator-controlled-upstream\n trustProxyIdentityHeaders: false\n maximumRequestBytes: 65536\n maximumConcurrentRequests: 64\n requestTimeoutMilliseconds: 10000\n shutdownGraceMilliseconds: 30000\nsecretProviders:\n file: {{root: {}}}\nsigner:\n kind: transit\n unixSocketPath: /run/registry-evidence/transit-proxy.sock\n mount: transit\n keyName: evidence-signing\n keyVersion: 7\n timeoutMilliseconds: 2000\naudit:\n path: /var/lib/registry-evidence/audit/evidence.jsonl\noutboundTls:\n systemRoots: true\n trustProfiles:\n private-pki: {{caBundleFile: {}}}\n", + "apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1\nkind: EvidenceRuntimeConfig\npackage:\n root: /etc/registry-evidence/bundle\nlistener:\n bind: 127.0.0.1:8080\n tlsTermination: operator-controlled-upstream\n trustProxyIdentityHeaders: false\n maximumRequestBytes: 65536\n maximumConcurrentRequests: 64\n requestTimeoutMilliseconds: 10000\n shutdownGraceMilliseconds: 30000\nsecretProviders:\n file: {{root: {}}}\nsigner:\n kind: transit\n unixSocketPath: /run/registry-evidence/transit-proxy.sock\n mount: transit\n keyName: evidence-signing\n keyVersion: 7\n timeoutMilliseconds: 2000\naudit:\n path: /var/lib/registry-evidence/audit/evidence.jsonl\noutboundTls:\n systemRoots: true\n trustProfiles:\n private-pki: {{caBundleFile: {}}}\n", secret_root.display(), ca_path.display() ), diff --git a/crates/registry-evidencectl/Cargo.toml b/crates/registry-evidencectl/Cargo.toml index c0b54e4481..7bc4fffa69 100644 --- a/crates/registry-evidencectl/Cargo.toml +++ b/crates/registry-evidencectl/Cargo.toml @@ -35,6 +35,7 @@ registry-evidence-client.workspace = true registry-language-server.workspace = true registry-platform-audit.workspace = true registry-platform-buildinfo.workspace = true +registry-platform-config.workspace = true registry-cli-reference.workspace = true registry-platform-crypto.workspace = true registry-thunderid-tooling.workspace = true diff --git a/crates/registry-evidencectl/src/audit_view.rs b/crates/registry-evidencectl/src/audit_view.rs index 0348feca8d..bfbc41564f 100644 --- a/crates/registry-evidencectl/src/audit_view.rs +++ b/crates/registry-evidencectl/src/audit_view.rs @@ -108,9 +108,10 @@ fn show(args: ShowArgs, format: OutputFormat) -> Result { /// audit or deployment detail from a substituted binary. fn inspect_core(evidence: &Path, runtime: &Path) -> Result> { let mut child = Command::new(evidence) - .arg("--runtime") - .arg(runtime) .arg("local-audit-last-operation") + .arg("--runtime-config") + .arg(runtime) + .env_remove("REGISTRY_EVIDENCE_RUNTIME") .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::null()) diff --git a/crates/registry-evidencectl/src/authoring.rs b/crates/registry-evidencectl/src/authoring.rs index e93e753796..1fd178595d 100644 --- a/crates/registry-evidencectl/src/authoring.rs +++ b/crates/registry-evidencectl/src/authoring.rs @@ -22,6 +22,7 @@ use registry_platform_crypto::{canonicalize_json, domain_separated_sha256}; use serde_json::{json, Map, Value}; use url::{Host, Url}; +use crate::evidence_binary::{EVIDENCE_RUNTIME_API_VERSION, EVIDENCE_RUNTIME_KIND}; use crate::suggest::{ narrow, openapi::Spec, @@ -48,7 +49,7 @@ pub(crate) use registry_evidence_authoring::{ collection_pointers, question_subjects, valid_local_identifier, validate_access_policy, validate_answer_schema_document, validate_question, }, - validate_authored_answer, Finding, + validate_answer_fact_reads, validate_authored_answer, Finding, }; const LOCAL_URI_PREFIX: &str = "urn:registrystack:evidence:local:"; @@ -430,12 +431,32 @@ pub(crate) fn compile_production_project( } /// Compile the target's complete governance under its declared assurance profile. +#[cfg(test)] pub(crate) fn compile_target_project( project_root: &Path, deployment_target_root: &Path, staging_root: &Path, governed_bundle: Value, evidence_bin: &Path, +) -> Result { + compile_target_project_with_revision( + project_root, + deployment_target_root, + staging_root, + governed_bundle, + evidence_bin, + None, + ) +} + +/// Compile a target package with optional operator revision metadata. +pub(crate) fn compile_target_project_with_revision( + project_root: &Path, + deployment_target_root: &Path, + staging_root: &Path, + governed_bundle: Value, + evidence_bin: &Path, + revision: Option<&str>, ) -> Result { validate_plain_path_components(project_root, "authoring project")?; let project_root = validate_project_root(project_root)?; @@ -481,6 +502,7 @@ pub(crate) fn compile_target_project( staging_root, &plan, evidence_bin, + revision, )?; let fixture_paths = plan .questions @@ -536,7 +558,7 @@ pub(crate) fn compile_check_project( )?; expand_check_signing_validity(&mut plan.bundle); validate_compiled_bundle_shape(&plan.bundle)?; - let bundle_path = write_bundle(&project_root, None, staging_root, &plan, evidence_bin)?; + let bundle_path = write_bundle(&project_root, None, staging_root, &plan, evidence_bin, None)?; let fixture_paths = plan .questions .iter() @@ -627,7 +649,7 @@ pub(crate) fn compile_fixture_project_with_connections( }, source_connections, )?; - let bundle_path = write_bundle(&project_root, None, staging_root, &plan, evidence_bin)?; + let bundle_path = write_bundle(&project_root, None, staging_root, &plan, evidence_bin, None)?; let fixture_paths = plan .questions .iter() @@ -1103,6 +1125,12 @@ fn read_inputs(project_root: &Path, require_local_secrets: bool) -> Result Result, + schemas: &BTreeMap, +) -> Option> { + let Some(source_id) = question.source.source_ref.as_deref() else { + return Some( + question + .source + .facts + .iter() + .map(|fact| fact.name.clone()) + .collect(), + ); + }; + let schema_key = sources + .get(source_id)? + .get("factSchema")? + .as_str()? + .strip_prefix(&format!("{SCHEMAS_DIRECTORY}/"))? + .strip_suffix(".yaml")?; + let schema = schemas.get(schema_key)?; + if schema.get("additionalProperties") != Some(&Value::Bool(false)) { + return None; + } + Some( + schema + .get("properties")? + .as_object()? + .keys() + .cloned() + .collect(), + ) +} + fn local_signing_public_jwk(project_root: &Path) -> Result<(String, Vec)> { let path = project_root .join(SECRETS_DIRECTORY) @@ -3733,18 +3804,19 @@ fn render_local_bundle( "jurisdictions": [local_uri("jurisdiction")], }, "authentication": { - "kind": "oidc-access-token", - "issuer": issuer_origin, - "audiences": [audience], - "tokenTypes": ["at+jwt"], - "algorithms": ["RS256"], - "jwksUri": format!("{issuer_origin}/oauth2/jwks"), - "principalClaim": "sub", - "requesterTagsClaim": "evidence_tags", - "evidenceAudienceClaim": "evidence_audience", - "requiredScopes": ["evidence:invoke"], - "maximumTokenLifetimeSeconds": 300, - "revokedKeyIds": [], + "oidc": { + "issuer": issuer_origin, + "audience": audience, + "jwksSource": {"kind": "uri", "uri": format!("{issuer_origin}/oauth2/jwks")}, + "tokenTypes": ["at+jwt"], + "algorithms": ["RS256"], + "principalClaim": "sub", + "requesterTagsClaim": "evidence_tags", + "evidenceAudienceClaim": "evidence_audience", + "requiredScopes": ["evidence:invoke"], + "maximumTokenLifetimeSeconds": 300, + "revokedKeyIds": [], + }, }, "audit": { "hashKeyRef": "secret:file/audit-hmac-key", @@ -3782,13 +3854,13 @@ fn render_local_bundle( bail!("task grant policies require active local clients"); } if !admission.allowed_clients.is_empty() { - bundle["authentication"]["allowedClients"] = json!(admission.allowed_clients); + bundle["authentication"]["oidc"]["allowedClients"] = json!(admission.allowed_clients); } else if has_task_grants { let admitted = active_client_policies.keys().collect::>(); - bundle["authentication"]["allowedClients"] = json!(admitted); + bundle["authentication"]["oidc"]["allowedClients"] = json!(admitted); } if !admission.assertion_issuers.is_empty() { - bundle["authentication"]["assertionIssuers"] = json!(admission.assertion_issuers); + bundle["authentication"]["oidc"]["assertionIssuers"] = json!(admission.assertion_issuers); } Ok(bundle) } @@ -4057,7 +4129,7 @@ fn write_plan( evidence_bin: &Path, outbound_tls: Value, ) -> Result { - write_bundle(project_root, None, staging_root, plan, evidence_bin)?; + write_bundle(project_root, None, staging_root, plan, evidence_bin, None)?; create_private_directory(&staging_root.join("audit"))?; let canonical_staging = fs::canonicalize(staging_root) @@ -4065,11 +4137,11 @@ fn write_plan( let secret_root = fs::canonicalize(project_root.join(SECRETS_DIRECTORY)) .context("resolving local secret directory")?; let runtime = json!({ - "version": 1, - "bundleDirectory": canonical_staging.join("bundle").to_string_lossy(), + "apiVersion": EVIDENCE_RUNTIME_API_VERSION, + "kind": EVIDENCE_RUNTIME_KIND, + "package": {"root": canonical_staging.join("bundle").to_string_lossy()}, "listener": { - "bindHost": "127.0.0.1", - "port": ports.evidence, + "bind": format!("127.0.0.1:{}", ports.evidence), "tlsTermination": "operator-controlled-upstream", "trustProxyIdentityHeaders": false, "maximumRequestBytes": 65536, @@ -4128,7 +4200,7 @@ fn write_plan( Ok(CompiledProject { runtime_path, questions, - local_audience: plan.bundle["authentication"]["audiences"][0] + local_audience: plan.bundle["authentication"]["oidc"]["audience"] .as_str() .context("local Evidence audience missing")? .to_owned(), @@ -4153,6 +4225,7 @@ fn write_bundle( staging_root: &Path, plan: &CompilePlan, evidence_bin: &Path, + revision: Option<&str>, ) -> Result { let bundle = staging_root.join("bundle"); create_private_directory(&bundle)?; @@ -4315,10 +4388,26 @@ fn write_bundle( write_private_file(&bundle.join(path), &bytes)?; } } + registry_platform_config::write_sum_file( + &bundle, + revision, + &package_limits(), + "evidencectl package", + )?; set_bundle_modes(&bundle, 0o500, 0o400)?; Ok(bundle) } +pub(crate) fn package_limits() -> registry_platform_config::PackageLimits { + registry_platform_config::PackageLimits { + max_files: 1_024, + max_file_bytes: MAX_SOURCE_ARTIFACT_BYTES, + max_total_bytes: 16 * 1024 * 1024, + max_depth: 3, + max_path_bytes: 128, + } +} + /// Refuse a missing governed public key as a field-addressed diagnostic. /// /// The compile copies the public key files governance names out of the @@ -4479,9 +4568,9 @@ fn set_bundle_modes(root: &Path, directory_mode: u32, file_mode: u32) -> Result< fn check_with_evidence(evidence_bin: &Path, runtime_path: &Path) -> Result<()> { let mut command = Command::new(evidence_bin); command - .arg("--runtime") - .arg(runtime_path) .arg("check") + .arg("--runtime-config") + .arg(runtime_path) .env_remove("REGISTRY_EVIDENCE_RUNTIME"); let run = run_bounded_evidence( command, @@ -5168,7 +5257,10 @@ properties: &fs::read(fixture.staging.join("bundle/evidence.yaml")).unwrap(), ) .unwrap(); - assert_eq!(bundle["authentication"]["audiences"], json!([audience])); + assert_eq!( + bundle["authentication"]["oidc"]["audience"], + json!(audience) + ); assert_eq!( bundle["service"]["providerId"], format!("{audience}:provider") @@ -5225,7 +5317,7 @@ properties: ) .unwrap(); assert_eq!( - bundle["authentication"]["allowedClients"], + bundle["authentication"]["oidc"]["allowedClients"], json!(["age-checker", "records-reader"]) ); assert!( @@ -5258,7 +5350,7 @@ properties: ) .unwrap(); assert_eq!( - bundle["authentication"]["assertionIssuers"], + bundle["authentication"]["oidc"]["assertionIssuers"], json!({"task-agent": ["https://casework.invalid"]}) ); } @@ -5314,6 +5406,7 @@ properties: vec![ "audit/", "bundle/", + "bundle/SHA256SUMS", "bundle/adapters/", "bundle/adapters/adult-status-source-extract.rhai", "bundle/adapters/adult-status-source-prepare.rhai", @@ -5595,6 +5688,90 @@ properties: ); } + fn undeclared_fact(error: &anyhow::Error) -> (&str, &str, &str) { + let diagnostic = error + .chain() + .find_map(|cause| cause.downcast_ref::()) + .expect("authored diagnostic"); + (&diagnostic.code, &diagnostic.path, &diagnostic.message) + } + + #[test] + fn a_derivation_reading_a_fact_the_operation_does_not_declare_is_refused() { + let renamed = ANSWER.replace("facts.date_of_birth", "facts.birth_date"); + let fixture = Fixture::new(OPENAPI, QUESTION, &renamed, true); + + let error = read_inputs(&fixture.project, false) + .map(drop) + .expect_err("undeclared fact"); + assert_eq!( + undeclared_fact(&error), + ( + "evidence.authoring.derivation-fact-undeclared", + "derivations/adult-status.rhai", + "authored derivation reads fact \"birth_date\", which the question's source does not declare", + ) + ); + // The structural check `source diff` and `source update` run is the + // same one, so a source change that drops a fact is refused there too. + let error = validate_source_artifact_graph(&fixture.project).expect_err("structural"); + assert_eq!( + undeclared_fact(&error).0, + "evidence.authoring.derivation-fact-undeclared" + ); + + fs::write( + fixture.project.join("derivations/adult-status.rhai"), + ANSWER, + ) + .expect("declared derivation"); + read_inputs(&fixture.project, false).expect("declared fact"); + } + + #[test] + fn a_derivation_reading_a_fact_the_referenced_source_no_longer_produces_is_refused() { + let question = QUESTION.replace( + "source:\n operation: getPerson\n facts:\n - name: date_of_birth\n path: /date_of_birth\n combine: exactly-one\n collectionBounds: {}\n", + "source:\n ref: people\n", + ); + let fixture = Fixture::new(OPENAPI, &question, ANSWER, true); + for directory in ["sources", "schemas"] { + fs::create_dir(fixture.project.join(directory)).expect("directory"); + } + fs::write( + fixture.project.join("sources/people.yaml"), + "transport: http-json\nfactSchema: schemas/people-facts.yaml\n", + ) + .expect("source"); + let facts = |closed: bool, name: &str| { + format!( + "type: object\nadditionalProperties: {}\nrequired: [{name}]\nproperties:\n {name}: {{type: string}}\n", + !closed + ) + }; + let schema = fixture.project.join("schemas/people-facts.yaml"); + + fs::write(&schema, facts(true, "birth_date")).expect("renamed facts"); + let error = read_inputs(&fixture.project, false) + .map(drop) + .expect_err("renamed fact"); + assert_eq!( + undeclared_fact(&error), + ( + "evidence.authoring.derivation-fact-undeclared", + "derivations/adult-status.rhai", + "authored derivation reads fact \"date_of_birth\", which the question's source does not declare", + ) + ); + + // An open fact schema names no closed set, so nothing is refused. + fs::write(&schema, facts(false, "birth_date")).expect("open facts"); + read_inputs(&fixture.project, false).expect("open fact schema"); + + fs::write(&schema, facts(true, "date_of_birth")).expect("declared facts"); + read_inputs(&fixture.project, false).expect("declared fact"); + } + #[test] fn refuses_an_open_object_answer_schema_naming_the_schema_file() { let fixture = Fixture::new( @@ -5856,6 +6033,14 @@ properties: let compiled = compile_local_project(&fixture.project, &fixture.staging, &fixture.evidence) .expect("controlled category compiles"); + let package = registry_platform_config::verify_package( + &fixture.staging.join("bundle"), + &package_limits(), + "evidencectl package", + ) + .expect("local development compilation produces a package"); + assert!(package.digest().starts_with("sha256:")); + assert_eq!( compiled.questions[0].concepts[0].concept_form, CompiledConceptForm::ControlledCategory @@ -6097,7 +6282,7 @@ factSchema: schemas/source-facts.schema.yaml ), ( "schemas/source-facts.schema.yaml", - "type: object\nadditionalProperties: false\nrequired: []\nproperties: {}\n", + "type: object\nadditionalProperties: false\nrequired: []\nproperties:\n date_of_birth: {type: string}\n dose_count: {type: integer}\n relationship_confirmed: {type: boolean}\n", ), ] { fs::write(fixture.project.join(path), contents).expect("source artifact"); @@ -6362,7 +6547,10 @@ factSchema: schemas/source-facts.schema.yaml let requirement = &bundle["requirements"][0]; assert_eq!(bundle["assuranceProfile"], "local"); - assert_eq!(bundle["authentication"]["issuer"], "http://127.0.0.1:8081"); + assert_eq!( + bundle["authentication"]["oidc"]["issuer"], + "http://127.0.0.1:8081" + ); assert_eq!(bundle["signing"]["algorithm"], "ES256"); let public_key = bundle["signing"]["activePublicJwkFile"] .as_str() @@ -7116,7 +7304,7 @@ factSchema: schemas/source-facts.schema.yaml ) .unwrap(); assert_eq!( - bundle["authentication"]["allowedClients"], + bundle["authentication"]["oidc"]["allowedClients"], json!(["task-agent"]) ); let profile = &bundle["authorityProfiles"][&policy.requester_tag]; @@ -7439,7 +7627,10 @@ fn prepare(selectors, context) { bundle["sources"]["people"]["request"]["concurrencyLimit"], 3 ); - assert_eq!(bundle["authentication"]["issuer"], "http://127.0.0.1:8081"); + assert_eq!( + bundle["authentication"]["oidc"]["issuer"], + "http://127.0.0.1:8081" + ); assert_eq!(bundle["assuranceProfile"], "local"); assert_eq!(runtime["outboundTls"], tls); assert_eq!(runtime["signer"]["kind"], "local-jwk"); @@ -7554,7 +7745,7 @@ factSchema: schemas/people-facts.schema.yaml ), ( "schemas/people-facts.schema.yaml", - "type: object\nadditionalProperties: false\nrequired: []\nproperties: {}\n", + "type: object\nadditionalProperties: false\nrequired: []\nproperties:\n date_of_birth: {type: string}\n", ), ] { fs::write(fixture.project.join(path), contents).expect("source artifact"); @@ -8211,7 +8402,7 @@ factSchema: schemas/family-facts.schema.yaml let fixture = Fixture::new(OPENAPI, QUESTION, ANSWER, false); fs::write( &fixture.evidence, - "#!/bin/sh\nif test \"$1\" = render-discovery-description; then printf '{}\\n'; exit 0; fi\nbundle=\"$(dirname \"$2\")/bundle\"\nchmod -R u+rwX \"$bundle\"\nrm -rf \"$bundle\"\necho 'script rejected' >&2\nexit 1\n", + "#!/bin/sh\nif test \"$1\" = render-discovery-description; then printf '{}\\n'; exit 0; fi\nbundle=\"$(dirname \"$3\")/bundle\"\nchmod -R u+rwX \"$bundle\"\nrm -rf \"$bundle\"\necho 'script rejected' >&2\nexit 1\n", ) .expect("stub that removes the generation it rejects"); @@ -8231,7 +8422,7 @@ factSchema: schemas/family-facts.schema.yaml let fixture = Fixture::new(OPENAPI, QUESTION, ANSWER, false); fs::write( &fixture.evidence, - "#!/bin/sh\nif test \"$1\" = render-discovery-description; then printf '{}\\n'; exit 0; fi\nrm -f \"$2\"\necho 'script rejected' >&2\nexit 1\n", + "#!/bin/sh\nif test \"$1\" = render-discovery-description; then printf '{}\\n'; exit 0; fi\nrm -f \"$3\"\necho 'script rejected' >&2\nexit 1\n", ) .expect("stub that removes the settings it rejects"); @@ -8435,7 +8626,7 @@ factSchema: schemas/family-facts.schema.yaml .open(&evidence) .expect("stub"); let script = if check_succeeds { - "#!/bin/sh\nif test \"$1\" = render-discovery-description; then printf '{}\\n'; exit 0; fi\ntest \"$1\" = --runtime && test \"$3\" = check\n" + "#!/bin/sh\nif test \"$1\" = render-discovery-description; then printf '{}\\n'; exit 0; fi\ntest \"$1\" = check && test \"$2\" = --runtime-config\n" } else { "#!/bin/sh\nif test \"$1\" = render-discovery-description; then printf '{}\\n'; exit 0; fi\necho 'script rejected' >&2\nexit 1\n" }; diff --git a/crates/registry-evidencectl/src/build.rs b/crates/registry-evidencectl/src/build.rs index 3159190145..bed4434d54 100644 --- a/crates/registry-evidencectl/src/build.rs +++ b/crates/registry-evidencectl/src/build.rs @@ -3,9 +3,9 @@ use std::{ collections::{BTreeMap, BTreeSet}, - fs::{self, File, OpenOptions}, - io::{Read as _, Write as _}, - os::unix::fs::{MetadataExt as _, OpenOptionsExt as _, PermissionsExt as _}, + fs::{self, File}, + io::Read as _, + os::unix::fs::{MetadataExt as _, PermissionsExt as _}, path::{Component, Path, PathBuf}, process::{Command, ExitCode, ExitStatus, Stdio}, sync::{ @@ -48,6 +48,10 @@ pub struct BuildArgs { /// New candidate directory to create. It must not already exist. #[arg(long)] pub output: PathBuf, + + /// Optional source or review revision recorded in `REVISION`. + #[arg(long)] + pub revision: Option, } #[derive(Debug, Deserialize)] @@ -148,21 +152,15 @@ impl TargetGovernance { } } -pub(crate) fn run_with_format(args: BuildArgs, format: OutputFormat) -> Result { - let interruption = BuildInterruption::install()?; - run_inner(args, &interruption, format, false) -} - pub(crate) fn run_package_with_format(args: BuildArgs, format: OutputFormat) -> Result { let interruption = BuildInterruption::install()?; - run_inner(args, &interruption, format, true) + run_inner(args, &interruption, format) } fn run_inner( args: BuildArgs, interruption: &BuildInterruption, format: OutputFormat, - require_deployable_assurance: bool, ) -> Result { interruption.check()?; let _project_lock = ProjectLock::acquire(&args.project) @@ -171,12 +169,11 @@ fn run_inner( // before the output location: a local target is named as such rather // than hidden behind an output refusal. let target = read_target_documents(&args.target)?; - if require_deployable_assurance - && target - .governed_bundle - .get("assuranceProfile") - .and_then(Value::as_str) - == Some("local") + if target + .governed_bundle + .get("assuranceProfile") + .and_then(Value::as_str) + == Some("local") { return Err(TargetDocumentDiagnostic { code: "evidence.package.production-profile-required", @@ -198,9 +195,7 @@ fn run_inner( if candidate.starts_with(&project) { bail!("candidate output must remain outside the editable project"); } - if require_deployable_assurance { - verify_bundle_directory_matches_candidate(&target.runtime, &candidate)?; - } + verify_stable_package_root(&target.runtime, &candidate)?; let evidence_bin = crate::evidence_binary::resolve_matching(None)?; interruption.check()?; @@ -216,8 +211,9 @@ fn run_inner( staging.path(), &evidence_bin, interruption, + args.revision.as_deref(), ); - let (revision, secret_references) = match result { + let (package_digest, secret_references) = match result { Ok(result) => result, Err(error) => { close_candidate_staging(staging)?; @@ -239,21 +235,25 @@ fn run_inner( "project": args.project, "target": args.target, "output": args.output, - "bundleRevision": revision, + "packageDigest": package_digest, + "revision": args.revision, "requiredSecrets": secret_references, "proofBoundary": "offline deployment candidate compilation and fixture validation" }))? ); return Ok(ExitCode::SUCCESS); } - println!("Bundle revision: {revision}"); - println!("Candidate: {}", args.output.display()); + println!("Package digest: {package_digest}"); + if let Some(revision) = &args.revision { + println!("Revision: {revision}"); + } + println!("Package: {}", args.output.display()); for reference in secret_references { println!("Provision {SECRET_PREFIX}{reference}"); } println!( - "Target runtime paths and deployment secret material remain unverified until `evidencectl doctor --runtime-config {}/runtime.yaml`.", - args.output.display(), + "Target runtime paths and deployment secret material remain unverified until `evidencectl doctor --runtime-config {}`.", + args.target.join("runtime.yaml").display(), ); Ok(ExitCode::SUCCESS) } @@ -309,7 +309,7 @@ pub(crate) fn compile_target_fixture_project( .context("sealing private target fixture compilation staging")?; let staging_path = fs::canonicalize(staging.path()) .context("resolving private target fixture compilation staging")?; - let compiled = compile_with_target(project, &target, &staging_path, evidence_bin)?; + let compiled = compile_with_target(project, &target, &staging_path, evidence_bin, None)?; Ok(TargetFixtureProject { bundle_path: compiled.bundle_path, fixture_paths: compiled.fixture_paths, @@ -433,13 +433,15 @@ pub(crate) fn compile_with_target( target: &TargetDocuments, staging_root: &Path, evidence_bin: &Path, + revision: Option<&str>, ) -> Result { - let compiled = authoring::compile_target_project( + let compiled = authoring::compile_target_project_with_revision( project, &target.root, staging_root, target.governed_bundle.clone(), evidence_bin, + revision, )?; Ok(TargetCompilation { bundle_path: compiled.bundle_path, @@ -462,18 +464,15 @@ fn prepare_candidate( staging_root: &Path, evidence_bin: &Path, interruption: &BuildInterruption, + revision: Option<&str>, ) -> Result<(String, Vec)> { - let compiled = compile_with_target(project, target, staging_root, evidence_bin)?; + let compiled = compile_with_target(project, target, staging_root, evidence_bin, revision)?; interruption.check()?; reject_review_markers(&compiled.bundle_path)?; reject_review_markers_in_bytes(&target.runtime, "runtime.yaml")?; - let runtime_path = staging_root.join("runtime.yaml"); - write_new_file(&runtime_path, &target.runtime, 0o600)?; - fs::set_permissions(&runtime_path, fs::Permissions::from_mode(0o400)) - .context("sealing the copied deployment runtime")?; - let secret_references = secret_references(&compiled.bundle)?; - let revision = run_bundle_check(evidence_bin, &compiled.bundle_path, project, interruption)?; + let package_digest = + run_bundle_check(evidence_bin, &compiled.bundle_path, project, interruption)?; for fixture in &compiled.fixture_paths { interruption.check()?; run_bundle_fixture( @@ -484,36 +483,34 @@ fn prepare_candidate( interruption, )?; } - Ok((revision, secret_references)) + Ok((package_digest, secret_references)) } -/// Refuse a packaged candidate whose deployment runtime would load a bundle -/// left behind by a different `package` invocation. `bundleDirectory` is the -/// one path the running process trusts at startup, so a candidate that copies -/// a `runtime.yaml` naming another directory would report this build's bundle -/// revision while quietly serving whatever bundle already sits at that other -/// path. This check only compares the value against the candidate this +/// Refuse a package whose deployment runtime would load a package +/// left behind by a different `package` invocation. `package.root` is the +/// one path the running process trusts at startup, so a runtime path nested in +/// one generated output could never remain stable across installations. This +/// check only compares the value against the package this /// invocation is producing; it leaves full runtime shape validation to /// `evidencectl doctor --runtime-config` and the `evidence` binary itself, so /// a runtime document that is otherwise malformed is still caught there /// rather than reported twice. -fn verify_bundle_directory_matches_candidate(runtime_bytes: &[u8], candidate: &Path) -> Result<()> { +fn verify_stable_package_root(runtime_bytes: &[u8], candidate: &Path) -> Result<()> { let Ok(document) = serde_norway::from_slice::(runtime_bytes) else { return Ok(()); }; - let Some(bundle_directory) = document.get("bundleDirectory").and_then(Value::as_str) else { + let Some(package_root) = document.pointer("/package/root").and_then(Value::as_str) else { return Ok(()); }; - let expected = candidate.join("bundle"); - if Path::new(bundle_directory) == expected { + if Path::new(package_root) != candidate && !Path::new(package_root).starts_with(candidate) { return Ok(()); } Err(TargetDocumentDiagnostic { - code: "evidence.package.bundle-directory-mismatch", - path: "runtime.yaml:/bundleDirectory".to_owned(), + code: "evidence.package.root-unstable", + path: "runtime.yaml:/package/root".to_owned(), message: format!( - "deployment runtime bundleDirectory {bundle_directory} does not resolve to this candidate's own bundle path {}", - expected.display() + "deployment runtime package.root {package_root} is inside this one package output; select a stable installed package path outside {}", + candidate.display() ), } .into()) @@ -525,7 +522,7 @@ fn run_bundle_check( project: &Path, interruption: &BuildInterruption, ) -> Result { - Ok(run_bundle_check_report(evidence_bin, bundle, project, interruption)?.bundle_revision) + Ok(run_bundle_check_report(evidence_bin, bundle, project, interruption)?.package_digest) } fn run_bundle_check_report( @@ -555,7 +552,7 @@ fn run_bundle_check_report( #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub(crate) struct BundleCheckReport { - pub(crate) bundle_revision: String, + pub(crate) package_digest: String, pub(crate) requirements: Vec, } @@ -581,8 +578,8 @@ pub(crate) fn check_compiled_bundle( fn parse_bundle_check_report(stdout: &[u8]) -> Result { let report: BundleCheckReport = serde_json::from_slice(stdout) - .context("Evidence check returned an invalid bundle revision report")?; - validate_digest(&report.bundle_revision, "bundleRevision")?; + .context("Evidence check returned an invalid package digest report")?; + validate_digest(&report.package_digest, "packageDigest")?; if report.requirements.is_empty() { bail!("Evidence check returned no requirement revisions"); } @@ -992,18 +989,6 @@ fn read_plain_file(path: &Path, maximum: u64, description: &str) -> Result Result<()> { - let mut file = OpenOptions::new() - .write(true) - .create_new(true) - .mode(mode) - .open(path) - .with_context(|| format!("creating {}", path.display()))?; - file.write_all(contents)?; - file.sync_all()?; - Ok(()) -} - fn make_tree_removable(root: &Path) -> Result<()> { for entry in fs::read_dir(root)? { let path = entry?.path(); @@ -1032,12 +1017,26 @@ fn close_candidate_staging(staging: tempfile::TempDir) -> Result<()> { } fn publish(staging: tempfile::TempDir, output: &Path) -> Result<()> { - let staged = staging.keep(); - if let Err(error) = rename_noreplace(&staged, output) { - let _ = make_tree_removable(&staged); - let _ = fs::remove_dir_all(&staged); + let staged_root = staging.keep(); + let staged_package = staged_root.join("bundle"); + // macOS refuses RENAME_EXCL for a directory whose own write bit has been + // removed, even though both parent directories remain writable. Restore + // owner access only on the package root for the atomic move; every package + // file and child directory remains sealed throughout publication. + fs::set_permissions(&staged_package, fs::Permissions::from_mode(0o700)) + .context("preparing the sealed package root for atomic publication")?; + if let Err(error) = rename_noreplace(&staged_package, output) { + let _ = make_tree_removable(&staged_root); + let _ = fs::remove_dir_all(&staged_root); return Err(error).context("publishing the deployment candidate without replacement"); } + if let Err(error) = fs::set_permissions(output, fs::Permissions::from_mode(0o500)) { + let _ = make_tree_removable(output); + let _ = fs::remove_dir_all(output); + let _ = fs::remove_dir(&staged_root); + return Err(error).context("sealing the published package root"); + } + fs::remove_dir(&staged_root).context("removing empty package staging directory")?; Ok(()) } @@ -1113,7 +1112,8 @@ authorityProfiles: .expect("governance"); fs::write( target.join("runtime.yaml"), - r#"version: 1 + r#"apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig outboundTls: systemRoots: true trustProfiles: @@ -1175,10 +1175,10 @@ authorityProfiles: } #[test] - fn revision_report_and_secret_reference_parsing_are_closed() { + fn package_report_and_secret_reference_parsing_are_closed() { let report = parse_bundle_check_report( serde_json::json!({ - "bundleRevision": format!("sha256:{}", "b".repeat(64)), + "packageDigest": format!("sha256:{}", "b".repeat(64)), "requirements": [ {"id": "registry-status", "configurationRevision": format!("sha256:{}", "c".repeat(64))} ] @@ -1187,14 +1187,14 @@ authorityProfiles: .as_bytes(), ) .expect("bundle-check JSON report"); - assert_eq!(report.bundle_revision, format!("sha256:{}", "b".repeat(64))); + assert_eq!(report.package_digest, format!("sha256:{}", "b".repeat(64))); assert_eq!( report.requirements[0].configuration_revision, format!("sha256:{}", "c".repeat(64)) ); assert!(parse_bundle_check_report( serde_json::json!({ - "bundleRevision": format!("sha256:{}", "b".repeat(64)), + "packageDigest": format!("sha256:{}", "b".repeat(64)), "requirements": [ {"id": "registry-status", "configurationRevision": format!("sha256:{}", "C".repeat(64))} ] @@ -1204,7 +1204,7 @@ authorityProfiles: ) .is_err()); assert!(parse_bundle_check_report( - br#"{"bundleRevision":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","requirements":[{"id":"dup","configurationRevision":"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"},{"id":"dup","configurationRevision":"sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"}]}"# + br#"{"packageDigest":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","requirements":[{"id":"dup","configurationRevision":"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"},{"id":"dup","configurationRevision":"sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"}]}"# ) .is_err()); diff --git a/crates/registry-evidencectl/src/check.rs b/crates/registry-evidencectl/src/check.rs index 2649b2a77b..50d32c1374 100644 --- a/crates/registry-evidencectl/src/check.rs +++ b/crates/registry-evidencectl/src/check.rs @@ -145,7 +145,7 @@ fn check_and_capture_target( let mut target_documents = None; let mut assurance_profile = None; - let mut bundle_revision = None; + let mut package_digest = None; if let Some(target) = target { match build::read_target_documents(target) { Ok(documents) => { @@ -201,7 +201,7 @@ fn check_and_capture_target( }; match checked { Ok(checked) => { - bundle_revision = Some(checked.bundle_revision); + package_digest = Some(checked.package_digest); } Err(error) => { return Err(classify_compiler_error( @@ -226,7 +226,7 @@ fn check_and_capture_target( "status": if complete { "complete" } else { "incomplete" }, "proof": if complete && target.is_some() { "deployment-closure" } else { "authoring" }, "assuranceProfile": assurance_profile, - "bundleRevision": bundle_revision, + "packageDigest": package_digest, "fixtureProof": false, "findings": findings, "offline": true, @@ -315,7 +315,7 @@ fn explain_captured(project: &Path, target: Option<&Path>, checked: CheckOutcome "target": target, "status": validation["status"], "proof": validation["proof"], - "bundleRevision": validation["bundleRevision"], + "packageDigest": validation["packageDigest"], "findings": validation["findings"], "questions": inventory.questions, "sources": inventory.sources, @@ -494,7 +494,7 @@ fn write_list(out: &mut dyn io::Write, label: &str, value: &Value) -> io::Result } struct CheckedBundle { - bundle_revision: String, + package_digest: String, } fn check_project_only(project: &Path, display_project: &Path) -> Result { @@ -509,7 +509,7 @@ fn check_project_only(project: &Path, display_project: &Path) -> Result) -> std::fmt::Result { write!( formatter, - "deployment runtime does not satisfy Version 1: {}", + "deployment runtime does not satisfy the published runtime contract: {}", self.rules.join("; ") ) } @@ -1620,7 +1621,7 @@ factSchema: schemas/record-status-facts.schema.yaml assert_eq!(report["status"], "incomplete"); assert_eq!(report["proof"], "authoring"); - assert_eq!(report["bundleRevision"], Value::Null); + assert_eq!(report["packageDigest"], Value::Null); assert_eq!(report["findings"].as_array().unwrap().len(), 1); assert_eq!( report["findings"][0]["code"], @@ -2273,9 +2274,12 @@ factSchema: schemas/record-status-facts.schema.yaml let runtime = include_str!( "../../../products/evidence/reference/deployment-targets/environments/production/evidence/runtime.yaml" ); - let runtime = runtime.replace("version: 1", "version: 1\nunknown: true"); + let runtime = runtime.replace( + "kind: EvidenceRuntimeConfig\n", + "kind: EvidenceRuntimeConfig\nunknown: true\n", + ); let error = validate_runtime_structure(runtime.as_bytes()).unwrap_err(); - assert!(format!("{error:#}").contains("Version 1")); + assert!(format!("{error:#}").contains("published runtime contract")); } #[test] diff --git a/crates/registry-evidencectl/src/dev.rs b/crates/registry-evidencectl/src/dev.rs index 5519421719..a8cd3f3dc4 100644 --- a/crates/registry-evidencectl/src/dev.rs +++ b/crates/registry-evidencectl/src/dev.rs @@ -1434,9 +1434,20 @@ fn state_matches_sealed_bundle( access_policies: &[AccessPolicyState], dev_root: &Path, ) -> Result { - let path = dev_root.join("bundle/evidence.yaml"); + let package_root = dev_root.join("bundle"); + let verified = registry_platform_config::verify_package( + &package_root, + &crate::authoring::package_limits(), + "evidencectl dev start", + )?; + let path = package_root.join("evidence.yaml"); require_owned_regular_file(&path, 0o400)?; let bytes = fs::read(&path).context("failed to read the sealed local bundle")?; + if verified.file_digest("evidence.yaml").as_deref() + != Some(registry_platform_config::sha256_uri(&bytes).as_str()) + { + bail!("the sealed local package changed after package verification"); + } if bytes.len() > 1024 * 1024 { return Ok(false); } @@ -2813,9 +2824,10 @@ fn spawn_evidence(binary: &Path, runtime: &Path, log: &Path) -> Result { let stdout = create_private_file(log)?; let stderr = stdout.try_clone()?; Command::new(binary) - .arg("--runtime") - .arg(runtime) .arg("serve") + .arg("--runtime-config") + .arg(runtime) + .env_remove("REGISTRY_EVIDENCE_RUNTIME") .stdin(Stdio::null()) .stdout(Stdio::from(stdout)) .stderr(Stdio::from(stderr)) @@ -3748,8 +3760,20 @@ requirements: "#, ) .expect("bundle config"); + registry_platform_config::write_sum_file( + &bundle, + None, + &crate::authoring::package_limits(), + "evidencectl dev start", + ) + .expect("seal local test package"); fs::set_permissions(&bundle_path, fs::Permissions::from_mode(0o400)) .expect("seal bundle config"); + fs::set_permissions( + bundle.join(registry_platform_config::SUM_FILE), + fs::Permissions::from_mode(0o400), + ) + .expect("seal package sum file"); fs::set_permissions(&bundle, fs::Permissions::from_mode(0o500)).expect("seal bundle"); let mut state = DevState { schema: STATE_SCHEMA.to_owned(), @@ -3832,8 +3856,26 @@ requirements: serde_norway::to_string(&explicit_bundle).expect("explicit bundle YAML"), ) .expect("write explicit bundle"); + fs::set_permissions(&bundle, fs::Permissions::from_mode(PRIVATE_DIR_MODE)) + .expect("unseal package directory for test update"); + fs::remove_file(bundle.join(registry_platform_config::SUM_FILE)) + .expect("remove stale package sum file"); + registry_platform_config::write_sum_file( + &bundle, + None, + &crate::authoring::package_limits(), + "evidencectl dev start", + ) + .expect("reseal local test package"); fs::set_permissions(&bundle_path, fs::Permissions::from_mode(0o400)) .expect("reseal bundle config"); + fs::set_permissions( + bundle.join(registry_platform_config::SUM_FILE), + fs::Permissions::from_mode(0o400), + ) + .expect("reseal package sum file"); + fs::set_permissions(&bundle, fs::Permissions::from_mode(0o500)) + .expect("reseal package directory"); state.caller = None; state.access_policies = vec![AccessPolicyState { id: "age-checks".to_owned(), diff --git a/crates/registry-evidencectl/src/doctor.rs b/crates/registry-evidencectl/src/doctor.rs index 52ad7ce0c2..32f3d7e3ef 100644 --- a/crates/registry-evidencectl/src/doctor.rs +++ b/crates/registry-evidencectl/src/doctor.rs @@ -59,9 +59,8 @@ const GATED_ACQUISITION_CAPABILITIES: [&str; 2] = ["search-then-fetch-set", "sou pub struct DoctorArgs { /// Evidence project directory; defaults to the current directory. /// - /// This command needs a deployment project: one holding runtime.yaml - /// beside bundle/. `evidencectl package` compiles an editable project into - /// one. + /// This command needs a deployment target whose runtime.yaml names the + /// installed package. `evidencectl package` writes that package separately. #[arg(long, default_value = ".")] pub project: PathBuf, @@ -186,12 +185,12 @@ fn missing_runtime_message(project: &Path, runtime_path: &Path) -> String { ); if project_is_editable(runtime_path) { format!( - "runtime configuration not found at {}; artifact inspection walks a deployment project and {project} is an editable project. Compile a candidate with {build}, or check the editable project as it stands with `evidencectl check {project}` and `evidencectl test {project}`", + "runtime configuration not found at {}; artifact inspection walks a deployment target and {project} is an editable project. Build the package separately with {build}, or check the editable project as it stands with `evidencectl check {project}` and `evidencectl test {project}`", runtime_path.display() ) } else { format!( - "runtime configuration not found at {}; doctor walks a deployment project, one holding runtime.yaml beside bundle/. Compile a candidate from an editable project with {build}", + "runtime configuration not found at {}; artifact inspection walks a deployment target whose runtime.yaml names the installed package. Build the package separately with {build}", runtime_path.display() ) } @@ -204,7 +203,7 @@ fn missing_runtime_diagnostic(project: &Path, runtime_path: &Path) -> DoctorDiag path: "runtime.yaml".to_owned(), message: missing_runtime_message(project, runtime_path), suggested_action: - "Select a deployment project, or use evidencectl package, check, and test to prepare and inspect an editable project." + "Select a deployment target whose runtime names the installed package, or use evidencectl package, check, and test to prepare and inspect an editable project." .to_owned(), } } @@ -987,13 +986,13 @@ fn resolve_bundle_directory( runtime_path: &Path, project: &Path, ) -> Result { - match runtime.get("bundleDirectory") { + match runtime + .get("package") + .and_then(|package| package.get("root")) + { Some(value) => { let value = value.as_str().ok_or_else(|| { - anyhow!( - "bundleDirectory in {} is not a string", - runtime_path.display() - ) + anyhow!("package.root in {} is not a string", runtime_path.display()) })?; Ok(resolve_against(runtime_path, project, Path::new(value))) } diff --git a/crates/registry-evidencectl/src/evidence_binary.rs b/crates/registry-evidencectl/src/evidence_binary.rs index c20821497c..5b3d4c15ff 100644 --- a/crates/registry-evidencectl/src/evidence_binary.rs +++ b/crates/registry-evidencectl/src/evidence_binary.rs @@ -13,6 +13,15 @@ use std::{ use anyhow::{Context as _, Result}; +/// The `apiVersion` of the runtime document the matching `evidence` binary +/// reads. evidencectl writes runtime documents; the runtime owns the grammar +/// and refuses any other envelope. +pub(crate) const EVIDENCE_RUNTIME_API_VERSION: &str = + "registry.registrystack.org/evidence-runtime/v1alpha1"; + +/// The `kind` of the runtime document the matching `evidence` binary reads. +pub(crate) const EVIDENCE_RUNTIME_KIND: &str = "EvidenceRuntimeConfig"; + /// How long a delegated `evidence` run may take before evidencectl stops it. /// /// Compilation and fixture evaluation are local, bounded work against a bundle diff --git a/crates/registry-evidencectl/src/fixtures.rs b/crates/registry-evidencectl/src/fixtures.rs index db47ca66fb..49e829bb74 100644 --- a/crates/registry-evidencectl/src/fixtures.rs +++ b/crates/registry-evidencectl/src/fixtures.rs @@ -313,9 +313,12 @@ impl FixtureTarget { fn check(&self, evidence_bin: &Path) -> StepOutcome { match self { - Self::Deployment { runtime_path, .. } => { - run_evidence_step(evidence_bin, &["--runtime"], Some(runtime_path), &["check"]) - } + Self::Deployment { runtime_path, .. } => run_evidence_step( + evidence_bin, + &["check", "--runtime-config"], + Some(runtime_path), + &[], + ), Self::Editable { compilation, .. } => run_evidence_step( evidence_bin, &["bundle-check", "--bundle"], @@ -346,12 +349,17 @@ impl FixtureTarget { // having evaluated nothing. let mut outcome = match self { Self::Deployment { runtime_path, .. } => { - let mut args = vec!["evaluate", "--fixture", fixture]; + let mut args = vec!["--fixture", fixture]; if let Some(case) = case { args.extend(["--case", case]); } args.extend(["--explain", "--explain-format", "json"]); - run_evidence_step(evidence_bin, &["--runtime"], Some(runtime_path), &args) + run_evidence_step( + evidence_bin, + &["evaluate", "--runtime-config"], + Some(runtime_path), + &args, + ) } Self::Editable { compilation, .. } => { let mut args = vec!["--fixture", fixture]; @@ -388,7 +396,7 @@ impl FixtureTarget { } /// Resolve the bundle directory a project's `runtime.yaml` names. A relative -/// `bundleDirectory` is resolved against the runtime file's own directory, an +/// `package.root` is resolved against the runtime file's own directory, an /// absolute one is used as-is, and `/bundle` is the default only when /// the key is absent. This is discovery, not validation: `evidence check` is /// left to reject a runtime configuration that is otherwise malformed. @@ -405,13 +413,13 @@ fn resolve_bundle_directory(runtime_path: &Path, project: &Path) -> Result { let value = value.as_str().ok_or_else(|| { - anyhow!( - "bundleDirectory in {} is not a string", - runtime_path.display() - ) + anyhow!("package.root in {} is not a string", runtime_path.display()) })?; let path = Path::new(value); if path.is_absolute() { @@ -473,7 +481,8 @@ fn discover_fixtures(bundle_config_path: &Path) -> Result> { Ok(fixture_paths) } -/// Run one `evidence --runtime ` invocation. +/// Run one `evidence ` invocation, such as +/// `evidence check --runtime-config `. /// /// Standard output and standard error are captured rather than inherited so /// steps never interleave, and any failure to even spawn the process is @@ -485,7 +494,7 @@ fn run_evidence_step( args: &[&str], ) -> StepOutcome { let mut command = Command::new(evidence_bin); - command.args(prefix); + command.args(prefix).env_remove("REGISTRY_EVIDENCE_RUNTIME"); if let Some(path) = path { command.arg(path); } diff --git a/crates/registry-evidencectl/src/lib.rs b/crates/registry-evidencectl/src/lib.rs index 1aab655024..ae33b0cfa3 100644 --- a/crates/registry-evidencectl/src/lib.rs +++ b/crates/registry-evidencectl/src/lib.rs @@ -81,7 +81,8 @@ enum Command { Jwks(jwks::JwksArgs), /// Start an editable Evidence Gateway project from OpenAPI, a starter, or a SQLite extract. New(scaffold::NewArgs), - /// Compile an editable project into a reviewed deployment candidate. + /// Retired spelling of `package`. + #[command(hide = true)] Build(build::BuildArgs), /// Drive the evidence binary across a project's bundle fixtures. #[command(subcommand)] @@ -178,6 +179,9 @@ struct PackageArgs { /// New candidate directory to create. #[arg(long)] output: PathBuf, + /// Optional source or review revision recorded in `REVISION`. + #[arg(long)] + revision: Option, } #[derive(Debug, Subcommand)] @@ -210,7 +214,7 @@ impl std::error::Error for SafeCliFailure {} #[derive(Debug, Args)] struct ArtifactInspectArgs { - /// Deployment project containing runtime.yaml beside bundle/. + /// Deployment target whose runtime.yaml names the installed package. project: PathBuf, } @@ -441,6 +445,7 @@ pub fn main_entry() -> ExitCode { project: args.project, target: args.target, output: args.output, + revision: args.revision, }, format, ), @@ -455,7 +460,18 @@ pub fn main_entry() -> ExitCode { Command::Keygen(command) => keygen::run(command, format), Command::Jwks(args) => jwks::run(args, format), Command::New(args) => scaffold::run_with_format(args, format), - Command::Build(args) => build::run_with_format(args, format), + Command::Build(args) => Err(SafeCliFailure { + operational: false, + code: "evidence.build.retired".to_owned(), + artifact: args.project.display().to_string(), + path: "$".to_owned(), + message: "The evidencectl build command was removed.".to_owned(), + suggested_action: + "Run evidencectl package with the same project, target, and new output directory." + .to_owned(), + cause: None, + } + .into()), Command::Fixtures(fixtures::FixturesCommand::Run(mut args)) => { args.json |= format == OutputFormat::Json; fixtures::run(fixtures::FixturesCommand::Run(args)) @@ -1656,7 +1672,11 @@ mod tests { let documented: Vec<_> = projects .iter() - .filter(|(_, argument)| !argument.is_hide_set()) + .filter(|(path, argument)| { + // Clap does not propagate a hidden command's state to its + // arguments. The retired build spelling is not public API. + path != "evidencectl build" && !argument.is_hide_set() + }) .collect(); assert_eq!( documented @@ -1664,7 +1684,6 @@ mod tests { .map(|(path, _)| path.as_str()) .collect::>(), std::collections::BTreeSet::from([ - "evidencectl build", "evidencectl fixtures run", "evidencectl source add", "evidencectl source suggest", @@ -1728,7 +1747,8 @@ mod tests { vec!["evidencectl", "fixtures", "run"], vec![ "evidencectl", - "build", + "package", + ".", "--target", "deployment/local", "--output", diff --git a/crates/registry-evidencectl/src/request.rs b/crates/registry-evidencectl/src/request.rs index 854caaf151..be32cd99a2 100644 --- a/crates/registry-evidencectl/src/request.rs +++ b/crates/registry-evidencectl/src/request.rs @@ -812,9 +812,10 @@ fn prepare_local_relying_procedure( input: &Path, ) -> Result { let mut child = Command::new(evidence) - .arg("--runtime") - .arg(runtime) .arg("prepare-local-relying-procedure") + .arg("--runtime-config") + .arg(runtime) + .env_remove("REGISTRY_EVIDENCE_RUNTIME") .arg("--input") .arg(input) .stdin(Stdio::null()) diff --git a/crates/registry-evidencectl/src/runtime.rs b/crates/registry-evidencectl/src/runtime.rs index a07b2c1cde..2d45210215 100644 --- a/crates/registry-evidencectl/src/runtime.rs +++ b/crates/registry-evidencectl/src/runtime.rs @@ -4,6 +4,8 @@ //! same dependency check without binding the public listener or sending an //! evidence-data request. Audit initialization may briefly create its //! operational lock file, but it never appends an application audit event. +//! With `--without-audit-lock` it leaves that lock to the running instance +//! that holds it and proves the rest of the audit destination read-only. use std::{ io::Write as _, @@ -43,6 +45,13 @@ pub(crate) struct DoctorArgs { /// Also prove that the audit destination resolves below this persistent root. #[arg(long, value_name = "ABSOLUTE_DIRECTORY", requires = "runtime_config")] require_audit_under: Option, + /// Prove the audit destination without taking its single-writer lock. + /// + /// For a candidate staged beside the running instance it will replace, + /// which holds that lock. Modes, write access, and a complete final entry + /// are still proved, and every other dependency is proved as without it. + #[arg(long, requires = "runtime_config")] + without_audit_lock: bool, /// Path to the matching Evidence runtime binary. #[arg(long, hide = true, requires = "runtime_config")] evidence_bin: Option, @@ -120,7 +129,7 @@ pub(crate) fn run(args: DoctorArgs, format: OutputFormat) -> Result { error } })?; - let base = invoke_check(&evidence, &runtime_config, false, None)?; + let base = invoke_check(&evidence, &runtime_config, false, None, false)?; if !base.status.success() { let dependency_failure = runtime_diagnostic_is_dependency_failure(&base.stderr); return render_refusal( @@ -145,6 +154,7 @@ pub(crate) fn run(args: DoctorArgs, format: OutputFormat) -> Result { &runtime_config, true, args.require_audit_under.as_deref(), + args.without_audit_lock, )?; if dependency.status.success() { @@ -152,7 +162,7 @@ pub(crate) fn run(args: DoctorArgs, format: OutputFormat) -> Result { operation: "doctor", status: "ready", runtime_config: &runtime_config, - proof_boundary: "live startup dependency preflight; no public listener, evidence-data request, or application audit event was produced", + proof_boundary: proof_boundary(args.without_audit_lock), diagnostics: Vec::new(), }; match format { @@ -163,7 +173,11 @@ pub(crate) fn run(args: DoctorArgs, format: OutputFormat) -> Result { "Dependency preflight passed for {}", runtime_config.display() ); - println!("Proof: startup dependencies were checked without opening the public listener, sending an evidence-data request, or appending an application audit event. Audit initialization may briefly hold its operational lock."); + println!("Proof: startup dependencies were checked without opening the public listener, sending an evidence-data request, or appending an application audit event. {}", if args.without_audit_lock { + "The audit writer lock was not taken, so a second writer is not detected. The audit destination's ownership, modes, write access, and complete final entry were checked." + } else { + "Audit initialization may briefly hold its operational lock." + }); } } return Ok(ExitCode::SUCCESS); @@ -179,6 +193,37 @@ pub(crate) fn run(args: DoctorArgs, format: OutputFormat) -> Result { ) } +const LOCKED_PROOF_BOUNDARY: &str = "live startup dependency preflight; no public listener, evidence-data request, or application audit event was produced"; + +const LOCK_FREE_PROOF_BOUNDARY: &str = "live startup dependency preflight without the audit writer lock; no public listener, evidence-data request, or application audit event was produced; the audit writer lock was not taken, so a second writer is not detected"; + +/// What a passing dependency preflight proved, for the report a consumer +/// reads. The lock-free form proves less, and says so. +fn proof_boundary(without_audit_lock: bool) -> &'static str { + if without_audit_lock { + LOCK_FREE_PROOF_BOUNDARY + } else { + LOCKED_PROOF_BOUNDARY + } +} + +const DEFAULT_ACTION: &str = "Read the value-free Evidence diagnostic, correct the selected runtime artifact or dependency, and rerun doctor."; + +const HELD_LOCK_ACTION: &str = "Another Evidence instance holds this audit destination's writer lock. To check a candidate staged beside it, rerun doctor with --without-audit-lock; otherwise stop the other writer first."; + +/// The next step for a refusal. A held writer lock is the one refusal a +/// candidate beside a running instance meets by design, so it names the +/// lock-free form; every other refusal is corrected where the diagnostic says. +fn suggested_action(runtime_diagnostic: &[u8]) -> &'static str { + if String::from_utf8_lossy(runtime_diagnostic) + .contains("another process holds the single-writer lock beside the audit file") + { + HELD_LOCK_ACTION + } else { + DEFAULT_ACTION + } +} + fn runtime_diagnostic_is_dependency_failure(diagnostic: &[u8]) -> bool { let diagnostic = String::from_utf8_lossy(diagnostic); [ @@ -205,15 +250,17 @@ fn invoke_check( runtime_config: &std::path::Path, dependencies: bool, audit_root: Option<&std::path::Path>, + without_audit_lock: bool, ) -> Result { let mut stdout = tempfile::tempfile().context("creating private Evidence doctor output")?; let mut stderr = tempfile::tempfile().context("creating private Evidence doctor diagnostics")?; let mut command = Command::new(evidence); command - .arg("--runtime") - .arg(runtime_config) .arg("check") + .arg("--runtime-config") + .arg(runtime_config) + .env_remove("REGISTRY_EVIDENCE_RUNTIME") .stdin(Stdio::null()) .stdout(Stdio::from(stdout.try_clone()?)) .stderr(Stdio::from(stderr.try_clone()?)); @@ -223,6 +270,9 @@ fn invoke_check( if let Some(root) = audit_root { command.arg("--require-audit-under").arg(root); } + if without_audit_lock { + command.arg("--without-audit-lock"); + } let mut child = command.spawn().with_context(|| { format!( "starting Evidence dependency preflight at {}", @@ -302,14 +352,18 @@ fn render_refusal( operation: "doctor", status, runtime_config, - proof_boundary: "live startup dependency preflight; no public listener, evidence-data request, or application audit event was produced", + proof_boundary: LOCKED_PROOF_BOUNDARY, diagnostics: vec![Diagnostic { severity: "error", code, artifact: runtime_config.display().to_string(), path: "$", - message: if detail.is_empty() { "Evidence runtime check failed without a diagnostic.".to_owned() } else { detail }, - suggested_action: "Read the value-free Evidence diagnostic, correct the selected runtime artifact or dependency, and rerun doctor.", + message: if detail.is_empty() { + "Evidence runtime check failed without a diagnostic.".to_owned() + } else { + detail + }, + suggested_action: suggested_action(runtime_diagnostic), }], }; match format { @@ -320,9 +374,13 @@ fn render_refusal( runtime_config.display() ); std::io::stderr().write_all(runtime_diagnostic)?; - eprintln!( - "Next: correct the selected runtime artifact or dependency and rerun doctor." - ); + if suggested_action(runtime_diagnostic) == HELD_LOCK_ACTION { + eprintln!("Next: {HELD_LOCK_ACTION}"); + } else { + eprintln!( + "Next: correct the selected runtime artifact or dependency and rerun doctor." + ); + } } } Ok(ExitCode::from(exit)) @@ -360,13 +418,14 @@ mod tests { .expect("script"); drop(script); - let outcome = invoke_check(&binary, &runtime, true, Some(root.path())).expect("preflight"); + let outcome = + invoke_check(&binary, &runtime, true, Some(root.path()), false).expect("preflight"); assert!(outcome.status.success()); let invoked = fs::read_to_string(arguments).expect("arguments"); assert_eq!( invoked, format!( - "--runtime\n{}\ncheck\n--require-runtime-dependencies\n--require-audit-under\n{}\n", + "check\n--runtime-config\n{}\n--require-runtime-dependencies\n--require-audit-under\n{}\n", runtime.display(), root.path().display() ) @@ -374,6 +433,43 @@ mod tests { assert!(!invoked.contains("serve")); assert!(!invoked.contains("evaluate")); assert!(!root.path().join("audit.jsonl").exists()); + + let outcome = invoke_check(&binary, &runtime, true, None, true).expect("preflight"); + assert!(outcome.status.success()); + assert_eq!( + fs::read_to_string(root.path().join("arguments")).expect("arguments"), + format!( + "check\n--runtime-config\n{}\n--require-runtime-dependencies\n--without-audit-lock\n", + runtime.display() + ) + ); + } + + /// An automated consumer reads `proofBoundary`, not the human lines, so + /// the lock-free form must say there what it left unproved. + #[test] + fn the_lock_free_form_states_its_own_proof_boundary() { + let locked = proof_boundary(false); + let lock_free = proof_boundary(true); + assert_ne!(locked, lock_free); + assert!(!locked.contains("lock")); + assert!(lock_free.contains("audit writer lock was not taken")); + assert!(lock_free.contains("second writer is not detected")); + } + + #[test] + fn a_held_audit_lock_points_at_the_lock_free_form() { + assert_eq!( + suggested_action( + b"evidence: runtime audit initialization failed: another process holds the \ + single-writer lock beside the audit file; stop it before starting this one\n" + ), + HELD_LOCK_ACTION + ); + assert_eq!( + suggested_action(b"evidence: runtime secret initialization failed\n"), + DEFAULT_ACTION + ); } #[test] diff --git a/crates/registry-evidencectl/src/source_cli.rs b/crates/registry-evidencectl/src/source_cli.rs index 2cb46f0025..805ea47004 100644 --- a/crates/registry-evidencectl/src/source_cli.rs +++ b/crates/registry-evidencectl/src/source_cli.rs @@ -302,6 +302,14 @@ mod tests { } fn export(&self, name: &str, extract: &str) -> PathBuf { + self.export_with_facts( + name, + extract, + "type: object\nproperties: {}\nadditionalProperties: false\n", + ) + } + + fn export_with_facts(&self, name: &str, extract: &str, facts: &str) -> PathBuf { let root = self.root.path().join(name); let artifacts = [ ("sources/lookup.yaml", SOURCE), @@ -317,10 +325,7 @@ mod tests { "schemas/lookup-response.yaml", "type: object\nproperties: {}\nadditionalProperties: false\n", ), - ( - "schemas/lookup-facts.yaml", - "type: object\nproperties: {}\nadditionalProperties: false\n", - ), + ("schemas/lookup-facts.yaml", facts), ( "adapters/lookup-prepare.rhai", "fn prepare(selectors, context) { #{query: [], body: ()} }\n", @@ -387,6 +392,51 @@ mod tests { ); } + #[test] + fn diff_and_apply_refuse_a_next_fact_schema_that_drops_a_fact_a_derivation_reads() { + let facts = |name: &str| { + format!( + "type: object\nrequired: [{name}]\nproperties:\n {name}: {{type: string}}\nadditionalProperties: false\n" + ) + }; + let fixture = Fixture::new(); + let first = fixture.export_with_facts("first", EXTRACT_ONE, &facts("status")); + review_with_revisions(fixture.args(first), true, &mut Vec::new(), no_target).unwrap(); + for (path, text) in [ + ( + "questions/record-active.yaml", + "id: record-active\nquestion: Is the record active?\npurpose: record-verification\nsubject:\n role: subject\n profiles: [record-code]\nsource:\n ref: lookup\nanswers:\n- concept: active\n type: boolean\nderivation: derivations/record-active.rhai\ndisclosure:\n allow: [active]\n", + ), + ( + "derivations/record-active.rhai", + "fn answer(facts, selectors, context) {\n let status = required(facts[\"status\"], \"status_missing\");\n #{active: status == \"active\"}\n}\n", + ), + ] { + fs::create_dir_all(fixture.project.join(path).parent().unwrap()).unwrap(); + fs::write(fixture.project.join(path), text).unwrap(); + } + let installed = fs::read(fixture.project.join("schemas/lookup-facts.yaml")).unwrap(); + + for apply in [false, true] { + let renamed = + fixture.export_with_facts("renamed", EXTRACT_ONE, &facts("lifecycle_status")); + let mut output = Vec::new(); + let error = review_with_revisions(fixture.args(renamed), apply, &mut output, no_target) + .expect_err("a renamed fact the derivation still reads"); + assert!( + format!("{error:#}").contains("reads fact \"status\""), + "error was: {error:#}" + ); + let report = parsed(&output); + assert_eq!(report["validation"]["status"], "failed"); + assert_eq!( + fs::read(fixture.project.join("schemas/lookup-facts.yaml")).unwrap(), + installed + ); + fs::remove_dir_all(fixture.root.path().join("renamed")).unwrap(); + } + } + #[test] fn diff_and_apply_validate_the_complete_graph_and_report_failure_without_installing_files() { for apply in [false, true] { @@ -458,8 +508,8 @@ mod tests { args.target = Some(target.clone()); let mut output = Vec::new(); let mut calls = Vec::new(); - // This test pins command-to-compiler routing. The build helper's tests - // pin the real runtime revision computation; this layer never hashes it. + // This test pins command-to-compiler routing. The package helper's + // tests pin configuration revisions; this layer never hashes them. review_with_revisions(args, false, &mut output, |project, selected| { assert_eq!(selected, target); calls.push(project.to_path_buf()); diff --git a/crates/registry-evidencectl/src/suggest/emit.rs b/crates/registry-evidencectl/src/suggest/emit.rs index c2d7c6028b..c224ea5cac 100644 --- a/crates/registry-evidencectl/src/suggest/emit.rs +++ b/crates/registry-evidencectl/src/suggest/emit.rs @@ -344,7 +344,7 @@ fn write_new_authoring_file(path: &Path, contents: &[u8]) -> Result<()> { .with_context(|| format!("persisting {}", path.display())) } -/// Run `evidence --runtime /runtime.yaml check` and classify the +/// Run `evidence check --runtime-config /runtime.yaml` and classify the /// result. `evidence_bin` resolves the same way as the other `evidencectl` /// subcommands that shell out to the runtime binary: an explicit path, else /// `EVIDENCE_BIN`, else the first `evidence` found on `PATH`, and the @@ -357,9 +357,10 @@ pub fn verify(project: &Path, evidence_bin: Option<&Path>) -> Result, @@ -342,9 +347,10 @@ pub(crate) fn create_local_target( let mut governance = crate::authoring::local_target_governance(&relative)?; governance["sourceConnections"] = source_connections; let mut runtime = serde_json::json!({ - "version": 1, + "apiVersion": EVIDENCE_RUNTIME_API_VERSION, + "kind": EVIDENCE_RUNTIME_KIND, "listener": { - "bindHost": "127.0.0.1", "port": 8080, + "bind": "127.0.0.1:8080", "tlsTermination": "operator-controlled-upstream", "trustProxyIdentityHeaders": false, "maximumRequestBytes": 65536, "maximumConcurrentRequests": 64, "requestTimeoutMilliseconds": 10000, "shutdownGraceMilliseconds": 30000, @@ -414,7 +420,7 @@ fn fill_local_paths(project: &Path, governance: &Value, runtime: &mut Value) -> let secrets = project.join(crate::authoring::SECRETS_DIRECTORY); let local = project.join(".evidence/dev"); for (components, path) in [ - (vec!["bundleDirectory"], local.join("bundle")), + (vec!["package", "root"], local.join("bundle")), (vec!["secretProviders", "file", "root"], secrets), (vec!["audit", "path"], local.join("audit/evidence.jsonl")), ] { @@ -463,8 +469,11 @@ fn validate_settings_documents(governance: &Value, runtime: &Value) -> Result<() .context("target settings governance is not the closed deployment governance shape")? .into_bundle() .context("target settings governance is not the closed deployment governance shape")?; - if runtime.get("version").and_then(Value::as_u64) != Some(1) { - bail!("target settings runtime.version must be 1"); + if runtime.get("apiVersion").and_then(Value::as_str) != Some(EVIDENCE_RUNTIME_API_VERSION) { + bail!("target settings runtime.apiVersion must be {EVIDENCE_RUNTIME_API_VERSION}"); + } + if runtime.get("kind").and_then(Value::as_str) != Some(EVIDENCE_RUNTIME_KIND) { + bail!("target settings runtime.kind must be {EVIDENCE_RUNTIME_KIND}"); } for section in [ "service", @@ -479,11 +488,12 @@ fn validate_settings_documents(governance: &Value, runtime: &Value) -> Result<() require_nonempty_mapping(governance, section, "target settings governance")?; } if runtime - .get("bundleDirectory") + .get("package") + .and_then(|package| package.get("root")) .and_then(Value::as_str) .is_none_or(str::is_empty) { - bail!("target settings runtime.bundleDirectory must be a string"); + bail!("target settings runtime.package.root must be a string"); } for section in [ "listener", @@ -926,7 +936,8 @@ governance: issuer: id: urn:example:issuer authentication: - kind: oidc-access-token + oidc: + issuer: https://issuer.example audit: hashKeyVersion: 1 subjectBinding: @@ -939,10 +950,12 @@ governance: local: kind: explicit-request runtime: - version: 1 - bundleDirectory: /tmp/evidence/bundle + apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 + kind: EvidenceRuntimeConfig + package: + root: /tmp/evidence/bundle listener: - bindHost: 127.0.0.1 + bind: 127.0.0.1:8080 secretProviders: file: root: /tmp/evidence/secrets @@ -965,7 +978,7 @@ runtime: let mut runtime = serde_json::json!({}); fill_local_paths(&project, &local, &mut runtime).unwrap(); assert_eq!( - runtime["bundleDirectory"], + runtime["package"]["root"], project .join(".evidence/dev/bundle") .to_string_lossy() @@ -982,7 +995,7 @@ runtime: .to_string_lossy() .as_ref() ); - runtime["bundleDirectory"] = Value::String("/srv/reviewed/bundle".to_owned()); + runtime["package"]["root"] = Value::String("/srv/reviewed/bundle".to_owned()); runtime["secretProviders"]["file"]["root"] = Value::String("/srv/reviewed/secrets".to_owned()); runtime["audit"]["path"] = Value::String("/srv/reviewed/audit.jsonl".to_owned()); @@ -1399,7 +1412,8 @@ governance: issuer: id: urn:example:issuer authentication: - kind: oidc-access-token + oidc: + issuer: https://issuer.example audit: hashKeyVersion: 1 subjectBinding: @@ -1413,7 +1427,8 @@ governance: local: kind: explicit-request runtime: - version: 1 + apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 + kind: EvidenceRuntimeConfig publicKeys: _QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json: active.jwk.json "#, @@ -1450,7 +1465,8 @@ governance: issuer: id: urn:example:issuer authentication: - kind: oidc-access-token + oidc: + issuer: https://issuer.example audit: hashKeyVersion: 1 subjectBinding: @@ -1465,7 +1481,8 @@ governance: kind: explicit-request unexpectedField: true runtime: - version: 1 + apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 + kind: EvidenceRuntimeConfig "#, ) .expect("settings"); @@ -1581,11 +1598,11 @@ runtime: /// A value the mirror can deserialize for the contract property `name`. The /// mirror leaves the interior of each section to the runtime, so only - /// `version` and `bundleDirectory` need a shape of their own. + /// `apiVersion` and `kind` need a shape of their own. fn sample_runtime_value(name: &str) -> Value { match name { - "version" => serde_json::json!(1), - "bundleDirectory" => serde_json::json!("/tmp/evidence/bundle"), + "apiVersion" => serde_json::json!(EVIDENCE_RUNTIME_API_VERSION), + "kind" => serde_json::json!(EVIDENCE_RUNTIME_KIND), _ => serde_json::json!({}), } } @@ -1606,7 +1623,8 @@ governance: issuer: id: urn:example:issuer authentication: - kind: oidc-access-token + oidc: + issuer: https://issuer.example audit: hashKeyVersion: 1 subjectBinding: @@ -1620,7 +1638,8 @@ governance: local: kind: explicit-request runtime: - version: 1 + apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 + kind: EvidenceRuntimeConfig "#, ) .expect("settings"); diff --git a/crates/registry-evidencectl/tests/audit_view.rs b/crates/registry-evidencectl/tests/audit_view.rs index 43a9d85b00..771f361539 100644 --- a/crates/registry-evidencectl/tests/audit_view.rs +++ b/crates/registry-evidencectl/tests/audit_view.rs @@ -75,13 +75,13 @@ fn successful_view_delegates_to_stopped_core_and_prints_only_aliases() { assert_eq!( arguments.lines().collect::>(), [ - "--runtime", + "local-audit-last-operation", + "--runtime-config", fs::canonicalize(&fixture.root) .expect("canonical project") .join(".evidence/dev/runtime.yaml") .to_str() .expect("runtime path"), - "local-audit-last-operation", ] ); let rendered = String::from_utf8_lossy(&output.stdout); @@ -200,6 +200,7 @@ fn structured_sd_jwt_release_uses_the_same_minimized_audit_view() { ) .expect("bundle writes"); fs::set_permissions(&bundle_path, fs::Permissions::from_mode(0o400)).expect("bundle mode"); + refresh_package_envelope(&fixture.root.join(".evidence/dev/bundle")); let mut view = successful_view(); view["events"][0]["responseProtection"] = json!("sd-jwt-vc"); @@ -264,6 +265,7 @@ fn multi_concept_release_requires_and_prints_the_exact_declared_list() { .expect("bundle writes"); fs::set_permissions(&bundle_path, fs::Permissions::from_mode(0o400)) .expect("seal updated bundle"); + refresh_package_envelope(&fixture.root.join(".evidence/dev/bundle")); let mut view = successful_view(); view["events"][1]["disclosedConcepts"] = json!([ @@ -629,6 +631,7 @@ impl Fixture { .as_bytes(), 0o400, ); + refresh_package_envelope(&root.join(".evidence/dev/bundle")); let evidence = temporary.path().join("evidence-stub"); executable( @@ -697,6 +700,32 @@ fn executable(path: &Path, contents: &[u8]) { private_file(path, contents, 0o700); } +/// Republish the local test package after an intended authored change. +fn refresh_package_envelope(root: &Path) { + fs::set_permissions(root, fs::Permissions::from_mode(0o700)) + .expect("open package directory for publication"); + let sum_file = root.join(registry_platform_config::SUM_FILE); + if sum_file.exists() { + fs::remove_file(&sum_file).expect("remove stale package sum file"); + } + registry_platform_config::write_sum_file( + root, + None, + ®istry_platform_config::PackageLimits { + max_files: 1_024, + max_file_bytes: 1024 * 1024, + max_total_bytes: 16 * 1024 * 1024, + max_depth: 3, + max_path_bytes: 128, + }, + "evidencectl package", + ) + .expect("publish local test package"); + fs::set_permissions(sum_file, fs::Permissions::from_mode(0o400)) + .expect("seal package sum file"); + fs::set_permissions(root, fs::Permissions::from_mode(0o500)).expect("seal package directory"); +} + fn assert_success(output: &Output) { assert!( output.status.success(), @@ -786,6 +815,11 @@ fn json_mode_embeds_the_validated_core_view_and_keeps_failures_value_free() { #[test] fn a_project_without_a_local_session_names_the_missing_session() { let fixture = Fixture::new(); + fs::set_permissions( + fixture.root.join(".evidence/dev/bundle"), + fs::Permissions::from_mode(0o700), + ) + .expect("unseal the package directory for removal"); fs::remove_dir_all(fixture.root.join(".evidence/dev")).expect("remove session"); fixture.write_core_json(&successful_view()); diff --git a/crates/registry-evidencectl/tests/cli_surface.rs b/crates/registry-evidencectl/tests/cli_surface.rs index b748d26ed4..f3333b04db 100644 --- a/crates/registry-evidencectl/tests/cli_surface.rs +++ b/crates/registry-evidencectl/tests/cli_surface.rs @@ -11,7 +11,7 @@ use std::process::Command; /// The complete top-level subcommand set. Adding a command means adding it /// here; the point of the list is that an omission fails rather than passes. const TOP_LEVEL_COMMANDS: [&str; 13] = [ - "client", "access", "keygen", "jwks", "new", "build", "fixtures", "source", "dev", "request", + "client", "access", "keygen", "jwks", "new", "package", "fixtures", "source", "dev", "request", "verify", "audit", "tooling", ]; diff --git a/crates/registry-evidencectl/tests/doctor.rs b/crates/registry-evidencectl/tests/doctor.rs index 71762be2d7..1935a532ec 100644 --- a/crates/registry-evidencectl/tests/doctor.rs +++ b/crates/registry-evidencectl/tests/doctor.rs @@ -659,7 +659,7 @@ const LOCAL_SIGNER: &str = const TRANSIT_SIGNER: &str = "signer:\n kind: transit\n unixSocketPath: /run/registry-evidence/transit-proxy.sock\n mount: transit\n keyName: evidence-signing\n keyVersion: 1\n timeoutMilliseconds: 2000\n"; fn runtime_document(signer: &str) -> String { - format!("bundleDirectory: bundle\nsecretProviders:\n file:\n root: secrets\n{signer}audit:\n path: audit/evidence.jsonl\n") + format!("package:\n root: bundle\nsecretProviders:\n file:\n root: secrets\n{signer}audit:\n path: audit/evidence.jsonl\n") } fn rewrite_runtime(project: &Path, signer: &str) { @@ -686,15 +686,17 @@ fn provision(project: &Path) { fs::write( project.join("bundle/evidence.yaml"), r#"authentication: - kind: oidc-access-token - issuer: https://identity.invalid - audiences: [evidence-scaffold] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: https://identity.invalid/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience + oidc: + issuer: https://identity.invalid + audience: evidence-scaffold + jwksSource: + kind: uri + uri: https://identity.invalid/.well-known/jwks.json + tokenTypes: [at+jwt] + algorithms: [ES256] + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience signing: secret:file/signing-p256-private-jwk audit: secret:file/audit-hmac-key subjectBinding: secret:file/subject-binding-hmac-key @@ -863,7 +865,7 @@ fn doctor_names_the_next_commands_when_the_project_is_still_editable() { assert!( !output.status.success(), - "doctor inspects a deployment project only" + "doctor inspects a deployment target only" ); let message = stderr_of(&output); assert!( @@ -881,7 +883,7 @@ fn doctor_names_the_next_commands_when_the_project_is_still_editable() { } #[test] -fn doctor_names_the_build_command_for_a_directory_that_is_neither_shape() { +fn doctor_names_the_package_command_for_a_directory_that_is_neither_shape() { let workspace = tempfile::tempdir().expect("tempdir"); let output = Command::new(env!("CARGO_BIN_EXE_evidencectl")) @@ -893,7 +895,7 @@ fn doctor_names_the_build_command_for_a_directory_that_is_neither_shape() { assert!(!output.status.success()); let message = stderr_of(&output); assert!( - message.contains("deployment project"), + message.contains("deployment target"), "the refusal must name the shape it needs: {message}" ); assert!( @@ -939,7 +941,7 @@ fn doctor_project_shape_refusal_is_one_value_safe_json_diagnostic() { assert!(diagnostic["message"] .as_str() .expect("message") - .contains("deployment project")); + .contains("deployment target")); assert!(diagnostic["suggestedAction"] .as_str() .expect("suggested action") diff --git a/crates/registry-evidencectl/tests/fixtures.rs b/crates/registry-evidencectl/tests/fixtures.rs index 7facfbc886..52ac157de4 100644 --- a/crates/registry-evidencectl/tests/fixtures.rs +++ b/crates/registry-evidencectl/tests/fixtures.rs @@ -183,11 +183,11 @@ fn happy_path_runs_check_then_each_fixture_and_reports_pass() { assert_eq!( invocations, vec![ - vec!["--runtime", runtime_path, "check"], + vec!["check", "--runtime-config", runtime_path], vec![ - "--runtime", - runtime_path, "evaluate", + "--runtime-config", + runtime_path, "--fixture", "fixtures/a.yaml", "--explain", @@ -195,9 +195,9 @@ fn happy_path_runs_check_then_each_fixture_and_reports_pass() { "json" ], vec![ - "--runtime", - runtime_path, "evaluate", + "--runtime-config", + runtime_path, "--fixture", "fixtures/b.yaml", "--explain", @@ -244,11 +244,11 @@ fn fixture_selection_runs_only_one_exact_referenced_fixture() { assert_eq!( read_argv_log(&argv_log), vec![ - vec!["--runtime", runtime_path, "check"], + vec!["check", "--runtime-config", runtime_path], vec![ - "--runtime", - runtime_path, "evaluate", + "--runtime-config", + runtime_path, "--fixture", "fixtures/a.yaml", "--case", @@ -912,11 +912,11 @@ fn explain_is_asked_of_every_evaluation_and_the_trace_is_relayed() { assert_eq!( invocations, vec![ - vec!["--runtime", runtime_path, "check"], + vec!["check", "--runtime-config", runtime_path], vec![ - "--runtime", - runtime_path, "evaluate", + "--runtime-config", + runtime_path, "--fixture", "fixtures/a.yaml", "--explain", @@ -924,9 +924,9 @@ fn explain_is_asked_of_every_evaluation_and_the_trace_is_relayed() { "json" ], vec![ - "--runtime", - runtime_path, "evaluate", + "--runtime-config", + runtime_path, "--fixture", "fixtures/b.yaml", "--explain", @@ -1066,13 +1066,13 @@ fn unresolvable_evidence_binary_errors_clearly() { } #[test] -fn fixtures_are_discovered_at_a_relative_bundle_directory_named_in_runtime_yaml() { +fn fixtures_are_discovered_at_a_relative_package_root_named_in_runtime_yaml() { let dir = tempfile::tempdir().expect("tempdir"); let project = dir.path().join("project"); fs::create_dir_all(project.join("custom-bundle")).expect("create custom bundle dir"); fs::write( project.join("runtime.yaml"), - b"placeholder: true\nbundleDirectory: custom-bundle\n", + b"placeholder: true\npackage:\n root: custom-bundle\n", ) .expect("write runtime.yaml"); fs::write( @@ -1102,7 +1102,7 @@ fn fixtures_are_discovered_at_a_relative_bundle_directory_named_in_runtime_yaml( } #[test] -fn fixtures_are_discovered_at_an_absolute_bundle_directory_named_in_runtime_yaml() { +fn fixtures_are_discovered_at_an_absolute_package_root_named_in_runtime_yaml() { let dir = tempfile::tempdir().expect("tempdir"); let project = dir.path().join("project"); fs::create_dir_all(&project).expect("create project dir"); @@ -1110,7 +1110,7 @@ fn fixtures_are_discovered_at_an_absolute_bundle_directory_named_in_runtime_yaml fs::create_dir_all(&bundle_directory).expect("create bundle dir"); let runtime_yaml = format!( - "placeholder: true\nbundleDirectory: {}\n", + "placeholder: true\npackage:\n root: {}\n", bundle_directory.to_str().expect("bundle directory is utf8") ); fs::write(project.join("runtime.yaml"), runtime_yaml).expect("write runtime.yaml"); @@ -1263,7 +1263,7 @@ fn a_fresh_local_target_journey_checks_and_explains_without_a_pretargets_directo "exit 0\n", ), version = registry_platform_buildinfo::DISPLAY_VERSION, - report = r#"{"bundleRevision":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","requirements":[{"id":"urn:example:requirement:record-status:v1","configurationRevision":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}]}"#, + report = r#"{"packageDigest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","requirements":[{"id":"urn:example:requirement:record-status:v1","configurationRevision":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}]}"#, ); fs::write(&stub, stub_script).expect("write check stub"); let mut permissions = fs::metadata(&stub).expect("stat stub").permissions(); diff --git a/crates/registry-evidencectl/tests/production_build.rs b/crates/registry-evidencectl/tests/production_build.rs index 08668e91fe..fb866dc3f8 100644 --- a/crates/registry-evidencectl/tests/production_build.rs +++ b/crates/registry-evidencectl/tests/production_build.rs @@ -22,6 +22,27 @@ const SECRET_CANARY: &str = "production-build-secret-canary"; const CHECK_DIAGNOSTIC: &str = "evidence bundle-check diagnostic: derivations/answer.rhai failed static compilation"; +#[test] +fn retired_build_command_refuses_with_package_replacement() { + let fixture = Fixture::new(); + let output = Command::new(env!("CARGO_BIN_EXE_evidencectl")) + .arg("build") + .arg("--project") + .arg(&fixture.project) + .arg("--target") + .arg(&fixture.target) + .arg("--output") + .arg(&fixture.output) + .output() + .expect("retired build command starts"); + + assert_failed(&output, "retired build command"); + assert!(!fixture.output.exists()); + let message = stderr(&output); + assert!(message.contains("evidence.build.retired"), "{message}"); + assert!(message.contains("evidencectl package"), "{message}"); +} + #[test] fn build_is_create_only_and_never_changes_an_existing_output() { let fixture = Fixture::new(); @@ -118,38 +139,30 @@ fn failed_runtime_check_leaves_no_output_or_private_staging() { #[test] fn a_rejected_bundle_and_fixture_name_the_command_that_shows_the_diagnosis() { let rejected_bundle = Fixture::new(); - let project = fs::canonicalize(&rejected_bundle.project).expect("canonical project"); let output = rejected_bundle.build_failing("check"); assert_failed(&output, "a rejected bundle must fail the build"); let message = stderr(&output); assert!( - message.contains(&format!( - "Run `evidencectl test {}` to read the diagnosis Evidence prints.", - project.display() - )), - "the refusal names the command that shows the diagnosis: {message}" + message.contains("evidence.package.failed"), + "the package refusal keeps its stable safe diagnostic: {message}" ); assert!( - message.contains(CHECK_DIAGNOSTIC), - "the refusal includes what Evidence printed: {message}" + !message.contains(CHECK_DIAGNOSTIC), + "the package refusal does not relay delegated stderr: {message}" ); assert_value_free(&output); let rejected_fixture = Fixture::new(); - let project = fs::canonicalize(&rejected_fixture.project).expect("canonical project"); let output = rejected_fixture.build_failing("fixture:fixtures/answer.yaml"); assert_failed(&output, "a rejected fixture must fail the build"); let message = stderr(&output); assert!( - message.contains(&format!( - "Run `evidencectl test {} --fixture fixtures/answer.yaml` to read the diagnosis Evidence prints.", - project.display() - )), - "the refusal names the rejected fixture with the command: {message}" + message.contains("evidence.package.failed"), + "the package refusal keeps its stable safe diagnostic: {message}" ); assert!( !message.contains("Evidence reported:"), @@ -167,16 +180,12 @@ fn a_long_bundle_check_diagnostic_is_bounded_and_says_so() { assert_failed(&output, "a long rejected bundle must still fail the build"); let message = stderr(&output); assert!( - message.contains("diagnostic trimmed to the last 40 lines"), - "a long diagnostic says it was trimmed: {message}" + message.contains("evidence.package.failed"), + "a long delegated diagnostic maps to the stable safe refusal: {message}" ); assert!( - message.contains("evidence bundle-check diagnostic line 200"), - "the most recent line survives the trim: {message}" - ); - assert!( - !message.contains("evidence bundle-check diagnostic line 1\n"), - "an earlier line does not survive the trim: {message}" + !message.contains("evidence bundle-check diagnostic line"), + "delegated stderr is not relayed by package: {message}" ); } @@ -236,37 +245,29 @@ fn an_evidence_binary_that_does_not_identify_itself_is_refused_before_any_step() } #[test] -fn successful_build_copies_runtime_exactly_and_excludes_local_and_validation_secrets() { +fn successful_package_excludes_runtime_local_state_and_validation_secrets() { let fixture = Fixture::new(); let local = fixture.project.join(".evidence/dev"); fs::create_dir_all(&local).expect("local state"); fs::write(local.join("disposable-private-key"), SECRET_CANARY).expect("local secret"); - let runtime = fs::read(&fixture.runtime).expect("target runtime"); - let output = fixture.build(); - assert_success(&output, "production build"); - assert_eq!( - fs::read(fixture.output.join("runtime.yaml")).unwrap(), - runtime - ); + assert_success(&output, "production package"); let snapshot = snapshot(&fixture.output); assert_eq!( snapshot.keys().cloned().collect::>(), vec![ - PathBuf::from("bundle/adapters/source-extract.rhai"), - PathBuf::from("bundle/adapters/source-prepare.rhai"), - PathBuf::from("bundle/catalog.jsonld"), - PathBuf::from("bundle/derivations/answer.rhai"), - PathBuf::from("bundle/evidence.yaml"), - PathBuf::from("bundle/fixtures/answer.yaml"), - PathBuf::from( - "bundle/public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json", - ), - PathBuf::from("bundle/schemas/facts.schema.yaml"), - PathBuf::from("bundle/schemas/parameters.schema.yaml"), - PathBuf::from("bundle/schemas/response.schema.yaml"), - PathBuf::from("runtime.yaml"), + PathBuf::from("SHA256SUMS"), + PathBuf::from("adapters/source-extract.rhai"), + PathBuf::from("adapters/source-prepare.rhai"), + PathBuf::from("catalog.jsonld"), + PathBuf::from("derivations/answer.rhai"), + PathBuf::from("evidence.yaml"), + PathBuf::from("fixtures/answer.yaml"), + PathBuf::from("public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json",), + PathBuf::from("schemas/facts.schema.yaml"), + PathBuf::from("schemas/parameters.schema.yaml"), + PathBuf::from("schemas/response.schema.yaml"), ] ); for (path, bytes) in snapshot { @@ -299,8 +300,8 @@ fn an_empty_publication_description_leaves_no_candidate() { ); let stderr = String::from_utf8_lossy(&output.stderr); assert!( - stderr.contains("publication"), - "the refusal names the publication the bundle declares: {stderr}" + stderr.contains("evidence.package.failed"), + "the refusal uses the stable package diagnostic: {stderr}" ); assert!(!fixture.output.exists()); fixture.assert_no_staging_residue(); @@ -317,38 +318,7 @@ fn a_bundle_without_publication_carries_no_catalog_description() { .expect("evidencectl build starts"); assert_success(&output, "build of a bundle that declares no publication"); - assert!(!fixture.output.join("bundle/catalog.jsonld").exists()); - fixture.assert_no_staging_residue(); -} - -#[test] -fn local_target_build_accepts_local_source_without_production_transport_requirements() { - let fixture = Fixture::new(); - fs::write( - &fixture.governance, - GOVERNANCE.replace("assuranceProfile: production", "assuranceProfile: local"), - ) - .expect("local target governance"); - fs::write( - fixture.project.join("sources/registry.yaml"), - SOURCE - .replace("baseUrl: https://registry.invalid", "baseUrl: http://127.0.0.1:8088") - .replace( - "authentication: {kind: static-authorization, tokenRef: 'secret:file/source-token'}", - "authentication: {kind: none}", - ), - ) - .expect("local source"); - - let output = fixture.build(); - - assert_success(&output, "local target build"); - assert_eq!(fixture.steps(), ["check", "evaluate:fixtures/answer.yaml"]); - let bundle = - fs::read_to_string(fixture.output.join("bundle/evidence.yaml")).expect("generated bundle"); - assert!(bundle.contains("assuranceProfile: local")); - assert!(bundle.contains("baseUrl: http://127.0.0.1:8088")); - assert!(!String::from_utf8_lossy(&output.stdout).contains("source-token")); + assert!(!fixture.output.join("catalog.jsonld").exists()); fixture.assert_no_staging_residue(); } @@ -448,44 +418,28 @@ fn package_names_a_local_target_before_judging_the_output_location() { } #[test] -fn package_refuses_a_bundle_directory_that_does_not_resolve_to_the_output() { +fn package_accepts_a_stable_package_root_outside_the_output() { for format in ["human", "json"] { let fixture = Fixture::new(); let output = fixture.package(format); - assert_failed(&output, "stale bundle directory"); - assert!(!fixture.output.exists()); - assert!(fixture.invocations().is_empty()); - assert_value_free(&output); - let expected_bundle = fixture.output.join("bundle").display().to_string(); + assert_success(&output, "stable package root"); + assert!(fixture.output.join("evidence.yaml").is_file()); match format { "human" => { - assert!(output.stdout.is_empty()); - let message = stderr(&output); - assert!(message.contains("evidence.package.bundle-directory-mismatch")); - assert!(message.contains("runtime.yaml:/bundleDirectory")); - assert!(message.contains("/srv/evidence/candidate/bundle")); - assert!(message.contains(&expected_bundle)); + assert!(stderr(&output).is_empty()); + assert!(String::from_utf8_lossy(&output.stdout).contains("Package digest: sha256:")); } "json" => { assert!(output.stderr.is_empty()); let report: serde_json::Value = - serde_json::from_slice(&output.stdout).expect("JSON refusal report"); - assert_eq!(report["status"], "domain-refusal"); - assert_eq!( - report["diagnostics"][0]["code"], - "evidence.package.bundle-directory-mismatch" - ); - assert_eq!( - report["diagnostics"][0]["path"], - "runtime.yaml:/bundleDirectory" - ); - let message = report["diagnostics"][0]["message"] + serde_json::from_slice(&output.stdout).expect("JSON package report"); + assert_eq!(report["status"], "packaged"); + assert!(report["packageDigest"] .as_str() - .expect("diagnostic message"); - assert!(message.contains("/srv/evidence/candidate/bundle")); - assert!(message.contains(&expected_bundle)); + .unwrap() + .starts_with("sha256:")); } _ => unreachable!(), } @@ -494,14 +448,14 @@ fn package_refuses_a_bundle_directory_that_does_not_resolve_to_the_output() { } #[test] -fn package_accepts_a_bundle_directory_that_resolves_to_the_output() { +fn package_refuses_a_package_root_inside_the_output() { let fixture = Fixture::new(); - let matching_bundle_directory = fixture.output.join("bundle").display().to_string(); + let unstable_package_directory = fixture.output.join("nested").display().to_string(); fs::write( &fixture.runtime, TARGET_RUNTIME.replacen( - "bundleDirectory: /srv/evidence/candidate/bundle", - &format!("bundleDirectory: {matching_bundle_directory}"), + "root: /srv/evidence/candidate/bundle", + &format!("root: {unstable_package_directory}"), 1, ), ) @@ -509,16 +463,12 @@ fn package_accepts_a_bundle_directory_that_resolves_to_the_output() { let output = fixture.package("human"); - assert_success(&output, "self-consistent package"); - assert!(fixture.output.join("bundle").is_dir()); - assert_eq!( - fs::read_to_string(fixture.output.join("runtime.yaml")).unwrap(), - TARGET_RUNTIME.replacen( - "bundleDirectory: /srv/evidence/candidate/bundle", - &format!("bundleDirectory: {matching_bundle_directory}"), - 1, - ) - ); + assert_failed(&output, "unstable package root"); + assert!(!fixture.output.exists()); + let message = stderr(&output); + assert!(message.contains("evidence.package.root-unstable")); + assert!(message.contains("runtime.yaml:/package/root")); + assert!(message.contains(&unstable_package_directory)); fixture.assert_no_staging_residue(); } @@ -531,20 +481,17 @@ fn sqlite_extract_build_copies_the_statement_without_http_only_artifacts() { assert_success(&output, "SQLite extract production build"); assert_eq!( - fs::read_to_string(fixture.output.join("bundle/queries/source.sql")).unwrap(), + fs::read_to_string(fixture.output.join("queries/source.sql")).unwrap(), "SELECT :reference <> '' AS allowed;\n" ); assert!(fixture .output - .join("bundle/adapters/source-extract.rhai") + .join("adapters/source-extract.rhai") .is_file()); + assert!(!fixture.output.join("adapters/source-prepare.rhai").exists()); assert!(!fixture .output - .join("bundle/adapters/source-prepare.rhai") - .exists()); - assert!(!fixture - .output - .join("bundle/schemas/parameters.schema.yaml") + .join("schemas/parameters.schema.yaml") .exists()); assert_eq!(fixture.steps(), ["check", "evaluate:fixtures/answer.yaml"]); fixture.assert_no_staging_residue(); @@ -709,16 +656,6 @@ fn unresolved_review_markers_and_unknown_target_fields_fail_closed() { fn package_names_the_file_and_rule_of_an_unresolved_review_marker() { for format in ["human", "json"] { let fixture = Fixture::new(); - let matching_bundle_directory = fixture.output.join("bundle").display().to_string(); - fs::write( - &fixture.runtime, - TARGET_RUNTIME.replacen( - "bundleDirectory: /srv/evidence/candidate/bundle", - &format!("bundleDirectory: {matching_bundle_directory}"), - 1, - ), - ) - .expect("self-consistent target runtime"); fs::write( fixture.project.join("fixtures/answer.yaml"), "fixture: TODO(evidencectl)\n", @@ -848,7 +785,7 @@ fn every_referenced_fixture_is_delegated_and_one_failure_prevents_publication() } #[test] -fn identical_inputs_produce_identical_bundle_bytes_revision_and_stable_report_shape() { +fn identical_inputs_produce_identical_package_bytes_digest_and_stable_report_shape() { let fixture = Fixture::new(); let first_output = fixture.root.join("candidate-one"); let second_output = fixture.root.join("candidate-two"); @@ -858,12 +795,9 @@ fn identical_inputs_produce_identical_bundle_bytes_revision_and_stable_report_sh assert_success(&first, "first deterministic build"); assert_success(&second, "second deterministic build"); - assert_eq!( - snapshot(&first_output.join("bundle")), - snapshot(&second_output.join("bundle")) - ); - assert_report(&first, &first_output); - assert_report(&second, &second_output); + assert_eq!(snapshot(&first_output), snapshot(&second_output)); + assert_report(&first, &first_output, &fixture.target); + assert_report(&second, &second_output, &fixture.target); assert_eq!(reported_revision(&first), reported_revision(&second)); } @@ -1128,8 +1062,7 @@ impl Fixture { fn command(&self, project: &Path, target: &Path, output: &Path) -> Command { let mut command = Command::new(env!("CARGO_BIN_EXE_evidencectl")); command - .arg("build") - .arg("--project") + .arg("package") .arg(project) .arg("--target") .arg(target) @@ -1373,14 +1306,14 @@ fn stderr(output: &Output) -> String { String::from_utf8_lossy(&output.stderr).into_owned() } -fn assert_report(output: &Output, candidate: &Path) { +fn assert_report(output: &Output, candidate: &Path, target: &Path) { let stdout = String::from_utf8_lossy(&output.stdout); assert_eq!( stdout, format!( - "Bundle revision: {REVISION}\nCandidate: {}\nProvision secret:file/audit-hmac-key\nProvision secret:file/source-token\nProvision secret:file/subject-binding-hmac-key\nTarget runtime paths and deployment secret material remain unverified until `evidencectl doctor --runtime-config {}/runtime.yaml`.\n", - candidate.display(), + "Package digest: {REVISION}\nPackage: {}\nProvision secret:file/audit-hmac-key\nProvision secret:file/source-token\nProvision secret:file/subject-binding-hmac-key\nTarget runtime paths and deployment secret material remain unverified until `evidencectl doctor --runtime-config {}/runtime.yaml`.\n", candidate.display(), + target.display(), ) ); } @@ -1388,8 +1321,8 @@ fn assert_report(output: &Output, candidate: &Path) { fn reported_revision(output: &Output) -> String { String::from_utf8_lossy(&output.stdout) .lines() - .find_map(|line| line.strip_prefix("Bundle revision: ")) - .expect("revision report") + .find_map(|line| line.strip_prefix("Package digest: ")) + .expect("package digest report") .to_owned() } @@ -1482,17 +1415,17 @@ assuranceProfile: production service: {providerId: urn:example:providers:evidence, trustDomain: urn:example:trust-domains:evidence} issuer: {id: urn:example:issuers:evidence} authentication: - kind: oidc-access-token - issuer: https://issuer.invalid - audiences: [evidence] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: https://issuer.invalid/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: https://issuer.invalid + audience: evidence + jwksSource: {kind: uri, uri: https://issuer.invalid/.well-known/jwks.json} + tokenTypes: [at+jwt] + algorithms: [ES256] + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: {hashKeyRef: 'secret:file/audit-hmac-key', hashKeyVersion: 1} subjectBinding: {secretRef: 'secret:file/subject-binding-hmac-key', keyVersion: 1} rateLimits: {requestsPerPrincipalPerMinute: 60, burstPerPrincipal: 10, failedSelectorAttemptsPerPrincipalAuthorityPerMinute: 10} @@ -1526,11 +1459,12 @@ const PUBLICATION: &str = r#"publication: jurisdictions: [urn:example:jurisdictions:governed] "#; -const TARGET_RUNTIME: &str = r#"version: 1 -bundleDirectory: /srv/evidence/candidate/bundle +const TARGET_RUNTIME: &str = r#"apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /srv/evidence/candidate/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 @@ -1604,7 +1538,7 @@ fi if [ -z "$fixture" ]; then if [ "$json" = '1' ]; then - printf '%s\n' '{"bundleRevision":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","requirements":[{"id":"urn:example:requirements:allowed:v1","configurationRevision":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}]}' + printf '%s\n' '{"packageDigest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","requirements":[{"id":"urn:example:requirements:allowed:v1","configurationRevision":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}]}' else printf '%s\n' 'Evidence bundle sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa passed check (2 requirements)' fi diff --git a/crates/registry-evidencectl/tests/production_handoff.rs b/crates/registry-evidencectl/tests/production_handoff.rs index d0e839ffb8..24041fad70 100644 --- a/crates/registry-evidencectl/tests/production_handoff.rs +++ b/crates/registry-evidencectl/tests/production_handoff.rs @@ -74,27 +74,23 @@ fn production_candidate_handoff_reaches_verified_assertion_and_audit() { "offline production check", ); let first = fixture.package(evidence); - let first_revision = bundle_revision(&first); + let first_revision = package_digest(&first); let first_bytes = snapshot_files(&fixture.candidate); fs::rename(&fixture.candidate, &fixture.first_candidate) .expect("archive the first create-only candidate"); let second = fixture.package(evidence); - let revision = bundle_revision(&second); + let revision = package_digest(&second); assert_eq!( revision, first_revision, - "bundle revision must be repeatable" + "package digest must be repeatable" ); assert_eq!( snapshot_files(&fixture.candidate), first_bytes, "identical inputs and output binding must reproduce every candidate file byte" ); - assert_eq!( - fs::read(&fixture.target_runtime).expect("target runtime"), - fs::read(fixture.candidate.join("runtime.yaml")).expect("copied runtime"), - "the target runtime must be copied byte-for-byte" - ); + assert!(!fixture.candidate.join("runtime.yaml").exists()); fixture.provision_target_secrets(); let audit_secret = fixture.secrets.join("audit-hmac-key"); @@ -103,7 +99,7 @@ fn production_candidate_handoff_reaches_verified_assertion_and_audit() { let unavailable = evidencectl() .arg("doctor") .arg("--runtime-config") - .arg(fixture.candidate.join("runtime.yaml")) + .arg(fixture.target_runtime.clone()) .env("EVIDENCE_BIN", evidence) .output() .expect("runtime doctor with unavailable secret starts"); @@ -119,8 +115,10 @@ fn production_candidate_handoff_reaches_verified_assertion_and_audit() { .expect("restore audit secret mode"); assert_success( evidencectl() - .args(["doctor", "--project"]) - .arg(&fixture.candidate) + .arg("doctor") + .arg("--runtime-config") + .arg(&fixture.target_runtime) + .env("EVIDENCE_BIN", evidence) .output() .expect("doctor starts"), "target-host doctor", @@ -128,13 +126,15 @@ fn production_candidate_handoff_reaches_verified_assertion_and_audit() { assert_success( evidencectl() .arg("test") - .arg(&fixture.candidate) + .arg(&fixture.project) + .arg("--target") + .arg(&fixture.target) .env("EVIDENCE_BIN", evidence) .output() .expect("fixture driver starts"), "target-host fixtures", ); - fixture.assert_compose_revision_distinction(evidence, &revision); + fixture.assert_compose_package_identity(evidence, &revision); let mut https = fixture.start_https(); fixture.wait_for_https(&mut https); @@ -142,7 +142,7 @@ fn production_candidate_handoff_reaches_verified_assertion_and_audit() { evidencectl() .arg("doctor") .arg("--runtime-config") - .arg(fixture.candidate.join("runtime.yaml")) + .arg(fixture.target_runtime.clone()) .env("EVIDENCE_BIN", evidence) .env("SSL_CERT_FILE", &fixture.ca) .output() @@ -152,6 +152,39 @@ fn production_candidate_handoff_reaches_verified_assertion_and_audit() { let mut service = fixture.start_evidence(evidence); fixture.wait_for_evidence(&mut service); + // A candidate staged beside the serving instance shares its audit path. + // The plain form meets that instance's writer lock and says how to check + // beside it; the lock-free form proves the candidate without taking it. + let beside = evidencectl() + .arg("doctor") + .arg("--runtime-config") + .arg(fixture.target_runtime.clone()) + .env("EVIDENCE_BIN", evidence) + .env("SSL_CERT_FILE", &fixture.ca) + .output() + .expect("runtime doctor beside the serving instance starts"); + assert_eq!( + beside.status.code(), + Some(3), + "the serving instance's writer lock must refuse the locking form" + ); + assert!( + String::from_utf8_lossy(&beside.stderr).contains("--without-audit-lock"), + "a held writer lock must name the lock-free form" + ); + assert_success( + evidencectl() + .arg("doctor") + .arg("--runtime-config") + .arg(fixture.target_runtime.clone()) + .arg("--without-audit-lock") + .env("EVIDENCE_BIN", evidence) + .env("SSL_CERT_FILE", &fixture.ca) + .output() + .expect("lock-free runtime doctor starts"), + "lock-free runtime doctor beside the serving instance", + ); + let token = fixture.access_token(); let published_revision = published_configuration_revision(fixture.evidence_port, &token); assert_ne!( @@ -239,8 +272,8 @@ fn production_candidate_handoff_reaches_verified_assertion_and_audit() { } #[test] -#[ignore = "exact gate: runs the real production builder and sibling Evidence bundle check"] -fn production_build_accepts_the_real_bundle_check_revision() { +#[ignore = "exact gate: runs the real production packager and sibling Evidence package check"] +fn production_package_accepts_the_real_package_check_digest() { let fixture = Fixture::new(); let evidence = evidence_binary(); fixture.stage_authoring_project(); @@ -249,9 +282,9 @@ fn production_build_accepts_the_real_bundle_check_revision() { fixture.authorize_four_shapes(); let output = fixture.build(evidence); - let revision = bundle_revision(&output); + let revision = package_digest(&output); assert!(revision.starts_with("sha256:")); - assert!(fixture.candidate.join("bundle/evidence.yaml").is_file()); + assert!(fixture.candidate.join("evidence.yaml").is_file()); } #[test] @@ -488,17 +521,17 @@ fn production_build_checks_and_evaluates_every_neutral_authoring_shape() { fixture.authorize_four_shapes(); let output = fixture.build(evidence); - let revision = bundle_revision(&output); + let revision = package_digest(&output); fixture.provision_target_secrets(); - let (checked_revision, _) = check_revisions( + let checked_revision = check_package( evidence, - &fixture.candidate.join("runtime.yaml"), + &fixture.target_runtime, "published four-shape production check", ); assert_eq!(checked_revision, revision); let bundle: Value = serde_norway::from_slice( - &fs::read(fixture.candidate.join("bundle/evidence.yaml")).expect("four-shape bundle"), + &fs::read(fixture.candidate.join("evidence.yaml")).expect("four-shape bundle"), ) .expect("four-shape bundle parses"); assert_eq!(bundle["assuranceProfile"], "production"); @@ -541,7 +574,7 @@ fn production_build_checks_and_evaluates_every_neutral_authoring_shape() { assert!( fixture .candidate - .join("bundle/fixtures") + .join("fixtures") .join(fixture_path) .is_file(), "the production candidate must capture fixture {fixture_path}" @@ -551,11 +584,7 @@ fn production_build_checks_and_evaluates_every_neutral_authoring_shape() { .as_str() .expect("compiled controlled-category codelist path"); assert!( - fixture - .candidate - .join("bundle") - .join(age_codelist) - .is_file(), + fixture.candidate.join(age_codelist).is_file(), "the governed controlled-category codelist must be captured" ); } @@ -651,7 +680,7 @@ fn public_lifecycle_keeps_local_dev_state_out_of_the_production_candidate() { let local_source_token = fs::read(fixture.project.join("secrets/source-token")).expect("local source token"); let build = fixture.build(evidence); - bundle_revision(&build); + package_digest(&build); assert_eq!( snapshot_files(&dev_root), stopped_dev, @@ -680,7 +709,7 @@ fn public_lifecycle_keeps_local_dev_state_out_of_the_production_candidate() { ); } let production_bundle = - fs::read(fixture.candidate.join("bundle/evidence.yaml")).expect("production bundle"); + fs::read(fixture.candidate.join("evidence.yaml")).expect("production bundle"); assert!( production_bundle .windows(REQUIREMENT.len()) @@ -1602,17 +1631,17 @@ assuranceProfile: production service: {{providerId: urn:example:providers:evidence, trustDomain: urn:example:trust-domains:acceptance, publicOrigin: https://evidence.example.test}} issuer: {{id: urn:example:issuers:evidence}} authentication: - kind: oidc-access-token - issuer: {identity} - audiences: [{TOKEN_AUDIENCE}] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: {identity}/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: {identity} + audience: {TOKEN_AUDIENCE} + jwksSource: {{kind: uri, uri: {identity}/.well-known/jwks.json}} + tokenTypes: [at+jwt] + algorithms: [ES256] + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: {{hashKeyRef: 'secret:file/audit-hmac-key', hashKeyVersion: 1}} subjectBinding: {{secretRef: 'secret:file/subject-binding-hmac-key', keyVersion: 1}} rateLimits: {{requestsPerPrincipalPerMinute: 60, burstPerPrincipal: 10, failedSelectorAttemptsPerPrincipalAuthorityPerMinute: 10}} @@ -1643,8 +1672,8 @@ authorityProfiles: fs::write( &self.target_runtime, format!( - "version: 1\nbundleDirectory: {bundle}\nlistener:\n bindHost: 127.0.0.1\n port: {port}\n tlsTermination: operator-controlled-upstream\n trustProxyIdentityHeaders: false\n maximumRequestBytes: 65536\n maximumConcurrentRequests: 64\n requestTimeoutMilliseconds: 10000\n shutdownGraceMilliseconds: 5000\nsecretProviders:\n file:\n root: {secrets}\nsigner:\n kind: transit\n unixSocketPath: {transit_socket}\n mount: transit\n keyName: evidence-signing\n keyVersion: 1\n timeoutMilliseconds: 2000\naudit:\n path: {audit}\noutboundTls:\n systemRoots: true\n trustProfiles: {{}}\n", - bundle = self.candidate.join("bundle").display(), + "apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1\nkind: EvidenceRuntimeConfig\npackage:\n root: {bundle}\nlistener:\n bind: 127.0.0.1:{port}\n tlsTermination: operator-controlled-upstream\n trustProxyIdentityHeaders: false\n maximumRequestBytes: 65536\n maximumConcurrentRequests: 64\n requestTimeoutMilliseconds: 10000\n shutdownGraceMilliseconds: 5000\nsecretProviders:\n file:\n root: {secrets}\nsigner:\n kind: transit\n unixSocketPath: {transit_socket}\n mount: transit\n keyName: evidence-signing\n keyVersion: 1\n timeoutMilliseconds: 2000\naudit:\n path: {audit}\noutboundTls:\n systemRoots: true\n trustProfiles: {{}}\n", + bundle = self.candidate.clone().display(), port = self.evidence_port, secrets = self.secrets.display(), transit_socket = self.root.join("transit-proxy.sock").display(), @@ -1748,10 +1777,10 @@ authorityProfiles: fn active_evidence_public_jwk(&self) -> PathBuf { let bundle: Value = serde_norway::from_slice( - &fs::read(self.candidate.join("bundle/evidence.yaml")).expect("candidate bundle"), + &fs::read(self.candidate.join("evidence.yaml")).expect("candidate bundle"), ) .expect("candidate bundle parses"); - self.candidate.join("bundle").join( + self.candidate.clone().join( bundle["signing"]["activePublicJwkFile"] .as_str() .expect("active public JWK file"), @@ -1769,13 +1798,11 @@ authorityProfiles: .to_owned() } - fn assert_compose_revision_distinction(&self, evidence: &Path, revision: &str) { - let (host_bundle, host_runtime) = check_revisions( - evidence, - &self.candidate.join("runtime.yaml"), - "host runtime check", + fn assert_compose_package_identity(&self, evidence: &Path, digest: &str) { + assert_eq!( + check_package(evidence, &self.target_runtime, "host runtime check"), + digest ); - assert_eq!(host_bundle, revision); let compose = self.root.join("compose-adapter"); fs::create_dir(&compose).expect("Compose adapter directory"); @@ -1783,10 +1810,10 @@ authorityProfiles: fs::write( &runtime, format!( - "version: 1\nbundleDirectory: {bundle}\nlistener:\n bindHost: 127.0.0.1\n port: {port}\n tlsTermination: operator-controlled-upstream\n trustProxyIdentityHeaders: false\n maximumRequestBytes: 131072\n maximumConcurrentRequests: 32\n requestTimeoutMilliseconds: 15000\n shutdownGraceMilliseconds: 10000\nsecretProviders:\n file:\n root: {secrets}\nsigner:\n kind: transit\n unixSocketPath: {transit_socket}\n mount: transit\n keyName: evidence-signing\n keyVersion: 1\n timeoutMilliseconds: 2000\naudit:\n path: {audit}\noutboundTls:\n systemRoots: true\n trustProfiles: {{}}\n", + "apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1\nkind: EvidenceRuntimeConfig\npackage:\n root: {bundle}\nlistener:\n bind: 127.0.0.1:{port}\n tlsTermination: operator-controlled-upstream\n trustProxyIdentityHeaders: false\n maximumRequestBytes: 131072\n maximumConcurrentRequests: 32\n requestTimeoutMilliseconds: 15000\n shutdownGraceMilliseconds: 10000\nsecretProviders:\n file:\n root: {secrets}\nsigner:\n kind: transit\n unixSocketPath: {transit_socket}\n mount: transit\n keyName: evidence-signing\n keyVersion: 1\n timeoutMilliseconds: 2000\naudit:\n path: {audit}\noutboundTls:\n systemRoots: true\n trustProfiles: {{}}\n", // This absolute host path stands for the unchanged read-only - // candidate/bundle mount in the container execution context. - bundle = self.candidate.join("bundle").display(), + // candidate package mount in the container execution context. + bundle = self.candidate.clone().display(), port = free_port(), secrets = self.secrets.display(), transit_socket = self.root.join("transit-proxy.sock").display(), @@ -1797,19 +1824,14 @@ authorityProfiles: fs::set_permissions(&runtime, fs::Permissions::from_mode(0o400)) .expect("seal Compose runtime"); - let unchanged_bundle = snapshot_files(&self.candidate.join("bundle")); - let (compose_bundle, compose_runtime) = - check_revisions(evidence, &runtime, "Compose-context runtime check"); + let unchanged_package = snapshot_files(&self.candidate); + let compose_digest = check_package(evidence, &runtime, "Compose-context runtime check"); assert_eq!( - snapshot_files(&self.candidate.join("bundle")), - unchanged_bundle, - "the Compose adapter must not edit the governed bundle" - ); - assert_eq!(compose_bundle, revision); - assert_ne!( - compose_runtime, host_runtime, - "environment-specific runtime bindings require an independent runtime revision" + snapshot_files(&self.candidate), + unchanged_package, + "the Compose adapter must not edit the governed package" ); + assert_eq!(compose_digest, digest); } fn start_https(&self) -> Child { @@ -1842,9 +1864,9 @@ authorityProfiles: fn start_evidence(&self, evidence: &Path) -> Child { let log = owner_only_log(&self.root.join("evidence.log")); Command::new(evidence) - .arg("--runtime") - .arg(self.candidate.join("runtime.yaml")) .arg("serve") + .arg("--runtime-config") + .arg(self.target_runtime.clone()) .env("SSL_CERT_FILE", &self.ca) .stdin(Stdio::null()) .stdout(Stdio::from(log.try_clone().expect("clone Evidence log"))) @@ -2152,16 +2174,16 @@ fn assert_success(output: Output, label: &str) -> Output { output } -fn bundle_revision(output: &Output) -> String { +fn package_digest(output: &Output) -> String { String::from_utf8_lossy(&output.stdout) .lines() - .find_map(|line| line.strip_prefix("Bundle revision: ")) + .find_map(|line| line.strip_prefix("Package digest: ")) .filter(|revision| { revision.len() == 71 && revision.starts_with("sha256:") && revision[7..].bytes().all(|byte| byte.is_ascii_hexdigit()) }) - .expect("build reports one bundle revision") + .expect("build reports one package digest") .to_owned() } @@ -2190,12 +2212,12 @@ fn assert_requirement_forms( ); } -fn check_revisions(evidence: &Path, runtime: &Path, label: &str) -> (String, String) { +fn check_package(evidence: &Path, runtime: &Path, label: &str) -> String { let output = assert_success( Command::new(evidence) - .arg("--runtime") - .arg(runtime) .arg("check") + .arg("--runtime-config") + .arg(runtime) .output() .expect("Evidence check starts"), label, @@ -2203,15 +2225,11 @@ fn check_revisions(evidence: &Path, runtime: &Path, label: &str) -> (String, Str let stdout = String::from_utf8(output.stdout).expect("Evidence check stdout"); let fields = stdout .lines() - .find(|line| line.starts_with("Evidence deployment ")) + .find(|line| line.starts_with("Evidence package ")) .expect("Evidence check report") .split_whitespace() .collect::>(); - assert_eq!(fields.get(3), Some(&"/"), "Evidence revision separator"); - ( - fields.get(2).expect("bundle revision").to_string(), - fields.get(4).expect("runtime revision").to_string(), - ) + fields.get(2).expect("package digest").to_string() } fn snapshot_files(root: &Path) -> BTreeMap> { @@ -2342,7 +2360,7 @@ fn post_evidence(port: u16, token: &str, nonce: &str) -> (u16, Vec) { /// The configuration revision discovery publishes for the fixture requirement. /// -/// It is requirement scoped, so it is not the deployment's bundle revision. +/// It is requirement scoped, so it is not the deployment's package digest. /// Reading it from discovery keeps the assertion check independent of the /// signed payload it is compared against. fn published_configuration_revision(port: u16, token: &str) -> String { diff --git a/crates/registry-evidencectl/tests/request_verify.rs b/crates/registry-evidencectl/tests/request_verify.rs index 6de762317d..2227353c56 100644 --- a/crates/registry-evidencectl/tests/request_verify.rs +++ b/crates/registry-evidencectl/tests/request_verify.rs @@ -233,8 +233,8 @@ fn prepare_and_verify_delegate_exactly_and_publish_only_safe_artifacts() { let evidence_args = fs::read_to_string(fixture.evidence.with_extension("prepare.args")) .expect("Evidence prepare argv"); let evidence_args = evidence_args.lines().collect::>(); - assert_eq!(evidence_args[0], "--runtime"); - assert_eq!(evidence_args[2], "prepare-local-relying-procedure"); + assert_eq!(evidence_args[0], "prepare-local-relying-procedure"); + assert_eq!(evidence_args[1], "--runtime-config"); assert_eq!(evidence_args[3], "--input"); assert!(evidence_args[4].ends_with("/procedure-input.json")); assert!(!evidence_args.join(" ").contains(TOKEN)); @@ -920,7 +920,7 @@ impl Fixture { let evidence = temporary.path().join("evidence-stub"); executable( &evidence, - b"#!/bin/sh\ncase \"$1\" in\n --runtime)\n printf '%s\\n' \"$@\" > \"$0.prepare.args\"\n [ \"$3\" = 'prepare-local-relying-procedure' ] || exit 40\n [ \"$4\" = '--input' ] || exit 41\n cp \"$5\" \"$0.input\" || exit 42\n if IFS= read -r unexpected; then exit 43; fi\n printf 'stdin-empty\\n' > \"$0.prepare.stdin\"\n [ ! -f \"$0.fail-prepare\" ] || exit 44\n cat \"$0.procedure\"\n ;;\n *) exit 45 ;;\nesac\n", + b"#!/bin/sh\ncase \"$1\" in\n prepare-local-relying-procedure)\n printf '%s\\n' \"$@\" > \"$0.prepare.args\"\n [ \"$2\" = '--runtime-config' ] || exit 40\n [ \"$4\" = '--input' ] || exit 41\n cp \"$5\" \"$0.input\" || exit 42\n if IFS= read -r unexpected; then exit 43; fi\n printf 'stdin-empty\\n' > \"$0.prepare.stdin\"\n [ ! -f \"$0.fail-prepare\" ] || exit 44\n cat \"$0.procedure\"\n ;;\n *) exit 45 ;;\nesac\n", ); Self { _temporary: temporary, @@ -1222,6 +1222,9 @@ fn write_sealed_bundle(root: &Path, state: &Value) { let bundle_directory = root.join(".evidence/dev/bundle"); if !bundle_directory.exists() { private_directory(&bundle_directory); + } else { + fs::set_permissions(&bundle_directory, fs::Permissions::from_mode(0o700)) + .expect("unseal bundle fixture directory"); } let questions = state["questions"].as_array().expect("state questions"); let mut selector_profiles = serde_json::Map::new(); @@ -1346,6 +1349,27 @@ fn write_sealed_bundle(root: &Path, state: &Value) { .as_bytes(), 0o400, ); + let sum_file = bundle_directory.join(registry_platform_config::SUM_FILE); + if sum_file.exists() { + fs::remove_file(&sum_file).expect("remove stale package sum file"); + } + registry_platform_config::write_sum_file( + &bundle_directory, + None, + ®istry_platform_config::PackageLimits { + max_files: 1_024, + max_file_bytes: 1024 * 1024, + max_total_bytes: 16 * 1024 * 1024, + max_depth: 3, + max_path_bytes: 128, + }, + "evidencectl package", + ) + .expect("publish local test package"); + fs::set_permissions(sum_file, fs::Permissions::from_mode(0o400)) + .expect("seal package sum file"); + fs::set_permissions(bundle_directory, fs::Permissions::from_mode(0o500)) + .expect("seal package directory"); } fn executable(path: &Path, contents: &[u8]) { diff --git a/crates/registry-evidencectl/tests/scaffold.rs b/crates/registry-evidencectl/tests/scaffold.rs index 882d94be92..efab8bdf13 100644 --- a/crates/registry-evidencectl/tests/scaffold.rs +++ b/crates/registry-evidencectl/tests/scaffold.rs @@ -129,7 +129,7 @@ for arg in "$@"; do done if [ "$step" = "bundle-check" ]; then - printf '{{"bundleRevision":"sha256:0000000000000000000000000000000000000000000000000000000000000000","requirements":[{{"id":"urn:example:stub:record-active","configurationRevision":"sha256:0000000000000000000000000000000000000000000000000000000000000000"}}]}}\n' + printf '{{"packageDigest":"sha256:0000000000000000000000000000000000000000000000000000000000000000","requirements":[{{"id":"urn:example:stub:record-active","configurationRevision":"sha256:0000000000000000000000000000000000000000000000000000000000000000"}}]}}\n' exit 0 fi diff --git a/crates/registry-language-server/tests/relay_v2_protocol.rs b/crates/registry-language-server/tests/relay_v2_protocol.rs index 5d23892696..c47aa8c7d1 100644 --- a/crates/registry-language-server/tests/relay_v2_protocol.rs +++ b/crates/registry-language-server/tests/relay_v2_protocol.rs @@ -159,8 +159,8 @@ async fn a_new_unsaved_runtime_document_contributes_compiler_diagnostics() { let invalid = fs::read_to_string(project.join("runtime.yaml")) .expect("the runtime fixture reads") .replacen( - "apiVersion: relay.registrystack.org/v2alpha1", - "apiVersion: relay.registrystack.org/unsupported", + "apiVersion: registry.registrystack.org/relay-runtime/v1alpha1", + "apiVersion: registry.registrystack.org/relay-runtime/unsupported", 1, ); fs::remove_file(project.join("runtime.yaml")).expect("the runtime starts absent"); diff --git a/crates/registry-platform-config/Cargo.toml b/crates/registry-platform-config/Cargo.toml index a2c257edbc..36652a3f3e 100644 --- a/crates/registry-platform-config/Cargo.toml +++ b/crates/registry-platform-config/Cargo.toml @@ -3,23 +3,32 @@ name = "registry-platform-config" version.workspace = true edition.workspace = true license.workspace = true -description = "Signed/governed runtime configuration verification contracts for Registry services." +description = "Shared runtime configuration loader, configuration blocks, and secret references for Registry Stack runtimes." repository.workspace = true publish = false [lints] workspace = true +[features] +schema = ["dep:schemars"] + [dependencies] -base64.workspace = true -registry-platform-crypto = { workspace = true } +registry-platform-canonical-json = { workspace = true } rustix.workspace = true +schemars = { workspace = true, optional = true } serde.workspace = true serde_json.workspace = true +serde_norway.workspace = true +serde_path_to_error.workspace = true sha2.workspace = true thiserror.workspace = true -time.workspace = true +url.workspace = true zeroize.workspace = true [dev-dependencies] tempfile.workspace = true + +[[example]] +name = "shared-blocks-schema" +required-features = ["schema"] diff --git a/crates/registry-platform-config/README.md b/crates/registry-platform-config/README.md index 32413bf0aa..d57e00361e 100644 --- a/crates/registry-platform-config/README.md +++ b/crates/registry-platform-config/README.md @@ -1,16 +1,122 @@ # registry-platform-config -Shared configuration parsing helpers for maintained Registry Stack products. +Shared runtime configuration for maintained Registry Stack runtimes: one +loader for the operator `runtime.yaml`, the configuration blocks every runtime +spells the same way, and the `secret:` reference resolver. Each product still +owns and validates the rest of its configuration contract. -The crate expands bounded environment references, reports deprecated field -names without exposing values, and produces canonical SHA-256 identifiers. -Each product still owns and validates its complete configuration contract. +## Loader -## Environment expansion +`RuntimeConfigLoader` reads one runtime configuration file named by an +absolute path. It refuses: -Shared configuration loaders expand `${VAR}` expressions before YAML parsing. -`${VAR}` requires `VAR` to be set to a non-empty value. `${VAR:-fallback}` -uses `fallback` when `VAR` is unset or empty, including `${VAR:-}` for an -explicit empty result. `${VAR:?message}` fails with `message` when `VAR` is -unset or empty. Whitespace-only values are non-empty. Diagnostics name the -variable or use the supplied message; they never include the variable value. +- a relative path, or one with `.` or `..` components; +- a symbolic link in any path component; +- anything but a regular file, an empty file, or one over the size bound + (1 MiB unless the product sets another); +- text that is not UTF-8, more than one YAML document, a root that is not a + mapping, a non-string key, a duplicate key, or a YAML tag; +- a key the product removed, naming the key that replaced it; +- an `apiVersion` or `kind` other than the product's literal envelope. + +A product may also require trusted ownership: every ancestor directory and +the file owned by root or the runtime user and not writable by group or +others, except a root-owned sticky directory. + +Every refusal names the file, the field, and the fix, and never repeats a +configured value. + +## Environment substitution + +After parsing, the loader substitutes environment expressions inside string +values of `runtime.yaml`: + +- `${VAR}` requires `VAR` to be set and non-empty; +- `${VAR:-default}` uses `default` when `VAR` is unset or empty; +- `${VAR:?message}` refuses when `VAR` is unset or empty. The refusal names + `VAR` and withholds `message`, because the message is configured text. + +Keys and comments are never substituted, a substituted value stays a string, +and substitution runs once, so a value that itself looks like an expression is +kept as written. There is no escape syntax: a literal `${` reaches the +configuration as the value of a variable. An expression inside a field whose +name ends in `Ref` or `Refs`, or anywhere beneath one, is refused: a secret +reference names a provider, and the provider reads the value. An expression +anywhere under `secretProviders` is refused too, because a provider setting +chooses which secret a reference resolves to. Authored package files are not +substituted; `reject_environment_expressions_in_authored_yaml` refuses an +authored document that carries an expression, and refuses text it cannot read +as YAML rather than letting it pass unchecked. Its refusals carry the +`authored_config.environment_expression` and `authored_config.syntax` codes. + +The effective digest of a loaded file is the `sha256:` label of the canonical +JSON of the substituted document, so a comment or formatting change does not +change it. + +## Shared blocks + +`blocks` holds the sections each runtime embeds unchanged: + +- `SecretProvidersConfig`: `file.root` enables `secret:file/name`, and + `environment: {}` enables `secret:env/NAME`. At least one must be declared, + and a reference to an undeclared provider is refused. +- `DatabaseConfig`: `runtimeUrlRef`, `migrationUrlRef`, and an optional + `trustedRootCertificateRef`. +- `JwksSource`: `kind: discovery` (the default) with no other member, + `kind: uri` with `uri`, or `kind: static` with `documentRef`. +- `PackageConfig`: an absolute `root` and an optional `expectedDigest` pin. +- `ListenerConfig` and `PrivateListenerConfig`: `bind` as `host:port` with an + IP address host, and for private listeners the declared TLS termination and + network exposure. + +## Package + +`package` is the one package format every runtime serves from +`package.root`. A product's `package` command either writes its files from +memory with `write_package`, or populates a new directory itself and calls +`write_sum_file`; `plan_package` reports the digest a package would have +without writing it. The package carries `SHA256SUMS` at the root: one line per +file in the `sha256sum` text format, sorted by path, LF line endings, listing +every file but itself. An optional `REVISION` file holds one free-text line +(at most 256 bytes) and is listed and hashed like every other file. The +package digest is the `sha256:` label of the `SHA256SUMS` bytes, and is what +`package.expectedDigest` pins. The same files and revision always give the +same digest, and `sha256sum -c SHA256SUMS` checks a package by hand. + +At startup `PackageConfig::verify_package` recomputes every digest and +refuses, in one message naming each file and the command that rebuilds the +package: + +- a changed, missing, or extra file, including a hidden file and an empty + directory; +- a missing or malformed `SHA256SUMS`, by line; +- a symbolic link or special file anywhere in the package, or a `package.root` + that is a link; +- a path with a backslash or control character, or two paths that differ only + in letter case; +- a package over its file count, per-file, total, depth, or path-length bound; +- a package whose digest is not `package.expectedDigest`, showing both + digests in the `PackageDigestMismatch` shape every runtime shares. + +Only file bytes are hashed. Modes, owners, and timestamps are not, because +copies, source control, and image layers do not preserve them the same way on +every platform, and hashing them would give one package several digests. +Line endings are not normalized. `is_envelope_file` names `SHA256SUMS` and +`REVISION` so a product loader that enumerates its package can skip them. + +With the `schema` feature, the `shared-blocks-schema` example writes the +canonical JSON Schema of these blocks to +`products/platform/generated/runtime-config-blocks.schema.json`: + +```bash +cargo run -p registry-platform-config --features schema \ + --example shared-blocks-schema -- --output products/platform/generated +``` + +`products/platform/scripts/check-config-conformance.py` holds each runtime that +reads `runtime.yaml` through `RuntimeConfigLoader` to this surface: its +generated runtime schema embeds the shared blocks it uses unchanged, it does +not call a text-level `expand_config_env_vars` expansion, and named tests prove a `*Ref` field and +an authored project file both refuse `${VAR}`. `--check-generated` also +regenerates the canonical schema and fails when the committed copy differs. A +product that adopts the loader adds a row to the gate. diff --git a/crates/registry-platform-config/examples/shared-blocks-schema.rs b/crates/registry-platform-config/examples/shared-blocks-schema.rs new file mode 100644 index 0000000000..e8606625c3 --- /dev/null +++ b/crates/registry-platform-config/examples/shared-blocks-schema.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Write the canonical JSON Schema of the shared runtime configuration blocks. +//! +//! The configuration conformance gate compares every runtime's generated +//! schema against this document, so a product that re-declares a shared block +//! instead of embedding it fails the gate. +//! +//! ```bash +//! cargo run -p registry-platform-config --features schema \ +//! --example shared-blocks-schema -- --output products/platform/generated +//! ``` + +fn main() -> std::process::ExitCode { + let mut arguments = std::env::args_os().skip(1); + let usage = || { + eprintln!("usage: shared-blocks-schema --output "); + std::process::ExitCode::from(2) + }; + if arguments.next().as_deref() != Some(std::ffi::OsStr::new("--output")) { + return usage(); + } + let Some(output) = arguments.next().map(std::path::PathBuf::from) else { + return usage(); + }; + if arguments.next().is_some() { + return usage(); + } + let rendered = match registry_platform_config::schema::shared_blocks_document() { + Ok(rendered) => rendered, + Err(error) => { + eprintln!("shared blocks schema generation failed: {error}"); + return std::process::ExitCode::FAILURE; + } + }; + let written = std::fs::create_dir_all(&output).and_then(|()| { + std::fs::write( + output.join(registry_platform_config::schema::SHARED_BLOCKS_SCHEMA_FILE), + rendered, + ) + }); + if let Err(error) = written { + eprintln!("shared blocks schema generation failed: {error}"); + return std::process::ExitCode::FAILURE; + } + std::process::ExitCode::SUCCESS +} diff --git a/crates/registry-platform-config/src/blocks.rs b/crates/registry-platform-config/src/blocks.rs new file mode 100644 index 0000000000..3ea775f4ec --- /dev/null +++ b/crates/registry-platform-config/src/blocks.rs @@ -0,0 +1,786 @@ +//! The runtime configuration blocks every Registry Stack runtime shares. +//! +//! A product's runtime configuration embeds these types under the same keys, +//! so an operator reads `secretProviders`, `database`, `listener.bind`, +//! `package`, `audit.hashKeyRef` and `authentication.oidc` the same way in +//! every product, and one implementation checks them. Product-specific +//! siblings stay in the product's own configuration types: the audit key and +//! the OIDC issuer and clients are embedded with `#[serde(flatten)]` beside +//! them. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::net::{IpAddr, Ipv6Addr, SocketAddr}; +use std::path::{Component, Path, PathBuf}; +use std::str::FromStr; + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use crate::{SecretError, SecretProvider, SecretReference, SecretResolver, MAX_SECRET_BYTES}; + +/// The schema pattern of a field that must name an enabled secret provider. +pub const SECRET_PROVIDER_PATTERN: &str = "^secret:(?:env|file)/"; + +/// The schema pattern of an exact secret reference. +pub const SECRET_REFERENCE_PATTERN: &str = + "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$"; + +/// Why a shared block was refused. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ConfigBlockErrorKind { + /// A path that must be absolute is not. + RelativePath, + /// `secretProviders` enables no provider. + NoSecretProvider, + /// A `*Ref` field is not an exact secret reference. + InvalidSecretReference, + /// A `*Ref` field names a provider `secretProviders` does not enable. + SecretProviderDisabled, + /// A required value is empty. + Empty, + /// `package.expectedDigest` is not a `sha256:` label. + InvalidDigest, + /// A JWKS URI or an OIDC issuer is not an absolute `https` URL. + InvalidUri, + /// An OIDC audience is empty, too long, or holds a control character. + InvalidAudience, + /// `assertionIssuers` exceeds a bound, names an empty value, or repeats + /// an issuer for one client. + InvalidAssertionIssuers, +} + +/// A shared block refusal naming its field and never a configured value. +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] +#[error("{message}")] +pub struct ConfigBlockError { + kind: ConfigBlockErrorKind, + field: String, + message: String, +} + +impl ConfigBlockError { + fn new(kind: ConfigBlockErrorKind, field: &str, message: String) -> Self { + Self { + kind, + field: field.to_owned(), + message, + } + } + + #[must_use] + pub const fn kind(&self) -> ConfigBlockErrorKind { + self.kind + } + + /// The dotted field the refusal concerns. + #[must_use] + pub fn field(&self) -> &str { + &self.field + } +} + +fn require_absolute(field: &str, path: &Path) -> Result<(), ConfigBlockError> { + let normal = path.is_absolute() + && path.components().all(|component| { + matches!( + component, + Component::Prefix(_) | Component::RootDir | Component::Normal(_) + ) + }); + if normal { + Ok(()) + } else { + Err(ConfigBlockError::new( + ConfigBlockErrorKind::RelativePath, + field, + format!("{field} must be an absolute path without . or .. components"), + )) + } +} + +/// The secret providers a runtime enables. A reference is resolved only by a +/// provider declared here: `secret:file/name` under `file.root`, and +/// `secret:env/NAME` only when `environment: {}` is present. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[cfg_attr( + feature = "schema", + schemars(extend("anyOf" = [ + {"required": ["file"], "properties": {"file": {"$ref": "#/$defs/FileSecretProviderConfig"}}}, + {"required": ["environment"], "properties": {"environment": {"$ref": "#/$defs/EnvironmentSecretProviderConfig"}}} + ])) +)] +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SecretProvidersConfig { + /// Enables `secret:file/name` references, read from files under `root`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub file: Option, + /// Enables `secret:env/NAME` references, read from the process + /// environment. Declared as an empty mapping: `environment: {}`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub environment: Option, +} + +/// The file secret provider. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct FileSecretProviderConfig { + /// Absolute directory holding one file per secret. Each file must be a + /// regular file owned by the runtime user, mode 0400 or 0600, with exactly + /// one hard link. + #[cfg_attr(feature = "schema", schemars(extend("pattern" = "^/")))] + pub root: PathBuf, +} + +/// The environment secret provider. It takes no settings. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentSecretProviderConfig {} + +impl SecretProvidersConfig { + /// At least one provider is enabled, and the file root is absolute. + pub fn check(&self) -> Result<(), ConfigBlockError> { + if self.file.is_none() && self.environment.is_none() { + return Err(ConfigBlockError::new( + ConfigBlockErrorKind::NoSecretProvider, + "secretProviders", + "secretProviders must enable file, environment, or both".to_owned(), + )); + } + if let Some(file) = &self.file { + require_absolute("secretProviders.file.root", &file.root)?; + } + Ok(()) + } + + /// `raw`, configured at `field`, is an exact secret reference whose + /// provider this block enables. + pub fn check_reference(&self, field: &str, raw: &str) -> Result<(), ConfigBlockError> { + let reference = SecretReference::parse(raw).map_err(|_| { + ConfigBlockError::new( + ConfigBlockErrorKind::InvalidSecretReference, + field, + format!("{field} must be an exact secret:env/NAME or secret:file/name reference"), + ) + })?; + let (enabled, provider) = match reference.provider() { + SecretProvider::File => (self.file.is_some(), "secretProviders.file"), + SecretProvider::Environment => { + (self.environment.is_some(), "secretProviders.environment") + } + }; + if enabled { + Ok(()) + } else { + Err(ConfigBlockError::new( + ConfigBlockErrorKind::SecretProviderDisabled, + field, + format!("{field} uses a provider that is not enabled; declare {provider}"), + )) + } + } + + /// The providers this block enables. + #[must_use] + pub fn providers(&self) -> Vec { + let mut providers = Vec::new(); + if self.file.is_some() { + providers.push(SecretProvider::File); + } + if self.environment.is_some() { + providers.push(SecretProvider::Environment); + } + providers + } + + /// A resolver for exactly the providers this block enables. + pub fn resolver(&self) -> Result { + SecretResolver::new( + self.providers(), + self.file + .as_ref() + .map_or_else(|| Path::new(""), |file| file.root.as_path()), + ) + } +} + +/// Explain one refused secret reference without disclosing what it protects. +/// +/// A valid reference is safe and useful to name, but invalid operator-authored +/// text might itself be a literal credential, so only its field is named. The +/// resolved bytes and opened path never appear. +#[must_use] +pub fn describe_secret_failure(field: &str, reference: &str, error: &SecretError) -> String { + let reason = match error { + SecretError::InvalidReference => { + "it is not an exact secret:env/NAME or secret:file/name reference".to_owned() + } + SecretError::ProviderDisabled => "its provider is not enabled for this runtime".to_owned(), + SecretError::InvalidProviderConfiguration => { + "the secret provider configuration is invalid".to_owned() + } + SecretError::Unavailable => { + "no readable secret of that name exists under the configured provider".to_owned() + } + SecretError::UnsafeFile => concat!( + "the secret file must be a regular file owned by the runtime user, ", + "with mode 0400 or 0600, and exactly one hard link" + ) + .to_owned(), + SecretError::Read => "the secret could not be read".to_owned(), + SecretError::InvalidValue => format!( + "the secret value must be non-empty text of at most {MAX_SECRET_BYTES} bytes \ + without NUL bytes" + ), + }; + if error == &SecretError::InvalidReference { + format!("the secret reference configured at {field} could not be resolved: {reason}") + } else { + format!("the secret reference {reference} could not be resolved: {reason}") + } +} + +/// The PostgreSQL connection a stateful runtime uses. Both URLs are secret +/// references and may name the same secret. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct DatabaseConfig { + /// Secret reference to the least-privileged runtime connection URL. + #[cfg_attr(feature = "schema", schemars(extend("pattern" = SECRET_PROVIDER_PATTERN)))] + pub runtime_url_ref: String, + /// Secret reference to the migration connection URL. + #[cfg_attr(feature = "schema", schemars(extend("pattern" = SECRET_PROVIDER_PATTERN)))] + pub migration_url_ref: String, + /// Secret reference to a PEM root certificate the connection trusts. + #[serde(default)] + #[cfg_attr(feature = "schema", schemars(extend("pattern" = SECRET_PROVIDER_PATTERN)))] + pub trusted_root_certificate_ref: Option, + /// Allow a plaintext connection. Refused outside test builds. + #[serde(default)] + pub test_only_plaintext: bool, +} + +impl fmt::Debug for DatabaseConfig { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DatabaseConfig") + .field("runtime_url_ref", &"") + .field("migration_url_ref", &"") + .field( + "trusted_root_certificate_ref", + &self + .trusted_root_certificate_ref + .as_ref() + .map(|_| ""), + ) + .field("test_only_plaintext", &self.test_only_plaintext) + .finish() + } +} + +impl DatabaseConfig { + /// Every secret reference this block configures, with its field. + #[must_use] + pub fn references(&self) -> Vec<(&'static str, &str)> { + let mut references = vec![ + ("database.runtimeUrlRef", self.runtime_url_ref.as_str()), + ("database.migrationUrlRef", self.migration_url_ref.as_str()), + ]; + if let Some(reference) = &self.trusted_root_certificate_ref { + references.push(("database.trustedRootCertificateRef", reference.as_str())); + } + references + } +} + +/// Where a runtime obtains the OIDC issuer's signing keys. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(tag = "kind", rename_all = "camelCase", deny_unknown_fields)] +pub enum JwksSource { + /// Read `jwks_uri` from the issuer's OpenID Connect discovery document. + // A struct variant, so `deny_unknown_fields` refuses a `uri` or a + // `documentRef` written beside `kind: discovery`; serde ignores extra + // members on a unit variant of an internally tagged enum. + Discovery {}, + /// Fetch the key set from this absolute `https` URI, skipping discovery. + Uri { + #[cfg_attr(feature = "schema", schemars(extend("pattern" = "^https?://")))] + uri: String, + }, + /// Read the key set from a secret, for deployments without network access + /// to the issuer. + Static { + #[serde(rename = "documentRef")] + #[cfg_attr(feature = "schema", schemars(extend("pattern" = SECRET_REFERENCE_PATTERN)))] + document_ref: String, + }, +} + +impl Default for JwksSource { + fn default() -> Self { + Self::Discovery {} + } +} + +impl JwksSource { + /// A `uri` source names an absolute `https` URL without credentials. A + /// loopback `http` URL is accepted only when `allow_loopback_http` is set, + /// for supervised local development. + pub fn check(&self, field: &str, allow_loopback_http: bool) -> Result<(), ConfigBlockError> { + match self { + Self::Discovery {} => Ok(()), + Self::Uri { uri } => { + let field = format!("{field}.uri"); + if valid_jwks_uri(uri, allow_loopback_http) { + Ok(()) + } else { + Err(ConfigBlockError::new( + ConfigBlockErrorKind::InvalidUri, + &field, + format!("{field} must be an absolute https URL without credentials"), + )) + } + } + Self::Static { document_ref } => { + if document_ref.is_empty() { + let field = format!("{field}.documentRef"); + Err(ConfigBlockError::new( + ConfigBlockErrorKind::Empty, + &field, + format!("{field} must be a secret reference"), + )) + } else { + Ok(()) + } + } + } + } + + /// The URI a `uri` source fetches from. + #[must_use] + pub fn uri(&self) -> Option<&str> { + match self { + Self::Uri { uri } => Some(uri), + _ => None, + } + } + + /// The secret reference a `static` source reads. + #[must_use] + pub fn document_ref(&self) -> Option<&str> { + match self { + Self::Static { document_ref } => Some(document_ref), + _ => None, + } + } +} + +fn valid_jwks_uri(value: &str, allow_loopback_http: bool) -> bool { + let Ok(parsed) = url::Url::parse(value) else { + return false; + }; + if !parsed.username().is_empty() || parsed.password().is_some() || parsed.host().is_none() { + return false; + } + match parsed.scheme() { + "https" => true, + "http" => { + allow_loopback_http + && matches!( + parsed.host(), + Some(url::Host::Ipv4(address)) if address.is_loopback() + ) + } + _ => false, + } +} + +/// The key for the keyed references an audit record carries in place of raw +/// identifiers, written `audit.hashKeyRef` beside the product's own audit +/// settings. +// A product's `audit` block embeds this with `#[serde(flatten)]`, so every +// product spells the key the same way and one implementation checks it. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AuditKeyConfig { + /// Secret reference to the audit hash key. + pub hash_key_ref: SecretReference, +} + +impl AuditKeyConfig { + /// The key names a provider `secret_providers` enables. + pub fn check(&self, secret_providers: &SecretProvidersConfig) -> Result<(), ConfigBlockError> { + secret_providers.check_reference("audit.hashKeyRef", self.hash_key_ref.as_str()) + } +} + +/// The longest `authentication.oidc.audience` accepted, in characters. +pub const MAX_OIDC_AUDIENCE_CHARACTERS: usize = 512; + +/// The OIDC issuer a runtime accepts access tokens from: the exact `iss` +/// value, the `aud` value a token must carry, and where the issuer's signing +/// keys come from, written under `authentication.oidc` beside the product's +/// own token rules. +// A product's `authentication.oidc` block embeds this with +// `#[serde(flatten)]`. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OidcIssuerConfig { + /// Exact issuer accepted in access-token `iss` claims, an absolute + /// `https` URL. + #[cfg_attr(feature = "schema", schemars(extend("pattern" = "^https?://")))] + pub issuer: String, + /// The audience every accepted access token must carry in `aud`. + #[cfg_attr(feature = "schema", schemars(length(min = 1, max = 512)))] + pub audience: String, + /// Where the issuer's signing keys come from. Absent reads the issuer's + /// OpenID Connect discovery document. + #[serde(default)] + pub jwks_source: JwksSource, +} + +impl OidcIssuerConfig { + /// The issuer is an absolute `https` URL without credentials, query, or + /// fragment, the audience is non-empty bounded text without control characters, and + /// the key source passes [`JwksSource::check`]. A loopback `http` issuer + /// or key URI is accepted only when `allow_loopback_http` is set, for + /// supervised local development. `field` is the dotted path of the block, + /// such as `authentication.oidc`. + pub fn check(&self, field: &str, allow_loopback_http: bool) -> Result<(), ConfigBlockError> { + let issuer_ok = valid_jwks_uri(&self.issuer, allow_loopback_http) + && url::Url::parse(&self.issuer) + .is_ok_and(|url| url.query().is_none() && url.fragment().is_none()); + if !issuer_ok { + let field = format!("{field}.issuer"); + return Err(ConfigBlockError::new( + ConfigBlockErrorKind::InvalidUri, + &field, + format!( + "{field} must be an absolute https URL without credentials, query, or fragment" + ), + )); + } + let audience_ok = !self.audience.is_empty() + && self.audience.chars().count() <= MAX_OIDC_AUDIENCE_CHARACTERS + && !self.audience.chars().any(char::is_control); + if !audience_ok { + let field = format!("{field}.audience"); + return Err(ConfigBlockError::new( + ConfigBlockErrorKind::InvalidAudience, + &field, + format!( + "{field} must be non-empty text of at most {MAX_OIDC_AUDIENCE_CHARACTERS} \ + characters without control characters" + ), + )); + } + self.jwks_source + .check(&format!("{field}.jwksSource"), allow_loopback_http) + } +} + +/// The most clients `authentication.oidc.assertionIssuers` may list. +pub const MAX_ASSERTION_ISSUER_CLIENTS: usize = 64; +/// The longest client key in `assertionIssuers`, in bytes. +pub const MAX_ASSERTION_ISSUER_CLIENT_BYTES: usize = 128; +/// The most assertion issuers one client may list. +pub const MAX_ASSERTION_ISSUERS_PER_CLIENT: usize = 16; +/// The longest assertion issuer, in bytes. +pub const MAX_ASSERTION_ISSUER_BYTES: usize = 512; + +/// The OAuth clients a runtime admits access tokens for, and the assertion +/// authorities each client may exchange a subject token from, written under +/// `authentication.oidc` beside the issuer. +// A product's `authentication.oidc` block embeds this with +// `#[serde(flatten)]`, so the bounds on the authored map hold in every +// product that exchanges tokens. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OidcClientsConfig { + /// Client identifiers whose access tokens are admitted. A runtime decides + /// whether an empty list is acceptable in production. + #[serde(default)] + pub allowed_clients: Vec, + /// Assertion authorities each client may exchange a subject token from, + /// keyed by client identifier. An empty map applies no rule. Once a + /// client is listed, a token it exchanged is accepted only for one of + /// that client's declared authorities. + #[serde(default)] + #[cfg_attr( + feature = "schema", + schemars(extend( + "maxProperties" = MAX_ASSERTION_ISSUER_CLIENTS, + "propertyNames" = {"minLength": 1, "maxLength": MAX_ASSERTION_ISSUER_CLIENT_BYTES}, + "additionalProperties" = { + "type": "array", + "maxItems": MAX_ASSERTION_ISSUERS_PER_CLIENT, + "uniqueItems": true, + "items": {"type": "string", "minLength": 1, "maxLength": MAX_ASSERTION_ISSUER_BYTES} + } + )) + )] + pub assertion_issuers: BTreeMap>, +} + +impl OidcClientsConfig { + /// Refuse an assertion-issuer map with too many clients, an empty or + /// oversized client key or issuer, too many issuers for one client, or an + /// issuer repeated within one client's list, so one operator document + /// cannot become an unbounded verifier input. `field` is the dotted path + /// of the block, such as `authentication.oidc`. + pub fn check(&self, field: &str) -> Result<(), ConfigBlockError> { + let within_bounds = self.assertion_issuers.len() <= MAX_ASSERTION_ISSUER_CLIENTS + && self.assertion_issuers.iter().all(|(client, issuers)| { + let mut seen = BTreeSet::new(); + !client.is_empty() + && client.len() <= MAX_ASSERTION_ISSUER_CLIENT_BYTES + && issuers.len() <= MAX_ASSERTION_ISSUERS_PER_CLIENT + && issuers.iter().all(|issuer| { + !issuer.is_empty() + && issuer.len() <= MAX_ASSERTION_ISSUER_BYTES + && seen.insert(issuer) + }) + }); + if within_bounds { + return Ok(()); + } + let field = format!("{field}.assertionIssuers"); + Err(ConfigBlockError::new( + ConfigBlockErrorKind::InvalidAssertionIssuers, + &field, + format!( + "{field} must list at most {MAX_ASSERTION_ISSUER_CLIENTS} non-empty client \ + identifiers of at most {MAX_ASSERTION_ISSUER_CLIENT_BYTES} bytes, each with at \ + most {MAX_ASSERTION_ISSUERS_PER_CLIENT} distinct non-empty issuers of at most \ + {MAX_ASSERTION_ISSUER_BYTES} bytes" + ), + )) + } +} + +/// The package a runtime serves: `root` is the absolute package directory, +/// and `expectedDigest`, when set, pins the package digest the runtime must +/// find there. The package digest is the digest of the package's +/// `SHA256SUMS` file; see [`crate::package`]. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PackageConfig { + /// Absolute path of the package directory. + #[cfg_attr(feature = "schema", schemars(extend("pattern" = "^/")))] + pub root: PathBuf, + /// `sha256:` label of the package digest, the digest of the package's + /// `SHA256SUMS` file. When set, the runtime refuses to start on any other + /// package. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "schema", schemars(extend("pattern" = "^sha256:[0-9a-f]{64}$")))] + pub expected_digest: Option, +} + +impl PackageConfig { + /// `root` is absolute and `expectedDigest`, if set, is a `sha256:` label. + pub fn check(&self) -> Result<(), ConfigBlockError> { + require_absolute("package.root", &self.root)?; + if let Some(digest) = &self.expected_digest { + if !is_sha256_label(digest) { + return Err(ConfigBlockError::new( + ConfigBlockErrorKind::InvalidDigest, + "package.expectedDigest", + "package.expectedDigest must be sha256: followed by 64 lowercase hex digits" + .to_owned(), + )); + } + } + Ok(()) + } + + /// Compare the digest of the package found at `root` with the pin. + pub fn verify_digest(&self, found: &str) -> Result<(), PackageDigestMismatch> { + match &self.expected_digest { + Some(expected) if expected != found => Err(PackageDigestMismatch { + expected: expected.clone(), + found: found.to_owned(), + }), + _ => Ok(()), + } + } +} + +/// The package at `package.root` is not the one `package.expectedDigest` pins. +/// Both values are package identities, not secrets. +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] +#[error( + "package.expectedDigest is {expected} but the package at package.root is {found}; \ + deploy the pinned package or update package.expectedDigest" +)] +pub struct PackageDigestMismatch { + pub expected: String, + pub found: String, +} + +/// Whether `value` is `sha256:` followed by 64 lowercase hex digits. +#[must_use] +pub fn is_sha256_label(value: &str) -> bool { + value.strip_prefix("sha256:").is_some_and(|hex| { + hex.len() == 64 + && hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) +} + +/// The longest `listener.bind` text accepted. A socket address parser accepts +/// any number of leading zeroes in the port, so the text is bounded first. +pub const MAX_LISTENER_BIND_CHARACTERS: usize = 128; + +/// A listener socket address written `host:port`, with an IP literal host +/// (`[addr]:port` for IPv6). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ListenerBind(pub SocketAddr); + +impl ListenerBind { + #[must_use] + pub const fn socket_addr(self) -> SocketAddr { + self.0 + } + + #[must_use] + pub const fn ip(self) -> IpAddr { + self.0.ip() + } +} + +impl FromStr for ListenerBind { + type Err = std::net::AddrParseError; + + fn from_str(value: &str) -> Result { + value.parse().map(Self) + } +} + +impl Serialize for ListenerBind { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for ListenerBind { + fn deserialize>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + let bounded = value.chars().count() <= MAX_LISTENER_BIND_CHARACTERS; + bounded + .then(|| value.parse().ok()) + .flatten() + .ok_or_else(|| { + serde::de::Error::custom( + "listener.bind must be host:port with an IP address host, such as \ + 127.0.0.1:8080 or [::1]:8080", + ) + }) + } +} + +#[cfg(feature = "schema")] +impl schemars::JsonSchema for ListenerBind { + fn schema_name() -> std::borrow::Cow<'static, str> { + "ListenerBind".into() + } + + fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "description": "Socket address the runtime listens on, written host:port with an IP address host ([addr]:port for IPv6).", + "type": "string", + "minLength": 1, + "maxLength": MAX_LISTENER_BIND_CHARACTERS + }) + } +} + +/// The listener of a runtime that declares no TLS or exposure settings. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ListenerConfig { + pub bind: ListenerBind, +} + +/// Declares the trusted transport boundary for a runtime's plaintext HTTP +/// listener. Production listeners require operator-controlled upstream TLS +/// termination; direct plaintext is limited to the explicit loopback-only +/// development mode. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum TlsTermination { + OperatorControlledUpstream, + DevelopmentLoopback, +} + +/// The operator-declared private network placement of an HTTP listener. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum ListenerNetworkExposure { + #[default] + PrivateAddress, + ContainerPrivate, +} + +/// The listener of a runtime that declares its TLS termination and network +/// exposure. +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PrivateListenerConfig { + pub bind: ListenerBind, + pub tls_termination: TlsTermination, + #[serde(default)] + pub network_exposure: ListenerNetworkExposure, +} + +impl PrivateListenerConfig { + /// Whether the bind address is allowed for the declared TLS termination + /// and network exposure: loopback only in development; a loopback or + /// private address behind an operator-controlled terminator; the + /// unspecified address only inside a private container network. + #[must_use] + pub fn is_valid(&self) -> bool { + let address = self.bind.ip(); + if address.is_multicast() { + return false; + } + if self.tls_termination == TlsTermination::DevelopmentLoopback { + return self.network_exposure == ListenerNetworkExposure::PrivateAddress + && address.is_loopback(); + } + match (address, self.network_exposure) { + (IpAddr::V4(address), ListenerNetworkExposure::PrivateAddress) => { + address.is_loopback() || address.is_private() + } + (IpAddr::V6(address), ListenerNetworkExposure::PrivateAddress) => { + address.is_loopback() || is_unique_local(address) + } + (IpAddr::V4(address), ListenerNetworkExposure::ContainerPrivate) => { + address.is_unspecified() || address.is_loopback() || address.is_private() + } + (IpAddr::V6(address), ListenerNetworkExposure::ContainerPrivate) => { + address.is_unspecified() || address.is_loopback() || is_unique_local(address) + } + } + } +} + +fn is_unique_local(address: Ipv6Addr) -> bool { + address.octets()[0] & 0xfe == 0xfc +} + +#[cfg(test)] +#[path = "blocks_tests.rs"] +mod tests; diff --git a/crates/registry-platform-config/src/blocks_tests.rs b/crates/registry-platform-config/src/blocks_tests.rs new file mode 100644 index 0000000000..06ef33eae7 --- /dev/null +++ b/crates/registry-platform-config/src/blocks_tests.rs @@ -0,0 +1,614 @@ +use super::*; + +fn providers(yaml: &str) -> SecretProvidersConfig { + serde_norway::from_str(yaml).expect("providers parse") +} + +#[test] +fn secret_providers_must_enable_at_least_one_provider_with_an_absolute_root() { + let error = providers("{}").check().expect_err("none refuses"); + assert_eq!(error.kind(), ConfigBlockErrorKind::NoSecretProvider); + assert_eq!(error.field(), "secretProviders"); + + let error = providers("file: {root: relative/secrets}") + .check() + .expect_err("relative root refuses"); + assert_eq!(error.kind(), ConfigBlockErrorKind::RelativePath); + assert_eq!(error.field(), "secretProviders.file.root"); + + providers("environment: {}").check().expect("env only"); + providers("file: {root: /run/secrets}\nenvironment: {}") + .check() + .expect("both"); + assert!(serde_norway::from_str::("environment: {x: 1}").is_err()); +} + +#[test] +fn a_reference_must_be_exact_and_its_provider_enabled() { + let file_only = providers("file: {root: /run/secrets}"); + file_only + .check_reference("audit.hashKeyRef", "secret:file/audit-key") + .expect("file reference"); + let error = file_only + .check_reference("audit.hashKeyRef", "secret:env/AUDIT_KEY") + .expect_err("env disabled"); + assert_eq!(error.kind(), ConfigBlockErrorKind::SecretProviderDisabled); + assert!(error.to_string().contains("secretProviders.environment")); + + let error = file_only + .check_reference("audit.hashKeyRef", "plaintext-literal-key") + .expect_err("invalid"); + assert_eq!(error.kind(), ConfigBlockErrorKind::InvalidSecretReference); + assert!(!error.to_string().contains("plaintext-literal-key")); +} + +#[test] +fn the_resolver_enables_exactly_the_declared_providers() { + let resolver = providers("environment: {}").resolver().expect("resolver"); + assert!(matches!( + resolver.resolve("secret:file/x"), + Err(SecretError::ProviderDisabled) + )); +} + +#[test] +fn database_debug_redacts_every_reference() { + let database: DatabaseConfig = serde_norway::from_str( + "runtimeUrlRef: secret:env/RUNTIME_URL\nmigrationUrlRef: secret:env/MIGRATION_URL\ntrustedRootCertificateRef: secret:file/ca", + ) + .expect("database parses"); + let rendered = format!("{database:?}"); + assert!(!rendered.contains("secret:"), "{rendered}"); + assert_eq!( + database.references(), + [ + ("database.runtimeUrlRef", "secret:env/RUNTIME_URL"), + ("database.migrationUrlRef", "secret:env/MIGRATION_URL"), + ("database.trustedRootCertificateRef", "secret:file/ca"), + ] + ); +} + +#[test] +fn a_discovery_jwks_source_refuses_the_members_of_the_other_kinds() { + for text in [ + "kind: discovery\nuri: https://keys.example.test/jwks", + "kind: discovery\ndocumentRef: secret:file/jwks", + ] { + let error = serde_norway::from_str::(text) + .expect_err("a discovery source carries no other member"); + assert!(error.to_string().contains("unknown field"), "{error}"); + } + let discovery: JwksSource = serde_norway::from_str("kind: discovery").unwrap(); + assert_eq!(discovery, JwksSource::Discovery {}); +} + +#[test] +fn jwks_source_has_three_kinds_and_defaults_to_discovery() { + assert_eq!(JwksSource::default(), JwksSource::Discovery {}); + let uri: JwksSource = + serde_norway::from_str("kind: uri\nuri: https://issuer.example.test/jwks").unwrap(); + assert_eq!(uri.uri(), Some("https://issuer.example.test/jwks")); + uri.check("authentication.oidc.jwksSource", false) + .expect("https uri"); + let static_source: JwksSource = + serde_norway::from_str("kind: static\ndocumentRef: secret:file/jwks").unwrap(); + assert_eq!(static_source.document_ref(), Some("secret:file/jwks")); + assert!(serde_norway::from_str::("kind: static\nuri: x").is_err()); + + for (uri, loopback) in [ + ("http://issuer.example.test/jwks", true), + ("http://127.0.0.1:1/jwks", false), + ("https://user:pass@issuer.example.test/jwks", false), + ("not a url", false), + ] { + let source = JwksSource::Uri { + uri: uri.to_owned(), + }; + let error = source + .check("authentication.oidc.jwksSource", loopback) + .expect_err("refused uri"); + assert_eq!(error.field(), "authentication.oidc.jwksSource.uri"); + } + JwksSource::Uri { + uri: "http://127.0.0.1:1/jwks".to_owned(), + } + .check("authentication.oidc.jwksSource", true) + .expect("supervised loopback"); +} + +#[test] +fn package_root_is_absolute_and_the_pin_is_a_sha256_label() { + let digest = format!("sha256:{}", "a".repeat(64)); + let package = PackageConfig { + root: PathBuf::from("/srv/package"), + expected_digest: Some(digest.clone()), + }; + package.check().expect("valid package"); + package.verify_digest(&digest).expect("matches"); + + let other = format!("sha256:{}", "b".repeat(64)); + let mismatch = package.verify_digest(&other).expect_err("mismatch"); + assert_eq!( + mismatch.to_string(), + format!( + "package.expectedDigest is {digest} but the package at package.root is {other}; \ + deploy the pinned package or update package.expectedDigest" + ) + ); + + PackageConfig { + root: PathBuf::from("/srv/package"), + expected_digest: None, + } + .verify_digest(&other) + .expect("no pin, no check"); + + let error = PackageConfig { + root: PathBuf::from("package"), + expected_digest: None, + } + .check() + .expect_err("relative"); + assert_eq!(error.field(), "package.root"); + + for bad in [ + "sha256:ABC", + "sha512:aa", + &format!("sha256:{}", "A".repeat(64)), + ] { + let error = PackageConfig { + root: PathBuf::from("/srv/package"), + expected_digest: Some(bad.to_owned()), + } + .check() + .expect_err("bad digest"); + assert_eq!(error.kind(), ConfigBlockErrorKind::InvalidDigest); + } +} + +#[test] +fn listener_bind_is_an_ip_socket_address() { + let listener: ListenerConfig = serde_norway::from_str("bind: \"[::1]:8080\"").unwrap(); + assert_eq!(listener.bind.socket_addr().port(), 8080); + for bad in ["localhost:8080", "127.0.0.1", "8080"] { + let error = serde_norway::from_str::(&format!("bind: \"{bad}\"")) + .expect_err("refused bind"); + assert!(error + .to_string() + .contains("listener.bind must be host:port")); + } + assert!(serde_norway::from_str::("address: 127.0.0.1:1").is_err()); + + let padded = |port: &str| { + let width = MAX_LISTENER_BIND_CHARACTERS - "127.0.0.1:".len(); + format!("bind: \"127.0.0.1:{port:0>width$}\"") + }; + serde_norway::from_str::(&padded("80")).expect("bound length"); + let overlong = padded("80").replacen(":0", ":00", 1); + let error = serde_norway::from_str::(&overlong).expect_err("overlong bind"); + assert!(error + .to_string() + .contains("listener.bind must be host:port")); +} + +#[test] +fn private_listener_follows_the_declared_boundary() { + let listener = |bind: &str, tls: TlsTermination, exposure: ListenerNetworkExposure| { + PrivateListenerConfig { + bind: bind.parse().unwrap(), + tls_termination: tls, + network_exposure: exposure, + } + .is_valid() + }; + use ListenerNetworkExposure::{ContainerPrivate, PrivateAddress}; + use TlsTermination::{DevelopmentLoopback, OperatorControlledUpstream}; + assert!(listener("127.0.0.1:1", DevelopmentLoopback, PrivateAddress)); + assert!(!listener("10.0.0.1:1", DevelopmentLoopback, PrivateAddress)); + assert!(!listener( + "127.0.0.1:1", + DevelopmentLoopback, + ContainerPrivate + )); + assert!(listener( + "10.0.0.1:1", + OperatorControlledUpstream, + PrivateAddress + )); + assert!(listener( + "[fd00::1]:1", + OperatorControlledUpstream, + PrivateAddress + )); + assert!(!listener( + "0.0.0.0:1", + OperatorControlledUpstream, + PrivateAddress + )); + assert!(listener( + "0.0.0.0:1", + OperatorControlledUpstream, + ContainerPrivate + )); + assert!(!listener( + "8.8.8.8:1", + OperatorControlledUpstream, + ContainerPrivate + )); + assert!(!listener( + "224.0.0.1:1", + OperatorControlledUpstream, + ContainerPrivate + )); +} + +#[test] +fn sha256_label_shape() { + assert!(is_sha256_label(&crate::sha256_uri(b"x"))); + assert!(!is_sha256_label("sha256:")); +} + +#[test] +fn blocks_serialize_back_to_the_form_they_were_read_from() { + let providers: SecretProvidersConfig = + serde_norway::from_str("file: {root: /run/secrets}\nenvironment: {}").unwrap(); + let package: PackageConfig = serde_norway::from_str("root: /srv/package").unwrap(); + let listener: ListenerConfig = serde_norway::from_str("bind: \"[::1]:8080\"").unwrap(); + let jwks: JwksSource = + serde_norway::from_str("kind: uri\nuri: https://issuer.example.test/jwks").unwrap(); + assert_eq!( + serde_json::to_value(&providers).unwrap(), + serde_json::json!({"file": {"root": "/run/secrets"}, "environment": {}}) + ); + assert_eq!( + serde_json::to_value(&package).unwrap(), + serde_json::json!({"root": "/srv/package"}) + ); + assert_eq!( + serde_json::to_value(listener).unwrap(), + serde_json::json!({"bind": "[::1]:8080"}) + ); + assert_eq!( + serde_json::to_value(&jwks).unwrap(), + serde_json::json!({"kind": "uri", "uri": "https://issuer.example.test/jwks"}) + ); + assert_eq!( + serde_json::to_value(JwksSource::Discovery {}).unwrap(), + serde_json::json!({"kind": "discovery"}) + ); + for (value, text) in [ + (serde_json::to_value(&providers).unwrap(), "providers"), + (serde_json::to_value(&package).unwrap(), "package"), + ] { + assert!(value.is_object(), "{text}"); + } + assert_eq!( + serde_json::from_value::(serde_json::to_value(&providers).unwrap()) + .unwrap(), + providers + ); +} + +/// A product audit block embedding the shared key beside its own sink setting. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ProductAudit { + path: String, + #[serde(flatten)] + key: AuditKeyConfig, +} + +#[test] +fn the_audit_key_embeds_beside_product_members_and_redacts_its_reference() { + let audit: ProductAudit = + serde_norway::from_str("path: /var/lib/audit\nhashKeyRef: secret:file/audit-key") + .expect("embedded audit key parses"); + assert_eq!(audit.path, "/var/lib/audit"); + assert_eq!(audit.key.hash_key_ref.as_str(), "secret:file/audit-key"); + assert!(!format!("{audit:?}").contains("audit-key"), "{audit:?}"); + + for (text, reason) in [ + ( + "path: /var/lib/audit\nhashKeyRef: secret:file/audit-key\nother: 1", + "an unknown member beside the embedded key", + ), + ( + "path: /var/lib/audit\nhashSecretRef: secret:file/audit-key", + "the key under another name", + ), + ("path: /var/lib/audit", "no key"), + ] { + assert!( + serde_norway::from_str::(text).is_err(), + "{reason} must be refused" + ); + } + + let error = serde_norway::from_str::( + "path: /var/lib/audit\nhashKeyRef: plaintext-literal-key", + ) + .expect_err("a literal is not a reference"); + assert!( + !error.to_string().contains("plaintext-literal-key"), + "{error}" + ); +} + +#[test] +fn the_audit_key_names_an_enabled_provider() { + let key: AuditKeyConfig = serde_norway::from_str("hashKeyRef: secret:env/AUDIT_KEY").unwrap(); + key.check(&providers("environment: {}")) + .expect("enabled provider"); + let error = key + .check(&providers("file: {root: /run/secrets}")) + .expect_err("disabled provider"); + assert_eq!(error.kind(), ConfigBlockErrorKind::SecretProviderDisabled); + assert_eq!(error.field(), "audit.hashKeyRef"); + assert_eq!( + serde_json::to_value(&key).unwrap(), + serde_json::json!({"hashKeyRef": "secret:env/AUDIT_KEY"}) + ); +} + +/// A product OIDC block embedding the shared issuer beside its own members. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ProductOidc { + #[serde(flatten)] + issuer: OidcIssuerConfig, + token_types: Vec, +} + +#[test] +fn the_oidc_issuer_embeds_beside_product_members() { + let oidc: ProductOidc = serde_norway::from_str( + "issuer: https://issuer.example.test\naudience: urn:example:api\n\ + jwksSource: {kind: uri, uri: https://issuer.example.test/jwks}\ntokenTypes: [at+jwt]", + ) + .expect("embedded issuer parses"); + assert_eq!(oidc.issuer.issuer, "https://issuer.example.test"); + assert_eq!(oidc.issuer.audience, "urn:example:api"); + assert_eq!( + oidc.issuer.jwks_source.uri(), + Some("https://issuer.example.test/jwks") + ); + assert_eq!(oidc.token_types, ["at+jwt"]); + + let defaulted: ProductOidc = serde_norway::from_str( + "issuer: https://issuer.example.test\naudience: urn:example:api\ntokenTypes: []", + ) + .unwrap(); + assert_eq!(defaulted.issuer.jwks_source, JwksSource::Discovery {}); + + for (text, reason) in [ + ( + "issuer: https://issuer.example.test\naudience: a\ntokenTypes: []\nother: 1", + "an unknown member", + ), + ( + "issuer: https://issuer.example.test\naudiences: [a]\ntokenTypes: []", + "a plural audience", + ), + ( + "issuer: https://issuer.example.test\naudience: a\njwksUri: https://issuer.example.test/jwks\ntokenTypes: []", + "a bare JWKS URI", + ), + ] { + assert!( + serde_norway::from_str::(text).is_err(), + "{reason} must be refused" + ); + } +} + +fn issuer(issuer: &str, audience: &str, jwks_source: JwksSource) -> OidcIssuerConfig { + OidcIssuerConfig { + issuer: issuer.to_owned(), + audience: audience.to_owned(), + jwks_source, + } +} + +#[test] +fn the_oidc_issuer_is_an_https_url_and_the_audience_is_bounded_text() { + let field = "authentication.oidc"; + issuer( + "https://issuer.example.test", + "urn:example:api", + JwksSource::default(), + ) + .check(field, false) + .expect("https issuer"); + issuer("http://127.0.0.1:8082", "api", JwksSource::default()) + .check(field, true) + .expect("supervised loopback issuer"); + + for (candidate, loopback, expected_field) in [ + ( + issuer("http://issuer.example.test", "api", JwksSource::default()), + true, + "authentication.oidc.issuer", + ), + ( + issuer("http://127.0.0.1:8082", "api", JwksSource::default()), + false, + "authentication.oidc.issuer", + ), + ( + issuer( + "https://user:pass@issuer.example.test", + "api", + JwksSource::default(), + ), + false, + "authentication.oidc.issuer", + ), + ( + issuer( + "https://issuer.example.test#fragment", + "api", + JwksSource::default(), + ), + false, + "authentication.oidc.issuer", + ), + ( + issuer("issuer.example.test", "api", JwksSource::default()), + false, + "authentication.oidc.issuer", + ), + ( + issuer("https://issuer.example.test", "", JwksSource::default()), + false, + "authentication.oidc.audience", + ), + ( + issuer("https://issuer.example.test", "a\nb", JwksSource::default()), + false, + "authentication.oidc.audience", + ), + ( + issuer( + "https://issuer.example.test", + &"a".repeat(MAX_OIDC_AUDIENCE_CHARACTERS + 1), + JwksSource::default(), + ), + false, + "authentication.oidc.audience", + ), + ( + issuer( + "https://issuer.example.test", + "api", + JwksSource::Uri { + uri: "http://keys.example.test/jwks".to_owned(), + }, + ), + false, + "authentication.oidc.jwksSource.uri", + ), + ] { + let error = candidate + .check(field, loopback) + .expect_err("refused issuer"); + assert_eq!(error.field(), expected_field, "{error}"); + assert!(!error.to_string().contains("user:pass"), "{error}"); + } +} + +#[test] +fn an_oidc_issuer_with_a_query_is_refused_without_echoing_it() { + let field = "authentication.oidc"; + issuer( + "https://issuer.example.test/realms/pilot", + "api", + JwksSource::default(), + ) + .check(field, false) + .expect("an issuer with a path"); + for candidate in [ + "https://issuer.example.test/realms/pilot?tenant=query-canary", + "https://issuer.example.test/?", + ] { + let error = issuer(candidate, "api", JwksSource::default()) + .check(field, false) + .expect_err("an issuer with a query is refused"); + assert_eq!(error.field(), "authentication.oidc.issuer", "{error}"); + assert!(error.to_string().contains("query"), "{error}"); + assert!(!error.to_string().contains("query-canary"), "{error}"); + } +} + +#[test] +fn a_secret_reference_reads_and_writes_as_its_text() { + let reference: SecretReference = + serde_json::from_value(serde_json::json!("secret:file/a")).expect("reference parses"); + assert_eq!( + serde_json::to_value(&reference).unwrap(), + serde_json::json!("secret:file/a") + ); + let error = serde_json::from_value::(serde_json::json!("literal-value")) + .expect_err("literal refused"); + assert!(!error.to_string().contains("literal-value"), "{error}"); + let ordered = std::collections::BTreeSet::from([ + SecretReference::parse("secret:file/b").unwrap(), + SecretReference::parse("secret:file/a").unwrap(), + ]); + assert_eq!( + ordered + .iter() + .map(SecretReference::as_str) + .collect::>(), + ["secret:file/a", "secret:file/b"] + ); +} + +fn clients(yaml: &str) -> OidcClientsConfig { + serde_norway::from_str(yaml).expect("clients parse") +} + +#[test] +fn oidc_clients_default_to_no_rule_and_bound_the_assertion_issuer_map() { + let field = "authentication.oidc"; + let empty = clients("{}"); + assert!(empty.allowed_clients.is_empty()); + assert!(empty.assertion_issuers.is_empty()); + empty.check(field).expect("no rule"); + clients("allowedClients: [portal]\nassertionIssuers: {portal: [https://assert.example.test]}") + .check(field) + .expect("one client, one authority"); + + let many_clients = (0..=MAX_ASSERTION_ISSUER_CLIENTS) + .map(|index| format!("c{index}: []")) + .collect::>() + .join(", "); + let many_issuers = (0..=MAX_ASSERTION_ISSUERS_PER_CLIENT) + .map(|index| format!("https://a{index}.example.test")) + .collect::>() + .join(", "); + for (reason, yaml) in [ + ( + "too many clients", + format!("assertionIssuers: {{{many_clients}}}"), + ), + ("an empty client", "assertionIssuers: {'': []}".to_owned()), + ( + "an oversized client", + format!( + "assertionIssuers: {{{}: []}}", + "c".repeat(MAX_ASSERTION_ISSUER_CLIENT_BYTES + 1) + ), + ), + ( + "too many issuers for one client", + format!("assertionIssuers: {{portal: [{many_issuers}]}}"), + ), + ( + "an empty issuer", + "assertionIssuers: {portal: ['']}".to_owned(), + ), + ( + "an oversized issuer", + format!( + "assertionIssuers: {{portal: [{}]}}", + "i".repeat(MAX_ASSERTION_ISSUER_BYTES + 1) + ), + ), + ( + "a repeated issuer", + "assertionIssuers: {portal: [https://a.example.test, https://a.example.test]}" + .to_owned(), + ), + ] { + let error = clients(&yaml).check(field).expect_err(reason); + assert_eq!( + error.kind(), + ConfigBlockErrorKind::InvalidAssertionIssuers, + "{reason}" + ); + assert_eq!(error.field(), "authentication.oidc.assertionIssuers"); + assert!(!error.to_string().contains("example.test"), "{reason}"); + } + assert!(serde_norway::from_str::("allowedClients: portal").is_err()); +} diff --git a/crates/registry-platform-config/src/lib.rs b/crates/registry-platform-config/src/lib.rs index d86387a118..ba4fa7d73e 100644 --- a/crates/registry-platform-config/src/lib.rs +++ b/crates/registry-platform-config/src/lib.rs @@ -1,137 +1,49 @@ -//! Governed runtime configuration verification contracts. - +//! Shared runtime configuration for Registry Stack runtimes. +//! +//! [`RuntimeConfigLoader`] reads an operator `runtime.yaml` under one set of +//! file, parsing and environment-substitution rules; the [`blocks`] types are +//! the configuration sections every runtime spells the same way; and +//! [`SecretResolver`] resolves the `secret:env/NAME` and `secret:file/name` +//! references those sections carry. [`package`] writes and verifies the +//! package directory `package.root` names. Each product still owns and +//! validates the rest of its configuration contract. + +pub mod blocks; +mod loader; +pub mod package; +#[cfg(feature = "schema")] +pub mod schema; mod secrets; -use serde_json::Value; use sha2::{Digest, Sha256}; +pub use blocks::{ + describe_secret_failure, is_sha256_label, AuditKeyConfig, ConfigBlockError, + ConfigBlockErrorKind, DatabaseConfig, EnvironmentSecretProviderConfig, + FileSecretProviderConfig, JwksSource, ListenerBind, ListenerConfig, ListenerNetworkExposure, + OidcClientsConfig, OidcIssuerConfig, PackageConfig, PackageDigestMismatch, + PrivateListenerConfig, SecretProvidersConfig, TlsTermination, MAX_ASSERTION_ISSUERS_PER_CLIENT, + MAX_ASSERTION_ISSUER_BYTES, MAX_ASSERTION_ISSUER_CLIENTS, MAX_ASSERTION_ISSUER_CLIENT_BYTES, + MAX_LISTENER_BIND_CHARACTERS, MAX_OIDC_AUDIENCE_CHARACTERS, +}; +pub use loader::{ + contains_environment_expression, reject_environment_expressions_in_authored_yaml, + LoadedRuntimeConfig, RemovedKey, RuntimeConfigError, RuntimeConfigErrorKind, + RuntimeConfigLoader, RuntimeEnvelope, DEFAULT_MAX_RUNTIME_CONFIG_BYTES, + MAX_RUNTIME_CONFIG_PATH_BYTES, REMOVED_OIDC_JWKS_URI, +}; +pub use package::{ + plan_package, verify_package, write_package, write_sum_file, PackageError, PackageErrorKind, + PackageLimits, VerifiedPackage, REVISION_FILE, SUM_FILE, +}; pub use secrets::{ ProtectedSecret, SecretError, SecretProvider, SecretReference, SecretResolver, MAX_SECRET_BYTES, }; -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct DeprecatedConfigField { - path: Vec, - replacement: Option, - message: Option, -} - -impl DeprecatedConfigField { - pub fn renamed(path: impl Into, replacement: impl Into) -> Self { - Self { - path: split_config_path(path), - replacement: Some(replacement.into()), - message: None, - } - } - - pub fn removed(path: impl Into, message: impl Into) -> Self { - Self { - path: split_config_path(path), - replacement: None, - message: Some(message.into()), - } - } - - pub fn path(&self) -> String { - self.path.join(".") - } -} - -#[derive(Debug, Clone, Eq, PartialEq, thiserror::Error)] -#[error("{message}")] -pub struct DeprecatedConfigFieldError { - field: String, - message: String, -} - -impl DeprecatedConfigFieldError { - pub fn field(&self) -> &str { - &self.field - } -} - -pub fn reject_deprecated_config_fields( - root: &Value, - fields: &[DeprecatedConfigField], -) -> Result<(), DeprecatedConfigFieldError> { - for field in fields { - if config_value_at_path(root, &field.path).is_some() { - let field_path = field.path(); - let message = if let Some(replacement) = &field.replacement { - format!("{field_path} has been renamed; use {replacement}") - } else if let Some(message) = &field.message { - format!("{field_path} has been removed; {message}") - } else { - format!("{field_path} has been removed") - }; - return Err(DeprecatedConfigFieldError { - field: field_path, - message, - }); - } - } - Ok(()) -} - -#[derive(Debug, Clone, Eq, PartialEq, thiserror::Error)] -#[error("{0}")] -pub struct ConfigEnvExpansionError(String); - -pub fn expand_config_env_vars(raw: &str) -> Result { - expand_config_env_vars_with(raw, |name| std::env::var(name).ok()) -} - -pub fn expand_config_env_vars_with( - raw: &str, - lookup: impl Fn(&str) -> Option, -) -> Result { - let mut expanded = String::with_capacity(raw.len()); - let mut rest = raw; - while let Some(start) = rest.find("${") { - expanded.push_str(&rest[..start]); - let after_start = &rest[start + 2..]; - let Some(end) = after_start.find('}') else { - return Err(ConfigEnvExpansionError( - "unterminated ${...} expression in config".to_string(), - )); - }; - let expression = &after_start[..end]; - let after_expression = &after_start[end + 1..]; - let (name, value) = resolve_config_env_expression(expression, &lookup)?; - if config_env_expression_is_whole_yaml_scalar(&expanded, after_expression) { - reject_config_env_nul(name, &value)?; - expanded.push_str(&yaml_double_quoted_scalar(&value)); - } else { - reject_unsafe_embedded_config_env_value(name, &value)?; - expanded.push_str(&value); - } - rest = after_expression; - } - expanded.push_str(rest); - Ok(expanded) -} - -fn split_config_path(path: impl Into) -> Vec { - path.into() - .split('.') - .filter(|segment| !segment.is_empty()) - .map(ToString::to_string) - .collect() -} - -fn config_value_at_path<'a>(root: &'a Value, path: &[String]) -> Option<&'a Value> { - let mut current = root; - for segment in path { - current = current.get(segment)?; - } - Some(current) -} - fn resolve_config_env_expression( expression: &str, lookup: impl Fn(&str) -> Option, -) -> Result<(&str, String), ConfigEnvExpansionError> { +) -> Result<(&str, String), String> { let (name, operator, fallback) = if let Some((name, fallback)) = expression.split_once(":-") { (name, ":-", fallback) } else if let Some((name, fallback)) = expression.split_once(":?") { @@ -139,10 +51,10 @@ fn resolve_config_env_expression( } else { (expression, "", "") }; + // Neither an invalid name nor a `:?` message is repeated: both are + // configured text, and a refusal reaches the operator's log. if !valid_env_key(name) { - return Err(ConfigEnvExpansionError(format!( - "invalid env var name in config expression: {name}" - ))); + return Err("a config expression names an invalid environment variable".to_string()); } match lookup(name) { @@ -150,118 +62,98 @@ fn resolve_config_env_expression( _ if operator == ":-" => Ok((name, fallback.to_string())), _ if operator == ":?" => { if fallback.trim().is_empty() { - Err(ConfigEnvExpansionError(format!( - "required env var {name} is unset or empty" - ))) + Err(format!("required env var {name} is unset or empty")) } else { - Err(ConfigEnvExpansionError(fallback.to_string())) + Err(format!( + "required env var {name} is unset or empty; its configured message is withheld" + )) } } - _ => Err(ConfigEnvExpansionError(format!( - "required env var {name} is unset or empty" - ))), + _ => Err(format!("required env var {name} is unset or empty")), } } -fn config_env_expression_is_whole_yaml_scalar(before: &str, after: &str) -> bool { - let line_prefix = before.rsplit_once('\n').map_or(before, |(_, line)| line); - let trimmed_prefix = line_prefix.trim_start(); - let prefix_is_scalar = trimmed_prefix.is_empty() - || trimmed_prefix.trim_end() == "-" - || trimmed_prefix.trim_end().ends_with(':'); - if !prefix_is_scalar { - return false; - } - - let line_suffix = after.split_once('\n').map_or(after, |(line, _)| line); - let trimmed_suffix = line_suffix.trim_start(); - trimmed_suffix.is_empty() || trimmed_suffix.starts_with('#') +fn valid_env_key(key: &str) -> bool { + let mut chars = key.chars(); + matches!(chars.next(), Some(c) if c == '_' || c.is_ascii_alphabetic()) + && chars.all(|c| c == '_' || c.is_ascii_alphanumeric()) } -fn yaml_double_quoted_scalar(value: &str) -> String { - let mut quoted = String::with_capacity(value.len() + 2); - quoted.push('"'); - for ch in value.chars() { - match ch { - '"' => quoted.push_str("\\\""), - '\\' => quoted.push_str("\\\\"), - '\n' => quoted.push_str("\\n"), - '\r' => quoted.push_str("\\r"), - '\t' => quoted.push_str("\\t"), - '\0' => quoted.push_str("\\0"), - ch if ch.is_control() => { - use std::fmt::Write; - let _ = write!(quoted, "\\x{:02X}", ch as u32); - } - ch => quoted.push(ch), - } - } - quoted.push('"'); - quoted +/// The `sha256:` label of `bytes`: `sha256:` followed by 64 lowercase hex +/// digits. +#[must_use] +pub fn sha256_uri(bytes: &[u8]) -> String { + format!("sha256:{}", hex_lower(&Sha256::digest(bytes))) } -fn reject_config_env_nul(name: &str, value: &str) -> Result<(), ConfigEnvExpansionError> { - if value.contains('\0') { - return Err(ConfigEnvExpansionError(format!( - "env var {name} contains characters that cannot be used in config expansion" - ))); +/// Keep the parts of a serde refusal an operator acts on, the member, the +/// reason and the location, while the refused value stays out of the message. +/// +/// serde reports the offending value inside an `invalid type:`, an +/// `invalid value:` or an `unknown variant` clause. Only the shape word that +/// opens such a clause survives, so the message still says a string arrived where a number was +/// required without repeating the string. A runtime configuration names +/// secret references, database URLs and destinations, and a startup refusal +/// is written to the operator's log. +#[must_use] +pub fn redact_refused_values(message: &str) -> String { + const CLAUSES: [&str; 3] = ["invalid type: ", "invalid value: ", "unknown variant"]; + let mut redacted = String::with_capacity(message.len()); + let mut rest = message; + loop { + let Some((start, len)) = CLAUSES + .iter() + .filter_map(|clause| rest.find(clause).map(|start| (start, clause.len()))) + .min_by_key(|(start, _)| *start) + else { + redacted.push_str(rest); + return redacted; + }; + let opened = start + len; + redacted.push_str(&rest[..opened]); + let (shape, tail) = split_refused_value(&rest[opened..]); + redacted.push_str(shape); + rest = tail; } - Ok(()) } -fn reject_unsafe_embedded_config_env_value( - name: &str, - value: &str, -) -> Result<(), ConfigEnvExpansionError> { - reject_config_env_nul(name, value)?; - if value.contains('\n') - || value.contains('\r') - // unsafe-libyaml treats NEL, LS, and PS as line breaks too. - || value.contains('\u{0085}') - || value.contains('\u{2028}') - || value.contains('\u{2029}') - || value.contains('"') - || value.contains('\'') - || value.contains('{') - || value.contains('}') - || value.contains('[') - || value.contains(']') - || value.contains(',') - || value.contains('|') - || value.contains('>') - || value.contains('`') - || value.contains(": ") - || value.contains(" #") - { - return Err(ConfigEnvExpansionError(format!( - "env var {name} contains characters that are unsafe in embedded config expansion" - ))); - } - let trimmed = value.trim_start(); - if trimmed.starts_with('#') - || trimmed.starts_with('&') - || trimmed.starts_with('*') - || trimmed.starts_with('!') - || trimmed.starts_with('%') - || trimmed.starts_with('@') - || trimmed.starts_with("---") - || trimmed.starts_with("...") - { - return Err(ConfigEnvExpansionError(format!( - "env var {name} contains characters that are unsafe in embedded config expansion" - ))); +/// Split the text after a clause marker into the shape word serde names and +/// the remainder that follows the refused value. +/// +/// serde renders the value with `Debug`, so it opens with a quote or a +/// backtick and may hold the comma that would otherwise end the clause. +fn split_refused_value(clause: &str) -> (&str, &str) { + let bytes = clause.as_bytes(); + let mut index = 0; + let mut shape_end = None; + while index < bytes.len() { + match bytes[index] { + delimiter @ (b'"' | b'`') => { + shape_end.get_or_insert(index); + index = skip_delimited(bytes, index, delimiter); + } + b',' => break, + _ => index += 1, + } } - Ok(()) -} - -fn valid_env_key(key: &str) -> bool { - let mut chars = key.chars(); - matches!(chars.next(), Some(c) if c == '_' || c.is_ascii_alphabetic()) - && chars.all(|c| c == '_' || c.is_ascii_alphanumeric()) + let shape_end = shape_end.unwrap_or(index); + (clause[..shape_end].trim_end(), &clause[index..]) } -pub fn sha256_uri(bytes: &[u8]) -> String { - format!("sha256:{}", hex_lower(&Sha256::digest(bytes))) +/// Return the offset just past the delimited run that opens at `open`. +/// +/// A delimiter inside a `Debug` rendering arrives escaped, so it does not end +/// the run. +fn skip_delimited(bytes: &[u8], open: usize, delimiter: u8) -> usize { + let mut index = open + 1; + while index < bytes.len() { + match bytes[index] { + b'\\' => index += 2, + byte if byte == delimiter => return index + 1, + _ => index += 1, + } + } + bytes.len() } fn hex_lower(bytes: &[u8]) -> String { @@ -277,56 +169,9 @@ fn hex_lower(bytes: &[u8]) -> String { #[cfg(test)] mod tests { use super::*; - use serde_json::json; - - #[test] - fn deprecated_config_field_detector_names_replacement() { - let root = json!({ - "auth": { - "oidc": { - "audience": ["registry-service"] - } - } - }); - - let err = reject_deprecated_config_fields( - &root, - &[DeprecatedConfigField::renamed( - "auth.oidc.audience", - "auth.oidc.audiences", - )], - ) - .expect_err("deprecated field is rejected"); - - assert_eq!(err.field(), "auth.oidc.audience"); - assert!(err.to_string().contains("auth.oidc.audiences")); - } - - #[test] - fn deprecated_config_field_detector_names_removal_rationale() { - let root = json!({ - "server": { - "cors": { - "allow_credentials": true - } - } - }); - - let err = reject_deprecated_config_fields( - &root, - &[DeprecatedConfigField::removed( - "server.cors.allow_credentials", - "credentials are always disabled", - )], - ) - .expect_err("removed field is rejected"); - - assert_eq!(err.field(), "server.cors.allow_credentials"); - assert!(err.to_string().contains("credentials are always disabled")); - } #[test] - fn config_env_expansion_distinguishes_unset_empty_and_whitespace_values() { + fn config_env_expressions_distinguish_unset_empty_and_whitespace_values() { struct Case { name: &'static str, expression: &'static str, @@ -337,186 +182,118 @@ mod tests { for case in [ Case { name: "plain expression rejects an unset value", - expression: "${VALUE}", + expression: "VALUE", value: None, expected: Err("required env var VALUE is unset or empty"), }, Case { name: "plain expression rejects an empty value", - expression: "${VALUE}", + expression: "VALUE", value: Some(""), expected: Err("required env var VALUE is unset or empty"), }, Case { name: "plain expression preserves whitespace-only value", - expression: "${VALUE}", + expression: "VALUE", value: Some(" "), - expected: Ok("\" \""), + expected: Ok(" "), }, Case { name: "fallback applies to an unset value", - expression: "${VALUE:-fallback}", + expression: "VALUE:-fallback", value: None, - expected: Ok("\"fallback\""), + expected: Ok("fallback"), }, Case { name: "fallback applies to an empty value", - expression: "${VALUE:-fallback}", + expression: "VALUE:-fallback", value: Some(""), - expected: Ok("\"fallback\""), + expected: Ok("fallback"), }, Case { name: "non-empty value wins over fallback", - expression: "${VALUE:-fallback}", + expression: "VALUE:-fallback", value: Some("configured"), - expected: Ok("\"configured\""), + expected: Ok("configured"), }, Case { name: "explicit empty fallback applies to an unset value", - expression: "${VALUE:-}", + expression: "VALUE:-", value: None, - expected: Ok("\"\""), + expected: Ok(""), }, Case { name: "explicit empty fallback applies to an empty value", - expression: "${VALUE:-}", + expression: "VALUE:-", value: Some(""), - expected: Ok("\"\""), + expected: Ok(""), }, Case { name: "required message applies to an unset value", - expression: "${VALUE:?configure VALUE}", + expression: "VALUE:?configure VALUE", value: None, - expected: Err("configure VALUE"), + expected: Err( + "required env var VALUE is unset or empty; its configured message is withheld", + ), }, Case { name: "required message applies to an empty value", - expression: "${VALUE:?configure VALUE}", + expression: "VALUE:?configure VALUE", value: Some(""), - expected: Err("configure VALUE"), + expected: Err( + "required env var VALUE is unset or empty; its configured message is withheld", + ), }, Case { name: "blank required message identifies an unset value", - expression: "${VALUE:?}", + expression: "VALUE:?", value: None, expected: Err("required env var VALUE is unset or empty"), }, Case { name: "blank required message identifies an empty value", - expression: "${VALUE:?}", + expression: "VALUE:?", value: Some(""), expected: Err("required env var VALUE is unset or empty"), }, Case { name: "unsupported syntax remains invalid", - expression: "${VALUE-fallback}", + expression: "VALUE-fallback", value: Some("configured"), - expected: Err("invalid env var name in config expression: VALUE-fallback"), + expected: Err("a config expression names an invalid environment variable"), }, ] { - let actual = expand_config_env_vars_with(case.expression, |_| { + let actual = resolve_config_env_expression(case.expression, |_| { case.value.map(ToString::to_string) }) - .map_err(|error| error.to_string()); - - let actual = actual - .as_ref() - .map(|value| value.as_str()) - .map_err(|error| error.as_str()); + .map(|(_, value)| value); + let actual = actual.as_ref().map(String::as_str).map_err(String::as_str); assert_eq!(actual, case.expected, "{}", case.name); } } +} - #[test] - fn config_env_expansion_scalarizes_whole_yaml_values() { - let expanded = - expand_config_env_vars_with("base: ${BASE_URL}\nflow: ${FLOW}\n", |name| match name { - "BASE_URL" => Some("https://registry.example\nadmin: false".to_string()), - "FLOW" => Some("{admin: false}".to_string()), - _ => None, - }) - .expect("whole-scalar config env vars are quoted"); - - assert!(expanded.contains("base: \"https://registry.example\\nadmin: false\"")); - assert!(expanded.contains("flow: \"{admin: false}\"")); - assert!(!expanded.contains("\nadmin: false")); - } - - #[test] - fn config_env_expansion_quotes_whole_scalar_yaml_syntax_values() { - let expanded = expand_config_env_vars_with( - "anchor: ${ANCHOR}\nalias: ${ALIAS}\ntag: ${TAG}\ncomment: ${COMMENT}\nblock: ${BLOCK}\nflow: ${FLOW}\n", - |name| match name { - "ANCHOR" => Some("&admin".to_string()), - "ALIAS" => Some("*admin".to_string()), - "TAG" => Some("!vault secret".to_string()), - "COMMENT" => Some("value # hidden".to_string()), - "BLOCK" => Some("line1\nline2".to_string()), - "FLOW" => Some("[admin, true]".to_string()), - _ => None, - }, - ) - .expect("whole-scalar config env vars are quoted"); - - assert!(expanded.contains("anchor: \"&admin\"")); - assert!(expanded.contains("alias: \"*admin\"")); - assert!(expanded.contains("tag: \"!vault secret\"")); - assert!(expanded.contains("comment: \"value # hidden\"")); - assert!(expanded.contains("block: \"line1\\nline2\"")); - assert!(expanded.contains("flow: \"[admin, true]\"")); - assert!(!expanded.contains("\nline2")); - } - - #[test] - fn config_env_expansion_rejects_unsafe_embedded_values() { - let err = expand_config_env_vars_with("base: https://${HOST}\n", |name| match name { - "HOST" => Some("registry.example\nadmin: false".to_string()), - _ => None, - }) - .expect_err("embedded newline cannot be expanded into YAML structure"); - - assert!(err.to_string().contains("HOST")); - assert!(!err.to_string().contains("admin")); - - let err = expand_config_env_vars_with("allowed: [${VALUE}]\n", |name| match name { - "VALUE" => Some("trusted, attacker".to_string()), - _ => None, - }) - .expect_err("embedded comma cannot expand into a YAML flow sequence"); - assert!(err.to_string().contains("VALUE")); - assert!(!err.to_string().contains("trusted")); - } +#[cfg(test)] +mod redaction_tests { + use super::redact_refused_values; #[test] - fn config_env_expansion_rejects_embedded_yaml_syntax_classes() { - for value in [ - "registry.example # hidden", - "admin: false", - "[admin]", - "trusted, attacker", - "line1\nline2", - "line1\u{0085}line2", - "evil.example\u{2028}admin:", - "evil.example\u{2028}---\u{2028}x:", - "line1\u{2029}line2", - "| block", - "> folded", - "&anchor", - "*alias", - "!tagged", - "%YAML 1.2", - "---", - "...", - ] { - let err = expand_config_env_vars_with("base: https://${VALUE}\n", |name| match name { - "VALUE" => Some(value.to_string()), - _ => None, - }) - .expect_err("embedded YAML syntax value must be rejected") - .to_string(); - - assert!(err.contains("VALUE")); - assert!(!err.contains(value)); - } + fn refused_values_are_reduced_to_their_shape() { + assert_eq!( + redact_refused_values( + "invalid type: string \"DO_NOT_DISCLOSE, still\", expected u16 at line 1" + ), + "invalid type: string, expected u16 at line 1" + ); + assert_eq!( + redact_refused_values("invalid value: integer `70000`, expected u16"), + "invalid value: integer, expected u16" + ); + assert_eq!( + redact_refused_values( + "mode: unknown variant `DO_NOT_DISCLOSE`, expected one of `a`, `b` at line 3" + ), + "mode: unknown variant, expected one of `a`, `b` at line 3" + ); } } diff --git a/crates/registry-platform-config/src/loader.rs b/crates/registry-platform-config/src/loader.rs new file mode 100644 index 0000000000..58fe302cbb --- /dev/null +++ b/crates/registry-platform-config/src/loader.rs @@ -0,0 +1,875 @@ +//! Bounded, strict loading of an operator runtime configuration document. +//! +//! Every Registry Stack runtime reads its `runtime.yaml` through +//! [`RuntimeConfigLoader`], so the same file rules, the same YAML reader and the +//! same environment substitution apply everywhere: +//! +//! 1. the path is absolute and lexically normal, and no component of it is a +//! symbolic link; +//! 2. the file is a regular file of at most the configured size, opened without +//! following a link and checked to be the same file before and after the +//! read; +//! 3. the bytes are UTF-8 and parse as exactly one YAML document with string +//! keys and no tags; +//! 4. removed keys are refused with a diagnostic naming their replacement; +//! 5. `apiVersion` and `kind` must be literally the product's envelope; +//! 6. `${VAR}`, `${VAR:-default}` and `${VAR:?message}` are substituted in +//! string values only, after parsing, and refused in every field whose name +//! ends in `Ref` or `Refs` and everywhere under `secretProviders`. There is +//! no escape syntax: a literal `${` reaches the configuration as the value of +//! a variable, because substitution is a single pass; +//! 7. the result is deserialized into the product's typed configuration. +//! +//! No refusal repeats a configured or substituted value. + +use std::fs; +use std::io::Read as _; +use std::path::{Component, Path, PathBuf}; + +use serde::de::DeserializeOwned; +use serde_json::{Map, Number, Value}; + +use crate::{redact_refused_values, resolve_config_env_expression, sha256_uri}; + +/// The default size cap for a runtime configuration document. +pub const DEFAULT_MAX_RUNTIME_CONFIG_BYTES: u64 = 1024 * 1024; + +/// The longest runtime configuration path the loader accepts, in bytes. +pub const MAX_RUNTIME_CONFIG_PATH_BYTES: usize = 4096; + +/// The `apiVersion` and `kind` a product's runtime configuration carries. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RuntimeEnvelope { + pub api_version: &'static str, + pub kind: &'static str, +} + +/// A key a runtime configuration no longer accepts, and what replaced it. +/// +/// `path` is dotted from the document root; a `*` segment matches any mapping +/// key or sequence index. `replacement` is the sentence an operator reads after +/// "`path` is no longer accepted; ". +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RemovedKey { + pub path: &'static str, + pub replacement: &'static str, +} + +/// The single-URL JWKS key the shared OIDC issuer block replaced with +/// `jwksSource`, for a runtime whose issuer block sits at `authentication.oidc`. +pub const REMOVED_OIDC_JWKS_URI: RemovedKey = RemovedKey { + path: "authentication.oidc.jwksUri", + replacement: "declare authentication.oidc.jwksSource with kind: uri and uri: ", +}; + +/// What kind of rule a runtime configuration broke. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RuntimeConfigErrorKind { + /// The configured path is not absolute and lexically normal. + Path, + /// A path component is a symbolic link, the file is not a regular file, or + /// it changed while it was read. + UnsafeFile, + /// The file could not be read. + Unavailable, + /// The file is empty or larger than the cap. + Bounds, + /// The file is not UTF-8. + Encoding, + /// The file is not exactly one YAML document with string keys and no tags. + Syntax, + /// A removed key is present. + RemovedKey, + /// `apiVersion` or `kind` is not the product's envelope. + Envelope, + /// An environment expression could not be substituted. + Substitution, + /// An environment expression appears in a secret-reference field. + SubstitutionInReference, + /// A value does not satisfy the product's typed configuration. + InvalidValue, + /// An authored project file holds an environment expression. + AuthoredExpression, + /// An authored project file is not YAML the environment-expression check + /// can read. + AuthoredSyntax, +} + +impl RuntimeConfigErrorKind { + /// The stable diagnostic code for this refusal. + #[must_use] + pub const fn code(self) -> &'static str { + match self { + Self::Path => "runtime_config.path", + Self::UnsafeFile => "runtime_config.unsafe_file", + Self::Unavailable => "runtime_config.unavailable", + Self::Bounds => "runtime_config.bounds", + Self::Encoding => "runtime_config.encoding", + Self::Syntax => "runtime_config.syntax", + Self::RemovedKey => "runtime_config.removed_key", + Self::Envelope => "runtime_config.envelope", + Self::Substitution => "runtime_config.substitution", + Self::SubstitutionInReference => "runtime_config.substitution_in_reference", + Self::InvalidValue => "runtime_config.invalid_value", + Self::AuthoredExpression => "authored_config.environment_expression", + Self::AuthoredSyntax => "authored_config.syntax", + } + } +} + +/// A refused runtime configuration: which rule broke, at which field, and a +/// message that names the field and never a configured value. +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] +#[error("{}", self.render())] +pub struct RuntimeConfigError { + kind: RuntimeConfigErrorKind, + file: Option, + field: String, + message: String, +} + +impl RuntimeConfigError { + fn new( + kind: RuntimeConfigErrorKind, + field: impl Into, + message: impl Into, + ) -> Self { + Self { + kind, + file: None, + field: field.into(), + message: message.into(), + } + } + + fn in_file(mut self, file: &Path) -> Self { + self.file = Some(file.to_owned()); + self + } + + #[must_use] + pub const fn kind(&self) -> RuntimeConfigErrorKind { + self.kind + } + + #[must_use] + pub const fn code(&self) -> &'static str { + self.kind.code() + } + + /// The runtime configuration file, when the refusal came from one. + #[must_use] + pub fn file(&self) -> Option<&Path> { + self.file.as_deref() + } + + /// The dotted field the refusal concerns; `/` for the whole document. + #[must_use] + pub fn field(&self) -> &str { + &self.field + } + + /// The refusal without the file prefix. + #[must_use] + pub fn message(&self) -> &str { + &self.message + } + + fn render(&self) -> String { + match &self.file { + Some(file) => format!("{}: {}", file.display(), self.message), + None => self.message.clone(), + } + } +} + +/// A loaded runtime configuration and the digest of what the runtime runs. +#[derive(Clone, Debug)] +pub struct LoadedRuntimeConfig { + pub config: T, + /// `sha256:` label over the RFC 8785 canonical JSON of the document after + /// substitution: two files that differ only in comments, layout or the + /// spelling of an environment expression that resolved to the same value + /// carry the same digest. + pub effective_digest: String, +} + +/// Loads one product's runtime configuration under the shared rules. +#[derive(Clone, Debug)] +pub struct RuntimeConfigLoader { + envelope: RuntimeEnvelope, + removed_keys: &'static [RemovedKey], + max_bytes: u64, + trusted_ownership: bool, +} + +impl RuntimeConfigLoader { + #[must_use] + pub const fn new(envelope: RuntimeEnvelope) -> Self { + Self { + envelope, + removed_keys: &[], + max_bytes: DEFAULT_MAX_RUNTIME_CONFIG_BYTES, + trusted_ownership: false, + } + } + + /// Refuse each of these keys with a diagnostic naming its replacement. + #[must_use] + pub const fn removed_keys(mut self, removed_keys: &'static [RemovedKey]) -> Self { + self.removed_keys = removed_keys; + self + } + + /// Lower or raise the size cap. + #[must_use] + pub const fn max_bytes(mut self, max_bytes: u64) -> Self { + self.max_bytes = max_bytes; + self + } + + /// Additionally require every directory above the file to be owned by root + /// or the effective user and not writable by group or others (a root-owned + /// sticky directory excepted), and the file itself to be owned by root or + /// the effective user and not writable by group or others. Platforms + /// without Unix ownership refuse every file under this rule. + #[must_use] + pub const fn require_trusted_ownership(mut self) -> Self { + self.trusted_ownership = true; + self + } + + #[must_use] + pub const fn envelope(&self) -> RuntimeEnvelope { + self.envelope + } + + /// Load `path`, substituting environment expressions from the process + /// environment. + pub fn load( + &self, + path: &Path, + ) -> Result, RuntimeConfigError> { + self.load_with(path, |name| std::env::var(name).ok()) + } + + /// Load `path`, substituting environment expressions from `lookup`. + pub fn load_with( + &self, + path: &Path, + lookup: impl Fn(&str) -> Option, + ) -> Result, RuntimeConfigError> { + let bytes = self.read(path).map_err(|error| error.in_file(path))?; + let text = std::str::from_utf8(&bytes).map_err(|_| { + RuntimeConfigError::new( + RuntimeConfigErrorKind::Encoding, + "/", + "the runtime configuration is not UTF-8 text", + ) + .in_file(path) + })?; + self.parse_str(text, lookup) + .map_err(|error| error.in_file(path)) + } + + /// Apply every rule after the file read to `text`. Used by authoring tools + /// that check an unsaved buffer, and by tests. + pub fn parse_str( + &self, + text: &str, + lookup: impl Fn(&str) -> Option, + ) -> Result, RuntimeConfigError> { + let mut document = parse_document(text)?; + self.reject_removed_keys(&document)?; + self.check_envelope(&document)?; + substitute_environment(&mut document, &lookup)?; + let canonical = + registry_platform_canonical_json::canonicalize_json(&document).map_err(|_| { + RuntimeConfigError::new( + RuntimeConfigErrorKind::InvalidValue, + "/", + "the runtime configuration holds a value that has no canonical JSON form", + ) + })?; + let effective_digest = sha256_uri(&canonical); + let config = serde_path_to_error::deserialize(document).map_err(|error| { + let field = error.path().to_string(); + let field = if field == "." { "/".to_owned() } else { field }; + let reason = redact_refused_values(&error.into_inner().to_string()); + RuntimeConfigError::new( + RuntimeConfigErrorKind::InvalidValue, + field.clone(), + format!("{field} is invalid: {reason}"), + ) + })?; + Ok(LoadedRuntimeConfig { + config, + effective_digest, + }) + } + + fn read(&self, path: &Path) -> Result, RuntimeConfigError> { + validate_absolute_lexical_path(path)?; + reject_symlink_components(path)?; + if self.trusted_ownership { + require_trusted_ownership(path)?; + } + read_bounded(path, self.max_bytes) + } + + fn reject_removed_keys(&self, document: &Value) -> Result<(), RuntimeConfigError> { + for removed in self.removed_keys { + let segments = removed.path.split('.').collect::>(); + if let Some(found) = find_path(document, &segments, &mut Vec::new()) { + return Err(RuntimeConfigError::new( + RuntimeConfigErrorKind::RemovedKey, + found.clone(), + format!("{found} is no longer accepted; {}", removed.replacement), + )); + } + } + Ok(()) + } + + fn check_envelope(&self, document: &Value) -> Result<(), RuntimeConfigError> { + for (field, expected) in [ + ("apiVersion", self.envelope.api_version), + ("kind", self.envelope.kind), + ] { + if document.get(field).and_then(Value::as_str) != Some(expected) { + return Err(RuntimeConfigError::new( + RuntimeConfigErrorKind::Envelope, + field, + format!("{field} must be exactly {expected}"), + )); + } + } + Ok(()) + } +} + +/// Whether `text` holds an environment expression the runtime loader would +/// substitute: `${NAME}`, `${NAME:-...}` or `${NAME:?...}` with `NAME` a valid +/// environment variable name. +#[must_use] +pub fn contains_environment_expression(text: &str) -> bool { + let mut rest = text; + while let Some(start) = rest.find("${") { + let after = &rest[start + 2..]; + let name_end = after + .find(|character: char| character != '_' && !character.is_ascii_alphanumeric()) + .unwrap_or(after.len()); + let name = &after[..name_end]; + let tail = &after[name_end..]; + if crate::valid_env_key(name) + && (tail.starts_with('}') || tail.starts_with(":-") || tail.starts_with(":?")) + { + return true; + } + rest = after; + } + false +} + +/// Refuse an authored project file that holds an environment expression in a +/// key or a string value. +/// +/// Environment substitution applies to `runtime.yaml` only. An authored file +/// is reviewed and packaged as written, so an expression in it would either be +/// taken literally or make the reviewed text differ from what runs. Text this +/// check cannot read as YAML is refused, so a file never passes unchecked +/// because a product's parser accepts what this reader does not. +/// +/// There is no escape for a literal `${NAME}` in an authored file. +pub fn reject_environment_expressions_in_authored_yaml( + text: &str, +) -> Result<(), RuntimeConfigError> { + let document = serde_norway::from_str::(text).map_err(|error| { + RuntimeConfigError::new( + RuntimeConfigErrorKind::AuthoredSyntax, + "/", + format!( + "the authored file is not valid YAML: {}", + redact_refused_values(&error.to_string()) + ), + ) + })?; + let mut path = Vec::new(); + match find_authored_expression(&document, &mut path) { + Some(field) => Err(RuntimeConfigError::new( + RuntimeConfigErrorKind::AuthoredExpression, + field.clone(), + format!( + "{field} holds an environment expression; ${{...}} substitution applies to \ + runtime.yaml only, so write the value in the authored file directly" + ), + )), + None => Ok(()), + } +} + +fn find_authored_expression(value: &serde_norway::Value, path: &mut Vec) -> Option { + match value { + serde_norway::Value::String(text) if contains_environment_expression(text) => { + Some(dotted(path)) + } + serde_norway::Value::Sequence(items) => { + items.iter().enumerate().find_map(|(index, item)| { + path.push(index.to_string()); + let found = find_authored_expression(item, path); + path.pop(); + found + }) + } + serde_norway::Value::Mapping(mapping) => mapping.iter().find_map(|(key, item)| { + let name = match key { + serde_norway::Value::String(name) => name.clone(), + _ => "?".to_owned(), + }; + path.push(name.clone()); + let found = if contains_environment_expression(&name) { + Some(dotted(path)) + } else { + find_authored_expression(item, path) + }; + path.pop(); + found + }), + serde_norway::Value::Tagged(tagged) => find_authored_expression(&tagged.value, path), + _ => None, + } +} + +fn dotted(path: &[String]) -> String { + if path.is_empty() { + "/".to_owned() + } else { + path.join(".") + } +} + +fn parse_document(text: &str) -> Result { + let document = serde_norway::from_str::(text).map_err(|error| { + RuntimeConfigError::new( + RuntimeConfigErrorKind::Syntax, + "/", + format!( + "the runtime configuration is not valid YAML: {}", + redact_refused_values(&error.to_string()) + ), + ) + })?; + let document = to_json(document, &mut Vec::new())?; + if !document.is_object() { + return Err(RuntimeConfigError::new( + RuntimeConfigErrorKind::Syntax, + "/", + "the runtime configuration must be a YAML mapping", + )); + } + Ok(document) +} + +fn to_json( + value: serde_norway::Value, + path: &mut Vec, +) -> Result { + Ok(match value { + serde_norway::Value::Null => Value::Null, + serde_norway::Value::Bool(value) => Value::Bool(value), + serde_norway::Value::Number(number) => { + if let Some(value) = number.as_u64() { + Value::Number(value.into()) + } else if let Some(value) = number.as_i64() { + Value::Number(value.into()) + } else { + let field = dotted(path); + number + .as_f64() + .and_then(Number::from_f64) + .map(Value::Number) + .ok_or_else(|| { + RuntimeConfigError::new( + RuntimeConfigErrorKind::Syntax, + field.clone(), + format!("{field} is not a finite number"), + ) + })? + } + } + serde_norway::Value::String(value) => Value::String(value), + serde_norway::Value::Sequence(items) => { + let mut converted = Vec::with_capacity(items.len()); + for (index, item) in items.into_iter().enumerate() { + path.push(index.to_string()); + converted.push(to_json(item, path)?); + path.pop(); + } + Value::Array(converted) + } + serde_norway::Value::Mapping(mapping) => { + let mut converted = Map::new(); + for (key, item) in mapping { + let serde_norway::Value::String(key) = key else { + let field = dotted(path); + return Err(RuntimeConfigError::new( + RuntimeConfigErrorKind::Syntax, + field.clone(), + format!("{field} has a key that is not a string"), + )); + }; + path.push(key.clone()); + let item = to_json(item, path)?; + path.pop(); + converted.insert(key, item); + } + Value::Object(converted) + } + serde_norway::Value::Tagged(_) => { + let field = dotted(path); + return Err(RuntimeConfigError::new( + RuntimeConfigErrorKind::Syntax, + field.clone(), + format!("{field} carries a YAML tag, which a runtime configuration does not use"), + )); + } + }) +} + +fn find_path(value: &Value, segments: &[&str], found: &mut Vec) -> Option { + let Some((segment, rest)) = segments.split_first() else { + return Some(found.join(".")); + }; + let children: Vec<(String, &Value)> = match value { + Value::Object(map) if *segment == "*" => { + map.iter().map(|(key, item)| (key.clone(), item)).collect() + } + Value::Object(map) => map + .get(*segment) + .map(|item| vec![((*segment).to_owned(), item)]) + .unwrap_or_default(), + Value::Array(items) if *segment == "*" => items + .iter() + .enumerate() + .map(|(index, item)| (index.to_string(), item)) + .collect(), + _ => Vec::new(), + }; + for (key, child) in children { + found.push(key); + if let Some(path) = find_path(child, rest, found) { + return Some(path); + } + found.pop(); + } + None +} + +/// Whether a field of this name holds secret references, where substitution +/// is refused. +fn is_reference_field(name: &str) -> bool { + name.ends_with("Ref") || name.ends_with("Refs") +} + +/// Whether a field of this name configures the secret providers. A provider +/// setting chooses which secret a reference resolves to, so substitution is +/// refused under it as it is in a reference. +fn is_secret_provider_field(name: &str) -> bool { + name == "secretProviders" +} + +/// Why substitution is refused at a field. +#[derive(Clone, Copy)] +enum Literal { + Reference, + SecretProvider, +} + +fn substitute_environment( + document: &mut Value, + lookup: &impl Fn(&str) -> Option, +) -> Result<(), RuntimeConfigError> { + substitute_value(document, &mut Vec::new(), None, lookup) +} + +fn substitute_value( + value: &mut Value, + path: &mut Vec, + literal: Option, + lookup: &impl Fn(&str) -> Option, +) -> Result<(), RuntimeConfigError> { + match value { + Value::String(text) if text.contains("${") => { + let field = dotted(path); + let message = match literal { + None => { + *text = substitute_string(text, &field, lookup)?; + return Ok(()); + } + Some(Literal::Reference) => format!( + "{field} is a secret reference and does not take ${{...}} substitution; \ + write secret:env/NAME or secret:file/name instead" + ), + Some(Literal::SecretProvider) => format!( + "{field} configures a secret provider and does not take ${{...}} \ + substitution; write the setting in runtime.yaml directly" + ), + }; + return Err(RuntimeConfigError::new( + RuntimeConfigErrorKind::SubstitutionInReference, + field, + message, + )); + } + Value::Array(items) => { + for (index, item) in items.iter_mut().enumerate() { + path.push(index.to_string()); + substitute_value(item, path, literal, lookup)?; + path.pop(); + } + } + Value::Object(map) => { + for (key, item) in map.iter_mut() { + path.push(key.clone()); + let literal = literal + .or_else(|| is_reference_field(key).then_some(Literal::Reference)) + .or_else(|| is_secret_provider_field(key).then_some(Literal::SecretProvider)); + substitute_value(item, path, literal, lookup)?; + path.pop(); + } + } + _ => {} + } + Ok(()) +} + +fn substitute_string( + text: &str, + field: &str, + lookup: &impl Fn(&str) -> Option, +) -> Result { + let mut substituted = String::with_capacity(text.len()); + let mut rest = text; + while let Some(start) = rest.find("${") { + substituted.push_str(&rest[..start]); + let after_start = &rest[start + 2..]; + let Some(end) = after_start.find('}') else { + return Err(RuntimeConfigError::new( + RuntimeConfigErrorKind::Substitution, + field, + format!("{field} has an unterminated ${{...}} expression"), + )); + }; + let (name, value) = + resolve_config_env_expression(&after_start[..end], lookup).map_err(|error| { + RuntimeConfigError::new( + RuntimeConfigErrorKind::Substitution, + field, + format!("{field} could not be substituted: {error}"), + ) + })?; + if value.contains('\0') { + return Err(RuntimeConfigError::new( + RuntimeConfigErrorKind::Substitution, + field, + format!("{field} could not be substituted: environment variable {name} holds a NUL byte"), + )); + } + substituted.push_str(&value); + rest = &after_start[end + 1..]; + } + substituted.push_str(rest); + Ok(substituted) +} + +fn validate_absolute_lexical_path(path: &Path) -> Result<(), RuntimeConfigError> { + let normal = !path.as_os_str().is_empty() + && path.is_absolute() + && path.as_os_str().len() <= MAX_RUNTIME_CONFIG_PATH_BYTES + && !has_current_directory_component(path) + && path.components().all(|component| { + matches!( + component, + Component::Prefix(_) | Component::RootDir | Component::Normal(_) + ) + }); + if normal { + Ok(()) + } else { + Err(RuntimeConfigError::new( + RuntimeConfigErrorKind::Path, + "/", + "the runtime configuration path must be absolute, without . or .. components", + )) + } +} + +/// `Path::components` drops interior `.` components, so they are found in the +/// raw path instead. +fn has_current_directory_component(path: &Path) -> bool { + path.as_os_str() + .as_encoded_bytes() + .split(|byte| std::path::is_separator(char::from(*byte))) + .any(|component| component == b".") +} + +fn unsafe_file(message: &str) -> RuntimeConfigError { + RuntimeConfigError::new(RuntimeConfigErrorKind::UnsafeFile, "/", message) +} + +fn unavailable() -> RuntimeConfigError { + RuntimeConfigError::new( + RuntimeConfigErrorKind::Unavailable, + "/", + "the runtime configuration could not be read", + ) +} + +fn out_of_bounds(maximum: u64) -> RuntimeConfigError { + RuntimeConfigError::new( + RuntimeConfigErrorKind::Bounds, + "/", + format!("the runtime configuration must be between 1 and {maximum} bytes"), + ) +} + +/// Refuse a path any of whose components is a symbolic link. +fn reject_symlink_components(path: &Path) -> Result<(), RuntimeConfigError> { + let mut checked = PathBuf::new(); + for component in path.components() { + checked.push(component.as_os_str()); + if matches!(component, Component::RootDir | Component::Prefix(_)) { + continue; + } + match fs::symlink_metadata(&checked) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(unsafe_file( + "the runtime configuration path must not pass through a symbolic link", + )) + } + Ok(_) => {} + Err(_) => return Err(unavailable()), + } + } + Ok(()) +} + +#[cfg(unix)] +fn require_trusted_ownership(path: &Path) -> Result<(), RuntimeConfigError> { + use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; + + let effective_user = rustix::process::geteuid().as_raw(); + let count = path.components().count(); + let mut current = PathBuf::new(); + for (index, component) in path.components().enumerate() { + current.push(component.as_os_str()); + let metadata = fs::symlink_metadata(¤t).map_err(|_| unavailable())?; + let last = index + 1 == count; + let owner = metadata.uid(); + let mode = metadata.permissions().mode(); + let trusted_owner = owner == 0 || owner == effective_user; + let not_writable_by_others = mode & 0o022 == 0; + let root_sticky = !last && owner == 0 && mode & 0o1000 != 0; + if !trusted_owner || !(not_writable_by_others || root_sticky) { + return Err(unsafe_file( + "the runtime configuration and every directory above it must be owned by root \ + or the runtime user and not writable by group or others", + )); + } + } + Ok(()) +} + +#[cfg(not(unix))] +fn require_trusted_ownership(_path: &Path) -> Result<(), RuntimeConfigError> { + Err(unsafe_file( + "trusted ownership of the runtime configuration cannot be checked on this platform", + )) +} + +fn read_bounded(path: &Path, maximum: u64) -> Result, RuntimeConfigError> { + let not_regular = || unsafe_file("the runtime configuration must be a regular file"); + let changed = || unsafe_file("the runtime configuration changed while it was read"); + let scanned = fs::symlink_metadata(path).map_err(|_| unavailable())?; + if scanned.file_type().is_symlink() || !scanned.is_file() { + return Err(not_regular()); + } + if scanned.len() == 0 || scanned.len() > maximum { + return Err(out_of_bounds(maximum)); + } + let file = open_no_follow(path)?; + let opened = file.metadata().map_err(|_| unavailable())?; + let current = fs::symlink_metadata(path).map_err(|_| unavailable())?; + if current.file_type().is_symlink() || !opened.is_file() { + return Err(not_regular()); + } + if !same_file(&scanned, &opened) || !same_file(&opened, ¤t) { + return Err(changed()); + } + let capacity = usize::try_from(opened.len()).map_err(|_| out_of_bounds(maximum))?; + let mut bytes = Vec::new(); + bytes + .try_reserve(capacity) + .map_err(|_| out_of_bounds(maximum))?; + let mut reader = file.take(maximum + 1); + reader.read_to_end(&mut bytes).map_err(|_| unavailable())?; + let after = reader.get_ref().metadata().map_err(|_| unavailable())?; + if bytes.is_empty() || bytes.len() as u64 > maximum { + return Err(out_of_bounds(maximum)); + } + if !same_file(&opened, &after) || bytes.len() as u64 != after.len() { + return Err(changed()); + } + Ok(bytes) +} + +fn open_no_follow(path: &Path) -> Result { + let mut options = fs::OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + + options.custom_flags( + (rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::CLOEXEC).bits() as i32, + ); + } + options.open(path).map_err(|_| { + fs::symlink_metadata(path).map_or_else( + |_| unavailable(), + |metadata| { + if metadata.file_type().is_symlink() || !metadata.is_file() { + unsafe_file("the runtime configuration must be a regular file") + } else { + unavailable() + } + }, + ) + }) +} + +#[cfg(unix)] +fn same_file(left: &fs::Metadata, right: &fs::Metadata) -> bool { + use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; + + left.dev() == right.dev() + && left.ino() == right.ino() + && left.len() == right.len() + && left.permissions().mode() == right.permissions().mode() + && left.mtime() == right.mtime() + && left.mtime_nsec() == right.mtime_nsec() + && left.ctime() == right.ctime() + && left.ctime_nsec() == right.ctime_nsec() +} + +#[cfg(not(unix))] +fn same_file(left: &fs::Metadata, right: &fs::Metadata) -> bool { + left.len() == right.len() + && left.permissions().readonly() == right.permissions().readonly() + && left.modified().ok() == right.modified().ok() + && left.created().ok() == right.created().ok() +} + +#[cfg(test)] +#[path = "loader_tests.rs"] +mod tests; diff --git a/crates/registry-platform-config/src/loader_tests.rs b/crates/registry-platform-config/src/loader_tests.rs new file mode 100644 index 0000000000..a06d24e85c --- /dev/null +++ b/crates/registry-platform-config/src/loader_tests.rs @@ -0,0 +1,595 @@ +use super::*; +use serde::Deserialize; +use std::collections::BTreeMap; + +const ENVELOPE: RuntimeEnvelope = RuntimeEnvelope { + api_version: "registry.registrystack.org/example-runtime/v1alpha1", + kind: "ExampleRuntimeConfig", +}; + +const REMOVED: &[RemovedKey] = &[ + RemovedKey { + path: "server.bind", + replacement: "use listener.bind", + }, + RemovedKey { + path: "sources.*.file", + replacement: "use sources..path", + }, +]; + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Example { + api_version: String, + kind: String, + #[serde(default)] + name: Option, + #[serde(default)] + port: Option, + #[serde(default)] + count: Option, + #[serde(default)] + flag: Option, + #[serde(default)] + list: Vec, + #[serde(default)] + audit: Option, + #[serde(default)] + sources: BTreeMap>, + #[serde(default)] + mode: Option, + #[serde(default)] + secret_providers: Option>>, +} + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +enum Mode { + Strict, + Relaxed, +} + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct Audit { + #[serde(default)] + path: Option, + #[serde(default)] + hash_key_ref: Option, + #[serde(default)] + allowed_key_refs: Vec, +} + +fn loader() -> RuntimeConfigLoader { + RuntimeConfigLoader::new(ENVELOPE).removed_keys(REMOVED) +} + +fn header() -> String { + format!( + "apiVersion: {}\nkind: {}\n", + ENVELOPE.api_version, ENVELOPE.kind + ) +} + +fn env(pairs: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { + move |name| { + pairs + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| (*value).to_owned()) + } +} + +fn parse(text: &str) -> Result, RuntimeConfigError> { + loader().parse_str(text, env(&[("NAME", "north"), ("EMPTY", "")])) +} + +#[test] +fn substitution_applies_to_string_values_after_parsing() { + let loaded = parse(&format!( + "{}name: \"${{NAME}}-counter\"\nlist: [\"${{NAME}}\", plain]\n", + header() + )) + .expect("substitutes"); + assert_eq!(loaded.config.name.as_deref(), Some("north-counter")); + assert_eq!(loaded.config.list, ["north", "plain"]); +} + +#[test] +fn substitution_leaves_comments_and_keys_untouched() { + // A comment naming an unset variable is not substituted, so it cannot + // refuse the document; a key is never substituted either, so the typed + // configuration refuses it as an unknown field rather than resolving it. + let loaded = parse(&format!("{}# ${{UNSET_IN_COMMENT}}\nname: x\n", header())) + .expect("comment is not substituted"); + assert_eq!(loaded.config.name.as_deref(), Some("x")); + + let error = parse(&format!("{}\"${{NAME}}\": x\n", header())).expect_err("key stays literal"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::InvalidValue); + assert!(error.message().contains("${NAME}"), "{error}"); +} + +#[test] +fn substituted_values_stay_strings_even_when_they_look_like_yaml() { + let loaded = loader() + .parse_str::( + &format!( + "{}port: ${{PORT}}\nflag: ${{FLAG}}\nname: ${{DOC}}\n", + header() + ), + env(&[ + ("PORT", "8080"), + ("FLAG", "true"), + ("DOC", "{a: [1, 2]}\n- x"), + ]), + ) + .expect("values stay strings"); + assert_eq!(loaded.config.port.as_deref(), Some("8080")); + assert_eq!(loaded.config.flag.as_deref(), Some("true")); + assert_eq!(loaded.config.name.as_deref(), Some("{a: [1, 2]}\n- x")); + + let error = loader() + .parse_str::( + &format!("{}count: ${{COUNT}}\n", header()), + env(&[("COUNT", "3")]), + ) + .expect_err("a substituted number is still a string"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::InvalidValue); + assert_eq!(error.field(), "count"); +} + +#[test] +fn substitution_supports_default_and_required_message_forms() { + let loaded = parse(&format!( + "{}name: \"${{MISSING:-fallback}}\"\nflag: \"${{EMPTY:-}}\"\n", + header() + )) + .expect("defaults apply"); + assert_eq!(loaded.config.name.as_deref(), Some("fallback")); + assert_eq!(loaded.config.flag.as_deref(), Some("")); + + let error = parse(&format!( + "{}name: \"${{MISSING:?set MISSING to the counter name}}\"\n", + header() + )) + .expect_err("required message form refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::Substitution); + assert_eq!(error.field(), "name"); + // The operator-authored message is configuration too, so the refusal + // names the variable and withholds the message. + assert!(error.message().contains("MISSING"), "{error}"); + assert!(error.message().contains("withheld"), "{error}"); + assert!(!error.message().contains("counter name"), "{error}"); +} + +#[test] +fn substitution_refuses_an_unset_or_empty_variable_without_default() { + for text in ["name: ${MISSING}\n", "name: ${EMPTY}\n"] { + let error = parse(&format!("{}{text}", header())).expect_err("unset refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::Substitution); + assert_eq!(error.code(), "runtime_config.substitution"); + assert_eq!(error.field(), "name"); + } +} + +#[test] +fn substitution_refuses_malformed_expressions() { + for text in [ + "name: \"${NAME\"\n", + "name: \"${1BAD}\"\n", + "name: \"${}\"\n", + ] { + let error = parse(&format!("{}{text}", header())).expect_err("malformed refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::Substitution, "{text}"); + } +} + +#[test] +fn substitution_is_single_pass() { + let loaded = loader() + .parse_str::( + &format!("{}name: ${{OUTER}}\n", header()), + env(&[("OUTER", "${INNER}"), ("INNER", "leak")]), + ) + .expect("substitutes once"); + assert_eq!(loaded.config.name.as_deref(), Some("${INNER}")); +} + +#[test] +fn substitution_refuses_a_nul_byte() { + let error = loader() + .parse_str::( + &format!("{}name: ${{NUL}}\n", header()), + env(&[("NUL", "a\0b")]), + ) + .expect_err("NUL refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::Substitution); +} + +#[test] +fn substitution_is_refused_in_every_reference_field() { + for (text, field) in [ + ("audit:\n hashKeyRef: ${NAME}\n", "audit.hashKeyRef"), + ( + "audit:\n hashKeyRef: \"secret:env/${NAME}\"\n", + "audit.hashKeyRef", + ), + ( + "audit:\n allowedKeyRefs: [\"secret:file/a\", \"${NAME}\"]\n", + "audit.allowedKeyRefs.1", + ), + // A reference field that the variable would resolve fine for is still + // refused: the rule is about the field, not the value. + ( + "audit:\n hashKeyRef: \"${MISSING:-secret:env/X}\"\n", + "audit.hashKeyRef", + ), + ] { + let error = parse(&format!("{}{text}", header())).expect_err("reference refuses"); + assert_eq!( + error.kind(), + RuntimeConfigErrorKind::SubstitutionInReference, + "{text}" + ); + assert_eq!(error.code(), "runtime_config.substitution_in_reference"); + assert_eq!(error.field(), field); + assert!(error.message().contains("secret:env/NAME"), "{error}"); + } + + let loaded = parse(&format!( + "{}audit:\n path: \"/var/${{NAME}}/audit\"\n hashKeyRef: secret:file/audit-key\n", + header() + )) + .expect("a sibling that is not a reference substitutes"); + assert_eq!( + loaded.config.audit.and_then(|audit| audit.path).as_deref(), + Some("/var/north/audit") + ); +} + +#[test] +fn removed_keys_are_refused_with_their_replacement_named() { + let error = parse(&format!("{}server:\n bind: 127.0.0.1:1\n", header())) + .expect_err("removed key refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::RemovedKey); + assert_eq!(error.code(), "runtime_config.removed_key"); + assert_eq!(error.field(), "server.bind"); + assert_eq!( + error.message(), + "server.bind is no longer accepted; use listener.bind" + ); + + let error = parse(&format!( + "{}sources:\n people:\n file: /data/people.sqlite\n", + header() + )) + .expect_err("wildcard removed key refuses"); + assert_eq!(error.field(), "sources.people.file"); + assert!(error.message().contains("sources..path")); +} + +#[test] +fn removed_keys_are_reported_before_the_envelope() { + let error = loader() + .parse_str::( + "apiVersion: old/v1\nkind: Old\nserver:\n bind: x\n", + env(&[]), + ) + .expect_err("removed key first"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::RemovedKey); +} + +#[test] +fn the_envelope_must_be_literal() { + for text in [ + "kind: ExampleRuntimeConfig\n".to_owned(), + format!("apiVersion: {}\nkind: Other\n", ENVELOPE.api_version), + format!("apiVersion: ${{API}}\nkind: {}\n", ENVELOPE.kind), + ] { + let error = loader() + .parse_str::(&text, env(&[("API", ENVELOPE.api_version)])) + .expect_err("envelope refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::Envelope, "{text}"); + } +} + +#[test] +fn the_document_must_be_one_mapping_with_string_keys_and_no_tags() { + for text in [ + format!("{}name: [unterminated\n", header()), + format!("{}---\n{}", header(), header()), + "- a\n- b\n".to_owned(), + format!("{}1: x\n", header()), + format!("{}name: !custom x\n", header()), + format!("{}name: a\nname: b\n", header()), + ] { + let error = parse(&text).expect_err("syntax refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::Syntax, "{text}"); + } +} + +#[test] +fn a_typed_refusal_names_the_field_without_the_value() { + let error = parse(&format!( + "{}count: DO_NOT_DISCLOSE_RUNTIME_VALUE\n", + header() + )) + .expect_err("type refuses"); + assert_eq!(error.field(), "count"); + assert!(!error.to_string().contains("DO_NOT_DISCLOSE"), "{error}"); +} + +#[test] +fn a_substitution_refusal_never_echoes_a_value() { + let error = loader() + .parse_str::( + &format!("{}count: ${{SECRETISH}}\n", header()), + env(&[("SECRETISH", "DO_NOT_DISCLOSE_RUNTIME_VALUE")]), + ) + .expect_err("typed refusal after substitution"); + assert!(!error.to_string().contains("DO_NOT_DISCLOSE"), "{error}"); +} + +#[test] +fn no_refusal_echoes_an_invalid_name_or_an_unknown_variant() { + let error = parse(&format!("{}name: \"${{DO_NOT DISCLOSE}}\"\n", header())) + .expect_err("invalid name refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::Substitution); + assert!(!error.to_string().contains("DISCLOSE"), "{error}"); + + let error = parse(&format!("{}mode: DO_NOT_DISCLOSE\n", header())) + .expect_err("unknown variant refuses"); + assert_eq!(error.field(), "mode"); + assert!(error.message().contains("unknown variant"), "{error}"); + assert!(error.message().contains("strict"), "{error}"); + assert!(!error.to_string().contains("DISCLOSE"), "{error}"); + + let error = loader() + .parse_str::( + &format!("{}mode: ${{MODE}}\n", header()), + env(&[("MODE", "DO_NOT_DISCLOSE")]), + ) + .expect_err("a substituted unknown variant refuses"); + assert!(!error.to_string().contains("DISCLOSE"), "{error}"); +} + +#[test] +fn substitution_is_refused_under_secret_providers() { + // A provider setting chooses which secret a reference resolves to, so the + // environment may not redirect it any more than it may rewrite a reference. + for (text, field) in [ + ( + "secretProviders:\n file:\n root: \"${NAME}\"\n", + "secretProviders.file.root", + ), + ( + "secretProviders:\n file:\n root: \"/run/${NAME}/secrets\"\n", + "secretProviders.file.root", + ), + ( + "secretProviders:\n env:\n prefix: \"${MISSING:-X}\"\n", + "secretProviders.env.prefix", + ), + ] { + let error = parse(&format!("{}{text}", header())).expect_err("provider refuses"); + assert_eq!( + error.kind(), + RuntimeConfigErrorKind::SubstitutionInReference, + "{text}" + ); + assert_eq!(error.field(), field); + assert!(error.message().contains("secret provider"), "{error}"); + } + let loaded = parse(&format!( + "{}secretProviders:\n file:\n root: /run/secrets\n", + header() + )) + .expect("a literal provider setting loads"); + assert!(loaded.config.secret_providers.is_some()); +} + +#[test] +fn the_effective_digest_ignores_layout_and_comments() { + let compact = parse(&format!("{}name: north\n", header())).expect("loads"); + let spaced = parse(&format!( + "# operator note\n{}\nname: \"${{NAME}}\"\n", + header() + )) + .expect("loads"); + assert_eq!(compact.effective_digest, spaced.effective_digest); + assert!(compact.effective_digest.starts_with("sha256:")); + let other = parse(&format!("{}name: south\n", header())).expect("loads"); + assert_ne!(compact.effective_digest, other.effective_digest); +} + +#[test] +fn environment_expressions_are_detected_in_authored_yaml() { + assert!(contains_environment_expression("${A}")); + assert!(contains_environment_expression("x ${A_1:-y} z")); + assert!(contains_environment_expression("${A:?m}")); + assert!(!contains_environment_expression("$A")); + assert!(!contains_environment_expression("${1A}")); + assert!(!contains_environment_expression("${a b}")); + assert!(!contains_environment_expression("cost: $5 {x}")); + + reject_environment_expressions_in_authored_yaml("a: plain\n# ${A}\n").expect("comment ok"); + let error = reject_environment_expressions_in_authored_yaml("a:\n b: [x, \"${HOST}\"]\n") + .expect_err("value refuses"); + assert_eq!(error.field(), "a.b.1"); + assert!(error.message().contains("runtime.yaml only")); + let error = reject_environment_expressions_in_authored_yaml("\"${HOST}\": x\n") + .expect_err("key refuses"); + assert_eq!(error.field(), "${HOST}"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::AuthoredExpression); + assert_eq!(error.code(), "authored_config.environment_expression"); +} + +#[test] +fn the_authored_check_fails_closed_on_text_that_does_not_parse() { + // An authored file the check cannot read is refused, never waved through + // to a parser that might accept what this reader could not. + for text in [ + "a: [unterminated ${HOST}\n", + "a: b\n c: d\n", + "a: 1\na: 2\n", + ] { + let error = + reject_environment_expressions_in_authored_yaml(text).expect_err("unparsed refuses"); + assert_eq!( + error.kind(), + RuntimeConfigErrorKind::AuthoredSyntax, + "{text}" + ); + assert_eq!(error.code(), "authored_config.syntax"); + assert!(!error.message().contains("HOST"), "{error}"); + } +} + +mod files { + use super::*; + + fn directory() -> (tempfile::TempDir, PathBuf) { + let directory = tempfile::tempdir().expect("tempdir"); + let root = directory.path().canonicalize().expect("canonical tempdir"); + (directory, root) + } + + #[test] + fn a_file_loads_and_errors_name_the_file() { + let (_guard, root) = directory(); + let path = root.join("runtime.yaml"); + std::fs::write(&path, format!("{}name: ${{MISSING}}\n", header())).unwrap(); + let error = loader() + .load_with::(&path, env(&[])) + .expect_err("unset refuses"); + assert_eq!(error.file(), Some(path.as_path())); + assert!(error.to_string().starts_with(&path.display().to_string())); + + std::fs::write(&path, format!("{}name: x\n", header())).unwrap(); + let loaded = loader() + .load_with::(&path, env(&[])) + .expect("loads"); + assert_eq!(loaded.config.name.as_deref(), Some("x")); + } + + #[test] + fn a_relative_or_unnormal_path_is_refused() { + let (_guard, root) = directory(); + for path in [ + PathBuf::from("runtime.yaml"), + root.join(".").join("runtime.yaml"), + root.join("..").join("runtime.yaml"), + ] { + let error = loader() + .load_with::(&path, env(&[])) + .expect_err("path refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::Path, "{path:?}"); + } + } + + #[cfg(unix)] + #[test] + fn a_symbolic_link_anywhere_in_the_path_is_refused() { + let (_guard, root) = directory(); + let real = root.join("real"); + std::fs::create_dir(&real).unwrap(); + std::fs::write(real.join("runtime.yaml"), header()).unwrap(); + std::os::unix::fs::symlink(&real, root.join("linked-dir")).unwrap(); + std::os::unix::fs::symlink(real.join("runtime.yaml"), root.join("linked.yaml")).unwrap(); + for path in [ + root.join("linked-dir").join("runtime.yaml"), + root.join("linked.yaml"), + ] { + let error = loader() + .load_with::(&path, env(&[])) + .expect_err("symlink refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::UnsafeFile, "{path:?}"); + } + } + + #[test] + fn a_directory_missing_file_empty_file_or_oversized_file_is_refused() { + let (_guard, root) = directory(); + let error = loader() + .load_with::(&root, env(&[])) + .expect_err("directory refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::UnsafeFile); + + let error = loader() + .load_with::(&root.join("absent.yaml"), env(&[])) + .expect_err("absent refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::Unavailable); + + let empty = root.join("empty.yaml"); + std::fs::write(&empty, "").unwrap(); + let error = loader() + .load_with::(&empty, env(&[])) + .expect_err("empty refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::Bounds); + + let large = root.join("large.yaml"); + std::fs::write(&large, format!("{}name: {}\n", header(), "x".repeat(200))).unwrap(); + let error = loader() + .max_bytes(128) + .load_with::(&large, env(&[])) + .expect_err("oversized refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::Bounds); + assert_eq!(error.code(), "runtime_config.bounds"); + } + + #[test] + fn a_file_that_is_not_utf8_is_refused() { + let (_guard, root) = directory(); + let path = root.join("runtime.yaml"); + std::fs::write(&path, b"apiVersion: \xff\n").unwrap(); + let error = loader() + .load_with::(&path, env(&[])) + .expect_err("encoding refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::Encoding); + } + + #[cfg(unix)] + #[test] + fn trusted_ownership_refuses_a_file_writable_by_others() { + use std::os::unix::fs::PermissionsExt as _; + + let (_guard, root) = directory(); + let path = root.join("runtime.yaml"); + std::fs::write(&path, header()).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); + loader() + .require_trusted_ownership() + .load_with::(&path, env(&[])) + .expect("owner-only file loads"); + + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o666)).unwrap(); + let error = loader() + .require_trusted_ownership() + .load_with::(&path, env(&[])) + .expect_err("world-writable refuses"); + assert_eq!(error.kind(), RuntimeConfigErrorKind::UnsafeFile); + loader() + .load_with::(&path, env(&[])) + .expect("the ownership rule is opt-in"); + } +} + +#[test] +fn the_shared_removed_jwks_uri_key_names_jwks_source() { + let loader = RuntimeConfigLoader::new(ENVELOPE) + .removed_keys(std::slice::from_ref(&REMOVED_OIDC_JWKS_URI)); + let error = loader + .parse_str::( + &format!( + "{}authentication:\n oidc:\n jwksUri: https://issuer.example.test/jwks\n", + header() + ), + env(&[]), + ) + .unwrap_err(); + assert_eq!(error.kind(), RuntimeConfigErrorKind::RemovedKey); + assert_eq!(error.field(), "authentication.oidc.jwksUri"); + assert!(error.to_string().contains("authentication.oidc.jwksSource")); + assert!(!error.to_string().contains("issuer.example.test")); +} diff --git a/crates/registry-platform-config/src/package.rs b/crates/registry-platform-config/src/package.rs new file mode 100644 index 0000000000..b230c4b426 --- /dev/null +++ b/crates/registry-platform-config/src/package.rs @@ -0,0 +1,975 @@ +//! The one package format every Registry Stack runtime serves. +//! +//! A package is a directory a product's `package` command produces. Its root +//! holds `SHA256SUMS`, one line per file in the `sha256sum` text format +//! (`<64 lowercase hex digits>`), sorted by path, +//! and optionally `REVISION`, one free-text line the operator chose (for +//! example a source-control revision). `REVISION` is listed and hashed like +//! every other file; `SHA256SUMS` lists everything but itself. The package +//! digest is the `sha256:` label of the `SHA256SUMS` bytes, which is what +//! `package.expectedDigest` pins. +//! +//! Only file bytes are hashed. File modes, owners and timestamps are not: +//! copying tools, source control and image layers do not carry them the same +//! way on every platform, so hashing them would give one package several +//! digests. Line endings are not normalized either, because the digest names +//! the exact bytes a runtime reads. +//! +//! [`write_sum_file`] finishes a directory a product has populated; +//! [`verify_package`] recomputes every digest at startup and refuses a +//! changed, missing or extra file by name. Both refuse symbolic links and +//! special files anywhere in the package and bound what they read. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::fs; +use std::io::{Read as _, Write as _}; +use std::path::{Path, PathBuf}; + +use sha2::{Digest as _, Sha256}; + +use crate::blocks::{PackageConfig, PackageDigestMismatch}; +use crate::{hex_lower, sha256_uri}; + +/// The file listing every other file's SHA-256. +pub const SUM_FILE: &str = "SHA256SUMS"; +/// The optional file holding the package revision. +pub const REVISION_FILE: &str = "REVISION"; +/// The longest revision a package records, in bytes. +pub const MAX_REVISION_BYTES: usize = 256; + +const HEX_DIGITS: usize = 64; +const SEPARATOR: &str = " "; +const READ_CHUNK: usize = 64 * 1024; + +/// Bounds on what a package may hold. A product may narrow them. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct PackageLimits { + /// Files listed in `SHA256SUMS`, `REVISION` included. + pub max_files: usize, + /// Bytes of any one file. + pub max_file_bytes: u64, + /// Bytes of every listed file together. + pub max_total_bytes: u64, + /// Path components of any one file, its own name included. + pub max_depth: usize, + /// Bytes of any one relative path. + pub max_path_bytes: usize, +} + +impl Default for PackageLimits { + fn default() -> Self { + Self { + max_files: 4_096, + max_file_bytes: 64 * 1024 * 1024, + max_total_bytes: 256 * 1024 * 1024, + max_depth: 16, + max_path_bytes: 512, + } + } +} + +/// A package whose every file matched its `SHA256SUMS` line when it was read. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct VerifiedPackage { + digest: String, + revision: Option, + files: BTreeMap, +} + +impl VerifiedPackage { + /// The `sha256:` label of the `SHA256SUMS` bytes. + #[must_use] + pub fn digest(&self) -> &str { + &self.digest + } + + /// The recorded revision, when the package has one. + #[must_use] + pub fn revision(&self) -> Option<&str> { + self.revision.as_deref() + } + + /// Every listed path, in `SHA256SUMS` order. + pub fn files(&self) -> impl Iterator { + self.files.keys().map(String::as_str) + } + + /// The `sha256:` label a listed file had when it was verified. + #[must_use] + pub fn file_digest(&self, path: &str) -> Option { + self.files.get(path).map(|hex| format!("sha256:{hex}")) + } +} + +/// Whether `path` names a file the package format owns rather than the +/// product. Product loaders that enumerate a package skip these. +#[must_use] +pub fn is_envelope_file(path: &str) -> bool { + path == SUM_FILE || path == REVISION_FILE +} + +/// A revision is one printable line of at most [`MAX_REVISION_BYTES`] bytes +/// without leading or trailing whitespace. +pub fn check_revision(revision: &str) -> Result<(), &'static str> { + if revision.is_empty() || revision.len() > MAX_REVISION_BYTES { + return Err("a revision is 1 to 256 bytes"); + } + if revision.chars().any(|character| { + character.is_control() + || matches!( + character, + '\u{200e}'..='\u{200f}' | '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}' + ) + }) { + return Err( + "a revision is one line without control or bidirectional formatting characters", + ); + } + if revision.trim() != revision { + return Err("a revision has no leading or trailing whitespace"); + } + Ok(()) +} + +impl PackageConfig { + /// Verify the package at `package.root` and, when `package.expectedDigest` + /// is set, that its digest is the pinned one. `fix` is the command that + /// builds the package, named in every refusal. + pub fn verify_package( + &self, + limits: &PackageLimits, + fix: &str, + ) -> Result { + let package = verify_package(&self.root, limits, fix)?; + self.verify_digest(package.digest()) + .map_err(|mismatch| PackageError::new(&self.root, fix, mismatch.into()))?; + Ok(package) + } +} + +/// Finish a populated package directory: write `REVISION` when `revision` is +/// given, then `SHA256SUMS` over every file, and return the verified result. +/// +/// The directory must not hold either file already; a package is written once. +pub fn write_sum_file( + root: &Path, + revision: Option<&str>, + limits: &PackageLimits, + fix: &str, +) -> Result { + let refuse = |kind| PackageError::new(root, fix, kind); + check_root(root).map_err(refuse)?; + for reserved in [SUM_FILE, REVISION_FILE] { + if fs::symlink_metadata(root.join(reserved)).is_ok() { + return Err(refuse(PackageErrorKind::Reserved { + path: reserved.to_owned(), + })); + } + } + if let Some(revision) = revision { + check_revision(revision).map_err(|reason| refuse(PackageErrorKind::Revision { reason }))?; + } + // Walk before writing so a refused tree leaves the directory untouched. + let mut walked = walk(root, limits).map_err(refuse)?; + if let Some(empty) = walked.empty_directories.into_iter().next() { + return Err(refuse(PackageErrorKind::UnsafeEntry { + path: empty, + reason: "is an empty directory", + })); + } + if walked.files.is_empty() { + return Err(refuse(PackageErrorKind::Empty)); + } + if let Some(revision) = revision { + write_new(root, REVISION_FILE, format!("{revision}\n").as_bytes()).map_err(refuse)?; + walked.files.insert(REVISION_FILE.to_owned()); + } + check_file_count(walked.files.len(), limits).map_err(refuse)?; + let mut total = 0_u64; + let mut sums = String::new(); + for path in &walked.files { + let (hex, length) = hash_file(root, path, limits, None).map_err(refuse)?; + total = add_bytes(total, length, limits).map_err(refuse)?; + push_sum_line(&mut sums, &hex, path); + } + write_new(root, SUM_FILE, sums.as_bytes()).map_err(refuse)?; + verify_package(root, limits, fix) +} + +/// The digest a package of exactly `files` and `revision` would have, without +/// writing anything. `files` maps relative paths to their bytes and never +/// holds `SHA256SUMS` or `REVISION`; `root` names the project or output in +/// refusals. +pub fn plan_package( + root: &Path, + files: &BTreeMap>, + revision: Option<&str>, + limits: &PackageLimits, + fix: &str, +) -> Result { + let sums = plan_sum_file(files, revision, limits) + .map_err(|kind| PackageError::new(root, fix, kind))?; + Ok(sha256_uri(sums.as_bytes())) +} + +/// Write `files` and `revision` as a package into `output`, a directory that +/// must not exist yet, and return the verified result. A failed write removes +/// the directory it created. +pub fn write_package( + output: &Path, + files: &BTreeMap>, + revision: Option<&str>, + limits: &PackageLimits, + fix: &str, +) -> Result { + let refuse = |kind| PackageError::new(output, fix, kind); + plan_sum_file(files, revision, limits).map_err(refuse)?; + if fs::symlink_metadata(output).is_ok() { + return Err(refuse(PackageErrorKind::OutputExists)); + } + if let Some(parent) = output + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent).map_err(|_| { + refuse(PackageErrorKind::Io { + path: "..".to_owned(), + }) + })?; + } + fs::create_dir(output).map_err(|_| refuse(PackageErrorKind::OutputExists))?; + let written = (|| { + for (relative, bytes) in files { + if let Some((directory, _)) = relative.rsplit_once('/') { + fs::create_dir_all(output.join(directory)).map_err(|_| { + refuse(PackageErrorKind::Io { + path: format!("{directory}/"), + }) + })?; + } + write_new(output, relative, bytes).map_err(refuse)?; + } + write_sum_file(output, revision, limits, fix) + })(); + if written.is_err() { + // Only the directory this call created is removed; a failure to + // remove it leaves an incomplete package the verifier refuses. + let _ = fs::remove_dir_all(output); + } + written +} + +/// Check `files` and `revision` as [`write_sum_file`] would and render the +/// `SHA256SUMS` text they produce. +fn plan_sum_file( + files: &BTreeMap>, + revision: Option<&str>, + limits: &PackageLimits, +) -> Result { + if files.is_empty() { + return Err(PackageErrorKind::Empty); + } + let revision_bytes = revision + .map(|revision| { + check_revision(revision) + .map(|()| format!("{revision}\n").into_bytes()) + .map_err(|reason| PackageErrorKind::Revision { reason }) + }) + .transpose()?; + let mut entries = BTreeMap::new(); + let mut folded = BTreeMap::new(); + for (path, bytes) in files { + if is_envelope_file(path) { + return Err(PackageErrorKind::Reserved { path: path.clone() }); + } + entries.insert(path.as_str(), bytes.as_slice()); + } + if let Some(bytes) = &revision_bytes { + entries.insert(REVISION_FILE, bytes.as_slice()); + } + check_file_count(entries.len(), limits)?; + let mut total = 0_u64; + let mut sums = String::new(); + for (path, bytes) in &entries { + check_path(path, limits)?; + // A file cannot also be a directory, and two names that differ only + // in letter case collide on a case-insensitive filesystem. + let mut prefix = String::new(); + for component in path.split('/') { + if !prefix.is_empty() { + prefix.push('/'); + } + prefix.push_str(component); + let is_file = prefix.len() == path.len(); + if let Some((previous, previous_is_file)) = + folded.insert(prefix.to_ascii_lowercase(), (prefix.clone(), is_file)) + { + if previous != prefix || previous_is_file || is_file { + return Err(PackageErrorKind::UnsafeEntry { + path: (*path).to_owned(), + reason: if previous == prefix { + "is both a file and a directory" + } else { + "differs from another name only in letter case" + }, + }); + } + } + } + let length = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + if length > limits.max_file_bytes { + return Err(PackageErrorKind::Bound { + path: Some((*path).to_owned()), + reason: "a file is larger than the package allows", + }); + } + total = add_bytes(total, length, limits)?; + push_sum_line(&mut sums, &hex_lower(&Sha256::digest(bytes)), path); + } + Ok(sums) +} + +fn push_sum_line(sums: &mut String, hex: &str, path: &str) { + sums.push_str(hex); + sums.push_str(SEPARATOR); + sums.push_str(path); + sums.push('\n'); +} + +/// Recompute every digest of the package at `root` and compare it with +/// `SHA256SUMS`. A changed, missing or extra file is refused by name, all of +/// them in one refusal. `fix` is the command that builds the package. +pub fn verify_package( + root: &Path, + limits: &PackageLimits, + fix: &str, +) -> Result { + let refuse = |kind| PackageError::new(root, fix, kind); + check_root(root).map_err(refuse)?; + let sums = read_sum_file(root, limits).map_err(refuse)?; + let listed = parse_sum_file(&sums, limits).map_err(refuse)?; + let walked = walk(root, limits).map_err(refuse)?; + + let mut changed = Vec::new(); + let mut missing = Vec::new(); + let mut total = 0_u64; + let mut revision_bytes = None; + for (path, expected) in &listed { + if !walked.files.contains(path) { + missing.push(path.clone()); + continue; + } + let keep = (path == REVISION_FILE).then_some(&mut revision_bytes); + let (found, length) = hash_file(root, path, limits, keep).map_err(refuse)?; + total = add_bytes(total, length, limits).map_err(refuse)?; + if &found != expected { + changed.push(path.clone()); + } + } + let mut extra = walked + .files + .iter() + .filter(|path| !listed.contains_key(*path)) + .cloned() + .chain(walked.empty_directories) + .collect::>(); + extra.sort(); + if !changed.is_empty() || !missing.is_empty() || !extra.is_empty() { + return Err(refuse(PackageErrorKind::Mismatch { + changed, + missing, + extra, + })); + } + let revision = revision_bytes + .map(|bytes| parse_revision(&bytes)) + .transpose() + .map_err(|reason| refuse(PackageErrorKind::Revision { reason }))?; + Ok(VerifiedPackage { + digest: sha256_uri(&sums), + revision, + files: listed, + }) +} + +/// Why a package was refused, and which package. +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] +pub struct PackageError { + root: PathBuf, + fix: String, + kind: PackageErrorKind, +} + +/// The refusal itself. Paths are relative to the package root; a directory +/// ends in `/`. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum PackageErrorKind { + /// `package.root` is absent, not a directory, or a symbolic link. + RootInvalid { reason: &'static str }, + /// The package directory has no `SHA256SUMS`. + SumFileMissing, + /// `SHA256SUMS` is not in the package format; `line` counts from 1. + SumFileInvalid { line: usize, reason: &'static str }, + /// The directory holds something other than what `SHA256SUMS` lists. + Mismatch { + changed: Vec, + missing: Vec, + extra: Vec, + }, + /// An entry the format cannot carry: a link, a special file, a name that + /// does not travel between platforms. + UnsafeEntry { path: String, reason: &'static str }, + /// A reserved file already exists where a package is being written. + Reserved { path: String }, + /// The directory a package is written into already exists. + OutputExists, + /// The revision is not one printable line. + Revision { reason: &'static str }, + /// The package exceeds a bound. + Bound { + path: Option, + reason: &'static str, + }, + /// A file could not be read or written. + Io { path: String }, + /// There is nothing to package. + Empty, + /// The package is not the one `package.expectedDigest` pins. + DigestMismatch(PackageDigestMismatch), +} + +impl From for PackageErrorKind { + fn from(mismatch: PackageDigestMismatch) -> Self { + Self::DigestMismatch(mismatch) + } +} + +impl PackageError { + fn new(root: &Path, fix: &str, kind: PackageErrorKind) -> Self { + Self { + root: root.to_path_buf(), + fix: fix.to_owned(), + kind, + } + } + + #[must_use] + pub fn kind(&self) -> &PackageErrorKind { + &self.kind + } + + /// The package directory the refusal is about. + #[must_use] + pub fn root(&self) -> &Path { + &self.root + } + + /// Name the package by `label`, such as the field `package.root`, instead + /// of its directory, for a product whose refusals never carry a + /// configured value. + #[must_use] + pub fn naming_root_as(mut self, label: &str) -> Self { + self.root = PathBuf::from(label); + self + } +} + +impl fmt::Display for PackageError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let root = self.root.display(); + let fix = &self.fix; + match &self.kind { + PackageErrorKind::RootInvalid { reason } => write!( + formatter, + "the package at {root} {reason}; build a package with `{fix}` and place its \ + directory there" + ), + PackageErrorKind::SumFileMissing => write!( + formatter, + "the directory at {root} has no {SUM_FILE}, so it is not a package; build one \ + with `{fix}`" + ), + PackageErrorKind::SumFileInvalid { line, reason } => write!( + formatter, + "the package at {root} has an invalid {SUM_FILE} at line {line}: {reason}; \ + rebuild the package with `{fix}`" + ), + PackageErrorKind::Mismatch { + changed, + missing, + extra, + } => { + write!( + formatter, + "the package at {root} does not match its {SUM_FILE}" + )?; + for (label, paths) in [("changed", changed), ("missing", missing), ("extra", extra)] + { + if !paths.is_empty() { + write!(formatter, "; {label}: {}", paths.join(", "))?; + } + } + write!( + formatter, + "; rebuild the package with `{fix}` and deploy the whole directory" + ) + } + PackageErrorKind::UnsafeEntry { path, reason } => write!( + formatter, + "the package at {root} holds {path}, which {reason}; a package holds only \ + regular files, rebuild it with `{fix}`" + ), + PackageErrorKind::Reserved { path } => write!( + formatter, + "{root} already holds {path}; `{fix}` writes a package into a new directory" + ), + PackageErrorKind::OutputExists => write!( + formatter, + "{root} already exists; `{fix}` writes a package into a new directory" + ), + PackageErrorKind::Revision { reason } => write!( + formatter, + "the package revision at {root} is refused: {reason}; pass one printable line \ + as the revision to `{fix}`" + ), + PackageErrorKind::Bound { path, reason } => { + write!(formatter, "the package at {root} is refused: {reason}")?; + if let Some(path) = path { + write!(formatter, " ({path})")?; + } + write!( + formatter, + "; reduce the project and rebuild it with `{fix}`" + ) + } + PackageErrorKind::Io { path } => write!( + formatter, + "the package at {root} could not read or write {path}; check that the file is \ + accessible and rebuild the package with `{fix}` if it is damaged" + ), + PackageErrorKind::Empty => write!( + formatter, + "the package at {root} holds no files; build it with `{fix}`" + ), + PackageErrorKind::DigestMismatch(mismatch) => write!(formatter, "{mismatch}"), + } + } +} + +fn check_root(root: &Path) -> Result<(), PackageErrorKind> { + let metadata = fs::symlink_metadata(root).map_err(|_| PackageErrorKind::RootInvalid { + reason: "does not exist or cannot be read", + })?; + if metadata.file_type().is_symlink() { + return Err(PackageErrorKind::RootInvalid { + reason: "is a symbolic link", + }); + } + if !metadata.is_dir() { + return Err(PackageErrorKind::RootInvalid { + reason: "is not a directory", + }); + } + Ok(()) +} + +struct Walked { + files: BTreeSet, + empty_directories: Vec, +} + +/// Every regular file under `root` except `SHA256SUMS` at the root, and every +/// directory that holds no file at any depth. +fn walk(root: &Path, limits: &PackageLimits) -> Result { + let mut walked = Walked { + files: BTreeSet::new(), + empty_directories: Vec::new(), + }; + // Directories and files together; a tree of empty directories is bounded + // as well as a tree of files. + let mut entries = 0_usize; + let entry_bound = limits.max_files.saturating_mul(2).saturating_add(1); + visit(root, "", limits, &mut walked, &mut entries, entry_bound)?; + check_file_count(walked.files.len(), limits)?; + Ok(walked) +} + +fn visit( + directory: &Path, + prefix: &str, + limits: &PackageLimits, + walked: &mut Walked, + entries: &mut usize, + entry_bound: usize, +) -> Result { + let unreadable = || PackageErrorKind::Io { + path: if prefix.is_empty() { + ".".to_owned() + } else { + format!("{prefix}/") + }, + }; + let mut files_below = 0_usize; + let mut folded = BTreeSet::new(); + for entry in fs::read_dir(directory).map_err(|_| unreadable())? { + let entry = entry.map_err(|_| unreadable())?; + *entries += 1; + if *entries > entry_bound { + return Err(PackageErrorKind::Bound { + path: None, + reason: "the package holds more entries than it allows", + }); + } + let name = entry.file_name(); + let relative = match name.to_str() { + Some(name) if prefix.is_empty() => name.to_owned(), + Some(name) => format!("{prefix}/{name}"), + None => { + return Err(PackageErrorKind::UnsafeEntry { + path: format!("{prefix}/{}", name.to_string_lossy()), + reason: "has a name that is not UTF-8", + }) + } + }; + if prefix.is_empty() && relative == SUM_FILE { + continue; + } + check_path(&relative, limits)?; + if !folded.insert(relative.to_ascii_lowercase()) { + return Err(PackageErrorKind::UnsafeEntry { + path: relative, + reason: "differs from another name only in letter case", + }); + } + let file_type = entry.file_type().map_err(|_| PackageErrorKind::Io { + path: relative.clone(), + })?; + if file_type.is_symlink() { + return Err(PackageErrorKind::UnsafeEntry { + path: relative, + reason: "is a symbolic link", + }); + } + if file_type.is_dir() { + let below = visit( + &entry.path(), + &relative, + limits, + walked, + entries, + entry_bound, + )?; + if below == 0 { + walked.empty_directories.push(format!("{relative}/")); + } + files_below += below; + } else if file_type.is_file() { + walked.files.insert(relative); + files_below += 1; + } else { + return Err(PackageErrorKind::UnsafeEntry { + path: relative, + reason: "is not a regular file or directory", + }); + } + } + Ok(files_below) +} + +/// A relative path the sum file can carry unescaped and every supported +/// platform can store. +fn check_path(path: &str, limits: &PackageLimits) -> Result<(), PackageErrorKind> { + if path.len() > limits.max_path_bytes { + return Err(PackageErrorKind::Bound { + path: Some(path.to_owned()), + reason: "a path is longer than the package allows", + }); + } + if path.split('/').count() > limits.max_depth { + return Err(PackageErrorKind::Bound { + path: Some(path.to_owned()), + reason: "a path is nested deeper than the package allows", + }); + } + let unsafe_entry = |reason| { + Err(PackageErrorKind::UnsafeEntry { + path: path.escape_debug().to_string(), + reason, + }) + }; + if path.chars().any(char::is_control) { + return unsafe_entry("has a control character in its name"); + } + if path.contains('\\') { + return unsafe_entry("has a backslash in its name"); + } + if path + .split('/') + .any(|component| component.is_empty() || component == "." || component == "..") + { + return unsafe_entry("is not a plain relative path"); + } + Ok(()) +} + +fn check_file_count(count: usize, limits: &PackageLimits) -> Result<(), PackageErrorKind> { + if count > limits.max_files { + return Err(PackageErrorKind::Bound { + path: None, + reason: "the package holds more files than it allows", + }); + } + Ok(()) +} + +fn add_bytes(total: u64, length: u64, limits: &PackageLimits) -> Result { + total + .checked_add(length) + .filter(|total| *total <= limits.max_total_bytes) + .ok_or(PackageErrorKind::Bound { + path: None, + reason: "the package is larger than it allows", + }) +} + +fn sum_file_bound(limits: &PackageLimits) -> u64 { + let line = HEX_DIGITS + SEPARATOR.len() + limits.max_path_bytes + 1; + u64::try_from(limits.max_files.saturating_mul(line)).unwrap_or(u64::MAX) +} + +fn read_sum_file(root: &Path, limits: &PackageLimits) -> Result, PackageErrorKind> { + let path = root.join(SUM_FILE); + let metadata = match fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(PackageErrorKind::SumFileMissing) + } + Err(_) => { + return Err(PackageErrorKind::Io { + path: SUM_FILE.to_owned(), + }) + } + }; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(PackageErrorKind::UnsafeEntry { + path: SUM_FILE.to_owned(), + reason: "is not a regular file", + }); + } + let bound = sum_file_bound(limits); + let mut bytes = Vec::new(); + read_bounded(root, SUM_FILE, bound, |chunk| { + bytes.extend_from_slice(chunk) + }) + .map_err(|error| match error { + PackageErrorKind::Bound { .. } => PackageErrorKind::Bound { + path: Some(SUM_FILE.to_owned()), + reason: "SHA256SUMS is larger than the package allows", + }, + other => other, + })?; + Ok(bytes) +} + +fn parse_sum_file( + bytes: &[u8], + limits: &PackageLimits, +) -> Result, PackageErrorKind> { + let invalid = |line, reason| PackageErrorKind::SumFileInvalid { line, reason }; + let text = std::str::from_utf8(bytes).map_err(|_| invalid(1, "it is not UTF-8"))?; + if text.is_empty() { + return Err(invalid(1, "it lists no files")); + } + let Some(body) = text.strip_suffix('\n') else { + let line = text.split('\n').count(); + return Err(invalid(line, "the last line does not end with a line feed")); + }; + let mut listed = BTreeMap::new(); + let mut previous: Option<&str> = None; + for (index, line) in body.split('\n').enumerate() { + let number = index + 1; + if listed.len() >= limits.max_files { + return Err(invalid( + number, + "it lists more files than the package allows", + )); + } + let (hex, path) = line + .split_at_checked(HEX_DIGITS) + .ok_or_else(|| invalid(number, "a line is not a digest, two spaces and a path"))?; + let path = path + .strip_prefix(SEPARATOR) + .ok_or_else(|| invalid(number, "a line is not a digest, two spaces and a path"))?; + if !hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(invalid(number, "a digest is not 64 lowercase hex digits")); + } + if path == SUM_FILE { + return Err(invalid(number, "SHA256SUMS lists itself")); + } + check_path(path, limits) + .map_err(|_| invalid(number, "a path is not a plain relative path"))?; + if previous.is_some_and(|previous| previous >= path) { + return Err(invalid(number, "paths are not sorted or a path repeats")); + } + previous = Some(path); + listed.insert(path.to_owned(), hex.to_owned()); + } + Ok(listed) +} + +/// Hash one listed file in bounded chunks. `keep` receives the bytes when the +/// caller also needs them. +fn hash_file( + root: &Path, + relative: &str, + limits: &PackageLimits, + keep: Option<&mut Option>>, +) -> Result<(String, u64), PackageErrorKind> { + let mut hasher = Sha256::new(); + let mut kept = keep.as_ref().map(|_| Vec::new()); + let length = read_bounded(root, relative, limits.max_file_bytes, |chunk| { + hasher.update(chunk); + if let Some(kept) = kept.as_mut() { + kept.extend_from_slice(chunk); + } + }) + .map_err(|error| match error { + PackageErrorKind::Bound { .. } => PackageErrorKind::Bound { + path: Some(relative.to_owned()), + reason: "a file is larger than the package allows", + }, + other => other, + })?; + if let Some(keep) = keep { + *keep = kept; + } + Ok((hex_lower(&hasher.finalize()), length)) +} + +/// Read `relative` under `root` without following a link, at most `bound` +/// bytes, and refuse a file whose identity or size changed during the read. +fn read_bounded( + root: &Path, + relative: &str, + bound: u64, + mut sink: impl FnMut(&[u8]), +) -> Result { + let path = root.join(relative); + let io = || PackageErrorKind::Io { + path: relative.to_owned(), + }; + let scanned = fs::symlink_metadata(&path).map_err(|_| io())?; + if scanned.file_type().is_symlink() { + return Err(PackageErrorKind::UnsafeEntry { + path: relative.to_owned(), + reason: "is a symbolic link", + }); + } + if !scanned.is_file() { + return Err(PackageErrorKind::UnsafeEntry { + path: relative.to_owned(), + reason: "is not a regular file", + }); + } + if scanned.len() > bound { + return Err(PackageErrorKind::Bound { + path: Some(relative.to_owned()), + reason: "a file is larger than the package allows", + }); + } + let file = open_no_follow(&path).map_err(|_| io())?; + let opened = file.metadata().map_err(|_| io())?; + if !opened.is_file() || !same_file(&scanned, &opened) { + return Err(changed_during_read(relative)); + } + let mut reader = file.take(bound.saturating_add(1)); + let mut buffer = vec![0_u8; READ_CHUNK]; + let mut length = 0_u64; + loop { + let read = reader.read(&mut buffer).map_err(|_| io())?; + if read == 0 { + break; + } + length += read as u64; + if length > bound { + return Err(PackageErrorKind::Bound { + path: Some(relative.to_owned()), + reason: "a file is larger than the package allows", + }); + } + sink(&buffer[..read]); + } + let after = reader.get_ref().metadata().map_err(|_| io())?; + if !same_file(&opened, &after) || length != after.len() { + return Err(changed_during_read(relative)); + } + Ok(length) +} + +fn changed_during_read(relative: &str) -> PackageErrorKind { + PackageErrorKind::UnsafeEntry { + path: relative.to_owned(), + reason: "changed while it was read", + } +} + +fn open_no_follow(path: &Path) -> std::io::Result { + let mut options = fs::OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + + options.custom_flags( + (rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::CLOEXEC).bits() as i32, + ); + } + options.open(path) +} + +#[cfg(unix)] +fn same_file(left: &fs::Metadata, right: &fs::Metadata) -> bool { + use std::os::unix::fs::MetadataExt as _; + + left.dev() == right.dev() + && left.ino() == right.ino() + && left.len() == right.len() + && left.mtime() == right.mtime() + && left.mtime_nsec() == right.mtime_nsec() +} + +#[cfg(not(unix))] +fn same_file(left: &fs::Metadata, right: &fs::Metadata) -> bool { + left.len() == right.len() && left.modified().ok() == right.modified().ok() +} + +fn write_new(root: &Path, relative: &str, bytes: &[u8]) -> Result<(), PackageErrorKind> { + let io = || PackageErrorKind::Io { + path: relative.to_owned(), + }; + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(root.join(relative)) + .map_err(|_| io())?; + file.write_all(bytes).map_err(|_| io())?; + file.sync_all().map_err(|_| io()) +} + +fn parse_revision(bytes: &[u8]) -> Result { + let text = std::str::from_utf8(bytes).map_err(|_| "REVISION is not UTF-8")?; + let revision = text + .strip_suffix('\n') + .ok_or("REVISION does not end with a line feed")?; + check_revision(revision)?; + Ok(revision.to_owned()) +} + +#[cfg(test)] +#[path = "package_tests.rs"] +mod tests; diff --git a/crates/registry-platform-config/src/package_tests.rs b/crates/registry-platform-config/src/package_tests.rs new file mode 100644 index 0000000000..3b7fd275f4 --- /dev/null +++ b/crates/registry-platform-config/src/package_tests.rs @@ -0,0 +1,539 @@ +use std::fs; +use std::path::Path; + +use super::*; +use crate::blocks::PackageConfig; + +const FIX: &str = "examplectl package"; + +fn write(root: &Path, relative: &str, bytes: &[u8]) { + let path = root.join(relative); + fs::create_dir_all(path.parent().expect("parent")).expect("directories"); + fs::write(path, bytes).expect("file"); +} + +fn populated() -> tempfile::TempDir { + let directory = tempfile::tempdir().expect("temporary directory"); + write(directory.path(), "policy.yaml", b"name: example\n"); + write(directory.path(), "rules/b.rhai", b"let b = 2;\n"); + write(directory.path(), "rules/a.rhai", b"let a = 1;\n"); + directory +} + +fn packaged(revision: Option<&str>) -> (tempfile::TempDir, VerifiedPackage) { + let directory = populated(); + let package = write_sum_file(directory.path(), revision, &PackageLimits::default(), FIX) + .expect("the package is written"); + (directory, package) +} + +fn verify(root: &Path) -> Result { + verify_package(root, &PackageLimits::default(), FIX) +} + +#[test] +fn the_sum_file_lists_every_file_sorted_in_the_sha256sum_format() { + let (directory, package) = packaged(None); + let sums = fs::read_to_string(directory.path().join(SUM_FILE)).expect("sum file"); + let expected = format!( + "{} policy.yaml\n{} rules/a.rhai\n{} rules/b.rhai\n", + hex(b"name: example\n"), + hex(b"let a = 1;\n"), + hex(b"let b = 2;\n"), + ); + assert_eq!(sums, expected); + assert_eq!(package.digest(), crate::sha256_uri(expected.as_bytes())); + assert_eq!(package.revision(), None); + assert_eq!( + package.files().collect::>(), + ["policy.yaml", "rules/a.rhai", "rules/b.rhai"] + ); +} + +#[test] +fn packaging_the_same_content_twice_gives_the_same_digest() { + let (_first, first) = packaged(Some("release 7")); + let (_second, second) = packaged(Some("release 7")); + assert_eq!(first.digest(), second.digest()); + let (_other, other) = packaged(Some("release 8")); + assert_ne!(first.digest(), other.digest()); + let (_none, none) = packaged(None); + assert_ne!(first.digest(), none.digest()); +} + +#[test] +fn the_revision_is_recorded_as_a_hashed_file_and_read_back() { + let (directory, package) = packaged(Some("0123abcd")); + assert_eq!( + fs::read(directory.path().join(REVISION_FILE)).expect("revision"), + b"0123abcd\n" + ); + assert_eq!(package.revision(), Some("0123abcd")); + let verified = verify(directory.path()).expect("the package verifies"); + assert_eq!(verified.revision(), Some("0123abcd")); + assert_eq!(verified.digest(), package.digest()); + assert!(verified.files().any(|path| path == REVISION_FILE)); +} + +#[test] +fn a_verified_package_reports_the_digest_the_writer_reported() { + let (directory, package) = packaged(None); + let verified = verify(directory.path()).expect("the package verifies"); + assert_eq!(verified, package); +} + +#[test] +fn a_changed_file_is_refused_by_name() { + let (directory, _) = packaged(None); + write(directory.path(), "rules/a.rhai", b"let a = 3;\n"); + let error = verify(directory.path()).expect_err("a changed file"); + assert_eq!( + error.kind(), + &PackageErrorKind::Mismatch { + changed: vec!["rules/a.rhai".to_owned()], + missing: Vec::new(), + extra: Vec::new(), + } + ); + let message = error.to_string(); + assert!(message.contains("changed: rules/a.rhai"), "{message}"); + assert!(message.contains("SHA256SUMS"), "{message}"); + assert!(message.contains(FIX), "{message}"); +} + +#[test] +fn a_missing_file_is_refused_by_name() { + let (directory, _) = packaged(None); + fs::remove_file(directory.path().join("rules/b.rhai")).expect("remove"); + let error = verify(directory.path()).expect_err("a missing file"); + assert_eq!( + error.kind(), + &PackageErrorKind::Mismatch { + changed: Vec::new(), + missing: vec!["rules/b.rhai".to_owned()], + extra: Vec::new(), + } + ); + assert!(error.to_string().contains("missing: rules/b.rhai")); +} + +#[test] +fn a_package_whose_every_listed_file_is_gone_names_each_missing_file() { + let (directory, _) = packaged(None); + fs::remove_dir_all(directory.path().join("rules")).expect("remove rules"); + fs::remove_file(directory.path().join("policy.yaml")).expect("remove policy"); + let error = verify(directory.path()).expect_err("only SHA256SUMS is left"); + assert!( + matches!(error.kind(), PackageErrorKind::Mismatch { missing, .. } if missing.len() == 3), + "{error}" + ); + assert!( + error.to_string().contains("missing: policy.yaml"), + "{error}" + ); +} + +#[test] +fn an_extra_file_or_empty_directory_is_refused_by_name() { + let (directory, _) = packaged(None); + write(directory.path(), "rules/.DS_Store", b"x"); + fs::create_dir(directory.path().join("empty")).expect("directory"); + let error = verify(directory.path()).expect_err("extra entries"); + assert_eq!( + error.kind(), + &PackageErrorKind::Mismatch { + changed: Vec::new(), + missing: Vec::new(), + extra: vec!["empty/".to_owned(), "rules/.DS_Store".to_owned()], + } + ); + assert!(error.to_string().contains("extra: empty/, rules/.DS_Store")); +} + +#[test] +fn every_discrepancy_is_reported_together() { + let (directory, _) = packaged(None); + write(directory.path(), "policy.yaml", b"name: other\n"); + fs::remove_file(directory.path().join("rules/a.rhai")).expect("remove"); + write(directory.path(), "notes.txt", b"x"); + let error = verify(directory.path()).expect_err("three discrepancies"); + let message = error.to_string(); + assert!(message.contains("changed: policy.yaml"), "{message}"); + assert!(message.contains("missing: rules/a.rhai"), "{message}"); + assert!(message.contains("extra: notes.txt"), "{message}"); +} + +#[test] +fn a_refusal_can_name_the_field_instead_of_the_directory() { + let (directory, _) = packaged(None); + write(directory.path(), "policy.yaml", b"name: other\n"); + let error = verify(directory.path()) + .expect_err("a changed file") + .naming_root_as("package.root"); + let message = error.to_string(); + assert!( + message.starts_with("the package at package.root does not match its SHA256SUMS"), + "{message}" + ); + assert!(message.contains("changed: policy.yaml"), "{message}"); + assert!(message.contains(FIX), "{message}"); + assert!( + !message.contains(&*directory.path().to_string_lossy()), + "{message}" + ); +} + +#[test] +fn a_directory_without_a_sum_file_is_refused_with_the_package_command() { + let directory = populated(); + let error = verify(directory.path()).expect_err("no sum file"); + assert_eq!(error.kind(), &PackageErrorKind::SumFileMissing); + assert!(error.to_string().contains(FIX)); + let absent = directory.path().join("absent"); + let error = verify(&absent).expect_err("no package directory"); + assert!(matches!(error.kind(), PackageErrorKind::RootInvalid { .. })); + assert!(error.to_string().contains(FIX)); +} + +#[test] +fn a_malformed_sum_file_is_refused_with_its_line() { + for (sums, line) in [ + ("not a sum line\n".to_owned(), 1), + (format!("{} a\n{} a\n", "0".repeat(64), "0".repeat(64)), 2), + (format!("{} b\n{} a\n", "0".repeat(64), "0".repeat(64)), 2), + (format!("{} a", "0".repeat(64)), 1), + (format!("{} a\r\n", "0".repeat(64)), 1), + (format!("{} *a\n", "0".repeat(64)), 1), + (format!("{} a\n", "A".repeat(64)), 1), + (format!("{} ../a\n", "0".repeat(64)), 1), + (format!("{} /a\n", "0".repeat(64)), 1), + (format!("{} SHA256SUMS\n", "0".repeat(64)), 1), + (String::new(), 1), + ] { + let directory = populated(); + fs::write(directory.path().join(SUM_FILE), &sums).expect("sum file"); + let error = verify(directory.path()).expect_err("a malformed sum file"); + match error.kind() { + PackageErrorKind::SumFileInvalid { line: found, .. } => { + assert_eq!(*found, line, "{sums:?}"); + } + other => panic!("{sums:?} gave {other:?}"), + } + assert!(error.to_string().contains(FIX)); + } +} + +#[test] +fn an_invalid_revision_file_is_refused() { + let (directory, _) = packaged(None); + fs::remove_file(directory.path().join(SUM_FILE)).expect("remove"); + fs::write(directory.path().join(REVISION_FILE), b"two\nlines\n").expect("revision"); + let error = write_sum_file(directory.path(), None, &PackageLimits::default(), FIX) + .expect_err("REVISION is reserved"); + assert!(matches!(error.kind(), PackageErrorKind::Reserved { .. })); +} + +#[test] +fn a_revision_must_be_one_printable_line() { + for revision in [ + "", + " leading", + "trailing ", + "two\nlines", + "tab\there", + "hidden\u{200e}mark", + "override\u{202e}text", + "isolate\u{2066}text", + &"x".repeat(MAX_REVISION_BYTES + 1), + ] { + let directory = populated(); + let error = write_sum_file( + directory.path(), + Some(revision), + &PackageLimits::default(), + FIX, + ) + .expect_err("an invalid revision"); + assert!( + matches!(error.kind(), PackageErrorKind::Revision { .. }), + "{revision:?}" + ); + assert!(!directory.path().join(REVISION_FILE).exists()); + assert!(!directory.path().join(SUM_FILE).exists()); + } + assert!(check_revision(&"x".repeat(MAX_REVISION_BYTES)).is_ok()); + assert!(check_revision("v1.2.3 (git 0123abcd)").is_ok()); + assert!(check_revision("إصدار-גרסה-1").is_ok()); +} + +#[test] +fn a_package_is_written_once() { + let (directory, _) = packaged(None); + let error = write_sum_file(directory.path(), None, &PackageLimits::default(), FIX) + .expect_err("already packaged"); + assert!(matches!(error.kind(), PackageErrorKind::Reserved { .. })); +} + +#[test] +fn an_empty_package_is_refused() { + let directory = tempfile::tempdir().expect("temporary directory"); + let error = write_sum_file(directory.path(), None, &PackageLimits::default(), FIX) + .expect_err("nothing to package"); + assert_eq!(error.kind(), &PackageErrorKind::Empty); +} + +#[test] +fn an_empty_directory_is_refused_when_packaging() { + let directory = populated(); + fs::create_dir(directory.path().join("empty")).expect("directory"); + let error = write_sum_file(directory.path(), None, &PackageLimits::default(), FIX) + .expect_err("an empty directory"); + assert_eq!( + error.kind(), + &PackageErrorKind::UnsafeEntry { + path: "empty/".to_owned(), + reason: "is an empty directory", + } + ); +} + +#[cfg(unix)] +#[test] +fn symbolic_links_are_refused_inside_and_at_the_root() { + let directory = populated(); + std::os::unix::fs::symlink("policy.yaml", directory.path().join("link.yaml")).expect("link"); + let error = write_sum_file(directory.path(), None, &PackageLimits::default(), FIX) + .expect_err("a symbolic link"); + assert_eq!( + error.kind(), + &PackageErrorKind::UnsafeEntry { + path: "link.yaml".to_owned(), + reason: "is a symbolic link", + } + ); + + let (packaged, _) = packaged(None); + let outside = tempfile::tempdir().expect("outside"); + std::os::unix::fs::symlink(packaged.path(), outside.path().join("current")).expect("link"); + let error = verify(&outside.path().join("current")).expect_err("a linked root"); + assert!(matches!(error.kind(), PackageErrorKind::RootInvalid { .. })); + + fs::remove_file(packaged.path().join("rules/a.rhai")).expect("remove"); + std::os::unix::fs::symlink("../policy.yaml", packaged.path().join("rules/a.rhai")) + .expect("link"); + let error = verify(packaged.path()).expect_err("a listed file became a link"); + assert_eq!( + error.kind(), + &PackageErrorKind::UnsafeEntry { + path: "rules/a.rhai".to_owned(), + reason: "is a symbolic link", + } + ); +} + +#[test] +fn names_that_cannot_travel_are_refused() { + for name in ["back\\slash", "new\nline", "UPPER.yaml"] { + let directory = populated(); + write(directory.path(), "upper.yaml", b"x"); + if fs::write(directory.path().join(name), b"x").is_err() { + continue; + } + if fs::read_dir(directory.path()).expect("listing").count() < 4 { + // A case-insensitive filesystem folded the two names into one file. + continue; + } + let error = write_sum_file(directory.path(), None, &PackageLimits::default(), FIX) + .expect_err("an unportable name"); + assert!( + matches!(error.kind(), PackageErrorKind::UnsafeEntry { .. }), + "{name:?}: {error}" + ); + } +} + +#[test] +fn limits_bound_files_bytes_and_depth() { + let directory = populated(); + let limits = PackageLimits { + max_files: 2, + ..PackageLimits::default() + }; + let error = write_sum_file(directory.path(), None, &limits, FIX).expect_err("too many files"); + assert!(matches!(error.kind(), PackageErrorKind::Bound { .. })); + + let directory = populated(); + let limits = PackageLimits { + max_file_bytes: 12, + ..PackageLimits::default() + }; + let error = write_sum_file(directory.path(), None, &limits, FIX).expect_err("a large file"); + assert_eq!( + error.kind(), + &PackageErrorKind::Bound { + path: Some("policy.yaml".to_owned()), + reason: "a file is larger than the package allows", + } + ); + + let directory = populated(); + let limits = PackageLimits { + max_total_bytes: 20, + ..PackageLimits::default() + }; + let error = write_sum_file(directory.path(), None, &limits, FIX).expect_err("too many bytes"); + assert!(matches!(error.kind(), PackageErrorKind::Bound { .. })); + + let directory = populated(); + write(directory.path(), "a/b/c/d.yaml", b"x"); + let limits = PackageLimits { + max_depth: 3, + ..PackageLimits::default() + }; + let error = write_sum_file(directory.path(), None, &limits, FIX).expect_err("too deep"); + assert!(matches!(error.kind(), PackageErrorKind::Bound { .. })); +} + +#[test] +fn the_package_config_verifies_the_directory_and_the_pin_together() { + let (directory, package) = packaged(None); + let unpinned = PackageConfig { + root: directory.path().to_path_buf(), + expected_digest: None, + }; + assert_eq!( + unpinned + .verify_package(&PackageLimits::default(), FIX) + .expect("no pin"), + package + ); + let pinned = PackageConfig { + expected_digest: Some(package.digest().to_owned()), + ..unpinned.clone() + }; + assert_eq!( + pinned + .verify_package(&PackageLimits::default(), FIX) + .expect("the pin matches"), + package + ); + let other = format!("sha256:{}", "0".repeat(64)); + let mismatched = PackageConfig { + expected_digest: Some(other.clone()), + ..unpinned + }; + let error = mismatched + .verify_package(&PackageLimits::default(), FIX) + .expect_err("the pin differs"); + assert_eq!( + error.kind(), + &PackageErrorKind::DigestMismatch(PackageDigestMismatch { + expected: other.clone(), + found: package.digest().to_owned(), + }) + ); + let message = error.to_string(); + assert!(message.contains(&other), "{message}"); + assert!(message.contains(package.digest()), "{message}"); + assert!(message.contains("package.expectedDigest"), "{message}"); +} + +#[test] +fn envelope_files_are_named_for_product_loaders() { + assert!(is_envelope_file(SUM_FILE)); + assert!(is_envelope_file(REVISION_FILE)); + assert!(!is_envelope_file("rules/SHA256SUMS")); + assert!(!is_envelope_file("policy.yaml")); +} + +fn in_memory() -> std::collections::BTreeMap> { + [ + ("policy.yaml", b"name: example\n".as_slice()), + ("rules/b.rhai", b"let b = 2;\n"), + ("rules/a.rhai", b"let a = 1;\n"), + ] + .into_iter() + .map(|(path, bytes)| (path.to_owned(), bytes.to_vec())) + .collect() +} + +#[test] +fn a_planned_package_has_the_digest_the_written_package_has() { + let limits = PackageLimits::default(); + for revision in [None, Some("release 7")] { + let (_directory, on_disk) = packaged(revision); + let planned = plan_package(Path::new("project"), &in_memory(), revision, &limits, FIX) + .expect("the plan is valid"); + assert_eq!(planned, on_disk.digest()); + + let parent = tempfile::tempdir().expect("temporary directory"); + let output = parent.path().join("nested/package"); + let written = write_package(&output, &in_memory(), revision, &limits, FIX) + .expect("the package is written"); + assert_eq!(written, on_disk); + assert_eq!(verify(&output).expect("verifies"), on_disk); + } +} + +#[test] +fn a_planned_package_refuses_what_the_writer_would_refuse() { + let limits = PackageLimits::default(); + let plan = |files: &[(&str, &[u8])], revision| { + let files = files + .iter() + .map(|(path, bytes)| ((*path).to_owned(), bytes.to_vec())) + .collect(); + plan_package(Path::new("project"), &files, revision, &limits, FIX) + .expect_err("refused") + .kind() + .clone() + }; + assert_eq!(plan(&[], None), PackageErrorKind::Empty); + assert!(matches!( + plan(&[("../a", b"")], None), + PackageErrorKind::UnsafeEntry { .. } + )); + assert!(matches!( + plan(&[("a", b""), ("a/b", b"")], None), + PackageErrorKind::UnsafeEntry { .. } + )); + assert!(matches!( + plan(&[("Policy.yaml", b""), ("policy.yaml", b"")], None), + PackageErrorKind::UnsafeEntry { .. } + )); + assert!(matches!( + plan(&[(SUM_FILE, b"")], None), + PackageErrorKind::Reserved { .. } + )); + assert!(matches!( + plan(&[(REVISION_FILE, b"")], None), + PackageErrorKind::Reserved { .. } + )); + assert!(matches!( + plan(&[("a", b"")], Some(" padded")), + PackageErrorKind::Revision { .. } + )); +} + +#[test] +fn a_package_is_written_into_a_new_directory_only() { + let existing = tempfile::tempdir().expect("temporary directory"); + let error = write_package( + existing.path(), + &in_memory(), + None, + &PackageLimits::default(), + FIX, + ) + .expect_err("an existing directory is refused"); + assert!(matches!(error.kind(), PackageErrorKind::OutputExists)); + assert!(error.to_string().contains(FIX), "{error}"); + assert_eq!(fs::read_dir(existing.path()).expect("list").count(), 0); +} + +fn hex(bytes: &[u8]) -> String { + crate::sha256_uri(bytes) + .strip_prefix("sha256:") + .expect("label") + .to_owned() +} diff --git a/crates/registry-platform-config/src/schema.rs b/crates/registry-platform-config/src/schema.rs new file mode 100644 index 0000000000..440cf09d06 --- /dev/null +++ b/crates/registry-platform-config/src/schema.rs @@ -0,0 +1,147 @@ +//! The canonical JSON Schema of the shared runtime configuration blocks. + +use serde_json::{Map, Value}; + +use crate::blocks::{ + AuditKeyConfig, DatabaseConfig, JwksSource, ListenerConfig, OidcClientsConfig, + OidcIssuerConfig, PackageConfig, PrivateListenerConfig, SecretProvidersConfig, +}; + +/// File name the shared-blocks generator writes. +pub const SHARED_BLOCKS_SCHEMA_FILE: &str = "runtime-config-blocks.schema.json"; + +/// Every shared block as a definition, keyed by the name a runtime schema +/// derived with schemars gives it. +#[derive(schemars::JsonSchema)] +#[allow(dead_code)] +struct SharedBlocks { + secret_providers: SecretProvidersConfig, + database: DatabaseConfig, + jwks_source: JwksSource, + package: PackageConfig, + listener: ListenerConfig, + private_listener: PrivateListenerConfig, + audit_key: AuditKeyConfig, + oidc_issuer: OidcIssuerConfig, + oidc_clients: OidcClientsConfig, +} + +/// The pretty-printed shared-blocks document, newline terminated. +pub fn shared_blocks_document() -> Result { + let derived = serde_json::to_value(schemars::schema_for!(SharedBlocks))?; + let definitions = derived.get("$defs").cloned().unwrap_or_default(); + let mut document = Map::new(); + document.insert( + "$schema".to_owned(), + Value::String("https://json-schema.org/draft/2020-12/schema".to_owned()), + ); + document.insert( + "title".to_owned(), + Value::String("Registry Stack shared runtime configuration blocks".to_owned()), + ); + document.insert("$defs".to_owned(), definitions); + let mut rendered = serde_json::to_string_pretty(&Value::Object(document))?; + rendered.push('\n'); + Ok(rendered) +} + +/// Root `allOf` members stating that a static JWKS document at +/// `authentication.oidc.jwksSource.documentRef` names a secret provider by +/// its prefix, so a configuration carrying one must enable that provider. +pub fn jwks_document_provider_requirements() -> Value { + serde_json::json!([ + secret_provider_requirement( + "^secret:env/", + "environment", + "EnvironmentSecretProviderConfig" + ), + secret_provider_requirement("^secret:file/", "file", "FileSecretProviderConfig"), + ]) +} + +fn secret_provider_requirement(reference_pattern: &str, provider: &str, definition: &str) -> Value { + serde_json::json!({ + "if": { + "properties": { + "authentication": { + "properties": { + "oidc": { + "properties": { + "jwksSource": { + "properties": { + "documentRef": {"pattern": reference_pattern} + }, + "required": ["documentRef"] + } + }, + "required": ["jwksSource"] + } + }, + "required": ["oidc"] + } + }, + "required": ["authentication"] + }, + "then": { + "properties": { + "secretProviders": { + "properties": { + provider: {"$ref": format!("#/$defs/{definition}")} + }, + "required": [provider] + } + } + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_static_jwks_document_requires_the_provider_its_reference_names() { + let requirement = |pattern: &str, provider: &str, definition: &str| { + serde_json::json!({ + "if": { + "properties": { + "authentication": { + "properties": { + "oidc": { + "properties": { + "jwksSource": { + "properties": {"documentRef": {"pattern": pattern}}, + "required": ["documentRef"] + } + }, + "required": ["jwksSource"] + } + }, + "required": ["oidc"] + } + }, + "required": ["authentication"] + }, + "then": { + "properties": { + "secretProviders": { + "properties": {provider: {"$ref": format!("#/$defs/{definition}")}}, + "required": [provider] + } + } + } + }) + }; + assert_eq!( + jwks_document_provider_requirements(), + serde_json::json!([ + requirement( + "^secret:env/", + "environment", + "EnvironmentSecretProviderConfig" + ), + requirement("^secret:file/", "file", "FileSecretProviderConfig"), + ]) + ); + } +} diff --git a/crates/registry-platform-config/src/secrets.rs b/crates/registry-platform-config/src/secrets.rs index 2f9be55477..fceef03f68 100644 --- a/crates/registry-platform-config/src/secrets.rs +++ b/crates/registry-platform-config/src/secrets.rs @@ -72,6 +72,56 @@ impl SecretReference { } } +impl PartialOrd for SecretReference { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for SecretReference { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.reference.cmp(&other.reference) + } +} + +impl std::hash::Hash for SecretReference { + fn hash(&self, state: &mut H) { + self.reference.hash(state); + } +} + +impl serde::Serialize for SecretReference { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.reference) + } +} + +impl<'de> serde::Deserialize<'de> for SecretReference { + fn deserialize>(deserializer: D) -> Result { + let value = ::deserialize(deserializer)?; + Self::parse(value).map_err(|_| { + serde::de::Error::custom( + "expected an exact secret:env/NAME or secret:file/name reference", + ) + }) + } +} + +#[cfg(feature = "schema")] +impl schemars::JsonSchema for SecretReference { + fn schema_name() -> std::borrow::Cow<'static, str> { + "SecretReference".into() + } + + fn json_schema(_generator: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "description": "An exact secret reference: secret:file/name, resolved under secretProviders.file.root, or secret:env/NAME, resolved only when secretProviders.environment is declared.", + "type": "string", + "pattern": crate::blocks::SECRET_REFERENCE_PATTERN + }) + } +} + impl fmt::Debug for SecretReference { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter diff --git a/crates/registry-platform-crypto/README.md b/crates/registry-platform-crypto/README.md index bfe5fad4c8..6ef34f737f 100644 --- a/crates/registry-platform-crypto/README.md +++ b/crates/registry-platform-crypto/README.md @@ -93,7 +93,10 @@ interoperability and security policy. sign-and-verify check before reporting ready. Signing inputs are SHA-256 hashed locally and sent with Transit `prehashed: true`, so assertion bytes do not cross the signing-provider boundary. The proxy owns authentication and - token renewal; the application never receives its token. + token renewal; the application never receives its token. A refused + initialization returns a `TransitInitializationError` naming one cause, such + as an unreachable socket or a pinned version below `min_encryption_version`, + and never a provider response, path, or key material. - Readiness-gated live apply should use `KeyReadinessSnapshot`; only `status = active` plus `readiness = ready` is accepted. Degraded, not-ready, unknown, publish-only, and disabled keys fail closed before diff --git a/crates/registry-platform-crypto/src/lib.rs b/crates/registry-platform-crypto/src/lib.rs index c26a438c0e..439cf699a4 100644 --- a/crates/registry-platform-crypto/src/lib.rs +++ b/crates/registry-platform-crypto/src/lib.rs @@ -9,17 +9,19 @@ pub mod transit_datakey; mod transit_mock; use async_trait::async_trait; -use aws_lc_rs::encoding::{AsBigEndian as _, AsDer as _, EcPrivateKeyBin, Pkcs8V1Der}; +use aws_lc_rs::encoding::{ + AsBigEndian as _, AsDer as _, Curve25519SeedBin, EcPrivateKeyBin, Pkcs8V1Der, +}; use aws_lc_rs::rand::SystemRandom; use aws_lc_rs::rsa::{ KeyPair as AwsRsaKeyPair, KeySize as AwsRsaKeySize, PublicKeyComponents as AwsRsaPublicKeyComponents, }; use aws_lc_rs::signature::{ - EcdsaKeyPair, KeyPair as _, RsaParameters, RsaSignatureEncoding, UnparsedPublicKey, - ECDSA_P256_SHA256_FIXED, ECDSA_P256_SHA256_FIXED_SIGNING, ECDSA_P384_SHA384_FIXED, - ECDSA_P384_SHA384_FIXED_SIGNING, RSA_PKCS1_2048_8192_SHA256, RSA_PKCS1_2048_8192_SHA384, - RSA_PKCS1_SHA256, RSA_PKCS1_SHA384, + EcdsaKeyPair, EcdsaSigningAlgorithm, Ed25519KeyPair, KeyPair as _, RsaParameters, + RsaSignatureEncoding, UnparsedPublicKey, ECDSA_P256_SHA256_FIXED, + ECDSA_P256_SHA256_FIXED_SIGNING, ECDSA_P384_SHA384_FIXED, ECDSA_P384_SHA384_FIXED_SIGNING, + RSA_PKCS1_2048_8192_SHA256, RSA_PKCS1_2048_8192_SHA384, RSA_PKCS1_SHA256, RSA_PKCS1_SHA384, }; #[cfg(feature = "transit")] use base64::engine::general_purpose::STANDARD; @@ -574,6 +576,103 @@ impl fmt::Debug for TransitSignerConfig { } } +/// Why a Transit signer refused to initialize. +/// +/// A socket that is not there, a key version the provider has retired, and a +/// key version it has not created yet are unrelated faults with unrelated +/// remedies, and from outside the process they look the same. Each cause is +/// reported by name. None carries a provider response, a path, or key material. +#[cfg(feature = "transit")] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +#[non_exhaustive] +pub enum TransitInitializationError { + /// The HTTP client for the Unix socket could not be built. + #[error("the Transit client could not be built for the configured Unix socket")] + Client, + /// Nothing answered on the socket in time: it is missing, refused the + /// connection, or the provider did not respond within the timeout. + #[error( + "the Transit provider did not answer on the configured Unix socket (missing socket, refused connection, or timeout)" + )] + Unavailable, + /// The provider refused the key metadata read with a client-error or + /// other non-success status below 500: the request it was sent is one it + /// will not serve. + #[error( + "the Transit provider refused the key metadata read (check the token policy, mount, and key name)" + )] + Refused, + /// The provider answered the key metadata read with a server-error + /// status: it is sealed, has no active backend, or failed internally. + #[error( + "the Transit provider failed the key metadata read with a server error (for example, sealed or without an active backend)" + )] + ProviderFailed, + /// The provider's answer was oversized, not strict JSON, or lacked the + /// version fields a key read carries. + #[error("the Transit provider response is malformed or too large")] + InvalidResponse, + /// The key is not a non-derived, non-exportable ECDSA P-256 signing key + /// without plaintext backup. + #[error( + "the Transit key is not a non-derived, non-exportable ecdsa-p256 signing key without plaintext backup" + )] + Custody, + /// The pinned key version is above the key's `latest_version`. + #[error("the configured Transit key version is above the key's latest_version")] + KeyVersionNotCreated, + /// The pinned key version is below the key's `min_encryption_version`, so + /// the provider no longer signs with it. + #[error( + "the configured Transit key version is below the key's min_encryption_version and can no longer sign" + )] + KeyVersionRetired, + /// The provider holds no public key for the pinned version, or holds one + /// that differs from the pinned public JWK. + #[error( + "the Transit public key for the configured key version is missing or does not match the pinned public JWK" + )] + PublicKeyMismatch, + /// The sign-and-verify self-test did not produce a verifying signature. + #[error("the Transit sign-and-verify self-test failed")] + SelfTest, +} + +/// How a single Transit request failed, before it is attributed to the +/// operation that sent it. +#[cfg(feature = "transit")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TransitRequestFault { + Unavailable, + Refused, + ProviderFailed, + InvalidResponse, +} + +#[cfg(feature = "transit")] +impl TransitRequestFault { + fn signing_error(self) -> SigningError { + match self { + Self::Unavailable | Self::Refused | Self::ProviderFailed => { + transit_error("transit provider request failed") + } + Self::InvalidResponse => transit_error("transit provider response is invalid"), + } + } +} + +#[cfg(feature = "transit")] +impl From for TransitInitializationError { + fn from(fault: TransitRequestFault) -> Self { + match fault { + TransitRequestFault::Unavailable => Self::Unavailable, + TransitRequestFault::Refused => Self::Refused, + TransitRequestFault::ProviderFailed => Self::ProviderFailed, + TransitRequestFault::InvalidResponse => Self::InvalidResponse, + } + } +} + /// Non-exportable ES256 signer backed by the common Vault/OpenBao Transit API. /// /// Construction validates provider custody metadata, the pinned version, and @@ -595,8 +694,11 @@ pub struct TransitSigner { impl TransitSigner { /// Connect to Transit, validate custody and public identity metadata, and /// prove signing access without exporting private material. - pub async fn initialize(config: TransitSignerConfig) -> Result { - let client = build_transit_client(&config.socket_path)?; + pub async fn initialize( + config: TransitSignerConfig, + ) -> Result { + let client = build_transit_client(&config.socket_path) + .map_err(|_| TransitInitializationError::Client)?; let signer = Self { client, metadata_url: format!( @@ -619,16 +721,13 @@ impl TransitSigner { readiness: AtomicU8::new(TRANSIT_READINESS_UNKNOWN), }; let metadata = signer - .request_json(reqwest::Method::GET, &signer.metadata_url, None) - .await - .map_err(|_| transit_error("transit signer metadata is unavailable"))?; - signer - .validate_metadata(&metadata) - .map_err(|_| transit_error("transit signer metadata is invalid"))?; + .send_json(reqwest::Method::GET, &signer.metadata_url, None) + .await?; + signer.validate_metadata(&metadata)?; signer .sign(TRANSIT_SELF_TEST_MESSAGE) .await - .map_err(|_| transit_error("transit signer self-test failed"))?; + .map_err(|_| TransitInitializationError::SelfTest)?; Ok(signer) } @@ -638,6 +737,17 @@ impl TransitSigner { url: &str, body: Option<&Value>, ) -> Result { + self.send_json(method, url, body) + .await + .map_err(TransitRequestFault::signing_error) + } + + async fn send_json( + &self, + method: reqwest::Method, + url: &str, + body: Option<&Value>, + ) -> Result { let mut request = self .client .request(method, url) @@ -649,19 +759,25 @@ impl TransitSigner { let response = request .send() .await - .map_err(|_| transit_error("transit provider request failed"))?; - if !response.status().is_success() { - return Err(transit_error("transit provider request failed")); + .map_err(|_| TransitRequestFault::Unavailable)?; + let status = response.status(); + if status.is_server_error() { + return Err(TransitRequestFault::ProviderFailed); } - let bytes = read_bounded_transit_response(response).await?; - parse_json_strict(&bytes).map_err(|_| transit_error("transit provider response is invalid")) + if !status.is_success() { + return Err(TransitRequestFault::Refused); + } + let bytes = read_bounded_transit_response(response) + .await + .map_err(|_| TransitRequestFault::InvalidResponse)?; + parse_json_strict(&bytes).map_err(|_| TransitRequestFault::InvalidResponse) } - fn validate_metadata(&self, document: &Value) -> Result<(), SigningError> { + fn validate_metadata(&self, document: &Value) -> Result<(), TransitInitializationError> { let data = document .get("data") .and_then(Value::as_object) - .ok_or_else(|| transit_error("transit provider metadata is invalid"))?; + .ok_or(TransitInitializationError::InvalidResponse)?; let required_false = ["derived", "exportable", "allow_plaintext_backup"]; if data.get("type").and_then(Value::as_str) != Some("ecdsa-p256") || data.get("supports_signing").and_then(Value::as_bool) != Some(true) @@ -669,21 +785,24 @@ impl TransitSigner { .iter() .any(|field| data.get(*field).and_then(Value::as_bool) != Some(false)) { - return Err(transit_error("transit provider custody is invalid")); + return Err(TransitInitializationError::Custody); } let latest_version = data .get("latest_version") .and_then(Value::as_u64) .and_then(|value| u32::try_from(value).ok()) - .ok_or_else(|| transit_error("transit provider version is invalid"))?; + .ok_or(TransitInitializationError::InvalidResponse)?; let minimum_signing_version = data .get("min_encryption_version") .and_then(Value::as_u64) .and_then(|value| u32::try_from(value).ok()) - .ok_or_else(|| transit_error("transit provider version is invalid"))?; - if self.key_version > latest_version || self.key_version < minimum_signing_version { - return Err(transit_error("transit provider version is invalid")); + .ok_or(TransitInitializationError::InvalidResponse)?; + if self.key_version > latest_version { + return Err(TransitInitializationError::KeyVersionNotCreated); + } + if self.key_version < minimum_signing_version { + return Err(TransitInitializationError::KeyVersionRetired); } let version = self.key_version.to_string(); @@ -694,8 +813,9 @@ impl TransitSigner { .and_then(Value::as_object) .and_then(|key| key.get("public_key")) .and_then(Value::as_str) - .ok_or_else(|| transit_error("transit provider public key is invalid"))?; + .ok_or(TransitInitializationError::PublicKeyMismatch)?; validate_transit_public_key(pem, &self.public_jwk) + .map_err(|_| TransitInitializationError::PublicKeyMismatch) } fn set_readiness(&self, readiness: KeyReadiness) { @@ -1306,13 +1426,19 @@ pub fn pairwise_subject_ref_hash( /// only [`sign`] and [`verify`] handle: asking for one of those is /// unrepresentable here rather than a runtime failure. /// -/// The pair is the one SMART on FHIR Backend Services settles on (SMART App -/// Launch v2.2.0, the `client-confidential-asymmetric` profile): an -/// authorization server there has to validate only one of RS384 and ES384, so a -/// generator offering only one of them would leave adopters unable to -/// authenticate against a conformant server that chose the other. +/// ES384 and RS384 are the pair SMART on FHIR Backend Services settles on +/// (SMART App Launch v2.2.0, the `client-confidential-asymmetric` profile): an +/// authorization server there has to validate only one of them, so a generator +/// offering only one would leave adopters unable to authenticate against a +/// conformant server that chose the other. Ed25519 and ES256 are the signing +/// keys the Registry Stack runtimes themselves use. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GeneratedKeyAlgorithm { + /// EdDSA over Ed25519, written as an OKP/Ed25519 JWK whose `d` is the + /// 32-byte seed. + Ed25519, + /// ECDSA over P-256 with SHA-256, written as an EC/P-256 JWK. + Es256, /// ECDSA over P-384 with SHA-384, written as an EC/P-384 JWK. Es384, /// RSASSA-PKCS1-v1_5 with SHA-384 over a 2048-bit modulus, written as an @@ -1325,6 +1451,8 @@ impl GeneratedKeyAlgorithm { #[must_use] pub const fn signing_algorithm(self) -> SigningAlgorithm { match self { + Self::Ed25519 => SigningAlgorithm::EdDsa, + Self::Es256 => SigningAlgorithm::Es256, Self::Es384 => SigningAlgorithm::Es384, Self::Rs384 => SigningAlgorithm::Rs384, } @@ -1336,6 +1464,9 @@ impl GeneratedKeyAlgorithm { /// costs generation time on every call for no interoperability gain. const GENERATED_RSA_KEY_SIZE: AwsRsaKeySize = AwsRsaKeySize::Rsa2048; +/// Width in bytes of a P-256 scalar and of each of its point coordinates. +const P256_COORDINATE_BYTES: usize = 32; + /// Width in bytes of a P-384 scalar and of each of its point coordinates. const P384_COORDINATE_BYTES: usize = 48; @@ -1348,7 +1479,17 @@ const P384_COORDINATE_BYTES: usize = 48; /// primes, so its cost varies from call to call and belongs off a request path. pub fn generate_private_jwk(algorithm: GeneratedKeyAlgorithm) -> Result { let mut jwk = match algorithm { - GeneratedKeyAlgorithm::Es384 => generate_es384_jwk(), + GeneratedKeyAlgorithm::Ed25519 => generate_ed25519_jwk(), + GeneratedKeyAlgorithm::Es256 => generate_ecdsa_jwk( + &ECDSA_P256_SHA256_FIXED_SIGNING, + "P-256", + P256_COORDINATE_BYTES, + ), + GeneratedKeyAlgorithm::Es384 => generate_ecdsa_jwk( + &ECDSA_P384_SHA384_FIXED_SIGNING, + "P-384", + P384_COORDINATE_BYTES, + ), GeneratedKeyAlgorithm::Rs384 => generate_rs384_jwk(), }?; jwk.alg = Some(algorithm.signing_algorithm().jwa_name().to_owned()); @@ -1359,32 +1500,63 @@ pub fn generate_private_jwk(algorithm: GeneratedKeyAlgorithm) -> Result Result { - let key_pair = EcdsaKeyPair::generate(&ECDSA_P384_SHA384_FIXED_SIGNING) - .map_err(|_| CryptoError::Crypto("ES384 key generation failed"))?; +fn generate_ed25519_jwk() -> Result { + let key_pair = Ed25519KeyPair::generate() + .map_err(|_| CryptoError::Crypto("Ed25519 key generation failed"))?; + // The exported seed zeroizes on drop, like the ECDSA scalar below, and is + // the 32-byte `d` that `sign_eddsa` reads back. + let seed: Curve25519SeedBin<'_> = key_pair + .seed() + .and_then(|seed| seed.as_be_bytes()) + .map_err(|_| CryptoError::Crypto("Ed25519 private key export failed"))?; + Ok(PrivateJwk { + kty: "OKP".to_owned(), + kid: None, + alg: None, + crv: Some("Ed25519".to_owned()), + d: Some(URL_SAFE_NO_PAD.encode(seed.as_ref())), + x: Some(URL_SAFE_NO_PAD.encode(key_pair.public_key().as_ref())), + y: None, + n: None, + e: None, + p: None, + q: None, + dp: None, + dq: None, + qi: None, + }) +} + +fn generate_ecdsa_jwk( + signing: &'static EcdsaSigningAlgorithm, + curve: &str, + coordinate_bytes: usize, +) -> Result { + let key_pair = EcdsaKeyPair::generate(signing) + .map_err(|_| CryptoError::Crypto("ECDSA key generation failed"))?; // aws-lc-rs zeroizes this buffer on drop, and the base64url encoding below // lands straight in the JWK member `PrivateJwk::drop` zeroizes, so the // scalar exists in exactly those two places. let scalar: EcPrivateKeyBin<'_> = key_pair .private_key() .as_be_bytes() - .map_err(|_| CryptoError::Crypto("ES384 private key export failed"))?; + .map_err(|_| CryptoError::Crypto("ECDSA private key export failed"))?; let point = key_pair.public_key().as_ref(); // The SEC 1 uncompressed point `0x04 || x || y` that - // `p384_uncompressed_point` assembles for signing, taken apart again into - // the JWK's two coordinates. - if point.len() != 1 + 2 * P384_COORDINATE_BYTES || point[0] != 0x04 { + // `p256_uncompressed_point` and `p384_uncompressed_point` assemble for + // signing, taken apart again into the JWK's two coordinates. + if point.len() != 1 + 2 * coordinate_bytes || point[0] != 0x04 { return Err(CryptoError::Crypto( - "ES384 public point is not uncompressed", + "ECDSA public point is not uncompressed", )); } - let (x, y) = point[1..].split_at(P384_COORDINATE_BYTES); + let (x, y) = point[1..].split_at(coordinate_bytes); Ok(PrivateJwk { kty: "EC".to_owned(), kid: None, alg: None, - crv: Some("P-384".to_owned()), + crv: Some(curve.to_owned()), d: Some(URL_SAFE_NO_PAD.encode(scalar.as_ref())), x: Some(URL_SAFE_NO_PAD.encode(x)), y: Some(URL_SAFE_NO_PAD.encode(y)), @@ -2470,23 +2642,41 @@ mod tests { for field in ["derived", "exportable", "allow_plaintext_backup"] { let mut metadata = transit_metadata(&pem); metadata["data"][field] = Value::Bool(true); - cases.push(metadata); + cases.push((metadata, TransitInitializationError::Custody)); } let mut wrong_type = transit_metadata(&pem); wrong_type["data"]["type"] = Value::String("ed25519".to_owned()); - cases.push(wrong_type); + cases.push((wrong_type, TransitInitializationError::Custody)); let mut no_signing = transit_metadata(&pem); no_signing["data"]["supports_signing"] = Value::Bool(false); - cases.push(no_signing); + cases.push((no_signing, TransitInitializationError::Custody)); let mut version_too_old = transit_metadata(&pem); version_too_old["data"]["min_encryption_version"] = json!(8); - cases.push(version_too_old); + cases.push(( + version_too_old, + TransitInitializationError::KeyVersionRetired, + )); + let mut version_too_new = transit_metadata(&pem); + version_too_new["data"]["latest_version"] = json!(6); + cases.push(( + version_too_new, + TransitInitializationError::KeyVersionNotCreated, + )); + let mut version_unreadable = transit_metadata(&pem); + version_unreadable["data"]["latest_version"] = json!("nine"); + cases.push(( + version_unreadable, + TransitInitializationError::InvalidResponse, + )); let mut missing_version = transit_metadata(&pem); missing_version["data"]["keys"] .as_object_mut() .expect("keys object") .remove("7"); - cases.push(missing_version); + cases.push(( + missing_version, + TransitInitializationError::PublicKeyMismatch, + )); let other_scalar = [42_u8; 32]; let other_signing = P256SigningKey::from_slice(&other_scalar).expect("second P-256 key"); @@ -2499,9 +2689,9 @@ mod tests { .expect("second public PEM"), ); wrong_public["data"]["latest_version"] = json!(7); - cases.push(wrong_public); + cases.push((wrong_public, TransitInitializationError::PublicKeyMismatch)); - for metadata in cases { + for (metadata, expected) in cases { let replies = vec![transit_reply("GET", METADATA_PATH, None, metadata)]; let (directory, socket_path, server) = spawn_transit_mock(replies); let config = TransitSignerConfig::new( @@ -2516,12 +2706,112 @@ mod tests { let error = TransitSigner::initialize(config) .await .expect_err("unsafe Transit metadata must reject"); - assert!(error.to_string().contains("metadata is invalid")); + assert_eq!(error, expected); server.await.expect("mock Transit server completed"); drop(directory); } } + /// A socket that is not there, a provider that answers with a refusal, and + /// a key whose signature does not verify are three unrelated faults. Each + /// initialization failure names its own, so an operator restarting after a + /// Transit-side change can tell which one to fix. + #[cfg(all(unix, feature = "transit"))] + #[tokio::test] + async fn transit_signer_initialization_names_the_fault_it_met() { + const METADATA_PATH: &str = "/v1/transit/keys/key"; + const SIGN_PATH: &str = "/v1/transit/sign/key/sha2-256"; + let private = PrivateJwk::parse(P256_JWK).expect("P-256 private JWK"); + let public = private.public(); + let config = |socket_path: PathBuf| { + TransitSignerConfig::new( + socket_path, + "transit", + "key", + 7, + public.clone(), + Duration::from_secs(1), + ) + .expect("Transit config") + }; + + let absent = tempfile::tempdir().expect("temporary Transit directory"); + let missing = TransitSigner::initialize(config(absent.path().join("transit.sock"))) + .await + .expect_err("a missing socket must reject"); + assert_eq!(missing, TransitInitializationError::Unavailable); + + let refused = MockTransitReply { + method: "GET", + path: METADATA_PATH, + body: None, + status: 403, + response: br#"{"errors":["permission denied"]}"#.to_vec(), + delay: Duration::ZERO, + }; + let (directory, socket_path, server) = spawn_transit_mock(vec![refused]); + let error = TransitSigner::initialize(config(socket_path)) + .await + .expect_err("a refused metadata read must reject"); + assert_eq!(error, TransitInitializationError::Refused); + server.await.expect("refusing mock completed"); + drop(directory); + + let failing = MockTransitReply { + method: "GET", + path: METADATA_PATH, + body: None, + status: 503, + response: br#"{"errors":["Vault is sealed"]}"#.to_vec(), + delay: Duration::ZERO, + }; + let (directory, socket_path, server) = spawn_transit_mock(vec![failing]); + let error = TransitSigner::initialize(config(socket_path)) + .await + .expect_err("a failing provider must reject"); + assert_eq!(error, TransitInitializationError::ProviderFailed); + server.await.expect("failing mock completed"); + drop(directory); + + let replies = vec![ + transit_reply( + "GET", + METADATA_PATH, + None, + transit_metadata(&p256_public_pem(&private)), + ), + transit_reply( + "POST", + SIGN_PATH, + Some(transit_request(TRANSIT_SELF_TEST_MESSAGE, 7)), + json!({"data": {"signature": "vault:v7:AAAA"}}), + ), + ]; + let (directory, socket_path, server) = spawn_transit_mock(replies); + let error = TransitSigner::initialize(config(socket_path)) + .await + .expect_err("a failing self-test must reject"); + assert_eq!(error, TransitInitializationError::SelfTest); + server.await.expect("self-test mock completed"); + drop(directory); + + for fault in [ + TransitInitializationError::Unavailable, + TransitInitializationError::Refused, + TransitInitializationError::ProviderFailed, + TransitInitializationError::InvalidResponse, + TransitInitializationError::Custody, + TransitInitializationError::KeyVersionNotCreated, + TransitInitializationError::KeyVersionRetired, + TransitInitializationError::PublicKeyMismatch, + TransitInitializationError::SelfTest, + ] { + let rendered = fault.to_string(); + assert!(!rendered.contains('\n')); + assert!(!rendered.contains(absent.path().to_string_lossy().as_ref())); + } + } + #[cfg(all(unix, feature = "transit"))] #[tokio::test] async fn transit_signer_fails_closed_without_leaking_provider_responses_then_recovers() { @@ -2613,7 +2903,7 @@ mod tests { let timeout = TransitSigner::initialize(config) .await .expect_err("slow Transit metadata times out"); - assert!(timeout.to_string().contains("metadata is unavailable")); + assert_eq!(timeout, TransitInitializationError::Unavailable); server.await.expect("slow mock completed"); drop(directory); @@ -2638,7 +2928,7 @@ mod tests { let oversized = TransitSigner::initialize(config) .await .expect_err("oversized Transit metadata rejects"); - assert!(oversized.to_string().contains("metadata is unavailable")); + assert_eq!(oversized, TransitInitializationError::InvalidResponse); server.await.expect("oversized mock completed"); drop(directory); } @@ -3773,6 +4063,103 @@ mod tests { ); } + // RFC 7638 section 3.1 publishes this vector for its RSA example key, and + // RFC 8037 appendix A.3 publishes the Ed25519 vector below for its example + // key. The ES256 vector was computed independently over the RFC 7517 + // appendix A.1 P-256 key. + #[test] + fn public_jwk_thumbprint_matches_rfc_7638_rsa_vector() { + let public = PublicJwk::parse( + r#"{"kty":"RSA","alg":"RS256","e":"AQAB","n":"0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAtVT86zwu1RK7aPFFxuhDR1L6tSoc_BJECPebWKRXjBZCiFV4n3oknjhMstn64tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6Cf0h4QyQ5v-65YGjQR0_FDW2QvzqY368QQMicAtaSqzs8KJZgnYb9c7d0zgdAZHzu6qMQvRL5hajrn1n91CbOpbISD08qNLyrdkt-bFTWhAI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINHaQ-G_xBniIqbw0Ls1jF44-csFCur-kEgU8awapJzKnqDKgw"}"#, + ) + .expect("RFC 7638 RSA key parses"); + assert_eq!( + public.jkt().expect("thumbprint computes"), + "NzbLsXh8uDCcd-6MNwXF4W_7noWXFZAfHkxZsRGC9Xs" + ); + } + + #[test] + fn public_jwk_thumbprint_matches_es256_vector() { + let public = PublicJwk::parse( + r#"{"kty":"EC","alg":"ES256","crv":"P-256","kid":"ignored","x":"MKBCTNIcKUSDii11ySs3526iDZ8AiTo7Tu6KPAqv7D4","y":"4Etl6SRW2YiLUrN5vfvVHuhp7x8PxltmWWlbbM4IFyM"}"#, + ) + .expect("RFC 7517 P-256 key parses"); + assert_eq!( + public.jkt().expect("thumbprint computes"), + "cn-I_WNMClehiVp51i_0VpOENW1upEerA8sEam5hn-s" + ); + } + + #[test] + fn public_jwk_thumbprint_matches_ed25519_vector() { + let public = PublicJwk::parse( + r#"{"kty":"OKP","crv":"Ed25519","x":"11qYAYKxCrfVS_7TyWQHOg7hcvPapiMlrwIaaPcHURo"}"#, + ) + .expect("RFC 8037 Ed25519 key parses"); + assert_eq!( + public.jkt().expect("thumbprint computes"), + "kPrK_qmxVWaYVA9wwBF6Iuo3vVzz7TxHCTwXBygrS4k" + ); + } + + #[test] + fn generated_ed25519_key_signs_and_verifies_under_its_public_half() { + let private = + generate_private_jwk(GeneratedKeyAlgorithm::Ed25519).expect("Ed25519 generates"); + assert_eq!(private.kty, "OKP"); + assert_eq!(private.crv.as_deref(), Some("Ed25519")); + assert_eq!(private.alg.as_deref(), Some("EdDSA")); + assert_eq!(private.y, None, "an OKP JWK carries no y coordinate"); + let public = private.public(); + assert_eq!( + private.kid.as_deref(), + Some(public.jkt().expect("thumbprint computes").as_str()) + ); + let rendered = serde_json::to_string(&private).expect("public half renders"); + PublicJwk::parse(&rendered).expect("generated public half validates"); + assert_eq!( + URL_SAFE_NO_PAD + .decode(private.d.as_deref().expect("generated key carries d")) + .expect("d is base64url") + .len(), + 32, + "an Ed25519 seed is 32 bytes wide" + ); + + let payload = b"registry-platform-crypto generated ed25519 round trip"; + let signature = sign(payload, &private).expect("generated key signs"); + verify(payload, &signature, &public).expect("generated public half verifies"); + assert!(matches!( + verify(b"tampered", &signature, &public), + Err(CryptoError::InvalidSignature) + )); + } + + #[test] + fn generated_es256_key_signs_and_verifies_under_its_public_half() { + let private = generate_private_jwk(GeneratedKeyAlgorithm::Es256).expect("ES256 generates"); + assert_eq!(private.kty, "EC"); + assert_eq!(private.crv.as_deref(), Some("P-256")); + assert_eq!(private.alg.as_deref(), Some("ES256")); + let public = private.public(); + assert_eq!( + private.kid.as_deref(), + Some(public.jkt().expect("thumbprint computes").as_str()) + ); + let rendered = serde_json::to_string(&private).expect("public half renders"); + PublicJwk::parse(&rendered).expect("generated public half validates"); + + let payload = b"registry-platform-crypto generated es256 round trip"; + let signature = sign(payload, &private).expect("generated key signs"); + assert_eq!(signature.len(), 64, "ES256 JWS signatures are raw r || s"); + verify(payload, &signature, &public).expect("generated public half verifies"); + assert!(matches!( + verify(b"tampered", &signature, &public), + Err(CryptoError::InvalidSignature) + )); + } + #[test] fn public_jwk_thumbprint_rejects_missing_required_members() { let mut public = PrivateJwk::parse(RAW_JWK) diff --git a/crates/registry-platform-hooks/src/delivery/service.rs b/crates/registry-platform-hooks/src/delivery/service.rs index b37eac3876..92c6d7af7a 100644 --- a/crates/registry-platform-hooks/src/delivery/service.rs +++ b/crates/registry-platform-hooks/src/delivery/service.rs @@ -678,14 +678,21 @@ impl DeliveryService { // lease's own fate. A lease that did commit sends nothing from // here and is answered when it expires; one that rolled back, or // one whose fate cannot be read, is answered now, since a second - // interrupted answer is harmless and none is not. + // interrupted answer is harmless and none is not. A lease of + // unknown fate may still be held, so its answer claims no + // database state. let lease_committed = self.transition_committed(&started, Some(lease_token)).await; self.record_resolved(committed).await; - if lease_committed != Some(true) { + let disposition = match lease_committed { + Some(true) => None, + Some(false) => Some(DeliveryAuditDisposition::RetryPending), + None => Some(DeliveryAuditDisposition::Unknown), + }; + if let Some(disposition) = disposition { let interrupted = PendingAudit { phase: DeliveryAuditPhase::Terminal, outcome: DeliveryAuditOutcome::WorkerInterrupted, - disposition: DeliveryAuditDisposition::RetryPending, + disposition, ..started }; // A refused entry has already stopped the product's writer, @@ -3956,9 +3963,9 @@ mod tests { recorded.contains(&( DeliveryAuditPhase::Terminal, DeliveryAuditOutcome::WorkerInterrupted, - DeliveryAuditDisposition::RetryPending, + DeliveryAuditDisposition::Unknown, )), - "the attempt of unknown fate is answered: {recorded:?}" + "the attempt of unknown fate is answered without claiming a retry: {recorded:?}" ); assert_eq!(recorded.len(), 3, "{recorded:?}"); } diff --git a/crates/registry-platform-httputil/README.md b/crates/registry-platform-httputil/README.md index 8bc2d2a4c1..5dee2d8013 100644 --- a/crates/registry-platform-httputil/README.md +++ b/crates/registry-platform-httputil/README.md @@ -84,6 +84,9 @@ async fn fetch_document() -> Result, Box> { - `ValidatedFetchUrl::immediate_get` applies a 30 second request timeout and a 10 second connect timeout by default. Use `immediate_get_with_timeout` or `RequestBuilder::timeout` for a tighter per-call bound. +- `ValidatedFetchUrl::immediate_get_with_additional_roots` trusts the given + certificate authorities beside the system roots for that one request. It + adds trust anchors and never removes one or disables hostname verification. - Requests built from `ValidatedFetchUrl` disable redirects and ignore proxy environment variables, so a redirect response cannot move the fetch to a URL that bypassed validation. diff --git a/crates/registry-platform-httputil/src/lib.rs b/crates/registry-platform-httputil/src/lib.rs index 3e164a29fd..d6e5f61df9 100644 --- a/crates/registry-platform-httputil/src/lib.rs +++ b/crates/registry-platform-httputil/src/lib.rs @@ -771,6 +771,20 @@ impl ValidatedFetchUrl { self.immediate_request_with_timeout(reqwest::Method::GET, timeout) } + /// Build an immediate GET that trusts `additional_roots` beside the + /// platform roots. + /// + /// For an endpoint served under a private certificate authority that the + /// deployment names explicitly. The roots are added to the platform roots, + /// never substituted for them, and certificate verification stays on. + pub fn immediate_get_with_additional_roots( + &self, + timeout: Duration, + additional_roots: &[reqwest::Certificate], + ) -> Result { + self.immediate_request_trusting(reqwest::Method::GET, timeout, additional_roots) + } + /// Build an immediate POST request from this validated URL. /// /// This is useful for token endpoints and other SSRF-sensitive POST @@ -853,22 +867,33 @@ impl ValidatedFetchUrl { &self, method: reqwest::Method, timeout: Duration, + ) -> Result { + self.immediate_request_trusting(method, timeout, &[]) + } + + fn immediate_request_trusting( + &self, + method: reqwest::Method, + timeout: Duration, + additional_roots: &[reqwest::Certificate], ) -> Result { let host = self .url .host_str() .ok_or(FetchUrlError::MissingHost)? .to_string(); - let client = reqwest::Client::builder() + let mut builder = reqwest::Client::builder() .timeout(timeout) .connect_timeout(timeout.min(DEFAULT_VALIDATED_FETCH_CONNECT_TIMEOUT)) .redirect(reqwest::redirect::Policy::none()) .no_proxy() .retry(reqwest::retry::never()) .pool_max_idle_per_host(0) - .resolve_to_addrs(&host, &self.resolved_addrs) - .build() - .map_err(FetchUrlError::ClientBuild)?; + .resolve_to_addrs(&host, &self.resolved_addrs); + for root in additional_roots { + builder = builder.add_root_certificate(root.clone()); + } + let client = builder.build().map_err(FetchUrlError::ClientBuild)?; Ok(client.request(method, self.url.clone()).timeout(timeout)) } } @@ -1070,6 +1095,91 @@ mod tests { addr } + /// A TLS listener on loopback whose certificate a private certificate + /// authority signed, answering every request with `private-ca`. Returns + /// its address and the authority's certificate. + async fn serve_under_private_ca() -> (SocketAddr, reqwest::Certificate) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio_rustls::rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; + + let mut authority = + rcgen::CertificateParams::new(Vec::::new()).expect("private CA parameters"); + authority.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let authority_key = rcgen::KeyPair::generate().expect("private CA key"); + let authority = authority + .self_signed(&authority_key) + .expect("private CA certificate"); + let server_key = rcgen::KeyPair::generate().expect("server key"); + let server = rcgen::CertificateParams::new(vec!["127.0.0.1".to_owned()]) + .expect("server parameters") + .signed_by(&server_key, &authority, &authority_key) + .expect("the private CA signs the server certificate"); + let server_config = tokio_rustls::rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert( + vec![server.der().clone()], + PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(server_key.serialize_der())), + ) + .expect("TLS server configuration"); + let acceptor = tokio_rustls::TlsAcceptor::from(std::sync::Arc::new(server_config)); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind TLS test server"); + let addr = listener.local_addr().expect("TLS test server address"); + tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let acceptor = acceptor.clone(); + tokio::spawn(async move { + let Ok(mut stream) = acceptor.accept(stream).await else { + return; + }; + let mut request = Vec::new(); + let mut chunk = [0_u8; 512]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + match stream.read(&mut chunk).await { + Ok(0) | Err(_) => return, + Ok(read) => request.extend_from_slice(&chunk[..read]), + } + } + let _ = stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\nConnection: close\r\n\r\nprivate-ca", + ) + .await; + let _ = stream.shutdown().await; + }); + } + }); + let authority = reqwest::Certificate::from_der(authority.der()) + .expect("the private CA certificate parses"); + (addr, authority) + } + + #[tokio::test] + async fn a_pinned_get_trusts_a_private_ca_only_when_it_is_named() { + let (addr, authority) = serve_under_private_ca().await; + let url = reqwest::Url::parse(&format!("https://{addr}/keys")).expect("target URL"); + let validated = FetchUrlPolicy::dev() + .validate_dns_pinned_for_immediate_fetch(&url) + .expect("loopback HTTPS target"); + + validated + .immediate_get_with_timeout(Duration::from_secs(5)) + .expect("pinned request") + .send() + .await + .expect_err("a certificate a private CA signed is refused by default"); + + let response = validated + .immediate_get_with_additional_roots(Duration::from_secs(5), &[authority]) + .expect("pinned request trusting the private CA") + .send() + .await + .expect("a certificate the named private CA signed is accepted"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.text().await.expect("response body"), "private-ca"); + } + #[tokio::test] async fn outbound_client_does_not_follow_redirects() { let base = serve( diff --git a/crates/registry-platform-oidc/src/lib.rs b/crates/registry-platform-oidc/src/lib.rs index 84e7628940..7daeb3573a 100644 --- a/crates/registry-platform-oidc/src/lib.rs +++ b/crates/registry-platform-oidc/src/lib.rs @@ -23,7 +23,9 @@ use base64::Engine; use jsonwebtoken::errors::ErrorKind as JwtErrorKind; use jsonwebtoken::jwk::{AlgorithmParameters, EllipticCurve, Jwk, JwkSet, KeyAlgorithm}; use jsonwebtoken::{decode, decode_header, Algorithm, DecodingKey, Validation}; -use registry_platform_httputil::{read_bounded, FetchUrlError, FetchUrlPolicy}; +use registry_platform_httputil::{ + read_bounded, FetchUrlError, FetchUrlPolicy, DEFAULT_VALIDATED_FETCH_TIMEOUT, +}; use reqwest::Url; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; @@ -207,6 +209,7 @@ enum JwksSource { Http { jwks_uri: String, fetch_url_policy: FetchUrlPolicy, + additional_roots: Vec, }, Static(JwkSet), } @@ -237,6 +240,34 @@ impl fmt::Debug for JwksFetcher { } } +/// Parse an operator-supplied static JWKS document for +/// [`JwksFetcher::new_static`]: a non-empty key set of RSA or elliptic-curve +/// keys, each with a non-empty `kid` no other key in the set repeats. A +/// symmetric key or an unnamed key is refused here, at startup, rather than +/// when a token first selects it. +pub fn parse_static_jwks(bytes: &[u8]) -> Result { + let jwks: JwkSet = serde_json::from_slice(bytes).map_err(|_| OidcError::Parse)?; + if jwks.keys.is_empty() { + return Err(OidcError::EmptyKeySet); + } + let mut kids = BTreeSet::new(); + let all_named_asymmetric = jwks.keys.iter().all(|key| { + matches!( + key.algorithm, + AlgorithmParameters::RSA(_) | AlgorithmParameters::EllipticCurve(_) + ) && key + .common + .key_id + .as_deref() + .is_some_and(|kid| !kid.is_empty() && kids.insert(kid)) + }); + if all_named_asymmetric { + Ok(jwks) + } else { + Err(OidcError::InvalidJwk) + } +} + impl JwksFetcher { #[must_use] pub fn new(jwks_uri: String, config: JwksFetcherConfig) -> Self { @@ -248,11 +279,28 @@ impl JwksFetcher { jwks_uri: String, config: JwksFetcherConfig, fetch_url_policy: FetchUrlPolicy, + ) -> Self { + Self::new_trusting_additional_roots(jwks_uri, config, fetch_url_policy, Vec::new()) + } + + /// Build a fetcher whose key-set requests also trust `additional_roots`, + /// beside the platform roots. + /// + /// For an issuer whose key set is served under a private certificate + /// authority the deployment names explicitly. The roots are added, never + /// substituted, and certificate verification stays on. + #[must_use] + pub fn new_trusting_additional_roots( + jwks_uri: String, + config: JwksFetcherConfig, + fetch_url_policy: FetchUrlPolicy, + additional_roots: Vec, ) -> Self { Self { source: JwksSource::Http { jwks_uri, fetch_url_policy, + additional_roots, }, config, state: RwLock::new(JwksState::default()), @@ -618,13 +666,17 @@ impl JwksFetcher { JwksSource::Http { jwks_uri, fetch_url_policy, + additional_roots, } => { let url = Url::parse(jwks_uri).map_err(|_| OidcError::InvalidUrl)?; let validated_url = fetch_url_policy .validate_for_immediate_fetch_with_timeout(&url, self.config.request_timeout) .await?; let resp = validated_url - .immediate_get()? + .immediate_get_with_additional_roots( + DEFAULT_VALIDATED_FETCH_TIMEOUT, + additional_roots, + )? .timeout(self.config.request_timeout) .send() .await @@ -4036,4 +4088,48 @@ mod tests { assert_eq!(document.issuer, "https://issuer.example"); assert_eq!(document.jwks_uri, "http://127.0.0.1/jwks"); } + + #[test] + fn a_static_key_set_requires_unique_named_asymmetric_keys() { + let keys = parse_static_jwks( + br#"{"keys":[{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"},{"kty":"EC","kid":"two","crv":"P-256","x":"f83OJ3D2xF4k1JQWctzS0r8uXH6Gz-l4WfXccj5WHv0","y":"x_FEzRu9dVvZt2pSuGQgH7u9tZxU7I5oUJu-4G8Azjo"}]}"#, + ) + .expect("RSA and EC keys with distinct kids"); + assert_eq!(keys.keys.len(), 2); + + for (reason, invalid, expected) in [ + ("no keys member", br#"{}"#.as_slice(), "parse"), + ("not JSON", b"not-json", "parse"), + ("an empty key set", br#"{"keys":[]}"#, "empty"), + ( + "a symmetric key", + br#"{"keys":[{"kty":"oct","kid":"one","k":"AA"}]}"#, + "jwk", + ), + ( + "a key without kid", + br#"{"keys":[{"kty":"RSA","n":"AQAB","e":"AQAB"}]}"#, + "jwk", + ), + ( + "an empty kid", + br#"{"keys":[{"kty":"RSA","kid":"","n":"AQAB","e":"AQAB"}]}"#, + "jwk", + ), + ( + "a repeated kid", + br#"{"keys":[{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"},{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"}]}"#, + "jwk", + ), + ] { + let error = parse_static_jwks(invalid).expect_err(reason); + let kind = match error { + OidcError::Parse => "parse", + OidcError::EmptyKeySet => "empty", + OidcError::InvalidJwk => "jwk", + other => panic!("{reason}: unexpected {other:?}"), + }; + assert_eq!(kind, expected, "{reason}"); + } + } } diff --git a/crates/registry-relay-v2/Cargo.toml b/crates/registry-relay-v2/Cargo.toml index f713241762..1d805cbba2 100644 --- a/crates/registry-relay-v2/Cargo.toml +++ b/crates/registry-relay-v2/Cargo.toml @@ -18,7 +18,7 @@ workspace = true [features] default = [] -schema = ["dep:schemars", "registry-platform-audit/schema"] +schema = ["dep:schemars", "registry-platform-audit/schema", "registry-platform-config/schema"] tooling = ["dep:tempfile", "registry-platform-sqlite/fixture"] [dependencies] diff --git a/crates/registry-relay-v2/src/authoring.rs b/crates/registry-relay-v2/src/authoring.rs index d67e420256..106996fa41 100644 --- a/crates/registry-relay-v2/src/authoring.rs +++ b/crates/registry-relay-v2/src/authoring.rs @@ -11,7 +11,7 @@ use crate::{ compiler::{compile_contract_with_governed_files, GovernedFileSet}, contract::{ contract_has_protected_access, runtime_cursor_configuration_is_valid, RegistryContract, - RelayRuntime, + RelayRuntime, RELAY_RUNTIME_API_VERSION, RELAY_RUNTIME_KIND, }, model::{CompileProfile, CompileReport, Diagnostic, DiagnosticSeverity}, }; @@ -31,13 +31,9 @@ pub fn check_project_documents( ) -> CompileReport { let contract = match RegistryContract::parse_yaml(registry_yaml) { Ok(contract) => contract, - Err(_) => { + Err(error) => { return CompileReport { - diagnostics: vec![diagnostic( - "contract.yaml_invalid", - "registry.yaml", - "the governed contract is not valid strict YAML", - )], + diagnostics: vec![error.diagnostic()], }; } }; @@ -45,12 +41,12 @@ pub fn check_project_documents( let runtime = match runtime_yaml { Some(yaml) => match RelayRuntime::parse_yaml(yaml) { Ok(runtime) => Some(runtime), - Err(_) => { + Err(refusal) => { return CompileReport { diagnostics: vec![diagnostic( "runtime.yaml_invalid", - "runtime.yaml", - "the deployment binding is not valid strict YAML", + &runtime_location(refusal.field()), + refusal.message(), )], }; } @@ -85,7 +81,7 @@ pub(crate) fn validate_runtime( let Some(runtime) = runtime else { return diagnostics; }; - if runtime.api_version != "relay.registrystack.org/v2alpha1" || runtime.kind != "RelayRuntime" { + if runtime.api_version != RELAY_RUNTIME_API_VERSION || runtime.kind != RELAY_RUNTIME_KIND { diagnostics.push(diagnostic( "runtime.identity_invalid", "runtime.yaml", @@ -108,10 +104,10 @@ pub(crate) fn validate_runtime( "a Registry with a paginated data or resource-metadata list requires an opaque-cursor key and age bound", )); } - if contract_has_protected_access(contract) && runtime.authentication.issuer.is_none() { + if contract_has_protected_access(contract) && runtime.authentication.oidc.is_none() { diagnostics.push(diagnostic( "runtime.issuer_missing", - "runtime.yaml.authentication.issuer", + "runtime.yaml.authentication.oidc", "a Registry with protected operations requires one configured issuer", )); } @@ -129,6 +125,15 @@ pub(crate) fn validate_runtime( diagnostics } +/// The diagnostic location of a refused runtime field: `runtime.yaml` for the +/// whole document, otherwise `runtime.yaml.`. +pub(crate) fn runtime_location(field: &str) -> String { + match field { + "" | "/" => "runtime.yaml".to_owned(), + field => format!("runtime.yaml.{field}"), + } +} + fn diagnostic(code: &str, location: &str, message: &str) -> Diagnostic { Diagnostic { severity: DiagnosticSeverity::Error, @@ -151,12 +156,12 @@ mod tests { #[test] fn runtime_source_bindings_are_checked_from_in_memory_documents() { - let runtime = r#"apiVersion: relay.registrystack.org/v2alpha1 -kind: RelayRuntime -server: {bind: '127.0.0.1:18080'} -packagePath: package + let runtime = r#"apiVersion: registry.registrystack.org/relay-runtime/v1alpha1 +kind: RelayRuntimeConfig +listener: {bind: '127.0.0.1:18080'} +package: {root: /srv/relay/package} +secretProviders: {environment: {}} sources: {other: {path: fixture.sqlite}} -authentication: {issuer: null} audit: {path: var/audit.jsonl} limits: {requestTimeoutMilliseconds: 1000, concurrentQueries: 1} "#; @@ -166,4 +171,26 @@ limits: {requestTimeoutMilliseconds: 1000, concurrentQueries: 1} .iter() .any(|item| item.code == "runtime.source_binding_mismatch")); } + + #[test] + fn a_refused_runtime_reports_the_field_and_its_replacement() { + let runtime = "apiVersion: registry.registrystack.org/relay-runtime/v1alpha1\nkind: RelayRuntimeConfig\nserver: {bind: '127.0.0.1:18080'}\n"; + let report = check_project_documents(valid_contract(), Some(runtime), &governed_files()); + let [diagnostic] = report.diagnostics.as_slice() else { + panic!("one refusal expected: {:?}", report.diagnostics); + }; + assert_eq!(diagnostic.code, "runtime.yaml_invalid"); + assert_eq!(diagnostic.location, "runtime.yaml.server"); + assert!( + diagnostic.message.contains("listener.bind"), + "{diagnostic:?}" + ); + + let secret_env = "apiVersion: registry.registrystack.org/relay-runtime/v1alpha1\nkind: RelayRuntimeConfig\nlistener: {bind: '127.0.0.1:18080'}\npackage: {root: /srv/relay/package}\nsecretProviders: {file: {root: /run/secrets/relay}}\nsources: {db: {path: fixture.sqlite}}\naudit: {path: var/audit.jsonl}\ncursor: {integrityKeyRef: secret:env/KEY, maximumAgeSeconds: 300}\nlimits: {requestTimeoutMilliseconds: 1000, concurrentQueries: 1}\n"; + let report = check_project_documents(valid_contract(), Some(secret_env), &governed_files()); + assert_eq!( + report.diagnostics[0].location, + "runtime.yaml.cursor.integrityKeyRef" + ); + } } diff --git a/crates/registry-relay-v2/src/cli.rs b/crates/registry-relay-v2/src/cli.rs index 3d31274036..f2fd839f13 100644 --- a/crates/registry-relay-v2/src/cli.rs +++ b/crates/registry-relay-v2/src/cli.rs @@ -6,7 +6,6 @@ use std::path::PathBuf; use clap::{CommandFactory, Parser, Subcommand}; const DEFAULT_HEALTHCHECK_URL: &str = "http://127.0.0.1:8080/health"; -const DEFAULT_RUNTIME_PATH: &str = "/etc/relay/runtime.yaml"; #[derive(Debug, Parser)] #[command( @@ -24,9 +23,10 @@ pub enum Command { /// Validate the sealed package and every deployment dependency without /// taking the listener socket. Check { - /// Strict deployment binding for the sealed package and local resources. - #[arg(long, env = "RELAY_RUNTIME", default_value = DEFAULT_RUNTIME_PATH)] - runtime: PathBuf, + /// Absolute path of the runtime configuration that binds the sealed + /// package to local resources. + #[arg(long = "runtime-config", value_name = "FILE")] + runtime_config: PathBuf, /// Also prove the configured audit file resolves inside this absolute /// directory, which the deployment declares persistent. /// @@ -41,9 +41,10 @@ pub enum Command { }, /// Verify and activate one sealed Registry package, then serve it. Serve { - /// Strict deployment binding for the sealed package and local resources. - #[arg(long, env = "RELAY_RUNTIME")] - runtime: PathBuf, + /// Absolute path of the runtime configuration that binds the sealed + /// package to local resources. + #[arg(long = "runtime-config", value_name = "FILE")] + runtime_config: PathBuf, }, /// Probe an unauthenticated Relay liveness endpoint. Healthcheck { @@ -92,6 +93,8 @@ mod tests { let parsed = Cli::try_parse_from([ "relay", "check", + "--runtime-config", + "/etc/relay/runtime.yaml", "--require-audit-under", "/var/lib/relay/audit", ]) @@ -109,7 +112,13 @@ mod tests { ); // The claim is an addition to the existing check, never a replacement. - let plain = Cli::try_parse_from(["relay", "check"]).expect("check parses without it"); + let plain = Cli::try_parse_from([ + "relay", + "check", + "--runtime-config", + "/etc/relay/runtime.yaml", + ]) + .expect("check parses without it"); assert!(matches!( plain.command, Command::Check { @@ -120,19 +129,28 @@ mod tests { } #[test] - fn check_uses_the_official_container_runtime_path_by_default() { - let command = command(); - let check = command - .find_subcommand("check") - .expect("check subcommand exists"); - let runtime = check - .get_arguments() - .find(|argument| argument.get_id() == "runtime") - .expect("check runtime argument exists"); - assert_eq!(runtime.get_env(), Some(OsStr::new("RELAY_RUNTIME"))); - assert_eq!( - runtime.get_default_values(), - [OsStr::new(DEFAULT_RUNTIME_PATH)] - ); + fn the_runtime_configuration_is_named_explicitly_on_every_command() { + for subcommand in ["check", "serve"] { + let command = command(); + let found = command + .find_subcommand(subcommand) + .expect("subcommand exists"); + let runtime = found + .get_arguments() + .find(|argument| argument.get_id() == "runtime_config") + .expect("runtime configuration argument exists"); + assert_eq!(runtime.get_long(), Some("runtime-config")); + assert_eq!(runtime.get_env(), None, "{subcommand}"); + assert!(runtime.get_default_values().is_empty(), "{subcommand}"); + assert!(runtime.is_required_set(), "{subcommand}"); + assert!(Cli::try_parse_from(["relay", subcommand]).is_err()); + assert!(Cli::try_parse_from([ + "relay", + subcommand, + "--runtime", + "/etc/relay/runtime.yaml" + ]) + .is_err()); + } } } diff --git a/crates/registry-relay-v2/src/compiler.rs b/crates/registry-relay-v2/src/compiler.rs index 94fc079509..a7800cc991 100644 --- a/crates/registry-relay-v2/src/compiler.rs +++ b/crates/registry-relay-v2/src/compiler.rs @@ -148,13 +148,8 @@ pub fn compile_yaml( observed: &[ObservedSourceSchema], profile: CompileProfile, ) -> Result { - let contract = RegistryContract::parse_yaml(yaml).map_err(|_| CompileReport { - diagnostics: vec![Diagnostic { - severity: DiagnosticSeverity::Error, - code: "contract.yaml_invalid".into(), - location: "registry.yaml".into(), - message: "the governed contract is not valid strict YAML".into(), - }], + let contract = RegistryContract::parse_yaml(yaml).map_err(|error| CompileReport { + diagnostics: vec![error.diagnostic()], })?; compile_contract(&contract, observed, profile) } diff --git a/crates/registry-relay-v2/src/contract.rs b/crates/registry-relay-v2/src/contract.rs index 20e490d0fe..eb2e17c162 100644 --- a/crates/registry-relay-v2/src/contract.rs +++ b/crates/registry-relay-v2/src/contract.rs @@ -3,11 +3,14 @@ use std::collections::HashSet; use std::fmt; -use std::net::SocketAddr; use std::ops::Deref; use std::path::{Component, Path, PathBuf}; use registry_platform_audit::{AuditDestination, AuditDestinationError, AuditDestinationKind}; +use registry_platform_config::{ + ConfigBlockError, JwksSource, ListenerConfig, PackageConfig, RemovedKey, RuntimeConfigError, + RuntimeConfigLoader, RuntimeEnvelope, SecretProvidersConfig, +}; use serde::de::{self, MapAccess, Visitor}; use serde::ser::SerializeMap; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -17,7 +20,6 @@ use url::Url; const OIDC_DISCOVERY_SUFFIX: &str = "/.well-known/openid-configuration"; pub(crate) const MAXIMUM_ACCESS_PROFILE_IDENTIFIER_BYTES: usize = 128; -pub(crate) const MAXIMUM_RUNTIME_BYTES: u64 = 1024 * 1024; // A JSON string byte can expand to six bytes (`\u00XX`). Capping the authored // audience at 8 KiB therefore leaves more than 16,000 bytes in the 64 KiB @@ -140,15 +142,64 @@ impl<'de, T: Deserialize<'de>> Deserialize<'de> for OrderedMap { } #[derive(Debug, Error)] -#[error("contract YAML is not valid")] -pub struct ContractParseError { - #[source] - source: serde_norway::Error, +pub enum ContractParseError { + #[error("contract YAML is not valid")] + Invalid(#[source] serde_norway::Error), + /// The contract holds a `${...}` environment expression, or text the + /// expression check cannot read. Substitution applies to `runtime.yaml` + /// only, so the reviewed contract is the one that runs. + #[error("{0}")] + EnvironmentExpression(registry_platform_config::RuntimeConfigError), } impl ContractParseError { - pub fn detail(&self) -> &serde_norway::Error { - &self.source + pub fn detail(&self) -> Option<&serde_norway::Error> { + match self { + Self::Invalid(source) => Some(source), + Self::EnvironmentExpression(_) => None, + } + } + + /// The dotted contract field holding an environment expression. + pub fn environment_expression_field(&self) -> Option<&str> { + match self { + Self::EnvironmentExpression(error) if !Self::is_unreadable(error) => { + Some(error.field()) + } + _ => None, + } + } + + fn is_unreadable(error: ®istry_platform_config::RuntimeConfigError) -> bool { + error.kind() == registry_platform_config::RuntimeConfigErrorKind::AuthoredSyntax + } + + /// The single diagnostic every authoring surface reports for this + /// refusal, located in `registry.yaml`. + pub(crate) fn diagnostic(&self) -> crate::model::Diagnostic { + let (code, location, message) = match self { + Self::EnvironmentExpression(error) if Self::is_unreadable(error) => ( + "contract.yaml_invalid", + "registry.yaml".to_owned(), + "the governed contract is not valid strict YAML".to_owned(), + ), + Self::Invalid(_) => ( + "contract.yaml_invalid", + "registry.yaml".to_owned(), + "the governed contract is not valid strict YAML".to_owned(), + ), + Self::EnvironmentExpression(error) => ( + "contract.environment_expression", + format!("registry.yaml.{}", error.field()), + error.message().to_owned(), + ), + }; + crate::model::Diagnostic { + severity: crate::model::DiagnosticSeverity::Error, + code: code.into(), + location, + message, + } } } @@ -194,7 +245,10 @@ pub struct Publication { impl RegistryContract { pub fn parse_yaml(input: &str) -> Result { - serde_norway::from_str(input).map_err(|source| ContractParseError { source }) + let contract = serde_norway::from_str(input).map_err(ContractParseError::Invalid)?; + registry_platform_config::reject_environment_expressions_in_authored_yaml(input) + .map_err(ContractParseError::EnvironmentExpression)?; + Ok(contract) } } @@ -1232,16 +1286,99 @@ pub enum Visibility { OperatorOnly, } +/// `apiVersion` of the Relay runtime configuration. +pub const RELAY_RUNTIME_API_VERSION: &str = "registry.registrystack.org/relay-runtime/v1alpha1"; +/// `kind` of the Relay runtime configuration. +pub const RELAY_RUNTIME_KIND: &str = "RelayRuntimeConfig"; + +/// The envelope every Relay runtime configuration carries. +pub const RELAY_RUNTIME_ENVELOPE: RuntimeEnvelope = RuntimeEnvelope { + api_version: RELAY_RUNTIME_API_VERSION, + kind: RELAY_RUNTIME_KIND, +}; + +/// Keys an earlier Relay runtime configuration accepted, each refused with the +/// key that replaced it. +pub const RELAY_REMOVED_RUNTIME_KEYS: &[RemovedKey] = &[ + RemovedKey { + path: "server", + replacement: "declare the listener address as listener.bind", + }, + RemovedKey { + path: "packagePath", + replacement: "declare package.root as the absolute path of the sealed package", + }, + RemovedKey { + path: "audit.integrityKeyRef", + replacement: "remove it; Relay audit entries are not hash-chained and no audit key is read", + }, + RemovedKey { + path: "audit.sink", + replacement: "declare audit.path as the audit file, absolute or relative to the runtime \ + configuration directory", + }, + RemovedKey { + path: "authentication.issuer", + replacement: "declare authentication.oidc with issuer, audience, jwksSource, \ + tokenTypes and algorithms", + }, +]; + +/// A refused Relay runtime configuration. It names the field and never a +/// configured value or a filesystem path. +#[derive(Clone, Debug, Error, PartialEq, Eq)] +#[error("{message}")] +pub struct RuntimeRefusal { + field: String, + message: String, +} + +impl RuntimeRefusal { + fn new(field: &str, message: impl Into) -> Self { + Self { + field: field.to_owned(), + message: message.into(), + } + } + + /// The dotted field the refusal concerns; `/` for the whole document. + #[must_use] + pub fn field(&self) -> &str { + &self.field + } + + #[must_use] + pub fn message(&self) -> &str { + &self.message + } +} + +impl From for RuntimeRefusal { + fn from(error: RuntimeConfigError) -> Self { + Self::new(error.field(), error.message()) + } +} + +impl From for RuntimeRefusal { + fn from(error: ConfigBlockError) -> Self { + Self::new(error.field(), error.to_string()) + } +} + /// Deployment-local bindings. No governed field is accepted here. #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct RelayRuntime { + #[cfg_attr(feature = "schema", schemars(extend("const" = RELAY_RUNTIME_API_VERSION)))] pub api_version: String, + #[cfg_attr(feature = "schema", schemars(extend("const" = RELAY_RUNTIME_KIND)))] pub kind: String, - pub server: ServerRuntime, - pub package_path: String, + pub listener: ListenerConfig, + pub package: PackageConfig, + pub secret_providers: SecretProvidersConfig, pub sources: OrderedMap, + #[serde(default)] pub authentication: AuthenticationRuntime, pub audit: AuditRuntime, #[serde(default)] @@ -1254,92 +1391,101 @@ pub struct RelayRuntime { } impl RelayRuntime { - pub fn parse_yaml(input: &str) -> Result { - let runtime: Self = - serde_norway::from_str(input).map_err(|source| ContractParseError { source })?; - if let Err(error) = runtime.audit.check_shape() { - return Err(ContractParseError { - source: ::custom(error), - }); - } - if runtime.is_valid() { - Ok(runtime) - } else { - Err(ContractParseError { - source: ::custom( - "the deployment binding violates the closed runtime profile", - ), - }) - } + /// The shared loader under Relay's envelope, removed keys, and trusted + /// ownership rule. + #[must_use] + pub const fn loader() -> RuntimeConfigLoader { + RuntimeConfigLoader::new(RELAY_RUNTIME_ENVELOPE) + .removed_keys(RELAY_REMOVED_RUNTIME_KEYS) + .require_trusted_ownership() } - fn is_valid(&self) -> bool { - if self.api_version != "relay.registrystack.org/v2alpha1" - || self.kind != "RelayRuntime" - || self.server.bind.parse::().is_err() - || self.package_path.trim().is_empty() - || self.sources.is_empty() - || self.audit.check_shape().is_err() - || self.limits.request_timeout_milliseconds == 0 - || self.limits.request_timeout_milliseconds > 120_000 - || self.limits.concurrent_queries == 0 - || self.limits.concurrent_queries > 256 - { - return false; + /// Parse an in-memory runtime document with an empty environment: an + /// environment expression takes its `:-` default, and one without a + /// default is refused. Authoring tools check unsaved buffers this way, so + /// what they report never depends on the checking process's environment. + pub fn parse_yaml(input: &str) -> Result { + Self::parse_yaml_with(input, |_| None) + } + + /// Parse an in-memory runtime document, substituting environment + /// expressions from `lookup`. + pub fn parse_yaml_with( + input: &str, + lookup: impl Fn(&str) -> Option, + ) -> Result { + let runtime: Self = Self::loader().parse_str(input, lookup)?.config; + runtime.check()?; + Ok(runtime) + } + + /// Check every rule the shared blocks and the closed Relay profile place + /// on a parsed document. + pub fn check(&self) -> Result<(), RuntimeRefusal> { + self.package.check()?; + self.secret_providers.check()?; + self.audit.check_shape().map_err(audit_refusal)?; + if let Some(cursor) = &self.cursor { + self.secret_providers + .check_reference("cursor.integrityKeyRef", &cursor.integrity_key_ref)?; + if cursor.maximum_age_seconds == 0 || cursor.maximum_age_seconds > 86_400 { + return Err(RuntimeRefusal::new( + "cursor.maximumAgeSeconds", + "cursor.maximumAgeSeconds must be between 1 and 86400", + )); + } } - if self - .sources - .iter() - .any(|(id, source)| !valid_runtime_id(id) || source.path.trim().is_empty()) + if self.sources.is_empty() { + return Err(RuntimeRefusal::new( + "sources", + "sources must bind at least one governed source", + )); + } + for (id, source) in self.sources.iter() { + if !valid_runtime_id(id) || source.path.trim().is_empty() { + return Err(RuntimeRefusal::new( + "sources", + "each source binding needs a lowercase identifier and a non-empty path", + )); + } + } + if self.limits.request_timeout_milliseconds == 0 + || self.limits.request_timeout_milliseconds > 120_000 { - return false; + return Err(RuntimeRefusal::new( + "limits.requestTimeoutMilliseconds", + "limits.requestTimeoutMilliseconds must be between 1 and 120000", + )); } - if self.cursor.as_ref().is_some_and(|cursor| { - !valid_secret_reference(&cursor.integrity_key_ref) - || cursor.maximum_age_seconds == 0 - || cursor.maximum_age_seconds > 86_400 - }) { - return false; + if self.limits.concurrent_queries == 0 || self.limits.concurrent_queries > 256 { + return Err(RuntimeRefusal::new( + "limits.concurrentQueries", + "limits.concurrentQueries must be between 1 and 256", + )); } if self.quotas.as_ref().is_some_and(|quota| { quota.requests_per_minute == 0 || quota.burst == 0 || quota.burst > 100_000 }) { - return false; + return Err(RuntimeRefusal::new( + "quotas", + "quotas.requestsPerMinute must be positive and quotas.burst between 1 and 100000", + )); } if self .shutdown .as_ref() .is_some_and(|shutdown| shutdown.grace_period_milliseconds == 0) { - return false; + return Err(RuntimeRefusal::new( + "shutdown.gracePeriodMilliseconds", + "shutdown.gracePeriodMilliseconds must be positive", + )); } - self.authentication - .issuer - .as_ref() - .is_none_or(|issuer| issuer.profile().is_some()) - } -} - -fn valid_secret_reference(value: &str) -> bool { - if let Some(name) = value.strip_prefix("secret:env/") { - let bytes = name.as_bytes(); - return matches!(bytes.first(), Some(b'A'..=b'Z')) - && bytes.len() <= 128 - && bytes[1..] - .iter() - .all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit() || *byte == b'_'); - } - if let Some(name) = value.strip_prefix("secret:file/") { - let bytes = name.as_bytes(); - return matches!(bytes.first(), Some(b'a'..=b'z')) - && bytes.len() <= 128 - && bytes[1..].iter().all(|byte| { - byte.is_ascii_lowercase() - || byte.is_ascii_digit() - || matches!(byte, b'.' | b'_' | b'-') - }); + if let Some(oidc) = &self.authentication.oidc { + oidc.check(false)?; + } + Ok(()) } - false } fn valid_runtime_id(value: &str) -> bool { @@ -1351,44 +1497,39 @@ fn valid_runtime_id(value: &str) -> bool { .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') } -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct ServerRuntime { - pub bind: String, -} - #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct RuntimeSource { + /// SQLite file of this source, absolute or relative to the directory of + /// the runtime configuration. pub path: String, } #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct AuthenticationRuntime { - pub issuer: Option, + /// The one OIDC issuer whose access tokens protected operations accept. + /// A Registry whose every operation is public declares none. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub oidc: Option, } #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct IssuerRuntime { - pub id: String, - /// Exact issuer accepted in access-token `iss` claims. - /// - /// Existing runtimes may omit this when `discoveryUrl` uses the canonical - /// issuer origin. A distinct discovery transport or direct JWKS transport - /// requires this field so network routing never changes token identity. - #[serde(default)] - pub trusted_issuer: Option, - #[serde(default)] - pub discovery_url: Option, - #[serde(default)] - pub jwks_url: Option, +pub struct OidcRuntime { + /// Exact issuer accepted in access-token `iss` claims, an absolute + /// `https` URL. + pub issuer: String, pub audience: String, + /// Where the issuer's signing keys come from: `{kind: discovery}` reads + /// the issuer's OpenID Connect discovery document, and + /// `{kind: uri, uri}` fetches the key set directly. Relay does not + /// accept `kind: static`. + #[serde(default)] + pub jwks_source: JwksSource, pub token_types: Vec, pub algorithms: Vec, } @@ -1412,70 +1553,91 @@ pub(crate) enum IssuerAlgorithm { Rs256, } -impl IssuerRuntime { +impl OidcRuntime { pub(crate) fn profile(&self) -> Option { - self.profile_with_supervised_loopback(false) + self.checked_profile(false).ok() } #[cfg(feature = "tooling")] pub(crate) fn supervised_local_profile(&self) -> Option { - self.profile_with_supervised_loopback(true) + self.checked_profile(true).ok() + } + + fn check(&self, allow_supervised_loopback: bool) -> Result<(), RuntimeRefusal> { + self.checked_profile(allow_supervised_loopback).map(|_| ()) } - fn profile_with_supervised_loopback( + fn checked_profile( &self, allow_supervised_loopback: bool, - ) -> Option { - if !valid_runtime_id(&self.id) - || self.audience.trim().is_empty() - || self.audience.len() > MAXIMUM_ISSUER_AUDIENCE_BYTES - || self.token_types.as_slice() != ["at+jwt"] - { - return None; + ) -> Result { + if self.audience.trim().is_empty() || self.audience.len() > MAXIMUM_ISSUER_AUDIENCE_BYTES { + return Err(RuntimeRefusal::new( + "authentication.oidc.audience", + "authentication.oidc.audience must be non-empty and at most 8192 bytes", + )); } - let algorithm = match self.algorithms.as_slice() { - [algorithm] if algorithm == "EdDSA" => IssuerAlgorithm::EdDsa, - [algorithm] if algorithm == "ES256" => IssuerAlgorithm::Es256, - [algorithm] if algorithm == "RS256" => IssuerAlgorithm::Rs256, - _ => return None, - }; - let (issuer_identifier, key_transport) = - match (self.discovery_url.as_deref(), self.jwks_url.as_deref()) { - (Some(discovery_url), None) => { - let discovery_url = - canonical_issuer_transport_url(discovery_url, allow_supervised_loopback)?; - if !discovery_url.path().ends_with(OIDC_DISCOVERY_SUFFIX) { - return None; - } - let discovery_url = discovery_url.to_string(); - let issuer_identifier = match self.trusted_issuer.as_deref() { - Some(issuer) => { - canonical_trusted_issuer(issuer, allow_supervised_loopback)? - } - None => discovery_url - .strip_suffix(OIDC_DISCOVERY_SUFFIX)? - .to_owned(), - }; - ( - issuer_identifier, - IssuerKeyTransport::Discovery(discovery_url), - ) - } - (None, Some(jwks_url)) => { - let issuer_identifier = canonical_trusted_issuer( - self.trusted_issuer.as_deref()?, - allow_supervised_loopback, - )?; - let jwks_url = - canonical_issuer_transport_url(jwks_url, allow_supervised_loopback)?; - ( - issuer_identifier, - IssuerKeyTransport::Jwks(jwks_url.to_string()), - ) - } - _ => return None, + if self.token_types.as_slice() != ["at+jwt"] { + return Err(RuntimeRefusal::new( + "authentication.oidc.tokenTypes", + "authentication.oidc.tokenTypes must be exactly [at+jwt]", + )); + } + let algorithm = + match self.algorithms.as_slice() { + [algorithm] if algorithm == "EdDSA" => IssuerAlgorithm::EdDsa, + [algorithm] if algorithm == "ES256" => IssuerAlgorithm::Es256, + [algorithm] if algorithm == "RS256" => IssuerAlgorithm::Rs256, + _ => return Err(RuntimeRefusal::new( + "authentication.oidc.algorithms", + "authentication.oidc.algorithms must list exactly one of EdDSA, ES256 or RS256", + )), }; - Some(IssuerProfile { + let issuer_identifier = canonical_trusted_issuer(&self.issuer, allow_supervised_loopback) + .ok_or_else(|| { + RuntimeRefusal::new( + "authentication.oidc.issuer", + "authentication.oidc.issuer must be an absolute https URL without \ + credentials, query or fragment", + ) + })?; + let key_transport = match &self.jwks_source { + JwksSource::Discovery {} => { + let discovery_url = format!( + "{}{OIDC_DISCOVERY_SUFFIX}", + issuer_identifier.trim_end_matches('/') + ); + canonical_issuer_transport_url(&discovery_url, allow_supervised_loopback) + .ok_or_else(|| { + RuntimeRefusal::new( + "authentication.oidc.issuer", + "authentication.oidc.issuer does not yield a canonical discovery URL", + ) + })?; + IssuerKeyTransport::Discovery(discovery_url) + } + JwksSource::Uri { uri } => { + self.jwks_source + .check("authentication.oidc.jwksSource", allow_supervised_loopback)?; + let uri = canonical_issuer_transport_url(uri, allow_supervised_loopback) + .ok_or_else(|| { + RuntimeRefusal::new( + "authentication.oidc.jwksSource.uri", + "authentication.oidc.jwksSource.uri must be a canonical absolute \ + https URL without credentials, query or fragment", + ) + })?; + IssuerKeyTransport::Jwks(uri.to_string()) + } + JwksSource::Static { .. } => { + return Err(RuntimeRefusal::new( + "authentication.oidc.jwksSource.kind", + "Relay does not accept authentication.oidc.jwksSource kind: static; \ + declare kind: discovery or kind: uri", + )) + } + }; + Ok(IssuerProfile { issuer_identifier, algorithm, key_transport, @@ -1574,6 +1736,17 @@ impl AuditRuntime { } } +/// A refused audit block, naming the field the destination error concerns. +fn audit_refusal(error: AuditDestinationError) -> RuntimeRefusal { + let field = match &error { + AuditDestinationError::FileOnlyField { field } => format!("audit.{field}"), + AuditDestinationError::RotateBytesOutOfRange { .. } => "audit.rotateBytes".to_owned(), + AuditDestinationError::RetainDaysOutOfRange { .. } => "audit.retainDays".to_owned(), + _ => "audit.path".to_owned(), + }; + RuntimeRefusal::new(&field, error.to_string()) +} + /// Whether `path` uses only the component shapes startup's `resolve_binding` /// accepts once a relative `audit.path` is joined to the runtime directory: /// an absolute path may use root, prefix, and normal components, and a @@ -1799,27 +1972,116 @@ metadataVisibility: {service: public, resources: public, semantics: public, clas assert!(RegistryContract::parse_yaml(input).is_err()); } + fn runtime_document(authentication: &str) -> String { + format!( + "apiVersion: registry.registrystack.org/relay-runtime/v1alpha1\nkind: RelayRuntimeConfig\nlistener: {{bind: '127.0.0.1:8080'}}\npackage: {{root: /srv/relay/package}}\nsecretProviders: {{environment: {{}}, file: {{root: /run/secrets/relay}}}}\nsources: {{db: {{path: /srv/registry.sqlite}}}}\n{authentication}audit: {{path: /var/log/relay.jsonl}}\nlimits: {{requestTimeoutMilliseconds: 1000, concurrentQueries: 4}}\n" + ) + } + + fn cursor_document(reference: &str) -> String { + format!( + "{}cursor: {{integrityKeyRef: {reference}, maximumAgeSeconds: 300}}\n", + runtime_document("") + ) + } + + fn oidc_runtime(oidc: &str) -> String { + runtime_document(&format!("authentication:\n oidc:\n{oidc}")) + } + + fn oidc_block(issuer: &str, jwks_source: &str, audience: &str, algorithms: &str) -> String { + format!( + " issuer: {issuer}\n{jwks_source} audience: '{audience}'\n tokenTypes: [at+jwt]\n algorithms: {algorithms}\n" + ) + } + + fn refusal(yaml: &str) -> RuntimeRefusal { + RelayRuntime::parse_yaml(yaml).expect_err("runtime refused") + } + + #[test] + fn runtime_accepts_the_shared_envelope_and_refuses_any_other() { + let valid = runtime_document(""); + let runtime = RelayRuntime::parse_yaml(&valid).expect("shared envelope"); + assert_eq!(runtime.api_version, RELAY_RUNTIME_API_VERSION); + assert_eq!(runtime.kind, RELAY_RUNTIME_KIND); + assert_eq!(runtime.listener.bind.socket_addr().port(), 8080); + assert_eq!( + runtime.package.root, + std::path::Path::new("/srv/relay/package") + ); + assert!(runtime.authentication.oidc.is_none()); + + let old_version = valid.replace( + "registry.registrystack.org/relay-runtime/v1alpha1", + "relay.registrystack.org/v2alpha1", + ); + let error = refusal(&old_version); + assert_eq!(error.field(), "apiVersion"); + assert!( + error.message().contains(RELAY_RUNTIME_API_VERSION), + "{error}" + ); + + let old_kind = valid.replace("kind: RelayRuntimeConfig", "kind: RelayRuntime"); + assert_eq!(refusal(&old_kind).field(), "kind"); + } + #[test] fn runtime_rejects_governed_override() { - let input = r#" -apiVersion: relay.registrystack.org/v2alpha1 -kind: RelayRuntime -server: {bind: "127.0.0.1:8080"} -packagePath: /srv/relay/package -sources: {db: {path: /srv/registry.sqlite}} -authentication: {issuer: null} -audit: {path: /var/log/relay.jsonl} -limits: {requestTimeoutMilliseconds: 1000, concurrentQueries: 4} -disclosureProfiles: {} -"#; - assert!(RelayRuntime::parse_yaml(input).is_err()); + let input = format!("{}disclosureProfiles: {{}}\n", runtime_document("")); + assert!(RelayRuntime::parse_yaml(&input).is_err()); + } + + #[test] + fn removed_runtime_keys_are_refused_with_their_replacement() { + let valid = runtime_document(""); + let cases = [ + ( + format!("{valid}server: {{bind: '127.0.0.1:8080'}}\n"), + "server", + "listener.bind", + ), + ( + format!("{valid}packagePath: package\n"), + "packagePath", + "package.root", + ), + ( + valid.replace( + "audit: {path: /var/log/relay.jsonl}", + "audit: {path: /var/log/relay.jsonl, integrityKeyRef: secret:env/RELAY_KEY}", + ), + "audit.integrityKeyRef", + "not hash-chained", + ), + ( + valid.replace( + "audit: {path: /var/log/relay.jsonl}", + "audit: {sink: /var/log/relay.jsonl}", + ), + "audit.sink", + "audit.path", + ), + ( + format!("{valid}authentication: {{issuer: null}}\n"), + "authentication.issuer", + "authentication.oidc", + ), + ]; + for (yaml, field, replacement) in cases { + let error = refusal(&yaml); + assert_eq!(error.field(), field, "{error}"); + assert!(error.message().contains(replacement), "{error}"); + } } #[test] fn runtime_audit_accepts_only_the_shared_destination_shape() { let template = |audit: &str| { - format!( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {{bind: '127.0.0.1:8080'}}\npackagePath: /srv/relay/package\nsources: {{db: {{path: /srv/registry.sqlite}}}}\nauthentication: {{issuer: null}}\naudit: {audit}\nlimits: {{requestTimeoutMilliseconds: 1000, concurrentQueries: 4}}\n" + runtime_document("").replace( + "audit: {path: /var/log/relay.jsonl}", + &format!("audit: {audit}"), ) }; for valid in [ @@ -1864,10 +2126,10 @@ disclosureProfiles: {} "{destination: syslog, path: /var/log/relay.jsonl}", "unknown variant", ), - ("{sink: /var/log/relay.jsonl}", "unknown field"), + ("{sink: /var/log/relay.jsonl}", "declare audit.path"), ( "{path: /var/log/relay.jsonl, integrityKeyRef: secret:env/RELAY_KEY}", - "unknown field", + "not hash-chained", ), ( "{path: /var/log/relay.jsonl, hashKeyRef: secret:env/RELAY_KEY}", @@ -1882,7 +2144,6 @@ disclosureProfiles: {} ] { let error = RelayRuntime::parse_yaml(&template(invalid)) .expect_err(invalid) - .detail() .to_string(); assert!(error.contains(reason), "{invalid}: {error}"); } @@ -1890,14 +2151,9 @@ disclosureProfiles: {} #[test] fn runtime_accepts_only_the_supported_secret_reference_grammars() { - let template = |reference: &str| { - format!( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {{bind: '127.0.0.1:8080'}}\npackagePath: /srv/relay/package\nsources: {{db: {{path: /srv/registry.sqlite}}}}\nauthentication: {{issuer: null}}\naudit: {{path: /var/log/relay.jsonl}}\ncursor: {{integrityKeyRef: {reference}, maximumAgeSeconds: 300}}\nlimits: {{requestTimeoutMilliseconds: 1000, concurrentQueries: 4}}\n" - ) - }; for valid in ["secret:env/RELAY_KEY", "secret:file/cursor-integrity-key"] { assert!( - RelayRuntime::parse_yaml(&template(valid)).is_ok(), + RelayRuntime::parse_yaml(&cursor_document(valid)).is_ok(), "{valid}" ); } @@ -1909,33 +2165,91 @@ disclosureProfiles: {} "secret:file/nested/key", "secret:vault/key", ] { - assert!( - RelayRuntime::parse_yaml(&template(invalid)).is_err(), - "{invalid}" - ); + let error = refusal(&cursor_document(invalid)); + assert_eq!(error.field(), "cursor.integrityKeyRef", "{invalid}"); + assert!(!error.message().contains(invalid), "{error}"); } } + #[test] + fn a_secret_reference_needs_its_provider_declared() { + let file_only = cursor_document("secret:env/RELAY_KEY").replace( + "secretProviders: {environment: {}, file: {root: /run/secrets/relay}}", + "secretProviders: {file: {root: /run/secrets/relay}}", + ); + let error = refusal(&file_only); + assert_eq!(error.field(), "cursor.integrityKeyRef"); + assert!( + error.message().contains("secretProviders.environment"), + "{error}" + ); + + let env_only = cursor_document("secret:file/cursor-key").replace( + "secretProviders: {environment: {}, file: {root: /run/secrets/relay}}", + "secretProviders: {environment: {}}", + ); + assert!(refusal(&env_only) + .message() + .contains("secretProviders.file")); + + let relative_root = + runtime_document("").replace("root: /run/secrets/relay", "root: secrets"); + assert_eq!(refusal(&relative_root).field(), "secretProviders.file.root"); + } + + #[test] + fn environment_substitution_never_reaches_a_secret_reference() { + let yaml = cursor_document("'${RELAY_CURSOR_REF}'"); + let error = RelayRuntime::parse_yaml_with(&yaml, |_| Some("secret:env/KEY".to_owned())) + .expect_err("substituted reference refused"); + assert_eq!(error.field(), "cursor.integrityKeyRef"); + + let bind = + runtime_document("").replace("'127.0.0.1:8080'", "'${RELAY_BIND:-127.0.0.1:9090}'"); + let runtime = RelayRuntime::parse_yaml(&bind).expect("default substitutes"); + assert_eq!(runtime.listener.bind.socket_addr().port(), 9090); + let runtime = RelayRuntime::parse_yaml_with(&bind, |name| { + (name == "RELAY_BIND").then(|| "127.0.0.1:9191".to_owned()) + }) + .expect("environment substitutes"); + assert_eq!(runtime.listener.bind.socket_addr().port(), 9191); + } + + #[test] + fn package_root_must_be_absolute() { + let relative = runtime_document("").replace("root: /srv/relay/package", "root: package"); + assert_eq!(refusal(&relative).field(), "package.root"); + } + #[test] fn runtime_accepts_exactly_one_startup_supported_issuer_algorithm() { let runtime = |algorithms: &str| { - format!( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {{bind: '127.0.0.1:8080'}}\npackagePath: /srv/relay/package\nsources: {{db: {{path: /srv/registry.sqlite}}}}\nauthentication:\n issuer:\n id: issuer\n discoveryUrl: https://issuer.example.invalid/.well-known/openid-configuration\n audience: registry\n tokenTypes: [at+jwt]\n algorithms: {algorithms}\naudit: {{path: /var/log/relay.jsonl}}\nlimits: {{requestTimeoutMilliseconds: 1000, concurrentQueries: 4}}\n" - ) + oidc_runtime(&oidc_block( + "https://issuer.example.invalid", + "", + "registry", + algorithms, + )) }; for algorithm in ["EdDSA", "ES256", "RS256"] { assert!(RelayRuntime::parse_yaml(&runtime(&format!("[{algorithm}]"))).is_ok()); } - assert!(RelayRuntime::parse_yaml(&runtime("[EdDSA, ES256]")).is_err()); + assert_eq!( + refusal(&runtime("[EdDSA, ES256]")).field(), + "authentication.oidc.algorithms" + ); } #[test] fn issuer_audience_is_bounded_inside_the_authentication_envelope() { let runtime = |audience: &str| { - format!( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {{bind: '127.0.0.1:8080'}}\npackagePath: /srv/relay/package\nsources: {{db: {{path: /srv/registry.sqlite}}}}\nauthentication:\n issuer:\n id: issuer\n discoveryUrl: https://issuer.example.invalid/.well-known/openid-configuration\n audience: '{audience}'\n tokenTypes: [at+jwt]\n algorithms: [EdDSA]\naudit: {{path: /var/log/relay.jsonl}}\nlimits: {{requestTimeoutMilliseconds: 1000, concurrentQueries: 4}}\n" - ) + oidc_runtime(&oidc_block( + "https://issuer.example.invalid", + "", + audience, + "[EdDSA]", + )) }; let boundary = "a".repeat(MAXIMUM_ISSUER_AUDIENCE_BYTES); @@ -1959,143 +2273,132 @@ disclosureProfiles: {} assert!(encoded_claims_upper_bound + 16 * 1024 < 128 * 1024); } + fn profile_of(yaml: &str) -> IssuerProfile { + RelayRuntime::parse_yaml(yaml) + .expect("runtime parses") + .authentication + .oidc + .as_ref() + .and_then(OidcRuntime::profile) + .expect("issuer profile") + } + #[test] - fn runtime_issuer_discovery_matches_the_exact_startup_profile() { - let runtime = |discovery_url: &str| { - format!( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {{bind: '127.0.0.1:8080'}}\npackagePath: /srv/relay/package\nsources: {{db: {{path: /srv/registry.sqlite}}}}\nauthentication:\n issuer:\n id: issuer\n discoveryUrl: {discovery_url}\n audience: registry\n tokenTypes: [at+jwt]\n algorithms: [EdDSA]\naudit: {{path: /var/log/relay.jsonl}}\nlimits: {{requestTimeoutMilliseconds: 1000, concurrentQueries: 4}}\n" - ) - }; - let valid = "https://identity.example.invalid/.well-known/openid-configuration"; - let parsed = RelayRuntime::parse_yaml(&runtime(valid)).expect("exact discovery URL"); - assert_eq!( - parsed - .authentication - .issuer - .as_ref() - .and_then(IssuerRuntime::profile) - .map(|profile| profile.issuer_identifier), - Some("https://identity.example.invalid".to_owned()) - ); + fn discovery_key_source_derives_the_issuer_discovery_document() { + for jwks_source in ["", " jwksSource: {kind: discovery}\n"] { + let profile = profile_of(&oidc_runtime(&oidc_block( + "https://identity.example.invalid", + jwks_source, + "registry", + "[EdDSA]", + ))); + assert_eq!( + profile.issuer_identifier, + "https://identity.example.invalid" + ); + assert_eq!( + profile.key_transport, + IssuerKeyTransport::Discovery( + "https://identity.example.invalid/.well-known/openid-configuration".to_owned() + ) + ); + } - for invalid in [ - "https://operator:credential@identity.example.invalid/.well-known/openid-configuration", - "https://identity.example.invalid/.well-known/openid-configuration?tenant=x", - "https://identity.example.invalid/.well-known/openid-configuration#fragment", - "https://identity.example.invalid/.well-known/oauth-authorization-server", - "https:///.well-known/openid-configuration", + for (issuer, discovery) in [ + ( + "https://issuer.example.invalid/", + "https://issuer.example.invalid/.well-known/openid-configuration", + ), + ( + "https://issuer.example.invalid/tenant/", + "https://issuer.example.invalid/tenant/.well-known/openid-configuration", + ), ] { - assert!( - RelayRuntime::parse_yaml(&runtime(invalid)).is_err(), - "{invalid}" + let profile = profile_of(&oidc_runtime(&oidc_block( + issuer, "", "registry", "[EdDSA]", + ))); + assert_eq!(profile.issuer_identifier, issuer); + assert_eq!( + profile.key_transport, + IssuerKeyTransport::Discovery(discovery.to_owned()) ); } + + for invalid in [ + "https://operator:credential@identity.example.invalid", + "https://identity.example.invalid/?tenant=x", + "https://identity.example.invalid/#fragment", + "http://identity.example.invalid", + "'http://127.0.0.1:8443'", + "https:///", + ] { + let error = refusal(&oidc_runtime(&oidc_block( + invalid, "", "registry", "[EdDSA]", + ))); + assert_eq!(error.field(), "authentication.oidc.issuer", "{invalid}"); + } } #[test] - fn runtime_separates_trusted_issuer_from_one_key_transport() { - let runtime = |transport: &str| { - format!( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {{bind: '127.0.0.1:8080'}}\npackagePath: /srv/relay/package\nsources: {{db: {{path: /srv/registry.sqlite}}}}\nauthentication:\n issuer:\n id: issuer\n trustedIssuer: https://issuer.example.invalid\n{transport} audience: registry\n tokenTypes: [at+jwt]\n algorithms: [EdDSA]\naudit: {{path: /var/log/relay.jsonl}}\nlimits: {{requestTimeoutMilliseconds: 1000, concurrentQueries: 4}}\n" - ) + fn uri_key_source_keeps_the_issuer_separate_from_the_key_transport() { + let runtime = |uri: &str| { + oidc_runtime(&oidc_block( + "https://issuer.example.invalid", + &format!(" jwksSource: {{kind: uri, uri: '{uri}'}}\n"), + "registry", + "[EdDSA]", + )) }; - - let discovery = RelayRuntime::parse_yaml( - &runtime( - " discoveryUrl: https://discovery.example.invalid/.well-known/openid-configuration\n", - ), - ) - .expect("distinct discovery transport parses"); - let profile = discovery - .authentication - .issuer - .as_ref() - .and_then(IssuerRuntime::profile) - .expect("distinct discovery transport validates"); - assert_eq!(profile.issuer_identifier, "https://issuer.example.invalid"); - assert_eq!( - profile.key_transport, - IssuerKeyTransport::Discovery( - "https://discovery.example.invalid/.well-known/openid-configuration".to_owned() - ) - ); - - let jwks = RelayRuntime::parse_yaml(&runtime( - " jwksUrl: https://keys.example.invalid/issuer.jwks.json\n", - )) - .expect("direct JWKS transport parses"); - let profile = jwks - .authentication - .issuer - .as_ref() - .and_then(IssuerRuntime::profile) - .expect("direct JWKS transport validates"); - assert_eq!(profile.issuer_identifier, "https://issuer.example.invalid"); - assert_eq!( - profile.key_transport, - IssuerKeyTransport::Jwks("https://keys.example.invalid/issuer.jwks.json".to_owned()) - ); - - let root_jwks = - RelayRuntime::parse_yaml(&runtime(" jwksUrl: https://keys.example.invalid/\n")) - .expect("canonical root JWKS transport parses"); - assert_eq!( - root_jwks - .authentication - .issuer - .as_ref() - .and_then(IssuerRuntime::profile) - .expect("canonical root JWKS transport validates") - .key_transport, - IssuerKeyTransport::Jwks("https://keys.example.invalid/".to_owned()) - ); - - for trusted_issuer in [ - "https://issuer.example.invalid/", - "https://issuer.example.invalid/tenant/", + for uri in [ + "https://keys.example.invalid/issuer.jwks.json", + "https://keys.example.invalid/", ] { - let trailing_slash = - runtime(" jwksUrl: https://keys.example.invalid/issuer.jwks.json\n").replace( - "trustedIssuer: https://issuer.example.invalid", - &format!("trustedIssuer: {trusted_issuer}"), - ); - let profile = RelayRuntime::parse_yaml(&trailing_slash) - .expect("canonical trailing-slash issuer parses") - .authentication - .issuer - .as_ref() - .and_then(IssuerRuntime::profile) - .expect("canonical trailing-slash issuer validates"); - assert_eq!(profile.issuer_identifier, trusted_issuer); + let profile = profile_of(&runtime(uri)); + assert_eq!(profile.issuer_identifier, "https://issuer.example.invalid"); + assert_eq!( + profile.key_transport, + IssuerKeyTransport::Jwks(uri.to_owned()) + ); } - for invalid in [ - runtime(""), - runtime( - " discoveryUrl: https://discovery.example.invalid/.well-known/openid-configuration\n jwksUrl: https://keys.example.invalid/issuer.jwks.json\n", - ), - runtime(" jwksUrl: http://keys.example.invalid/issuer.jwks.json\n"), + "http://keys.example.invalid/issuer.jwks.json", + "https://keys.example.invalid/issuer.jwks.json?tenant=x", + "https://keys.example.invalid", ] { + let error = refusal(&runtime(invalid)); assert!( - RelayRuntime::parse_yaml(&invalid).is_err(), - "runtime accepted an invalid issuer transport contract" + error.field().starts_with("authentication.oidc.jwksSource"), + "{invalid}: {error}" ); } + } - let missing_trusted_issuer = - runtime(" jwksUrl: https://keys.example.invalid/issuer.jwks.json\n") - .replace(" trustedIssuer: https://issuer.example.invalid\n", ""); - assert!(RelayRuntime::parse_yaml(&missing_trusted_issuer).is_err()); + #[test] + fn static_key_source_is_refused() { + let error = refusal(&oidc_runtime(&oidc_block( + "https://issuer.example.invalid", + " jwksSource: {kind: static, documentRef: secret:file/issuer-jwks}\n", + "registry", + "[EdDSA]", + ))); + assert_eq!(error.field(), "authentication.oidc.jwksSource.kind"); + assert!(error.message().contains("kind: uri"), "{error}"); + } + + #[test] + fn refusals_name_the_field_and_never_the_value() { + let yaml = runtime_document("").replace("concurrentQueries: 4", "concurrentQueries: 999"); + let error = refusal(&yaml); + assert_eq!(error.field(), "limits.concurrentQueries"); + assert!(!error.message().contains("999"), "{error}"); } #[test] fn cursor_requirement_counts_only_potentially_visible_metadata_resources() { let mut contract = RegistryContract::parse_yaml(crate::compiler::tests::valid_contract()) .expect("base contract"); - let mut runtime = RelayRuntime::parse_yaml( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {bind: '127.0.0.1:8080'}\npackagePath: package\nsources: {db: {path: fixture.sqlite}}\nauthentication: {issuer: null}\naudit: {path: var/audit.jsonl}\nlimits: {requestTimeoutMilliseconds: 1000, concurrentQueries: 1}\n", - ) - .expect("runtime without cursor"); + let mut runtime = + RelayRuntime::parse_yaml(&runtime_document("")).expect("runtime without cursor"); let mut protected_resource = contract.resources[0].clone(); protected_resource.id = "protected-record".into(); protected_resource.operations.read = Some( @@ -2174,4 +2477,45 @@ disclosureProfiles: {} ); } } + + #[test] + fn an_authored_contract_carrying_an_environment_expression_is_refused() { + let yaml = crate::compiler::tests::valid_contract(); + let error = RegistryContract::parse_yaml(&yaml.replacen( + "title: Records}", + "title: \"${REGISTRY_TITLE}\"}", + 1, + )) + .expect_err("an environment expression in the contract is refused"); + assert_eq!(error.environment_expression_field(), Some("metadata.title")); + assert!(error.detail().is_none()); + + let report = crate::compiler::compile_yaml( + &yaml.replacen("name: Records", "name: ${REGISTRY_NAME:-Records}", 1), + &[], + crate::model::CompileProfile::Authoring, + ) + .expect_err("compilation refuses it"); + assert_eq!(report.diagnostics.len(), 1); + let diagnostic = &report.diagnostics[0]; + assert_eq!(diagnostic.code, "contract.environment_expression"); + assert_eq!(diagnostic.location, "registry.yaml.registry.name"); + assert!(diagnostic.message.contains("runtime.yaml only")); + + let invalid = RegistryContract::parse_yaml("kind: [").expect_err("invalid YAML"); + assert!(invalid.environment_expression_field().is_none()); + assert!(invalid.detail().is_some()); + } + + #[test] + fn text_the_expression_check_cannot_read_is_reported_as_invalid_yaml() { + let unreadable = ContractParseError::EnvironmentExpression( + registry_platform_config::reject_environment_expressions_in_authored_yaml("kind: [") + .expect_err("the check fails closed"), + ); + assert!(unreadable.environment_expression_field().is_none()); + let diagnostic = unreadable.diagnostic(); + assert_eq!(diagnostic.code, "contract.yaml_invalid"); + assert_eq!(diagnostic.location, "registry.yaml"); + } } diff --git a/crates/registry-relay-v2/src/lib.rs b/crates/registry-relay-v2/src/lib.rs index 4ccbed57d0..2a9a81e66c 100644 --- a/crates/registry-relay-v2/src/lib.rs +++ b/crates/registry-relay-v2/src/lib.rs @@ -35,5 +35,7 @@ pub mod transform; pub use cli::command; pub use compiler::{classification_inventory_digest, compile, CompileError}; -pub use contract::{RegistryContract, RelayRuntime}; +pub use contract::{ + RegistryContract, RelayRuntime, RuntimeRefusal, RELAY_RUNTIME_API_VERSION, RELAY_RUNTIME_KIND, +}; pub use model::{CompileProfile, CompiledRegistry, ObservedSourceSchema}; diff --git a/crates/registry-relay-v2/src/main.rs b/crates/registry-relay-v2/src/main.rs index f12d73bb06..4da7bddc16 100644 --- a/crates/registry-relay-v2/src/main.rs +++ b/crates/registry-relay-v2/src/main.rs @@ -11,10 +11,14 @@ async fn main() -> ExitCode { install_operational_logging(); let result = match Cli::parse().command { Command::Check { - runtime, + runtime_config, require_audit_under, - } => registry_relay_v2::startup::check(&runtime, require_audit_under.as_deref()).await, - Command::Serve { runtime } => registry_relay_v2::startup::serve(&runtime).await, + } => { + registry_relay_v2::startup::check(&runtime_config, require_audit_under.as_deref()).await + } + Command::Serve { runtime_config } => { + registry_relay_v2::startup::serve(&runtime_config).await + } Command::Healthcheck { url } => registry_relay_v2::startup::healthcheck(&url).await, }; match result { @@ -27,7 +31,9 @@ async fn main() -> ExitCode { } /// Install bounded structured operational logs on stderr. Relay-owned events -/// deliberately carry only fixed messages and value-free dimensions. +/// deliberately carry only fixed messages and value-free dimensions; a +/// refused runtime configuration adds the refused field's name and the rule it +/// broke, never a configured value or the file's path. fn install_operational_logging() { let configured = std::env::var("RELAY_LOG").ok(); let filter = diff --git a/crates/registry-relay-v2/src/package.rs b/crates/registry-relay-v2/src/package.rs index c06033d0b0..1489bf3c08 100644 --- a/crates/registry-relay-v2/src/package.rs +++ b/crates/registry-relay-v2/src/package.rs @@ -1,18 +1,21 @@ // SPDX-License-Identifier: Apache-2.0 -//! Deterministic sealed package construction from one compiled Registry. +//! Deterministic Relay packages from one compiled Registry, written and +//! verified in the shared Registry Stack package format. use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::path::{Component, Path}; use registry_platform_canonical_json::canonicalize_json; +use registry_platform_config::package::{ + self as shared, is_envelope_file, PackageLimits, SUM_FILE, +}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use thiserror::Error; use crate::artifacts::{ - generate_artifacts, ArtifactAccessBinding, ArtifactSet, GeneratedArtifact, - OperationArtifactBindings, + generate_artifacts, ArtifactAccessBinding, ArtifactSet, OperationArtifactBindings, }; use crate::compiler::{ compile_contract_with_governed_files, referenced_governed_files, GovernedFileSet, @@ -22,51 +25,69 @@ use crate::model::{ CompileProfile, CompiledClassificationReview, CompiledRegistry, ObservedSourceSchema, }; -const PACKAGE_VERSION: &str = "relay.registrystack.org/package/v1alpha3"; +/// The command that builds a Relay package, named in every refusal. +pub const PACKAGE_COMMAND: &str = "relayctl package"; +/// The field a runtime refusal names instead of the configured directory. +const PACKAGE_ROOT_FIELD: &str = "package.root"; +/// The self-hashing manifest earlier packages carried instead of `SHA256SUMS`. +const RETIRED_MANIFEST_PATH: &str = "relay-package.json"; +const REGISTRY_PATH: &str = "registry.yaml"; const COMPILED_REGISTRY_PATH: &str = "compiled/registry.json"; +const GOVERNED_PREFIX: &str = "governed/"; +const GENERATED_PREFIX: &str = "generated/"; const MAX_AUTHORED_FILES: usize = 256; const MAX_AUTHORED_BYTES: u64 = 16 * 1024 * 1024; const MAX_PACKAGE_FILES: usize = 1_024; const MAX_PACKAGE_BYTES: u64 = 64 * 1024 * 1024; -const MAX_MANIFEST_BYTES: u64 = 4 * 1024 * 1024; +/// The bounds a Relay package is written and verified under. +#[must_use] +pub fn package_limits() -> PackageLimits { + PackageLimits { + max_files: MAX_PACKAGE_FILES, + max_file_bytes: MAX_PACKAGE_BYTES, + max_total_bytes: MAX_PACKAGE_BYTES, + ..PackageLimits::default() + } +} + +/// What `relayctl package` wrote, or would write under `--dry-run`. #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct PackageManifest { - pub package_version: String, - pub package_revision: String, +pub struct PackageSummary { + /// The `sha256:` digest of the package's `SHA256SUMS`, the value + /// `package.expectedDigest` pins. + pub package_digest: String, + pub revision: Option, + pub dry_run: bool, pub contract_revision: String, pub source_schema_fingerprints: BTreeMap, - pub source_schemas: BTreeMap, - pub artifacts: Vec, - pub operation_artifact_bindings: Vec, + /// Every Relay file of the package; `SHA256SUMS` and `REVISION` are the + /// shared format's own. pub files: Vec, + /// The exposure of every generated artifact, as the runtime derives it + /// from the compiled Registry. The package does not store it. + pub artifacts: Vec, } #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct PackageArtifact { - pub id: String, +pub struct PackageFile { pub path: String, - pub media_type: String, - pub visibility: Visibility, - /// Ownership is retained for operation-bound Record artifacts and every - /// statistical structure artifact, including public or operator-only ones. - pub operation_identifier: Option, - /// The closed ownership mechanism paired with `operation_identifier`. - pub access_binding: Option, pub sha256: String, + pub bytes: u64, } #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct PackageFile { +pub struct PackageArtifact { + pub id: String, pub path: String, - pub size: u64, - pub sha256: String, pub media_type: String, pub visibility: Visibility, - pub generated: bool, + pub operation_identifier: Option, + pub access_binding: Option, + pub sha256: String, } #[derive(Debug, Error)] @@ -75,44 +96,106 @@ pub enum PackageError { UnsafeClosure, #[error("the project closure exceeds package bounds")] ClosureBound, - #[error("the package destination is not empty")] - DestinationExists, #[error("a package file could not be read")] Read, - #[error("the sealed package could not be written")] + #[error("the package could not be written")] Write, #[error("the package manifest could not be canonicalized")] CanonicalJson, - #[error("the sealed package failed verification")] + #[error("the compiled project could not be packaged")] Verification, + /// The shared package format refused the directory: a missing + /// `SHA256SUMS`, a changed, missing, or extra file, or a bound. + #[error("{0}")] + Package(#[from] shared::PackageError), + #[error( + "the package at {PACKAGE_ROOT_FIELD} holds {RETIRED_MANIFEST_PATH}, which Relay no \ + longer reads; rebuild the package with `{PACKAGE_COMMAND}`, which writes {SUM_FILE}" + )] + RetiredManifest, + #[error( + "the package at {PACKAGE_ROOT_FIELD}, or a directory above it, holds a symbolic link or \ + special file, is not owned by the effective user or root, or is writable by others; \ + deploy the package built by `{PACKAGE_COMMAND}` read-only" + )] + UnsafePermissions, + #[error( + "the package at {PACKAGE_ROOT_FIELD} changed while it was read ({path}); deploy the \ + whole package built by `{PACKAGE_COMMAND}` and restart" + )] + ChangedWhileRead { path: String }, + #[error( + "the package at {PACKAGE_ROOT_FIELD} does not hold the files its compiled registry \ + names{}; rebuild it with `{PACKAGE_COMMAND}`", + describe_contents(.missing, .extra) + )] + Contents { + missing: Vec, + extra: Vec, + }, + #[error( + "the package at {PACKAGE_ROOT_FIELD} holds {path}, which its registry.yaml and governed \ + files do not reproduce; rebuild it with `{PACKAGE_COMMAND}`" + )] + Derivation { path: String }, } +fn describe_contents(missing: &[String], extra: &[String]) -> String { + let mut described = String::new(); + for (label, paths) in [("missing", missing), ("extra", extra)] { + if !paths.is_empty() { + described.push_str(&format!("; {label}: {}", paths.join(", "))); + } + } + described +} + +/// A package whose every byte matched its `SHA256SUMS` and whose compiled +/// registry and generated artifacts were reproduced from its sources. #[derive(Clone, Debug)] pub struct VerifiedPackage { - pub manifest: PackageManifest, + /// The `sha256:` digest of `SHA256SUMS`. + pub digest: String, + pub revision: Option, pub contract: RegistryContract, pub registry: CompiledRegistry, pub artifacts: ArtifactSet, } -/// Construct a new package directory. Existing destinations are refused so a -/// failed run can never leave a mixture of package revisions. +impl VerifiedPackage { + /// The source schemas the package was compiled against, by source. + #[must_use] + pub fn source_schemas(&self) -> BTreeMap { + self.registry + .sources + .iter() + .filter_map(|source| { + source + .observed_schema + .clone() + .map(|schema| (source.id.clone(), schema)) + }) + .collect() + } +} + +/// Construct a package of one compiled project. With `output_dir`, write it +/// into that new directory; without, report the digest it would have. An +/// existing destination is refused so a run can never mix package contents. pub fn build_package( project_root: &Path, - output_dir: &Path, + output_dir: Option<&Path>, + revision: Option<&str>, contract: &RegistryContract, compiled: &CompiledRegistry, artifacts: &ArtifactSet, -) -> Result { - if output_dir.exists() { - return Err(PackageError::DestinationExists); - } +) -> Result { let authored = capture_governed_closure( project_root, contract, compiled.classification_review.as_ref(), )?; - let registry_bytes = read_regular(&project_root.join("registry.yaml"))?; + let registry_bytes = read_regular(&project_root.join(REGISTRY_PATH))?; let packaged_contract = RegistryContract::parse_yaml( std::str::from_utf8(®istry_bytes).map_err(|_| PackageError::Verification)?, ) @@ -121,150 +204,81 @@ pub fn build_package( return Err(PackageError::Verification); } validate_build_inputs(contract, compiled, artifacts, &authored)?; - let mut files = Vec::new(); - files.push(file_entry( - "registry.yaml", - ®istry_bytes, - "application/yaml", - Visibility::OperatorOnly, - false, - )); - for (relative, content) in &authored { - files.push(file_entry( - &format!("governed/{relative}"), - content, - media_type(relative), - Visibility::OperatorOnly, - false, - )); + + let mut files = BTreeMap::new(); + files.insert(REGISTRY_PATH.to_owned(), registry_bytes); + for (relative, content) in authored { + files.insert(format!("{GOVERNED_PREFIX}{relative}"), content); } - let compiled_bytes = canonicalize_json( - &serde_json::to_value(compiled).map_err(|_| PackageError::CanonicalJson)?, - ) - .map_err(|_| PackageError::CanonicalJson)?; - files.push(file_entry( - COMPILED_REGISTRY_PATH, - &compiled_bytes, - "application/json", - Visibility::OperatorOnly, - true, - )); + files.insert( + COMPILED_REGISTRY_PATH.to_owned(), + canonical_compiled(compiled)?, + ); for artifact in &artifacts.artifacts { - files.push(file_entry( - &format!("generated/{}", artifact.path), - &artifact.content, - &artifact.media_type, - artifact.visibility, - true, - )); + files.insert( + format!("{GENERATED_PREFIX}{}", artifact.path), + artifact.content.clone(), + ); } - files.sort_by(|left, right| left.path.cmp(&right.path)); - let packaged_artifacts = artifacts - .artifacts - .iter() - .map(|artifact| PackageArtifact { - id: artifact.id.clone(), - path: format!("generated/{}", artifact.path), - media_type: artifact.media_type.clone(), - visibility: artifact.visibility, - operation_identifier: artifact.operation_identifier.clone(), - access_binding: artifact.access_binding.clone(), - sha256: artifact.sha256.clone(), - }) - .collect::>(); - let source_schema_fingerprints = compiled - .sources - .iter() - .map(|source| { + let limits = package_limits(); + let (package_digest, revision) = match output_dir { + None => ( + shared::plan_package(project_root, &files, revision, &limits, PACKAGE_COMMAND)?, + revision.map(str::to_owned), + ), + Some(output_dir) => { + let written = + shared::write_package(output_dir, &files, revision, &limits, PACKAGE_COMMAND)?; + harden_package_permissions(output_dir)?; ( - source.id.clone(), - source.expected_schema_fingerprint.clone(), + written.digest().to_owned(), + written.revision().map(str::to_owned), ) - }) - .collect(); - let source_schemas = compiled - .sources - .iter() - .map(|source| { - source - .observed_schema - .clone() - .map(|schema| (source.id.clone(), schema)) - .ok_or(PackageError::Verification) - }) - .collect::, _>>()?; - let unsigned = UnsignedManifest { - package_version: PACKAGE_VERSION, - contract_revision: &compiled.contract_revision, - source_schema_fingerprints: &source_schema_fingerprints, - source_schemas: &source_schemas, - artifacts: &packaged_artifacts, - operation_artifact_bindings: &artifacts.operation_bindings, - files: &files, + } }; - let manifest_value = serde_json::to_value(unsigned).map_err(|_| PackageError::CanonicalJson)?; - let manifest_bytes = - canonicalize_json(&manifest_value).map_err(|_| PackageError::CanonicalJson)?; - let package_revision = digest(&manifest_bytes); - let manifest = PackageManifest { - package_version: PACKAGE_VERSION.into(), - package_revision, + Ok(PackageSummary { + package_digest, + revision, + dry_run: output_dir.is_none(), contract_revision: compiled.contract_revision.clone(), - source_schema_fingerprints, - source_schemas, - artifacts: packaged_artifacts, - operation_artifact_bindings: artifacts.operation_bindings.clone(), - files, - }; - let final_manifest = canonicalize_json( - &serde_json::to_value(&manifest).map_err(|_| PackageError::CanonicalJson)?, - ) - .map_err(|_| PackageError::CanonicalJson)?; - validate_package_bounds(&manifest.files, final_manifest.len())?; - - fs::create_dir(output_dir).map_err(|_| PackageError::Write)?; - let write_result = (|| { - write_new_file(&output_dir.join("registry.yaml"), ®istry_bytes)?; - for (relative, content) in &authored { - write_new_file(&output_dir.join("governed").join(relative), content)?; - } - write_new_file(&output_dir.join(COMPILED_REGISTRY_PATH), &compiled_bytes)?; - for artifact in &artifacts.artifacts { - write_generated(output_dir, artifact)?; - } - write_new_file(&output_dir.join("relay-package.json"), &final_manifest) - })(); - if write_result.is_err() { - // Do not remove a partially written directory here. A caller can - // inspect it, and a subsequent package attempt will refuse it rather - // than silently overwriting evidence. - return Err(PackageError::Write); - } - harden_package_permissions(output_dir)?; - Ok(manifest) + source_schema_fingerprints: compiled + .sources + .iter() + .map(|source| { + ( + source.id.clone(), + source.expected_schema_fingerprint.clone(), + ) + }) + .collect(), + files: files + .iter() + .map(|(path, content)| PackageFile { + path: path.clone(), + sha256: digest(content), + bytes: content.len() as u64, + }) + .collect(), + artifacts: artifacts + .artifacts + .iter() + .map(|artifact| PackageArtifact { + id: artifact.id.clone(), + path: format!("{GENERATED_PREFIX}{}", artifact.path), + media_type: artifact.media_type.clone(), + visibility: artifact.visibility, + operation_identifier: artifact.operation_identifier.clone(), + access_binding: artifact.access_binding.clone(), + sha256: artifact.sha256.clone(), + }) + .collect(), + }) } -fn validate_package_bounds( - files: &[PackageFile], - manifest_bytes: usize, -) -> Result<(), PackageError> { - if files.len() > MAX_PACKAGE_FILES { - return Err(PackageError::ClosureBound); - } - let manifest_bytes = u64::try_from(manifest_bytes).map_err(|_| PackageError::ClosureBound)?; - if manifest_bytes > MAX_MANIFEST_BYTES { - return Err(PackageError::ClosureBound); - } - let total = files.iter().try_fold(manifest_bytes, |total, file| { - total - .checked_add(file.size) - .ok_or(PackageError::ClosureBound) - })?; - if total > MAX_PACKAGE_BYTES { - return Err(PackageError::ClosureBound); - } - Ok(()) +fn canonical_compiled(compiled: &CompiledRegistry) -> Result, PackageError> { + canonicalize_json(&serde_json::to_value(compiled).map_err(|_| PackageError::CanonicalJson)?) + .map_err(|_| PackageError::CanonicalJson) } fn validate_build_inputs( @@ -274,31 +288,50 @@ fn validate_build_inputs( governed: &GovernedFileSet, ) -> Result<(), PackageError> { if artifacts.contract_revision != compiled.contract_revision - || compiled.contract_id != contract.metadata.id - || compiled.contract_version != contract.metadata.version - || compiled.registry_identifier != contract.registry.registry_identifier + || !compiled_matches_contract(compiled, contract) { return Err(PackageError::Verification); } - let observed = compiled + let observed = observed_source_schemas(compiled).ok_or(PackageError::Verification)?; + verify_compiled_derivation(contract, compiled, governed, &observed)?; + verify_artifact_derivation(compiled, artifacts)?; + if !valid_artifact_set(compiled, artifacts) { + return Err(PackageError::Verification); + } + Ok(()) +} + +fn compiled_matches_contract(compiled: &CompiledRegistry, contract: &RegistryContract) -> bool { + compiled.contract_id == contract.metadata.id + && compiled.contract_version == contract.metadata.version + && compiled.registry_identifier == contract.registry.registry_identifier +} + +/// The schema each source was compiled against, when every source records +/// one that names it. +fn observed_source_schemas(compiled: &CompiledRegistry) -> Option> { + compiled .sources .iter() .map(|source| { - source - .observed_schema - .clone() - .ok_or(PackageError::Verification) + source.observed_schema.clone().filter(|schema| { + schema.source == source.id + && schema.fingerprint == source.expected_schema_fingerprint + }) }) - .collect::, _>>()?; - verify_compiled_derivation(contract, compiled, governed, &observed)?; - verify_artifact_derivation(compiled, artifacts)?; + .collect() +} + +/// Every artifact has a unique identifier and safe relative path, and every +/// ownership and operation binding is exactly the one the Registry declares. +fn valid_artifact_set(compiled: &CompiledRegistry, artifacts: &ArtifactSet) -> bool { let expected_operation_access_profiles = operation_access_profile_pairs(compiled); let expected_fixed_operations = fixed_statistical_operations(compiled); let mut artifact_ids = BTreeSet::new(); let mut artifact_paths = BTreeSet::new(); for artifact in &artifacts.artifacts { - validate_relative(&artifact.path)?; - if !artifact_ids.insert(artifact.id.as_str()) + if validate_relative(&artifact.path).is_err() + || !artifact_ids.insert(artifact.id.as_str()) || !artifact_paths.insert(artifact.path.as_str()) || artifact.sha256 != digest(&artifact.content) || !valid_artifact_access_binding( @@ -309,17 +342,14 @@ fn validate_build_inputs( &expected_fixed_operations, ) { - return Err(PackageError::Verification); + return false; } } - if !valid_operation_artifact_bindings( + valid_operation_artifact_bindings( &artifacts.operation_bindings, &expected_operation_access_profiles, &artifact_paths, - ) { - return Err(PackageError::Verification); - } - Ok(()) + ) } fn verify_compiled_derivation( @@ -345,7 +375,7 @@ fn verify_artifact_derivation( compiled: &CompiledRegistry, artifacts: &ArtifactSet, ) -> Result<(), PackageError> { - // `packageRevision` is an integrity digest, not an authenticity proof. A + // The package digest is an integrity digest, not an authenticity proof. A // caller can recalculate it, so acceptance must reproduce every artifact // byte and its release metadata from the already rederived Registry. let reproduced = generate_artifacts(compiled).map_err(|_| PackageError::Verification)?; @@ -355,298 +385,146 @@ fn verify_artifact_derivation( Ok(()) } -/// Load and verify a sealed package before any listener, issuer, audit sink, -/// or SQLite source is activated. +/// Load and verify a package before any listener, issuer, audit sink, or +/// SQLite source is activated. +/// +/// The shared format proves every byte matches `SHA256SUMS`. Relay then +/// proves the files are exactly the ones its compiled registry names, and +/// reproduces the compiled registry from `registry.yaml`, the governed files, +/// and the recorded source schemas, and every generated artifact from the +/// compiled registry. pub fn load_package(package_path: &Path) -> Result { - reject_symlink_path(package_path)?; - let package_metadata = fs::symlink_metadata(package_path).map_err(|_| PackageError::Read)?; - if !package_metadata.is_dir() || !safe_permissions(&package_metadata) { - return Err(PackageError::Verification); - } - let manifest_path = package_path.join("relay-package.json"); - let manifest_metadata = fs::symlink_metadata(&manifest_path).map_err(|_| PackageError::Read)?; - if !manifest_metadata.is_file() - || manifest_metadata.len() > MAX_MANIFEST_BYTES - || !safe_permissions(&manifest_metadata) - { - return Err(PackageError::Verification); - } - let manifest_bytes = read_regular(&manifest_path)?; - let manifest: PackageManifest = - serde_json::from_slice(&manifest_bytes).map_err(|_| PackageError::Verification)?; - if manifest.package_version != PACKAGE_VERSION - || manifest.files.is_empty() - || manifest.files.len() > MAX_PACKAGE_FILES - { - return Err(PackageError::Verification); + match reject_symlink_path(package_path) { + // An absent root is refused by the shared format, naming the command. + Ok(()) | Err(PackageError::Read) => {} + Err(_) => return Err(PackageError::UnsafePermissions), } - let canonical_manifest = canonicalize_json( - &serde_json::to_value(&manifest).map_err(|_| PackageError::CanonicalJson)?, - ) - .map_err(|_| PackageError::CanonicalJson)?; - if canonical_manifest != manifest_bytes { - return Err(PackageError::Verification); + if fs::symlink_metadata(package_path.join(RETIRED_MANIFEST_PATH)).is_ok() { + return Err(PackageError::RetiredManifest); } - let unsigned = UnsignedManifest { - package_version: PACKAGE_VERSION, - contract_revision: &manifest.contract_revision, - source_schema_fingerprints: &manifest.source_schema_fingerprints, - source_schemas: &manifest.source_schemas, - artifacts: &manifest.artifacts, - operation_artifact_bindings: &manifest.operation_artifact_bindings, - files: &manifest.files, - }; - let unsigned_bytes = canonicalize_json( - &serde_json::to_value(unsigned).map_err(|_| PackageError::CanonicalJson)?, - ) - .map_err(|_| PackageError::CanonicalJson)?; - if digest(&unsigned_bytes) != manifest.package_revision { - return Err(PackageError::Verification); + let verified = shared::verify_package(package_path, &package_limits(), PACKAGE_COMMAND) + .map_err(|error| error.naming_root_as(PACKAGE_ROOT_FIELD))?; + + // The shared format hashes bytes only; Relay also requires every entry to + // be owned by the effective user or root and not writable by others. + let present = enumerate_package_files(package_path)?; + let listed = verified + .files() + .chain(std::iter::once(SUM_FILE)) + .map(str::to_owned) + .collect::>(); + if let Some(path) = present.symmetric_difference(&listed).next() { + return Err(PackageError::ChangedWhileRead { path: path.clone() }); } - let mut listed = BTreeSet::new(); + // Read each file once more and hold exactly the bytes that were verified. let mut loaded = BTreeMap::new(); - let mut total = manifest_bytes.len() as u64; - for entry in &manifest.files { - validate_relative(&entry.path)?; - if !listed.insert(entry.path.as_str()) { - return Err(PackageError::Verification); - } - reject_relative_symlinks(package_path, Path::new(&entry.path))?; - let path = package_path.join(&entry.path); - let metadata = fs::symlink_metadata(&path).map_err(|_| PackageError::Read)?; - if !metadata.is_file() - || metadata.file_type().is_symlink() - || !safe_permissions(&metadata) - || metadata.len() != entry.size - { - return Err(PackageError::Verification); - } - total = total - .checked_add(metadata.len()) - .ok_or(PackageError::ClosureBound)?; - if total > MAX_PACKAGE_BYTES { - return Err(PackageError::ClosureBound); + for path in verified.files().filter(|path| !is_envelope_file(path)) { + let content = + read_regular(&package_path.join(path)).map_err(|_| PackageError::ChangedWhileRead { + path: path.to_owned(), + })?; + if verified.file_digest(path).as_deref() != Some(digest(&content).as_str()) { + return Err(PackageError::ChangedWhileRead { + path: path.to_owned(), + }); } - let content = read_regular(&path)?; - if digest(&content) != entry.sha256 { - return Err(PackageError::Verification); - } - loaded.insert(entry.path.clone(), content); - } - let actual = enumerate_package_files(package_path)?; - let mut expected = listed - .iter() - .map(|path| (*path).to_owned()) - .collect::>(); - expected.insert("relay-package.json".into()); - if actual != expected { - return Err(PackageError::Verification); + loaded.insert(path.to_owned(), content); } - let registry_entry = manifest - .files + let required = [REGISTRY_PATH, COMPILED_REGISTRY_PATH]; + let missing = required .iter() - .filter(|entry| entry.path == "registry.yaml") + .filter(|path| !loaded.contains_key(**path)) + .map(|path| (*path).to_owned()) .collect::>(); - if registry_entry.len() != 1 - || registry_entry[0].generated - || registry_entry[0].visibility != Visibility::OperatorOnly - { - return Err(PackageError::Verification); + if !missing.is_empty() { + return Err(PackageError::Contents { + missing, + extra: Vec::new(), + }); } - let contract_bytes = loaded - .get("registry.yaml") - .ok_or(PackageError::Verification)?; - let contract_text = - std::str::from_utf8(contract_bytes).map_err(|_| PackageError::Verification)?; - let contract = - RegistryContract::parse_yaml(contract_text).map_err(|_| PackageError::Verification)?; - if manifest.source_schemas.keys().collect::>() - != manifest - .source_schema_fingerprints - .keys() - .collect::>() - || manifest.source_schemas.iter().any(|(id, schema)| { - schema.source != *id || schema.fingerprint != manifest.source_schema_fingerprints[id] + let derivation = |path: &str| PackageError::Derivation { + path: path.to_owned(), + }; + let contract = std::str::from_utf8(&loaded[REGISTRY_PATH]) + .ok() + .and_then(|text| RegistryContract::parse_yaml(text).ok()) + .ok_or_else(|| derivation(REGISTRY_PATH))?; + let compiled_bytes = &loaded[COMPILED_REGISTRY_PATH]; + let registry = serde_json::from_slice::(compiled_bytes) + .ok() + .filter(|registry| { + canonical_compiled(registry).is_ok_and(|bytes| bytes == *compiled_bytes) + && compiled_matches_contract(registry, &contract) }) - { - return Err(PackageError::Verification); - } + .ok_or_else(|| derivation(COMPILED_REGISTRY_PATH))?; + let artifacts = + generate_artifacts(®istry).map_err(|_| derivation(COMPILED_REGISTRY_PATH))?; - let compiled_entry = manifest - .files + let expected = required .iter() - .filter(|entry| entry.path == COMPILED_REGISTRY_PATH) - .collect::>(); - if compiled_entry.len() != 1 - || !compiled_entry[0].generated - || compiled_entry[0].visibility != Visibility::OperatorOnly - || compiled_entry[0].media_type != "application/json" - { - return Err(PackageError::Verification); - } - let compiled_bytes = loaded - .get(COMPILED_REGISTRY_PATH) - .ok_or(PackageError::Verification)?; - let compiled_value: serde_json::Value = - serde_json::from_slice(compiled_bytes).map_err(|_| PackageError::Verification)?; - if canonicalize_json(&compiled_value).map_err(|_| PackageError::CanonicalJson)? - != *compiled_bytes - { - return Err(PackageError::Verification); - } - let registry: CompiledRegistry = - serde_json::from_value(compiled_value).map_err(|_| PackageError::Verification)?; - let reproduced_compiled = canonicalize_json( - &serde_json::to_value(®istry).map_err(|_| PackageError::CanonicalJson)?, - ) - .map_err(|_| PackageError::CanonicalJson)?; - if reproduced_compiled != *compiled_bytes { - return Err(PackageError::Verification); - } - if registry.contract_revision != manifest.contract_revision - || registry.contract_id != contract.metadata.id - || registry.contract_version != contract.metadata.version - || registry.registry_identifier != contract.registry.registry_identifier - || registry - .sources - .iter() - .map(|source| { - ( - source.id.clone(), - source.expected_schema_fingerprint.clone(), - ) - }) - .collect::>() - != manifest.source_schema_fingerprints - { - return Err(PackageError::Verification); + .map(|path| (*path).to_owned()) + .chain( + registry + .governed_files + .iter() + .map(|file| format!("{GOVERNED_PREFIX}{}", file.path)), + ) + .chain( + artifacts + .artifacts + .iter() + .map(|artifact| format!("{GENERATED_PREFIX}{}", artifact.path)), + ) + .collect::>(); + let found = loaded.keys().cloned().collect::>(); + if expected != found { + return Err(PackageError::Contents { + missing: expected.difference(&found).cloned().collect(), + extra: found.difference(&expected).cloned().collect(), + }); } let governed = loaded .iter() .filter_map(|(path, content)| { - path.strip_prefix("governed/") + path.strip_prefix(GOVERNED_PREFIX) .map(|relative| (relative.to_owned(), content.clone())) }) .collect::(); - let observed = manifest - .source_schemas - .values() - .cloned() - .collect::>(); - verify_compiled_derivation(&contract, ®istry, &governed, &observed)?; + if let Some(file) = registry.governed_files.iter().find(|file| { + governed + .get(&file.path) + .is_none_or(|content| digest(content) != file.sha256) + }) { + return Err(derivation(&format!("{GOVERNED_PREFIX}{}", file.path))); + } + let observed = + observed_source_schemas(®istry).ok_or_else(|| derivation(COMPILED_REGISTRY_PATH))?; + verify_compiled_derivation(&contract, ®istry, &governed, &observed) + .map_err(|_| derivation(COMPILED_REGISTRY_PATH))?; - let governed_paths = registry - .governed_files - .iter() - .map(|file| file.path.as_str()) - .collect::>(); - let loaded_governed_paths = loaded - .keys() - .filter_map(|path| path.strip_prefix("governed/")) - .collect::>(); - if governed_paths != loaded_governed_paths - || registry.governed_files.iter().any(|file| { - loaded - .get(&format!("governed/{}", file.path)) - .is_none_or(|content| digest(content) != file.sha256) - }) + if artifacts.contract_revision != registry.contract_revision + || !valid_artifact_set(®istry, &artifacts) { - return Err(PackageError::Verification); + return Err(derivation(COMPILED_REGISTRY_PATH)); } - - let expected_operation_access_profiles = operation_access_profile_pairs(®istry); - let expected_fixed_operations = fixed_statistical_operations(®istry); - let mut artifact_ids = BTreeSet::new(); - let mut artifact_paths = BTreeSet::new(); - let mut generated_artifacts = Vec::with_capacity(manifest.artifacts.len()); - for artifact in &manifest.artifacts { - let relative_path = artifact - .path - .strip_prefix("generated/") - .ok_or(PackageError::Verification)?; - if relative_path.is_empty() - || !artifact_ids.insert(artifact.id.as_str()) - || !artifact_paths.insert(relative_path) - || !valid_artifact_access_binding( - artifact.visibility, - artifact.operation_identifier.as_deref(), - artifact.access_binding.as_ref(), - &expected_operation_access_profiles, - &expected_fixed_operations, - ) - { - return Err(PackageError::Verification); - } - let file_entries = manifest - .files - .iter() - .filter(|entry| entry.path == artifact.path) - .collect::>(); - if file_entries.len() != 1 - || !file_entries[0].generated - || file_entries[0].media_type != artifact.media_type - || file_entries[0].visibility != artifact.visibility - || file_entries[0].sha256 != artifact.sha256 - { - return Err(PackageError::Verification); + for artifact in &artifacts.artifacts { + let path = format!("{GENERATED_PREFIX}{}", artifact.path); + if loaded[&path] != artifact.content { + return Err(derivation(&path)); } - let content = loaded - .get(&artifact.path) - .ok_or(PackageError::Verification)? - .clone(); - generated_artifacts.push(GeneratedArtifact { - id: artifact.id.clone(), - path: relative_path.to_owned(), - media_type: artifact.media_type.clone(), - visibility: artifact.visibility, - operation_identifier: artifact.operation_identifier.clone(), - access_binding: artifact.access_binding.clone(), - sha256: artifact.sha256.clone(), - content, - }); } - let loaded_generated_paths = loaded - .keys() - .filter_map(|path| path.strip_prefix("generated/")) - .collect::>(); - if artifact_paths != loaded_generated_paths - || !valid_operation_artifact_bindings( - &manifest.operation_artifact_bindings, - &expected_operation_access_profiles, - &artifact_paths, - ) - { - return Err(PackageError::Verification); - } - let artifacts = ArtifactSet { - contract_revision: registry.contract_revision.clone(), - artifacts: generated_artifacts, - operation_bindings: manifest.operation_artifact_bindings.clone(), - }; - verify_artifact_derivation(®istry, &artifacts)?; Ok(VerifiedPackage { - manifest, + digest: verified.digest().to_owned(), + revision: verified.revision().map(str::to_owned), contract, registry, artifacts, }) } -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -struct UnsignedManifest<'a> { - package_version: &'static str, - contract_revision: &'a str, - source_schema_fingerprints: &'a BTreeMap, - source_schemas: &'a BTreeMap, - artifacts: &'a [PackageArtifact], - operation_artifact_bindings: &'a [OperationArtifactBindings], - files: &'a [PackageFile], -} - fn valid_operation_artifact_bindings( bindings: &[OperationArtifactBindings], expected_operation_access_profiles: &BTreeSet<(&str, &str)>, @@ -849,14 +727,14 @@ fn enumerate_package_files(root: &Path) -> Result, PackageError ) -> Result<(), PackageError> { let metadata = fs::symlink_metadata(directory).map_err(|_| PackageError::Read)?; if metadata.file_type().is_symlink() || !metadata.is_dir() || !safe_permissions(&metadata) { - return Err(PackageError::Verification); + return Err(PackageError::UnsafePermissions); } for entry in fs::read_dir(directory).map_err(|_| PackageError::Read)? { let entry = entry.map_err(|_| PackageError::Read)?; let path = entry.path(); let metadata = fs::symlink_metadata(&path).map_err(|_| PackageError::Read)?; if metadata.file_type().is_symlink() || !safe_permissions(&metadata) { - return Err(PackageError::Verification); + return Err(PackageError::UnsafePermissions); } if metadata.is_dir() { visit(root, &path, files)?; @@ -870,7 +748,7 @@ fn enumerate_package_files(root: &Path) -> Result, PackageError return Err(PackageError::ClosureBound); } } else { - return Err(PackageError::Verification); + return Err(PackageError::UnsafePermissions); } } Ok(()) @@ -965,156 +843,390 @@ fn harden_package_permissions(_root: &Path) -> Result<(), PackageError> { Ok(()) } -fn write_generated(root: &Path, artifact: &GeneratedArtifact) -> Result<(), PackageError> { - validate_relative(&artifact.path)?; - write_new_file( - &root.join("generated").join(&artifact.path), - &artifact.content, - ) -} - -fn write_new_file(path: &Path, content: &[u8]) -> Result<(), PackageError> { - if path.exists() { - return Err(PackageError::Write); - } - if let Some(parent) = path.parent() { - fs::create_dir_all(parent).map_err(|_| PackageError::Write)?; - } - fs::write(path, content).map_err(|_| PackageError::Write) -} - -fn file_entry( - path: &str, - content: &[u8], - media_type: &str, - visibility: Visibility, - generated: bool, -) -> PackageFile { - PackageFile { - path: path.into(), - size: content.len() as u64, - sha256: digest(content), - media_type: media_type.into(), - visibility, - generated, - } -} - fn digest(content: &[u8]) -> String { format!("sha256:{}", hex::encode(Sha256::digest(content))) } -fn media_type(path: &str) -> &'static str { - match Path::new(path) - .extension() - .and_then(|extension| extension.to_str()) - { - Some("json") | Some("jsonld") => "application/json", - Some("ttl") => "text/turtle", - _ => "application/yaml", - } -} - #[cfg(test)] mod tests { use super::*; - fn bounded_file(size: u64) -> PackageFile { - PackageFile { - path: "bounded-fixture".into(), - size, - sha256: digest(b""), - media_type: "application/octet-stream".into(), - visibility: Visibility::OperatorOnly, - generated: true, + struct Fixture { + temporary: tempfile::TempDir, + project: std::path::PathBuf, + contract: RegistryContract, + registry: CompiledRegistry, + artifacts: ArtifactSet, + } + + impl Fixture { + fn new() -> Self { + let temporary = tempfile::tempdir().expect("temporary project"); + let project = temporary.path().join("project"); + fs::create_dir(&project).expect("project directory"); + fs::write( + project.join("registry.yaml"), + crate::compiler::tests::valid_contract(), + ) + .expect("registry contract"); + let governed = crate::compiler::tests::governed_files(); + for (relative, content) in &governed { + let path = project.join(relative); + fs::create_dir_all(path.parent().expect("parent")).expect("governed directory"); + fs::write(path, content).expect("governed file"); + } + let contract = RegistryContract::parse_yaml(crate::compiler::tests::valid_contract()) + .expect("strict contract"); + let registry = compile_contract_with_governed_files( + &contract, + &[crate::compiler::tests::observed_schema()], + CompileProfile::Production, + &governed, + ) + .expect("compiled Registry"); + let artifacts = generate_artifacts(®istry).expect("artifacts"); + Self { + temporary, + project, + contract, + registry, + artifacts, + } + } + + /// Build a package named `name` and return its resolved path. macOS + /// places temporary directories below `/var`, which is itself a + /// symlink, and the loader refuses symlink traversal. + fn package( + &self, + name: &str, + revision: Option<&str>, + ) -> (std::path::PathBuf, PackageSummary) { + let output = self.temporary.path().join(name); + let summary = self + .build(Some(&output), revision, &self.registry, &self.artifacts) + .expect("package builds"); + ( + output.canonicalize().expect("resolved package path"), + summary, + ) + } + + fn build( + &self, + output: Option<&Path>, + revision: Option<&str>, + registry: &CompiledRegistry, + artifacts: &ArtifactSet, + ) -> Result { + build_package( + &self.project, + output, + revision, + &self.contract, + registry, + artifacts, + ) } } - #[test] - fn package_file_count_bound_matches_the_loader() { - let files = vec![bounded_file(0); MAX_PACKAGE_FILES]; - assert!(validate_package_bounds(&files, 1).is_ok()); + /// Rewrite `SHA256SUMS` over whatever the directory now holds, as a + /// forger who can recompute digests would. + fn reseal(package_path: &Path) { + fs::remove_file(package_path.join(SUM_FILE)).expect("sum file removes"); + shared::write_sum_file(package_path, None, &package_limits(), PACKAGE_COMMAND) + .expect("forged sum file writes"); + } + + fn refusal(package_path: &Path) -> String { + load_package(package_path) + .expect_err("the package is refused") + .to_string() + } - let files = vec![bounded_file(0); MAX_PACKAGE_FILES + 1]; + #[test] + fn package_limits_bound_files_and_bytes() { + let limits = package_limits(); + assert_eq!(limits.max_files, MAX_PACKAGE_FILES); + assert_eq!(limits.max_total_bytes, MAX_PACKAGE_BYTES); + let files = (0..=MAX_PACKAGE_FILES) + .map(|index| (format!("generated/{index}.json"), Vec::new())) + .collect::>(); + let error = shared::plan_package(Path::new("."), &files, None, &limits, PACKAGE_COMMAND) + .expect_err("one file too many"); assert!(matches!( - validate_package_bounds(&files, 1), - Err(PackageError::ClosureBound) + error.kind(), + shared::PackageErrorKind::Bound { .. } )); } #[test] - fn package_manifest_byte_bound_matches_the_loader() { - let files = [bounded_file(0)]; - assert!(validate_package_bounds( - &files, - usize::try_from(MAX_MANIFEST_BYTES).expect("manifest cap fits usize") + fn a_package_is_the_shared_format_and_reproduces() { + let fixture = Fixture::new(); + let (package_path, summary) = fixture.package("package", None); + let sums = fs::read(package_path.join(SUM_FILE)).expect("sum file"); + assert_eq!(summary.package_digest, digest(&sums)); + assert!(!summary.dry_run); + assert!(!package_path.join(RETIRED_MANIFEST_PATH).exists()); + assert!(summary + .files + .iter() + .any(|file| file.path == COMPILED_REGISTRY_PATH)); + assert_eq!( + summary + .source_schema_fingerprints + .keys() + .collect::>(), + fixture + .registry + .sources + .iter() + .map(|source| &source.id) + .collect::>() + ); + + let verified = load_package(&package_path).expect("verified package"); + assert_eq!(verified.digest, summary.package_digest); + assert_eq!(verified.revision, None); + assert_eq!(verified.registry, fixture.registry); + assert_eq!(verified.artifacts, fixture.artifacts); + assert_eq!( + verified.source_schemas().into_values().collect::>(), + [crate::compiler::tests::observed_schema()] + ); + + let (_again, again) = fixture.package("again", None); + assert_eq!(again.package_digest, summary.package_digest); + let planned = fixture + .build(None, None, &fixture.registry, &fixture.artifacts) + .expect("dry run"); + assert!(planned.dry_run); + assert_eq!(planned.package_digest, summary.package_digest); + assert_eq!(planned.files, summary.files); + + let (revised_path, revised) = fixture.package("revised", Some("release 7")); + assert_ne!(revised.package_digest, summary.package_digest); + assert_eq!(revised.revision.as_deref(), Some("release 7")); + assert_eq!( + load_package(&revised_path) + .expect("revised package") + .revision + .as_deref(), + Some("release 7") + ); + + let error = fixture + .build( + Some(&fixture.temporary.path().join("package")), + None, + &fixture.registry, + &fixture.artifacts, + ) + .expect_err("an existing output is refused"); + assert!( + matches!(&error, PackageError::Package(error) + if matches!(error.kind(), shared::PackageErrorKind::OutputExists)), + "{error}" + ); + } + + #[test] + fn a_changed_missing_or_extra_file_is_refused_by_name() { + let fixture = Fixture::new(); + let (package_path, summary) = fixture.package("package", None); + let artifact = summary + .files + .iter() + .find(|file| file.path.starts_with(GENERATED_PREFIX)) + .expect("generated file") + .path + .clone(); + + let original = fs::read(package_path.join(&artifact)).expect("artifact bytes"); + fs::write(package_path.join(&artifact), b"changed").expect("change a file"); + let message = refusal(&package_path); + assert!( + message.contains(&format!("changed: {artifact}")), + "{message}" + ); + assert!(message.contains("package.root"), "{message}"); + assert!(message.contains(PACKAGE_COMMAND), "{message}"); + assert!( + !message.contains(&*package_path.to_string_lossy()), + "{message}" + ); + fs::write(package_path.join(&artifact), &original).expect("restore"); + + fs::remove_file(package_path.join(REGISTRY_PATH)).expect("remove a file"); + let message = refusal(&package_path); + assert!(message.contains("missing: registry.yaml"), "{message}"); + assert!(message.contains(PACKAGE_COMMAND), "{message}"); + fs::write( + package_path.join(REGISTRY_PATH), + crate::compiler::tests::valid_contract(), ) - .is_ok()); - assert!(matches!( - validate_package_bounds( - &files, - usize::try_from(MAX_MANIFEST_BYTES + 1).expect("manifest cap plus one fits usize") - ), - Err(PackageError::ClosureBound) - )); + .expect("restore"); + + fs::write(package_path.join("notes.txt"), b"extra").expect("add a file"); + let message = refusal(&package_path); + assert!(message.contains("extra: notes.txt"), "{message}"); + assert!(message.contains(PACKAGE_COMMAND), "{message}"); + fs::remove_file(package_path.join("notes.txt")).expect("remove the extra file"); + + load_package(&package_path).expect("the restored package verifies"); } #[test] - fn package_total_byte_bound_matches_the_loader() { - let at_cap = [bounded_file(MAX_PACKAGE_BYTES - 1)]; - assert!(validate_package_bounds(&at_cap, 1).is_ok()); + fn a_directory_that_is_not_a_package_is_refused_with_the_command() { + let fixture = Fixture::new(); + let project = fixture.project.canonicalize().expect("resolved project"); + let message = refusal(&project); + assert!(message.contains("has no SHA256SUMS"), "{message}"); + assert!(message.contains(PACKAGE_COMMAND), "{message}"); + + let (package_path, _) = fixture.package("package", None); + fs::write(package_path.join(RETIRED_MANIFEST_PATH), b"{}").expect("retired manifest"); + let message = refusal(&package_path); + assert!(message.contains(RETIRED_MANIFEST_PATH), "{message}"); + assert!(message.contains(PACKAGE_COMMAND), "{message}"); + } - let above_cap = [bounded_file(MAX_PACKAGE_BYTES)]; - assert!(matches!( - validate_package_bounds(&above_cap, 1), - Err(PackageError::ClosureBound) - )); + #[test] + fn a_resealed_package_must_hold_exactly_what_its_registry_names() { + let fixture = Fixture::new(); + + let (package_path, _) = fixture.package("extra", None); + fs::write(package_path.join("governed/unreferenced.yaml"), b"x: 1\n") + .expect("unreferenced governed file"); + reseal(&package_path); + let message = refusal(&package_path); + assert!( + message.contains("extra: governed/unreferenced.yaml"), + "{message}" + ); + + let (package_path, summary) = fixture.package("missing", None); + let artifact = summary + .files + .iter() + .find(|file| file.path.starts_with(GENERATED_PREFIX)) + .expect("generated file") + .path + .clone(); + fs::remove_file(package_path.join(&artifact)).expect("remove an artifact"); + reseal(&package_path); + let message = refusal(&package_path); + assert!( + message.contains(&format!("missing: {artifact}")), + "{message}" + ); } - fn reseal_manifest(package_path: &Path, manifest: &mut PackageManifest) { - let unsigned = UnsignedManifest { - package_version: PACKAGE_VERSION, - contract_revision: &manifest.contract_revision, - source_schema_fingerprints: &manifest.source_schema_fingerprints, - source_schemas: &manifest.source_schemas, - artifacts: &manifest.artifacts, - operation_artifact_bindings: &manifest.operation_artifact_bindings, - files: &manifest.files, - }; - let unsigned_bytes = canonicalize_json( - &serde_json::to_value(unsigned).expect("unsigned manifest serializes"), + #[test] + fn a_resealed_package_must_reproduce_its_compiled_registry_and_artifacts() { + let fixture = Fixture::new(); + + let (package_path, _) = fixture.package("forged-compiled-registry", None); + let compiled_path = package_path.join(COMPILED_REGISTRY_PATH); + let mut forged: CompiledRegistry = + serde_json::from_slice(&fs::read(&compiled_path).expect("compiled Registry bytes")) + .expect("compiled Registry parses"); + forged.registry_name = "Forged Registry semantics".into(); + fs::write( + &compiled_path, + canonical_compiled(&forged).expect("forged bytes"), ) - .expect("unsigned manifest canonicalizes"); - manifest.package_revision = digest(&unsigned_bytes); - let manifest_bytes = - canonicalize_json(&serde_json::to_value(manifest).expect("sealed manifest serializes")) - .expect("sealed manifest canonicalizes"); - fs::write(package_path.join("relay-package.json"), manifest_bytes) - .expect("forged manifest writes"); + .expect("forged Registry writes"); + reseal(&package_path); + let message = refusal(&package_path); + assert!( + message.contains(&format!("holds {COMPILED_REGISTRY_PATH}")), + "{message}" + ); + assert!(message.contains(PACKAGE_COMMAND), "{message}"); + + let (package_path, _) = fixture.package("non-canonical-registry", None); + let compiled_path = package_path.join(COMPILED_REGISTRY_PATH); + let mut bytes = fs::read(&compiled_path).expect("compiled Registry bytes"); + bytes.push(b'\n'); + fs::write(&compiled_path, bytes).expect("non-canonical Registry writes"); + reseal(&package_path); + assert!(refusal(&package_path).contains(COMPILED_REGISTRY_PATH)); + + let (package_path, summary) = fixture.package("forged-artifact", None); + let artifact = summary + .files + .iter() + .find(|file| file.path.starts_with(GENERATED_PREFIX)) + .expect("generated file") + .path + .clone(); + let mut content = fs::read(package_path.join(&artifact)).expect("artifact bytes"); + content.extend_from_slice(b"tampered"); + fs::write(package_path.join(&artifact), content).expect("forged artifact writes"); + reseal(&package_path); + let message = refusal(&package_path); + assert!(message.contains(&format!("holds {artifact}")), "{message}"); + + let (package_path, _) = fixture.package("forged-governed", None); + let governed = fixture + .registry + .governed_files + .first() + .expect("governed file"); + let governed_path = format!("{GOVERNED_PREFIX}{}", governed.path); + let mut content = fs::read(package_path.join(&governed_path)).expect("governed bytes"); + content.extend_from_slice(b"\n# tampered\n"); + fs::write(package_path.join(&governed_path), content).expect("forged governed writes"); + reseal(&package_path); + let message = refusal(&package_path); + assert!( + message.contains(&format!("holds {governed_path}")), + "{message}" + ); } - fn assert_resealed_package_rejected( - root: &Path, - project: &Path, - name: &str, - contract: &RegistryContract, - registry: &CompiledRegistry, - artifacts: &ArtifactSet, - mutate: impl FnOnce(&Path, &mut PackageManifest), - ) { - let package_path = root.join(name); - let mut manifest = build_package(project, &package_path, contract, registry, artifacts) - .expect("forgery fixture"); - mutate(&package_path, &mut manifest); - reseal_manifest(&package_path, &mut manifest); - assert!(matches!( - load_package( - &package_path - .canonicalize() - .expect("forged package resolves") - ), - Err(PackageError::Verification) - )); + #[test] + fn inconsistent_build_inputs_are_refused() { + let fixture = Fixture::new(); + let artifacts = &fixture.artifacts; + let registry = &fixture.registry; + let refused = |registry: &CompiledRegistry, artifacts: &ArtifactSet| { + matches!( + fixture.build(None, None, registry, artifacts), + Err(PackageError::Verification) + ) + }; + + let mut mismatched = artifacts.clone(); + mismatched.contract_revision = "sha256:mismatched".into(); + assert!(refused(registry, &mismatched)); + + let mut tampered = artifacts.clone(); + let artifact = tampered.artifacts.first_mut().expect("generated artifact"); + artifact.content.extend_from_slice(b"tampered"); + artifact.sha256 = digest(&artifact.content); + assert!(refused(registry, &tampered)); + + let mut tampered = artifacts.clone(); + let artifact = tampered.artifacts.first_mut().expect("generated artifact"); + artifact.visibility = match artifact.visibility { + Visibility::OperatorOnly => Visibility::Public, + Visibility::Public | Visibility::OperationBound => Visibility::OperatorOnly, + }; + assert!(refused(registry, &tampered)); + + let mut tampered = artifacts.clone(); + let binding = tampered + .operation_bindings + .first_mut() + .expect("operation artifact binding"); + binding.context_path = binding.vocabulary_path.clone(); + assert!(refused(registry, &tampered)); + + let mut mismatched_registry = registry.clone(); + mismatched_registry.registry_name = "Different Registry semantics".into(); + assert!(refused(&mismatched_registry, artifacts)); } #[test] @@ -1201,20 +1313,27 @@ mod tests { fs::write(path, content).expect("governed file"); } let package_path = temporary.path().join("package"); - let manifest = build_package(&project, &package_path, &contract, ®istry, &artifacts) - .expect("multi-profile package builds"); + build_package( + &project, + Some(&package_path), + None, + &contract, + ®istry, + &artifacts, + ) + .expect("multi-profile package builds"); let verified = load_package( &package_path .canonicalize() .expect("multi-profile package resolves"), ) .expect("multi-profile package loads"); - assert_eq!(verified.manifest, manifest); + assert_eq!(verified.artifacts, artifacts); assert_eq!(verified.artifacts.operation_bindings.len(), 3); } #[test] - fn statistical_structure_artifacts_are_exactly_bound_in_v1alpha3_package() { + fn statistical_structure_artifacts_are_exactly_bound_in_a_package() { let yaml = crate::compiler::tests::statistical_contract() .replace( " access: public\n query:", @@ -1247,9 +1366,15 @@ mod tests { } let package_path = temporary.path().join("package"); - let manifest = build_package(&project, &package_path, &contract, ®istry, &artifacts) - .expect("statistical package builds"); - assert_eq!(manifest.package_version, PACKAGE_VERSION); + build_package( + &project, + Some(&package_path), + None, + &contract, + ®istry, + &artifacts, + ) + .expect("statistical package builds"); let operation_identifier = registry.statistical_datasets[0].operation_identifier(); let record_bindings = BTreeSet::new(); let fixed_operations = fixed_statistical_operations(®istry); @@ -1266,7 +1391,7 @@ mod tests { "labour-rates-sdmx-dataflow-structure", "labour-rates-sdmx-datastructure-structure", ] { - let packaged = manifest + let packaged = artifacts .artifacts .iter() .find(|artifact| artifact.id == id) @@ -1280,26 +1405,12 @@ mod tests { packaged.access_binding, Some(ArtifactAccessBinding::FixedOperation) ); - let generated = artifacts - .artifacts - .iter() - .find(|artifact| artifact.id == id) - .expect("generated structure artifact"); assert_eq!( - fs::read(package_path.join(&packaged.path)).expect("packaged structure bytes"), - generated.content + fs::read(package_path.join(GENERATED_PREFIX).join(&packaged.path)) + .expect("packaged structure bytes"), + packaged.content ); } - let manifest_value = serde_json::to_value(&manifest).expect("manifest serializes"); - assert!(manifest_value["artifacts"] - .as_array() - .expect("package artifacts") - .iter() - .filter(|artifact| { - artifact["id"] == "labour-rates-sdmx-dataflow-structure" - || artifact["id"] == "labour-rates-sdmx-datastructure-structure" - }) - .all(|artifact| artifact.get("accessProfileIdentifier").is_none())); let verified = load_package( &package_path @@ -1307,7 +1418,6 @@ mod tests { .expect("statistical package resolves"), ) .expect("statistical package loads"); - assert_eq!(verified.manifest, manifest); assert_eq!(verified.artifacts, artifacts); } @@ -1405,254 +1515,4 @@ mod tests { Err(PackageError::UnsafeClosure) )); } - - #[test] - fn sealed_package_reproduces_and_tampering_is_refused() { - let temporary = tempfile::tempdir().expect("temporary project"); - let project = temporary.path().join("project"); - fs::create_dir(&project).expect("project directory"); - fs::write( - project.join("registry.yaml"), - crate::compiler::tests::valid_contract(), - ) - .expect("registry contract"); - let governed = crate::compiler::tests::governed_files(); - for (relative, content) in &governed { - let path = project.join(relative); - fs::create_dir_all(path.parent().expect("parent")).expect("governed directory"); - fs::write(path, content).expect("governed file"); - } - let contract = RegistryContract::parse_yaml(crate::compiler::tests::valid_contract()) - .expect("strict contract"); - let registry = compile_contract_with_governed_files( - &contract, - &[crate::compiler::tests::observed_schema()], - CompileProfile::Production, - &governed, - ) - .expect("compiled Registry"); - let artifacts = generate_artifacts(®istry).expect("artifacts"); - let mut mismatched_artifacts = artifacts.clone(); - mismatched_artifacts.contract_revision = "sha256:mismatched".into(); - assert!(matches!( - build_package( - &project, - &temporary.path().join("rejected-package"), - &contract, - ®istry, - &mismatched_artifacts, - ), - Err(PackageError::Verification) - )); - let mut tampered_artifact_bytes = artifacts.clone(); - let tampered_artifact = tampered_artifact_bytes - .artifacts - .first_mut() - .expect("generated artifact"); - tampered_artifact.content.extend_from_slice(b"tampered"); - tampered_artifact.sha256 = digest(&tampered_artifact.content); - assert!(matches!( - build_package( - &project, - &temporary.path().join("rejected-artifact-bytes"), - &contract, - ®istry, - &tampered_artifact_bytes, - ), - Err(PackageError::Verification) - )); - let mut tampered_artifact_visibility = artifacts.clone(); - let tampered_artifact = tampered_artifact_visibility - .artifacts - .first_mut() - .expect("generated artifact"); - tampered_artifact.visibility = match tampered_artifact.visibility { - Visibility::OperatorOnly => Visibility::Public, - Visibility::Public | Visibility::OperationBound => Visibility::OperatorOnly, - }; - assert!(matches!( - build_package( - &project, - &temporary.path().join("rejected-artifact-visibility"), - &contract, - ®istry, - &tampered_artifact_visibility, - ), - Err(PackageError::Verification) - )); - let mut tampered_artifact_binding = artifacts.clone(); - let binding = tampered_artifact_binding - .operation_bindings - .first_mut() - .expect("operation artifact binding"); - binding.context_path = binding.vocabulary_path.clone(); - assert!(matches!( - build_package( - &project, - &temporary.path().join("rejected-artifact-binding"), - &contract, - ®istry, - &tampered_artifact_binding, - ), - Err(PackageError::Verification) - )); - let mut mismatched_registry = registry.clone(); - mismatched_registry.registry_name = "Different Registry semantics".into(); - assert!(matches!( - build_package( - &project, - &temporary.path().join("rejected-compiled-registry"), - &contract, - &mismatched_registry, - &artifacts, - ), - Err(PackageError::Verification) - )); - let output = temporary.path().join("package"); - let manifest = build_package(&project, &output, &contract, ®istry, &artifacts) - .expect("sealed package"); - // macOS places temporary directories below `/var`, which is itself a - // symlink. The production loader correctly refuses paths containing - // symlink traversal, so exercise it with the resolved package path. - let resolved_output = output.canonicalize().expect("resolved package path"); - let verified = load_package(&resolved_output).expect("verified package"); - assert_eq!(verified.manifest, manifest); - assert_eq!(verified.registry, registry); - assert_eq!(verified.artifacts, artifacts); - assert!(manifest - .files - .iter() - .any(|file| file.path == COMPILED_REGISTRY_PATH)); - assert_eq!( - manifest.operation_artifact_bindings, - artifacts.operation_bindings - ); - let serialized_manifest = serde_json::to_value(&manifest).expect("manifest serializes"); - assert!(serialized_manifest["artifacts"] - .as_array() - .expect("artifact array") - .iter() - .filter(|artifact| artifact["operationIdentifier"].is_string()) - .all(|artifact| artifact.get("accessBinding").is_some() - && artifact.get("accessProfileIdentifier").is_none() - && artifact.get("representationIdentifier").is_none())); - - assert_resealed_package_rejected( - temporary.path(), - &project, - "forged-compiled-registry", - &contract, - ®istry, - &artifacts, - |package_path, manifest| { - let compiled_path = package_path.join(COMPILED_REGISTRY_PATH); - let mut forged: CompiledRegistry = serde_json::from_slice( - &fs::read(&compiled_path).expect("compiled Registry bytes"), - ) - .expect("compiled Registry parses"); - forged.registry_name = "Forged Registry semantics".into(); - let forged_bytes = canonicalize_json( - &serde_json::to_value(forged).expect("forged Registry serializes"), - ) - .expect("forged Registry canonicalizes"); - fs::write(&compiled_path, &forged_bytes).expect("forged Registry writes"); - let file = manifest - .files - .iter_mut() - .find(|file| file.path == COMPILED_REGISTRY_PATH) - .expect("compiled Registry package file"); - file.size = forged_bytes.len() as u64; - file.sha256 = digest(&forged_bytes); - }, - ); - assert_resealed_package_rejected( - temporary.path(), - &project, - "forged-artifact-content", - &contract, - ®istry, - &artifacts, - |package_path, manifest| { - let artifact = manifest.artifacts.first_mut().expect("generated artifact"); - let file_path = artifact.path.clone(); - let mut content = fs::read(package_path.join(&file_path)).expect("artifact bytes"); - content.extend_from_slice(b"tampered"); - fs::write(package_path.join(&file_path), &content).expect("forged artifact bytes"); - let content_digest = digest(&content); - artifact.sha256 = content_digest.clone(); - let file = manifest - .files - .iter_mut() - .find(|file| file.path == file_path) - .expect("generated package file"); - file.size = content.len() as u64; - file.sha256 = content_digest; - }, - ); - assert_resealed_package_rejected( - temporary.path(), - &project, - "forged-artifact-visibility", - &contract, - ®istry, - &artifacts, - |_package_path, manifest| { - let artifact = manifest.artifacts.first_mut().expect("generated artifact"); - let file_path = artifact.path.clone(); - let visibility = match artifact.visibility { - Visibility::OperatorOnly => Visibility::Public, - Visibility::Public | Visibility::OperationBound => Visibility::OperatorOnly, - }; - artifact.visibility = visibility; - manifest - .files - .iter_mut() - .find(|file| file.path == file_path) - .expect("generated package file") - .visibility = visibility; - }, - ); - assert_resealed_package_rejected( - temporary.path(), - &project, - "forged-swapped-binding", - &contract, - ®istry, - &artifacts, - |_package_path, manifest| { - let binding = manifest - .operation_artifact_bindings - .first_mut() - .expect("operation artifact binding"); - let vocabulary_path = binding.vocabulary_path.clone(); - binding.vocabulary_path = binding.context_path.clone(); - binding.context_path = vocabulary_path; - }, - ); - assert_resealed_package_rejected( - temporary.path(), - &project, - "forged-repeated-binding", - &contract, - ®istry, - &artifacts, - |_package_path, manifest| { - let binding = manifest - .operation_artifact_bindings - .first_mut() - .expect("operation artifact binding"); - binding.context_path = binding.vocabulary_path.clone(); - }, - ); - - let compiled_path = output.join(COMPILED_REGISTRY_PATH); - let compiled_bytes = fs::read(&compiled_path).expect("compiled Registry bytes"); - fs::write(&compiled_path, b"{}").expect("tamper compiled Registry"); - assert!(load_package(&resolved_output).is_err()); - fs::write(&compiled_path, compiled_bytes).expect("restore compiled Registry"); - - let artifact = &manifest.artifacts[0]; - fs::write(output.join(&artifact.path), b"tampered").expect("tamper fixture"); - assert!(load_package(&resolved_output).is_err()); - } } diff --git a/crates/registry-relay-v2/src/schema.rs b/crates/registry-relay-v2/src/schema.rs index a0dcf11a5d..673fafab3f 100644 --- a/crates/registry-relay-v2/src/schema.rs +++ b/crates/registry-relay-v2/src/schema.rs @@ -117,4 +117,18 @@ mod tests { assert!(!validator.is_valid(&refused), "{refused}"); } } + + #[test] + fn the_runtime_schema_pins_the_envelope() { + let documents = documents().unwrap(); + let runtime: Value = serde_json::from_str(&documents[RUNTIME_SCHEMA_FILE]).unwrap(); + assert_eq!( + runtime["properties"]["apiVersion"]["const"], + crate::contract::RELAY_RUNTIME_API_VERSION + ); + assert_eq!( + runtime["properties"]["kind"]["const"], + crate::contract::RELAY_RUNTIME_KIND + ); + } } diff --git a/crates/registry-relay-v2/src/source_observation.rs b/crates/registry-relay-v2/src/source_observation.rs index b89de2de9e..6b5e270c66 100644 --- a/crates/registry-relay-v2/src/source_observation.rs +++ b/crates/registry-relay-v2/src/source_observation.rs @@ -108,7 +108,7 @@ mod tests { #[test] fn governed_source_observation_uses_the_runtime_request_timeout() { let runtime = RelayRuntime::parse_yaml( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {bind: '127.0.0.1:8080'}\npackagePath: package\nsources: {db: {path: fixture.sqlite}}\nauthentication: {issuer: null}\naudit: {path: var/audit.jsonl}\nlimits: {requestTimeoutMilliseconds: 37, concurrentQueries: 1}\n", + "apiVersion: registry.registrystack.org/relay-runtime/v1alpha1\nkind: RelayRuntimeConfig\nlistener: {bind: '127.0.0.1:8080'}\npackage: {root: /srv/relay/package}\nsecretProviders: {environment: {}}\nsources: {db: {path: fixture.sqlite}}\naudit: {path: var/audit.jsonl}\nlimits: {requestTimeoutMilliseconds: 37, concurrentQueries: 1}\n", ) .expect("runtime parses"); diff --git a/crates/registry-relay-v2/src/startup.rs b/crates/registry-relay-v2/src/startup.rs index acba5fb4a9..5f8635ae7e 100644 --- a/crates/registry-relay-v2/src/startup.rs +++ b/crates/registry-relay-v2/src/startup.rs @@ -4,7 +4,6 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::future::IntoFuture; -use std::io::Read as _; use std::net::SocketAddr; use std::path::{Component, Path, PathBuf}; use std::sync::Arc; @@ -15,7 +14,7 @@ use jsonwebtoken::Algorithm; use registry_platform_audit::{ require_audit_under, AuditDestination, AuditWriter, PersistentRootFault, }; -use registry_platform_config::{SecretProvider, SecretResolver}; +use registry_platform_config::SecretResolver; use registry_platform_httputil::FetchUrlPolicy; use registry_platform_oidc::{ fetch_discovery_at_with_policy, fetch_discovery_with_policy, JwksFetcher, JwksFetcherConfig, @@ -29,8 +28,7 @@ use crate::audit::RelayAudit; use crate::auth::RelayAuthenticator; use crate::contract::{ contract_has_protected_access, runtime_cursor_configuration_is_valid, IssuerAlgorithm, - IssuerKeyTransport, IssuerProfile, IssuerRuntime, RegistryContract, RelayRuntime, - MAXIMUM_RUNTIME_BYTES, + IssuerKeyTransport, IssuerProfile, OidcRuntime, RegistryContract, RelayRuntime, RuntimeRefusal, }; use crate::cursor::CursorKey; use crate::package::{load_package, VerifiedPackage}; @@ -49,14 +47,22 @@ const HEALTHCHECK_TIMEOUT: Duration = Duration::from_secs(5); const MAXIMUM_HEALTH_BODY_BYTES: usize = 128; const HEALTH_BODY: &[u8] = br#"{"status":"ok"}"#; -#[derive(Debug, Error, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Error, Clone, PartialEq, Eq)] pub enum StartupError { #[error("the runtime configuration could not be loaded")] RuntimeLoad, #[error("the runtime configuration is invalid")] RuntimeInvalid, - #[error("the sealed package could not be verified")] - PackageInvalid, + /// The shared runtime-configuration loader or a shared block refused the + /// file. The message names the field and never a configured value or the + /// runtime file's path. + #[error("the runtime configuration is refused: {0}")] + RuntimeRefused(String), + /// The package at `package.root` was refused. The message names the + /// field, the package files involved, and `relayctl package`, never the + /// configured directory. + #[error("the package is refused: {0}")] + PackageRefused(String), #[error("a runtime source could not be verified")] SourceInvalid, #[error("the configured issuer is not ready")] @@ -132,7 +138,12 @@ async fn prepare_loaded(loaded: LoadedRuntime) -> Result Result Result Result<(), StartupError> { Ok(()) } +/// Read the runtime once through the shared loader: an absolute, lexically +/// normal path with no symbolic-link component, a trusted owner and mode on +/// the file and every ancestor, and a bounded size. The directory holding the +/// file is the root that relative source and audit bindings resolve against. fn load_runtime(path: &Path) -> Result<(PathBuf, RelayRuntime), StartupError> { - let path_metadata = validate_runtime_path(path)?; - let mut file = fs::File::open(path).map_err(|_| StartupError::RuntimeLoad)?; - let opened_metadata = file.metadata().map_err(|_| StartupError::RuntimeLoad)?; - if !opened_metadata.is_file() - || opened_metadata.len() == 0 - || opened_metadata.len() > MAXIMUM_RUNTIME_BYTES - || !same_file(&path_metadata, &opened_metadata) - || !safe_runtime_permissions(&opened_metadata) - { - return Err(StartupError::RuntimeInvalid); - } - let mut bytes = Vec::with_capacity(opened_metadata.len() as usize); - file.by_ref() - .take(MAXIMUM_RUNTIME_BYTES.saturating_add(1)) - .read_to_end(&mut bytes) - .map_err(|_| StartupError::RuntimeLoad)?; - let final_metadata = validate_runtime_path(path)?; - if bytes.len() as u64 > MAXIMUM_RUNTIME_BYTES || !same_file(&final_metadata, &opened_metadata) { - return Err(StartupError::RuntimeInvalid); - } - let yaml = std::str::from_utf8(&bytes).map_err(|_| StartupError::RuntimeInvalid)?; - let runtime = RelayRuntime::parse_yaml(yaml).map_err(|_| StartupError::RuntimeInvalid)?; - let parent = path + let runtime: RelayRuntime = RelayRuntime::loader() + .load(path) + .map_err(|error| StartupError::RuntimeRefused(RuntimeRefusal::from(error).to_string()))? + .config; + runtime + .check() + .map_err(|error| StartupError::RuntimeRefused(error.to_string()))?; + let root = path .parent() - .filter(|value| !value.as_os_str().is_empty()) - .unwrap_or(Path::new(".")); - let root = parent - .canonicalize() - .map_err(|_| StartupError::RuntimeLoad)?; + .ok_or(StartupError::RuntimeLoad)? + .to_path_buf(); Ok((root, runtime)) } -#[cfg(unix)] -fn validate_runtime_path(path: &Path) -> Result { - use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; - - let absolute = if path.is_absolute() { - path.to_owned() - } else { - std::env::current_dir() - .map_err(|_| StartupError::RuntimeLoad)? - .join(path) - }; - let effective_user = rustix::process::geteuid().as_raw(); - let component_count = absolute.components().count(); - let mut current = PathBuf::new(); - let mut final_metadata = None; - for (index, component) in absolute.components().enumerate() { - current.push(component.as_os_str()); - let metadata = fs::symlink_metadata(¤t).map_err(|_| StartupError::RuntimeLoad)?; - let final_component = index + 1 == component_count; - if metadata.file_type().is_symlink() - || if final_component { - !metadata.is_file() - || !trusted_unix_owner_and_mode( - metadata.uid(), - metadata.permissions().mode(), - effective_user, - false, - ) - } else { - !metadata.is_dir() - || !trusted_unix_owner_and_mode( - metadata.uid(), - metadata.permissions().mode(), - effective_user, - true, - ) - } - { - return Err(StartupError::RuntimeInvalid); - } - if final_component { - final_metadata = Some(metadata); - } - } - final_metadata.ok_or(StartupError::RuntimeInvalid) -} - -#[cfg(unix)] -fn trusted_unix_owner_and_mode( - owner: u32, - mode: u32, - effective_user: u32, - allow_root_sticky: bool, -) -> bool { - let trusted_owner = owner == 0 || owner == effective_user; - let not_writable_by_others = mode & 0o022 == 0; - let protected_shared_ancestor = allow_root_sticky && owner == 0 && mode & 0o1000 != 0; - trusted_owner && (not_writable_by_others || protected_shared_ancestor) -} - -#[cfg(not(unix))] -fn validate_runtime_path(_path: &Path) -> Result { - // This trust contract depends on Unix ownership and sticky-directory - // semantics. Platforms without an equivalent implementation fail closed. - Err(StartupError::RuntimeInvalid) -} - -#[cfg(unix)] -fn same_file(path_metadata: &fs::Metadata, opened_metadata: &fs::Metadata) -> bool { - use std::os::unix::fs::MetadataExt as _; - - path_metadata.dev() == opened_metadata.dev() && path_metadata.ino() == opened_metadata.ino() -} - -#[cfg(unix)] -fn safe_runtime_permissions(metadata: &fs::Metadata) -> bool { - use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; - - trusted_unix_owner_and_mode( - metadata.uid(), - metadata.permissions().mode(), - rustix::process::geteuid().as_raw(), - false, - ) -} - -#[cfg(not(unix))] -fn same_file(path_metadata: &fs::Metadata, opened_metadata: &fs::Metadata) -> bool { - path_metadata.len() == opened_metadata.len() - && path_metadata.modified().ok() == opened_metadata.modified().ok() -} - -#[cfg(not(unix))] -fn safe_runtime_permissions(_metadata: &fs::Metadata) -> bool { - false -} - struct RuntimePaths { package: PathBuf, sources: BTreeMap, @@ -478,7 +378,7 @@ struct RuntimePaths { impl RuntimePaths { fn resolve(root: &Path, runtime: &RelayRuntime) -> Result { - let package = resolve_binding(root, &runtime.package_path)?; + let package = runtime.package.root.clone(); reject_existing_symlink_components(&package)?; let mut sources = BTreeMap::new(); for (identifier, source) in runtime.sources.iter() { @@ -565,7 +465,7 @@ fn validate_runtime_contract( if !runtime_cursor_configuration_is_valid(contract, runtime) { return Err(StartupError::CursorInvalid); } - if contract_has_protected_access(contract) && runtime.authentication.issuer.is_none() { + if contract_has_protected_access(contract) && runtime.authentication.oidc.is_none() { return Err(StartupError::IssuerUnavailable); } let has_lookup = contract @@ -586,14 +486,14 @@ fn require_packaged_source_schemas( .iter() .map(|schema| (schema.source.clone(), schema.clone())) .collect::>(); - if observed != package.manifest.source_schemas { + if observed != package.source_schemas() { return Err(StartupError::SourceInvalid); } Ok(()) } async fn build_authenticator( - issuer: Option<&IssuerRuntime>, + issuer: Option<&OidcRuntime>, ) -> Result, StartupError> { let Some(issuer) = issuer else { return Ok(None); @@ -605,7 +505,7 @@ async fn build_authenticator( } async fn build_authenticator_with_profile( - issuer: &IssuerRuntime, + issuer: &OidcRuntime, profile: IssuerProfile, fetch_url_policy: &FetchUrlPolicy, ) -> Result { @@ -676,7 +576,7 @@ async fn build_authenticator_with_profile( /// issuer without weakening the production HTTPS and SSRF policy. #[cfg(feature = "tooling")] pub async fn build_authenticator_for_supervised_local_development( - issuer: &IssuerRuntime, + issuer: &OidcRuntime, ) -> Result { let profile = issuer .supervised_local_profile() @@ -684,7 +584,7 @@ pub async fn build_authenticator_for_supervised_local_development( build_authenticator_with_profile(issuer, profile, &FetchUrlPolicy::dev()).await } -fn verifier_issuer_profile(issuer: &IssuerRuntime) -> Result { +fn verifier_issuer_profile(issuer: &OidcRuntime) -> Result { issuer.profile().ok_or(StartupError::RuntimeInvalid) } @@ -703,13 +603,13 @@ async fn build_audit(destination: AuditDestination) -> Result Result<(Option>, Duration), StartupError> { let Some(cursor) = &runtime.cursor else { return Ok((None, DEFAULT_CURSOR_MAXIMUM_AGE)); }; - let secret = resolve_secret(runtime_root, &cursor.integrity_key_ref)?; + let secret = resolve_secret(secrets, &cursor.integrity_key_ref)?; let key = CursorKey::new(secret.expose_secret().to_vec()).map_err(|_| StartupError::CursorInvalid)?; Ok(( @@ -718,16 +618,14 @@ fn build_cursor( )) } +/// Resolve one reference through exactly the providers `secretProviders` +/// declares; a `secret:file/` reference resolves under +/// `secretProviders.file.root`. fn resolve_secret( - runtime_root: &Path, + secrets: &SecretResolver, reference: &str, ) -> Result { - let resolver = SecretResolver::new( - [SecretProvider::Environment, SecretProvider::File], - runtime_root, - ) - .map_err(|_| StartupError::RuntimeInvalid)?; - resolver + secrets .resolve(reference) .map_err(|_| StartupError::SecretUnavailable) } @@ -785,42 +683,94 @@ async fn shutdown_signal() { #[cfg(test)] mod tests { use super::*; + use registry_platform_config::DEFAULT_MAX_RUNTIME_CONFIG_BYTES; use std::io::Write as _; use tokio::io::AsyncWriteExt as _; + fn runtime_text(bind: &str, package_root: &str, source: &str, audit: &str) -> String { + format!( + "apiVersion: registry.registrystack.org/relay-runtime/v1alpha1\nkind: RelayRuntimeConfig\nlistener: {{bind: '{bind}'}}\npackage: {{root: {package_root}}}\nsecretProviders: {{environment: {{}}}}\nsources: {{{source}: {{path: fixture.sqlite}}}}\naudit: {audit}\nlimits: {{requestTimeoutMilliseconds: 1000, concurrentQueries: 1}}\n" + ) + } + + fn closed_runtime(source: &str) -> RelayRuntime { + RelayRuntime::parse_yaml(&runtime_text( + "127.0.0.1:18081", + "/srv/relay/package", + source, + "{path: var/audit.jsonl}", + )) + .expect("closed runtime") + } + + fn resolver(providers: &str) -> SecretResolver { + serde_norway::from_str::(providers) + .expect("secret providers parse") + .resolver() + .expect("secret resolver") + } + #[test] - fn environment_and_owner_only_file_secrets_use_the_closed_resolver() { + fn secrets_resolve_only_through_the_declared_providers() { const VARIABLE: &str = "RELAY_V2_SECRET_RESOLVER_TEST"; std::env::set_var(VARIABLE, "synthetic-test-key-material-32-bytes-long"); let temporary = tempfile::tempdir().expect("temporary root"); + let root = temporary.path().canonicalize().expect("canonical root"); + let secrets_root = root.join("secrets"); + fs::create_dir(&secrets_root).expect("secrets root"); + let both = resolver(&format!( + "environment: {{}}\nfile: {{root: {}}}", + secrets_root.display() + )); assert_eq!( - resolve_secret(temporary.path(), &format!("secret:env/{VARIABLE}")) + resolve_secret(&both, &format!("secret:env/{VARIABLE}")) .expect("environment secret") .expose_secret(), b"synthetic-test-key-material-32-bytes-long" ); - let path = temporary.path().join("cursor-integrity-key"); + // A file reference resolves under secretProviders.file.root, never + // beside the runtime configuration. + let beside_runtime = root.join("cursor-integrity-key"); + fs::write( + &beside_runtime, + b"synthetic-other-key-material-32-bytes-long", + ) + .expect("decoy writes"); + let path = secrets_root.join("cursor-integrity-key"); fs::write(&path, b"synthetic-file-key-material-32-bytes-long").expect("secret writes"); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt as _; - fs::set_permissions(&path, fs::Permissions::from_mode(0o600)) - .expect("secret becomes owner-only"); + for file in [&path, &beside_runtime] { + fs::set_permissions(file, fs::Permissions::from_mode(0o600)) + .expect("secret becomes owner-only"); + } } assert_eq!( - resolve_secret(temporary.path(), "secret:file/cursor-integrity-key") + resolve_secret(&both, "secret:file/cursor-integrity-key") .expect("file secret") .expose_secret(), b"synthetic-file-key-material-32-bytes-long" ); + let environment_only = resolver("environment: {}"); + assert_eq!( + resolve_secret(&environment_only, "secret:file/cursor-integrity-key").err(), + Some(StartupError::SecretUnavailable) + ); + let file_only = resolver(&format!("file: {{root: {}}}", secrets_root.display())); + assert_eq!( + resolve_secret(&file_only, &format!("secret:env/{VARIABLE}")).err(), + Some(StartupError::SecretUnavailable) + ); + #[cfg(unix)] { use std::os::unix::fs::PermissionsExt as _; fs::set_permissions(&path, fs::Permissions::from_mode(0o640)) .expect("secret becomes unsafe"); - assert!(resolve_secret(temporary.path(), "secret:file/cursor-integrity-key").is_err()); + assert!(resolve_secret(&both, "secret:file/cursor-integrity-key").is_err()); } } @@ -845,26 +795,31 @@ mod tests { #[test] fn issuer_discovery_is_one_exact_https_profile() { - let issuer = |discovery_url: &str| { - serde_norway::from_str::(&format!( - "id: issuer\ndiscoveryUrl: {discovery_url}\naudience: registry\ntokenTypes: [at+jwt]\nalgorithms: [EdDSA]\n" + let issuer = |issuer: &str| { + serde_norway::from_str::(&format!( + "issuer: '{issuer}'\naudience: registry\ntokenTypes: [at+jwt]\nalgorithms: [EdDSA]\n" )) .expect("issuer shape parses") }; - let valid = "https://identity.example.invalid/.well-known/openid-configuration"; - let profile = verifier_issuer_profile(&issuer(valid)).expect("issuer profile validates"); + let profile = verifier_issuer_profile(&issuer("https://identity.example.invalid")) + .expect("issuer profile validates"); assert_eq!( profile.issuer_identifier, "https://identity.example.invalid" ); assert_eq!(profile.algorithm, IssuerAlgorithm::EdDsa); + assert_eq!( + profile.key_transport, + IssuerKeyTransport::Discovery( + "https://identity.example.invalid/.well-known/openid-configuration".to_owned() + ) + ); for invalid in [ - "https://operator:credential@identity.example.invalid/.well-known/openid-configuration", - "https://identity.example.invalid/.well-known/openid-configuration?tenant=x", - "https://identity.example.invalid/.well-known/openid-configuration#fragment", - "https://identity.example.invalid/.well-known/oauth-authorization-server", - "https:///.well-known/openid-configuration", + "https://operator:credential@identity.example.invalid", + "https://identity.example.invalid/?tenant=x", + "https://identity.example.invalid/#fragment", + "http://identity.example.invalid", ] { assert!(matches!( verifier_issuer_profile(&issuer(invalid)), @@ -874,7 +829,7 @@ mod tests { #[cfg(feature = "tooling")] { - let loopback = issuer("http://127.0.0.1:18080/.well-known/openid-configuration"); + let loopback = issuer("http://127.0.0.1:18080"); assert!(matches!( verifier_issuer_profile(&loopback), Err(StartupError::RuntimeInvalid) @@ -884,9 +839,9 @@ mod tests { .expect("tooling accepts one canonical loopback issuer"); assert_eq!(profile.issuer_identifier, "http://127.0.0.1:18080"); for invalid in [ - "http://localhost:18080/.well-known/openid-configuration", - "http://127.0.0.1/.well-known/openid-configuration", - "http://10.0.0.1:18080/.well-known/openid-configuration", + "http://localhost:18080", + "http://127.0.0.1", + "http://10.0.0.1:18080", ] { assert!(issuer(invalid).supervised_local_profile().is_none()); } @@ -908,49 +863,7 @@ mod tests { #[cfg(unix)] #[test] - fn runtime_trust_rejects_foreign_owners_and_limits_the_sticky_exception() { - let effective_user = 1000; - assert!(trusted_unix_owner_and_mode( - effective_user, - 0o100600, - effective_user, - false - )); - assert!(trusted_unix_owner_and_mode( - 0, - 0o100644, - effective_user, - false - )); - assert!(!trusted_unix_owner_and_mode( - effective_user + 1, - 0o100600, - effective_user, - false - )); - assert!(trusted_unix_owner_and_mode( - 0, - 0o041777, - effective_user, - true - )); - assert!(!trusted_unix_owner_and_mode( - 0, - 0o041777, - effective_user, - false - )); - assert!(!trusted_unix_owner_and_mode( - effective_user, - 0o041777, - effective_user, - true - )); - } - - #[cfg(unix)] - #[test] - fn a_runtime_below_a_writable_ancestor_is_rejected() { + fn a_runtime_below_a_writable_ancestor_is_refused_without_naming_the_path() { use std::os::unix::fs::PermissionsExt as _; let temporary = tempfile::tempdir().expect("temporary root"); @@ -960,12 +873,29 @@ mod tests { fs::set_permissions(&writable, fs::Permissions::from_mode(0o777)) .expect("ancestor becomes unsafe"); let runtime = writable.join("runtime.yaml"); - fs::write(&runtime, b"runtime").expect("runtime fixture"); + fs::write( + &runtime, + runtime_text( + "127.0.0.1:0", + "/srv/relay/package", + "db", + "{path: var/audit.jsonl}", + ), + ) + .expect("runtime fixture"); - assert_eq!( - validate_runtime_path(&runtime).err(), - Some(StartupError::RuntimeInvalid) - ); + let Err(StartupError::RuntimeRefused(message)) = load_runtime(&runtime) else { + panic!("a runtime below a writable ancestor must be refused"); + }; + assert!(!message.contains(&*root.to_string_lossy()), "{message}"); + } + + #[test] + fn a_relative_runtime_path_is_refused() { + assert!(matches!( + load_runtime(Path::new("runtime.yaml")), + Err(StartupError::RuntimeRefused(_)) + )); } #[tokio::test] @@ -1028,23 +958,28 @@ mod tests { drop(reservation); let temporary = tempfile::tempdir().expect("temporary root"); - let path = temporary + let root = temporary .path() .canonicalize() - .expect("canonical temporary root") - .join("runtime.yaml"); + .expect("canonical temporary root"); + let path = root.join("runtime.yaml"); fs::write( &path, - format!( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {{bind: '{address}'}}\npackagePath: missing-package\nsources: {{db: {{path: source.sqlite}}}}\nauthentication: {{issuer: null}}\naudit: {{path: var/audit.jsonl}}\nlimits: {{requestTimeoutMilliseconds: 1000, concurrentQueries: 1}}\n" + runtime_text( + &address.to_string(), + &root.join("missing-package").display().to_string(), + "db", + "{path: var/audit.jsonl}", ), ) .expect("write runtime"); - assert_eq!( - prepare(&path).await.err(), - Some(StartupError::PackageInvalid) - ); + let Some(StartupError::PackageRefused(message)) = prepare(&path).await.err() else { + panic!("an absent package is refused"); + }; + assert!(message.contains("package.root"), "{message}"); + assert!(message.contains("relayctl package"), "{message}"); + assert!(!message.contains(&*root.to_string_lossy()), "{message}"); let listener = TcpListener::bind(address) .await .expect("startup did not bind before readiness"); @@ -1102,10 +1037,7 @@ metadataVisibility: {service: public, resources: public, semantics: public, clas let protected = contract( "{read: {defaultAccessProfile: default, accessProfiles: {default: {access: {scope: registry:record:read}, disclosureProfile: default}}}}", ); - let protected_runtime = RelayRuntime::parse_yaml( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {bind: '127.0.0.1:18081'}\npackagePath: package\nsources: {records: {path: fixture.sqlite}}\nauthentication: {issuer: null}\naudit: {path: var/audit.jsonl}\nlimits: {requestTimeoutMilliseconds: 1000, concurrentQueries: 1}\n", - ) - .expect("closed runtime"); + let protected_runtime = closed_runtime("records"); assert_eq!( validate_runtime_contract(&protected_runtime, &protected), Err(StartupError::IssuerUnavailable) @@ -1130,10 +1062,7 @@ metadataVisibility: {service: public, resources: public, semantics: public, clas protected_statistics.statistical_datasets[0].access = serde_norway::from_str("{scope: registry:statistics:read}") .expect("protected statistical access"); - let protected_statistics_runtime = RelayRuntime::parse_yaml( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {bind: '127.0.0.1:18084'}\npackagePath: package\nsources: {db: {path: fixture.sqlite}}\nauthentication: {issuer: null}\naudit: {path: var/audit.jsonl}\nlimits: {requestTimeoutMilliseconds: 1000, concurrentQueries: 1}\n", - ) - .expect("closed runtime"); + let protected_statistics_runtime = closed_runtime("db"); assert_eq!( validate_runtime_contract(&protected_statistics_runtime, &protected_statistics), Err(StartupError::IssuerUnavailable) @@ -1142,10 +1071,7 @@ metadataVisibility: {service: public, resources: public, semantics: public, clas let list = contract( "{list: {defaultAccessProfile: default, accessProfiles: {default: {access: public, disclosureProfile: default}}, filters: [], allowUnfiltered: true, orderBy: [id], pagination: {defaultPageSize: 10, maximumPageSize: 20}}}", ); - let list_runtime = RelayRuntime::parse_yaml( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {bind: '127.0.0.1:18082'}\npackagePath: package\nsources: {records: {path: fixture.sqlite}}\nauthentication: {issuer: null}\naudit: {path: var/audit.jsonl}\nlimits: {requestTimeoutMilliseconds: 1000, concurrentQueries: 1}\n", - ) - .expect("closed runtime"); + let list_runtime = closed_runtime("records"); assert_eq!( validate_runtime_contract(&list_runtime, &list), Err(StartupError::CursorInvalid) @@ -1154,10 +1080,7 @@ metadataVisibility: {service: public, resources: public, semantics: public, clas let lookup = contract( "{lookups: [{id: by-label, requestBody: {maximumBytes: 128, selectors: {label: {sourceColumn: label, type: string, minimumBytes: 1, maximumBytes: 32}}}, defaultAccessProfile: default, accessProfiles: {default: {access: public, disclosureProfile: default}}}]}", ); - let mut lookup_runtime = RelayRuntime::parse_yaml( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {bind: '127.0.0.1:18083'}\npackagePath: package\nsources: {records: {path: fixture.sqlite}}\nauthentication: {issuer: null}\naudit: {path: var/audit.jsonl}\nlimits: {requestTimeoutMilliseconds: 1000, concurrentQueries: 1}\n", - ) - .expect("closed runtime"); + let mut lookup_runtime = closed_runtime("records"); assert_eq!( validate_runtime_contract(&lookup_runtime, &lookup), Err(StartupError::RuntimeInvalid) @@ -1176,10 +1099,7 @@ metadataVisibility: {service: public, resources: public, semantics: public, clas let mut second_resource = contract.resources[0].clone(); second_resource.id = "second-record".into(); contract.resources.push(second_resource); - let mut runtime = RelayRuntime::parse_yaml( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {bind: '127.0.0.1:18084'}\npackagePath: package\nsources: {db: {path: fixture.sqlite}}\nauthentication: {issuer: null}\naudit: {path: var/audit.jsonl}\nlimits: {requestTimeoutMilliseconds: 1000, concurrentQueries: 1}\n", - ) - .expect("closed runtime"); + let mut runtime = closed_runtime("db"); assert_eq!( validate_runtime_contract(&runtime, &contract), @@ -1196,9 +1116,9 @@ metadataVisibility: {service: public, resources: public, semantics: public, clas ) .expect("protected read operation"), ); - runtime.authentication.issuer = Some( + runtime.authentication.oidc = Some( serde_norway::from_str( - "id: issuer\ndiscoveryUrl: https://issuer.example.invalid/.well-known/openid-configuration\naudience: registry\ntokenTypes: [at+jwt]\nalgorithms: [EdDSA]\n", + "issuer: https://issuer.example.invalid\naudience: registry\ntokenTypes: [at+jwt]\nalgorithms: [EdDSA]\n", ) .expect("issuer runtime"), ); @@ -1222,9 +1142,7 @@ metadataVisibility: {service: public, resources: public, semantics: public, clas let path = root.join("runtime.yaml"); fs::write( &path, - format!( - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {{bind: '127.0.0.1:0'}}\npackagePath: package\nsources: {{db: {{path: source.sqlite}}}}\nauthentication: {{issuer: null}}\naudit: {audit}\nlimits: {{requestTimeoutMilliseconds: 1000, concurrentQueries: 1}}\n" - ), + runtime_text("127.0.0.1:0", "/srv/relay/package", "db", audit), ) .expect("write runtime"); path @@ -1386,13 +1304,33 @@ metadataVisibility: {service: public, resources: public, semantics: public, clas #[test] fn a_runtime_file_is_bounded_and_strict() { let temporary = tempfile::tempdir().expect("temporary root"); - let path = temporary.path().join("runtime.yaml"); + let root = temporary.path().canonicalize().expect("canonical root"); + let path = root.join("runtime.yaml"); let mut file = fs::File::create(&path).expect("runtime file"); writeln!( file, - "apiVersion: relay.registrystack.org/v2alpha1\nkind: RelayRuntime\nserver: {{bind: '127.0.0.1:0'}}\npackagePath: package\nsources: {{db: {{path: source.sqlite}}}}\nauthentication: {{issuer: null}}\naudit: {{path: var/audit.jsonl}}\nlimits: {{requestTimeoutMilliseconds: 1000, concurrentQueries: 1}}\nunknown: true" + "{}unknown: true", + runtime_text( + "127.0.0.1:0", + "/srv/relay/package", + "db", + "{path: var/audit.jsonl}" + ) ) .expect("write runtime"); - assert_eq!(load_runtime(&path), Err(StartupError::RuntimeInvalid)); + assert!(matches!( + load_runtime(&path), + Err(StartupError::RuntimeRefused(_)) + )); + + fs::write( + &path, + vec![b' '; DEFAULT_MAX_RUNTIME_CONFIG_BYTES as usize + 1], + ) + .expect("oversized"); + assert!(matches!( + load_runtime(&path), + Err(StartupError::RuntimeRefused(_)) + )); } } diff --git a/crates/registry-relay-v2/src/tooling.rs b/crates/registry-relay-v2/src/tooling.rs index 9b4fd5a82a..1d9acaa29b 100644 --- a/crates/registry-relay-v2/src/tooling.rs +++ b/crates/registry-relay-v2/src/tooling.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use std::time::Duration; use registry_platform_audit::{AuditDestination, AuditWriter, FileDestination}; +use registry_platform_config::DEFAULT_MAX_RUNTIME_CONFIG_BYTES; use registry_platform_sqlite::{ inspect_schema as inspect_sqlite_schema, materialize_fixture, CapturedSnapshot, DatabaseProfile, LiveDatabaseFile, SchemaObjectKind, @@ -30,7 +31,7 @@ use crate::contract::{ RegistryContract, RelayRuntime, ResourceSource, ReviewStatus, SdmxBindingDefinition, StatisticalAttributeDefinition, StatisticalBindings, StatisticalDimensionDefinition, StatisticalMeasureDefinition, StatisticalPublication, StatisticalQueryProfile, - StatisticalValueType, MAXIMUM_RUNTIME_BYTES, + StatisticalValueType, }; use crate::cursor::CursorKey; use crate::diff::{diff_registries, ChangeImpactReport}; @@ -48,7 +49,7 @@ use crate::identification::{ use crate::model::{ CompileProfile, CompileReport, CompiledRegistry, Diagnostic, DiagnosticSeverity, }; -use crate::package::{build_package, PackageManifest}; +use crate::package::{build_package, PackageError, PackageSummary}; use crate::server::{ router, AlignmentMetadata, InstitutionMetadata, QuotaConfig, RelayService, ServiceMetadata, }; @@ -106,7 +107,11 @@ pub struct DiffOptions { #[derive(Clone, Debug)] pub struct PackageOptions { pub project_root: PathBuf, - pub output_dir: PathBuf, + /// The new directory the package is written into; `None` reports the + /// package without writing it. + pub output_dir: Option, + /// One printable line recorded as the package's `REVISION`. + pub revision: Option, } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -176,7 +181,7 @@ pub enum ToolingDetails { report: Option, }, Package { - manifest: Option, + package: Option, }, } @@ -983,7 +988,7 @@ pub fn package_project(options: &PackageOptions) -> Result { return Ok(ToolingReport::refused( report.diagnostics, - ToolingDetails::Package { manifest: None }, + ToolingDetails::Package { package: None }, )); } }; @@ -991,17 +996,36 @@ pub fn package_project(options: &PackageOptions) -> Result Ok(ToolingReport::success(ToolingDetails::Package { + package: Some(package), + })), + // A refusal of the package format names files relative to the + // package and the fixing command; the directory is named by role. + Err(PackageError::Package(error)) => Ok(ToolingReport::refused( + vec![diagnostic( + "package.refused", + ".", + &error + .naming_root_as(if options.output_dir.is_some() { + "the --output directory" + } else { + "the project" + }) + .to_string(), + )], + ToolingDetails::Package { package: None }, + )), + Err(_) => Err(ToolingError::Package), + } } #[derive(Serialize)] @@ -1264,13 +1288,9 @@ fn compile_project( let contract_yaml = read_utf8(&root.join("registry.yaml"))?; let contract = match RegistryContract::parse_yaml(&contract_yaml) { Ok(contract) => contract, - Err(_) => { + Err(error) => { return Ok(ProjectCompilation::Refused(CompileReport { - diagnostics: vec![diagnostic( - "contract.yaml_invalid", - "registry.yaml", - "the governed contract is not valid strict YAML", - )], + diagnostics: vec![error.diagnostic()], })); } }; @@ -1279,12 +1299,12 @@ fn compile_project( let yaml = read_runtime_utf8(&runtime_path)?; match RelayRuntime::parse_yaml(&yaml) { Ok(runtime) => Some(runtime), - Err(_) => { + Err(refusal) => { return Ok(ProjectCompilation::Refused(CompileReport { diagnostics: vec![diagnostic( "runtime.yaml_invalid", - "runtime.yaml", - "the deployment binding is not valid strict YAML", + &crate::authoring::runtime_location(refusal.field()), + refusal.message(), )], })); } @@ -1512,10 +1532,10 @@ fn read_runtime_utf8(path: &Path) -> Result { let mut file = fs::File::open(path).map_err(|_| ToolingError::Read)?; let mut bytes = Vec::new(); std::io::Read::by_ref(&mut file) - .take(MAXIMUM_RUNTIME_BYTES.saturating_add(1)) + .take(DEFAULT_MAX_RUNTIME_CONFIG_BYTES.saturating_add(1)) .read_to_end(&mut bytes) .map_err(|_| ToolingError::Read)?; - if bytes.len() as u64 > MAXIMUM_RUNTIME_BYTES { + if bytes.len() as u64 > DEFAULT_MAX_RUNTIME_CONFIG_BYTES { return Err(ToolingError::Read); } String::from_utf8(bytes).map_err(|_| ToolingError::Read) @@ -1580,12 +1600,12 @@ resources: metadataVisibility: {service: public, resources: operation-bound, semantics: operation-bound, classifications: operator-only, processing: operation-bound} "#; -const STARTER_RUNTIME: &str = r#"apiVersion: relay.registrystack.org/v2alpha1 -kind: RelayRuntime -server: {bind: "127.0.0.1:8080"} -packagePath: package +const STARTER_RUNTIME: &str = r#"apiVersion: registry.registrystack.org/relay-runtime/v1alpha1 +kind: RelayRuntimeConfig +listener: {bind: "127.0.0.1:8080"} +package: {root: "${RELAY_PACKAGE_ROOT:-/srv/relay/package}"} +secretProviders: {environment: {}} sources: {registry: {path: registry.sqlite}} -authentication: {issuer: null} audit: {destination: file, path: var/audit.jsonl} limits: {requestTimeoutMilliseconds: 1500, concurrentQueries: 8} "#; @@ -1620,6 +1640,21 @@ mod tests { runtime } + #[test] + fn the_starter_runtime_takes_its_package_root_from_the_environment() { + let authored = RelayRuntime::parse_yaml(STARTER_RUNTIME).expect("starter parses"); + assert_eq!(authored.package.root, Path::new("/srv/relay/package")); + + let deployed = RelayRuntime::parse_yaml_with(STARTER_RUNTIME, |name| { + (name == "RELAY_PACKAGE_ROOT").then(|| "/home/reader/registry/package".to_owned()) + }) + .expect("starter parses with the package root set"); + assert_eq!( + deployed.package.root, + Path::new("/home/reader/registry/package") + ); + } + #[test] fn errors_never_render_paths() { for error in [ @@ -1695,15 +1730,15 @@ mod tests { #[test] fn tooling_runtime_loading_matches_the_startup_issuer_profile() { - let runtime = |discovery_url: &str| { + let runtime = |issuer: &str| { STARTER_RUNTIME.replace( - "authentication: {issuer: null}", + "audit:", &format!( - "authentication:\n issuer:\n id: issuer\n discoveryUrl: {discovery_url}\n audience: registry\n tokenTypes: [at+jwt]\n algorithms: [EdDSA]" + "authentication:\n oidc:\n issuer: '{issuer}'\n audience: registry\n tokenTypes: [at+jwt]\n algorithms: [EdDSA]\naudit:" ), ) }; - let valid = "https://identity.example.invalid/.well-known/openid-configuration"; + let valid = "https://identity.example.invalid"; assert!(RelayRuntime::parse_yaml(&runtime(valid)).is_ok()); let temporary = tempfile::tempdir().expect("temporary root"); @@ -1713,11 +1748,10 @@ mod tests { ) .expect("contract writes"); for invalid in [ - "https://operator:credential@identity.example.invalid/.well-known/openid-configuration", - "https://identity.example.invalid/.well-known/openid-configuration?tenant=x", - "https://identity.example.invalid/.well-known/openid-configuration#fragment", - "https://identity.example.invalid/.well-known/oauth-authorization-server", - "https:///.well-known/openid-configuration", + "https://operator:credential@identity.example.invalid", + "https://identity.example.invalid/?tenant=x", + "https://identity.example.invalid/#fragment", + "http://identity.example.invalid", ] { fs::write(temporary.path().join("runtime.yaml"), runtime(invalid)) .expect("runtime writes"); @@ -1730,7 +1764,8 @@ mod tests { assert!(report .diagnostics .iter() - .any(|item| item.code == "runtime.yaml_invalid")); + .any(|item| item.code == "runtime.yaml_invalid" + && item.location == "runtime.yaml.authentication.oidc.issuer")); } } @@ -1738,17 +1773,20 @@ mod tests { fn tooling_runtime_reads_match_the_startup_byte_ceiling() { let temporary = tempfile::tempdir().expect("temporary root"); let path = temporary.path().join("runtime.yaml"); - fs::write(&path, vec![b' '; MAXIMUM_RUNTIME_BYTES as usize]) + fs::write(&path, vec![b' '; DEFAULT_MAX_RUNTIME_CONFIG_BYTES as usize]) .expect("boundary runtime writes"); assert_eq!( read_runtime_utf8(&path) .expect("the exact startup byte ceiling reads") .len(), - MAXIMUM_RUNTIME_BYTES as usize + DEFAULT_MAX_RUNTIME_CONFIG_BYTES as usize ); - fs::write(&path, vec![b' '; MAXIMUM_RUNTIME_BYTES as usize + 1]) - .expect("oversized runtime writes"); + fs::write( + &path, + vec![b' '; DEFAULT_MAX_RUNTIME_CONFIG_BYTES as usize + 1], + ) + .expect("oversized runtime writes"); assert!(matches!(read_runtime_utf8(&path), Err(ToolingError::Read))); fs::write( temporary.path().join("registry.yaml"), @@ -1769,7 +1807,7 @@ mod tests { serde_norway::from_str("{scope: registry:statistics:read}") .expect("protected statistical access"); - let diagnostics = validate_runtime(&contract, Some(&runtime("{issuer: null}"))); + let diagnostics = validate_runtime(&contract, Some(&runtime("{}"))); assert!(diagnostics .iter() .any(|item| item.code == "runtime.issuer_missing")); @@ -1988,8 +2026,7 @@ mod tests { ) .expect("lookup parses"), ); - let lookup_diagnostics = - validate_runtime(&lookup_contract, Some(&runtime("{issuer: null}"))); + let lookup_diagnostics = validate_runtime(&lookup_contract, Some(&runtime("{}"))); assert!(lookup_diagnostics .iter() .any(|item| item.code == "runtime.lookup_quota_missing")); @@ -2000,15 +2037,13 @@ mod tests { let mut second = template; second.id = "second-record".into(); metadata_contract.resources.push(second); - let metadata_diagnostics = - validate_runtime(&metadata_contract, Some(&runtime("{issuer: null}"))); + let metadata_diagnostics = validate_runtime(&metadata_contract, Some(&runtime("{}"))); assert!(metadata_diagnostics .iter() .any(|item| item.code == "runtime.cursor_missing")); metadata_contract.metadata_visibility.resources = crate::contract::Visibility::OperatorOnly; - let operator_only_diagnostics = - validate_runtime(&metadata_contract, Some(&runtime("{issuer: null}"))); + let operator_only_diagnostics = validate_runtime(&metadata_contract, Some(&runtime("{}"))); assert!(!operator_only_diagnostics .iter() .any(|item| item.code == "runtime.cursor_missing")); @@ -2020,8 +2055,7 @@ mod tests { ) .expect("protected read operation"), ); - let one_public_diagnostics = - validate_runtime(&metadata_contract, Some(&runtime("{issuer: null}"))); + let one_public_diagnostics = validate_runtime(&metadata_contract, Some(&runtime("{}"))); assert!(!one_public_diagnostics .iter() .any(|item| item.code == "runtime.cursor_missing")); @@ -2029,7 +2063,7 @@ mod tests { metadata_contract.metadata_visibility.resources = crate::contract::Visibility::OperationBound; let operation_bound_diagnostics = - validate_runtime(&metadata_contract, Some(&runtime("{issuer: null}"))); + validate_runtime(&metadata_contract, Some(&runtime("{}"))); assert!(operation_bound_diagnostics .iter() .any(|item| item.code == "runtime.cursor_missing")); diff --git a/crates/registry-relay-v2/tests/acceptance_http.rs b/crates/registry-relay-v2/tests/acceptance_http.rs index e89d1cd53b..a25ee69698 100644 --- a/crates/registry-relay-v2/tests/acceptance_http.rs +++ b/crates/registry-relay-v2/tests/acceptance_http.rs @@ -969,10 +969,11 @@ async fn stock_issuer_registered_authority_drives_a_protected_relay_lookup() { let mut issuer = relay .runtime .authentication - .issuer + .oidc .clone() .expect("social-assistance declares an issuer"); - issuer.discovery_url = Some(format!("{}/.well-known/openid-configuration", stock.issuer)); + issuer.issuer = stock.issuer.clone(); + issuer.jwks_source = registry_platform_config::JwksSource::Discovery {}; issuer.audience = audience; issuer.algorithms = vec!["RS256".into()]; let authenticator = build_authenticator_for_supervised_local_development(&issuer) @@ -1928,7 +1929,7 @@ async fn real_jwt_path_rejects_malformed_audience_time_and_expired_tokens() { let expected_audience = &harness .runtime .authentication - .issuer + .oidc .as_ref() .expect("issuer exists") .audience; @@ -2015,7 +2016,7 @@ async fn real_jwt_path_rejects_malformed_audience_time_and_expired_tokens() { &harness .runtime .authentication - .issuer + .oidc .as_ref() .expect("issuer exists") .audience, @@ -2076,12 +2077,11 @@ async fn real_jwt_path_uses_trusted_issuer_not_the_jwks_transport_host() { let mut issuer = harness .runtime .authentication - .issuer + .oidc .clone() .expect("social-assistance declares an issuer"); - issuer.trusted_issuer = Some(TRUSTED_ISSUER.into()); - issuer.discovery_url = None; - issuer.jwks_url = Some(jwks_url); + issuer.issuer = TRUSTED_ISSUER.into(); + issuer.jwks_source = registry_platform_config::JwksSource::Uri { uri: jwks_url }; issuer.algorithms = vec!["EdDSA".into()]; let audience = issuer.audience.clone(); let authenticator = build_authenticator_for_supervised_local_development(&issuer) @@ -3813,7 +3813,7 @@ impl ProjectHarness { ); let audit = RelayAudit::new(audit.unwrap_or_else(|| AuditLines::recording().writer())); - let (authenticator, idp) = if let Some(issuer) = runtime.authentication.issuer.as_ref() { + let (authenticator, idp) = if let Some(issuer) = runtime.authentication.oidc.as_ref() { let idp = MockIdp::start().await; let fetcher = Arc::new(JwksFetcher::new_with_fetch_url_policy( idp.jwks_uri(), @@ -4004,7 +4004,7 @@ impl ProjectHarness { let audience = &self .runtime .authentication - .issuer + .oidc .as_ref() .expect("runtime has issuer") .audience; diff --git a/crates/registry-relay-v2/tests/process_http.rs b/crates/registry-relay-v2/tests/process_http.rs index 8f25b37a51..fa1a198a7f 100644 --- a/crates/registry-relay-v2/tests/process_http.rs +++ b/crates/registry-relay-v2/tests/process_http.rs @@ -8,7 +8,7 @@ use std::fs; use std::io::Read as _; use std::net::TcpListener; use std::os::unix::fs::PermissionsExt as _; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; use std::time::{Duration, Instant}; @@ -38,7 +38,7 @@ impl RelayProcess { fn spawn(runtime: &Path) -> Self { let child = Command::new(env!("CARGO_BIN_EXE_relay")) .arg("serve") - .arg("--runtime") + .arg("--runtime-config") .arg(runtime) .stdin(Stdio::null()) .stdout(Stdio::null()) @@ -100,88 +100,150 @@ impl Drop for RelayProcess { } } -#[tokio::test] -async fn built_relay_serves_a_sealed_package_over_real_tcp_and_shuts_down() { - let temporary = tempfile::tempdir().expect("temporary image layout"); - let image_root = temporary - .path() - .canonicalize() - .expect("temporary image root canonicalizes"); - let etc = image_root.join("etc/relay"); - let data = image_root.join("var/lib/relay/data"); - let audit = image_root.join("var/lib/relay/audit"); - fs::create_dir_all(&etc).expect("runtime directory creates"); - fs::create_dir_all(&data).expect("data directory creates"); - fs::create_dir_all(&audit).expect("audit directory creates"); - fs::set_permissions(&audit, fs::Permissions::from_mode(0o700)) - .expect("audit directory becomes owner-only"); - copy_tree(Path::new(BUSINESS_PROJECT), &etc); - - let source = data.join("business-registry.sqlite"); - materialize_fixture( - &source, - &fs::read_to_string(etc.join("fixture.sql")).expect("fixture SQL reads"), - ) - .expect("fixture materializes"); - make_business_project_public_only(&etc, &source); +/// One copied acceptance project laid out like the official image, with its +/// sealed package built and its runtime configuration written. +struct BusinessImage { + _temporary: tempfile::TempDir, + runtime_path: PathBuf, + runtime: RelayRuntime, + address: std::net::SocketAddr, +} - let package = data.join("business-registry-package"); - let runtime_path = etc.join("runtime.yaml"); - let mut runtime = RelayRuntime::parse_yaml( - &fs::read_to_string(&runtime_path).expect("acceptance runtime reads"), - ) - .expect("acceptance runtime parses"); - let reservation = TcpListener::bind("127.0.0.1:0").expect("loopback port reserves"); - let address = reservation.local_addr().expect("reserved address"); - drop(reservation); - runtime.server.bind = address.to_string(); - runtime.package_path = package.to_string_lossy().into_owned(); - runtime.authentication.issuer = None; - let mut runtime_value = serde_json::to_value(&runtime).expect("runtime becomes a value"); - *runtime_value - .pointer_mut("/sources/companies/path") - .expect("business source binding") = Value::String(source.to_string_lossy().into_owned()); - runtime = serde_json::from_value(runtime_value).expect("modified runtime remains valid"); - runtime.audit.path = Some(audit.join("events.jsonl").to_string_lossy().into_owned()); - runtime - .cursor - .as_mut() - .expect("business cursor") - .integrity_key_ref = "secret:file/cursor-integrity-key".into(); +impl BusinessImage { + fn prepare() -> Self { + let temporary = tempfile::tempdir().expect("temporary image layout"); + let image_root = temporary + .path() + .canonicalize() + .expect("temporary image root canonicalizes"); + let etc = image_root.join("etc/relay"); + let data = image_root.join("var/lib/relay/data"); + let audit = image_root.join("var/lib/relay/audit"); + let secrets = image_root.join("run/secrets/relay"); + fs::create_dir_all(&etc).expect("runtime directory creates"); + fs::create_dir_all(&data).expect("data directory creates"); + fs::create_dir_all(&audit).expect("audit directory creates"); + fs::create_dir_all(&secrets).expect("secrets directory creates"); + fs::set_permissions(&audit, fs::Permissions::from_mode(0o700)) + .expect("audit directory becomes owner-only"); + copy_tree(Path::new(BUSINESS_PROJECT), &etc); + + let source = data.join("business-registry.sqlite"); + materialize_fixture( + &source, + &fs::read_to_string(etc.join("fixture.sql")).expect("fixture SQL reads"), + ) + .expect("fixture materializes"); + make_business_project_public_only(&etc, &source); + + let package = data.join("business-registry-package"); + let runtime_path = etc.join("runtime.yaml"); + let mut runtime = RelayRuntime::parse_yaml( + &fs::read_to_string(&runtime_path).expect("acceptance runtime reads"), + ) + .expect("acceptance runtime parses"); + let reservation = TcpListener::bind("127.0.0.1:0").expect("loopback port reserves"); + let address = reservation.local_addr().expect("reserved address"); + drop(reservation); + let mut runtime_value = serde_json::to_value(&runtime).expect("runtime becomes a value"); + *runtime_value + .pointer_mut("/listener/bind") + .expect("listener binding") = Value::String(address.to_string()); + *runtime_value + .pointer_mut("/package/root") + .expect("package root") = Value::String(package.to_string_lossy().into_owned()); + *runtime_value + .pointer_mut("/sources/companies/path") + .expect("business source binding") = + Value::String(source.to_string_lossy().into_owned()); + // File secrets resolve under secretProviders.file.root, a directory + // apart from the runtime configuration. + *runtime_value + .pointer_mut("/secretProviders") + .expect("secret providers") = serde_json::json!({ + "file": {"root": secrets.to_string_lossy()} + }); + runtime_value + .as_object_mut() + .expect("runtime object") + .remove("authentication"); + runtime = serde_json::from_value(runtime_value).expect("modified runtime remains valid"); + runtime.audit.path = Some(audit.join("events.jsonl").to_string_lossy().into_owned()); + runtime + .cursor + .as_mut() + .expect("business cursor") + .integrity_key_ref = "secret:file/cursor-integrity-key".into(); + runtime + .check() + .expect("modified runtime passes the shared blocks"); + write_runtime(&runtime_path, &runtime); + write_secret( + &secrets.join("cursor-integrity-key"), + b"a-32-byte-minimum-synthetic-cursor-key", + ); + + let report = package_project(&PackageOptions { + project_root: etc.clone(), + output_dir: Some(package), + revision: None, + }) + .expect("package operation succeeds"); + assert!( + report.is_success(), + "acceptance project packages: {report:?}" + ); + Self { + _temporary: temporary, + runtime_path, + runtime, + address, + } + } +} + +fn write_runtime(path: &Path, runtime: &RelayRuntime) { fs::write( - &runtime_path, - serde_norway::to_string(&runtime).expect("runtime serializes"), + path, + serde_norway::to_string(runtime).expect("runtime serializes"), ) .expect("absolute runtime writes"); - write_secret( - &etc.join("cursor-integrity-key"), - b"a-32-byte-minimum-synthetic-cursor-key", - ); +} - let report = package_project(&PackageOptions { - project_root: etc.clone(), - output_dir: package, - }) - .expect("sealed package operation succeeds"); - assert!( - report.is_success(), - "acceptance project packages: {report:?}" - ); +fn relay_check(runtime: &Path) -> std::process::Output { + Command::new(env!("CARGO_BIN_EXE_relay")) + .arg("check") + .arg("--runtime-config") + .arg(runtime) + .stdin(Stdio::null()) + .output() + .expect("built relay check runs") +} +#[tokio::test] +async fn built_relay_serves_a_sealed_package_over_real_tcp_and_shuts_down() { + let image = BusinessImage::prepare(); let client = Client::builder() .no_proxy() .timeout(Duration::from_secs(1)) .build() .expect("HTTP client builds"); - let base = format!("http://{address}"); - let first = serve_one_lifecycle(&runtime_path, &client, &base).await; - let second = serve_one_lifecycle(&runtime_path, &client, &base).await; + let base = format!("http://{}", image.address); + let first = serve_one_lifecycle(&image.runtime_path, &client, &base).await; + let second = serve_one_lifecycle(&image.runtime_path, &client, &base).await; assert_eq!( first, second, "the same sealed package and snapshot must serialize identically after restart" ); - let audit_log = audit.join("events.jsonl"); + let audit_log = PathBuf::from( + image + .runtime + .audit + .path + .as_deref() + .expect("the image writes a file audit destination"), + ); assert_eq!( fs::metadata(&audit_log) .expect("audit log exists") @@ -213,6 +275,79 @@ async fn built_relay_serves_a_sealed_package_over_real_tcp_and_shuts_down() { assert_ne!(entries[0]["correlation"], entries[2]["correlation"]); } +#[test] +fn built_relay_check_honors_a_package_digest_pin() { + let image = BusinessImage::prepare(); + let digest = registry_platform_config::sha256_uri( + &fs::read(image.runtime.package.root.join("SHA256SUMS")).expect("package sums read"), + ); + + let mut pinned = image.runtime.clone(); + pinned.package.expected_digest = Some(digest.clone()); + write_runtime(&image.runtime_path, &pinned); + let output = relay_check(&image.runtime_path); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + + let wrong = format!("sha256:{}", "0".repeat(64)); + pinned.package.expected_digest = Some(wrong.clone()); + write_runtime(&image.runtime_path, &pinned); + let output = relay_check(&image.runtime_path); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains(&format!( + "package.expectedDigest is {wrong} but the package at package.root is {digest}" + )), + "{stderr}" + ); +} + +#[test] +fn built_relay_check_refuses_a_changed_package_file_by_name_without_a_pin() { + let image = BusinessImage::prepare(); + assert_eq!(image.runtime.package.expected_digest, None); + let contract = image.runtime.package.root.join("registry.yaml"); + let mut bytes = fs::read(&contract).expect("packaged contract reads"); + bytes.extend_from_slice(b"\n"); + fs::write(&contract, bytes).expect("packaged contract changes"); + + let output = relay_check(&image.runtime_path); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("changed: registry.yaml"), "{stderr}"); + assert!(stderr.contains("package.root"), "{stderr}"); + assert!(stderr.contains("relayctl package"), "{stderr}"); + assert!( + !stderr.contains(&*image.runtime.package.root.to_string_lossy()), + "{stderr}" + ); +} + +#[test] +fn built_relay_refuses_removed_keys_and_relative_paths_by_field() { + let image = BusinessImage::prepare(); + let text = fs::read_to_string(&image.runtime_path).expect("runtime reads"); + fs::write(&image.runtime_path, format!("{text}packagePath: package\n")) + .expect("runtime with a removed key writes"); + let output = relay_check(&image.runtime_path); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("packagePath"), "{stderr}"); + assert!(stderr.contains("package.root"), "{stderr}"); + assert!( + !stderr.contains(&*image.runtime_path.to_string_lossy()), + "{stderr}" + ); + + let output = relay_check(Path::new("runtime.yaml")); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("runtime configuration is refused")); +} + fn make_business_project_public_only(project: &Path, source: &Path) { let contract_path = project.join("registry.yaml"); let mut value: Value = serde_norway::from_str( diff --git a/crates/registry-relay-v2/tests/sdmx_http.rs b/crates/registry-relay-v2/tests/sdmx_http.rs index 3a1754f8ac..828ca7e118 100644 --- a/crates/registry-relay-v2/tests/sdmx_http.rs +++ b/crates/registry-relay-v2/tests/sdmx_http.rs @@ -778,7 +778,7 @@ impl Harness { fetcher.ensure_key_set().await.expect("fixture JWKS loads"); let issuer = runtime .authentication - .issuer + .oidc .as_ref() .expect("statistical runtime has issuer"); let mut verifier = oidc_verifier_config(idp.issuer(), vec![issuer.audience.clone()]); @@ -878,7 +878,7 @@ impl Harness { let audience = &self .runtime .authentication - .issuer + .oidc .as_ref() .expect("runtime has issuer") .audience; diff --git a/crates/registry-relayctl/schemas/authoring/runtime.schema.json b/crates/registry-relayctl/schemas/authoring/runtime.schema.json index cf791859e1..6d34a1b5a1 100644 --- a/crates/registry-relayctl/schemas/authoring/runtime.schema.json +++ b/crates/registry-relayctl/schemas/authoring/runtime.schema.json @@ -123,15 +123,16 @@ "AuthenticationRuntime": { "additionalProperties": false, "properties": { - "issuer": { + "oidc": { "anyOf": [ { - "$ref": "#/$defs/IssuerRuntime" + "$ref": "#/$defs/OidcRuntime" }, { "type": "null" } - ] + ], + "description": "The one OIDC issuer whose access tokens protected operations accept.\nA Registry whose every operation is public declares none." } }, "type": "object" @@ -154,7 +155,103 @@ ], "type": "object" }, - "IssuerRuntime": { + "EnvironmentSecretProviderConfig": { + "additionalProperties": false, + "description": "The environment secret provider. It takes no settings.", + "type": "object" + }, + "FileSecretProviderConfig": { + "additionalProperties": false, + "description": "The file secret provider.", + "properties": { + "root": { + "description": "Absolute directory holding one file per secret. Each file must be a\nregular file owned by the runtime user, mode 0400 or 0600, with exactly\none hard link.", + "pattern": "^/", + "type": "string" + } + }, + "required": [ + "root" + ], + "type": "object" + }, + "JwksSource": { + "description": "Where a runtime obtains the OIDC issuer's signing keys.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Read `jwks_uri` from the issuer's OpenID Connect discovery document.", + "properties": { + "kind": { + "const": "discovery", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Fetch the key set from this absolute `https` URI, skipping discovery.", + "properties": { + "kind": { + "const": "uri", + "type": "string" + }, + "uri": { + "pattern": "^https?://", + "type": "string" + } + }, + "required": [ + "kind", + "uri" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Read the key set from a secret, for deployments without network access\nto the issuer.", + "properties": { + "documentRef": { + "pattern": "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$", + "type": "string" + }, + "kind": { + "const": "static", + "type": "string" + } + }, + "required": [ + "kind", + "documentRef" + ], + "type": "object" + } + ] + }, + "ListenerBind": { + "description": "Socket address the runtime listens on, written host:port with an IP address host ([addr]:port for IPv6).", + "maxLength": 128, + "minLength": 1, + "type": "string" + }, + "ListenerConfig": { + "additionalProperties": false, + "description": "The listener of a runtime that declares no TLS or exposure settings.", + "properties": { + "bind": { + "$ref": "#/$defs/ListenerBind" + } + }, + "required": [ + "bind" + ], + "type": "object" + }, + "OidcRuntime": { "additionalProperties": false, "properties": { "algorithms": { @@ -166,43 +263,52 @@ "audience": { "type": "string" }, - "discoveryUrl": { - "default": null, - "type": [ - "string", - "null" - ] - }, - "id": { + "issuer": { + "description": "Exact issuer accepted in access-token `iss` claims, an absolute\n`https` URL.", "type": "string" }, - "jwksUrl": { - "default": null, - "type": [ - "string", - "null" - ] + "jwksSource": { + "$ref": "#/$defs/JwksSource", + "default": { + "kind": "discovery" + }, + "description": "Where the issuer's signing keys come from: `{kind: discovery}` reads\nthe issuer's OpenID Connect discovery document, and\n`{kind: uri, uri}` fetches the key set directly. Relay does not\naccept `kind: static`." }, "tokenTypes": { "items": { "type": "string" }, "type": "array" - }, - "trustedIssuer": { - "default": null, - "description": "Exact issuer accepted in access-token `iss` claims.\n\nExisting runtimes may omit this when `discoveryUrl` uses the canonical\nissuer origin. A distinct discovery transport or direct JWKS transport\nrequires this field so network routing never changes token identity.", + } + }, + "required": [ + "issuer", + "audience", + "tokenTypes", + "algorithms" + ], + "type": "object" + }, + "PackageConfig": { + "additionalProperties": false, + "description": "The package a runtime serves: `root` is the absolute package directory,\nand `expectedDigest`, when set, pins the package digest the runtime must\nfind there. The package digest is the digest of the package's\n`SHA256SUMS` file; see [`crate::package`].", + "properties": { + "expectedDigest": { + "description": "`sha256:` label of the package digest, the digest of the package's\n`SHA256SUMS` file. When set, the runtime refuses to start on any other\npackage.", + "pattern": "^sha256:[0-9a-f]{64}$", "type": [ "string", "null" ] + }, + "root": { + "description": "Absolute path of the package directory.", + "pattern": "^/", + "type": "string" } }, "required": [ - "id", - "audience", - "tokenTypes", - "algorithms" + "root" ], "type": "object" }, @@ -250,6 +356,7 @@ "additionalProperties": false, "properties": { "path": { + "description": "SQLite file of this source, absolute or relative to the directory of\nthe runtime configuration.", "type": "string" } }, @@ -258,16 +365,55 @@ ], "type": "object" }, - "ServerRuntime": { + "SecretProvidersConfig": { "additionalProperties": false, + "anyOf": [ + { + "properties": { + "file": { + "$ref": "#/$defs/FileSecretProviderConfig" + } + }, + "required": [ + "file" + ] + }, + { + "properties": { + "environment": { + "$ref": "#/$defs/EnvironmentSecretProviderConfig" + } + }, + "required": [ + "environment" + ] + } + ], + "description": "The secret providers a runtime enables. A reference is resolved only by a\nprovider declared here: `secret:file/name` under `file.root`, and\n`secret:env/NAME` only when `environment: {}` is present.", "properties": { - "bind": { - "type": "string" + "environment": { + "anyOf": [ + { + "$ref": "#/$defs/EnvironmentSecretProviderConfig" + }, + { + "type": "null" + } + ], + "description": "Enables `secret:env/NAME` references, read from the process\nenvironment. Declared as an empty mapping: `environment: {}`." + }, + "file": { + "anyOf": [ + { + "$ref": "#/$defs/FileSecretProviderConfig" + }, + { + "type": "null" + } + ], + "description": "Enables `secret:file/name` references, read from files under `root`." } }, - "required": [ - "bind" - ], "type": "object" }, "ShutdownRuntime": { @@ -291,13 +437,15 @@ "description": "Deployment-local bindings. No governed field is accepted here.", "properties": { "apiVersion": { + "const": "registry.registrystack.org/relay-runtime/v1alpha1", "type": "string" }, "audit": { "$ref": "#/$defs/AuditRuntime" }, "authentication": { - "$ref": "#/$defs/AuthenticationRuntime" + "$ref": "#/$defs/AuthenticationRuntime", + "default": {} }, "cursor": { "anyOf": [ @@ -311,13 +459,17 @@ "default": null }, "kind": { + "const": "RelayRuntimeConfig", "type": "string" }, "limits": { "$ref": "#/$defs/RuntimeLimits" }, - "packagePath": { - "type": "string" + "listener": { + "$ref": "#/$defs/ListenerConfig" + }, + "package": { + "$ref": "#/$defs/PackageConfig" }, "quotas": { "anyOf": [ @@ -330,8 +482,8 @@ ], "default": null }, - "server": { - "$ref": "#/$defs/ServerRuntime" + "secretProviders": { + "$ref": "#/$defs/SecretProvidersConfig" }, "shutdown": { "anyOf": [ @@ -354,10 +506,10 @@ "required": [ "apiVersion", "kind", - "server", - "packagePath", + "listener", + "package", + "secretProviders", "sources", - "authentication", "audit", "limits" ], diff --git a/crates/registry-relayctl/src/lib.rs b/crates/registry-relayctl/src/lib.rs index 9ee5d2f7c8..5f975d9923 100644 --- a/crates/registry-relayctl/src/lib.rs +++ b/crates/registry-relayctl/src/lib.rs @@ -173,9 +173,22 @@ struct PackageArgs { #[arg(value_name = "PROJECT")] project: std::path::PathBuf, - /// New sealed package directory. - #[arg(long, required = true, value_name = "DIRECTORY")] - output: std::path::PathBuf, + /// New package directory; it must not exist yet. + #[arg( + long, + value_name = "DIRECTORY", + required_unless_present = "dry_run", + conflicts_with = "dry_run" + )] + output: Option, + + /// Report the package digest and files without writing anything. + #[arg(long)] + dry_run: bool, + + /// One printable line recorded in the package as REVISION. + #[arg(long, value_name = "TEXT")] + revision: Option, } /// Parse process arguments, run one shared-library operation, and return the @@ -444,6 +457,18 @@ mod tests { error.kind(), clap::error::ErrorKind::MissingRequiredArgument ); + Cli::try_parse_from(["relayctl", "package", "project", "--dry-run"]) + .expect("a dry run needs no destination"); + let error = Cli::try_parse_from([ + "relayctl", + "package", + "project", + "--dry-run", + "--output", + "package", + ]) + .expect_err("a dry run writes nothing"); + assert_eq!(error.kind(), clap::error::ErrorKind::ArgumentConflict); } #[test] @@ -575,7 +600,7 @@ mod tests { " {\n", " \"severity\": \"error\",\n", " \"code\": \"runtime.issuer_missing\",\n", - " \"location\": \"runtime.yaml.authentication.issuer\",\n", + " \"location\": \"runtime.yaml.authentication.oidc\",\n", " \"message\": \"a Registry with protected operations requires one configured issuer\"\n", " }\n", " ],\n", @@ -663,7 +688,7 @@ mod tests { " ],\n", " \"details\": {\n", " \"kind\": \"package\",\n", - " \"manifest\": null\n", + " \"package\": null\n", " }\n", "}" ), diff --git a/crates/registry-relayctl/src/report.rs b/crates/registry-relayctl/src/report.rs index 753359436d..e50c4ae5bb 100644 --- a/crates/registry-relayctl/src/report.rs +++ b/crates/registry-relayctl/src/report.rs @@ -12,7 +12,7 @@ use serde::Serialize; use crate::shared::{ ChangeImpact, ChangeImpactReport, Diagnostic, DiagnosticSeverity, FixturePlanReport, - InspectedObject, PackageManifest, ToolingDetails, ToolingReport, ToolingStatus, + InspectedObject, PackageSummary, ToolingDetails, ToolingReport, ToolingStatus, }; /// Ordinary detail indent, and the width of one nesting level. @@ -106,14 +106,19 @@ fn lead(report: &ToolingReport, refused: bool) -> String { ), (false, None) => "Change classification reported nothing.".to_owned(), }, - ToolingDetails::Package { manifest } => match (refused, manifest) { + ToolingDetails::Package { package } => match (refused, package) { (true, _) => "Packaging refused.".to_owned(), - (false, Some(manifest)) => format!( - "Sealed a deployment package. {}, {}.", - counted(manifest.artifacts.len(), "artifact"), - counted(manifest.files.len(), "file") + (false, Some(package)) => format!( + "{} {}, {}.", + if package.dry_run { + "Planned a deployment package without writing it." + } else { + "Wrote a deployment package." + }, + counted(package.artifacts.len(), "artifact"), + counted(package.files.len(), "file") ), - (false, None) => "Sealed a deployment package.".to_owned(), + (false, None) => "Wrote a deployment package.".to_owned(), }, } } @@ -214,9 +219,9 @@ fn detail_lines( push_changes(lines, impact)?; } } - ToolingDetails::Package { manifest } => { - if let Some(manifest) = manifest { - push_manifest(lines, manifest); + ToolingDetails::Package { package } => { + if let Some(package) = package { + push_package(lines, package); } } } @@ -363,26 +368,19 @@ fn push_changes( Ok(()) } -fn push_manifest(lines: &mut Vec, manifest: &PackageManifest) { - push_pairs( - lines, - INDENT, - &[ - ("package version", manifest.package_version.clone()), - ("package revision", manifest.package_revision.clone()), - ("contract revision", manifest.contract_revision.clone()), - ( - "artifact bindings", - manifest.operation_artifact_bindings.len().to_string(), - ), - ], - ); - if manifest.source_schema_fingerprints.is_empty() { +fn push_package(lines: &mut Vec, package: &PackageSummary) { + let mut pairs = vec![("package digest", package.package_digest.clone())]; + if let Some(revision) = &package.revision { + pairs.push(("revision", revision.clone())); + } + pairs.push(("contract revision", package.contract_revision.clone())); + push_pairs(lines, INDENT, &pairs); + if package.source_schema_fingerprints.is_empty() { return; } lines.push(String::new()); lines.push(format!("{INDENT}source schema fingerprints")); - let pairs = manifest + let pairs = package .source_schema_fingerprints .iter() .map(|(source, fingerprint)| (source.as_str(), fingerprint.clone())) @@ -544,7 +542,7 @@ mod tests { { "severity": "error", "code": "runtime.issuer_missing", - "location": "runtime.yaml.authentication.issuer", + "location": "runtime.yaml.authentication.oidc", "message": "a Registry with protected operations requires one configured issuer" }, { @@ -678,62 +676,50 @@ mod tests { "diagnostics": [], "details": { "kind": "package", - "manifest": { - "packageVersion": "relay.registrystack.org/package/v1alpha3", - "packageRevision": "sha256:3333", + "package": { + "packageDigest": "sha256:3333", + "revision": null, + "dryRun": false, "contractRevision": "sha256:4444", "sourceSchemaFingerprints": {"records": "sha256:5555"}, - "sourceSchemas": { - "records": { - "source": "records", - "fingerprint": "sha256:5555", - "views": [ - { - "name": "relay_records", - "columns": [ - {"name": "record_identifier", "declaredType": "TEXT", "nullable": true, "primaryKey": false} - ] - } - ] - } - }, + "files": [ + {"path": "compiled/registry.json", "sha256": "sha256:6666", "bytes": 2048}, + {"path": "registry.yaml", "sha256": "sha256:7777", "bytes": 512} + ], "artifacts": [ { - "id": "capability-inventory", - "path": "generated/artifacts/capabilities.json", + "id": "openapi-public", + "path": "generated/openapi.public.json", "mediaType": "application/json", "visibility": "public", "operationIdentifier": null, "accessBinding": null, - "sha256": "sha256:6666" - } - ], - "operationArtifactBindings": [ - { - "operationIdentifier": "record.list", - "accessProfileIdentifier": "public-view", - "vocabularyPath": "artifacts/record--list.vocabulary.jsonld", - "contextPath": "artifacts/record--list.context.jsonld", - "accessProfileSchemaPath": "artifacts/record--list.schema.json", - "accessProfileShaclPath": "artifacts/record--list.shacl.ttl", - "classificationPath": "artifacts/record--list.classifications.json", - "processingPath": "artifacts/record--list.processing.json" - } - ], - "files": [ - { - "path": "generated/artifacts/capabilities.json", - "size": 512, - "sha256": "sha256:6666", - "mediaType": "application/json", - "visibility": "public", - "generated": true + "sha256": "sha256:8888" } ] } } }"#; + const PACKAGE_PLANNED: &str = r#"{ + "status": "success", + "diagnostics": [], + "details": { + "kind": "package", + "package": { + "packageDigest": "sha256:3333", + "revision": "release 7", + "dryRun": true, + "contractRevision": "sha256:4444", + "sourceSchemaFingerprints": {}, + "files": [ + {"path": "registry.yaml", "sha256": "sha256:7777", "bytes": 512} + ], + "artifacts": [] + } + } + }"#; + const PACKAGE_REFUSED: &str = r#"{ "status": "refused", "diagnostics": [ @@ -744,12 +730,12 @@ mod tests { "message": "production compilation requires reviewed classification" } ], - "details": {"kind": "package", "manifest": null} + "details": {"kind": "package", "package": null} }"#; /// Every fixture above is one report the shared library can return, so the /// rendering tests never describe a shape the JSON contract does not have. - const EVERY_FIXTURE: [&str; 11] = [ + const EVERY_FIXTURE: [&str; 12] = [ INITIALIZED, INITIALIZATION_REFUSED, INSPECTION, @@ -760,6 +746,7 @@ mod tests { DIFF_CHANGED, DIFF_UNCHANGED, PACKAGED, + PACKAGE_PLANNED, PACKAGE_REFUSED, ]; @@ -829,7 +816,7 @@ mod tests { concat!( "Production check refused.\n", "\n", - " error runtime.issuer_missing runtime.yaml.authentication.issuer\n", + " error runtime.issuer_missing runtime.yaml.authentication.oidc\n", " a Registry with protected operations requires one configured issuer\n", " error source.schema_observation_missing sources.registry\n", " production compilation requires the observed source schema\n", @@ -963,24 +950,31 @@ artifacts/long.json\n", } #[test] - fn a_sealed_package_renders_its_revisions_and_source_fingerprints() { + fn a_package_renders_its_digest_revisions_and_source_fingerprints() { assert_eq!( rendered(PACKAGED), concat!( - "Sealed a deployment package. 1 artifact, 1 file.\n", - " package version relay.registrystack.org/package/v1alpha3\n", - " package revision sha256:3333\n", + "Wrote a deployment package. 1 artifact, 2 files.\n", + " package digest sha256:3333\n", " contract revision sha256:4444\n", - " artifact bindings 1\n", "\n", " source schema fingerprints\n", " records sha256:5555\n", ) ); + assert_eq!( + rendered(PACKAGE_PLANNED), + concat!( + "Planned a deployment package without writing it. 0 artifacts, 1 file.\n", + " package digest sha256:3333\n", + " revision release 7\n", + " contract revision sha256:4444\n", + ) + ); } #[test] - fn a_refused_package_renders_the_refusal_without_a_manifest() { + fn a_refused_package_renders_the_refusal_without_a_package() { assert_eq!( rendered(PACKAGE_REFUSED), concat!( diff --git a/crates/registry-relayctl/src/shared.rs b/crates/registry-relayctl/src/shared.rs index a7ba01d23c..fbbf18a275 100644 --- a/crates/registry-relayctl/src/shared.rs +++ b/crates/registry-relayctl/src/shared.rs @@ -12,7 +12,7 @@ pub(crate) use registry_relay_v2::{ diff::{ChangeImpact, ChangeImpactReport}, fixtures::FixturePlanReport, model::{Diagnostic, DiagnosticSeverity}, - package::PackageManifest, + package::PackageSummary, tooling::{InspectedObject, ToolingDetails, ToolingReport, ToolingStatus}, }; @@ -54,6 +54,7 @@ pub(crate) fn execute(command: Command) -> Result { Command::Package(args) => tooling::package_project(&PackageOptions { project_root: args.project, output_dir: args.output, + revision: args.revision, }), Command::Tooling(_) => { unreachable!("tooling commands are handled before the product facade") diff --git a/crates/registry-render/src/bundle.rs b/crates/registry-render/src/bundle.rs index 4f89a64f66..d9bdf453df 100644 --- a/crates/registry-render/src/bundle.rs +++ b/crates/registry-render/src/bundle.rs @@ -1,7 +1,8 @@ -//! Bundle loading, verification, and sealing. A bundle is a directory: one -//! `manifest.yaml` plus the templates, labels, schemas, fonts, and vendored -//! packages it governs. Sealing hashes every governed file; serving requires -//! a sealed bundle, compiling does not. +//! Bundle loading and package binding. An authored bundle is a directory with +//! one `manifest.yaml` plus the templates, labels, schemas, fonts, and +//! vendored Typst packages it governs. A deployment package adds the shared +//! `SHA256SUMS` envelope. Runtime loads bind the exact captured bytes back to +//! that verified envelope before parsing or rendering any product content. use std::collections::BTreeMap; use std::path::{Component, Path, PathBuf}; @@ -29,8 +30,7 @@ pub struct LoadedDocument { pub schema: Option, } -/// A loaded bundle. `Bundle::load` accepts unsealed bundles (for compile and -/// authoring); `Bundle::load_sealed` is what `serve` uses. +/// A loaded authored bundle or verified deployment package. #[derive(Debug, Clone)] pub struct Bundle { /// Retained only to redact any host path a future diagnostic might @@ -39,15 +39,17 @@ pub struct Bundle { pub manifest: Manifest, /// Exact bytes of the loaded `manifest.yaml`. pub manifest_bytes: Vec, - /// sha256 of the manifest bytes; identifies the sealed content set. + /// Hex SHA-256 identity. For a deployment package this is the shared + /// package digest without its `sha256:` label; raw authoring loads use the + /// manifest digest only for preview output. pub bundle_hash: String, pub documents: BTreeMap, /// The binary's baseline set (`typst-assets` order) first, then bundle /// fonts sorted by path — the same book order the Typst CLI builds, so /// library and CLI renders agree byte for byte. pub fonts: Vec, - /// Immutable bytes read once and, for a sealed bundle, verified against - /// the manifest before any consumer parses or renders them. + /// Immutable bytes read once and, for a deployment package, bound to the + /// shared package envelope before any consumer parses or renders them. pub(crate) snapshot: BundleSnapshot, } @@ -60,10 +62,7 @@ pub(crate) struct BundleSnapshot { } impl BundleSnapshot { - fn load( - root: &Path, - require_sealed: bool, - ) -> Result<(Self, Manifest, Vec, std::fs::File), RenderProblem> { + fn load(root: &Path) -> Result<(Self, Manifest, Vec), RenderProblem> { #[cfg(any(target_os = "linux", target_vendor = "apple"))] { use std::ffi::OsStr; @@ -75,12 +74,6 @@ impl BundleSnapshot { &root.join(MANIFEST_FILE), )?; let manifest = Manifest::parse(&manifest_bytes)?; - if require_sealed && !manifest.is_sealed() { - return Err(RenderProblem::new( - ProblemKind::BundleUnsealed, - "serve requires a sealed bundle; run `registry-render seal` first", - )); - } let mut files = BTreeMap::new(); files.insert(MANIFEST_FILE.to_owned(), Bytes::new(manifest_bytes.clone())); @@ -91,13 +84,11 @@ impl BundleSnapshot { }, manifest, manifest_bytes, - directory, )) } #[cfg(not(any(target_os = "linux", target_vendor = "apple")))] { - let _ = require_sealed; Err(RenderProblem::new( ProblemKind::ManifestInvalid, format!( @@ -108,14 +99,41 @@ impl BundleSnapshot { } } - fn hashes(&self) -> BTreeMap { - let mut files = BTreeMap::new(); - for (path, bytes) in self.iter() { - if path != MANIFEST_FILE { - files.insert(path.clone(), sha256_hex(bytes.as_slice())); - } + /// Capture a package without interpreting product bytes. The caller + /// binds this snapshot to shared verification before parsing the + /// manifest or constructing any other product consumer. + fn load_unparsed(root: &Path) -> Result<(Self, Vec), RenderProblem> { + #[cfg(any(target_os = "linux", target_vendor = "apple"))] + { + use std::ffi::OsStr; + + let directory = open_bundle_root(root)?; + let manifest_bytes = read_bundle_file( + &directory, + OsStr::new(MANIFEST_FILE), + &root.join(MANIFEST_FILE), + )?; + let mut files = BTreeMap::new(); + files.insert(MANIFEST_FILE.to_owned(), Bytes::new(manifest_bytes.clone())); + capture_bundle_directory(&directory, Path::new(""), root, &mut files)?; + Ok(( + Self { + files: Arc::new(files), + }, + manifest_bytes, + )) + } + + #[cfg(not(any(target_os = "linux", target_vendor = "apple")))] + { + Err(RenderProblem::new( + ProblemKind::ManifestInvalid, + format!( + "cannot securely snapshot bundle {} on this platform", + root.display() + ), + )) } - files } pub(crate) fn get(&self, path: &str) -> Option { @@ -130,6 +148,25 @@ impl BundleSnapshot { fn iter(&self) -> impl Iterator { self.files.iter() } + + fn product_files(&self) -> Self { + let files = self + .files + .iter() + .filter(|(path, _)| !registry_platform_config::package::is_envelope_file(path)) + .map(|(path, bytes)| (path.clone(), bytes.clone())) + .collect(); + Self { + files: Arc::new(files), + } + } + + fn package_inputs(&self) -> BTreeMap> { + self.files + .iter() + .map(|(path, bytes)| (path.clone(), bytes.to_vec())) + .collect() + } } #[cfg(any(target_os = "linux", target_vendor = "apple"))] @@ -315,21 +352,79 @@ fn capture_bundle_directory( } impl Bundle { - /// Load a bundle without requiring it to be sealed. If it is sealed, the - /// hashes are verified. + /// Load raw authoring source. Deployment envelope files are refused so an + /// already-built package cannot silently become the source of another. pub fn load(root: &Path) -> Result { - let (snapshot, manifest, manifest_bytes, _directory) = BundleSnapshot::load(root, false)?; - if manifest.is_sealed() { - verify_hashes(&snapshot, &manifest)?; + let (snapshot, manifest, manifest_bytes) = BundleSnapshot::load(root)?; + if let Some(path) = snapshot + .iter() + .map(|(path, _)| path.as_str()) + .find(|path| registry_platform_config::package::is_envelope_file(path)) + { + return Err(RenderProblem::new( + ProblemKind::InvalidArgument, + format!( + "authoring bundle contains package envelope file {path}; edit the source bundle and build a new directory with `registry-render package --bundle --output `" + ), + )); + } + let bundle_hash = sha256_hex(&manifest_bytes); + Self::assemble(root, manifest, manifest_bytes, snapshot, bundle_hash) + } + + /// Load source for authoring commands, or verify and load a current + /// package when the directory has the shared envelope. This lets an + /// operator inspect or compile the exact package they will deploy while + /// keeping `package` itself source-only and write-once. + pub fn load_for_preview(root: &Path) -> Result { + match std::fs::symlink_metadata(root.join(registry_platform_config::package::SUM_FILE)) { + Ok(_) => { + let verified = registry_platform_config::package::verify_package( + root, + &crate::runtime::package_limits(), + "registry-render package", + ) + .map_err(crate::runtime::package_problem)?; + Self::load_package(root, &verified) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Self::load(root), + Err(error) => Err(RenderProblem::new( + ProblemKind::ManifestInvalid, + format!( + "cannot inspect package envelope {}: {error}", + root.join(registry_platform_config::package::SUM_FILE) + .display() + ), + )), } - Self::assemble(root, manifest, manifest_bytes, snapshot) } - /// Load a bundle and require a verified seal. - pub fn load_sealed(root: &Path) -> Result { - let (snapshot, manifest, manifest_bytes, _directory) = BundleSnapshot::load(root, true)?; - verify_hashes(&snapshot, &manifest)?; - Self::assemble(root, manifest, manifest_bytes, snapshot) + /// Capture a package once, prove those consumed bytes are the bytes the + /// shared verifier accepted, and assemble only product-owned content. + pub fn load_package( + root: &Path, + verified: ®istry_platform_config::package::VerifiedPackage, + ) -> Result { + let (snapshot, manifest_bytes) = BundleSnapshot::load_unparsed(root)?; + bind_verified_snapshot(&snapshot, verified)?; + let manifest = Manifest::parse(&manifest_bytes)?; + let bundle_hash = verified + .digest() + .strip_prefix("sha256:") + .expect("the shared verifier returns a sha256 label") + .to_owned(); + Self::assemble( + root, + manifest, + manifest_bytes, + snapshot.product_files(), + bundle_hash, + ) + } + + /// Exact authored files supplied to the shared package writer. + pub(crate) fn package_inputs(&self) -> BTreeMap> { + self.snapshot.package_inputs() } fn assemble( @@ -337,8 +432,8 @@ impl Bundle { manifest: Manifest, manifest_bytes: Vec, snapshot: BundleSnapshot, + bundle_hash: String, ) -> Result { - let bundle_hash = sha256_hex(&manifest_bytes); let mut documents = BTreeMap::new(); for spec in manifest.documents.clone() { let mut labels = BTreeMap::new(); @@ -432,113 +527,57 @@ impl Bundle { ) }) } - - /// Compute and write per-file hashes into the bundle's manifest, making - /// it sealed. Returns the new manifest. - pub fn seal(root: &Path) -> Result { - let (snapshot, mut manifest, _, directory) = BundleSnapshot::load(root, false)?; - manifest.hashes = Some(snapshot.hashes()); - let serialized = serde_norway::to_string(&manifest).map_err(|err| { - RenderProblem::new( - ProblemKind::Internal, - format!("cannot serialize manifest: {err}"), - ) - })?; - write_manifest(&directory, root, serialized.as_bytes())?; - Ok(manifest) - } } -#[cfg(any(target_os = "linux", target_vendor = "apple"))] -fn write_manifest( - directory: &std::fs::File, - root: &Path, - bytes: &[u8], +fn bind_verified_snapshot( + snapshot: &BundleSnapshot, + verified: ®istry_platform_config::package::VerifiedPackage, ) -> Result<(), RenderProblem> { - use std::io::Write as _; - - use rustix::fs::{openat, Mode, OFlags}; - - let flags = OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::NONBLOCK | OFlags::CLOEXEC; - let descriptor = openat(directory, MANIFEST_FILE, flags, Mode::empty()).map_err(|err| { - RenderProblem::new( - ProblemKind::ManifestInvalid, - format!( - "cannot open {} for writing: {err}", - root.join(MANIFEST_FILE).display() - ), - ) - })?; - let mut file = std::fs::File::from(descriptor); - let metadata = file.metadata().map_err(|err| { - RenderProblem::new( - ProblemKind::ManifestInvalid, - format!( - "cannot inspect bundle entry {}: {err}", - root.join(MANIFEST_FILE).display() - ), - ) - })?; - if !metadata.is_file() { + let sums = snapshot.get(registry_platform_config::package::SUM_FILE); + let captured_digest = sums + .as_ref() + .map(|bytes| registry_platform_config::sha256_uri(bytes.as_slice())); + if captured_digest.as_deref() != Some(verified.digest()) { return Err(RenderProblem::new( - ProblemKind::ManifestInvalid, - format!( - "bundle entry is not a regular file: {}", - root.join(MANIFEST_FILE).display() - ), - )); + ProblemKind::BundleTampered, + "SHA256SUMS changed after package verification; rebuild the package with `registry-render package` and deploy the whole directory", + ) + .with_locations(vec![registry_platform_config::package::SUM_FILE.to_owned()])); } - file.set_len(0) - .and_then(|()| file.write_all(bytes)) - .map_err(|err| { - RenderProblem::new( - ProblemKind::ManifestInvalid, - format!("cannot write {}: {err}", root.join(MANIFEST_FILE).display()), - ) - }) -} -#[cfg(not(any(target_os = "linux", target_vendor = "apple")))] -fn write_manifest( - _directory: &std::fs::File, - root: &Path, - _bytes: &[u8], -) -> Result<(), RenderProblem> { - Err(RenderProblem::new( - ProblemKind::ManifestInvalid, - format!( - "cannot securely seal bundle {} on this platform", - root.display() - ), - )) -} - -fn verify_hashes(snapshot: &BundleSnapshot, manifest: &Manifest) -> Result<(), RenderProblem> { - let declared = manifest.hashes.as_ref().expect("caller checked is_sealed"); - let mut mismatches = Vec::new(); - for (rel, want) in declared { - match snapshot.get(rel) { - Some(bytes) if sha256_hex(bytes.as_slice()) == *want => {} - Some(_) => mismatches.push(format!("{rel} (content changed)")), - None => mismatches.push(format!("{rel} (missing)")), - } - } - for (rel, _) in snapshot.iter() { - if rel != MANIFEST_FILE && !declared.contains_key(rel) { - mismatches.push(format!("{rel} (not covered by the manifest)")); + let captured_files = snapshot + .iter() + .map(|(path, _)| path.as_str()) + .filter(|path| *path != registry_platform_config::package::SUM_FILE) + .collect::>(); + let verified_files = verified.files().collect::>(); + let mut mismatches = captured_files + .symmetric_difference(&verified_files) + .map(|path| (*path).to_owned()) + .collect::>(); + for path in verified.files() { + let Some(bytes) = snapshot.get(path) else { + continue; + }; + let captured = registry_platform_config::sha256_uri(bytes.as_slice()); + if verified.file_digest(path).as_deref() != Some(captured.as_str()) { + mismatches.push(path.to_owned()); } } - if !mismatches.is_empty() { - return Err(RenderProblem::new( + mismatches.sort(); + mismatches.dedup(); + if mismatches.is_empty() { + Ok(()) + } else { + Err(RenderProblem::new( ProblemKind::BundleTampered, format!( - "sealed bundle does not match its manifest: {}", - mismatches.join("; ") + "package content changed after verification: {}; rebuild the package with `registry-render package` and deploy the whole directory", + mismatches.join(", ") ), ) - .with_locations(mismatches)); + .with_locations(mismatches)) } - Ok(()) } fn require_string_map(table: &Value, locale: &str) -> Result { @@ -659,9 +698,7 @@ mod tests { let direct_link = base.join("current"); symlink(&direct_target, &direct_link).unwrap(); let trailing_slash = PathBuf::from(format!("{}/", direct_link.display())); - let error = BundleSnapshot::load(&trailing_slash, false).unwrap_err(); - assert_eq!(error.kind, ProblemKind::ManifestInvalid); - let error = Bundle::seal(&trailing_slash).unwrap_err(); + let error = BundleSnapshot::load(&trailing_slash).unwrap_err(); assert_eq!(error.kind, ProblemKind::ManifestInvalid); let ancestor_target = base.join("ancestor-target"); @@ -669,15 +706,13 @@ mod tests { let ancestor_link = base.join("deploy"); symlink(&ancestor_target, &ancestor_link).unwrap(); let ancestor_bundle = ancestor_link.join("bundle"); - let error = BundleSnapshot::load(&ancestor_bundle, false).unwrap_err(); - assert_eq!(error.kind, ProblemKind::ManifestInvalid); - let error = Bundle::seal(&ancestor_bundle).unwrap_err(); + let error = BundleSnapshot::load(&ancestor_bundle).unwrap_err(); assert_eq!(error.kind, ProblemKind::ManifestInvalid); } #[cfg(any(target_os = "linux", target_vendor = "apple"))] #[test] - fn sealed_load_checks_the_manifest_before_capturing_descendants() { + fn source_load_checks_the_manifest_before_capturing_descendants() { use std::os::unix::fs::symlink; let dir = tempfile::tempdir().unwrap(); @@ -685,12 +720,12 @@ mod tests { let root = root_path.as_path(); symlink(root, root.join("descendant-link")).unwrap(); - let missing = Bundle::load_sealed(root).unwrap_err(); + let missing = Bundle::load(root).unwrap_err(); assert_eq!(missing.kind, ProblemKind::ManifestInvalid); assert!(missing.detail.contains(MANIFEST_FILE)); std::fs::write(root.join(MANIFEST_FILE), "not: [valid").unwrap(); - let malformed = Bundle::load_sealed(root).unwrap_err(); + let malformed = Bundle::load(root).unwrap_err(); assert_eq!(malformed.kind, ProblemKind::ManifestInvalid); assert!(malformed.detail.contains("manifest.yaml is not valid")); @@ -699,8 +734,9 @@ mod tests { "apiVersion: render.registrystack.org/v1alpha1\nkind: RenderBundle\nbundleVersion: 1\ndocuments: []\n", ) .unwrap(); - let unsealed = Bundle::load_sealed(root).unwrap_err(); - assert_eq!(unsealed.kind, ProblemKind::BundleUnsealed); + let unsafe_entry = Bundle::load(root).unwrap_err(); + assert_eq!(unsafe_entry.kind, ProblemKind::ManifestInvalid); + assert!(unsafe_entry.detail.contains("without following links")); } #[test] @@ -728,10 +764,7 @@ mod tests { include_bytes!("../assets/starter-fonts/NotoSans-Regular.ttf"), ) .unwrap(); - Bundle::seal(root).unwrap(); - - let (snapshot, manifest, manifest_bytes, _directory) = - BundleSnapshot::load(root, false).unwrap(); + let (snapshot, manifest, manifest_bytes) = BundleSnapshot::load(root).unwrap(); let expected_hash = sha256_hex(&manifest_bytes); // Every path used by assembly changes after capture. Assembly must @@ -742,7 +775,14 @@ mod tests { std::fs::write(&schema, "not json").unwrap(); std::fs::write(&font, "not a font").unwrap(); - let bundle = Bundle::assemble(root, manifest, manifest_bytes, snapshot).unwrap(); + let bundle = Bundle::assemble( + root, + manifest, + manifest_bytes, + snapshot, + expected_hash.clone(), + ) + .unwrap(); let document = bundle.document("notice").unwrap(); assert_eq!(document.labels["en"]["title"], "Captured"); assert_eq!(document.schema.as_ref().unwrap()["type"], "object"); @@ -771,9 +811,7 @@ mod tests { r#"{"type":"object"}"#, ) .unwrap(); - Bundle::seal(root).unwrap(); - - let bundle = Bundle::load_sealed(root).unwrap(); + let bundle = Bundle::load(root).unwrap(); let document = bundle.document("notice").unwrap(); let request = crate::render::RenderRequest { locale: None, @@ -786,4 +824,67 @@ mod tests { assert_eq!(rendered.deps, vec!["templates/notice.typ"]); assert_eq!(document.schema.as_ref().unwrap()["type"], "object"); } + + #[test] + fn package_load_refuses_bytes_replaced_after_shared_verification() { + let source = tempfile::tempdir().unwrap(); + std::fs::create_dir(source.path().join("templates")).unwrap(); + std::fs::write( + source.path().join(MANIFEST_FILE), + "apiVersion: render.registrystack.org/v1alpha1\nkind: RenderBundle\nbundleVersion: 1\ndocuments:\n - id: notice\n version: 1\n entry: templates/notice.typ\n", + ) + .unwrap(); + std::fs::write(source.path().join("templates/notice.typ"), "= Accepted").unwrap(); + let source_root = source.path().canonicalize().unwrap(); + let authored = Bundle::load(&source_root).unwrap(); + + let parent = tempfile::tempdir().unwrap(); + let package = parent.path().canonicalize().unwrap().join("package"); + registry_platform_config::package::write_package( + &package, + &authored.package_inputs(), + None, + &crate::runtime::package_limits(), + "registry-render package", + ) + .unwrap(); + let verified = registry_platform_config::package::verify_package( + &package, + &crate::runtime::package_limits(), + "registry-render package", + ) + .unwrap(); + std::fs::write(package.join("templates/notice.typ"), "= Replaced").unwrap(); + + let problem = Bundle::load_package(&package, &verified) + .expect_err("captured replacement must not be consumed"); + assert_eq!(problem.kind, ProblemKind::BundleTampered); + assert!(problem.detail.contains("templates/notice.typ")); + + let manifest_package = parent + .path() + .canonicalize() + .unwrap() + .join("manifest-package"); + registry_platform_config::package::write_package( + &manifest_package, + &authored.package_inputs(), + None, + &crate::runtime::package_limits(), + "registry-render package", + ) + .unwrap(); + let verified = registry_platform_config::package::verify_package( + &manifest_package, + &crate::runtime::package_limits(), + "registry-render package", + ) + .unwrap(); + std::fs::write(manifest_package.join(MANIFEST_FILE), "not: [valid").unwrap(); + + let problem = Bundle::load_package(&manifest_package, &verified) + .expect_err("manifest replacement must be bound before it is parsed"); + assert_eq!(problem.kind, ProblemKind::BundleTampered); + assert!(problem.detail.contains(MANIFEST_FILE)); + } } diff --git a/crates/registry-render/src/check.rs b/crates/registry-render/src/check.rs index 9f54677ce8..745c67fbdf 100644 --- a/crates/registry-render/src/check.rs +++ b/crates/registry-render/src/check.rs @@ -1,5 +1,5 @@ -//! `registry-render check`: the plain-language preflight. Verifies structure and -//! seal hashes, label-script font coverage, and per-locale label key sets. +//! `registry-render check`: the plain-language authoring preflight. Verifies +//! structure, label-script font coverage, and per-locale label key sets. //! The rendered file closure is governed where a render exists to capture //! it: the golden suite pins each acceptance bundle's closure and proves //! every file it reads is manifest-governed, and `compile --json` reports @@ -39,25 +39,15 @@ pub fn run( root.display() ); } - // Verify everything first; only a bundle that fully verifies gets - // sealed. Sealing a broken bundle would lend it an unearned seal. - let bundle = Bundle::load(bundle_dir)?; - if seal && bundle.manifest.is_sealed() { + if seal { return Err(RenderProblem::new( ProblemKind::InvalidArgument, - "bundle is already sealed; edit, then run `registry-render seal` to re-seal", + "`registry-render check --seal` is no longer accepted; run `registry-render check`, then build a new directory with `registry-render package --bundle --output `", )); } + let bundle = Bundle::load_for_preview(bundle_dir)?; check_script_coverage(&bundle)?; check_label_key_sets(&bundle)?; - if seal { - Bundle::seal(bundle_dir)?; - println!("sealed (bundle hashes written to manifest.yaml)"); - } else if !bundle.manifest.is_sealed() { - println!( - "note: bundle is unsealed; compile works, serve does not (run `registry-render seal`)" - ); - } for document in bundle.documents.values() { let labels = document .spec @@ -79,12 +69,7 @@ pub fn run( .unwrap_or_else(|| "plain".to_owned()) ); } - let governed = bundle - .manifest - .hashes - .as_ref() - .map(|h| h.len()) - .unwrap_or(0); + let governed = bundle.package_inputs().len(); println!( "bundle {} v{} hash {} ({} fonts, {} documents, {} governed files)", bundle_dir.display(), diff --git a/crates/registry-render/src/cli.rs b/crates/registry-render/src/cli.rs index c0888f5512..f40c05e4df 100644 --- a/crates/registry-render/src/cli.rs +++ b/crates/registry-render/src/cli.rs @@ -1,4 +1,4 @@ -//! The `registry-render` CLI: init, check, validate, seal, compile, +//! The `registry-render` CLI: init, check, validate, package, compile, //! serve, healthcheck. Exit codes come from the problem model. use std::collections::BTreeMap; @@ -45,27 +45,26 @@ pub enum Command { #[arg(long = "labels", value_delimiter = ',')] labels: Vec, }, - /// Verify a bundle (structure, hashes, label coverage), optionally - /// sealing it only after verification passes. + /// Verify source or package structure, labels, fonts, and schemas. Check { - /// Bundle directory (default: current directory). + /// Authored bundle or package directory (default: current directory). #[arg(long, default_value = ".")] bundle: PathBuf, - /// Rewrite the manifest hashes, sealing the bundle. - #[arg(long)] + /// Retired. Use `registry-render package` after check succeeds. + #[arg(long, hide = true)] seal: bool, /// Runtime file whose audit file is proven to resolve under the /// given root (the container preflight proof). A stdout audit /// destination is refused. - #[arg(long)] + #[arg(long = "runtime-config", value_name = "FILE")] runtime: Option, - /// Root the audit file must resolve under (with --runtime). + /// Root the audit file must resolve under (with --runtime-config). #[arg(long)] require_audit_under: Option, }, /// Dry-run request data against a document's schema, without rendering. Validate { - /// Bundle directory. + /// Authored bundle or package directory. #[arg(long)] bundle: PathBuf, /// Document type id, as declared in the manifest. @@ -84,15 +83,28 @@ pub enum Command { #[arg(long)] json: bool, }, - /// Write per-file hashes into the manifest, sealing the bundle. + /// Build a new deployment package with the shared checksum envelope. + Package { + /// Authored bundle directory. + #[arg(long)] + bundle: PathBuf, + /// New package directory; it must not exist yet. + #[arg(long)] + output: PathBuf, + /// One printable line recorded in REVISION and covered by the digest. + #[arg(long, value_name = "TEXT")] + revision: Option, + }, + /// Retired spelling. Use `registry-render package`. + #[command(hide = true)] Seal { - /// Bundle directory (default: current directory). - #[arg(long, default_value = ".")] + /// Authored bundle directory. + #[arg(long, default_value = ".", hide = true)] bundle: PathBuf, }, /// Render one document offline. Compile { - /// Bundle directory. + /// Authored bundle or package directory. #[arg(long)] bundle: PathBuf, /// Document type id, as declared in the manifest. @@ -124,8 +136,8 @@ pub enum Command { /// uses). #[arg(long = "timeout")] timeout: Option, - /// Keep rendering on bundle changes (authoring loop; unsealed - /// bundles are fine, poll-based). Never returns, so it refuses + /// Keep rendering on bundle changes (authoring loop; raw source + /// directories are fine, poll-based). Never returns, so it refuses /// `--emit-envelope` rather than ignoring it. #[arg(long, conflicts_with = "emit_envelope")] watch: bool, @@ -142,15 +154,16 @@ pub enum Command { }, /// Serve the HTTP rendering API (see the runtime YAML). Serve { - /// Runtime file naming the bundle, bind address, secrets, and limits. - #[arg(long, env = "REGISTRY_RENDER_RUNTIME")] - runtime: Option, + /// Absolute path of the runtime file naming the package, listener, + /// secrets, and limits. + #[arg(long = "runtime-config", value_name = "FILE")] + runtime: PathBuf, }, /// Probe a running server's /health. Healthcheck { - /// Runtime file naming the server to probe. - #[arg(long, env = "REGISTRY_RENDER_RUNTIME")] - runtime: Option, + /// Absolute path of the runtime file naming the server to probe. + #[arg(long = "runtime-config", value_name = "FILE")] + runtime: PathBuf, }, /// Hidden: one supervised render (used by `registry-render serve`). #[command(hide = true, name = "__worker")] @@ -217,11 +230,42 @@ fn run_inner(cli: Cli) -> Result { runtime.as_deref(), require_audit_under.as_deref(), ), - Command::Seal { bundle } => { - Bundle::seal(&bundle)?; - println!("sealed {}", bundle.display()); + Command::Package { + bundle, + output, + revision, + } => { + let loaded = Bundle::load(&bundle)?; + crate::check::check_script_coverage(&loaded)?; + crate::check::check_label_key_sets(&loaded)?; + let written = registry_platform_config::package::write_package( + &output, + &loaded.package_inputs(), + revision.as_deref(), + &crate::runtime::package_limits(), + "registry-render package", + ) + .map_err(|error| { + RenderProblem::new( + crate::problem::ProblemKind::InvalidArgument, + error.to_string(), + ) + })?; + println!( + "packaged {} as {}", + output.display(), + written.digest() + ); Ok(0) } + Command::Seal { bundle } => Err(RenderProblem::new( + crate::problem::ProblemKind::InvalidArgument, + format!( + "`registry-render seal --bundle {}` is no longer accepted; build a new directory with `registry-render package --bundle {} --output `", + bundle.display(), + bundle.display() + ), + )), Command::Validate { bundle, document, @@ -229,7 +273,7 @@ fn run_inner(cli: Cli) -> Result { locale, json, } => { - let bundle = Bundle::load(&bundle)?; + let bundle = Bundle::load_for_preview(&bundle)?; let document = bundle.document(&document)?; // Validate is a schema dry-run; it needs no issuance time and // renders nothing. @@ -274,15 +318,9 @@ fn run_inner(cli: Cli) -> Result { if watch { return watch_loop(&bundle, &document, &request, strict, timeout, &out); } - let loaded = Bundle::load(&bundle).map_err(recovery_hint)?; - if !loaded.manifest.is_sealed() && !json { - eprintln!( - "note: bundle is unsealed; compile is fine, serve is not (run `registry-render seal` when ready)" - ); - } + let loaded = Bundle::load_for_preview(&bundle)?; let document_spec = loaded.document(&document)?.clone(); - let rendered = compile_once(&bundle, &document, &request, strict, timeout) - .map_err(recovery_hint)?; + let rendered = compile_once(&bundle, &document, &request, strict, timeout)?; let pdf = base64::Engine::decode( &base64::engine::general_purpose::STANDARD, &rendered.pdf_base64, @@ -356,11 +394,11 @@ fn run_inner(cli: Cli) -> Result { Ok(0) } Command::Serve { runtime } => { - let code = crate::server::serve(runtime.as_deref())?; + let code = crate::server::serve(&runtime)?; Ok(code) } Command::Healthcheck { runtime } => { - let code = crate::server::healthcheck(runtime.as_deref())?; + let code = crate::server::healthcheck(&runtime)?; Ok(code) } Command::Worker => Ok(crate::worker::worker_main()), @@ -475,7 +513,7 @@ fn compile_once( assets: request.assets.clone(), issued_at: request.issued_at.to_rfc3339(), strict, - require_sealed: false, + require_package: false, max_output_bytes: crate::render::DEFAULT_MAX_OUTPUT_BYTES, memory_limit_bytes: 512 * 1024 * 1024, }; @@ -496,21 +534,6 @@ pub fn flush() { let _ = std::io::stdout().flush(); } -/// Compile-time wrapper that keeps the authoring loop actionable: a -/// drifted seal after an intentional edit should tell the author how to -/// recover, not accuse them of tampering. -fn recovery_hint(problem: RenderProblem) -> RenderProblem { - if problem.kind == crate::ProblemKind::BundleTampered { - let mut problem = problem; - problem.detail.push_str( - "; if these edits are yours, run `registry-render seal` to re-seal the bundle", - ); - problem - } else { - problem - } -} - /// The authoring loop: render now, then re-render whenever the bundle's /// files change. Poll-based on purpose — no filesystem-event dependency, /// works everywhere the CLI works, and the loop is for humans, not CI. @@ -525,7 +548,7 @@ fn watch_loop( out: &Path, ) -> Result { eprintln!( - "watching {} (ctrl-c to stop); unsealed bundles are fine while authoring", + "watching {} (ctrl-c to stop); package after authoring is complete", bundle_dir.display() ); let mut last_fingerprint: Option> = None; diff --git a/crates/registry-render/src/hash.rs b/crates/registry-render/src/hash.rs index 19dbc27e49..278d8aaf54 100644 --- a/crates/registry-render/src/hash.rs +++ b/crates/registry-render/src/hash.rs @@ -1,4 +1,4 @@ -//! Hash helpers shared by manifest sealing, envelope hashing, and audit. +//! Hash helpers shared by request envelopes, rendered output, and audit. use sha2::{Digest, Sha256}; diff --git a/crates/registry-render/src/init.rs b/crates/registry-render/src/init.rs index 03b137ee64..e9d3efe5bf 100644 --- a/crates/registry-render/src/init.rs +++ b/crates/registry-render/src/init.rs @@ -85,11 +85,9 @@ const FIXTURE_DATA: &str = const README: &str = r#"# Render bundle -- `manifest.yaml` — document types, versions, labels, and (after - `registry-render seal`) per-file hashes. +- `manifest.yaml` — document types, versions, labels, and PDF constraints. - `templates/` — Typst entry points. Author them with any upstream Typst - tooling; `registry-render compile` (and `--watch`) run unsealed bundles with a - notice. + tooling; `registry-render compile` (and `--watch`) run the authored bundle. - `schemas/` — the JSON Schema each request's `data` must satisfy. - `labels/` — one flat YAML string map per locale; seeded in English, localize at will. A script beyond Latin needs a font in `fonts/` (for @@ -105,7 +103,8 @@ const README: &str = r#"# Render bundle Workflow: edit → `registry-render compile --bundle . --type letter --data fixtures/data.json --issued-at … --out letter.pdf` (or `--watch`) → -`registry-render seal` when ready → `registry-render serve` requires the sealed bundle. +`registry-render check` → `registry-render package --bundle . --output ../letter-package`. +Point `package.root` at the new package directory before serving. Before writing a template that prints registry data, agree the field list (the disclosure gate): what appears on paper leaves every access profile diff --git a/crates/registry-render/src/lib.rs b/crates/registry-render/src/lib.rs index 31fffe4903..fe7a377d86 100644 --- a/crates/registry-render/src/lib.rs +++ b/crates/registry-render/src/lib.rs @@ -1,7 +1,7 @@ //! `registry-render` — governed, byte-stable PDF documents from registry //! data, rendered with Typst in library mode. //! -//! The crate is a pure function at heart: a sealed template bundle plus a +//! The crate is a pure function at heart: a verified template package plus a //! validated request in, an identical PDF plus hashes out, every time. The //! world a template sees contains exactly the bundle, the request's decoded //! assets, and the vendored packages — no host fonts, no network, no clock @@ -19,8 +19,8 @@ //! bundle fonts sorted by path — mirroring the Typst CLI's book so //! library and CLI renders agree byte for byte. Never filesystem //! iteration order. -//! - **Verified bytes are consumed bytes**: bundle files are captured once, -//! seals are verified over that immutable snapshot, and the same bytes feed +//! - **Verified bytes are consumed bytes**: package files are captured once, +//! shared checksums are bound to that immutable snapshot, and the same bytes feed //! label/schema parsing, font loading, and every Typst source/file lookup. //! - **Path safety is world-enforced**: every resolution rejects `..` and //! absolute components, then must name an exact immutable snapshot key. diff --git a/crates/registry-render/src/manifest.rs b/crates/registry-render/src/manifest.rs index 6833a40273..655863536a 100644 --- a/crates/registry-render/src/manifest.rs +++ b/crates/registry-render/src/manifest.rs @@ -2,9 +2,8 @@ //! directory of Typst files, fonts, labels, schemas, and packages into //! governed content. -use std::collections::BTreeMap; use std::fmt; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use serde::{Deserialize, Serialize}; @@ -132,21 +131,44 @@ pub struct Manifest { pub bundle_version: u32, #[serde(default, rename = "documents")] pub documents: Vec, - /// Per-file sha256 hex digests, relative to the bundle root, slash - /// separated. Present iff the bundle is sealed. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub hashes: Option>, } impl Manifest { /// Parse and structurally validate manifest bytes. pub fn parse(bytes: &[u8]) -> Result { + let value: serde_norway::Value = serde_norway::from_slice(bytes).map_err(|err| { + RenderProblem::new( + ProblemKind::ManifestInvalid, + format!("manifest.yaml is not valid: {err}"), + ) + })?; + if value + .as_mapping() + .is_some_and(|mapping| mapping.contains_key("hashes")) + { + return Err(RenderProblem::new( + ProblemKind::ManifestInvalid, + "manifest.yaml key hashes is no longer accepted; remove it and build a deployment package with `registry-render package --bundle --output `", + )); + } let manifest: Manifest = serde_norway::from_slice(bytes).map_err(|err| { RenderProblem::new( ProblemKind::ManifestInvalid, format!("manifest.yaml is not valid: {err}"), ) })?; + let text = std::str::from_utf8(bytes).map_err(|_| { + RenderProblem::new( + ProblemKind::ManifestInvalid, + "manifest.yaml is not valid UTF-8", + ) + })?; + registry_platform_config::reject_environment_expressions_in_authored_yaml(text).map_err( + |error| { + RenderProblem::new(ProblemKind::ManifestInvalid, error.message()) + .with_locations(vec![MANIFEST_FILE.to_owned()]) + }, + )?; if manifest.api_version != MANIFEST_API_VERSION { return Err(RenderProblem::new( ProblemKind::ManifestInvalid, @@ -207,7 +229,7 @@ impl Manifest { )); } // The schema gets the same containment rule as the entry: a - // schema outside the bundle would sit outside the seal. + // schema outside the bundle would sit outside package governance. if let Some(schema) = doc.schema.as_ref() { let schema = schema.to_string_lossy(); if schema.is_empty() @@ -251,81 +273,6 @@ impl Manifest { ) }) } - - pub fn is_sealed(&self) -> bool { - self.hashes.as_ref().is_some_and(|h| !h.is_empty()) - } - - /// Compute the per-file sha256 map over every governed file in the - /// bundle. `manifest.yaml` itself is excluded: it carries the hashes - /// and cannot hash itself; the bundle id is the sha256 of the sealed - /// manifest bytes. Deterministic: sorted relative paths, - /// slash-separated. - pub fn compute_hashes(root: &Path) -> Result, RenderProblem> { - let mut map = BTreeMap::new(); - let mut stack = vec![root.to_path_buf()]; - while let Some(dir) = stack.pop() { - let entries = std::fs::read_dir(&dir).map_err(|err| { - RenderProblem::new( - ProblemKind::ManifestInvalid, - format!("cannot read bundle directory {}: {err}", dir.display()), - ) - })?; - for entry in entries { - let entry = entry.map_err(|err| { - RenderProblem::new( - ProblemKind::ManifestInvalid, - format!("cannot read bundle directory {}: {err}", dir.display()), - ) - })?; - let path = entry.path(); - let meta = std::fs::symlink_metadata(&path).map_err(|err| { - RenderProblem::new( - ProblemKind::ManifestInvalid, - format!("cannot stat {}: {err}", path.display()), - ) - })?; - let file_type = meta.file_type(); - if file_type.is_symlink() { - // A symlink inside the bundle could point outside the - // root; the render world refuses it, so a seal that - // silently hashed its target would produce a bundle - // that verifies but cannot render. - return Err(RenderProblem::new( - ProblemKind::ManifestInvalid, - format!( - "bundle contains a symlink, which cannot be sealed: {}", - path.display() - ), - )); - } - if file_type.is_dir() { - stack.push(path); - } else { - let bytes = std::fs::read(&path).map_err(|err| { - RenderProblem::new( - ProblemKind::ManifestInvalid, - format!("cannot read {}: {err}", path.display()), - ) - })?; - let rel = path - .strip_prefix(root) - .expect("walk stays under root") - .to_string_lossy() - .replace('\\', "/"); - // Only the bundle's own root manifest is excluded (it - // carries these hashes); a nested manifest.yaml is - // ordinary governed content. - if rel == MANIFEST_FILE { - continue; - } - let digest = crate::hash::sha256_hex(&bytes); - map.insert(rel, digest); - } - } - } - Ok(map) - } } #[cfg(test)] @@ -342,7 +289,7 @@ mod tests { #[test] fn schema_path_is_validated_like_the_entry() { - // A schema outside the bundle sits outside the seal; the entry rule + // A schema outside the bundle sits outside the package; the entry rule // (no `..`, no absolute, correct suffix) applies to it too. let doc = |schema: &str| { format!("apiVersion: render.registrystack.org/v1alpha1\nkind: RenderBundle\nbundleVersion: 1\ndocuments:\n - id: d\n version: 1\n entry: templates/d.typ\n schema: {schema}\n") @@ -359,10 +306,17 @@ mod tests { let m = Manifest::parse(good).expect("parses"); assert_eq!(m.documents.len(), 1); assert_eq!(m.documents[0].labels, vec!["ar", "fr"]); - assert!(!m.is_sealed()); assert_eq!(m.documents[0].pdf_standard.map(|s| s.to_string()), None); } + #[test] + fn retired_manifest_hashes_name_the_package_replacement() { + let manifest = b"apiVersion: render.registrystack.org/v1alpha1\nkind: RenderBundle\nbundleVersion: 1\ndocuments: []\nhashes:\n templates/a.typ: aaaa\n"; + let problem = Manifest::parse(manifest).expect_err("self-hashing manifest is retired"); + assert!(problem.detail.contains("hashes")); + assert!(problem.detail.contains("registry-render package")); + } + #[test] fn pdf_standard_roundtrip_kebab() { let good = b"apiVersion: render.registrystack.org/v1alpha1\nkind: RenderBundle\nbundleVersion: 1\ndocuments:\n - id: cert\n version: 1\n entry: templates/cert.typ\n pdfStandard: a-4\n"; @@ -374,4 +328,22 @@ mod tests { typst_pdf::PdfStandard::V_1_7 ); } + + #[test] + fn an_authored_manifest_carrying_an_environment_expression_is_refused() { + let manifest = b"apiVersion: render.registrystack.org/v1alpha1\nkind: RenderBundle\nbundleVersion: 1\ndocuments:\n - id: receipt\n version: 1\n entry: templates/${DOCUMENT}.typ\n"; + let problem = Manifest::parse(manifest).expect_err("expression refused"); + assert!(matches!(problem.kind, ProblemKind::ManifestInvalid)); + assert!( + problem.detail.contains("documents.0.entry"), + "{}", + problem.detail + ); + assert!( + problem.detail.contains("runtime.yaml only"), + "{}", + problem.detail + ); + assert_eq!(problem.locations, vec![MANIFEST_FILE.to_owned()]); + } } diff --git a/crates/registry-render/src/openapi.rs b/crates/registry-render/src/openapi.rs index 13a48f7886..e57accb831 100644 --- a/crates/registry-render/src/openapi.rs +++ b/crates/registry-render/src/openapi.rs @@ -26,7 +26,7 @@ pub const OPENAPI_JSON: &str = r#"{ }, "/v1/documents": { "get": { - "summary": "List document types in the sealed bundle", + "summary": "List document types in the verified package", "security": [{ "bearerAuth": [] }], "responses": { "200": { "description": "document inventory" } } } diff --git a/crates/registry-render/src/problem.rs b/crates/registry-render/src/problem.rs index 9607c92fc9..5f355462ad 100644 --- a/crates/registry-render/src/problem.rs +++ b/crates/registry-render/src/problem.rs @@ -14,9 +14,9 @@ pub enum ProblemKind { InvalidArgument, /// The bundle manifest is missing or structurally invalid. ManifestInvalid, - /// A sealed bundle's file hashes do not match the manifest. + /// A package does not match its shared checksum envelope. BundleTampered, - /// An operation requires a sealed bundle and the bundle is unsealed. + /// A runtime package lacks the shared checksum envelope. BundleUnsealed, /// A requested document type or locale is not in the bundle. UnknownDocument, diff --git a/crates/registry-render/src/runtime.rs b/crates/registry-render/src/runtime.rs index 7e6673c458..7f7aae3017 100644 --- a/crates/registry-render/src/runtime.rs +++ b/crates/registry-render/src/runtime.rs @@ -1,63 +1,85 @@ //! The serve runtime: one strict YAML file with everything a deployment -//! owns — bind address, sealed bundle path, caller key, limits, audit. -//! Nothing here can override governed (bundle) behavior. +//! owns: listener, verified bundle package, secret providers, caller key, +//! limits, audit. Nothing here can override governed (bundle) behavior. +//! +//! The file is read through the shared Registry Stack runtime configuration +//! loader, so its envelope, size bound, path rules, `${VAR}` substitution, +//! and secret provider declarations match every other runtime. use std::path::{Path, PathBuf}; -use registry_platform_audit::{AuditDestination, AuditDestinationError, AuditDestinationKind}; -use serde::{Deserialize, Serialize}; +use registry_platform_audit::{AuditDestination, AuditDestinationKind}; +use registry_platform_config::{ + ListenerBind, PackageConfig, RemovedKey, RuntimeConfigLoader, RuntimeEnvelope, + SecretProvidersConfig, +}; +use serde::Deserialize; use crate::problem::{ProblemKind, RenderProblem}; -pub const RUNTIME_API_VERSION: &str = "render.registrystack.org/v1alpha1"; -pub const RUNTIME_KIND: &str = "RenderRuntime"; - -#[derive(Debug, Clone, Serialize, Deserialize)] +pub const RUNTIME_API_VERSION: &str = "registry.registrystack.org/render-runtime/v1alpha1"; +pub const RUNTIME_KIND: &str = "RenderRuntimeConfig"; + +const RENDER_RUNTIME_ENVELOPE: RuntimeEnvelope = RuntimeEnvelope { + api_version: RUNTIME_API_VERSION, + kind: RUNTIME_KIND, +}; + +/// Keys an earlier runtime file carried, each refused with its replacement. +const RENDER_REMOVED_KEYS: &[RemovedKey] = &[ + RemovedKey { + path: "server", + replacement: "declare listener.bind and listener.shutdownGraceSeconds instead", + }, + RemovedKey { + path: "bundle", + replacement: "declare package.root as the absolute path of the package directory", + }, + RemovedKey { + path: "audit.directory", + replacement: "declare audit.path as the absolute path of the active audit file instead", + }, + RemovedKey { + path: "audit.integrityKeyRef", + replacement: "remove it; audit entries are not hash-chained, so no integrity key is read", + }, + RemovedKey { + path: "audit.maxSegmentBytes", + replacement: "declare audit.rotateBytes instead", + }, +]; + +#[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct RenderRuntime { pub api_version: String, pub kind: String, - #[serde(default)] - pub server: ServerRuntime, - pub bundle: BundleRuntime, + pub listener: ListenerRuntime, + /// The verified bundle package Render serves. + pub package: PackageConfig, + pub secret_providers: SecretProvidersConfig, pub auth: AuthRuntime, #[serde(default)] pub limits: LimitsRuntime, pub audit: AuditRuntime, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct ServerRuntime { - /// Loopback or private address to listen on; defaults to loopback. - /// Public and all-interfaces binds are refused at startup — see - /// [`validate_bind`]. TLS is the proxy's job, not Render's. - #[serde(default = "default_bind")] - pub bind: String, +pub struct ListenerRuntime { + /// Loopback or private address to listen on. Public and all-interfaces + /// binds are refused at startup, see [`validate_bind`]. TLS is the + /// proxy's job, not Render's. + pub bind: ListenerBind, /// Grace period for in-flight renders at shutdown. #[serde(default = "default_shutdown_grace_seconds")] pub shutdown_grace_seconds: u64, } -/// The default listener: loopback, fixed port. Deployments behind a proxy -/// set their own private address explicitly. -pub fn default_bind() -> String { - "127.0.0.1:8080".to_owned() -} - -impl Default for ServerRuntime { - fn default() -> Self { - Self { - bind: default_bind(), - shutdown_grace_seconds: default_shutdown_grace_seconds(), - } - } -} - /// Render never listens on a public address: TLS termination and network /// position belong to the deployment's proxy. Loopback, private, and /// link-local addresses pass; unspecified (all interfaces) and public -/// addresses are refused with a named startup problem — an explicit refusal +/// addresses are refused with a named startup problem, an explicit refusal /// instead of an accidental exposure. pub fn validate_bind(addr: std::net::SocketAddr) -> Result<(), RenderProblem> { let allowed = match addr { @@ -76,8 +98,8 @@ pub fn validate_bind(addr: std::net::SocketAddr) -> Result<(), RenderProblem> { Err(RenderProblem::new( ProblemKind::RuntimeInvalid, format!( - "server.bind {addr} is not a loopback or private address; Render never \ - listens publicly or on all interfaces — terminate TLS and position the \ + "listener.bind {addr} is not a loopback or private address; Render never \ + listens publicly or on all interfaces; terminate TLS and position the \ service on a proxy, and bind its private address here" ), )) @@ -93,22 +115,14 @@ fn default_shutdown_grace_seconds() -> u64 { 30 } -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct BundleRuntime { - /// Path to a sealed bundle directory. Relative paths anchor to the - /// runtime file's directory (see [`load`]), not the working directory. - pub path: PathBuf, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct AuthRuntime { - /// `secret:file/…` or `secret:env/…` reference to the caller API key. + /// Secret reference to the caller API key, under a declared provider. pub api_key_ref: String, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct LimitsRuntime { /// Wall-clock budget per render; the worker is killed at this bound. @@ -154,21 +168,20 @@ pub fn default_max_concurrency() -> usize { /// The audit block every Registry Stack product shares: a `file` (the /// default) or `stdout` destination, with rotation and retention for a file. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields, rename_all = "camelCase")] pub struct AuditRuntime { /// Where audit lines go: `file` or `stdout`. #[serde(default)] pub destination: AuditDestinationKind, - /// The active audit file, for a `file` destination. Relative paths - /// anchor to the runtime file's directory (see [`load`]). - #[serde(default, skip_serializing_if = "Option::is_none")] + /// The absolute path of the active audit file, for a `file` destination. + #[serde(default)] pub path: Option, /// Size at which the active file rotates, for a `file` destination. - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(default)] pub rotate_bytes: Option, /// Days a rotated file is kept, for a `file` destination. - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(default)] pub retain_days: Option, } @@ -185,146 +198,99 @@ impl AuditRuntime { } } -/// Load and validate a runtime file, expanding bounded `${VAR}` references. -/// Relative `bundle.path` and `audit.path` values are anchored to the -/// runtime file's directory — the same anchor `secret:file/…` refs use — so -/// one runtime file behaves identically regardless of the working directory -/// it is loaded from. Returns the runtime and its canonical sha256 config id. +fn invalid(message: impl Into) -> RenderProblem { + RenderProblem::new(ProblemKind::RuntimeInvalid, message) +} + +/// Load and validate a runtime file through the shared loader. Returns the +/// runtime and the `sha256:` digest of its effective configuration. pub fn load(path: &Path) -> Result<(RenderRuntime, String), RenderProblem> { - let raw = std::fs::read_to_string(path).map_err(|err| { - RenderProblem::new( - ProblemKind::RuntimeInvalid, - format!("cannot read runtime {}: {err}", path.display()), - ) - })?; - let expanded = registry_platform_config::expand_config_env_vars(&raw) - .map_err(|err| RenderProblem::new(ProblemKind::RuntimeInvalid, format!("{err}")))?; - let mut runtime: RenderRuntime = serde_norway::from_str(&expanded).map_err(|err| { - RenderProblem::new( - ProblemKind::RuntimeInvalid, - format!("runtime YAML invalid: {err}"), - ) - })?; - if runtime.api_version != RUNTIME_API_VERSION { - return Err(RenderProblem::new( - ProblemKind::RuntimeInvalid, - format!( - "runtime apiVersion must be {RUNTIME_API_VERSION}, found {}", - runtime.api_version - ), - )); - } - if runtime.kind != RUNTIME_KIND { - return Err(RenderProblem::new( - ProblemKind::RuntimeInvalid, - format!( - "runtime kind must be {RUNTIME_KIND}, found {}", - runtime.kind - ), - )); - } + let loaded = RuntimeConfigLoader::new(RENDER_RUNTIME_ENVELOPE) + .removed_keys(RENDER_REMOVED_KEYS) + .load::(path) + .map_err(|error| invalid(error.to_string()))?; + let runtime = loaded.config; + let in_file = |message: String| invalid(format!("{}: {message}", path.display())); + runtime + .package + .check() + .map_err(|error| in_file(error.to_string()))?; + runtime + .secret_providers + .check() + .map_err(|error| in_file(error.to_string()))?; + runtime + .secret_providers + .check_reference("auth.apiKeyRef", &runtime.auth.api_key_ref) + .map_err(|error| in_file(error.to_string()))?; + runtime + .audit + .destination() + .map_err(|error| in_file(error.detail))?; if runtime.limits.max_output_bytes > crate::render::DEFAULT_MAX_OUTPUT_BYTES { - return Err(RenderProblem::new( - ProblemKind::RuntimeInvalid, - format!( - "maxOutputBytes {} exceeds the hard ceiling {}", - runtime.limits.max_output_bytes, - crate::render::DEFAULT_MAX_OUTPUT_BYTES - ), - )); + return Err(invalid(format!( + "maxOutputBytes {} exceeds the hard ceiling {}", + runtime.limits.max_output_bytes, + crate::render::DEFAULT_MAX_OUTPUT_BYTES + ))); } if runtime.limits.max_concurrency == 0 || runtime.limits.max_concurrency > 64 { - return Err(RenderProblem::new( - ProblemKind::RuntimeInvalid, - "maxConcurrency must be between 1 and 64", - )); + return Err(invalid("maxConcurrency must be between 1 and 64")); } if runtime.limits.max_request_body_bytes > 64 * 1024 * 1024 { - return Err(RenderProblem::new( - ProblemKind::RuntimeInvalid, + return Err(invalid( "maxRequestBodyBytes exceeds the 64 MiB hard ceiling", )); } // Zero would drop renders in flight the moment SIGTERM arrives; a value // past the hour is not a deployment intent, and the bounded wait the // shutdown path computes from it must stay representable. - if !(1..=MAX_SHUTDOWN_GRACE_SECONDS).contains(&runtime.server.shutdown_grace_seconds) { - return Err(RenderProblem::new( - ProblemKind::RuntimeInvalid, - format!( - "shutdownGraceSeconds must be between 1 and {MAX_SHUTDOWN_GRACE_SECONDS}, found {}", - runtime.server.shutdown_grace_seconds - ), - )); - } - let anchor = runtime_anchor(path); - runtime.bundle.path = anchored(&anchor, &runtime.bundle.path); - if let Some(audit_path) = runtime.audit.path.take() { - // An empty path would anchor to the runtime directory itself. - if audit_path.as_os_str().is_empty() { - return Err(RenderProblem::new( - ProblemKind::RuntimeInvalid, - AuditDestinationError::MissingPath.to_string(), - )); - } - runtime.audit.path = Some(anchored(&anchor, &audit_path)); + if !(1..=MAX_SHUTDOWN_GRACE_SECONDS).contains(&runtime.listener.shutdown_grace_seconds) { + return Err(invalid(format!( + "listener.shutdownGraceSeconds must be between 1 and {MAX_SHUTDOWN_GRACE_SECONDS}, found {}", + runtime.listener.shutdown_grace_seconds + ))); } - runtime.audit.destination()?; - let id = crate::hash::sha256_hex(expanded.as_bytes()); - Ok((runtime, id)) + Ok((runtime, loaded.effective_digest)) } -/// The directory runtime-relative values anchor to: the runtime file's own -/// directory, canonicalized when possible. A bare filename anchors to the -/// current directory (its parent is empty). -fn runtime_anchor(runtime_path: &Path) -> PathBuf { - let parent = runtime_path - .parent() - .filter(|p| !p.as_os_str().is_empty()) - .map(Path::to_path_buf) - .unwrap_or_else(|| std::env::current_dir().unwrap_or_else(|_| PathBuf::from("/"))); - std::fs::canonicalize(&parent).unwrap_or(parent) +/// Verify the shared package envelope, apply the optional digest pin, then +/// bind the exact captured product bytes that rendering will consume. +pub fn load_package(runtime: &RenderRuntime) -> Result { + let verified = runtime + .package + .verify_package(&package_limits(), "registry-render package") + .map_err(package_problem)?; + crate::Bundle::load_package(&runtime.package.root, &verified) } -/// Join a configured path onto the anchor when relative, collapsing `.` and -/// `..` lexically so anchored paths stay readable in problems and logs. -/// Absolute paths pass through verbatim. -fn anchored(anchor: &Path, field: &Path) -> PathBuf { - if field.is_absolute() { - return field.to_path_buf(); - } - let mut out = PathBuf::new(); - if anchor.is_absolute() { - out.push(std::path::Component::RootDir.as_os_str()); - } - let mut parts: Vec = Vec::new(); - for component in anchor.join(field).components() { - match component { - std::path::Component::CurDir | std::path::Component::RootDir => {} - std::path::Component::ParentDir => { - parts.pop(); - } - std::path::Component::Prefix(prefix) => parts.push(prefix.as_os_str().to_owned()), - std::path::Component::Normal(segment) => parts.push(segment.to_owned()), - } - } - for part in parts { - out.push(part); - } - out +/// Product bounds for authored and deployed Render packages. +pub fn package_limits() -> registry_platform_config::package::PackageLimits { + registry_platform_config::package::PackageLimits::default() } -/// Resolve a `secret:…` reference relative to the runtime file's directory, -/// the same provider pair Relay allows (environment and runtime-rooted files). -pub fn resolve_secret(runtime_path: &Path, reference: &str) -> Result, RenderProblem> { - use registry_platform_config::{SecretProvider, SecretResolver}; - let root = runtime_anchor(runtime_path); - let resolver = - SecretResolver::new([SecretProvider::Environment, SecretProvider::File], root) - .map_err(|err| RenderProblem::new(ProblemKind::RuntimeInvalid, format!("{err}")))?; +pub(crate) fn package_problem( + error: registry_platform_config::package::PackageError, +) -> RenderProblem { + use registry_platform_config::package::PackageErrorKind; + + let kind = match error.kind() { + PackageErrorKind::SumFileMissing => ProblemKind::BundleUnsealed, + PackageErrorKind::DigestMismatch(_) => ProblemKind::RuntimeInvalid, + _ => ProblemKind::BundleTampered, + }; + RenderProblem::new(kind, error.to_string()) +} + +/// Resolve a secret reference under the providers the runtime declares. +pub fn resolve_secret(runtime: &RenderRuntime, reference: &str) -> Result, RenderProblem> { + let resolver = runtime + .secret_providers + .resolver() + .map_err(|err| invalid(format!("secretProviders: {err}")))?; let secret = resolver .resolve(reference) - .map_err(|err| RenderProblem::new(ProblemKind::RuntimeInvalid, format!("{err}")))?; + .map_err(|err| invalid(format!("{err}")))?; Ok(secret.expose_secret().to_vec()) } @@ -333,120 +299,220 @@ mod tests { use super::*; use std::net::SocketAddr; - fn runtime_yaml(body: &str) -> RenderRuntime { - let text = - format!("apiVersion: render.registrystack.org/v1alpha1\nkind: RenderRuntime\n{body}"); - serde_norway::from_str(&text).expect("runtime parses") + /// A canonical temporary directory: the loader refuses a path through a + /// symbolic link, and the system temporary directory is one on some + /// hosts. + fn home() -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().expect("deployment home"); + let path = dir.path().canonicalize().expect("canonical home"); + (dir, path) } - #[test] - fn bind_defaults_to_loopback() { - let runtime = runtime_yaml( - "bundle:\n path: /b\nauth:\n apiKeyRef: secret:file/k\naudit:\n path: /a/render.jsonl\n", - ); - assert_eq!(runtime.server.bind, "127.0.0.1:8080"); - } + const HEAD: &str = "apiVersion: registry.registrystack.org/render-runtime/v1alpha1\nkind: RenderRuntimeConfig\n"; - #[test] - fn loopback_private_and_link_local_binds_are_allowed() { - for bind in [ - "127.0.0.1:8080", - "10.0.0.5:8080", - "192.168.1.10:8080", - "172.16.0.1:8080", - "169.254.7.7:8080", - "[::1]:8080", - "[fe80::1]:8080", - "[fd00::5]:8080", - ] { - let addr: SocketAddr = bind.parse().unwrap(); - validate_bind(addr).unwrap_or_else(|err| panic!("{bind} must be allowed: {err}")); - } + fn minimal(home: &Path) -> String { + format!( + "{HEAD}listener:\n bind: 127.0.0.1:8080\npackage:\n root: /srv/render/package\nsecretProviders:\n file:\n root: {secrets}\nauth:\n apiKeyRef: secret:file/api.key\naudit:\n path: /var/lib/render/audit/render.jsonl\n", + secrets = home.join("secrets").display(), + ) } - /// `load` on a runtime file written under a temporary directory, so the - /// checks that only `load` performs are exercised. - fn load_yaml(body: &str) -> Result { - load_yaml_with_audit(" path: /a/render.jsonl\n", body) + fn load_text(home: &Path, text: &str) -> Result { + let file = home.join("runtime.yaml"); + std::fs::write(&file, text).expect("runtime file"); + load(&file).map(|(runtime, _)| runtime) } - fn load_yaml_with_audit(audit: &str, body: &str) -> Result { - let home = tempfile::tempdir().expect("deployment home"); - let file = home.path().join("runtime.yaml"); - std::fs::write( - &file, - format!( - "apiVersion: render.registrystack.org/v1alpha1\nkind: RenderRuntime\nbundle:\n path: /b\nauth:\n apiKeyRef: secret:file/api.key\naudit:\n{audit}{body}" + #[test] + fn the_runtime_configuration_is_read_through_the_shared_loader() { + let (_dir, home) = home(); + let file = home.join("runtime.yaml"); + std::fs::write(&file, minimal(&home)).expect("runtime file"); + let (runtime, digest) = load(&file).expect("the minimal runtime loads"); + assert_eq!(runtime.listener.bind.socket_addr().port(), 8080); + assert_eq!(runtime.package.root, PathBuf::from("/srv/render/package")); + assert!( + registry_platform_config::is_sha256_label(&digest), + "{digest}" + ); + + let error = load_text( + &home, + &minimal(&home).replace( + "apiVersion: registry.registrystack.org/render-runtime/v1alpha1", + "apiVersion: render.registrystack.org/v1alpha1", ), ) - .expect("runtime file"); - load(&file).map(|(runtime, _)| runtime) + .expect_err("the retired envelope is refused"); + assert!( + error.detail.contains( + "apiVersion must be exactly registry.registrystack.org/render-runtime/v1alpha1" + ), + "{}", + error.detail + ); } #[test] - fn shutdown_grace_outside_the_supported_range_is_refused() { - // A zero grace drops renders in flight on SIGTERM, and a grace past - // the hour overflows the bounded wait the shutdown path computes. - for grace in ["0", "3601", "10000000000000000000"] { - let err = load_yaml(&format!("server:\n shutdownGraceSeconds: {grace}\n")) - .expect_err("out-of-range grace must be refused"); - assert_eq!(err.kind, ProblemKind::RuntimeInvalid, "grace {grace}"); + fn the_listener_bind_is_required() { + let (_dir, home) = home(); + let text = minimal(&home).replace("listener:\n bind: 127.0.0.1:8080\n", ""); + let error = load_text(&home, &text).expect_err("no listener"); + assert_eq!(error.kind, ProblemKind::RuntimeInvalid); + assert!(error.detail.contains("listener"), "{}", error.detail); + } + + #[test] + fn removed_keys_name_their_replacements() { + let (_dir, home) = home(); + for (text, key, replacement) in [ + ( + minimal(&home).replace("listener:\n bind:", "server:\n bind:"), + "server", + "listener.bind", + ), + ( + minimal(&home).replace("package:\n root:", "bundle:\n path:"), + "bundle", + "package.root", + ), + ( + minimal(&home).replace(" path: /var/lib/render/audit/render.jsonl\n", " directory: /var/lib/render/audit\n"), + "audit.directory", + "audit.path", + ), + ( + minimal(&home).replace( + " path: /var/lib/render/audit/render.jsonl\n", + " path: /var/lib/render/audit/render.jsonl\n integrityKeyRef: secret:file/audit.key\n", + ), + "audit.integrityKeyRef", + "not hash-chained", + ), + ( + minimal(&home).replace(" path: /var/lib/render/audit/render.jsonl\n", " path: /var/lib/render/audit/render.jsonl\n maxSegmentBytes: 67108864\n"), + "audit.maxSegmentBytes", + "audit.rotateBytes", + ), + ] { + let error = load_text(&home, &text).expect_err(key); assert!( - err.detail.contains("shutdownGraceSeconds"), - "grace {grace}: {}", - err.detail + error + .detail + .contains(&format!("{key} is no longer accepted")) + && error.detail.contains(replacement), + "{key}: {}", + error.detail ); } } #[test] - fn shutdown_grace_at_the_range_ends_is_accepted() { - for grace in [1, 3600] { - let runtime = load_yaml(&format!("server:\n shutdownGraceSeconds: {grace}\n")) - .expect("grace at the range end loads"); - assert_eq!(runtime.server.shutdown_grace_seconds, grace); + fn package_and_audit_paths_must_be_absolute() { + let (_dir, home) = home(); + for (text, field) in [ + ( + minimal(&home).replace("root: /srv/render/package", "root: ../package"), + "package.root", + ), + ( + minimal(&home).replace( + "path: /var/lib/render/audit/render.jsonl", + "path: audit/render.jsonl", + ), + "audit.path", + ), + ] { + let error = load_text(&home, &text).expect_err(field); + assert!(error.detail.contains(field), "{field}: {}", error.detail); } } #[test] - fn relative_bundle_and_audit_paths_anchor_to_the_runtime_file() { - let home = tempfile::tempdir().expect("deployment home"); - let deploy = home.path().join("deploy"); - std::fs::create_dir_all(&deploy).expect("deploy directory"); - let file = deploy.join("runtime.yaml"); - std::fs::write( - &file, - "apiVersion: render.registrystack.org/v1alpha1\nkind: RenderRuntime\nbundle:\n path: ../bundle\nauth:\n apiKeyRef: secret:file/api.key\naudit:\n path: ../audit/render.jsonl\n", - ) - .expect("runtime file"); - let (runtime, _) = load(&file).expect("runtime loads"); - let home = std::fs::canonicalize(home.path()).expect("canonical home"); - assert_eq!(runtime.bundle.path, home.join("bundle")); - assert_eq!(runtime.audit.path, Some(home.join("audit/render.jsonl"))); + fn a_secret_reference_must_name_a_declared_provider() { + let (_dir, home) = home(); + let text = minimal(&home).replace( + "apiKeyRef: secret:file/api.key", + "apiKeyRef: secret:env/RENDER_API_KEY", + ); + let error = load_text(&home, &text).expect_err("environment provider not declared"); + assert!(error.detail.contains("auth.apiKeyRef"), "{}", error.detail); + assert!( + error.detail.contains("secretProviders.environment"), + "{}", + error.detail + ); + + let text = text.replace( + "secretProviders:\n", + "secretProviders:\n environment: {}\n", + ); + load_text(&home, &text).expect("the declared environment provider admits the reference"); + + let text = minimal(&home).replace( + &format!( + "secretProviders:\n file:\n root: {}\n", + home.join("secrets").display() + ), + "secretProviders: {}\n", + ); + let error = load_text(&home, &text).expect_err("no provider declared"); + assert!(error.detail.contains("secretProviders"), "{}", error.detail); } #[test] - fn absolute_bundle_and_audit_paths_are_kept_verbatim() { - let home = tempfile::tempdir().expect("deployment home"); - let file = home.path().join("runtime.yaml"); + fn environment_expressions_substitute_values_but_never_secret_references() { + let (_dir, home) = home(); + let file = home.join("runtime.yaml"); std::fs::write( &file, - "apiVersion: render.registrystack.org/v1alpha1\nkind: RenderRuntime\nbundle:\n path: /srv/render/bundle\nauth:\n apiKeyRef: secret:file/api.key\naudit:\n path: /var/lib/render/audit/render.jsonl\n", + minimal(&home).replace( + "root: /srv/render/package", + "root: ${RENDER_PACKAGE_ROOT:-/srv/render/default}", + ), ) .expect("runtime file"); - let (runtime, _) = load(&file).expect("runtime loads"); - assert_eq!(runtime.bundle.path, PathBuf::from("/srv/render/bundle")); + let (runtime, _) = load(&file).expect("a defaulted expression loads"); + assert_eq!(runtime.package.root, PathBuf::from("/srv/render/default")); + + let text = minimal(&home).replace( + "apiKeyRef: secret:file/api.key", + "apiKeyRef: ${RENDER_API_KEY_REF:-secret:file/api.key}", + ); + let error = load_text(&home, &text).expect_err("an expression in a *Ref field is refused"); + assert!(error.detail.contains("auth.apiKeyRef"), "{}", error.detail); + } + + #[test] + fn secrets_resolve_under_the_declared_file_root_only() { + let (_dir, home) = home(); + let secrets = home.join("secrets"); + std::fs::create_dir_all(&secrets).expect("secret root"); + write_secret(&secrets.join("api.key"), b"from-the-declared-root"); + // A file beside the runtime file is not a secret root. + write_secret(&home.join("audit.key"), b"beside-the-runtime-file"); + let runtime = load_text(&home, &minimal(&home)).expect("runtime loads"); assert_eq!( - runtime.audit.path, - Some(PathBuf::from("/var/lib/render/audit/render.jsonl")) + resolve_secret(&runtime, &runtime.auth.api_key_ref).expect("declared root"), + b"from-the-declared-root" ); + resolve_secret(&runtime, "secret:file/audit.key") + .expect_err("a file outside the declared root does not resolve"); + } + + fn with_audit(home: &Path, audit: &str) -> String { + minimal(home).replace(" path: /var/lib/render/audit/render.jsonl\n", audit) } #[test] fn the_audit_block_takes_the_shared_destination_shape() { - let file = load_yaml_with_audit( - " path: /a/render.jsonl\n rotateBytes: 1048576\n retainDays: 30\n", - "", + let (_dir, home) = home(); + let file = load_text( + &home, + &with_audit( + &home, + " path: /a/render.jsonl\n rotateBytes: 1048576\n retainDays: 30\n", + ), ) .expect("a file destination with rotation and retention loads"); match file.audit.destination().expect("destination") { @@ -459,7 +525,7 @@ mod tests { panic!("file is the default destination") } } - let stdout = load_yaml_with_audit(" destination: stdout\n", "") + let stdout = load_text(&home, &with_audit(&home, " destination: stdout\n")) .expect("a stdout destination loads"); assert_eq!( stdout.audit.destination().expect("destination"), @@ -469,9 +535,10 @@ mod tests { #[test] fn an_audit_block_outside_the_shared_shape_is_refused() { + let (_dir, home) = home(); for (audit, expected) in [ (" destination: file\n", "audit.path is required"), - (" path: \"\"\n", "audit.path is required"), + (" path: \"\"\n", "audit.path"), ( " destination: stdout\n path: /a/render.jsonl\n", "audit.path applies only", @@ -484,18 +551,71 @@ mod tests { " path: /a/render.jsonl\n retainDays: 0\n", "audit.retainDays must be between", ), - (" directory: /a\n", "unknown field `directory`"), - ( - " path: /a/render.jsonl\n integrityKeyRef: secret:file/audit.key\n", - "unknown field `integrityKeyRef`", - ), + (" path: /a/../render.jsonl\n", "audit.path"), ] { - let err = load_yaml_with_audit(audit, "").expect_err(audit); + let err = load_text(&home, &with_audit(&home, audit)).expect_err(audit); assert_eq!(err.kind, ProblemKind::RuntimeInvalid, "{audit}"); assert!(err.detail.contains(expected), "{audit}: {}", err.detail); } } + fn write_secret(path: &Path, bytes: &[u8]) { + use std::os::unix::fs::PermissionsExt as _; + std::fs::write(path, bytes).expect("secret file"); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .expect("secret mode"); + } + + #[test] + fn loopback_private_and_link_local_binds_are_allowed() { + for bind in [ + "127.0.0.1:8080", + "10.0.0.5:8080", + "192.168.1.10:8080", + "172.16.0.1:8080", + "169.254.7.7:8080", + "[::1]:8080", + "[fe80::1]:8080", + "[fd00::5]:8080", + ] { + let addr: SocketAddr = bind.parse().unwrap(); + validate_bind(addr).unwrap_or_else(|err| panic!("{bind} must be allowed: {err}")); + } + } + + #[test] + fn shutdown_grace_outside_the_supported_range_is_refused() { + // A zero grace drops renders in flight on SIGTERM, and a grace past + // the hour overflows the bounded wait the shutdown path computes. + let (_dir, home) = home(); + for grace in ["0", "3601", "10000000000000000000"] { + let text = minimal(&home).replace( + " bind: 127.0.0.1:8080\n", + &format!(" bind: 127.0.0.1:8080\n shutdownGraceSeconds: {grace}\n"), + ); + let err = load_text(&home, &text).expect_err("out-of-range grace must be refused"); + assert_eq!(err.kind, ProblemKind::RuntimeInvalid, "grace {grace}"); + assert!( + err.detail.contains("shutdownGraceSeconds"), + "grace {grace}: {}", + err.detail + ); + } + } + + #[test] + fn shutdown_grace_at_the_range_ends_is_accepted() { + let (_dir, home) = home(); + for grace in [1, 3600] { + let text = minimal(&home).replace( + " bind: 127.0.0.1:8080\n", + &format!(" bind: 127.0.0.1:8080\n shutdownGraceSeconds: {grace}\n"), + ); + let runtime = load_text(&home, &text).expect("grace at the range end loads"); + assert_eq!(runtime.listener.shutdown_grace_seconds, grace); + } + } + #[test] fn public_and_unspecified_binds_are_refused() { for bind in [ @@ -508,6 +628,11 @@ mod tests { let addr: SocketAddr = bind.parse().unwrap(); let problem = validate_bind(addr).expect_err(bind); assert_eq!(problem.kind, ProblemKind::RuntimeInvalid, "{bind}"); + assert!( + problem.detail.contains("listener.bind"), + "{}", + problem.detail + ); } } } diff --git a/crates/registry-render/src/server.rs b/crates/registry-render/src/server.rs index ca088b1ddf..cf5fee5c4c 100644 --- a/crates/registry-render/src/server.rs +++ b/crates/registry-render/src/server.rs @@ -55,22 +55,19 @@ struct RenderHttpRequest { assets: Option>, } -pub fn serve(runtime_path: Option<&Path>) -> Result { - let runtime_path = runtime_path - .map(Path::to_path_buf) - .unwrap_or_else(|| PathBuf::from("/etc/registry-render/runtime.yaml")); +pub fn serve(runtime_path: &Path) -> Result { let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() .map_err(|err| RenderProblem::new(ProblemKind::Internal, format!("runtime: {err}")))?; - runtime.block_on(serve_async(&runtime_path)) + runtime.block_on(serve_async(runtime_path)) } async fn serve_async(runtime_path: &Path) -> Result { init_tracing(); let (runtime, config_id) = runtime::load(runtime_path)?; let api_key = normalize_api_key(runtime::resolve_secret( - runtime_path, + &runtime, &runtime.auth.api_key_ref, )?)?; if api_key.len() < MIN_API_KEY_ENTROPY_BYTES { @@ -88,17 +85,9 @@ async fn serve_async(runtime_path: &Path) -> Result { // The address is settled before the steps with side effects (opening // the audit destination creates its directory), so a refused bind // leaves nothing half-made behind. - let bind: SocketAddr = runtime.server.bind.parse().map_err(|err| { - RenderProblem::new( - ProblemKind::RuntimeInvalid, - format!( - "server.bind {:?} is not an address: {err}", - runtime.server.bind - ), - ) - })?; + let bind: SocketAddr = runtime.listener.bind.socket_addr(); runtime::validate_bind(bind)?; - let bundle = Bundle::load_sealed(&runtime.bundle.path)?; + let bundle = runtime::load_package(&runtime)?; crate::check::check_script_coverage(&bundle)?; crate::check::check_label_key_sets(&bundle)?; let audit = RenderAudit::open(runtime.audit.destination()?).await?; @@ -110,7 +99,7 @@ async fn serve_async(runtime_path: &Path) -> Result { audit, api_key, limits: runtime.limits.clone(), - bundle_path: runtime.bundle.path.clone(), + bundle_path: runtime.package.root.clone(), concurrency: Arc::new(tokio::sync::Semaphore::new(runtime.limits.max_concurrency)), max_concurrency: runtime.limits.max_concurrency, }); @@ -124,7 +113,7 @@ async fn serve_async(runtime_path: &Path) -> Result { RenderProblem::new(ProblemKind::RuntimeInvalid, format!("bind {bind}: {err}")) })?; let app = router(Arc::clone(&service)).layer(axum::middleware::from_fn(lifecycle_log)); - let grace = Duration::from_secs(runtime.server.shutdown_grace_seconds); + let grace = Duration::from_secs(runtime.listener.shutdown_grace_seconds); let drain_service = Arc::clone(&service); // The stop signal is observed once and broadcast, so the drain and the // hard bound below race the same event. @@ -765,7 +754,7 @@ fn prepare_render( assets: request.assets.unwrap_or_default(), issued_at, strict: false, - require_sealed: true, + require_package: true, max_output_bytes: service.limits.max_output_bytes, memory_limit_bytes: 512 * 1024 * 1024, }; @@ -788,13 +777,13 @@ async fn run_render( let result = worker::supervise(worker_request, timeout).await; drop(permit); // The worker already rendered from one verified immutable snapshot. Its - // manifest identity must also be the bundle serve started with, not merely - // a different valid sealed bundle installed during the request. + // package digest must also be the one serve started with, not merely + // a different valid package installed during the request. match result { Ok(rendered) if rendered.bundle_hash != service.bundle.bundle_hash => { Err(RenderProblem::new( ProblemKind::BundleTampered, - "the bundle changed under serve; refusing the render from a drifted bundle", + "the package changed under serve; refusing the render from a drifted package", )) } other => other, @@ -876,12 +865,9 @@ fn insert_header(map: &mut header::HeaderMap, name: &str, value: &str) { } /// `registry-render healthcheck`: one plain HTTP GET against /health. -pub fn healthcheck(runtime_path: Option<&Path>) -> Result { - let runtime_path = runtime_path - .map(Path::to_path_buf) - .unwrap_or_else(|| PathBuf::from("/etc/registry-render/runtime.yaml")); - let (runtime, _) = runtime::load(&runtime_path)?; - let address = runtime.server.bind.clone(); +pub fn healthcheck(runtime_path: &Path) -> Result { + let (runtime, _) = runtime::load(runtime_path)?; + let address = runtime.listener.bind.socket_addr().to_string(); let stream = std::net::TcpStream::connect(&address).map_err(|err| { RenderProblem::new( ProblemKind::RuntimeInvalid, @@ -983,13 +969,20 @@ mod tests { } } - /// A service over the sealed receipt bundle with `permits` render slots. + /// A service over the packaged receipt bundle with `permits` render slots. /// Zero slots means a render can never start: any request that reaches /// the render step waits there forever. fn service(lines: &AuditLines, permits: usize) -> Arc { let bundle_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../products/render/bundles/receipt"); - let bundle = Bundle::load_sealed(&bundle_path).expect("sealed receipt bundle"); + let verified = registry_platform_config::package::verify_package( + &bundle_path, + &runtime::package_limits(), + "registry-render package", + ) + .expect("packaged receipt bundle"); + let bundle = + Bundle::load_package(&bundle_path, &verified).expect("packaged receipt bundle"); Arc::new(Service { bundle, audit: RenderAudit::new(AuditWriter::from_line_sink(Box::new(lines.clone()))), diff --git a/crates/registry-render/src/worker.rs b/crates/registry-render/src/worker.rs index f4789d7bd5..080cb17e7e 100644 --- a/crates/registry-render/src/worker.rs +++ b/crates/registry-render/src/worker.rs @@ -33,11 +33,10 @@ pub struct WorkerRequest { pub issued_at: String, #[serde(default)] pub strict: bool, - /// Serve mode pins the seal per request, not just at startup: the - /// worker must load sealed and the response must carry the startup - /// bundle hash. + /// Serve mode pins the package per request, not just at startup: the + /// worker must verify it and the response must carry the package digest. #[serde(default)] - pub require_sealed: bool, + pub require_package: bool, #[serde(default = "default_max_output")] pub max_output_bytes: usize, #[serde(default = "default_memory_limit")] @@ -218,7 +217,7 @@ fn cap_address_space(limit: u64) { } /// Bundle-load problems format host paths (the manifest it cannot read, a -/// symlink the seal walk refuses), and in serve mode they cross the pipe to +/// symlink the package walk refuses), and in serve mode they cross the pipe to /// the caller. The bundle root is replaced with `` the way render /// diagnostics already are, in both its given and its canonical spelling. fn redact_bundle_root(mut problem: RenderProblem, root: &Path) -> RenderProblem { @@ -244,14 +243,27 @@ fn redact_bundle_root(mut problem: RenderProblem, root: &Path) -> RenderProblem } fn render_in_worker(request: &WorkerRequest) -> WorkerResponse { - // Serve pins the seal per request: the worker loads one immutable bundle - // snapshot every time, verifies the seal over those exact bytes, and - // renders only from that snapshot. Drift present before capture is - // refused here; later path changes cannot affect this render. - let bundle = if request.require_sealed { - crate::bundle::Bundle::load_sealed(&request.bundle) + // Serve pins the package per request: the worker verifies the common + // envelope, captures one immutable product snapshot, binds those consumed + // bytes to the verified digests, and renders only from that snapshot. + let bundle = if request.require_package { + registry_platform_config::package::verify_package( + &request.bundle, + &crate::runtime::package_limits(), + "registry-render package", + ) + .map_err(|error| { + let kind = match error.kind() { + registry_platform_config::package::PackageErrorKind::SumFileMissing => { + ProblemKind::BundleUnsealed + } + _ => ProblemKind::BundleTampered, + }; + RenderProblem::new(kind, error.to_string()) + }) + .and_then(|verified| crate::bundle::Bundle::load_package(&request.bundle, &verified)) } else { - crate::bundle::Bundle::load(&request.bundle) + crate::bundle::Bundle::load_for_preview(&request.bundle) }; let bundle = match bundle { Ok(bundle) => bundle, diff --git a/crates/registry-render/src/world.rs b/crates/registry-render/src/world.rs index d4e925f50a..5db53ee925 100644 --- a/crates/registry-render/src/world.rs +++ b/crates/registry-render/src/world.rs @@ -113,7 +113,7 @@ impl RenderWorld { } /// Normalize a Typst virtual path into the slash-separated spelling used - /// by the sealed snapshot. No resolution ever consults the filesystem. + /// by the verified package snapshot. No resolution ever consults the filesystem. fn normalize_virtual_path(vpath: &str) -> FileResult { let mut clean = PathBuf::new(); for component in Path::new(vpath).components() { diff --git a/crates/registry-render/tests/golden.rs b/crates/registry-render/tests/golden.rs index ec67f9b26f..b654eaa3cf 100644 --- a/crates/registry-render/tests/golden.rs +++ b/crates/registry-render/tests/golden.rs @@ -69,8 +69,18 @@ fn issued_at() -> chrono::DateTime { Utc.with_ymd_and_hms(2026, 9, 16, 10, 32, 0).unwrap() } +fn load_package(root: &Path) -> registry_render::Bundle { + let verified = registry_platform_config::package::verify_package( + root, + ®istry_render::runtime::package_limits(), + "registry-render package", + ) + .expect("shared package envelope"); + registry_render::Bundle::load_package(root, &verified).expect("bound package bytes") +} + fn render_case(case: &Case) -> registry_render::Rendered { - let bundle = registry_render::Bundle::load_sealed(&case.bundle).expect("sealed bundle"); + let bundle = load_package(&case.bundle); let document = bundle.document(case.document).expect("document"); let data: Value = serde_json::from_str( &std::fs::read_to_string(case.bundle.join("fixtures/data.json")).expect("fixture"), @@ -113,6 +123,12 @@ fn golden_hashes_match() { "golden envelope hash drifted for {}", case.name ); + assert_eq!( + rendered.bundle_hash, + pinned["bundleHash"].as_str().expect("pinned package hash"), + "golden package hash drifted for {}", + case.name + ); assert!( rendered.warnings.is_empty(), "{} rendered with warnings: {:?}", @@ -120,7 +136,7 @@ fn golden_hashes_match() { rendered.warnings ); // Closure drift gate: the pinned file closure must match exactly, - // and every file it names must be governed by the manifest's seal + // and every file it names must be governed by the package envelope // (bundle files by their virtual path, package files by their // packages/… spelling). let pinned_deps: Option> = pinned["deps"].as_array().map(|deps| { @@ -135,7 +151,7 @@ fn golden_hashes_match() { different file set than the reviewed one", case.name ); - let governed = manifest_hashes(&case.bundle); + let governed = package_files(&case.bundle); for dep in &rendered.deps { // Virtual request assets are scoped to the request and covered // by dataSha256, not by the bundle's manifest. @@ -156,8 +172,8 @@ fn golden_hashes_match() { None => dep.clone(), }; assert!( - governed.contains_key(&governed_name), - "{} closure reads {}, which the manifest does not govern", + governed.contains(&governed_name), + "{} closure reads {}, which the package does not govern", case.name, governed_name ); @@ -165,11 +181,17 @@ fn golden_hashes_match() { } } -/// The sealed manifest's per-file hash map, parsed by the crate's own model. -fn manifest_hashes(bundle: &Path) -> std::collections::BTreeMap { - let bytes = std::fs::read(bundle.join("manifest.yaml")).expect("manifest bytes"); - let manifest = registry_render::manifest::Manifest::parse(&bytes).expect("manifest parses"); - manifest.hashes.expect("example bundles are sealed") +/// The paths governed by the shared package envelope. +fn package_files(bundle: &Path) -> std::collections::BTreeSet { + registry_platform_config::package::verify_package( + bundle, + ®istry_render::runtime::package_limits(), + "registry-render package", + ) + .expect("example package verifies") + .files() + .map(str::to_owned) + .collect() } #[test] @@ -241,7 +263,7 @@ fn pdf_bytes_carry_no_renderer_version() { #[test] fn issued_at_changes_bytes_and_is_the_only_knob() { let case = cases().into_iter().find(|c| c.name == "receipt").unwrap(); - let bundle = registry_render::Bundle::load_sealed(&case.bundle).unwrap(); + let bundle = load_package(&case.bundle); let document = bundle.document("receipt").unwrap().clone(); let data: Value = serde_json::from_str( &std::fs::read_to_string(case.bundle.join("fixtures/data.json")).unwrap(), @@ -262,37 +284,30 @@ fn issued_at_changes_bytes_and_is_the_only_knob() { } #[test] -fn tampered_sealed_bundle_is_refused() { +fn tampered_package_is_refused() { let case = cases().into_iter().find(|c| c.name == "receipt").unwrap(); let (_copy, copy) = physical_tempdir(); copy_bundle(&case.bundle, ©); - // Tamper with a governed file after sealing. + // Tamper with a governed file after packaging. let labels = copy.join("labels/ar.yaml"); let mut text = std::fs::read_to_string(&labels).unwrap(); text.push_str("extra: tampered\n"); std::fs::write(&labels, text).unwrap(); - let problem = - registry_render::Bundle::load_sealed(©).expect_err("tampered bundle must be refused"); - assert_eq!( - problem.kind, - registry_render::ProblemKind::BundleTampered, - "{problem}" - ); - assert!( - problem.locations.iter().any(|l| l.contains("ar.yaml")), - "the problem names the drifted file: {problem}" - ); + let problem = registry_platform_config::package::verify_package( + ©, + ®istry_render::runtime::package_limits(), + "registry-render package", + ) + .expect_err("tampered package must be refused"); + assert!(problem.to_string().contains("labels/ar.yaml"), "{problem}"); } #[test] -fn sealed_template_and_package_bytes_are_bound_to_the_loaded_snapshot() { - let (_dir, bundle_dir) = physical_tempdir(); - for sub in [ - "templates", - "fonts", - "labels", - "packages/preview/notice/0.1.0/src", - ] { +fn verified_template_and_typst_package_bytes_are_bound_to_the_loaded_snapshot() { + let (_dir, root) = physical_tempdir(); + let bundle_dir = root.join("source"); + std::fs::create_dir(&bundle_dir).unwrap(); + for sub in ["templates", "packages/preview/notice/0.1.0/src"] { std::fs::create_dir_all(bundle_dir.join(sub)).unwrap(); } std::fs::write( @@ -301,10 +316,10 @@ fn sealed_template_and_package_bytes_are_bound_to_the_loaded_snapshot() { ) .unwrap(); let template = "#import \"@preview/notice:0.1.0\": message\n#message #read(\"value.txt\")\n"; - let package = "#let message = [sealed content]\n"; + let package = "#let message = [verified content]\n"; std::fs::write(bundle_dir.join("templates/notice.typ"), template).unwrap(); let file_path = bundle_dir.join("templates/value.txt"); - std::fs::write(&file_path, "sealed file content\n").unwrap(); + std::fs::write(&file_path, "verified file content\n").unwrap(); std::fs::write( bundle_dir.join("packages/preview/notice/0.1.0/typst.toml"), "[package]\nname = \"notice\"\nversion = \"0.1.0\"\nentrypoint = \"src/lib.typ\"\n", @@ -313,8 +328,17 @@ fn sealed_template_and_package_bytes_are_bound_to_the_loaded_snapshot() { let package_path = bundle_dir.join("packages/preview/notice/0.1.0/src/lib.typ"); std::fs::write(&package_path, package).unwrap(); - registry_render::Bundle::seal(&bundle_dir).expect("bundle seals"); - let bundle = registry_render::Bundle::load_sealed(&bundle_dir).expect("sealed bundle loads"); + registry_render::Bundle::load(&bundle_dir).expect("authored bundle"); + let package_dir = root.join("package"); + copy_bundle(&bundle_dir, &package_dir); + registry_platform_config::package::write_sum_file( + &package_dir, + None, + ®istry_render::runtime::package_limits(), + "registry-render package", + ) + .expect("package writes"); + let bundle = load_package(&package_dir); let document = bundle.document("notice").unwrap().clone(); let request = registry_render::RenderRequest { locale: None, @@ -324,11 +348,11 @@ fn sealed_template_and_package_bytes_are_bound_to_the_loaded_snapshot() { }; let baseline = registry_render::render(&bundle, &document, &request, false).unwrap(); - // After the seal has been verified, replace the exact template path + // After the package has been verified, replace the exact template path // before Typst asks the world for it. The already loaded bundle must // still render only its verified snapshot bytes. std::fs::write( - bundle_dir.join("templates/notice.typ"), + package_dir.join("templates/notice.typ"), "tampered template content\n", ) .unwrap(); @@ -339,8 +363,13 @@ fn sealed_template_and_package_bytes_are_bound_to_the_loaded_snapshot() { // Package source is resolved through a different Typst virtual root and // must be bound to the same immutable snapshot too. - std::fs::write(bundle_dir.join("templates/notice.typ"), template).unwrap(); - std::fs::write(&package_path, "#let message = [tampered package content]\n").unwrap(); + std::fs::write(package_dir.join("templates/notice.typ"), template).unwrap(); + let deployed_package_path = package_dir.join("packages/preview/notice/0.1.0/src/lib.typ"); + std::fs::write( + &deployed_package_path, + "#let message = [tampered package content]\n", + ) + .unwrap(); let after_package_replace = registry_render::render(&bundle, &document, &request, false).unwrap(); assert_eq!(after_package_replace.pdf, baseline.pdf); @@ -348,37 +377,40 @@ fn sealed_template_and_package_bytes_are_bound_to_the_loaded_snapshot() { // Non-source reads use `World::file`; those bytes must not be reopened // either. Restore the package path so this assertion isolates that path. - std::fs::write(&package_path, package).unwrap(); - std::fs::write(&file_path, "tampered file content\n").unwrap(); + std::fs::write(&deployed_package_path, package).unwrap(); + std::fs::write( + package_dir.join("templates/value.txt"), + "tampered file content\n", + ) + .unwrap(); let after_file_replace = registry_render::render(&bundle, &document, &request, false).unwrap(); assert_eq!(after_file_replace.pdf, baseline.pdf); assert_eq!(after_file_replace.bundle_hash, baseline.bundle_hash); } #[test] -fn unsealed_bundle_is_refused_for_serving() { +fn directory_without_sum_file_is_not_a_package_but_remains_authoring_source() { let case = cases() .into_iter() .find(|c| c.name == "certificate") .unwrap(); let (_copy, copy) = physical_tempdir(); copy_bundle(&case.bundle, ©); - let manifest_path = copy.join("manifest.yaml"); - let manifest = std::fs::read_to_string(&manifest_path).unwrap(); - // Strip the hashes block: everything from `hashes:` to EOF. - let stripped = manifest.split("hashes:").next().unwrap().to_owned(); - std::fs::write(&manifest_path, stripped).unwrap(); - let problem = registry_render::Bundle::load_sealed(©) - .expect_err("unsealed bundle must be refused for serve"); - assert_eq!(problem.kind, registry_render::ProblemKind::BundleUnsealed); - // But plain compile loading still works unsealed. - registry_render::Bundle::load(©).expect("compile accepts unsealed"); + std::fs::remove_file(copy.join("SHA256SUMS")).unwrap(); + let problem = registry_platform_config::package::verify_package( + ©, + ®istry_render::runtime::package_limits(), + "registry-render package", + ) + .expect_err("source directory is not a package"); + assert!(problem.to_string().contains("has no SHA256SUMS")); + registry_render::Bundle::load(©).expect("compile accepts raw authoring source"); } #[test] fn schema_violations_carry_json_pointers() { let case = cases().into_iter().find(|c| c.name == "receipt").unwrap(); - let bundle = registry_render::Bundle::load_sealed(&case.bundle).unwrap(); + let bundle = load_package(&case.bundle); let document = bundle.document("receipt").unwrap().clone(); let mut data: Value = serde_json::from_str( &std::fs::read_to_string(case.bundle.join("fixtures/data.json")).unwrap(), @@ -419,7 +451,7 @@ fn schema_violations_carry_json_pointers() { #[test] fn bad_locale_is_refused_with_a_pointer() { let case = cases().into_iter().find(|c| c.name == "card").unwrap(); - let bundle = registry_render::Bundle::load_sealed(&case.bundle).unwrap(); + let bundle = load_package(&case.bundle); let document = bundle.document("beneficiary-card").unwrap().clone(); let data: Value = serde_json::from_str( &std::fs::read_to_string(case.bundle.join("fixtures/data.json")).unwrap(), diff --git a/crates/registry-render/tests/scaffold.rs b/crates/registry-render/tests/scaffold.rs index 5ca87e80b7..6a79503a24 100644 --- a/crates/registry-render/tests/scaffold.rs +++ b/crates/registry-render/tests/scaffold.rs @@ -1,7 +1,6 @@ //! Scaffold and CLI regression tests: the first-hour path, exercised as a //! user drives it — `registry-render init`, first compile, validate without a -//! clock, the unsealed notice, edit-after-seal recovery, and verify-then- -//! seal ordering. +//! clock, package creation, repeatability, and verify-before-write ordering. use std::path::{Path, PathBuf}; use std::process::{Command, Output}; @@ -224,42 +223,111 @@ fn validate_is_a_dry_run_and_needs_no_clock() { } #[test] -fn compile_prints_the_unsealed_notice() { +fn retired_seal_command_names_the_package_replacement() { let dir = tempdir(); run(&["init", dir.to_str().unwrap()]); - let compiled = compile(&dir); - let stderr = String::from_utf8_lossy(&compiled.stderr); + let refused = run(&["seal", "--bundle", dir.to_str().unwrap()]); + assert_eq!( + refused.status.code(), + Some(registry_render::ProblemKind::InvalidArgument.exit_code()) + ); + let stderr = String::from_utf8_lossy(&refused.stderr); assert!( - stderr.contains("unsealed"), - "compile must say when a bundle is unsealed: {stderr}" + stderr.contains("registry-render package") && stderr.contains("--output"), + "retired command must name its replacement: {stderr}" ); } #[test] -fn edit_after_seal_names_the_recovery_path() { +fn package_command_writes_repeatable_verified_envelopes_and_revision() { let dir = tempdir(); run(&["init", dir.to_str().unwrap()]); - let sealed = run(&["seal", "--bundle", dir.to_str().unwrap()]); - assert!(sealed.status.success()); - // An intentional edit after sealing… - let template = dir.join("templates/letter.typ"); - let mut text = std::fs::read_to_string(&template).unwrap(); - text.push_str("// edited\n"); - std::fs::write(&template, text).unwrap(); - let compiled = compile(&dir); + let packages = tempdir(); + let first = packages.join("letter-package-a"); + let second = packages.join("letter-package-b"); + for output in [&first, &second] { + let packaged = run(&[ + "package", + "--bundle", + dir.to_str().unwrap(), + "--output", + output.to_str().unwrap(), + "--revision", + "revision-1", + ]); + assert!( + packaged.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&packaged.stdout), + String::from_utf8_lossy(&packaged.stderr) + ); + } assert_eq!( - compiled.status.code(), - Some(registry_render::ProblemKind::BundleTampered.exit_code()) + std::fs::read(first.join("SHA256SUMS")).unwrap(), + std::fs::read(second.join("SHA256SUMS")).unwrap(), + "the same authored bytes and revision produce the same package digest" ); - let stderr = String::from_utf8_lossy(&compiled.stderr); + assert_eq!( + std::fs::read_to_string(first.join("REVISION")).unwrap(), + "revision-1\n" + ); + let verified = registry_platform_config::package::verify_package( + &first, + ®istry_render::runtime::package_limits(), + "registry-render package", + ) + .expect("written package verifies"); + assert_eq!(verified.revision(), Some("revision-1")); + + let checked = run(&["check", "--bundle", first.to_str().unwrap()]); + assert!( + checked.status.success(), + "a deployment package remains inspectable: {}", + String::from_utf8_lossy(&checked.stderr) + ); + let validated = run(&[ + "validate", + "--bundle", + first.to_str().unwrap(), + "--type", + "letter", + "--data", + dir.join("fixtures/data.json").to_str().unwrap(), + ]); assert!( - stderr.contains("registry-render seal"), - "the drift message must point at recovery: {stderr}" + validated.status.success(), + "a deployment package remains validatable: {}", + String::from_utf8_lossy(&validated.stderr) ); - // …and recovery actually works. - let resealed = run(&["seal", "--bundle", dir.to_str().unwrap()]); - assert!(resealed.status.success()); - assert!(compile(&dir).status.success()); + let rendered = run(&[ + "compile", + "--bundle", + first.to_str().unwrap(), + "--type", + "letter", + "--data", + dir.join("fixtures/data.json").to_str().unwrap(), + "--issued-at", + "2026-01-01T00:00:00Z", + "--out", + packages.join("letter.pdf").to_str().unwrap(), + ]); + assert!( + rendered.status.success(), + "the CLI consumes the package it produced: {}", + String::from_utf8_lossy(&rendered.stderr) + ); + assert!(String::from_utf8_lossy( + &run(&[ + "package", + "--bundle", + dir.to_str().unwrap(), + "--output", + first.to_str().unwrap(), + ]) + .stderr + ) + .contains("new directory")); } #[test] @@ -440,27 +508,33 @@ fn check_names_a_locale_missing_a_label_key() { } #[test] -fn check_seal_refuses_to_seal_a_broken_bundle() { +fn package_refuses_a_broken_bundle_before_writing_output() { let dir = tempdir(); run(&["init", dir.to_str().unwrap()]); // Break script coverage: a label value in a script no font covers, // without adding any font to the bundle. std::fs::write(dir.join("labels/en.yaml"), "title: 你好\n").unwrap(); - let out = run(&["check", "--bundle", dir.to_str().unwrap(), "--seal"]); + let output = tempdir().join("broken-package"); + let out = run(&[ + "package", + "--bundle", + dir.to_str().unwrap(), + "--output", + output.to_str().unwrap(), + ]); assert_eq!( out.status.code(), Some(registry_render::ProblemKind::FontInvalid.exit_code()), - "verify must run before sealing" + "authoring validation must run before package output" ); - let manifest = std::fs::read_to_string(dir.join("manifest.yaml")).unwrap(); assert!( - !manifest.contains("hashes:"), - "a failed check must not leave a seal behind" + !output.exists(), + "a failed package leaves no output directory" ); } #[test] -fn bundle_with_symlink_cannot_be_sealed() { +fn bundle_with_symlink_cannot_be_packaged() { let dir = tempdir(); run(&["init", dir.to_str().unwrap()]); let outside = tempfile::tempdir().unwrap(); @@ -468,7 +542,14 @@ fn bundle_with_symlink_cannot_be_sealed() { #[cfg(unix)] std::os::unix::fs::symlink(outside.path().join("x.txt"), dir.join("templates/link.typ")) .unwrap(); - let out = run(&["seal", "--bundle", dir.to_str().unwrap()]); + let output = tempdir().join("symlink-package"); + let out = run(&[ + "package", + "--bundle", + dir.to_str().unwrap(), + "--output", + output.to_str().unwrap(), + ]); #[cfg(unix)] assert_eq!( out.status.code(), @@ -520,7 +601,7 @@ fn check_audit_under(runtime: &Path, bundle: &Path, root: &Path) -> Output { "check", "--bundle", bundle.to_str().unwrap(), - "--runtime", + "--runtime-config", runtime.to_str().unwrap(), "--require-audit-under", root.to_str().unwrap(), @@ -624,8 +705,9 @@ fn serve_deployment() -> (PathBuf, PathBuf, u16) { .unwrap() .port(); let runtime = format!( - "apiVersion: render.registrystack.org/v1alpha1\nkind: RenderRuntime\nserver:\n bind: 127.0.0.1:{port}\nbundle:\n path: {}\nauth:\n apiKeyRef: secret:file/api.key\nlimits:\n renderTimeoutSeconds: 20\naudit:\n path: {}\n", + "apiVersion: registry.registrystack.org/render-runtime/v1alpha1\nkind: RenderRuntimeConfig\nlistener:\n bind: 127.0.0.1:{port}\npackage:\n root: {}\nsecretProviders:\n file:\n root: {}\nauth:\n apiKeyRef: secret:file/api.key\nlimits:\n renderTimeoutSeconds: 20\naudit:\n path: {}\n", bundle.display(), + home.display(), home.join("audit/render.jsonl").display() ); let runtime_path = home.join("runtime.yaml"); @@ -647,7 +729,7 @@ impl Drop for Server { fn start_server(runtime_path: &Path) -> Server { let mut child = render_bin() - .args(["serve", "--runtime", runtime_path.to_str().unwrap()]) + .args(["serve", "--runtime-config", runtime_path.to_str().unwrap()]) .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() diff --git a/crates/registry-render/tests/serve.rs b/crates/registry-render/tests/serve.rs index 5ab10bf3ca..b4f9d6315e 100644 --- a/crates/registry-render/tests/serve.rs +++ b/crates/registry-render/tests/serve.rs @@ -72,7 +72,7 @@ fn copy_dir(from: &Path, to: &Path) { } } -/// Build a deployment home: sealed receipt bundle, key file, audit dir, +/// Build a deployment home: verified Render package, key file, audit dir, /// runtime.yaml. Returns (home, runtime_path, port). fn deployment(limits: &str, bundle_source: &Path) -> (PathBuf, PathBuf, u16) { let (home_guard, home) = physical_tempdir(); @@ -82,8 +82,9 @@ fn deployment(limits: &str, bundle_source: &Path) -> (PathBuf, PathBuf, u16) { std::fs::create_dir(home.join("audit")).unwrap(); let port = free_port(); let runtime = format!( - "apiVersion: render.registrystack.org/v1alpha1\nkind: RenderRuntime\nserver:\n bind: 127.0.0.1:{port}\n shutdownGraceSeconds: 5\nbundle:\n path: {}\nauth:\n apiKeyRef: secret:file/api.key\n{limits}audit:\n path: {}\n", + "apiVersion: registry.registrystack.org/render-runtime/v1alpha1\nkind: RenderRuntimeConfig\nlistener:\n bind: 127.0.0.1:{port}\n shutdownGraceSeconds: 5\npackage:\n root: {}\nsecretProviders:\n file:\n root: {}\nauth:\n apiKeyRef: secret:file/api.key\n{limits}audit:\n path: {}\n", bundle.display(), + home.display(), audit_file(&home).display() ); let runtime_path = home.join("runtime.yaml"); @@ -127,7 +128,7 @@ fn spawn_server(runtime_path: &Path, current_dir: Option<&Path>, stdout: Stdio) let mut command = Command::new(env!("CARGO_BIN_EXE_registry-render")); command .arg("serve") - .arg("--runtime") + .arg("--runtime-config") .arg(runtime_path) .stdout(stdout) .stderr(Stdio::null()); @@ -269,6 +270,18 @@ fn serve_health_and_ready() { DEFAULT_LIMITS, &repo_root().join("products/render/bundles/receipt"), ); + let digest = registry_platform_config::sha256_uri( + &std::fs::read(home.join("bundle/SHA256SUMS")).unwrap(), + ); + let text = std::fs::read_to_string(&runtime).unwrap(); + std::fs::write( + &runtime, + text.replace( + "package:\n", + &format!("package:\n expectedDigest: {digest}\n"), + ), + ) + .unwrap(); let server = start_server(&runtime); let health = request(server.port, "GET", "/health", &[], None); assert_eq!(health.status, 200); @@ -296,12 +309,11 @@ fn serve_health_and_ready() { } #[test] -fn relative_runtime_paths_anchor_to_the_runtime_files_directory() { +fn the_runtime_file_serves_the_same_from_any_working_directory() { // The natural deployment layout: runtime file and key files together in // deploy/, bundle and audit as siblings one level up. Every path in the - // runtime file is relative to it, so the file works from any working - // directory — before anchoring, a relative audit directory killed serve - // at startup and a relative bundle path silently depended on the CWD. + // runtime file is absolute and the key files resolve under the declared + // secretProviders.file.root, so the working directory never matters. let (_home_guard, home) = physical_tempdir(); let bundle = home.join("bundle"); copy_dir( @@ -316,7 +328,10 @@ fn relative_runtime_paths_anchor_to_the_runtime_files_directory() { std::fs::write( deploy.join("runtime.yaml"), format!( - "apiVersion: render.registrystack.org/v1alpha1\nkind: RenderRuntime\nserver:\n bind: 127.0.0.1:{port}\nbundle:\n path: ../bundle\nauth:\n apiKeyRef: secret:file/api.key\naudit:\n path: ../audit/render.jsonl\n" + "apiVersion: registry.registrystack.org/render-runtime/v1alpha1\nkind: RenderRuntimeConfig\nlistener:\n bind: 127.0.0.1:{port}\npackage:\n root: {bundle}\nsecretProviders:\n file:\n root: {deploy}\nauth:\n apiKeyRef: secret:file/api.key\naudit:\n path: {audit}\n", + bundle = bundle.display(), + deploy = deploy.display(), + audit = home.join("audit/render.jsonl").display(), ), ) .unwrap(); @@ -344,7 +359,7 @@ fn relative_runtime_paths_anchor_to_the_runtime_files_directory() { let lines = audit_lines(&home); assert!( lines.iter().any(|l| l.contains("\"outcome\":\"rendered\"")), - "the render was audited into the anchored audit file: {lines:?}" + "the render was audited into the configured audit file: {lines:?}" ); } @@ -389,7 +404,7 @@ fn api_key_with_stray_whitespace_is_refused_at_startup() { // startup error, not a silent permanent 401 with /health green. write_secret(&home.join("api.key"), &format!(" {API_KEY}")); let mut child = Command::new(env!("CARGO_BIN_EXE_registry-render")) - .args(["serve", "--runtime", runtime.to_str().unwrap()]) + .args(["serve", "--runtime-config", runtime.to_str().unwrap()]) .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() @@ -461,7 +476,7 @@ fn a_refused_bind_is_caught_before_startup_touches_the_filesystem() { ); std::fs::write(&runtime, text).unwrap(); let mut child = Command::new(env!("CARGO_BIN_EXE_registry-render")) - .args(["serve", "--runtime", runtime.to_str().unwrap()]) + .args(["serve", "--runtime-config", runtime.to_str().unwrap()]) .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() @@ -478,6 +493,50 @@ fn a_refused_bind_is_caught_before_startup_touches_the_filesystem() { ); } +#[test] +fn serve_startup_package_digest_mismatch_uses_common_expected_and_found_shape() { + let (home, runtime, _) = deployment( + DEFAULT_LIMITS, + &repo_root().join("products/render/bundles/receipt"), + ); + let audit = home.join("audit-elsewhere"); + let expected = format!("sha256:{}", "0".repeat(64)); + let found = registry_platform_config::sha256_uri( + &std::fs::read(home.join("bundle/SHA256SUMS")).unwrap(), + ); + let text = std::fs::read_to_string(&runtime).unwrap(); + let text = text + .replace( + "package:\n", + &format!("package:\n expectedDigest: {expected}\n"), + ) + .replace( + &format!("path: {}", audit_file(&home).display()), + &format!("path: {}", audit.join("render.jsonl").display()), + ); + std::fs::write(&runtime, text).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_registry-render")) + .args(["serve", "--runtime-config", runtime.to_str().unwrap()]) + .output() + .unwrap(); + assert_eq!( + output.status.code(), + Some(registry_render::ProblemKind::RuntimeInvalid.exit_code()), + "a pinned digest naming another bundle must refuse startup" + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains(&format!( + "package.expectedDigest is {expected} but the package at package.root is {found}" + )), + "{stderr}" + ); + assert!( + !audit.exists(), + "the refusal lands before the audit directory opens" + ); +} + fn wait_for_exit(child: &mut Child) -> i32 { let deadline = Instant::now() + Duration::from_secs(30); loop { @@ -888,26 +947,43 @@ fn audit_events_are_value_free() { } #[test] -fn tampered_bundle_refuses_to_serve() { - let (home, runtime, _) = deployment( - DEFAULT_LIMITS, - &repo_root().join("products/render/bundles/receipt"), - ); - let labels = home.join("bundle/labels/ar.yaml"); - let mut text = std::fs::read_to_string(&labels).unwrap(); - text.push_str("extra: tampered\n"); - std::fs::write(&labels, text).unwrap(); - let mut child = Command::new(env!("CARGO_BIN_EXE_registry-render")) - .args(["serve", "--runtime", runtime.to_str().unwrap()]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .unwrap(); - let code = wait_for_exit(&mut child); - assert_eq!( - code, - registry_render::ProblemKind::BundleTampered.exit_code() - ); +fn package_changed_missing_and_extra_files_refuse_startup_by_name() { + for (label, mutate, path) in [ + ("changed", "changed", "labels/ar.yaml"), + ("missing", "missing", "labels/ar.yaml"), + ("extra", "extra", "unexpected.txt"), + ] { + let (home, runtime, _) = deployment( + DEFAULT_LIMITS, + &repo_root().join("products/render/bundles/receipt"), + ); + match mutate { + "changed" => { + let file = home.join("bundle").join(path); + let mut text = std::fs::read_to_string(&file).unwrap(); + text.push_str("extra: tampered\n"); + std::fs::write(file, text).unwrap(); + } + "missing" => std::fs::remove_file(home.join("bundle").join(path)).unwrap(), + "extra" => std::fs::write(home.join("bundle").join(path), "extra").unwrap(), + _ => unreachable!(), + } + let output = Command::new(env!("CARGO_BIN_EXE_registry-render")) + .args(["serve", "--runtime-config", runtime.to_str().unwrap()]) + .output() + .unwrap(); + assert_eq!( + output.status.code(), + Some(registry_render::ProblemKind::BundleTampered.exit_code()), + "{label}: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains(label) && stderr.contains(path), + "{label}: {stderr}" + ); + } } #[test] @@ -923,7 +999,7 @@ fn bundle_drift_after_serve_starts_is_refused_per_render() { ("Content-Type", "application/json"), ]; - // Content drift after startup: the per-request seal check catches it. + // Content drift after startup: the per-request package check catches it. let labels = home.join("bundle/labels/ar.yaml"); let mut text = std::fs::read_to_string(&labels).unwrap(); text.push_str("extra: tampered\n"); @@ -947,8 +1023,8 @@ fn bundle_drift_after_serve_starts_is_refused_per_render() { String::from_utf8_lossy(&tampered.body) ); - // A bundle-load failure that formats a host path (here: a symlink the - // seal walk refuses) reaches the caller as a problem, so the bundle + // A package-load failure that names a path (here: a symlink the package + // walk refuses) reaches the caller as a problem, so the bundle // root must be redacted the way render diagnostics already are. #[cfg(unix)] { @@ -964,10 +1040,10 @@ fn bundle_drift_after_serve_starts_is_refused_per_render() { std::fs::remove_file(&escape).unwrap(); let body = String::from_utf8_lossy(&symlinked.body).into_owned(); assert_eq!(symlinked.status, 400, "{body}"); - assert!(body.contains("manifest-invalid"), "{body}"); + assert!(body.contains("bundle-tampered"), "{body}"); assert!( - body.contains("/escape"), - "the detail names the offending file under the redaction marker: {body}" + body.contains("escape") && body.contains("symbolic link"), + "the detail names the offending package entry: {body}" ); for root in [ home.to_string_lossy().into_owned(), @@ -983,13 +1059,9 @@ fn bundle_drift_after_serve_starts_is_refused_per_render() { } } - // Unsealing after startup (hashes stripped, content otherwise intact): - // the worker must load sealed, not merely verify-if-sealed. - let manifest_path = home.join("bundle/manifest.yaml"); - let manifest = std::fs::read_to_string(&manifest_path).unwrap(); - let stripped = manifest.split("hashes:").next().unwrap().to_owned(); - std::fs::write(&manifest_path, stripped).unwrap(); - let unsealed = request( + // Removing the envelope after startup is refused by every worker. + std::fs::remove_file(home.join("bundle/SHA256SUMS")).unwrap(); + let unpackaged = request( server.port, "POST", "/v1/render/receipt", @@ -997,15 +1069,15 @@ fn bundle_drift_after_serve_starts_is_refused_per_render() { Some(&receipt_body()), ); assert_eq!( - unsealed.status, + unpackaged.status, 400, "{}", - String::from_utf8_lossy(&unsealed.body) + String::from_utf8_lossy(&unpackaged.body) ); assert!( - String::from_utf8_lossy(&unsealed.body).contains("bundle-unsealed"), - "the worker must refuse an unsealed bundle per request: {}", - String::from_utf8_lossy(&unsealed.body) + String::from_utf8_lossy(&unpackaged.body).contains("bundle-unsealed"), + "the worker must refuse a directory without SHA256SUMS per request: {}", + String::from_utf8_lossy(&unpackaged.body) ); } @@ -1144,10 +1216,7 @@ fn pathological_renders_are_bounded_and_the_service_recovers() { "apiVersion: render.registrystack.org/v1alpha1\nkind: RenderBundle\nbundleVersion: 1\ndocuments:\n - id: heavy\n version: 1\n entry: templates/heavy.typ\n - id: healthy\n version: 1\n entry: templates/healthy.typ\n", ) .unwrap(); - for dir in ["templates", "fonts", "labels", "schemas"] { - std::fs::create_dir_all(bundle.join(dir)).unwrap(); - } - std::fs::create_dir_all(bundle.join("packages/preview")).unwrap(); + std::fs::create_dir_all(bundle.join("templates")).unwrap(); std::fs::write( bundle.join("templates/heavy.typ"), "#let payload = json(bytes(sys.inputs.data))\n#let x = range(20000000).fold(0, (a, b) => a + b)\n#x\n", @@ -1158,16 +1227,13 @@ fn pathological_renders_are_bounded_and_the_service_recovers() { "#let payload = json(bytes(sys.inputs.data))\n= Worker recovered\n", ) .unwrap(); - // Seal the heavy bundle so serve accepts it. - let sealed = Command::new(env!("CARGO_BIN_EXE_registry-render")) - .args(["seal", "--bundle", bundle.to_str().unwrap()]) - .output() - .unwrap(); - assert!( - sealed.status.success(), - "{}", - String::from_utf8_lossy(&sealed.stderr) - ); + registry_platform_config::package::write_sum_file( + &bundle, + None, + ®istry_render::runtime::package_limits(), + "registry-render package", + ) + .unwrap(); let (home, runtime, _) = deployment( "limits:\n renderTimeoutSeconds: 2\n maxOutputBytes: 8388608\n maxRequestBodyBytes: 8388608\n maxConcurrency: 2\n", @@ -1259,25 +1325,19 @@ fn shutdown_is_bounded_by_grace_even_with_renders_in_flight() { "apiVersion: render.registrystack.org/v1alpha1\nkind: RenderBundle\nbundleVersion: 1\ndocuments:\n - id: heavy\n version: 1\n entry: templates/heavy.typ\n", ) .unwrap(); - for dir in [ - "templates", - "fonts", - "labels", - "schemas", - "packages/preview", - ] { - std::fs::create_dir_all(bundle.join(dir)).unwrap(); - } + std::fs::create_dir_all(bundle.join("templates")).unwrap(); std::fs::write( bundle.join("templates/heavy.typ"), "#let x = range(200000000).fold(0, (a, b) => a + b)\n#x\n", ) .unwrap(); - let sealed = Command::new(env!("CARGO_BIN_EXE_registry-render")) - .args(["seal", "--bundle", bundle.to_str().unwrap()]) - .output() - .unwrap(); - assert!(sealed.status.success()); + registry_platform_config::package::write_sum_file( + &bundle, + None, + ®istry_render::runtime::package_limits(), + "registry-render package", + ) + .unwrap(); let (home, runtime, port) = deployment( "limits:\n renderTimeoutSeconds: 120\n maxOutputBytes: 8388608\n maxRequestBodyBytes: 8388608\n maxConcurrency: 2\n", @@ -1291,7 +1351,7 @@ fn shutdown_is_bounded_by_grace_even_with_renders_in_flight() { ) .unwrap(); let mut child = Command::new(env!("CARGO_BIN_EXE_registry-render")) - .args(["serve", "--runtime", runtime.to_str().unwrap()]) + .args(["serve", "--runtime-config", runtime.to_str().unwrap()]) .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() diff --git a/crates/registry-scheduling-core/src/naming.rs b/crates/registry-scheduling-core/src/naming.rs index 4883c5091c..48956dbfdc 100644 --- a/crates/registry-scheduling-core/src/naming.rs +++ b/crates/registry-scheduling-core/src/naming.rs @@ -9,9 +9,6 @@ pub const SCHEDULING_POLICY_API_VERSION: &str = /// Kind of an authored scheduling policy package. pub const SCHEDULING_POLICY_KIND: &str = "SchedulingPolicyPackage"; -/// File name of the package manifest written beside the authored policy. -pub const SCHEDULING_PACKAGE_MANIFEST_FILE: &str = "scheduling.package.json"; - /// File name of the authored policy inside a scheduling project. pub const AUTHORED_POLICY_FILE: &str = "scheduling.yaml"; @@ -96,7 +93,6 @@ mod tests { "registry.registrystack.org/scheduling-policy-package/v1alpha1" ); assert_eq!(SCHEDULING_POLICY_KIND, "SchedulingPolicyPackage"); - assert_eq!(SCHEDULING_PACKAGE_MANIFEST_FILE, "scheduling.package.json"); assert_eq!(AUTHORED_POLICY_FILE, "scheduling.yaml"); assert_eq!( SCHEDULING_RUNTIME_API_VERSION, diff --git a/crates/registry-scheduling/Cargo.toml b/crates/registry-scheduling/Cargo.toml index b9cb2d1342..a5283dc48e 100644 --- a/crates/registry-scheduling/Cargo.toml +++ b/crates/registry-scheduling/Cargo.toml @@ -20,7 +20,7 @@ required-features = ["postgres-test"] [features] default = [] postgres-test = [] -schema = ["dep:schemars", "registry-platform-audit/schema"] +schema = ["dep:schemars", "registry-platform-audit/schema", "registry-platform-config/schema"] [lints] workspace = true diff --git a/crates/registry-scheduling/examples/scheduling-auth-probe.rs b/crates/registry-scheduling/examples/scheduling-auth-probe.rs index 559972127d..f29c93ebea 100644 --- a/crates/registry-scheduling/examples/scheduling-auth-probe.rs +++ b/crates/registry-scheduling/examples/scheduling-auth-probe.rs @@ -14,7 +14,7 @@ use std::sync::Arc; use jsonwebtoken::{jwk::JwkSet, Algorithm}; use registry_platform_oidc::{JwksFetcher, JwksFetcherConfig, TokenVerifierConfig}; use registry_scheduling::auth::SchedulingAuthenticator; -use registry_scheduling::config::{OidcConfig, OidcJwksSource}; +use registry_scheduling::config::{JwksSource, OidcClientsConfig, OidcConfig, OidcIssuerConfig}; use serde::Deserialize; const MAXIMUM_INPUT_BYTES: u64 = 1024 * 1024; @@ -86,12 +86,15 @@ async fn authenticate(input: ProbeInput) -> Result { let assertion_issuers = BTreeMap::from([(input.client.clone(), vec![input.assertion_issuer.clone()])]); let oidc = OidcConfig { - allowed_clients: vec![input.client.clone()], - assertion_issuers: assertion_issuers.clone(), - issuer: input.issuer.clone(), - audience: input.audience.clone(), - jwks_uri: None, - jwks_source: OidcJwksSource::Discovery, + provider: OidcIssuerConfig { + issuer: input.issuer.clone(), + audience: input.audience.clone(), + jwks_source: JwksSource::Discovery {}, + }, + clients: OidcClientsConfig { + allowed_clients: vec![input.client.clone()], + assertion_issuers: assertion_issuers.clone(), + }, scope_claim: "scope".to_owned(), reads_scope: "scheduling-read".to_owned(), explain_scope: "scheduling-explain".to_owned(), diff --git a/crates/registry-scheduling/src/auth.rs b/crates/registry-scheduling/src/auth.rs index 3ce25ee291..734161ecf7 100644 --- a/crates/registry-scheduling/src/auth.rs +++ b/crates/registry-scheduling/src/auth.rs @@ -56,10 +56,10 @@ impl SchedulingAuthenticator { Self { verifier: TokenVerifier::new(verifier, keys), claim_names, - audience: oidc.audience.clone(), + audience: oidc.provider.audience.clone(), reads_scope: oidc.reads_scope.clone(), explain_scope: oidc.explain_scope.clone(), - binds_assertion_issuers: !oidc.assertion_issuers.is_empty(), + binds_assertion_issuers: !oidc.clients.assertion_issuers.is_empty(), } } @@ -251,12 +251,15 @@ mod tests { fn oidc() -> OidcConfig { OidcConfig { - allowed_clients: vec![CLIENT.to_owned()], - assertion_issuers: std::collections::BTreeMap::new(), - issuer: ISSUER.to_owned(), - audience: AUDIENCE.to_owned(), - jwks_uri: None, - jwks_source: crate::config::OidcJwksSource::Discovery, + provider: crate::config::OidcIssuerConfig { + issuer: ISSUER.to_owned(), + audience: AUDIENCE.to_owned(), + jwks_source: crate::config::JwksSource::Discovery {}, + }, + clients: crate::config::OidcClientsConfig { + allowed_clients: vec![CLIENT.to_owned()], + assertion_issuers: std::collections::BTreeMap::new(), + }, scope_claim: "registry_scopes".to_owned(), reads_scope: "scheduling-read".to_owned(), explain_scope: "scheduling-explain".to_owned(), @@ -299,7 +302,7 @@ mod tests { ) .with_scope_claim("registry_scopes") .with_allowed_clients(vec![CLIENT.to_owned()]) - .with_assertion_issuers(oidc.assertion_issuers.clone()); + .with_assertion_issuers(oidc.clients.assertion_issuers.clone()); SchedulingAuthenticator::new(&oidc, verifier, keys) } @@ -504,7 +507,7 @@ mod tests { ); let mut declared = oidc(); - declared.assertion_issuers = + declared.clients.assertion_issuers = std::collections::BTreeMap::from([(CLIENT.to_owned(), vec![AUTHORITY.to_owned()])]); authenticator_with(declared.clone()) .authenticate_read(&credential) diff --git a/crates/registry-scheduling/src/config.rs b/crates/registry-scheduling/src/config.rs index 53f8f2e573..62ae6c2e35 100644 --- a/crates/registry-scheduling/src/config.rs +++ b/crates/registry-scheduling/src/config.rs @@ -1,84 +1,48 @@ // SPDX-License-Identifier: Apache-2.0 -//! The operator runtime configuration document and the policy package -//! identity it verifies. +//! The operator runtime configuration document and the package it +//! verifies. use std::collections::{BTreeMap, BTreeSet}; -use std::net::{IpAddr, Ipv6Addr, SocketAddr}; use std::path::{Path, PathBuf}; use std::time::Duration; -use jsonwebtoken::jwk::{AlgorithmParameters, JwkSet}; use jsonwebtoken::Algorithm; use registry_platform_audit::{AuditDestination, AuditDestinationError, AuditDestinationKind}; +pub(crate) use registry_platform_config::describe_secret_failure; +use registry_platform_config::package::is_envelope_file; use registry_platform_config::{ - SecretError, SecretProvider, SecretReference, SecretResolver, MAX_SECRET_BYTES, + redact_refused_values, reject_environment_expressions_in_authored_yaml, sha256_uri, + ConfigBlockError, PackageDigestMismatch, PackageError, PackageErrorKind, PackageLimits, + RemovedKey, RuntimeConfigErrorKind, RuntimeConfigLoader, RuntimeEnvelope, SecretResolver, + VerifiedPackage, REMOVED_OIDC_JWKS_URI, +}; +pub use registry_platform_config::{ + AuditKeyConfig, DatabaseConfig, EnvironmentSecretProviderConfig, FileSecretProviderConfig, + JwksSource, ListenerNetworkExposure, OidcClientsConfig, OidcIssuerConfig, PackageConfig, + PrivateListenerConfig as ListenerConfig, SecretProvidersConfig, TlsTermination, }; use registry_platform_oidc::{ - access_token_typ_set, fetch_discovery, JwksFetcher, JwksFetcherConfig, OidcDiscoveryConfig, - TokenVerifierConfig, + access_token_typ_set, fetch_discovery, parse_static_jwks, JwksFetcher, JwksFetcherConfig, + OidcDiscoveryConfig, TokenVerifierConfig, }; use registry_scheduling_core::{ - parse_policy_yaml, SchedulingPolicy, AUTHORED_POLICY_FILE, SCHEDULING_PACKAGE_MANIFEST_FILE, - SCHEDULING_RUNTIME_API_VERSION, SCHEDULING_RUNTIME_KIND, + parse_policy_yaml, SchedulingPolicy, AUTHORED_POLICY_FILE, SCHEDULING_RUNTIME_API_VERSION, + SCHEDULING_RUNTIME_KIND, }; -use serde::{Deserialize, Serialize}; -use sha2::{Digest as _, Sha256}; +use serde::Deserialize; use thiserror::Error; -/// Explain one refused secret reference without disclosing what it protects. -/// -/// A startup refusal reaches an operator as a single line, and the resolver -/// reports only which rule broke. A valid reference is safe and useful to name, -/// but invalid operator-authored text might itself be a literal credential, so -/// only its field is named. The resolved bytes and opened path never appear. -pub(crate) fn describe_secret_failure( - field: &'static str, - reference: &str, - error: &SecretError, -) -> String { - let reason = match error { - SecretError::InvalidReference => { - "it is not an exact secret:env/NAME or secret:file/name reference".to_owned() - } - SecretError::ProviderDisabled => "its provider is not enabled for this runtime".to_owned(), - SecretError::InvalidProviderConfiguration => { - "the secret provider configuration is invalid".to_owned() - } - SecretError::Unavailable => { - "no readable secret of that name exists under the configured provider".to_owned() - } - SecretError::UnsafeFile => concat!( - "the secret file must be a regular file owned by the runtime user, ", - "with mode 0400 or 0600, and exactly one hard link" - ) - .to_owned(), - SecretError::Read => "the secret could not be read".to_owned(), - SecretError::InvalidValue => format!( - "the secret value must be non-empty text of at most {MAX_SECRET_BYTES} bytes \ - without NUL bytes" - ), - }; - if error == &SecretError::InvalidReference { - format!("the secret reference configured at {field} could not be resolved: {reason}") - } else { - format!("the secret reference {reference} could not be resolved: {reason}") - } -} - const MAXIMUM_POLICY_FILE_BYTES: usize = 1024 * 1024; -const MAXIMUM_PACKAGE_MANIFEST_BYTES: usize = 1024 * 1024; -/// apiVersion of the package manifest written beside the authored policy. -/// -/// The manifest names a package identity; the authored policy names a policy. -/// They are two documents with two readers, so they carry two names and -/// neither reader accepts the other's document. -pub const SCHEDULING_PACKAGE_MANIFEST_API_VERSION: &str = - "registry.registrystack.org/scheduling-policy-package-manifest/v1alpha1"; +/// The command that writes a Scheduling package, named by every package +/// refusal. +pub const PACKAGE_COMMAND: &str = "schedulingctl package"; -/// Kind of the package manifest written beside the authored policy. -pub const SCHEDULING_PACKAGE_MANIFEST_KIND: &str = "SchedulingPolicyPackageManifest"; +/// The manifest file an earlier `schedulingctl package` wrote beside the +/// policy. A package root that still holds it is refused with the command +/// that rebuilds the package. +pub const RETIRED_PACKAGE_MANIFEST_FILE: &str = "scheduling.package.json"; /// The default number of days a stored idempotency receipt is replayable /// before the retention sweep erases it. The default is a floor, not a @@ -86,6 +50,16 @@ pub const SCHEDULING_PACKAGE_MANIFEST_KIND: &str = "SchedulingPolicyPackageManif /// value, and the deployed value is what the audit entries record. pub const DEFAULT_ATTEMPT_RECEIPT_DAYS: u16 = 7; +/// The envelope every Scheduling runtime configuration carries. +pub const SCHEDULING_RUNTIME_ENVELOPE: RuntimeEnvelope = RuntimeEnvelope { + api_version: SCHEDULING_RUNTIME_API_VERSION, + kind: SCHEDULING_RUNTIME_KIND, +}; + +/// Keys an earlier Scheduling runtime configuration accepted, each refused +/// with the key that replaced it. +pub const SCHEDULING_REMOVED_KEYS: &[RemovedKey] = &[REMOVED_OIDC_JWKS_URI]; + /// The operator runtime configuration document. #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Clone, Debug, Deserialize)] @@ -93,7 +67,7 @@ pub const DEFAULT_ATTEMPT_RECEIPT_DAYS: u16 = 7; pub struct RuntimeConfig { pub api_version: String, pub kind: String, - pub package: RuntimePackageConfig, + pub package: PackageConfig, pub listener: ListenerConfig, pub secret_providers: SecretProvidersConfig, pub database: DatabaseConfig, @@ -104,75 +78,11 @@ pub struct RuntimeConfig { pub retention: RetentionConfig, } -/// The root of an authored scheduling project, holding `scheduling.yaml` and -/// its optional package manifest. -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct RuntimePackageConfig { - pub root: PathBuf, -} - -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct ListenerConfig { - #[serde(default = "default_listener_bind")] - #[cfg_attr(feature = "schema", schemars(with = "String"))] - pub bind: SocketAddr, - pub tls_termination: TlsTermination, - #[serde(default)] - pub network_exposure: ListenerNetworkExposure, -} - -fn default_listener_bind() -> SocketAddr { - "127.0.0.1:8105" - .parse() - .expect("valid Scheduling listener default") -} - -/// Declares the trusted transport boundary for the runtime's plaintext HTTP -/// listener. Production listeners require operator-controlled upstream TLS -/// termination; direct plaintext is limited to the explicit loopback-only -/// development mode. -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] -#[serde(rename_all = "kebab-case")] -pub enum TlsTermination { - OperatorControlledUpstream, - DevelopmentLoopback, -} - -/// The operator-declared private network placement of the HTTP listener. -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] -#[serde(rename_all = "kebab-case")] -pub enum ListenerNetworkExposure { - #[default] - PrivateAddress, - ContainerPrivate, -} - -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct SecretProvidersConfig { - #[serde(default)] - pub file: Option, - #[serde(default)] - pub environment: Option, -} - -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Debug, Default, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EnvironmentSecretProviderConfig {} - -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct FileSecretProviderConfig { - pub root: PathBuf, +/// The policy and package identity produced by one verified package load. +#[derive(Clone, Debug)] +pub struct LoadedSchedulingPolicy { + pub policy: SchedulingPolicy, + pub package_digest: String, } #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] @@ -182,58 +92,23 @@ pub struct AuthenticationConfig { pub oidc: OidcConfig, } -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct DatabaseConfig { - pub runtime_url_ref: String, - pub migration_url_ref: String, - #[serde(default)] - pub trusted_root_certificate_ref: Option, - #[serde(default)] - pub test_only_plaintext: bool, -} - -impl std::fmt::Debug for DatabaseConfig { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("DatabaseConfig") - .field("runtime_url_ref", &"") - .field("migration_url_ref", &"") - .field( - "trusted_root_certificate_ref", - &self - .trusted_root_certificate_ref - .as_ref() - .map(|_| ""), - ) - .field("test_only_plaintext", &self.test_only_plaintext) - .finish() - } -} - +/// The access tokens this runtime accepts: the issuer and its keys, the +/// clients admitted, and the scopes each route family requires. #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct OidcConfig { - #[serde(default)] - pub allowed_clients: Vec, - /// The assertion authorities each client may exchange a subject token - /// from, keyed by client identifier. - /// - /// A deployment that performs no token exchange leaves this empty. Once a - /// client is listed, a token it exchanged is accepted only for one of that - /// client's declared authorities, so an assertion minted by an unrelated - /// authority the issuer happens to federate cannot become a booking - /// credential here. - #[serde(default)] - pub assertion_issuers: BTreeMap>, - pub issuer: String, - pub audience: String, - #[serde(default)] - pub jwks_uri: Option, - #[serde(default)] - pub jwks_source: OidcJwksSource, + /// The exact issuer, the one audience every token carries, and where the + /// issuer's signing keys come from. + #[serde(flatten)] + pub provider: OidcIssuerConfig, + /// The clients admitted and the assertion authorities each may exchange + /// a subject token from. A deployment that performs no token exchange + /// leaves `assertionIssuers` empty; once a client is listed, an assertion + /// minted by an unrelated authority the issuer happens to federate cannot + /// become a booking credential here. + #[serde(flatten)] + pub clients: OidcClientsConfig, #[serde(default = "default_scope_claim")] pub scope_claim: String, /// The scope every listing and availability read requires. @@ -249,14 +124,6 @@ pub struct OidcConfig { /// it to an ordinary JWT. const SCHEDULING_ACCESS_TOKEN_TYPE: &str = "at+jwt"; -/// Bounds on the authored assertion-issuer map, matching the Casework -/// runtime's. They keep one operator document from becoming an unbounded -/// verifier input. -pub(crate) const MAXIMUM_ASSERTION_ISSUER_CLIENTS: usize = 64; -pub(crate) const MAXIMUM_ASSERTION_ISSUER_CLIENT_BYTES: usize = 128; -pub(crate) const MAXIMUM_ASSERTION_ISSUERS_PER_CLIENT: usize = 16; -pub(crate) const MAXIMUM_ASSERTION_ISSUER_BYTES: usize = 512; - fn default_scope_claim() -> String { "registry_scopes".to_owned() } @@ -267,23 +134,14 @@ fn default_explain_scope() -> String { "scheduling-explain".to_owned() } -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[derive(Clone, Debug, Default, Deserialize)] -#[serde(tag = "kind", rename_all = "camelCase", deny_unknown_fields)] -pub enum OidcJwksSource { - #[default] - Discovery, - Static { - #[serde(rename = "documentRef")] - document_ref: String, - }, -} - +/// The audit journal: where it is written and the key its hashes use. #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct AuditConfig { - pub hash_key_ref: String, + /// The secret keying the audit journal's hashes, `hashKeyRef`. + #[serde(flatten)] + pub key: AuditKeyConfig, /// Where audit entries go: a rotated `file` (the default) or `stdout`. #[serde(default)] pub destination: AuditDestinationKind, @@ -393,125 +251,58 @@ const fn default_hook_payload_days() -> u16 { 7 } -/// The immutable identity of a packaged policy: the digest of the package -/// envelope over its exact authored files. A scheduling package is one -/// authored policy file, so the manifest is small, but it is still a -/// separate document the runtime verifies rather than trusts. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct PolicyPackageManifest { - pub api_version: String, - pub kind: String, - pub policy_digest: String, - pub files: Vec, -} - -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -pub struct PolicyPackageFile { - pub path: String, - pub sha256: String, - pub bytes: u64, -} - -impl PolicyPackageManifest { - /// Build the immutable identity for an already validated policy text. - pub fn build(policy_text: &str) -> Result { - let bytes = policy_text.as_bytes(); - if bytes.len() > MAXIMUM_POLICY_FILE_BYTES { - return Err(PolicyPackageError::Invalid); - } - let files = vec![PolicyPackageFile { - path: AUTHORED_POLICY_FILE.to_owned(), - sha256: sha256_bytes(bytes), - bytes: u64::try_from(bytes.len()).map_err(|_| PolicyPackageError::Invalid)?, - }]; - Ok(Self { - api_version: SCHEDULING_PACKAGE_MANIFEST_API_VERSION.to_owned(), - kind: SCHEDULING_PACKAGE_MANIFEST_KIND.to_owned(), - policy_digest: package_digest(&files)?, - files, - }) - } - - fn verify(&self, policy_text: &str) -> Result<(), PolicyPackageError> { - let expected = Self::build(policy_text)?; - if self.api_version != expected.api_version - || self.kind != expected.kind - || self.files != expected.files - || self.policy_digest != expected.policy_digest - { - return Err(PolicyPackageError::Invalid); - } - Ok(()) +/// The bounds a Scheduling package holds to: each file at most one MiB. +#[must_use] +pub fn package_limits() -> PackageLimits { + PackageLimits { + max_file_bytes: MAXIMUM_POLICY_FILE_BYTES as u64, + ..PackageLimits::default() } } -/// Verify the package beside `scheduling.yaml`. An absent manifest is -/// distinguished so local authored development remains usable. -pub fn verify_policy_package( - policy_path: &Path, - policy_text: &str, -) -> Result, PolicyPackageError> { - if policy_path.file_name().and_then(|name| name.to_str()) != Some(AUTHORED_POLICY_FILE) { - return Err(PolicyPackageError::Invalid); - } - let root = policy_path.parent().ok_or(PolicyPackageError::Invalid)?; - let manifest_path = root.join(SCHEDULING_PACKAGE_MANIFEST_FILE); - if !manifest_path.exists() { - return Ok(None); +/// Verify the package at `package.root`: its `SHA256SUMS`, the +/// `package.expectedDigest` pin, and that it holds exactly `scheduling.yaml`. +/// Every listener mode serves a package: a directory without `SHA256SUMS` is +/// refused with the packaging command, whether or not a digest is pinned. +pub fn verify_scheduling_package( + package: &PackageConfig, +) -> Result { + if std::fs::symlink_metadata(package.root.join(RETIRED_PACKAGE_MANIFEST_FILE)).is_ok() { + return Err(RuntimeConfigError::RetiredPackageManifest); } - let metadata = std::fs::symlink_metadata(&manifest_path).map_err(PolicyPackageError::Read)?; - if !metadata.file_type().is_file() - || metadata.file_type().is_symlink() - || metadata.len() > MAXIMUM_PACKAGE_MANIFEST_BYTES as u64 - { - return Err(PolicyPackageError::Invalid); + let verified = package + .verify_package(&package_limits(), PACKAGE_COMMAND) + .map_err(|error| match error.kind() { + PackageErrorKind::DigestMismatch(mismatch) => { + RuntimeConfigError::PackageDigest(mismatch.clone()) + } + _ => RuntimeConfigError::Package(error), + })?; + let extra = verified + .files() + .filter(|path| !is_envelope_file(path) && *path != AUTHORED_POLICY_FILE) + .map(ToOwned::to_owned) + .collect::>(); + if verified.file_digest(AUTHORED_POLICY_FILE).is_none() || !extra.is_empty() { + return Err(RuntimeConfigError::PackageContents { extra }); } - let bytes = std::fs::read(&manifest_path).map_err(PolicyPackageError::Read)?; - let manifest: PolicyPackageManifest = - serde_json::from_slice(&bytes).map_err(|_| PolicyPackageError::Invalid)?; - manifest.verify(policy_text)?; - Ok(Some(manifest.policy_digest)) -} - -fn package_digest(files: &[PolicyPackageFile]) -> Result { - let identity = serde_json::json!({ - "apiVersion": SCHEDULING_PACKAGE_MANIFEST_API_VERSION, - "kind": SCHEDULING_PACKAGE_MANIFEST_KIND, - "files": files, - }); - let canonical = registry_platform_canonical_json::canonicalize_json(&identity) - .map_err(|_| PolicyPackageError::Invalid)?; - Ok(sha256_bytes(&canonical)) -} - -fn sha256_bytes(bytes: &[u8]) -> String { - let digest = Sha256::digest(bytes); - format!( - "sha256:{}", - digest - .iter() - .map(|byte| format!("{byte:02x}")) - .collect::() - ) + Ok(verified) } impl RuntimeConfig { /// Load and validate the operator document, reading the authored policy /// beside it. The bytes never re-enter a parser after startup. pub fn load(path: impl AsRef) -> Result { - if !path.as_ref().is_absolute() { - return Err(RuntimeConfigError::RelativeRuntimePath); - } - let bytes = std::fs::read(path.as_ref()).map_err(RuntimeConfigError::Read)?; - let deserializer = serde_norway::Deserializer::from_slice(&bytes); - let config: Self = serde_path_to_error::deserialize(deserializer).map_err(|error| { - let (path, cause) = refused_yaml(error); - RuntimeConfigError::Parse { path, cause } - })?; - config.check()?; - Ok(config) + let loaded = Self::loader().load::(path.as_ref())?; + loaded.config.check()?; + Ok(loaded.config) + } + + /// The shared runtime configuration loader under Scheduling's envelope + /// and removed keys. + #[must_use] + pub const fn loader() -> RuntimeConfigLoader { + RuntimeConfigLoader::new(SCHEDULING_RUNTIME_ENVELOPE).removed_keys(SCHEDULING_REMOVED_KEYS) } #[must_use] @@ -519,90 +310,91 @@ impl RuntimeConfig { self.package.root.join(AUTHORED_POLICY_FILE) } - /// Read and validate the authored policy this deployment runs. - pub fn load_policy(&self) -> Result { + /// Read and validate the packaged policy this deployment runs. + pub fn load_policy(&self) -> Result { + let package = verify_scheduling_package(&self.package)?; let policy_text = std::fs::read_to_string(self.policy_path()).map_err(RuntimeConfigError::PolicyRead)?; let policy = parse_policy_yaml(&policy_text).map_err(|error| { let (path, cause) = refused_yaml(error); RuntimeConfigError::PolicyParse { path, cause } })?; + reject_environment_expressions_in_authored_yaml(&policy_text).map_err(|error| { + if error.kind() == RuntimeConfigErrorKind::AuthoredSyntax { + RuntimeConfigError::PolicyParse { + path: error.field().to_owned(), + cause: error.message().to_owned(), + } + } else { + RuntimeConfigError::PolicyEnvironmentExpression { + field: error.field().to_owned(), + } + } + })?; if !policy.check().is_empty() { return Err(RuntimeConfigError::PolicyFindings); } - // A present manifest is verified against the exact policy text; the - // digest the runtime publishes is the policy's own, not the manifest's. - verify_policy_package(&self.policy_path(), &policy_text) - .map_err(RuntimeConfigError::PolicyPackage)?; - Ok(policy) + // The policy is served only when the text just parsed is the text the + // package lists; the digest the runtime publishes is the policy's + // own, not the package's. + if package.file_digest(AUTHORED_POLICY_FILE).as_deref() + != Some(sha256_uri(policy_text.as_bytes()).as_str()) + { + return Err(RuntimeConfigError::PolicyChanged); + } + Ok(LoadedSchedulingPolicy { + policy, + package_digest: package.digest().to_owned(), + }) } - /// Return the verified package identity, if this is a packaged - /// deployment. Production configurations always have one. - pub fn policy_package_digest(&self) -> Result, RuntimeConfigError> { - let policy_text = - std::fs::read_to_string(self.policy_path()).map_err(RuntimeConfigError::PolicyRead)?; - verify_policy_package(&self.policy_path(), &policy_text) - .map_err(RuntimeConfigError::PolicyPackage) + /// Return the digest of the verified package this deployment serves. + pub fn package_digest(&self) -> Result { + Ok(verify_scheduling_package(&self.package)? + .digest() + .to_owned()) } pub fn check(&self) -> Result<(), RuntimeConfigError> { - if self.api_version != SCHEDULING_RUNTIME_API_VERSION { - return Err(RuntimeConfigError::InvalidApiVersion); - } - if self.kind != SCHEDULING_RUNTIME_KIND { - return Err(RuntimeConfigError::InvalidKind); - } - if !self.package.root.is_absolute() { - return Err(RuntimeConfigError::RelativeOperatedPath("package.root")); - } - if self - .secret_providers - .file - .as_ref() - .is_some_and(|file| !file.root.is_absolute()) + if self.api_version != SCHEDULING_RUNTIME_API_VERSION + || self.kind != SCHEDULING_RUNTIME_KIND { - return Err(RuntimeConfigError::RelativeOperatedPath( - "secretProviders.file.root", - )); + return Err(RuntimeConfigError::InvalidEnvelope); } + self.package.check()?; + self.secret_providers.check()?; self.audit.destination()?; - if self.secret_providers.file.is_none() && self.secret_providers.environment.is_none() { - return Err(RuntimeConfigError::InvalidSecretProviders); - } - if !valid_listener( - self.listener.bind.ip(), - self.listener.network_exposure, - self.listener.tls_termination, - ) { + if !self.listener.is_valid() { return Err(RuntimeConfigError::InvalidListener); } - if self.authentication.oidc.issuer.is_empty() - || self.authentication.oidc.audience.is_empty() - || self.authentication.oidc.scope_claim.is_empty() + self.authentication.oidc.provider.check( + "authentication.oidc", + self.listener.tls_termination == TlsTermination::DevelopmentLoopback, + )?; + self.authentication + .oidc + .clients + .check("authentication.oidc")?; + if self.authentication.oidc.scope_claim.is_empty() || self.authentication.oidc.reads_scope.is_empty() || self.authentication.oidc.explain_scope.is_empty() || self.authentication.oidc.explain_scope == self.authentication.oidc.reads_scope { return Err(RuntimeConfigError::InvalidOidc); } - self.validate_assertion_issuers()?; // An empty client list admits every client the issuer verifies, so a // deployment that simply forgot the field would accept a token minted // for an unrelated application in the same realm. Development loopback // keeps that convenience; a deployment behind an operator-controlled // terminator must name the clients it admits. if self.listener.tls_termination == TlsTermination::OperatorControlledUpstream - && self.authentication.oidc.allowed_clients.is_empty() + && self.authentication.oidc.clients.allowed_clients.is_empty() { return Err(RuntimeConfigError::InvalidOidc); } if self.database.runtime_url_ref.is_empty() || self.database.migration_url_ref.is_empty() { return Err(RuntimeConfigError::InvalidDatabaseReference); } - if self.audit.hash_key_ref.is_empty() { - return Err(RuntimeConfigError::InvalidAuditReference); - } if self.retention.attempt_receipt_days == 0 || !(1..=30).contains(&self.retention.hook_payload_days) { @@ -627,7 +419,7 @@ impl RuntimeConfig { } self.validate_secret_references()?; - let policy = self.load_policy()?; + let policy = self.load_policy()?.policy; let declared_hook_destinations = policy .hooks .iter() @@ -643,11 +435,6 @@ impl RuntimeConfig { if !declared_hook_destinations.is_subset(&configured_hook_destinations) { return Err(RuntimeConfigError::HookDestinationInventoryMismatch); } - if self.listener.tls_termination == TlsTermination::OperatorControlledUpstream - && self.policy_package_digest()?.is_none() - { - return Err(RuntimeConfigError::ProductionPolicyPackageRequired); - } #[cfg(not(feature = "postgres-test"))] if self.database.test_only_plaintext { return Err(RuntimeConfigError::PlaintextDatabase); @@ -655,52 +442,18 @@ impl RuntimeConfig { Ok(()) } - /// Refuse an assertion-issuer map with too many clients, an oversized - /// client key or issuer string, too many issuers listed for one client, or - /// a repeated issuer within one client's list. This runs at configuration - /// load, before any verifier is built, so an operator sees the refusal - /// without the runtime ever starting. - fn validate_assertion_issuers(&self) -> Result<(), RuntimeConfigError> { - let assertion_issuers = &self.authentication.oidc.assertion_issuers; - if assertion_issuers.len() > MAXIMUM_ASSERTION_ISSUER_CLIENTS { - return Err(RuntimeConfigError::InvalidOidc); - } - for (client, issuers) in assertion_issuers { - if client.is_empty() - || client.len() > MAXIMUM_ASSERTION_ISSUER_CLIENT_BYTES - || issuers.len() > MAXIMUM_ASSERTION_ISSUERS_PER_CLIENT - { - return Err(RuntimeConfigError::InvalidOidc); - } - let mut seen = BTreeSet::new(); - for issuer in issuers { - if issuer.is_empty() - || issuer.len() > MAXIMUM_ASSERTION_ISSUER_BYTES - || !seen.insert(issuer) - { - return Err(RuntimeConfigError::InvalidOidc); - } - } - } - Ok(()) - } - fn validate_secret_references(&self) -> Result<(), RuntimeConfigError> { - let mut references = vec![ - ( - "database.runtimeUrlRef".to_owned(), - &self.database.runtime_url_ref, - ), - ( - "database.migrationUrlRef".to_owned(), - &self.database.migration_url_ref, - ), - ("audit.hashKeyRef".to_owned(), &self.audit.hash_key_ref), - ]; - if let Some(reference) = &self.database.trusted_root_certificate_ref { - references.push(("database.trustedRootCertificateRef".to_owned(), reference)); - } - if let OidcJwksSource::Static { document_ref } = &self.authentication.oidc.jwks_source { + let mut references: Vec<(String, &str)> = self + .database + .references() + .into_iter() + .map(|(field, reference)| (field.to_owned(), reference)) + .collect(); + references.push(( + "audit.hashKeyRef".to_owned(), + self.audit.key.hash_key_ref.as_str(), + )); + if let Some(document_ref) = self.authentication.oidc.provider.jwks_source.document_ref() { references.push(( "authentication.oidc.jwksSource.documentRef".to_owned(), document_ref, @@ -720,16 +473,8 @@ impl RuntimeConfig { &destination.hmac_sha256_key_ref, )); } - for (path, raw) in references { - let reference = SecretReference::parse(raw.clone()) - .map_err(|_| RuntimeConfigError::InvalidSecretReference { path: path.clone() })?; - let enabled = match reference.provider() { - SecretProvider::File => self.secret_providers.file.is_some(), - SecretProvider::Environment => self.secret_providers.environment.is_some(), - }; - if !enabled { - return Err(RuntimeConfigError::SecretProviderRequired { path }); - } + for (field, raw) in references { + self.secret_providers.check_reference(&field, raw)?; } Ok(()) } @@ -738,20 +483,21 @@ impl RuntimeConfig { &self, secrets: &SecretResolver, ) -> Result<(TokenVerifierConfig, std::sync::Arc), RuntimeConfigError> { - let discovery_config = OidcDiscoveryConfig { - issuer: self.authentication.oidc.issuer.clone(), - jwks_uri_override: self.authentication.oidc.jwks_uri.clone(), - discovery_timeout: Duration::from_secs(5), - max_doc_bytes: 1024 * 1024, - }; - let fetcher = match &self.authentication.oidc.jwks_source { - OidcJwksSource::Discovery => { + let fetcher = match &self.authentication.oidc.provider.jwks_source { + JwksSource::Uri { uri } => JwksFetcher::new(uri.clone(), JwksFetcherConfig::defaults()), + JwksSource::Discovery {} => { + let discovery_config = OidcDiscoveryConfig { + issuer: self.authentication.oidc.provider.issuer.clone(), + jwks_uri_override: None, + discovery_timeout: Duration::from_secs(5), + max_doc_bytes: 1024 * 1024, + }; let discovery = fetch_discovery(&discovery_config) .await .map_err(|_| RuntimeConfigError::Oidc)?; JwksFetcher::new(discovery.jwks_uri, JwksFetcherConfig::defaults()) } - OidcJwksSource::Static { document_ref } => { + JwksSource::Static { document_ref } => { let document = secrets.resolve(document_ref).map_err(|error| { RuntimeConfigError::OidcJwksSecret(describe_secret_failure( "authentication.oidc.jwksSource.documentRef", @@ -759,7 +505,8 @@ impl RuntimeConfig { &error, )) })?; - let jwks = parse_static_jwks(document.expose_secret())?; + let jwks = parse_static_jwks(document.expose_secret()) + .map_err(|_| RuntimeConfigError::Oidc)?; JwksFetcher::new_static(jwks, JwksFetcherConfig::defaults()) } }; @@ -778,14 +525,14 @@ impl RuntimeConfig { /// another purpose book, reschedule or cancel an appointment. pub(crate) fn verifier_profile(&self) -> TokenVerifierConfig { TokenVerifierConfig::access_token_profile( - self.authentication.oidc.issuer.clone(), - vec![self.authentication.oidc.audience.clone()], + self.authentication.oidc.provider.issuer.clone(), + vec![self.authentication.oidc.provider.audience.clone()], vec![Algorithm::RS256, Algorithm::ES256], access_token_typ_set(SCHEDULING_ACCESS_TOKEN_TYPE), ) .with_scope_claim(self.authentication.oidc.scope_claim.clone()) - .with_allowed_clients(self.authentication.oidc.allowed_clients.clone()) - .with_assertion_issuers(self.authentication.oidc.assertion_issuers.clone()) + .with_allowed_clients(self.authentication.oidc.clients.allowed_clients.clone()) + .with_assertion_issuers(self.authentication.oidc.clients.assertion_issuers.clone()) } } @@ -834,57 +581,6 @@ fn valid_logical_destination_id(value: &str) -> bool { }) } -fn valid_listener( - address: IpAddr, - exposure: ListenerNetworkExposure, - tls_termination: TlsTermination, -) -> bool { - if address.is_multicast() { - return false; - } - if tls_termination == TlsTermination::DevelopmentLoopback { - return exposure == ListenerNetworkExposure::PrivateAddress && address.is_loopback(); - } - match (address, exposure) { - (IpAddr::V4(address), ListenerNetworkExposure::PrivateAddress) => { - address.is_loopback() || address.is_private() - } - (IpAddr::V6(address), ListenerNetworkExposure::PrivateAddress) => { - address.is_loopback() || is_unique_local(address) - } - (IpAddr::V4(address), ListenerNetworkExposure::ContainerPrivate) => { - address.is_unspecified() || address.is_loopback() || address.is_private() - } - (IpAddr::V6(address), ListenerNetworkExposure::ContainerPrivate) => { - address.is_unspecified() || address.is_loopback() || is_unique_local(address) - } - } -} - -fn is_unique_local(address: Ipv6Addr) -> bool { - address.octets()[0] & 0xfe == 0xfc -} - -fn parse_static_jwks(bytes: &[u8]) -> Result { - let jwks: JwkSet = serde_json::from_slice(bytes).map_err(|_| RuntimeConfigError::Oidc)?; - let mut kids = BTreeSet::new(); - if jwks.keys.is_empty() - || jwks.keys.iter().any(|key| { - !matches!( - key.algorithm, - AlgorithmParameters::RSA(_) | AlgorithmParameters::EllipticCurve(_) - ) || key - .common - .key_id - .as_ref() - .is_none_or(|kid| kid.is_empty() || !kids.insert(kid.clone())) - }) - { - return Err(RuntimeConfigError::Oidc); - } - Ok(jwks) -} - /// Name where a YAML document was refused and why, so an operator reading a /// startup failure can open the document at the place that failed. /// @@ -898,115 +594,45 @@ fn refused_yaml(error: serde_path_to_error::Error) -> (Stri (path, redact_refused_values(&error.into_inner().to_string())) } -/// Keep the parts of a refusal an operator acts on, the member, the reason -/// and the location, while the refused value stays out of the message. -/// -/// serde reports the offending value inside an `invalid type:` or an -/// `invalid value:` clause. Only the shape word that opens such a clause -/// survives, so the message still says a string arrived where a number was -/// required without repeating the string. A runtime configuration names -/// secret references, database URLs and destinations, and a startup refusal -/// is written to the operator's log. -fn redact_refused_values(message: &str) -> String { - const CLAUSES: [&str; 2] = ["invalid type: ", "invalid value: "]; - let mut redacted = String::with_capacity(message.len()); - let mut rest = message; - loop { - let Some((start, len)) = CLAUSES - .iter() - .filter_map(|clause| rest.find(clause).map(|start| (start, clause.len()))) - .min_by_key(|(start, _)| *start) - else { - redacted.push_str(rest); - return redacted; - }; - let opened = start + len; - redacted.push_str(&rest[..opened]); - let (shape, tail) = split_refused_value(&rest[opened..]); - redacted.push_str(shape); - rest = tail; - } -} - -/// Split the text after a clause marker into the shape word serde names and -/// the remainder that follows the refused value. -/// -/// serde renders the value with `Debug`, so it opens with a quote or a -/// backtick and may hold the comma that would otherwise end the clause. -fn split_refused_value(clause: &str) -> (&str, &str) { - let bytes = clause.as_bytes(); - let mut index = 0; - let mut shape_end = None; - while index < bytes.len() { - match bytes[index] { - delimiter @ (b'"' | b'`') => { - shape_end.get_or_insert(index); - index = skip_delimited(bytes, index, delimiter); - } - b',' => break, - _ => index += 1, - } - } - let shape_end = shape_end.unwrap_or(index); - (clause[..shape_end].trim_end(), &clause[index..]) -} - -/// Return the offset just past the delimited run that opens at `open`. -/// -/// A delimiter inside a `Debug` rendering arrives escaped, so it does not end -/// the run. -fn skip_delimited(bytes: &[u8], open: usize, delimiter: u8) -> usize { - let mut index = open + 1; - while index < bytes.len() { - match bytes[index] { - b'\\' => index += 2, - byte if byte == delimiter => return index + 1, - _ => index += 1, - } - } - bytes.len() -} - -#[derive(Debug, Error)] -pub enum PolicyPackageError { - #[error("the Scheduling policy package could not be read")] - Read(#[source] std::io::Error), - #[error("the Scheduling policy package is invalid or does not match its exact inputs")] - Invalid, -} - #[derive(Debug, Error)] pub enum RuntimeConfigError { - #[error("the Scheduling runtime configuration could not be read")] - Read(#[source] std::io::Error), - #[error("the Scheduling runtime configuration is not valid YAML at {path}: {cause}")] - Parse { path: String, cause: String }, + #[error(transparent)] + Load(#[from] registry_platform_config::RuntimeConfigError), + #[error(transparent)] + Block(#[from] ConfigBlockError), + #[error(transparent)] + PackageDigest(#[from] PackageDigestMismatch), #[error( - "unsupported Scheduling runtime apiVersion; expected registry.registrystack.org/scheduling-runtime/v1alpha1" + "apiVersion and kind must be exactly registry.registrystack.org/scheduling-runtime/v1alpha1 and SchedulingRuntimeConfig" )] - InvalidApiVersion, - #[error("unsupported Scheduling runtime kind; expected SchedulingRuntimeConfig")] - InvalidKind, + InvalidEnvelope, #[error("the operated runtime path {0} must be absolute")] RelativeOperatedPath(&'static str), - #[error("the selected Scheduling runtime configuration path must be absolute")] - RelativeRuntimePath, - #[error("secretProviders must explicitly enable file, environment, or both")] - InvalidSecretProviders, - #[error("{path} is not a valid secret reference")] - InvalidSecretReference { path: String }, - #[error("{path} uses a secret provider that is not explicitly enabled")] - SecretProviderRequired { path: String }, #[error("the authored scheduling policy could not be read")] PolicyRead(#[source] std::io::Error), + #[error( + "{field} in the authored scheduling policy holds an environment expression; ${{...}} substitution applies to runtime.yaml only, so write the value in scheduling.yaml directly" + )] + PolicyEnvironmentExpression { field: String }, #[error("the authored scheduling policy is not valid YAML at {path}: {cause}")] PolicyParse { path: String, cause: String }, #[error("the authored scheduling policy does not pass its checks")] PolicyFindings, - #[error("the Scheduling policy package is invalid")] - PolicyPackage(#[source] PolicyPackageError), - #[error("operator-controlled production requires a verified Scheduling policy package")] - ProductionPolicyPackageRequired, + #[error(transparent)] + Package(PackageError), + #[error( + "the package at package.root must hold exactly scheduling.yaml{}; rebuild it with `schedulingctl package`", + extra_files(extra) + )] + PackageContents { extra: Vec }, + #[error( + "package.root holds scheduling.package.json, which Scheduling no longer reads; rebuild the package with `schedulingctl package`, which writes SHA256SUMS" + )] + RetiredPackageManifest, + #[error( + "package.root/scheduling.yaml changed after its package was verified; deploy the whole package again" + )] + PolicyChanged, #[error("listener is not valid for its declared TLS termination and network exposure")] InvalidListener, #[error("authentication.oidc is invalid")] @@ -1015,8 +641,6 @@ pub enum RuntimeConfigError { "database.runtimeUrlRef and database.migrationUrlRef must be non-empty secret references" )] InvalidDatabaseReference, - #[error("audit.hashKeyRef must be a non-empty secret reference")] - InvalidAuditReference, #[error("{0}")] InvalidAuditDestination(#[source] AuditDestinationError), #[error("retention.attemptReceiptDays must be at least one day")] @@ -1039,18 +663,15 @@ impl RuntimeConfigError { #[must_use] pub fn path(&self) -> &str { match self { - Self::InvalidApiVersion => "apiVersion", - Self::InvalidKind => "kind", + Self::Load(error) => error.field(), + Self::Block(error) => error.field(), + Self::PackageDigest(_) => "package.expectedDigest", + Self::InvalidEnvelope => "apiVersion", Self::RelativeOperatedPath(path) => path, - Self::RelativeRuntimePath | Self::Read(_) => "/", - Self::Parse { path, .. } => path, - Self::InvalidSecretProviders => "secretProviders", - Self::InvalidSecretReference { path } | Self::SecretProviderRequired { path } => path, Self::InvalidOidc | Self::Oidc => "authentication.oidc", Self::OidcJwksSecret(_) => "authentication.oidc.jwksSource.documentRef", Self::InvalidListener => "listener", Self::InvalidDatabaseReference | Self::PlaintextDatabase => "database", - Self::InvalidAuditReference => "audit.hashKeyRef", Self::InvalidAuditDestination(error) => match error { AuditDestinationError::MissingPath | AuditDestinationError::RelativePath => { "audit.path" @@ -1071,18 +692,36 @@ impl RuntimeConfigError { Self::InvalidHookDestination | Self::HookDestinationInventoryMismatch => { "destinations.hooks" } - Self::PolicyRead(_) | Self::PolicyParse { .. } => "package.root/scheduling.yaml", - Self::PolicyFindings | Self::PolicyPackage(_) => "package.root", - Self::ProductionPolicyPackageRequired => "package.root", + Self::PolicyRead(_) + | Self::PolicyParse { .. } + | Self::PolicyEnvironmentExpression { .. } => "package.root/scheduling.yaml", + Self::PolicyChanged => "package.root/scheduling.yaml", + Self::PolicyFindings + | Self::Package(_) + | Self::PackageContents { .. } + | Self::RetiredPackageManifest => "package.root", } } } +fn extra_files(extra: &[String]) -> String { + if extra.is_empty() { + String::new() + } else { + format!(", not {}", extra.join(", ")) + } +} + #[cfg(test)] mod tests { use super::*; + use registry_platform_config::package::write_sum_file; + use registry_platform_config::SUM_FILE; + use registry_platform_config::{ + MAX_ASSERTION_ISSUERS_PER_CLIENT, MAX_ASSERTION_ISSUER_BYTES, MAX_ASSERTION_ISSUER_CLIENTS, + MAX_ASSERTION_ISSUER_CLIENT_BYTES, + }; use registry_platform_oidc::is_access_token_typ_pair; - use registry_scheduling_core::{SCHEDULING_POLICY_API_VERSION, SCHEDULING_POLICY_KIND}; #[test] fn a_stdout_destination_accepts_an_explicit_null_path() { @@ -1135,9 +774,26 @@ offerings: holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} "#; + /// A temporary directory named by its canonical path: the loader refuses + /// a runtime configuration reached through a symbolic link, and the + /// system temporary directory is one on some platforms. + fn canonical_tempdir() -> tempfile::TempDir { + tempfile::tempdir_in(std::fs::canonicalize(std::env::temp_dir()).unwrap()).unwrap() + } + fn write_policy(root: &Path) { + package_policy(root, POLICY); + } + + /// Write `policy` as the package's `scheduling.yaml` and list it in a + /// fresh `SHA256SUMS`, as `schedulingctl package` does. + fn package_policy(root: &Path, policy: &str) { std::fs::create_dir_all(root).unwrap(); - std::fs::write(root.join(AUTHORED_POLICY_FILE), POLICY).unwrap(); + std::fs::write(root.join(AUTHORED_POLICY_FILE), policy).unwrap(); + if root.join(SUM_FILE).exists() { + std::fs::remove_file(root.join(SUM_FILE)).unwrap(); + } + write_sum_file(root, None, &package_limits(), PACKAGE_COMMAND).unwrap(); } fn operator_value(package: &Path, tls: &str) -> serde_json::Value { @@ -1171,7 +827,7 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} #[test] fn a_development_configuration_loads_and_exposes_its_defaults() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); write_policy(&package); let operator = write_operator( @@ -1189,13 +845,14 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} assert_eq!(config.retention.hook_payload_days, 7); assert!(config.destinations.reminders.is_none()); assert!(config.destinations.hooks.is_empty()); - let policy = config.load_policy().expect("policy loads"); + let loaded = config.load_policy().expect("policy loads"); + let policy = loaded.policy; assert_eq!(policy.scheduling.id, "standalone-exact-time"); } #[test] fn the_audit_block_takes_a_file_or_stdout_destination() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); write_policy(&package); let audit_file = root.path().join("audit.ndjson"); @@ -1256,45 +913,73 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} } #[test] - fn production_requires_a_verified_package_while_loopback_accepts_authoring() { - let root = tempfile::tempdir().unwrap(); + fn one_verified_load_supplies_the_policy_and_its_package_identity() { + let root = canonical_tempdir(); let package = root.path().join("package"); write_policy(&package); let operator = write_operator( root.path(), operator_value(&package, "operator-controlled-upstream"), ); - assert!(matches!( - RuntimeConfig::load(&operator), - Err(RuntimeConfigError::ProductionPolicyPackageRequired) - )); + let config = RuntimeConfig::load(&operator).expect("configuration is accepted"); - let manifest = PolicyPackageManifest::build(POLICY).unwrap(); - std::fs::write( - package.join(SCHEDULING_PACKAGE_MANIFEST_FILE), - serde_json::to_vec_pretty(&manifest).unwrap(), - ) - .unwrap(); - let config = RuntimeConfig::load(&operator).expect("packaged production is accepted"); + let loaded = config + .load_policy() + .expect("policy and package load together"); + assert_eq!(loaded.policy.scheduling.id, "standalone-exact-time"); assert_eq!( - config.policy_package_digest().unwrap(), - Some(manifest.policy_digest) + loaded.package_digest, + sha256_uri(&std::fs::read(package.join(SUM_FILE)).unwrap()) ); + } - std::fs::write( - package.join(AUTHORED_POLICY_FILE), - POLICY.replace("30-minute", "31-minute"), - ) - .unwrap(); - assert!(matches!( - RuntimeConfig::load(&operator), - Err(RuntimeConfigError::PolicyPackage(_)) - )); + #[test] + fn every_listener_mode_verifies_the_package_without_a_pin() { + for tls in ["development-loopback", "operator-controlled-upstream"] { + let root = canonical_tempdir(); + let package = root.path().join("package"); + write_policy(&package); + let operator = write_operator(root.path(), operator_value(&package, tls)); + let config = RuntimeConfig::load(&operator).expect("the package is admitted"); + assert_eq!( + config.package_digest().unwrap(), + sha256_uri(&std::fs::read(package.join(SUM_FILE)).unwrap()) + ); + + let refused = |named: &str| { + let error = RuntimeConfig::load(&operator).unwrap_err(); + let message = error.to_string(); + assert_eq!(error.path(), "package.root", "{tls}: {message}"); + assert!(message.contains(named), "{tls}: {message}"); + assert!(message.contains(PACKAGE_COMMAND), "{tls}: {message}"); + }; + + let policy = package.join(AUTHORED_POLICY_FILE); + std::fs::write(&policy, format!("{POLICY}# changed\n")).unwrap(); + refused("changed: scheduling.yaml"); + std::fs::write(&policy, POLICY).unwrap(); + + std::fs::write(package.join("notes.txt"), "stale\n").unwrap(); + refused("extra: notes.txt"); + std::fs::remove_file(package.join("notes.txt")).unwrap(); + + std::fs::remove_file(&policy).unwrap(); + refused("missing: scheduling.yaml"); + std::fs::write(&policy, POLICY).unwrap(); + RuntimeConfig::load(&operator).expect("the restored package is admitted"); + + std::fs::write(package.join(RETIRED_PACKAGE_MANIFEST_FILE), "{}").unwrap(); + refused(RETIRED_PACKAGE_MANIFEST_FILE); + std::fs::remove_file(package.join(RETIRED_PACKAGE_MANIFEST_FILE)).unwrap(); + + std::fs::remove_file(package.join(SUM_FILE)).unwrap(); + refused("has no SHA256SUMS"); + } } #[test] fn explain_and_read_scopes_must_differ_and_retention_must_be_positive() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); write_policy(&package); for (patch, expected) in [ @@ -1339,13 +1024,13 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} #[test] fn a_hook_policy_requires_and_accepts_its_deployment_binding() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); write_policy(&package); let hooked = format!( "{POLICY}hooks:\n - id: appointment-observer\n phase: after\n trigger: appointment.confirmed\n projection: []\n handler:\n kind: url\n destinationId: appointment-events\n" ); - std::fs::write(package.join(AUTHORED_POLICY_FILE), hooked).unwrap(); + package_policy(&package, &hooked); let mut document = operator_value(&package, "development-loopback"); let operator = write_operator(root.path(), document.clone()); assert!(matches!( @@ -1366,7 +1051,7 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} #[test] fn typed_parse_path_does_not_echo_the_rejected_value() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); write_policy(&package); let canary = "DO_NOT_DISCLOSE_RUNTIME_VALUE"; @@ -1386,7 +1071,7 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} #[test] fn a_runtime_document_refused_whole_is_reported_at_the_root() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let operator = root.path().join("runtime.yaml"); let canary = "DO_NOT_DISCLOSE_RUNTIME_VALUE"; std::fs::write(&operator, format!("{canary}\n")).unwrap(); @@ -1397,10 +1082,147 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} // document, and "." names nothing an operator can look up. assert_eq!(error.path(), "/"); assert!(!message.contains(" at ."), "{message}"); - assert!(message.contains("invalid type: string"), "{message}"); + assert!(message.contains("must be a YAML mapping"), "{message}"); assert!(!message.contains(canary), "{message}"); } + #[test] + fn the_runtime_configuration_is_read_through_the_shared_loader() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + write_policy(&package); + + let error = RuntimeConfig::load("runtime.yaml").unwrap_err(); + assert!(matches!( + &error, + RuntimeConfigError::Load(load) if load.code() == "runtime_config.path" + )); + + let mut document = operator_value(&package, "development-loopback"); + document["listener"].as_object_mut().unwrap().remove("bind"); + let error = RuntimeConfig::load(write_operator(root.path(), document)).unwrap_err(); + assert_eq!(error.path(), "listener"); + assert!(error.to_string().contains("bind"), "{error}"); + } + + #[test] + fn a_removed_jwks_uri_names_its_replacement() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + write_policy(&package); + let mut document = operator_value(&package, "development-loopback"); + document["authentication"]["oidc"]["jwksUri"] = + serde_json::json!("https://identity.example.test/jwks"); + let error = RuntimeConfig::load(write_operator(root.path(), document)).unwrap_err(); + assert_eq!(error.path(), "authentication.oidc.jwksUri"); + assert!( + error.to_string().contains("authentication.oidc.jwksSource"), + "{error}" + ); + + let mut document = operator_value(&package, "development-loopback"); + document["authentication"]["oidc"]["jwksSource"] = + serde_json::json!({"kind": "uri", "uri": "https://identity.example.test/jwks"}); + let config = RuntimeConfig::load(write_operator(root.path(), document)).unwrap(); + assert_eq!( + config.authentication.oidc.provider.jwks_source.uri(), + Some("https://identity.example.test/jwks") + ); + } + + #[test] + fn environment_expressions_substitute_values_but_never_secret_references() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + write_policy(&package); + let mut document = operator_value(&package, "development-loopback"); + document["authentication"]["oidc"]["audience"] = + serde_json::json!("${SCHEDULING_TEST_AUDIENCE:-urn:example:substituted}"); + let config = RuntimeConfig::load(write_operator(root.path(), document)).unwrap(); + assert_eq!( + config.authentication.oidc.provider.audience, + "urn:example:substituted" + ); + + let mut document = operator_value(&package, "development-loopback"); + document["database"]["runtimeUrlRef"] = + serde_json::json!("${SCHEDULING_TEST_REFERENCE:-secret:env/RUNTIME}"); + let error = RuntimeConfig::load(write_operator(root.path(), document)).unwrap_err(); + assert!(matches!( + &error, + RuntimeConfigError::Load(load) + if load.code() == "runtime_config.substitution_in_reference" + )); + assert_eq!(error.path(), "database.runtimeUrlRef"); + } + + #[test] + fn an_authored_policy_carrying_an_environment_expression_is_refused() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + package_policy( + &package, + &POLICY.replace( + " because: test\nofferings:", + " because: ${OPENING_REASON}\nofferings:", + ), + ); + let operator = write_operator( + root.path(), + operator_value(&package, "development-loopback"), + ); + let error = RuntimeConfig::load(&operator).unwrap_err(); + assert!(matches!( + &error, + RuntimeConfigError::PolicyEnvironmentExpression { field } if field == "openings.0.because" + )); + assert!(error.to_string().contains("runtime.yaml only"), "{error}"); + } + + #[test] + fn a_malformed_authored_policy_is_a_parse_refusal() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + package_policy(&package, &format!("{POLICY}services: [unterminated\n")); + let operator = write_operator( + root.path(), + operator_value(&package, "development-loopback"), + ); + let error = RuntimeConfig::load(&operator).unwrap_err(); + assert!( + matches!(&error, RuntimeConfigError::PolicyParse { .. }), + "{error}" + ); + } + + #[test] + fn a_pinned_package_digest_must_match_the_verified_package() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + write_policy(&package); + let digest = sha256_uri(&std::fs::read(package.join(SUM_FILE)).unwrap()); + + let mut document = operator_value(&package, "operator-controlled-upstream"); + document["package"]["expectedDigest"] = serde_json::json!(digest); + let config = + RuntimeConfig::load(write_operator(root.path(), document)).expect("the pin matches"); + assert_eq!(config.package_digest().unwrap(), digest); + + let mut document = operator_value(&package, "operator-controlled-upstream"); + let expected = format!("sha256:{}", "0".repeat(64)); + document["package"]["expectedDigest"] = serde_json::json!(&expected); + let error = RuntimeConfig::load(write_operator(root.path(), document)).unwrap_err(); + assert!(matches!(&error, RuntimeConfigError::PackageDigest(_))); + assert_eq!(error.path(), "package.expectedDigest"); + assert_eq!( + error.to_string(), + format!( + "package.expectedDigest is {expected} but the package at package.root is \ + {digest}; deploy the pinned package or update package.expectedDigest" + ) + ); + } + #[test] fn a_refused_value_never_survives_the_clause_that_names_it() { // The value serde renders may hold the comma that ends the clause, @@ -1421,7 +1243,7 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} #[test] fn a_runtime_document_the_reader_stops_on_names_the_line_and_column() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let operator = root.path().join("runtime.yaml"); // A tab can never open an indented line, so the reader stops on it. std::fs::write(&operator, "apiVersion: v1\n\tkind: x\n").unwrap(); @@ -1434,7 +1256,7 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} #[test] fn a_rejected_runtime_member_names_the_cause_without_the_value() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); write_policy(&package); let canary = "DO_NOT_DISCLOSE_RUNTIME_VALUE"; @@ -1456,14 +1278,12 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} #[test] fn a_refused_authored_policy_names_the_member_and_the_cause() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); - std::fs::create_dir_all(&package).unwrap(); - std::fs::write( - package.join(AUTHORED_POLICY_FILE), + package_policy( + &package, "scheduling: {id: standalone-exact-time, version: one}\n", - ) - .unwrap(); + ); let operator = write_operator( root.path(), operator_value(&package, "development-loopback"), @@ -1478,56 +1298,11 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} assert!(message.contains("column"), "{message}"); } - // The package manifest and the policy it identifies are two documents - // with two readers: the manifest is verified by the runtime at startup, - // the policy is parsed by the authoring grammar. A reader that accepts - // one must be able to refuse the other on its declared names alone. - #[test] - fn a_package_manifest_is_a_different_document_from_the_policy_it_identifies() { - let manifest = PolicyPackageManifest::build(POLICY).expect("the policy is packaged"); - assert_ne!(manifest.api_version, SCHEDULING_POLICY_API_VERSION); - assert_ne!(manifest.kind, SCHEDULING_POLICY_KIND); - assert_eq!( - manifest.api_version, - SCHEDULING_PACKAGE_MANIFEST_API_VERSION - ); - assert_eq!(manifest.kind, SCHEDULING_PACKAGE_MANIFEST_KIND); - } - - // A deployment identity is only an identity if a document of another kind - // cannot stand in for it. The manifest is verified against its own names, - // so an authored policy's names never carry a package identity. - #[test] - fn a_manifest_wearing_the_authored_policy_names_is_refused() { - let root = tempfile::tempdir().unwrap(); - let package = root.path().join("package"); - write_policy(&package); - let manifest = PolicyPackageManifest::build(POLICY).expect("the policy is packaged"); - let mut document = serde_json::to_value(&manifest).unwrap(); - document["apiVersion"] = serde_json::json!(SCHEDULING_POLICY_API_VERSION); - document["kind"] = serde_json::json!(SCHEDULING_POLICY_KIND); - std::fs::write( - package.join(SCHEDULING_PACKAGE_MANIFEST_FILE), - serde_json::to_vec_pretty(&document).unwrap(), - ) - .unwrap(); - assert!(matches!( - verify_policy_package(&package.join(AUTHORED_POLICY_FILE), POLICY), - Err(PolicyPackageError::Invalid) - )); - } - #[test] fn a_production_deployment_must_name_the_clients_it_admits() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); write_policy(&package); - let manifest = PolicyPackageManifest::build(POLICY).unwrap(); - std::fs::write( - package.join(SCHEDULING_PACKAGE_MANIFEST_FILE), - serde_json::to_vec_pretty(&manifest).unwrap(), - ) - .unwrap(); let mut document = operator_value(&package, "operator-controlled-upstream"); document["authentication"]["oidc"] @@ -1562,7 +1337,7 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} #[test] fn a_declared_assertion_authority_binds_the_client_that_may_exchange_from_it() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); write_policy(&package); let mut document = operator_value(&package, "development-loopback"); @@ -1583,13 +1358,13 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} #[test] fn an_assertion_issuer_map_outside_its_bounds_is_refused() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); write_policy(&package); - let oversized_client = "c".repeat(MAXIMUM_ASSERTION_ISSUER_CLIENT_BYTES + 1); - let oversized_issuer = format!("https://{}", "a".repeat(MAXIMUM_ASSERTION_ISSUER_BYTES)); + let oversized_client = "c".repeat(MAX_ASSERTION_ISSUER_CLIENT_BYTES + 1); + let oversized_issuer = format!("https://{}", "a".repeat(MAX_ASSERTION_ISSUER_BYTES)); let too_many_clients: serde_json::Map = (0 - ..=MAXIMUM_ASSERTION_ISSUER_CLIENTS) + ..=MAX_ASSERTION_ISSUER_CLIENTS) .map(|index| { ( format!("client-{index}"), @@ -1597,7 +1372,7 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} ) }) .collect(); - let too_many_issuers: Vec = (0..=MAXIMUM_ASSERTION_ISSUERS_PER_CLIENT) + let too_many_issuers: Vec = (0..=MAX_ASSERTION_ISSUERS_PER_CLIENT) .map(|index| format!("https://authority-{index}.test")) .collect(); for refused in [ @@ -1612,17 +1387,42 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} let mut document = operator_value(&package, "development-loopback"); document["authentication"]["oidc"]["assertionIssuers"] = refused.clone(); let operator = write_operator(root.path(), document); - let outcome = RuntimeConfig::load(&operator); - assert!( - matches!(outcome, Err(RuntimeConfigError::InvalidOidc)), + let error = RuntimeConfig::load(&operator).expect_err(&format!( "an assertion-issuer map outside its bounds was accepted: {refused}" + )); + assert!( + matches!(error, RuntimeConfigError::Block(_)), + "{refused}: {error:?}" ); + assert_eq!(error.path(), "authentication.oidc.assertionIssuers"); } } + #[test] + fn the_oidc_issuer_and_clients_are_the_shared_blocks() { + let root = canonical_tempdir(); + let package = root.path().join("package"); + write_policy(&package); + + let mut document = operator_value(&package, "development-loopback"); + document["authentication"]["oidc"]["issuer"] = + serde_json::json!("http://identity.example.test"); + let operator = write_operator(root.path(), document); + let error = RuntimeConfig::load(&operator).expect_err("a remote http issuer"); + assert_eq!(error.path(), "authentication.oidc.issuer"); + + // The flattened blocks leave the enclosing block closed. + let mut document = operator_value(&package, "development-loopback"); + document["authentication"]["oidc"]["issuerr"] = + serde_json::json!("https://identity.example.test"); + let operator = write_operator(root.path(), document); + let error = RuntimeConfig::load(&operator).expect_err("an unknown oidc member"); + assert!(error.to_string().contains("issuerr"), "{error}"); + } + #[test] fn the_access_token_profile_admits_only_the_rfc_9068_pair() { - let root = tempfile::tempdir().unwrap(); + let root = canonical_tempdir(); let package = root.path().join("package"); write_policy(&package); let operator = write_operator( @@ -1648,43 +1448,101 @@ holdPolicy: {ttlMinutes: 10, maxPerCaller: 2, because: test} #[test] fn listener_requires_a_private_address_or_explicit_container_network() { - use std::net::{Ipv4Addr, Ipv6Addr}; - assert!(valid_listener( - IpAddr::V4(Ipv4Addr::LOCALHOST), + let listener = |bind: &str, exposure, tls_termination| ListenerConfig { + bind: bind.parse().unwrap(), + tls_termination, + network_exposure: exposure, + }; + assert!(listener( + "127.0.0.1:8105", ListenerNetworkExposure::PrivateAddress, TlsTermination::OperatorControlledUpstream, - )); - assert!(!valid_listener( - "203.0.113.10".parse::().unwrap(), + ) + .is_valid()); + assert!(!listener( + "203.0.113.10:8105", ListenerNetworkExposure::ContainerPrivate, TlsTermination::OperatorControlledUpstream, - )); - assert!(valid_listener( - IpAddr::V6(Ipv6Addr::UNSPECIFIED), + ) + .is_valid()); + assert!(listener( + "[::]:8105", ListenerNetworkExposure::ContainerPrivate, TlsTermination::OperatorControlledUpstream, - )); - assert!(!valid_listener( - "10.20.30.40".parse::().unwrap(), + ) + .is_valid()); + assert!(!listener( + "10.20.30.40:8105", ListenerNetworkExposure::PrivateAddress, TlsTermination::DevelopmentLoopback, + ) + .is_valid()); + + // The runtime refuses the same listener at load. + let root = canonical_tempdir(); + let package = root.path().join("package"); + write_policy(&package); + let mut document = operator_value(&package, "development-loopback"); + document["listener"]["bind"] = serde_json::json!("10.20.30.40:8105"); + let operator = write_operator(root.path(), document); + assert!(matches!( + RuntimeConfig::load(&operator), + Err(RuntimeConfigError::InvalidListener) )); } - #[test] - fn static_jwks_requires_unique_named_asymmetric_keys() { - assert!(parse_static_jwks( - br#"{"keys":[{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"}]}"# - ) - .is_ok()); - for invalid in [ - br#"{}"#.as_slice(), - br#"{"keys":[]}"#, - br#"{"keys":[{"kty":"oct","kid":"one","k":"AA"}]}"#, - br#"{"keys":[{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"},{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"}]}"#, - b"not-json", + /// The runtime's static JWKS arm hands the resolved document to the + /// shared parser and refuses a key set it cannot trust before any + /// verifier exists: a symmetric key, an unnamed key, two keys sharing a + /// name, or no key at all. + #[cfg(unix)] + #[tokio::test] + async fn static_jwks_requires_unique_named_asymmetric_keys() { + use std::os::unix::fs::PermissionsExt as _; + + let root = canonical_tempdir(); + let package = root.path().join("package"); + write_policy(&package); + let secrets_root = root.path().join("secrets"); + std::fs::create_dir(&secrets_root).unwrap(); + let mut document = operator_value(&package, "development-loopback"); + document["authentication"]["oidc"]["jwksSource"] = + serde_json::json!({"kind": "static", "documentRef": "secret:file/jwks.json"}); + let config = RuntimeConfig::load(write_operator(root.path(), document)).unwrap(); + let secrets = config.secret_providers.resolver().unwrap(); + let jwks = secrets_root.join("jwks.json"); + + let write_jwks = |bytes: &[u8]| { + std::fs::write(&jwks, bytes).unwrap(); + std::fs::set_permissions(&jwks, std::fs::Permissions::from_mode(0o600)).unwrap(); + }; + + write_jwks(br#"{"keys":[{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"}]}"#); + config + .oidc_verifier(&secrets) + .await + .expect("one named asymmetric key is accepted"); + + for (case, invalid) in [ + ("symmetric", br#"{"keys":[{"kty":"oct","kid":"one","k":"AA"}]}"#.as_slice()), + ("unnamed", br#"{"keys":[{"kty":"RSA","n":"AQAB","e":"AQAB"}]}"#), + ( + "duplicate name", + br#"{"keys":[{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"},{"kty":"RSA","kid":"one","n":"AQAB","e":"AQAB"}]}"#, + ), + ("empty", br#"{"keys":[]}"#), ] { - assert!(parse_static_jwks(invalid).is_err()); + write_jwks(invalid); + let error = config + .oidc_verifier(&secrets) + .await + .map(|_| ()) + .expect_err(case); + assert!( + matches!(error, RuntimeConfigError::Oidc), + "{case}: {error}" + ); + assert_eq!(error.path(), "authentication.oidc", "{case}"); } } } diff --git a/crates/registry-scheduling/src/runtime.rs b/crates/registry-scheduling/src/runtime.rs index 4410715197..d12deaa870 100644 --- a/crates/registry-scheduling/src/runtime.rs +++ b/crates/registry-scheduling/src/runtime.rs @@ -28,7 +28,7 @@ use chrono::{DateTime, TimeDelta, Utc}; use clap::{Arg, Command}; use registry_platform_audit::{AuditProfile, AuditWriter}; use registry_platform_canonical_json::canonicalize_json; -use registry_platform_config::{ProtectedSecret, SecretProvider, SecretResolver}; +use registry_platform_config::{ProtectedSecret, SecretResolver}; use registry_platform_httputil::destination::{ DataDestinationPolicy, DataDestinationRequestTemplate, DestinationAuthorizationTemplate, DestinationAuthorizationValue, DestinationBodyTemplate, DestinationMethod, DestinationProfile, @@ -112,7 +112,7 @@ fn database_step(stage: &'static str) -> impl Fn(StoreError) -> RuntimeError { pub async fn migrate_from_path(path: impl AsRef) -> Result<(), RuntimeError> { let config = RuntimeConfig::load(path)?; - let policy = config.load_policy()?; + let policy = config.load_policy()?.policy; let secrets = secret_resolver(&config)?; let store = PostgresStore::connect_migration(&config.database, &secrets) .map_err(database_step("migration database configuration"))?; @@ -132,14 +132,10 @@ pub async fn migrate_from_path(path: impl AsRef) -> Result<(), RuntimeErro pub async fn serve_from_path(path: impl AsRef) -> Result<(), RuntimeError> { let config = RuntimeConfig::load(path)?; - match config.policy_package_digest()? { - Some(digest) => tracing::info!( - policy_package_digest = %digest, - "verified Scheduling policy package" - ), - None => tracing::info!("loading authored Scheduling policy for loopback development"), - } - let policy = config.load_policy()?; + let loaded = config.load_policy()?; + let package_digest = loaded.package_digest; + tracing::info!(package_digest = %package_digest, "verified Scheduling package"); + let policy = loaded.policy; let scheduling_id = policy.scheduling.id.clone(); let policy_digest = policy.policy_digest(); let secrets = secret_resolver(&config)?; @@ -372,7 +368,7 @@ pub async fn serve_from_path(path: impl AsRef) -> Result<(), RuntimeError> authenticator, store, }); - let listener = tokio::net::TcpListener::bind(config.listener.bind).await?; + let listener = tokio::net::TcpListener::bind(config.listener.bind.socket_addr()).await?; let served = serve_until_worker_stops(listener, app, worker_stops).await; for worker in workers { worker.abort(); @@ -495,7 +491,7 @@ pub async fn open_audit( secrets: &SecretResolver, process: Option<&str>, ) -> Result<(registry_platform_audit::AuditKeyHasher, SchedulingAudit), RuntimeError> { - let audit_secret = resolve_audit_secret(secrets, &config.audit.hash_key_ref)?; + let audit_secret = resolve_audit_secret(secrets, config.audit.key.hash_key_ref.as_str())?; let audit_profile = AuditProfile::production_from_secret_bytes(audit_secret.expose_secret().to_vec().into()) .map_err(|_| RuntimeError::Audit)?; @@ -532,22 +528,10 @@ fn resolve_audit_secret( } pub fn secret_resolver(config: &RuntimeConfig) -> Result { - let mut providers = Vec::new(); - if config.secret_providers.file.is_some() { - providers.push(SecretProvider::File); - } - if config.secret_providers.environment.is_some() { - providers.push(SecretProvider::Environment); - } - SecretResolver::new( - providers, - config - .secret_providers - .file - .as_ref() - .map_or_else(|| Path::new(""), |file| file.root.as_path()), - ) - .map_err(|_| RuntimeError::SecretConfiguration) + config + .secret_providers + .resolver() + .map_err(|_| RuntimeError::SecretConfiguration) } /// The pool anchors the policy's exact-time offerings name. diff --git a/crates/registry-scheduling/src/schema.rs b/crates/registry-scheduling/src/schema.rs index 944bc71b53..8c97083182 100644 --- a/crates/registry-scheduling/src/schema.rs +++ b/crates/registry-scheduling/src/schema.rs @@ -23,13 +23,9 @@ use registry_scheduling_core::{ SCHEDULING_RUNTIME_SCHEMA_ID, }; -use crate::config::{ - RuntimeConfig, MAXIMUM_ASSERTION_ISSUERS_PER_CLIENT, MAXIMUM_ASSERTION_ISSUER_BYTES, - MAXIMUM_ASSERTION_ISSUER_CLIENTS, MAXIMUM_ASSERTION_ISSUER_CLIENT_BYTES, -}; +use registry_platform_config::blocks::SECRET_PROVIDER_PATTERN; -const SECRET_REFERENCE_SCHEMA_PATTERN: &str = - "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$"; +use crate::config::RuntimeConfig; pub fn runtime_documents() -> Result, serde_json::Error> { let mut derived = serde_json::to_value(schemars::schema_for!(RuntimeConfig))?; @@ -58,23 +54,12 @@ pub fn runtime_documents() -> Result, serde_json: } /// State in the schema the bounds `RuntimeConfig::check` and -/// `validate_secret_references` enforce at load: operated paths are absolute, -/// every secret field is a secret reference, a static JWKS document names its -/// provider, and at least one secret provider is configured. +/// `validate_secret_references` enforce at load beyond the shared blocks, +/// which carry their own: the audit destination has the shape +/// `AuditConfig::destination` requires and its file is absolute with no `..` +/// segment, every Scheduling secret field is a secret reference, and a static +/// JWKS document names an enabled provider. fn install_runtime_constraints(schema: &mut Value) { - for (definition, property) in [ - ("RuntimePackageConfig", "root"), - ("FileSecretProviderConfig", "root"), - ] { - set_definition_property( - schema, - definition, - property, - "pattern", - Value::String("^/".to_owned()), - ); - } - // The audit file is also refused with a `..` segment. set_definition_property( schema, "AuditConfig", @@ -83,43 +68,22 @@ fn install_runtime_constraints(schema: &mut Value) { Value::String(registry_platform_audit::ABSOLUTE_AUDIT_PATH_PATTERN.to_owned()), ); for (definition, property) in [ - ("DatabaseConfig", "runtimeUrlRef"), - ("DatabaseConfig", "migrationUrlRef"), - ("DatabaseConfig", "trustedRootCertificateRef"), - ("AuditConfig", "hashKeyRef"), ("ReminderDestinationConfig", "bearerTokenRef"), + ("HookDestinationConfig", "hmacSha256KeyRef"), ] { set_definition_property( schema, definition, property, "pattern", - Value::String("^secret:(?:env|file)/".to_owned()), + Value::String(SECRET_PROVIDER_PATTERN.to_owned()), ); } - set_jwks_document_reference_constraints(schema); - set_assertion_issuer_bounds(schema); set_audit_destination_constraints(schema); - if let Some(providers) = schema - .get_mut("$defs") - .and_then(|definitions| definitions.get_mut("SecretProvidersConfig")) - .and_then(Value::as_object_mut) - { - providers.insert( - "anyOf".to_owned(), - serde_json::json!([ - {"required": ["file"], "properties": {"file": {"$ref": "#/$defs/FileSecretProviderConfig"}}}, - {"required": ["environment"], "properties": {"environment": {"$ref": "#/$defs/EnvironmentSecretProviderConfig"}}} - ]), - ); - } if let Some(root) = schema.as_object_mut() { root.insert( "allOf".to_owned(), - serde_json::json!([ - secret_provider_requirement("^secret:env/", "environment"), - secret_provider_requirement("^secret:file/", "file") - ]), + registry_platform_config::schema::jwks_document_provider_requirements(), ); } } @@ -189,106 +153,6 @@ fn set_audit_destination_constraints(schema: &mut Value) { } } -/// A static JWKS document reference is the one secret field whose full -/// reference grammar the schema states: the other fields need only name a -/// provider, while this one an operator authors directly. -fn set_jwks_document_reference_constraints(schema: &mut Value) { - if let Some(variants) = schema - .pointer_mut("/$defs/OidcJwksSource/oneOf") - .and_then(Value::as_array_mut) - { - for variant in variants { - if let Some(document_reference) = variant - .pointer_mut("/properties/documentRef") - .and_then(Value::as_object_mut) - { - document_reference.insert( - "pattern".to_owned(), - Value::String(SECRET_REFERENCE_SCHEMA_PATTERN.to_owned()), - ); - } - } - } -} - -/// State the assertion-issuer map's authored bounds, the ones -/// `RuntimeConfig::check` refuses a document for exceeding. -fn set_assertion_issuer_bounds(schema: &mut Value) { - if let Some(property) = schema - .pointer_mut("/$defs/OidcConfig/properties/assertionIssuers") - .and_then(Value::as_object_mut) - { - property.insert( - "maxProperties".to_owned(), - Value::from(MAXIMUM_ASSERTION_ISSUER_CLIENTS), - ); - property.insert( - "propertyNames".to_owned(), - serde_json::json!({ - "minLength": 1, - "maxLength": MAXIMUM_ASSERTION_ISSUER_CLIENT_BYTES, - }), - ); - property.insert( - "additionalProperties".to_owned(), - serde_json::json!({ - "type": "array", - "maxItems": MAXIMUM_ASSERTION_ISSUERS_PER_CLIENT, - "uniqueItems": true, - "items": { - "type": "string", - "minLength": 1, - "maxLength": MAXIMUM_ASSERTION_ISSUER_BYTES, - }, - }), - ); - } -} - -/// A static JWKS document reference names its provider by its prefix, so a -/// configuration carrying one must enable that provider. -fn secret_provider_requirement(reference_pattern: &str, provider: &str) -> Value { - serde_json::json!({ - "if": { - "properties": { - "authentication": { - "properties": { - "oidc": { - "properties": { - "jwksSource": { - "properties": { - "documentRef": {"pattern": reference_pattern} - }, - "required": ["documentRef"] - } - }, - "required": ["jwksSource"] - } - }, - "required": ["oidc"] - } - }, - "required": ["authentication"] - }, - "then": { - "properties": { - "secretProviders": { - "properties": { - provider: { - "$ref": format!("#/$defs/{}SecretProviderConfig", match provider { - "environment" => "Environment", - "file" => "File", - _ => unreachable!("closed secret provider schema"), - }) - } - }, - "required": [provider] - } - } - } - }) -} - fn set_definition_property( schema: &mut Value, definition: &str, @@ -320,6 +184,7 @@ fn set_const(schema: &mut Value, property: &str, expected: &str) { #[cfg(test)] mod tests { use super::*; + use registry_platform_config::blocks::SECRET_REFERENCE_PATTERN; #[test] fn runtime_schema_is_deterministic_and_versioned() { @@ -351,7 +216,7 @@ mod tests { let documents = runtime_documents().unwrap(); let document: Value = serde_json::from_str(&documents[RUNTIME_SCHEMA_FILE]).unwrap(); for (definition, property) in [ - ("RuntimePackageConfig", "root"), + ("PackageConfig", "root"), ("FileSecretProviderConfig", "root"), ] { assert_eq!( @@ -368,8 +233,8 @@ mod tests { ("DatabaseConfig", "runtimeUrlRef"), ("DatabaseConfig", "migrationUrlRef"), ("DatabaseConfig", "trustedRootCertificateRef"), - ("AuditConfig", "hashKeyRef"), ("ReminderDestinationConfig", "bearerTokenRef"), + ("HookDestinationConfig", "hmacSha256KeyRef"), ] { assert_eq!( document["$defs"][definition]["properties"][property]["pattern"], @@ -386,8 +251,25 @@ mod tests { "at least one secret provider must be configured" ); assert_eq!( - document["$defs"]["OidcJwksSource"]["oneOf"][1]["properties"]["documentRef"]["pattern"], - SECRET_REFERENCE_SCHEMA_PATTERN + document["$defs"]["JwksSource"]["oneOf"][2]["properties"]["documentRef"]["pattern"], + SECRET_REFERENCE_PATTERN + ); + // The shared blocks carry their own bounds into this schema. + assert_eq!( + document["$defs"]["AuditConfig"]["properties"]["hashKeyRef"]["$ref"], + "#/$defs/SecretReference" + ); + let assertion_issuers = &document["$defs"]["OidcConfig"]["properties"]["assertionIssuers"]; + assert_eq!(assertion_issuers["maxProperties"], 64); + assert_eq!(assertion_issuers["propertyNames"]["maxLength"], 128); + assert_eq!(assertion_issuers["additionalProperties"]["maxItems"], 16); + assert_eq!( + assertion_issuers["additionalProperties"]["items"]["maxLength"], + 512 + ); + assert_eq!( + document["$defs"]["OidcConfig"]["properties"]["issuer"]["pattern"], + "^https?://" ); let audit = &document["$defs"]["AuditConfig"]; assert_eq!( diff --git a/crates/registry-scheduling/tests/postgres_commitments.rs b/crates/registry-scheduling/tests/postgres_commitments.rs index 558f7e9a43..c78372c3c7 100644 --- a/crates/registry-scheduling/tests/postgres_commitments.rs +++ b/crates/registry-scheduling/tests/postgres_commitments.rs @@ -25,7 +25,9 @@ use registry_platform_hooks::{EnvelopeLimits, HookEnvelope, HookHandlerSource}; use registry_platform_oidc::{JwksFetcher, JwksFetcherConfig, TokenVerifierConfig}; use registry_scheduling::audit::{AuditCapture, SchedulingAudit, SCHEDULING_AUDIT_SCHEMA}; use registry_scheduling::auth::SchedulingAuthenticator; -use registry_scheduling::config::{DatabaseConfig, OidcConfig, OidcJwksSource}; +use registry_scheduling::config::{ + DatabaseConfig, JwksSource, OidcClientsConfig, OidcConfig, OidcIssuerConfig, +}; use registry_scheduling::config::{HookDestinationConfig, ReminderDestinationConfig}; use registry_scheduling::hooks::{ActivatedHooks, HookRuntimeIdentity}; use registry_scheduling::http::{router, HttpState}; @@ -416,12 +418,15 @@ async fn fixture_publishing_with_hook_url( fn authenticator() -> SchedulingAuthenticator { let oidc = OidcConfig { - allowed_clients: vec![CLIENT.to_owned()], - assertion_issuers: std::collections::BTreeMap::new(), - issuer: ISSUER.to_owned(), - audience: AUDIENCE.to_owned(), - jwks_uri: None, - jwks_source: OidcJwksSource::Discovery, + provider: OidcIssuerConfig { + issuer: ISSUER.to_owned(), + audience: AUDIENCE.to_owned(), + jwks_source: JwksSource::Discovery {}, + }, + clients: OidcClientsConfig { + allowed_clients: vec![CLIENT.to_owned()], + assertion_issuers: std::collections::BTreeMap::new(), + }, scope_claim: "registry_scopes".to_owned(), reads_scope: "scheduling-read".to_owned(), explain_scope: "scheduling-explain".to_owned(), @@ -3300,6 +3305,13 @@ fn unreachable_deployment(root: &std::path::Path) -> std::path::PathBuf { POLICY, ) .expect("the authored policy"); + registry_platform_config::package::write_sum_file( + &package, + None, + ®istry_scheduling::config::package_limits(), + registry_scheduling::config::PACKAGE_COMMAND, + ) + .expect("the package is sealed"); let secret_name = format!("SCHEDULING_CLOSED_{}", Uuid::new_v4().simple()).to_ascii_uppercase(); std::env::set_var( &secret_name, @@ -3336,7 +3348,8 @@ fn unreachable_deployment(root: &std::path::Path) -> std::path::PathBuf { #[tokio::test] async fn a_database_that_refuses_at_startup_names_the_step_and_the_cause() { - let root = tempfile::tempdir().expect("a temporary deployment root"); + let root = tempfile::tempdir_in(std::env::temp_dir().canonicalize().unwrap()) + .expect("a temporary deployment root"); let operator = unreachable_deployment(root.path()); let failure = registry_scheduling::runtime::migrate_from_path(&operator) .await diff --git a/crates/registry-schedulingctl/src/lib.rs b/crates/registry-schedulingctl/src/lib.rs index 7dc829d9ee..1b61e0889e 100644 --- a/crates/registry-schedulingctl/src/lib.rs +++ b/crates/registry-schedulingctl/src/lib.rs @@ -50,8 +50,8 @@ enum Command { Test(ProjectArgs), /// Explain the checked policy offline: what the runtime would publish. Explain(ProjectArgs), - /// Write the verified policy package manifest beside the authored policy. - Package(ProjectArgs), + /// Write the checked policy into a new package directory the runtime verifies. + Package(PackageArgs), /// Apply the live environment records of a deployment. Records(RecordsArgs), /// List delivery intents a deployment's sweep has stopped carrying. @@ -85,6 +85,22 @@ struct ProjectArgs { project: PathBuf, } +#[derive(Debug, Args)] +struct PackageArgs { + /// Authored scheduling project directory. + #[arg(value_name = "PROJECT")] + project: PathBuf, + /// New directory for the verified package. Required unless --dry-run. + #[arg(long, value_name = "DIRECTORY", required_unless_present = "dry_run")] + output: Option, + /// Report the same packageDigest and files a package would produce, without writing one. + #[arg(long, conflicts_with = "output", required_unless_present = "output")] + dry_run: bool, + /// Free-text revision recorded in the package's REVISION file and covered by its digest. + #[arg(long, value_name = "TEXT")] + revision: Option, +} + #[derive(Debug, Args)] struct RecordsArgs { #[command(subcommand)] @@ -214,7 +230,10 @@ fn run(cli: Cli) -> Result { Command::Check(args) => project::check(&args.project), Command::Test(args) => project::test(&args.project), Command::Explain(args) => project::explain(&args.project), - Command::Package(args) => project::package(&args.project), + Command::Package(args) => match args.output { + Some(output) => project::package(&args.project, &output, args.revision.as_deref()), + None => project::package_dry_run(&args.project, args.revision.as_deref()), + }, Command::Records(args) => match args.command { RecordsCommand::Apply(apply) => records::apply(&apply.config, &apply.records), }, @@ -396,9 +415,10 @@ fn human_lead(report: &Value) -> String { "Offline synthetic fixtures passed with incomplete authored inputs.".to_owned() } ("test", _, _) => "Offline synthetic fixtures passed.".to_owned(), - ("package", _, _) => { - "Policy package manifest written beside the authored policy.".to_owned() + ("package", _, _) if report["dryRun"] == true => { + "Package planned; nothing was written.".to_owned() } + ("package", _, _) => "Package written.".to_owned(), ("records-apply", _, _) => "Environment records applied.".to_owned(), ("intents", _, _) => "Undelivered delivery intents listed.".to_owned(), _ => format!("{command} succeeded."), @@ -460,7 +480,11 @@ mod tests { } fn initialized(template: &str) -> (tempfile::TempDir, PathBuf) { - let root = tempfile::tempdir().unwrap(); + // Canonical, because the runtime configuration loader refuses a path + // reached through a symbolic link and the system temporary + // directory is one on some platforms. + let root = + tempfile::tempdir_in(std::fs::canonicalize(std::env::temp_dir()).unwrap()).unwrap(); let project = root.path().join("project"); project::init(&project, template).unwrap(); (root, project) @@ -522,14 +546,49 @@ mod tests { }; assert_eq!(args.template, "standalone-exact-time"); - for command in ["test", "explain", "package"] { + for command in ["test", "explain"] { let cli = Cli::try_parse_from(["schedulingctl", command, "/tmp/project"]).unwrap(); match command { "test" => assert!(matches!(cli.command, Command::Test(_))), - "explain" => assert!(matches!(cli.command, Command::Explain(_))), - _ => assert!(matches!(cli.command, Command::Package(_))), + _ => assert!(matches!(cli.command, Command::Explain(_))), } } + // `package` writes a new directory or plans one, never both and + // never beside the project. + assert!(Cli::try_parse_from(["schedulingctl", "package", "/tmp/project"]).is_err()); + assert!(Cli::try_parse_from([ + "schedulingctl", + "package", + "/tmp/project", + "--output", + "/tmp/package", + "--dry-run", + ]) + .is_err()); + let Command::Package(args) = Cli::try_parse_from([ + "schedulingctl", + "package", + "/tmp/project", + "--output", + "/tmp/package", + "--revision", + "change 42", + ]) + .unwrap() + .command + else { + panic!("expected package") + }; + assert_eq!(args.output, Some(PathBuf::from("/tmp/package"))); + assert_eq!(args.revision.as_deref(), Some("change 42")); + let Command::Package(args) = + Cli::try_parse_from(["schedulingctl", "package", "/tmp/project", "--dry-run"]) + .unwrap() + .command + else { + panic!("expected package") + }; + assert!(args.dry_run && args.output.is_none()); let cli = Cli::try_parse_from([ "schedulingctl", @@ -910,53 +969,85 @@ mod tests { } #[test] - fn package_writes_a_verifiable_manifest_and_refuses_replacement() { - let (_root, project) = initialized("standalone-exact-time"); - let (exit, report, stderr) = run_json(&["package", project.to_str().unwrap()]); - assert_eq!(exit, ExitCode::SUCCESS); + fn package_writes_a_package_the_runtime_verifies_and_refuses_replacement() { + let (root, project) = initialized("standalone-exact-time"); + let output = root.path().join("package"); + let (exit, report, stderr) = run_json(&[ + "package", + project.to_str().unwrap(), + "--output", + output.to_str().unwrap(), + ]); + assert_eq!(exit, ExitCode::SUCCESS, "{report}"); assert!(stderr.is_empty()); assert_eq!(report["command"], "package"); - assert!(report["packageDigest"] - .as_str() - .unwrap() - .starts_with("sha256:")); - assert!(report["policyDigest"] - .as_str() - .unwrap() - .starts_with("sha256:")); - // The package identity and the policy's own digest are different - // documents and must never be conflated. - assert_ne!(report["packageDigest"], report["policyDigest"]); + assert_eq!(report["dryRun"], false); + assert_eq!(report["revision"], Value::Null); assert_eq!(report["files"][0]["path"], "scheduling.yaml"); + assert_eq!(report["files"].as_array().unwrap().len(), 1); assert_eq!(report["runtimeConfigurationIncluded"], false); assert_eq!(report["secretsIncluded"], false); + assert!(report.get("policyDigest").is_none(), "{report}"); + // The package digest is the digest of SHA256SUMS, the file the + // runtime verifies the package against. + let sums = std::fs::read(output.join("SHA256SUMS")).unwrap(); + assert_eq!( + report["packageDigest"], + registry_platform_config::sha256_uri(&sums) + ); + let verified = registry_scheduling::config::verify_scheduling_package( + ®istry_platform_config::PackageConfig { + root: output.clone(), + expected_digest: None, + }, + ) + .expect("the runtime verifies the written package"); + assert_eq!(verified.digest(), report["packageDigest"]); + assert_eq!( + std::fs::read(output.join("scheduling.yaml")).unwrap(), + std::fs::read(project.join("scheduling.yaml")).unwrap() + ); + assert!(!project.join("SHA256SUMS").exists()); - let manifest_path = project.join("scheduling.package.json"); - assert!(manifest_path.is_file()); - // The written manifest is exactly the one the runtime's own verifier - // accepts against the same policy text. - let policy_path = project.join("scheduling.yaml"); - let policy_text = std::fs::read_to_string(&policy_path).unwrap(); - let verified = - registry_scheduling::config::verify_policy_package(&policy_path, &policy_text) - .unwrap() - .expect("the written manifest verifies"); - // `verify_policy_package` returns the manifest's own byte-exact - // digest, which the on-disk manifest calls `policyDigest`; the - // report must mirror that naming, not the policy's semantic digest. - assert_eq!(verified, report["policyDigest"].as_str().unwrap()); - // Pretty-printed, newline-terminated: a text document an operator - // diffs. - let bytes = std::fs::read(&manifest_path).unwrap(); - assert_eq!(bytes.last(), Some(&b'\n')); - - // Repackaging is a deliberate act: the existing manifest is refused, - // never silently replaced. - let (exit, report, stderr) = run_json(&["package", project.to_str().unwrap()]); + // A dry run reports the digest the written package carries. + let (exit, planned, _) = run_json(&["package", project.to_str().unwrap(), "--dry-run"]); + assert_eq!(exit, ExitCode::SUCCESS); + assert_eq!(planned["dryRun"], true); + assert_eq!(planned["packageDigest"], report["packageDigest"]); + + // Packaging the same project twice gives the same digest; a revision + // is covered by it. + let again = root.path().join("again"); + let (_, repeated, _) = run_json(&[ + "package", + project.to_str().unwrap(), + "--output", + again.to_str().unwrap(), + ]); + assert_eq!(repeated["packageDigest"], report["packageDigest"]); + let revised = root.path().join("revised"); + let (_, revised_report, _) = run_json(&[ + "package", + project.to_str().unwrap(), + "--output", + revised.to_str().unwrap(), + "--revision", + "change 42", + ]); + assert_eq!(revised_report["revision"], "change 42"); + assert_ne!(revised_report["packageDigest"], report["packageDigest"]); + + // A package is written once: an existing output is refused. + let (exit, report, stderr) = run_json(&[ + "package", + project.to_str().unwrap(), + "--output", + output.to_str().unwrap(), + ]); assert_eq!(exit, ExitCode::from(DOMAIN_REFUSAL_EXIT)); assert!(stderr.is_empty()); let message = report["diagnostics"][0]["message"].as_str().unwrap(); - assert!(message.contains("already exists"), "{message}"); + assert!(message.contains("schedulingctl package"), "{message}"); } #[test] @@ -969,19 +1060,50 @@ mod tests { 1, ); std::fs::write(&policy_path, broken).unwrap(); - let (exit, report, stderr) = run_json(&["package", project.to_str().unwrap()]); + let output = project.with_file_name("package"); + let (exit, report, stderr) = run_json(&[ + "package", + project.to_str().unwrap(), + "--output", + output.to_str().unwrap(), + ]); assert_eq!(exit, ExitCode::from(DOMAIN_REFUSAL_EXIT)); assert!(stderr.is_empty()); assert!(report["diagnostics"][0]["message"] .as_str() .unwrap() .contains("finding")); - assert!(!project.join("scheduling.package.json").exists()); + assert!(!output.exists()); + } + + #[test] + fn an_authored_policy_carrying_an_environment_expression_is_refused() { + let (_root, project) = initialized("standalone-exact-time"); + let policy_path = project.join(AUTHORED_POLICY_FILE); + let policy = std::fs::read_to_string(&policy_path).unwrap(); + let (line, _) = policy + .lines() + .find_map(|line| { + line.trim_start() + .strip_prefix("because: ") + .map(|v| (line, v)) + }) + .expect("the template states a reason"); + let indent = &line[..line.len() - line.trim_start().len()]; + std::fs::write( + &policy_path, + policy.replacen(line, &format!("{indent}because: ${{REASON}}"), 1), + ) + .unwrap(); + let (exit, report, _) = run_json(&["check", project.to_str().unwrap()]); + assert_eq!(exit, ExitCode::from(DOMAIN_REFUSAL_EXIT)); + let message = report["diagnostics"][0]["message"].as_str().unwrap(); + assert!(message.contains("runtime.yaml only"), "{message}"); } #[test] fn runtime_configuration_and_store_failures_map_to_their_own_diagnostics() { - let config_error = anyhow::Error::new(RuntimeConfigError::RelativeRuntimePath); + let config_error = anyhow::Error::new(RuntimeConfigError::InvalidEnvelope); let (exit, diagnostic) = classify_failure(&config_error); assert_eq!(exit, DOMAIN_REFUSAL_EXIT); assert_eq!( @@ -1013,6 +1135,8 @@ mod tests { #[test] fn records_apply_refuses_an_invalid_document_before_touching_a_database() { let (root, project) = initialized("standalone-exact-time"); + let package = root.path().join("package"); + project::package(&project, &package, None).unwrap(); let config_path = root.path().join("runtime.yaml"); std::fs::write( &config_path, @@ -1029,7 +1153,7 @@ mod tests { audit:\n path: {}/audit.jsonl\n\ \x20 hashKeyRef: secret:env/SCHEDULINGCTL_TEST_AUDIT\n\ retention:\n attemptReceiptDays: 2\n", - project.display(), + package.display(), root.path().display() ), ) diff --git a/crates/registry-schedulingctl/src/project.rs b/crates/registry-schedulingctl/src/project.rs index 572952a691..fbf5f3819e 100644 --- a/crates/registry-schedulingctl/src/project.rs +++ b/crates/registry-schedulingctl/src/project.rs @@ -11,13 +11,16 @@ use std::fs; use std::path::{Path, PathBuf}; use anyhow::{anyhow, bail, Context, Result}; -use registry_scheduling::config::{verify_policy_package, PolicyPackageManifest}; +use registry_platform_config::package::{plan_package, write_package}; +use registry_platform_config::sha256_uri; +use registry_scheduling::config::{package_limits, PACKAGE_COMMAND}; use registry_scheduling_core::{ parse_fixture_yaml, parse_policy_yaml, CaseStatus, FixtureExpectation, ReplayError, SchedulingDiagnostic, SchedulingFacts, SchedulingFixture, SchedulingPolicy, - AUTHORED_POLICY_FILE, SCHEDULING_PACKAGE_MANIFEST_FILE, + AUTHORED_POLICY_FILE, }; use serde_json::{json, Value}; +use std::collections::BTreeMap; use crate::templates; @@ -251,12 +254,16 @@ pub(super) fn explain(project: &Path) -> Result { })) } -/// Write the verified package manifest beside the authored policy. The -/// manifest is the deployment identity the runtime verifies at startup, so -/// repackaging after an edit is a deliberate act: an existing manifest is -/// never silently replaced, and the written manifest is proven to verify -/// against the exact policy text before the command reports success. -pub(super) fn package(project: &Path) -> Result { +/// The project and the exact inputs a package of it carries. +struct PackageContents { + project: PathBuf, + inputs: BTreeMap>, +} + +/// Canonicalize the project, check the authored policy, and assemble the one +/// file a package carries. Performs no writes, so both `package` and +/// `package_dry_run` share it. +fn compute_package(project: &Path) -> Result { let project = fs::canonicalize(project).context("resolving the Scheduling authoring project")?; let policy_text = read_authoring_input(&project.join(AUTHORED_POLICY_FILE))?; @@ -269,39 +276,70 @@ pub(super) fn package(project: &Path) -> Result { findings.len() ); } - let manifest = PolicyPackageManifest::build(&policy_text) - .context("building the Scheduling policy package identity")?; - let manifest_path = project.join(SCHEDULING_PACKAGE_MANIFEST_FILE); - if manifest_path.exists() { - bail!( - "{} already exists; remove it deliberately before repackaging", - manifest_path.display() - ); - } - let mut bytes = - serde_json::to_vec_pretty(&manifest).context("encoding the package manifest")?; - bytes.push(b'\n'); - fs::write(&manifest_path, bytes) - .with_context(|| format!("writing {}", manifest_path.display()))?; - let verified = verify_policy_package(&project.join(AUTHORED_POLICY_FILE), &policy_text) - .with_context(|| format!("verifying the written {}", manifest_path.display()))?; - match verified { - Some(digest) if digest == manifest.policy_digest => {} - _ => bail!("the written package manifest does not verify against the authored policy"), - } + let inputs = BTreeMap::from([(AUTHORED_POLICY_FILE.to_owned(), policy_text.into_bytes())]); + Ok(PackageContents { project, inputs }) +} + +fn package_files(inputs: &BTreeMap>) -> Vec { + inputs + .iter() + .map(|(path, bytes)| { + json!({ + "path": path, + "sha256": sha256_uri(bytes), + "bytes": bytes.len(), + }) + }) + .collect() +} + +/// Write the checked policy into `output`, a new directory, as the package +/// the runtime verifies at startup. The package is written once: an existing +/// output is refused, so each candidate lands in its own directory. +pub(super) fn package(project: &Path, output: &Path, revision: Option<&str>) -> Result { + let PackageContents { project, inputs } = compute_package(project)?; + let written = write_package( + output, + &inputs, + revision, + &package_limits(), + PACKAGE_COMMAND, + )?; Ok(json!({ "ok": true, "command": "package", "project": project, - "manifest": manifest_path, - // The manifest on disk names its own byte-exact digest `policyDigest` - // (see `PolicyPackageManifest`); mirror that naming here instead of - // reporting the policy's separate semantic digest under the same - // key. `packageDigest` is that semantic digest, the one `explain` - // also reports. - "policyDigest": manifest.policy_digest, - "packageDigest": policy.policy_digest(), - "files": manifest.files, + "output": output, + "dryRun": false, + "packageDigest": written.digest(), + "revision": written.revision(), + "files": package_files(&inputs), + "runtimeConfigurationIncluded": false, + "secretsIncluded": false, + "networkAccess": false, + "databaseAccess": false, + })) +} + +/// Report the exact `packageDigest` and `files` a package of this project +/// would carry, without writing anything. +pub(super) fn package_dry_run(project: &Path, revision: Option<&str>) -> Result { + let PackageContents { project, inputs } = compute_package(project)?; + let digest = plan_package( + &project, + &inputs, + revision, + &package_limits(), + PACKAGE_COMMAND, + )?; + Ok(json!({ + "ok": true, + "command": "package", + "project": project, + "dryRun": true, + "packageDigest": digest, + "revision": revision, + "files": package_files(&inputs), "runtimeConfigurationIncluded": false, "secretsIncluded": false, "networkAccess": false, @@ -394,12 +432,21 @@ fn load_fixture(path: &Path) -> Result { parse_fixture_yaml(&bytes).with_context(|| format!("parsing fixture {}", path.display())) } +/// Read one authored input. Authored files are package content, so an +/// environment expression in one is refused rather than left for a reader to +/// mistake for substitution, which applies to `runtime.yaml` only. pub(super) fn read_authoring_input(path: &Path) -> Result { let bytes = fs::read(path).with_context(|| format!("reading {}", path.display()))?; if bytes.len() > MAXIMUM_INPUT_BYTES { bail!("{} exceeds the one MiB authoring limit", path.display()); } - String::from_utf8(bytes).with_context(|| format!("reading {}", path.display())) + let text = String::from_utf8(bytes).with_context(|| format!("reading {}", path.display()))?; + if let Err(error) = + registry_platform_config::reject_environment_expressions_in_authored_yaml(&text) + { + bail!("{}: {}", path.display(), error.message()); + } + Ok(text) } /// Every check finding as its path and closed reason, ready for a report. @@ -505,25 +552,27 @@ mod tests { assert_eq!(examples[0], examples[1]); let root = tempfile::tempdir().unwrap(); - let project = root.path().join("project"); + // The loader refuses a path through a symbolic link, and the system + // temporary directory is one on some hosts. + let base = root.path().canonicalize().unwrap(); + let project = base.join("project"); init(&project, "standalone-exact-time").unwrap(); + let output = base.join("package"); + package(&project, &output, None).unwrap(); let text = fs::read_to_string(project.join("runtime.example.yaml")) .unwrap() .replace( EXAMPLE_PACKAGE_ROOT, - project.to_str().expect("utf-8 project path"), + output.to_str().expect("utf-8 package path"), ) - .replace( - EXAMPLE_STATE_ROOT, - root.path().to_str().expect("utf-8 root path"), - ); - let runtime = root.path().join("runtime.yaml"); + .replace(EXAMPLE_STATE_ROOT, base.to_str().expect("utf-8 root path")); + let runtime = base.join("runtime.yaml"); fs::write(&runtime, &text).unwrap(); let config = RuntimeConfig::load(&runtime).expect("the emitted example loads"); assert_eq!( config.policy_path(), - project.join(AUTHORED_POLICY_FILE), - "the example selects the project beside it" + output.join(AUTHORED_POLICY_FILE), + "the example selects the package it names" ); assert_eq!(config.retention.attempt_receipt_days, 7); assert!(config.destinations.reminders.is_none()); diff --git a/crates/registry-schedulingctl/src/records.rs b/crates/registry-schedulingctl/src/records.rs index 7958c1c993..588edb9f84 100644 --- a/crates/registry-schedulingctl/src/records.rs +++ b/crates/registry-schedulingctl/src/records.rs @@ -39,7 +39,7 @@ pub fn apply(config_path: &Path, records_path: &Path) -> Result { .with_context(|| format!("loading {}", config.policy_path().display()))?; let text = crate::project::read_authoring_input(&records_path)?; let facts = parse_records(&text)?; - validate(&facts, &policy)?; + validate(&facts, &policy.policy)?; let counts = counts(&facts); let resolver = secret_resolver(&config)?; let store = PostgresStore::connect_migration(&config.database, &resolver) @@ -67,7 +67,7 @@ pub fn apply(config_path: &Path, records_path: &Path) -> Result { }), )) .context("writing the records.apply request audit entry; nothing was replaced")?; - match runtime.block_on(store.replace_facts(&policy.scheduling.id, &facts)) { + match runtime.block_on(store.replace_facts(&policy.policy.scheduling.id, &facts)) { Ok(()) => {} Err(store_error) => { let answer = unanswered_swap(&store_error); diff --git a/crates/registry-schedulingctl/src/templates.rs b/crates/registry-schedulingctl/src/templates.rs index ecd14c0cd8..116f5b7fb5 100644 --- a/crates/registry-schedulingctl/src/templates.rs +++ b/crates/registry-schedulingctl/src/templates.rs @@ -586,17 +586,28 @@ pub(super) const RUNTIME_EXAMPLE: &str = r#"# A complete Scheduling runtime conf # location, and point the secret references at secrets the configured provider # can resolve. Scheduling reads the authored policy at # package.root/scheduling.yaml and refuses to start when that file is missing -# or does not pass its checks. +# or does not pass its checks. Every configured path must be absolute and must +# not pass through a symbolic link. +# +# String values in this file may use ${VAR}, ${VAR:-default}, or ${VAR:?message} +# to take a deployment value from the environment when the runtime starts. +# Substitution never applies to a *Ref field, whose value must be written as a +# literal secret reference, nor under secretProviders, nor to the authored +# scheduling.yaml. apiVersion: registry.registrystack.org/scheduling-runtime/v1alpha1 kind: SchedulingRuntimeConfig package: - # The selected package always contains the policy at scheduling.yaml; no - # second selector can override it. Under - # listener.tlsTermination: operator-controlled-upstream the directory must - # also contain the scheduling.package.json manifest `schedulingctl package` - # writes; development loopback may select an unpackaged project directory. + # The package `schedulingctl package --output` writes: scheduling.yaml and + # the SHA256SUMS file that lists it. The runtime verifies it at every start, + # in every listener mode, and refuses a changed, missing, or extra file; no + # second selector can override the policy it holds. root: /srv/registry-scheduling/package + # Pin the package the runtime must serve: the packageDigest + # `schedulingctl package` prints, the SHA-256 digest of SHA256SUMS. Any + # other package is a startup refusal. + # expectedDigest: sha256:<64 lowercase hex digits> listener: + # Required: the IP address and port the runtime listens on. bind: 127.0.0.1:8105 tlsTermination: development-loopback networkExposure: private-address @@ -663,6 +674,8 @@ authentication: # jwksSource: # kind: static # documentRef: secret:file/jwks.json + # `kind: uri` with `uri: https://...` fetches the key set from a fixed + # address instead of the one discovery names. audit: # One single-writer destination. `file`, the default, appends to the # absolute path, rotates at rotateBytes (default 100 MiB), and deletes diff --git a/crates/registry-schedulingctl/tests/intents_postgres.rs b/crates/registry-schedulingctl/tests/intents_postgres.rs index 57f3b397f8..e0cf1d3718 100644 --- a/crates/registry-schedulingctl/tests/intents_postgres.rs +++ b/crates/registry-schedulingctl/tests/intents_postgres.rs @@ -150,10 +150,17 @@ async fn intents_lists_local_and_failed_oldest_due_first_and_respects_limit() { .await .expect("a disposable schema is created"); - let root = tempfile::tempdir().unwrap(); + let root = tempfile::tempdir_in(std::env::temp_dir().canonicalize().unwrap()).unwrap(); let project = root.path().join("project"); std::fs::create_dir_all(&project).unwrap(); std::fs::write(project.join("scheduling.yaml"), POLICY).unwrap(); + registry_platform_config::package::write_sum_file( + &project, + None, + ®istry_scheduling::config::package_limits(), + registry_scheduling::config::PACKAGE_COMMAND, + ) + .expect("the package is sealed"); std::fs::write( root.path().join("runtime.yaml"), format!( diff --git a/crates/registry-schedulingctl/tests/records_apply_postgres.rs b/crates/registry-schedulingctl/tests/records_apply_postgres.rs index 00aa9e0573..367b6771a8 100644 --- a/crates/registry-schedulingctl/tests/records_apply_postgres.rs +++ b/crates/registry-schedulingctl/tests/records_apply_postgres.rs @@ -193,10 +193,17 @@ async fn records_apply_replaces_facts_wholesale_and_audits_each_write() { .await .expect("a disposable schema is created"); - let root = tempfile::tempdir().unwrap(); + let root = tempfile::tempdir_in(std::env::temp_dir().canonicalize().unwrap()).unwrap(); let project = root.path().join("project"); std::fs::create_dir_all(&project).unwrap(); std::fs::write(project.join("scheduling.yaml"), POLICY).unwrap(); + registry_platform_config::package::write_sum_file( + &project, + None, + ®istry_scheduling::config::package_limits(), + registry_scheduling::config::PACKAGE_COMMAND, + ) + .expect("the package is sealed"); std::fs::write(root.path().join("first.yaml"), FIRST_RECORDS).unwrap(); std::fs::write(root.path().join("second.yaml"), SECOND_RECORDS).unwrap(); std::fs::write( @@ -334,7 +341,7 @@ async fn records_apply_rejects_a_different_deployment_identity_without_writing() .await .expect("a disposable schema is created"); - let root = tempfile::tempdir().unwrap(); + let root = tempfile::tempdir_in(std::env::temp_dir().canonicalize().unwrap()).unwrap(); let adopted_project = root.path().join("adopted-project"); let other_project = root.path().join("other-project"); std::fs::create_dir_all(&adopted_project).unwrap(); @@ -345,6 +352,20 @@ async fn records_apply_rejects_a_different_deployment_identity_without_writing() POLICY.replacen(" id: registry-updates\n", " id: permit-renewals\n", 1), ) .unwrap(); + registry_platform_config::package::write_sum_file( + &adopted_project, + None, + ®istry_scheduling::config::package_limits(), + registry_scheduling::config::PACKAGE_COMMAND, + ) + .expect("the package is sealed"); + registry_platform_config::package::write_sum_file( + &other_project, + None, + ®istry_scheduling::config::package_limits(), + registry_scheduling::config::PACKAGE_COMMAND, + ) + .expect("the package is sealed"); std::fs::write(root.path().join("first.yaml"), FIRST_RECORDS).unwrap(); std::fs::write(root.path().join("second.yaml"), SECOND_RECORDS).unwrap(); diff --git a/crates/registry-stack-client-node/breg/client.d.ts b/crates/registry-stack-client-node/breg/client.d.ts index bcfe85b372..2327cf1b5c 100644 --- a/crates/registry-stack-client-node/breg/client.d.ts +++ b/crates/registry-stack-client-node/breg/client.d.ts @@ -608,6 +608,7 @@ export type BRegIngestionAttemptOutcome = | 'invalidItem' | 'refused' | 'bindingChanged' + | 'importAuthorityClosed' | 'chunkMismatch' | 'runNotOpen' | 'unavailable' diff --git a/docker/Dockerfile b/docker/Dockerfile index eb33b3654a..33e32497d9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -69,13 +69,12 @@ WORKDIR /var/lib/registry-evidence # deployment artifacts; mount them read-only under /etc/registry-evidence. # Point the runtime's audit destination under /var/lib/registry-evidence, # which is writable by the nonroot user. -ENV REGISTRY_EVIDENCE_RUNTIME=/etc/registry-evidence/runtime.yaml EXPOSE 8080 # No HEALTHCHECK: distroless has no shell or curl and the binary has no # healthcheck subcommand. The service serves GET /health for HTTP probes. ENTRYPOINT ["/usr/local/bin/evidence"] -CMD ["serve"] +CMD ["serve", "--runtime-config", "/etc/registry-evidence/runtime.yaml"] FROM chef AS casework-builder ARG SOURCE_DATE_EPOCH diff --git a/docker/README.md b/docker/README.md index 22e974f827..76d055ab42 100644 --- a/docker/README.md +++ b/docker/README.md @@ -69,8 +69,8 @@ the machine-readable `org.registrystack.runtime.uid` and The runtime file, governed bundle, and secret root are startup-only artifacts; mount them read-only under `/etc/registry-evidence`. The image -expects the operator runtime at `/etc/registry-evidence/runtime.yaml` -(`REGISTRY_EVIDENCE_RUNTIME`). The runtime's `bundleDirectory`, secret +expects the operator runtime at `/etc/registry-evidence/runtime.yaml`, which +its default command passes as `--runtime-config`. The runtime's `package.root`, secret provider `root`, audit `path`, and any `trustProfiles.*.caBundleFile` are validated absolute paths, interpreted inside the container: every one of them must resolve to a mount. Point the audit destination under @@ -96,7 +96,7 @@ docker run --rm \ registry-evidence ``` -with `listener.bindHost: 0.0.0.0` and +with `listener.bind: 0.0.0.0:8080` and `listener.networkExposure: container-private` in `runtime.yaml`. TLS and public exposure are upstream concerns by design; front this listener with your operator-network proxy. `evidence check` validates the bundle without serving. @@ -106,7 +106,8 @@ writability, signer readiness, source credentials, and JWKS reachability: ```sh docker run --rm -v "$PWD/deploy/evidence:/etc/registry-evidence:ro" \ -v evidence-audit:/var/lib/registry-evidence \ - registry-evidence check --require-runtime-dependencies \ + registry-evidence check --runtime-config /etc/registry-evidence/runtime.yaml \ + --require-runtime-dependencies \ --require-audit-under /var/lib/registry-evidence ``` @@ -119,7 +120,7 @@ declared root, and an existing symlink inside the root that leads out of it, both fail closed. The option proves containment only; the destination writability checks still have to pass. -Relay provides the equivalent `relay check --runtime +Relay provides the equivalent `relay check --runtime-config /etc/relay/runtime.yaml`, including the same `--require-audit-under` option. For a Compose deployment containing Evidence, Relay, or both, use `docker/runtime-preflight.py` to verify the common container posture first and diff --git a/docker/compose/README.md b/docker/compose/README.md index 137c1c4343..d15375b519 100644 --- a/docker/compose/README.md +++ b/docker/compose/README.md @@ -1,14 +1,14 @@ # Evidence candidate Compose adapter This directory is an operator-owned Docker Compose adapter for a reviewed Evidence candidate. -It is not generated by `evidencectl build`, and it does not turn a local authoring project into a +It is not generated by `evidencectl package`, and it does not turn a local authoring project into a deployment. ## Inputs Set the following absolute paths before starting Compose: -- `EVIDENCE_CANDIDATE_DIR`, an approved candidate containing `bundle/`. +- `EVIDENCE_CANDIDATE_DIR`, the approved package directory containing `SHA256SUMS`. - `EVIDENCE_RUNTIME_FILE`, a container-specific runtime document. - `EVIDENCE_SECRET_ROOT`, owner-only Evidence secret files. - `EVIDENCE_TRANSIT_SOCKET_DIR`, the dedicated directory containing `transit-proxy.sock`. @@ -51,7 +51,7 @@ target-context check before starting the service: ```sh docker compose -f docker-compose.yaml run --rm evidence \ - --runtime /etc/registry-evidence/runtime.yaml check \ + check --runtime-config /etc/registry-evidence/runtime.yaml \ --require-runtime-dependencies \ --require-audit-under /var/lib/registry-evidence ``` @@ -105,8 +105,8 @@ changes to the source Compose or environment files cannot change the checked containers between phases. Host storage durability, daemon state, and changes made after preflight remain operator responsibilities. -The bundle revision remains unchanged when only the container runtime changes. Run fixtures again -only when the governed bundle changes. The runtime and bundle being read-only does not waive secret +The package digest remains unchanged when only the container runtime changes. Run fixtures again +only when governed package content changes. The runtime and package being read-only does not waive secret owner or mode checks for the container service identity. This adapter does not establish image provenance, TLS, routing, client registration, or secret diff --git a/docker/compose/docker-compose.yaml b/docker/compose/docker-compose.yaml index 625109c407..bd560df1fa 100644 --- a/docker/compose/docker-compose.yaml +++ b/docker/compose/docker-compose.yaml @@ -1,5 +1,5 @@ # Operator-owned Compose adapter for an approved Evidence candidate. It is not -# generated by evidencectl build. Operators select reviewed image digests and +# generated by evidencectl package. Operators select reviewed image digests and # supply the candidate, runtime, and secrets separately. # # Required environment: @@ -19,7 +19,7 @@ services: security_opt: [no-new-privileges:true] restart: unless-stopped volumes: - - ${EVIDENCE_CANDIDATE_DIR:?set EVIDENCE_CANDIDATE_DIR to the approved candidate}/bundle:/etc/registry-evidence/bundle:ro + - ${EVIDENCE_CANDIDATE_DIR:?set EVIDENCE_CANDIDATE_DIR to the approved package}:/etc/registry-evidence/bundle:ro - ${EVIDENCE_RUNTIME_FILE:?set EVIDENCE_RUNTIME_FILE to a container runtime}:/etc/registry-evidence/runtime.yaml:ro - ${EVIDENCE_SECRET_ROOT:?set EVIDENCE_SECRET_ROOT to the Evidence secret root}:/run/secrets/registry-evidence:ro # Evidence receives only the workload-local proxy socket, never its diff --git a/docker/compose/runtime.docker.yaml b/docker/compose/runtime.docker.yaml index c4640305f1..239eb468fa 100644 --- a/docker/compose/runtime.docker.yaml +++ b/docker/compose/runtime.docker.yaml @@ -1,21 +1,22 @@ # Example container-shaped runtime configuration for the operator-owned Compose # adapter. It is not governed content and is never generated by evidencectl -# build. Copy it into an operator runtime file and bind it to the approved -# candidate bundle mounted by docker-compose.yaml. +# package. Copy it into an operator runtime file and bind it to the approved +# package mounted by docker-compose.yaml. # # Every path below is a container path: the approved bundle lives under # /etc/registry-evidence, secrets under /run/secrets/registry-evidence, and # writable audit state under /var/lib/registry-evidence. The explicit # container-private mode permits the wildcard bind inside the isolated Compose # network; it does not authorize public or direct-TLS exposure. -version: 1 +apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig -bundleDirectory: /etc/registry-evidence/bundle +package: + root: /etc/registry-evidence/bundle listener: - bindHost: 0.0.0.0 + bind: 0.0.0.0:8080 networkExposure: container-private - port: 8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 diff --git a/docker/runtime-preflight.py b/docker/runtime-preflight.py index 3e0a070af4..80c1b6ef26 100755 --- a/docker/runtime-preflight.py +++ b/docker/runtime-preflight.py @@ -84,16 +84,16 @@ # product configuration to answer that question. NATIVE_CHECKS = { "evidence": [ - "--runtime", - "/etc/registry-evidence/runtime.yaml", "check", + "--runtime-config", + "/etc/registry-evidence/runtime.yaml", "--require-runtime-dependencies", AUDIT_CONTAINMENT_FLAG, AUDIT_PREFIXES["evidence"], ], "relay": [ "check", - "--runtime", + "--runtime-config", "/etc/relay/runtime.yaml", AUDIT_CONTAINMENT_FLAG, AUDIT_PREFIXES["relay"], @@ -511,19 +511,11 @@ def validate_service(selection: ServiceSelection, document: dict[str, Any]) -> N name in environment for name in ("LD_AUDIT", "LD_LIBRARY_PATH", "LD_PRELOAD") ): raise PreflightError("service must not override dynamic-loader behavior") - fixed_config = { - "evidence": ( - "REGISTRY_EVIDENCE_RUNTIME", - "/etc/registry-evidence/runtime.yaml", - ), - }.get(selection.product) - if fixed_config is not None: - name, expected = fixed_config - configured = environment.get(name) - if configured is not None and configured != expected: - raise PreflightError( - "service must use the official runtime configuration path" - ) + if selection.product == "evidence" and "REGISTRY_EVIDENCE_RUNTIME" in environment: + raise PreflightError( + "service must not set REGISTRY_EVIDENCE_RUNTIME; the official image " + "passes --runtime-config" + ) if "ALL" not in require_string_list(service, "cap_drop"): raise PreflightError("service must drop all Linux capabilities") cap_add = service.get("cap_add", []) diff --git a/docker/test_runtime_preflight.py b/docker/test_runtime_preflight.py index d48dd89385..9a2ddf25fd 100644 --- a/docker/test_runtime_preflight.py +++ b/docker/test_runtime_preflight.py @@ -673,26 +673,19 @@ def test_writable_mounts_cannot_overlap_configuration_or_secrets(self) -> None: deployment({"evidence": selected}), ) - def test_official_configuration_paths_may_not_be_overridden(self) -> None: - fixed = { - "evidence": ( - "REGISTRY_EVIDENCE_RUNTIME", - "/etc/registry-evidence/runtime.yaml", - ), - } - for product, (name, expected) in fixed.items(): - with self.subTest(product=product): - selected = service(product) - selected["environment"] = {name: expected} - self.module.validate_service( - self.module.ServiceSelection(product, product), - deployment({product: selected}), - ) - selected["environment"] = {name: "/tmp/alternate.yaml"} - with self.assertRaises(self.module.PreflightError): + def test_removed_evidence_runtime_variable_is_refused(self) -> None: + # The official image passes --runtime-config itself, and Evidence + # refuses to start while the removed variable is set, whatever it says. + for value in ("/etc/registry-evidence/runtime.yaml", "/tmp/alternate.yaml"): + with self.subTest(value=value): + selected = service("evidence") + selected["environment"] = {"REGISTRY_EVIDENCE_RUNTIME": value} + with self.assertRaisesRegex( + self.module.PreflightError, "REGISTRY_EVIDENCE_RUNTIME" + ): self.module.validate_service( - self.module.ServiceSelection(product, product), - deployment({product: selected}), + self.module.ServiceSelection("evidence", "evidence"), + deployment({"evidence": selected}), ) def test_native_failure_is_value_free(self) -> None: @@ -729,7 +722,7 @@ def test_an_image_without_the_containment_flag_is_named_as_the_cause(self) -> No native_returncode=2, native_stderr=( "error: unexpected argument '--require-audit-under' found\n" - "\nUsage: relay check --runtime \n" + "\nUsage: relay check --runtime-config \n" ), argv=[ "--compose-file", diff --git a/docs/site/astro.config.mjs b/docs/site/astro.config.mjs index fc66cd4b9d..4660065a0c 100644 --- a/docs/site/astro.config.mjs +++ b/docs/site/astro.config.mjs @@ -596,7 +596,7 @@ export default defineConfig({ items: [ { label: 'How the index works', slug: 'explanation/discovery-as-an-index' }, { label: 'Publish and consume an index', slug: 'tutorials/publish-and-consume-discovery-index' }, - { label: 'Build and run an index', slug: 'configure/discovery' }, + { label: 'Package and run an index', slug: 'configure/discovery' }, ], }, { diff --git a/docs/site/scripts/check-current-doc-cutover.test.mjs b/docs/site/scripts/check-current-doc-cutover.test.mjs index 865d6166ac..bf5e06cfdb 100644 --- a/docs/site/scripts/check-current-doc-cutover.test.mjs +++ b/docs/site/scripts/check-current-doc-cutover.test.mjs @@ -158,7 +158,7 @@ test('accepts the Relay V2 command surface', async () => { '`relayctl test`', '`relayctl diff`', '`relayctl package`', - '`relay serve --runtime runtime.yaml`', + '`relay serve --runtime-config /etc/relay/runtime.yaml`', '`relay healthcheck --url http://127.0.0.1:8080/health`', ].join('\n'), ); diff --git a/docs/site/scripts/check-discovery-tutorial.sh b/docs/site/scripts/check-discovery-tutorial.sh index fcc0a7d794..314ac56c20 100755 --- a/docs/site/scripts/check-discovery-tutorial.sh +++ b/docs/site/scripts/check-discovery-tutorial.sh @@ -81,7 +81,7 @@ expected_output=( '[provider] evidence.jsonld sha256=fc96f3a8cb0d82239425ea5712dceca975a5899e5528616648174da661fae905' '[provider] relay.jsonld sha256=5a34fa469803b7c28b3d5e7134a42398e326a2f173aacae9090d29787bc8f4d7' '[operator] offline check: valid origins=2 mappings=1' - '[operator] explicit build: built catalogRevision=sha256:b4b7195f36691c245bf49a88a248049ed899c0c41dbf1a87a386571c0dbfba0f mappingRevision=sha256:332004ca3920c498539180946e8f2637e9998ba7e49cd98f31e19d6f818857ac' + '[operator] explicit package: packaged packageDigest=sha256:' '[consumer] resolved evidenceType=urn:example:evidence-type:adult-status alternatives=1' '[consumer] selected evidence recordId=urn:registrystack:discovery:record:sha256:676659c10ce5cc9d353f4fd2816673c7947e612151efbbe1cc4d42372d9be9d5' '[consumer] selected relay recordId=urn:registrystack:discovery:record:sha256:aa220c11f493c266bc22adf5dc7ca82fb7a83842e6e887dc0e8e5680f4f84244' diff --git a/docs/site/src/content/docs/changelog.mdx b/docs/site/src/content/docs/changelog.mdx index 4a037cda17..d12ee9d0de 100644 --- a/docs/site/src/content/docs/changelog.mdx +++ b/docs/site/src/content/docs/changelog.mdx @@ -5,7 +5,7 @@ status: current owner: registry-docs source_repos: - registry-docs -last_reviewed: "2026-09-15" +last_reviewed: "2026-09-25" doc_type: reference locale: en standards_referenced: [] @@ -91,6 +91,488 @@ relevant product pages on this site rather than duplicating release notes. [Evidence audit](../operate/evidence-audit/), and [audit upgrades](../operate/retention-and-persistent-state/#upgrade-casework-and-scheduling-audit). +- BREAKING: published Base Registry Engine packages now carry the shared + `SHA256SUMS` envelope. `bregctl package` reports its `packageDigest` and can + add a hash-covered `REVISION` with `--revision`. `breg` verifies every listed + file at startup in addition to the existing BReg signatures and deployment + bindings, and `package.expectedDigest` can pin the shared digest. Rebuild old + package directories with `bregctl package` before starting this version. + +{/* Evidence: crates/registry-breg/src/package.rs, PreparedPackage::publish_to_directory_with_revision; + crates/registry-breg/src/runtime_config.rs, RuntimeConfig::verify_package_envelope; + crates/registry-breg/tests/runtime_config.rs, shared_package_envelope_and_pin_are_checked_before_startup. */} + +- BREAKING: `evidencectl package` now publishes one environment-neutral package + directory with `SHA256SUMS` and an optional hash-covered `REVISION`; it no + longer copies `runtime.yaml` into the output. Evidence Gateway verifies every + listed file at startup and `package.expectedDigest` pins the package digest. + The separate Evidence bundle and runtime revisions are removed, while each + requirement's package-derived `configurationRevision` remains. The retired + `evidencectl build` spelling is refused with `evidencectl package` as the fix. + +{/* Evidence: crates/registry-evidence/src/bundle.rs, DeploymentInputs::load; + crates/registry-evidencectl/src/build.rs, run_package_with_format; + crates/registry-evidencectl/src/authoring.rs, write_bundle. */} + +- `evidence check --require-runtime-dependencies` and `evidence serve` name + the fault that stopped a Transit signer from initializing, after the + unchanged `evidence: runtime signing initialization failed:` prefix: no + answer on the Unix socket, a refused key read, a provider server error such + as a sealed provider, a malformed response, a key whose custody is unsafe, a + `keyVersion` above the key's `latest_version` or below its + `min_encryption_version`, a public key that is not the governed one, or a + failed self-test. No cause carries a path, provider response, or key + material. See + [activate the new version](../tutorials/rotate-evidence-signing-keys/#activate-the-new-version). + +{/* Evidence: crates/registry-evidence/src/runtime.rs, SigningInitializationFault::cause(); + crates/registry-platform-crypto/src/lib.rs, TransitInitializationError and transit_signer_initialization_names_the_fault_it_met; + crates/registry-evidence/src/runtime_tests.rs, signing_initialization_faults_name_distinct_causes. */} + +- An Evidence access-token issuer that serves its key set under a private + certificate authority can be trusted through a named `tlsTrustProfile` on + the bundle's `authentication` block, bound in `runtime.yaml` exactly like a + source's profile. The CA is trusted beside the system roots for the + `jwksUri` connection alone, hostname verification stays on, and a local + HTTP `jwksUri` cannot name a profile. A profile bound on one side only is + refused, and `evidence check --require-runtime-dependencies` fetches the key + set through it before the listener binds. See + [an issuer behind a private CA](../configure/evidence/#an-issuer-behind-a-private-ca). + +{/* Evidence: crates/registry-evidence/src/runtime.rs, issuer_trust_roots(); + crates/registry-evidence/src/bundle.rs, the_issuer_trust_profile_is_bound_exactly_like_a_source_profile; + crates/registry-evidence/src/config.rs, an_issuer_trust_profile_is_a_local_id_for_an_https_key_set_only; + crates/registry-evidence/tests/cli.rs, dependency_check_trusts_a_private_ca_issuer_only_through_its_named_profile; + crates/registry-platform-httputil/src/lib.rs, a_pinned_get_trusts_a_private_ca_only_when_it_is_named. */} + +- `evidencectl doctor --runtime-config --without-audit-lock`, and the + runtime's `evidence check --require-runtime-dependencies + --without-audit-lock`, check a candidate staged beside the Evidence instance + it will replace. Without the option the check opens the audit destination as + startup does, so it refuses while that instance holds the destination's + single-writer lock, and the refusal now names the option. With it, the audit + hash key is checked as startup checks it, and the audit directory and files + are checked for ownership, mode, write access, and a complete final entry, + without taking the lock. A second writer is not detected; the JSON + `proofBoundary` says so. See + [evidencectl](../reference/evidencectl/#project-workflow). + +{/* Evidence: crates/registry-platform-audit/src/writer.rs, FileDestination::check_writable(); + crates/registry-evidence/src/runtime.rs, assemble() and check_dependencies_without_audit_lock(); + crates/registry-evidence/src/audit.rs, EvidenceAuditLog::preflight(); + crates/registry-evidencectl/src/runtime.rs, the_lock_free_form_states_its_own_proof_boundary; + crates/registry-evidence/tests/cli.rs, dependency_check_without_the_audit_lock_passes_beside_a_running_writer. */} + +- BREAKING: `evidencectl` refuses a derivation whose `answer` reads a fact + its question's source does not declare, as + `evidence.authoring.derivation-fact-undeclared` against the derivation + file. The declared facts are an inline operation's `source.facts[].name`, + or the properties of a referenced source's closed `factSchema`. `check`, + `test`, `package`, `source diff`, and `source update` all refuse, as do the + compatibility spellings `fixtures run` and `build`, so a Base Registry + Engine (BReg) field rename that a derivation still reads under its old name + is refused before `source update` installs it or a candidate is packaged. + The check reads the project, not the running registry: a field renamed in a + registry that already serves a deployed candidate still fails every request + that reads it while `/ready` reports ready, so change the registry and + deploy the rebuilt candidate together. Only literal reads such as + `facts["status"]` and `facts.status` are checked, and a `??` fallback + between names passes when any of them is declared. See + [evidencectl](../reference/evidencectl/#project-workflow). + +{/* Evidence: crates/registry-evidence-authoring/src/derivation.rs, validate_answer_fact_reads(); + crates/registry-evidencectl/src/authoring.rs, declared_fact_names() and read_inputs(); + crates/registry-evidence-authoring/tests/derivation_fact_reads.rs; + crates/registry-evidencectl/src/source_cli.rs, diff_and_apply_refuse_a_next_fact_schema_that_drops_a_fact_a_derivation_reads; + products/breg/evidence/tests/verify-composition.py, verify(). */} + +- BREAKING: `bregctl test --baseline-runtime-config` rehearses the successor + migration before it writes a receipt. It rebuilds the active package's + schema from its signed sources on the disposable test database, runs the + migration in apply order, requires the candidate schema, and rolls back. + A plan that used to fail only at `apply` now fails `test`, so `package` + cannot sign it, and a predecessor this `bregctl` cannot rebuild is refused + as `migration.rehearsal.baseline_unavailable` or + `migration.rehearsal.baseline_not_reproducible`. The rehearsal runs over + empty tables, so it does not prove data-dependent steps. A failure reports + the SQLSTATE and the object names, never the PostgreSQL message. See + [what `test` rehearses](../operate/breg-changes/#what-test-rehearses). + +{/* Evidence: crates/registry-breg/src/postgres/rehearsal.rs; + crates/registry-bregctl/src/test_lifecycle.rs; + crates/registry-breg/tests/postgres_migration.rs, real_postgres_rehearsal_refuses_a_reviewed_plan_activation_would_refuse. */} + +- BREAKING: when PostgreSQL refuses a statement after `bregctl apply` began + maintenance, the report is `apply.migration.statement_failed` instead of + `apply.migration.failed`, and its message names the SQLSTATE, its class, + and the table, column, or constraint PostgreSQL reported, never the + PostgreSQL message or a stored value. `registry-breg` reports it as + `MigrationError::StatementFailed`, from the new + `PostgresKernelError::Statement` variant, so an exhaustive match on either + enum needs the new arm; `PostgresFailure` is exported from + `registry_breg::postgres` under every feature set. A lost connection, + including a session the server ended (SQLSTATE class 08, 57P01 to 57P05, + 25P03), still reports `apply.migration.failed`, while a compiler or + reviewed statement that times out is a refused statement with SQLSTATE + 57014. The target stays pinned as before. See + [troubleshooting](../operate/breg-changes/#troubleshooting). + +{/* Evidence: crates/registry-breg/src/postgres/mod.rs, PostgresKernelError::from_statement_error(), a_server_ended_session_is_a_connection_failure_not_a_refused_statement; + crates/registry-breg/src/migration.rs, MigrationError::StatementFailed; + crates/registry-bregctl/src/lib.rs, apply_reports_a_refused_statement_with_its_sqlstate_and_objects; + crates/registry-breg/tests/postgres_migration.rs, refused_step_reports_its_sqlstate. */} + +- BREAKING: `bregctl history rebaseline` no longer refuses a registry holding + more than 1,000 live rows. It verifies every live row against its journal + head in pages of 1,000 inside one transaction, so a registry of any size + can regain snapshot coverage after an erasure. The codes + `history.rebaseline.live_rows.budget_exceeded` and + `field_encryption.erase_history.rebaseline.live_rows_budget_exceeded` are + gone, and so are `HistoryRebaselineError::LiveRowBudgetExceeded` and + `MAX_REBASELINE_LIVE_ROWS` in `registry-breg`. The run holds every entity + table exclusively, reads included, until it commits, so a larger registry + means a longer read and write outage: size a maintenance window from a + rehearsal. See + [restore snapshot coverage after an erasure](../operate/breg-retention/#restore-snapshot-coverage-after-an-erasure). + +{/* Evidence: crates/registry-breg/src/history_migration.rs, verify_every_live_row_matches_its_journal_head(); + crates/registry-breg/tests/postgres_history_rebaseline.rs, rebaseline_refuses_a_mismatch_on_a_later_page. */} + +- Raising a `text` field's `maxLength` is a compatible additive change, + `field_length_widened`, applied live: the successor replaces the column's + length check and validates the stored rows, with no reviewed migration. + Revisions recorded under the lower limit stay readable, and an action over + the entity stays additive as `action_target_fields_widened`. A `string` + field's `maxLength` is its column type, so raising it still needs a + reviewed migration. See + [review and apply changes](../operate/breg-changes/#test-and-package-the-successor). + +{/* Evidence: crates/registry-breg/src/package.rs, CompiledRegistryChangeCode::FieldLengthWidened; + crates/registry-breg/src/generated_ddl.rs, replace_length_check_statement(); + crates/registry-breg/tests/package_change_plan.rs, text_length_widening_is_additive_and_replaces_the_length_check; + crates/registry-breg/tests/action_vocabulary_codes.rs, a_raised_text_limit_on_a_targeted_entity_keeps_the_action_contracts; + crates/registry-breg/tests/postgres_compiled_schema.rs, text_length_widening_replaces_the_length_check_and_keeps_existing_rows. */} + +- Lowering a `string` field's `minLength` under the same `maxLength` is also + `field_length_widened`, applied live: the successor replaces the column's + minimum check, or drops it when the minimum falls to 0, with no reviewed + migration. Revisions recorded under the higher minimum stay readable, and an + action over the entity stays additive. Raising a `minLength`, changing a + `string` field's `maxLength`, and widening a `decimal` still need a reviewed + migration. See + [review and apply changes](../operate/breg-changes/#test-and-package-the-successor). + +{/* Evidence: crates/registry-breg/src/contract.rs, FieldTypeSource::lowers_string_min_length_of(); + crates/registry-breg/src/generated_ddl.rs, replace_length_check_statement(); + crates/registry-breg/tests/package_change_plan.rs, string_minimum_lowering_is_additive_and_replaces_or_drops_the_length_check; + crates/registry-breg/tests/action_vocabulary_codes.rs, a_lowered_string_minimum_on_a_targeted_entity_keeps_the_action_contracts; + crates/registry-breg/tests/postgres_compiled_schema.rs, string_minimum_lowering_replaces_or_drops_the_length_check_and_keeps_existing_rows. */} + +- `bregctl diff` reports every removed field or entity as + `diff.history.removed_values_retained`. Removing one from the package drops + the live column or table, but the values stay in every revision snapshot + recorded before the change and in every database backup; only + `bregctl history erase` removes them, whole revisions of one record at a + time. See + [review and apply changes](../operate/breg-changes/#compare-the-candidate-with-the-active-package). + +{/* Evidence: crates/registry-bregctl/src/lib.rs, removed_value_findings(); + crates/registry-bregctl/tests/diff.rs, a_removed_field_is_reported_as_retained_in_history_not_erased. */} + +- BREAKING: a reviewed `chunked_backfill` step now appends one history + revision for every row each chunk changes, in the same commit as the chunk, + so snapshot and as-of reads agree with the backfilled rows. Its `chunkSize` + cap falls from 10,000 to 1,000, and its SQL must be a single `UPDATE` of the declared + entity that names no record metadata column, uses no Unicode-escape + identifier or string, and names no other statement word outside comments + and plain string literals; the journal also refuses a step that changed + `created_at` or `updated_at`. A plan outside those limits that `test` and `package` + used to accept is now refused, and `test` reports a journaled step whose + SQL the history journal refuses as + `migration.rehearsal.history_step_refused`. See + [reviewed migration files](../operate/breg-changes/#reviewed-migration-files). + +{/* Evidence: crates/registry-breg/src/history_migration.rs, check_reviewed_history_step(), a_reviewed_step_that_changes_any_record_metadata_is_detected; + crates/registry-breg/src/postgres/interlock.rs, execute_reviewed_chunk(); + crates/registry-breg/src/migration_plan.rs, MAX_CHUNK_SIZE; + crates/registry-breg/tests/migration_plan.rs, reviewed_chunked_backfill_refuses_a_chunk_size_beyond_the_commit_budget; + crates/registry-breg/tests/postgres_migration.rs, reviewed_migration_history(). */} + +- Base Registry Engine (BReg) reports an accepted review whose result lookup + answers an empty `404` as `recovery.code: result-unknown-to-authority` + instead of leaving the code empty. The result poller serves a review that has a pending webhook + completion first and one its authority no longer knows last, and a pending + completion makes its review due at once. The new + `bregctl review-recovery resubmit` submits the exact retained review + request again under its original idempotency key, and + `bregctl review-recovery close` stops waiting on a review the authority will + never answer, reporting `operator-closed`. See + [operate Base Registry Engine](../operate/breg/#resubmit-or-close-a-review-the-authority-lost). + +{/* Evidence: crates/registry-breg/src/review_store.rs, poll_one_result and receive_completion; + crates/registry-breg/src/review_recovery.rs; + crates/registry-breg/tests/postgres_review_executor.rs, a_live_review_is_polled_before_one_its_authority_does_not_know and a_webhook_completion_makes_its_review_due_and_first_in_the_poll_queue; + crates/registry-breg/tests/postgres_change_requests.rs, an_operator_resubmits_or_closes_a_review_its_authority_lost. */} + +- BREAKING: Evidence Gateway reads `runtime.yaml` through the shared + runtime configuration loader and declares its access-token issuer and + audit key through the shared blocks. The runtime file now opens with + `apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1` and + `kind: EvidenceRuntimeConfig` instead of `version: 1`; + `bundleDirectory` is `package.root`, with an optional + `package.expectedDigest` pin; and `listener.bindHost` with + `listener.port` is one `listener.bind: host:port` value, as it is for + `metricsListener`. `secretProviders.environment: {}` enables + `secret:env/NAME` references beside `secret:file/name`, and runtime string + values accept `${VAR}` outside secret references and `secretProviders`, + while a `${...}` expression in the governed bundle is refused with the + field that holds it. + In the governed bundle, the flat `authentication` fields move under + `authentication.oidc`, `kind: oidc-access-token` is dropped, the one-entry + `audiences` list is a single `audience`, `jwksUri` is + `jwksSource: {kind: uri, uri: ...}`, and `audit.hashSecretRef` is + `audit.hashKeyRef`; `audit.hashKeyVersion` is unchanged. Every subcommand + that reads the runtime file takes the required + `--runtime-config FILE` after the subcommand; the global `--runtime` + flag, the `REGISTRY_EVIDENCE_RUNTIME` variable, and the default path are + gone, and the container image passes the flag in its default command. A + removed key, flag, or variable is refused with the name of its + replacement. To migrate, rewrite the project's `governance.yaml` and + `runtime.yaml` with the new keys, rebuild the candidate with the matching + `evidencectl`, and replace `--runtime FILE` with + ` --runtime-config FILE` in service definitions. See + [configure Evidence Gateway](../configure/evidence/#the-runtime-file). + +{/* Evidence: crates/registry-evidence/src/config.rs, EVIDENCE_RUNTIME_REMOVED_KEYS, + every_removed_runtime_key_is_refused_with_its_replacement_named, + every_removed_bundle_authentication_and_audit_key_is_refused_with_its_replacement_named, + environment_substitution_fills_values_and_never_a_secret_reference and + the_environment_secret_provider_is_enabled_only_by_declaration and + an_authored_bundle_carrying_an_environment_expression_is_refused; + crates/registry-evidence/tests/cli.rs, the_removed_runtime_inputs_are_refused_with_their_replacement_named; + release/docker/Dockerfile.evidence. */} + +- BREAKING: Registry Casework reads `runtime.yaml` through the shared + runtime configuration loader and declares its secret providers, database, + listener bind, OpenID Connect issuer and clients, and audit key through the + shared blocks. The runtime file may not pass through a symbolic link and is + at most 1 MiB. `${VAR}`, `${VAR:-default}`, and `${VAR:?message}` are + substituted in string values after parsing; an expression in a field ending + in `Ref` or under `secretProviders` is refused, and one in `casework.yaml` + is refused by the runtime and by `caseworkctl check`. `listener.bind` is + required. `authentication.oidc.issuer` must be an `https` URL without + credentials, query, or fragment, with plain `http` accepted only on an IPv4 + loopback host under `development-loopback`, and `audience` is at most 512 + characters. Under `operator-controlled-upstream`, + `authentication.oidc.allowedClients` must name at least one client, because + an empty list admits every client the issuer verifies; + `development-loopback` still accepts an empty list. + `authentication.oidc.jwksUri` is removed: declare `jwksSource` + with `kind: uri` and the same URL as `uri`. `audit.hashKeyRef` must be an + exact secret reference. To migrate, add `listener.bind` if it was omitted, + list the admitted clients in `allowedClients` for a production deployment, + replace `jwksUri`, and move any environment expression out of + `casework.yaml`. See + [configure the runtime](../operate/casework/). + +{/* Evidence: crates/registry-casework/src/config.rs, the_runtime_configuration_is_read_through_the_shared_loader, + a_removed_jwks_uri_names_its_replacement, the_oidc_issuer_and_clients_are_the_shared_blocks, + production_names_its_allowed_clients_while_loopback_may_leave_them_empty, + environment_expressions_substitute_values_but_never_secret_references and + an_authored_project_carrying_an_environment_expression_is_refused; + crates/registry-caseworkctl/src/lib.rs, removed_runtime_keys_name_the_replacement_path_and_action and + check_refuses_an_environment_expression_in_the_authored_project. */} + +- BREAKING: Base Registry Engine reads `runtime.yaml` through the shared + runtime configuration loader and declares its OpenID Connect issuer, audit + key, listeners, and secret providers through the shared blocks. `breg` + takes the runtime file as `--runtime-config FILE`; `--config` is refused + with the replacement named, `bregctl request-retention cleanup-attachments` + no longer accepts its `--config` alias, and the container image passes the + new flag in its default command. `${VAR}`, `${VAR:-default}`, and + `${VAR:?message}` are substituted in string values after parsing, so a + substituted value is always text and never a number, boolean, or YAML + structure; a `${...}` expression in a field ending in `Ref` or under + `secretProviders` is refused as + `runtime_config.substitution_in_reference`, and one in the authored project + or module files is refused as `source.environment_expression`. An empty + YAML scalar is now null rather than an empty string, so quote `""` where an + empty string is meant. `authentication.oidc.issuer` must be an `https` URL + without credentials, query, or fragment, with plain `http` accepted only on + an IPv4 loopback host, `audience` is at most 512 characters, and `jwksSource` + gains `kind: uri` beside `discovery` and `static`. A file missing + `apiVersion` or `kind` is refused as `runtime_config.invalid_api_version` + or `runtime_config.invalid_kind`. To migrate, replace `--config FILE` with + `--runtime-config FILE` in service definitions and write any number or + boolean that came from a variable directly in the file. See + [deploy a registry](../operate/breg/#write-the-runtime-configuration). + +{/* Evidence: crates/registry-breg/src/runtime_config.rs, parse_runtime_config_with_env and runtime_config_error_from_loader; + crates/registry-breg/src/cli.rs, removed_config_flag; + crates/registry-breg/src/contract.rs, reject_authored_environment_expression; + crates/registry-breg/tests/runtime_config.rs, a_substitution_inside_a_secret_reference_is_refused, + substituted_values_stay_strings, a_substituted_value_cannot_inject_document_structure, + oidc_issuer_and_audience_follow_the_shared_issuer_block and + jwks_source_uri_kind_skips_discovery_under_the_shared_rules; + crates/registry-breg/tests/compiler_contract.rs, an_authored_project_carrying_an_environment_expression_is_refused; + release/docker/Dockerfile.breg. */} + +- BREAKING: Registry Relay reads `runtime.yaml` through the shared + runtime configuration loader. The file now opens with + `apiVersion: registry.registrystack.org/relay-runtime/v1alpha1` and + `kind: RelayRuntimeConfig`; `server.bind` is `listener.bind`, + `packagePath` is an absolute `package.root` with an optional + `package.expectedDigest` pin, and `authentication.issuer` is `authentication.oidc` + with `issuer` and a `jwksSource` of `kind: discovery` or `kind: uri`. A + `secret:file/` reference resolves under the declared + `secretProviders.file.root` instead of beside the runtime file, and + `secret:env/` needs `secretProviders.environment`. `relay check` and + `relay serve` take the required absolute `--runtime-config FILE`; the + `--runtime` flag, the `RELAY_RUNTIME` variable, and the default path are + gone. A removed key is refused with the name of its replacement, and an + environment expression such as `${VAR}` in the authored `registry.yaml` + is refused as `contract.environment_expression`. See + [operate Registry Relay](../operate/relay/#bind-deployment-inputs-without-editing-the-package). + +{/* Evidence: crates/registry-relay-v2/src/contract.rs, RELAY_REMOVED_RUNTIME_KEYS and + removed_runtime_keys_are_refused_with_their_replacement; + crates/registry-relay-v2/src/cli.rs, the_runtime_configuration_is_named_explicitly_on_every_command; + crates/registry-relay-v2/src/contract.rs, an_authored_contract_carrying_an_environment_expression_is_refused; + release/docker/Dockerfile.relay. */} + +- BREAKING: Registry Discovery reads `runtime.yaml` through the shared + runtime configuration loader. The file now opens with + `apiVersion: registry.registrystack.org/discovery-runtime/v1alpha1` and + `kind: DiscoveryRuntimeConfig` instead of `schemaVersion`, and + `listener.address` is `listener.bind`. The binary takes + `--runtime-config FILE`, an absolute path free of symbolic links, in + place of `--runtime`, and the container image passes it in its default + command. A removed key is refused with the name of its replacement. See + [configure Registry Discovery](../configure/discovery/#deploy-and-restart-the-service). + +{/* Evidence: crates/registry-discovery/src/startup.rs, load_runtime() and + removed_runtime_keys_name_their_replacements; + release/docker/Dockerfile.discovery. */} + +- BREAKING: Registry Render reads `runtime.yaml` through the shared + runtime configuration loader. The file now opens with + `apiVersion: registry.registrystack.org/render-runtime/v1alpha1` and + `kind: RenderRuntimeConfig`; `server.bind` is `listener.bind` and is + required, `server.shutdownGraceSeconds` is + `listener.shutdownGraceSeconds`, `bundle.path` is `package.root`, and + secret providers are declared under `secretProviders`. Paths, including + `audit.path`, must be absolute, and a removed key, among them + `audit.directory`, `audit.integrityKeyRef`, and `audit.maxSegmentBytes`, + is refused with the name of its replacement. `serve`, `healthcheck`, and + `check` take `--runtime-config FILE`, and none reads a default + path or the `REGISTRY_RENDER_RUNTIME` variable. An environment + expression such as `${VAR}` in the authored bundle `manifest.yaml` is + refused with the field that holds it. See + [run Registry Render in serve mode](../operate/registry-render/). + +{/* Evidence: crates/registry-render/src/runtime.rs, RENDER_REMOVED_KEYS and load(); + crates/registry-render/src/cli.rs, Command::Serve; + crates/registry-render/src/runtime.rs, removed_keys_name_their_replacements; + crates/registry-render/src/manifest.rs, an_authored_manifest_carrying_an_environment_expression_is_refused. */} + +- BREAKING: Registry Scheduling reads `runtime.yaml` through the shared + runtime configuration loader. `authentication.oidc.jwksUri` is removed + and refused with the name of its replacement, a `jwksSource` of + `kind: uri`; `listener.bind` is required and no longer defaults to a + loopback port; the file and every configured path are refused when they + pass through a symbolic link; and an environment expression such as + `${VAR}` in the authored `scheduling.yaml`, records, or fixtures is + refused with the field that holds it. `runtime.yaml` string values + accept `${VAR}` outside `*Ref` fields and `secretProviders`, and an optional + `package.expectedDigest` pins the policy the runtime starts on. + +{/* Evidence: crates/registry-scheduling/src/config.rs, SCHEDULING_REMOVED_KEYS and + a_removed_jwks_uri_names_its_replacement; products/scheduling/RUNTIME-CONFIG.md. */} + +- Base Registry Engine (BReg) adds the `import` operation and import authorities, so a governed + entity can take a bulk load without granting `batch`, `create`, or `patch` + to a direct-write profile. `import` is create-only, is served only through + ingestion runs, and is not a direct write for change control, so + `change_control.direct_write_grant` accepts it on a controlled entity. An + import run is created only inside an open import authority that an operator + opens with `bregctl import-authority open` for one entity and one profile, + with a window of at most 30 days, a maximum item count, and optional pinned + input digests. Every chunk counts against the authority; closing it, its + expiry, its exhaustion, or a successor package activation stops the run with + `blockedReason` `importAuthorityClosed`. Opening, closing, expiry, + exhaustion, and supersession each append an audit record, and every run + audit record carries `importAuthorityId`. The compiler refuses `import` + without entity batch bounds (`import.batch_bounds.required`), without an + authenticated principal (`import.principal.required`), and beside a `batch` + grant on the same entity (`import.batch.redundant`). `bregctl data + validate` reports the source's `inputDigest` for pinning; the digest is a + label the client announces with the run and the server records, never + recomputed over the written items. The runtime + installs a new internal table and a run column on its next apply. See + [load a governed entity through an import window](../operate/breg-data/#load-a-governed-entity-through-an-import-window). + +- BREAKING: the ingestion-run `blockedReason` gains the value + `importAuthorityClosed`, and the Rust `BRegIngestionBlockedReason` enum + gains `ImportAuthorityClosed`; a client that matched the only previous value + exhaustively must handle the new one. The attempt that blocks a run on its + authority is recorded as `lastAttempt.outcome` `importAuthorityClosed`, not + `bindingChanged`, so the Rust `BRegIngestionAttemptOutcome` enum and the + Node.js `BRegIngestionAttemptOutcome` type gain the same value. The `ingestion.run_blocked` problem + detail now reads "The ingestion run is blocked and refuses further chunks." + because the run's `blockedReason`, not the detail, names the cause. + +- BREAKING: `bregctl data import` reports a run creation the server refuses + with `412` as `data.import.ingestion_run.import_authority_required` when the + grant is `import`, and `data.import.ingestion_run.precondition_failed` + otherwise, instead of `data.import.operation.refused`. A blocked run reports + `data.import.ingestion_run.import_authority_closed` or + `data.import.ingestion_run.blocked` with its reason. + +{/* Evidence: crates/registry-breg/src/import_authority.rs, admit_run() and admit_chunk(); + crates/registry-breg/src/compiler.rs, import checks; + crates/registry-breg/tests/postgres_import_authority.rs; + crates/registry-breg/tests/import_grant_compiler.rs; + crates/registry-bregctl/src/import_authority_lifecycle.rs; + crates/registry-bregctl/src/data_lifecycle.rs, a_run_blocked_by_its_import_authority_names_that_reason. */} + +- BREAKING: the Base Registry Engine runtime refuses to serve a database that + is not the one its instance claim names. The claim records the PostgreSQL + system identifier and database object identifier at the first apply, so a + logically restored copy (`pg_dump` and `pg_restore`) or a registry moved by + `pg_upgrade` no longer starts: `breg` refuses with the new `StartupError` + variant `InstanceClaimMismatch`, `GET /ready` answers `503`, and + `bregctl doctor` reports `startup.instance_claim.mismatch`. This keeps a + restored copy from serving beside its original as a divergent writer of + the same registry. Once the original is stopped for good, the new command + `bregctl instance-claim adopt --acknowledge-original-retired` moves the + claim to the copy with a raised epoch and, once that commits, appends an + audit entry naming the previous and the adopted claim; + `bregctl instance-claim status` reports whether the claim matches. A + physical copy (base backup, point-in-time recovery, snapshot, or promoted + replica) keeps its identity and is not refused. The claim is recorded only + into a fresh database, one with no committed revision and no commit head, + so an existing registry + records none when its next apply installs the new internal table: run + `instance-claim adopt` once after that apply, before starting the server. + Adopting also supersedes every open import authority, with its own audit + record, so a restored backup brings no authority back. On a managed + PostgreSQL service that withholds `pg_control_system()`, the claim compares + the database object identifier alone. The claim is checked at startup and + on readiness, not per request. A Rust `match` over `StartupError` must + handle the new variant. See + [back up and restore the database](../operate/breg-changes/#back-up-and-restore-the-database). + +{/* Evidence: crates/registry-breg/src/instance_claim.rs; + crates/registry-breg/src/startup.rs, verify_instance_claim(); + crates/registry-breg/src/import_authority.rs, supersede_every_open(); + crates/registry-breg/tests/postgres_startup.rs, a_restored_copy_refuses_to_serve_until_adopted + and a_database_that_withholds_its_system_identifier_still_serves_and_refuses_a_copy; + crates/registry-breg/tests/postgres_import_authority.rs, + installing_the_claim_beside_committed_history_leaves_the_database_to_adopt and + adopting_a_restored_copy_supersedes_every_open_authority; + crates/registry-bregctl/src/instance_claim_lifecycle.rs; + crates/registry-bregctl/tests/instance_claim.rs. */} + ## v0.34.0 beta-46 - `bregctl apply` reports a migration database it could not reach, or an @@ -275,7 +757,7 @@ relevant product pages on this site rather than duplicating release notes. See [change an active registry](../operate/breg-changes/). {/* Evidence: crates/registry-breg/src/package.rs, FieldVocabularyCodesAdded and ActionVocabularyCodesAdded; - crates/registry-breg/src/immediate_actions.rs, contract_only_adds_vocabulary_codes(); + crates/registry-breg/src/immediate_actions.rs, contract_only_widens(); crates/registry-breg/src/tooling.rs, ACTION_CODES_ADDED. */} - BREAKING: `bregctl init --from publicschema` and the four BReg starters diff --git a/docs/site/src/content/docs/configure/breg-change-control.mdx b/docs/site/src/content/docs/configure/breg-change-control.mdx index f6a022f9c0..01692b8101 100644 --- a/docs/site/src/content/docs/configure/breg-change-control.mdx +++ b/docs/site/src/content/docs/configure/breg-change-control.mdx @@ -5,7 +5,7 @@ status: current owner: registry-docs source_repos: - registry-stack -last_reviewed: "2026-09-04" +last_reviewed: "2026-09-25" doc_type: how-to locale: en standards_referenced: [] @@ -86,13 +86,34 @@ entities: | Member | Effect | |---|---| -| `changeControl.requiredFor` | The operations on the target that must go through a request. With `[patch]`, a direct patch of a placement is refused even for a profile that holds `patch`: the operation is absent from ordinary permissions, and only applying a request performs it. | +| `changeControl.requiredFor` | The operations on the target that must go through a request. With `[patch]`, a direct patch of a placement is refused even for a profile that holds `patch`: the operation is absent from ordinary permissions, and only applying a request performs it. A profile that still holds an operation `requiredFor` names, or holds `batch` on an entity controlled for `create` or `patch`, is refused by `check` with `change_control.direct_write_grant`. `import` is not a direct write and stays allowed (see below). | | `changeRequest.effects` | What applying the request writes. `target` names the record from a reference field, `operation` is `create` or `patch`, `set` maps target fields to request fields, and `clear` lists target fields to null. A request entity declares either `effects` or a `planner`, never both. | | `review` | The logical external review authority and policy frozen into the submitted proposal. Casework owns the policy, review stages, reviewer independence, and terminal result. A request that intentionally needs no external review declares `review: {mode: none}`. | | `onApproved` | What happens after BReg has reconciled an exact approved result. `manual` waits for a currently authorized caller to use `apply_request`. `automatic` requires a logical `executor`; runtime configuration binds that executor to a separate ordinary credential that uses the same source action and guards. | | `application` | Optional application preconditions over frozen request facts, current target facts, or signed Evidence. It does not choose review or application authority. | | `retention.mode` | `retain` or `operator_erase`. With `operator_erase`, an operator can erase the proposal detail of an applied or canceled request while the record and its state remain. | +A governed entity still needs a way to receive its initial records, and a later district or +program adds many more. Grant `import` on the entity for that, never `batch`: + +```yaml +accessProfiles: + - id: loader + operations: [import] +``` + +`import` is create only and is served only by the durable ingestion-run routes: it declares no +item route and no raw batch route, so it can never change an existing record, and change control +does not count it as a direct write. It loads nothing until an operator opens an import authority +for the entity and profile, bounded by volume, expiry, and optionally the input digests a run may +announce, as +[move data in bulk](../../operate/breg-data/#load-a-governed-entity-through-an-import-window) +describes. The profile needs an authenticated principal and the entity needs `batch` bounds; +`import` is refused beside `batch` on the same entity, on a change-request or consent-record +entity, and for a task-grant profile. Declaring `import` before change control is added keeps the +same route when the successor package governs the entity, so the package diff reports no route +removal. + Decide `retention.mode` before the first request is submitted, because a proposal stored under `retain` keeps its detail for as long as the record exists; [retain, erase, and audit](../../operate/breg-retention/) covers what an operator can erase later. ### Connect the Casework authority @@ -523,6 +544,8 @@ The report lists every compiled action with its inputs, effects, grants, contrac |---|---| | `check` reports a `change_request.*` error. | `effects` and `planner` are exclusive; `review` must be either `{mode: none}` or an authority and policy id; manual application must omit an executor; automatic application must name one. `check` does not run the script; evaluate it with `project planner-test`. | | A direct patch of a controlled entity is refused although the profile holds `patch`. | The entity's `changeControl.requiredFor` includes `patch`, so the write must arrive as an applied request. Submit a request of one of the `eligibleRequestTypes` that `explain change-requests` lists. | +| `check` reports `change_control.direct_write_grant`. | A profile still holds a controlled operation, or `batch` on an entity controlled for `create` or `patch`. Remove it; for bulk loads of new records, grant `import` in its place and load through an import authority. | +| `check` reports `import.batch_bounds.required`, `import.principal.required`, or `import.batch.redundant`. | An `import` grant needs the entity's `batch` bounds, a non-anonymous profile, and no `batch` grant on the same entity in any profile. | | `project planner-test` fails, or a submit is refused with `request.invalid`. | The script failed, returned an effect outside `writes`, attempted to return an obsolete disposition, or broke a bound. Read the message, fix the script or the ceiling, and rerun `project planner-test` with the same request file. | | `revise_request` answers `409 mutation.conflict`. | The settled review result rules the shape out: a rejected request can only be cancelled, and a send-back or an expired approval is answered by a revision, not a rebase. Read the request again and follow the action it offers. | | An invoke fails with `precondition.failed`. | A target changed between the target-conditions request and the invoke. Ask for target conditions again and invoke with the new ETags. | diff --git a/docs/site/src/content/docs/configure/casework.mdx b/docs/site/src/content/docs/configure/casework.mdx index 83a76cd20a..ea41557056 100644 --- a/docs/site/src/content/docs/configure/casework.mdx +++ b/docs/site/src/content/docs/configure/casework.mdx @@ -14,7 +14,7 @@ standards_referenced: You have decided an item in [Decide your first Casework item](../../tutorials/first-casework/), and now you want to author the policy a real team will work under. This page covers the project file: the access profiles that separate people from producer services, the queues work waits in, the unified review policies, the routing rules and clocks that move work over time, and the source declaration that binds a Base Registry Engine (BReg) register. -At the end, `caseworkctl package` writes a directory whose manifest an operator verifies before a runtime serves it. +At the end, `caseworkctl package` writes a directory whose `SHA256SUMS` an operator verifies before a runtime serves it. If `caseworkctl` is not installed yet, the release installer places `casework` and `caseworkctl` together in `~/.local/bin` after checking the release `SHA256SUMS`: @@ -43,7 +43,7 @@ caseworkctl init ./decisions --template standalone-decision `init` refuses a destination that already exists; it never overwrites a project. Point it at a new directory, or edit the project you already have. -`casework.yaml` is the policy a deployment serves: `runtime.example.yaml` is a starting point for the deployment's own file, `dev-clients.yaml` describes local callers, `fixtures/` holds the synthetic cases `caseworkctl test` runs, and `sources/` waits for the imported description of any source you connect. The project also includes an editor schema and VS Code settings for the runtime file. +`casework.yaml` is the policy a deployment serves: `runtime.example.yaml` is a starting point for the deployment's own file and serves the package `caseworkctl package . --output .casework/package` writes, `dev-clients.yaml` describes local callers, `fixtures/` holds the synthetic cases `caseworkctl test` runs, and `sources/` waits for the imported description of any source you connect. The project also includes an editor schema and VS Code settings for the runtime file. {/* Evidence: crates/registry-caseworkctl/src/project.rs, init() and CASEWORK_YAML; products/casework/examples/standalone-decision/casework.yaml. */} @@ -499,20 +499,22 @@ caseworkctl test ./decisions caseworkctl package ./decisions --output ./decisions-package ``` -The output is a directory, not an archive: `casework.yaml`, every source description the policy names, and `casework.package.json`, a manifest carrying a `policyDigest` over the sorted list of path, sha256, and byte count for each file. -The report repeats the digest and states `secretsIncluded: false` and `runtimeConfigurationIncluded: false`, because the runtime file and its secrets stay outside the package and outside review. +The output is a directory, not an archive: `casework.yaml`, every source description the policy names, and `SHA256SUMS`, one `sha256sum` line per file sorted by path. +The package digest is the SHA-256 digest of `SHA256SUMS` itself, so it covers every file, and `sha256sum -c SHA256SUMS` checks the directory by hand. +`--revision TEXT` adds a `REVISION` file with one free-text line, such as a ticket or a date, which the digest covers too. +The report states the `packageDigest` and states `secretsIncluded: false` and `runtimeConfigurationIncluded: false`, because the runtime file and its secrets stay outside the package and outside review. Packaging refuses an existing output directory, so each candidate lands in its own new directory. :::caution[A packaged policy is verified by byte, not by intent] -The runtime rebuilds the manifest from the directory at startup and refuses any difference: a changed byte, a missing file, or an extra file left beside the policy. Adding one blank line to a packaged `casework.yaml` is enough to make `casework` exit with `casework: the Casework policy package is invalid`, and so is dropping an unrelated note into the package root. A running process keeps the policy it verified at startup, so editing in place changes nothing it is serving and breaks the next restart. +The runtime rechecks every line of `SHA256SUMS` at startup and refuses any difference, naming each file: a changed byte, a missing file, or an extra file left beside the policy. Adding one blank line to a packaged `casework.yaml` is enough to make `casework` exit with `the package at … does not match its SHA256SUMS; changed: casework.yaml`, and so is dropping an unrelated note into the package root. A running process keeps the policy it verified at startup, so editing in place changes nothing it is serving and breaks the next restart. ::: Edit the authoring project instead, run the loop again, and package into a new directory. -`caseworkctl check` on a package directory validates the policy file and says nothing about the manifest, so treat a package as read-only once it exists. +`caseworkctl check` on a package directory validates the policy file and says nothing about `SHA256SUMS`, so treat a package as read-only once it exists. {/* Evidence: crates/registry-caseworkctl/src/project.rs, check(), explain(), simulate(), - test(), and package(); crates/registry-casework/src/config.rs, PolicyPackageManifest and - verify_policy_package(). */} + test(), and package(); crates/registry-casework/src/config.rs, verify_casework_package(); + crates/registry-platform-config/src/package.rs, write_package() and verify_package(). */} ## Connect a Base Registry Engine source @@ -647,15 +649,15 @@ An operator receives three things from you and nothing else. | Artifact | Purpose | | ------------------------ | ------------------------------------------------------------ | | The package directory | The policy the runtime verifies and serves | -| The manifest digest | The one value that says which policy this is | +| The package digest | The one value that says which policy this is | | `runtime.example.yaml` | The starting point for the deployment's own file | -Send the `policyDigest` out of band and have the operator confirm it against `casework.package.json` in the directory they received, because that digest is what distinguishes the reviewed policy from a copy of it. +Send the `packageDigest` out of band and have the operator set it as `package.expectedDigest`, because that digest is what distinguishes the reviewed policy from a copy of it. The runtime file, the database credentials, the identity provider, the audit key, and the source bindings are theirs, and none of them belongs in your project or your package. Tell them which queues need teams, since the inbox stays closed until every declared queue has one. {/* Evidence: crates/registry-caseworkctl/src/project.rs, package(); - crates/registry-casework/src/config.rs, PolicyPackageManifest and RuntimeConfig::check; + crates/registry-casework/src/config.rs, verify_casework_package() and RuntimeConfig::check; products/casework/examples/professional-review/runtime.example.yaml. */} ## Next diff --git a/docs/site/src/content/docs/configure/discovery.mdx b/docs/site/src/content/docs/configure/discovery.mdx index 4e21212e94..191b569240 100644 --- a/docs/site/src/content/docs/configure/discovery.mdx +++ b/docs/site/src/content/docs/configure/discovery.mdx @@ -1,6 +1,6 @@ --- -title: Build and run a Registry Discovery index -description: Build a bounded Registry Discovery index from approved provider descriptions and run its read-only service. +title: Package and run a Registry Discovery index +description: Package a bounded Registry Discovery index from approved provider descriptions and run its read-only service. status: draft owner: registry-docs source_repos: @@ -16,12 +16,12 @@ standards_referenced: --- Use this guide when you operate a Registry Discovery catalog for known Evidence Gateway and -Registry Relay providers. You maintain provider URLs and evidence-type mappings, build one immutable -index on demand, then restart the read-only service with that index. +Registry Relay providers. You maintain provider URLs and evidence-type mappings, package one +immutable index on demand, then restart the read-only service with that package. {/* Evidence: products/discovery/README.md describes the operator flow as offline `check`, one - explicit `build`, immutable-index deployment, then restart. `crates/registry-discoveryctl/src/lib.rs` - exposes only `check` and `build`; `crates/registry-discovery/src/server.rs` fixes the public routes. */} + explicit `package`, immutable-package deployment, then restart. `crates/registry-discoveryctl/src/lib.rs` + exposes only `check` and `package`; `crates/registry-discovery/src/server.rs` fixes the public routes. */} ## When to use this @@ -40,8 +40,8 @@ You need these inputs: - A provider-supplied public description URL, such as `https:///catalog.jsonld`. Record the URL out of band with the provider. -- A local directory for the catalog project and a separate deployment directory for the index and - runtime file. +- A local directory for the catalog project, a new output directory for the package, and a separate + deployment runtime file. - The `discoveryctl` and `discovery` binaries. The provider description uses the closed Registry Discovery JSON-LD profile. The profile is a @@ -55,7 +55,7 @@ It does not claim complete DCAT-AP or BRegDCAT-AP conformance. ## Add the approved provider URLs -Create `/origins.yaml`. Each enabled origin is fetched only during a build. +Create `/origins.yaml`. Each enabled origin is fetched only during packaging. ```yaml schemaVersion: registry-discovery/origins/v1alpha1 @@ -101,7 +101,7 @@ declares one. ## Check the authoring project -Run the offline check before any build. +Run the offline check before packaging. ```sh discoveryctl check --project "" @@ -119,42 +119,52 @@ The check reads the two authoring inputs but does not contact the provider URL. `crates/registry-discoveryctl/src/project.rs`, test `check_is_offline_and_accepts_an_unreachable_https_origin`, proves that validation is offline. */} -## Build one bounded index +## Package one bounded index -Build into the deployment directory when the approved provider descriptions are available. +Package into a new directory when the approved provider descriptions are available. ```sh -discoveryctl build \ +discoveryctl package \ --project "" \ - --output "/discovery-index.json" + --output "/discovery-package" \ + --revision "" ``` -Expected output reports independent semantic revisions: +Expected output reports the package digest and independent semantic revisions: ```text -built catalogRevision=sha256:<64-lowercase-hex-digits> mappingRevision=sha256:<64-lowercase-hex-digits> +packaged packageDigest=sha256:<64-lowercase-hex-digits> catalogRevision=sha256:<64-lowercase-hex-digits> mappingRevision=sha256:<64-lowercase-hex-digits> ``` -The build fetches every enabled origin once, records the exact fetched-byte digest and fetch time, -and records `builtAt` after every origin has been collected and the semantic revisions compile. -The builder validates and syncs the complete file before replacing the active index, then syncs the -deployment directory. Run the build again only when you intend to publish a new index revision. +The command fetches every enabled origin once, records the exact fetched-byte digest and fetch time, +and records `builtAt` after every origin has been collected and the semantic revisions compile. It +writes `discovery-index.json`, optional `REVISION`, and `SHA256SUMS` into a directory that must not +already exist. The package digest is the SHA-256 digest of the exact `SHA256SUMS` bytes. -{/* Evidence: `crates/registry-discoveryctl/src/build.rs`, `fetch_origins()` and `atomic_replace()`; +Packaging the same compiled index bytes and revision twice produces the same package digest. A new +collection records new `originFetchedAt` and `builtAt` provenance, so its package digest may change +even when `catalogRevision` and `mappingRevision` stay the same. Build a new directory and deploy it +as a unit. Do not edit a package in place. + +{/* Evidence: `crates/registry-discoveryctl/src/build.rs`, `fetch_origins()`, `write_index_package()`, + and `packaging_the_same_compiled_index_twice_is_repeatable`; `crates/registry-discoveryctl/tests/build.rs`, - `build_fetches_each_origin_once_and_preserves_semantic_revisions` and - `production_build_time_is_captured_after_origin_collection`; the production `atomic_replace()` - path calls `sync_all()` for the staged file and its parent directory. */} + `package_fetches_each_origin_once_and_preserves_semantic_revisions` and + `production_package_time_is_captured_after_origin_collection`. */} ## Deploy and restart the service -Place this `runtime.yaml` beside `discovery-index.json` in the deployment directory. +Place this `runtime.yaml` outside the package directory. Set `package.root` to its absolute path and +copy the reported package digest into `package.expectedDigest` when you want to pin the deployment. ```yaml -schemaVersion: registry-discovery/runtime/v1alpha1 +apiVersion: registry.registrystack.org/discovery-runtime/v1alpha1 +kind: DiscoveryRuntimeConfig listener: - address: 127.0.0.1:8080 -indexPath: discovery-index.json + bind: 127.0.0.1:8080 +package: + root: /srv/registry-discovery/package + expectedDigest: sha256: limits: maximumRequestBytes: 65536 maximumResponseBytes: 1048576 @@ -165,14 +175,22 @@ limits: logLevel: info ``` -Start the new revision after replacing the runtime directory contents. +Start the new revision after deploying the complete package directory. ```sh -discovery --runtime "/runtime.yaml" +discovery --runtime-config "/runtime.yaml" ``` -The process remains running and serves the index named by the relative `indexPath`. Verify readiness -from a network location that can reach the listener. +The runtime file is read through the shared runtime configuration loader, as the other Registry +Stack runtimes read theirs. Its path must be absolute and free of symbolic links, `listener.bind` is +required and takes an IP address host, and a string value may name an environment variable as +`${NAME}`, `${NAME:-default}`, or `${NAME:?message}`. A key from the earlier grammar, such as +`schemaVersion`, `listener.address`, or `indexPath`, is refused with the name of its replacement. + +Before binding the listener, the runtime recomputes every `SHA256SUMS` entry, refuses changed, +missing, or extra files by name, checks `package.expectedDigest` when set, then parses the exact +verified `discovery-index.json` bytes. The process remains running with that captured index. Verify +readiness from a network location that can reach the listener. ```sh curl --fail-with-body "http://127.0.0.1:8080/ready" @@ -182,14 +200,19 @@ curl --fail-with-body "http://127.0.0.1:8080/ready" {"status":"ready"} ``` -The runtime accepts only its listener, relative index path, limits, and log level. It does not carry +The runtime accepts only its listener, package location and optional pin, limits, and log level. It does not carry provider credentials, origin-fetch configuration, mappings, or application trust configuration. The runtime acquires one of four request-body permits before reading a body and holds the permit through request handling. It also rejects malformed percent escapes, invalid UTF-8 query values, duplicate media-type headers, and request media parameters other than UTF-8 JSON. {/* Evidence: `crates/registry-discovery/src/startup.rs`, `RuntimeConfig`, `load_runtime()`, and - test `runtime_is_closed_and_contains_no_origin_mapping_trust_or_fetch_configuration`; + tests `runtime_is_closed_and_contains_no_origin_mapping_trust_or_fetch_configuration`, + `the_runtime_file_is_read_through_the_shared_loader`, + `removed_runtime_keys_name_their_replacements`, + `the_listener_bind_is_required_and_substitutes_from_the_environment`, + `startup_verifies_package_and_refuses_expected_digest_mismatch_with_common_shape`, and + `exact_consumed_index_bytes_remain_bound_to_the_verified_package`; `crates/registry-discovery/src/server.rs`, tests `request_body_capacity_is_acquired_before_buffering_and_recovers` and `request_media_type_accepts_only_bare_or_utf8_json`; `crates/registry-discovery/src/query.rs`, @@ -252,7 +275,9 @@ credentials or native request data to Registry Discovery. | Symptom | Cause | Fix | | --- | --- | --- | | `discoveryctl check` fails | The origins or mapping document is not in the closed authoring shape. | Correct the schema version, identifiers, and duplicate values, then run the offline check again. | -| `discoveryctl build` fails | An enabled provider URL could not be fetched safely or did not return the exact profile media type. | Confirm the URL with the provider and its public description deployment. Keep the previous index running until a complete build succeeds. | +| `discoveryctl package` fails | An enabled provider URL could not be fetched safely, did not return the exact profile media type, or the output directory already exists. | Confirm the provider's public description deployment and choose a new output directory. Keep the previous package running until packaging succeeds. | +| Startup says the package does not match `SHA256SUMS` | A listed file changed, is missing, or an extra file appeared in the package directory. | Redeploy the whole directory produced by `discoveryctl package`. | +| Startup reports different expected and found package digests | `package.expectedDigest` pins a different package. | Deploy the pinned package or update the pin to the reviewed `packageDigest`. | | A search returns no items | The index contains no advertisement that matches every supplied filter. | Inspect the advertised capability IDs and use a narrower or correct filter set. | | Native connection is refused | The application did not accept the selected provider under its local product trust configuration. | Resolve the native Evidence Gateway or Registry Relay trust decision before retrying the direct connection. | diff --git a/docs/site/src/content/docs/configure/enable-sd-jwt-vc.mdx b/docs/site/src/content/docs/configure/enable-sd-jwt-vc.mdx index 279024809b..9704bad27e 100644 --- a/docs/site/src/content/docs/configure/enable-sd-jwt-vc.mdx +++ b/docs/site/src/content/docs/configure/enable-sd-jwt-vc.mdx @@ -56,8 +56,8 @@ HTTP 406. Run the candidate checks and deploy the exact reviewed revision: ```sh -evidencectl test "" -evidence check --runtime /runtime.yaml +evidencectl test "" --target "" +evidence check --runtime-config "/runtime.yaml" ``` ## Choose root or structured disclosures diff --git a/docs/site/src/content/docs/configure/evidence.mdx b/docs/site/src/content/docs/configure/evidence.mdx index 52e261da42..f6a5b96750 100644 --- a/docs/site/src/content/docs/configure/evidence.mdx +++ b/docs/site/src/content/docs/configure/evidence.mdx @@ -87,7 +87,7 @@ evidencectl check "" \ ``` This applies the target's declared production or evidence-grade assurance profile and validates -its governance, runtime structure, public keys, source connections, and governed bundle. It does +its governance, runtime structure, public keys, source connections, and governed package. It does not require target-host paths or secrets to exist, run fixtures, or prove live readiness. {/* Evidence: crates/registry-evidencectl/src/check.rs, check() and explain(); @@ -95,31 +95,33 @@ not require target-host paths or secrets to exist, run fixtures, or prove live r ## The runtime file -`runtime.yaml` binds one governed bundle to one process: where the bundle lives, what the -listener binds to, where secrets sit, where audit entries go, and which private CA files a source may -trust. It is process-local operator configuration, not part of the reviewed bundle, and +`runtime.yaml` binds one governed package to one process: where the package lives, what the +listener binds to, where secrets sit, where audit entries go, and which private CA files a source or the +access-token issuer may trust. It is process-local operator configuration, not part of the reviewed package, and `products/evidence/contracts/runtime.schema.yaml` is its schema of record. Top-level required fields: | Field | Binds | | --- | --- | -| `version` | Fixed at `1`. | -| `bundleDirectory` | Absolute path to the governed `bundle/` directory. | -| `listener` | `bindHost` (loopback, private IPv4, or unique-local IPv6 only), `port`, `tlsTermination` (fixed at `operator-controlled-upstream`: TLS terminates ahead of Evidence Gateway), `trustProxyIdentityHeaders` (fixed `false`: proxy-supplied identity is never trusted), plus request-size, concurrency, timeout, and shutdown-grace bounds. | -| `secretProviders.file.root` | The owner-only directory the file secret provider resolves every `secret:file/` reference beneath. | +| `apiVersion` | Fixed at `registry.registrystack.org/evidence-runtime/v1alpha1`. | +| `kind` | Fixed at `EvidenceRuntimeConfig`. | +| `package` | `root`, the stable absolute path to the governed package directory, plus an optional `expectedDigest` that must match the digest of its verified `SHA256SUMS` file. | +| `listener` | `bind` as one `host:port` value (loopback, private IPv4, or unique-local IPv6 only; an IPv6 host goes in brackets), `tlsTermination` (fixed at `operator-controlled-upstream`: TLS terminates ahead of Evidence Gateway), `trustProxyIdentityHeaders` (fixed `false`: proxy-supplied identity is never trusted), plus request-size, concurrency, timeout, and shutdown-grace bounds. | +| `secretProviders` | The secret providers the deployment enables, at least one. `file.root` is the owner-only directory every `secret:file/` reference resolves beneath; `environment: {}` enables `secret:env/` references to operator-named variables, and any reference in the reviewed bundle may then name any variable in the process environment; prefer the file provider, or give the process a dedicated environment holding only the secrets it needs. A reference to a provider that is not declared is refused. | | `audit` | `destination` (`file` by default, or `stdout`); for `file`, the absolute `path` of the active audit file, `rotateBytes` (100 MiB by default), and `retainDays` (90 by default). | -| `outboundTls` | `systemRoots` (fixed `true`) plus named `trustProfiles`, each a local id bound to one CA bundle file a source can select through `tlsTrustProfile`. | +| `outboundTls` | `systemRoots` (fixed `true`) plus named `trustProfiles`, each a local id bound to one CA bundle file a source or `authentication` can select through `tlsTrustProfile`. | -An optional `metricsListener` (`bindHost`, `port`) serves `GET /metrics` on a second private +An optional `metricsListener` (`bind`) serves `GET /metrics` on a second private binding; absent by default, it is documented in `products/evidence/OPERATOR-CONTRACT.md` under Metrics reference rather than in the public Evidence Gateway contract. -`runtime.yaml` cannot override anything the bundle governs. The schema's `ownership` block closes -the allowed set to the bundle directory, listener binding and process limits, the optional -metrics listener binding, the file-secret root, the audit destination with its path, rotation size, and retention, and logical +`runtime.yaml` cannot override anything the package governs. The schema's `ownership` block closes +the allowed set to the package root and optional expected digest, listener binding and +process limits, the optional metrics listener binding, the enabled secret providers and file-secret +root, the audit destination with its path, rotation size, and retention, and logical private-CA file bindings. Every other field, including service identity, authentication, -authority, sources, disclosure, and signing policy, belongs to the bundle alone. +authority, sources, disclosure, and signing policy, belongs to the package alone. ## The bundle @@ -129,7 +131,7 @@ atomic revision (`products/evidence/contracts/bundle.schema.yaml`). Its top-leve | Section | Declares | | --- | --- | | `service`, `issuer` | The technical provider and the legal issuing authority, both URIs. | -| `authentication` | The one trusted OIDC access-token profile: issuer, audiences, token type, algorithms, JWKS URI, maximum token lifetime, revoked key identifiers, and the claim names for principal, requester tags, evidence audience, grant id, and grant authority. | +| `authentication` | `oidc`, the one trusted OIDC access-token issuer: issuer, the single accepted `audience`, token type, algorithms, `jwksSource` (only `kind: uri` with a fixed `uri`), maximum token lifetime, revoked key identifiers, and the claim names for principal, requester tags, evidence audience, grant id, and grant authority. An optional `tlsTrustProfile` names the private CA the key set is served under; see [An issuer behind a private CA](#an-issuer-behind-a-private-ca). | | `audit` | Audit pseudonyms: `hashKeyRef`, the secret the pseudonym key is derived from, and `hashKeyVersion`, stamped into every pseudonym. Every audit gate is fail closed. | | `subjectBinding` | The secret reference and key version behind the audience-scoped entity-reference HMAC. | | `rateLimits` | Per-principal request, burst, and failed-selector-attempt bounds. | @@ -142,7 +144,7 @@ atomic revision (`products/evidence/contracts/bundle.schema.yaml`). Its top-leve Configure the OpenID Connect (OIDC) issuer to include `registry_actor_kind` on every Evidence access token. The claim name defaults to `registry_actor_kind` and can be changed with -`authentication.claims.actorKind`. Its value must be exactly `human`, `agent`, or `service`; +`authentication.oidc.claims.actorKind`. Its value must be exactly `human`, `agent`, or `service`; a missing or malformed value causes authentication to fail before Evidence evaluates authority or accesses a source. @@ -181,6 +183,44 @@ complete path segment. Neither a response nor a script can select a source, orig or additional call. The requirement's effective acquisition posture is the weaker posture of its two sources (`products/evidence/contracts/source-contract.yaml`, invariant `V1-I40`). +### An issuer behind a private CA + +When the access-token issuer serves its key set under a certificate authority that is not in the +system trust store, name a trust profile on `authentication.oidc` and bind it in `runtime.yaml`, exactly +as a source does: + +```yaml +# bundle (governance.yaml in a production project) +authentication: + oidc: + jwksSource: + kind: uri + uri: https://issuer.internal.example/oauth2/jwks + tlsTrustProfile: issuer-pki +``` + +```yaml +# runtime.yaml +outboundTls: + systemRoots: true + trustProfiles: + issuer-pki: {caBundleFile: /etc/registry-evidence/ca/issuer-pki.pem} +``` + +The CA file is trusted beside the system roots for the `jwksSource.uri` connection alone; it widens no +source connection and no other process. Hostname verification stays on, and there is no setting +that disables verification. The file follows the same rules as a source CA file: an absolute path, +no write permission for anyone, and a bounded PEM bundle. Changing it requires a restart but does +not change the package digest. The binding is exact in both directions, so a runtime file that binds the +profile without the bundle naming it, or the reverse, is refused. A source and the issuer may share +one profile. A local HTTP `jwksSource.uri` cannot name a profile. `evidence check +--require-runtime-dependencies` fetches the key set through the profile, so a missing or wrong CA +is reported before the listener binds. + +{/* Evidence: crates/registry-evidence/src/runtime.rs, issuer_trust_roots(); + crates/registry-evidence/src/bundle.rs, validate_runtime_bindings(); + products/evidence/contracts/bundle.schema.yaml, authentication.oidc.tlsTrustProfile. */} + ## What Rhai scripts may and may not do A deployment project supplies three kinds of Rhai script, one per source or requirement, and Rust @@ -344,26 +384,29 @@ Version 1 permits no reload, merge, mutation, governed-field override, or fallba runtime file: a project is either the exact bytes Evidence Gateway loaded at startup, or it is a different revision that requires a restart. -Freeze the project before validating or serving it: +Freeze the installed package and deployment target before validating or serving them: ```sh -chmod -R a-w bundle && chmod 444 runtime.yaml +chmod -R a-w "" && chmod 444 "/runtime.yaml" ``` -The bundle directory and every file beneath it carry no write bits, `runtime.yaml` is mode `444`, +The package directory and every file beneath it carry no write bits, the target's `runtime.yaml` +is mode `444`, and the secret root stays owner-only at mode `0700` with mode `0600` secret files. Evidence Gateway Version 1 supports Unix targets only because these invariants rely on owner, mode, no-follow, link-count, and open-file-identity checks the platform provides (`products/evidence/contracts/runtime.schema.yaml`, `platform`). To edit the project again, -restore write permission, make the change, and freeze it again before the next `evidence check`. +make the change in the editable project, publish a new package, and freeze the new package before +the next `evidence check`. ## Validating a project Two offline commands prove a project before it ever binds a port: ```sh -evidence --runtime runtime.yaml check -evidence --runtime runtime.yaml evaluate --fixture "bundle/fixtures/.yaml" +evidence check --runtime-config "/runtime.yaml" +evidence evaluate --runtime-config "/runtime.yaml" \ + --fixture "fixtures/.yaml" ``` `evidence check` loads, compiles, and validates the complete bundle and runtime file together: @@ -381,16 +424,13 @@ never a response, fact, derived, or selector value, and each fixture's own `diagnostics_exclude` canaries are checked against it before it is printed. The flag is offline only: `evidence serve` has no equivalent. -`evidencectl test ` accepts an editable or deployment project. For a -deployment project, it runs `check`, then `evaluate` against every fixture path -the bundle's requirements reference, and reports `PASS` or `FAIL` per step +`evidencectl test [--target ]` privately compiles the +editable project, runs the runtime-owned bundle checks, then evaluates every fixture the +questions reference and reports `PASS` or `FAIL` per step (`crates/registry-evidencectl/src/fixtures.rs`). Add `--explain` to relay each fixture's trace with -its step, or as that fixture's `trace` field under `--format json`. The incomplete workspace created by -the OpenAPI form of `evidencectl init` has no fixture cases, so it cannot pass -this gate until the author adds them. For an editable project with fixtures, -Evidencectl privately compiles a bundle before invoking the same runtime-owned -bundle checks. Both `check` and `test` must pass before `evidence serve` runs -the revision. +its step, or as that fixture's `trace` field under `--format json`. The incomplete workspace created +by the OpenAPI form of `evidencectl init` has no fixture cases, so it cannot pass this gate until +the author adds them. Both `check` and `test` must pass before `evidence serve` runs the revision. ## Build a production candidate @@ -421,12 +461,12 @@ Build a new candidate directory with explicit target and output paths: ```sh evidencectl package "" \ --target "/environments/production/evidence" \ - --output "" + --output "" ``` -The output is create-only and contains `runtime.yaml` and a closed `bundle/`. It contains no -secret values, source responses, local request state, or audit records. The compiler validates -the resulting bundle with the `evidence` binary and every referenced fixture before publishing. +The output is one create-only package with `SHA256SUMS` at its root. Runtime configuration stays in +the target. The package contains no secret values, source responses, local request state, or audit +records. The compiler validates it with the `evidence` binary and every referenced fixture before publishing. It does not contact an identity provider or a source endpoint. Use [Build and deploy an Evidence Gateway project](../../tutorials/build-and-deploy-evidence-project/) diff --git a/docs/site/src/content/docs/explanation/architecture.mdx b/docs/site/src/content/docs/explanation/architecture.mdx index 0d0809b53e..6a92959048 100644 --- a/docs/site/src/content/docs/explanation/architecture.mdx +++ b/docs/site/src/content/docs/explanation/architecture.mdx @@ -225,10 +225,11 @@ decision. fixed route set the package compiled. There is no adaptation step: the runtime executes only the statements the compiler produced, over the source columns the contract named, and returns only the properties a disclosure profile declares. The one runtime file (`runtime.yaml`, - `kind: RelayRuntime`) binds local paths, the audit sink, secret references, limits, and an optional + `kind: RelayRuntimeConfig`) binds local paths, the audit destination, secret references, limits, and an optional OIDC issuer, and cannot widen anything the package sealed. - {/* Evidence: the RelayRuntime struct is a closed deny_unknown_fields schema over server, - packagePath, sources, authentication, audit, cursor, limits, quotas, and shutdown only, + {/* Evidence: the RelayRuntime struct is a closed deny_unknown_fields schema over listener, + package, secretProviders, sources, authentication, audit, cursor, limits, quotas, and + shutdown only, crates/registry-relay-v2/src/contract.rs; startup verifies the package before opening any other resource, crates/registry-relay-v2/src/startup.rs:96-104. */} 7. Evidence Gateway evaluates one predefined, versioned requirement per request, aligned with CCCEV through diff --git a/docs/site/src/content/docs/explanation/discovery-as-an-index.mdx b/docs/site/src/content/docs/explanation/discovery-as-an-index.mdx index aa0b1ea168..7cbb327939 100644 --- a/docs/site/src/content/docs/explanation/discovery-as-an-index.mdx +++ b/docs/site/src/content/docs/explanation/discovery-as-an-index.mdx @@ -27,14 +27,23 @@ index, and an application decides whether to trust and directly use a selected p A provider publishes one closed JSON-LD description containing its public service advertisement. The catalog operator keeps an explicit allowlist of those description URLs and performs a bounded, -one-shot build. The build records each origin URL, fetched-byte digest, and fetch time beside every -indexed service record. +one-shot package operation. That operation records each origin URL, fetched-byte digest, and fetch +time beside every indexed service record. {/* Evidence: `crates/registry-discovery-profile/src/lib.rs`, `DiscoveryDescription` and `ServiceDescription`, define the closed provider publication. `crates/registry-discoveryctl/src/project.rs`, `ApprovedOrigin`, defines the explicit origin list. `crates/registry-discoveryctl/src/build.rs`, `fetch_origins()`, populates `OriginSummary` and `ServiceRecord` provenance. */} +The deployable result is a directory containing the canonical index and `SHA256SUMS`, plus an +optional operator revision. `package.root` always names that directory. At startup the runtime +verifies every listed file, checks the optional `package.expectedDigest` pin, and retains the exact +verified index bytes for its query directory before it opens the listener. + +{/* Evidence: `crates/registry-discoveryctl/src/build.rs`, `write_index_package()` writes the shared + package; `crates/registry-discovery/src/startup.rs`, `prepare()` and `load_verified_index()` + bind the runtime consumer to the verified package bytes. */} + The profile uses a selected set of Data Catalog Vocabulary (DCAT) 3, DCAT-AP 3.0.1, and BRegDCAT-AP terms. Registry Discovery does not claim full DCAT-AP or BRegDCAT-AP conformance. Its offline tooling transforms only this pinned context and evaluates a selected Shapes Constraint @@ -340,7 +349,7 @@ input and output. ## Why the split reduces maintenance The provider maintains one public description URL. The catalog operator maintains a small explicit -origins file, any evidence-type mappings, and an intentional build-and-restart loop. The application +origins file, any evidence-type mappings, and an intentional package-and-restart loop. The application maintains native provider trust where it already belongs. No component must synchronize a central provider registration database, shared credentials, or a proxy policy. @@ -352,6 +361,6 @@ provider registration database, shared credentials, or a proxy policy. ## Related - [Publish and consume a Registry Discovery index](../../tutorials/publish-and-consume-discovery-index/) -- [Build and run a Registry Discovery index](../../configure/discovery/) +- [Package and run a Registry Discovery index](../../configure/discovery/) - [Records stay home](../records-stay-home/) - [Architecture](../architecture/) diff --git a/docs/site/src/content/docs/explanation/dpi-safeguards-alignment.mdx b/docs/site/src/content/docs/explanation/dpi-safeguards-alignment.mdx index 39f34f7a9a..eb79cbc2a0 100644 --- a/docs/site/src/content/docs/explanation/dpi-safeguards-alignment.mdx +++ b/docs/site/src/content/docs/explanation/dpi-safeguards-alignment.mdx @@ -86,7 +86,7 @@ framework. | Who can read protected data? | Each operation Relay compiles carries an access profile that is either public or protected. A protected operation requires a bearer token from the one configured OpenID Connect (OIDC) issuer and the scope the contract names; there is no API-key mode and no static-credential mode. Evidence Gateway independently authenticates the assertion caller against its own one configured OIDC issuer and matches one complete entitlement before it contacts any source. | Deployment policy decides who receives scopes, grants, and workload authority. The two products do not share an issuer, an authorization model, or a decision. | | Is the exchange purpose-bound? | A Relay access profile may declare a purpose constraint: a claim name plus a closed list of allowed values, read from the verified token and never from a request field. It may also declare an authority row binding, so every returned row is pinned to a value the token carries. Denials use a closed set of 26 [stable problem codes](../../reference/errors/) under `https://id.registrystack.org/problems/registry-relay/`. | A declared purpose is enforcement only where an operation carries the constraint and the issuer actually populates the claim. Relay performs no consent check and no jurisdiction check; the `processingDescriptions` block it seals is a reviewed declaration, not a runtime gate. | | Is data minimized? | Relay returns only the properties a compiled disclosure profile declares, and can coarsen a returned value through exactly two fixed transforms: partial-string masking and date-precision reduction to year or year-month. Evidence Gateway returns only the concepts a predefined requirement declares, each in one of twelve closed [value forms](../disclosure-modes-and-computed-answers/). | The operator still chooses the properties, the requirements, and how coarse each declared value form is. Relay minimization is fixed per operation at compile time, not negotiated per caller. | -| Can the exchange be reviewed later? | Relay writes one audit record per served data request naming the resource, the operation, the contract revision, the access and disclosure profiles, the transforms applied, and the principal kind, and returns 503 rather than answering when the audit sink is unavailable. Evidence Gateway writes keyed audit records at two durable gates. Package and configuration digests and Selective Disclosure JWT Verifiable Credentials (SD-JWT VC) add artifact-level review evidence. | Audit records support accountability. They are not accountability by themselves. A Relay `packageRevision` is an integrity digest over the sealed package, not an authenticity proof, so it shows that a package is intact and not who produced it. | +| Can the exchange be reviewed later? | Relay writes one audit record per served data request naming the resource, the operation, the contract revision, the access and disclosure profiles, the transforms applied, and the principal kind, and returns 503 rather than answering when the audit sink is unavailable. Evidence Gateway writes keyed audit records at two durable gates. Package and configuration digests and Selective Disclosure JWT Verifiable Credentials (SD-JWT VC) add artifact-level review evidence. | Audit records support accountability. They are not accountability by themselves. A Relay package digest is an integrity digest over the sealed package, not an authenticity proof, so it shows that a package is intact and not who produced it. | | Can other systems interoperate? | Manifest emits standards-shaped metadata; Evidence Gateway publishes a product-level OpenAPI document, and Relay serves an OpenAPI description generated from the deployment's own compiled contract at `GET /openapi.json`; Evidence Gateway can serialize one assertion as an SD-JWT VC under a frozen local profile. | These are scoped adoption claims, not blanket conformance to every named standard. Relay's OpenAPI describes one deployment, so two deployments do not share an API document. The SD-JWT VC profile is a second encoding of one response, and it excludes OpenID for Verifiable Credential Issuance (OID4VCI) in every part. | {/* Evidence: PurposeConstraint { claim, allowed } and AuthorityRowBinding over a token claim or the @@ -98,9 +98,9 @@ framework. ProblemCode::AuditUnavailable (503) with no configuration to disable it, crates/registry-relay-v2/src/api.rs; the OpenAPI document is generated from the compiled contract and served at GET /openapi.json by the fixed router, - crates/registry-relay-v2/src/server.rs; packageRevision is documented in the source as an + crates/registry-relay-v2/src/server.rs; the package digest is documented in the source as an integrity digest and not an authenticity proof, - crates/registry-relay-v2/src/package.rs:348-350. */} + crates/registry-relay-v2/src/package.rs. */} ## Project roles diff --git a/docs/site/src/content/docs/explanation/known-limitations.mdx b/docs/site/src/content/docs/explanation/known-limitations.mdx index 76ff448a57..4d095cae07 100644 --- a/docs/site/src/content/docs/explanation/known-limitations.mdx +++ b/docs/site/src/content/docs/explanation/known-limitations.mdx @@ -133,8 +133,8 @@ the product does not do. must name a backup artifact that the tooling validates but never creates. - Erasure runs one way and stops early: `history erase` has no dry run and no undo, and refuses more than 10,000 revisions in one transaction. `history rebaseline`, the only way to restore - snapshot coverage after an erasure, refuses a registry holding more than 1,000 live rows, so a - larger registry does not get that coverage back. + snapshot coverage after an erasure, verifies every live row in one transaction and holds writes + to every entity table until it commits. - No request-rate limit: the runtime enforces operator-set timeouts and connection-pool bounds, but no rate limit of its own. A registry that admits anonymous reads belongs behind an upstream limiter. @@ -391,8 +391,8 @@ that narrowness rather than an unfinished feature. Its boundaries are covered in ### What the guarantees do not cover -- A package is verified, not authenticated: `packageRevision` is a SHA-256 integrity digest over - the canonical manifest. Startup re-runs the compiler and the artifact generator and requires +- A package is verified, not authenticated: the package digest is a SHA-256 integrity digest over + the package's `SHA256SUMS`. Startup re-runs the compiler and the artifact generator and requires byte-for-byte equality, which detects drift and tampering. It cannot distinguish a legitimate package from a well-formed forgery, because nothing signs a package. Package provenance rests on the delivery path and the filesystem ownership checks, not on the format. diff --git a/docs/site/src/content/docs/explanation/publishing-pipeline.mdx b/docs/site/src/content/docs/explanation/publishing-pipeline.mdx index 02ecbb65ec..6e90e0d3f5 100644 --- a/docs/site/src/content/docs/explanation/publishing-pipeline.mdx +++ b/docs/site/src/content/docs/explanation/publishing-pipeline.mdx @@ -132,8 +132,8 @@ The full pipeline, from an existing SQLite database to a running service: | Prove behavior offline | `relayctl test ` | The project's fixture cases run through the same kernel the server uses. | | Inspect the artifacts | `relayctl generate --output ` | The deterministic generated inventory: OpenAPI, JSON Schema, SHACL, JSON-LD context and vocabulary. | | Classify the change | `relayctl diff ` | Every difference, each carrying a change class and an impact. | -| Seal it | `relayctl package --output ` | A sealed package directory with a revision digest over its own contents. | -| Serve it | `relay serve --runtime ` | One process serving one package, with deployment-local settings supplied separately. | +| Seal it | `relayctl package --output ` | A sealed package directory whose `SHA256SUMS` lists every file, and the package digest over it. | +| Serve it | `relay serve --runtime-config ` | One process serving one package, with deployment-local settings supplied separately. | Two properties of this pipeline matter more than the command list. @@ -167,28 +167,28 @@ A global `--json` flag emits the shared report for local automation. `relayctl package` writes a directory containing the authored `registry.yaml`, the governed input files it referenced under `governed/`, the compiled model at `compiled/registry.json`, the generated artifacts under `generated/`, and a -`relay-package.json` manifest that lists every file with its digest. -The manifest's `packageRevision` is a digest over the canonical form of that list. -The command refuses to write into a directory that already exists, and it leaves a -partially written directory in place rather than deleting evidence of a failed attempt. +`SHA256SUMS` file that lists every other file with its digest. +The package digest is the digest of `SHA256SUMS`, the value `package.expectedDigest` pins. +The command refuses to write into a directory that already exists, and a failed write +removes the directory it created. -At startup, Relay re-derives that digest from the files on disk and compares it byte for +At startup, Relay checks every file against `SHA256SUMS`, refuses a changed, missing, or +extra file by name, and recompiles the contract and regenerates every artifact byte for byte before it serves anything. This is an integrity check, not an authenticity check. -`packageRevision` proves that the package you are serving is the package that was built. +The package digest proves that the package you are serving is the package that was built. It does not prove who built it. Relay packages are not signed, and Relay signs no response. If you need an authenticity proof, it has to come from the channel you distributed the package over, or from your own signing step around it. -{/* Evidence: crates/registry-relay-v2/src/package.rs:25-26 declares - PACKAGE_VERSION relay.registrystack.org/package/v1alpha3 and - COMPILED_REGISTRY_PATH compiled/registry.json; :224-236 writes - registry.yaml, governed/, compiled/registry.json, generated/, and - relay-package.json, and fs::create_dir refuses an existing output - directory; the comment at :348-350 records that packageRevision is an - integrity digest, not an authenticity proof. */} +{/* Evidence: crates/registry-relay-v2/src/package.rs declares + COMPILED_REGISTRY_PATH compiled/registry.json; build_package() writes + registry.yaml, governed/, compiled/registry.json, and generated/ through + registry_platform_config::package, which writes SHA256SUMS and refuses an + existing output directory; verify_artifact_derivation() records that the + package digest is an integrity digest, not an authenticity proof. */} The deployment-local half stays outside the package entirely. Listener address, source file paths, issuer, audit sink, secrets, limits, and quotas live diff --git a/docs/site/src/content/docs/explanation/records-stay-home.mdx b/docs/site/src/content/docs/explanation/records-stay-home.mdx index 59492aab44..9f12d9a336 100644 --- a/docs/site/src/content/docs/explanation/records-stay-home.mdx +++ b/docs/site/src/content/docs/explanation/records-stay-home.mdx @@ -248,13 +248,13 @@ Security material. Registry Relay enforces these: - The package is verified before anything else opens: at startup Relay re-derives the compiled registry and every generated artifact from the sealed authored inputs and requires byte-for-byte equality, before it opens a source, an audit sink, an issuer, or a listener. - Read `packageRevision` as an integrity digest, not an authenticity proof: the package is not + Read the package digest as an integrity digest, not an authenticity proof: the package is not signed, and a caller who can rewrite the package can recompute it. {/* Evidence: prepare() loads and verifies the package first, crates/registry-relay-v2/src/startup.rs:96-104; verify_compiled_derivation and verify_artifact_derivation re-run the compiler and artifact generator and compare bytes, crates/registry-relay-v2/src/package.rs; the crate's own comment states that - "packageRevision is an integrity digest, not an authenticity proof". */} + "The package digest is an integrity digest, not an authenticity proof". */} - Audit fails closed, with no switch: if the audit sink cannot durably accept the attempt or terminal record, the request is refused with a 503 `audit.unavailable` problem rather than answered. This is not configurable, and readiness re-checks the sink, so replacing the audit diff --git a/docs/site/src/content/docs/explanation/render-determinism.mdx b/docs/site/src/content/docs/explanation/render-determinism.mdx index 943b4a6a3b..cace73644e 100644 --- a/docs/site/src/content/docs/explanation/render-determinism.mdx +++ b/docs/site/src/content/docs/explanation/render-determinism.mdx @@ -18,7 +18,7 @@ nothing to run here; read it to understand what you are trusting when you store ## What fixes a document -A rendered document is a function of five inputs: the sealed bundle, the document type, the +A rendered document is a function of five inputs: the verified package, the document type, the locale, the request, and the issuance time. The request is two parts, `data` (validated against the document's JSON Schema) and `assets` (base64 images decoded and served to the template as virtual files). Nothing else enters: no host fonts, no environment, no machine clock. @@ -82,13 +82,17 @@ the deflate stack is a reviewed golden-hash change, never a silent pass. {/* Evidence: products/render/EVIDENCE.md; .github/workflows/render-golden.yml. */} -The bundle is sealed before serving: the manifest lists every governed file with its sha256, and -a served render re-verifies the seal per request, in the worker, not only at startup. A bundle -that drifted, or whose seal was stripped while the service was running, is refused with a named -problem and an audit event. - -{/* Evidence: crates/registry-render/src/worker.rs, render_in_worker; - crates/registry-render/src/manifest.rs, compute_hashes; +The template source is built with `registry-render package` before serving. The shared +`SHA256SUMS` envelope covers every product file and optional `REVISION`; the package digest is +the sha256 of those exact sum-file bytes. Startup verifies the envelope and optional +`package.expectedDigest`, then binds every captured file back to the recorded digest before +validation or rendering. Each worker repeats that process per request, and the parent requires +the worker's package digest to equal startup. Changed, missing, and extra files are refused by +name, including drift introduced after the service began. + +{/* Evidence: crates/registry-render/src/runtime.rs, load_package; + crates/registry-render/src/bundle.rs, load_package and bind_verified_snapshot; + crates/registry-render/src/worker.rs, render_in_worker; crates/registry-render/tests/serve.rs, bundle_drift_after_serve_starts_is_refused_per_render. */} ## Where the proof runs @@ -109,7 +113,7 @@ Store both hashes with the record the document was made from: `pdfSha256` for th `dataSha256` for the exact request the artifact renders. A later re-render from the same stored inputs reproduces the first byte for byte, which is what makes a dead-letter replay safe and what lets an auditor compare the paper against the audit log. If a hash changes, one of the five inputs -changed, the bundle's seal changed, or a dependency upgrade changed output, and the last case +changed, the package digest changed, or a dependency upgrade changed output, and the last case arrives as a golden-hash review, not silently. {/* Evidence: products/render/integrations/openfn/JOURNEY.md; diff --git a/docs/site/src/content/docs/explanation/threat-model.mdx b/docs/site/src/content/docs/explanation/threat-model.mdx index 02ba69d4e1..7d19434cec 100644 --- a/docs/site/src/content/docs/explanation/threat-model.mdx +++ b/docs/site/src/content/docs/explanation/threat-model.mdx @@ -271,7 +271,9 @@ through a governed read, and privacy regressions that expose raw subject identif one HTTP source that resolves none is the `none` authentication kind, and it is credential-free by allowance rather than by construction: only a `local` assurance bundle may declare it, only at a canonical numeric-loopback origin, and production and evidence-grade bundles reject that kind. - {/* Evidence: valid_secret_reference(), crates/registry-relay-v2/src/contract.rs; + {/* Evidence: RelayRuntime::check() refuses a reference whose provider is not declared, + crates/registry-relay-v2/src/contract.rs, through check_reference() in + crates/registry-platform-config/src/blocks.rs; SecretReference, validate_file_metadata(), and the tests references_use_only_the_two_exact_contract_grammars, file_secret_accepts_only_owner_read_and_optional_owner_write_modes, and @@ -362,7 +364,7 @@ through a governed read, and privacy regressions that expose raw subject identif no origin, no scheme, no host, no port, and no network path into a registry data tier, so there is nothing for a URL security rule or a redirect denial to govern and no destination an attacker could steer. - {/* Evidence: IssuerRuntime::profile() and canonical_issuer_transport_url() discovery-URL + {/* Evidence: OidcRuntime::checked_profile() and canonical_issuer_transport_url() key-URL constraints, crates/registry-relay-v2/src/contract.rs; healthcheck() rejects userinfo, password, query, and fragment and uses .no_proxy() with redirect::Policy::none(), crates/registry-relay-v2/src/startup.rs. The statement transport's absences, contract @@ -526,7 +528,7 @@ These are the risks the design does *not* close: bucket. Treat them as an availability control, not an enumeration defense. {/* Evidence: QuotaLimiter is an in-memory per-operation token bucket, crates/registry-relay-v2/src/server.rs. */} -- A Relay package is not authenticated: `packageRevision` is a SHA-256 integrity digest that any +- A Relay package is not authenticated: its package digest is a SHA-256 integrity digest that any holder of the package can recompute. Startup detects drift and tampering by re-deriving every artifact, but it cannot tell a legitimate package from a well-formed forgery. Package provenance is an operator duty: the delivery path, the filesystem ownership checks, and the @@ -603,8 +605,8 @@ conformance claim: and open-file identity checks. Relay refuses to start on a non-Unix target outright, because its package and runtime trust checks have no equivalent there, and its published binary is built for Linux amd64 only. - {/* Evidence: #[cfg(not(unix))] validate_runtime_path returns Err(StartupError::RuntimeInvalid) - and safe_runtime_permissions returns false, crates/registry-relay-v2/src/startup.rs; the + {/* Evidence: #[cfg(not(unix))] require_trusted_ownership returns an error, so the loader + refuses every runtime configuration, crates/registry-platform-config/src/loader.rs; the non-Unix safe_permissions in crates/registry-relay-v2/src/package.rs returns false unconditionally; the installer refuses any platform other than Linux amd64, proven by unsupported_platform_fails_before_download_or_install, diff --git a/docs/site/src/content/docs/generated-artifacts/index.mdx b/docs/site/src/content/docs/generated-artifacts/index.mdx index 6411c5b305..d69b2e4e48 100644 --- a/docs/site/src/content/docs/generated-artifacts/index.mdx +++ b/docs/site/src/content/docs/generated-artifacts/index.mdx @@ -37,33 +37,12 @@ always the product of exactly one run. | Path | What it carries | | --- | --- | -| `openapi.full.yaml` | Every compiled operation, including operator-only ones | -| `openapi.public.json` | The description Relay serves to unauthenticated callers | -| `artifacts/` | Capability inventories, JSON Schema, SHACL shapes, JSON-LD vocabularies and contexts, classification and processing descriptions, the audit event schema, and SDMX structures | -| `reports/identification-report.json` | Which source columns the identification pack flagged, and why | -| `reports/classification-inventory.json` | The effective classification of every published property | -| `reports/access-profile-report.json` | What each access profile lets a caller reach | -| `reports/contextual-review-findings.json` | Combinations a reviewer should look at before approving | -| `governance/classification-review-starter.yaml` | A starting point for the review record the production profile requires | - -Every one of these is derived from the contract and the observed source -structure. -None of them carries a source row value. - -## The sealed package - -`relayctl package` writes one new directory and refuses a destination that -already exists. -It recompiles the project under the production profile first, so a revision that -would fail `relayctl check --production` cannot be sealed. - -| Path | What it carries | -| --- | --- | -| `relay-package.json` | The manifest: package version, `packageRevision`, contract revision, source schema fingerprints and observed schemas, the artifact list with its operation bindings, and one entry per file | +| `SHA256SUMS` | The digest of every other file, one line per file, sorted by path | +| `REVISION` | The optional operator label given with `--revision`; absent without it | | `registry.yaml` | The contract exactly as authored | | `governed/` | Every governance file the contract references, plus the classification review rationale and its accepted identification report | | `compiled/registry.json` | The canonical compiled Registry | -| `generated/` | The artifact set, keyed to the manifest entries | +| `generated/` | The artifact set | The package's `generated/` directory holds the artifact set only. The review reports and the review starter stay in the authoring project: they @@ -72,32 +51,38 @@ are inputs to approval, not deployment material. A package carries no database, no `runtime.yaml`, no secret, and no fixture. The operator supplies those at the deployment. Packaging refuses oversized input instead of truncating it: at most 256 -referenced governance files totalling 16 MiB, 1024 package files totalling -64 MiB, and a 4 MiB manifest. +referenced governance files totalling 16 MiB, and 1024 package files totalling +64 MiB. -## What packageRevision proves +## What the package digest proves -`packageRevision` is a SHA-256 digest over the canonicalized manifest. +The package digest is the SHA-256 digest of `SHA256SUMS`, so it covers every +file's bytes and the exact file set. +`relayctl package` reports it, `relayctl package --dry-run` reports it without +writing, and `package.expectedDigest` in `runtime.yaml` pins it. It detects a modified, truncated, or reassembled package. It is not a signature, and anyone who alters a package can recompute it, so authenticity stays a property of how the institution transfers, stores, and restricts the directory. Acceptance does not stop at the digest. -Loading a package re-reads every listed file, compares each size and SHA-256, -recompiles the Registry from the governed files under the production profile, -regenerates the whole artifact set, and requires byte-for-byte equality before -the service opens a listener. - -{/* Evidence: crates/registry-relay-v2/src/package.rs verify_artifact_derivation(): - "`packageRevision` is an integrity digest, not an authenticity proof." - load_package() rechecks every file entry, then reproduces the compiled +Loading a package refuses a changed, missing, or extra file by name, re-reads +every file against its listed digest, recompiles the Registry from the governed +files under the production profile, regenerates the whole artifact set, and +requires byte-for-byte equality before the service opens a listener. + +{/* Evidence: crates/registry-relay-v2/src/package.rs load_package(): + verifies SHA256SUMS through registry_platform_config::package, re-reads + every file against its verified digest, then reproduces the compiled Registry and every artifact byte. */} -## Visibility travels with each file +## Visibility is derived, not stored -Every manifest entry records whether the file is `public`, `operation-bound`, or -`operator-only`, and whether it was generated or authored. +Relay derives whether each artifact is `public`, `operation-bound`, or +`operator-only` from the compiled Registry at startup; the package stores no +visibility of its own. +`relayctl package` reports the derived exposure of every artifact so a reviewer +can compare it across revisions. The split is plain in the two OpenAPI documents: `openapi.public.json` is what Relay returns from `/openapi.json`, while `openapi.full.yaml` describes every compiled operation, including the ones no anonymous caller can reach. diff --git a/docs/site/src/content/docs/operate/advanced/inspect-and-diagnose.mdx b/docs/site/src/content/docs/operate/advanced/inspect-and-diagnose.mdx index 4edc0036a2..e200153c31 100644 --- a/docs/site/src/content/docs/operate/advanced/inspect-and-diagnose.mdx +++ b/docs/site/src/content/docs/operate/advanced/inspect-and-diagnose.mdx @@ -22,8 +22,7 @@ ambiguity, or audit-write failure without collecting source rows or secret value - Use the protected operator network and a least-privilege posture or API credential. - Know the active product and instance for Relay: its runtime file path, bound source path, and - package `packageRevision` digest. Know the deployment revision for Evidence Gateway: its runtime - file path and bundle revision. + package digest. Know the Evidence Gateway runtime file path and package digest. - Keep public problem responses, operator diagnostics, and protected audit records in separate access classes. - Reproduce with synthetic identifiers unless separate authority permits another probe. @@ -110,11 +109,12 @@ needs no separate CLI catalog because the codes are fixed by the frozen public c ### Source access and startup refusal Relay V2 verifies its package, source, and runtime bindings before it starts listening. `relay -serve --runtime ` refuses an unsafe path (a symbolic link, or a component with the wrong -owner or mode), a package whose re-derived bytes do not match what is installed at `packagePath`, +serve --runtime-config ` refuses an unsafe path (a symbolic link, or a component with the wrong +owner or mode), a package whose re-derived bytes do not match what is installed at `package.root`, a source whose SQLite schema fingerprint has drifted from what the package pinned, a missing mandatory audit input, or a runtime binding incompatible with the package, such as a protected -access profile paired with `authentication.issuer: null`. Correct the mismatched package, source, +access profile with no `authentication.oidc` block. A removed or unknown runtime key is refused +with the field name and, for a removed key, its replacement. Correct the mismatched package, source, or runtime file and restart. {/* TODO[evidence]: confirm whether relay serve reports a stable, documented code per @@ -192,8 +192,9 @@ A reproduction that fails names the contract that broke, which says a case faile stage it stopped at. Ask the Evidence Gateway fixture run to explain itself: ```sh -evidence --runtime runtime.yaml evaluate --fixture "bundle/fixtures/.yaml" --explain -evidencectl test "" --explain +evidence evaluate --runtime-config "/runtime.yaml" \ + --fixture "fixtures/.yaml" --explain +evidencectl test "" --target "" --explain ``` The trace records, for every case, each stage it reached (`prepare`, `acquire`, `extract`, @@ -235,7 +236,7 @@ Relay V2 deployment: - **Signed-bundle rejection codes.** `relay.startup.bundle_signature_rejected`, `_binding_rejected`, `_validation_rejected`, and `_rollback_rejected` assumed a signed product bundle and an anti-rollback ratchet. Relay V2 packages are integrity-digested through - `packageRevision`, not signed, and carry no anti-rollback state, so there is no signature, + their package digest, not signed, and carry no anti-rollback state, so there is no signature, binding, or rollback check to reject. ## Expected evidence @@ -244,7 +245,7 @@ Retain: - Timestamped health and readiness status without response secrets. - Stable public diagnostic codes and status classes. -- Relay's package `packageRevision` digest and runtime file path identifying the deployed +- Relay's package digest and runtime file path identifying the deployed instance, rather than a signed bundle sequence. - Evidence Gateway's `traceId` from a problem response or the W3C `traceparent` response header, and the shipped audit entries for the operation when a signing or diff --git a/docs/site/src/content/docs/operate/advanced/rotate-credentials-and-trust.mdx b/docs/site/src/content/docs/operate/advanced/rotate-credentials-and-trust.mdx index dd3df9d7fa..ec35e41d52 100644 --- a/docs/site/src/content/docs/operate/advanced/rotate-credentials-and-trust.mdx +++ b/docs/site/src/content/docs/operate/advanced/rotate-credentials-and-trust.mdx @@ -24,7 +24,7 @@ where that authority actually lives. - Identify the material, every consumer, its current secret or trust reference, and its expiry. - Preserve a verified recovery set for the current Relay product configuration, and keep the - prior Evidence Gateway bundle revision each rotation replaces. + prior Evidence Gateway package digest each rotation replaces. - Keep caller traffic outside the staged Relay instance until its checks pass; restart Evidence Gateway only after its offline checks pass first. - Use synthetic or institution-approved canaries. Do not use personal data for a rotation probe. @@ -45,7 +45,7 @@ The deployment operator owns secret storage, certificates, the OIDC issuer Relay admission, and revocation. Relay V2 packages carry no signature and no trust anchor. `relayctl package` produces a package -identified by an integrity digest, `packageRevision`; `relay serve` re-derives that package from +identified by an integrity digest, its package digest; `relay serve` re-derives that package from its inputs and refuses to start on a mismatch. There is no signing key, no anchor, no lane, and no anti-rollback ratchet to rotate for Relay V2, unlike Relay V1's `registryctl`-built and `registryctl`-verified `relay-public` and `relay-consultation` lanes. @@ -111,16 +111,16 @@ effect where you expect. ## Rotate caller keys Relay V2 has no caller-key management of its own: it has no `generate-api-key` command, no -API-key store, and no per-key revocation list. `authentication.issuer` in `runtime.yaml` names an -OIDC issuer, its discovery URL, audience, accepted token types, and accepted algorithms; every +API-key store, and no per-key revocation list. `authentication.oidc` in `runtime.yaml` names an +OIDC issuer, its key source (`jwksSource`), audience, accepted token types, and accepted algorithms; every caller authenticates with a bearer token from that issuer, verified against the issuer's own published keys. Caller-key rotation for Relay V2 happens entirely at the OIDC issuer: rotate the issuer's signing keys through the issuer's own key-rollover procedure, and rotate an individual caller's credential (client secret, certificate, or key) through whatever mechanism that issuer or its client registry uses. Relay V2 has no compromised-key denylist of its own, unlike Evidence Gateway's `authentication.revokedKeyIds`; a compromised caller credential must be revoked at the -issuer, and Relay only needs a new deployment when the issuer binding itself changes (`id`, -`discoveryUrl`, `audience`, `tokenTypes`, or `algorithms`). +issuer, and Relay only needs a new deployment when the issuer binding itself changes (`issuer`, +`jwksSource`, `audience`, `tokenTypes`, or `algorithms`). ## Configuration signing and trust anchors do not apply to Relay V2 @@ -128,7 +128,7 @@ Relay V1 built and verified two independently signed lanes, `relay-public` and `relay-consultation`, through `registryctl trust anchor rotate`, `registryctl trust bundle sign`, `registryctl trust bundle verify`, and `registryctl trust approved-set assemble`. Relay V2 has none of this. A package built by `relayctl package` is identified by an integrity digest -(`packageRevision`) that `relay serve` re-derives and compares byte-for-byte at startup; it is not +(its package digest) that `relay serve` re-derives and compares byte-for-byte at startup; it is not signed, has no trust anchor, no lane, no approved set, and no anti-rollback sequence. There is no anchor-rotation procedure to run, no signer key to rotate, and no bundle-verification command to invoke for Relay V2: replacing a package is the same @@ -162,7 +162,7 @@ long as pseudonyms from its version must be recomputable. Retain: - The `relayctl check` (and `relayctl check --production`) report for a Relay V2 project change, - and the package's `packageRevision` digest for the staged candidate. + and the package digest for the staged candidate. - The `evidence check` output and the readiness result for a rotated Evidence Gateway signing key. - Health and readiness results from the staged Relay instance; Relay V2 has no redacted posture report to retain alongside them. diff --git a/docs/site/src/content/docs/operate/breg-changes.mdx b/docs/site/src/content/docs/operate/breg-changes.mdx index 941fc34dcc..3cfeff61f4 100644 --- a/docs/site/src/content/docs/operate/breg-changes.mdx +++ b/docs/site/src/content/docs/operate/breg-changes.mdx @@ -1,11 +1,11 @@ --- title: Change an active registry -description: Compare an edited project with the active package, test and sign the successor against the active baseline, activate it with reviewed migration evidence, recover a registry that a failed activation pinned, and undo a change by rolling forward or restoring a backup. +description: Compare an edited project with the active package, test and sign the successor against the active baseline, activate it with reviewed migration evidence, recover a registry that a failed activation pinned, undo a change by rolling forward or restoring a backup, and adopt a restored database before it serves. status: current owner: registry-docs source_repos: - registry-stack -last_reviewed: "2026-09-24" +last_reviewed: "2026-09-25" doc_type: how-to persona: - operator @@ -45,11 +45,24 @@ as widening or narrowing, and mixed changes are marked for review. A change to c recipients, or a consent-issuing action carries a `reason` when it alters who holds an existing consent, such as a client added to a recipient organization or a gated profile widened in place. The `reason` is a review aid in the diff output, not a runtime guarantee: the successor enforces -exactly what its package declares. Set `package.sequence` in +exactly what its package declares. Replacing a `batch` grant with `import`, which a successor that +puts an entity under change control needs, removes the entity's `:batch` route, and the diff +reports it as a route removal; an entity whose loader already held `import` keeps the same route +when change control is added. + +Removing a field or an entity is not erasure. The successor drops the live column or table, but +every revision snapshot recorded before it still holds the removed values in +`registry_internal.registry_revisions`, and so does every database backup. The diff reports each +removal as `diff.history.removed_values_retained` so the reviewer sees this before signing. When +the removal is for data minimization or a legal reason, erase the retained revisions of each +affected record with [`history erase`](../breg-retention/#erase-retained-history) as well; there is +no command that removes one field from history and keeps the rest of each snapshot. + +Set `package.sequence` in `registry.yaml` to the next sequence. `registry.version` is bound to the database for its lifetime: a package that changes it can only initialize a new database. -{/* Evidence: crates/registry-bregctl/src/lib.rs, DiffArgs; +{/* Evidence: crates/registry-bregctl/src/lib.rs, DiffArgs and removed_value_findings(); crates/registry-breg/src/tooling.rs, AccessChangeDirection and GATED_SCOPE_WIDENED; crates/registry-breg/src/migration.rs, apply_verified_package(). */} @@ -58,9 +71,15 @@ lifetime: a package that changes it can only initialize a new database. Pass the active runtime file as `--baseline-runtime-config` to both `test` and `package`, so the receipt and the package bind to the baseline they will succeed. A change the migration planner classifies as compatible additive, such as a new optional field or a code added to a vocabulary a -field uses (`field_vocabulary_codes_added`), needs no further evidence. The diff classifies a field -code addition as `lock_or_rewrite_risk`, because the migration replaces the column check under an -exclusive table lock and validates every stored row. A widened grant is an access change: the +field uses (`field_vocabulary_codes_added`), a higher `maxLength` on a `text` field, or a lower +`minLength` on a `string` field under the same `maxLength` (both `field_length_widened`), needs no +further evidence. The diff classifies these as `lock_or_rewrite_risk`, because the migration +replaces the column check under an exclusive table lock and validates every stored row; a +`minLength` lowered to 0 drops the check instead. A `string` field's `maxLength` is its column type, +so raising it is `field_type_changed` and needs a reviewed migration, as does lowering any +`maxLength`, raising a `minLength`, or widening a `decimal`. An action that sets or requires the +entity is reported as `action_target_fields_widened` when a relaxed length bound is its only +change, and stays additive. A widened grant is an access change: the planner accepts it as metadata-only, but it still needs a reviewed migration descriptor and rehearsal evidence, as the metadata-only case below describes. @@ -109,6 +128,74 @@ bregctl --format json test ./my-registry \ --output /srv/registry/schema-test-receipt-2.json ``` +### Reviewed migration files + +Every file is canonical JSON or SQL, and every path inside a descriptor is relative to the +reviewed directory root, starting with `modules//migrations//`. + +- `descriptor.json` holds `id`, `changeClass`, `covers` (each change `code` and `target` + from `diff --format json` that is not compatible additive), `recovery` (`exact_target_resume`), + `lockTimeoutMs`, `statementTimeoutMs`, `steps`, `preAssertions`, `postAssertions`, + `rehearsalReceiptPath`, and, for a destructive change, `backupBindingPath`. +- A step is one of three kinds. `transactional_sql` runs one statement from `sqlPath` in its own + transaction, which also records the step's ledger progress, so a later step or post-assertion + that fails does not roll it back and `apply` resumes after it; DML must declare `affectedRows` + bounds, and every row it changes gets a history revision. `chunked_backfill` runs one `UPDATE` of its declared entity once per chunk of + record identifiers, bound as a UUID array, within `chunkSize` (at most 1,000) and + `maxTotalRows`; each chunk commits on its own and appends one history revision for every row it + changed, in one history commit, so snapshot and as-of reads agree with the backfilled rows. + `field_encryption_backfill` carries no SQL; the engine seals the covered fields chunk by chunk + and journals each chunk the same way. A journaled step is one `UPDATE`, after any leading + comments, and may not name a record metadata column (`record_revision`, `record_lifecycle`, + `active_package_revision`, `created_at`, `updated_at`) anywhere in the statement, even to read + it. Outside comments and plain string literals, its text may not contain a second statement or + another statement word such as `drop` or `delete`, and a `transactional_sql` step may not name + `record_id`. A dollar-quoted body or a literal holding a backslash is read as written, so a + statement word inside it refuses the step. A Unicode-escape identifier or string (`U&"..."`, + `U&'...'`, with or without `UESCAPE`) refuses the step wherever it appears. The journal also + refuses a step that changed any record metadata column. + Each step lists the `objects` (schema, table, entity, kind, member, physical name) its SQL may + touch. +- An assertion is one `SELECT` returning exactly one boolean column. Pre-assertions run against + the predecessor tables before any step, post-assertions against the successor tables after + every step, and activation stops on a false result. +- `rehearsal.json` binds the prior revision and schema fingerprint, the descriptor and SQL + digests, the fixture inventory, the PostgreSQL major version, the affected row counts your + rehearsal observed, the final schema fingerprint, and the lock-timeout and resume proofs. +- `backup.json` binds the backup file `apply --backup` will present: `databaseId`, + `priorRevision`, `priorSchemaFingerprint`, `sha256`, `byteLength`, `createdAt`, and + `maxAgeSeconds`. + +`apply` accepts a backup only when all of these hold: the binding names the active database, +revision, and schema fingerprint; `createdAt` is no older than `maxAgeSeconds` and not in the +future; the path given to `--backup` is absolute and is a regular file rather than a symbolic +link, owned by the user running `apply`, with mode `0600` and exactly one hard link; its length +and SHA-256 digest equal the binding. Take the backup with your PostgreSQL tooling immediately +before `apply`, `chmod 600` it, and record its digest and length. A backup that fails any check +refuses the apply before maintenance begins. + +### What `test` rehearses + +With `--baseline-runtime-config`, `test` rebuilds the predecessor schema from the verified active +package's signed sources on the disposable database, checks that it reproduces the predecessor's +schema fingerprint, then runs the successor migration over it in the order `apply` would: +pre-assertions, the compiler's statements, your reviewed steps, the deferred constraints and +views, and post-assertions. It then requires the result to match the candidate's schema +fingerprint, and rolls everything back before the ordinary journey test runs. A statement +PostgreSQL refuses, an assertion that is not a single boolean column, or a plan that does not reach +the candidate schema fails `test`, and so does a journaled step whose SQL the history journal +refuses. `package`, which requires that receipt, therefore cannot sign a plan `apply` would +reject. The report names the migration, step or assertion, the SQLSTATE and its +class, and the table, column, or constraint PostgreSQL named. It never includes a PostgreSQL +message, because those can quote row values. + +The rehearsal runs over empty tables, so it proves that the SQL is valid, ordered, and reaches +the target schema. It does not prove that your data satisfies a step or an assertion: a duplicate +key, a row count outside `affectedRows`, or an assertion that is false over real rows is found +only by your own rehearsal on a restored copy of production and by `apply` itself. Field-encryption +backfill steps are skipped, because they need key material and rows. Reviewed fixture files are +bound by digest only; the rehearsal does not load them. + Package and sign as [deploy a registry](../breg/) describes, with the same `--baseline-runtime-config` and `--reviewed-migrations` arguments. If any source, reviewed artifact, baseline, or signature-policy input changes afterwards, repeat the test and the signing. @@ -121,10 +208,18 @@ Reviewed artifacts cannot authorize a change the planner classifies as unsupport {/* Evidence: crates/registry-bregctl/src/lib.rs, PackageCandidateArgs; crates/registry-bregctl/src/reviewed_migrations.rs; crates/registry-breg/src/migration_plan.rs; - crates/registry-breg/src/immediate_actions.rs, contract_only_adds_vocabulary_codes(); + crates/registry-breg/src/immediate_actions.rs, contract_only_widens(); + crates/registry-breg/src/contract.rs, FieldTypeSource::widens_text_length_of(); + crates/registry-breg/tests/postgres_compiled_schema.rs, text_length_widening_replaces_the_length_check_and_keeps_existing_rows; crates/registry-breg/src/tooling.rs, classify_change(); crates/registry-breg/tests/action_vocabulary_codes.rs; crates/registry-bregctl/tests/cli/reviewed_migrations.rs; + crates/registry-breg/src/postgres/rehearsal.rs; + crates/registry-breg/src/history_migration.rs, check_reviewed_history_step(); + crates/registry-breg/src/postgres/interlock.rs, execute_reviewed_chunk(); + crates/registry-breg/tests/postgres_migration.rs, reviewed_migration_history(); + crates/registry-breg/src/migration.rs, open_bound_backup(); + crates/registry-breg/tests/postgres_migration.rs, real_postgres_rehearsal_refuses_a_reviewed_plan_activation_would_refuse; products/breg/scripts/test-adopter-workflow.sh. */} ## Activate the successor @@ -171,6 +266,13 @@ package, by revision digest, schema fingerprint, sequence, and deployment identi no maintenance pending. It exits 0 with `activation: already_active` and changes nothing. Another package that only shares the active sequence still refuses with `apply.package.binding_mismatch`. +An activation supersedes every open import authority, because each is bound to the package +revision it opened under: from the moment the successor is active, no chunk or new run is +admitted under it, and an `import` run in progress stops as `blocked`. The first load or +`bregctl import-authority` command that observes it records the supersession. Open a new +authority under the successor to continue a load, as +[move data in bulk](../breg-data/#load-a-governed-entity-through-an-import-window) describes. + {/* Evidence: crates/registry-bregctl/src/lib.rs, ApplyArgs; crates/registry-bregctl/src/apply_lifecycle.rs, confirm_already_active(); crates/registry-breg/src/migration.rs, apply_verified_package(), confirm_active_package(); @@ -246,7 +348,12 @@ reported `unresolvable`, or a reviewed migration changed rows in a way no succes including operator companion files, in your append-only archive. 2. Restore the database from the backup you took before the `apply`, with your PostgreSQL tooling. 3. Point the runtime file at the package that was active when the backup was taken, with its - directory, revision, and sequence, run `bregctl verify`, and start the server. + directory, revision, and sequence, and run `bregctl verify`. +4. A logical restore is a new database, so the server refuses to serve it until you adopt it: + run `bregctl instance-claim adopt` as + [back up and restore the database](#back-up-and-restore-the-database) describes, then start + the server. The adopt supersedes every import authority the backup held open; open again + only the ones you still need. A restore discards every database write and activation committed after the backup, and it brings back history that a later `history erase` removed: repeat those erasures before the @@ -260,6 +367,117 @@ next successor needs a sequence above the restored active package, not above the crates/registry-bregctl/tests/cli/reviewed_migrations.rs, apply_refuses_an_older_package_and_points_at_the_roll_forward_procedure. */} +## Back up and restore the database + +Take a logical backup of the serving database with the PostgreSQL tools, as an administrator, before +every activation: + +```sh +pg_dump --format=custom --file=/srv/backup/registry-before-build-2.dump registry +``` + +To restore it, stop every `breg` process that serves the registry, then recreate the database with +the database-level statements from [provision PostgreSQL](../breg/#provision-postgresql), which a +dump does not carry, and restore the dump into it as an administrator: + +```sql +CREATE DATABASE registry_restored; +REVOKE ALL ON DATABASE registry_restored FROM PUBLIC; +GRANT CONNECT ON DATABASE registry_restored TO registry_migration, registry_runtime; +``` + +```sh +pg_restore --exit-on-error --dbname=registry_restored /srv/backup/registry-before-build-2.dump +``` + +For a spatial project, also repeat the `REVOKE CREATE ON DATABASE` statement. Point the runtime +file's database references at the restored database, then check it before anything serves from +it: + +```sh +bregctl verify --runtime-config /etc/breg/runtime.yaml +bregctl --format json instance-claim status --runtime-config /etc/breg/runtime.yaml +``` + +If the restored database is pinned in maintenance, run +[`migration reconcile`](#recover-a-failed-activation) before going further. + +Every registry database holds an instance claim: the PostgreSQL system identifier and database +object identifier of the database the registry serves from, recorded at the first apply. A restored +copy carries the claim of the database it was dumped from, so `instance-claim status` reports +`"matches": false`, `breg` refuses to start with `startup.instance_claim.mismatch`, and `GET /ready` +answers `503`. The refusal exists because two databases serving one registry become divergent writers: each +accepts writes, admits imports under the authorities the backup held open, and delivers the same +outbox work from the backup onward, and nothing can merge the two histories afterwards. + +The first apply records the claim only into a fresh database, one with no committed revision and +no commit head. A registry that already holds committed history when the claim table is first +installed, which is every registry +upgraded from a release without the claim and every copy restored from a backup taken before it, +records none: `instance-claim status` reports that no claim is recorded, and `breg` refuses in the +same way until you adopt it once. After upgrading, run the adopt below after the first `apply` +and before starting the server. + +The claim is checked when `breg` starts and on every `GET /ready`, not on each request. A process +that keeps running while its database host name is repointed at a restored copy keeps serving +requests on the connections it opens until something acts on its readiness `503`. Gate traffic +on `GET /ready`, or restart every `breg` process, whenever the database behind the registry +changes. + +The system identifier comes from `pg_control_system()`. PostgreSQL grants it to every role, but a +managed service may withhold it. When the migration role cannot call it, the claim is recorded +without a system identifier; when either side lacks one, the claim compares the database object +identifier alone, and `instance-claim status` shows the system identifier as not readable. The +object identifier alone still tells another database in the same cluster apart, but a copy +restored into a new cluster can receive the same object identifier and is not refused. On such a +service the rule that the original is stopped before a copy serves is the only control, so keep +it, and grant `EXECUTE` on `pg_control_system()` to both roles where the service allows it. + +Once the database the claim names is stopped for good, adopt the copy: + +```sh +bregctl --format json instance-claim adopt \ + --runtime-config /etc/breg/runtime.yaml \ + --acknowledge-original-retired +``` + +`adopt` runs under the migration credential. Under the same lock an activation takes, it moves the +claim to this database with its epoch raised by one, supersedes every open +[import authority](../breg-data/#load-a-governed-entity-through-an-import-window), all in one +transaction. Once that commits, it appends each superseded authority's transition record and an +audit entry naming the previous and the adopted claim and the authorities it superseded to the +`bregctl` companion of the configured audit destination. A backup holds the authorities that were open +when it was taken, including any you closed afterwards, so no authority survives an adopt: open +again the ones you still need. It refuses with +`instance_claim.already_current` when the claim already names this database, and without +`--acknowledge-original-retired` it refuses before it opens a connection. Then run `bregctl doctor`, +start the server, and check `GET /ready`. + +:::caution[Adopting does not stop the original] +The claim tells a logical restore from its original, not a physical one. A base backup, a +point-in-time recovery, a volume snapshot, or a promoted replica keeps the original's system +identifier and database object identifier, so it serves without an adopt: keeping the original +stopped, or fenced from clients, while such a copy serves is your job. An adopt changes only the +copy's claim, so an original that is still running keeps serving. Pass +`--acknowledge-original-retired` only once no client can reach it. +::: + +An upgrade with `pg_upgrade` moves the registry into a new cluster with a new system identifier, +so it needs the same adopt once the old cluster is stopped. A failover to a streaming replica +keeps the system identifier and needs none. + +{/* Evidence: crates/registry-breg/src/instance_claim.rs, install(), check(), live_identity(), + adopt_in(); + crates/registry-breg/src/import_authority.rs, supersede_every_open(); + crates/registry-breg/src/instance_claim.rs, InstanceClaimService::adopt(); + crates/registry-breg/src/startup.rs, verify_instance_claim(); + crates/registry-bregctl/src/lib.rs, InstanceClaimAdoptArgs and instance_claim_failure(); + crates/registry-breg/tests/postgres_startup.rs, a_restored_copy_refuses_to_serve_until_adopted + and a_database_that_withholds_its_system_identifier_still_serves_and_refuses_a_copy; + crates/registry-breg/tests/postgres_import_authority.rs, + installing_the_claim_beside_committed_history_leaves_the_database_to_adopt and + adopting_a_restored_copy_supersedes_every_open_authority. */} + ## Inspect the migration plan `migration explain --runtime-config ` describes the configured package's migration plan @@ -285,6 +503,11 @@ server binary, and earlier packages keep verifying under their recorded rules. | `test` or `package` reports `migration.review.closure_refused` | The set of files does not close: a descriptor the directory names is missing, a path appears twice, or there are too many artifacts. | | `test` or `package` reports `migration.review.refused` | A precondition failed before the reviewed directory was read, such as the prior revision or the baseline binding. Check the baseline arguments and the fingerprints in the rehearsal receipt. | | `test` or `package` reports `migration.review.fingerprint_mismatch` | The rehearsal receipt was taken for a different candidate or baseline. Rehearse the exact candidate again. | +| `test` reports `migration.rehearsal.step_failed`, `migration.rehearsal.assertion_failed`, or `migration.rehearsal.compiler_statement_failed` | PostgreSQL refused the named step, assertion, or generated statement when `test` rehearsed the successor over the predecessor schema, so `apply` would refuse it too. The message names the SQLSTATE, its class, and the object. Correct the reviewed SQL or the project, then repeat test and package. | +| `test` reports `migration.rehearsal.history_step_refused` | The named step changes rows, and `apply` journals every row it changes, but the step's SQL is not an update the journal can record: it does not start with `UPDATE` once leading comments are set aside, it holds a second statement or a semicolon before its end, it names a record metadata column (`record_revision`, `record_lifecycle`, `active_package_revision`, `created_at`, `updated_at`) anywhere, even in a read, it uses a Unicode-escape identifier or string (`U&"..."` or `U&'...'`), a transactional step names `record_id`, it uses a refused statement word (`insert`, `delete`, `truncate`, `alter`, `drop`, `create`, `merge`) outside a comment or plain string literal, or inside a dollar-quoted body or a literal holding a backslash, or its objects span more than the step's entity. Correct the reviewed SQL, then repeat test and package. | +| `test` reports `migration.rehearsal.schema_mismatch` | The rehearsed migration does not reach the candidate schema, for example a required field the reviewed steps never constrain. Activation would refuse it. | +| `test` reports `migration.rehearsal.baseline_unavailable` or `migration.rehearsal.baseline_not_reproducible` | This `bregctl` cannot rebuild the active package's schema from its signed sources. Run `test` with a `bregctl` release that compiles the predecessor, or report the mismatch. | +| `apply` reports `apply.migration.statement_failed` | PostgreSQL refused a statement after maintenance began. The message names the SQLSTATE, its class, and the table, column, or constraint PostgreSQL reported, for example `23502` for stored rows a new `NOT NULL` refuses or `22P02` for a value a reviewed step cannot convert. Fix that cause and retry the same target, or run `migration reconcile` as for `apply.migration.failed`. | | `apply` reports `apply.migration.failed` | The apply failed after maintenance began. Read the report, fix the cause, and retry the same target. If that is not available, run `migration reconcile` to assess the pinned target, execute the transition it names, and restore the pre-activation backup only when it reports `unresolvable`. | | `apply` reports `apply.database.unavailable` | The migration database could not be reached, or another apply held the migration lock past the lock timeout, before maintenance began. Check that the database is reachable and accepts the migration role, then retry the same apply. Nothing was changed, so there is nothing to reconcile. | | `apply` reports `apply.package.older_than_active` | The target package is older than the active one, and packages apply forward only. Build a successor that reverts the change, or restore the pre-activation backup, as [roll back by rolling forward](#roll-back-by-rolling-forward) describes. Nothing was changed. | @@ -292,6 +515,8 @@ server binary, and earlier packages keep verifying under their recorded rules. | `apply` reports `apply.package.binding_mismatch` | The package, or the active package at `package.root`, is bound to a different deployment than the runtime file. The report path names the runtime key that differs (`identity.environment`, `identity.databaseInitializationEnvironment`, `identity.instanceId`, `identity.databaseId`, `package.compilerSourceRevision`, `package.activeRevision`, or `package.activeSequence`) and never either value. A `package.activeSequence` mismatch means the target is a different package at the active sequence: build it as a successor of the active package. Nothing was changed. | | `apply` reports `apply.package.active_mismatch` | The runtime file names the target as the active package, but the database does not record it as active and ready. Set `package.activeRevision` and `package.activeSequence` to the package the database runs and apply again. If the database has never been activated, apply with `--initial`. If the database is pinned in maintenance, run `migration reconcile`. Nothing was changed. | | `apply` reports `apply.history.coverage_incomplete` | Retained history coverage does not admit a successor: a `field-encryption erase-history` run has not finished, or an erasure reached the coverage baseline or left a gap the dedicated history-erasure coverage table does not record. Finish the erase-history run, or run [`history rebaseline`](../breg-retention/#restore-snapshot-coverage-after-an-erasure), then apply the same package again. Maintenance state was not changed, so there is nothing to reconcile. | +| `breg` refuses to start, or `doctor` reports `startup.instance_claim.mismatch` | The database is not the one the registry's instance claim names, as after a logical restore or a `pg_upgrade`, or it holds no claim, as after the first apply of an upgrade from a release without one. Stop the database the claim names for good, then run `instance-claim adopt` as [back up and restore the database](#back-up-and-restore-the-database) describes. | +| `instance-claim adopt` reports `instance_claim.already_current` | The claim already names this database, so it serves without an adopt. Nothing was changed. | | `migration reconcile` reports `in_progress` | Another session holds the migration lock. Wait for it to finish, then assess again. | | The process still serves the old package after `apply` | Activation changes the database, not the process. Update the runtime file and restart the server. | diff --git a/docs/site/src/content/docs/operate/breg-data.mdx b/docs/site/src/content/docs/operate/breg-data.mdx index 5c4c9af8d6..3cb82cd0c3 100644 --- a/docs/site/src/content/docs/operate/breg-data.mdx +++ b/docs/site/src/content/docs/operate/breg-data.mdx @@ -5,7 +5,7 @@ status: current owner: registry-docs source_repos: - registry-stack -last_reviewed: "2026-09-04" +last_reviewed: "2026-09-25" doc_type: how-to persona: - operator @@ -22,8 +22,9 @@ interrupted and resumed from its checkpoint. `data import` and `data export` drive the ordinary authenticated routes, so every row passes the same grants, validation, and audit as an interactive client: the import commits its chunks through the entity's ingestion-run routes, and the export pages through the list route. Nothing -here bypasses a profile: an import needs a profile that may create or patch the entity, and an -export needs one that is export-enabled. A chunk is one batch request the import commits as a +here bypasses a profile: an import needs a profile that may create or patch the entity, or one +that holds `import` on it inside an open import authority, and an export needs one that is +export-enabled. A chunk is one batch request the import commits as a unit; a page is one list response the export appends as a unit; a checkpoint is the file each command writes after every unit so a rerun continues where the last one stopped. @@ -62,8 +63,8 @@ bregctl data validate \ ``` The report names the package revision and schema fingerprint it validated against, the entity, -profile, and operation, the input length, and the item and chunk counts the import would use. A -line that does not fit the entity's shape or the profile's grants is reported without record +profile, and operation, the input length and its SHA-256 `inputDigest`, and the item and chunk +counts the import would use. A line that does not fit the entity's shape or the profile's grants is reported without record values. {/* Evidence: crates/registry-bregctl/src/data_lifecycle.rs; @@ -107,6 +108,73 @@ run stays inspectable. crates/registry-bregctl/src/lib.rs, DataImportArgs; crates/registry-bregctl/tests/cli.rs. */} +## Load a governed entity through an import window + +Change control refuses `create` and `batch` as direct writes on an entity it governs, so a +governed entity receives its initial or additional records through an `import` grant instead. An +`import` grant is create only and is served only by the ingestion-run routes, so it never changes +an existing record. It loads nothing on its own: an operator first opens an import authority, a +window bounded by entity, profile, item volume, expiry, and optionally the input digests a run may +announce, over the migration connection that no API caller holds. The authority is bound to the package revision +active when it opened. + +Validate the file, then open the window with the digest the report names. `--expires-in` takes a +whole number of minutes, hours, or days (`90m`, `12h`, `7d`), defaults to `7d`, and is at most +`30d`; there is no extension, a longer load opens a second authority. `--input-sha256` may repeat +up to 16 times; without it any input within the volume is admitted. An input digest is a label the +client computes over the file it reads and announces with the run, recorded on the run and in its +audit; the server never receives the file and does not recompute it. A pinned digest therefore +names the file the operator expects to be loaded, not proof of what the chunks write: the item +volume is the bound the server enforces. + +```sh +bregctl import-authority open \ + --runtime-config /etc/registry/runtime.yaml \ + --entity record --profile loader --max-items 10000 --expires-in 2d \ + --input-sha256 \ + --operator-reference change-1482 --reason "District 4 initial load" +``` + +Then run `data import` with the `loader` profile exactly as above. Close the window when the load +is done, and list authorities to see what is open: + +```sh +bregctl import-authority close --runtime-config /etc/registry/runtime.yaml \ + --authority-id --operator-reference change-1482 --reason "District 4 loaded" +bregctl import-authority list --runtime-config /etc/registry/runtime.yaml +``` + +One entity holds at most one open authority. It stops admitting work when the operator closes it, +when its expiry passes, when its committed items reach the volume (`exhausted`), or when another +package revision is activated (`superseded`). Run `bregctl import-authority close-expired` on a +schedule, or after an activation, to record the expiries and supersessions no load has observed +yet. `list` is a read: it takes no Registry lock, so it never holds back a write and still answers +during an interrupted apply, and it records nothing, but it shows an authority whose expiry or +package has passed with the status it has reached. The operator reference and reason are +stored and audited only as keyed hashes, so do not rely on reading them back; every opening, +close, expiry, exhaustion, and supersession appends one record to the audit journal, and every +committed chunk's run record names the authority it consumed. Hooks and events fire for imported +records exactly as for a batch create; pause event destinations for the window if a flood of +events is a concern. + +A run is created only when an open authority for its entity and profile has room for the whole +input and, if digests are pinned, one of them is the digest the run announces. Otherwise `data import` +reports `data.import.ingestion_run.import_authority_required` and no run exists. Every chunk +rechecks the authority inside its own transaction and counts its items against the volume, so a +close or expiry during a load stops the next chunk: the run is `blocked` with reason +`importAuthorityClosed`, `data import` reports +`data.import.ingestion_run.import_authority_closed`, and the chunks already committed stay. A +blocked run is final. To load the rest, open a new authority and import only the uncommitted lines +under a fresh checkpoint path; the run's `committedItems` says how many lines committed. Import is +create only, so re-running an import of lines that already committed, under a fresh checkpoint or +a second authority with room, creates their records again. + +{/* Evidence: crates/registry-breg/src/import_authority.rs, MAX_IMPORT_AUTHORITY_WINDOW, + MAX_PINNED_INPUT_DIGESTS, admit_run(), and admit_chunk(); + crates/registry-bregctl/src/import_authority_lifecycle.rs, parse_expires_in(); + crates/registry-bregctl/src/data_lifecycle.rs, IngestionRunPrecondition; + crates/registry-breg/tests/postgres_import_authority.rs. */} + ## Resume a load through a durable ingestion run The checkpoint of an import is a file beside the client, so the process that resumes must reach @@ -118,16 +186,16 @@ stores no source rows; reading one returns operational metadata and bounded fail classifications only. One run covers one input file. Create it on the entity's `ingestion-runs` route: the create or -patch operation, the profile, the package revision and schema fingerprint, the input length, and +patch operation (create for an `import` grant), the profile, the package revision and schema fingerprint, the input length, and the item and chunk counts are what `data validate` reports for the same input; the run also needs the source digest and the chunking algorithm `greedy-canonical-http-batch-v1`. Submit each chunk in order, naming its index, its digest, and the digest of the source prefix it ends at. -Every submission rechecks the profile against the entity's batch route, so a run id alone grants -nothing. +Every submission rechecks the profile against the entity's batch or import grant, so a run id +alone grants nothing. A run is `open` while chunks are due, `complete` when the last chunk commits, `cancelled` after -an explicit cancel, and `blocked` once the active package no longer matches its binding. Each -failure has one recovery: +an explicit cancel, and `blocked` once the active package no longer matches its binding or, for +an `import` run, once its import authority stops admitting chunks. Each failure has one recovery: | Failure | Next move | | --- | --- | @@ -137,6 +205,7 @@ failure has one recovery: | A chunk holds an invalid item or a business refusal | The checkpoint stays where it is. Start a successor run for the remainder; skipping the refused rows is never a default recovery. | | Authorization is lost | Progress refuses until the selected profile satisfies the batch route again. | | The package or schema changed | The run reports `blocked` with reason `activePackageChanged` and stays inspectable. Start a successor run under the new binding. | +| The import authority closed, expired, or ran out | The run reports `blocked` with reason `importAuthorityClosed` and stays inspectable. Open a new authority and start a successor run for the remainder. | | You stop the load on purpose | Cancel the run. Its counts and audit are preserved. | {/* Evidence: crates/registry-breg/src/ingestion_store.rs, IngestionRunStatus, IngestionBlockedReason, and IngestionAttemptOutcome; @@ -188,7 +257,11 @@ server response is bounded at 2 MiB. | An export refuses to start | Exactly one of the output and checkpoint files exists, or the existing output no longer matches its checkpoint. Keep both files together and unedited, or start both afresh. | | `data export` is refused with `data.export.checkpoint.refused` | The output file and its checkpoint no longer describe one another: the output is shorter than the checkpoint, its checkpointed prefix changed, more than one page follows the checkpoint, or the checkpoint belongs to another export. Keep both files and resume from copies you trust. A rerun discards at most the single page an interrupted run left unrecorded. | | The report says the run is incomplete | `--max-chunks` or `--max-pages` bounded it. Run the same command again to continue. | -| A durable ingestion run reports `blocked` | The active package no longer matches the binding the run was created under. Read the run, then start a successor run under the new package; the blocked run stays inspectable. `data import` surfaces the same state as `data.import.ingestion_run.blocked`. | +| A durable ingestion run reports `blocked` | Read the run's `blockedReason`. `activePackageChanged`: the active package no longer matches the binding the run was created under; start a successor run under the new package. `importAuthorityClosed`: see the next row. The blocked run stays inspectable. `data import` surfaces the first as `data.import.ingestion_run.blocked`. | +| An import reports `data.import.ingestion_run.import_authority_closed` | The run's import authority was closed, expired, reached its volume, or was superseded by an activation. The committed chunks stay. Open a new authority and import only the uncommitted lines under a fresh checkpoint path. | +| An import reports `data.import.ingestion_run.import_authority_required` | No open authority admits the run: none is open for the entity, it names another profile, it expired, its remaining volume is smaller than the input, or it pins other digests. Check `bregctl import-authority list`, then open one that covers this input. | +| `import-authority open` reports `import_authority.already_open` | The entity already has an open authority. Close it, or wait for it to expire, before opening another. | +| `import-authority open` reports `import_authority.grant.not_importable` | The profile holds no `import` grant on the entity in the active package. | ## Next diff --git a/docs/site/src/content/docs/operate/breg-requirements.mdx b/docs/site/src/content/docs/operate/breg-requirements.mdx index 36565446a9..34983f9fdf 100644 --- a/docs/site/src/content/docs/operate/breg-requirements.mdx +++ b/docs/site/src/content/docs/operate/breg-requirements.mdx @@ -34,13 +34,19 @@ on your machine. signer produces a detached Ed25519 signature over it, and the runtime accepts a package only when enough of the keys in its trust anchor have signed. Who holds those keys, and how they sign, is your decision. -- **A backup of the database before every activation.** The product restores nothing itself. +- **A backup of the database before every activation.** The product restores nothing itself, + and it refuses to serve a logically restored copy until an operator adopts it, as + [back up and restore the database](../breg-changes/#back-up-and-restore-the-database) + describes. That check reads the cluster's system identifier through `pg_control_system()`, + which PostgreSQL grants to every role; on a managed service that withholds it, the check falls + back to the database object identifier alone and still starts. {/* Evidence: crates/registry-breg/src/postgres/schema.rs; crates/registry-breg/src/postgres/roles.rs, verify_postgres_16_or_newer and postgis_version_supported; crates/registry-breg/src/runtime_config.rs, SqlRoles and PublicOrigin; crates/registry-breg/src/auth.rs, RegistryAuthenticator; - crates/registry-breg/src/package.rs, validate_signature_policy; products/breg/README.md; + crates/registry-breg/src/package.rs, validate_signature_policy; + crates/registry-breg/src/instance_claim.rs; products/breg/README.md; products/breg/DECISIONS.md. */} ## What it does not need @@ -94,16 +100,15 @@ You ship the streams from every process to append-only storage and verify delive retention expires. The product does not provide tamper evidence. **Retention and erasure.** Erasing a record's history is a bounded operator command with no undo. -It removes at most 10,000 revisions per transaction, and restoring snapshot coverage afterwards is -refused for a registry with more than 1,000 live rows, so a large registry cannot regain snapshot -coverage after an erasure. Retained webhook payloads expire after a configurable number of days, -at most 30. +It removes at most 10,000 revisions per transaction. Restoring snapshot coverage afterwards +verifies every live row in one transaction that holds writes until it commits. Retained webhook +payloads expire after a configurable number of days, at most 30. {/* Evidence: crates/registry-breg/src/runtime_config.rs, AuditConfig, CursorConfig, DatabaseConfig, and MAX_WEBHOOK_PAYLOAD_RETENTION_DAYS; crates/registry-bregctl/src/lib.rs, ApplyArgs and MigrationCommand; crates/registry-breg/src/migration_reconcile.rs; crates/registry-breg/src/audit.rs; crates/registry-breg/src/history_erasure.rs, MAX_ERASURE_REVISIONS; - crates/registry-breg/src/history_rebaseline.rs, MAX_REBASELINE_LIVE_ROWS. */} + crates/registry-breg/src/history_rebaseline.rs. */} ## Capacity diff --git a/docs/site/src/content/docs/operate/breg-retention.mdx b/docs/site/src/content/docs/operate/breg-retention.mdx index 8ac4f5bef8..15f3c63f2e 100644 --- a/docs/site/src/content/docs/operate/breg-retention.mdx +++ b/docs/site/src/content/docs/operate/breg-retention.mdx @@ -35,6 +35,11 @@ reference is not a way back either: an erased revision is gone from the registry, and a snapshot reference that named it can no longer be served. +A package change removes nothing from history either. A successor that drops a field or an entity +leaves the removed values in every revision snapshot recorded before it, and `bregctl diff` +reports each such removal as `diff.history.removed_values_retained`. `history erase` is the only +command that removes them, and it removes whole revisions of one record, not one field. + Every command on this page reads the runtime file and connects to PostgreSQL directly with the migration credential, and the erasures and the rebaseline take the exclusive registry lock, so run them from the operator host while no activation is in progress. When the database @@ -71,8 +76,8 @@ Confirm the record and the approved revision boundary before running the command backups, saved exports, and copies already delivered to consumers. A saved snapshot reference cannot restore erased bytes. There is no dry run. Erasing baseline data also makes snapshot coverage unavailable until [`history rebaseline`](#restore-snapshot-coverage-after-an-erasure) -restores it, and that command refuses a registry holding more than 1,000 live rows: if this -registry is at or near that size, confirm the rebaseline is possible before you erase, not after. +restores it, and that command holds a write lock on every entity table until it has verified +every live row, so plan the erasure for a window in which writes can wait. ::: **`history erase` refuses to run without `--acknowledge-irreversible`, because erasure cannot be @@ -106,7 +111,9 @@ apply the same package again once the rebaseline completes. {/* Evidence: crates/registry-breg/src/history_erasure.rs; crates/registry-bregctl/src/history_erasure_lifecycle.rs; crates/registry-bregctl/src/lib.rs, HistoryEraseArgs; - crates/registry-breg/src/history_rebaseline.rs, MAX_REBASELINE_LIVE_ROWS; + crates/registry-breg/src/history_rebaseline.rs; + crates/registry-breg/src/history_migration.rs, verify_every_live_row_matches_its_journal_head; + crates/registry-breg/tests/postgres_history_rebaseline.rs, rebaseline_verifies_more_live_rows_than_one_commit_indexes; crates/registry-breg/src/postgres/interlock.rs, history_coverage_admits_successor; crates/registry-breg/tests/postgres_history_erasure.rs; crates/registry-breg/tests/postgres_history_migration.rs; @@ -141,11 +148,32 @@ is already complete, when the registry is not ready, when a retained journal hea by a commit, and when a live row has no retained journal head that reproduces it. The result and the audit record carry counts and positions, not values. -That one transaction verifies at most 1,000 live rows, and a registry holding more is refused -with `history.rebaseline.live_rows.budget_exceeded`. The limit is fixed and the refusal is not -retryable: a registry over that limit cannot restore snapshot coverage with this command, so -treat the code as a report that the erasure left coverage unrecoverable rather than as a -transient failure. +The transaction has no live-row limit. It reads each entity's live rows, their journal heads, and +the retained heads in the same record-identifier range in pages of 1,000 records, and it proves +each entity holds exactly one retained journal head per live row. Memory stays flat as the +registry grows, and each statement is bounded by one page of records and their retained +revisions, which is not a fixed cost: a page of records that carry thousands of revisions each +reads far more than an average page, and the last statement for each entity counts every +retained head past its last live row. + +The run is a read outage for the whole registry. To read rows under forced row security, the +migration role lifts that force on every entity table before the first page, and lifting it takes +each table exclusively until the transaction commits or rolls back. API reads and writes of every +entity wait for the whole run, and a request that waits past +`operationalTimeouts.httpRequestMilliseconds` fails. The migration role deliberately has no +`BYPASSRLS` authority, which is what would let it read without that lock. Plan the run as a +maintenance window: + +- Size the window from a rehearsal. Restore a recent backup to a scratch database, erase there, + and time `history rebaseline` against it. The run grows with the number of live rows and + retained revisions across all entities; allow at least twice the rehearsed time. +- Stop or drain API traffic for the window, or announce that reads and writes fail during it. +- Set `operationalTimeouts.migrationLockMilliseconds` long enough to wait out in-flight requests + at the start. Set `operationalTimeouts.migrationStatementMilliseconds` long enough for one scan + of the whole revision journal: before the entity tables are locked, one statement proves every + retained journal head is indexed by a commit. Every later statement reads one page, so the same + timeout must also cover the page whose records hold the most retained revisions; the rehearsal + shows whether it does. `history.rebaseline.live_rows.unverified` reports one live row that its journal head does not reproduce, and names neither the entity nor the record, because operator refusals here carry no @@ -404,7 +432,7 @@ outside the new writer's retention directory. | Symptom | Next move | | --- | --- | | A saved snapshot returns `503 source.unavailable` | Retained history no longer covers it, after an erasure or an incompatible successor. Take a fresh reference. | -| A fresh snapshot reference is refused after an erasure | Coverage ends before the erased commit. Run `history rebaseline` to cover the current state again; earlier references stay unavailable. It verifies at most 1,000 live rows, and refuses a larger registry outright. | +| A fresh snapshot reference is refused after an erasure | Coverage ends before the erased commit. Run `history rebaseline` to cover the current state again; earlier references stay unavailable. Reads and writes of every entity wait while it runs; see [the outage note](#restore-snapshot-coverage-after-an-erasure). | | `history rebaseline` reports `history.rebaseline.live_rows.unverified` | A live row and its journal head disagree, and the refusal names no record. Read the revisions of the records you suspect, or compare live rows with their journal heads under the migration role. | | `history erase` reports `history.erase.acknowledgement.required` | The command was run without `--acknowledge-irreversible`. Confirm the record and the revision boundary, then run it again with the flag. Nothing was read or erased. | | `history.erase.request_file.refused` or `history.rebaseline.request_file.refused` | The request file must be an absolute path to an owner-only regular file, not a symbolic link, holding the documented fields and nothing else. | diff --git a/docs/site/src/content/docs/operate/breg.mdx b/docs/site/src/content/docs/operate/breg.mdx index 12a865ec9f..cff227f77b 100644 --- a/docs/site/src/content/docs/operate/breg.mdx +++ b/docs/site/src/content/docs/operate/breg.mdx @@ -5,7 +5,7 @@ status: current owner: registry-docs source_repos: - registry-stack -last_reviewed: "2026-09-08" +last_reviewed: "2026-09-25" doc_type: how-to persona: - operator @@ -100,9 +100,9 @@ an older public image tag cannot serve this configuration. The release image is distroless nonroot for `linux/amd64`; see [platform support](../../explanation/known-limitations/#platform-support) for the artifact matrix. Its entrypoint is `/usr/local/bin/breg`, its default arguments are -`--config /etc/breg/runtime.yaml`, and it exposes port 8080. The image carries no shell, no writable -directory, no healthcheck subcommand, and no `bregctl`. Mount a writable volume for the file audit -destination; mount the runtime configuration and its package and secret files read-only under +`--runtime-config /etc/breg/runtime.yaml`, and it exposes port 8080. The image carries no shell, no +writable directory, no healthcheck subcommand, and no `bregctl`. Mount a writable volume for the +file audit destination; mount the runtime configuration and its package and secret files read-only under `/etc/breg`. The orchestrator probes `GET /health` or `GET /healthz` for liveness and `GET /ready` for readiness. Activation runs on a separate operator host with the matching `bregctl`. @@ -311,6 +311,7 @@ database: runtime: registry_runtime package: root: /var/lib/breg/packages/build-1/package + expectedDigest: sha256: trustAnchorPath: /etc/breg/package-trust-anchor.json compilerSourceRevision: civil-registry-0.1.0 activeRevision: sha256: @@ -341,6 +342,16 @@ cursor: eventDestinations: {} ``` +`package.expectedDigest` is optional. When you set it, copy the `packageDigest` +reported by the successful `bregctl package` publication. Startup checks that +digest before the existing BReg signature, environment, database, sequence, and +active-revision checks. Rebuild a candidate with `bregctl package`; do not edit +`SHA256SUMS`, `REVISION`, or files below `package.root` in place. + +{/* Evidence: crates/registry-breg/src/runtime_config.rs, RuntimeConfig::verify_package_envelope; + crates/registry-breg/src/package.rs, PreparedPackage::publish_to_directory_with_revision; + crates/registry-breg/src/startup.rs, prepare(). */} + `publicOrigin` may include a deployment path prefix, such as `https://registry.example.org/registry-a`. Discovery, paging, and schema links preserve the configured prefix and never derive their authority from request headers. @@ -398,6 +409,13 @@ runtime file's `identity.databaseInitializationEnvironment` is `local`, and `bre field_encryption_refusal(); crates/registry-breg/src/startup.rs, FieldEncryptionCustody; crates/registry-bregctl/src/doctor.rs, startup_diagnostic(). */} +`issuer` must be an `https` URL without credentials, query, or fragment; plain `http` is accepted +only on an IPv4 loopback host, for local development or an issuer or proxy the operator runs on the +same host. Use an `https` issuer in production. `audience` is at most 512 characters. `jwksSource` +takes one of three kinds: `discovery` (the default) reads the key set location from the issuer's +discovery document, `uri` names it directly with a `uri` member under the same URL rule, and +`static` pins a document through `documentRef`. + `leewayMilliseconds` must be a whole number of seconds and at most 300000. Static keys load at startup; rotate them through a configuration change and restart: @@ -713,7 +731,7 @@ fingerprint. Confirm the configuration before starting the process: ```sh bregctl verify --runtime-config /etc/breg/runtime.yaml bregctl doctor --runtime-config /etc/breg/runtime.yaml -breg --config /etc/breg/runtime.yaml +breg --runtime-config /etc/breg/runtime.yaml ``` `verify` opens no runtime dependency. It proves that the configured package verifies against the @@ -827,21 +845,70 @@ the database directly. | --- | --- | | `remote-uncertain`, `cancellation-uncertain`, `result-lookup-uncertain`, `source-precondition-changed` | Wait through `application.nextAttemptAt` where present and re-read the request. The durable worker retries automatically. A later pending answer or a reconciled result clears `result-lookup-uncertain`. | | `token-unavailable` | Restore the configured review-authority credential provider, then let the durable worker retry. | -| `submission-recovery-expired` | Reconcile the Casework request by the recorded idempotency and digest bindings before starting a new proposal. | +| `submission-recovery-expired` | Reconcile the Casework request by the recorded idempotency and digest bindings. If the authority never received it, `bregctl review-recovery resubmit` submits the same retained request again; otherwise start a new proposal. | | `remote-refused` | Correct the producer, policy, or submitted contract reported by the review authority before submitting a new proposal. | | `cancellation-recovery-expired`, `cancellation-attempts-exhausted` | Reconcile the exact Casework request and cancellation idempotency key. If cancellation did not complete, correct the binding before starting a fresh proposal. | | `result-expired` | Start a new review; the authority no longer promises the result payload. | -| `result-poll-attempts-exhausted` | The result lookup failed, or answered an empty `404`, until the attempt budget ran out; a review the authority keeps answering as pending is never failed. Reconcile the accepted Casework request by its retained binding. If the authority still holds a result, repair the result endpoint and start a new proposal; the retained binding identifies the review to close. | +| `result-unknown-to-authority` | The authority answered the result lookup with an empty `404`: it no longer holds the accepted review, typically because the review environment was restored from an older backup or replaced. The engine keeps polling, after every review the authority still knows, until the attempt budget runs out. Once the authority is back in its intended state, run `bregctl review-recovery resubmit` to submit the same retained request again, or `bregctl review-recovery close` to stop waiting. | +| `result-poll-attempts-exhausted` | The result lookup failed, or answered an empty `404`, until the attempt budget ran out; a review the authority keeps answering as pending is never failed. Reconcile the accepted Casework request by its retained binding. If the authority lost the review, `bregctl review-recovery resubmit` submits the same retained request again. If the authority still holds a result, repair the result endpoint and start a new proposal; the retained binding identifies the review to close. | +| `operator-closed` | An operator closed the review with `bregctl review-recovery close`. Start a new proposal, or run `bregctl review-recovery resubmit` if the authority can take the same request again. | | `executor-unconfigured` | Restore the named executor in runtime configuration, then use the advertised authorized manual apply action for the same approved proposal. | | `executor-denied` | Restore the executor's current application grant, then use the advertised authorized manual apply action for the same approved proposal. | | `source-action-unavailable` | Restore the compiled source apply action and its access profile, then use the advertised authorized manual apply action for the same approved proposal. | | `source-response-invalid` | Repair the source deployment so its read and apply responses match the compiled contract, then use the advertised authorized manual apply action. | | `application-attempts-exhausted` | Inspect the source and executor logs using the request and application ids, then apply manually or create a new proposal. | +### Resubmit or close a review the authority lost + +A review environment restored from an older backup, or replaced by a fresh one, no longer holds +the reviews the engine submitted to it. Two commands recover one exact request proposal version. Both +need the migration authority of the runtime file, take the registry lock, and append one audit +record that names the request only by its keyed record reference. + +```bash +bregctl --format json review-recovery resubmit \ + --runtime-config /etc/registry/breg/runtime.yaml \ + --request-entity correction-request \ + --request-id 00000000-0000-4000-8000-000000000001 \ + --proposal-version 1 +``` + +`resubmit` accepts an accepted review coded `result-unknown-to-authority`, or a failed one coded +`result-poll-attempts-exhausted`, `submission-recovery-expired`, or `operator-closed`. It releases +the recorded authority binding, resets the attempt counters, restarts the recovery deadline with +the configured window, and queues the retained request for submission under its original +idempotency key. An authority that still holds the request answers with the same binding; one that +lost it creates the review again. A result the old binding delivers later is recorded as +unmatched. + +Resubmitting relies on the authority still honouring that idempotency key. Before you resubmit a +review coded `result-poll-attempts-exhausted` or `operator-closed`, confirm at the authority that +it no longer holds the review: an authority whose idempotency retention has lapsed opens a second +review while a reviewer may still be working the first. + +`close` accepts an accepted review without a result. It marks the submission failed with +`operator-closed` and keeps the binding for reconciliation, and a result the authority delivers +for it afterwards is refused. It does not contact the authority, so cancel the review there as +well if it still exists. + +Both refuse a withdrawn proposal, a review whose result is already recorded, and any other state +with `review_recovery.submission.ineligible`, whose message names the reason, state, and code. +`resubmit` also refuses once request retention erased the review request (`request-erased`), or +when the request no longer awaits review for that proposal version (`proposal-not-submitted`). + +Each operation writes an audit `request` entry to the `bregctl` companion audit file before it +changes the submission, and its `response` after the change commits. An audit destination that +refuses the request entry stops the operation before it changes anything. One that refuses the +response after the commit is reported as `review_recovery.recovery.unaudited`: the recovery took +effect, so read the submission's state before retrying. + Retain the project sources, module locks, generated files, test receipt, signature documents, and runtime file for each activation. Store secrets separately. {/* Evidence: crates/registry-bregctl/src/apply_lifecycle.rs; + crates/registry-breg/src/review_recovery.rs, ReviewRecoveryOperatorService; + crates/registry-breg/tests/postgres_change_requests.rs, an_operator_resubmits_or_closes_a_review_its_authority_lost; + crates/registry-bregctl/src/review_recovery.rs; crates/registry-breg/src/review_store.rs, read_projection(); crates/registry-bregctl/src/doctor.rs, startup_diagnostic(); crates/registry-bregctl/src/lib.rs, VerifyArgs, verify(), write_doctor_success(), @@ -917,7 +984,11 @@ rather than reinterpreted, and a request file that is not owner-only is refused connection is opened. For every record still holding pre-flip plaintext, the lifecycle erases its retained history through the same path [`history erase`](../breg-retention/#erase-retained-history) uses and restores snapshot coverage -with one rebaseline, then writes one audit record that carries counts, not values. +with one rebaseline, then writes one audit record that carries counts, not values. That rebaseline +blocks reads and writes of every entity until it commits, so run the lifecycle in the maintenance +window the +[rebaseline outage guidance](../breg-retention/#restore-snapshot-coverage-after-an-erasure) +describes. `bregctl field-encryption keygen --output ` writes one fresh base64 data key for the local file provider with owner-only permissions, never prints it, and refuses to overwrite an @@ -1039,6 +1110,7 @@ behind an upstream rate limit. | A secret reference is refused | The file under the secret root must be a regular file owned by the running user, with mode `0400` or `0600`, one link, and no symbolic link, and its name must follow the naming rule in [Create the secret files](#create-the-secret-files). | | `package` refuses the receipt | The receipt binds sources, baseline, fingerprint, and signature policy. Rerun `test` for the exact candidate. | | `apply --initial` reports a binding error | The runtime file must already name the target package's revision and sequence one. | +| `doctor` reports `startup.instance_claim.mismatch` | The database is a copy the registry's instance claim does not name, as after a logical restore. Adopt it once the original is stopped for good, as [back up and restore the database](../breg-changes/#back-up-and-restore-the-database) describes. | | `doctor` reports `startup.oidc.refused` | Check the issuer, discovery reachability or the static JWKS document, the algorithm, and the leeway bound. | | `doctor` reports `startup.package.refused` with `the package compiler derivation failed` after an engine upgrade | The upgraded compiler derives a different schema from the active package, for example a new reference index. Build and apply a successor package with the upgraded `bregctl` before starting the upgraded `breg`, as [Change an active registry](../breg-changes/#test-and-package-the-successor) describes. `test` and `package` accept the active package through `--baseline-runtime-config`; `verify` and `diff --runtime-config` refuse it until the successor is active. | | `doctor` or startup warns `postgres.connections.pool_over_half` | Lower `database.pool.maxSize` or raise `max_connections` until every replica, operator command, and monitoring session fits; see [Set a PostgreSQL baseline](#set-a-postgresql-baseline). | diff --git a/docs/site/src/content/docs/operate/casework.mdx b/docs/site/src/content/docs/operate/casework.mdx index 6047cb17c8..9b229028c3 100644 --- a/docs/site/src/content/docs/operate/casework.mdx +++ b/docs/site/src/content/docs/operate/casework.mdx @@ -159,8 +159,8 @@ kind: CaseworkRuntimeConfig package: # The selected package always contains the policy at casework.yaml. root: /etc/registry-casework/package - # Optional: the policyDigest of the package you reviewed; any other package is refused. - expectedPolicyDigest: sha256:0000000000000000000000000000000000000000000000000000000000000000 + # Optional: the packageDigest of the package you reviewed; any other package is refused. + expectedDigest: sha256:0000000000000000000000000000000000000000000000000000000000000000 listener: # The proxy in front of this private listener terminates TLS. bind: 10.42.0.7:8100 @@ -181,6 +181,8 @@ authentication: oidc: issuer: https://identity.example.org/realms/registry audience: urn:example:casework + # Every client whose tokens this deployment admits; required in production. + allowedClients: [casework-console] scopeClaim: scope humanIdentity: claim: registry_actor_kind @@ -205,8 +207,9 @@ sources: | Section | What it binds | | --- | --- | -| `package` | The absolute root of the package containing `casework.yaml`, its manifest, and exact source descriptions, and optionally the one policy digest the runtime may load from it. | +| `package` | The absolute root of the package containing `casework.yaml`, its `SHA256SUMS`, and exact source descriptions, and optionally the one package digest the runtime may load from it. | | `listener` | The listener address and the transport boundary the deployment declares for it. | +| `metricsListener` | Optional. A second, operator-private address for `/metrics` and `/version`; see [Scrape metrics and the running version](#scrape-metrics-and-the-running-version). | | `secretProviders` | The explicitly enabled file and environment secret providers. | | `database` | Secret references for the runtime and migration connection URLs, and an optional trusted root certificate for the PostgreSQL connection. | | `authentication` | The OpenID Connect issuer, audience, claim names, human-identity assertion, and the source of the issuer's keys. | @@ -216,13 +219,19 @@ sources: The package root, file secret root, and audit path must be absolute. A `stdout` destination refuses `path`, `rotateBytes`, and `retainDays`, and leaves collection and retention to the platform that reads the stream. An environment reference is -valid only when `secretProviders.environment: {}` explicitly enables that provider. +valid only when `secretProviders.environment: {}` explicitly enables that provider. The runtime file +itself may not pass through a symbolic link and is at most 1 MiB. A string value may take a +deployment value from the environment at startup, written `${VAR}`, `${VAR:-default}`, or +`${VAR:?message}`; the substituted value is always text. Substitution is refused in a field whose +name ends in `Ref` and beneath `secretProviders`, because a secret reference is written literally, +and it never applies to `casework.yaml`, where an expression is refused by the runtime and by +`caseworkctl check`. The listener boundary is checked, not advisory. With `operator-controlled-upstream` and `private-address`, the address must be loopback or private (IPv4 private range or IPv6 unique local); `container-private` also accepts a wildcard bind for a container on a private network. `development-loopback` accepts only a loopback address with `private-address`, and it is the one -mode that serves an authored project with no manifest beside it. Apply HSTS on the proxy's TLS +mode that serves an authored project with no `SHA256SUMS`. Apply HSTS on the proxy's TLS responses; the runtime adds its remaining security headers and `Cache-Control: no-store` to every response it returns. @@ -237,10 +246,21 @@ media type spelled `at+jwt` or `application/at+jwt`. A token carrying any other `JWT` included, is refused, so an issuer that mints ordinary JWTs for this audience must be configured to mint the access-token type. -With the discovery source, the runtime reads the issuer's discovery document at startup and -fetches its keys from the `jwks_uri` that document names; an optional `jwksUri` in this file -overrides that one value and leaves the rest of discovery in place. Prefer the static alternative -when the Casework host cannot reach discovery, or when you pin the issuer's keys deliberately: +`issuer` must be an `https` URL without credentials, query, or fragment; plain `http` is accepted +only for an IPv4 loopback address under `development-loopback`. `audience` is at most 512 characters. + +`allowedClients` names the clients whose tokens the runtime admits, matched against the token's +`azp` claim or, when it has none, its `client_id` claim. With `operator-controlled-upstream` the list +must name at least one client, and the runtime refuses to start at +`authentication.oidc.allowedClients` otherwise: an empty list admits every client the issuer +verifies, including an unrelated application registered in the same realm. `development-loopback` +still accepts an empty list. A configured `taskAuthority` requires a non-empty list in either mode. + +With the discovery source, the default, the runtime reads the issuer's discovery document at +startup and fetches its keys from the `jwks_uri` that document names. `kind: uri` with `uri` fetches +the keys from that fixed address instead and skips discovery; the removed `jwksUri` key is refused +with that replacement named. Prefer the static alternative when the Casework host cannot reach the +issuer at all, or when you pin the issuer's keys deliberately: ```yaml jwksSource: @@ -282,8 +302,9 @@ before they become the anchor for verification. ::: {/* Evidence: crates/registry-casework/src/config.rs, RuntimeConfig, TlsTermination, - ListenerNetworkExposure, OidcConfig, HumanIdentityConfig, OidcJwksSource, valid_listener(), - and parse_static_jwks(); + ListenerNetworkExposure, OidcConfig, HumanIdentityConfig, JwksSource, parse_static_jwks(), + and AllowedClientsRequired; + crates/registry-platform-config/src/blocks.rs, PrivateListenerConfig and is_valid(); crates/registry-casework-breg/src/config.rs, BregBinding, binding_generation(), generation_ignores_credentials_transport_and_presentation_settings(), DEFAULT_REQUEST_TIMEOUT_MILLISECONDS, and MAXIMUM_TIMEOUT_MILLISECONDS; @@ -332,47 +353,88 @@ entries must stay comparable. ## Verify the package before it opens the listener The runtime verifies the package every time it loads the runtime file, before it opens the -database, contacts the issuer, or binds the listener. It reads `casework.package.json` beside -`casework.yaml`, recomputes the policy digest over the sorted file list, checks every file's -SHA-256 digest and byte count, and refuses a package directory holding any file the manifest does -not declare or any symbolic link. A start that verifies the package records the policy digest in -the runtime log before anything else happens. - -With `tlsTermination: operator-controlled-upstream`, an absent manifest is a refusal rather than a -fallback to the authored project. - -A verified package proves its files match its own manifest, not that it is the package you -reviewed. Set `package.expectedPolicyDigest` to the `policyDigest` that `caseworkctl package` -reported for the reviewed package, and the runtime starts only on that package: a package naming -another digest, or a directory with no manifest, is refused with both digests named, so a -replaced package directory cannot change the policy a restart loads. +database, contacts the issuer, or binds the listener. It reads `SHA256SUMS` at `package.root`, +recomputes the SHA-256 digest of every file it lists, and refuses a changed, missing, or extra +file by name, and any symbolic link. The package must hold exactly `casework.yaml` and the source +descriptions it names. A start that verifies the package records the package digest, the SHA-256 +digest of `SHA256SUMS`, in the runtime log before anything else happens. + +With `tlsTermination: operator-controlled-upstream`, an absent `SHA256SUMS` is a refusal rather +than a fallback to the authored project. A directory that still holds `casework.package.json` is +refused and names `caseworkctl package` as the command that rebuilds it. + +A verified package proves its files match its own `SHA256SUMS`, not that it is the package you +reviewed. Set `package.expectedDigest` to the `packageDigest` that `caseworkctl package` reported +for the reviewed package, and the runtime starts only on that package: any other package, or a +directory with no `SHA256SUMS`, is refused with both digests named, so a replaced package directory +cannot change the policy a restart loads. Each of these refusals is one run's entire output on standard error, written before the listener binds, and each exits non-zero: ```text -casework: operator-controlled production requires a verified Casework policy package -casework: the Casework policy package is invalid -casework: package.expectedPolicyDigest is sha256:4f0c…, but package.root holds the package with policy digest sha256:9b2e… +casework: the directory at /etc/registry-casework/package has no SHA256SUMS, so it is not a package; build one with `caseworkctl package` +casework: the package at /etc/registry-casework/package does not match its SHA256SUMS; changed: casework.yaml; rebuild the package with `caseworkctl package` and deploy the whole directory +casework: package.expectedDigest is sha256:4f0c… but the package at package.root is sha256:9b2e…; deploy the pinned package or update package.expectedDigest casework: the Casework runtime configuration is not valid YAML casework: the Casework runtime configuration is invalid casework: the Casework secret-provider configuration is invalid; secretProviders.file.root must be an absolute path ``` -The first line reports a production configuration with no manifest. The second reports a manifest -whose digests, byte counts, or file set do not match the directory. The third reports a verified -package other than the one `package.expectedPolicyDigest` names, shown here with both digests +The first line reports a production configuration with no `SHA256SUMS`. The second reports a +package whose files do not match its `SHA256SUMS`, naming each changed, missing, or extra file. The +third reports a verified package other than the one `package.expectedDigest` names, shown here with both digests shortened. The fourth and fifth report the runtime file itself: unparseable YAML, and a configuration the checks refuse, which covers an invalid listener boundary, an empty issuer or claim name, a human-identity claim equal to the scope claim, an access profile using the human-identity claim as its principal claim, and a `sources` map that does not match the declared source ids exactly. The sixth reports a secret provider root the resolver cannot use. -{/* Evidence: crates/registry-casework/src/config.rs, verify_policy_package(), - PolicyPackageManifest, RuntimeConfig::check(), and RuntimeConfigError; +{/* Evidence: crates/registry-casework/src/config.rs, verify_casework_package(), + RuntimeConfig::check(), and RuntimeConfigError; + crates/registry-platform-config/src/package.rs, verify_package() and PackageError; crates/registry-casework/src/runtime.rs, serve_from_path() and RuntimeError; crates/registry-casework/src/main.rs. */} +## Refuse a package that strands pinned work + +A review request pins its kind's policy when it is admitted: the stages, their queues, the +profiles that decide them, and the display schema reviewers read against. An open work item keeps +the queue it was routed to. A later package can remove one of those queues or profiles, change a +kind's content without changing its version, or change a source read so that the pinned display +schema no longer accepts it. Each of those would leave in-flight work that no team serves, that no +reviewer can decide, or whose context every reviewer is refused. + +After it connects to the database and before it registers any source generation, the runtime +compares the package it is about to activate with the work the database retains and refuses a +package that would strand any of it. The refusal names each conflict with its counts, never a +subject: + +```text +casework: the policy package would strand work pinned under an earlier package: 3 in-flight reviews and 2 open work items are in queue intake, which the package no longer declares. Let that work finish under the earlier package, or set package.acknowledgeStrandedWork to sha256:9b2e… to activate this package anyway +``` + +The other conflicts read the same way: an access profile the package no longer declares, a review +kind version declared with different content, a removed source that source-context reviews or open +work items still need (no action on such an item can reach its source), and a field a source read +would add to, or drop from, what the pinned display schema allows. Give a +changed review kind a new version rather than editing one that work still pins. Keep the earlier +package active until the named work finishes; if the work may stay hidden or orphaned, set +`package.acknowledgeStrandedWork` to the digest the refusal names. The acknowledgement admits only +that package, so the next package is compared afresh. + +To preview the comparison before a restart, point `caseworkctl doctor` at a runtime file whose +`package.root` holds the next package. Its `pinnedWork` check fails with the same sentence, and its +report lists the conflicts it found under `pinnedWork.conflicts` with the verdict `clear` or +`acknowledged`. + +{/* Evidence: crates/registry-casework/src/pinned_work.rs, stranded_pinned_work(), + pinned_work_verdict(), and stranded_work_refusal(); + crates/registry-casework/src/runtime.rs, check_pinned_work() and serve_from_path(); + crates/registry-caseworkctl/src/project.rs, doctor_pinned_work(); + crates/registry-casework/tests/review_postgres.rs, + activation_preflight_counts_in_flight_reviews_a_package_would_strand(). */} + ## Migrate and serve Apply the migrations with the migration credential, from the operator host: @@ -418,6 +480,41 @@ source-backed work before upgrading. crates/registry-casework/tests/postgres_transactions.rs, returning_to_an_earlier_binding_generation_opens_a_fresh_occurrence(). */} +### Upgrade Casework and BReg in lock-step + +Casework and every BReg source it reads run the same release. BReg names its release in the +`Registry-Engine-Version` header of `GET /v1/registry`, and Casework compares it with its own on +every registry contract read. It refuses any other release, and an engine that reports no version, +rather than guessing what an older or newer contract means: a member such as a change request's +`effects` that one release sends and another omits must never read as empty. + +A build made without the release marker reports its version followed by `-dev`, as +`breg --version` and `casework --version` show. Casework sets one trailing `-dev` aside on each +side, so a release build matches a development build of the same version and logs a warning that +it did; every other part of the version, a prerelease tag included, must match exactly. `0.34.0` +matches `0.34.0-dev`, and `0.34.0-rc.1` matches only `0.34.0-rc.1` and `0.34.0-rc.1-dev`. A +development build is matched by its version alone, not by the source revision it was built from, +so run release builds of both in production. + +To upgrade, upgrade each BReg source first, then Casework, to the same release: + +1. Upgrade and start BReg. Until Casework matches it, Casework refuses that source's reads by name: + work items report a source outage, `GET /ready` fails once reconciliation keeps failing, and the + runtime log names both versions. +2. Migrate and start Casework of the same release. It resumes source reads on the first matching + contract read, with no further step, and logs `Casework reads a BReg source on its own release` + with the engine version. + +A reverse proxy between Casework and BReg must pass the `Registry-Engine-Version` response header +through unchanged. + +{/* Evidence: crates/registry-breg/src/api/mod.rs, registry_metadata(); + crates/registry-breg-client/src/client.rs, registry_contract_and_engine_version(); + crates/registry-casework-breg/src/lib.rs, BregAdapter::metadata() and PeerVersionMismatch; + crates/registry-caseworkctl/src/project.rs, source_readiness_failure(); + crates/registry-casework-breg/tests/source_boundary.rs, + a_breg_engine_from_another_release_is_refused_naming_both_versions(). */} + ### Upgrade from a release that chained the audit file Casework 0.34.0 and earlier kept pending audit records in the `casework_audit_outbox` table and @@ -492,8 +589,52 @@ reporting the stopped worker, because a deployment whose maintenance, review com source reconciliation loop is gone keeps neither its deadlines nor its source state current. Probe `GET /health` for liveness: it answers `200` as long as the process runs. Probe `GET /ready` -before routing traffic: it answers `503` while the audit writer is not ready or PostgreSQL is -unreachable, and `200` otherwise. Readiness checks the audit writer first, then the store. +before routing traffic: it answers `503` while the audit writer is not ready, PostgreSQL is +unreachable or its schema is not current, or a source's reconciliation has failed five consecutive +passes, and `200` otherwise. Readiness checks the audit writer first, then the store, then +reconciliation health. The `503` body names none of these; the runtime log and `caseworkctl doctor` +do. + +### Scrape metrics and the running version + +Set `metricsListener` to serve telemetry on a second address that the proxy in front of the API +never routes to: + +```yaml +metricsListener: + bind: 127.0.0.1:9100 +``` + +The address must be loopback or private with a nonzero port, never a wildcard, and never the address +and port the API listener occupies; the runtime refuses anything else at `metricsListener` before +it opens a connection. Both addresses bind before either serves, so a metrics address already in +use refuses startup, and the metrics listener stops with the API listener. Without the block no +telemetry socket opens. + +`GET /version` answers JSON with the running `version` and the `packageDigest` of the verified +policy package, so a rollout can confirm which package each +replica serves. `GET /metrics` answers the Prometheus text format, read at scrape time: + +| Series | Meaning | +| --- | --- | +| `casework_build_info{version,package_digest}` | Always `1`; the labels carry the build and the package digest. | +| `casework_database_up` | `1` when this scrape read PostgreSQL. The series below appear only then. | +| `casework_source_reconciliation_consecutive_failures{source_id}` | Failed reconciliation passes in a row for each configured source. | +| `casework_source_reconciliation_last_success_age_seconds{source_id}` | Seconds since the source last reconciled; absent until its first success. | + +Alert on a reconciliation age well beyond the source's interval: that age catches a pass that +hangs, which readiness does not. Audit writer health is reported by `/ready`, not by a series. The only label values are the configured source identifiers, the version, and +the package digest. + +The listener takes no credentials, so the database series do not follow the scrape rate: every +scrape within five seconds of a database reading reuses it, scrapes that arrive during a reading +wait for that one reading, and a reading that takes longer than five seconds reports +`casework_database_up 0`. A burst of scrapes therefore holds at most one database connection at a +time and runs at most one reading every five seconds. + +{/* Evidence: crates/registry-casework/src/config.rs, MetricsListenerConfig; + crates/registry-casework/src/metrics.rs, metrics_router(), MetricsInner::readings() and render(); + crates/registry-casework/src/runtime.rs, serve_from_path() and serve_until_worker_stops(). */} Each audited call writes a request entry, schema `registry-casework-audit/v1`, before it opens its transaction, and its response entries after the transaction commits, all sharing one correlation. @@ -517,6 +658,17 @@ from the registry contract, readiness, or a list is. The next successful reader again, not that reconciliation has caught up: a pass that still cannot apply the source logs `Casework reconciliation pass did not complete`. +A reconciliation pass that cannot apply one item keeps going with the rest, logs +`Casework reconciliation could not apply every claimed subject` with how many it could not apply, +and retries each one once its 30-second claim lapses. The runtime records every pass's outcome in +the database: consecutive failures, the time of the last pass that succeeded and of the last that +failed, and the class of the last failure (`source-unavailable`, `source-refused`, `store`, or +`configuration`). The fifth consecutive failed pass for a source logs +`Casework reconciliation keeps failing; readiness fails until a pass succeeds`, and every replica's +readiness fails until one pass for that source succeeds, which resets the count. Readiness does not +see a pass that hangs without finishing; the time of the last pass that succeeded, which `doctor` +prints, does. + Between a source change and the reconciliation that applies it, an item whose binding moved within the same source generation stays in the inbox, the next-item result, and holdings, and its view, history, and clocks stay readable with the binding Casework last applied. It offers no actions. @@ -595,16 +747,27 @@ caseworkctl doctor --runtime-config /etc/registry-casework/runtime.yaml ``` It checks the configuration, the exact imported source descriptions, each source connection and its -reader grants, that the audit destination is writable by the user running it, the database, the -OIDC issuer, and directory readiness, and it stops at the first one that refuses and names it. Run -it as the runtime user. The audit check refuses an audit directory that is missing and cannot be -created, that another user owns, or that is group- or world-writable, and an existing audit file -that is not owner-only. It takes no lock, so it passes beside a running service. Its secret -preflight resolves the audit reference before it opens anything, and a reference that cannot -resolve is reported by name with the rule it broke. Directory +reader grants, that the audit destination is writable by the user running it, the database and its +schema version, the in-flight work the package would strand (see +[Refuse a package that strands pinned work](#refuse-a-package-that-strands-pinned-work)), the OIDC +issuer, directory readiness, and each source's reconciliation health, and it +stops at the first one that refuses. A refusal carries the code `casework.doctor.check-failed`, +names the check in its `path` (for example `doctor:/checks/database`), says what failed, and +suggests the next step. It never echoes a connection string or a source response. Run it as the +runtime user. The audit check refuses an audit directory that is missing and cannot be created, that +another user owns, or that is group- or world-writable, and an existing audit file that is not +owner-only. It takes no lock, so it passes beside a running service. Its secret preflight resolves +the audit reference before it opens anything, and a reference that cannot resolve is reported by +name with the rule it broke. Directory readiness requires a team serving every queue the package declares, and a gap is reported as an -instruction to complete the queue assignments as an Administrator. A run that reaches the end -prints a report naming each check and each source it contacted. +instruction to complete the queue assignments as an Administrator. Reconciliation health refuses a +source whose last five or more passes failed, naming the source, the count, the last failure's +class, and when a pass last succeeded. The audit check verifies the sealed audit files, and the +active file too when no runtime holds its lock, so run it with the runtime stopped to prove a key +against the whole chain; it refuses a chain the configured key does not verify. A run that reaches +the end prints a report naming each check, the digest of the package `package.root` holds +(`packageDigest`), each source it contacted with its +reconciliation health, and the audit records and files it verified. `doctor` answers whether the deployment is ready to do work. It does not summarize an individual review. Inspect the Casework request, task, context, result, and result-feed resources for review @@ -615,8 +778,11 @@ or applying. Use `application.state` with `recovery.code` for recovery decisions either product's database directly. {/* Evidence: crates/registry-caseworkctl/src/project.rs, doctor(), load_runtime(), - and check_source_descriptions(); + check_source_descriptions(), doctor_dependency_failure(), and reconciliation_failure_message(); + crates/registry-caseworkctl/src/lib.rs, classify_failure(); crates/registry-platform-audit/src/writer.rs, FileDestination::check_writable(); + crates/registry-casework/src/service.rs, reconcile_source() and RECONCILIATION_FAILURE_THRESHOLD; + crates/registry-casework/src/store.rs, record_reconciliation_outcome() and reconciliation_health(); crates/registry-casework/tests/secret_diagnostics.rs. */} ## Change the active package @@ -632,7 +798,7 @@ the generation and the work items. :::caution[Editing a package in place changes nothing the process is serving] A running process keeps the policy it verified at startup, so files replaced under it leave the -deployment serving a package that no longer matches its own manifest, and the next restart refuses +deployment serving a package that no longer matches its own `SHA256SUMS`, and the next restart refuses the directory. Install the successor beside the current package and restart. ::: @@ -647,7 +813,7 @@ bound to the actor and selected profile for 15 minutes; an expired preview retur {/* Evidence: products/casework/README.md; products/casework/generated/registry-casework.openapi.json; - crates/registry-casework/src/config.rs, verify_policy_package(); + crates/registry-casework/src/config.rs, verify_casework_package(); crates/registry-casework/migrations/0016_occurrence_identity_excludes_superseded.sql; crates/registry-casework/tests/postgres_transactions.rs, returning_to_an_earlier_binding_generation_opens_a_fresh_occurrence; @@ -658,19 +824,23 @@ bound to the actor and selected profile for 15 minutes; an expired preview retur | Symptom | Next move | | --- | --- | -| `operator-controlled production requires a verified Casework policy package`, or `the Casework policy package is invalid` | The first names a production configuration with no manifest, the second a package directory whose files disagree with the manifest. Point `package.root` at the installed package, and reinstall it from the reviewed artifact rather than editing it. | +| `has no SHA256SUMS, so it is not a package`, or `does not match its SHA256SUMS` | The first names a configuration whose `package.root` is not a package, such as an authored project, the second a package directory whose files disagree with `SHA256SUMS`, naming each changed, missing, or extra file. Point `package.root` at the installed package, and reinstall it from the reviewed artifact rather than editing it. | | `the Casework runtime configuration is invalid` | Check the listener against `tlsTermination` and `networkExposure`, the non-empty issuer and claim names, the human-identity claim against `scopeClaim` and the package's profiles, and `sources` against the declared source ids. | | `the Casework secret-provider configuration is invalid` | The file provider root is relative or unusable. Write an absolute `root`. | | A secret reference is refused | The named file must be a regular file owned by the running user, mode `0400` or `0600`, one link, no symbolic link, non-empty, and free of NUL bytes. | | Startup refuses the database or `doctor` stops at it | Check the resolved URL's user and database name, TLS on the server, and the trusted root reference when the server uses a private authority. | | Startup refuses the OIDC issuer | Check discovery reachability from the Casework host, or the pinned JWKS document's keys and their distinct `kid` values. | +| `the Casework database schema is not current` | The database was never migrated, or an earlier release migrated it. Run `casework migrate` or `caseworkctl db migrate` with the migration credential, then start the runtime. | | `the Casework database schema version N is newer than this binary supports (M)` | A newer release already migrated this database. Run that release or a later one; Casework does not migrate a schema down. | | `the Casework database holds hosted work that schema migration 15 would drop` | The database still holds hosted work from Casework 0.32.0 or earlier. Keep it with that release until its work is exported, then migrate a fresh database as [Migrate and serve](#migrate-and-serve) describes. | | `the Casework database holds N audit record(s) that schema migration 17 would drop` | The earlier release has not published every audit record. Run it until its publisher drains the outbox, then migrate as [the upgrade steps](#upgrade-from-a-release-that-chained-the-audit-file) describe. | | `the Casework audit destination could not be opened: another process holds the single-writer lock beside the audit file` | Another `casework` process writes this file. Stop it, or give this process its own `audit.path`. | | `the Casework audit destination could not be opened: the audit file could not be opened: ...` | The rest of the message names the rule the audit file or its directory broke and the fix, for example `chown it to the service user and chmod it 0700`, or `archive it and restart with a fresh path` for a file whose last entry is incomplete. Apply it and start again. | | `caseworkctl doctor` stops at `the Casework audit destination is not writable by this user` | Run `doctor` as the runtime user, and fix the audit directory's owner and mode, or the audit file's, as the message names. | -| `GET /ready` returns `503` while `GET /health` returns `200` | The audit writer stopped or PostgreSQL is failing. Read the runtime log for `audit file write failed`, check the audit file, its lock, the free space, and the database, then restart the process. | +| `GET /ready` returns `503` while `GET /health` returns `200` | The audit writer stopped, PostgreSQL is failing, or a source's reconciliation is failing. Run `caseworkctl doctor`, which names the check, then read the runtime log for `audit file write failed` or the reconciliation entry, and check the audit file, its lock, the free space, and the database. | +| `doctor` refuses at `doctor:/checks/reconciliation` | The named source's last passes failed. `source-unavailable` means the source did not answer; `source-refused` means it answered with a refusal or a response Casework cannot use; `store` and `configuration` point at the database and the binding. Repair the cause; the next pass that succeeds restores readiness. | +| `doctor` refuses at `doctor:/checks/pinnedWork`, or startup refuses with `would strand work pinned under an earlier package` | The package removes a queue, profile, or source that in-flight work still needs, edits a pinned review kind without a new version, or changes what a source read discloses. Keep the earlier package until that work finishes, or set `package.acknowledgeStrandedWork` to the digest the refusal names. | +| `BReg source ID runs engine version X and this Casework runs Y`, or `BReg source ID does not report its engine version` | Casework and that BReg source are on different releases, or a proxy removes the `Registry-Engine-Version` header. Upgrade the one that is behind as [the lock-step upgrade](#upgrade-casework-and-breg-in-lock-step) describes, or let the header through; source reads resume on the next matching read. | | `profile.not-human` on a token you believe is human | The issuer did not add the configured human-identity claim to that session. Check the claim name and value against the issuer's mapping for interactive sessions. | | Work items report a source outage | The bound source is unreachable or refusing the reader. Run `caseworkctl doctor`, which names the failing source, read the runtime log's `Casework source reader request to BReg failed` entry for the cause, then check that binding's base URL, token endpoint, and reader grants. | diff --git a/docs/site/src/content/docs/operate/evidence-audit.mdx b/docs/site/src/content/docs/operate/evidence-audit.mdx index 015126d767..1eaba23672 100644 --- a/docs/site/src/content/docs/operate/evidence-audit.mdx +++ b/docs/site/src/content/docs/operate/evidence-audit.mdx @@ -61,7 +61,8 @@ Authentication, malformed-request, and invalid-selector failures happen before E enough privacy-safe context and do not fabricate an audit entry. Do not expect every HTTP request to produce two entries. -Authorized-material records can carry the governed requirement, purpose, bundle revision, +Authorized-material records can carry the governed requirement, purpose, package digest +(in the frozen `bundleRevision` field), requester pseudonym, subject role and selector-profile pseudonym, source and adapter identifiers, response protection, decision, safe failure class, disclosed concept IDs, evidence ID, and signing key ID. A standalone authorization-refusal record instead carries a requester pseudonym and closed diff --git a/docs/site/src/content/docs/operate/index.mdx b/docs/site/src/content/docs/operate/index.mdx index f58e3a07f6..09b9a64eca 100644 --- a/docs/site/src/content/docs/operate/index.mdx +++ b/docs/site/src/content/docs/operate/index.mdx @@ -26,7 +26,7 @@ change none of the meaning inside them. | Replay fixtures | `relayctl test ` | Data publisher | | Seal the revision | `relayctl package --output ` | Data publisher | | Bind the deployment | `runtime.yaml` | Deployment operator | -| Run | `relay serve --runtime ` | Deployment operator | +| Run | `relay serve --runtime-config ` | Deployment operator | | Confirm | `GET /health` and `GET /ready` | Deployment operator | Packaging recompiles the project under the production profile, so a package @@ -59,7 +59,7 @@ the verified directory. `~/.local/bin`. The container image is `ghcr.io/registrystack/relay:v0.26.1`, built on -distroless nonroot. It exposes port 8080, runs `relay serve --runtime +distroless nonroot. It exposes port 8080, runs `relay serve --runtime-config /etc/relay/runtime.yaml` by default, and probes itself with `relay healthcheck`. `relayctl` also ships for each platform in @@ -82,7 +82,7 @@ Operators who prefer to place `relay` themselves take the Before the first activation, record: -- The package directory, its `packageRevision`, and the contract revision it seals +- The package directory, its package digest, and the contract revision it seals - The SQLite source path and the source profile the package already chose - The token issuer identity, discovery URL, audience, and accepted algorithms, or the decision that the deployment is fully anonymous @@ -94,12 +94,13 @@ Before the first activation, record: Relay resolves secrets through `secret:env/` or `secret:file/` references in `runtime.yaml`, where `` is a single flat lowercase -filename. Secret values never belong in the package, and the +filename under the declared `secretProviders.file.root`. Secret values never belong in the package, and the package never travels with the database. ## The package proves integrity, not authenticity -`packageRevision` is a SHA-256 digest over the canonicalized package manifest. +The package digest is the SHA-256 digest of the package's `SHA256SUMS`, which +lists the digest of every other file, and `package.expectedDigest` pins it. It detects a modified or truncated package. It is not a signature, and Registry Relay does not sign packages or responses. Authenticity is whatever the institution's transfer, storage, and access diff --git a/docs/site/src/content/docs/operate/registry-render.mdx b/docs/site/src/content/docs/operate/registry-render.mdx index a35dc3e6e5..e3c8c2a493 100644 --- a/docs/site/src/content/docs/operate/registry-render.mdx +++ b/docs/site/src/content/docs/operate/registry-render.mdx @@ -11,8 +11,8 @@ locale: en standards_referenced: [] --- -You have a sealed template bundle and callers that need PDFs, and you want `registry-render serve` -answering them. At the end of this page one `registry-render` process serves your bundle on a private +You have a verified template package and callers that need PDFs, and you want `registry-render serve` +answering them. At the end of this page one `registry-render` process serves your package on a private address behind your TLS proxy, answers `GET /health` and `GET /ready`, renders under API-key authentication with a per-render audit trail, and shuts down within a bounded window. @@ -24,14 +24,15 @@ checklist. products/render/DEFINITION-OF-DONE.md. */} [Rendering your first document](../../tutorials/first-render-document/) covers the bundle side: -scaffolding, authoring, and sealing. This page starts from a sealed bundle directory. +scaffolding, authoring, and packaging. This page starts from a package directory. ## What you provide -- **One sealed bundle directory, mounted read-only.** `registry-render seal` writes per-file - sha256 hashes into the bundle's `manifest.yaml`, and serve verifies the seal at startup and - again in the worker on every request, so a bundle that drifts while the service runs is - refused, not rendered. Treat the directory as content the deployment does not write. +- **One package directory, mounted read-only.** `registry-render package` validates authoring + source and writes a new directory with the shared `SHA256SUMS` envelope plus optional + `REVISION`. Serve verifies the envelope at startup and again in the worker on every request, + then binds the exact captured files to the recorded digests before any validation or render. + A package that drifts while the service runs is refused, not rendered. - **One secret in an owner-only file: the caller API key.** The key is at least 32 bytes of ASCII material; serve trims exactly one trailing line ending from the file and refuses to start if the key carries any other whitespace, because a silently mis-armed key would reject every @@ -51,7 +52,7 @@ scaffolding, authoring, and sealing. This page starts from a sealed bundle direc crates/registry-render/src/server.rs, normalize_api_key; crates/registry-render/src/worker.rs, supervise and cap_address_space; crates/registry-render/src/audit.rs, RenderAudit; - crates/registry-render/src/bundle.rs, load_sealed. */} + crates/registry-render/src/bundle.rs, load_package and bind_verified_snapshot. */} ## Build the binary @@ -65,22 +66,30 @@ target/release/registry-render --version The version line names the release and the Typst pin, for example `registry-render (typst 0.15.1)`. Record it: every audit event carries the same string, and a stored PDF's provenance is -this binary plus the bundle hash. +this binary plus the package hash. {/* Evidence: crates/registry-render/src/lib.rs, display_version and TYPST_PIN. */} ## Write the runtime configuration -Serve mode reads one YAML file. Every field is a contract, and unknown fields are refused: +Serve mode reads one YAML file, selected with `--runtime-config`. Every field is a contract, and +unknown fields are refused: ```yaml -apiVersion: render.registrystack.org/v1alpha1 -kind: RenderRuntime -server: +apiVersion: registry.registrystack.org/render-runtime/v1alpha1 +kind: RenderRuntimeConfig +listener: bind: 127.0.0.1:8080 shutdownGraceSeconds: 30 -bundle: - path: /var/lib/registry-render/bundle +package: + # The directory written by `registry-render package`. + root: /var/lib/registry-render/package + # Optional: refuse to start on any other package digest. + # expectedDigest: sha256:<64 lowercase hex digits> +secretProviders: + file: + root: /run/secrets/registry-render + # environment: {} auth: apiKeyRef: secret:file/render-api-key limits: @@ -94,9 +103,18 @@ audit: retainDays: 90 ``` -Secret references resolve against files beside the runtime file or named environment variables. -Relative `bundle.path` and `audit.path` values anchor to the runtime file's directory too, so -the same file behaves identically wherever the process is launched from. +The runtime file path, `package.root`, `audit.path`, and `secretProviders.file.root` are +absolute, and the runtime file path must not pass through a symbolic link, so the same file behaves +identically wherever the process is launched from. `listener.bind` is required. A `secret:file/name` +reference resolves under `secretProviders.file.root`, and a `secret:env/NAME` reference only when +`secretProviders.environment: {}` is declared; a reference to an undeclared provider refuses +startup. String values may take a deployment value from the environment with `${VAR}`, +`${VAR:-default}`, or `${VAR:?message}`, except in a `*Ref` field, which must be written as a +literal secret reference, and under `secretProviders`, which must be written as it is meant. +`package.expectedDigest`, when set, is compared with the package digest. A mismatch uses the +same expected/found message as every other Registry Stack runtime. `GET /health` reports the +same digest without the `sha256:` label as `bundleHash` for compatibility with Render's response +and audit fields. `audit.destination` is `file` by default; set it to `stdout`, without `path`, `rotateBytes`, or `retainDays`, to hand the entries to your log collector instead. The limits carry hard ceilings the runtime refuses to exceed: the output cap cannot exceed 8 MiB @@ -114,15 +132,15 @@ drop the renders it exists to protect, and startup refuses both ends. Start the process with the runtime file, and check both endpoints before sending traffic: ```sh -target/release/registry-render serve --runtime /etc/registry-render/runtime.yaml & +target/release/registry-render serve --runtime-config /etc/registry-render/runtime.yaml & curl -s http://127.0.0.1:8080/health curl -s http://127.0.0.1:8080/ready ``` -`GET /health` answers with the bundle version and hash and the renderer version, so you can +`GET /health` answers with the bundle version and package hash and the renderer version, so you can reconcile the running service against what you deployed. `GET /ready` includes the audit writer: if it has stopped, readiness fails. Both are value-free. -`registry-render healthcheck --runtime ` performs the same probe for a supervisor script. +`registry-render healthcheck --runtime-config ` performs the same probe for a supervisor script. {/* Evidence: crates/registry-render/src/server.rs, health and healthcheck; crates/registry-render/src/openapi.rs, OPENAPI_JSON. */} @@ -192,7 +210,7 @@ files, or the `stdout` stream, to append-only storage before retention deletes t that held the older chained ledger is not an audit path to reuse: archive its segments first and point `audit.path` at a fresh directory. -`registry-render check --bundle --runtime --require-audit-under ` proves the +`registry-render check --bundle --runtime-config --require-audit-under ` proves the audit file resolves under a persistent root, as a container preflight, and refuses a `stdout` destination. @@ -207,8 +225,9 @@ destination. A Render binary is exact source plus the Typst pin plus the lockfile, and any change to the Typst pin, the compression stack, or a template changes output bytes by design. Treat every upgrade as a golden-hash review: the two-OS job in CI fails on any byte difference and the diff is the -review. Never overwrite a deployment's bundle content without resealing, because the seal, not -the directory listing, is what serve trusts. +review. Build every intentional change into a new output directory and switch the deployment to +that immutable package. The package writer refuses an existing output directory, so a published +digest never gains different bytes. {/* Evidence: .github/workflows/render-golden.yml; products/render/EVIDENCE.md. */} diff --git a/docs/site/src/content/docs/operate/relay.mdx b/docs/site/src/content/docs/operate/relay.mdx index 08a2c0c840..8a098ee4f6 100644 --- a/docs/site/src/content/docs/operate/relay.mdx +++ b/docs/site/src/content/docs/operate/relay.mdx @@ -51,12 +51,13 @@ checks. The runtime file names local deployment bindings and must not restate or override governed policy: ```yaml -apiVersion: relay.registrystack.org/v2alpha1 -kind: RelayRuntime -server: {bind: "127.0.0.1:8080"} -packagePath: /etc/relay/business/package +apiVersion: registry.registrystack.org/relay-runtime/v1alpha1 +kind: RelayRuntimeConfig +listener: {bind: "127.0.0.1:8080"} +package: {root: /etc/relay/business/package} +secretProviders: + file: {root: /run/secrets/relay} sources: {companies: {path: /srv/registries/business.sqlite}} -authentication: {issuer: null} audit: path: /var/lib/relay/business/audit.jsonl cursor: {integrityKeyRef: secret:file/cursor-integrity-key, maximumAgeSeconds: 300} @@ -64,9 +65,33 @@ limits: {requestTimeoutMilliseconds: 1500, concurrentQueries: 32} quotas: {requestsPerMinute: 120, burst: 20} ``` -`authentication.issuer: null` is valid only when every compiled access rule is public, including -Record access profiles and fixed statistical-dataset access rules. A package with either kind of -protected access needs the configured issuer at startup. +`package.root` is an absolute path. Add `package.expectedDigest` with the package digest +`relayctl package` reported, the `sha256:` digest of the package's `SHA256SUMS`, to refuse any +other package at that path. A `secret:file/` reference resolves +under `secretProviders.file.root`, never beside the runtime file; a `secret:env/` reference needs +`secretProviders.environment: {}`. Other values may use `${VAR}` or `${VAR:-default}` +substitution, but a field ending in `Ref` and every value under `secretProviders` refuse it. A removed key such as `server`, +`packagePath`, `audit.sink`, or `authentication.issuer` is refused with the name of its +replacement, and `audit.integrityKeyRef` with a diagnostic saying to remove it. + +A package with protected access declares its one issuer: + +```yaml +authentication: + oidc: + issuer: https://identity.example.org + audience: relay-business + tokenTypes: [at+jwt] + algorithms: [ES256] +``` + +`jwksSource` defaults to `kind: discovery`, which reads the issuer's +`/.well-known/openid-configuration`. Use `jwksSource: {kind: uri, uri: }` when the +keys are served elsewhere; token `iss` validation stays bound to `issuer` either way. + +Leaving `authentication.oidc` out is valid only when every compiled access rule is public, +including Record access profiles and fixed statistical-dataset access rules. A package with either +kind of protected access needs the configured issuer at startup. The issuer's verified claims may establish scopes, purpose, and row authority, but cannot enable an operation or access profile the package did not compile. A syntactically valid unknown access profile and a valid principal without its scope receive the @@ -79,8 +104,9 @@ order, selected fields, authorization context, optional bbox, wire format, forma expiry. Treat cursors as opaque continuation tokens even though they contain no plaintext filter, order, or bbox values. -`relay serve --runtime ` opens only the sealed package at `packagePath`. -It rejects unsafe paths, a package inventory mismatch, source-schema drift, missing mandatory audit +`relay serve --runtime-config ` opens only the sealed package at `package.root`. +It rejects unsafe paths, a changed, missing, or extra package file (naming the file and +`relayctl package`), source-schema drift, missing mandatory audit inputs, and incompatible runtime bindings before listening. There is no deployment switch that disables fail-closed audit behavior. @@ -123,7 +149,7 @@ change its time granularity, or add another statistical format or query. Start the process using the exact runtime file: ```sh -sudo -u /usr/local/bin/relay serve --runtime /etc/relay/business/runtime.yaml +sudo -u /usr/local/bin/relay serve --runtime-config /etc/relay/business/runtime.yaml ``` Relay reports its listener only after it verifies the package, source, issuer when configured, diff --git a/docs/site/src/content/docs/operate/retention-and-persistent-state.mdx b/docs/site/src/content/docs/operate/retention-and-persistent-state.mdx index cc01d65842..940b2bf32a 100644 --- a/docs/site/src/content/docs/operate/retention-and-persistent-state.mdx +++ b/docs/site/src/content/docs/operate/retention-and-persistent-state.mdx @@ -44,7 +44,7 @@ Two consequences follow from that boundary: | Base Registry Engine PostgreSQL database | The registry's own database. `registry_data` holds the current row of every record, `registry_internal` holds the retained revisions, history-erasure coverage, the webhook outbox, the change-request proposal and target snapshots, and the cached idempotency results, and `registry_source`, `registry_derived`, and `registry_context` hold views and functions the compiler generates. Records, their revisions, and proposal snapshots carry whatever the project's entities declare, personal data included. | Only a retained webhook payload expires on a schedule: `eventDelivery.payloadRetentionDays` in `runtime.yaml` defaults to `7` days and is capped at `30`, and the runtime clears a payload on the first successful delivery or once its expiry has passed. Revisions and proposal snapshots are kept until an operator command removes them. | Back up the database; Base Registry Engine writes no local copy of it. `bregctl history erase` and `request-retention erase` remove retained history and proposal detail; audit files need separate shipping and backup. [Retain, erase, and audit](../breg-retention/) states what each one destroys and what it leaves. | | Base Registry Engine audit files | Minimized request, response, refusal, and maintenance entries with keyed references, separate from the database. Runtime instances and operator companion commands have separate files. | The shared writer rotates at `audit.rotateBytes` (100 MiB by default) and deletes sealed files older than `audit.retainDays` (90 days by default) on open or rotation. | Give every process its own absolute path, and ship runtime and `bregctl` companion files before expiry. With `destination: stdout`, the collector owns durability, rotation, and retention, and `bregctl` companion commands write their entries to stderr so their own report keeps stdout; collect both streams. | | Render audit file | A `request` entry accepted before a render's worker starts and a `response` entry accepted before the response leaves, both sharing one `correlation`, a random id the server draws for every call. The caller's `Idempotency-Key` is recorded only as the record's `correlationId`, since two calls may carry the same key. Entries carry hashes, versions, the renderer and Typst pin, caller fingerprint, and trace/correlation ids, never record data values or asset bytes. | Render writes through the shared writer Evidence Gateway and Relay use. The active file rotates to `.` when an append would pass `audit.rotateBytes` (100 MiB by default), and sealed files last modified more than `audit.retainDays` ago (90 by default) are deleted when the writer opens or rotates. A `stdout` destination leaves rotation and retention to the log collector. | Configure `audit.destination`, `audit.path`, and optionally `audit.rotateBytes` and `audit.retainDays` in the runtime file. Ship sealed files off host before retention deletes them; the mechanics are identical to [Ship Evidence Gateway audit records off host](#ship-evidence-gateway-audit-records-off-host), since all three products share the same writer and file naming. | -| Operator-owned config, source, and secret paths | Relay's `runtime.yaml`, the sealed package at `packagePath`, and the bound SQLite source path; Evidence Gateway's governed bundle and runtime document; Base Registry Engine's signed package root, runtime file, project sources, module locks, generated files, test receipt, signature documents, and the `bregctl data` checkpoint, sidecar, and export output files on the operator host; secret references for all three. Source files and a `bregctl data export` output can contain personal data. | Registry Stack does not expire these files, except through the specific audit and cache mechanics listed for each store. | Mount source data read-only where possible; back up config, packages, and secrets through your platform controls. | +| Operator-owned config, source, and secret paths | Relay's `runtime.yaml`, the sealed package at `package.root`, the secret files under `secretProviders.file.root`, and the bound SQLite source path; Evidence Gateway's governed bundle and runtime document; Base Registry Engine's signed package root, runtime file, project sources, module locks, generated files, test receipt, signature documents, and the `bregctl data` checkpoint, sidecar, and export output files on the operator host; secret references for all three. Source files and a `bregctl data export` output can contain personal data. | Registry Stack does not expire these files, except through the specific audit and cache mechanics listed for each store. | Mount source data read-only where possible; back up config, packages, and secrets through your platform controls. | Relay V2 keeps no other durable, product-owned state beyond the audit file and the operator-owned paths in [Durable state and externally retained records](#durable-state-and-externally-retained-records): @@ -56,8 +56,8 @@ anti-rollback state, and no separate consultation database. Relay V1 had all fou | Store | What it can contain | Expiry or rotation | Operator control | |---|---|---|---| -| Relay OIDC JWKS cache | Issuer signing keys and negative lookup entries, not subject records. Applies only when `authentication.issuer` is configured; a package where every access profile is public can run with `authentication.issuer: null` and no cache at all. | In process only. Relay V2 exposes no cache-lifetime override, so the shared platform defaults apply: `600` second positive cache TTL and `60` second negative cache TTL. | Restart clears the cache. | -| Evidence Gateway OIDC JWKS cache | Issuer signing keys and negative lookup entries for the configured `authentication.jwksUri`, not subject records. | In process only. Evidence Gateway exposes no product-level override, so the shared platform defaults apply: `600` second positive cache TTL and `60` second negative cache TTL. | Restart clears the cache. An unreachable key set is retried and reported at a bounded interval rather than silently ignored. | +| Relay OIDC JWKS cache | Issuer signing keys and negative lookup entries, not subject records. Applies only when `authentication.oidc` is configured; a package where every access profile is public can run without it and with no cache at all. | In process only. Relay V2 exposes no cache-lifetime override, so the shared platform defaults apply: `600` second positive cache TTL and `60` second negative cache TTL. | Restart clears the cache. | +| Evidence Gateway OIDC JWKS cache | Issuer signing keys and negative lookup entries for the configured `authentication.oidc.jwksSource.uri`, not subject records. | In process only. Evidence Gateway exposes no product-level override, so the shared platform defaults apply: `600` second positive cache TTL and `60` second negative cache TTL. | Restart clears the cache. An unreachable key set is retried and reported at a bounded interval rather than silently ignored. | | Relay response headers | No stored response cache. Relay validates certain public snapshot responses with a strong `ETag` and answers a matching conditional request with `304 Not Modified`; every other response, including metadata, defaults to `Cache-Control: no-store`. | Not time-bound; validators change when the underlying package, source revision, or requested representation changes. | Not configurable. Behavior follows the source profile and access profile compiled into the package. | | Relay pagination cursors | Client-held, authenticated, and encrypted cursor payloads binding the source and contract revisions, operation, access profile, disclosure profile, filters, order, selected fields, and authorization context. No plaintext filter, order, or bbox value. | Not stored server-side. Each cursor carries an expiry checked against `cursor.maximumAgeSeconds`, which the runtime defaults to `300` seconds only when the whole `cursor` section is absent, and becomes invalid immediately if the bound request context no longer matches. Cursors apply to snapshot-source list and search operations alike; live read-only sources do not support pagination. | Configure `cursor.integrityKeyRef` and, optionally, `cursor.maximumAgeSeconds` in `runtime.yaml`. Treat cursor tokens as opaque, client-held request context. | diff --git a/docs/site/src/content/docs/reference/api-stability.mdx b/docs/site/src/content/docs/reference/api-stability.mdx index d89a05adb6..673e2b7ce5 100644 --- a/docs/site/src/content/docs/reference/api-stability.mdx +++ b/docs/site/src/content/docs/reference/api-stability.mdx @@ -46,7 +46,7 @@ The enforcement column names the repository CI checks so the mechanism is audita | Surface | Contract artifact | Enforcement today | | --- | --- | --- | | Relay HTTP API (the fixed route inventory and the response envelope it carries) | The closed route list built by `router` in `crates/registry-relay-v2/src/server.rs` and the generated-artifact contract `products/relay-v2/contracts/artifact-inventory.yaml`. Relay has no product-level OpenAPI document to pin, because the resources a document describes come from the adopter's own Registry contract; each deployment serves its own description at `GET /openapi.json`. See [API references](../apis/) | Root CI's `relay-v2-contracts` job runs `products/relay-v2/scripts/check-contracts.sh` and `products/relay-v2/scripts/test-http.sh`, which replay the four coequal acceptance projects under `products/relay-v2/acceptance/` against their recorded `expected-http.yaml` exchanges | -| Relay authoring grammar and sealed package format | The Registry contract grammar `relay.registrystack.org/v2alpha1`, covering `registry.yaml` (`kind: RegistryContract`) and `runtime.yaml` (`kind: RelayRuntime`), and the sealed package format `relay.registrystack.org/package/v1alpha3` described by `products/relay-v2/contracts/package-layout.yaml`. Both carry pre-1.0 version markers today; the promise attaches to the version each one carries at `v1.0.0` | `RegistryContract::parse_yaml` and `RelayRuntime::parse_yaml` reject unknown keys rather than ignoring them (`crates/registry-relay-v2/src/contract.rs`); `products/relay-v2/scripts/check-configs.sh` holds each acceptance project against its recorded contract revision, package revision, and artifact digests in `products/relay-v2/contracts/generated-baselines.yaml`; `relay serve` verifies a package before it activates | +| Relay authoring grammar and sealed package format | The Registry contract grammar `relay.registrystack.org/v2alpha1`, covering `registry.yaml` (`kind: RegistryContract`), the deployment binding grammar `registry.registrystack.org/relay-runtime/v1alpha1` covering `runtime.yaml` (`kind: RelayRuntimeConfig`), and the sealed package in the shared Registry Stack package format. The grammars carry pre-1.0 version markers today; the promise attaches to the version each one carries at `v1.0.0` | `RegistryContract::parse_yaml` and `RelayRuntime::parse_yaml` reject unknown keys rather than ignoring them (`crates/registry-relay-v2/src/contract.rs`); `products/relay-v2/scripts/check-configs.sh` holds each acceptance project against its recorded contract revision, package digest, and artifact digests in `products/relay-v2/contracts/generated-baselines.yaml`; `relay serve` verifies a package before it activates | | Evidence Gateway HTTP API and its Version 1 contracts | The frozen Version 1 source contracts under `products/evidence/contracts/`, indexed by `products/evidence/contracts/README.md`, and the artifacts generated from them under `products/evidence/generated/`, including `registry-evidence.openapi.json` | Root CI's `evidence-contracts` job runs `products/evidence/scripts/check-contracts.sh`, which regenerates every contract artifact and fails on any byte difference from the committed copies, plus `products/evidence/scripts/check-source-neutrality.sh` and `products/evidence/scripts/check-verifier-portability.sh` | | Error contract and stable identifiers | RFC 9457 problem shape with the stable `code` member, the [error registry](../errors/), and the `https://id.registrystack.org/` identifier space. Relay's set is closed by `ProblemCode` in `crates/registry-relay-http-contract/src/lib.rs`. Evidence Gateway's closed problem set has its own frozen contract, `products/evidence/contracts/problem-contract.yaml`, documented at [Evidence Gateway problem types](../evidence-problems/) | Relay's code, status, title, detail, and type URI come from one exhaustive catalog, and product tests pin the rendered body and headers; the `evidence-contracts` job regenerates and byte-diffs the committed Evidence Gateway problem schema against runtime-owned generation | | Configuration formats and documented environment variables | Relay's `registry.yaml` and `runtime.yaml` grammars, the frozen Evidence Gateway schemas `products/evidence/contracts/runtime.schema.yaml` and `products/evidence/contracts/bundle.schema.yaml`, plus the [environment variable reference](../environment-variables/) | Relay and Evidence Gateway both parse with `deny_unknown_fields`, so a retired or misspelled key is a startup refusal rather than an ignored field; Evidence Gateway's config parser is tested against the frozen contract schemas (`crates/registry-evidence/src/config.rs`), and `products/evidence/scripts/check-config-key-paths.sh` holds its configuration reference in exact parity with them | @@ -131,10 +131,11 @@ None of the three products this page covers, Relay, Evidence Gateway, and Regist migrated state or a migration command. Relay owns no database: it reads the adopter's SQLite source through a read-only boundary and writes the audit log named by the `audit` block in its runtime file. Evidence Gateway writes the audit log named by the `audit` -block in its runtime file. Relay's sealed package is an input to verify rather than state to migrate. `relay` reads the -package format version out of `relay-package.json` and refuses to activate a package whose version -string is not the one that binary was built against, so a package-format change is a recompile of -the authoring project rather than a conversion of an installed package. +block in its runtime file. Relay's sealed package is an input to verify rather than state to migrate. `relay` recompiles the +packaged contract and regenerates its artifacts at startup, and refuses to activate a package they +do not reproduce, so a package-format change is a repackage of the authoring project rather than a +conversion of an installed package. A directory that still carries `relay-package.json` instead of +`SHA256SUMS` is refused with a message naming `relayctl package`. Base Registry Engine, Casework, and Scheduling are separate runtime products outside this page's covered surfaces, and each owns migrated PostgreSQL state with its own migration command: @@ -195,7 +196,7 @@ The promise is machine-checked where a checker exists: - Relay: root CI's `relay-v2-contracts` job runs `products/relay-v2/scripts/check-contracts.sh`, which validates the product contracts, the SDMX profile lock, and source neutrality, then holds - each acceptance project against its recorded contract revision, package revision, and artifact + each acceptance project against its recorded contract revision, package digest, and artifact digests. The same job runs `products/relay-v2/scripts/test-http.sh`, which replays the recorded HTTP journeys for the four acceptance projects and the SDMX read profile against the runtime. - Evidence Gateway: root CI's `evidence-contracts` job regenerates every contract artifact from the code diff --git a/docs/site/src/content/docs/reference/breg-api.mdx b/docs/site/src/content/docs/reference/breg-api.mdx index 9b5ae6cee1..ad05a34064 100644 --- a/docs/site/src/content/docs/reference/breg-api.mdx +++ b/docs/site/src/content/docs/reference/breg-api.mdx @@ -73,6 +73,15 @@ same concealed 404 as an unauthorized request. crates/registry-breg/src/compiler.rs, route_shape() and query_kind_id(); crates/registry-breg/src/model.rs, MAX_REVISION_HISTORY_RECORDS. */} +A successful `GET /v1/registry` response carries a `Registry-Engine-Version` header with the +engine release that served it, the value `breg --version` prints. It grants nothing, and the +concealed 404 does not carry it. Casework reads it to hold its BReg sources to its own release, as +[Upgrade Casework and BReg in lock-step](../../operate/casework/#upgrade-casework-and-breg-in-lock-step) +describes. + +{/* Evidence: crates/registry-breg/src/api/mod.rs, registry_metadata(); + crates/registry-breg/tests/startup_http.rs. */} + ## Record envelope Reads, creates, patches, tombstones, revisions, and snapshots return one shape, the Registry @@ -284,7 +293,7 @@ A run response carries operational metadata and bounded failure classifications source rows, committed record values, or chunk bodies. {/* Evidence: crates/registry-breg/src/ingestion_store.rs, IngestionRunStatus and IngestionRefusal; - crates/registry-breg/src/data.rs, ingestion_batch_route() and ingestion_chunk_idempotency_key(); + crates/registry-breg/src/data.rs, ingestion_route() and ingestion_chunk_idempotency_key(); crates/registry-breg/src/api/mod.rs, router(). */} ### Create a run @@ -310,6 +319,19 @@ source as 64 lowercase hexadecimal characters. The chunking contract is `greedy-canonical-http-batch-v1`; a run refuses any other algorithm, so its remaining source bytes are never reinterpreted under a different chunking contract. +A profile that holds `import` on the entity announces `"operation": "create"` here; an import +grant has no other write route. Its run is created only while the entity holds an open import +authority for that profile: unexpired, opened under the active package revision, with remaining +volume that covers the whole announced `itemCount`, and, when the authority pins input digests, +listing this `inputDigest`. Otherwise the answer is `412 precondition.failed` and no run exists. +`inputDigest` is the caller's label for its source, recorded with the run; the server never +receives the source and does not recompute it, so a pinned digest does not prove which items the +chunks carry. +Every chunk of an import run rechecks the authority in its own transaction and counts its +committed items against the authority's volume. Operators open and close authorities with +`bregctl import-authority`; see +[Load a governed entity through an import window](../../operate/breg-data/#load-a-governed-entity-through-an-import-window). + ### The run document Every run response is one document: @@ -318,7 +340,7 @@ Every run response is one document: | --- | --- | | `runId` | The run identifier. | | `status` | `open`, `complete`, `cancelled`, or `blocked`. | -| `blockedReason` | Present when the run is `blocked`: `activePackageChanged`. | +| `blockedReason` | Present when the run is `blocked`: `activePackageChanged` or `importAuthorityClosed`. | | `entityId`, `operation`, `profileId` | The bound entity, its create-or-patch operation, and the selected profile. | | `packageRevision`, `schemaFingerprint` | The binding the run was created under. | | `inputDigest`, `inputLength`, `itemCount`, `chunkCount` | The announced source: digest, byte length, item count, and chunk count. | @@ -333,10 +355,15 @@ Every run response is one document: An open run whose binding no longer matches the active package reports `blocked` with `blockedReason` `activePackageChanged`. A blocked run is retained and inspectable, and it never reinterprets its remaining bytes under a new binding: continue in a successor run created -under the new package. +under the new package. An import run whose authority was closed, expired, exhausted, or +superseded by a package activation reports `blocked` with `blockedReason` `importAuthorityClosed` +at the next chunk, which commits nothing and answers `409 ingestion.run_blocked`; continue the +uncommitted remainder in a successor run under a new authority. `lastAttempt.outcome` classifies the last submission as `committed`, `replayed`, `invalidItem`, -`refused`, `bindingChanged`, `chunkMismatch`, `runNotOpen`, or `unavailable`. `invalidItem` and +`refused`, `bindingChanged`, `importAuthorityClosed`, `chunkMismatch`, `runNotOpen`, or +`unavailable`. `bindingChanged` is the attempt that blocked a run on a package activation, and +`importAuthorityClosed` the attempt that blocked it on its import authority. `invalidItem` and `refused` leave the checkpoint where it was; the corrective move is a successor run, not a retry with different bytes and not a skipped row. @@ -386,7 +413,7 @@ re-sending the chunk. A receipt is erased with the record history it describes; then answers `410 ingestion.receipt_erased`, and the chunk's counts stay in the run. {/* Evidence: crates/registry-breg/src/ingestion_store.rs, StoredChunkReceipt and IngestionBlockedReason; - crates/registry-breg/src/data.rs, ingestion_batch_route(). */} + crates/registry-breg/src/data.rs, ingestion_route(). */} ### Recovery @@ -402,7 +429,7 @@ then answers `410 ingestion.receipt_erased`, and the chunk's counts stay in the A run drives the same compiled batch route one chunk at a time, so an existing direct batch client keeps its contract unchanged. -{/* Evidence: crates/registry-breg/src/data.rs, ingestion_batch_route(). */} +{/* Evidence: crates/registry-breg/src/data.rs, ingestion_route(). */} ## History reads diff --git a/docs/site/src/content/docs/reference/contracts.mdx b/docs/site/src/content/docs/reference/contracts.mdx index 79337a22d6..f54941f227 100644 --- a/docs/site/src/content/docs/reference/contracts.mdx +++ b/docs/site/src/content/docs/reference/contracts.mdx @@ -76,12 +76,13 @@ served by that deployment at `GET /openapi.json`. See Two Relay surfaces belong in this table instead of an OpenAPI artifact: - The Registry contract grammar, `relay.registrystack.org/v2alpha1`, covering `registry.yaml` - (`kind: RegistryContract`) and `runtime.yaml` (`kind: RelayRuntime`). Both are closed shapes: an + (`kind: RegistryContract`) and `runtime.yaml` (`apiVersion: + registry.registrystack.org/relay-runtime/v1alpha1`, `kind: RelayRuntimeConfig`). Both are closed shapes: an unrecognized key is a parse failure rather than an ignored field. Compatibility for recognized keys follows the release policy. -- The sealed package format, `relay.registrystack.org/package/v1alpha3`, whose manifest is - `relay-package.json`. `relayctl package` produces it and `relay serve` verifies it before - activation. Its `packageRevision` is an integrity digest over canonical JSON, not an +- The sealed package format: a directory whose `SHA256SUMS` file lists the digest of every other + file. `relayctl package` produces it and `relay serve` verifies it before activation. Its + package digest, the `sha256:` digest of `SHA256SUMS`, is an integrity digest, not an authenticity proof: packages are unsigned, so the operator's own transport and storage are what bind a package to its author. @@ -146,5 +147,6 @@ authority that runs the registry. ## Compatibility boundaries -- Relay's authoring grammar and sealed package format are versioned `v2alpha1` and `v1alpha3`. Both - are pre-1.0 and can change with a documented migration in a release note. +- Relay's authoring grammar is versioned `v2alpha1`, and its sealed package follows the shared + Registry Stack package format. Both are pre-1.0 and can change with a documented migration in a + release note. diff --git a/docs/site/src/content/docs/reference/environment-variables.mdx b/docs/site/src/content/docs/reference/environment-variables.mdx index dd305f19b1..4cd4368251 100644 --- a/docs/site/src/content/docs/reference/environment-variables.mdx +++ b/docs/site/src/content/docs/reference/environment-variables.mdx @@ -7,7 +7,7 @@ source_repos: - registry-evidence - registry-relay - registry-stack -last_reviewed: "2026-08-11" +last_reviewed: "2026-09-25" doc_type: reference locale: en standards_referenced: [] @@ -17,11 +17,13 @@ This page lists Registry Stack's supported fixed environment-variable interfaces adopter tooling, and installers. It also records how Relayctl keeps fixture scratch state independent of host temporary-directory configuration. -The stack's other kind of environment variable is operator-named. Secret material such as a cursor integrity key is not read from a fixed variable name: a configuration field carries a reference that names the variable, and the operator chooses the name. Relay and Base Registry Engine are the products that work that way, and the reference grammar is closed rather than free-form. Evidence Gateway reads no secret from the environment under either kind of name, as its section records. +The stack's other kind of environment variable is operator-named. Secret material such as a cursor integrity key is not read from a fixed variable name: a configuration field carries a reference that names the variable, and the operator chooses the name. Relay, Base Registry Engine, and Evidence Gateway work that way, and the reference grammar is closed rather than free-form. Each reads a secret from an operator-named variable only when its runtime file opts in to the environment provider, as its section records. ## Configuration expansion -Relay and Evidence Gateway expand no environment reference inside their configuration documents. Each parses its documents as written: Relay in `RelayRuntime::parse_yaml` and `RegistryContract::parse_yaml` (`crates/registry-relay-v2/src/contract.rs`), and Evidence Gateway in `RuntimeConfig::parse_yaml` (`crates/registry-evidence/src/config.rs`). Base Registry Engine is the exception: `breg` expands `${NAME}` expressions in its runtime file at load, as [its section](#base-registry-engine) records. +Evidence Gateway substitutes `${VAR}` and `${VAR:-default}` in string values of its runtime file after parsing, in `RuntimeConfig::parse_yaml` (`crates/registry-evidence/src/config.rs`), and refuses a `${...}` expression in a secret reference or under `secretProviders`, and refuses any `${...}` expression in the governed bundle, so the reviewed bundle is the one that runs. Relay substitutes `${VAR}` and `${VAR:-default}` in string values of its runtime file after parsing, in `RelayRuntime::parse_yaml` (`crates/registry-relay-v2/src/contract.rs`), and refuses any `${...}` expression in the authored `registry.yaml` in `RegistryContract::parse_yaml`, so the reviewed contract is the one that runs. Base Registry Engine substitutes `${VAR}`, `${VAR:-default}`, and `${VAR:?message}` in string values of its runtime file after parsing, through the shared configuration loader, and refuses any `${...}` expression in the authored project or module files, as [its section](#base-registry-engine) records. + +There is no escape for a literal `${` in a runtime file that substitutes. Put the literal text in a variable and reference that variable: a substituted value is not expanded again. ## Relay @@ -29,7 +31,6 @@ The `relay` binary reads these variables. | Name | Purpose | Default or required | | --- | --- | --- | -| `RELAY_RUNTIME` | Path to the deployment binding that names the sealed package, the local sources, authentication, audit, and limits. Equivalent to `--runtime` on `relay check` and `relay serve`. | Required for `relay serve`, by flag or by variable. `relay check` defaults to `/etc/relay/runtime.yaml`. | | `RELAY_HEALTHCHECK_URL` | Complete HTTP or HTTPS URL of the unauthenticated `/health` endpoint to probe. Equivalent to the `--url` flag on `relay healthcheck`. | Defaults to `http://127.0.0.1:8080/health`. | | `RELAY_LOG` | Level for the JSON operational records the process writes on standard error. | Defaults to `info`. | @@ -37,14 +38,14 @@ The `relay` binary reads these variables. ### Secret references in `runtime.yaml` -Relay resolves secrets through two providers, environment and file, rooted at the directory holding the runtime file. One field takes a reference: `cursor.integrityKeyRef`, which applies when the deployment enables cursors. +Relay reads no variable for the runtime path: `relay check` and `relay serve` take it as the required absolute `--runtime-config ` flag. Relay resolves secrets through the providers `runtime.yaml` declares under `secretProviders`: `environment: {}` enables `secret:env/` references and `file: {root: }` enables `secret:file/` references. One field takes a reference: `cursor.integrityKeyRef`, which applies when the deployment enables cursors. Other runtime values may use `${VAR}` or `${VAR:-default}` substitution; a field ending in `Ref` and every value under `secretProviders` refuse it, so the environment cannot choose which secret is read or where it comes from. | Grammar | Resolves to | Accepted name | | --- | --- | --- | -| `secret:env/` | The value of the environment variable `` | Starts with an uppercase ASCII letter, then uppercase ASCII letters, digits, or `_`, up to 128 characters | -| `secret:file/` | A file named `` under the runtime file's directory | Starts with a lowercase ASCII letter, then lowercase ASCII letters, digits, `.`, `_`, or `-`, up to 128 characters | +| `secret:env/` | The value of the environment variable ``, when `secretProviders.environment` is declared | Starts with an uppercase ASCII letter, then uppercase ASCII letters, digits, or `_`, up to 128 characters | +| `secret:file/` | A file named `` under `secretProviders.file.root` | Starts with a lowercase ASCII letter, then lowercase ASCII letters, digits, `.`, `_`, or `-`, up to 128 characters | -A reference that matches neither grammar makes the runtime document invalid, so the process refuses to start rather than serving with an unresolved secret. The variable names themselves are the operator's choice and appear nowhere in Relay's source. See [Configure Relay](../../configure/relay/) for the fields around them. +A reference that matches neither grammar, or names a provider the document does not declare, makes the runtime document invalid, so the process refuses to start rather than serving with an unresolved secret. The variable names themselves are the operator's choice and appear nowhere in Relay's source. See [Configure Relay](../../configure/relay/) for the fields around them. ### Relay installer @@ -75,20 +76,29 @@ governed behavior. ## Evidence Gateway -The runtime-path option is global. `EVIDENCE_LOG` is read by `evidence serve` only. +`evidence` reads one fixed variable, `EVIDENCE_LOG`, and only in `evidence serve`. | Name | Purpose | Default or required | | --- | --- | --- | -| `REGISTRY_EVIDENCE_RUNTIME` | Absolute path to the one operator runtime file that binds the governed bundle. Equivalent to the global `--runtime` flag. | Defaults to `/etc/registry-evidence/runtime.yaml`. | | `EVIDENCE_LOG` | Tracing filter for the operational records the serving process writes as line-delimited JSON on standard output. | Defaults to `info`. Read by `evidence serve` only; offline commands install no log subscriber. | -Evidence Gateway reads no secret from an environment variable. A configured secret reference uses -one grammar, `secret:file/`, enforced by `SecretRef::parse` in -`crates/registry-evidence/src/config.rs`, and resolves through `registry-platform-config` to an -owner-only regular file under the `secretProviders.file.root` directory named in `runtime.yaml`. -Neither the runtime file nor the governed bundle can name an environment variable to read a -credential from. See [Configure Evidence Gateway](../../configure/evidence/) for the runtime file -and the bundle it binds. +Evidence Gateway reads no variable for the runtime path. Every subcommand that reads the runtime +file takes it as the required `--runtime-config ` flag after the subcommand, for example +`evidence check --runtime-config /etc/registry-evidence/runtime.yaml`. The global `--runtime` flag +and the `REGISTRY_EVIDENCE_RUNTIME` variable are removed: `evidence` refuses either one at startup +and names `--runtime-config` as the replacement, rather than silently ignoring it. + +### Secret references + +Evidence Gateway resolves secrets through the providers `runtime.yaml` declares under +`secretProviders`: `file: {root: }` enables `secret:file/` references, which +resolve to an owner-only regular file under that directory, and `environment: {}` enables +`secret:env/` references, which resolve to the operator-named environment variable. The +grammars and accepted names are the ones in [the Relay table](#secret-references-in-runtimeyaml). +A reference in the runtime file or the governed bundle that names a provider the runtime file does +not declare is refused at startup, so an environment-backed secret is always an explicit operator +choice. See [Configure Evidence Gateway](../../configure/evidence/) for the runtime file and the +bundle it binds. ## Evidencectl @@ -130,7 +140,7 @@ The `breg` binary reads one fixed variable. ### Expansion and secret references in `runtime.yaml` -`breg` expands `${NAME}` expressions in `runtime.yaml` before parsing it. A bare `${NAME}` refuses the file when the variable is unset or empty, `${NAME:-fallback}` substitutes the fallback in that case, and `${NAME:?message}` refuses the file with that message. A value that fills a whole YAML scalar is inserted as a quoted scalar. A value embedded in a longer scalar is refused when it carries anything YAML could read as structure: a line break, a quote or backtick, a brace or bracket, a comma, `|` or `>`, a colon followed by a space, a space followed by `#`, or a leading `#`, `&`, `*`, `!`, `%`, `@`, `---`, or `...`. A refused expansion stops the process at startup. +`breg` substitutes `${NAME}` expressions in the string values of `runtime.yaml` after parsing it, so a substituted value is always text: it cannot add a key, change a number or boolean, or carry YAML structure, and a line break inside it stays part of the string. A bare `${NAME}` refuses the file when the variable is unset or empty, `${NAME:-fallback}` substitutes the fallback in that case, and `${NAME:?message}` refuses the file without repeating the message. A field whose name ends in `Ref` and every value under `secretProviders` refuse an expression, so the environment cannot choose which secret is read or where it comes from. The authored project and module files refuse any `${...}` expression, so the reviewed package is the one that runs. A refused substitution stops the process at startup. Secret references use the same two grammars as Relay: `secret:env/` names an operator-chosen environment variable and `secret:file/` a file under the `secretProviders.file.root` directory. `secret:env/` resolves only when `runtime.yaml` declares `secretProviders.environment`; without that declaration the reference is refused. See [Deploy a registry](../../operate/breg/#write-the-runtime-configuration) for the fields that take a reference. @@ -162,8 +172,8 @@ preserves the previous set. It does not verify release authenticity. The fixed environment variable names in this reference are transcribed from the CLI definitions, binary entry points, and install scripts. Relay CLI ownership is in `crates/registry-relay-v2/src/cli.rs`, with logging in `main.rs`, secret grammar in `contract.rs`, and resolution in `startup.rs`. Evidence CLI ownership is in -`crates/registry-evidence/src/cli.rs`, logging in `main.rs`, and file-secret resolution in -`registry-platform-config`. Evidencectl binary selection is in +`crates/registry-evidence/src/cli.rs`, logging in `main.rs`, runtime-file substitution and the +secret grammar in `config.rs`, and secret resolution in `registry-platform-config`. Evidencectl binary selection is in `crates/registry-evidencectl/src/evidence_binary.rs`; its installer is `crates/registry-evidencectl/install.sh`. OID4VCI CLI ownership is in `crates/registry-evidence-oid4vci/src/cli.rs`, with logging and dispatch in `main.rs`. Base Registry Engine logging is in diff --git a/docs/site/src/content/docs/reference/evidence-configuration.mdx b/docs/site/src/content/docs/reference/evidence-configuration.mdx index e21077254d..7671f12440 100644 --- a/docs/site/src/content/docs/reference/evidence-configuration.mdx +++ b/docs/site/src/content/docs/reference/evidence-configuration.mdx @@ -120,7 +120,7 @@ the authoring keys in the context of a whole project. Start with complete authoring and deployment path. A key path that this page lists is not a deployment that runs. `evidence check` is the authoritative -runtime acceptance gate. `evidencectl artifact inspect ` reports artifact custody, and +runtime acceptance gate. `evidencectl artifact inspect ` reports custody for the target and installed package, and `evidencectl doctor --runtime-config ` performs the live startup dependency preflight without opening the public listener or sending an Evidence request. `evidencectl test` invokes the real Evidence checks before evaluating fixtures offline. diff --git a/docs/site/src/content/docs/reference/evidencectl.mdx b/docs/site/src/content/docs/reference/evidencectl.mdx index 1d43767ec4..6d6af85802 100644 --- a/docs/site/src/content/docs/reference/evidencectl.mdx +++ b/docs/site/src/content/docs/reference/evidencectl.mdx @@ -33,17 +33,40 @@ The canonical adopter commands do not change the numeric Evidence Gateway Versio | `evidencectl check --target ` | Editable project and one explicit target | Adds the target's governance, runtime structure, public keys, source connections, and governed-bundle validation | Require target-host paths or secrets to exist, run fixtures, or contact a dependency | | `evidencectl check --target --production` | Editable project and one explicit production or evidence-grade target | Refuses incomplete deployment closure under that target's own assurance profile | Select a target, upgrade its profile, or prove live readiness | | `evidencectl explain [--target ]` | Editable project and optional explicit target | Applies the same offline authoring validation, then reports status, findings, revision, and the authored inventory; includes target governance only when selected | Contact sources or expose secret and subject values | -| `evidencectl package --target --output ` | Editable project and one explicit production target | Creates a new candidate with `runtime.yaml` and closed `bundle/` | Overwrite output, contact source services or OIDC, create production secrets, or start a listener | -| `evidencectl test ` | A deployment project with `runtime.yaml`, or an editable project with `questions/` and `sources/`, plus referenced synthetic fixtures | Uses runtime `check` and `evaluate` for a deployment; privately compiles an editable project, then uses runtime `bundle-check` and `bundle-evaluate` | Start HTTP, call a source, or write a production audit entry | -| `evidencectl test --explain` | The same, and asks each evaluation to explain itself | Relays each fixture's stage trace beside its step, or as that fixture's `trace` field under `--format json` | Print a response, fact, derived, or selector value, or explain a served request | +| `evidencectl package --target --output ` | Editable project and one explicit production target | Creates one closed package with `SHA256SUMS`; runtime remains in the target | Overwrite output, contact source services or OIDC, create production secrets, or start a listener | +| `evidencectl test [--target ]` | An editable project with `questions/`, `sources/`, and referenced synthetic fixtures; an optional target supplies complete deployment governance | Privately compiles the project, then uses runtime `bundle-check` and `bundle-evaluate` | Start HTTP, call a source, or write a production audit entry | +| `evidencectl test --target --explain` | The same, and asks each evaluation to explain itself | Relays each fixture's stage trace beside its step, or as that fixture's `trace` field under `--format json` | Print a response, fact, derived, or selector value, or explain a served request | | `evidencectl doctor --runtime-config ` | Runtime file on its target host | Runs the runtime-owned startup dependency preflight without opening the public listener | Send an Evidence request or establish that a fixture passed | -| `evidencectl artifact inspect ` | Completed candidate | Inspects artifact custody without contacting dependencies | Establish live dependency readiness | +| `evidencectl artifact inspect ` | Deployment target whose runtime names the installed package | Inspects artifact custody without contacting dependencies | Establish live dependency readiness | + +`doctor --runtime-config` opens the configured audit destination as startup does, so it refuses +while another Evidence instance holds that destination's single-writer lock. To check a candidate +staged beside the instance it will replace, add `--without-audit-lock`: the audit hash key is +checked as startup checks it, and the audit directory and files are checked for ownership, mode, +write access, and a complete final entry. The lock stays with the running writer, so a second +writer is not detected. The JSON `proofBoundary` states this. Every other dependency is checked in +the same way. The refusal for a held lock names this option. `package` is create-only. It rejects an existing output directory, unauthenticated or non-HTTPS production HTTP sources, source transports with no stated production conditions, missing governance metadata or fixtures, unresolved review markers, unknown fields, symlink traversal, and references outside the allowed project directories. A failed package publishes no candidate. +`check`, `test`, `package`, `source diff`, and `source update`, plus the compatibility spelling +`fixtures run`, refuse a derivation whose `answer` reads a fact its question's source +does not declare, with `evidence.authoring.derivation-fact-undeclared` against the derivation +file. A source declares an inline operation's `source.facts[].name`, or the properties of a +referenced source's closed `factSchema`. A registry field rename that a derivation still reads +under its old name therefore fails before `source update` installs anything. The check reads the +project, not the running registry: a candidate already deployed against a registry whose field is +then renamed still fails every request that reads it while `/ready` reports ready. Only literal +reads on the first `answer` parameter, such as `facts["status"]` and `facts.status`, are checked. +The operands of one `??` fallback are read together, so `facts.new ?? facts.old` passes while +either name is declared. A computed key such as `facts[key]` is not a literal read and is how a +derivation reads a fact the check should not see. It, a read inside a helper function, an +`answer` that rebinds or writes its first parameter, and an open fact schema are left to the +fixtures. + An ordinary `check` can succeed with `status: incomplete` and visible findings. Add `--deny-findings` when any finding must refuse the command. All findings carry `severity`, `code`, `artifact`, `path`, `message`, and `suggestedAction`. Human output is the default. Add the global @@ -58,14 +81,17 @@ they do not change the Evidence Gateway runtime verifier's frozen exit contract. {/* Evidence: crates/registry-evidencectl/src/lib.rs, Cli, Command, OutputFormat, run_check_command(), and run_explain_command(); crates/registry-evidencectl/src/check.rs, check() and explain(); - crates/registry-evidencectl/src/runtime.rs, DoctorArgs and run(). */} + crates/registry-evidencectl/src/runtime.rs, DoctorArgs and run(); + crates/registry-evidencectl/src/authoring.rs, read_inputs() and declared_fact_names(); + crates/registry-evidence-authoring/src/derivation.rs, validate_answer_fact_reads(). */} ## Compatibility spellings -The released `new`, `fixtures run`, and `build` spellings remain available during the command -transition. New projects, examples, and current procedures use `init`, `test`, and `package`. -The `doctor --project ` compatibility form retains artifact inspection; -use `artifact inspect ` for that operation and reserve `doctor --runtime-config` for +The released `new` and `fixtures run` spellings remain available during the command +transition. The retired `build` spelling refuses the request and names `package` as its replacement. +New projects, examples, and current procedures use `init`, `test`, and `package`. +The `doctor --project ` compatibility form retains artifact inspection; +use `artifact inspect ` for that operation and reserve `doctor --runtime-config` for live startup dependency checks. Do not combine a canonical positional project with its former `--project` flag; conflicting old and new forms are usage errors. Request preparation uses `--response-format signed-jws|sd-jwt-vc` so the diff --git a/docs/site/src/content/docs/reference/glossary.mdx b/docs/site/src/content/docs/reference/glossary.mdx index 5b54bc4a1f..5926b0b1fe 100644 --- a/docs/site/src/content/docs/reference/glossary.mdx +++ b/docs/site/src/content/docs/reference/glossary.mdx @@ -51,7 +51,7 @@ Product names are always in English, including on future translated pages.
A named pairing of one access rule and one disclosure profile on a Registry Relay resource. The access rule is either `public` or a protected rule naming the required scope and, optionally, a purpose constraint and an authority row binding. A caller selects a profile with the `accessProfile` request parameter, and the profile it selects is both the authorization decision and the maximum disclosure the response may carry: no parameter, header, or token claim widens it. Statistical datasets carry one fixed access rule instead and do not accept the parameter. Declared in `registry.yaml` (`crates/registry-relay-v2/src/contract.rs`). Base Registry Engine uses the same two words and the same `accessProfile` request parameter for a different object: a named grant inside the registry project stating which scopes, purposes, and claims a token must carry and which operations, fields, and rows the caller may reach. See Control access per profile.
access token
-
The bearer credential a caller presents to Evidence Gateway. Evidence Gateway runs one authentication kind, `oidc-access-token`, whose issuer, audiences, accepted token types, algorithms, JWKS URI, and claim names are fixed in the immutable bundle, and it reads authority only from those configured claim names. Any issuer may be used only when it satisfies that complete configured token profile and signs with a key accepted from the configured JWKS; matching claim names alone is insufficient.
+
The bearer credential a caller presents to Evidence Gateway. Evidence Gateway accepts access tokens from one OIDC issuer, `authentication.oidc`, whose issuer, single audience, accepted token types, algorithms, fixed JWKS URI, and claim names are fixed in the immutable bundle, and it reads authority only from those configured claim names. Any issuer may be used only when it satisfies that complete configured token profile and signs with a key accepted from the configured JWKS; matching claim names alone is insufficient.
accountability record
The protected state Registry Casework keeps beside a terminal review decision: the deciding person's issuer-qualified identity, profile, private reason, and result digest, held for the review kind's `accountabilityDays`. Producer-visible results carry none of that identity or private reason. An authorized Supervisor resolves one event through the separate accountability route, and that read is audited. See Retain, erase, and settle.
@@ -105,7 +105,7 @@ Product names are always in English, including on future translated pages.
In Registry Casework, a policy rule that says when work is late. A subject clock measures a whole request from a source timestamp to a source event and can pause while the source waits on someone outside the team. An activity clock measures one stage in working days against a calendar and can record a reminder or, when due, release the holder and move the item, under the system actor, after a fresh source read. A clock decides no outcome and sends nothing outward. A project declares at most 32 clocks.
`configurationRevision`
-
The `sha256:` digest an Evidence Gateway assertion carries for the configuration and artifact closure that requirement's evaluation reached (`products/evidence/contracts/evidence.schema.yaml`). It is scoped to the requirement, not to the whole deployment, so an edit that cannot change an assertion leaves that assertion's revision alone. A signing-key rotation or revocation leaves it alone too, because key trust travels in the JWKS and the verification policy's denylist, and so does revoking a caller token key, which decides who may call rather than what an assertion means. Changing container paths or listener bindings moves the runtime revision instead.
+
The `sha256:` digest an Evidence Gateway assertion carries for the package configuration and artifact closure that requirement's evaluation reached (`products/evidence/contracts/evidence.schema.yaml`). It is scoped to the requirement, not to the whole package, so an edit that cannot change an assertion leaves that assertion's revision alone. A signing-key rotation or revocation leaves it alone too, because key trust travels in the JWKS and the verification policy's denylist, and so does revoking a caller token key, which decides who may call rather than what an assertion means. Runtime-only path or listener changes leave both the package digest and requirement revisions unchanged.
consultation
Registry Relay's Record-oriented read surface: the operations that answer a question about governed Registry Records, as against the SDMX statistical surface. One consultation follows one of three patterns, `list`, `retrieve`, or `search`. Consultation-specific refusals use `consultation.invalid_request`, `consultation.denied`, `consultation.unresolved`, `consultation.response_too_large`, or `consultation.rate_limited`. Shared request, authentication, resource, format, source, audit, service, and internal failures retain their own code families. Consultations belong to Relay and are not an Evidence Gateway source contract.
@@ -141,7 +141,7 @@ Product names are always in English, including on future translated pages.
An operated set of Registry Relay or Evidence Gateway product instances. Each product stages and activates its own separately verified configuration; there is no cross-product activation coordinator, and this is not atomic project activation.
deployment project
-
The directory an Evidence Gateway operator mounts: a `runtime.yaml` file holding process-local bindings, and a `bundle/` directory holding the governed configuration, scripts, schemas, codelists, and fixtures. It is the compiled output an operator deploys, not the thing an author edits: that is the authoring workspace, and `evidencectl package` compiles one workspace and one explicit target into a production candidate. Evidence Gateway loads a completed bundle read-only at startup; a new revision is a new deployment, not a live change.
+
An Evidence Gateway input form used by deployment inspection and fixture tooling: a directory with `runtime.yaml` whose `package.root` names the installed package. The current authoring flow keeps these inputs separate: `evidencectl package` compiles an authoring workspace and explicit target into one closed package, while the target retains `runtime.yaml`. Evidence Gateway verifies the package and captures the runtime read-only at startup; a new package is a new deployment, not a live change.
decision owner
The institution accountable for the requirements, rules, decisions, and actions that use evidence. The decision owner can operate the evidence consumer directly or rely on a separate caller or intermediary.
@@ -260,8 +260,8 @@ Product names are always in English, including on future translated pages.
package
The compiled and verified form of one Base Registry Engine registry project: `bregctl` builds it, `breg` serves it, and a production package carries detached signatures from keys the deployment's trust anchor names. `breg` verifies the active package at startup and answers only the routes that package compiled. Registry Relay's counterpart is the sealed package, which `relayctl` builds and which is unsigned.
-
package revision
-
The `packageRevision` member of a sealed package's `relay-package.json`: a digest over the canonical JSON of the manifest without that member. It identifies the exact bytes of one sealed package, so two operators can confirm they run the same reviewed contract. It is an integrity digest, not an authenticity proof; Relay packages are unsigned, and the operator's own transport and storage are what bind a package to its author.
+
package digest
+
The `sha256:` digest of a package's `SHA256SUMS` file, which lists the digest of every other file in the package. Every product that serves a package reports this value and accepts it as `package.expectedDigest`, so two operators can confirm they run the same reviewed bytes. It is an integrity digest, not an authenticity proof: Relay packages are unsigned, and the operator's own transport and storage are what bind a package to its author.
PROV-O
W3C Provenance Ontology. Currently listed as design influence (`inspired_by`). Provenance-shaped concepts appear in audit fields, but no PROV-O vocabulary terms are emitted as JSON-LD at reviewed commits.
@@ -309,7 +309,7 @@ Product names are always in English, including on future translated pages.
The Base Registry Engine maintenance run that restores snapshot coverage after a history erasure, by proving every live row still matches its retained journal head and installing one baseline commit at that head.
policy package
-
In Registry Casework, the directory `caseworkctl package` writes from a verified project: the policy file, the exact imported source descriptions, and a manifest of their digests. The `casework` runtime rebuilds the manifest at startup and refuses to serve a package with a changed byte, a missing file, or an extra file, so the package is what an operator deploys and what a policy change replaces whole.
+
In Registry Casework, the directory `caseworkctl package` writes from a verified project: the policy file, the exact imported source descriptions, and a `SHA256SUMS` file listing their digests. The `casework` runtime rechecks every digest at startup and refuses to serve a package with a changed byte, a missing file, or an extra file, so the package is what an operator deploys and what a policy change replaces whole.
record context
The four bindings every Registry Relay resource declares so a caller can tell one answer from another over time: the record identifier, the revision identifier, the lifecycle state with the codelist that constrains it, and the recorded-at instant. Record context is required, not optional metadata: a resource cannot be published without it, and it is what lets a consumer say which version of a record it acted on.
@@ -321,7 +321,7 @@ Product names are always in English, including on future translated pages.
The coordinated inbox and unified-review product in this monorepo: authorized human teams claim, draft, and decide source work items or review requests from admitted producers. It verifies bearer tokens and issues none, requires the issuer's human-identity assertion for human roles, and leaves source eligibility, visibility, and mutation with the source. Crates: `crates/registry-casework` (runtime, binary `casework`) and `crates/registry-caseworkctl` (adopter tooling, binary `caseworkctl`); client module `casework` in the unified clients. Short form on a page after first use: Casework. See the Registry Casework overview.
Registry Discovery
-
The read-only public index in this monorepo. A catalog operator maintains provider URLs and evidence-type mappings for the Evidence Gateway and Registry Relay providers that operator has chosen to index, `discoveryctl check` and `discoveryctl build` produce one immutable index offline, and the `discovery` binary serves that index until it is restarted with a newer one. Discovery carries inert public metadata: it makes no authorization decision, serves no record, and answers no question. Crate: `crates/registry-discovery`; binaries `discovery` and `discoveryctl`; repo slug: `registry-discovery`. Write `Registry Discovery` on a page's first use and `Discovery` after it. See Build and run a Registry Discovery index.
+
The read-only public index in this monorepo. A catalog operator maintains provider URLs and evidence-type mappings for the Evidence Gateway and Registry Relay providers that operator has chosen to index, `discoveryctl check` and `discoveryctl package` produce one immutable index package offline, and the `discovery` binary serves that package until it is restarted with a newer one. Discovery carries inert public metadata: it makes no authorization decision, serves no record, and answers no question. Crate: `crates/registry-discovery`; binaries `discovery` and `discoveryctl`; repo slug: `registry-discovery`. Write `Registry Discovery` on a page's first use and `Discovery` after it. See Package and run a Registry Discovery index.
registry document
Disambiguation. Base Registry Engine and Registry Relay each compile a file named `registry.yaml`, and the two grammars are unrelated. Relay's is the Registry contract (`kind: RegistryContract`), compiled by `relayctl`; BReg's is the root of a registry project, compiled by `bregctl`, and declares entities, fields, relationships, access profiles, change requests, and events. Both use the words access profile for different objects. Where both products are in view, prose writes the BReg registry document and the Relay registry document. A snippet copied from the other product's page does not compile.
@@ -333,7 +333,7 @@ Product names are always in English, including on future translated pages.
Shared Rust crates for registry security and operational primitives, including auth helpers, OIDC verification, audit envelopes, HTTP security, outbound HTTP policy, crypto, SD-JWT VC helpers, and test fixtures. Their APIs are workspace-internal and not published compatibility contracts.
Registry contract
-
The document an institution authors to describe its own registry: `registry.yaml`, `kind: RegistryContract`, `apiVersion: relay.registrystack.org/v2alpha1`. It names the resources, their properties and classifications, the operations Registry Relay will answer, the access and disclosure profiles that bound each answer, and any statistical datasets. It is a closed shape: an unrecognized key is a parse failure, not an ignored field. Its deployment-local half, `runtime.yaml` (`kind: RelayRuntime`), stays separate so the reviewed contract carries no secret and no local path.
+
The document an institution authors to describe its own registry: `registry.yaml`, `kind: RegistryContract`, `apiVersion: relay.registrystack.org/v2alpha1`. It names the resources, their properties and classifications, the operations Registry Relay will answer, the access and disclosure profiles that bound each answer, and any statistical datasets. It is a closed shape: an unrecognized key is a parse failure, not an ignored field. Its deployment-local half, `runtime.yaml` (`kind: RelayRuntimeConfig`), stays separate so the reviewed contract carries no secret and no local path.
Registry Record
The unit Registry Relay serves on its consultation surface: one row of a governed resource, projected through a disclosure profile and carrying its record context. It is not a copy of the source row. A property reaches a Registry Record only when the Registry contract declares it and the selected access profile admits it. Base Registry Engine serves the records of its own registry under the same shared Registry Record profile, in the Registry Record envelope.
@@ -369,7 +369,7 @@ Product names are always in English, including on future translated pages.
A rule in a Base Registry Engine access profile that binds one record field to a verified claim in the caller's token with `equals` or `in`, so a request reaches only the rows whose field matches what the token carries.
sealed package
-
The directory `relayctl package` produces and the `relay` binary verifies before it activates anything: the compiled contract, the generated artifacts, the observed source schemas, and a `relay-package.json` manifest carrying the package format version, the contract revision, the package revision, and a digest for every file. Format version `relay.registrystack.org/package/v1alpha3`. Relay refuses a package whose format version is not the one that binary was built against, and it refuses one whose recomputed digests do not match. Packages are unsigned: verification proves integrity, not authorship.
+
The directory `relayctl package` produces and the `relay` binary verifies before it activates anything: the authored `registry.yaml` and governed files, the compiled contract, the generated artifacts, and a `SHA256SUMS` file listing the digest of every other file. Relay refuses a package with a changed, missing, or extra file, and it recompiles the packaged contract and regenerates the artifacts at startup, refusing a package whose compiled contract or artifacts the packaged inputs do not reproduce. Packages are unsigned: verification proves integrity, not authorship. See package digest.
source profile
How a Registry Relay source is expected to behave while it is being read, declared per source as `snapshot` or `live-read-only`. Under `snapshot` the database is a fixed artifact published for reading, so a response is reproducible for the life of that snapshot. Under `live-read-only` the database is the operational registry, still opened read-only through the shared SQLite boundary, so answers move as the registry moves. Neither profile permits a write, and both pin the expected schema fingerprint.
diff --git a/docs/site/src/content/docs/reference/relayctl.mdx b/docs/site/src/content/docs/reference/relayctl.mdx index 0d085265a1..03f0fa8b3c 100644 --- a/docs/site/src/content/docs/reference/relayctl.mdx +++ b/docs/site/src/content/docs/reference/relayctl.mdx @@ -40,7 +40,7 @@ subcommands. | `relayctl generate ` | Authoring project directory | Writes the deterministic artifact set and five authoring reports, each with its SHA-256 digest | Seal a package or write into a destination that already has entries | | `relayctl test ` | Project fixture inputs | Runs the project's offline fixture cases through the shared kernel | Start a listener, contact a network source, or start `relay` | | `relayctl diff ` | Two project directories | Classifies meaning, disclosure, and security changes between the reviewed project and the candidate | Add or remove change classes, or decide whether the change is acceptable | -| `relayctl package --output ` | Project that compiles under the production profile | Builds a sealed package directory carrying `relay-package.json` | Sign the package, deploy it, or write into a destination that already exists | +| `relayctl package --output ` | Project that compiles under the production profile; `--revision` records an optional operator label, `--dry-run` replaces `--output` | Builds a sealed package directory whose `SHA256SUMS` lists every file, and reports the package digest `package.expectedDigest` pins; `--dry-run` reports the same digest without writing | Sign the package, deploy it, or write into a destination that already exists | | `relayctl tooling editor [PROJECT]` | Authoring project directory, defaulting to the current directory | Writes the project-local schema mappings VS Code and Zed read, and reports all six managed files | Install an extension, change editor settings outside the project, or read any database | | `relayctl tooling language-server` | No arguments; speaks the Language Server Protocol over standard input and output | Reports Relay V2 authoring diagnostics from the entry documents and exact governed closure held for the workspace | Open a socket, observe SQLite, read source values, or adopt an unrelated file merely because a recursive watcher reports it | @@ -221,8 +221,9 @@ release: - `inspect` offers no row or value sampling surface. Its help states that it inspects structure without reading row values, and the test refuses the options `--sample`, `--rows`, `--values`, and `--limit`. -- A usage error discloses no project content. `relayctl package ` without `--output` exits - `2` with empty standard output and an error that names `--output` and nothing about the project. +- A usage error discloses no project content. `relayctl package ` without `--output` or + `--dry-run` exits `2` with empty standard output and an error that names `--output` and nothing + about the project. - All seven flat workflow commands and both tooling subcommands print help successfully with an empty standard error, so the adopter workflow is exposed by one binary. diff --git a/docs/site/src/content/docs/security/hardening-checklist.mdx b/docs/site/src/content/docs/security/hardening-checklist.mdx index d30dd03d9b..d0db8a217e 100644 --- a/docs/site/src/content/docs/security/hardening-checklist.mdx +++ b/docs/site/src/content/docs/security/hardening-checklist.mdx @@ -59,6 +59,13 @@ bundle, or the sealed Relay package. It assumes you have already configured your files below that root, each owned by the service identity at mode `0600`, and checks owner, mode, no-follow, link count, and open-file identity. Keep the secret root itself operator-only. Strict deployments receive only a Transit Unix socket for signing. +- Evidence Gateway: prefer the file provider with a secret root dedicated to this deployment. Enable + `secretProviders.environment: {}` only when the platform injects secrets as environment variables, + and then run the process in a dedicated, curated environment holding only the secrets this + deployment needs. Enabling the provider lets a reference in the reviewed bundle, a source + credential included, name any variable in the process environment, so review the bundle's + `secret:env/` references as you review its other secret references. A bundle cannot enable + a provider itself; only `runtime.yaml` does. - Evidence Gateway: give the audit hash key and the subject-binding key independently generated raw key material of at least 32 bytes each. The file provider does not base64-decode them, so write raw bytes rather than an encoded string. @@ -92,7 +99,8 @@ bundle, or the sealed Relay package. It assumes you have already configured your `secret:env/` or `secret:file/`, and never as a literal value. `` is uppercase ASCII, digits, and underscores starting with a letter; `` is a single flat lowercase filename with no directory component, so a nested or traversing path is rejected rather than - resolved. No secret may exceed 64 KiB. + resolved. A file secret resolves under `secretProviders.file.root`, and an environment secret + needs `secretProviders.environment: {}`. No secret may exceed 64 KiB. - Relay: on Unix, a `secret:file/` target must be a regular file owned by the running user with mode `0400` or `0600` and a link count of one; a hard-linked file is refused under every name. @@ -103,7 +111,7 @@ bundle, or the sealed Relay package. It assumes you have already configured your independently generated random material and hold it under your own custody and retention controls. - Relay: pick the single OIDC signing algorithm the deployment actually uses. - `authentication.issuer.algorithms` must contain exactly one of `EdDSA`, `ES256`, or `RS256`, and + `authentication.oidc.algorithms` must contain exactly one of `EdDSA`, `ES256`, or `RS256`, and `tokenTypes` must be exactly `["at+jwt"]`. There is no allowlist to prune and no `HS*` or `none` to exclude: anything else fails the closed runtime profile before the process serves. Evidence Gateway's own access-token allowlist is configured per bundle from the same three, while its @@ -122,10 +130,10 @@ bundle, or the sealed Relay package. It assumes you have already configured your TransitDataKeyClient; crates/registry-breg/src/startup.rs, FieldEncryptionCustody; crates/registry-bregctl/src/doctor.rs, startup_diagnostic() and CHECKED_DEPENDENCIES. */} -## Freeze the Evidence Gateway deployment project +## Freeze the Evidence Gateway package and target -- Mount the governed `bundle/` directory and every artifact inside it, `runtime.yaml`, and each - named CA bundle file read-only and non-writable to the service process, and keep the secret root +- Mount the installed package and every artifact inside it, the deployment target's `runtime.yaml`, + and each named CA bundle file read-only and non-writable to the service process, and keep the secret root at mode `0700` or tighter. Evidence Gateway reports a non-immutable-input error and refuses to start rather than serving from an input it could write to, and applies the same refusal to the secret root when it is reachable by group or other even if it is not itself writable. Keep every @@ -142,12 +150,13 @@ bundle, or the sealed Relay package. It assumes you have already configured your writable bound source extract as BundleError::InvalidArtifact rather than NotImmutable, crates/registry-evidence/src/bundle.rs:852-864, :80-81, :863-864. */} -- Freeze the project before validating or serving it, then validate the frozen bytes: +- Freeze the package and target before validating or serving them, then validate the frozen bytes: ```sh - chmod -R a-w bundle && chmod 444 runtime.yaml - evidence --runtime runtime.yaml check - evidence --runtime runtime.yaml evaluate --fixture "bundle/fixtures/.yaml" + chmod -R a-w "" && chmod 444 "/runtime.yaml" + evidence check --runtime-config "/runtime.yaml" + evidence evaluate --runtime-config "/runtime.yaml" \ + --fixture "fixtures/.yaml" ``` - Review the complete simultaneously enabled bundle as one disclosure surface before deployment, @@ -165,10 +174,10 @@ bundle, or the sealed Relay package. It assumes you have already configured your ## Freeze the Relay source and package -- Mount the sealed package at `packagePath` read-only. Relay re-runs the compiler and the artifact +- Mount the sealed package at `package.root` read-only. Relay re-runs the compiler and the artifact generator over the sealed inputs at startup and compares the results byte for byte, so a modified package fails to start rather than serving quietly. That check proves integrity, not - authenticity: `packageRevision` is a digest and nothing signs a Relay package, so provenance is + authenticity: the package digest is a digest and nothing signs a Relay package, so provenance is whatever your build and distribution pipeline can prove, not something the runtime verifies. - Decide the source profile deliberately per binding. A `snapshot` source is pinned to the exact bytes present at startup; a `live-read-only` source reflects the file as other processes commit @@ -187,8 +196,8 @@ bundle, or the sealed Relay package. It assumes you have already configured your of it, so a review that reads only the runtime file has reviewed almost nothing. {/* Evidence: verify_compiled_derivation() and verify_artifact_derivation() re-run the compiler and - generator and compare bytes, and packageRevision is documented as an integrity digest and not - an authenticity proof, crates/registry-relay-v2/src/package.rs:348-350; SourceProfile has + generator and compare bytes, and the package digest is documented as an integrity digest and + not an authenticity proof, crates/registry-relay-v2/src/package.rs; SourceProfile has exactly two variants, crates/registry-relay-v2/src/contract.rs:323-325; SNAPSHOT_SIDECARS, symlink refusal, the read-only-filesystem-or-non-writable requirement, same_file() over dev/ino/len/mode/mtime/ctime, and the verify_unchanged_until comment about a privileged writer, @@ -298,8 +307,8 @@ serve, and a startup that fails rather than degrades. quotas; a contract that declares a list or a search operation, or that shows more than one resource in its resource listing, requires a cursor key. None of these is a warning you can accept, and none of them can be waived. -- Relay: record the `packageRevision` you deployed, the digest of the package file, and the - `relayctl` project revision that produced it. That triple plus the runtime file is the only +- Relay: record the package digest you deployed, pin it as `package.expectedDigest`, and record the + `relayctl` project revision that produced it. That record plus the runtime file is the only description of what the process is enforcing, and the runtime file alone is not enough because it binds paths and limits and nothing about disclosure. - Relay: the metadata a deployment exposes about itself is governed, not operational. Service, @@ -308,14 +317,15 @@ serve, and a startup that fails rather than degrades. deliberately rather than accepting whatever the starter contract produced, because they decide what an unauthenticated reader can learn about the registry's shape. - Evidence Gateway has no equivalent declaration and no posture route either. Its posture is the exact - bytes it loaded: one governed bundle and one closed runtime file, each with its own content - digest, mounted read-only, with no reload, merge, mutation, governed-field override, or fallback - path. Record the bundle revision you deployed, because that identifier plus the digests is the - only description of what the process is enforcing. - -{/* Evidence: RelayRuntime is a closed deny_unknown_fields schema over apiVersion, kind, server, - packagePath, sources, authentication, audit, cursor, limits, quotas, and shutdown only, with - ServerRuntime carrying exactly one field (bind), crates/registry-relay-v2/src/contract.rs:1062-1180; + bytes it loaded: one verified package and one closed runtime file, mounted read-only, with no + reload, merge, mutation, governed-field override, or fallback path. Record and pin the package + digest you deployed. Keep the runtime file with the deployment record because it binds the + package path and environment-specific resources without becoming part of the package. + +{/* Evidence: RelayRuntime is a closed deny_unknown_fields schema over apiVersion, kind, listener, + package, secretProviders, sources, authentication, audit, cursor, limits, quotas, and shutdown + only, with the shared ListenerConfig carrying exactly one field (bind), + crates/registry-relay-v2/src/contract.rs; validate_runtime_contract() with the test protected_contracts_require_issuer_lists_require_cursor_and_lookups_require_quota, crates/registry-relay-v2/src/startup.rs:469-492; QuotaLimiter is an in-memory per-operation @@ -392,7 +402,7 @@ serve, and a startup that fails rather than degrades. | Relay refuses to start on a contract it served before | The contract now has protected access with no configured issuer, a lookup with no quotas, or a list with no cursor key | Add the missing runtime block; none of the three is waivable | | Relay refuses to start and names the source | The snapshot sits on writable storage, is a symlink, or has a `-wal` or `-journal` sidecar beside it | Place the file on a read-only filesystem or make it non-writable, resolve the symlink, and remove the sidecar by checkpointing the database before you stage it | | Relay returns `503 audit.unavailable` and then fails readiness | The audit file is unwritable, or the file the writer was bound to was replaced | Restore writable durable storage at `audit.path` and restart; the failed requests returned no data | -| Evidence Gateway refuses to start on a non-immutable input | The bundle directory, `runtime.yaml`, a captured artifact, or a named CA bundle file is writable by the service process, or the secret root is reachable by group or other | Re-freeze the project (`chmod -R a-w bundle && chmod 444 runtime.yaml`), tighten the secret root to `0700`, and restart | +| Evidence Gateway refuses to start on a non-immutable input | The installed package, target `runtime.yaml`, a captured artifact, or a named CA bundle file is writable by the service process, or the secret root is reachable by group or other | Re-freeze the package and target (`chmod -R a-w "" && chmod 444 "/runtime.yaml"`), tighten the secret root to `0700`, and restart | | Evidence Gateway refuses to start on an invalid bound source extract | The source extract the runtime file names is writable; a distinct refusal from the non-immutable-input row above | Make the extract file non-writable and restart | | A second Evidence Gateway process fails at startup with a sink-locked error | Two processes point at the same `audit.path` | Run one writer per audit path; use active/passive with restart-on-failure rather than a second replica | | Evidence Gateway returns `403 evidence.denied` for a request you expected to work | Any one of the audience, grant, authority, or response-format checks failed; the response never says which | Work through the ordered checks in [Incident response](#incident-response) against local configuration, then read the refusal phase from the audit log | diff --git a/docs/site/src/content/docs/security/index.mdx b/docs/site/src/content/docs/security/index.mdx index 4ea57f8656..7643fc9094 100644 --- a/docs/site/src/content/docs/security/index.mdx +++ b/docs/site/src/content/docs/security/index.mdx @@ -38,7 +38,7 @@ evidence](openssf-evidence/). | Person-level access is written to an audit log | Built in, and fail-closed with no switch to turn it off | | Three disclosure modes, with `redacted` revealing neither value nor answer | Built in (Evidence Gateway) | | Governed contract verified before the process serves anything | Built in (Relay re-derives its package and compares bytes at startup) | -| A signature over the governed Relay package | Not provided; `packageRevision` is an integrity digest, not an authenticity proof | +| A signature over the governed Relay package | Not provided; the package digest is an integrity digest, not an authenticity proof | | Selective-disclosure SD-JWT VC serialization of an assertion | Built in (a second encoding, not a credential lifecycle) | | Holder-bound subject binding, proven at presentation by a key-binding JWT | Declared per requirement, verified by the relying party (frozen profile) | | Replay prevention for a presented credential | Not provided anywhere; the challenge lifecycle is the relying party's | @@ -78,9 +78,9 @@ authenticate, authorize, serve, audit. rather than trusting the check it did at startup (REQ-SEC-G-009). Entries are not chained or signed; tamper evidence comes from shipping them to append-only storage. -{/* Evidence: IssuerRuntime::profile() requires token_types == ["at+jwt"], exactly one of - EdDSA/ES256/RS256, and one canonical https discovery URL, - crates/registry-relay-v2/src/contract.rs:1215-1255; PurposeConstraint and AuthorityRowBinding +{/* Evidence: OidcRuntime::checked_profile() requires token_types == ["at+jwt"], exactly one of + EdDSA/ES256/RS256, a canonical https issuer, and keys from its discovery document or one + canonical https JWKS URI, crates/registry-relay-v2/src/contract.rs:1516-1591; PurposeConstraint and AuthorityRowBinding are read from the verified principal, crates/registry-relay-v2/src/auth.rs:204-257; every audit failure maps to ProblemCode::AuditUnavailable (503), crates/registry-relay-v2/src/api.rs, with the test @@ -167,7 +167,7 @@ process does not serve. Those checks are the security surface worth reviewing: - The package is verified first. Before Relay opens a source, a sink, or a socket, it re-runs the compiler and the artifact generator over the sealed inputs and compares the results byte for byte against what the package contains. A package that does not re-derive does not start. -- The runtime file cannot widen the package. `runtime.yaml` (`kind: RelayRuntime`) is a closed +- The runtime file cannot widen the package. `runtime.yaml` (`kind: RelayRuntimeConfig`) is a closed schema that rejects unknown fields, and it binds only local concerns: listen address, package path, source paths, the audit sink, an optional OIDC issuer, cursor settings, limits, quotas, and shutdown. Nothing in it can add an operation, a readable column, or a disclosure. @@ -175,8 +175,8 @@ process does not serve. Those checks are the security surface worth reviewing: issuer, and a contract that declares any lookup requires quotas. Neither is a warning. - Secrets are referenced, never inlined. The runtime file accepts exactly two reference grammars, `secret:env/` and `secret:file/`, and nothing else. A file secret must be a single - flat filename, owned by the running user, with mode `0400` or `0600` and a link count of one, - and no secret may exceed 64 KiB. + flat filename under the declared `secretProviders.file.root`, owned by the running user, with + mode `0400` or `0600` and a link count of one, and no secret may exceed 64 KiB. - The source is opened read-only and pinned. Relay opens SQLite with `SQLITE_OPEN_READ_ONLY | SQLITE_OPEN_URI | SQLITE_OPEN_NO_MUTEX`, refuses a symlink, refuses a `-wal` or `-journal` sidecar beside a snapshot, and requires a snapshot to sit on a read-only @@ -185,17 +185,18 @@ process does not serve. Those checks are the security surface worth reviewing: `debug`, or `trace`; any other value falls back to the crate's own `info` filter rather than enabling a dependency's logging. - Unix only, and it fails closed off it. On a non-Unix target Relay's permission check returns - false unconditionally and runtime-path validation returns an error, so the process refuses to - start rather than running without the ownership and mode checks it relies on. + false unconditionally and the runtime configuration loader refuses to read `runtime.yaml`, so + the process refuses to start rather than running without the ownership and mode checks it relies + on. {/* Evidence: verify_compiled_derivation() and verify_artifact_derivation() re-run the compiler and generator and compare bytes, and safe_permissions() returns false unconditionally under #[cfg(not(unix))], crates/registry-relay-v2/src/package.rs; prepare() verifies the package first, validate_runtime_contract() requires an issuer for protected access and quotas for - lookups, and #[cfg(not(unix))] validate_runtime_path returns StartupError::RuntimeInvalid, - crates/registry-relay-v2/src/startup.rs:96-104 and :469-492; RelayRuntime is a closed - deny_unknown_fields schema, crates/registry-relay-v2/src/contract.rs:1062-1079; the two secret - grammars, the 64 KiB bound, and the uid, 0400-or-0600 mode, and nlink==1 file checks with + lookups, crates/registry-relay-v2/src/startup.rs; #[cfg(not(unix))] + require_trusted_ownership returns an error, crates/registry-platform-config/src/loader.rs; + RelayRuntime is a closed deny_unknown_fields schema, crates/registry-relay-v2/src/contract.rs; + the two secret grammars, the 64 KiB bound, and the uid, 0400-or-0600 mode, and nlink==1 file checks with their tests references_use_only_the_two_exact_contract_grammars and file_secret_accepts_only_owner_read_and_optional_owner_write_modes, crates/registry-platform-config/src/secrets.rs; the open flags, diff --git a/docs/site/src/content/docs/spec/rs-arc-g.mdx b/docs/site/src/content/docs/spec/rs-arc-g.mdx index b2e9019f32..3c9f43db2a 100644 --- a/docs/site/src/content/docs/spec/rs-arc-g.mdx +++ b/docs/site/src/content/docs/spec/rs-arc-g.mdx @@ -178,7 +178,7 @@ The following ordered flow describes how a request moves through the stack from 3. **Contract authoring.** An adopter authors a Relay project: the governed contract, the governance review documents that record institutional judgment, and the codelists the contract references. `relayctl` compiles the project, reports diagnostics, and generates artifacts. Under the production profile an unreviewed classification or an unobserved source schema is an error rather than a warning. -4. **Sealing.** `relayctl` builds a sealed package containing the authored contract, the governed closure, the canonical compiled model, the generated artifacts, and a manifest that digests every file. Deployment-local bindings are excluded by construction. +4. **Sealing.** `relayctl` builds a sealed package containing the authored contract, the governed closure, the canonical compiled model, the generated artifacts, and a `SHA256SUMS` file that digests every file. Deployment-local bindings are excluded by construction. 5. **Serving.** A Relay process verifies the package by re-deriving it, opens its issuer, audit sink, source, and listener only after that verification succeeds, and serves read-only consultation routes. Every attempt is recorded to the audit log before the source is read, and the request is refused when the audit destination cannot accept the entry. @@ -268,7 +268,7 @@ This specification is `verified`: it is distilled from published artifacts a rea {/* Evidence: crates/registry-relay-v2/src/startup.rs verifies the sealed package before opening issuer, audit, source, or listener resources. */} {/* Evidence: crates/registry-relay-v2/src/package.rs re-derives the compiled model and every - artifact, and records that the package revision is an integrity digest, not an authenticity + artifact, and records that the package digest is an integrity digest, not an authenticity proof. */} {/* Evidence: crates/registry-relay-v2/src/server.rs defines the fixed route set with no administrative or reload route. */} diff --git a/docs/site/src/content/docs/spec/rs-op-posture.mdx b/docs/site/src/content/docs/spec/rs-op-posture.mdx index ebcb4d7312..8ba335f9e9 100644 --- a/docs/site/src/content/docs/spec/rs-op-posture.mdx +++ b/docs/site/src/content/docs/spec/rs-op-posture.mdx @@ -239,10 +239,10 @@ can carry request URLs and headers, remain off at every level. A Relay-owned log record MUST carry fixed messages and value-free dimensions, and MUST NOT carry record values, credentials, or token contents. -`RELAY_LOG` and `RELAY_RUNTIME`, which supplies the runtime document path, are the environment -variables the `relay` process reads by name, alongside `RELAY_HEALTHCHECK_URL` for the healthcheck -subcommand. It also reads whatever variable a `secret:env/` reference in the runtime document -names. The `relayctl` authoring tool defines no product-specific environment-variable configuration; +`RELAY_LOG` is the environment variable the `relay` process reads by name, alongside +`RELAY_HEALTHCHECK_URL` for the healthcheck subcommand; the runtime document path is the required +`--runtime-config` flag. It also reads whatever variable a `secret:env/` reference or a +`${NAME}` substitution in the runtime document names. The `relayctl` authoring tool defines no product-specific environment-variable configuration; its fixture runner may use only the host's temporary-directory selection for internal materialization, per [RS-PR-RELAYCTL](../rs-pr-relayctl/). @@ -253,13 +253,12 @@ verification passes. REQ-OP-POSTURE-115: The runtime MUST verify the sealed package before it opens an issuer, audit, source, or listener resource. -Verification MUST re-derive the package rather than trust its manifest: it MUST recompute the digest -of every listed file, recompute the manifest's `packageRevision` over the canonical JSON of the -manifest's own unsigned content, re-canonicalize the compiled registry and compare it byte for byte -with the packaged bytes, and re-derive the compiled contract from the packaged `registry.yaml`, -governed files, and recorded source schemas. -The package directory's file set MUST equal the manifest's file list exactly, so an added or removed -file fails verification. +Verification MUST re-derive the package rather than trust its `SHA256SUMS`: it MUST recompute the +digest of every listed file, re-canonicalize the compiled registry and compare it byte for byte with +the packaged bytes, re-derive the compiled contract from the packaged `registry.yaml`, governed +files, and recorded source schemas, and regenerate every artifact byte for byte. +The package directory's file set MUST equal the `SHA256SUMS` list exactly, and the listed file set +MUST equal what the compiled contract names, so an added or removed file fails verification. A package containing a symlink, a non-regular file, or an unsafe permission bit MUST be refused. REQ-OP-POSTURE-116: After package verification, the runtime MUST observe each configured source and @@ -312,8 +311,8 @@ runtime and exercised by its tests. - `verify_readiness_sources` in `crates/registry-relay-v2/src/sqlite_runtime.rs` re-verifies snapshot immutability and schema fingerprints under the request timeout, and its readiness gate coalesces concurrent probes (Section 3). -- `RelayRuntime::is_valid` and `valid_secret_reference` in - `crates/registry-relay-v2/src/contract.rs` enforce the numeric bounds and the secret-reference +- `RelayRuntime::check` in `crates/registry-relay-v2/src/contract.rs` and `SecretProvidersConfig` + in `crates/registry-platform-config` enforce the numeric bounds and the secret-reference forms (Section 5). - `QuotaLimiter` in `crates/registry-relay-v2/src/server.rs` implements the per-operation token bucket, and `crates/registry-relay-v2/src/problem.rs` fixes the refusal codes and statuses diff --git a/docs/site/src/content/docs/spec/rs-pr-evidence.mdx b/docs/site/src/content/docs/spec/rs-pr-evidence.mdx index d6ebc4a8ec..c1bd5ef3a4 100644 --- a/docs/site/src/content/docs/spec/rs-pr-evidence.mdx +++ b/docs/site/src/content/docs/spec/rs-pr-evidence.mdx @@ -171,8 +171,8 @@ governed verifier configuration (`products/evidence/contracts/jws-profile.yaml`, ## 3. Authentication and authorization -Evidence Gateway runs one authentication kind, `oidc-access-token`, whose issuer, audiences, accepted token -types, algorithms, JWKS URI, and claim paths are fixed in the immutable bundle +Evidence Gateway accepts access tokens from one OIDC issuer, `authentication.oidc`, whose issuer, single +audience, accepted token types, algorithms, fixed JWKS URI, and claim paths are fixed in the immutable bundle (`products/evidence/contracts/bundle.schema.yaml`). REQ-PR-EVIDENCE-005: Both evidence-production operations and definition discovery MUST require @@ -711,7 +711,8 @@ Identity MUST travel only as domain-separated keyed pseudonyms (invariant `V1-I1 REQ-PR-EVIDENCE-050: Every authorization refusal after successful authentication MUST durably append exactly one standalone minimal native event before Evidence Gateway returns the generic `403`. -That event MUST identify the event and operation, assurance profile, bundle revision, requester +That event MUST identify the event and operation, assurance profile, package digest in the frozen +`bundleRevision` field, requester pseudonym, optional actor pseudonym, closed denial category and decision, timestamp, and duration. It MUST NOT contain the requested requirement, purpose, subjects, unmatched authority, selector information, response protection, source, or evaluation material. If the event cannot be durably @@ -734,7 +735,7 @@ ships audit entries to, not to the serving process ## 10. Immutability of deployment input REQ-PR-EVIDENCE-052: Configuration MUST be immutable for the serving process lifetime. -Evidence Gateway MUST load one read-only atomic governed bundle and one separately digested closed runtime +Evidence Gateway MUST verify and load one read-only atomic governed package plus one closed runtime file at startup; runtime override, reload, merge, fallback, and mutation paths MUST NOT exist (invariant `V1-I18`, `products/evidence/contracts/runtime.schema.yaml`). @@ -743,17 +744,18 @@ REQ-PR-EVIDENCE-053: The runtime file MUST own only the process-local bindings e governed semantics or source authority. Unknown keys MUST be rejected at every level. -REQ-PR-EVIDENCE-054: A missing, writable, or unreviewed bundle MUST NOT be treated as trusted +REQ-PR-EVIDENCE-054: A missing, writable, unverified, or unreviewed package MUST NOT be treated as trusted configuration. Version 1 has no in-bundle trust override, and absence or a failed immutability check MUST fail readiness (`products/evidence/contracts/security-invariant-matrix.yaml`, cross-cutting config trust). REQ-PR-EVIDENCE-055: One process MUST serve one operator-controlled trust domain, with one service -trust domain, issuer governance boundary, bundle lifecycle, signer, and audit boundary +trust domain, issuer governance boundary, package lifecycle, signer, and audit boundary (invariant `V1-I19`). -REQ-PR-EVIDENCE-056: The bundle revision MUST be a digest over the complete atomic bundle bytes and -layout manifest and MUST be carried in every native audit event. +REQ-PR-EVIDENCE-056: The package digest MUST be the digest of the deterministic `SHA256SUMS` file +covering the complete atomic package and MUST be carried in every native audit event through the +frozen `bundleRevision` field. Every assertion MUST instead carry the configuration revision for its requirement, computed over that requirement's configuration and artifact closure and published with the requirement through authenticated definition discovery, in the forms fixed by @@ -933,7 +935,7 @@ An Evidence Gateway deployment conforms to this specification when it: REQ-PR-EVIDENCE-047, REQ-PR-EVIDENCE-048, REQ-PR-EVIDENCE-049, REQ-PR-EVIDENCE-050, REQ-PR-EVIDENCE-051); - treats deployment input as immutable, restricts the runtime file to process-local bindings, fails - readiness on an untrusted bundle, serves one trust domain, carries the bundle revision in every + readiness on an untrusted package, serves one trust domain, carries the package digest in every native audit event, and carries the requirement-scoped configuration revision in every assertion (REQ-PR-EVIDENCE-052, REQ-PR-EVIDENCE-053, REQ-PR-EVIDENCE-054, REQ-PR-EVIDENCE-055, REQ-PR-EVIDENCE-056); diff --git a/docs/site/src/content/docs/spec/rs-pr-relay.mdx b/docs/site/src/content/docs/spec/rs-pr-relay.mdx index c708b0451b..8f4134dd71 100644 --- a/docs/site/src/content/docs/spec/rs-pr-relay.mdx +++ b/docs/site/src/content/docs/spec/rs-pr-relay.mdx @@ -73,15 +73,15 @@ Relay returns unsigned responses. Portable, signed, minimum-disclosure assertion ## 2. Sealed deployment model -A Relay service serves exactly one sealed package. The package is the governed trust root: it carries the authored contract, the compiled Registry derived from it, and every generated artifact, each bound to a SHA-256 digest over its canonical bytes. +A Relay service serves exactly one sealed package. The package is the governed trust root: it carries the authored contract, the compiled Registry derived from it, and every generated artifact, each bound to a SHA-256 digest listed in the package's `SHA256SUMS`. REQ-PR-RELAY-101: A Relay process MUST verify its sealed package before it opens an identity issuer connection, an audit sink, a SQLite source, or a listener. A package that fails verification MUST prevent the process from serving rather than degrade it. REQ-PR-RELAY-102: Package verification MUST re-derive the compiled Registry from the authored contract and the packaged source schema observations, and MUST re-derive every generated artifact from that compiled Registry, comparing both byte for byte against the packaged bytes. A mismatch MUST fail verification. -REQ-PR-RELAY-103: The package revision MUST be a SHA-256 digest over the canonical JSON of the package manifest. It is an integrity digest, not an authenticity proof: any party can recompute it. Relay packages are unsigned, and a deployment MUST NOT treat a matching package revision as evidence of who produced the package. +REQ-PR-RELAY-103: The package digest MUST be the SHA-256 digest of the package's `SHA256SUMS`, which lists the digest of every other package file. Verification MUST refuse a changed, missing, or extra file and name it, and MUST refuse a package whose digest differs from a configured `package.expectedDigest`. The digest is an integrity digest, not an authenticity proof: any party can recompute it. Relay packages are unsigned, and a deployment MUST NOT treat a matching package digest as evidence of who produced the package. -REQ-PR-RELAY-104: Governed contract input and deployment binding MUST be separate closed documents. The contract is `kind: RegistryContract`, the binding is `kind: RelayRuntime`, both at `apiVersion: relay.registrystack.org/v2alpha1`, and both MUST reject unknown fields at every nested structure rather than ignore them. A governed field MUST NOT be accepted from the deployment binding. +REQ-PR-RELAY-104: Governed contract input and deployment binding MUST be separate closed documents. The contract is `kind: RegistryContract` at `apiVersion: relay.registrystack.org/v2alpha1`, the binding is `kind: RelayRuntimeConfig` at `apiVersion: registry.registrystack.org/relay-runtime/v1alpha1`, and both MUST reject unknown fields at every nested structure rather than ignore them. A governed field MUST NOT be accepted from the deployment binding. REQ-PR-RELAY-105: Every source MUST be a SQLite database read through the shared bounded read-only boundary, opened read-only, and MUST declare exactly one source profile, either `snapshot` or `live-read-only`. Each resource MUST bind to one named view in one declared source. Table names, SQL text, and hidden source columns MUST NOT appear on the public surface. @@ -209,7 +209,7 @@ REQ-PR-RELAY-142: Relay MUST apply a per-operation quota where the deployment co These constraints are stated so a reader does not infer a capability from the route list that the reviewed implementation does not provide. - No writes and no provisioning: Relay reads governed SQLite views and nothing else (REQ-PR-RELAY-106, REQ-PR-RELAY-135). -- No signatures: Relay responses are unsigned, and the package revision proves integrity, not authorship (REQ-PR-RELAY-103, REQ-PR-RELAY-123). +- No signatures: Relay responses are unsigned, and the package digest proves integrity, not authorship (REQ-PR-RELAY-103, REQ-PR-RELAY-123). - No general geospatial API: the Point profile covers exact CRS84 points and one inclusive bounding-box predicate (REQ-PR-RELAY-128, REQ-PR-RELAY-129). - No SDMX registry: the statistical surface is a four-resource read profile over pre-aggregated datasets (REQ-PR-RELAY-131, REQ-PR-RELAY-133). - No record linkage: the lookup route matches declared values exactly and returns no similarity score (REQ-PR-RELAY-108). @@ -244,7 +244,7 @@ This specification is `verified`: every requirement describes shipped behavior a (Sections 3, 5, 6, 7, and 11). */} {/* Evidence: crates/registry-relay-v2/src/startup.rs verifies the sealed package before it opens an issuer, an audit sink, a source, or a listener, and package.rs re-derives the compiled Registry - and every generated artifact and records the package revision as an integrity digest rather than + and every generated artifact and records the package digest as an integrity digest rather than an authenticity proof (Section 2). contract.rs carries the two closed documents, RegistryContract and RelayRuntime (REQ-PR-RELAY-104). */} {/* Evidence: crates/registry-relay-v2/src/server.rs assembles the fixed router with router() and diff --git a/docs/site/src/content/docs/spec/rs-pr-relayctl.mdx b/docs/site/src/content/docs/spec/rs-pr-relayctl.mdx index 0e493eab2d..f540412eb9 100644 --- a/docs/site/src/content/docs/spec/rs-pr-relayctl.mdx +++ b/docs/site/src/content/docs/spec/rs-pr-relayctl.mdx @@ -51,7 +51,7 @@ This specification covers: - Authoring and production compilation profiles. - Deterministic generated artifacts and offline fixture execution. - Change classification between two project revisions. -- The sealed deployment package and its manifest. +- The sealed deployment package and its `SHA256SUMS`. - Report presentation and process exit statuses. - The process boundary between `relayctl` and `relay`. @@ -215,23 +215,23 @@ Packaging MUST re-derive the compiled registry and every generated artifact from and refuse when a re-derived byte differs. REQ-PR-RELAYCTL-022: The package root MUST contain `registry.yaml`, the authored `governed/` -closure, the canonical `compiled/registry.json`, the `generated/` artifacts, and the -`relay-package.json` manifest. +closure, the canonical `compiled/registry.json`, the `generated/` artifacts, and a `SHA256SUMS` +file in the shared Registry Stack package format, plus a `REVISION` file when `--revision` is given. `runtime.yaml` MUST NOT be packaged: deployment-local bindings stay with the deployment. -REQ-PR-RELAYCTL-023: The manifest MUST declare the package version, the package revision, the -contract revision, the expected source schema fingerprint and observed schema for every source, the -artifact inventory with per-artifact visibility and digest, the operation-to-artifact bindings, and -one entry per packaged file with its size and digest. -The package revision MUST be the SHA-256 digest of the canonical manifest without that field. -It is an integrity digest, not an authenticity proof: any party can recompute it, so acceptance MUST -NOT rest on the revision alone. +REQ-PR-RELAYCTL-023: `SHA256SUMS` MUST list the SHA-256 digest of every other package file, and the +package digest MUST be the SHA-256 digest of `SHA256SUMS`. +The package report MUST state the package digest, the contract revision, the expected source schema +fingerprint for every source, one entry per packaged file with its size and digest, and the derived +exposure of every generated artifact; `--dry-run` MUST report the same digest without writing. +The package digest is an integrity digest, not an authenticity proof: any party can recompute it, +so acceptance MUST NOT rest on the digest alone. -REQ-PR-RELAYCTL-024: Packaging MUST enforce closure bounds on file count, manifest size, and total +REQ-PR-RELAYCTL-024: Packaging MUST enforce closure bounds on file count, file size, and total package size, and MUST refuse a symbolic link inside the closure. On a Unix host it MUST leave directories at mode `0755` and files at mode `0644`. -A partially written destination MUST be left in place for inspection rather than removed, and a -later packaging attempt MUST refuse it. +A failed write MUST remove only the destination directory it created, and a packaging attempt MUST +refuse a destination that already exists. ## 10. Reports and output @@ -263,8 +263,8 @@ closure MUST be discarded. The project workflow JSON report carries no schema-version field and is not a versioned automation contract. -Automation that must survive a release upgrade uses the sealed package and its manifest, which are -versioned, rather than the report shape. +Automation that must survive a release upgrade uses the sealed package and its `SHA256SUMS`, which +follow the shared Registry Stack package format, rather than the report shape. ## 11. Exit statuses @@ -316,7 +316,7 @@ A `relayctl` release conforms to this specification when it: (REQ-PR-RELAYCTL-013 through REQ-PR-RELAYCTL-017). - Classifies change impact including disclosure and access widening, without mutating either project (REQ-PR-RELAYCTL-018 and REQ-PR-RELAYCTL-019). -- Builds a re-derived, bounded, hardened sealed package with a complete manifest and no +- Builds a re-derived, bounded, hardened sealed package with a complete `SHA256SUMS` and no `runtime.yaml` (REQ-PR-RELAYCTL-020 through REQ-PR-RELAYCTL-024). - Keeps workflow reports, editor reports, and LSP output distinct and emits value-free diagnostics under both command output modes (REQ-PR-RELAYCTL-025 through REQ-PR-RELAYCTL-027). @@ -353,7 +353,7 @@ tests. {/* Evidence: crates/registry-relay-v2/src/compiler.rs holds the authoring and production severity split for unobserved schemas, unreviewed classification, and review-document status. */} {/* Evidence: crates/registry-relay-v2/src/package.rs implements re-derivation, the package layout, - the canonical manifest, the closure bounds, symlink refusal, and permission hardening. */} + the shared package format, the closure bounds, symlink refusal, and permission hardening. */} {/* Evidence: crates/registry-relay-v2/src/fixtures.rs plans only the selected step, accumulates observations as the journey runs, and treats an absent prior observation as a mismatch. */} {/* Evidence: crates/registry-relay-v2/src/diff.rs implements the change classes, the impact scale, diff --git a/docs/site/src/content/docs/spec/rs-terms.mdx b/docs/site/src/content/docs/spec/rs-terms.mdx index 36edac962f..8c788c2773 100644 --- a/docs/site/src/content/docs/spec/rs-terms.mdx +++ b/docs/site/src/content/docs/spec/rs-terms.mdx @@ -128,13 +128,13 @@ The registry stack comprises four formal products: Registry Platform, Registry R **governed contract**: The authored `kind: RegistryContract` document, at `apiVersion: relay.registrystack.org/v2alpha1`, that declares one Relay deployment's registry identity, sources, resources, operations, access profiles, disclosure profiles, and statistical datasets. It rejects unknown fields at every nested structure. Registry Stack product term. -**deployment binding**: The deployment-local half of a Relay deployment, authored as a closed `kind: RelayRuntime` document at the same `apiVersion`. It carries the listener, the identity issuer, source file paths, the audit destination, and quotas, and it rejects every governed field, so an operator cannot widen governed intent through deployment configuration. Replaces the Relay 1.0 term runtime binding. Registry Stack product term. +**deployment binding**: The deployment-local half of a Relay deployment, authored as a closed `kind: RelayRuntimeConfig` document at `apiVersion: registry.registrystack.org/relay-runtime/v1alpha1`. It carries the listener, the identity issuer, source file paths, the audit destination, and quotas, and it rejects every governed field, so an operator cannot widen governed intent through deployment configuration. Replaces the Relay 1.0 term runtime binding. Registry Stack product term. **compiled Registry**: The canonical model derived from a governed contract together with the reviewed source schema observations recorded beside it. Every generated artifact derives from the compiled Registry, and the runtime re-derives both at startup rather than trusting the packaged copy. Registry Stack product term. -**sealed package**: The one directory a Relay process serves: the authored contract, the compiled Registry derived from it, and every generated artifact, each bound to a SHA-256 digest over its canonical bytes under one manifest. It is the governed trust root, so Relay verifies it, and re-derives the compiled Registry and each artifact byte for byte, before it opens an issuer connection, an audit sink, a source, or a listener. Registry Stack product term. +**sealed package**: The one directory a Relay process serves: the authored contract, the compiled Registry derived from it, and every generated artifact, each bound to a SHA-256 digest listed in the package's `SHA256SUMS`. It is the governed trust root, so Relay verifies it, and re-derives the compiled Registry and each artifact byte for byte, before it opens an issuer connection, an audit sink, a source, or a listener. Registry Stack product term. -**package revision**: The SHA-256 digest over the canonical JSON of a sealed package manifest. It is an integrity digest, not an authenticity proof: any party can recompute it, Relay packages are unsigned, and a matching revision is no evidence of who produced the package. Registry Stack product term. +**package digest**: The SHA-256 digest of a package's `SHA256SUMS`, which lists the digest of every other file in the package. It is an integrity digest, not an authenticity proof: any party can recompute it, Relay packages are unsigned, and a matching digest is no evidence of who produced the package. Registry Stack product term. **source profile**: Which of two reading disciplines a declared Relay source follows. `snapshot` names a source whose bytes are fixed for the life of the deployment and confirmed unchanged at startup; `live-read-only` names a source that may change underneath a running process and is still read read-only. The vocabulary is closed at these two values, and every source declares exactly one. Registry Stack product term. diff --git a/docs/site/src/content/docs/start/casework.mdx b/docs/site/src/content/docs/start/casework.mdx index 826eaa58b9..6161568c60 100644 --- a/docs/site/src/content/docs/start/casework.mdx +++ b/docs/site/src/content/docs/start/casework.mdx @@ -79,7 +79,7 @@ caseworkctl dev tutorial-work/casework The installer verifies the release `SHA256SUMS` before it installs `casework` and `caseworkctl` into `~/.local/bin`. The `standalone-decision` template writes a project that declares one review kind and admitted producer and needs no registry behind it. `caseworkctl dev` starts PostgreSQL in a container, -starts the maintained stock identity provider and `casework`, seeds the directory, and prints a +packages the project, starts the maintained stock identity provider and `casework`, seeds the directory, and prints a report carrying a ready status, the service URL, the token endpoint, and one credential directory per local client. diff --git a/docs/site/src/content/docs/start/registry-render.mdx b/docs/site/src/content/docs/start/registry-render.mdx index 255b848748..3ebed529cf 100644 --- a/docs/site/src/content/docs/start/registry-render.mdx +++ b/docs/site/src/content/docs/start/registry-render.mdx @@ -13,7 +13,7 @@ standards_referenced: [] If you print registry data on paper, or hand a PDF to a person who was never one of your systems, Registry Render turns that data into a governed document. Render is a deterministic PDF renderer: -the same sealed template bundle and the same request produce the same PDF bytes, every time, with +the same verified template package and the same request produce the same PDF bytes, every time, with the hashes and the audit trail an institution needs to stand behind the printed artifact. {/* Evidence: crates/registry-render/src/lib.rs, render and RenderRequest; @@ -26,7 +26,7 @@ different documents for one decision. [How Registry Render stays byte-stable](../../explanation/render-determinism/) explains the mechanisms behind that promise. Render holds no database and makes no outbound calls. Everything a template can read comes from two -places: the sealed bundle on disk (templates, labels, schemas, fonts, vendored packages) and the +places: the verified package on disk (templates, labels, schemas, fonts, vendored packages) and the request itself (the data, the images, and the issuance time). The issuance time, `issuedAt`, is the document's own claim about when it was issued and is the only clock a render ever sees; nothing in the output depends on the machine's wall clock. @@ -60,15 +60,15 @@ mode](../../operate/registry-render/) covers the deployment contract for the ser Render renders documents; it does not govern records. It is not a document store, an e-signature or verifiable-credential issuer, an HTML converter, or an imposition engine, and it makes no authorization decisions about who may see which field. The field list a template prints is a -disclosure decision that belongs to the bundle author: everything placed on paper leaves every +disclosure decision that belongs to the template author: everything placed on paper leaves every access profile behind, and the QR a template prints can point a member of the public at a -verification page. Render's own surface is the rendering contract, sealed and audited. +verification page. Render's own surface is the rendering contract, packaged and audited. {/* Evidence: products/render/DEFINITION-OF-DONE.md; products/render/integrations/app-kit/README.md. */} -Templates are plain Typst, authored with upstream tooling, and sealed into a bundle with -per-file hashes before serving. A bundle plus a render binary is the whole deployment surface: +Templates are plain Typst, authored with upstream tooling, and built into the shared Registry +Stack package envelope before serving. A package plus a render binary is the whole deployment surface: no migration, no broker, no cache. ## Next diff --git a/docs/site/src/content/docs/tutorials/build-and-deploy-evidence-project.mdx b/docs/site/src/content/docs/tutorials/build-and-deploy-evidence-project.mdx index 1fd976d7d0..e106fe1886 100644 --- a/docs/site/src/content/docs/tutorials/build-and-deploy-evidence-project.mdx +++ b/docs/site/src/content/docs/tutorials/build-and-deploy-evidence-project.mdx @@ -118,11 +118,12 @@ real identifiers. limits, signing, optional response formats, and authority profiles. Secret references use `secret:file/` only. Do not place secret values or absolute secret paths in this file. -`runtime.yaml` is the ordinary Evidence Gateway runtime document. It binds the final absolute candidate -bundle path, private listener, secret root, audit path, and optional private certificate authority -files. The build copies its bytes unchanged, so the target host remains the authority for path, -ownership, permission, secret, and trust validation. -Set `bundleDirectory` to `/bundle`, and keep the listener on a numeric +`runtime.yaml` is the ordinary Evidence Gateway runtime document. It binds the stable absolute +installed package path, private listener, secret root, audit path, and optional private certificate +authority files. It stays in the deployment target and is never copied into the package, so the +target host remains the authority for path, ownership, permission, secret, and trust validation. +Set `package.root` to the stable installation path where this and future approved packages will be +placed, and keep the listener on a numeric loopback or private address. `listener.networkExposure` declares which addresses that means. It defaults to `private-address`, which accepts loopback, RFC 1918 private IPv4, or RFC 4193 unique-local IPv6. A container listener that must bind the wildcard `0.0.0.0` inside an isolated @@ -149,40 +150,43 @@ evidencectl package "/shared/evidence-project" \ --output "" ``` -The candidate contains the runtime document and closed bundle: +The output is one closed package. Environment-specific runtime configuration stays in the +deployment target: ```text / - runtime.yaml - bundle/ - evidence.yaml - adapters/ - derivations/ - schemas/ - fixtures/ - public-keys/ + SHA256SUMS + evidence.yaml + adapters/ + derivations/ + schemas/ + fixtures/ + public-keys/ ``` -The build validates the generated bundle through the real `evidence` binary and every referenced +The package command validates the generated package through the real `evidence` binary and every referenced fixture before it publishes the candidate. It asks that binary for its version first and refuses one that is not this evidencectl's, so the candidate is always the one the matching runtime shaped: `evidencectl package` takes the binary from `EVIDENCE_BIN` or the first `evidence` on `PATH`, and `evidencectl test` accepts `--evidence-bin` for the same handshake. A bundle that declares a -`publication` also has to render the description that advertises it, and a build whose description +`publication` also has to render the description that advertises it, and a package whose description comes back empty is refused rather than written, because `the candidate would carry no -catalog.jsonld`. The build does not contact an identity provider or a source endpoint. It +catalog.jsonld`. The package does not contain `runtime.yaml` or secret material. Packaging does not contact an identity provider or a source endpoint. It validates governed public-key semantics without contacting Transit or generating an unrelated signing key. The target-host check performs the provider self-test. Record -the printed bundle revision with the approved candidate path. +the printed package digest with the approved package path. Pass `--revision ` +when you also want one printable source or review reference recorded in `REVISION`. ## Provision the target host -Transfer the exact candidate. The operator provisions the audit HMAC key, subject-binding HMAC key, +Transfer the exact package to the stable absolute path named by the deployment target's +`runtime.yaml` at `package.root`. Replacing the package does not require editing that runtime path. +The operator provisions the audit HMAC key, subject-binding HMAC key, and source credentials beneath the runtime secret root. The workload-local Transit proxy holds the provider token and auto-auth state; Evidence Gateway receives only access to the configured Unix socket. Its non-exportable signing key remains in Transit. -Make the candidate runtime and bundle non-writable to the Evidence Gateway service identity, for -example `chmod -R a-w bundle && chmod 444 runtime.yaml`. `evidencectl test` below runs +Make the installed package and target runtime non-writable to the Evidence Gateway service identity, +for example `chmod -R a-w "" && chmod 444 "/runtime.yaml"`. `evidencectl test` below runs `evidence check` first, which refuses a writable runtime file, bundle artifact, or CA bundle file, a secret root reachable by group or other, or a writable extract, before it evaluates any fixture. @@ -190,14 +194,14 @@ Start the workload-local Transit proxy. Run the grouped offline ceremony once af runtime bindings, trust files, secrets, and proxy socket are in place: ```sh -evidencectl doctor --runtime-config "/runtime.yaml" -evidencectl test "" +evidencectl doctor --runtime-config "/runtime.yaml" +evidencectl test "" --target "" ``` Start Evidence Gateway only after both commands pass: ```sh -evidence --runtime "/runtime.yaml" serve +evidence serve --runtime-config "/runtime.yaml" ``` Route traffic through operator-controlled TLS only after `GET /ready` succeeds. The listener stays @@ -258,8 +262,8 @@ identifier. ## Expected result You have a retained signed response that verifies under the independent production policy, audit -entries for the synthetic request in append-only storage, and a recorded bundle revision for the -exact candidate serving the synthetic request. +entries for the synthetic request in append-only storage, and a recorded package digest for the +exact package serving the synthetic request. ## Clean up the request credential diff --git a/docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx b/docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx index 04495f22b3..6ef3c83047 100644 --- a/docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx +++ b/docs/site/src/content/docs/tutorials/connect-a-sqlite-extract.mdx @@ -139,16 +139,16 @@ Never replace bytes behind a running immutable connection. Publish a new path an Create a complete governed target by following [Build and deploy an Evidence Gateway project](../build-and-deploy-evidence-project/). Then build the editable project and run the checks -against the exact candidate and mounted extract: +against the exact package, target runtime, and mounted extract: ```sh test-skip="needs the complete governed deployment target from Build and deploy an Evidence Gateway project" evidencectl package registry-status \ --target registry-status/deployment-targets/staging \ --output candidate-staging -evidencectl doctor --runtime-config "$(pwd)/candidate-staging/runtime.yaml" -evidencectl test candidate-staging -evidence --runtime candidate-staging/runtime.yaml check +evidencectl doctor --runtime-config "$(pwd)/registry-status/deployment-targets/staging/runtime.yaml" +evidencectl test registry-status --target registry-status/deployment-targets/staging +evidence check --runtime-config registry-status/deployment-targets/staging/runtime.yaml ``` The last command opens the bound extract, validates its metadata and fixed statement, and refuses diff --git a/docs/site/src/content/docs/tutorials/deploy-evidence-from-breg.mdx b/docs/site/src/content/docs/tutorials/deploy-evidence-from-breg.mdx index 64457b629b..e281ab8bb8 100644 --- a/docs/site/src/content/docs/tutorials/deploy-evidence-from-breg.mdx +++ b/docs/site/src/content/docs/tutorials/deploy-evidence-from-breg.mdx @@ -94,7 +94,7 @@ existing configuration, not an overlay. Review every environment-dependent field | Question governance | The requirement owner approves requirement, concept, framework, Evidence Type and disclosure-family identifiers. Replace example identifiers consistently when using institutional meaning. | | Source connection | The registry administrator supplies the HTTPS BReg origin, token endpoint, source scope, applicable resource audience and dedicated credential references. | | Signing | The signing operator supplies the public JWK and exact non-exportable Transit key version and workload-local proxy binding. | -| Runtime | The deployment operator supplies the absolute candidate, secret and audit paths, private listener, HTTPS reverse proxy and any private CA files. | +| Runtime | The deployment operator supplies the absolute installed package, secret and audit paths, private listener, HTTPS reverse proxy and any private CA files. | Use a compatible OpenID Connect issuer for an operated deployment. Its source registration and Evidence caller registration have different audiences and grants. A source credential must not @@ -167,8 +167,8 @@ signer: ``` Use the operator's exact selected version, which may differ from this example. -Set `runtime.bundleDirectory` to the final deployment location, such as -`/srv/evidence/candidates/operated-001/bundle`. The local build output may live +Set `runtime.package.root` to the final deployment location, such as +`/srv/evidence/packages/operated-001`. The local package output may live elsewhere before transport. Bind production secret and audit paths separately. ```sh @@ -192,25 +192,23 @@ ordinary files or choose another directory. ```sh evidencectl package ./evidence --target ./evidence/targets/production \ --output ./candidate-operated-001 -evidence --runtime ./candidate-operated-001/runtime.yaml check +evidence check --runtime-config ./evidence/targets/production/runtime.yaml ``` -Build uses the matching Evidence runtime to validate the bundle and required -fixtures before publishing. A refused build reports which check objected -(`Evidence rejected the generated deployment bundle` or `Evidence rejected a -deployment fixture`) and names the exact `evidencectl test` command to -rerun for the full diagnosis; it publishes nothing at `--output` until every -check passes. The check then validates and compiles the complete -immutable bundle and validates the mounted secret material exactly as startup -does, and reports the bundle revision, the runtime revision and the number of -requirements it compiled. Run it in a context where the runtime file's absolute -`bundleDirectory` and secret root already resolve: a build host that does not -carry those paths fails the check until the candidate is placed. Neither command +Package uses the matching Evidence runtime to validate the generated package and required +fixtures before publishing. A refused package publishes nothing at `--output`; rerun the exact +`evidencectl test` command for the complete fixture diagnosis. The check then verifies and compiles +the installed immutable package and validates mounted secret material exactly as startup does. It +reports the package digest and the number of requirements it compiled. Run it in a context where +the target runtime file's absolute `package.root` and secret root already resolve: a packaging host +that does not carry those paths cannot run the target-host check until the package is installed. +Neither command contacts the source, caller issuer or Transit service. Hand over the authored revision, operated source-export baseline, complete target, -exact candidate and revisions, required secret references, and the approved -synthetic verification case. The candidate contains `runtime.yaml` and `bundle/`. +exact package digest and package, the separate target runtime, required secret references, and +the approved synthetic verification case. The package contains `SHA256SUMS` and governed Evidence +artifacts. It contains no `runtime.yaml`. Development services, credentials, tokens and live responses stay outside it. Named connections share an HTTP client, eligible OAuth token cache and bounded @@ -227,8 +225,8 @@ identity uses the [source credential rotation procedure](../../products/registry ## Provision and check the target host -The operator places the exact candidate at the configured location and makes its -runtime and bundle non-writable to the service identity. Provision owner-only +The operator places the exact package at the configured `package.root` and makes the +package and separately supplied runtime non-writable to the service identity. Provision owner-only source client files, independent audit and subject-binding HMAC secrets, durable audit storage, private CA files when needed, and the workload-local Transit proxy. @@ -243,12 +241,12 @@ as disclosed and replaced through the operator's own provisioning. Under the intended service identity, after those dependencies exist: ```sh -evidencectl doctor --runtime-config /srv/evidence/candidates/operated-001/runtime.yaml -evidence --runtime /srv/evidence/candidates/operated-001/runtime.yaml check --require-runtime-dependencies -evidence --runtime /srv/evidence/candidates/operated-001/runtime.yaml serve +evidencectl doctor --runtime-config '/runtime.yaml' +evidence check --runtime-config '/runtime.yaml' --require-runtime-dependencies +evidence serve --runtime-config '/runtime.yaml' ``` -Use your configured candidate path. `evidencectl doctor` reports every artifact +Use your configured deployment target. `evidencectl doctor` reports every artifact the runtime would otherwise refuse at startup for wrong permissions, ownership or secret exposure, all in one pass instead of one restart at a time; fix each named artifact and rerun it until it exits zero. If `check diff --git a/docs/site/src/content/docs/tutorials/evidence-from-breg.mdx b/docs/site/src/content/docs/tutorials/evidence-from-breg.mdx index e9229a984f..1bea277a00 100644 --- a/docs/site/src/content/docs/tutorials/evidence-from-breg.mdx +++ b/docs/site/src/content/docs/tutorials/evidence-from-breg.mdx @@ -317,6 +317,17 @@ contract, export to a fresh directory, review `evidencectl source diff`, then ru and match its runtime path. A shared folder grants no source authority, selects no production target, and deploys neither component. +A renamed or removed field changes the fact the export produces. When a +derivation still reads the old fact name, `source diff`, `source update`, and +`check` refuse with `evidence.authoring.derivation-fact-undeclared` and name +the derivation file. Update the derivation to the new fact name in the same +change. The check reads the project, not the running registry: a field renamed +in a registry that already serves a deployed candidate still fails that +candidate's requests, so change the registry contract and deploy the rebuilt +candidate together. + +{/* Evidence: crates/registry-evidence-authoring/src/derivation.rs; crates/registry-evidencectl/src/authoring.rs. */} + The local `dev` run rehearses the source connection with generated caller authority. For a complete production target and coordinated cutover, follow [Deploy Evidence with a BReg source](../deploy-evidence-from-breg/). diff --git a/docs/site/src/content/docs/tutorials/first-render-document.mdx b/docs/site/src/content/docs/tutorials/first-render-document.mdx index a426fc97d2..64608c5436 100644 --- a/docs/site/src/content/docs/tutorials/first-render-document.mdx +++ b/docs/site/src/content/docs/tutorials/first-render-document.mdx @@ -1,6 +1,6 @@ --- title: Render your first document -description: Scaffold a Registry Render bundle, render offline, seal it, serve it on loopback, render over HTTP, and check that both renders are the same bytes. +description: Scaffold Registry Render source, render offline, package it, serve it on loopback, render over HTTP, and check that both renders are the same bytes. status: draft owner: registry-docs source_repos: @@ -14,7 +14,7 @@ standards_referenced: [] --- If you are evaluating Registry Render as the renderer for your registry's printed documents, start -with one bundle and one letter. You will scaffold a working bundle, render it offline, seal it, +with one bundle and one letter. You will scaffold working source, render it offline, package it, serve it on your machine, render the same document over HTTP, and check that the served PDF is byte-identical to the offline one. Along the way you will see the two hashes worth storing on a record and the audit trail the service keeps. @@ -77,33 +77,42 @@ the output line with both hashes: wrote render-work/letter.pdf (13292 bytes, pdf sha256 1e2dc9c981d19ef62549c3100395a106bc8fb30e01bef21c26844e45a0c29f9f, data sha256 ad8d63dc40da68da4b3bc96144969c6ec85000cb460d9cbfe37c82688a944523) ``` -The compile also prints a note that the bundle is unsealed. That is expected while authoring: -compile accepts unsealed bundles, serve does not. If it fails instead, the problem names the file -and the line, with exit code 12. +Raw source is expected here: compile, validate, and check are authoring commands as well as package +inspection commands. Serve is the boundary that always requires the shared package envelope. If +compile fails, the problem names the file and the line, with exit code 12. {/* Evidence: crates/registry-render/src/cli.rs, Compile and parse_issued_at; crates/registry-render/src/problem.rs, exit_code. */} -## Seal the bundle +## Build the package ```sh -target/release/registry-render seal --bundle render-work/bundle target/release/registry-render check --bundle render-work/bundle +target/release/registry-render package \ + --bundle render-work/bundle \ + --output render-work/package \ + --revision tutorial-1 +target/release/registry-render check --bundle render-work/package ``` -`registry-render seal` writes a per-file sha256 for every governed file into the bundle's -manifest, and `registry-render check` verifies structure, hashes, label script coverage, and -label key sets. You know both worked when check prints the document line and the bundle line: +The first check verifies manifest structure, label script coverage, and label key sets before you +package anything. `registry-render package` copies those exact validated source files into a new +directory and writes sorted `SHA256SUMS`; `--revision` adds operator metadata that is hashed like +every other package file. The package command prints its `sha256:` digest. The second check +verifies that envelope before inspecting the same product content. You know it worked when check +prints the document line and the bundle line: ```text document letter v1 entry templates/letter.typ labels [en] pdf plain -bundle render-work/bundle v1 hash 1c15a689885df6ac (19 fonts, 1 documents, 14 governed files) +bundle render-work/package v1 hash (19 fonts, 1 documents, 14 governed files) ``` -The seal is what serve trusts, not the directory listing: after sealing, a changed file is refused -by name. Seal again after any intentional template edit. +The shared envelope is what serve trusts, not the directory listing: a changed, missing, or extra +file is refused by name. After an intentional template edit, build a new output directory. The +package command refuses to replace an existing directory. -{/* Evidence: crates/registry-render/src/bundle.rs, seal and load_sealed; +{/* Evidence: crates/registry-render/src/cli.rs, Package; + crates/registry-render/src/bundle.rs, load_package and bind_verified_snapshot; crates/registry-render/src/check.rs, run. */} ## Serve on your machine @@ -117,18 +126,22 @@ openssl rand -hex 32 > render-work/deploy/api.key chmod 600 render-work/deploy/api.key ``` -Then write `render-work/deploy/runtime.yaml`. Every path in it is relative to the file's own -directory, the same anchor the `secret:file/…` references use, so the file works from any working -directory: +Then write `render-work/deploy/runtime.yaml`. Every path in it is absolute and free of symbolic +links, so the file works from any working directory, and the `secret:file/…` references resolve +under the declared `secretProviders.file.root`: ```sh -cat > render-work/deploy/runtime.yaml <<'EOF' -apiVersion: render.registrystack.org/v1alpha1 -kind: RenderRuntime -server: +work="$(cd render-work && pwd -P)" +cat > render-work/deploy/runtime.yaml <","bundleVersion":1,"rendererVersion":" (typst 0.15.1)","status":"ok","typstPin":"0.15.1"} ``` -If startup fails instead, check the two relative paths: serve resolves `bundle.path` and -`audit.path` from the runtime file's directory, wherever you launched it from. +If startup fails instead, the refusal names the file and the field: check that `package.root` is +the package directory and that the API key file sits under `secretProviders.file.root`. {/* Evidence: crates/registry-render/src/runtime.rs, RenderRuntime and validate_bind; crates/registry-render/src/audit.rs, RenderAudit. */} diff --git a/docs/site/src/content/docs/tutorials/integrate-evidence-candidate-with-docker-compose.mdx b/docs/site/src/content/docs/tutorials/integrate-evidence-candidate-with-docker-compose.mdx index 0cc026d88c..6085104877 100644 --- a/docs/site/src/content/docs/tutorials/integrate-evidence-candidate-with-docker-compose.mdx +++ b/docs/site/src/content/docs/tutorials/integrate-evidence-candidate-with-docker-compose.mdx @@ -21,7 +21,7 @@ import QuickstartMeta from '../../../components/QuickstartMeta.astro'; Start with the approved candidate created in [Build and deploy an Evidence Gateway project](../build-and-deploy-evidence-project/). This guide runs that candidate in an existing Docker Compose application. Compose is an operator-owned deployment -adapter, not output from `evidencectl package`. The candidate bundle stays unchanged across host and +adapter, not output from `evidencectl package`. The package stays unchanged across host and container deployments. /etc/registry-evidence/bundle read-only +candidate -> /etc/registry-evidence/package read-only runtime.docker.yaml -> /etc/registry-evidence/runtime.yaml read-only Evidence secret root -> /run/secrets/registry-evidence read-only Evidence audit volume -> /var/lib/registry-evidence writable Transit socket directory -> /run/registry-evidence socket access ``` -Keep the bundle and runtime read-only. A read-only mount establishes their immutability, but does +Keep the package and runtime read-only. A read-only mount establishes their immutability, but does not waive secret ownership or mode validation. The maintained Evidence Gateway image runs as UID and GID `65532`; pin that identity in the Compose file and make every secret file acceptable to it with the required owner-only permissions. A different reviewed image requires an explicitly reviewed UID @@ -89,7 +89,7 @@ services: user: "65532:65532" read_only: true volumes: - - /bundle:/etc/registry-evidence/bundle:ro + - :/etc/registry-evidence/package:ro - ./runtime.docker.yaml:/etc/registry-evidence/runtime.yaml:ro - :/run/secrets/registry-evidence:ro - evidence-audit:/var/lib/registry-evidence @@ -112,23 +112,24 @@ ownership, paths, and trust files are in place: ```sh docker compose run --rm evidence \ - --runtime /etc/registry-evidence/runtime.yaml check --require-runtime-dependencies + check --runtime-config /etc/registry-evidence/runtime.yaml --require-runtime-dependencies ``` It prints one line and exits zero: ```text -Evidence deployment / passed check ( requirements) +Evidence package passed check ( requirements) ``` -Both revisions are your own candidate's, and the count is the number of requirements in your -bundle, so the line will not match anyone else's. +The digest is your own package's, and the count is the number of requirements in that package, so +the line will not match anyone else's. -Both forms of the command compile the bundle, compile the source plans, refuse a mounted extract +Both forms of the command compile the verified package, compile the source plans, refuse a mounted extract already older than its source allows, validate the mounted secret material, and initialize the configured signer, which signs a self-test message through the Transit proxy and verifies it against the governed public JWK. That is why the proxy starts first: without a reachable socket -either form stops at `evidence: runtime signing initialization failed`. +either form stops at `evidence: runtime signing initialization failed: the Transit provider did not +answer on the configured Unix socket (missing socket, refused connection, or timeout)`. `--require-runtime-dependencies` adds the dependencies a served request would need. It opens the configured audit writer exactly as `serve` would: creating the audit file and its lock if they are @@ -137,8 +138,16 @@ access-token issuer key set fail-closed and resolves every configured source cre no audit event of its own, but run beside a running service it refuses instead, because the writer is a single-writer destination and the running service already holds its lock. -Changing only the container runtime does not change the governed bundle, so it does not require the -fixture suite to run again. Run fixtures again when the bundle changes. +Opening the audit path takes its single-writer lock, so a candidate that shares that path with a +running instance is refused with `another process holds the single-writer lock beside the audit +file`. To +check the candidate before cutover without stopping that instance, add `--without-audit-lock`. The +audit hash key is still checked, and the audit directory and files are still checked for +ownership, mode, write access, and a complete final entry. The lock stays with the running +instance. + +Changing only the container runtime does not change the governed package, so it does not require the +fixture suite to run again. Run fixtures again when the package changes. ## Start the service @@ -147,8 +156,8 @@ docker compose up -d docker compose ps ``` -The service reads its runtime from `REGISTRY_EVIDENCE_RUNTIME`, which the maintained image already -points at `/etc/registry-evidence/runtime.yaml`, and its default command is `serve`. No further +The maintained image's default command is `serve --runtime-config +/etc/registry-evidence/runtime.yaml`, the path the runtime file is mounted at. No further arguments are needed. Evidence Gateway writes line-delimited JSON to stdout. It announces the listener only after every @@ -158,7 +167,7 @@ listener is bound, so the announcement means the port is this deployment's and n docker compose logs evidence | grep 'evidence service listening' ``` -That record carries the bundle revision, the runtime revision, the bind host, and the port. If no +That record carries the package digest, the bind host, and the port. If no such line appears, read the whole log: startup failures are reported there, and the container will have exited. @@ -166,14 +175,14 @@ The runtime in this guide binds a Compose-network address with no published port reachable from another service on the same Compose network and not from your host. Publish a port only behind the TLS-terminating service you control. -## Keep revisions distinct +## Keep package and requirement digests distinct -The bundle revision covers exact bundle bytes and remains the same in host and Compose deployments. -The runtime revision covers exact runtime bytes and bound private CA files, so it changes with -container paths, listener bindings, or trust files. A configuration revision is narrower than either: -it covers one requirement's own configuration and artifacts. Signed assertions carry the revision of -the requirement they answer as `configurationRevision`, never the runtime or bundle revision. No -revision contains secret values or audit contents. +The package digest covers the exact `SHA256SUMS` bytes and remains the same in host and Compose +deployments. Runtime paths, listener bindings, trust files, secrets, and audit contents remain +outside that package identity. A configuration revision is narrower than the package digest: it +covers one requirement's own configuration and artifacts. Signed assertions carry the revision of +the requirement they answer as `configurationRevision`, never the whole-package digest. Neither +digest contains secret values or audit contents. ## Stop without deleting the audit history diff --git a/docs/site/src/content/docs/tutorials/move-evidence-to-production-signing.mdx b/docs/site/src/content/docs/tutorials/move-evidence-to-production-signing.mdx index 99035780d2..4a859b6f27 100644 --- a/docs/site/src/content/docs/tutorials/move-evidence-to-production-signing.mdx +++ b/docs/site/src/content/docs/tutorials/move-evidence-to-production-signing.mdx @@ -265,7 +265,7 @@ Start the proxy before checking Evidence Gateway. Run the checks from the final the commands see the same socket, public keys, paths, ownership, and secret roots as the service: ```sh -evidence check --runtime "/runtime.yaml" +evidence check --runtime-config "/runtime.yaml" ``` Each check reads provider metadata, verifies the pinned version and custody controls, compares the diff --git a/docs/site/src/content/docs/tutorials/prove-an-evidence-project.mdx b/docs/site/src/content/docs/tutorials/prove-an-evidence-project.mdx index 63ef88b979..0c67abb5e8 100644 --- a/docs/site/src/content/docs/tutorials/prove-an-evidence-project.mdx +++ b/docs/site/src/content/docs/tutorials/prove-an-evidence-project.mdx @@ -161,12 +161,12 @@ public problem and the audit log instead. ## Check the deployable inputs -An editable project has no runtime document to check. Once `evidencectl package` has produced a -candidate, that candidate is what carries `runtime.yaml` beside `bundle/`, and `evidence check` -runs against it: +An editable project has no runtime document to check. Its deployment target keeps `runtime.yaml` +outside the package and points `package.root` at the installed package. Run `evidence check` +against that target runtime after installing the package: ```sh -evidence check --runtime "/runtime.yaml" +evidence check --runtime-config "/runtime.yaml" ``` This validates the complete captured bundle, scripts, schemas, codelists, key references, and @@ -186,13 +186,14 @@ from the editable project and one explicit production target. Build and hand off only when these facts are recorded together: -1. The fixture suite passed against the editable inputs and the generated candidate. -2. `evidence check` accepted the exact candidate runtime and bundle. +1. The fixture suite passed against the editable inputs and generated package. +2. `evidence check` accepted the exact installed package and target runtime. 3. A reviewer approved the source projection, cardinality mapping, requirement, derivation, codelists, purposes, audiences, and privacy expectations. 4. The environment supplies a governed public signing key and matching Transit binding, plus independent owner-only subject-binding and audit secrets. -5. The deployed bundle revision matches the reviewed candidate. +5. The deployed package digest matches the reviewed package and the runtime pins it with + `package.expectedDigest`. 6. Readiness passes, followed by one authorized HTTP-path check using synthetic data. Build a complete governed bundle for each environment. Each target owns its identities, endpoints, diff --git a/docs/site/src/content/docs/tutorials/publish-and-consume-discovery-index.mdx b/docs/site/src/content/docs/tutorials/publish-and-consume-discovery-index.mdx index 44b2d407f7..fda4d4b708 100644 --- a/docs/site/src/content/docs/tutorials/publish-and-consume-discovery-index.mdx +++ b/docs/site/src/content/docs/tutorials/publish-and-consume-discovery-index.mdx @@ -1,6 +1,6 @@ --- title: Publish and consume a Registry Discovery index -description: Build a local Discovery index from Evidence and Relay advertisements, select exact records, and hand them to native product trust. +description: Package a local Discovery index from Evidence and Relay advertisements, select exact records, and hand them to native product trust. status: draft owner: registry-docs source_repos: @@ -20,12 +20,12 @@ standards_referenced: import QuickstartMeta from '../../../components/QuickstartMeta.astro'; -Build a local Registry Discovery index from exact Evidence Gateway and Registry Relay publication +Package a local Registry Discovery index from exact Evidence Gateway and Registry Relay publication bytes, resolve and select one record for each product, then verify both selections through the maintained native clients and adopter-owned trust. catalogRevision=sha256:b4b7195f36691c245bf49a88a248049ed899c0c41dbf1a87a386571c0dbfba0f mappingRevision=sha256:332004ca3920c498539180946e8f2637e9998ba7e49cd98f31e19d6f818857ac [operator] readiness: {"status":"ready"} [consumer] resolved evidenceType=urn:example:evidence-type:adult-status alternatives=1 [consumer] selected evidence recordId=urn:registrystack:discovery:record:sha256:676659c10ce5cc9d353f4fd2816673c7947e612151efbbe1cc4d42372d9be9d5 @@ -160,6 +161,11 @@ tuple and records one exact Relay result. The saved typed selections contain pub complete resolution context, capability matches, and revision provenance only. Rust, Node.js, and Python applications perform the same workflow through their maintained Discovery client binding. +The package digest covers the exact canonical index and optional `REVISION` through `SHA256SUMS`. +The semantic revisions above remain stable for the checked-in descriptions and mappings. The +package digest can change on a later run because the index truthfully records that run's +`originFetchedAt` and `builtAt` provenance. + The final two handoff checks revalidate each persisted selection, then repeat the exact-selection boundary against publications derived by the native products. The adopter's existing Evidence and Relay trust configurations must accept a saved selection before a credential is created or a native @@ -182,16 +188,17 @@ a collection directly. Discovery is stopped before those direct calls. | --- | --- | --- | --- | | Assertion provider | Packaged Evidence public description | Public Evidence endpoint, profile, evidence type, or issuer role changes | Publish the exact packaged bytes at the approved URL | | Data publisher | Packaged Relay public description | Public Relay endpoint, profile, semantic class, operation family, or authority role changes | Publish the exact sealed artifact bytes at the approved URL | -| Operator | `origins.yaml`, mapping files, immutable index, and `runtime.yaml` | An approved origin or mapping changes | Run offline `check`, explicit `build`, deploy the complete index, then restart Discovery | +| Operator | `origins.yaml`, mapping files, immutable package, and `runtime.yaml` | An approved origin or mapping changes | Run offline `check`, explicit `package`, deploy the complete package, pin its digest when required, then restart Discovery | | Consumer or verifier | Saved exact selection and native product trust | A catalog revision, mapping revision, origin digest, binding, or local trust decision changes | Re-evaluate local trust, then call the selected native endpoint directly | -The operator does not run a synchronization scheduler or maintain a writable catalog. A failed build -does not replace the deployed index. Deploy a complete new index and restart the process when the -change is intentional. +The operator does not run a synchronization scheduler or maintain a writable catalog. A failed +package operation does not replace the deployed package. Deploy a complete new package and restart +the process when the change is intentional. -{/* Tier-C evidence: `products/discovery/README.md` defines check, explicit build, deployment, and restart +{/* Tier-C evidence: `products/discovery/README.md` defines check, explicit package, deployment, and restart as the maintenance loop. `crates/registry-discoveryctl/tests/build.rs`, test - `failed_origin_fetch_leaves_the_previous_output_untouched`, proves atomic replacement. + `failed_origin_fetch_leaves_the_previous_output_untouched`, proves that a failed collection + leaves the prior output unchanged. `crates/registry-discovery/src/startup.rs`, test `runtime_is_closed_and_contains_no_origin_mapping_trust_or_fetch_configuration`, defines the runtime configuration boundary. `crates/registry-discovery-client/tests/native_journey.rs`, test @@ -209,7 +216,7 @@ remains active. ## What you built - The provider roles supplied two deterministic public descriptions. -- The operator checked explicit origins and mappings offline, built one immutable index, started +- The operator checked explicit origins and mappings offline, packaged one immutable index, started Discovery, and waited for `{"status":"ready"}`. - The consumer resolved one Evidence requirement and made exact Evidence and Relay selections. - The verifier applied adopter-owned native trust and used the existing clients without routing @@ -231,7 +238,7 @@ remains active. ## Next -- [Build and run a Registry Discovery index](../../configure/discovery/) with your approved HTTPS +- [Package and run a Registry Discovery index](../../configure/discovery/) with your approved HTTPS origins, deployment directory, and runtime limits. - [Registry Discovery is an index](../../explanation/discovery-as-an-index/) explains why trust and invocation stay in the native products. diff --git a/docs/site/src/content/docs/tutorials/publish-governed-sqlite-registry.mdx b/docs/site/src/content/docs/tutorials/publish-governed-sqlite-registry.mdx index 61b528a101..fd8d4f669a 100644 --- a/docs/site/src/content/docs/tutorials/publish-governed-sqlite-registry.mdx +++ b/docs/site/src/content/docs/tutorials/publish-governed-sqlite-registry.mdx @@ -586,54 +586,59 @@ a different digest here than the authoring check printed above. The revision cov files the contract points at, so recording the review changed it, and every answer the service gives will carry this value rather than the earlier one. -## Seal the package +## Package the project ```sh relayctl package . --output package ``` ```text test-expect -Sealed a deployment package. artifacts, files. - package version relay.registrystack.org/package/v1alpha3 - package revision sha256: +Wrote a deployment package. artifacts, files. + package digest sha256: contract revision sha256: - artifact bindings source schema fingerprints registry sha256:b3c73e50829bf63f8034bac74ce23c9b387fa4e84ca0afc27bb98d5eccc0fe18 ``` -That report summarizes `package/relay-package.json`, which names every file and artifact in -the package with a digest for each and records the full observed schema of every source. The -package holds the compiled contract, the generated artifacts, and the governed files. It -does not hold the database. Packaging recompiles under the production profile, so a package -cannot be produced from a revision that would fail `check --production`. +The package holds the compiled contract, the generated artifacts, and the governed files, plus a +`SHA256SUMS` file that lists the digest of every other file. The package digest is the digest of +`SHA256SUMS`, and it is the value `package.expectedDigest` pins. The package does not hold the +database. Packaging recompiles under the production profile, so a package cannot be produced from +a revision that would fail `check --production`. ## Serve it -`runtime.yaml` from `relayctl init` already points at `registry.sqlite` and `package`, so it -needs no edit. Read it once: +`runtime.yaml` from `relayctl init` already points at `registry.sqlite`, so it needs no edit. +Read it once: ```yaml test-excerpt="runtime.yaml" -apiVersion: relay.registrystack.org/v2alpha1 -kind: RelayRuntime -server: {bind: "127.0.0.1:8080"} -packagePath: package +apiVersion: registry.registrystack.org/relay-runtime/v1alpha1 +kind: RelayRuntimeConfig +listener: {bind: "127.0.0.1:8080"} +package: {root: "${RELAY_PACKAGE_ROOT:-/srv/relay/package}"} +secretProviders: {environment: {}} sources: {registry: {path: registry.sqlite}} -authentication: {issuer: null} audit: {destination: file, path: var/audit.jsonl} limits: {requestTimeoutMilliseconds: 1500, concurrentQueries: 8} ``` -`authentication: {issuer: null}` works here only because every access profile in the contract is -public. Add one protected profile and Relay refuses to start without a reachable token issuer, -even for requests that would have been anonymous. +`package.root` must be absolute. It reads the `RELAY_PACKAGE_ROOT` environment variable and falls +back to `/srv/relay/package` when that is unset, so you point it at the package you just wrote +without editing the file. The audit key is a secret reference, never a substituted value, and +`secretProviders.environment` is what lets it resolve from the environment. + +There is no `authentication` block because every access profile in the contract is public. Add +one protected profile and Relay refuses to start without a reachable token issuer, even for +requests that would have been anonymous. -Create the audit directory, then start the service: +Create the audit directory, then start the service. The runtime path is +absolute too: ```sh test-background="http://127.0.0.1:8080/ready" mkdir -m 700 var -relay serve --runtime runtime.yaml +export RELAY_PACKAGE_ROOT="$PWD/package" +relay serve --runtime-config "$PWD/runtime.yaml" ``` Mode `700` is required, not tidiness. The audit directory must be owner-only, and Relay refuses @@ -886,6 +891,7 @@ rm -rf business-registry | --- | --- | --- | | `project.destination_not_empty` from `relayctl init` | The target directory already has files | Initialize into a new directory name. | | `contract.yaml_invalid` from `relayctl check` | A required key is missing, an unknown key is present, or the YAML does not parse | Every key in the contract above is required. `registry.alignmentTargets` needs at least one entry. | +| `contract.environment_expression` from `relayctl check` | A value in `registry.yaml` holds an environment expression such as `${VAR}` | Substitution applies to `runtime.yaml` only. Write the value in `registry.yaml` directly. | | `resource.view_unknown` | `source.view` names something that is not a view in the database | Relay binds to views only. Add a `CREATE VIEW` for the columns you intend to publish. | | `source.schema_fingerprint_invalid` | `expectedSchemaFingerprint` is not a `sha256:` digest, usually because it was only partly pasted | Run `relayctl inspect registry.sqlite` and paste the whole value it prints. | | `source.schema_fingerprint_mismatch` | The contract holds a fingerprint from a different schema, usually because the SQL was retyped rather than copied | Run `relayctl inspect registry.sqlite` again and paste the current value. | diff --git a/docs/site/src/content/docs/tutorials/request-a-holder-bound-credential.mdx b/docs/site/src/content/docs/tutorials/request-a-holder-bound-credential.mdx index f6e32c7c16..7318f5a423 100644 --- a/docs/site/src/content/docs/tutorials/request-a-holder-bound-credential.mdx +++ b/docs/site/src/content/docs/tutorials/request-a-holder-bound-credential.mdx @@ -109,7 +109,7 @@ to and a holder-bound assertion names no audience. Check the candidate before deploying it: ```sh -evidence check --runtime /runtime.yaml +evidence check --runtime-config "/runtime.yaml" ``` A holder-bound requirement that the bundle or every grant leaves unreachable fails here, at diff --git a/docs/site/src/content/docs/tutorials/review-breg-changes-in-casework.mdx b/docs/site/src/content/docs/tutorials/review-breg-changes-in-casework.mdx index 48ec21dae9..1154488626 100644 --- a/docs/site/src/content/docs/tutorials/review-breg-changes-in-casework.mdx +++ b/docs/site/src/content/docs/tutorials/review-breg-changes-in-casework.mdx @@ -184,7 +184,7 @@ casework --runtime-config /etc/registry-casework/runtime.yaml migrate bregctl doctor --runtime-config /etc/registry-breg/runtime.yaml caseworkctl doctor --runtime-config /etc/registry-casework/runtime.yaml -breg --config /etc/registry-breg/runtime.yaml +breg --runtime-config /etc/registry-breg/runtime.yaml casework --runtime-config /etc/registry-casework/runtime.yaml serve curl --fail --silent https://registry.example.org/ready diff --git a/docs/site/src/content/docs/tutorials/rotate-evidence-signing-keys.mdx b/docs/site/src/content/docs/tutorials/rotate-evidence-signing-keys.mdx index 4a91ffc77d..f10f6c4722 100644 --- a/docs/site/src/content/docs/tutorials/rotate-evidence-signing-keys.mdx +++ b/docs/site/src/content/docs/tutorials/rotate-evidence-signing-keys.mdx @@ -47,7 +47,7 @@ has no export or plaintext-backup permission. Do not overwrite an active public key file. A candidate revision must be complete and checkable before activation. -Publishing, activating, retiring, and revoking a key each change the bundle revision you record at +Publishing, activating, retiring, and revoking a key each change the package digest you record at approval, but no requirement's `configurationRevision`. Which keys sign is trust consumers take from the JWKS and their own denylist, so they update their pinned key set and `revokedKeyIds`, not the revisions their verification policies pin. @@ -79,9 +79,10 @@ Build a new candidate, run its fixtures, and run `evidence check` in the target ```sh evidencectl package "/shared/evidence-project" \ --target "/environments//evidence" \ - --output "" -evidencectl test "" -evidence check --runtime "/runtime.yaml" + --output "" +evidencectl test "/shared/evidence-project" \ + --target "/environments//evidence" +evidence check --runtime-config "/environments//evidence/runtime.yaml" ``` Give consumers the new public key, `kid`, provider identity, activation window, and overlap period @@ -116,6 +117,15 @@ authorized synthetic request and verify its `kid` with the consumer's approved o Missing, unreadable, or mismatched signing material fails closed. Evidence Gateway never falls back to an unsigned success. +A replica that refuses to start names the cause after `evidence: runtime signing initialization +failed:`. The two version causes are the ones a rotation reaches: + +- `the runtime keyVersion is above the Transit key's latest_version`: the provider has not created + the version `signer.keyVersion` pins yet. Rotate the Transit key first, or pin the version it has. +- `the runtime keyVersion is below the Transit key's min_encryption_version and can no longer sign`: + the provider has retired the pinned version. Pin a version at or above the minimum, or lower the + minimum through the provider-administration identity. + After every replica uses the new version and the validity-plus-skew window has elapsed, remove the old version from `allowed_parameters.key_version` as part of retiring the old public key. diff --git a/docs/site/src/data/breg-api.yaml b/docs/site/src/data/breg-api.yaml index f92ab9c50c..119b35adec 100644 --- a/docs/site/src/data/breg-api.yaml +++ b/docs/site/src/data/breg-api.yaml @@ -67,7 +67,7 @@ - ['`idempotency.conflict`', '409', 'The `Idempotency-Key` was reused with a different request.'] - ['`ingestion.profile_mismatch`', '403', 'The selected profile is not the one the run bound, or does not satisfy the compiled batch route for the bound operation. A run id grants nothing on its own.'] - ['`ingestion.run_not_open`', '409', 'The run completed or was cancelled, so it accepts no chunk and no cancel. The run stays readable.'] - - ['`ingestion.run_blocked`', '409', 'The active package revision or schema fingerprint no longer matches the run binding. The run stays blocked and inspectable; continue in a successor run.'] + - ['`ingestion.run_blocked`', '409', 'The run is blocked: the active package revision or schema fingerprint no longer matches the run binding, or the import authority the run consumes closed, expired, or has too little volume left for the chunk. The run''s `blockedReason` names which. The run stays blocked and inspectable; continue in a successor run.'] - ['`ingestion.chunk_mismatch`', '409', 'The chunk index, chunk digest, or rolling prefix digest does not match the run''s expected next chunk. Nothing was written; reread the run and resubmit the exact chunk.'] - ['`ingestion.receipt_erased`', '410', 'The chunk receipt was erased with the record history it describes. The chunk and its counts stay visible in the run.'] - ['`request.timeout`', '504', 'The request exceeded `operationalTimeouts.httpRequestMilliseconds`.'] diff --git a/docs/site/src/data/cli-reference.yaml b/docs/site/src/data/cli-reference.yaml index 43d7be5751..5f928a5d90 100644 --- a/docs/site/src/data/cli-reference.yaml +++ b/docs/site/src/data/cli-reference.yaml @@ -6,7 +6,7 @@ # while this record remains a draft. schema_version: registry.cli-reference-review/v3 status: current -last_reviewed: 2026-09-26 +last_reviewed: 2026-09-27 reviewed_source_version: "0.34.0" -reviewed_catalog_sha256: 6c4b93a034e52be740434b7a2e9d2f1c5e915b3c52771224c3893d27aabf7474 -reviewed_content_sha256: aae6f3c15d74a4c4e3c9b75a13dc59bf87b42f9170af3aca459b6b34f0070d66 +reviewed_catalog_sha256: 84da470c8c2a7ba3d2a7d0cf55fd03e60d4e75da183c13dbd72bc8d5eb883a95 +reviewed_content_sha256: ce618deff6faf95a099e4173d8bb62c3a092af5851f727cb6b5d2f394377257f diff --git a/docs/site/src/data/contracts.yaml b/docs/site/src/data/contracts.yaml index 5a252870f0..ef481aa6f0 100644 --- a/docs/site/src/data/contracts.yaml +++ b/docs/site/src/data/contracts.yaml @@ -2,7 +2,7 @@ name: Relay Registry Contract Grammar owner: registry-relay status: current-source - surface: "Closed YAML grammar `relay.registrystack.org/v2alpha1` for the two documents an adopter authors: `registry.yaml` (`kind: RegistryContract`), declaring registry identity, resources, operations, access profiles, disclosure, and the closed public discovery publication; and `runtime.yaml` (`kind: RelayRuntime`), binding one deployment's package, SQLite source, authentication, audit, secrets, and listener." + surface: "Closed YAML grammars for the two documents an adopter authors: `registry.yaml` (`apiVersion: relay.registrystack.org/v2alpha1`, `kind: RegistryContract`), declaring registry identity, resources, operations, access profiles, disclosure, and the closed public discovery publication; and `runtime.yaml` (`apiVersion: registry.registrystack.org/relay-runtime/v1alpha1`, `kind: RelayRuntimeConfig`), binding one deployment's package, SQLite source, authentication, audit, secret providers, and listener." source_of_truth: label: Registry Stack contract reference url: /reference/contracts/ @@ -17,16 +17,16 @@ name: Relay Sealed Package Format owner: registry-relay status: current-source - surface: "Versioned sealed directory `relay.registrystack.org/package/v1alpha3`: the `relay-package.json` manifest with its governed-file digests, media types, and artifact visibility, alongside the compiled registry, the governed inputs, and the generated artifacts a Relay process verifies before it activates." + surface: "Sealed directory in the shared Registry Stack package format: a `SHA256SUMS` file listing the digest of every other file, alongside the authored registry, the governed inputs, the compiled registry, and the generated artifacts a Relay process verifies and re-derives before it activates." source_of_truth: - label: Relay V2 package manifest - url: https://github.com/registrystack/registry-stack/blob/2573f77d0fb59cbdae10c3b554c1b2328eeb32be/crates/registry-relay-v2/src/package.rs + label: Relay adopter tooling specification + url: /spec/rs-pr-relayctl/ consumer_note: >- `relayctl package` produces the directory and Relay verifies it before it - opens the database, audit file, token issuer, or listener. Its - `packageRevision` is an integrity digest over canonical JSON, not an - authenticity proof: packages are unsigned, so the operator's own transport - and storage are what bind a package to its author. + opens the database, audit file, token issuer, or listener. Its package + digest, the `sha256:` digest of `SHA256SUMS`, is an integrity digest, not + an authenticity proof: packages are unsigned, so the operator's own + transport and storage are what bind a package to its author. - id: registry-manifest.metadata-yaml name: Metadata Manifest diff --git a/products/breg/HISTORY.md b/products/breg/HISTORY.md index 12c04d8f08..f29effcc89 100644 --- a/products/breg/HISTORY.md +++ b/products/breg/HISTORY.md @@ -321,10 +321,15 @@ field makes the query unavailable instead of inventing today's default. Reviewed bounded data migrations append internal `migration` revisions and commit membership. They do not fabricate user patches or business events. -The supported data step is a direct reviewed `UPDATE` on one retained entity +A transactional data step is a direct reviewed `UPDATE` on one retained entity table with explicit affected-row bounds. The complete table must fit that bound, up to 1,000 rows, so before/after capture remains bounded within the transaction. -Establishing an existing-data baseline also has a 1,000-row limit. +A chunked backfill is a direct reviewed `UPDATE` of one retained entity per +chunk of at most 1,000 record identifiers; each chunk commits on its own with +one migration commit for the rows it changed, so a resumed backfill never +journals a committed chunk twice and the table size is bounded only by the +step's `maxTotalRows`. Establishing an existing-data baseline also has a +1,000-row limit. Unsupported data-changing migrations refuse before changing records. Runtime credentials do not acquire journal UPDATE or DELETE authority. diff --git a/products/breg/INGESTION-RUNS.md b/products/breg/INGESTION-RUNS.md index e46f6a6f7c..a49eafaba9 100644 --- a/products/breg/INGESTION-RUNS.md +++ b/products/breg/INGESTION-RUNS.md @@ -41,6 +41,29 @@ selected profile cannot execute to the end of every chunk, decided exactly as an import binding decides it: the profile's operations, the item route for the profile, and patch only on a mutable entity. The refused run never exists. +A profile holding the `import` operation drives the same runs, create only: +`import` mounts no item route and no batch route, so an ingestion run is the +only path an import grant can write through. Creating an import run also +requires the entity's open import authority for that profile. The authority +must be unexpired, opened under the active package revision, hold remaining +volume that covers the run's whole announced item count, and, when it pins +input digests, list the run's input digest. The input digest is the label the +caller announces for its source and the run records; the server never +receives the source and does not recompute it, so a pinned digest names the +expected file rather than proving what the chunks write. Without one the run is refused +with `precondition.failed` and never exists; with one the run binds the +authority's id. Every chunk rechecks the bound authority under a row lock in +the chunk's own transaction and counts its committed items against the +authority's volume in that transaction. A chunk the authority no longer +admits (closed, expired, exhausted, or superseded by a successor activation) +blocks the run with reason `importAuthorityClosed` and commits nothing. An +authority is opened and closed only by the migration role through +`bregctl import-authority`; the runtime role can read it and advance an open +authority's counter or record its expiry, exhaustion, or supersession, never +open, close, or reopen one. Each transition appends a +`breg-import-authority-audit/v1` record, and every run audit record under an +authority carries `importAuthorityId`. + ## Chunk protocol A submission names the run, the expected chunk index, the chunk digest, and the @@ -98,7 +121,9 @@ run. A run is `open`, `complete`, `cancelled`, or `blocked`. An open run whose package or schema binding no longer matches the active package reports -`blocked` with reason `activePackageChanged`; the report and the blocked +`blocked` with reason `activePackageChanged`; an import run whose bound +import authority no longer admits a chunk is `blocked` with reason +`importAuthorityClosed`. For `activePackageChanged`, the report and the blocked transition answer to the binding the database holds active, so a serving instance a successor activation has left stale reports and blocks the run the same way the successor does. The blocking transition verifies it changed the @@ -118,7 +143,10 @@ stale instance answers an outage instead of closing a run its successor can still resume. The last attempt is classified as `committed`, `replayed`, `invalidItem`, `refused`, `bindingChanged`, -`chunkMismatch`, `runNotOpen`, or `unavailable`. +`importAuthorityClosed`, `chunkMismatch`, `runNotOpen`, or `unavailable`. +`bindingChanged` and `importAuthorityClosed` are the attempts that blocked a +run, and match its `activePackageChanged` and `importAuthorityClosed` blocked +reasons. ## Value-free surfaces @@ -159,6 +187,7 @@ refusals. | Invalid item or business refusal | Keep the checkpoint and start a successor run. Row skipping is never a default recovery. | | Authorization lost | Refuse progress. | | Package or schema binding changed | Block the run; continue in a successor run. | +| Import authority closed, expired, exhausted, or superseded | Block the run; open a new authority and continue the uncommitted remainder in a successor run. | | Operator stop | Explicit cancel, preserving counts and audit. | ## Retention diff --git a/products/breg/README.md b/products/breg/README.md index f48ebc2633..ee97aca753 100644 --- a/products/breg/README.md +++ b/products/breg/README.md @@ -140,13 +140,16 @@ configured domain or a compatible additive schema change. 3. `bregctl package` reproduces that tested candidate and stops at `awaiting_signatures`. An external signer reviews and signs the exact `signing-input.json`; rerunning `package` with the detached signature - document publishes the verified package. The CLI accepts no private signing - key. + document publishes the verified package with `SHA256SUMS` and reports its + shared package digest. `--revision ` optionally records a source label + in the hash-covered `REVISION` file. The CLI accepts no private signing key. 4. An operator with the migration database credential runs `bregctl apply --runtime-config --package ` and then `bregctl verify --runtime-config `. Initial activation also requires `--initial`. -5. `breg --config ` serves the active package. Authorized +5. `breg --runtime-config ` verifies `SHA256SUMS`, the optional + `package.expectedDigest` pin, and BReg's existing signatures and deployment + bindings before it serves the active package. Authorized bulk operations use `bregctl data validate`, `data import`, and `data export`, which reuse the packaged plans and normal authenticated API paths. @@ -171,7 +174,10 @@ refused before initial production control-plane state or DDL is created. OIDC key resolution is deployment configuration, not governed package content. If `authentication.oidc.jwksSource` is omitted, discovery is used. An operator -can instead pin a static document through a protected secret reference: +can name the key set location directly with `kind: uri` and an `https` `uri` +(plain `http` only on an IPv4 loopback host), which the verifier fetches and +refreshes the same way it does a discovered key set, or pin a static document +through a protected secret reference: ```yaml authentication: diff --git a/products/breg/SECURITY-REVIEW-NOTES.md b/products/breg/SECURITY-REVIEW-NOTES.md new file mode 100644 index 0000000000..0f39354b4c --- /dev/null +++ b/products/breg/SECURITY-REVIEW-NOTES.md @@ -0,0 +1,238 @@ +# Base Registry Engine security review notes + +Review notes for security-sensitive Base Registry Engine changes, held in +tracked material because commit messages do not survive a squash. Each +section names the change, the threat it answers, the defaults it ships, where +Rust enforces it, the tests that pin it, and the residual risk it accepts. The +security-invariant matrix in `contracts/security-invariant-matrix.yaml` is the +row-per-invariant baseline; this file is the narrative behind the decisions +and the residuals the matrix rows do not state. + +## Import authorities + +The change adds the `import` operation and the operator-opened import +authority it requires (`crates/registry-breg/src/import_authority.rs`, +`bregctl import-authority open|close|close-expired|list`). + +### Threat + +Change control refuses `create` and `batch` as direct writes on a governed +entity, so a governed entity needs a way to receive its initial load. Without +a bound, an `import` grant would be a standing direct write that change +control does not see. The threats: + +1. An import grant writes with no operator-opened window (BREG-SEC-109). +2. An import run keeps writing after its window closed, expired, ran out, or + was superseded by a package activation (BREG-SEC-110). +3. A load writes more records than the operator approved (BREG-SEC-111). +4. The runtime role opens, closes, or reopens a window for itself + (BREG-SEC-112). +5. An import grant reaches an item or batch route outside an ingestion run + (BREG-SEC-113). + +### Enforcement and defaults + +- Run creation calls `import_authority::admit_run` in the run-creation + transaction; every chunk calls `admit_chunk` under the authority's row lock + in the chunk transaction, before any item is written, and counts the + chunk's committed items against the volume there. +- Only the migration role opens or closes an authority. The runtime role has + `SELECT` and an `UPDATE` of the counter and terminal status of an open row, + under row-level security that refuses reopening. +- One open authority per entity (partial unique index). The window defaults + to 7 days and is at most 30, with no extension. At most 16 input digests. + The authority binds the package revision active when it opened, and an + activation supersedes it. +- The operator reference and reason are stored and audited only as keyed + hashes. Every transition is collected in its transaction and appended + through the process audit writer after the commit. +- `import` mounts no item or batch route; the compiler refuses it beside + `batch`, without entity batch bounds, and without an authenticated + principal. + +### Tests + +`crates/registry-breg/tests/postgres_import_authority.rs`: +`an_import_run_is_refused_without_an_open_authority`, +`closing_the_authority_blocks_the_next_chunk_and_keeps_committed_ones`, +`every_chunk_counts_against_the_authority_until_it_is_exhausted`, +`the_runtime_role_cannot_open_close_or_reopen_an_authority`. +`crates/registry-breg/tests/import_grant_compiler.rs`: +`batch_is_still_refused_on_a_controlled_entity_and_the_message_suggests_import`. + +### Accepted residuals + +- **Input digests are labels, not verification.** The client computes the + input digest over the file it reads and announces it with the run; the + server never receives the file and does not recompute it. A pinned digest + names the file the operator expects, and a holder of the import grant can + announce a pinned digest over other items. The item volume is the bound the + server enforces. The docs and `--input-sha256` help say so. +- **Admission reserves no volume, and import is create only.** Two runs under + one authority are each admitted against the remaining volume and stop when + the counter fills; re-running lines that already committed creates their + records again. The import guide states the duplicate behaviour. +- **The runtime role can lower `committed_items`.** Its column grant and + update policy check the volume bounds, not monotonic growth, so a + compromised runtime could refill an authority's volume. This is treated as + equivalent to the runtime role's existing `INSERT` power over records. +- **Operator hash context.** The reference and reason are keyed with the + package revision as context, so one operator's reference hashes differently + across activations; `--operator-reference` and `--reason` are argv values + visible on the local host while the command runs. + +## Instance claim + +The change records which PostgreSQL database a registry serves from and +refuses a copy until an operator adopts it +(`crates/registry-breg/src/instance_claim.rs`, `bregctl instance-claim +status|adopt`). + +### Threat + +A logical restore carries a registry's committed state into another +database. Without a claim, the copy serves beside its original and the two +become divergent writers of one registry: both accept writes, admit imports, +and deliver outbox work from the same history (BREG-SEC-114). A copy could +also adopt itself through the runtime role (BREG-SEC-115), carry an import +window the operator closed after the backup (BREG-SEC-116), or, for a +registry that predates the claim, claim itself on its first apply +(BREG-SEC-117). + +### Enforcement and defaults + +- Startup and every readiness probe compare the claim with the live + database: the system identifier when both expose it, the database oid + alone otherwise. A mismatch refuses startup and answers readiness 503 with + `startup.instance_claim.mismatch`. +- The runtime role holds `SELECT` only on the claim. +- The claim is recorded only into a fresh database, one with no committed + revision and no commit head, so an existing registry must be adopted once + after its first apply on this release (a documented breaking step). +- `adopt --acknowledge-original-retired` moves the claim under the migration + role, raises its epoch, and supersedes every open import authority in one + transaction, then appends the audit response. + +### Tests + +`crates/registry-breg/tests/postgres_startup.rs`: +`a_restored_copy_refuses_to_serve_until_adopted`. +`crates/registry-breg/tests/postgres_import_authority.rs`: +`the_runtime_role_cannot_rewrite_or_remove_the_instance_claim`, +`adopting_a_restored_copy_supersedes_every_open_authority`, +`installing_the_claim_beside_committed_history_leaves_the_database_to_adopt`. + +### Accepted residuals + +- **Physical copies are not detected.** Point-in-time recovery, storage + snapshots, and base backups keep the system identifier and database oid, so + the copy matches the claim and serves without adoption, and an import + authority closed after the backup point is open again on it. BREG-SEC-116 + holds for logical restores only. Fencing the original stays with the + operator; after a physical restore, list and close every open import + authority before serving. + +## Audit retention and prune + +This change adds no BReg audit prune, export, or retention floor. BReg +audit retention is the platform audit writer's file rotation and +`retainDays`, and tamper evidence is shipping the stream to append-only +storage, as the operator documentation describes. + +## Review recovery + +The change adds `bregctl review-recovery resubmit|close` for a change-request +review its authority will not answer +(`crates/registry-breg/src/review_recovery.rs`), and orders the result poller +so fresh and webhook-signalled reviews go before reviews the authority +reported unknown. + +### Threat + +An operator action changes the state of a governed review. The threats are +resubmitting a review the authority already decided, opening a second review +beside a live one, and an unaccountable operator change to review state. + +### Enforcement and defaults + +- Both operations run in one verified migration transaction under the + registry lock, behind the operator boundary request retention uses: package, + database identity, and migration role are verified first. +- An audit `request` entry is accepted before the transaction opens and its + `response` is written after the commit, naming the request only by its + keyed reference. A commit whose response the audit destination refuses is + reported as unaudited. +- Resubmission is limited to failure codes that mean BReg stopped waiting + (`result-poll-attempts-exhausted`, `submission-recovery-expired`, + `operator-closed`) and resends the exact retained request under its + original idempotency key. A withdrawn proposal, a recorded result, an + erased request, or a proposal no longer submitted is refused by a closed + reason naming the state and code. + +### Tests + +`crates/registry-breg/tests/postgres_change_requests.rs`: +`an_operator_resubmits_or_closes_a_review_its_authority_lost`. +`crates/registry-breg/tests/postgres_review_executor.rs`: +`a_webhook_completion_makes_its_review_due_and_first_in_the_poll_queue`. + +### Accepted residuals + +- **Close does not withdraw the review at the authority.** `close` is + allowed on any accepted review without a result; BReg sends nothing to the + authority, so a reviewer there may still decide, and that late result is + then refused. `operator-closed` is resubmittable, so the close is + reversible. +- **Resubmission relies on the authority's idempotency.** If the authority + no longer honours the original key, resubmitting opens a second review. + The operator documentation says to confirm the authority lost the review + first. +- Reviews the authority reported unknown still poll until the attempt + budget fails them, ordered last. + +## Package building and byte binding + +The change wraps the signed BReg package in the shared package envelope +(`SHA256SUMS`, optional `REVISION`) and binds every consumer to the bytes the +shared verification checked (`crates/registry-breg/src/package.rs`, +`crates/registry-breg/src/runtime_config.rs`, `bregctl package`). + +### Threat + +A package file changes between verification and use, a package is swapped for +another under the same path, or an operator runs a package other than the one +they reviewed (release provenance). + +### Enforcement and defaults + +- Startup, `apply`, and operator tooling verify the shared envelope before + any database authority is used: every listed file is re-hashed, and a + changed, missing, or extra file, a symbolic link, or a special file is + refused by name. `package.expectedDigest`, when set, must equal the package + digest. +- The active and predecessor package loads read each file once and bind it + to the per-file digest the shared verification recorded, so the signature, + trust, environment, database, revision, and sequence checks run over the + verified bytes. +- `bregctl package` writes the envelope deterministically and still requires + the `bregctl test` receipt. + +### Tests + +`crates/registry-breg/tests/runtime_config.rs`: +`shared_package_envelope_and_pin_are_checked_before_startup`. +`crates/registry-bregctl/tests/cli.rs`: +`apply_refuses_a_stale_shared_envelope_before_database_authority`. +`crates/registry-breg/tests/postgres_package.rs`: +`package_builder_is_deterministic_and_local_publication_loads`, +`local_unsigned_package_rederives_every_artifact_and_refuses_filesystem_tampering`. +`crates/registry-platform-config/src/package_tests.rs` covers the shared +writer and verifier. + +### Accepted residuals + +- A freshly supplied successor package is loaded without the active + package's byte binding, because it verifies its own envelope on load. +- The OIDC issuer and JWKS URI accept loopback `http` in every environment, + as before this change; `operate/breg.mdx` recommends an `https` issuer for + production. diff --git a/products/breg/acceptance/farmer-landholding-evidence/evidence/provider/SHA256SUMS b/products/breg/acceptance/farmer-landholding-evidence/evidence/provider/SHA256SUMS new file mode 100644 index 0000000000..6b6af64035 --- /dev/null +++ b/products/breg/acceptance/farmer-landholding-evidence/evidence/provider/SHA256SUMS @@ -0,0 +1,15 @@ +860d4d10be781b4c56fd1246fbea00c205e668ca45a16be98e4f3f1901ce6bc1 adapters/source-a-prepare.rhai +7ea2668068188528ec6dd49c65de34a62ed446f543b518498c8e8ae2b5bb66eb adapters/source-a.rhai +332bd188bfe02a25fd75ee66b08d16e29fad3dee553964b2cfbac50e337dc0e0 codelists/categories.yaml +811c275b2b3a0bbda745000120a5db61a1e6b765250fe2707ce2544b62650bd0 derivations/category.rhai +e66ffc888f46253fb8323ada8f6600a4827c8b8d15187e56e7b70aa0943df317 derivations/status.rhai +62dfe6949230fe180fcde454f420abcab5821ef46f101f368f99f46f15df26a7 evidence.yaml +5a7741a8add2fbd6652298257022ac7c9aef19924daf4a17d29feefa902dbad6 fixtures/category.yaml +48c004da29c3ff18b4b45a2351f2e633273a3cef90babe83264fd360328de7eb fixtures/status.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +bc458a6d275d119ed9ed5bd686dc7ab81e9fc94134341e88303f1c68d29c3a50 schemas/category-facts.schema.yaml +9371cf2f375874ab9ec18693c68849c6a5d0293af264a7ee7aae463ed263e405 schemas/category-parameters.schema.yaml +eaa1dc09beecfd0156d7547fe6508fdd9443ae736038995feb56ab885c94247e schemas/category-response.schema.yaml +01ee3663ebc7e5f294e1e9db4b5bf9fe4efc669a039e3a8e37cf2f38d29b4564 schemas/status-facts.schema.yaml +d24b3d3f168261b4e114cf8e43b4616c8f0d4cb6baee578c4def3574788ac367 schemas/status-parameters.schema.yaml +692b27024f9d11420defb118684c315e2403bdcde01ec63e9f2fc8bc15f91f8a schemas/status-response.schema.yaml diff --git a/products/breg/acceptance/farmer-landholding-evidence/evidence/provider/evidence.yaml b/products/breg/acceptance/farmer-landholding-evidence/evidence/provider/evidence.yaml index 24a7c6806a..ea0e8e2e82 100644 --- a/products/breg/acceptance/farmer-landholding-evidence/evidence/provider/evidence.yaml +++ b/products/breg/acceptance/farmer-landholding-evidence/evidence/provider/evidence.yaml @@ -7,20 +7,21 @@ service: issuer: id: urn:example:farmer-authority authentication: - kind: oidc-access-token - issuer: http://127.0.0.1:9 - audiences: - - farmer-evidence-trial - tokenTypes: - - at+jwt - algorithms: - - ES256 - jwksUri: http://127.0.0.1:9/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: http://127.0.0.1:9 + audience: farmer-evidence-trial + tokenTypes: + - at+jwt + algorithms: + - ES256 + jwksSource: + kind: uri + uri: http://127.0.0.1:9/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: hashKeyRef: secret:file/audit-hash-key hashKeyVersion: 1 diff --git a/products/breg/acceptance/farmer-landholding-evidence/evidence/run-provider.py b/products/breg/acceptance/farmer-landholding-evidence/evidence/run-provider.py index 4ea707d8ba..d859ad03b0 100644 --- a/products/breg/acceptance/farmer-landholding-evidence/evidence/run-provider.py +++ b/products/breg/acceptance/farmer-landholding-evidence/evidence/run-provider.py @@ -7,6 +7,7 @@ import argparse import base64 import hashlib +import importlib.util import json import os from pathlib import Path @@ -25,6 +26,17 @@ import yaml +sys.dont_write_bytecode = True +PACKAGE_SUMS_PATH = ( + Path(__file__).resolve().parents[5] + / "products/evidence/scripts/generate-package-sums.py" +) +PACKAGE_SUMS_SPEC = importlib.util.spec_from_file_location("evidence_package_sums", PACKAGE_SUMS_PATH) +assert PACKAGE_SUMS_SPEC is not None and PACKAGE_SUMS_SPEC.loader is not None +PACKAGE_SUMS = importlib.util.module_from_spec(PACKAGE_SUMS_SPEC) +PACKAGE_SUMS_SPEC.loader.exec_module(PACKAGE_SUMS) + + def b64(data): return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") @@ -155,8 +167,8 @@ def do_POST(self): origin = "http://127.0.0.1:" + str(port) config = yaml.safe_load((bundle / "evidence.yaml").read_text()) config["service"]["publicOrigin"] = origin - config["authentication"]["issuer"] = issuer - config["authentication"]["jwksUri"] = issuer + "/.well-known/jwks.json" + config["authentication"]["oidc"]["issuer"] = issuer + config["authentication"]["oidc"]["jwksSource"] = {"kind": "uri", "uri": issuer + "/.well-known/jwks.json"} for source_config in config["sources"].values(): source_config["baseUrl"] = issuer for path in (bundle / "public-keys").iterdir(): @@ -165,8 +177,11 @@ def do_POST(self): write_json(bundle / public_path, signing_public) config["signing"]["activePublicJwkFile"] = public_path (bundle / "evidence.yaml").write_text(yaml.safe_dump(config, sort_keys=False)) - runtime = {"version": 1, "bundleDirectory": str(bundle), "listener": { - "bindHost": "127.0.0.1", "port": port, "tlsTermination": "operator-controlled-upstream", "trustProxyIdentityHeaders": False, + # Publish the dynamically authored package through the maintained checksum renderer. + (bundle / "SHA256SUMS").write_bytes(PACKAGE_SUMS.rendered_sum(bundle)) + runtime = {"apiVersion": "registry.registrystack.org/evidence-runtime/v1alpha1", "kind": "EvidenceRuntimeConfig", + "package": {"root": str(bundle)}, "listener": { + "bind": "127.0.0.1:" + str(port), "tlsTermination": "operator-controlled-upstream", "trustProxyIdentityHeaders": False, "maximumRequestBytes": 65536, "maximumConcurrentRequests": 16, "requestTimeoutMilliseconds": 10000, "shutdownGraceMilliseconds": 1000}, "secretProviders": {"file": {"root": str(secrets)}}, "signer": {"kind": "local-jwk", "privateKeyRef": "secret:file/signing-key"}, "audit": {"path": str(output / "audit.jsonl")}, "outboundTls": {"systemRoots": True, "trustProfiles": {}}} @@ -200,7 +215,7 @@ def stdin_stop(): stop.set() threading.Thread(target=stdin_stop, daemon=True).start() with (output / "service.log").open("wb") as log: - process = subprocess.Popen([str(args.evidence.resolve()), "--runtime", str(runtime_path), "serve"], stdout=log, stderr=log) + process = subprocess.Popen([str(args.evidence.resolve()), "serve", "--runtime-config", str(runtime_path)], stdout=log, stderr=log) try: deadline = time.monotonic() + 15 while True: diff --git a/products/breg/acceptance/farmer-landholding-evidence/tests/live_registration.py b/products/breg/acceptance/farmer-landholding-evidence/tests/live_registration.py index c810630b4f..27c6d2a82f 100644 --- a/products/breg/acceptance/farmer-landholding-evidence/tests/live_registration.py +++ b/products/breg/acceptance/farmer-landholding-evidence/tests/live_registration.py @@ -86,7 +86,7 @@ def call_count() -> int | None: return len(args.requests.read_text().splitlines()) if args.requests.exists() else 0 with (output / "server.log").open("wb") as log: - server = subprocess.Popen([str(args.breg), "--config", str(active_runtime)], stdout=log, stderr=log) + server = subprocess.Popen([str(args.breg), "--runtime-config", str(active_runtime)], stdout=log, stderr=log) try: deadline = time.monotonic() + 20 while True: diff --git a/products/breg/acceptance/person-registration-rhai/tests/live_registration.py b/products/breg/acceptance/person-registration-rhai/tests/live_registration.py index 7eae2bfd84..6382f8ac80 100644 --- a/products/breg/acceptance/person-registration-rhai/tests/live_registration.py +++ b/products/breg/acceptance/person-registration-rhai/tests/live_registration.py @@ -106,7 +106,7 @@ def request(path: str, role: str, body: dict | None = None, key: str | None = No return response.status, json.load(response) with (output / "server.log").open("wb") as log: - server = subprocess.Popen([str(args.breg), "--config", str(active_runtime)], stdout=log, stderr=log) + server = subprocess.Popen([str(args.breg), "--runtime-config", str(active_runtime)], stdout=log, stderr=log) try: deadline = time.monotonic() + 20 while True: diff --git a/products/breg/contracts/client-capabilities.json b/products/breg/contracts/client-capabilities.json index c4afe73d0d..53d35d2f33 100644 --- a/products/breg/contracts/client-capabilities.json +++ b/products/breg/contracts/client-capabilities.json @@ -8,6 +8,7 @@ "Patch", "Tombstone", "Batch", + "Import", "Revisions", "Snapshot", "SubmitRequest", diff --git a/products/breg/contracts/package-layout.yaml b/products/breg/contracts/package-layout.yaml index b2bbe3c9be..28c21f153e 100644 --- a/products/breg/contracts/package-layout.yaml +++ b/products/breg/contracts/package-layout.yaml @@ -2,6 +2,8 @@ apiVersion: registry.registrystack.org/product-contract/v1 product: breg packageVersion: v1 entries: + - {path: SHA256SUMS, role: shared-checksum-envelope, required: true} + - {path: REVISION, role: operator-revision, required: false} - {path: package.json, role: identity, required: true} - {path: effective-model.json, role: governed-model, required: true} - {path: inventories/physical-names.json, role: physical-name-inventory, required: true} diff --git a/products/breg/contracts/security-invariant-matrix.yaml b/products/breg/contracts/security-invariant-matrix.yaml index eb4796acea..a54d9c7902 100644 --- a/products/breg/contracts/security-invariant-matrix.yaml +++ b/products/breg/contracts/security-invariant-matrix.yaml @@ -109,3 +109,12 @@ invariants: - {id: BREG-SEC-106, state: enforced, targetWave: W5, threat: "Operator request-detail erasure erases change-request detail while the audit writer is unavailable.", enforcementPoint: "the request retention erase path appends a minimized retention request entry before its verified transaction opens", refusal: "Refuse the erasure as unavailable and leave the request detail unchanged when the audit writer does not accept the request entry.", negativeId: BREG-NEG-106, negativeTest: {path: crates/registry-breg/tests/postgres_request_read_retention.rs, name: request_detail_erasure_changes_nothing_when_the_audit_writer_refuses_its_request_entry}} - {id: BREG-SEC-107, state: enforced, targetWave: W5, threat: "Migration reconciliation completes or reverts a pinned target while the audit writer is unavailable.", enforcementPoint: "reconcile_under_lock appends a minimized reconciliation request entry before activating or reverting the pinned target", refusal: "Refuse the reconciliation as unavailable and leave the maintenance state, ledger, and steps unchanged when the audit writer does not accept the request entry.", negativeId: BREG-NEG-107, negativeTest: {path: crates/registry-breg/tests/postgres_migration.rs, name: real_postgres_reconciliation_changes_nothing_when_the_audit_writer_refuses_its_request_entry}} - {id: BREG-SEC-108, state: enforced, targetWave: W5, threat: "The field-encryption erase-history lifecycle scrubs request snapshots or erases history while the audit writer is unavailable.", enforcementPoint: "scrub_plaintext_request_snapshots appends a minimized lifecycle request entry once a recorded erase-and-rebaseline flip makes the run a lifecycle and before its first scrub", refusal: "Refuse the lifecycle as unavailable and leave request snapshots, record history, lifecycle progress, and coverage unchanged when the audit writer does not accept the request entry.", negativeId: BREG-NEG-108, negativeTest: {path: crates/registry-breg/tests/postgres_history_erasure.rs, name: field_encryption_erasure_resumes_rebaseline_after_final_erase_crash}} + - {id: BREG-SEC-109, state: enforced, targetWave: W5, threat: "An import grant writes a governed entity with no operator-opened window, bypassing change control.", enforcementPoint: "import run creation in the run-creation transaction (import_authority::admit_run)", refusal: "Refuse an import run with precondition.failed unless the entity holds an open, unexpired, current-revision import authority for the selected profile whose remaining volume covers the announced item count and whose pinned digests, when present, list the input digest the run announces (a client-supplied label the server records and does not recompute); the refused run never exists.", negativeId: BREG-NEG-109, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: an_import_run_is_refused_without_an_open_authority}} + - {id: BREG-SEC-110, state: enforced, targetWave: W5, threat: "An import run keeps writing after its authority closed, expired, ran out, or was superseded by a package activation.", enforcementPoint: "the ingestion chunk transaction, after the run lock and before item writes (import_authority::admit_chunk under a row lock)", refusal: "Block the run with importAuthorityClosed and commit nothing from the refused chunk; committed chunks stay.", negativeId: BREG-NEG-110, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: closing_the_authority_blocks_the_next_chunk_and_keeps_committed_ones}} + - {id: BREG-SEC-111, state: enforced, targetWave: W5, threat: "A chunk writes more items than the operator approved for the window.", enforcementPoint: "the ingestion chunk transaction counting committed items against the authority under a row lock", refusal: "Refuse a chunk whose items exceed the authority's remaining volume and record the authority exhausted.", negativeId: BREG-NEG-111, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: every_chunk_counts_against_the_authority_until_it_is_exhausted}} + - {id: BREG-SEC-112, state: enforced, targetWave: W5, threat: "The runtime role opens, closes, or reopens an import authority for itself.", enforcementPoint: "row-level security and column grants on registry_import_authorities", refusal: "Grant the runtime role only SELECT and an UPDATE of the counter and terminal status of an open row; opening and closing remain with the migration role.", negativeId: BREG-NEG-112, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: the_runtime_role_cannot_open_close_or_reopen_an_authority}} + - {id: BREG-SEC-113, state: enforced, targetWave: W5, threat: "An import grant reaches an item or batch route and writes outside an ingestion run.", enforcementPoint: "the compiler's route mounting and change-control direct-write check", refusal: "Mount no item or batch route for import, and keep refusing batch, create, or patch grants on an entity controlled for them.", negativeId: BREG-NEG-113, negativeTest: {path: crates/registry-breg/tests/import_grant_compiler.rs, name: batch_is_still_refused_on_a_controlled_entity_and_the_message_suggests_import}} + - {id: BREG-SEC-114, state: enforced, targetWave: W5, threat: "A restored copy of the database serves beside its original, so the two become divergent writers of one Registry: both accept writes, admit imports, and deliver outbox work from the same history.", enforcementPoint: "startup and readiness, against the instance claim in registry_instance_claim; the system identifier is compared when both the claim and the live database expose it, and the database oid alone otherwise", refusal: "Refuse to start and answer readiness 503 with startup.instance_claim.mismatch until an operator adopts the copy under the migration role.", negativeId: BREG-NEG-114, negativeTest: {path: crates/registry-breg/tests/postgres_startup.rs, name: a_restored_copy_refuses_to_serve_until_adopted}} + - {id: BREG-SEC-115, state: enforced, targetWave: W5, threat: "The runtime role rewrites, inserts, or removes the instance claim so a copy adopts itself.", enforcementPoint: "table grants on registry_instance_claim", refusal: "Grant the runtime role only SELECT; adopting remains with the migration role.", negativeId: BREG-NEG-115, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: the_runtime_role_cannot_rewrite_or_remove_the_instance_claim}} + - {id: BREG-SEC-116, state: enforced, targetWave: W5, threat: "A restored copy brings back an import authority the operator closed after the backup was taken, so an import grant writes through a window that no longer exists.", enforcementPoint: "InstanceClaimService::adopt, inside the adopt transaction (import_authority::supersede_every_open)", refusal: "Supersede every open import authority in the transaction that moves the claim, appending each transition record once it commits, so an import run on the adopted copy is refused until an operator opens a new authority.", negativeId: BREG-NEG-116, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: adopting_a_restored_copy_supersedes_every_open_authority}} + - {id: BREG-SEC-117, state: enforced, targetWave: W5, threat: "A restored copy of a registry that predates the claim claims itself on its first apply and serves beside its original.", enforcementPoint: "instance_claim install, in the apply that creates the claim table", refusal: "Record the claim only into a fresh database, one with no committed revision and no commit head, so a database that already holds committed history refuses to serve until an operator adopts it.", negativeId: BREG-NEG-117, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: installing_the_claim_beside_committed_history_leaves_the_database_to_adopt}} diff --git a/products/breg/contracts/security-test-traceability.yaml b/products/breg/contracts/security-test-traceability.yaml index 01175a7686..e2f8f92dec 100644 --- a/products/breg/contracts/security-test-traceability.yaml +++ b/products/breg/contracts/security-test-traceability.yaml @@ -109,3 +109,12 @@ traceability: - {id: BREG-SEC-106, state: enforced, negativeId: BREG-NEG-106, negativeTest: {path: crates/registry-breg/tests/postgres_request_read_retention.rs, name: request_detail_erasure_changes_nothing_when_the_audit_writer_refuses_its_request_entry}} - {id: BREG-SEC-107, state: enforced, negativeId: BREG-NEG-107, negativeTest: {path: crates/registry-breg/tests/postgres_migration.rs, name: real_postgres_reconciliation_changes_nothing_when_the_audit_writer_refuses_its_request_entry}} - {id: BREG-SEC-108, state: enforced, negativeId: BREG-NEG-108, negativeTest: {path: crates/registry-breg/tests/postgres_history_erasure.rs, name: field_encryption_erasure_resumes_rebaseline_after_final_erase_crash}} + - {id: BREG-SEC-109, state: enforced, negativeId: BREG-NEG-109, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: an_import_run_is_refused_without_an_open_authority}} + - {id: BREG-SEC-110, state: enforced, negativeId: BREG-NEG-110, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: closing_the_authority_blocks_the_next_chunk_and_keeps_committed_ones}} + - {id: BREG-SEC-111, state: enforced, negativeId: BREG-NEG-111, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: every_chunk_counts_against_the_authority_until_it_is_exhausted}} + - {id: BREG-SEC-112, state: enforced, negativeId: BREG-NEG-112, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: the_runtime_role_cannot_open_close_or_reopen_an_authority}} + - {id: BREG-SEC-113, state: enforced, negativeId: BREG-NEG-113, negativeTest: {path: crates/registry-breg/tests/import_grant_compiler.rs, name: batch_is_still_refused_on_a_controlled_entity_and_the_message_suggests_import}} + - {id: BREG-SEC-114, state: enforced, negativeId: BREG-NEG-114, negativeTest: {path: crates/registry-breg/tests/postgres_startup.rs, name: a_restored_copy_refuses_to_serve_until_adopted}} + - {id: BREG-SEC-115, state: enforced, negativeId: BREG-NEG-115, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: the_runtime_role_cannot_rewrite_or_remove_the_instance_claim}} + - {id: BREG-SEC-116, state: enforced, negativeId: BREG-NEG-116, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: adopting_a_restored_copy_supersedes_every_open_authority}} + - {id: BREG-SEC-117, state: enforced, negativeId: BREG-NEG-117, negativeTest: {path: crates/registry-breg/tests/postgres_import_authority.rs, name: installing_the_claim_beside_committed_history_leaves_the_database_to_adopt}} diff --git a/products/breg/evidence/default-starter/README.md b/products/breg/evidence/default-starter/README.md index 1b0c54293f..ff8ef5d67e 100644 --- a/products/breg/evidence/default-starter/README.md +++ b/products/breg/evidence/default-starter/README.md @@ -52,23 +52,25 @@ which only `source add --apply` (or the hand `source import`) provides, so `check` reports it missing and the run refuses. The copied `targets/local/settings.yaml` is the explicit loopback teaching -target for a candidate. Review its fixed authority and connection, then use -`pwd -P` to replace its absolute runtime paths: set the bundle directory to your -chosen candidate's `bundle/`, the file secret root to this project's `secrets/`, +target for development. Review its fixed authority and connection, then use +`pwd -P` to replace its absolute runtime paths: set `package.root` to your +chosen package directory, the file secret root to this project's `secrets/`, and audit storage to `audit/evidence.jsonl`. Its source endpoints use BReg's default ports `8090` and `8091`; match any ports you selected on the registry's -first start. Then build the candidate and serve the local rehearsal: +first start. Then test the editable project and serve the local rehearsal: ```sh evidencectl target new ./targets/configured --settings ./targets/local/settings.yaml \ --signing-public-key ./secrets/signing-p256-public.jwk.json -evidencectl build --project . --target ./targets/configured --output ../candidate +evidencectl test . --target ./targets/configured bregctl dev start ../registry evidencectl dev --target ./targets/local --detach ``` `dev --target` rehearses the source connection using a separate generated local caller authority. It does not serve the target's complete caller governance. +Create a separate reviewed production or evidence-grade target with HTTPS +authorities and a production signer before running `evidencectl package`. Use the maintained “Answer questions from Base Registry Engine” tutorial for record creation, requests, verification, and shutdown. diff --git a/products/breg/evidence/default-starter/targets/local/settings.yaml b/products/breg/evidence/default-starter/targets/local/settings.yaml index 322d46cd4e..8ac8754751 100644 --- a/products/breg/evidence/default-starter/targets/local/settings.yaml +++ b/products/breg/evidence/default-starter/targets/local/settings.yaml @@ -11,20 +11,21 @@ governance: issuer: id: urn:example:issuer:record-evidence authentication: - kind: oidc-access-token - issuer: http://127.0.0.1:8082 - audiences: - - urn:example:audience:record-evidence - tokenTypes: - - at+jwt - algorithms: - - ES256 - jwksUri: http://127.0.0.1:8082/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: http://127.0.0.1:8082 + audience: urn:example:audience:record-evidence + tokenTypes: + - at+jwt + algorithms: + - ES256 + jwksSource: + kind: uri + uri: http://127.0.0.1:8082/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: hashKeyRef: secret:file/audit-hmac-key hashKeyVersion: 1 @@ -73,11 +74,12 @@ governance: selectorProfile: breg-8-registry-6-record-7-by-code valueOrigin: request runtime: - version: 1 - bundleDirectory: /absolute/path/to/candidate/bundle + apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 + kind: EvidenceRuntimeConfig + package: + root: /absolute/path/to/package listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 diff --git a/products/breg/evidence/offline-starter/targets/local/settings.yaml b/products/breg/evidence/offline-starter/targets/local/settings.yaml index 8203f1a959..f8ff131c9f 100644 --- a/products/breg/evidence/offline-starter/targets/local/settings.yaml +++ b/products/breg/evidence/offline-starter/targets/local/settings.yaml @@ -13,20 +13,21 @@ governance: issuer: id: urn:example:issuer:record-evidence authentication: - kind: oidc-access-token - issuer: http://127.0.0.1:8082 - audiences: - - urn:example:audience:record-evidence - tokenTypes: - - at+jwt - algorithms: - - ES256 - jwksUri: http://127.0.0.1:8082/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: http://127.0.0.1:8082 + audience: urn:example:audience:record-evidence + tokenTypes: + - at+jwt + algorithms: + - ES256 + jwksSource: + kind: uri + uri: http://127.0.0.1:8082/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: hashKeyRef: secret:file/audit-hmac-key hashKeyVersion: 1 @@ -75,11 +76,12 @@ governance: selectorProfile: record-by-code-v1 valueOrigin: request runtime: - version: 1 - bundleDirectory: /absolute/path/to/candidate/bundle + apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 + kind: EvidenceRuntimeConfig + package: + root: /absolute/path/to/package listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 diff --git a/products/breg/evidence/starter/README.md b/products/breg/evidence/starter/README.md index cb8aa85f2b..8742cba444 100644 --- a/products/breg/evidence/starter/README.md +++ b/products/breg/evidence/starter/README.md @@ -46,7 +46,7 @@ evidencectl target new ./targets/configured --settings ./targets/local/settings. --signing-public-key ./secrets/signing-p256-public.jwk.json evidencectl source import ../exports/registry-status --project . --target ./targets/configured evidencectl fixtures run --project . --target ./targets/configured -evidencectl build --project . --target ./targets/configured --output ../candidate +evidencectl test . --target ./targets/configured ``` `evidencectl source add ./registry --project ./evidence` connects a stopped @@ -64,9 +64,10 @@ This starts Evidence and a separate stock caller issuer using generated local au it reuses the target's source connections and outbound TLS settings. The target's caller authentication and service identity remain the explicit build settings. -Fixtures and build replay recorded synthetic responses once the reviewed export is +Fixtures and tests replay recorded synthetic responses once the reviewed export is imported, and then need neither a running BReg nor source credentials to be present; before that import they refuse on the missing `registry-status` source. -Serving the candidate requires the separate caller issuer, BReg endpoint, source -credentials, signing key and audit storage named in the reviewed target. A -production deployment needs its own reviewed HTTPS target and transit signer. +Serving the generated development package requires the separate caller issuer, +BReg endpoint, source credentials, signing key and audit storage named in the +reviewed target. A production deployment needs its own reviewed HTTPS target +and transit signer before `evidencectl package` creates its immutable package. diff --git a/products/breg/evidence/starter/targets/local/settings.yaml b/products/breg/evidence/starter/targets/local/settings.yaml index 37f9702ec3..38d34fef8c 100644 --- a/products/breg/evidence/starter/targets/local/settings.yaml +++ b/products/breg/evidence/starter/targets/local/settings.yaml @@ -11,20 +11,21 @@ governance: issuer: id: urn:example:issuer:record-evidence authentication: - kind: oidc-access-token - issuer: http://127.0.0.1:8082 - audiences: - - urn:example:audience:record-evidence - tokenTypes: - - at+jwt - algorithms: - - ES256 - jwksUri: http://127.0.0.1:8082/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: http://127.0.0.1:8082 + audience: urn:example:audience:record-evidence + tokenTypes: + - at+jwt + algorithms: + - ES256 + jwksSource: + kind: uri + uri: http://127.0.0.1:8082/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: hashKeyRef: secret:file/audit-hmac-key hashKeyVersion: 1 @@ -100,11 +101,12 @@ governance: selectorProfile: breg-8-registry-6-record-22-by-registration-number valueOrigin: request runtime: - version: 1 - bundleDirectory: /absolute/path/to/candidate/bundle + apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 + kind: EvidenceRuntimeConfig + package: + root: /absolute/path/to/package listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 diff --git a/products/breg/evidence/tests/verify-composition.py b/products/breg/evidence/tests/verify-composition.py index 78a580e54d..ed146756a4 100644 --- a/products/breg/evidence/tests/verify-composition.py +++ b/products/breg/evidence/tests/verify-composition.py @@ -7,6 +7,7 @@ from __future__ import annotations import argparse +import copy import hashlib import json import os @@ -21,6 +22,37 @@ INPUTS = Path(__file__).resolve().parents[1] +def production_settings( + local_settings: dict[str, object], *, project: Path +) -> dict[str, object]: + """Turn a starter's loopback rehearsal target into an offline package target.""" + settings = copy.deepcopy(local_settings) + governance = settings["governance"] + governance["assuranceProfile"] = "production" + governance["service"]["publicOrigin"] = "https://evidence.example.test" + oidc = governance["authentication"]["oidc"] + oidc["issuer"] = "https://issuer.example.test" + oidc["jwksSource"]["uri"] = "https://issuer.example.test/.well-known/jwks.json" + connection = governance["sourceConnections"]["registry"] + connection["baseUrl"] = "https://registry.example.test" + connection["authentication"]["tokenEndpoint"] = ( + "https://issuer.example.test/oauth2/token" + ) + runtime = settings["runtime"] + runtime["package"]["root"] = "/srv/registry-evidence/package" + runtime["secretProviders"]["file"]["root"] = str(project / "secrets") + runtime["signer"] = { + "kind": "transit", + "unixSocketPath": "/run/registry-evidence/transit-proxy.sock", + "mount": "transit", + "keyName": "evidence-signing", + "keyVersion": 1, + "timeoutMilliseconds": 2000, + } + runtime["audit"]["path"] = str(project / "audit/evidence.jsonl") + return settings + + def run(binary: Path, *args: object, environment: dict[str, str]) -> str: result = subprocess.run( [str(binary), *(str(arg) for arg in args)], @@ -39,6 +71,23 @@ def run(binary: Path, *args: object, environment: dict[str, str]) -> str: return result.stdout +def run_refused(binary: Path, *args: object, environment: dict[str, str]) -> str: + result = subprocess.run( + [str(binary), *(str(arg) for arg in args)], + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=60, + check=False, + ) + if not result.returncode: + raise RuntimeError( + f"{binary.name} {args[0]} unexpectedly succeeded:\n" + result.stdout[-8192:] + ) + return result.stderr + result.stdout + + def verify(workspace: Path, binaries: dict[str, Path]) -> dict[str, object]: environment = dict(os.environ) environment.pop("REGISTRY_EVIDENCE_RUNTIME", None) @@ -71,9 +120,10 @@ def verify(workspace: Path, binaries: dict[str, Path]) -> dict[str, object]: run(binaries["evidencectl"], "new", project, "--starter", INPUTS / "starter", "--profile", "local", environment=environment) assert not list((project / "sources").iterdir()), "starter must not hold a generated copy" - settings = yaml.safe_load((project / "targets/local/settings.yaml").read_text()) - settings["runtime"]["bundleDirectory"] = str(candidate / "bundle") - settings["runtime"]["secretProviders"]["file"]["root"] = str(project / "secrets") + settings = production_settings( + yaml.safe_load((project / "targets/local/settings.yaml").read_text()), + project=project, + ) settings["runtime"]["audit"]["path"] = str(workspace / "audit/evidence.jsonl") settings_path = workspace / "resolved-settings.json" settings_path.write_text(json.dumps(settings)) @@ -89,9 +139,9 @@ def verify(workspace: Path, binaries: dict[str, Path]) -> dict[str, object]: assert len(fixtures["fixtures"]) == 2, "both questions need their own executed fixture" assert all(fixture["passed"] and fixture["evaluated_cases"] == 11 for fixture in fixtures["fixtures"]), fixtures - run(binaries["evidencectl"], "build", "--project", project, "--target", target, + run(binaries["evidencectl"], "package", project, "--target", target, "--output", candidate, environment=environment) - bundle = yaml.safe_load((candidate / "bundle/evidence.yaml").read_text()) + bundle = yaml.safe_load((candidate / "evidence.yaml").read_text()) assert list(bundle["sources"]) == ["registry-status"], "questions must reuse one source" assert list(bundle["sourceConnections"]) == ["registry"] source = bundle["sources"]["registry-status"] @@ -104,10 +154,10 @@ def verify(workspace: Path, binaries: dict[str, Path]) -> dict[str, object]: assert all(requirement["acquisition"]["source"] == "registry-status" for requirement in bundle["requirements"]) assert source["behaviorRevision"] == manifest["provenance"]["behaviorRevision"] - fact_schema = yaml.safe_load((candidate / "bundle" / source["factSchema"]).read_text()) + fact_schema = yaml.safe_load((candidate / source["factSchema"]).read_text()) assert list(fact_schema["properties"]) == ["status"], "identity must not become a fact" report = json.loads(run(binaries["evidence"], "bundle-check", "--bundle", - candidate / "bundle", "--json", environment=environment)) + candidate, "--json", environment=environment)) assert len(report["requirements"]) == 2 # Full package provenance can move without changing the consumed lookup. model_path = registry / "registry.yaml" @@ -149,15 +199,46 @@ def verify(workspace: Path, binaries: dict[str, Path]) -> dict[str, object]: )) assert not updated["conflicts"] assert all(item["change"] == "changed" for item in updated["questionRevisions"]) + # A renamed consumed field changes the fact the export produces. Both + # starter derivations still read the old fact, so review and update refuse + # the export before any file changes, rather than every request failing. + field = next(item for item in model["entities"][0]["fields"] if item["id"] == "status") + field["id"] = "lifecycle-status" + field["apiName"] = "lifecycleStatus" + for profile in model["accessProfiles"]: + for permission in profile["permissions"]: + for key in ("readableFields", "writableFields"): + if key in permission: + permission[key] = [ + "lifecycle-status" if name == "status" else name + for name in permission[key] + ] + model_path.write_text(yaml.safe_dump(model, sort_keys=False)) + renamed_export = workspace / "renamed-export" + renamed_arguments = tuple( + "lifecycle-status" if argument == "status" else argument + for argument in export_arguments + ) + run(binaries["bregctl"], *renamed_arguments, "--output", renamed_export, + environment=environment) + installed_facts = (project / source["factSchema"]).read_bytes() + for command in ("diff", "update"): + refused = run_refused( + binaries["evidencectl"], "source", command, renamed_export, + "--project", project, "--target", target, environment=environment, + ) + assert 'reads fact "status"' in refused, refused + assert (project / source["factSchema"]).read_bytes() == installed_facts return { "exportArtifacts": len(manifest["artifacts"]), "fixtureCases": sum(item["evaluated_cases"] for item in fixtures["fixtures"]), "behaviorRevision": source["behaviorRevision"], - "bundleRevision": report["bundleRevision"], + "packageDigest": report["packageDigest"], "questions": len(report["requirements"]), "provenanceOnlyRevisions": "unchanged", "consumedChangeRevisions": "both changed", "nativeSourceUpdate": "passed", + "renamedFactUpdate": "refused", } @@ -184,10 +265,10 @@ def verify_default_init(workspace: Path, binaries: dict[str, Path]) -> dict[str, assert len({client["claims"]["registry_principal"] for client in clients}) == len(clients) run(binaries["evidencectl"], "new", project, "--starter", INPUTS / "default-starter", "--profile", "local", environment=environment) - settings = yaml.safe_load((project / "targets/local/settings.yaml").read_text()) - settings["runtime"]["bundleDirectory"] = str(candidate / "bundle") - settings["runtime"]["secretProviders"]["file"]["root"] = str(project / "secrets") - settings["runtime"]["audit"]["path"] = str(project / "audit/evidence.jsonl") + settings = production_settings( + yaml.safe_load((project / "targets/local/settings.yaml").read_text()), + project=project, + ) settings_path = workspace / "resolved-settings.json" settings_path.write_text(json.dumps(settings)) run(binaries["evidencectl"], "target", "new", target, "--settings", settings_path, @@ -204,7 +285,7 @@ def verify_default_init(workspace: Path, binaries: dict[str, Path]) -> dict[str, assert len(fixtures["fixtures"]) == 1 assert fixtures["fixtures"][0]["passed"] assert fixtures["fixtures"][0]["evaluated_cases"] == 11 - run(binaries["evidencectl"], "build", "--project", project, "--target", target, + run(binaries["evidencectl"], "package", project, "--target", target, "--output", candidate, environment=environment) return {"questions": 1, "fixtureCases": 11, "defaultSourceClient": "dedicated"} @@ -550,9 +631,9 @@ def history() -> bytes: assert pair == [(project / "secrets" / name).read_bytes() for name in ["registry-client-id", "registry-client-key"]] assert registrations == (state_root / "clients.json").read_bytes() - settings = yaml.safe_load((project / "targets/local/settings.yaml").read_text()) - settings["governance"]["sourceConnections"]["registry"]["baseUrl"] = session["bregUrl"] - source_authentication = settings["governance"]["sourceConnections"]["registry"][ + local_settings = yaml.safe_load((project / "targets/local/settings.yaml").read_text()) + local_settings["governance"]["sourceConnections"]["registry"]["baseUrl"] = session["bregUrl"] + source_authentication = local_settings["governance"]["sourceConnections"]["registry"][ "authentication" ] source_authentication["tokenEndpoint"] = session["tokenEndpoint"] @@ -560,18 +641,27 @@ def history() -> bytes: source_authentication["audience"] = session["audience"] source_authentication["resource"] = session["resource"] source_authentication["scope"] = " ".join(source["scopes"]) - settings["runtime"]["bundleDirectory"] = str(candidate / "bundle") - settings["runtime"]["secretProviders"]["file"]["root"] = str(project / "secrets") - settings["runtime"]["audit"]["path"] = str(project / "audit/evidence.jsonl") + local_settings["runtime"]["package"]["root"] = str(candidate) + local_settings["runtime"]["secretProviders"]["file"]["root"] = str(project / "secrets") + local_settings["runtime"]["audit"]["path"] = str(project / "audit/evidence.jsonl") settings_path = workspace / "settings.json" - settings_path.write_text(json.dumps(settings)) + settings_path.write_text(json.dumps(local_settings)) command("evidencectl", "target", "new", target, "--settings", settings_path, "--signing-public-key", project / "secrets/signing-p256-public.jwk.json") command("evidencectl", "source", "import", exported, "--project", project, "--target", target) command("evidencectl", "fixtures", "run", "--project", project, "--target", target) - command("evidencectl", "build", "--project", project, "--target", target, "--output", candidate) + package_target = project / "targets/package" + package_settings_path = workspace / "package-settings.json" + package_settings_path.write_text(json.dumps(production_settings( + local_settings, project=project + ))) + command("evidencectl", "target", "new", package_target, "--settings", + package_settings_path, "--signing-public-key", + project / "secrets/signing-p256-public.jwk.json") + command("evidencectl", "package", project, "--target", package_target, + "--output", candidate) compiled_authentication = yaml.safe_load( - (candidate / "bundle/evidence.yaml").read_text() + (candidate / "evidence.yaml").read_text() )["sourceConnections"]["registry"]["authentication"] assert ( compiled_authentication["clientAssertionAudience"] diff --git a/products/breg/generated/authoring/registry-module.schema.json b/products/breg/generated/authoring/registry-module.schema.json index bd2de79123..278ac86918 100644 --- a/products/breg/generated/authoring/registry-module.schema.json +++ b/products/breg/generated/authoring/registry-module.schema.json @@ -2783,23 +2783,32 @@ ] }, "Operation": { - "enum": [ - "create", - "get", - "lookup", - "list", - "patch", - "tombstone", - "batch", - "revisions", - "snapshot", - "submit_request", - "revise_request", - "cancel_request", - "apply_request", - "invoke" - ], - "type": "string" + "oneOf": [ + { + "enum": [ + "create", + "get", + "lookup", + "list", + "patch", + "tombstone", + "batch", + "revisions", + "snapshot", + "submit_request", + "revise_request", + "cancel_request", + "apply_request", + "invoke" + ], + "type": "string" + }, + { + "const": "import", + "description": "Create records through a durable ingestion run, and nothing else. The\ngrant is enabled only while an operator-opened import authority is open\nfor the entity and profile. It declares no item route and no raw batch\nroute, and change control does not count it as a direct write.", + "type": "string" + } + ] }, "ReadPathSource": { "additionalProperties": false, diff --git a/products/breg/generated/authoring/registry-project.schema.json b/products/breg/generated/authoring/registry-project.schema.json index 0ab9f121f7..666d3c5029 100644 --- a/products/breg/generated/authoring/registry-project.schema.json +++ b/products/breg/generated/authoring/registry-project.schema.json @@ -3664,23 +3664,32 @@ ] }, "Operation": { - "enum": [ - "create", - "get", - "lookup", - "list", - "patch", - "tombstone", - "batch", - "revisions", - "snapshot", - "submit_request", - "revise_request", - "cancel_request", - "apply_request", - "invoke" - ], - "type": "string" + "oneOf": [ + { + "enum": [ + "create", + "get", + "lookup", + "list", + "patch", + "tombstone", + "batch", + "revisions", + "snapshot", + "submit_request", + "revise_request", + "cancel_request", + "apply_request", + "invoke" + ], + "type": "string" + }, + { + "const": "import", + "description": "Create records through a durable ingestion run, and nothing else. The\ngrant is enabled only while an operator-opened import authority is open\nfor the entity and profile. It declares no item route and no raw batch\nroute, and change control does not count it as a direct write.", + "type": "string" + } + ] }, "PackageIdentitySource": { "additionalProperties": false, diff --git a/products/breg/generated/runtime/runtime.schema.json b/products/breg/generated/runtime/runtime.schema.json index 37df6c3e66..b6413aebbe 100644 --- a/products/breg/generated/runtime/runtime.schema.json +++ b/products/breg/generated/runtime/runtime.schema.json @@ -23,6 +23,11 @@ ], "type": "string" }, + "EnvironmentSecretProviderConfig": { + "additionalProperties": false, + "description": "The environment secret provider. It takes no settings.", + "type": "object" + }, "EventDestinationDnsFamily": { "enum": [ "dualStackStrict", @@ -148,6 +153,84 @@ ], "type": "object" }, + "FileSecretProviderConfig": { + "additionalProperties": false, + "description": "The file secret provider.", + "properties": { + "root": { + "description": "Absolute directory holding one file per secret. Each file must be a\nregular file owned by the runtime user, mode 0400 or 0600, with exactly\none hard link.", + "pattern": "^/", + "type": "string" + } + }, + "required": [ + "root" + ], + "type": "object" + }, + "JwksSource": { + "description": "Where a runtime obtains the OIDC issuer's signing keys.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Read `jwks_uri` from the issuer's OpenID Connect discovery document.", + "properties": { + "kind": { + "const": "discovery", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Fetch the key set from this absolute `https` URI, skipping discovery.", + "properties": { + "kind": { + "const": "uri", + "type": "string" + }, + "uri": { + "pattern": "^https?://", + "type": "string" + } + }, + "required": [ + "kind", + "uri" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Read the key set from a secret, for deployments without network access\nto the issuer.", + "properties": { + "documentRef": { + "pattern": "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$", + "type": "string" + }, + "kind": { + "const": "static", + "type": "string" + } + }, + "required": [ + "kind", + "documentRef" + ], + "type": "object" + } + ] + }, + "ListenerBind": { + "description": "Socket address the runtime listens on, written host:port with an IP address host ([addr]:port for IPv6).", + "maxLength": 128, + "minLength": 1, + "type": "string" + }, "OidcAlgorithm": { "enum": [ "EdDSA", @@ -309,6 +392,7 @@ }, "RawAuditConfig": { "additionalProperties": false, + "description": "The key for the keyed references an audit record carries in place of raw\nidentifiers, written `audit.hashKeyRef` beside the product's own audit\nsettings.", "else": { "properties": { "path": { @@ -337,10 +421,8 @@ "description": "`file` (the default) writes a durable, rotated JSON Lines file at\n`path`; `stdout` writes one JSON line per entry to standard output." }, "hashKeyRef": { - "maxLength": 140, - "minLength": 1, - "pattern": "^(secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$", - "type": "string" + "$ref": "#/$defs/SecretReference", + "description": "Secret reference to the audit hash key." }, "path": { "default": null, @@ -638,10 +720,6 @@ ], "type": "object" }, - "RawEnvironmentSecretProviderConfig": { - "additionalProperties": false, - "type": "object" - }, "RawEventDeliveryConfig": { "additionalProperties": false, "properties": { @@ -885,20 +963,6 @@ } ] }, - "RawFileSecretProviderConfig": { - "additionalProperties": false, - "properties": { - "root": { - "maxLength": 512, - "minLength": 1, - "type": "string" - } - }, - "required": [ - "root" - ], - "type": "object" - }, "RawJwksCacheConfig": { "additionalProperties": false, "properties": { @@ -957,7 +1021,7 @@ "additionalProperties": false, "properties": { "bind": { - "type": "string" + "$ref": "#/$defs/ListenerBind" }, "publicOrigin": { "description": "Canonical HTTPS origin (loopback HTTP for local development) for QGIS\ndiscovery and pagination. Required when the registry exposes GIS collections.", @@ -979,8 +1043,8 @@ "additionalProperties": false, "properties": { "bind": { - "description": "Operator-private loopback or private numeric address and named port,\nfor example `127.0.0.1:9100`.", - "type": "string" + "$ref": "#/$defs/ListenerBind", + "description": "Operator-private loopback or private numeric address and named port,\nfor example `127.0.0.1:9100`." } }, "required": [ @@ -988,45 +1052,9 @@ ], "type": "object" }, - "RawOidcJwksSource": { - "oneOf": [ - { - "additionalProperties": false, - "properties": { - "kind": { - "const": "discovery", - "type": "string" - } - }, - "required": [ - "kind" - ], - "type": "object" - }, - { - "additionalProperties": false, - "properties": { - "documentRef": { - "maxLength": 140, - "minLength": 1, - "pattern": "^(secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$", - "type": "string" - }, - "kind": { - "const": "static", - "type": "string" - } - }, - "required": [ - "kind", - "documentRef" - ], - "type": "object" - } - ] - }, "RawOidcVerifierConfig": { "additionalProperties": false, + "description": "The OIDC issuer a runtime accepts access tokens from: the exact `iss`\nvalue, the `aud` value a token must carry, and where the issuer's signing\nkeys come from, written under `authentication.oidc` beside the product's\nown token rules.", "properties": { "accessTokenType": { "description": "The one admitted access-token `typ` semantics. Configuring the\nRFC 9068 access-token media type as `at+jwt` or\n`application/at+jwt` admits both spellings of that one type; any\nother value (for example `JWT`) admits only that token type, matched\ncase-insensitively.", @@ -1071,7 +1099,8 @@ "type": "object" }, "audience": { - "maxLength": 2048, + "description": "The audience every accepted access token must carry in `aud`.", + "maxLength": 512, "minLength": 1, "pattern": "^[^\\s\\x00-\\x1F\\x7F](?:[^\\x00-\\x1F\\x7F]*[^\\s\\x00-\\x1F\\x7F])?$", "type": "string" @@ -1088,9 +1117,10 @@ "uniqueItems": true }, "issuer": { + "description": "Exact issuer accepted in access-token `iss` claims, an absolute\n`https` URL.", "maxLength": 2048, "minLength": 1, - "pattern": "^[^\\s\\x00-\\x1F\\x7F](?:[^\\x00-\\x1F\\x7F]*[^\\s\\x00-\\x1F\\x7F])?$", + "pattern": "^https?://", "type": "string" }, "jwksCache": { @@ -1106,14 +1136,8 @@ "description": "Optional JWKS fetch and cache tuning. Defaults to bounded cache behavior." }, "jwksSource": { - "anyOf": [ - { - "$ref": "#/$defs/RawOidcJwksSource" - }, - { - "type": "null" - } - ] + "$ref": "#/$defs/JwksSource", + "description": "Where the issuer's signing keys come from. Absent reads the issuer's\nOpenID Connect discovery document." }, "leewayMilliseconds": { "format": "uint64", @@ -1200,6 +1224,7 @@ }, "RawPackageConfig": { "additionalProperties": false, + "description": "The package a runtime serves: `root` is the absolute package directory,\nand `expectedDigest`, when set, pins the package digest the runtime must\nfind there. The package digest is the digest of the package's\n`SHA256SUMS` file; see [`crate::package`].", "properties": { "activeRevision": { "maxLength": 256, @@ -1218,9 +1243,19 @@ "pattern": "^[^\\s\\x00-\\x1F\\x7F](?:[^\\x00-\\x1F\\x7F]*[^\\s\\x00-\\x1F\\x7F])?$", "type": "string" }, + "expectedDigest": { + "description": "`sha256:` label of the package digest, the digest of the package's\n`SHA256SUMS` file. When set, the runtime refuses to start on any other\npackage.", + "pattern": "^sha256:[0-9a-f]{64}$", + "type": [ + "string", + "null" + ] + }, "root": { + "description": "Absolute path of the package directory.", "maxLength": 512, "minLength": 1, + "pattern": "^/", "type": "string" }, "trustAnchorPath": { @@ -1402,32 +1437,6 @@ ], "type": "object" }, - "RawSecretProvidersConfig": { - "additionalProperties": false, - "properties": { - "environment": { - "anyOf": [ - { - "$ref": "#/$defs/RawEnvironmentSecretProviderConfig" - }, - { - "type": "null" - } - ] - }, - "file": { - "anyOf": [ - { - "$ref": "#/$defs/RawFileSecretProviderConfig" - }, - { - "type": "null" - } - ] - } - }, - "type": "object" - }, "RawSqlRoles": { "additionalProperties": false, "properties": { @@ -1477,6 +1486,62 @@ }, "type": "object" }, + "SecretProvidersConfig": { + "additionalProperties": false, + "anyOf": [ + { + "properties": { + "file": { + "$ref": "#/$defs/FileSecretProviderConfig" + } + }, + "required": [ + "file" + ] + }, + { + "properties": { + "environment": { + "$ref": "#/$defs/EnvironmentSecretProviderConfig" + } + }, + "required": [ + "environment" + ] + } + ], + "description": "The secret providers a runtime enables. A reference is resolved only by a\nprovider declared here: `secret:file/name` under `file.root`, and\n`secret:env/NAME` only when `environment: {}` is present.", + "properties": { + "environment": { + "anyOf": [ + { + "$ref": "#/$defs/EnvironmentSecretProviderConfig" + }, + { + "type": "null" + } + ], + "description": "Enables `secret:env/NAME` references, read from the process\nenvironment. Declared as an empty mapping: `environment: {}`." + }, + "file": { + "anyOf": [ + { + "$ref": "#/$defs/FileSecretProviderConfig" + }, + { + "type": "null" + } + ], + "description": "Enables `secret:file/name` references, read from files under `root`." + } + }, + "type": "object" + }, + "SecretReference": { + "description": "An exact secret reference: secret:file/name, resolved under secretProviders.file.root, or secret:env/NAME, resolved only when secretProviders.environment is declared.", + "pattern": "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$", + "type": "string" + }, "TaskGrantStatusConfig": { "additionalProperties": false, "properties": { @@ -1635,7 +1700,7 @@ "type": "object" }, "secretProviders": { - "$ref": "#/$defs/RawSecretProvidersConfig" + "$ref": "#/$defs/SecretProvidersConfig" }, "taskGrantStatus": { "default": [], diff --git a/products/breg/metadata.md b/products/breg/metadata.md index 98fa15730a..c30f29fc8f 100644 --- a/products/breg/metadata.md +++ b/products/breg/metadata.md @@ -272,3 +272,9 @@ servers, but a missing executable request contract returns an unsupported selection error. Older strict clients may reject added metadata members, so upgrade clients and servers together. These descriptors expose existing runtime capabilities without changing permissions, history retention, or database schema. + +A successful `GET /v1/registry` response carries a `Registry-Engine-Version` +header naming the engine release that served it, the same value `breg --version` +prints. It grants nothing and is absent from the concealed 404. Peers that run +in lock-step with the engine, such as a Casework source adapter, compare it with +their own release and refuse another release by name. diff --git a/products/breg/scripts/test-adopter-workflow.sh b/products/breg/scripts/test-adopter-workflow.sh index 179f771a0c..0c602dfb16 100755 --- a/products/breg/scripts/test-adopter-workflow.sh +++ b/products/breg/scripts/test-adopter-workflow.sh @@ -720,7 +720,7 @@ render_runtime_config "$temporary_root/runtime-server-v1.yaml" "$temporary_root/ "$package_revision_v1" 1 60000 "secret:file/production-runtime-url" \ "secret:file/production-migration-url" "$listener" \ "asset-site-placement-acceptance-0.1.0" -BREG_LOG=error "$breg" --config "$temporary_root/runtime-server-v1.yaml" >"$temporary_root/server-v1.log" 2>&1 & +BREG_LOG=error "$breg" --runtime-config "$temporary_root/runtime-server-v1.yaml" >"$temporary_root/server-v1.log" 2>&1 & breg_pid=$! wait_ready_status "${server_url}ready" 200 @@ -939,7 +939,7 @@ render_runtime_config "$temporary_root/runtime-server-v2.yaml" "$temporary_root/ "$package_revision_v2" 2 60000 "secret:file/production-runtime-url" \ "secret:file/production-migration-url" "$listener" \ "asset-site-placement-acceptance-0.1.0" -BREG_LOG=error "$breg" --config "$temporary_root/runtime-server-v2.yaml" >"$temporary_root/server-v2.log" 2>&1 & +BREG_LOG=error "$breg" --runtime-config "$temporary_root/runtime-server-v2.yaml" >"$temporary_root/server-v2.log" 2>&1 & breg_pid=$! wait_ready_status "${server_url}ready" 200 @@ -1114,7 +1114,7 @@ breg_pid="" render_runtime_config "$temporary_root/runtime-server-v3.yaml" "$temporary_root/build-v3/package" \ "$package_revision_v3" 3 60000 "secret:file/production-runtime-url" \ "secret:file/production-migration-url" "$listener" "asset-site-placement-acceptance-0.1.0" -BREG_LOG=error "$breg" --config "$temporary_root/runtime-server-v3.yaml" >"$temporary_root/server-v3.log" 2>&1 & +BREG_LOG=error "$breg" --runtime-config "$temporary_root/runtime-server-v3.yaml" >"$temporary_root/server-v3.log" 2>&1 & breg_pid=$! wait_ready_status "${server_url}ready" 200 printf '%s\n' '{"operation":"create","data":{"assetCode":"ASSET-PUBLIC-002","label":"Reviewed field asset","assetClass":"equipment","maintenanceNote":"Synthetic maintenance note"}}' >"$temporary_root/assets-create-v3.jsonl" diff --git a/products/breg/scripts/test-historical-workflow.sh b/products/breg/scripts/test-historical-workflow.sh index 2eb6440bf2..03352be468 100755 --- a/products/breg/scripts/test-historical-workflow.sh +++ b/products/breg/scripts/test-historical-workflow.sh @@ -700,7 +700,7 @@ start_server() { local config=$1 local url=$2 local log=$3 - BREG_LOG=error "$breg" --config "$config" >"$log" 2>&1 & + BREG_LOG=error "$breg" --runtime-config "$config" >"$log" 2>&1 & breg_pid=$! wait_ready_status "${url}ready" 200 } diff --git a/products/breg/scripts/test-postgres.sh b/products/breg/scripts/test-postgres.sh index e0109347f6..b868c3b451 100755 --- a/products/breg/scripts/test-postgres.sh +++ b/products/breg/scripts/test-postgres.sh @@ -76,6 +76,7 @@ if [[ "$lane" == all || "$lane" == postgres ]]; then --test postgres_ingestion_runs \ --test postgres_ingestion_contract \ --test postgres_ingestion_receipts \ + --test postgres_import_authority \ --test postgres_change_requests \ --test postgres_task_grants \ --test postgres_request_receipts \ diff --git a/products/breg/scripts/test-request-attachments.py b/products/breg/scripts/test-request-attachments.py index 78310a1b75..e692968fec 100755 --- a/products/breg/scripts/test-request-attachments.py +++ b/products/breg/scripts/test-request-attachments.py @@ -200,7 +200,7 @@ def do_POST(self): runtime = str(temporary / "verified-runtime.json") Path(runtime).write_text(json.dumps(configured), encoding="utf-8") configured_log = (temporary / "verified-runtime.log").open("wb") - configured_process = subprocess.Popen([str(binaries / "breg"), "--config", runtime], + configured_process = subprocess.Popen([str(binaries / "breg"), "--runtime-config", runtime], env=environment, stdout=configured_log, stderr=subprocess.STDOUT) base = f"http://127.0.0.1:{configured_port}" deadline = time.monotonic() + 30 diff --git a/products/breg/scripts/test_generated_gates.py b/products/breg/scripts/test_generated_gates.py index 03c46d40c9..9d6873b0a0 100755 --- a/products/breg/scripts/test_generated_gates.py +++ b/products/breg/scripts/test_generated_gates.py @@ -169,7 +169,7 @@ def test_adopter_workflow_uses_public_binaries_database_and_recovery(self) -> No "apply.database_configuration.refused", "author refusal changed the production database state", "apply --runtime-config", - '"$breg" --config', + '"$breg" --runtime-config', "data validate", "data import", '"assetCode":"ASSET-PUBLIC-001"', diff --git a/products/breg/scripts/test_validate_product.py b/products/breg/scripts/test_validate_product.py index 6f3a352db7..60a38659b7 100644 --- a/products/breg/scripts/test_validate_product.py +++ b/products/breg/scripts/test_validate_product.py @@ -71,7 +71,7 @@ def test_security_range_is_closed_through_field_encryption_invariants(self) -> N ) extension_rows = matrix["invariants"][24:] self.assertEqual( - [f"BREG-NEG-{index:02d}" for index in range(25, 109)], + [f"BREG-NEG-{index:02d}" for index in range(25, 118)], [invariant["negativeId"] for invariant in extension_rows], ) for invariant in extension_rows: @@ -122,7 +122,6 @@ def test_w0_crate_boundary_includes_clients_and_opt_in_runtime(self) -> None: "dep:registry-platform-audit", "dep:registry-platform-authcommon", "dep:registry-platform-buildinfo", - "dep:registry-platform-config", "dep:registry-platform-crypto", "registry-platform-crypto/transit", "registry-platform-hooks/postgres", @@ -158,7 +157,6 @@ def test_w0_crate_boundary_includes_clients_and_opt_in_runtime(self) -> None: "registry-platform-audit", "registry-platform-authcommon", "registry-platform-buildinfo", - "registry-platform-config", "registry-platform-crypto", "registry-platform-httpsec", "registry-platform-httputil", @@ -174,6 +172,14 @@ def test_w0_crate_boundary_includes_clients_and_opt_in_runtime(self) -> None: "zeroize", ): self.assertTrue(breg["dependencies"][dependency]["optional"]) + # The compiler refuses an environment expression in an authored + # project or module through the shared configuration crate, so that + # crate is part of the default compiler surface. It carries no async + # runtime, network, or database dependency. + self.assertEqual( + {"workspace": True}, + breg["dependencies"]["registry-platform-config"], + ) ctl = tomllib.loads( ( @@ -755,6 +761,30 @@ def load_with_wrong_fixture_role(path: Path): any("unexpected entry tuples" in error for error in errors), errors ) + def test_package_layout_requires_the_shared_envelope_and_optional_revision(self) -> None: + original = VALIDATOR.load_yaml + + def load_without_shared_envelope(path: Path): + value = copy.deepcopy(original(path)) + if path.name == "package-layout.yaml": + value["entries"] = [ + entry + for entry in value["entries"] + if entry["path"] not in {"SHA256SUMS", "REVISION"} + ] + return value + + errors: list[str] = [] + with mock.patch.object( + VALIDATOR, "load_yaml", side_effect=load_without_shared_envelope + ): + VALIDATOR.validate_package_layout(errors) + missing = "\n".join(errors) + self.assertIn("SHA256SUMS", missing) + self.assertIn("shared-checksum-envelope", missing) + self.assertIn("REVISION", missing) + self.assertIn("operator-revision", missing) + def test_package_layout_binds_action_inventory_and_schemas_as_optional_generated_outputs( self, ) -> None: diff --git a/products/breg/scripts/validate_product.py b/products/breg/scripts/validate_product.py index 195ae1079a..e5d401b6cb 100644 --- a/products/breg/scripts/validate_product.py +++ b/products/breg/scripts/validate_product.py @@ -30,7 +30,7 @@ CONTRACT_STATES = {"enforced", "partial", "planned"} V1_REQUIREMENT_IDS = tuple(f"BREG-V1-{index:02d}" for index in range(1, 45)) ACCEPTANCE_JOURNEY_IDS = tuple(f"BREG-J{index:02d}" for index in range(1, 24)) -SECURITY_INVARIANT_IDS = tuple(f"BREG-SEC-{index:02d}" for index in range(1, 109)) +SECURITY_INVARIANT_IDS = tuple(f"BREG-SEC-{index:02d}" for index in range(1, 118)) ACCEPTANCE_FIXTURES = { "BREG-J01": ("asset-site-placement", "acceptance/asset-site-placement"), "BREG-J02": ("asset-site-placement", "acceptance/asset-site-placement"), @@ -50,6 +50,8 @@ re.MULTILINE, ) PACKAGE_LAYOUT_ENTRIES = { + ("SHA256SUMS", "shared-checksum-envelope", True), + ("REVISION", "operator-revision", False), ("package.json", "identity", True), ("effective-model.json", "governed-model", True), ("inventories/physical-names.json", "physical-name-inventory", True), @@ -128,6 +130,7 @@ "cargo test --locked -p registry-breg --features postgres-test,tooling,schema --test postgres_ingestion_runs", "cargo test --locked -p registry-breg --features postgres-test,tooling,schema --test postgres_ingestion_contract", "cargo test --locked -p registry-breg --features postgres-test,tooling,schema --test postgres_ingestion_receipts", + "cargo test --locked -p registry-breg --features postgres-test,tooling,schema --test postgres_import_authority", "cargo test --locked -p registry-breg --features postgres-test,tooling,schema --test postgres_change_requests", "cargo test --locked -p registry-breg --features postgres-test,tooling,schema --test postgres_task_grants", "cargo test --locked -p registry-breg --features postgres-test,tooling,schema --test postgres_request_receipts", diff --git a/products/breg/wasm-handler-sdk/Cargo.lock b/products/breg/wasm-handler-sdk/Cargo.lock index edae2604e4..77bfd7a485 100644 --- a/products/breg/wasm-handler-sdk/Cargo.lock +++ b/products/breg/wasm-handler-sdk/Cargo.lock @@ -2763,6 +2763,7 @@ dependencies = [ "registry-evidence-verifier", "registry-manifest-core", "registry-platform-canonical-json", + "registry-platform-config", "registry-platform-hooks", "registry-platform-script", "rhai", @@ -2861,6 +2862,22 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "registry-platform-config" +version = "0.34.0" +dependencies = [ + "registry-platform-canonical-json", + "rustix 1.1.5", + "serde", + "serde_json", + "serde_norway", + "serde_path_to_error", + "sha2 0.11.0", + "thiserror 2.0.20", + "url", + "zeroize", +] + [[package]] name = "registry-platform-crypto" version = "0.34.0" diff --git a/products/casework/CHANGELOG.md b/products/casework/CHANGELOG.md index 2bd0cb3aa7..a5fbed826c 100644 --- a/products/casework/CHANGELOG.md +++ b/products/casework/CHANGELOG.md @@ -2,6 +2,68 @@ ## Unreleased +- BREAKING: Casework reads `runtime.yaml` through the shared Registry Stack + runtime configuration loader and declares its secret providers, database, + listener bind, OpenID Connect issuer and clients, and audit key through the + shared blocks. The runtime file may not pass through a symbolic link and is + at most 1 MiB. `${VAR}`, `${VAR:-default}`, and `${VAR:?message}` are + substituted in string values after parsing, so a substituted value is + always text; an expression in a field ending in `Ref` or under + `secretProviders` is refused, and one in `casework.yaml` is refused by the + runtime and by `caseworkctl check` with the path of the field that holds it. +- BREAKING: `listener.bind` is required; the `127.0.0.1:8100` default is + removed. +- BREAKING: `authentication.oidc.issuer` must be an absolute `https` URL + without credentials, query, or fragment, or IPv4-loopback `http` under + `development-loopback`, and `authentication.oidc.audience` is at most 512 + characters. +- BREAKING: under `operator-controlled-upstream`, + `authentication.oidc.allowedClients` must name at least one client; an + empty list admitted every client the issuer verifies and is now refused at + that field. `development-loopback` still accepts an empty list, and a + configured `taskAuthority` still requires a non-empty list in either mode. +- BREAKING: `authentication.oidc.jwksUri` is removed. Declare + `jwksSource` with `kind: uri` and the same `https` URL as `uri`; that + source fetches the key set from the fixed address without reading the + discovery document. The removed key is refused with the replacement named, + and `caseworkctl` reports it at `runtime.yaml:/authentication/oidc/jwksUri`. +- BREAKING: `audit.hashKeyRef` must be an exact secret reference when the + document is read. A refused `authentication.oidc.assertionIssuers` map is + reported at that field; its bounds are unchanged. +- BREAKING: a Casework package is the shared Registry Stack package format. + `caseworkctl package` writes `SHA256SUMS`, one `sha256sum` line per file + sorted by path, in place of `casework.package.json`, and reports + `packageDigest`, the SHA-256 digest of `SHA256SUMS`, in place of + `policyDigest`. `--revision TEXT` records one free-text line in a + `REVISION` file the digest covers. At startup the runtime refuses a changed, + missing, or extra file by name, and refuses a `package.root` that still + holds `casework.package.json`, naming `caseworkctl package`; rebuild every + deployed package with it. +- BREAKING: `package.expectedPolicyDigest` is renamed `package.expectedDigest` + and pins the package digest. The retired key is refused with its + replacement named, and a mismatch is refused in the shape every Registry + Stack runtime shares: + `package.expectedDigest is but the package at package.root is `. +- BREAKING: the runtime verifies `package.root` as a package in every listener + mode, with or without `package.expectedDigest`. A local + `development-loopback` runtime no longer serves an authored project: a + directory without `SHA256SUMS` is refused naming `caseworkctl package`. + `caseworkctl dev` packages the authored project under `.casework/dev/package` + on every start and serves that package, and the `runtime.example.yaml` + `caseworkctl init` writes serves `.casework/package`. The doctor + `pinnedWork` verdict `development` is removed, and `/version` and doctor + always report `packageDigest` as a digest, never `null`. +- The metrics listener no longer lets the scrape rate set the database load. + Scrapes within five seconds of a database reading reuse it, concurrent + scrapes wait for the one reading in flight, and a reading that takes longer + than five seconds reports `casework_database_up 0`. + +- Casework database connections now use `connect_timeout=5`, + `keepalives_idle=15`, `keepalives_interval=5`, `keepalives_retries=3`, and + `tcp_user_timeout=30`, all in seconds, unless the database URL sets them, + so a connection to a server that stopped answering fails within seconds + instead of the operating system's hours. + - Answer every audited request entry. An operation whose change is not known to have committed (a refusal, a failure, a canceled request, or a commit whose acknowledgment was lost and whose outcome could not be read @@ -51,6 +113,59 @@ `audit.path` (`audit.caseworkctl.ndjson` beside `audit.ndjson`). - The retention report no longer carries an `auditRecords` count. +- The optional `metricsListener.bind` runtime setting serves `/metrics` + (Prometheus text: build and package digest, database reachability, and + per-source reconciliation failures and last-success age) and `/version` (running version and package digest) on a second, + operator-private address. It must be loopback or private, never a wildcard, + and never the API listener's address and port. Without it no telemetry + socket opens, and the API listener is unchanged. +- BREAKING: Casework and its BReg sources run in lock-step. BReg now names its + release in a `Registry-Engine-Version` header on `GET /v1/registry`, and the + Casework BReg adapter refuses a source whose engine reports another release, + or no release, as a source outage. One trailing `-dev`, which a build + without the release marker appends, is set aside on each side, so a release + build matches a development build of the same version (with a warning); + every other part of the version, a prerelease tag included, must match. The runtime log names the source and both + versions, and `caseworkctl doctor` refuses at `sourceConnections` with the + same message and the upgrade step. Upgrade each BReg source first, then + Casework, to the same release; reads resume on the next matching contract + read. The adapter logs the engine version once when it first reads it, and + a caller's read of a registry contract that does not decode is now logged + with its route and metadata error kind instead of passing silently. +- BREAKING: the runtime refuses to activate a policy package that would strand + in-flight work pinned under an earlier package, and names each conflict with + its counts: a queue or access profile that pinned reviews or open work items + still need, a pinned review kind version declared with different content, a + removed source that source-context reviews or open work items still need, + or a source read the pinned display schema would refuse. Let that work finish, or set the new + `package.acknowledgeStrandedWork` to the exact package digest the refusal + names. `caseworkctl doctor` runs the same comparison as its `pinnedWork` + check and reports the conflicts under `pinnedWork`, so it previews the + refusal against the next package before a restart. +- BREAKING: `caseworkctl doctor` names the check that failed instead of + reporting every dependency failure as "A Casework runtime dependency check + failed." A refusal carries the code `casework.doctor.check-failed`, names the + check in its `path` (`doctor:/checks/database`, `sourceConnections`, + `directory`, `reconciliation`, `audit`, and so on), says what failed + without echoing a connection string or a source response, and suggests the + next step. `doctor` also checks each source's reconciliation health, and + its report adds `packageDigest`, the `reconciliation` readiness key, and + each source's reconciliation health. Because pinned + objects gained fields, the `caseworkctl --format json` wire contract moves + from `caseworkctl/v1alpha1` to `caseworkctl/v1alpha2`; a consumer that + matches the version must accept the new one. +- BREAKING: `GET /ready` answers `503` once a source's reconciliation has + failed five consecutive passes, and until one pass for that source succeeds. + Schema migration 18 records each pass's outcome, so every replica and + `doctor` see the same health; run `casework migrate` before serving this + release. A database that was never migrated, or that an earlier release + migrated, is now named as `the Casework database schema is not current` + with the migrate instruction, instead of as invalid stored data. +- One item a reconciliation or event-synchronization pass cannot apply no + longer stops the rest of the pass. The pass applies every other claimed + item, logs how many it could not apply, and retries each after its claim + lapses. + ## v0.34.0 - 2026-09-25 - BREAKING: give each paired BReg request entity its own lifecycle hook, so diff --git a/products/casework/README.md b/products/casework/README.md index 1f95562d6c..bc9c67a32e 100644 --- a/products/casework/README.md +++ b/products/casework/README.md @@ -387,15 +387,17 @@ caseworkctl package ./casework --output ./casework-policy-package ``` The package contains only `casework.yaml`, its exact declared source -descriptions, and `casework.package.json`. The v1alpha1 manifest records the -policy digest and a sorted path, SHA-256 digest, and byte count for every -included file. Keep operator bindings and secrets outside the package. In the -production runtime configuration, set `package.root` to the absolute path of -the installed package directory. The runtime selects only the fixed -`casework.yaml` inside that directory. The service requires and verifies the -adjacent manifest when `listener.tlsTermination` is -`operator-controlled-upstream`. Local `development-loopback` can use the -authored project directory as `package.root`. `source add --apply` updates reviewed authoring +descriptions, and `SHA256SUMS`, one `sha256sum` line per file sorted by path. +The package digest is the SHA-256 digest of `SHA256SUMS`, and `--revision TEXT` +records one free-text line in a `REVISION` file the digest covers. Keep operator bindings and secrets outside the package. In the +runtime configuration, set `package.root` to the absolute path of the +installed package directory. The runtime selects only the fixed +`casework.yaml` inside that directory, and verifies `SHA256SUMS` at every +start in every listener mode; it refuses an authored project directory and +names `caseworkctl package`. The `runtime.example.yaml` that `caseworkctl init` +writes selects `.casework/package` in the project, which +`caseworkctl package . --output .casework/package` builds, and +`caseworkctl dev` packages the project on each start. `source add --apply` updates reviewed authoring inputs; it does not activate a production package. The deployment operator installs and atomically selects the reviewed package, then restarts or rolls out Casework. Activating a new package does not rewrite running clock occurrences; diff --git a/products/casework/RUNTIME-CONFIG.md b/products/casework/RUNTIME-CONFIG.md index 5be84a8dd6..dd72dcae21 100644 --- a/products/casework/RUNTIME-CONFIG.md +++ b/products/casework/RUNTIME-CONFIG.md @@ -2,8 +2,21 @@ Casework reads one versioned operator document selected with `casework --runtime-config ABSOLUTE_FILE serve` or `migrate`. The selected file -path and every operated resource path are absolute. Local development tooling -may resolve paths before it writes the file. +path and every operated resource path are absolute, and the selected file may +not pass through a symbolic link. Local development tooling may resolve paths +before it writes the file. The file is read through the shared Registry Stack +runtime configuration loader: it must be a YAML mapping of at most 1 MiB, and +unknown keys are refused with the path of the offending field. + +String values in `runtime.yaml` may take a deployment value from the +environment when the runtime starts: `${VAR}` requires `VAR`, `${VAR:-default}` +falls back to `default`, and `${VAR:?message}` refuses to start with `message` +when `VAR` is unset. Substitution never applies to a field whose name ends in +`Ref` or `Refs`, or to any value beneath one, because a secret reference must +be written literally and resolved by a declared provider. It never applies to +the authored `casework.yaml` either: an environment expression there is +refused, by the runtime and by `caseworkctl check`, with the path of the field +that holds it. The closed envelope is: @@ -13,28 +26,58 @@ kind: CaseworkRuntimeConfig ``` `package.root` selects one directory. The runtime always loads -`package.root/casework.yaml`; no second project selector can override it. With -`listener.tlsTermination: operator-controlled-upstream`, the directory must -also contain a matching `casework.package.json`. Development loopback may -select an authored project directory without that manifest. - -`package.expectedPolicyDigest` is optional. When set, it is `sha256:` followed -by 64 lowercase hexadecimal digits, and the runtime starts only on the verified -package whose manifest names that `policyDigest`. A package naming any other -digest, or a directory without `casework.package.json`, is refused before the -runtime starts, and the refusal names the expected digest and the one found. +`package.root/casework.yaml`; no second project selector can override it. In +every listener mode, with or without `package.expectedDigest`, the directory +must be a package `caseworkctl package` wrote: a `SHA256SUMS` file listing +exactly `casework.yaml` and the source descriptions it names, each matching its +digest. A directory without `SHA256SUMS`, such as an authored project, is +refused and names `caseworkctl package`; so is a directory that holds the +retired `casework.package.json`. `caseworkctl dev` packages the authored +project on every start and serves that package. + +`package.expectedDigest` is optional. When set, it is `sha256:` followed by 64 +lowercase hexadecimal digits, and the runtime starts only on the verified +package whose package digest, the SHA-256 digest of its `SHA256SUMS`, is that +value. Any other package, or a directory without `SHA256SUMS`, is refused +before the runtime starts, and the refusal names the expected digest and the +one found. The retired `package.expectedPolicyDigest` is refused with +`package.expectedDigest` named as its replacement. Set it to the digest printed by `caseworkctl package` for the package you reviewed, so that replacing the files under `package.root` cannot change the policy a restart loads. -`listener` is required. `listener.bind` is one numeric socket address, including -bracketed IPv6 forms, and defaults to `127.0.0.1:8100` when omitted from the -listener block. `listener.tlsTermination` is required. Use +`package.acknowledgeStrandedWork` is optional and takes the same digest form. +Before it registers any source generation, the runtime compares the package it +is about to activate with the in-flight work retained in the database. Review +requests still under review pinned their kind's policy when they were +admitted, and open work items keep the queue they were routed to. The runtime +refuses a package that removes a queue or access profile that work still +needs, that declares a pinned review kind version with different content, that +removes the source of a source-context review, or whose source read would +disclose a field the pinned display schema does not declare or omit one it +requires. The refusal names each conflict with its counts and the package +digest. Let that work finish under the earlier package, or set +`package.acknowledgeStrandedWork` to that exact digest to activate the package +anyway; the acknowledgement admits only the package it names, so it never +carries over to a later one. +`caseworkctl doctor` runs the same comparison as its `pinnedWork` check, so +pointing it at a runtime file whose `package.root` holds the next package +previews the refusal before a restart. + +`listener` is required. `listener.bind` is required and is one numeric socket +address, including bracketed IPv6 forms. `listener.tlsTermination` is required. Use `operator-controlled-upstream` behind an operator-managed TLS edge or `development-loopback` for direct local development. `listener.networkExposure` defaults to `private-address`; `container-private` permits an unspecified bind only for a listener kept on a private container network. +`metricsListener` is optional and absent by default. `metricsListener.bind` is +one numeric socket address with a nonzero port, loopback or private (IPv4 +private range or IPv6 unique local), never a wildcard, and never the address +and port the API listener occupies; an IPv6 wildcard API listener counts as +occupying that port on both families. When set, the runtime serves `/metrics` +and `/version` there and nowhere else. + `secretProviders` explicitly enables each accepted reference form. Declare `file: {root: ABSOLUTE_DIRECTORY}` before using `secret:file/name`. Declare `environment: {}` before using `secret:env/NAME`. The runtime does not fall back @@ -44,10 +87,29 @@ from one provider to another. The maintained example uses mounted files. `database.migrationUrlRef` supplies the operator-run migration connection. `database.trustedRootCertificateRef` is optional. Plaintext PostgreSQL is available only to builds with the `postgres-test` feature and an explicit -`testOnlyPlaintext: true` setting. - -`authentication.oidc` requires `issuer` and `audience`. `jwksSource` defaults to -discovery and can instead select a static `documentRef`. `scopeClaim` defaults +`testOnlyPlaintext: true` setting. Unless a database URL sets them, every +Casework connection uses `connect_timeout=5`, `keepalives_idle=15`, +`keepalives_interval=5`, `keepalives_retries=3`, and `tcp_user_timeout=30`, +all in seconds as Casework reads the URL (unlike libpq, which reads +`tcp_user_timeout` in milliseconds), so a connection to a server that stopped +answering fails within seconds. The TCP user timeout applies on Linux only. + +`authentication.oidc` requires `issuer` and `audience`. The issuer is an exact +`https` URL without credentials, query, or fragment; plain `http` is accepted only for +an IPv4 loopback address under development loopback, for the issuer and for a +`kind: uri` key set alike. The audience is at most 512 +characters. `jwksSource` defaults to `kind: discovery`. `kind: uri` with `uri` +fetches the key set from a fixed HTTPS address instead of the one discovery +names. `kind: static` with `documentRef` reads a pinned key set, which does no +rotation of its own: rolling a key means replacing the referenced document and +restarting Casework. The removed `jwksUri` key is refused with a diagnostic +naming `jwksSource` `kind: uri` as its replacement. `allowedClients` lists the +client identifiers whose tokens the runtime admits, matched against the token's +`azp` claim or, when it has none, its `client_id` claim. Under +`operator-controlled-upstream` the list must name at least one client, because an +empty list admits every client the issuer verifies; development loopback keeps +an empty list as a local convenience. A configured `taskAuthority` requires a +non-empty list in either mode. `scopeClaim` defaults to `registry_scopes` for compatibility with existing deployments. Stock ThunderID emits `scope`, so the maintained example and `caseworkctl init` set that explicit override. `humanIdentity` defaults to claim @@ -58,7 +120,8 @@ Principal selection belongs exclusively to each authored `audit` selects where Casework writes its audit entries and the key that pseudonymizes the principals and identifiers they name. `audit.hashKeyRef` is -that key's secret reference. `audit.destination` is `file` (the default) or +that key's secret reference and must be an exact `secret:env/NAME` or +`secret:file/name` reference. `audit.destination` is `file` (the default) or `stdout`. A `file` destination requires the absolute `audit.path` of the active file and accepts `audit.rotateBytes` (default 104857600, at least 1048576, at most 4294967295) and `audit.retainDays` (default 90, at most 36500); `stdout` diff --git a/products/casework/contracts/cli/AttemptSettlementReport.schema.json b/products/casework/contracts/cli/AttemptSettlementReport.schema.json index 16670e3ba8..a7fba455a1 100644 --- a/products/casework/contracts/cli/AttemptSettlementReport.schema.json +++ b/products/casework/contracts/cli/AttemptSettlementReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/AttemptSettlementReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/AttemptSettlementReport.schema.json", "title": "AttemptSettlementReport", "description": "Versioned JSON report emitted by caseworkctl for attempt settle.", "oneOf": [ @@ -18,7 +18,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "AttemptSettlementReport" @@ -51,7 +51,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "AttemptSettlementReport" diff --git a/products/casework/contracts/cli/AttemptUncertainMarkingReport.schema.json b/products/casework/contracts/cli/AttemptUncertainMarkingReport.schema.json index 817f9b2dad..97d96fa82a 100644 --- a/products/casework/contracts/cli/AttemptUncertainMarkingReport.schema.json +++ b/products/casework/contracts/cli/AttemptUncertainMarkingReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/AttemptUncertainMarkingReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/AttemptUncertainMarkingReport.schema.json", "title": "AttemptUncertainMarkingReport", "description": "Versioned JSON report emitted by caseworkctl for attempt mark-uncertain.", "oneOf": [ @@ -18,7 +18,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "AttemptUncertainMarkingReport" @@ -51,7 +51,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "AttemptUncertainMarkingReport" diff --git a/products/casework/contracts/cli/CheckReport.schema.json b/products/casework/contracts/cli/CheckReport.schema.json index c1e578ab71..88ad8aee90 100644 --- a/products/casework/contracts/cli/CheckReport.schema.json +++ b/products/casework/contracts/cli/CheckReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/CheckReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/CheckReport.schema.json", "title": "CheckReport", "description": "Versioned JSON report emitted by caseworkctl for check.", "oneOf": [ @@ -22,7 +22,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "CheckReport" @@ -73,7 +73,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "CheckReport" diff --git a/products/casework/contracts/cli/DatabaseMigrationReport.schema.json b/products/casework/contracts/cli/DatabaseMigrationReport.schema.json index 25c334d9b7..50937c1944 100644 --- a/products/casework/contracts/cli/DatabaseMigrationReport.schema.json +++ b/products/casework/contracts/cli/DatabaseMigrationReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/DatabaseMigrationReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/DatabaseMigrationReport.schema.json", "title": "DatabaseMigrationReport", "description": "Versioned JSON report emitted by caseworkctl for db migrate.", "oneOf": [ @@ -18,7 +18,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DatabaseMigrationReport" @@ -51,7 +51,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DatabaseMigrationReport" diff --git a/products/casework/contracts/cli/DevEventsReport.schema.json b/products/casework/contracts/cli/DevEventsReport.schema.json index d55dac9592..3b3ea847b5 100644 --- a/products/casework/contracts/cli/DevEventsReport.schema.json +++ b/products/casework/contracts/cli/DevEventsReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/DevEventsReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/DevEventsReport.schema.json", "title": "DevEventsReport", "description": "Versioned JSON report emitted by caseworkctl for dev events.", "oneOf": [ @@ -19,7 +19,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DevEventsReport" @@ -58,7 +58,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DevEventsReport" diff --git a/products/casework/contracts/cli/DevGrantReport.schema.json b/products/casework/contracts/cli/DevGrantReport.schema.json index ba8505bc11..3db8b5ba98 100644 --- a/products/casework/contracts/cli/DevGrantReport.schema.json +++ b/products/casework/contracts/cli/DevGrantReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/DevGrantReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/DevGrantReport.schema.json", "title": "DevGrantReport", "description": "Versioned JSON report emitted by caseworkctl for dev grant.", "oneOf": [ @@ -17,7 +17,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DevGrantReport" @@ -47,7 +47,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DevGrantReport" diff --git a/products/casework/contracts/cli/DevIdentityReport.schema.json b/products/casework/contracts/cli/DevIdentityReport.schema.json index 83fea6a3a7..bd8761004b 100644 --- a/products/casework/contracts/cli/DevIdentityReport.schema.json +++ b/products/casework/contracts/cli/DevIdentityReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/DevIdentityReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/DevIdentityReport.schema.json", "title": "DevIdentityReport", "description": "Versioned JSON report emitted by caseworkctl for dev identity.", "oneOf": [ @@ -17,7 +17,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DevIdentityReport" @@ -47,7 +47,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DevIdentityReport" diff --git a/products/casework/contracts/cli/DevReport.schema.json b/products/casework/contracts/cli/DevReport.schema.json index a95e23f667..8788538fdb 100644 --- a/products/casework/contracts/cli/DevReport.schema.json +++ b/products/casework/contracts/cli/DevReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/DevReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/DevReport.schema.json", "title": "DevReport", "description": "Versioned JSON report emitted by caseworkctl for dev.", "oneOf": [ @@ -29,7 +29,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DevReport" @@ -118,7 +118,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DevReport" diff --git a/products/casework/contracts/cli/DevTokenReport.schema.json b/products/casework/contracts/cli/DevTokenReport.schema.json index e2412b1867..3c92a92ff6 100644 --- a/products/casework/contracts/cli/DevTokenReport.schema.json +++ b/products/casework/contracts/cli/DevTokenReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/DevTokenReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/DevTokenReport.schema.json", "title": "DevTokenReport", "description": "Versioned JSON report emitted by caseworkctl for dev token.", "oneOf": [ @@ -16,7 +16,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DevTokenReport" @@ -43,7 +43,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DevTokenReport" diff --git a/products/casework/contracts/cli/DoctorReport.schema.json b/products/casework/contracts/cli/DoctorReport.schema.json index 9cdbadbd88..6095276e83 100644 --- a/products/casework/contracts/cli/DoctorReport.schema.json +++ b/products/casework/contracts/cli/DoctorReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/DoctorReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/DoctorReport.schema.json", "title": "DoctorReport", "description": "Versioned JSON report emitted by caseworkctl for doctor.", "oneOf": [ @@ -15,13 +15,15 @@ "runtimeConfig", "packageRoot", "checks", + "packageDigest", "secretFileChecks", "sourceChecks", + "pinnedWork", "eventWiringGuidance" ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DoctorReport" @@ -38,6 +40,10 @@ "packageRoot": { "type": "string" }, + "packageDigest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, "checks": { "$ref": "#/$defs/doctorChecks" }, @@ -53,6 +59,28 @@ "type": "object" } }, + "pinnedWork": { + "type": "object", + "additionalProperties": false, + "required": [ + "verdict", + "conflicts" + ], + "properties": { + "verdict": { + "enum": [ + "clear", + "acknowledged" + ] + }, + "conflicts": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, "eventWiringGuidance": { "type": "string" } @@ -69,7 +97,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "DoctorReport" @@ -142,8 +170,10 @@ "sourceConnections", "audit", "database", + "pinnedWork", "oidcIssuer", - "directory" + "directory", + "reconciliation" ], "properties": { "configuration": { @@ -164,11 +194,17 @@ "database": { "const": "ready" }, + "pinnedWork": { + "const": "ready" + }, "oidcIssuer": { "const": "ready" }, "directory": { "const": "ready" + }, + "reconciliation": { + "const": "ready" } } } diff --git a/products/casework/contracts/cli/ExplainReport.schema.json b/products/casework/contracts/cli/ExplainReport.schema.json index 94ffde2f90..a72240a354 100644 --- a/products/casework/contracts/cli/ExplainReport.schema.json +++ b/products/casework/contracts/cli/ExplainReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/ExplainReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/ExplainReport.schema.json", "title": "ExplainReport", "description": "Versioned JSON report emitted by caseworkctl for explain.", "oneOf": [ @@ -25,7 +25,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "ExplainReport" @@ -97,7 +97,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "ExplainReport" diff --git a/products/casework/contracts/cli/InitReport.schema.json b/products/casework/contracts/cli/InitReport.schema.json index 0d2ddcedea..a22f93b96f 100644 --- a/products/casework/contracts/cli/InitReport.schema.json +++ b/products/casework/contracts/cli/InitReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/InitReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/InitReport.schema.json", "title": "InitReport", "description": "Versioned JSON report emitted by caseworkctl for init.", "oneOf": [ @@ -19,7 +19,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "InitReport" @@ -64,7 +64,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "InitReport" diff --git a/products/casework/contracts/cli/LifecycleReport.schema.json b/products/casework/contracts/cli/LifecycleReport.schema.json index a345c5429c..1a44febe28 100644 --- a/products/casework/contracts/cli/LifecycleReport.schema.json +++ b/products/casework/contracts/cli/LifecycleReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/LifecycleReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/LifecycleReport.schema.json", "title": "LifecycleReport", "description": "Versioned JSON report emitted by caseworkctl for lifecycle.", "oneOf": [ @@ -16,7 +16,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "LifecycleReport" @@ -48,7 +48,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "LifecycleReport" diff --git a/products/casework/contracts/cli/PackageReport.schema.json b/products/casework/contracts/cli/PackageReport.schema.json index e57d5e63b1..316007c07b 100644 --- a/products/casework/contracts/cli/PackageReport.schema.json +++ b/products/casework/contracts/cli/PackageReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/PackageReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/PackageReport.schema.json", "title": "PackageReport", "description": "Versioned JSON report emitted by caseworkctl for package.", "oneOf": [ @@ -14,7 +14,8 @@ "command", "project", "dryRun", - "policyDigest", + "packageDigest", + "revision", "files", "runtimeConfigurationIncluded", "secretsIncluded", @@ -23,7 +24,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "PackageReport" @@ -43,9 +44,15 @@ "dryRun": { "type": "boolean" }, - "policyDigest": { + "packageDigest": { "type": "string", - "pattern": "^sha256:" + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "revision": { + "type": [ + "string", + "null" + ] }, "files": { "type": "array", @@ -78,7 +85,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "PackageReport" diff --git a/products/casework/contracts/cli/README.md b/products/casework/contracts/cli/README.md index ad7f2019aa..a5a9a22fe5 100644 --- a/products/casework/contracts/cli/README.md +++ b/products/casework/contracts/cli/README.md @@ -5,7 +5,7 @@ carries this top-level envelope: ```json { - "apiVersion": "registry.registrystack.org/caseworkctl/v1alpha1", + "apiVersion": "registry.registrystack.org/caseworkctl/v1alpha2", "kind": "CheckReport", "ok": true, "command": "check" diff --git a/products/casework/contracts/cli/RetentionEraseReport.schema.json b/products/casework/contracts/cli/RetentionEraseReport.schema.json index 62311825be..fb9c19e35a 100644 --- a/products/casework/contracts/cli/RetentionEraseReport.schema.json +++ b/products/casework/contracts/cli/RetentionEraseReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/RetentionEraseReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/RetentionEraseReport.schema.json", "title": "RetentionEraseReport", "description": "Versioned JSON report emitted by caseworkctl for retention erase.", "oneOf": [ @@ -18,7 +18,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "RetentionEraseReport" @@ -51,7 +51,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "RetentionEraseReport" diff --git a/products/casework/contracts/cli/SimulationReport.schema.json b/products/casework/contracts/cli/SimulationReport.schema.json index 95b7e344a8..995cffac7d 100644 --- a/products/casework/contracts/cli/SimulationReport.schema.json +++ b/products/casework/contracts/cli/SimulationReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/SimulationReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/SimulationReport.schema.json", "title": "SimulationReport", "description": "Versioned JSON report emitted by caseworkctl for simulate.", "oneOf": [ @@ -24,7 +24,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "SimulationReport" @@ -90,7 +90,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "SimulationReport" diff --git a/products/casework/contracts/cli/SourceAddReport.schema.json b/products/casework/contracts/cli/SourceAddReport.schema.json index 1792af8eb1..517cae7ae9 100644 --- a/products/casework/contracts/cli/SourceAddReport.schema.json +++ b/products/casework/contracts/cli/SourceAddReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/SourceAddReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/SourceAddReport.schema.json", "title": "SourceAddReport", "description": "Versioned JSON report emitted by caseworkctl for source add.", "oneOf": [ @@ -27,7 +27,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "SourceAddReport" @@ -96,7 +96,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "SourceAddReport" diff --git a/products/casework/contracts/cli/TestReport.schema.json b/products/casework/contracts/cli/TestReport.schema.json index 0bf29596fc..8726182a2c 100644 --- a/products/casework/contracts/cli/TestReport.schema.json +++ b/products/casework/contracts/cli/TestReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/TestReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/TestReport.schema.json", "title": "TestReport", "description": "Versioned JSON report emitted by caseworkctl for test.", "oneOf": [ @@ -23,7 +23,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "TestReport" @@ -78,7 +78,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "TestReport" diff --git a/products/casework/contracts/cli/UsageReport.schema.json b/products/casework/contracts/cli/UsageReport.schema.json index 80ad6d8420..154c393ded 100644 --- a/products/casework/contracts/cli/UsageReport.schema.json +++ b/products/casework/contracts/cli/UsageReport.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://registrystack.org/caseworkctl/v1alpha1/UsageReport.schema.json", + "$id": "https://registrystack.org/caseworkctl/v1alpha2/UsageReport.schema.json", "title": "UsageReport", "description": "Versioned JSON report emitted by caseworkctl for usage.", "oneOf": [ @@ -15,7 +15,7 @@ ], "properties": { "apiVersion": { - "const": "registry.registrystack.org/caseworkctl/v1alpha1" + "const": "registry.registrystack.org/caseworkctl/v1alpha2" }, "kind": { "const": "UsageReport" diff --git a/products/casework/generated/runtime/runtime.schema.json b/products/casework/generated/runtime/runtime.schema.json index 3b52351a02..60d85998b1 100644 --- a/products/casework/generated/runtime/runtime.schema.json +++ b/products/casework/generated/runtime/runtime.schema.json @@ -2,6 +2,7 @@ "$defs": { "AuditConfig": { "additionalProperties": false, + "description": "The audit journal: where it is written and the key its hashes use.", "else": { "properties": { "path": { @@ -29,8 +30,8 @@ "description": "Where audit entries go: a rotated `file` (the default) or `stdout`." }, "hashKeyRef": { - "pattern": "^secret:(?:env|file)/", - "type": "string" + "$ref": "#/$defs/SecretReference", + "description": "Secret reference to the audit hash key." }, "path": { "default": null, @@ -230,21 +231,26 @@ }, "DatabaseConfig": { "additionalProperties": false, + "description": "The PostgreSQL connection a stateful runtime uses. Both URLs are secret\nreferences and may name the same secret.", "properties": { "migrationUrlRef": { + "description": "Secret reference to the migration connection URL.", "pattern": "^secret:(?:env|file)/", "type": "string" }, "runtimeUrlRef": { + "description": "Secret reference to the least-privileged runtime connection URL.", "pattern": "^secret:(?:env|file)/", "type": "string" }, "testOnlyPlaintext": { "default": false, + "description": "Allow a plaintext connection. Refused outside test builds.", "type": "boolean" }, "trustedRootCertificateRef": { "default": null, + "description": "Secret reference to a PEM root certificate the connection trusts.", "pattern": "^secret:(?:env|file)/", "type": [ "string", @@ -260,12 +266,15 @@ }, "EnvironmentSecretProviderConfig": { "additionalProperties": false, + "description": "The environment secret provider. It takes no settings.", "type": "object" }, "FileSecretProviderConfig": { "additionalProperties": false, + "description": "The file secret provider.", "properties": { "root": { + "description": "Absolute directory holding one file per secret. Each file must be a\nregular file owned by the runtime user, mode 0400 or 0600, with exactly\none hard link.", "pattern": "^/", "type": "string" } @@ -289,38 +298,97 @@ }, "type": "object" }, - "ListenerConfig": { - "additionalProperties": false, - "properties": { - "bind": { - "default": "127.0.0.1:8100", - "type": "string" + "JwksSource": { + "description": "Where a runtime obtains the OIDC issuer's signing keys.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Read `jwks_uri` from the issuer's OpenID Connect discovery document.", + "properties": { + "kind": { + "const": "discovery", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" }, - "networkExposure": { - "$ref": "#/$defs/ListenerNetworkExposure" + { + "additionalProperties": false, + "description": "Fetch the key set from this absolute `https` URI, skipping discovery.", + "properties": { + "kind": { + "const": "uri", + "type": "string" + }, + "uri": { + "pattern": "^https?://", + "type": "string" + } + }, + "required": [ + "kind", + "uri" + ], + "type": "object" }, - "tlsTermination": { - "$ref": "#/$defs/TlsTermination" + { + "additionalProperties": false, + "description": "Read the key set from a secret, for deployments without network access\nto the issuer.", + "properties": { + "documentRef": { + "pattern": "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$", + "type": "string" + }, + "kind": { + "const": "static", + "type": "string" + } + }, + "required": [ + "kind", + "documentRef" + ], + "type": "object" } - }, - "required": [ - "tlsTermination" - ], - "type": "object" + ] + }, + "ListenerBind": { + "description": "Socket address the runtime listens on, written host:port with an IP address host ([addr]:port for IPv6).", + "maxLength": 128, + "minLength": 1, + "type": "string" }, "ListenerNetworkExposure": { - "description": "The operator-declared private network placement of the HTTP listener.", + "description": "The operator-declared private network placement of an HTTP listener.", "enum": [ "private-address", "container-private" ], "type": "string" }, + "MetricsListenerConfig": { + "additionalProperties": false, + "description": "Operator-private listener for `/metrics` and `/version`.\n\nIt is separate from the API listener so the counters and the active\npackage digest never appear on the surface the public contract describes.", + "properties": { + "bind": { + "type": "string" + } + }, + "required": [ + "bind" + ], + "type": "object" + }, "OidcConfig": { "additionalProperties": false, + "description": "The access tokens this runtime accepts: the issuer and its keys, the\nclients admitted, the scope claim, and the claim that marks a human actor.", "properties": { "allowedClients": { "default": [], + "description": "Client identifiers whose access tokens are admitted. A runtime decides\nwhether an empty list is acceptable in production.", "items": { "type": "string" }, @@ -338,33 +406,34 @@ "uniqueItems": true }, "default": {}, - "description": "Assertion authorities each client may exchange a subject token from,\nkeyed by client identifier. An empty map applies no rule; see\n[`registry_platform_oidc::TokenVerifierConfig::assertion_issuers`].", + "description": "Assertion authorities each client may exchange a subject token from,\nkeyed by client identifier. An empty map applies no rule. Once a\nclient is listed, a token it exchanged is accepted only for one of\nthat client's declared authorities.", "maxProperties": 64, "propertyNames": { "maxLength": 128, - "minLength": 1, - "type": "string" + "minLength": 1 }, "type": "object" }, "audience": { + "description": "The audience every accepted access token must carry in `aud`.", + "maxLength": 512, + "minLength": 1, "type": "string" }, "humanIdentity": { "$ref": "#/$defs/HumanIdentityConfig" }, "issuer": { + "description": "Exact issuer accepted in access-token `iss` claims, an absolute\n`https` URL.", + "pattern": "^https?://", "type": "string" }, "jwksSource": { - "$ref": "#/$defs/OidcJwksSource" - }, - "jwksUri": { - "default": null, - "type": [ - "string", - "null" - ] + "$ref": "#/$defs/JwksSource", + "default": { + "kind": "discovery" + }, + "description": "Where the issuer's signing keys come from. Absent reads the issuer's\nOpenID Connect discovery document." }, "scopeClaim": { "default": "registry_scopes", @@ -377,40 +446,26 @@ ], "type": "object" }, - "OidcJwksSource": { - "oneOf": [ - { - "additionalProperties": false, - "properties": { - "kind": { - "const": "discovery", - "type": "string" - } - }, - "required": [ - "kind" - ], - "type": "object" + "PrivateListenerConfig": { + "additionalProperties": false, + "description": "The listener of a runtime that declares its TLS termination and network\nexposure.", + "properties": { + "bind": { + "$ref": "#/$defs/ListenerBind" }, - { - "additionalProperties": false, - "properties": { - "documentRef": { - "pattern": "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$", - "type": "string" - }, - "kind": { - "const": "static", - "type": "string" - } - }, - "required": [ - "kind", - "documentRef" - ], - "type": "object" + "networkExposure": { + "$ref": "#/$defs/ListenerNetworkExposure", + "default": "private-address" + }, + "tlsTermination": { + "$ref": "#/$defs/TlsTermination" } - ] + }, + "required": [ + "bind", + "tlsTermination" + ], + "type": "object" }, "ReviewCompletionAuthConfig": { "additionalProperties": false, @@ -517,9 +572,17 @@ "RuntimePackageConfig": { "additionalProperties": false, "properties": { - "expectedPolicyDigest": { + "acknowledgeStrandedWork": { "default": null, - "description": "The `policyDigest` of the one reviewed package this runtime may load.\nWhen set, a package whose manifest names any other digest, or a\ndirectory with no manifest, is refused before the runtime starts.", + "description": "The package digest of a package the operator has accepted will strand\nin-flight work pinned under an earlier package. Startup and `doctor`\nrefuse such a package unless this names its exact digest, so an\nacknowledgement never carries over to a later package.", + "type": [ + "string", + "null" + ] + }, + "expectedDigest": { + "default": null, + "description": "`sha256:` label of the package digest, the digest of the package's\n`SHA256SUMS` file. When set, the runtime refuses to start on any other\npackage, and on an authored project that has no `SHA256SUMS`.", "pattern": "^sha256:[0-9a-f]{64}$", "type": [ "string", @@ -527,6 +590,7 @@ ] }, "root": { + "description": "Absolute path of the package directory.", "pattern": "^/", "type": "string" } @@ -560,6 +624,7 @@ ] } ], + "description": "The secret providers a runtime enables. A reference is resolved only by a\nprovider declared here: `secret:file/name` under `file.root`, and\n`secret:env/NAME` only when `environment: {}` is present.", "properties": { "environment": { "anyOf": [ @@ -569,7 +634,8 @@ { "type": "null" } - ] + ], + "description": "Enables `secret:env/NAME` references, read from the process\nenvironment. Declared as an empty mapping: `environment: {}`." }, "file": { "anyOf": [ @@ -579,11 +645,17 @@ { "type": "null" } - ] + ], + "description": "Enables `secret:file/name` references, read from files under `root`." } }, "type": "object" }, + "SecretReference": { + "description": "An exact secret reference: secret:file/name, resolved under secretProviders.file.root, or secret:env/NAME, resolved only when secretProviders.environment is declared.", + "pattern": "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$", + "type": "string" + }, "TaskAuthorityConfig": { "additionalProperties": false, "properties": { @@ -614,7 +686,7 @@ "type": "object" }, "TlsTermination": { - "description": "Declares the trusted transport boundary for the runtime's plaintext HTTP listener.\n\nProduction listeners require operator-controlled upstream TLS termination.\nDirect plaintext is limited to the explicit loopback-only development mode.", + "description": "Declares the trusted transport boundary for a runtime's plaintext HTTP\nlistener. Production listeners require operator-controlled upstream TLS\ntermination; direct plaintext is limited to the explicit loopback-only\ndevelopment mode.", "enum": [ "operator-controlled-upstream", "development-loopback" @@ -740,7 +812,17 @@ "type": "string" }, "listener": { - "$ref": "#/$defs/ListenerConfig" + "$ref": "#/$defs/PrivateListenerConfig" + }, + "metricsListener": { + "anyOf": [ + { + "$ref": "#/$defs/MetricsListenerConfig" + }, + { + "type": "null" + } + ] }, "package": { "$ref": "#/$defs/RuntimePackageConfig" diff --git a/products/casework/scripts/check-checkpoint.sh b/products/casework/scripts/check-checkpoint.sh index 850e79a4f0..1099a8738f 100755 --- a/products/casework/scripts/check-checkpoint.sh +++ b/products/casework/scripts/check-checkpoint.sh @@ -79,10 +79,10 @@ if "output" in dry_run: sys.exit("dry-run report must omit output") if packaged["dryRun"] is not False: sys.exit("written package report must set dryRun: false") -if dry_run["policyDigest"] != packaged["policyDigest"]: +if dry_run["packageDigest"] != packaged["packageDigest"]: sys.exit( - "dry-run policyDigest %r does not match the written package's %r" - % (dry_run["policyDigest"], packaged["policyDigest"]) + "dry-run packageDigest %r does not match the written package's %r" + % (dry_run["packageDigest"], packaged["packageDigest"]) ) if dry_run["files"] != packaged["files"]: sys.exit("dry-run files do not match the written package's files") diff --git a/products/casework/scripts/generate_cli_schemas.py b/products/casework/scripts/generate_cli_schemas.py index 41a02b5ed7..50a5dc9ba3 100644 --- a/products/casework/scripts/generate_cli_schemas.py +++ b/products/casework/scripts/generate_cli_schemas.py @@ -10,7 +10,7 @@ from pathlib import Path -API_VERSION = "registry.registrystack.org/caseworkctl/v1alpha1" +API_VERSION = "registry.registrystack.org/caseworkctl/v1alpha2" OUTPUT = Path(__file__).resolve().parents[1] / "contracts" / "cli" STRING = {"type": "string"} @@ -70,16 +70,28 @@ "runtimeConfig", "packageRoot", "checks", + "packageDigest", "secretFileChecks", "sourceChecks", + "pinnedWork", "eventWiringGuidance", ], "properties": { "runtimeConfig": STRING, "packageRoot": STRING, + "packageDigest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, "checks": {"$ref": "#/$defs/doctorChecks"}, "secretFileChecks": OBJECT_ARRAY, "sourceChecks": OBJECT_ARRAY, + "pinnedWork": { + "type": "object", + "additionalProperties": False, + "required": ["verdict", "conflicts"], + "properties": { + "verdict": {"enum": ["clear", "acknowledged"]}, + "conflicts": OBJECT_ARRAY, + }, + }, "eventWiringGuidance": STRING, }, "defs": { @@ -93,8 +105,10 @@ "sourceConnections", "audit", "database", + "pinnedWork", "oidcIssuer", "directory", + "reconciliation", ], "properties": { key: {"const": "ready"} @@ -105,8 +119,10 @@ "sourceConnections", "audit", "database", + "pinnedWork", "oidcIssuer", "directory", + "reconciliation", ] }, } @@ -237,7 +253,8 @@ "required": [ "project", "dryRun", - "policyDigest", + "packageDigest", + "revision", "files", "runtimeConfigurationIncluded", "secretsIncluded", @@ -248,7 +265,8 @@ "project": STRING, "output": STRING, "dryRun": BOOLEAN, - "policyDigest": {"type": "string", "pattern": "^sha256:"}, + "packageDigest": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "revision": {"type": ["string", "null"]}, "files": OBJECT_ARRAY, "runtimeConfigurationIncluded": {"const": False}, "secretsIncluded": {"const": False}, @@ -497,7 +515,7 @@ def schema(kind: str, report: dict) -> dict: defs.update(report.get("defs", {})) return { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": f"https://registrystack.org/caseworkctl/v1alpha1/{kind}.schema.json", + "$id": f"https://registrystack.org/caseworkctl/v1alpha2/{kind}.schema.json", "title": kind, "description": f"Versioned JSON report emitted by caseworkctl for {report['command']}.", "oneOf": variants, diff --git a/products/discovery/ACCEPTANCE-JOURNEYS.md b/products/discovery/ACCEPTANCE-JOURNEYS.md index 9da8855774..09dec96121 100644 --- a/products/discovery/ACCEPTANCE-JOURNEYS.md +++ b/products/discovery/ACCEPTANCE-JOURNEYS.md @@ -7,8 +7,8 @@ become enforced bindings in the security matrix. ## Evidence 1. A validated Evidence deployment derives and packages a public description. -2. `discoveryctl build` reads that exact description from an approved local - fixture origin and writes one immutable index. +2. `discoveryctl package` reads that exact description from an approved local + fixture origin and writes one package containing the index and `SHA256SUMS`. 3. A relying application resolves an explicit requirement to evidence types, searches that type, and selects one exact record. 4. Existing application-owned Evidence trust accepts the selection. @@ -19,7 +19,7 @@ become enforced bindings in the security matrix. 1. A validated Relay deployment packages its public description as a sealed public artifact. -2. The one-shot build indexes it as a separate origin record. +2. The one-shot package operation indexes it as a separate origin record. 3. A relying application searches an exact public semantic class or operation family, explicitly selects the record, and applies existing native Relay trust. @@ -32,5 +32,5 @@ become enforced bindings in the security matrix. product-kind capability is refused before a description can reach an index. - A hostile origin, resource-bound breach, collision, mapping failure, validation failure, canonicalization failure, or output-bound failure emits - no new visible index. + no new visible package. - A local trust refusal occurs before native credential creation or traffic. diff --git a/products/discovery/AGENTS.md b/products/discovery/AGENTS.md index bb6fae4989..17d5291731 100644 --- a/products/discovery/AGENTS.md +++ b/products/discovery/AGENTS.md @@ -16,9 +16,10 @@ queries, and client handoff belong outside that crate. The profile's pinned JSON-LD context is a local contract resource; parsing does not fetch contexts, expand RDF, resolve remote references, or merge graphs. -`discoveryctl` builds indexes outside the serving runtime, fetching only exact -operator-approved origins within fixed bounds. A failure leaves the previous -output intact. +`discoveryctl package` builds index packages outside the serving runtime, +fetching only exact operator-approved origins within fixed bounds. A failure +leaves an existing output untouched. The runtime verifies the shared package +envelope and consumes only index bytes bound to its recorded file digest. The runtime serves one immutable index until restart. Keep records scoped to their origin, preserve exact capability pairs and mapping provenance, and reject ambiguous selection. Evidence matching retains the complete AND-list; diff --git a/products/discovery/README.md b/products/discovery/README.md index d5f0edc376..98a113963a 100644 --- a/products/discovery/README.md +++ b/products/discovery/README.md @@ -30,4 +30,6 @@ deployment. The Python distribution imports as `registry_client`. Run `scripts/check-contracts.sh` to validate the resources and their traceability. The normal operator flow, owned by `discoveryctl`, is offline -`check`, one explicit `build`, deployment of an immutable index, then restart. +`check`, one explicit `package`, deployment of the immutable package directory, +then restart. The runtime verifies `SHA256SUMS` and an optional +`package.expectedDigest` pin before consuming the exact packaged index bytes. diff --git a/products/discovery/contracts/definition-of-done.yaml b/products/discovery/contracts/definition-of-done.yaml index b377b52b09..732b449ef9 100644 --- a/products/discovery/contracts/definition-of-done.yaml +++ b/products/discovery/contracts/definition-of-done.yaml @@ -6,11 +6,11 @@ {"id": "discovery-dod-16-1-product-scope", "section": "16.1", "requirement": "Registry Discovery naming, the five-route public surface, and the absence of scheduler, writable catalog, proxy, trust store, procedure, ranking, and mutation concerns are consistent.", "requiredEvidence": [{"path": "products/discovery/DECISIONS.md"}, {"path": "products/discovery/PR-DESCRIPTION.md"}, {"path": "crates/registry-discovery/openapi.json"}, {"path": "products/discovery/SALVAGE-LEDGER.md"}]}, {"id": "discovery-dod-16-2-standards-profile", "section": "16.2", "requirement": "One pinned JSON-LD context, schema, deterministic renderer, independently verified RDF projection, and selected offline SHACL subset implement the accurately recorded selected standards terms without a full application-profile claim.", "requiredEvidence": [{"path": "products/discovery/contracts/standards-profile.yaml"}, {"path": "products/discovery/profile/rdf/provenance.json"}, {"path": "products/discovery/scripts/validate_profile_rdf.py"}, {"path": "crates/registry-discovery-profile/src/lib.rs", "name": "render_and_parse_are_deterministic"}, {"path": "products/discovery/scripts/test_standards_oracle.py", "name": "test_json_ld_and_shacl_oracles_validate_every_profile_fixture"}, {"path": "products/discovery/scripts/test_standards_oracle.py", "name": "test_distinct_binding_nodes_preserve_repeated_service_id_capability_correlation"}]}, {"id": "discovery-dod-16-3-provider-publication", "section": "16.3", "requirement": "Evidence and Relay derive deterministic public descriptions, preserve exact capability correlation under distinct binding identities, package exact bytes, exclude private or protected data through closed public projections, reject invalid authored projection values, and serve no dynamically reconstructed provider data.", "requiredEvidence": [{"path": "crates/registry-evidence/src/discovery.rs", "name": "provider_discovery_description_preserves_evidence_type_profile_correlation"}, {"path": "crates/registry-relay-v2/src/artifacts.rs", "name": "provider_discovery_description_preserves_semantic_class_operation_family_correlation"}, {"path": "products/discovery/contracts/security-test-traceability.yaml"}]}, - {"id": "discovery-dod-16-4-origin-build", "section": "16.4", "requirement": "The offline checked origin and mapping project feeds one bounded exact-target build that preserves provenance, origin isolation, and atomic, durably synced output replacement. Production builtAt is captured after all origin fetches complete. catalogRevision covers the normalized semantic service projection and excludes per-build originContentDigest, originFetchedAt, and builtAt values, so identical semantic inputs preserve record identities, semantic fields, and revisions while provenance timestamps may change. Every authored and compiled fixture satisfies its Draft 2020-12 schema and closed Rust parser.", "requiredEvidence": [{"path": "products/discovery/schemas/origins.schema.json"}, {"path": "products/discovery/schemas/evidence-mapping.schema.json"}, {"path": "products/discovery/schemas/runtime.schema.json"}, {"path": "products/discovery/schemas/index.schema.json"}, {"path": "products/discovery/fixtures/project/origins.yaml"}, {"path": "products/discovery/fixtures/project/discovery-index.json"}, {"path": "products/discovery/fixtures/schema-negative-corpus.json"}, {"path": "crates/registry-discoveryctl/tests/schema_contract.rs", "name": "every_positive_fixture_satisfies_draft_2020_12_and_the_closed_rust_parser"}, {"path": "crates/registry-discoveryctl/tests/schema_contract.rs", "name": "shared_negative_corpus_is_refused_by_both_schema_and_rust"}, {"path": "crates/registry-discoveryctl/src/project.rs", "name": "check_is_offline_and_accepts_an_unreachable_https_origin"}, {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "build_fetches_each_origin_once_and_preserves_semantic_revisions"}, {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "production_build_time_is_captured_after_origin_collection"}, {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "failed_origin_fetch_leaves_the_previous_output_untouched"}, {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "write_failure_preserves_previous_output_and_leaves_no_visible_temporary_file"}]}, + {"id": "discovery-dod-16-4-origin-package", "section": "16.4", "requirement": "The offline checked origin and mapping project feeds one bounded exact-target package operation that preserves provenance and origin isolation, writes the canonical index under the shared SHA256SUMS envelope, and never replaces an existing output directory. Production builtAt is captured after all origin fetches complete. catalogRevision covers the normalized semantic service projection and excludes per-build originContentDigest, originFetchedAt, and builtAt values, so identical semantic inputs preserve record identities, semantic fields, and revisions while provenance timestamps may change. The same compiled index bytes and optional revision produce the same package digest. Every authored and compiled fixture satisfies its Draft 2020-12 schema and closed Rust parser.", "requiredEvidence": [{"path": "products/discovery/schemas/origins.schema.json"}, {"path": "products/discovery/schemas/evidence-mapping.schema.json"}, {"path": "products/discovery/schemas/runtime.schema.json"}, {"path": "products/discovery/schemas/index.schema.json"}, {"path": "products/discovery/fixtures/project/origins.yaml"}, {"path": "products/discovery/fixtures/project/discovery-index.json"}, {"path": "products/discovery/fixtures/schema-negative-corpus.json"}, {"path": "crates/registry-discoveryctl/tests/schema_contract.rs", "name": "every_positive_fixture_satisfies_draft_2020_12_and_the_closed_rust_parser"}, {"path": "crates/registry-discoveryctl/tests/schema_contract.rs", "name": "shared_negative_corpus_is_refused_by_both_schema_and_rust"}, {"path": "crates/registry-discoveryctl/src/project.rs", "name": "check_is_offline_and_accepts_an_unreachable_https_origin"}, {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "package_fetches_each_origin_once_and_preserves_semantic_revisions"}, {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "production_package_time_is_captured_after_origin_collection"}, {"path": "crates/registry-discoveryctl/src/build.rs", "name": "packaging_the_same_compiled_index_twice_is_repeatable"}, {"path": "crates/registry-discovery/src/startup.rs", "name": "startup_verifies_package_and_refuses_expected_digest_mismatch_with_common_shape"}, {"path": "crates/registry-discovery/src/startup.rs", "name": "exact_consumed_index_bytes_remain_bound_to_the_verified_package"}, {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "failed_origin_fetch_leaves_the_previous_output_untouched"}, {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "package_refuses_an_existing_output_without_changing_it"}]}, {"id": "discovery-dod-16-5-evidence-resolver", "section": "16.5", "requirement": "Requirement resolution preserves exact jurisdiction, AND within a list, OR between alternatives, mapping provenance, empty success, and complete-or-refused bounds without provider resolution.", "requiredEvidence": [{"path": "crates/registry-discovery/src/query.rs", "name": "resolver_preserves_and_within_lists_or_across_alternatives_and_refuses_over_bound"}, {"path": "crates/registry-discovery/src/model.rs"}]}, {"id": "discovery-dod-16-6-runtime-query-api", "section": "16.6", "requirement": "Startup loads one strict immutable index and the runtime exposes only value-free probes, deterministic generated OpenAPI, exact bounded service search, and evidence-type resolution, with no metrics route. Runtime admission bounds request bodies before buffering, rejects malformed form encoding and request media parameters, and bounds synchronous query work.", "requiredEvidence": [{"path": "crates/registry-discovery/src/startup.rs", "name": "runtime_is_closed_and_contains_no_origin_mapping_trust_or_fetch_configuration"}, {"path": "crates/registry-discovery/src/server.rs", "name": "request_body_capacity_is_acquired_before_buffering_and_recovers"}, {"path": "crates/registry-discovery/src/server.rs", "name": "request_media_type_accepts_only_bare_or_utf8_json"}, {"path": "crates/registry-discovery/src/query.rs", "name": "malformed_percent_encoding_and_invalid_utf8_are_refused"}, {"path": "crates/registry-discovery/src/server.rs", "name": "real_trace_and_problem_output_exclude_request_canaries"}, {"path": "crates/registry-discovery/src/server.rs", "name": "real_router_exposes_only_the_fixed_read_only_surface"}, {"path": "crates/registry-discovery/src/openapi.rs", "name": "committed_openapi_is_the_deterministic_generator_output"}, {"path": "crates/registry-discovery/openapi.json"}]}, {"id": "discovery-dod-16-7-client-trust-invocation", "section": "16.7", "requirement": "The Rust, Node, and Python clients resolve, search, and explicitly select public metadata only. Structural validation proves closed shape and capability binding, not trust or currentness. An ephemeral accepted handoff exists only after adopter-owned local acceptance; credentials and native product I/O occur only after that acceptance. Online renewal may refresh provenance only for unchanged trust-relevant service semantics, and every semantic change requires a new local acceptance decision.", "requiredEvidence": [{"path": "crates/registry-discovery-client/src/selection.rs", "name": "structural_validation_does_not_turn_descriptive_metadata_into_binding_authority"}, {"path": "crates/registry-discovery-client/src/selection.rs", "name": "unchanged_renewal_refreshes_provenance_but_requires_new_acceptance_for_semantic_change"}, {"path": "crates/registry-discovery-client/src/selection.rs", "name": "discovery_metadata_has_no_trust_or_native_io_capability"}, {"path": "crates/registry-discovery-client/tests/native_journey.rs", "name": "complete_evidence_and_relay_journeys_build_select_trust_and_invoke_natively"}, {"path": "crates/registry-discovery-client-node/__test__/surface.test.js"}, {"path": "crates/registry-discovery-client-py/tests/python/test_client.py", "name": "test_exact_local_acceptance_precedes_credentials_and_native_io"}, {"path": "crates/registry-discovery-client-py/tests/python/test_client.py", "name": "test_renewal_only_updates_provenance_for_the_same_accepted_subject"}, {"path": "products/discovery/scripts/test-adopter-tutorial.sh"}, {"path": "crates/registry-discovery-client/src/client.rs", "name": "response_bytes_are_bounded"}]}, - {"id": "discovery-dod-16-8-adopter-maintenance-ux", "section": "16.8", "requirement": "Provider, catalog operator, and consumer/verifier adoption is documented and exercised from a clean source tree as a small public publication block, origins and mappings, offline check, explicit build, exact selection, native trust handoff, deployment, and restart loop.", "requiredEvidence": [{"path": "products/discovery/README.md"}, {"path": "products/discovery/ACCEPTANCE-JOURNEYS.md"}, {"path": "products/discovery/contracts/implementation-schedule.yaml"}, {"path": "docs/site/src/content/docs/tutorials/publish-and-consume-discovery-index.mdx"}, {"path": "products/discovery/scripts/test-adopter-tutorial.sh"}]}, + {"id": "discovery-dod-16-8-adopter-maintenance-ux", "section": "16.8", "requirement": "Provider, catalog operator, and consumer/verifier adoption is documented and exercised from a clean source tree as a small public publication block, origins and mappings, offline check, explicit package operation, exact selection, native trust handoff, deployment, and restart loop.", "requiredEvidence": [{"path": "products/discovery/README.md"}, {"path": "products/discovery/ACCEPTANCE-JOURNEYS.md"}, {"path": "products/discovery/contracts/implementation-schedule.yaml"}, {"path": "docs/site/src/content/docs/tutorials/publish-and-consume-discovery-index.mdx"}, {"path": "products/discovery/scripts/test-adopter-tutorial.sh"}]}, {"id": "discovery-dod-16-9-acceptance-journeys", "section": "16.9", "requirement": "Evidence and Relay publication, build, exact selection, existing local trust, and direct native invocation are exercised as deterministic local journeys, including invalid origin and untrusted selection failures.", "requiredEvidence": [{"path": "products/discovery/ACCEPTANCE-JOURNEYS.md"}, {"path": "products/discovery/scripts/test-http.sh"}, {"path": "products/discovery/fixtures/descriptions/evidence.jsonld"}, {"path": "products/discovery/fixtures/descriptions/relay.jsonld"}]}, {"id": "discovery-dod-16-10-security-ci", "section": "16.10", "requirement": "Every named Discovery security threat has a refusal and executable traceability; profile, schema, standards, contract, publication, build, runtime, and client gates are selected and run without secrets or live network fixtures.", "requiredEvidence": [{"path": "products/discovery/contracts/security-invariant-matrix.yaml"}, {"path": "products/discovery/contracts/security-test-traceability.yaml"}, {"path": "products/discovery/scripts/check-contracts.sh"}, {"path": "products/discovery/scripts/test-http.sh"}, {"path": "crates/registry-discoveryctl/tests/schema_contract.rs", "name": "shared_negative_corpus_is_refused_by_both_schema_and_rust"}, {"path": "products/discovery/scripts/test_standards_oracle.py", "name": "test_shacl_oracle_rejects_missing_endpoint"}]}, {"id": "discovery-dod-16-11-pr-reduction", "section": "16.11", "requirement": "Wrong-boundary Federation registration, routing, trust, procedures, bindings, release surfaces, and dormant code are removed while useful deferred work remains recoverable outside production scope.", "requiredEvidence": [{"path": "products/discovery/SALVAGE-LEDGER.md"}, {"path": "products/discovery/FUTURE-WORK.md"}, {"path": "products/discovery/PR-DESCRIPTION.md"}]} diff --git a/products/discovery/contracts/implementation-schedule.yaml b/products/discovery/contracts/implementation-schedule.yaml index 548eaab8b0..411a931de0 100644 --- a/products/discovery/contracts/implementation-schedule.yaml +++ b/products/discovery/contracts/implementation-schedule.yaml @@ -3,7 +3,7 @@ "phases": [ {"id": "phase-0", "name": "Contract reset", "exit": "The salvage ledger, future-work ledger, product boundary, and supported public surface are reviewed."}, {"id": "phase-1", "name": "Profile and publication", "exit": "Evidence and Relay generate deterministic closed profile bytes from validated public inputs and package them."}, - {"id": "phase-2", "name": "Immutable build", "exit": "discoveryctl validates origins and mappings offline, performs a bounded one-shot build, and emits an atomic index."}, + {"id": "phase-2", "name": "Immutable package", "exit": "discoveryctl validates origins and mappings offline, performs a bounded one-shot collection, and emits a verified package containing the canonical index."}, {"id": "phase-3", "name": "Read-only discovery", "exit": "The runtime serves only probes, OpenAPI, exact service search, and evidence-type resolution from one immutable index."}, {"id": "phase-4", "name": "Native journeys", "exit": "One Evidence and one Relay journey prove explicit selection, existing local trust, and direct native invocation."} ] diff --git a/products/discovery/contracts/security-invariant-matrix.yaml b/products/discovery/contracts/security-invariant-matrix.yaml index 180ed99185..5ea32db5dd 100644 --- a/products/discovery/contracts/security-invariant-matrix.yaml +++ b/products/discovery/contracts/security-invariant-matrix.yaml @@ -49,14 +49,23 @@ "binding": {"path": "crates/registry-discovery/src/model.rs", "name": "records_from_distinct_origins_never_merge"} }, { - "id": "sec-atomic-index-build", + "id": "sec-atomic-index-package", "status": "enforced", - "threat": "A failure during fetch, parse, normalization, mapping, serialization, or write exposes a partial index.", - "enforcementPoint": "discoveryctl staged build and atomic activation.", - "requiredNegativeBehavior": "Leave the previous index unchanged and create no visible replacement.", - "negativeTest": "failed-build-leaves-no-new-visible-index", + "threat": "A failure during fetch, parse, normalization, mapping, serialization, or write exposes a partial package or changes an existing output.", + "enforcementPoint": "discoveryctl package construction through the shared package writer.", + "requiredNegativeBehavior": "Leave an existing output unchanged and remove a newly created incomplete package directory.", + "negativeTest": "failed-package-leaves-no-new-visible-package", "binding": {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "failed_origin_fetch_leaves_the_previous_output_untouched"} }, + { + "id": "sec-package-integrity", + "status": "enforced", + "threat": "Discovery serves index bytes that differ from the verified package, accepts a changed, missing, or extra package file, or starts on a package other than the optional deployment pin.", + "enforcementPoint": "Runtime package verification and exact index-byte capture before listener binding.", + "requiredNegativeBehavior": "Refuse startup by file name or with the shared expected and found digest shape before constructing the query directory.", + "negativeTest": "package-envelope-pin-and-consumed-index-bytes-are-bound", + "binding": {"path": "crates/registry-discovery/src/startup.rs", "name": "startup_verifies_package_and_refuses_expected_digest_mismatch_with_common_shape"} + }, { "id": "sec-discovery-not-trust", "status": "enforced", diff --git a/products/discovery/contracts/security-test-traceability.yaml b/products/discovery/contracts/security-test-traceability.yaml index fb1688c829..3bfad08a0f 100644 --- a/products/discovery/contracts/security-test-traceability.yaml +++ b/products/discovery/contracts/security-test-traceability.yaml @@ -70,9 +70,9 @@ ] }, { - "id": "sec-atomic-index-build", - "negativeTest": "failed-build-leaves-no-new-visible-index", - "testId": "discoveryctl::atomic_index_build", + "id": "sec-atomic-index-package", + "negativeTest": "failed-package-leaves-no-new-visible-package", + "testId": "discoveryctl::atomic_index_package", "tests": [ {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "failed_origin_fetch_leaves_the_previous_output_untouched"}, {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "wrong_profile_media_type_is_refused_without_replacing_the_output"}, @@ -82,7 +82,19 @@ {"path": "crates/registry-discoveryctl/src/build.rs", "name": "origin_fetch_timeout_leaves_the_previous_output_untouched"}, {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "compiled_service_bound_leaves_the_previous_output_untouched"}, {"path": "crates/registry-discoveryctl/src/build.rs", "name": "compiled_index_byte_overflow_preserves_the_previous_output"}, - {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "write_failure_preserves_previous_output_and_leaves_no_visible_temporary_file"} + {"path": "crates/registry-discoveryctl/tests/build.rs", "name": "package_refuses_an_existing_output_without_changing_it"} + ] + }, + { + "id": "sec-package-integrity", + "negativeTest": "package-envelope-pin-and-consumed-index-bytes-are-bound", + "testId": "discovery_runtime::package_integrity", + "tests": [ + {"path": "crates/registry-discovery/src/startup.rs", "name": "startup_verifies_package_and_refuses_expected_digest_mismatch_with_common_shape"}, + {"path": "crates/registry-discovery/src/startup.rs", "name": "startup_refuses_changed_missing_and_extra_package_files_by_name"}, + {"path": "crates/registry-discovery/src/startup.rs", "name": "startup_refuses_a_hash_covered_non_index_file_by_name"}, + {"path": "crates/registry-discovery/src/startup.rs", "name": "exact_consumed_index_bytes_remain_bound_to_the_verified_package"}, + {"path": "crates/registry-discovery/tests/http_journey.rs", "name": "verified_package_startup_serves_the_packaged_index_through_the_real_router"} ] }, { diff --git a/products/discovery/fixtures/project/README.md b/products/discovery/fixtures/project/README.md index 6829aa99cd..e62f37d25e 100644 --- a/products/discovery/fixtures/project/README.md +++ b/products/discovery/fixtures/project/README.md @@ -10,4 +10,4 @@ cargo run --locked -p registry-discoveryctl -- check \ The two `.invalid` description URLs are deliberate. `check` performs no network access. The product HTTP journey replaces them with bounded local -providers, builds the index, and drives the real runtime and client. +providers, packages the index, and drives the real runtime and client. diff --git a/products/discovery/fixtures/project/runtime.yaml b/products/discovery/fixtures/project/runtime.yaml index e6daad494f..79f092559d 100644 --- a/products/discovery/fixtures/project/runtime.yaml +++ b/products/discovery/fixtures/project/runtime.yaml @@ -1,7 +1,9 @@ -schemaVersion: registry-discovery/runtime/v1alpha1 +apiVersion: registry.registrystack.org/discovery-runtime/v1alpha1 +kind: DiscoveryRuntimeConfig listener: - address: 127.0.0.1:8080 -indexPath: discovery-index.json + bind: 127.0.0.1:8080 +package: + root: /srv/registry-discovery/package limits: maximumRequestBytes: 65536 maximumResponseBytes: 1048576 diff --git a/products/discovery/fixtures/schema-negative-corpus.json b/products/discovery/fixtures/schema-negative-corpus.json index c4e31c560d..db477344b7 100644 --- a/products/discovery/fixtures/schema-negative-corpus.json +++ b/products/discovery/fixtures/schema-negative-corpus.json @@ -76,20 +76,44 @@ { "name": "runtime-refuses-non-socket-listener", "contract": "runtime", - "pointer": "/listener/address", + "pointer": "/listener/bind", "value": "localhost" }, { "name": "runtime-refuses-out-of-range-socket-literal", "contract": "runtime", - "pointer": "/listener/address", + "pointer": "/listener/bind", "value": "999.999.999.999:99999" }, { - "name": "runtime-refuses-parent-traversal-index-path", + "name": "runtime-refuses-removed-schema-version", + "contract": "runtime", + "pointer": "/schemaVersion", + "value": "registry-discovery/runtime/v1alpha1" + }, + { + "name": "runtime-refuses-removed-listener-address", + "contract": "runtime", + "pointer": "/listener/address", + "value": "127.0.0.1:8080" + }, + { + "name": "runtime-refuses-foreign-envelope", + "contract": "runtime", + "pointer": "/kind", + "value": "RelayRuntimeConfig" + }, + { + "name": "runtime-refuses-relative-package-root", + "contract": "runtime", + "pointer": "/package/root", + "value": "../package" + }, + { + "name": "runtime-refuses-non-normal-package-root", "contract": "runtime", - "pointer": "/indexPath", - "value": "../discovery-index.json" + "pointer": "/package/root", + "value": "/srv/current/../package" }, { "name": "index-refuses-unknown-root-member", diff --git a/products/discovery/profile/rdf/provenance.json b/products/discovery/profile/rdf/provenance.json index d8e3f9659d..f85ad6b481 100644 --- a/products/discovery/profile/rdf/provenance.json +++ b/products/discovery/profile/rdf/provenance.json @@ -74,14 +74,14 @@ {"path": "profile/shapes/registry-discovery-v1alpha1.shacl.ttl", "version": "v1alpha1", "sha256": "19890b75f14c5292c38317f12b10272c508c49936dba9918f9970f75242bcec1"}, {"path": "schemas/origins.schema.json", "version": "v1alpha1", "sha256": "3060a0edad49b261e3c237236a9b5ebed08ee9e22ebba0b501337c0d8c7b4b70"}, {"path": "schemas/evidence-mapping.schema.json", "version": "v1alpha1", "sha256": "14e25f7931f0d519b3fc09ce0e7b8b7c9f783b343779f73dfa31db5a31a3d70d"}, - {"path": "schemas/runtime.schema.json", "version": "v1alpha1", "sha256": "12a4d0ac52602fa55989f7db630e91903a9f8ca07a601480fe45e618c0173cc5"}, + {"path": "schemas/runtime.schema.json", "version": "v1alpha1", "sha256": "4919dec68a1cbd0a2349f9753c97700e455cdbc4e82f49a59a85d104f0e860aa"}, {"path": "schemas/index.schema.json", "version": "v1alpha1", "sha256": "306296f37852d2f789a2843a8409dd644b65f030b17a94d2c6a119d33904c7a0"}, {"path": "scripts/test_standards_oracle.py", "version": "v1alpha1", "sha256": "b2f1d81056d605888703c7134c3f93345af4636b68c7cfa806347b16f9400c10"}, {"path": "standards-oracle/pyproject.toml", "version": "locked", "sha256": "78b53a2fa3a496aeceafcee189a0974e820e05c5c23bd63dd898d896ae89eb3e"}, {"path": "standards-oracle/uv.lock", "version": "revision-3", "sha256": "970302805249159017e5f72c434a2e00d099f6a95ee87838ad67927bec8da35d"} ], "fixtures": [ - {"path": "fixtures/schema-negative-corpus.json", "sha256": "634f0ae46f0269b4a65f50ab9cc97540ae1098cf0e2a6bb8105f8d95aa479476"}, + {"path": "fixtures/schema-negative-corpus.json", "sha256": "6fa3bcc6b535df0ba9b30cc95f24e55ff3cad76888888b4ff9837ee652f99fdf"}, {"path": "fixtures/descriptions/repeated-service-bindings.jsonld", "sha256": "03d281647751c60489d1b2940c3a79bba2cfaa932b186daccb0b0ac310f17f24"}, {"path": "fixtures/rdf/repeated-service-bindings.nt", "sha256": "3fa531e2b9a27eab260ae3e2838124e5886e1d22b717322f37bf3e0d27099746"} ] diff --git a/products/discovery/schemas/runtime.schema.json b/products/discovery/schemas/runtime.schema.json index 52285268d0..1690aa96bf 100644 --- a/products/discovery/schemas/runtime.schema.json +++ b/products/discovery/schemas/runtime.schema.json @@ -4,15 +4,16 @@ "title": "Registry Discovery runtime configuration", "type": "object", "additionalProperties": false, - "required": ["schemaVersion", "listener", "indexPath", "limits", "logLevel"], + "required": ["apiVersion", "kind", "listener", "package", "limits", "logLevel"], "properties": { - "schemaVersion": {"const": "registry-discovery/runtime/v1alpha1"}, + "apiVersion": {"const": "registry.registrystack.org/discovery-runtime/v1alpha1"}, + "kind": {"const": "DiscoveryRuntimeConfig"}, "listener": { "type": "object", "additionalProperties": false, - "required": ["address"], + "required": ["bind"], "properties": { - "address": { + "bind": { "type": "string", "minLength": 1, "maxLength": 128, @@ -30,13 +31,23 @@ } } }, - "indexPath": { - "type": "string", - "minLength": 1, - "maxLength": 4096, - "pattern": "^(?:[^/]+/)*[^/]+$", - "not": {"pattern": "(^|/)\\.{1,2}(/|$)"}, - "description": "Relative normal path resolved beside this runtime file." + "package": { + "type": "object", + "additionalProperties": false, + "required": ["root"], + "properties": { + "root": { + "type": "string", + "pattern": "^/", + "not": {"pattern": "(^|/)\\.{1,2}(/|$)"}, + "description": "Absolute path of the package directory." + }, + "expectedDigest": { + "type": ["string", "null"], + "pattern": "^sha256:[0-9a-f]{64}$", + "description": "Optional digest of the package SHA256SUMS file." + } + } }, "limits": { "type": "object", diff --git a/products/discovery/scripts/test-adopter-tutorial.sh b/products/discovery/scripts/test-adopter-tutorial.sh index b9a65276d1..cc0360f05a 100755 --- a/products/discovery/scripts/test-adopter-tutorial.sh +++ b/products/discovery/scripts/test-adopter-tutorial.sh @@ -5,7 +5,7 @@ repository=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) . "$repository/scripts/cargo-runtime-library-path.sh" target_dir="${DISCOVERY_TUTORIAL_TARGET_DIR:-$repository/target/discovery-tutorial-source}" profile="${DISCOVERY_TUTORIAL_CARGO_PROFILE:-ci}" -work_root=$(mktemp -d "${TMPDIR:-/tmp}/discovery-adopter-tutorial.XXXXXX") +work_root=$(cd "$(mktemp -d "${TMPDIR:-/tmp}/discovery-adopter-tutorial.XXXXXX")" && pwd -P) publication_pid="" discovery_pid="" @@ -114,17 +114,19 @@ if ! curl --fail --silent --output /dev/null "http://127.0.0.1:38090/relay.jsonl exit 1 fi -build_output=$("$DISCOVERYCTL_BIN" build \ +package_output=$("$DISCOVERYCTL_BIN" package \ --project "$work_root" \ - --output "$work_root/discovery-index.json" \ + --output "$work_root/package" \ + --revision tutorial-source \ --allow-loopback) -if [[ ! "$build_output" =~ ^built\ catalogRevision=sha256:[0-9a-f]{64}\ mappingRevision=sha256:[0-9a-f]{64}$ ]]; then - printf 'unexpected discoveryctl build output: %s\n' "$build_output" >&2 +if [[ ! "$package_output" =~ ^packaged\ packageDigest=sha256:[0-9a-f]{64}\ catalogRevision=sha256:[0-9a-f]{64}\ mappingRevision=sha256:[0-9a-f]{64}$ ]]; then + printf 'unexpected discoveryctl package output: %s\n' "$package_output" >&2 exit 1 fi -printf '[operator] explicit build: %s\n' "$build_output" +printf '[operator] explicit package: %s\n' "$package_output" -"$DISCOVERY_BIN" --runtime "$work_root/runtime.yaml" \ +DISCOVERY_PACKAGE_ROOT="$work_root/package" \ +"$DISCOVERY_BIN" --runtime-config "$work_root/runtime.yaml" \ >"$work_root/discovery.log" 2>&1 & discovery_pid=$! diff --git a/products/discovery/scripts/validate_contract_artifacts.py b/products/discovery/scripts/validate_contract_artifacts.py index 8d5f6217b7..af489963c5 100755 --- a/products/discovery/scripts/validate_contract_artifacts.py +++ b/products/discovery/scripts/validate_contract_artifacts.py @@ -31,12 +31,12 @@ REQUIRED_INVARIANTS = { "sec-provider-public-projection", "sec-origin-target-confinement", "sec-profile-parser-confinement", "sec-build-resource-bounds", - "sec-origin-record-isolation", "sec-atomic-index-build", + "sec-origin-record-isolation", "sec-atomic-index-package", "sec-package-integrity", "sec-discovery-not-trust", "sec-query-and-log-minimization", } REQUIRED_DOD_IDS = { "discovery-dod-16-1-product-scope", "discovery-dod-16-2-standards-profile", - "discovery-dod-16-3-provider-publication", "discovery-dod-16-4-origin-build", + "discovery-dod-16-3-provider-publication", "discovery-dod-16-4-origin-package", "discovery-dod-16-5-evidence-resolver", "discovery-dod-16-6-runtime-query-api", "discovery-dod-16-7-client-trust-invocation", "discovery-dod-16-8-adopter-maintenance-ux", "discovery-dod-16-9-acceptance-journeys", "discovery-dod-16-10-security-ci", diff --git a/products/discovery/tutorial/project/runtime.yaml b/products/discovery/tutorial/project/runtime.yaml index 32e4bc1f82..c1f8810c40 100644 --- a/products/discovery/tutorial/project/runtime.yaml +++ b/products/discovery/tutorial/project/runtime.yaml @@ -1,7 +1,9 @@ -schemaVersion: registry-discovery/runtime/v1alpha1 +apiVersion: registry.registrystack.org/discovery-runtime/v1alpha1 +kind: DiscoveryRuntimeConfig listener: - address: 127.0.0.1:38080 -indexPath: discovery-index.json + bind: 127.0.0.1:38080 +package: + root: ${DISCOVERY_PACKAGE_ROOT} limits: maximumRequestBytes: 65536 maximumResponseBytes: 1048576 diff --git a/products/evidence/CONCEPT.md b/products/evidence/CONCEPT.md index 551be1dcdf..63b6caf03f 100644 --- a/products/evidence/CONCEPT.md +++ b/products/evidence/CONCEPT.md @@ -141,7 +141,7 @@ The query-platform line deserves stating precisely, because a source may execute a reviewed SQL statement against a mounted extract. A query platform is a system whose purpose is to accept a query at request time; Evidence accepts none, over any transport. A statement is written into the bundle, covered by -the bundle hash, and reviewed as part of one disclosure surface. A caller +the package sum file, and reviewed as part of one disclosure surface. A caller cannot write one, name a table or a column, add a predicate, widen a join, or choose among statements beyond selecting a requirement it is already authorized to invoke. This is the same reasoning that already admits a fixed HTTP request: @@ -837,10 +837,10 @@ sources: factSchema: schemas/subject-extract-facts.schema.yaml ``` -The statement is a bundle artifact covered by the bundle hash. The file is not: +The statement is a bundle artifact covered by the package sum file. The file is not: the bundle names a logical extract and the closed runtime file binds it to a path, exactly as it binds a private certificate authority. Republishing an -extract therefore leaves the bundle revision a relying party pinned unchanged, +extract therefore leaves the package digest a relying party pinned unchanged, which is why publication metadata and a declared maximum age are mandatory rather than advisory. Binding a path is not a capability gate: a deployment that mounts no extract runs no extract source, and no runtime switch turns this @@ -931,7 +931,7 @@ a world that has moved. The comparison runs for every evaluation, not only at startup, so a process that has been running longer than its own tolerance refuses rather than serving out of a file it has already outlived. -Staleness tolerance is bundle-declared because it is a property of the +Staleness tolerance is package-declared because it is a property of the question, not of the deployment. A fact that cannot change once it is recorded is indifferent to a file published a week ago. A status that can be revoked at any moment is not, and the same week makes the same answer wrong. An @@ -1017,7 +1017,7 @@ Rhai receives no ambient access to: - audit sinks; - signing keys. -Scripts compile at startup and are identified by bundle hash. Each invocation receives fresh local state. Explicit limits apply to operations, call depth, strings, collections, modules, and result size. +Scripts compile at startup and are identified by package sum file. Each invocation receives fresh local state. Explicit limits apply to operations, call depth, strings, collections, modules, and result size. A future `plan(context) -> SourceCall` hook requires a separate design and a demonstrated source that cannot use a fixed request. It is not a hidden extension point in version one. @@ -1416,7 +1416,7 @@ server or source network. ### 11.6 Discovery and publication The set of definitions a requester may use is the intersection of the exact -deployed bundle revision and the caller's verified authority context. It +deployed package digest and the caller's verified authority context. It depends on requirement, purpose, audience, the complete role/profile/origin tuple, and any token-owned selector values together. A process-wide catalog would overstate availability and reveal definitions or selector structure that @@ -1540,7 +1540,7 @@ shipping the stream to append-only storage outside the service's write authority Authorized-material audit events contain only reviewed fields: - operation identifier and phase; -- requirement and bundle revision; +- requirement and package digest (carried in the frozen `bundleRevision` field); - purpose code; - pseudonymized requester and optional actor; - selector profile identifiers and one pseudonymized complete selector bundle @@ -1579,7 +1579,8 @@ After successful authentication, an authorization refusal produces a separate minimal native event with the `registry.evidence.audit.authorization-refusal/v1` discriminator before Evidence returns the generic `403`. That event contains only the operation and -event identifiers, assurance profile, bundle revision, a scoped requester +event identifiers, assurance profile, package digest in the frozen +`bundleRevision` field, a scoped requester pseudonym, an optional actor pseudonym, the closed `not-authorized` decision and safe error category, and timestamp and duration. It omits the untrusted requested requirement, purpose, subjects, unmatched @@ -2358,7 +2359,7 @@ mandatory default and includes: - two coequal generic evidence-data transports: one fixed HTTP JSON request executor, and one reviewed-statement executor over a read-only mounted SQLite extract with a prepare-time authorizer verdict, required publication - metadata, a bundle-declared maximum extract age, declared row and step + metadata, a package-declared maximum extract age, declared row and step bounds, and the reserved evaluation instant in place of an ambient clock; - generic Basic, static Authorization header, static API-key header, and OAuth 2.0 client-credentials authentication for HTTP sources using secret @@ -2404,7 +2405,7 @@ mandatory default and includes: and confirmation that the audit entries were written; - independently configured OIDC authentication with an explicit issuer, resource, scope and claim contract; -- a documented Docker Compose adapter that mounts the candidate bundle +- a documented Docker Compose adapter that mounts the candidate package unchanged without generating Compose, container, or cloud deployment output; - deterministic source-contract mocks for flat REST, DHIS2 Tracker-style REST, OpenCRVS Version 2 Event Search-style JSON, and a sanitized SQLite extract; @@ -2704,7 +2705,7 @@ This concept fixes the following decisions: 22. Evidence-definition discovery uses authenticated `GET /v1/evidence-definitions`, which returns only complete request shapes matching exactly one authority path for the verified caller and exact - bundle revision. Static onboarding owns token acquisition, human and legal + package digest. Static onboarding owns token acquisition, human and legal context, and verifier trust; OpenAPI describes the wire contract, RFC 9728 metadata binds the public resource to its authorization server and JWKS, and JWKS provides key discovery. Discovery metadata never creates @@ -2727,7 +2728,7 @@ This concept fixes the following decisions: extract. The statement is a bundle artifact, a prepare-time authorizer proves it cannot write or leave its file, Rust supplies the only clock through a reserved parameter, and an extract without publication metadata - or past its bundle-declared maximum age is refused before any row is read. + or past its package-declared maximum age is refused before any row is read. 27. `POST /v1/evidence/batch` is a bounded audience-scoped evaluation envelope, not the holder-bound issuance batch. It returns ordered signed JWS or `evidence_not_available` outcomes, applies all admission gates before I/O, diff --git a/products/evidence/IMPLEMENTATION.md b/products/evidence/IMPLEMENTATION.md index 947398c258..2d01f89616 100644 --- a/products/evidence/IMPLEMENTATION.md +++ b/products/evidence/IMPLEMENTATION.md @@ -148,7 +148,7 @@ must use them: | Subject selector | Closed identifier or compound field set with deployment-defined names, scalar types, bounds, and fixed source placements | | Lookup outcome | Provider-owned `match`, `no_match`, or `ambiguous`; facts exist only on `match` | | Source | One fixed HTTP JSON data request using field projection and denied redirects, or one reviewed SQL statement over a read-only SQLite extract the runtime mounts | -| Statement source | A reviewed statement artifact covered by the bundle hash, a prepare-time authorizer verdict, required extract publication metadata, a bundle-declared maximum extract age, and declared row, statement-step, cell, and time bounds | +| Statement source | A reviewed statement artifact covered by the package sum file, a prepare-time authorizer verdict, required extract publication metadata, a package-declared maximum extract age, and declared row, statement-step, cell, and time bounds | | Source authentication | Secret-referenced Basic, static Authorization header, static API-key header, or OAuth 2.0 client credentials by client secret or private-key JWT assertion; explicit local authoring may use no credential only at a canonical numeric-loopback HTTP origin. A statement source presents no credential at all | | Audit | `registry-platform-audit` JSONL sink on explicitly durable storage, fail-closed | | Signing | Flattened JWS JSON with one active ES256/P-256 key, RFC 7638 `kid`, explicit published and revoked sets, and a public JWKS endpoint | @@ -394,7 +394,7 @@ application-level proxy and ignores ambient HTTP proxy environment variables. A `sqlite-extract` source has no origin, no credential, and no transport security, because it opens one local file. Its Rust-fixed boundary is the -reviewed statement artifact under `queries/`, covered by the bundle hash and +reviewed statement artifact under `queries/`, covered by the package sum file and holding exactly one statement; the declared result columns in result order; the declared parameter bindings, each with exactly one origin, either an authorized selector or the optional preparation script; and its `maximumRows`, @@ -420,8 +420,8 @@ The bundle names a logical `extractProfile` and the closed runtime file binds it to one absolute path under `sourceExtracts`, exactly as it binds a private certificate authority. Startup refuses a profile the runtime did not bind and a binding no source names. The bound file must be a regular, non-symlink file -this process cannot write; it is digested into the computed runtime revision -and opened read-only and immutable. Its reserved `evidence_extract` table must +this process cannot write; its digest and file identity are captured and rechecked during startup, +and it is opened read-only and immutable. Its reserved `evidence_extract` table must carry exactly one publication row, and the source's `maximumExtractAgeSeconds` is compared against the evaluation instant before a single row is read. The statement's select list is its projection, so Rust maps the bounded result set @@ -758,11 +758,11 @@ fixtures, and production Rust contains no case-specific branch or type. concurrency bounds. - Bind each logical extract profile to one runtime-named file, refuse an unbound profile and a binding no source names, refuse a symlinked, - non-regular, or writable file, digest the bound file into the computed - runtime revision, and prove the statement's result columns and parameters + non-regular, or writable file, capture its digest and identity for startup, + and prove the statement's result columns and parameters against the real extract at startup. - Require the reserved publication-metadata row and refuse an extract past its - bundle-declared maximum age against the evaluation instant, before any row is + package-declared maximum age against the evaluation instant, before any row is read. - Run one acceptance definition over the statement transport against a real extract materialized from a committed text seed, and keep statement text, @@ -913,9 +913,9 @@ follow-up issue. | Values and validation | Every Version 1 Supported Value form declared in `CONCEPT.md` passes positive, negative, boundary, size, cardinality, Evidence construction, JWS serialization, and verification tests. The four initial assertion cases exercise boolean, controlled-code, time-bucket, multiple-concept, and multi-subject behavior through the full service. | | Selector and matching boundary | Identifier-only, compound no-identifier, additional-disambiguator, and multi-role selector profiles pass the complete service. Each profile has one exact field set. Missing, extra, unknown, mistyped, oversized, unauthorized, or wrong-origin values, and values a selector-bound source path segment cannot carry, fail as `request.selector_invalid` before the access-attempt audit, credential acquisition, and source access. Provider results are limited to `match`, `no_match`, and `ambiguous`; Evidence never performs broad candidate retrieval, scoring, or selection. Reviewed deterministic derivation may compare authorized selectors with facts from one unique authoritative record. Explicit false relationship evidence requires a complete valid relationship set. A source that lacks count metadata may return at most two minimally projected results solely to distinguish ambiguity. | | Source minimization | Rust executes only the requirement's closed `single` or `search-then-fetch` acquisition, or a kind added after that surface froze where the bundle declares it and the operator separately enabled it. Each stage has fixed transport authority, a fixed or closed selector/prior-fact-bound path, fixed non-secret headers, bounded reviewed query/body rendering, explicit response projection, one durable pre-access audit, and no retry. Search facts are schema-validated before every fixed fetch and never persist; a fetch reads only the prior facts its acquisition gives that stage; no response can choose transport or add a call the configuration did not fix. Request batches run sequentially in order unless both capability gates and one fixed-path HTTP source batch block authorize exactly one optimized call within its ceiling. Strategy is fixed before I/O, and optimized failure never fans out. The effective posture is the weakest among the acquisition's sources. Basic, static Authorization header, static API-key, and OAuth client-credentials authentication and all three postures pass generic contract tests through the same HTTP executor. Credential-free execution is a separate local-only exception pinned to an exact numeric-loopback HTTP origin. | -| Statement source minimization | A `sqlite-extract` source executes exactly one bundle-fixed reviewed statement, held to one statement per artifact and covered by the bundle hash, against the one extract file the runtime bound, and Rust binds every value into it by index so no value is ever rendered into statement text. SQLite's authorizer decides every action the compiled statement would take while it is prepared, permitting reads and refusing every write, schema, and control action, `ATTACH`, `DETACH`, `PRAGMA`, extension loading, non-deterministic functions, and the whole clock family; a denied action fails the bundle at load rather than at request time. The reserved `evidence_now` parameter carries the same evaluation instant the assertion reports, and a bundle declaring that name is refused. Every declared parameter has exactly one origin, so a preparation script cannot fill a selector parameter, return a name the source never declared, reach the reserved name, or leave a declared prepared parameter unfilled, and a preparation script and a prepared parameter are refused unless declared together. Startup proves the statement's real result columns and parameters against the bundle over the extract it will read, refuses an extract profile the runtime did not bind and a binding no source names, and refuses a symbolic link, a non-regular file, a file this process could write, and a path replaced before it was opened; the bound file's digest enters the computed runtime revision. The reserved `evidence_extract` table must carry exactly one publication row, and the declared `maximumExtractAgeSeconds` is compared against the evaluation instant before a single row is read. Row, cell, and response-byte bounds are enforced as the result is read, the statement-step and time bounds by the progress handler inside the engine, and a cancelled request returns its connection and its permit. The transport holds no credential of any kind, and no diagnostic, log, snapshot, or audit record carries statement text, a bound or result value, the extract path, or engine message text. | +| Statement source minimization | A `sqlite-extract` source executes exactly one package-fixed reviewed statement, held to one statement per artifact and covered by the package sum file, against the one extract file the runtime bound, and Rust binds every value into it by index so no value is ever rendered into statement text. SQLite's authorizer decides every action the compiled statement would take while it is prepared, permitting reads and refusing every write, schema, and control action, `ATTACH`, `DETACH`, `PRAGMA`, extension loading, non-deterministic functions, and the whole clock family; a denied action fails the package at load rather than at request time. The reserved `evidence_now` parameter carries the same evaluation instant the assertion reports, and a package declaring that name is refused. Every declared parameter has exactly one origin, so a preparation script cannot fill a selector parameter, return a name the source never declared, reach the reserved name, or leave a declared prepared parameter unfilled, and a preparation script and a prepared parameter are refused unless declared together. Startup proves the statement's real result columns and parameters against the package over the extract it will read, refuses an extract profile the runtime did not bind and a binding no source names, and refuses a symbolic link, a non-regular file, a file this process could write, and a path replaced before it was opened; the bound file's digest and file identity are captured and rechecked during startup. The reserved `evidence_extract` table must carry exactly one publication row, and the declared `maximumExtractAgeSeconds` is compared against the evaluation instant before a single row is read. Row, cell, and response-byte bounds are enforced as the result is read, the statement-step and time bounds by the progress handler inside the engine, and a cancelled request returns its connection and its permit. The transport holds no credential of any kind, and no diagnostic, log, snapshot, or audit record carries statement text, a bound or result value, the extract path, or engine message text. | | Authentication and authority | Strict OIDC verification and the configured principal claim fail closed. One authorization decision binds requester, optional actor, requirement revision, purpose, every role's selector profile and value origin, subject authority path, audience, and requested response format. Possessing selector values or discovery metadata, or choosing an API media type, creates no authority. Authenticated discovery lists only complete shapes matching exactly one authority path and valid token-owned selector material; unentitled, ambiguous, and invalid-context shapes are absent. Every denial occurs before credential acquisition or source access. | -| Privacy and audit | After successful authentication, every authorization refusal is durably accepted as a standalone minimal denial event before the generic `403`; sink failure returns the generic `503`. The event contains only the operation and event identifiers, assurance profile, bundle revision, scoped requester pseudonym, optional actor pseudonym, closed denial category and decision, timestamp, and duration. The pseudonym scope binds operator trust domain, requested purpose, and authenticated audience while omitting those inputs. The event omits untrusted requested requirement, purpose, subjects, unmatched authority, selector information, response protection, source, and evaluation material. Authentication, malformed-request, and invalid-selector failures remain operational-only. One access-attempt audit is durably accepted before every actual source stage. Rust serializes final immutable response bytes, durably accepts disclosure-release audit, then releases those exact bytes. Request batches use their distinct audit schema, one access event per physical call with bounded item groups by authority and subject set, and one terminal release with every ordered outcome or one value-free terminal failure. An all-unavailable release carries no signing key id. Sink failure blocks the applicable step. Audit records stage source identity but never prior facts or intermediate identifiers, records the closed response-protection mode and a signing key only for a release that signed at least one assertion, and uses at most one scoped keyed pseudonym over each complete canonical role and selector bundle. Neither audit, logs, errors, metrics, nor traces contain credentials, tokens, request nonces, raw selector values, per-field quasi-identifier hashes, source values, Supported Values, signed material, or raw subject identifiers. | +| Privacy and audit | After successful authentication, every authorization refusal is durably accepted as a standalone minimal denial event before the generic `403`; sink failure returns the generic `503`. The event contains only the operation and event identifiers, assurance profile, package digest, scoped requester pseudonym, optional actor pseudonym, closed denial category and decision, timestamp, and duration. The pseudonym scope binds operator trust domain, requested purpose, and authenticated audience while omitting those inputs. The event omits untrusted requested requirement, purpose, subjects, unmatched authority, selector information, response protection, source, and evaluation material. Authentication, malformed-request, and invalid-selector failures remain operational-only. One access-attempt audit is durably accepted before every actual source stage. Rust serializes final immutable response bytes, durably accepts disclosure-release audit, then releases those exact bytes. Request batches use their distinct audit schema, one access event per physical call with bounded item groups by authority and subject set, and one terminal release with every ordered outcome or one value-free terminal failure. An all-unavailable release carries no signing key id. Sink failure blocks the applicable step. Audit records stage source identity but never prior facts or intermediate identifiers, records the closed response-protection mode and a signing key only for a release that signed at least one assertion, and uses at most one scoped keyed pseudonym over each complete canonical role and selector bundle. Neither audit, logs, errors, metrics, nor traces contain credentials, tokens, request nonces, raw selector values, per-field quasi-identifier hashes, source values, Supported Values, signed material, or raw subject identifiers. | | Evidence and response integrity | Rust alone constructs Evidence, signed flattened JWS, the unsigned envelope, and the request-batch envelope. Signed JWS is mandatory and default, uses ES256/P-256, RFC 7638 service key identifiers, allowlisted protected headers and trusted key resolution, has verifiable nonce, independently expected subjects and output contract, audience, policy, and validity, and publishes usable active and planned-rotation public keys while revoked identifiers override cached selection. Request-batch available items are signed JWS only, stay in request order, and the exact complete envelope is bounded to 1 MiB, pre-audited, and returned unchanged. Deployable assurance uses a pinned non-exportable Transit signer whose public key matches the governed active JWK and passes startup sign-and-verify. Unsigned JSON is self-identifying, requires bundle and complete matched grant permission plus exact singular API selection, and makes no later-verification claim. Signed failure never falls back to unsigned or partial batch release. | | Failure and operations | Stable safe errors, reviewed existence-disclosure semantics, public collapse of `no_match` and `ambiguous` by default, request limits, per-principal and failed-selector-attempt rate controls, authenticated requester-scoped discovery, unauthenticated closed provider publication, health, readiness, dependency timeouts, and graceful shutdown work without exposing protected data. Discovery performs no source access and exposes no source plan, scripts, credentials, internal authority metadata, selector values, codelist values, protected operations, or unrelated definitions. Readiness fails for missing bundle, selector binding, credential, audit, or signing dependencies required by the configured deployment. | | Multiple definitions | All four definitions run concurrently in one process and one trust domain without script state, limits, identifiers, subjects, source responses, audit context, or results crossing definition boundaries. Unsafe combined disclosure and mutually distrustful issuer configurations are rejected. | @@ -925,7 +925,7 @@ follow-up issue. | Production build | An editable project remains local until its author supplies exact governance metadata, stable concept identifiers, and one synthetic fixture per question. `evidencectl package` consumes one explicit closed production target, follows no symlink or outside-project reference, creates no secret or runtime residue, delegates bundle validation and every fixture to the real `evidence` binary, atomically publishes only a complete candidate, and reproduces identical bundle bytes and revision from identical inputs. It creates no keys, callers, approvals, deployments, or network side effects. | | Target-host handoff | A reviewed candidate with independently provisioned owner-only production secrets passes `evidencectl doctor --runtime-config `, `evidencectl test`, and real startup. One authorized synthetic-subject HTTP request yields a signed assertion that `evidence verify` accepts only under independent `production` policy and trusted keys; the resulting access and disclosure audit entries are written under the `registry.evidence.audit/v2` envelope. | | Issuer handoff | External HTTPS OIDC is independently configured. Registered-client token acquisition and Evidence acceptance pass; issuer, audience, scope and grant-boundary mismatches fail without credentials, selectors, or source values in output. Local tooling uses pinned stock ThunderID. | -| Compose and bare-binary journey | The maintained Compose guidance mounts the candidate bundle unchanged and read-only, uses a distinct container runtime revision, separate read-only secrets, persistent audit storage, a private listener, and operator TLS. It documents service UID and secret modes, public HTTPS issuer routing, and image provenance without generating Compose output. The production tutorials execute from released bare binaries and include a real Curl boundary. | +| Compose and bare-binary journey | The maintained Compose guidance mounts the candidate package unchanged and read-only, uses a separate container runtime document, separate read-only secrets, persistent audit storage, a private listener, and operator TLS. It documents service UID and secret modes, public HTTPS issuer routing, and image provenance without generating Compose output. The production tutorials execute from released bare binaries and include a real Curl boundary. | | Stop boundary | No deferred capability from `CONCEPT.md` section 4 or section 15 is implemented or stubbed beyond the explicitly closed acquisition kinds, each of which fixes every call it may make in configuration before any call is made. This includes document evidence, credential lifecycle, OID4VCI, status lists, presentation verification, nonce or replay storage beyond stateless request-nonce echo and comparison, OOTS XML or AS4, citizen-to-agent delegation, agent runtimes or MCP, federation, workflow, a public requester-entitlement or definition catalog, searchable, mutable, aggregate, or federated catalogs, runtime bundle mutation, script-selected transport, response-led or general multi-call planning, an evidence-data call no declared acquisition fixed, response-led multi-source fulfillment, a policy engine, application database, message broker, or worker process. The package-derived public provider advertisement remains inside the boundary as a closed publication for external indexing, not a catalog runtime. | ## Required Version 1 acceptance tests @@ -1000,14 +1000,14 @@ At minimum, pin these acceptance and negative cases: cross-requirement match; and closed-projection negatives prove source, authorization, credential, signing, audit, and internal deployment fields are absent. Publication-only changes move the - bundle revision but not any assertion-semantic requirement revision. + package digest but not any assertion-semantic requirement revision. 28b. Each step of a planned signing-key rotation, publishing the next public key, activating it, and retiring the previous one, and an emergency - revocation through `signing.revokedKeyIds` move the bundle revision but not + revocation through `signing.revokedKeyIds` move the package digest but not any requirement revision, because key trust travels in the JWKS and the policy denylist rather than in the revision a relying party pins. An emergency identity-provider key revocation through - `authentication.revokedKeyIds` likewise moves the bundle revision but not + `authentication.revokedKeyIds` likewise moves the package digest but not any requirement revision, because it changes which callers are accepted, not what an assertion means. 29. Every declared Supported Value form rejects wrong scalar types, unknown @@ -1018,7 +1018,7 @@ At minimum, pin these acceptance and negative cases: 30. `source-derived`, `field-projected`, and `record-transformed` definitions use the same executor and report their acquisition guarantees honestly. 31. A serving process cannot reload, mutate, merge, or fall back to another - bundle revision at runtime. + package digest at runtime. 32. No test, log, trace, metric, audit event, snapshot, panic, or failure artifact contains the canary credentials, raw selector values, source facts, or Supported Values used by the acceptance suite. @@ -1058,8 +1058,8 @@ At minimum, pin these acceptance and negative cases: sanitized extract, the real router, both audit gates, signed JWS, explicitly authorized unsigned output, and strict verification on one revision. -45. Governed bundle and runtime configuration have separate closed schemas, - independent startup digests, read-only lifetime enforcement, and negative +45. Governed package and runtime configuration have separate closed schemas, + package-digest verification, one-time read-only runtime capture, and negative tests proving runtime fields cannot override sources, authorization, disclosure, limits, signing, or audit policy. 46. Fixed paths and tagged selector or fetch prior-fact path templates pass exact encoding tests. @@ -1159,7 +1159,7 @@ At minimum, pin these acceptance and negative cases: `evidencectl doctor`. The declared acquisition ceiling bounds the source exchanges and the transitions between stages as a dependency failure under its own safe category, without ever cancelling a durable audit append. -67. A `sqlite-extract` source executes one bundle-fixed reviewed statement +67. A `sqlite-extract` source executes one package-fixed reviewed statement against the extract the runtime bound, returns the declared columns in result order beside the extract's own publication row, and presents no credential of any kind. @@ -1190,7 +1190,7 @@ At minimum, pin these acceptance and negative cases: owned value is built. A cancelled request gives back its connection and its permit. 73. A bound extract must be a regular, non-symlink file this process cannot - write, and its digest enters the computed runtime revision. An extract + write, and its digest and file identity are captured and rechecked during startup. An extract profile the runtime did not bind, a binding no source names, and a path replaced between its digest and its opening each fail at startup by name and cause. diff --git a/products/evidence/OPERATOR-CONTRACT.md b/products/evidence/OPERATOR-CONTRACT.md index 375236bef8..d9c7d809b0 100644 --- a/products/evidence/OPERATOR-CONTRACT.md +++ b/products/evidence/OPERATOR-CONTRACT.md @@ -113,34 +113,32 @@ not an approval, promotion, deployment, key-generation, caller-registration, or service-start command. It runs the real `evidence` binary through its bundle-only validation entry point and evaluates every referenced fixture without generating a temporary signing key or other validation secret. It then -atomically publishes a candidate with a copied `runtime.yaml` and one closed -`bundle/`. The candidate contains no production private key, credential, token, +atomically publishes one closed package with `SHA256SUMS` and an optional +`REVISION`. Runtime configuration remains in the deployment target. The package contains no production private key, credential, token, local request, audit entry, or source response. -The operator reviews and transfers the exact candidate, records its bundle -revision, and independently provisions the signing key, audit HMAC key, +The operator reviews and transfers the exact package, records its package +digest, and independently provisions the signing key, audit HMAC key, subject-binding HMAC key, and source credentials below the runtime's secret root. Secret ownership and mode requirements remain unchanged: each referenced secret is a regular owner-only file accepted by the eventual service identity. -The bundle and runtime must be non-writable to that identity. The copied -runtime is target-specific; its revision and bound private-CA bytes are not the -bundle revision, and signed assertions continue to carry only a configuration +The package and runtime must be non-writable to that identity. The +runtime is target-specific and remains outside the package. Signed assertions continue to carry only a configuration revision as `configurationRevision`. That value is scoped to the one requirement -the assertion answers, not to the whole deployment, so it is neither the runtime -revision nor the bundle revision. The public signing keys and +the assertion answers, not to the whole package digest. The public signing keys and `signing.revokedKeyIds` are outside it: publishing, activating, retiring, or -revoking a key changes the bundle revision and the JWKS but no +revoking a key changes the package digest and the JWKS but no `configurationRevision`. So is `authentication.revokedKeyIds`: revoking an -identity-provider key changes which caller tokens are accepted and the bundle -revision, but no `configurationRevision`. +identity-provider key changes which caller tokens are accepted and the package +digest, but no `configurationRevision`. Run the following grouped handoff after provisioning and whenever candidate bytes, runtime bindings, trust files, or secrets change: ```sh -evidencectl doctor --runtime-config '/runtime.yaml' -evidencectl test '' -evidence --runtime '/runtime.yaml' serve +evidencectl doctor --runtime-config '/runtime.yaml' +evidencectl test '' --target '' +evidence serve --runtime-config '/runtime.yaml' ``` `doctor` delegates the runtime-owned startup dependency preflight without @@ -153,15 +151,15 @@ independently prepared `production` policy and trusted keys, and confirm that its access and disclosure audit entries reached the audit destination. Configure an HTTPS OIDC issuer independently of Evidence. Its client registration -must bind the approved resource and scopes; the Evidence runtime pins issuer, -JWKS URI, audiences, allowed algorithms, token types, and claim mappings. -Inspect the candidate with `evidencectl artifact inspect ` and verify +must bind the approved resource and scopes; the governed bundle pins issuer, +JWKS URI, audience, allowed algorithms, token types, and claim mappings. +Inspect the installed package and target with `evidencectl artifact inspect ` and verify an actual issuer-to-resource request at handoff. Inspection does not register a client, decide authority, or issue a token. Maintained local tooling uses pinned stock ThunderID. Docker Compose remains a documented deployment adapter, never build output. -It mounts the approved candidate bundle unchanged and read-only, supplies a +It mounts the approved package unchanged and read-only, supplies a separate container runtime file and owner-readable secret mounts, gives only the audit path persistent writable storage, binds Evidence privately, and keeps public TLS and routing operator-controlled. The OIDC issuer retains its @@ -222,7 +220,7 @@ Discovery uses five separately trusted surfaces: | RFC 9728 protected-resource metadata | Binds the exact configured public Evidence origin to one authorization-server issuer, the Evidence JWKS location, and header-only bearer transport. | It contains no requester-scoped definition or entitlement data and does not replace HTTPS or an out-of-band trust pin. | | Generated Evidence OpenAPI | Describes `GET /v1/evidence-definitions`, `POST /v1/evidence`, `POST /v1/evidence/batch`, operational routes, envelopes, media types, and safe problems. | It contains no deployment definitions or entitlements. | | Public provider advertisement | Serves the exact packaged `catalog.jsonld` bytes at `GET /catalog.jsonld`, with public service identity and one distinct binding for each exact Evidence Type and compatible response profile. | It contains no requester-specific request shape, entitlement, source configuration, credential, or trust decision. | -| Authenticated definition response | Lists the exact complete request shapes available to this verified token at this bundle revision, each with the configuration revision an assertion for that one requirement carries. | It performs no provider access, does not grant authority, and is not a global catalog. | +| Authenticated definition response | Lists the exact complete request shapes available to this verified token from the loaded package, each with the configuration revision an assertion for that one requirement carries. | It performs no provider access, does not grant authority, and is not a global catalog. | | Static onboarding material | Gives an approved consumer token-acquisition instructions, human descriptions, legal context, endpoint trust, and verifier policy through the existing API catalog, developer portal, configuration repository, or bilateral process. | It is not accepted by the runtime and grants no authority. | | Evidence JWKS | Publishes the active and retained public verification keys. | It is not a trust anchor and contains no definition or entitlement metadata. | @@ -268,7 +266,7 @@ The publication workflow is: 1. Review the complete bundle and its combined disclosure surface. 2. Run `evidence check` and every referenced fixture, and record the exact - governed bundle revision. + governed package digest. 3. Run the production `evidencectl package` flow, which generates and seals `catalog.jsonld`, then publish the generic OpenAPI, provider advertisement, and static onboarding material. Configure token issuance and verifier trust @@ -619,9 +617,12 @@ integer slots paired with minimized selectors and closed parameters. It must return an exact slot bijection over ordinary lookup results. Missing, duplicate, extra, negative, or out-of-range slots abort the whole request. -A source may name a logical TLS trust profile. `runtime.yaml` binds it to one -bounded PEM CA file. Hostname and fixed-origin verification remain mandatory; -there is no insecure or trust-all mode. Version 1 ignores `HTTP_PROXY`, +A source may name a logical TLS trust profile, and so may `authentication` for +the connection that fetches the access-token issuer's key set. `runtime.yaml` +binds each name to one bounded PEM CA file, trusted beside the system roots for +the connections of the source or issuer that names it and no other. Hostname +and fixed-origin verification remain mandatory; there is no insecure or +trust-all mode. Version 1 ignores `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, and `NO_PROXY` and has no application-level proxy. ## Audit and operational data @@ -868,14 +869,13 @@ absent serves none of it. ```yaml metricsListener: - bindHost: 127.0.0.1 - port: 9090 + bind: 127.0.0.1:9090 ``` -`bindHost` accepts a numeric loopback, RFC 1918 private IPv4, or RFC 4193 -unique-local IPv6 address. Hostnames and unspecified, multicast, and public -addresses are rejected at startup, as is a `bindHost` and `port` pair that -repeats the evidence listener binding. Both listeners bind before either +`bind` is a `host:port` socket address whose host is a numeric loopback, RFC +1918 private IPv4, or RFC 4193 unique-local IPv6 address. Hostnames and +unspecified, multicast, and public addresses are rejected at startup, as are +port `0` and an address that repeats the evidence listener binding. Both listeners bind before either serves, so a rejected telemetry binding fails startup rather than leaving a service that reports healthy while publishing nothing. The two share one lifecycle: the telemetry listener cannot outlive a failed evidence listener. @@ -951,7 +951,7 @@ the registry even though no individual request is described. The accepted address range is therefore a floor, not a boundary. Startup rejects the mistake that actually exposes telemetry, a public or unspecified -`bindHost`, but an accepted RFC 1918 or unique-local address only means the +`bind` host, but an accepted RFC 1918 or unique-local address only means the endpoint is unreachable from the public internet. On a flat pod network or a shared VPC every workload already holds such an address, so binding one there makes the endpoint scrapable by every neighbouring workload. `127.0.0.1` with @@ -1001,11 +1001,13 @@ summary line's verdict and evaluated-case count move inside the document rather than trailing it, and the exit code and the operator message on standard error are unchanged. See the fixture reference for what it prints. -All commands accept `--runtime `. The same path may be supplied -through `REGISTRY_EVIDENCE_RUNTIME`; the reference default is +Every command that reads a deployment takes `--runtime-config ` +after the subcommand; the maintained container image passes `/etc/registry-evidence/runtime.yaml`. That file supplies the absolute -`bundleDirectory`. Command-line or environment values cannot override governed -bundle fields. The runtime file, bundle directory, and every captured artifact must +`package.root`. The earlier `--runtime` flag and `REGISTRY_EVIDENCE_RUNTIME` +variable are refused with the replacement named, so a stale invocation fails +instead of silently reading another file. Command-line or environment values +cannot override governed bundle fields. The runtime file, bundle directory, and every captured artifact must be non-writable to the service process. Evidence Version 1 supports Unix targets only because its secret and audit invariants require owner, mode, no-follow, link-count, and open-file identity checks. A read-only mount is preferred; @@ -1065,8 +1067,21 @@ the deployment's responsibility, not Evidence's; Evidence resolves its own configured destination and never inspects mounts. Failures name which side failed and no path. +Adding `--without-audit-lock` checks a candidate staged beside the running +instance it will replace, which shares its audit path and so holds the +single-writer lock by design. The option requires +`--require-runtime-dependencies`. The audit boundary is proved without taking +that lock: the audit destination settings and hash key, an owner-controlled +directory the service user can write, and an existing active file and lock +companion that are owner-only, singly linked, and writable, with an active file +whose final entry is complete. Every other dependency is proved as without the +option, and no audit entry is appended. It does not detect a second writer, so +`serve` still refuses to start while another instance holds the lock; without +the option, a held lock refuses the check with `another process holds the +single-writer lock beside the audit file`. + For `assuranceProfile: local`, a supervised issuer may use the exact canonical -issuer origin `http://127.0.0.1:` only when `jwksUri` is the +issuer origin `http://127.0.0.1:` only when `jwksSource.uri` is the same origin plus `/.well-known/jwks.json` or `/oauth2/jwks`. Production and evidence-grade, and every other authentication location, remain HTTPS-only. @@ -1127,10 +1142,10 @@ to publish a fresh extract and restart. Startup itself does not refuse an already-stale extract because a restart racing a republish would otherwise crashloop. -The access-token issuer's `jwksUri` is retrieved once at startup and again on +The access-token issuer's `jwksSource.uri` is retrieved once at startup and again on each readiness check, subject to the verifier cache lifecycle and a short suppression interval after a failure. Both report and neither refuses: a -`jwksUri` that cannot be used is named in the log at startup rather than +`jwksSource.uri` that cannot be used is named in the log at startup rather than discovered one rejected request at a time, but the issuer is a shared dependency this deployment does not own, so an issuer outage does not withhold its readiness or prevent it from starting. A key set already retrieved keeps diff --git a/products/evidence/README.md b/products/evidence/README.md index b519bb5ce2..03e1e7991f 100644 --- a/products/evidence/README.md +++ b/products/evidence/README.md @@ -179,7 +179,7 @@ applies the same authoring validation and reports its status, findings, and revision with the authored inventory. Add `--target ` to include that target's governance. -`evidencectl package --target --output ` +`evidencectl package --target --output ` is create-only. It reads regular files without following symlinks, compiles one closed bundle, and delegates its internal bundle-only check and every referenced fixture to the real `evidence` @@ -189,16 +189,16 @@ request, opens no listener, writes no production audit event, and never copies local `.evidence` state, credentials, tokens, responses, or private keys into the candidate. -The candidate contains `runtime.yaml` and `bundle/`; the bundle may contain -adapters, derivations, schemas, codelists, fixtures, and public keys where -referenced. The operator independently provisions the Transit key and +The package contains `SHA256SUMS` at its root and may contain adapters, +derivations, schemas, codelists, fixtures, and public keys where referenced. +Runtime configuration remains in the environment target. The operator independently provisions the Transit key and workload-local proxy, plus audit, subject-binding, and source secrets, then runs one grouped handoff: ```sh -evidencectl doctor --runtime-config '/runtime.yaml' -evidencectl test '' -evidence --runtime '/runtime.yaml' serve +evidencectl doctor --runtime-config '/runtime.yaml' +evidencectl test '' --target '' +evidence serve --runtime-config '/runtime.yaml' ``` Doctor delegates the runtime-owned live startup dependency preflight without @@ -214,7 +214,7 @@ algorithms, token type, and claim mappings declared by the Evidence runtime. Register the workload's client, resource and scopes at that issuer. The maintained local development tooling uses pinned stock ThunderID; production issuer registration remains an operator responsibility. Inspect the deployment with -`evidencectl artifact inspect ` and verify its actual token and +`evidencectl artifact inspect ` and verify its actual token and resource journey before handoff. Registry Stack publishes the `ghcr.io/registrystack/evidence` runtime image. diff --git a/products/evidence/contracts/acceptance-test-traceability.yaml b/products/evidence/contracts/acceptance-test-traceability.yaml index c3b3af790c..aeadc5d6c2 100644 --- a/products/evidence/contracts/acceptance-test-traceability.yaml +++ b/products/evidence/contracts/acceptance-test-traceability.yaml @@ -206,10 +206,10 @@ entries: - {file: crates/registry-evidence/tests/source_contracts.rs, name: every_acquisition_posture_fixture_executes_with_one_bounded_request} - {file: crates/registry-evidence/tests/source_contracts.rs, name: every_frozen_source_shape_executes_through_production_materialization_and_projection} - id: acceptance-row-31 - summary: A serving process cannot reload, mutate, merge, or fall back to another bundle revision at runtime. + summary: A serving process cannot reload, mutate, merge, or fall back to another verified package at runtime. tests: - - {file: crates/registry-evidence/src/runtime_tests.rs, name: serving_runtime_never_reloads_merges_or_falls_back_after_bundle_capture} - - {file: crates/registry-evidence/src/bundle.rs, name: revision_binds_paths_and_exact_bytes_deterministically} + - {file: crates/registry-evidence/src/runtime_tests.rs, name: serving_runtime_never_reloads_merges_or_falls_back_after_package_capture} + - {file: crates/registry-evidence/src/bundle.rs, name: package_digest_binds_paths_and_exact_bytes_deterministically} - {file: crates/registry-evidence/src/bundle.rs, name: writable_bundle_and_unknown_files_fail_closed} - id: acceptance-row-32 summary: No test, log, trace, metric, audit event, snapshot, panic, or failure artifact contains the canary credentials, raw selector values, source facts, or Supported Values used by the acceptance suite. @@ -295,10 +295,10 @@ entries: - {file: crates/registry-evidence/src/main.rs, name: offline_cli_evaluates_the_combined_acceptance_bundle} - {file: crates/registry-evidence/src/kernel.rs, name: all_four_acceptance_bundles_use_the_same_kernel} - id: acceptance-row-45 - summary: Governed bundle and runtime configuration have separate closed schemas, independent startup digests, read-only lifetime enforcement, and negative tests proving runtime fields cannot override sources, authorization, disclosure, limits, signing, or audit policy. + summary: Governed package and runtime configuration have separate closed schemas, package-digest verification, one-time read-only runtime capture, and negative tests proving runtime fields cannot override sources, authorization, disclosure, limits, signing, or audit policy. tests: - {file: crates/registry-evidence/src/config.rs, name: runtime_document_is_closed_and_contains_no_governed_override_surface} - - {file: crates/registry-evidence/src/bundle.rs, name: runtime_and_ca_bytes_are_captured_under_an_independent_read_only_revision} + - {file: crates/registry-evidence/src/bundle.rs, name: runtime_and_ca_bytes_are_captured_from_independent_read_only_inputs} - {file: crates/registry-evidence/src/bundle.rs, name: writable_bundle_and_unknown_files_fail_closed} - {file: crates/registry-evidence/src/bundle.rs, name: symlinked_artifact_fails_before_file_access} - id: acceptance-row-46 @@ -484,10 +484,10 @@ entries: - {file: crates/registry-evidence/src/main.rs, name: offline_cli_evaluates_the_holder_bound_acceptance_bundle} - {file: crates/registry-evidence/src/model.rs, name: the_offline_stand_in_holder_key_is_a_key_the_request_boundary_would_accept} - id: acceptance-row-73 - summary: A bound extract is a regular non-symlink file the process cannot write, its digest enters the runtime revision, binding names are exact in both directions, and replacement between digest and open fails startup. + summary: A bound extract is a regular non-symlink file the process cannot write, its digest and file identity are captured, binding names are exact in both directions, and replacement between digest and open fails startup. tests: - {file: crates/registry-evidence/src/bundle.rs, name: every_unusable_extract_is_refused_by_its_own_name_and_cause} - - {file: crates/registry-evidence/src/bundle.rs, name: an_extract_reaches_the_runtime_revision_as_a_digest_of_its_bytes} + - {file: crates/registry-evidence/src/bundle.rs, name: an_extract_is_bound_as_a_digest_of_its_bytes} - {file: crates/registry-evidence/src/bundle.rs, name: a_bound_extract_refuses_a_path_replaced_before_it_was_opened} - {file: crates/registry-evidence/src/bundle.rs, name: a_source_extract_binding_must_be_exact_in_both_directions} - id: acceptance-row-74 diff --git a/products/evidence/contracts/audit-event.schema.yaml b/products/evidence/contracts/audit-event.schema.yaml index b8d4e7ebe0..2afa684b01 100644 --- a/products/evidence/contracts/audit-event.schema.yaml +++ b/products/evidence/contracts/audit-event.schema.yaml @@ -158,7 +158,7 @@ allOf: audit_rules: envelope: Each record is the record member of one platform audit entry with the members schema, eventId, time, phase, correlation, and record, written as one JSON line by the shared platform audit writer to the operator-configured file or stdout destination. The entry schema is registry.evidence.audit/v2 for an authorized-material record and registry.evidence.audit.authorization-refusal/v2 for an authorization-refusal record; the record itself carries no schema member. The entry phase is request for access-attempt and response for disclosure-release, denial, and transient-failure. The entry correlation is the record's operation, so the access-attempt entries of a multi-stage acquisition share the correlation of the one terminal entry that closes it. authorization_refusal_gate: After successful authentication, an authorization refusal is durably accepted before the generic HTTP 403 is returned. Audit failure changes the outward result to service.unavailable with HTTP 503. - authorization_refusal_minimization: The refusal event contains only its event identifier, server-minted operation, timestamp and duration, assurance profile, bundle revision, actor kind, scoped requester and optional client, grant, or actor pseudonyms, and closed denial reason, category, and decision. The public trace identifier never becomes this operation identity. Pseudonym scope binds the operator trust domain, requested purpose, and authenticated audience, while those scope inputs remain omitted from the event. It omits the untrusted requested requirement, purpose, subjects, unmatched authority, selector information, response protection, and source or evaluation material. + authorization_refusal_minimization: The refusal event contains only its event identifier, server-minted operation, timestamp and duration, assurance profile, the package digest carried in bundleRevision, actor kind, scoped requester and optional client, grant, or actor pseudonyms, and closed denial reason, category, and decision. The public trace identifier never becomes this operation identity. Pseudonym scope binds the operator trust domain, requested purpose, and authenticated audience, while those scope inputs remain omitted from the event. It omits the untrusted requested requirement, purpose, subjects, unmatched authority, selector information, response protection, and source or evaluation material. access_gate: One access-attempt is durably accepted after authorization and before credential acquisition or source access for every actual source stage; search-then-fetch therefore records search access and, only after a unique validated match, fetch access, and search-then-fetch-set records search access followed by one access per declared fetch member in declared order, each reached only after every earlier stage resolved. A stage that is never reached records nothing. unresolved_decision: A configured exact source-level unresolved Problem Details outcome records the neutral denial decision unresolved only when it terminates a singular acquisition or search stage. It does not assert no-match or ambiguity, and the event records none of the upstream problem body, type, code, detail, or trace. The same outcome after a unique search is a dependency failure because the acquisition has already committed to a fetch or member reference. release_gate: disclosure-release is durably accepted after the final immutable response bytes are serialized and before those exact bytes are released. One operation has exactly one terminal release event, whatever it released; a release carrying more than one assertion names the complete released set in evidenceIds, in release order, instead of naming a single assertion in evidenceId. Splitting a batch into one event per member is not available, because that would make one request either several operations or one operation with several terminal events, and the audit contract accepts neither. diff --git a/products/evidence/contracts/bundle.schema.yaml b/products/evidence/contracts/bundle.schema.yaml index cc711134e2..c69b22b509 100644 --- a/products/evidence/contracts/bundle.schema.yaml +++ b/products/evidence/contracts/bundle.schema.yaml @@ -1,6 +1,6 @@ $schema: https://json-schema.org/draft/2020-12/schema $id: https://registrystack.org/schemas/evidence/bundle-v1.json -title: Evidence immutable deployment bundle Version 1 +title: Evidence governed package configuration Version 1 type: object additionalProperties: false required: @@ -20,16 +20,16 @@ required: properties: version: description: >- - Selects the bundle grammar the rest of this document is read against, and a version + Selects the package configuration grammar the rest of this document is read against, and a version the runtime does not implement is rejected at startup. const: 1 assuranceProfile: description: >- - Declares the governed assurance boundary of the whole bundle, and the value is visible + Declares the governed assurance boundary of the whole package, and the value is visible in every assertion, so an authentic `local` assertion never satisfies a relying procedure expecting a deployable profile. `local` is an authoring profile that lets a requirement omit `fixtures` and disables no other runtime boundary; the deployable profiles require - every requirement to reference a fixture suite with complete coverage before the bundle + every requirement to reference a fixture suite with complete coverage before the package loads. enum: - local @@ -324,14 +324,18 @@ allOf: properties: authentication: properties: - issuer: - pattern: ^(?:https://.*|http://127\.0\.0\.1:[1-9][0-9]{0,4})$ - jwksUri: - # Local development permits a supervised issuer on the exact - # canonical loopback origin, serving its JWKS from that origin at - # any absolute path without query or fragment. The route is the - # issuer's to choose; the origin is not. - pattern: ^(?:https://.*|http://127\.0\.0\.1:[1-9][0-9]{0,4}/[^\s?#]*)$ + oidc: + properties: + issuer: + pattern: ^(?:https://.*|http://127\.0\.0\.1:[1-9][0-9]{0,4})$ + jwksSource: + properties: + uri: + # Local development permits a supervised issuer on the exact + # canonical loopback origin, serving its JWKS from that origin at + # any absolute path without query or fragment. The route is the + # issuer's to choose; the origin is not. + pattern: ^(?:https://.*|http://127\.0\.0\.1:[1-9][0-9]{0,4}/[^\s?#]*)$ - if: properties: assuranceProfile: @@ -342,10 +346,14 @@ allOf: properties: authentication: properties: - issuer: - pattern: ^https:// - jwksUri: - pattern: ^https:// + oidc: + properties: + issuer: + pattern: ^https:// + jwksSource: + properties: + uri: + pattern: ^https:// requirements: items: required: @@ -454,7 +462,7 @@ $defs: - holder-bound secret-ref: type: string - pattern: ^secret:file/[a-z][a-z0-9._-]{0,127}$ + pattern: ^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$ relative-path: type: string pattern: ^(adapters|derivations|schemas|codelists|fixtures)/[A-Za-z0-9._/-]+$ @@ -471,22 +479,28 @@ $defs: type: object additionalProperties: false required: - - kind + - oidc + properties: + oidc: + description: >- + The one OpenID Connect issuer whose access tokens this deployment accepts, and the + rules a token from it must satisfy. + $ref: '#/$defs/oidc-authentication' + oidc-authentication: + type: object + additionalProperties: false + required: - issuer - - audiences + - audience - tokenTypes - algorithms - - jwksUri + - jwksSource - principalClaim - requesterTagsClaim - evidenceAudienceClaim - maximumTokenLifetimeSeconds - revokedKeyIds properties: - kind: - description: >- - Selects the inbound credential model, and Version 1 defines exactly one such profile. - const: oidc-access-token issuer: description: >- Exact token issuer an inbound access token must declare, with path-based issuers @@ -494,18 +508,13 @@ $defs: type: string pattern: ^https?:// maxLength: 512 - audiences: + audience: description: >- - Exact allowlist of audience values an inbound access token must carry to be accepted. - type: array - minItems: 1 - maxItems: 16 - uniqueItems: true - items: - description: One exact audience value an inbound access token may carry. - type: string - minLength: 1 - maxLength: 512 + The one exact audience value an inbound access token must carry to be accepted; it is + also the resource this deployment advertises in its protected-resource metadata. + type: string + minLength: 1 + maxLength: 512 tokenTypes: description: >- Allowlist of the exact `typ` value an inbound token's protected JWT header must @@ -535,14 +544,36 @@ $defs: - EdDSA - ES256 - RS256 - jwksUri: + jwksSource: description: >- - Fixed endpoint the verifier fetches issuer signing keys from; the endpoint may resolve - to a public or private address, so DNS is pinned for each fetch, ambient proxies - are disabled, and cloud-metadata destinations remain prohibited. - type: string - pattern: ^https?:// - maxLength: 512 + Where the verifier fetches issuer signing keys. Evidence accepts only `kind: uri`, a + fixed endpoint named in the governed bundle; discovery and static key sets are + refused. + type: object + additionalProperties: false + required: + - kind + - uri + properties: + kind: + description: >- + Selects the fixed-endpoint key source, the only one Evidence accepts. + const: uri + uri: + description: >- + Fixed endpoint the verifier fetches issuer signing keys from; the endpoint may + resolve to a public or private address, so DNS is pinned for each fetch, ambient + proxies are disabled, and cloud-metadata destinations remain prohibited. + type: string + pattern: ^https?:// + maxLength: 512 + tlsTrustProfile: + description: >- + Logical name resolved to an exact private certificate authority file bound in + `runtime.yaml`, trusted beside the system roots for the `jwksSource.uri` connection + alone. Omission trusts only the system roots, and the profile is rejected outright + when `jwksSource.uri` is a local HTTP origin. + $ref: '#/$defs/local-id' principalClaim: description: >- The only claim read as the principal, whose absence denies the request; `client_id`, @@ -651,6 +682,18 @@ $defs: minLength: 1 maxLength: 256 pattern: ^[\x21\x23-\x5B\x5D-\x7E]+$ + if: + required: + - jwksSource + properties: + jwksSource: + properties: + uri: + pattern: ^http:// + then: + not: + required: + - tlsTrustProfile audit: type: object additionalProperties: false @@ -660,8 +703,8 @@ $defs: properties: hashKeyRef: description: >- - Points at the file secret Rust derives the audit pseudonym key from; the same - reference cannot also serve `subjectBinding.secretRef`. Every audit gate is fail + Points at the secret Rust derives the audit pseudonym key from; the same reference + cannot also serve `subjectBinding.secretRef`. Every audit gate is fail closed whatever this section says, so an authorization refusal, access attempt, or disclosure release whose entry cannot be written stops the response instead of being served unaudited. @@ -682,7 +725,7 @@ $defs: properties: secretRef: description: >- - Points at the file secret keying every subject binding; that reference and its resolved + Points at the secret keying every subject binding; that reference and its resolved bytes must both differ from the audit master, and a match fails startup. $ref: '#/$defs/secret-ref' keyVersion: @@ -3231,7 +3274,9 @@ startup_checks: and cross-referenced to the exact authorized role/profile/field set. secret_policy: permitted: >- - File-provider logical names matching the exact secret-reference grammar only. + Secret references matching the exact secret-reference grammar only: `secret:file/name` + logical names, and `secret:env/NAME` variable names, which resolve only when the runtime + enables secretProviders.environment. prohibited: >- private key, password, bearer token, client secret, or expanded environment value in any bundle file @@ -3240,8 +3285,16 @@ secret_policy: There is no query-string placement, so no credential can reach a token URL, and the token URL, body, response, and debug output are still fully redacted. file_provider_rule: >- - Resolve secret:file logical names beneath the configured owner-controlled fileRoot, reject + Resolve secret:file logical names beneath the configured owner-controlled file root, reject symlinks and path traversal, require regular owner-only files, and parse values as data. + environment_provider_rule: >- + Resolve secret:env names from the process environment only when the runtime declares + secretProviders.environment; a bundle reference naming the environment is refused at + startup otherwise, and values are parsed as data. + environment_expression_rule: >- + The bundle is parsed as written. A `${...}` environment expression in any bundle key or + string value is refused at startup, naming the field and never the expression; + substitution applies to runtime.yaml only. deployment_rules: authentication: >- Evidence independently validates the bearer token against exact configured issuer, audience, diff --git a/products/evidence/contracts/rhai-abi.yaml b/products/evidence/contracts/rhai-abi.yaml index 663f2148eb..e2e55200e2 100644 --- a/products/evidence/contracts/rhai-abi.yaml +++ b/products/evidence/contracts/rhai-abi.yaml @@ -171,7 +171,7 @@ capabilities: lifecycle: compilation: Startup only; each ordinary script must expose exactly one public entry point at the required arity before readiness; extraction alone permits either arity two or three, never both. Each batch block names distinct preparation and extraction scripts exposing exactly prepare_batch/2 and extract_batch/2 respectively. state: 'Fresh invocation state and fresh input copies; only the immutable schema-validated search FactSet crosses into a fetch stage, whole or projected onto the allowlist that stage declares, with no cross-request or cross-definition mutable state. One fetch stage never observes another fetch stage: facts flow forward from the search into each member and forward again into derivation, never sideways.' - identity: Script path and exact bytes are covered by the governed bundle revision. + identity: Script path and exact bytes are listed in the governed package's SHA256SUMS file. failure: preparation: adapter_input_error extraction: source_protocol_error diff --git a/products/evidence/contracts/runtime.schema.yaml b/products/evidence/contracts/runtime.schema.yaml index a65d18a910..ff76757849 100644 --- a/products/evidence/contracts/runtime.schema.yaml +++ b/products/evidence/contracts/runtime.schema.yaml @@ -4,24 +4,44 @@ title: Evidence closed operator runtime configuration Version 1 type: object additionalProperties: false required: - - version - - bundleDirectory + - apiVersion + - kind + - package - listener - secretProviders - signer - audit - outboundTls properties: - version: + apiVersion: description: >- - Pins the runtime document to Version 1 of this grammar; the loader rejects any other + Names the grammar this runtime document is written in; the loader refuses any other value, so a future incompatible runtime shape is never read as if it were this one. - const: 1 - bundleDirectory: + const: registry.registrystack.org/evidence-runtime/v1alpha1 + kind: description: >- - Selects the one governed bundle directory this process loads at startup; no alternate, - overlay, or fallback bundle path exists for a running deployment. - $ref: '#/$defs/absolute-path' + Names the document type; together with `apiVersion` it tells an Evidence runtime file + apart from any other product's runtime configuration. + const: EvidenceRuntimeConfig + package: + description: >- + Selects the one governed package this process verifies and loads at startup; no alternate, + overlay, or fallback package path exists for a running deployment. + type: object + additionalProperties: false + required: + - root + properties: + root: + description: >- + Absolute path of the governed package directory this process verifies and loads. + $ref: '#/$defs/absolute-path' + expectedDigest: + description: >- + Optional `sha256:` digest of the package's `SHA256SUMS` file. When set, startup is + refused unless the verified package found at `package.root` has exactly this digest. + type: string + pattern: ^sha256:[0-9a-f]{64}$ listener: description: >- Groups the network binding and per-request admission limits for the process's evidence @@ -30,8 +50,7 @@ properties: type: object additionalProperties: false required: - - bindHost - - port + - bind - tlsTermination - trustProxyIdentityHeaders - maximumRequestBytes @@ -39,19 +58,20 @@ properties: - requestTimeoutMilliseconds - shutdownGraceMilliseconds properties: - bindHost: + bind: description: >- - Numeric listener address. The default `private-address` exposure accepts loopback, - RFC 1918 private IPv4, or RFC 4193 unique-local IPv6. The explicit + Numeric socket address the evidence API listens on, written `host:port`, with an + IPv6 host in brackets (`[addr]:port`). The default `private-address` exposure accepts + a loopback, RFC 1918 private IPv4, or RFC 4193 unique-local IPv6 host. The explicit `container-private` exposure additionally accepts the unspecified IPv4 or IPv6 wildcard for a container network confined by the operator. Multicast, public, and - hostname values are prohibited in both modes. - type: string - minLength: 2 - maxLength: 64 + hostname values are prohibited in both modes, and port `0`, which would ask the + kernel for an arbitrary ephemeral port instead of naming one, is refused. + $ref: '#/$defs/socket-address' x-runtime-validation: >- - Parsed as an IP address and checked together with listener.networkExposure. Wildcards - require the explicit container-private value; public addresses remain prohibited. + Parsed as a socket address and its host checked together with + listener.networkExposure. Wildcards require the explicit container-private value; + public addresses remain prohibited. networkExposure: description: >- Declares the operator-owned network boundary around the listener. Absent preserves @@ -63,14 +83,6 @@ properties: - private-address - container-private default: private-address - port: - description: >- - TCP port the process binds for the evidence API alongside `listener.bindHost`; port - `0`, which would ask the kernel for an arbitrary ephemeral port instead of naming - one, is refused. - type: integer - minimum: 1 - maximum: 65535 tlsTermination: description: >- Declares that TLS terminates at an operator-controlled upstream in front of this @@ -122,43 +134,34 @@ properties: type: object additionalProperties: false required: - - bindHost - - port + - bind properties: - bindHost: + bind: description: >- - Numeric loopback, RFC 1918 private IPv4, or RFC 4193 unique-local IPv6 address. - Unspecified, multicast, public, and hostname values are prohibited. - type: string - minLength: 2 - maxLength: 64 + Numeric socket address for the optional metrics listener, written `host:port`. The + host must be a loopback, RFC 1918 private IPv4, or RFC 4193 unique-local IPv6 + address; unspecified, multicast, public, and hostname values are prohibited, port + `0` is refused, and the address must not repeat the evidence listener's binding. + $ref: '#/$defs/socket-address' x-runtime-validation: >- - Parsed as an IP address and accepted only when Rust classifies it as loopback, private - IPv4, or unique-local IPv6. - port: - description: >- - TCP port for the optional metrics listener; paired with `metricsListener.bindHost`, - it must not repeat the evidence listener's exact host-and-port binding. - type: integer - minimum: 1 - maximum: 65535 + Parsed as a socket address and accepted only when Rust classifies its host as + loopback, private IPv4, or unique-local IPv6. x-runtime-validation: >- - Rejected at startup when bindHost and port together repeat the evidence listener binding. + Rejected at startup when the bind address repeats the evidence listener binding. secretProviders: description: >- - Configures how a `secret:file/...` reference resolves to bytes at startup; it supplies - the storage location only, and which secret each governed field names is fixed by the - bundle, not by this section. + Enables the providers a secret reference may resolve through; it supplies storage + locations only, and which secret each governed field names is fixed by the bundle, not + by this section. At least one provider is declared, and a bundle reference naming a + provider this section does not enable is refused at startup. type: object additionalProperties: false - required: - - file + minProperties: 1 properties: file: description: >- - Declares the file-based secret provider and its root directory; Version 1 defines - no other provider kind, so every `secret:file/name` reference resolves through this - one path. + Enables the file-based secret provider and names its root directory; every + `secret:file/name` reference resolves beneath this one path. type: object additionalProperties: false required: @@ -171,6 +174,14 @@ properties: opened file is checked for type, ownership, mode, and link count before its bytes are read. $ref: '#/$defs/absolute-path' + environment: + description: >- + Enables `secret:env/NAME` references, read from the process environment when a + request needs them. It takes no settings and is written `environment: {}`; absent + means no reference may name the environment. + type: object + additionalProperties: false + maxProperties: 0 signer: description: >- Binds the process to the private-key transport that produces every signature over the @@ -310,23 +321,25 @@ properties: systemRoots: description: >- Fixes the system certificate authority store as always trusted for outbound calls; - because Version 1 has no way to disable it, a source's public certificate must still - chain to a trusted root even when a private trust profile also applies. + because Version 1 has no way to disable it, a source's or the access-token issuer's + public certificate must still chain to a trusted root even when a private trust + profile also applies. const: true trustProfiles: description: >- - Closed map of private certificate-authority bundles a source may reference by logical - id; the set of ids present here must exactly match the `tlsTrustProfile` names the - bundle's sources declare, with no extra or missing profile. + Closed map of private certificate-authority bundles a source or the access-token + issuer may reference by logical id; the set of ids present here must exactly match + the `tlsTrustProfile` names the bundle's sources and `authentication` declare, with + no extra or missing profile. type: object maxProperties: 64 propertyNames: $ref: '#/$defs/local-id' additionalProperties: description: >- - One named private trust profile, binding a logical id a source's `tlsTrustProfile` - reference names to the certificate-authority bundle used only for that source's - outbound connections. + One named private trust profile, binding a logical id a `tlsTrustProfile` + reference names to the certificate-authority bundle used only for the outbound + connections of the source or access-token issuer that names it. type: object additionalProperties: false required: @@ -335,8 +348,8 @@ properties: caBundleFile: description: >- Absolute path to one bounded PEM file of trust anchors, loaded and validated - at startup; changing its bytes requires a restart and changes the computed - runtime digest. + at startup; changing its bytes requires a restart before the new trust anchors + can be used. $ref: '#/$defs/absolute-path' sourceExtracts: description: >- @@ -362,8 +375,8 @@ properties: description: >- Absolute path to one read-only regular file, validated at startup and digested without being read into memory; it must be neither a symbolic link nor writable, - because the statement executor opens it as immutable, and its digest changes the - computed runtime digest. + because the statement executor opens it as immutable. Startup binds the exact file + identity and content digest for the process lifetime. $ref: '#/$defs/absolute-path' acquisitionCapabilities: description: >- @@ -391,7 +404,15 @@ $defs: pattern: ^[a-z][a-z0-9._-]{0,127}$ secret-ref: type: string - pattern: ^secret:file/[a-z][a-z0-9._-]{0,127}$ + pattern: ^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$ + socket-address: + description: >- + A numeric `host:port` socket address with a non-zero port, the host in brackets when + it is IPv6. + type: string + minLength: 1 + maxLength: 128 + pattern: ^(?:[0-9]{1,3}(?:\.[0-9]{1,3}){3}|\[[0-9A-Fa-f:.]+\]):(?!0+$)[0-9]{1,5}$ absolute-path: type: string minLength: 2 @@ -400,10 +421,10 @@ $defs: ownership: governed_fields: prohibited allowed: - - bundle directory + - bundle package root and optional expected revision - listener binding and process limits - optional operator metrics listener binding - - file-secret root + - enabled secret providers and the file-secret root - signer binding to a local private JWK or workload-local Transit proxy - audit destination, path, rotation size, and retention - logical private-CA file bindings @@ -417,14 +438,15 @@ startup: mutability: >- runtime.yaml and bound CA files are captured read-only once; reload, merge, fallback, and partial serving are prohibited - digest: >- - independent SHA-256 revision over exact runtime.yaml bytes plus logical trust-profile - names and exact CA bytes + substitution: >- + `${NAME}` and `${NAME:-default}` expressions in string values are substituted from the + process environment after parsing; they are refused inside secret references and inside + secretProviders trust_profiles: >- - names must exactly equal the logical tlsTrustProfile names used by the governed bundle + names must exactly equal the logical tlsTrustProfile names the governed bundle's sources + and authentication use secrets: >- - values and provider tokens are never parsed into or included in the runtime document or - digest + values and provider tokens are never parsed into or included in the runtime document signer: >- local assurance requires local-jwk; production and evidence-grade require Transit through an absolute Unix socket. Transit key version is pinned and no provider token is supplied @@ -436,5 +458,5 @@ platform: supported: Unix reason: >- Version 1 requires Unix owner, mode, no-follow, link-count, and file-identity guarantees - for the runtime file, the bundle directory and its artifacts, named CA bundle files, the + for the runtime file, the package directory and its artifacts, named CA bundle files, the secret root, source extracts, individual secrets, and the audit file destination. diff --git a/products/evidence/contracts/security-invariant-matrix.yaml b/products/evidence/contracts/security-invariant-matrix.yaml index b8953ba2fb..db946fa206 100644 --- a/products/evidence/contracts/security-invariant-matrix.yaml +++ b/products/evidence/contracts/security-invariant-matrix.yaml @@ -90,7 +90,7 @@ invariants: - id: V1-I18 rule: Configuration is immutable for the serving process lifetime. threat: Unreviewed hot mutation, partial revisions, rollback, or inconsistent audit provenance. - enforcement: One read-only atomic governed bundle and one separately digested closed runtime file at startup; runtime overrides, reload, merge, fallback, and mutation paths do not exist. + enforcement: One read-only package verified from its SHA256SUMS and one closed runtime file captured at startup; runtime overrides, reload, merge, fallback, and mutation paths do not exist. negative_test: sec-runtime-bundle-mutation-absent - id: V1-I19 rule: One process serves one operator-controlled trust domain. @@ -215,7 +215,7 @@ invariants: - id: V1-I43 rule: A sqlite-extract source executes exactly one bundle-fixed reviewed statement, against one read-only extract file the runtime bound, under its declared row, cell, statement-step, time, and concurrency bounds, and neither a caller, a response, nor a script can change the statement, the file, or those bounds. threat: The statement transport becomes a query channel into a registry data tier, a statement writes to or reaches outside the file it was prepared against, an unbounded result moves a whole register into the process, a statement reads a clock of its own and becomes non-deterministic, or an unattributed or stale extract answers as though it were current. - enforcement: The statement is a hash-covered bundle artifact holding exactly one statement, and Rust binds every value into it by index so no value is ever rendered into statement text. SQLite's authorizer decides every action of the compiled statement while it is prepared, allowing only reads and refusing every write, schema, control, attach, detach, and pragma action plus a closed denied function list that includes the whole clock family; its final match arm denies, so an action a later dependency version introduces refuses the statement rather than widening the boundary silently. The runtime binds its own evaluation instant to the reserved evidence_now parameter, which a bundle may not declare. Every parameter declares one origin and exactly one, so a preparation script fills the parameters the bundle declared prepared and cannot return a value for a parameter the bundle fills from an authorized selector, return a name no binding declares, reach the reserved name, or leave a declared prepared parameter unfilled; each of those is a request failure naming the statement artifact, and a preparation script and a prepared parameter are refused at startup unless declared together. Refusals settleable without data are startup failures through the offline check, and opening the extract additionally proves the statement's result columns and parameters against the bundle at startup. The extract is opened read-only and immutable after startup refuses a symbolic link, a non-regular file, and a file this process could write, and its digest enters the runtime revision. Its reserved evidence_extract table must carry exactly one publication row, and the declared maximumExtractAgeSeconds is checked against the evaluation instant before a single row is read. Row, cell, and response-byte bounds are enforced as the result is read, and the step and time bounds are enforced by a progress handler inside the engine, which is the only cancellation a blocking execution has. Every failure carries one closed cause and its artifact, and never statement text, a bound or result value, the extract path, or a message SQLite wrote. + enforcement: The statement is a hash-covered package artifact holding exactly one statement, and Rust binds every value into it by index so no value is ever rendered into statement text. SQLite's authorizer decides every action of the compiled statement while it is prepared, allowing only reads and refusing every write, schema, control, attach, detach, and pragma action plus a closed denied function list that includes the whole clock family; its final match arm denies, so an action a later dependency version introduces refuses the statement rather than widening the boundary silently. The runtime binds its own evaluation instant to the reserved evidence_now parameter, which a package may not declare. Every parameter declares one origin and exactly one, so a preparation script fills the parameters the package declared prepared and cannot return a value for a parameter the package fills from an authorized selector, return a name no binding declares, reach the reserved name, or leave a declared prepared parameter unfilled; each of those is a request failure naming the statement artifact, and a preparation script and a prepared parameter are refused at startup unless declared together. Refusals settleable without data are startup failures through the offline check, and opening the extract additionally proves the statement's result columns and parameters against the package at startup. The extract is opened read-only and immutable after startup refuses a symbolic link, a non-regular file, and a file this process could write, and its digest and file identity are captured for the process. Its reserved evidence_extract table must carry exactly one publication row, and the declared maximumExtractAgeSeconds is checked against the evaluation instant before a single row is read. Row, cell, and response-byte bounds are enforced as the result is read, and the step and time bounds are enforced by a progress handler inside the engine, which is the only cancellation a blocking execution has. Every failure carries one closed cause and its artifact, and never statement text, a bound or result value, the extract path, or a message SQLite wrote. negative_test: sec-statement-source-bounded - id: V1-I44 rule: Under the holder-bound subject binding, every subject binding is derived from the presented holder key's canonical RFC 7638 thumbprint under a domain separate from the audience-scoped one, and from nothing about the requester. One holder receives one binding however many relying parties collect the assertion on its behalf, two holders never receive the same binding for the same subject, and no key thumbprint can be made to collide with any audience. @@ -330,11 +330,11 @@ cross_cutting: negative_test: sec-request-preparation-closed runtime_ownership_split: threat: An environment-specific runtime file silently changes governed authorization, disclosure, source authority, signing, or audit policy. - enforcement: Closed runtime.yaml accepts only process-local listener, optional metrics-listener, path, secret-root, audit destination, signer transport and pinned version, logical private-CA bindings, and logical source-extract file bindings; a bound extract file no source names and a source extract profile the runtime did not bind are both refused, and the signer must exactly match the governed active public JWK and the runtime has an independent immutable digest. + enforcement: Closed runtime.yaml accepts only process-local listener, optional metrics-listener, path, secret-root, audit destination, signer transport and pinned version, logical private-CA bindings, and logical source-extract file bindings; a bound extract file no source names and a source extract profile the runtime did not bind are both refused, and the signer must exactly match the governed active public JWK, and runtime-owned bytes are captured once and held read-only for the process lifetime without a separate public runtime revision. negative_test: sec-runtime-cannot-override-governed-bundle outbound_tls_and_proxy: threat: A mutable or untrusted CA or ambient proxy redirects credentials and protected source queries to another authority. - enforcement: Runtime captures validated private-CA bytes at startup, fixed-origin hostname verification stays enabled, and both evidence-data and OAuth clients ignore ambient proxy variables. + enforcement: Runtime captures validated private-CA bytes at startup, fixed-origin hostname verification stays enabled, and both evidence-data and OAuth clients ignore ambient proxy variables. The access-token issuer's key-set connection trusts a private CA only through the one profile its authentication block names, beside the system roots and for that connection alone, and the runtime binds that profile exactly as it binds a source's. negative_test: sec-tls-and-proxy-authority-fixed subject_role_order: threat: Caller-controlled array position substitutes one subject role for another or changes the signed binding order. diff --git a/products/evidence/contracts/security-test-traceability.yaml b/products/evidence/contracts/security-test-traceability.yaml index 42378bfef8..339f621289 100644 --- a/products/evidence/contracts/security-test-traceability.yaml +++ b/products/evidence/contracts/security-test-traceability.yaml @@ -141,9 +141,9 @@ entries: tests: [{file: crates/registry-evidence/src/binding.rs, name: every_subject_binding_scope_component_is_cryptographically_bound}] - id: sec-runtime-bundle-mutation-absent tests: - - {file: crates/registry-evidence/src/bundle.rs, name: revision_binds_paths_and_exact_bytes_deterministically} + - {file: crates/registry-evidence/src/bundle.rs, name: package_digest_binds_paths_and_exact_bytes_deterministically} - {file: crates/registry-evidence/src/bundle.rs, name: requirement_projection_prunes_unreached_connections_and_covers_selected_behavior} - - {file: crates/registry-evidence/src/runtime_tests.rs, name: serving_runtime_never_reloads_merges_or_falls_back_after_bundle_capture} + - {file: crates/registry-evidence/src/runtime_tests.rs, name: serving_runtime_never_reloads_merges_or_falls_back_after_package_capture} - id: sec-mutually-distrustful-configuration tests: [{file: crates/registry-evidence/src/config.rs, name: one_trust_domain_and_native_token_identity_are_closed_configuration}] - id: sec-rate-limit-does-not-legalize-ladder @@ -238,13 +238,18 @@ entries: - id: sec-runtime-cannot-override-governed-bundle tests: - {file: crates/registry-evidence/src/config.rs, name: runtime_document_is_closed_and_contains_no_governed_override_surface} - - {file: crates/registry-evidence/src/bundle.rs, name: runtime_and_ca_bytes_are_captured_under_an_independent_read_only_revision} + - {file: crates/registry-evidence/src/bundle.rs, name: runtime_and_ca_bytes_are_captured_from_independent_read_only_inputs} - id: sec-tls-and-proxy-authority-fixed tests: - {file: crates/registry-evidence/tests/source_contracts.rs, name: private_ca_tls_handshake_succeeds_and_hostname_mismatch_fails} - {file: crates/registry-evidence/tests/source_contracts.rs, name: private_ca_plan_rejects_unbound_missing_and_malformed_captures} - {file: crates/registry-evidence/tests/source_contracts.rs, name: runtime_ca_capture_rejects_symlink_malformed_and_mutable_files} - {file: crates/registry-evidence/tests/source_contracts.rs, name: ambient_proxy_variables_are_ignored_in_an_isolated_process} + - {file: crates/registry-evidence/tests/cli.rs, name: dependency_check_trusts_a_private_ca_issuer_only_through_its_named_profile} + - {file: crates/registry-evidence/tests/cli.rs, name: dependency_check_refuses_an_issuer_the_named_profile_does_not_vouch_for} + - {file: crates/registry-evidence/tests/cli.rs, name: dependency_check_refuses_an_issuer_profile_that_is_not_a_certificate_bundle} + - {file: crates/registry-evidence/src/config.rs, name: an_issuer_trust_profile_is_a_local_id_for_an_https_key_set_only} + - {file: crates/registry-evidence/src/bundle.rs, name: the_issuer_trust_profile_is_bound_exactly_like_a_source_profile} - id: sec-subject-array-order-nonsemantic tests: - {file: crates/registry-evidence/src/runtime_tests.rs, name: multi_role_request_order_is_not_semantic_and_output_uses_declaration_order} @@ -422,7 +427,7 @@ entries: - {file: crates/registry-evidence/src/config.rs, name: a_prepared_statement_parameter_name_is_held_to_the_preparation_abi} - {file: crates/registry-evidence/tests/statement_source.rs, name: a_parameter_is_filled_from_its_one_declared_origin} - {file: crates/registry-evidence/src/bundle.rs, name: every_unusable_extract_is_refused_by_its_own_name_and_cause} - - {file: crates/registry-evidence/src/bundle.rs, name: an_extract_reaches_the_runtime_revision_as_a_digest_of_its_bytes} + - {file: crates/registry-evidence/src/bundle.rs, name: an_extract_is_bound_as_a_digest_of_its_bytes} - {file: crates/registry-evidence/src/bundle.rs, name: a_bound_extract_refuses_a_path_replaced_before_it_was_opened} - {file: crates/registry-evidence/src/bundle.rs, name: a_source_extract_binding_must_be_exact_in_both_directions} - id: sec-holder-bound-binding-derivation diff --git a/products/evidence/contracts/sqlite-extract-source-contract.yaml b/products/evidence/contracts/sqlite-extract-source-contract.yaml index b2b280f97b..191ff9ce58 100644 --- a/products/evidence/contracts/sqlite-extract-source-contract.yaml +++ b/products/evidence/contracts/sqlite-extract-source-contract.yaml @@ -103,10 +103,10 @@ extract: file: form: One regular file. A symbolic link and a non-regular file are refused at startup. writability: A file this process could write is refused at startup, and a file on a read-only filesystem satisfies the check outright. - digest: The file is digested at startup in bounded chunks with identity checks bracketing the read, and the digest enters the computed runtime revision under source-extract/. There is no byte cap, because an extract is a register rather than an artifact. - still_bound: The digest and the SQLite open are not the same moment, because the bundle, the kernel, and the audit log are read in between. The file identity the digest was taken over is checked again once the connections are open, so a path refreshed inside that window fails startup rather than serving bytes the runtime revision does not name. This narrows the window rather than closing it, and it does not replace the deployment guarantee stated under connection.immutability. + digest: The file is digested at startup in bounded chunks with identity checks bracketing the read. There is no byte cap, because an extract is a register rather than a package artifact. + still_bound: The digest and the SQLite open are not the same moment, because the package, the kernel, and the audit log are read in between. The file identity the digest was taken over is checked again once the connections are open, so a path refreshed inside that window fails startup rather than serving bytes from another file. This narrows the window rather than closing it, and it does not replace the deployment guarantee stated under connection.immutability. checkpointed: A published extract is one whole file. An extract with a write-ahead log or a rollback journal beside it is refused at startup, because immutable=1 skips change detection and therefore skips those files. Committed frames in a log are read straight past, and a journal left by a writer that died mid-transaction is read as though the rows it never committed were authoritative, where an ordinary read-only opener refuses the file outright. A leftover shared-memory file is not refused, because one survives a clean checkpoint and says nothing about the snapshot. This detects a publishing mistake rather than preventing one, since a publisher who copies only the main file out of a live database leaves nothing beside it to find, which is why publishing a checkpointed single file is stated as an obligation and not only checked. - bundle_revision: The extract is not part of the bundle hash. Republishing one leaves a pinned bundle revision unchanged, which is why publication metadata and a declared maximum age are mandatory rather than advisory. + package_digest: The extract is process-local and does not enter the package digest. Republishing one leaves a pinned package unchanged, which is why publication metadata and a declared maximum age are mandatory rather than advisory. connection: uri_parameters: [mode=ro, immutable=1] open_flags: [SQLITE_OPEN_READ_ONLY, SQLITE_OPEN_URI, SQLITE_OPEN_NO_MUTEX] diff --git a/products/evidence/fixtures/acceptance/adult-status/SHA256SUMS b/products/evidence/fixtures/acceptance/adult-status/SHA256SUMS new file mode 100644 index 0000000000..1c97dcc3be --- /dev/null +++ b/products/evidence/fixtures/acceptance/adult-status/SHA256SUMS @@ -0,0 +1,10 @@ +d40e55f3f5e4b294abc9b87f4a95041557ad27128641ea83ef7a1bf7a29e0552 adapters/source-a-prepare.rhai +328fb4ead93fe33fb3a03d0b02fd3408fdb863bac35bb202c5cef09fcbd29d28 adapters/source-a.rhai +eeee8281409c627979b6124338c9160c298c6e9e1207e9e0cdaada7dca50fb8b catalog.jsonld +a7e447541c5da6a656e29e350a50969aa554ffdb1070c8b89c7fb6ae58a44681 derivations/adult-status.rhai +c2910dc3f34194fdaff1db0ba78a1d6bcbc53c2584aca7b6862a3ca7f7a780a3 evidence.yaml +a812e1a48ac2453441928390d12a45ddc185e149ed9aa4094e325ff260b5875a fixtures/cases.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +fb99d8ef387b4a2c24917aa852e7fddd358fa76ef8c50a22019da0de1a31978b schemas/adapter-parameters.schema.yaml +abefb41557b92264da888e86d7e5e1a833a73bb76e8b83470372f59dbabf8f37 schemas/facts.schema.yaml +592dee37dbc52d4f73a24f26a48b8832c99aac9b0136e3f71f7399140eb221e3 schemas/response.schema.yaml diff --git a/products/evidence/fixtures/acceptance/adult-status/evidence.yaml b/products/evidence/fixtures/acceptance/adult-status/evidence.yaml index 7832cf66f2..bc64a8c033 100644 --- a/products/evidence/fixtures/acceptance/adult-status/evidence.yaml +++ b/products/evidence/fixtures/acceptance/adult-status/evidence.yaml @@ -4,17 +4,19 @@ service: {providerId: urn:example:fixture:provider:evidence, publicOrigin: https issuer: {id: urn:example:fixture:issuer:authority} publication: {serviceId: urn:example:fixture:service:evidence, title: Synthetic Evidence service, description: Synthetic minimum-disclosure assertions for acceptance testing, endpointUrl: https://evidence.example.invalid, jurisdictions: [urn:example:jurisdiction:acceptance]} authentication: - kind: oidc-access-token - issuer: https://identity.invalid - audiences: [evidence-fixture] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: https://identity.invalid/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: https://identity.invalid + audience: evidence-fixture + tokenTypes: [at+jwt] + algorithms: [ES256] + jwksSource: + kind: uri + uri: https://identity.invalid/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: {hashKeyRef: secret:file/audit-hash-key, hashKeyVersion: 1} subjectBinding: {secretRef: secret:file/subject-binding-key, keyVersion: 1} rateLimits: {requestsPerPrincipalPerMinute: 60, burstPerPrincipal: 10, failedSelectorAttemptsPerPrincipalAuthorityPerMinute: 10} diff --git a/products/evidence/fixtures/acceptance/all-definitions/SHA256SUMS b/products/evidence/fixtures/acceptance/all-definitions/SHA256SUMS new file mode 100644 index 0000000000..77e3ad47b9 --- /dev/null +++ b/products/evidence/fixtures/acceptance/all-definitions/SHA256SUMS @@ -0,0 +1,34 @@ +f6aa0dba39caf4eb75c2205ffddebae899f23329b3fd199e0982b2f294ee53e2 adapters/adult-status-prepare.rhai +712035e748a8acb8e0b33294af9b8358b79a79e676586917715f2c9e8c0fdd23 adapters/adult-status-source.rhai +a6f4ec091085f3470deff419dbe022eb3f0368d8cf22be812c2cc0f8ee0c140a adapters/legal-parent-relationship-prepare.rhai +3657aa452e74bfc3fa9bf0e32a7905f591fc1bbf7d9c3aeb7ef9195ae595779a adapters/legal-parent-relationship-source.rhai +c7feb77fbc1ccc8020f013011a3d64df7999d64b1f01a0896bdbb32fde08a742 adapters/professional-licence-prepare.rhai +35cbb3e8b0b4f7d57bc8bcb51092ec16bad84bf48f2fd59a9eb8eb09267bcdc3 adapters/professional-licence-source.rhai +b158771cae78aa88d93a7b9217f49066b6bee56f3417c01e3f76b018cd9d5020 adapters/residence-region-prepare.rhai +27bbb0cd7e36133c0282eb632dcc183e7379dce96314c8a872caeadcb2344c31 adapters/residence-region-source.rhai +750a202654cc02437898efc7c80effa225a4f8bb0a83665fb7525b7292ac46db catalog.jsonld +fbc20764b24549898faacbe7399fe78795f2ee123fa82c5904954d56fcdec5fe codelists/professional-expiry-categories.yaml +3ad31cf9eeb227d838202d1af923a47538aba528bb3ec73cfabe790725d2b47f codelists/professional-registry-regions.yaml +026e5a406cd076a7bcdb92b5aa021e57e119c5fa5d84f840b7806b3b733bfa2f codelists/residence-region-map.yaml +a7e447541c5da6a656e29e350a50969aa554ffdb1070c8b89c7fb6ae58a44681 derivations/adult-status.rhai +e1e8c52a2c109a4649ee3097022c4ee6fdc2f8c9814aa17016044289fc7dbfe6 derivations/legal-parent-relationship.rhai +9c238b8d6e2b4af76f9a6bc536cad0adb7a698c03085799796ad4cbbbc44bfa5 derivations/professional-licence.rhai +76d13e1539d01316b1f79e59bd5bb5937afac3393491fa36a800f0ee96769a59 derivations/residence-region.rhai +620e04726c7c46dd844373929325a7c0248250facffab285fc2a45e13aa0c8ea evidence.yaml +4711240408b271db3dcbc89afe956542be8f66434df27d9e425ebf8db1ef53c4 fixtures/adult-status-cases.yaml +0bdfa63a413f415ea9b6c144886b93195b687c05c980dbe69aa8309675d4fca5 fixtures/legal-parent-relationship-cases.yaml +ed1fb6c099333226bce08ff0cae725ceaccea919f3e8f92553c550a71c15ebd5 fixtures/professional-licence-cases.yaml +105d1c431b47844d0e9868948ac8a90e28239dab273a50dd6c627ed3846b2c25 fixtures/residence-region-cases.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +84b20ba0e05562fb4edb10aebaed712224cfcf332b2c51467e14600124754977 schemas/adult-status-adapter-parameters.schema.yaml +abefb41557b92264da888e86d7e5e1a833a73bb76e8b83470372f59dbabf8f37 schemas/adult-status-facts.schema.yaml +592dee37dbc52d4f73a24f26a48b8832c99aac9b0136e3f71f7399140eb221e3 schemas/adult-status-response.schema.yaml +498cad52e7ab5a87ba4353141ddffb1b5b4fa811b1f63a2aa2951df39ba8cf58 schemas/legal-parent-relationship-adapter-parameters.schema.yaml +7881acb037e63a961ce5b111066dbb820066dee41f95463387bba559341ff6d6 schemas/legal-parent-relationship-facts.schema.yaml +9a4a25a6c78f75d911599f4e9d0d044e1cb52398404d1227c4990ff460e14326 schemas/legal-parent-relationship-response.schema.yaml +2d60df56edc63e3e33ab168ed57970696413d196f3faaed6d86ba25789464216 schemas/professional-licence-adapter-parameters.schema.yaml +1c55d98b7497a99c8514a1ad32888f423bd9e10010babfd2647e557bd0e3539e schemas/professional-licence-facts.schema.yaml +d8594b939cbf1f437de8576c9f255cae94bc7928f0f064a048058374453942a5 schemas/professional-licence-response.schema.yaml +ad06fc28a274075ab7b54c4aeaee08ceef7a4951c5fbe154e8bf5a7a71c45e8a schemas/residence-region-adapter-parameters.schema.yaml +1069510dc7a95f9951b65bf63029b3b68863cc257f51a02276a739c2cb763cb2 schemas/residence-region-facts.schema.yaml +afdc01bad5711fc536dfc955abec2368615194176842e0c953847964c232eb4a schemas/residence-region-response.schema.yaml diff --git a/products/evidence/fixtures/acceptance/all-definitions/evidence.yaml b/products/evidence/fixtures/acceptance/all-definitions/evidence.yaml index 139877547c..117e5fd548 100644 --- a/products/evidence/fixtures/acceptance/all-definitions/evidence.yaml +++ b/products/evidence/fixtures/acceptance/all-definitions/evidence.yaml @@ -7,18 +7,20 @@ service: issuer: {id: urn:example:fixture:issuer:authority} publication: {serviceId: urn:example:fixture:service:evidence, title: Synthetic Evidence service, description: Synthetic minimum-disclosure assertions for acceptance testing, endpointUrl: https://evidence.example.invalid, jurisdictions: [urn:example:jurisdiction:acceptance]} authentication: - kind: oidc-access-token - issuer: https://identity.invalid - audiences: [evidence-fixture] - allowedClients: [evidence-task-agent] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: https://identity.invalid/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: https://identity.invalid + audience: evidence-fixture + allowedClients: [evidence-task-agent] + tokenTypes: [at+jwt] + algorithms: [ES256] + jwksSource: + kind: uri + uri: https://identity.invalid/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: {hashKeyRef: secret:file/audit-hash-key, hashKeyVersion: 1} subjectBinding: {secretRef: secret:file/subject-binding-key, keyVersion: 1} rateLimits: {requestsPerPrincipalPerMinute: 60, burstPerPrincipal: 10, failedSelectorAttemptsPerPrincipalAuthorityPerMinute: 10} diff --git a/products/evidence/fixtures/acceptance/holder-bound/SHA256SUMS b/products/evidence/fixtures/acceptance/holder-bound/SHA256SUMS new file mode 100644 index 0000000000..698b5ada8c --- /dev/null +++ b/products/evidence/fixtures/acceptance/holder-bound/SHA256SUMS @@ -0,0 +1,34 @@ +f6aa0dba39caf4eb75c2205ffddebae899f23329b3fd199e0982b2f294ee53e2 adapters/adult-status-prepare.rhai +712035e748a8acb8e0b33294af9b8358b79a79e676586917715f2c9e8c0fdd23 adapters/adult-status-source.rhai +a6f4ec091085f3470deff419dbe022eb3f0368d8cf22be812c2cc0f8ee0c140a adapters/legal-parent-relationship-prepare.rhai +3657aa452e74bfc3fa9bf0e32a7905f591fc1bbf7d9c3aeb7ef9195ae595779a adapters/legal-parent-relationship-source.rhai +c7feb77fbc1ccc8020f013011a3d64df7999d64b1f01a0896bdbb32fde08a742 adapters/professional-licence-prepare.rhai +35cbb3e8b0b4f7d57bc8bcb51092ec16bad84bf48f2fd59a9eb8eb09267bcdc3 adapters/professional-licence-source.rhai +b158771cae78aa88d93a7b9217f49066b6bee56f3417c01e3f76b018cd9d5020 adapters/residence-region-prepare.rhai +27bbb0cd7e36133c0282eb632dcc183e7379dce96314c8a872caeadcb2344c31 adapters/residence-region-source.rhai +e48aaf37464412e2fb5e26316a136d31dff77a83857d345badda80bb647d93dc catalog.jsonld +fbc20764b24549898faacbe7399fe78795f2ee123fa82c5904954d56fcdec5fe codelists/professional-expiry-categories.yaml +3ad31cf9eeb227d838202d1af923a47538aba528bb3ec73cfabe790725d2b47f codelists/professional-registry-regions.yaml +026e5a406cd076a7bcdb92b5aa021e57e119c5fa5d84f840b7806b3b733bfa2f codelists/residence-region-map.yaml +a7e447541c5da6a656e29e350a50969aa554ffdb1070c8b89c7fb6ae58a44681 derivations/adult-status.rhai +e1e8c52a2c109a4649ee3097022c4ee6fdc2f8c9814aa17016044289fc7dbfe6 derivations/legal-parent-relationship.rhai +9c238b8d6e2b4af76f9a6bc536cad0adb7a698c03085799796ad4cbbbc44bfa5 derivations/professional-licence.rhai +76d13e1539d01316b1f79e59bd5bb5937afac3393491fa36a800f0ee96769a59 derivations/residence-region.rhai +9e1fc8c270ea95bff00ee048f6e0fa3e3d90c21b3f0e517758e6eda1f4fff07e evidence.yaml +4711240408b271db3dcbc89afe956542be8f66434df27d9e425ebf8db1ef53c4 fixtures/adult-status-cases.yaml +0bdfa63a413f415ea9b6c144886b93195b687c05c980dbe69aa8309675d4fca5 fixtures/legal-parent-relationship-cases.yaml +ed1fb6c099333226bce08ff0cae725ceaccea919f3e8f92553c550a71c15ebd5 fixtures/professional-licence-cases.yaml +105d1c431b47844d0e9868948ac8a90e28239dab273a50dd6c627ed3846b2c25 fixtures/residence-region-cases.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +84b20ba0e05562fb4edb10aebaed712224cfcf332b2c51467e14600124754977 schemas/adult-status-adapter-parameters.schema.yaml +abefb41557b92264da888e86d7e5e1a833a73bb76e8b83470372f59dbabf8f37 schemas/adult-status-facts.schema.yaml +592dee37dbc52d4f73a24f26a48b8832c99aac9b0136e3f71f7399140eb221e3 schemas/adult-status-response.schema.yaml +498cad52e7ab5a87ba4353141ddffb1b5b4fa811b1f63a2aa2951df39ba8cf58 schemas/legal-parent-relationship-adapter-parameters.schema.yaml +7881acb037e63a961ce5b111066dbb820066dee41f95463387bba559341ff6d6 schemas/legal-parent-relationship-facts.schema.yaml +9a4a25a6c78f75d911599f4e9d0d044e1cb52398404d1227c4990ff460e14326 schemas/legal-parent-relationship-response.schema.yaml +2d60df56edc63e3e33ab168ed57970696413d196f3faaed6d86ba25789464216 schemas/professional-licence-adapter-parameters.schema.yaml +1c55d98b7497a99c8514a1ad32888f423bd9e10010babfd2647e557bd0e3539e schemas/professional-licence-facts.schema.yaml +d8594b939cbf1f437de8576c9f255cae94bc7928f0f064a048058374453942a5 schemas/professional-licence-response.schema.yaml +ad06fc28a274075ab7b54c4aeaee08ceef7a4951c5fbe154e8bf5a7a71c45e8a schemas/residence-region-adapter-parameters.schema.yaml +1069510dc7a95f9951b65bf63029b3b68863cc257f51a02276a739c2cb763cb2 schemas/residence-region-facts.schema.yaml +afdc01bad5711fc536dfc955abec2368615194176842e0c953847964c232eb4a schemas/residence-region-response.schema.yaml diff --git a/products/evidence/fixtures/acceptance/holder-bound/evidence.yaml b/products/evidence/fixtures/acceptance/holder-bound/evidence.yaml index fc3faa100f..aefb362876 100644 --- a/products/evidence/fixtures/acceptance/holder-bound/evidence.yaml +++ b/products/evidence/fixtures/acceptance/holder-bound/evidence.yaml @@ -24,18 +24,20 @@ service: issuer: {id: urn:example:fixture:issuer:authority} publication: {serviceId: urn:example:fixture:service:evidence, title: Synthetic Evidence service, description: Synthetic minimum-disclosure assertions for acceptance testing, endpointUrl: https://evidence.example.invalid, jurisdictions: [urn:example:jurisdiction:acceptance]} authentication: - kind: oidc-access-token - issuer: https://identity.invalid - audiences: [evidence-fixture] - allowedClients: [evidence-task-agent] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: https://identity.invalid/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: https://identity.invalid + audience: evidence-fixture + allowedClients: [evidence-task-agent] + tokenTypes: [at+jwt] + algorithms: [ES256] + jwksSource: + kind: uri + uri: https://identity.invalid/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: {hashKeyRef: secret:file/audit-hash-key, hashKeyVersion: 1} subjectBinding: {secretRef: secret:file/subject-binding-key, keyVersion: 1} rateLimits: {requestsPerPrincipalPerMinute: 60, burstPerPrincipal: 10, failedSelectorAttemptsPerPrincipalAuthorityPerMinute: 10} diff --git a/products/evidence/fixtures/acceptance/legal-parent-relationship/SHA256SUMS b/products/evidence/fixtures/acceptance/legal-parent-relationship/SHA256SUMS new file mode 100644 index 0000000000..b2ce76168a --- /dev/null +++ b/products/evidence/fixtures/acceptance/legal-parent-relationship/SHA256SUMS @@ -0,0 +1,10 @@ +169fde4aadbbc353116b5081b52fccfea9b204eded22e444fa7d42abe542215b adapters/source-d-prepare.rhai +f0bf977be2abe238812e00e67fc9c83123888fc9f47ebb0e2d9023eb145bd6f7 adapters/source-d.rhai +77127a3d43affc3adb1bec7969078e96d3952e3710cbd0b8316b7a17e7421dc1 catalog.jsonld +e1fff4020193041f9d70e93f140c79b263cb1dabab6e5f605e3d67a378b9a6b6 derivations/legal-parent-relationship.rhai +42076fe7c17bffb182eb0df6d372cd10012a09285fe1e0fcec6a20f6e0a84e4f evidence.yaml +5236b1eff0b8b9c44e1ca80937ace1c7a2b5878d1d3d0e5c8b6fd3860cb8d29e fixtures/cases.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +c2dbb295f40531781558aed948d869617bc7ed3de2dd702da2f5c05444d5ffa9 schemas/adapter-parameters.schema.yaml +7881acb037e63a961ce5b111066dbb820066dee41f95463387bba559341ff6d6 schemas/facts.schema.yaml +9a4a25a6c78f75d911599f4e9d0d044e1cb52398404d1227c4990ff460e14326 schemas/response.schema.yaml diff --git a/products/evidence/fixtures/acceptance/legal-parent-relationship/evidence.yaml b/products/evidence/fixtures/acceptance/legal-parent-relationship/evidence.yaml index 6acb72a88e..03829567bf 100644 --- a/products/evidence/fixtures/acceptance/legal-parent-relationship/evidence.yaml +++ b/products/evidence/fixtures/acceptance/legal-parent-relationship/evidence.yaml @@ -3,7 +3,8 @@ assuranceProfile: evidence-grade service: {providerId: urn:example:fixture:provider:evidence, publicOrigin: https://evidence.invalid, trustDomain: urn:example:fixture:trust-domain:acceptance} issuer: {id: urn:example:fixture:issuer:authority} publication: {serviceId: urn:example:fixture:service:evidence, title: Synthetic Evidence service, description: Synthetic minimum-disclosure assertions for acceptance testing, endpointUrl: https://evidence.example.invalid, jurisdictions: [urn:example:jurisdiction:acceptance]} -authentication: {kind: oidc-access-token, issuer: https://identity.invalid, audiences: [evidence-fixture], allowedClients: [evidence-task-agent], tokenTypes: [at+jwt], algorithms: [ES256], jwksUri: https://identity.invalid/.well-known/jwks.json, principalClaim: sub, requesterTagsClaim: evidence_tags, evidenceAudienceClaim: evidence_audience, maximumTokenLifetimeSeconds: 300, revokedKeyIds: []} +authentication: + oidc: {issuer: https://identity.invalid, audience: evidence-fixture, allowedClients: [evidence-task-agent], tokenTypes: [at+jwt], algorithms: [ES256], jwksSource: {kind: uri, uri: https://identity.invalid/.well-known/jwks.json}, principalClaim: sub, requesterTagsClaim: evidence_tags, evidenceAudienceClaim: evidence_audience, maximumTokenLifetimeSeconds: 300, revokedKeyIds: []} audit: {hashKeyRef: secret:file/audit-hash-key, hashKeyVersion: 1} subjectBinding: {secretRef: secret:file/subject-binding-key, keyVersion: 1} rateLimits: {requestsPerPrincipalPerMinute: 60, burstPerPrincipal: 10, failedSelectorAttemptsPerPrincipalAuthorityPerMinute: 10} diff --git a/products/evidence/fixtures/acceptance/professional-licence/SHA256SUMS b/products/evidence/fixtures/acceptance/professional-licence/SHA256SUMS new file mode 100644 index 0000000000..2ec0451c71 --- /dev/null +++ b/products/evidence/fixtures/acceptance/professional-licence/SHA256SUMS @@ -0,0 +1,12 @@ +1ef5cbb9c73f9455a0500b9336fb93daa58ff80e0b52bb5015a336360e60f5e7 adapters/source-c-prepare.rhai +d30e104b95049b360fa526ca30a8a48aa7a4eef2b52cafad0adbabf791813cc0 adapters/source-c.rhai +ae41b82d4e8551a6807f7b0c237b675565f3e233ab004c4ae3c3425b9d2f602d catalog.jsonld +fbc20764b24549898faacbe7399fe78795f2ee123fa82c5904954d56fcdec5fe codelists/expiry-categories.yaml +3ad31cf9eeb227d838202d1af923a47538aba528bb3ec73cfabe790725d2b47f codelists/registry-regions.yaml +9c238b8d6e2b4af76f9a6bc536cad0adb7a698c03085799796ad4cbbbc44bfa5 derivations/professional-licence.rhai +53acd458ccfe613689e698cbe758d42356079ad1225a62e6023aa29147a8a05a evidence.yaml +7126efc67f9d31e36d89a7253ed076dac6d205ec1475118ec0d83136d11c762d fixtures/cases.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +2d60df56edc63e3e33ab168ed57970696413d196f3faaed6d86ba25789464216 schemas/adapter-parameters.schema.yaml +1c55d98b7497a99c8514a1ad32888f423bd9e10010babfd2647e557bd0e3539e schemas/facts.schema.yaml +d8594b939cbf1f437de8576c9f255cae94bc7928f0f064a048058374453942a5 schemas/response.schema.yaml diff --git a/products/evidence/fixtures/acceptance/professional-licence/evidence.yaml b/products/evidence/fixtures/acceptance/professional-licence/evidence.yaml index 85546a7da8..3ce10c7fb0 100644 --- a/products/evidence/fixtures/acceptance/professional-licence/evidence.yaml +++ b/products/evidence/fixtures/acceptance/professional-licence/evidence.yaml @@ -3,7 +3,8 @@ assuranceProfile: evidence-grade service: {providerId: urn:example:fixture:provider:evidence, publicOrigin: https://evidence.invalid, trustDomain: urn:example:fixture:trust-domain:acceptance} issuer: {id: urn:example:fixture:issuer:authority} publication: {serviceId: urn:example:fixture:service:evidence, title: Synthetic Evidence service, description: Synthetic minimum-disclosure assertions for acceptance testing, endpointUrl: https://evidence.example.invalid, jurisdictions: [urn:example:jurisdiction:acceptance]} -authentication: {kind: oidc-access-token, issuer: https://identity.invalid, audiences: [evidence-fixture], tokenTypes: [at+jwt], algorithms: [ES256], jwksUri: https://identity.invalid/.well-known/jwks.json, principalClaim: sub, requesterTagsClaim: evidence_tags, evidenceAudienceClaim: evidence_audience, maximumTokenLifetimeSeconds: 300, revokedKeyIds: []} +authentication: + oidc: {issuer: https://identity.invalid, audience: evidence-fixture, tokenTypes: [at+jwt], algorithms: [ES256], jwksSource: {kind: uri, uri: https://identity.invalid/.well-known/jwks.json}, principalClaim: sub, requesterTagsClaim: evidence_tags, evidenceAudienceClaim: evidence_audience, maximumTokenLifetimeSeconds: 300, revokedKeyIds: []} audit: {hashKeyRef: secret:file/audit-hash-key, hashKeyVersion: 1} subjectBinding: {secretRef: secret:file/subject-binding-key, keyVersion: 1} rateLimits: {requestsPerPrincipalPerMinute: 60, burstPerPrincipal: 10, failedSelectorAttemptsPerPrincipalAuthorityPerMinute: 10} diff --git a/products/evidence/fixtures/acceptance/residence-region/SHA256SUMS b/products/evidence/fixtures/acceptance/residence-region/SHA256SUMS new file mode 100644 index 0000000000..3c758e99a8 --- /dev/null +++ b/products/evidence/fixtures/acceptance/residence-region/SHA256SUMS @@ -0,0 +1,11 @@ +de3793074e6cdea98456fe4fcf47b61b4dcb9e384456934a681eabb38fe4f912 adapters/source-b-prepare.rhai +9140d0841ab09c3c72f776cbcc4cad68f33e5001a9255f1ea4e9c6567c6dadce adapters/source-b.rhai +8cbeb75b527c948b35c9b2667451d482612f72f0a71bd8fcdd5d8b4f5dd401bd catalog.jsonld +026e5a406cd076a7bcdb92b5aa021e57e119c5fa5d84f840b7806b3b733bfa2f codelists/region-map.yaml +d2d2a759ae867f403b601d3f318474f486664450c47d7e8fc5b7411e30a64bb4 derivations/residence-region.rhai +c09dcb8861013e45348214e36523197411629b9cdcd3049da142fefd7253c989 evidence.yaml +6449a9bfd9e004297c3e9f06b2d33a242506f5cb562c6936cc10b679b3515c44 fixtures/cases.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +40a1a1b1dfcefd4ec16dcdae92447269b0ce36471a8b76d6e44a2b016a350bd3 schemas/adapter-parameters.schema.yaml +1069510dc7a95f9951b65bf63029b3b68863cc257f51a02276a739c2cb763cb2 schemas/facts.schema.yaml +afdc01bad5711fc536dfc955abec2368615194176842e0c953847964c232eb4a schemas/response.schema.yaml diff --git a/products/evidence/fixtures/acceptance/residence-region/evidence.yaml b/products/evidence/fixtures/acceptance/residence-region/evidence.yaml index bc97ffe1e1..b7029eb5ae 100644 --- a/products/evidence/fixtures/acceptance/residence-region/evidence.yaml +++ b/products/evidence/fixtures/acceptance/residence-region/evidence.yaml @@ -3,7 +3,8 @@ assuranceProfile: evidence-grade service: {providerId: urn:example:fixture:provider:evidence, publicOrigin: https://evidence.invalid, trustDomain: urn:example:fixture:trust-domain:acceptance} issuer: {id: urn:example:fixture:issuer:authority} publication: {serviceId: urn:example:fixture:service:evidence, title: Synthetic Evidence service, description: Synthetic minimum-disclosure assertions for acceptance testing, endpointUrl: https://evidence.example.invalid, jurisdictions: [urn:example:jurisdiction:acceptance]} -authentication: {kind: oidc-access-token, issuer: https://identity.invalid, audiences: [evidence-fixture], tokenTypes: [at+jwt], algorithms: [ES256], jwksUri: https://identity.invalid/.well-known/jwks.json, principalClaim: sub, requesterTagsClaim: evidence_tags, evidenceAudienceClaim: evidence_audience, maximumTokenLifetimeSeconds: 300, revokedKeyIds: []} +authentication: + oidc: {issuer: https://identity.invalid, audience: evidence-fixture, tokenTypes: [at+jwt], algorithms: [ES256], jwksSource: {kind: uri, uri: https://identity.invalid/.well-known/jwks.json}, principalClaim: sub, requesterTagsClaim: evidence_tags, evidenceAudienceClaim: evidence_audience, maximumTokenLifetimeSeconds: 300, revokedKeyIds: []} audit: {hashKeyRef: secret:file/audit-hash-key, hashKeyVersion: 1} subjectBinding: {secretRef: secret:file/subject-binding-key, keyVersion: 1} rateLimits: {requestsPerPrincipalPerMinute: 60, burstPerPrincipal: 10, failedSelectorAttemptsPerPrincipalAuthorityPerMinute: 10} diff --git a/products/evidence/fixtures/acceptance/surviving-spouse-status/SHA256SUMS b/products/evidence/fixtures/acceptance/surviving-spouse-status/SHA256SUMS new file mode 100644 index 0000000000..0916ff8e34 --- /dev/null +++ b/products/evidence/fixtures/acceptance/surviving-spouse-status/SHA256SUMS @@ -0,0 +1,20 @@ +8fadda5ea32c2efb55e69b516c50e8f64987559f605d62f9fe7ba1c885ac11c9 adapters/civil-record-search-prepare.rhai +b0757b843c6a02b47859049d71152f3e0fc31a0d463faefb20138c504d4c1fde adapters/civil-record-search.rhai +c11a2cdd58ccb5330e48c8a3332efea85e04b7f78f1c2a958c0fb19486944575 adapters/death-register-prepare.rhai +4ae9df4a6cff864d54f12349d624a6cf50aef873d6a8e1f6b663ba8e517dc9e5 adapters/death-register.rhai +6c211d4a7f78d3c2388d1ae7acefefd9b5049abc6bd3471e0567e5cd39915c73 adapters/union-register-prepare.rhai +def5ee7d05eb92dda2a97c0ea65d90ed743ed53d0d75e3d0e6475cb1cb23793a adapters/union-register.rhai +20fcdb561d0eb3acd477a657cda18e3672fb53cf132e731daf8ae1dd1d35d768 catalog.jsonld +e4c815082dc57b32eebe9d12883c9546c44cdbc5712d2b32073f63e5649247f2 derivations/surviving-spouse-status.rhai +270a073a54eac84dc1a0686f6ac5e09c4071d25986e5c344a0708f05c7e69b09 evidence.yaml +b2ef3f210126d960ebe3a23ccd762bae91c31032dd42547f17e5cfd26543426d fixtures/cases.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +0e368251522d81297eb5dadeef33ea4614f31f83251859d96e8583c5386299b7 schemas/civil-record-search.adapter-parameters.schema.yaml +e698f71e7d15fa5ac03b6b45606add0349de4c355753030348458e1ff765f2af schemas/civil-record-search.facts.schema.yaml +bf916984ab4c5054ea4627d2b6311790b216a8951068b834a968bb7d0769a09b schemas/civil-record-search.response.schema.yaml +a9fa00e567f9dfc4bf9247fb0c2338d9637c1643f61baee006bbcb8b4e614985 schemas/death-register.adapter-parameters.schema.yaml +0c3c4b24d88311afc66c1e9e97046d32d54ad8519354f9d22acd9cf3575360cc schemas/death-register.facts.schema.yaml +51be0d07344925e2f436e8d136807c7deb02c896de63aab5adee3e83c5124f4d schemas/death-register.response.schema.yaml +10ac6858cd9761e512c0ee88a01a3cdd3175a744276f97e2d705580c55ee4ee2 schemas/union-register.adapter-parameters.schema.yaml +f7f76d766a18d52b4ca5dec6c28ce31f7dbb7ed46283102c957667cb8050273d schemas/union-register.facts.schema.yaml +b30711c2c838e4835bbd18226d0af923036459529b5e7b397fb011da602239c1 schemas/union-register.response.schema.yaml diff --git a/products/evidence/fixtures/acceptance/surviving-spouse-status/evidence.yaml b/products/evidence/fixtures/acceptance/surviving-spouse-status/evidence.yaml index 92d2c2f926..16d6080701 100644 --- a/products/evidence/fixtures/acceptance/surviving-spouse-status/evidence.yaml +++ b/products/evidence/fixtures/acceptance/surviving-spouse-status/evidence.yaml @@ -3,7 +3,8 @@ assuranceProfile: evidence-grade service: {providerId: urn:example:fixture:provider:evidence, publicOrigin: https://evidence.invalid, trustDomain: urn:example:fixture:trust-domain:acceptance} issuer: {id: urn:example:fixture:issuer:authority} publication: {serviceId: urn:example:fixture:service:evidence, title: Synthetic Evidence service, description: Synthetic minimum-disclosure assertions for acceptance testing, endpointUrl: https://evidence.example.invalid, jurisdictions: [urn:example:jurisdiction:acceptance]} -authentication: {kind: oidc-access-token, issuer: https://identity.invalid, audiences: [evidence-fixture], tokenTypes: [at+jwt], algorithms: [ES256], jwksUri: https://identity.invalid/.well-known/jwks.json, principalClaim: sub, requesterTagsClaim: evidence_tags, evidenceAudienceClaim: evidence_audience, maximumTokenLifetimeSeconds: 300, revokedKeyIds: []} +authentication: + oidc: {issuer: https://identity.invalid, audience: evidence-fixture, tokenTypes: [at+jwt], algorithms: [ES256], jwksSource: {kind: uri, uri: https://identity.invalid/.well-known/jwks.json}, principalClaim: sub, requesterTagsClaim: evidence_tags, evidenceAudienceClaim: evidence_audience, maximumTokenLifetimeSeconds: 300, revokedKeyIds: []} audit: {hashKeyRef: secret:file/audit-hash-key, hashKeyVersion: 1} subjectBinding: {secretRef: secret:file/subject-binding-key, keyVersion: 1} rateLimits: {requestsPerPrincipalPerMinute: 60, burstPerPrincipal: 10, failedSelectorAttemptsPerPrincipalAuthorityPerMinute: 10} diff --git a/products/evidence/fixtures/conformance/selectors/SHA256SUMS b/products/evidence/fixtures/conformance/selectors/SHA256SUMS new file mode 100644 index 0000000000..f998b9b81d --- /dev/null +++ b/products/evidence/fixtures/conformance/selectors/SHA256SUMS @@ -0,0 +1,22 @@ +d55df6ece3882b0c46051daa0db7cee436809f3b7397d3d716ca3c422ac78a39 adapters/classification-source-prepare.rhai +f9032b9420bd2e72fc0530a529bf963657e6e9da385ac00dda44bbe588c7ced3 adapters/classification-source.rhai +cefe877cb53e81adeba5b42f4ae2af2b4d4a5a9a1fa17a58e4983e6301d72e4a adapters/context-source-prepare.rhai +f9032b9420bd2e72fc0530a529bf963657e6e9da385ac00dda44bbe588c7ced3 adapters/context-source.rhai +527fd3416a53bf72f64adc400e5dda5f578e07af986e0c78b0ba1498d36f9a48 adapters/grant-source-prepare.rhai +f9032b9420bd2e72fc0530a529bf963657e6e9da385ac00dda44bbe588c7ced3 adapters/grant-source.rhai +c40d053b7d800dfd03d1eef2bc81c77a92812993cdf6687615cdfda43be94adc adapters/opaque-source-prepare.rhai +f9032b9420bd2e72fc0530a529bf963657e6e9da385ac00dda44bbe588c7ced3 adapters/opaque-source.rhai +01516e8e08497355c4dbc7980a7922ad1ee1b50c5424e730df09e0c5a3026b3f adapters/relationship-source-prepare.rhai +f9032b9420bd2e72fc0530a529bf963657e6e9da385ac00dda44bbe588c7ced3 adapters/relationship-source.rhai +7b6d5e26affcfe364a9a3212e8896ab1c1a152b05b88cfebf27c328d5a5617b5 codelists/opaque-codes.yaml +94e8a1f6377fcbc17c0492098350a732028e3dc426fe5648e2d6f6e15cf8ac7d derivations/classification.rhai +fbd21304bd2dd16e30db2b6facf39d232919599610b8c562cfe3ecf28ffe6729 derivations/opaque.rhai +d8b091b8b72a8247f843ad00ba195e8bc93d18d5c6752f12148dc4c5fb0eb9a9 derivations/property-with-event.rhai +861ce7ed9526929f5b6bbaf1fb00579da714d34b546fb2b878ea1857515c9d36 derivations/property.rhai +40076d967deef9aa9c693cb31900f57985c81f979545f5967648c902d411fcbc derivations/relationship.rhai +7e72e4fdc8548072b8b197bd607788098850b2ed223ca44183a22783857460d5 evidence.yaml +564f66e7c0c4024dbcd5701ee16fc8f4a4cc8c1a9a5fa53aa8428c292cab5d66 fixtures/cases.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +5f29176540b8f3f84ffe50d3bdf6041bb778741516368f10ba700b71fcb19ce5 schemas/adapter-parameters.schema.yaml +31ef747c04cc877b4e9b1c722ffb82aa3f21b4832d0e85a895c430ef768b3856 schemas/facts.schema.yaml +7e20a76c2193175cf95a9afba7019e3b1a131f8d99123fc7ee63f9e8780dd66a schemas/response.schema.yaml diff --git a/products/evidence/fixtures/conformance/selectors/evidence.yaml b/products/evidence/fixtures/conformance/selectors/evidence.yaml index 89b2025cae..b3871d7ba6 100644 --- a/products/evidence/fixtures/conformance/selectors/evidence.yaml +++ b/products/evidence/fixtures/conformance/selectors/evidence.yaml @@ -6,18 +6,20 @@ service: trustDomain: urn:example:fixture:trust-domain:selector-conformance issuer: {id: urn:example:fixture:issuer:selector-conformance} authentication: - kind: oidc-access-token - issuer: https://identity.invalid - audiences: [selector-conformance] - allowedClients: [evidence-task-agent] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: https://identity.invalid/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: https://identity.invalid + audience: selector-conformance + allowedClients: [evidence-task-agent] + tokenTypes: [at+jwt] + algorithms: [ES256] + jwksSource: + kind: uri + uri: https://identity.invalid/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: hashKeyRef: secret:file/audit-key hashKeyVersion: 1 diff --git a/products/evidence/fixtures/conformance/supported-values/SHA256SUMS b/products/evidence/fixtures/conformance/supported-values/SHA256SUMS new file mode 100644 index 0000000000..13df769216 --- /dev/null +++ b/products/evidence/fixtures/conformance/supported-values/SHA256SUMS @@ -0,0 +1,14 @@ +7f71b3b367b21f9a6384e1bcb6fe370695540c25027460dab194aedfca034146 adapters/source-prepare.rhai +50ff6e11dac634ae108c8bff3664ccc34a80663f9eb9a271ee08c39d0ca137b9 adapters/source.rhai +0394846c3c420deefa5c846807485be9a581054e0ef7a9ea7fbb8e34376f9123 codelists/categories.yaml +c47c2226db730d1ce5092a098c666247af879579489e57077b190528a89ea76d codelists/date-buckets.yaml +c1e5cc8c9e3cf28a74818923853c022d2d69d40c065435972e72c55ef2535f03 codelists/synthetic-codes.yaml +6bd65d33c3ef5fcb90bcb8c82b2f438920ad9648399ad731a421fea2b3f1e6b2 codelists/time-buckets.yaml +a08546e52460b7c1c186a55ce5e9ad610122e4532635a3b9ca6698a06ad0fdac derivations/values.rhai +170de926b2583e0698c14383074b2f2f0f0732080b94fceae58c45087c09a84d evidence.yaml +2c2655976b3c7d82da1c1ce218ea39712a703b455175bdfee9b6b24a3ad64aa3 fixtures/cases.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +e0a2f9111055f749652f83ed29f35d12ef0d37e55199b75d09b343632f582c4a schemas/adapter-parameters.schema.yaml +c3babf375038db441d9b99c7a00956bd237a55d9b1e49115101500c76bdebd56 schemas/closed-structure.schema.yaml +f9d7b3edf6d305c49d4b0dd3b10d90084b9a419efef4bafb5c6b012e12086a5f schemas/facts.schema.yaml +a06ed23ba50020c65db92c350dd668cc16a6f9c63e39fcc0b51932f326883efa schemas/response.schema.yaml diff --git a/products/evidence/fixtures/conformance/supported-values/evidence.yaml b/products/evidence/fixtures/conformance/supported-values/evidence.yaml index 30ab0cd0ff..f98e82db26 100644 --- a/products/evidence/fixtures/conformance/supported-values/evidence.yaml +++ b/products/evidence/fixtures/conformance/supported-values/evidence.yaml @@ -6,17 +6,19 @@ service: trustDomain: urn:example:fixture:trust-domain:supported-values issuer: {id: urn:example:fixture:issuer:authority} authentication: - kind: oidc-access-token - issuer: https://identity.invalid - audiences: [evidence-fixture] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: https://identity.invalid/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: https://identity.invalid + audience: evidence-fixture + tokenTypes: [at+jwt] + algorithms: [ES256] + jwksSource: + kind: uri + uri: https://identity.invalid/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: hashKeyRef: secret:file/audit-hash-key hashKeyVersion: 1 diff --git a/products/evidence/reference/authoring-projects/CONFIG.md b/products/evidence/reference/authoring-projects/CONFIG.md index 410db7624b..b34ccaee47 100644 --- a/products/evidence/reference/authoring-projects/CONFIG.md +++ b/products/evidence/reference/authoring-projects/CONFIG.md @@ -500,6 +500,29 @@ declares exactly one `answer(facts, selectors, context)` with those three parameters. Function discovery reads the parsed syntax tree, so a name inside a string or a comment is not an entry point. +The same syntax tree holds the facts `answer` reads. A read of its first +parameter with a literal key, `facts["status"]` or `facts.status` (the first key +of a longer chain), must name a fact the question's source declares: a +`source.facts[].name` for an inline operation, or a property of a referenced +source's `factSchema` when that schema is a closed object +(`additionalProperties: false`). A read of any other name is refused as +`evidence.authoring.derivation-fact-undeclared` against the derivation file, by +`check`, fixture runs, `build`, `package`, and the structural check `source +diff` and `source update` run. That is what turns a source rename, such as a +regenerated export whose fact changed name, into an authoring refusal rather +than a failure of every request. The operands of one `??` fallback are read +together: when any of them names a declared fact, none of them is refused, so +`facts.new ?? facts.old` stays valid while a source moves from one name to the +other. The check reads only what it can read without running the program: a +computed key, a read inside another function the facts are passed to, an +`answer` that rebinds or writes its first parameter (through `let`, `const`, an +assignment to it or to one of its keys, a `for` loop variable, or a `catch` +variable), and a referenced source whose fact schema is open are left to the +fixtures. A computed key such as `facts[key]` is therefore how a derivation +reads a fact this check should not see. The check reads the project, not the +running source: a candidate already deployed against a source whose fact is +then renamed still fails every request that reads it. + ```text fn answer(facts, selectors, context) { let born = parse_date(required(facts.date_of_birth, "date_of_birth_missing")); diff --git a/products/evidence/reference/authoring-projects/SOURCE-EXPORT.md b/products/evidence/reference/authoring-projects/SOURCE-EXPORT.md index c0c6efad93..a5a3100a5e 100644 --- a/products/evidence/reference/authoring-projects/SOURCE-EXPORT.md +++ b/products/evidence/reference/authoring-projects/SOURCE-EXPORT.md @@ -98,7 +98,11 @@ normal compiler/build path to compare actual question revisions under that complete target. Without it, the report contains structural impact only. Structural validation checks the complete source artifact graph and any existing questions without requiring a target, local credentials, or a first -question. The report identifies its validation kind explicitly. Target +question. It includes each existing derivation's fact reads, so a next fact +schema that no longer declares a fact a derivation reads fails structural +validation with `evidence.authoring.derivation-fact-undeclared` +([CONFIG.md](CONFIG.md#the-derivation-program)). The report identifies its +validation kind explicitly. Target validation runs the ordinary compiled-bundle and fixture checks. If the current project cannot yet compile with that target, as before an initial source import, `previousValidation.status` is `unavailable`; the report carries only diff --git a/products/evidence/reference/deployment-targets/environments/local/evidence/governance.yaml b/products/evidence/reference/deployment-targets/environments/local/evidence/governance.yaml index 6d96ee26e5..fa7a7cd919 100644 --- a/products/evidence/reference/deployment-targets/environments/local/evidence/governance.yaml +++ b/products/evidence/reference/deployment-targets/environments/local/evidence/governance.yaml @@ -4,17 +4,19 @@ service: {providerId: urn:example:local:evidence, publicOrigin: http://127.0.0.1 issuer: {id: urn:example:issuer:authority} publication: {serviceId: urn:example:local:evidence-service, title: Local Evidence service, description: Local minimum-disclosure Evidence development service, endpointUrl: http://127.0.0.1:8080, jurisdictions: [urn:example:jurisdiction:local]} authentication: - kind: oidc-access-token - issuer: http://127.0.0.1:8081 - audiences: [evidence.local] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: http://127.0.0.1:8081/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: http://127.0.0.1:8081 + audience: evidence.local + tokenTypes: [at+jwt] + algorithms: [ES256] + jwksSource: + kind: uri + uri: http://127.0.0.1:8081/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: {hashKeyRef: secret:file/evidence-audit-hmac, hashKeyVersion: 1} subjectBinding: {secretRef: secret:file/evidence-subject-binding, keyVersion: 1} rateLimits: {requestsPerPrincipalPerMinute: 60, burstPerPrincipal: 10, failedSelectorAttemptsPerPrincipalAuthorityPerMinute: 10} diff --git a/products/evidence/reference/deployment-targets/environments/local/evidence/runtime.yaml b/products/evidence/reference/deployment-targets/environments/local/evidence/runtime.yaml index f6d5fa0b50..6b63d88651 100644 --- a/products/evidence/reference/deployment-targets/environments/local/evidence/runtime.yaml +++ b/products/evidence/reference/deployment-targets/environments/local/evidence/runtime.yaml @@ -1,8 +1,9 @@ -version: 1 -bundleDirectory: /tmp/registry-evidence-local/bundle +apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /tmp/registry-evidence-local/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 diff --git a/products/evidence/reference/deployment-targets/environments/production/evidence/governance.yaml b/products/evidence/reference/deployment-targets/environments/production/evidence/governance.yaml index 536806f234..01fd18f25d 100644 --- a/products/evidence/reference/deployment-targets/environments/production/evidence/governance.yaml +++ b/products/evidence/reference/deployment-targets/environments/production/evidence/governance.yaml @@ -4,17 +4,19 @@ service: {providerId: https://evidence.example.org, publicOrigin: https://eviden issuer: {id: https://authority.example.org} publication: {serviceId: urn:example:production:evidence-service, title: Production Evidence service, description: Production minimum-disclosure Evidence service, endpointUrl: https://evidence.example.org, jurisdictions: [urn:example:jurisdiction:production]} authentication: - kind: oidc-access-token - issuer: https://issuer.example.org - audiences: [evidence.example.org] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: https://issuer.example.org/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: https://issuer.example.org + audience: evidence.example.org + tokenTypes: [at+jwt] + algorithms: [ES256] + jwksSource: + kind: uri + uri: https://issuer.example.org/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: {hashKeyRef: secret:file/evidence-audit-hmac, hashKeyVersion: 1} subjectBinding: {secretRef: secret:file/evidence-subject-binding, keyVersion: 1} rateLimits: {requestsPerPrincipalPerMinute: 60, burstPerPrincipal: 10, failedSelectorAttemptsPerPrincipalAuthorityPerMinute: 10} diff --git a/products/evidence/reference/deployment-targets/environments/production/evidence/runtime.yaml b/products/evidence/reference/deployment-targets/environments/production/evidence/runtime.yaml index 466cc93568..e7f69cce1d 100644 --- a/products/evidence/reference/deployment-targets/environments/production/evidence/runtime.yaml +++ b/products/evidence/reference/deployment-targets/environments/production/evidence/runtime.yaml @@ -1,8 +1,9 @@ -version: 1 -bundleDirectory: /srv/registry-evidence/production/bundle +apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /srv/registry-evidence/production/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 diff --git a/products/evidence/reference/deployment-targets/environments/staging/evidence/governance.yaml b/products/evidence/reference/deployment-targets/environments/staging/evidence/governance.yaml index 818c9d9474..db08631ae5 100644 --- a/products/evidence/reference/deployment-targets/environments/staging/evidence/governance.yaml +++ b/products/evidence/reference/deployment-targets/environments/staging/evidence/governance.yaml @@ -4,17 +4,19 @@ service: {providerId: https://evidence.staging.example.org, publicOrigin: https: issuer: {id: https://authority.example.org} publication: {serviceId: urn:example:staging:evidence-service, title: Staging Evidence service, description: Staging minimum-disclosure Evidence service, endpointUrl: https://evidence.staging.example.org, jurisdictions: [urn:example:jurisdiction:staging]} authentication: - kind: oidc-access-token - issuer: https://issuer.staging.example.org - audiences: [evidence.staging.example.org] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: https://issuer.staging.example.org/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: https://issuer.staging.example.org + audience: evidence.staging.example.org + tokenTypes: [at+jwt] + algorithms: [ES256] + jwksSource: + kind: uri + uri: https://issuer.staging.example.org/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: {hashKeyRef: secret:file/evidence-audit-hmac, hashKeyVersion: 1} subjectBinding: {secretRef: secret:file/evidence-subject-binding, keyVersion: 1} rateLimits: {requestsPerPrincipalPerMinute: 60, burstPerPrincipal: 10, failedSelectorAttemptsPerPrincipalAuthorityPerMinute: 10} diff --git a/products/evidence/reference/deployment-targets/environments/staging/evidence/runtime.yaml b/products/evidence/reference/deployment-targets/environments/staging/evidence/runtime.yaml index 61e19fc55f..023d274f99 100644 --- a/products/evidence/reference/deployment-targets/environments/staging/evidence/runtime.yaml +++ b/products/evidence/reference/deployment-targets/environments/staging/evidence/runtime.yaml @@ -1,8 +1,9 @@ -version: 1 -bundleDirectory: /srv/registry-evidence/staging/bundle +apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /srv/registry-evidence/staging/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 diff --git a/products/evidence/reference/request-adapter/README.md b/products/evidence/reference/request-adapter/README.md index 00b7411327..428da04c50 100644 --- a/products/evidence/reference/request-adapter/README.md +++ b/products/evidence/reference/request-adapter/README.md @@ -45,9 +45,9 @@ provider can satisfy the Version 1 request and cardinality boundary. 8. Write sanitized [fixtures](deployment-projects/FIXTURES.md) covering positive, false-as-success, boundary, no-match, ambiguity, missing data, protocol failure, privacy canaries, and exact request transport. -9. Run `evidence check --runtime ` to validate the +9. Run `evidence check --runtime-config ` to validate the complete immutable bundle and runtime bindings. -10. Run `evidence evaluate --runtime --fixture +10. Run `evidence evaluate --runtime-config --fixture ` for every referenced fixture before deployment. Add `--explain` to a run that failed to see the stages each case reached and how each one ended, described in diff --git a/products/evidence/reference/request-adapter/deployment-projects/CONFIG.md b/products/evidence/reference/request-adapter/deployment-projects/CONFIG.md index ebc4cb1365..5f03b0f706 100644 --- a/products/evidence/reference/request-adapter/deployment-projects/CONFIG.md +++ b/products/evidence/reference/request-adapter/deployment-projects/CONFIG.md @@ -11,9 +11,9 @@ Evidence starts from two closed, startup-only inputs: TLS trust files. Both inputs are reviewed, validated completely before readiness, mounted -read-only, and immutable for the process lifetime. Evidence computes stable -bundle and runtime revisions at startup; audit events carry the governed bundle -revision. Runtime configuration is not an override layer. It cannot +read-only, and immutable for the process lifetime. Evidence verifies the +package and computes its stable digest at startup; audit events carry that +digest in the frozen `bundleRevision` field. Runtime configuration is not an override layer. It cannot change a source origin, request, credential kind, requirement, authority, selector, disclosure rule, rate limit, signing policy, or audit fail-closed policy. @@ -165,32 +165,54 @@ artifact, or alternate evaluator is introduced by the assurance profile. | `service.publicOrigin` | yes | Exact canonical public Evidence origin used by RFC 9728 metadata. HTTPS is required outside the numeric-loopback local profile. | | `service.trustDomain` | yes | One operator-controlled trust-domain URI for the process. | | `issuer.id` | yes | Legal issuer URI placed in evidence. Governance must authorize the provider to act for it. | -| `authentication.kind` | yes | Exactly `oidc-access-token`. | -| `authentication.issuer`, `authentication.jwksUri` | yes | Exact HTTPS token issuer and JWKS endpoint. Path-based issuers are supported. The fixed JWKS endpoint may resolve to a public or private HTTPS address; DNS is pinned for each fetch, ambient proxies are disabled, and cloud-metadata destinations remain prohibited. | -| `authentication.audiences` | yes | Non-empty exact audience allowlist. | -| `authentication.tokenTypes` | yes | Non-empty allowlist containing only `at+jwt` and/or `application/at+jwt`. | -| `authentication.algorithms` | yes | Non-empty allowlist containing only `EdDSA`, `ES256`, and/or `RS256`. No algorithm fallback is permitted. | -| `authentication.principalClaim` | yes | The only claim used for the principal. Its absence denies; `client_id`, `azp`, request data, and proxy headers are not fallbacks. | -| `authentication.requesterTagsClaim` | yes | Claim containing the requester tags matched against an authority profile. | -| `authentication.evidenceAudienceClaim` | yes | Claim containing the exact evidence audience. The public request cannot choose another audience. | -| `authentication.claims` | no | Direct claim-name mapping for shared actor, purpose, grant id, authority, source issuer, client, resource, expiration, bounds, and approver claims. Omission uses the documented `registry_*` names. The names must be distinct from each other, Evidence product claims, and registered authentication claims. `registry_assertion_issuer` is deliberately not among them, and is reserved against being used as one of these names: the issuer derives it from the verified subject token rather than any deployment minting it, so there is no foreign vocabulary to adapt to, and a name changed at one end only would leave `authentication.assertionIssuers` reading a claim the token does not carry, which applies no rule rather than refusing. | -| `authentication.maximumTokenLifetimeSeconds` | yes | Positive maximum accepted `exp - iat`, up to 86,400 seconds. Its presence requires `iat`, `exp > iat`, and an interval within the maximum. | -| `authentication.revokedKeyIds` | yes | Explicit emergency denylist, including an empty list. It is checked before cached JWKS key selection. | -| `authentication.allowedClients` | no | Explicit machine-client admission, matched against the verified token's `client_id`/`azp` and never `sub`. Omission keeps the issuer-vouched-client behavior. A stated list must be non-empty, unique, and bounded (at most 32 entries of 1..=128 bytes). Audience plus static issuer-governed attributes alone cannot establish that a client was granted this resource's permission, which is what `requiredScopes` closes. | -| `authentication.assertionIssuers` | no | Per-client assertion-authority admission for a token carrying the platform verifier's `registry_assertion_issuer` claim, keyed by the client the token's `client_id`/`azp` names and naming the issuers that client may present the claim as. Omission applies no rule, so a claim-bearing token is admitted regardless of its value. A stated map must be non-empty and bounded (at most 32 client keys of 1..=128 bytes), and each client's issuer list must be non-empty, unique, and bounded (at most 8 entries of 1..=512 bytes). A token carrying no such claim is never affected by this admission. | -| `authentication.requiredScopes` | no | Scopes every inbound token must carry, read from the verified token's scope set after signature verification and before any authority claim is read. Omission keeps the no-scope-gate behavior. A stated list must be non-empty, unique RFC 6749 scope-tokens (at most 32 entries of 1..=256 bytes). A missing scope is never inferred from tags, principal, roles, `sub`, or request fields. | -| `authentication.actorClaim` | no | Optional verified actor claim. Omission does not enable a fallback actor source. | +| `authentication.oidc` | yes | The one OpenID Connect issuer whose access tokens this deployment accepts, and the rules a token from it must satisfy. Evidence accepts no other inbound credential. | +| `authentication.oidc.issuer`, `authentication.oidc.jwksSource` | yes | Exact HTTPS token issuer and the fixed JWKS endpoint, written `jwksSource: {kind: uri, uri: }`. `uri` is the only key-source kind Evidence accepts; discovery and static key sets are refused. Path-based issuers are supported. The fixed JWKS endpoint may resolve to a public or private HTTPS address; DNS is pinned for each fetch, ambient proxies are disabled, and cloud-metadata destinations remain prohibited. | +| `authentication.oidc.audience` | yes | The one exact audience an inbound token must carry. It is also the resource the deployment advertises in its protected-resource metadata. | +| `authentication.oidc.tokenTypes` | yes | Non-empty allowlist containing only `at+jwt` and/or `application/at+jwt`. | +| `authentication.oidc.algorithms` | yes | Non-empty allowlist containing only `EdDSA`, `ES256`, and/or `RS256`. No algorithm fallback is permitted. | +| `authentication.oidc.principalClaim` | yes | The only claim used for the principal. Its absence denies; `client_id`, `azp`, request data, and proxy headers are not fallbacks. | +| `authentication.oidc.requesterTagsClaim` | yes | Claim containing the requester tags matched against an authority profile. | +| `authentication.oidc.evidenceAudienceClaim` | yes | Claim containing the exact evidence audience. The public request cannot choose another audience. | +| `authentication.oidc.claims` | no | Direct claim-name mapping for shared actor, purpose, grant id, authority, source issuer, client, resource, expiration, bounds, and approver claims. Omission uses the documented `registry_*` names. The names must be distinct from each other, Evidence product claims, and registered authentication claims. `registry_assertion_issuer` is deliberately not among them, and is reserved against being used as one of these names: the issuer derives it from the verified subject token rather than any deployment minting it, so there is no foreign vocabulary to adapt to, and a name changed at one end only would leave `authentication.oidc.assertionIssuers` reading a claim the token does not carry, which applies no rule rather than refusing. | +| `authentication.oidc.maximumTokenLifetimeSeconds` | yes | Positive maximum accepted `exp - iat`, up to 86,400 seconds. Its presence requires `iat`, `exp > iat`, and an interval within the maximum. | +| `authentication.oidc.revokedKeyIds` | yes | Explicit emergency denylist, including an empty list. It is checked before cached JWKS key selection. | +| `authentication.oidc.allowedClients` | no | Explicit machine-client admission, matched against the verified token's `client_id`/`azp` and never `sub`. Omission keeps the issuer-vouched-client behavior. A stated list must be non-empty, unique, and bounded (at most 32 entries of 1..=128 bytes). Audience plus static issuer-governed attributes alone cannot establish that a client was granted this resource's permission, which is what `requiredScopes` closes. | +| `authentication.oidc.assertionIssuers` | no | Per-client assertion-authority admission for a token carrying the platform verifier's `registry_assertion_issuer` claim, keyed by the client the token's `client_id`/`azp` names and naming the issuers that client may present the claim as. Omission applies no rule, so a claim-bearing token is admitted regardless of its value. A stated map must be non-empty and bounded (at most 32 client keys of 1..=128 bytes), and each client's issuer list must be non-empty, unique, and bounded (at most 8 entries of 1..=512 bytes). A token carrying no such claim is never affected by this admission. | +| `authentication.oidc.requiredScopes` | no | Scopes every inbound token must carry, read from the verified token's scope set after signature verification and before any authority claim is read. Omission keeps the no-scope-gate behavior. A stated list must be non-empty, unique RFC 6749 scope-tokens (at most 32 entries of 1..=256 bytes). A missing scope is never inferred from tags, principal, roles, `sub`, or request fields. | +| `authentication.oidc.actorClaim` | no | Optional verified actor claim. Omission does not enable a fallback actor source. | +| `authentication.oidc.tlsTrustProfile` | no | Logical profile name bound by `runtime.yaml` to a private CA file trusted beside the system roots for the `jwksSource.uri` connection alone. Omission uses system roots only. Refused when `jwksSource.uri` is a local HTTP origin. | + +A bundle that still writes the access-token rules directly under +`authentication` (`authentication.kind`, `authentication.issuer`, +`authentication.audiences`, `authentication.jwksUri`, and the rest), or that +writes `authentication.oidc.audiences` or `authentication.oidc.jwksUri`, is +refused at startup. The refusal names the key that replaced it; no old +spelling is read as an alias. ### Audit, subject binding, rates, and signing | Section | Required fields and rule | |---|---| -| `audit` | File-only `hashKeyRef` and positive `hashKeyVersion`, both required. The referenced master contains at least 32 raw secret bytes; Rust derives the audit pseudonym key from it, and `hashKeyVersion` is stamped into every audit pseudonym so pseudonyms under different key material stay distinguishable. Every audit gate is fail closed whatever this section says. The runtime file owns the audit destination. | -| `subjectBinding` | File-only `secretRef` and positive `keyVersion`. The referenced master contains at least 32 raw secret bytes, uses a distinct reference, and must resolve to bytes distinct from the audit master. Rust derives purpose-scoped bindings over the complete canonical role/profile/value bundle, never per-field hashes. The remaining scope input is the requirement's binding mode: the authenticated audience for an audience-scoped requirement, the presented holder key thumbprint for a holder-bound one. The two derivations are domain-separated, so one mode's binding can never be read as the other's. | +| `audit` | `hashKeyRef` and positive `hashKeyVersion`, both required. The referenced master contains at least 32 raw secret bytes; Rust derives the audit pseudonym key from it, and `hashKeyVersion` is stamped into every audit pseudonym so pseudonyms under different key material stay distinguishable. Every audit gate is fail closed whatever this section says. The runtime file owns the audit destination. | +| `subjectBinding` | `secretRef` and positive `keyVersion`. The referenced master contains at least 32 raw secret bytes, uses a distinct reference, and must resolve to bytes distinct from the audit master. Rust derives purpose-scoped bindings over the complete canonical role/profile/value bundle, never per-field hashes. The remaining scope input is the requirement's binding mode: the authenticated audience for an audience-scoped requirement, the presented holder key thumbprint for a holder-bound one. The two derivations are domain-separated, so one mode's binding can never be read as the other's. | | `rateLimits` | Positive `requestsPerPrincipalPerMinute`, `burstPerPrincipal`, and `failedSelectorAttemptsPerPrincipalAuthorityPerMinute`. Raw selector values never become rate-limit labels. | | `signing` | Exact keys are `format: flattened-jws-json`, `algorithm: ES256`, `activePublicJwkFile`, `publishedPublicJwkFiles`, `revokedKeyIds`, fixed `jwksPath`, `maximumAssertionValiditySeconds`, and `verifierClockSkewSeconds`. Every exact public EC P-256 JWK has a 43-character RFC 7638 thumbprint `kid`; active, published, and revoked sets are disjoint. Missing signing material fails readiness; there is no unsigned fallback. | | `responseFormats` | Closed unique list of 1 through 3 entries drawn from `signed-jws`, `unsigned-json`, and `sd-jwt-vc`. `signed-jws` must always be present; a bundle that omits it is rejected at startup. Every other format additionally requires the matched grant to permit it, and signing material must still be ready even for an unsigned response. | +`audit.hashKeyRef` replaced `audit.hashSecretRef`, which is refused at startup +with its replacement named. + +Every secret reference in the bundle uses one of two forms. `secret:file/` +resolves to a file beneath the runtime's `secretProviders.file.root`. +`secret:env/` resolves to a process environment variable, and only when +the runtime enables `secretProviders.environment`; otherwise startup refuses the +bundle. Either way the bundle carries a logical name, never the secret value. + +The bundle is parsed as written: a `${...}` environment expression in any bundle +key or string value is refused at startup, naming the field and never the +expression. Substitution applies to `runtime.yaml` only, so the reviewed bundle +is the one that runs. + ### Selector profiles Each `selectorProfiles.` declares `maximumAggregateBytes` and one exact @@ -945,8 +967,8 @@ sound only because the read-only check passed. A file the process or anything else can still write makes the promise false, and an immutable connection over a file that changes is undefined behaviour rather than a stale read, because SQLite is entitled to trust pages it has already cached. Publish a new extract -as a new file, mount it read-only, and restart. Startup digests each bound file -into the runtime digest, so a replacement shows up in the digest instead of +as a new file, mount it read-only, and restart. Startup captures each bound +file's identity and content digest, so a replacement is refused instead of passing silently. Digesting the file and opening it are not the same moment: the bundle, the @@ -973,7 +995,7 @@ against a deployment project like this one, it instead runs `evidence check` with the runtime file's real bindings, which is the check described above. The internal `evidence bundle-check --bundle --json` tooling seam -returns `bundleRevision` and a `requirements` array of `id` and +returns `packageDigest` and a `requirements` array of `id` and `configurationRevision` after validation. Those revisions come from the same bundle closure used by signed answers. The document contains identifiers and digests only; the default command output remains a human-readable check result. @@ -1153,7 +1175,7 @@ The supported publication handoff is deliberately explicit because and mounted extract. The check refuses an extract that is already older than its source permits. Then run every referenced fixture, real startup, and `/ready` before routing traffic. - Retain the runtime revision and governed extract profile in the deployment + Retain the runtime file and governed extract profile in the deployment record. Do not copy publisher-controlled metadata values into logs, audit, or error tickets. @@ -1207,11 +1229,12 @@ files. Private signing material is never a bundle artifact. `runtime.yaml` contains only process-local bindings: ```yaml -version: 1 -bundleDirectory: /etc/registry-evidence/bundle +apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /etc/registry-evidence/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 @@ -1245,19 +1268,22 @@ sourceExtracts: | Key | Required | Meaning and Version 1 bounds | |---|---|---| -| `version` | yes | Literal integer `1`. | -| `bundleDirectory` | yes | Absolute path to the single governed bundle directory. No alternate, overlay, or fallback bundle exists. | -| `listener.bindHost` | yes | Numeric listener address, 2 through 64 bytes. Under the default `private-address` exposure, only loopback, RFC 1918 private IPv4, or RFC 4193 unique-local IPv6 is accepted. Under explicit `container-private`, unspecified IPv4 or IPv6 wildcard is also accepted. Hostnames, concrete public addresses, and multicast are always rejected. | +| `apiVersion` | yes | Literal `registry.registrystack.org/evidence-runtime/v1alpha1`. Names the grammar the document is written in. | +| `kind` | yes | Literal `EvidenceRuntimeConfig`. Together with `apiVersion` it tells an Evidence runtime file apart from another product's runtime configuration. | +| `package.root` | yes | Absolute path to the single governed package directory. No alternate, overlay, or fallback package exists. | +| `package.expectedDigest` | no | The `sha256:` package digest the operator approved. When set, startup is refused unless the package at `package.root` computes exactly this digest. Omission still verifies and loads the package found there. | +| `listener.bind` | yes | Numeric socket address written `host:port`, with an IPv6 host in brackets (`[addr]:port`). Port `0` is refused. Under the default `private-address` exposure, the host must be loopback, RFC 1918 private IPv4, or RFC 4193 unique-local IPv6. Under explicit `container-private`, the unspecified IPv4 or IPv6 wildcard is also accepted. Hostnames, concrete public addresses, and multicast are always rejected. | | `listener.networkExposure` | no | `private-address` by default. `container-private` permits a wildcard bind only when the operator confines the container network and terminates TLS upstream. It does not enable direct public exposure. | -| `listener.port` | yes | TCP port 1 through 65535. | | `listener.tlsTermination` | yes | Literal `operator-controlled-upstream`. | | `listener.trustProxyIdentityHeaders` | yes | Literal `false`; proxy headers never supply authenticated identity or authority. | | `listener.maximumRequestBytes` | yes | 1,024 through 1,048,576 bytes. | | `listener.maximumConcurrentRequests` | yes | 1 through 4,096. | | `listener.requestTimeoutMilliseconds` | yes | 1 through 30,000 milliseconds for admission, concurrency queueing, and request-body collection. Once protected evaluation starts, this timer does not cancel it; source and OIDC boundaries have their own bounds, and the runtime preserves fail-closed audit and release ordering. | | `listener.shutdownGraceMilliseconds` | yes | 1 through 120,000 milliseconds. | -| `metricsListener` | no | Optional operator-only telemetry listener serving `GET /metrics`, a binding separate from the evidence listener above and absent from the public evidence contract. Absence is the default and serves no metrics endpoint. `bindHost` accepts only loopback, RFC 1918 private IPv4, or RFC 4193 unique-local IPv6, and `bindHost`/`port` together must not repeat the evidence listener's exact binding. | -| `secretProviders.file.root` | yes | Absolute root for logical `secret:file/...` references. Only regular, non-symlink, single-link files owned by the service identity with exact mode `0400` or `0600` are accepted. | +| `metricsListener` | no | Optional operator-only telemetry listener serving `GET /metrics`, a binding separate from the evidence listener above and absent from the public evidence contract. Absence is the default and serves no metrics endpoint. `metricsListener.bind` is a `host:port` socket address whose host must be loopback, RFC 1918 private IPv4, or RFC 4193 unique-local IPv6; port `0` is refused, and it must not repeat the evidence listener's binding. | +| `secretProviders` | yes | The providers a secret reference may resolve through, at least one. A bundle reference naming a provider this section does not enable is refused at startup. | +| `secretProviders.file.root` | when the file provider is used | Absolute root for logical `secret:file/...` references. Only regular, non-symlink, single-link files owned by the service identity with exact mode `0400` or `0600` are accepted. | +| `secretProviders.environment` | no | Written `environment: {}`, with no settings. Enables `secret:env/NAME` references, read from the process environment. Absence means no reference may name the environment. Use it where the platform injects secrets as environment variables; the file provider keeps its ownership and mode checks, which the environment cannot offer. Once enabled, any reference in the reviewed bundle, a source credential included, may name any variable in the process environment, so give the process a dedicated environment holding only the secrets this deployment needs, and review each `secret:env/NAME` reference as part of the bundle. A bundle cannot enable a provider itself. | | `signer` | yes | Closed runtime signer union. `production` and `evidence-grade` require a pinned Transit signer over a workload-local Unix socket. `local` requires `kind: local-jwk` with `privateKeyRef: secret:file/evidence-signing`. Startup validates provider controls and exact public-key agreement, then signs and verifies a challenge. | | `audit` | yes | Where this process writes its audit entries through the shared platform audit writer, one JSON line per entry with the members `schema`, `eventId`, `time`, `phase`, `correlation`, and `record`. Entries are not chained; ship them to append-only storage for tamper evidence. Every audit gate is fail closed on either destination. | | `audit.destination` | no | `file` by default, or `stdout`. `file` appends each entry durably (the append returns only after `fsync`) under a single-writer lock at `.lock`, so a second process on the same path fails startup. `stdout` writes each entry as one line on standard output for a collector that owns durability, rotation, and retention; `check --require-audit-under` refuses it, and local audit inspection cannot read it. | @@ -1265,13 +1291,26 @@ sourceExtracts: | `audit.rotateBytes` | no | 1,048,576 through 4,294,967,295 bytes; 104,857,600 (100 MiB) by default. Reaching it seals the active file under an ascending sequence number and opens a fresh one at `path`. A write or sync failure, not rotation itself, is what fails closed. Refused for `stdout`. | | `audit.retainDays` | no | 1 through 36,500 days; 90 by default. When the writer opens or rotates, sealed files last modified longer ago than this are deleted; the active file never is. Refused for `stdout`. | | `outboundTls.systemRoots` | yes | Literal `true`. | -| `outboundTls.trustProfiles` | yes | Closed map of at most 64 logical profile ids. It may be empty when no source names a private trust profile. | +| `outboundTls.trustProfiles` | yes | Closed map of at most 64 logical profile ids. It may be empty when neither a source nor `authentication` names a private trust profile. | | `outboundTls.trustProfiles..caBundleFile` | for each profile | Absolute path to one bounded PEM CA file. Profile names must exactly match bundle `tlsTrustProfile` references. | | `sourceExtracts` | no | Closed map of at most 64 logical extract names. Omission binds none, which is what a runtime file for a bundle with no extract source says. | | `sourceExtracts..path` | for each name | Absolute path to one read-only regular file. Names must exactly match bundle `extractProfile` references. | | `acquisitionCapabilities` | no | Gated acquisition kinds and source optimizations this deployment enables, at most two entries: `search-then-fetch-set` and `source-batch`. Omission and `[]` both enable nothing, so a bundle carrying a source batch block or needing a gated kind is refused before the listener binds. | -`bundleDirectory`, secret roots, audit destinations, and CA files must be +A runtime file written in the earlier grammar is refused before anything else +is read, with the replacement named: `version` gives way to `apiVersion` and +`kind`, `bundleDirectory` to `package.root`, and `listener.bindHost` with +`listener.port` (and the same pair under `metricsListener`) to one `bind` +address. No old spelling is read as an alias. + +A string value may carry `${NAME}` or `${NAME:-default}`, substituted from the +process environment after the document is parsed. Substitution is refused +inside a secret reference and anywhere under `secretProviders`, so an +environment variable can move a path or an address but can never choose which +secret a field names. The runtime captures the substituted document once at +startup, so a changed variable takes effect only after restart. + +`package.root`, secret roots, audit destinations, and CA files must be absolute paths. The runtime rejects symlinks, insecure ownership/modes, missing required logical bindings, mutable files, and files outside the configured roots according to the operator contract. @@ -1312,11 +1351,15 @@ initialization failed`, naming neither the artifact nor the cause. Write `evidence check` before `evidence serve` so a fault surfaces with its artifact and cause instead of only as `runtime bundle initialization failed`. -A bundle source may name one `tlsTrustProfile`. The corresponding bounded PEM -file is loaded and validated at startup. Hostname verification and source-origin -checks remain mandatory. There is no `insecure`, `skipVerification`, or -`trustAll` setting. Changing a trust file requires restart and changes the -runtime digest. +A bundle source may name one `tlsTrustProfile`, and so may +`authentication.oidc` for the connection that fetches the access-token issuer's +key set from `jwksSource.uri`. The corresponding bounded PEM file is loaded and validated at +startup and is trusted beside the system roots, only for the connection of the +source or issuer that names it. A source and the issuer may name the same +profile. Hostname verification and source-origin checks remain mandatory. +There is no `insecure`, `skipVerification`, or `trustAll` setting, and the +issuer needs no process-wide trust store change such as `SSL_CERT_FILE`. +Changing a trust file requires restart before the new bytes can be used. A bundle source that reads an extract names one `extractProfile` and never a filesystem location, so the operator decides where the file sits without @@ -1324,8 +1367,8 @@ editing reviewed material. Each bound file must be a regular, non-symlink, read-only file. Read-only is a correctness requirement rather than hygiene: the statement executor opens the file as immutable, and an immutable connection over a file that can change is undefined behaviour. Startup digests each file -without reading it into memory and folds that digest into the runtime digest, -so replacing an extract requires restart and changes the digest. Startup +without reading it into memory and retains that content digest with the opened +extract, so replacing an extract requires restart. Startup refuses a profile the bundle names and the runtime does not bind, and a profile the runtime binds and no source reads, naming the profile in each case. @@ -1423,14 +1466,14 @@ makes no identity-provider or source-data call; opens no listener; and writes no production audit event. The editable project and `.evidence` local state remain unchanged. -The candidate contains `runtime.yaml` and `bundle/`, including only referenced -adapters, derivations, schemas, codelists, fixtures, and public keys. Given -identical authoring files, target governance, runtime bytes, and toolset -release, bundle bytes and revision are identical. The copied runtime is -environment-specific and has its own revision; it is not part of the bundle -revision or signed `configurationRevision`. +The package contains `SHA256SUMS` at its root with only referenced adapters, +derivations, schemas, codelists, fixtures, and public keys. Given identical +authoring files, target governance, and toolset release, package bytes and +digest are identical. Environment-specific runtime configuration remains in +the target and is not part of the package digest or signed +`configurationRevision`. -The bundle revision identifies the whole reviewed bundle and is what an operator +The package digest identifies the whole reviewed package and is what an operator records at approval. A signed `configurationRevision` is narrower: it covers only the configuration and artifacts one requirement depends on. Editing a requirement, its source, one of its selector profiles, an authority grant naming @@ -1439,35 +1482,35 @@ requirement's revision. The public verification keys and `signing.revokedKeyIds` are in no requirement's closure: which keys are published, which one signs, and which are revoked is trust a relying party takes from the JWKS and its verification policy's denylist, so publishing, -activating, retiring, or revoking a key changes the bundle revision but no +activating, retiring, or revoking a key changes the package digest but no requirement's revision. `authentication.revokedKeyIds` is in no requirement's closure either: it decides which caller tokens are accepted, not what an -assertion means, so revoking an identity-provider key changes the bundle -revision but no requirement's revision. Every other `authentication` member +assertion means, so revoking an identity-provider key changes the package +digest but no requirement's revision. Every other `authentication` member stays in every requirement's closure. An edit outside a requirement's closure leaves its revision unchanged, so it does not force every relying party to re-review. -After approval, transfer the exact candidate, provision independent owner-only -secrets under the configured secret root, make bundle and runtime non-writable +After approval, transfer the exact package to the stable `package.root`, provision independent owner-only +secrets under the configured secret root, make the package and runtime non-writable to the service identity, and run the grouped handoff once whenever candidate bytes, runtime bindings, trust files, or secrets change: ```sh -evidencectl doctor --runtime-config '/runtime.yaml' -evidencectl test '' -evidence --runtime '/runtime.yaml' check --require-runtime-dependencies -evidence --runtime '/runtime.yaml' serve +evidencectl doctor --runtime-config '/runtime.yaml' +evidencectl test '' --target '' +evidence check --runtime-config '/runtime.yaml' --require-runtime-dependencies +evidence serve --runtime-config '/runtime.yaml' ``` Then route only after `/ready`, retain one authorized synthetic-subject response, verify it under independently prepared production policy and trusted keys, and confirm the audit entries reached the operator's append-only store. A provider API or governance change produces -a newly reviewed bundle revision and reruns the fixture matrix. +a newly reviewed package digest and reruns the fixture matrix. Configure an OIDC issuer independently and register each workload with the exact resource, scopes, client identity, and public key required by its approved -journey. Match the runtime's issuer, JWKS URI, audiences, allowed algorithms, +journey. Match the bundle's issuer, JWKS URI, audience, allowed algorithms, accepted token types, and principal, requester-tag, Evidence-audience, grant, and optional actor claim mappings. The maintained local tooling uses stock ThunderID. Deployment inspection does not register callers, decide authority, @@ -1475,10 +1518,10 @@ or provision an issuer; verify the configured issuer-to-resource journey as part of the handoff. Docker Compose is a documented adapter rather than build output. It mounts the -candidate bundle unchanged and read-only; mounts a distinct container runtime, +approved package unchanged and read-only; mounts a distinct container runtime, secrets, and persistent audit storage separately; binds Evidence privately; and -keeps TLS and public routing operator-controlled. The Compose runtime has its -own revision while assertions continue to carry their unchanged per-requirement +keeps TLS and public routing operator-controlled. The Compose runtime remains +outside the package while assertions continue to carry their per-requirement configuration revisions. Retain the configured issuer's public HTTPS identity and JWKS URI when services share a network; internal plain-HTTP service names do not replace them. @@ -1523,39 +1566,41 @@ audit audit.hashKeyRef audit.hashKeyVersion authentication -authentication.actorClaim -authentication.algorithms -authentication.algorithms[] -authentication.allowedClients -authentication.allowedClients[] -authentication.assertionIssuers -authentication.assertionIssuers.* -authentication.assertionIssuers.*[] -authentication.audiences -authentication.audiences[] -authentication.claims -authentication.claims.actorKind -authentication.claims.approver -authentication.claims.grantBounds -authentication.claims.grantClient -authentication.claims.grantExp -authentication.claims.grantId -authentication.claims.grantResource -authentication.claims.grantSourceIssuer -authentication.claims.purpose -authentication.evidenceAudienceClaim -authentication.issuer -authentication.jwksUri -authentication.kind -authentication.maximumTokenLifetimeSeconds -authentication.principalClaim -authentication.requesterTagsClaim -authentication.requiredScopes -authentication.requiredScopes[] -authentication.revokedKeyIds -authentication.revokedKeyIds[] -authentication.tokenTypes -authentication.tokenTypes[] +authentication.oidc +authentication.oidc.actorClaim +authentication.oidc.algorithms +authentication.oidc.algorithms[] +authentication.oidc.allowedClients +authentication.oidc.allowedClients[] +authentication.oidc.assertionIssuers +authentication.oidc.assertionIssuers.* +authentication.oidc.assertionIssuers.*[] +authentication.oidc.audience +authentication.oidc.claims +authentication.oidc.claims.actorKind +authentication.oidc.claims.approver +authentication.oidc.claims.grantBounds +authentication.oidc.claims.grantClient +authentication.oidc.claims.grantExp +authentication.oidc.claims.grantId +authentication.oidc.claims.grantResource +authentication.oidc.claims.grantSourceIssuer +authentication.oidc.claims.purpose +authentication.oidc.evidenceAudienceClaim +authentication.oidc.issuer +authentication.oidc.jwksSource +authentication.oidc.jwksSource.kind +authentication.oidc.jwksSource.uri +authentication.oidc.maximumTokenLifetimeSeconds +authentication.oidc.principalClaim +authentication.oidc.requesterTagsClaim +authentication.oidc.requiredScopes +authentication.oidc.requiredScopes[] +authentication.oidc.revokedKeyIds +authentication.oidc.revokedKeyIds[] +authentication.oidc.tlsTrustProfile +authentication.oidc.tokenTypes +authentication.oidc.tokenTypes[] authorityProfiles authorityProfiles.* authorityProfiles.*.actorKind @@ -1850,31 +1895,34 @@ version ```text acquisitionCapabilities acquisitionCapabilities[] +apiVersion audit audit.destination audit.path audit.retainDays audit.rotateBytes -bundleDirectory +kind listener -listener.bindHost +listener.bind listener.maximumConcurrentRequests listener.maximumRequestBytes listener.networkExposure -listener.port listener.requestTimeoutMilliseconds listener.shutdownGraceMilliseconds listener.tlsTermination listener.trustProxyIdentityHeaders metricsListener -metricsListener.bindHost -metricsListener.port +metricsListener.bind outboundTls outboundTls.systemRoots outboundTls.trustProfiles outboundTls.trustProfiles.* outboundTls.trustProfiles.*.caBundleFile +package +package.expectedDigest +package.root secretProviders +secretProviders.environment secretProviders.file secretProviders.file.root signer @@ -1888,6 +1936,5 @@ signer.unixSocketPath sourceExtracts sourceExtracts.* sourceExtracts.*.path -version ``` diff --git a/products/evidence/reference/request-adapter/deployment-projects/FIXTURES.md b/products/evidence/reference/request-adapter/deployment-projects/FIXTURES.md index 068d6d0488..06b2eaefb7 100644 --- a/products/evidence/reference/request-adapter/deployment-projects/FIXTURES.md +++ b/products/evidence/reference/request-adapter/deployment-projects/FIXTURES.md @@ -23,11 +23,11 @@ reference is absent or its captured suite is incomplete. For production compilation, each editable question names one regular, project-relative `fixtures/.yaml` file. The build copies that file into -the candidate bundle and delegates its execution to `evidence evaluate`; it +the package and delegates its execution to `evidence evaluate`; it does not reinterpret fixture semantics or manufacture cases. Missing, symlinked, outside-project, duplicate, or incomplete fixture references stop the build before publication. The build's private validation secrets and -runtime are not fixture inputs and never appear in the candidate. +runtime are not fixture inputs and never appear in the package. ## File shape @@ -452,12 +452,12 @@ arbitrary string remains only the shape `string`; the trace never substitutes raw category text or a concept identifier for an ordinal. ```sh -evidence --runtime "/runtime.yaml" \ - evaluate --fixture "" --explain --explain-format json \ +evidence evaluate --runtime-config "/runtime.yaml" \ + --fixture "" --explain --explain-format json \ | jq -r '.cases[] | "\(.id)\t\(.failure // "passed")"' ``` -`evidencectl test --explain` asks the same of +`evidencectl test --target --explain` asks the same of every fixture a project references using the JSON form. The human report pretty-prints each value-free document under its step line; `--format json` places the same document at that fixture's `trace` field. The driver totals diff --git a/products/evidence/reference/request-adapter/deployment-projects/README.md b/products/evidence/reference/request-adapter/deployment-projects/README.md index a6a001f01e..3081f23e23 100644 --- a/products/evidence/reference/request-adapter/deployment-projects/README.md +++ b/products/evidence/reference/request-adapter/deployment-projects/README.md @@ -57,7 +57,7 @@ Follow the [authoring and production-build workflow](CONFIG.md#authoring-and-pro when adapting a project. Use these complete bundles as reference material, not as local state to copy or promote. An editable project gains its own reviewed governance metadata and fixtures, then `evidencectl package` produces one closed -candidate. Keep that candidate bundle unchanged across environments and bind +package. Keep that package unchanged across environments and bind each environment through its own runtime file and secret mounts. ## Security boundary diff --git a/products/evidence/reference/request-adapter/deployment-projects/dhis2-tracker-evidence/bundle/SHA256SUMS b/products/evidence/reference/request-adapter/deployment-projects/dhis2-tracker-evidence/bundle/SHA256SUMS new file mode 100644 index 0000000000..e53365968f --- /dev/null +++ b/products/evidence/reference/request-adapter/deployment-projects/dhis2-tracker-evidence/bundle/SHA256SUMS @@ -0,0 +1,16 @@ +d7b46b6f29ee0b9df2c483ac53b3681355108605ff1f2b72c22fcfa41b707db6 adapters/adult-status-extract.rhai +b09a9b02623b7ec40b6a0337846df5df46f79232bafeee263c4d3de2f1bd82ed adapters/prepare.rhai +d7e74ef39eb05528366d61fc46d196a2e063fd69aba3ce88538ba6ef8fb9dcd9 adapters/professional-licence-extract.rhai +ec548ec26efb9f9036d767d729157a9eaa3898a390763655970dc33ffd7e16d1 codelists/licence-expiry-categories.yaml +d3f78c0172e190590d04d6f31b698651ce1d79096bc97a488c775d7054a58b07 derivations/adult-status.rhai +c44b700c4cd49c75fd9e1e3353f6bb95d959801e0035de5d15cfe6d7b177ea64 derivations/professional-licence.rhai +c61fe245cda2ad04384029c168b885a7445d022e08dcfe5ffbf9d8f98820ffac evidence.yaml +b3fbbcca2f6303410ebe3c9ac8a0e1ba69efcc69668b1ba0da4332960480aadc fixtures/adult-status-cases.yaml +f2b4763885dddec8761d227f918a42221b78e8f54b51db04b357da6813f38d30 fixtures/professional-licence-cases.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +e4423b206d3b98330910b98abe3e3a522da1996f9c225c58f719dd4ef179d1ad schemas/adult-status-adapter-parameters.schema.yaml +0d4ac5584f263f954d58882d50e12fc7f6085ac4726cf30f897a1b5e2f24085c schemas/adult-status-facts.schema.yaml +fc504ffc698efa2368db242ef1b7280a689970a7d0b30e81d0bde303c24ddc29 schemas/adult-status-response.schema.yaml +edb56957616696f879150c008cb7b180802e6ce50bf0ac17b07291389df5353c schemas/professional-licence-adapter-parameters.schema.yaml +2a711c050dec92a9574b951b99ed22bbcc0fd72a89c3f993f7d8c2c158c088ed schemas/professional-licence-facts.schema.yaml +93e5ba1cf79329879422beac55fef167b578ec65dfc407d7528d60d9898a8db5 schemas/professional-licence-response.schema.yaml diff --git a/products/evidence/reference/request-adapter/deployment-projects/dhis2-tracker-evidence/bundle/evidence.yaml b/products/evidence/reference/request-adapter/deployment-projects/dhis2-tracker-evidence/bundle/evidence.yaml index 0725d357d0..d115cd19bc 100644 --- a/products/evidence/reference/request-adapter/deployment-projects/dhis2-tracker-evidence/bundle/evidence.yaml +++ b/products/evidence/reference/request-adapter/deployment-projects/dhis2-tracker-evidence/bundle/evidence.yaml @@ -7,17 +7,19 @@ service: issuer: id: urn:gov:example:issuer:population-authority authentication: - kind: oidc-access-token - issuer: https://identity.gov.example - audiences: [registry-evidence] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: https://identity.gov.example/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: https://identity.gov.example + audience: registry-evidence + tokenTypes: [at+jwt] + algorithms: [ES256] + jwksSource: + kind: uri + uri: https://identity.gov.example/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: hashKeyRef: secret:file/audit-hmac-key hashKeyVersion: 1 diff --git a/products/evidence/reference/request-adapter/deployment-projects/dhis2-tracker-evidence/runtime.yaml b/products/evidence/reference/request-adapter/deployment-projects/dhis2-tracker-evidence/runtime.yaml index 7a81cdf8ce..0f863f638c 100644 --- a/products/evidence/reference/request-adapter/deployment-projects/dhis2-tracker-evidence/runtime.yaml +++ b/products/evidence/reference/request-adapter/deployment-projects/dhis2-tracker-evidence/runtime.yaml @@ -1,8 +1,9 @@ -version: 1 -bundleDirectory: /etc/registry-evidence/bundle +apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /etc/registry-evidence/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 diff --git a/products/evidence/reference/request-adapter/deployment-projects/opencrvs-family-evidence/bundle/SHA256SUMS b/products/evidence/reference/request-adapter/deployment-projects/opencrvs-family-evidence/bundle/SHA256SUMS new file mode 100644 index 0000000000..f1b9fa32a1 --- /dev/null +++ b/products/evidence/reference/request-adapter/deployment-projects/opencrvs-family-evidence/bundle/SHA256SUMS @@ -0,0 +1,17 @@ +4611e0a2402c6f98fa8bf51553bcf67668da0a63d6904e15c00c07fad63933e0 adapters/birth-adult-extract.rhai +3f96a64883bb00fe3319c6fd3d8b894cbd4f4c4168e111a70fdcb9dbbbb180a1 adapters/birth-event-prepare.rhai +981c173e9f9cc1b1ee56efa7f00396a59c26a3b0daee75582a78984b5dabae33 adapters/birth-parents-extract.rhai +e1bd75af85bb151b544113ec1e4fa14917b8a8f86d1b7aa09b81aa6addb20499 derivations/adult-status.rhai +3e862b55edc5f8c9fa1e84c79c3e9fcaf5f77c7b1cfcb9241b9e0c7490a9c769 derivations/registered-parent-references.rhai +05a0b1c671de7a4ddc59c4ad898d5445f18a20535705601194a1c179c4467217 derivations/registered-parent-relationship.rhai +4411155d5be64af9a036def6631940e4b3a4848b8b93bac14188cca00e8f507b evidence.yaml +1aaa9aeada3f23341320d2f01f8f48a38b7f3434abe969b2087c888128b55c9c fixtures/adult-status-cases.yaml +f710d439dc99bdd129af9eee101099861bbf9b4546ce9624d00bbac0baa2b7cc fixtures/registered-parent-references-cases.yaml +c3d8c4908733a60dbd7b6ff35c877ed0889acc91098ff738f4a0dd42f2487d7b fixtures/registered-parent-relationship-cases.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +9ea8fa9321ade2e62392f62b78d9011e96e4eecb9a0e5d776dcf07cfbc8f81a0 schemas/birth-adult-facts.schema.yaml +04cc14f27567ec3872444612983567543f0925ed7a6ad1e17fc9efc75bc77275 schemas/birth-adult-parameters.schema.yaml +4b9f40a95b6976f3682d118ae5e5849354639c6a968d34642926a9981c485c6a schemas/birth-adult-response.schema.yaml +e791d2a0f738c26d2c3df25cf8000a94f08876219ca695a64c420a57252bf981 schemas/birth-parents-facts.schema.yaml +7b75e8922ca3e91849f7aaa0b5f554c27d9abf0ab13644acee117088a6a3183e schemas/birth-parents-parameters.schema.yaml +b099850625eab27b49552d9d72b3dc641e970aa9279b0182b7a7d047f459e635 schemas/birth-parents-response.schema.yaml diff --git a/products/evidence/reference/request-adapter/deployment-projects/opencrvs-family-evidence/bundle/evidence.yaml b/products/evidence/reference/request-adapter/deployment-projects/opencrvs-family-evidence/bundle/evidence.yaml index 9a3a4fa3ae..dca61b07ad 100644 --- a/products/evidence/reference/request-adapter/deployment-projects/opencrvs-family-evidence/bundle/evidence.yaml +++ b/products/evidence/reference/request-adapter/deployment-projects/opencrvs-family-evidence/bundle/evidence.yaml @@ -7,18 +7,20 @@ service: issuer: id: urn:gov:example:issuer:civil-registration-authority authentication: - kind: oidc-access-token - issuer: https://identity.gov.example - audiences: [registry-evidence] - allowedClients: [casework-agent] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: https://identity.gov.example/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: https://identity.gov.example + audience: registry-evidence + allowedClients: [casework-agent] + tokenTypes: [at+jwt] + algorithms: [ES256] + jwksSource: + kind: uri + uri: https://identity.gov.example/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: hashKeyRef: secret:file/audit-hmac-key hashKeyVersion: 1 diff --git a/products/evidence/reference/request-adapter/deployment-projects/opencrvs-family-evidence/runtime.yaml b/products/evidence/reference/request-adapter/deployment-projects/opencrvs-family-evidence/runtime.yaml index 46728efe07..da74d8be05 100644 --- a/products/evidence/reference/request-adapter/deployment-projects/opencrvs-family-evidence/runtime.yaml +++ b/products/evidence/reference/request-adapter/deployment-projects/opencrvs-family-evidence/runtime.yaml @@ -1,8 +1,9 @@ -version: 1 -bundleDirectory: /etc/registry-evidence/bundle +apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /etc/registry-evidence/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 diff --git a/products/evidence/reference/request-adapter/deployment-projects/protected-read-evidence/bundle/SHA256SUMS b/products/evidence/reference/request-adapter/deployment-projects/protected-read-evidence/bundle/SHA256SUMS new file mode 100644 index 0000000000..ab50035232 --- /dev/null +++ b/products/evidence/reference/request-adapter/deployment-projects/protected-read-evidence/bundle/SHA256SUMS @@ -0,0 +1,10 @@ +7e9984ff3f3f2fc68999b6b7d4c83f6411861181061bb02d3a8521e8167ce1cd adapters/prepare.rhai +5e74708232c7f64e0111d60b08c05876baa38d3bea7d56c0646863f4b30960d5 adapters/residence-region-extract.rhai +657197b5d26d31af456b5896d0186e179e0949a4b315c3ae95838fcbb2ab8423 codelists/residence-regions.yaml +5f5790690f31f284aeddbbad330ab5ba7e3465a18bfe4ebc271b016bc0c5390a derivations/residence-region.rhai +d6c447f9c8e3fd25b2db9f9b12c2b497ef67572089bb82c2a26caa088ff96cb4 evidence.yaml +602a5c13d195c26e5b8b4194a5ff6420c263ead3aca1dac2e74e0f9f92b87d51 fixtures/residence-region-cases.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +a0d996c162f2e79e721f766abba2fc6b2cd7d88ca93010a3c9762e6e84291350 schemas/residence-region-adapter-parameters.schema.yaml +f9793498f5df20a093b8a135f7459cd39dd99731ccadd40e5784ca8d3f271987 schemas/residence-region-facts.schema.yaml +1897749194a690e233a5e353eb78d1ae3e0b5d44fb8c6d87af4c7e5f4aca7801 schemas/residence-region-response.schema.yaml diff --git a/products/evidence/reference/request-adapter/deployment-projects/protected-read-evidence/bundle/evidence.yaml b/products/evidence/reference/request-adapter/deployment-projects/protected-read-evidence/bundle/evidence.yaml index cfae6c1564..15cf27c2c5 100644 --- a/products/evidence/reference/request-adapter/deployment-projects/protected-read-evidence/bundle/evidence.yaml +++ b/products/evidence/reference/request-adapter/deployment-projects/protected-read-evidence/bundle/evidence.yaml @@ -10,17 +10,19 @@ issuer: # issuer. Evidence and the protected registry API verify that issuer under # their independently configured resource-server policies. authentication: - kind: oidc-access-token - issuer: https://tokens.gov.example - audiences: [registry-evidence] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: https://tokens.gov.example/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: https://tokens.gov.example + audience: registry-evidence + tokenTypes: [at+jwt] + algorithms: [ES256] + jwksSource: + kind: uri + uri: https://tokens.gov.example/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: hashKeyRef: secret:file/audit-hmac-key hashKeyVersion: 1 diff --git a/products/evidence/reference/request-adapter/deployment-projects/protected-read-evidence/runtime.yaml b/products/evidence/reference/request-adapter/deployment-projects/protected-read-evidence/runtime.yaml index 262f2163e1..89e413ce63 100644 --- a/products/evidence/reference/request-adapter/deployment-projects/protected-read-evidence/runtime.yaml +++ b/products/evidence/reference/request-adapter/deployment-projects/protected-read-evidence/runtime.yaml @@ -1,8 +1,9 @@ -version: 1 -bundleDirectory: /etc/registry-evidence/bundle +apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /etc/registry-evidence/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 diff --git a/products/evidence/reference/request-adapter/deployment-projects/sqlite-extract-evidence/bundle/SHA256SUMS b/products/evidence/reference/request-adapter/deployment-projects/sqlite-extract-evidence/bundle/SHA256SUMS new file mode 100644 index 0000000000..25d5f9979d --- /dev/null +++ b/products/evidence/reference/request-adapter/deployment-projects/sqlite-extract-evidence/bundle/SHA256SUMS @@ -0,0 +1,8 @@ +9b35109d9a5bcc9dbd638c6ac6c1eecfa9fc97ca1614b781544ac3380cc9d95e adapters/professional-licence-extract.rhai +f1335bcf04387f107f014aa872258a7cc2d7fc09e9e9ba5c77a25756929a9b5c derivations/professional-licence.rhai +d150b33075c7d557e1b15db8904194744b7aa777022b46ddac83152bfb470c34 evidence.yaml +0c636c7f283e235d3a651ab7979f76cbbb09b283a22585bd2efa141ae25051a1 fixtures/professional-licence-cases.yaml +a99518bc301f34140faab29d7533e97607ce4c82682537965160f4f6499e8c9b public-keys/_QkPweRjMZxmIHnz7v8tj3coTKx-90L2LRsZbkeP_Bo.jwk.json +e23bfab4ba941d8cee235ce8764337d174a2c1ce982cea360582befe2483f068 queries/professional-licence-lookup.sql +44d7ff6361cf58717d7cfe74a4a180d7d47befde5b7a832d10c2847c50467f25 schemas/professional-licence-facts.schema.yaml +dc5ce5a4d2bfea2387853cac9a16e983218e9300ba56d0284a2bed6581174751 schemas/professional-licence-response.schema.yaml diff --git a/products/evidence/reference/request-adapter/deployment-projects/sqlite-extract-evidence/bundle/evidence.yaml b/products/evidence/reference/request-adapter/deployment-projects/sqlite-extract-evidence/bundle/evidence.yaml index cf933a4111..33bb7ab121 100644 --- a/products/evidence/reference/request-adapter/deployment-projects/sqlite-extract-evidence/bundle/evidence.yaml +++ b/products/evidence/reference/request-adapter/deployment-projects/sqlite-extract-evidence/bundle/evidence.yaml @@ -10,17 +10,19 @@ issuer: # issuer. The extract is a local file, so this is the only network credential # contract in the project. authentication: - kind: oidc-access-token - issuer: https://tokens.gov.example - audiences: [registry-evidence] - tokenTypes: [at+jwt] - algorithms: [ES256] - jwksUri: https://tokens.gov.example/.well-known/jwks.json - principalClaim: sub - requesterTagsClaim: evidence_tags - evidenceAudienceClaim: evidence_audience - maximumTokenLifetimeSeconds: 300 - revokedKeyIds: [] + oidc: + issuer: https://tokens.gov.example + audience: registry-evidence + tokenTypes: [at+jwt] + algorithms: [ES256] + jwksSource: + kind: uri + uri: https://tokens.gov.example/.well-known/jwks.json + principalClaim: sub + requesterTagsClaim: evidence_tags + evidenceAudienceClaim: evidence_audience + maximumTokenLifetimeSeconds: 300 + revokedKeyIds: [] audit: hashKeyRef: secret:file/audit-hmac-key hashKeyVersion: 1 diff --git a/products/evidence/reference/request-adapter/deployment-projects/sqlite-extract-evidence/runtime.yaml b/products/evidence/reference/request-adapter/deployment-projects/sqlite-extract-evidence/runtime.yaml index fe2615b263..5131e8479f 100644 --- a/products/evidence/reference/request-adapter/deployment-projects/sqlite-extract-evidence/runtime.yaml +++ b/products/evidence/reference/request-adapter/deployment-projects/sqlite-extract-evidence/runtime.yaml @@ -1,8 +1,9 @@ -version: 1 -bundleDirectory: /etc/registry-evidence/bundle +apiVersion: registry.registrystack.org/evidence-runtime/v1alpha1 +kind: EvidenceRuntimeConfig +package: + root: /etc/registry-evidence/bundle listener: - bindHost: 127.0.0.1 - port: 8080 + bind: 127.0.0.1:8080 tlsTermination: operator-controlled-upstream trustProxyIdentityHeaders: false maximumRequestBytes: 65536 diff --git a/products/evidence/scripts/check-contracts.sh b/products/evidence/scripts/check-contracts.sh index d3251c744c..2adcda8d8a 100755 --- a/products/evidence/scripts/check-contracts.sh +++ b/products/evidence/scripts/check-contracts.sh @@ -13,6 +13,7 @@ if [ ! -d "$committed_root" ]; then fi cd "$repository_root" +python3 products/evidence/scripts/generate-package-sums.py CARGO_INCREMENTAL=0 CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0 \ cargo test --locked --quiet -p registry-evidence --test security_contract_traceability CARGO_INCREMENTAL=0 CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0 \ diff --git a/products/evidence/scripts/generate-package-sums.py b/products/evidence/scripts/generate-package-sums.py new file mode 100755 index 0000000000..2411f4e528 --- /dev/null +++ b/products/evidence/scripts/generate-package-sums.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Generate or check shared package sum files for committed Evidence bundles.""" + +from __future__ import annotations + +import argparse +import hashlib +from pathlib import Path + + +REPOSITORY = Path(__file__).resolve().parents[3] +EVIDENCE_ROOT = REPOSITORY / "products" / "evidence" +EXTRA_PACKAGES = ( + REPOSITORY + / "products" + / "breg" + / "acceptance" + / "farmer-landholding-evidence" + / "evidence" + / "provider", +) + + +def package_roots() -> list[Path]: + roots = {path.parent for path in EVIDENCE_ROOT.rglob("evidence.yaml")} + roots.update(path for path in EXTRA_PACKAGES if (path / "evidence.yaml").is_file()) + return sorted(roots) + + +def rendered_sum(root: Path) -> bytes: + lines = [] + for path in sorted(path for path in root.rglob("*") if path.is_file()): + relative = path.relative_to(root).as_posix() + if relative == "SHA256SUMS": + continue + digest = hashlib.sha256(path.read_bytes()).hexdigest() + lines.append(f"{digest} {relative}\n") + return "".join(lines).encode() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--write", action="store_true") + args = parser.parse_args() + drift = [] + for root in package_roots(): + output = root / "SHA256SUMS" + expected = rendered_sum(root) + if args.write: + output.write_bytes(expected) + elif not output.is_file() or output.read_bytes() != expected: + drift.append(root.relative_to(REPOSITORY).as_posix()) + if drift: + for root in drift: + print(f"Evidence package sum file is stale: {root}") + print("Run products/evidence/scripts/generate-package-sums.py --write") + return 1 + print(f"Evidence package sum files reproduce exactly ({len(package_roots())} packages).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/products/identifiers/generated/catalog.v1.json b/products/identifiers/generated/catalog.v1.json index bacc5acd75..a70512e121 100644 --- a/products/identifiers/generated/catalog.v1.json +++ b/products/identifiers/generated/catalog.v1.json @@ -43,7 +43,7 @@ "description": "The declared Evidence dependency could not be accepted.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "action.evidence_failed", @@ -62,7 +62,7 @@ "description": "The action handler could not produce an accepted result.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "action.handler_failed", @@ -81,7 +81,7 @@ "description": "The action was refused by a declared business rule.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "action.refused", @@ -100,7 +100,7 @@ "description": "The bearer credential is missing or refused.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "authentication.refused", @@ -119,7 +119,7 @@ "description": "The idempotency key is bound to another request.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "idempotency.conflict", @@ -138,7 +138,7 @@ "description": "The chunk does not match the expected next chunk.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "ingestion.chunk_mismatch", @@ -157,7 +157,7 @@ "description": "The selected access profile does not match the run's bound profile.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "ingestion.profile_mismatch", @@ -176,7 +176,7 @@ "description": "The stored receipt of the chunk was erased.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "ingestion.receipt_erased", @@ -192,10 +192,10 @@ "compatibilityLine": "v1alpha1", "owner": "breg", "title": "Conflict", - "description": "The active package no longer matches the run binding.", + "description": "The ingestion run is blocked and refuses further chunks.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "ingestion.run_blocked", @@ -214,7 +214,7 @@ "description": "The ingestion run is not open for this transition.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "ingestion.run_not_open", @@ -233,7 +233,7 @@ "description": "The lookup did not resolve exactly one record.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "lookup.unresolved", @@ -252,7 +252,7 @@ "description": "The mutation conflicts with current state.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "mutation.conflict", @@ -271,7 +271,7 @@ "description": "The mutation precondition failed.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "precondition.failed", @@ -290,7 +290,7 @@ "description": "The mutation precondition is required.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "precondition.required", @@ -309,7 +309,7 @@ "description": "The query cursor is invalid.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "query.cursor_invalid", @@ -328,7 +328,7 @@ "description": "The query request is invalid.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "query.invalid", @@ -347,7 +347,7 @@ "description": "The request is invalid.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "request.invalid", @@ -366,7 +366,7 @@ "description": "The change-request planner refused the submission, naming the failure kind.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "request.plan_refused", @@ -385,7 +385,7 @@ "description": "The request timed out.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "request.timeout", @@ -404,7 +404,7 @@ "description": "The requested resource was not found.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "resource.not_found", @@ -423,7 +423,7 @@ "description": "The Registry field-encryption service is unavailable.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "runtime.field_encryption.unavailable", @@ -442,7 +442,7 @@ "description": "Registry runtime is not ready.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "runtime.not_ready", @@ -461,7 +461,7 @@ "description": "The Registry mutation service is unavailable.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "service.unavailable", @@ -480,7 +480,7 @@ "description": "The Registry data service is unavailable.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "source.unavailable", @@ -499,7 +499,7 @@ "description": "The request media type is not supported.", "source": { "path": "crates/registry-breg/src/problem.rs", - "sha256": "1103252370b62dfc248689f7baf08f0479e4d7dd762a491ceafbd8341816b540" + "sha256": "a1a1640628a5fdf95767f82c347f90bf38af9c30867882ad0bdd18eb3601a28a" }, "problem": { "code": "unsupported.media_type", @@ -2763,7 +2763,7 @@ }, "artifact": { "path": "products/breg/generated/authoring/registry-module.schema.json", - "sha256": "285b54526deb920eaa6db8a93b87eb25c2a6a81e90b0743aa18e082b8cea6f7a", + "sha256": "532f2e49816a38ac82595c32d1cff93c53bfce72edb7f84ce2012d580e883338", "mediaType": "application/schema+json" } }, @@ -2781,7 +2781,7 @@ }, "artifact": { "path": "products/breg/generated/authoring/registry-project.schema.json", - "sha256": "39cec102bdf8febeae25aba599a115b9d46208f19529d69bcbd849ddc3c99f62", + "sha256": "f74838de0bcf456337f94c2040594a51c15e7a74f6be61abc37ba13dba08d813", "mediaType": "application/schema+json" } }, @@ -2799,7 +2799,7 @@ }, "artifact": { "path": "products/breg/generated/runtime/runtime.schema.json", - "sha256": "f75317190ca7be1c5354612a08b213495e6fd41ddf5db565e01106760da318b2", + "sha256": "76ef4e12ee92a32f8d7ac2f4b3445581af6701f56323ede6da6d1ccedfc39ed7", "mediaType": "application/schema+json" } }, @@ -2813,11 +2813,11 @@ "description": "Registry Casework runtime configuration JSON Schema.", "source": { "path": "crates/registry-casework/src/schema.rs", - "sha256": "d0f25762e9c7259afcc4f1b21259f3bce94f82958fd3c56d8c33f02fe4c37a3f" + "sha256": "3e7923770567f0339ba3e0173f4ac7735f2bb0f5133bc72488e6f61b99a2804e" }, "artifact": { "path": "products/casework/generated/runtime/runtime.schema.json", - "sha256": "8dbf377a18ec92abef929c991509e72ee33657b652f4b34222fcc46839e391c7", + "sha256": "8b4dc0fbd680c404011403031e5865829a934758ea6cb36b11da0f8007e756fd", "mediaType": "application/schema+json" } }, @@ -2903,11 +2903,11 @@ "description": "Relay V2 authoring JSON Schema.", "source": { "path": "crates/registry-relayctl/schemas/authoring/runtime.schema.json", - "sha256": "08e6bacc0adb91489e6d85b6366c9d43536d6b8d36ded8175d491c1ac9503c6b" + "sha256": "e789f3f61af4d7527e0a309ee7512a40f984096abe031a205398c14d5f2c452c" }, "artifact": { "path": "crates/registry-relayctl/schemas/authoring/runtime.schema.json", - "sha256": "08e6bacc0adb91489e6d85b6366c9d43536d6b8d36ded8175d491c1ac9503c6b", + "sha256": "e789f3f61af4d7527e0a309ee7512a40f984096abe031a205398c14d5f2c452c", "mediaType": "application/schema+json" } }, @@ -2921,11 +2921,11 @@ "description": "Registry Scheduling runtime configuration JSON Schema.", "source": { "path": "crates/registry-scheduling/src/schema.rs", - "sha256": "1cbcdc86074891a5ef5c9237b43e492967b1cc23eb917438a953905613327bc6" + "sha256": "868c84dec7bd0d5cbef44ac42e29a35a0fbc313dd92f5e12ff76e339a04a5dc0" }, "artifact": { "path": "products/scheduling/generated/runtime/runtime.schema.json", - "sha256": "3df3a8ea822cc2db0b82c15815490620a5e5b6b7882f0306588c7829828eb120", + "sha256": "b51c462eba370acb45ef5f20a09aead5cab3d0d98d42f55947ab7352eed77ccf", "mediaType": "application/schema+json" } }, diff --git a/products/platform/AGENTS.md b/products/platform/AGENTS.md index 7fab7b9755..a630dac5d0 100644 --- a/products/platform/AGENTS.md +++ b/products/platform/AGENTS.md @@ -67,7 +67,9 @@ workspace package ran. Use `products/platform/scripts/check-hygiene-alignment.sh` for shared lint/format templates and `cargo deny check` for dependency or advisory policy -changes. Coverage and fuzz definitions live in CI and +changes. Run `products/platform/scripts/check-config-conformance.py +--check-generated` after changing a shared runtime configuration block or a +runtime that reads `runtime.yaml` through `RuntimeConfigLoader`. Coverage and fuzz definitions live in CI and [fuzz/README.md](fuzz/README.md). Run the affected gate locally when changing it or when CI cannot supply the required proof; routine edits do not require a new full-workspace assurance exercise. diff --git a/products/platform/CHANGELOG.md b/products/platform/CHANGELOG.md index 76835b9652..e50868f66f 100644 --- a/products/platform/CHANGELOG.md +++ b/products/platform/CHANGELOG.md @@ -49,6 +49,81 @@ `ChainAssertionError`. Keyed audit references from `AuditProfile` and `AuditKeyHasher` are unchanged byte for byte. +- BREAKING: `OidcIssuerConfig::check` refuses an issuer carrying a query + component, as it already refused credentials and a fragment. The refusal + names the field and does not repeat the configured value. +- BREAKING: `TransitSigner::initialize` returns `TransitInitializationError`, + which names the fault it met: an unreachable socket, a refused metadata read, + a provider server error on that read, a malformed response, unsafe custody, + a key version above `latest_version`, a key version below + `min_encryption_version`, a public key mismatch, or a failed self-test. The + causes carry no path, provider response, or key material. +- Add `ValidatedFetchUrl::immediate_get_with_additional_roots` and + `JwksFetcher::new_trusting_additional_roots`, so a key set served under a + private certificate authority is fetched with that authority trusted beside + the system roots for that connection alone, without a process-wide trust + store change. + +- Add the shared runtime configuration loader: a bounded, strict YAML reader + for `runtime.yaml` that refuses symbolic links, removed keys, and a wrong + envelope, then substitutes `${VAR}`, `${VAR:-default}`, and + `${VAR:?message}` inside string values after parsing. Substitution is + refused inside `*Ref` fields, under `secretProviders`, and in authored + package files; the authored-file check refuses text it cannot read. No + refusal repeats a configured value: a `:?` message, an invalid variable + name, and a refused enum variant are all withheld. +- Add the shared configuration blocks (`secretProviders`, `database`, + `jwksSource`, `package`, `listener`) and their canonical JSON Schema under + `products/platform/generated/`. The blocks also serialize back to the form + they were read from, so a runtime that renders its configuration keeps the + same key names. +- Add `registry_platform_config::package`, the one package format every + runtime serves. A package directory holds `SHA256SUMS`, one `sha256sum` + line per file sorted by path, and an optional one-line `REVISION` that is + listed and hashed like any other file. The package digest is the `sha256:` + label of the `SHA256SUMS` bytes. `write_package` and `write_sum_file` write + a package, `plan_package` reports its digest without writing, and + `verify_package` recomputes every digest and refuses a changed, missing, or + extra file by name, a symbolic link, a special file, or a package over its + `PackageLimits`. `PackageConfig::verify_package` also compares the digest + with `package.expectedDigest` when it is set, with the same expected and + found message in every runtime. +- Add the runtime configuration conformance gate, + `products/platform/scripts/check-config-conformance.py`, run in root CI. It + fails when a runtime's generated schema re-declares a shared block, when a + runtime without a generated schema stops holding a shared block type in its + runtime struct, when a hand-written schema copy of a shared block widens it, + when no non-test code reads `runtime.yaml` through the shared loader, when a + runtime still expands its configuration outside the shared loader, when the + tests proving `*Ref` fields and authored files refuse `${VAR}` go missing, or + when the committed canonical schema differs from its generator. +- Add the shared `audit.hashKeyRef` and `authentication.oidc` blocks + (`AuditKeyConfig` and `OidcIssuerConfig`). The OIDC block holds the exact + issuer URL, one bounded audience, and the `jwksSource`; a product embeds it + beside its own token rules. `SecretReference` now reads and writes as its + reference text, so a configuration struct can hold one directly. +- Add the shared `authentication.oidc` client block (`OidcClientsConfig`): + `allowedClients` and the bounded `assertionIssuers` map, at most 64 clients + of at most 128 bytes, each with at most 16 distinct issuers of at most 512 + bytes. Add `registry_platform_oidc::parse_static_jwks`, which refuses a + static key set that is empty, holds a symmetric key, or leaves a key without + a unique `kid`. +- Add `REMOVED_OIDC_JWKS_URI`, the removed `authentication.oidc.jwksUri` key + with its `jwksSource` replacement, and, behind the `schema` feature, + `schema::jwks_document_provider_requirements`, the root `allOf` rule that a + static `jwksSource.documentRef` enables the secret provider it names. +- Add Ed25519 and ES256 private key generation beside ES384, with RFC 7638 + thumbprint tests for RSA, EC, and OKP keys. +- BREAKING: remove `reject_deprecated_config_fields`; runtimes declare removed + keys on the loader instead. +- BREAKING: `JwksSource::Discovery` is an empty struct variant, so a `uri` or + a `documentRef` written beside `kind: discovery` is refused. +- BREAKING: `expand_config_env_vars` no longer repeats a `${VAR:?message}` + message or an invalid variable name in its refusal. +- BREAKING: remove `expand_config_env_vars`, `expand_config_env_vars_with`, and + `ConfigEnvExpansionError`. Every runtime reads `runtime.yaml` through + `RuntimeConfigLoader`, which substitutes `${VAR}` inside parsed string values. + ## v0.34.0 - 2026-09-25 - The shared platform crates have no user-visible changes in this release. diff --git a/products/platform/README.md b/products/platform/README.md index efd644ca42..7e9f075309 100644 --- a/products/platform/README.md +++ b/products/platform/README.md @@ -89,6 +89,7 @@ cargo test --locked -p 'registry-platform-*' --all-targets --all-features cargo llvm-cov --locked -p 'registry-platform-*' --all-features --fail-under-lines 80 cargo deny check products/platform/scripts/check-hygiene-alignment.sh +products/platform/scripts/check-config-conformance.py --check-generated gitleaks dir --config .gitleaks.toml --no-banner --redact --timeout 120 . ``` diff --git a/products/platform/generated/runtime-config-blocks.schema.json b/products/platform/generated/runtime-config-blocks.schema.json new file mode 100644 index 0000000000..d28f61b463 --- /dev/null +++ b/products/platform/generated/runtime-config-blocks.schema.json @@ -0,0 +1,328 @@ +{ + "$defs": { + "AuditKeyConfig": { + "description": "The key for the keyed references an audit record carries in place of raw\nidentifiers, written `audit.hashKeyRef` beside the product's own audit\nsettings.", + "properties": { + "hashKeyRef": { + "$ref": "#/$defs/SecretReference", + "description": "Secret reference to the audit hash key." + } + }, + "required": [ + "hashKeyRef" + ], + "type": "object" + }, + "DatabaseConfig": { + "additionalProperties": false, + "description": "The PostgreSQL connection a stateful runtime uses. Both URLs are secret\nreferences and may name the same secret.", + "properties": { + "migrationUrlRef": { + "description": "Secret reference to the migration connection URL.", + "pattern": "^secret:(?:env|file)/", + "type": "string" + }, + "runtimeUrlRef": { + "description": "Secret reference to the least-privileged runtime connection URL.", + "pattern": "^secret:(?:env|file)/", + "type": "string" + }, + "testOnlyPlaintext": { + "default": false, + "description": "Allow a plaintext connection. Refused outside test builds.", + "type": "boolean" + }, + "trustedRootCertificateRef": { + "default": null, + "description": "Secret reference to a PEM root certificate the connection trusts.", + "pattern": "^secret:(?:env|file)/", + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "runtimeUrlRef", + "migrationUrlRef" + ], + "type": "object" + }, + "EnvironmentSecretProviderConfig": { + "additionalProperties": false, + "description": "The environment secret provider. It takes no settings.", + "type": "object" + }, + "FileSecretProviderConfig": { + "additionalProperties": false, + "description": "The file secret provider.", + "properties": { + "root": { + "description": "Absolute directory holding one file per secret. Each file must be a\nregular file owned by the runtime user, mode 0400 or 0600, with exactly\none hard link.", + "pattern": "^/", + "type": "string" + } + }, + "required": [ + "root" + ], + "type": "object" + }, + "JwksSource": { + "description": "Where a runtime obtains the OIDC issuer's signing keys.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Read `jwks_uri` from the issuer's OpenID Connect discovery document.", + "properties": { + "kind": { + "const": "discovery", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Fetch the key set from this absolute `https` URI, skipping discovery.", + "properties": { + "kind": { + "const": "uri", + "type": "string" + }, + "uri": { + "pattern": "^https?://", + "type": "string" + } + }, + "required": [ + "kind", + "uri" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "Read the key set from a secret, for deployments without network access\nto the issuer.", + "properties": { + "documentRef": { + "pattern": "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$", + "type": "string" + }, + "kind": { + "const": "static", + "type": "string" + } + }, + "required": [ + "kind", + "documentRef" + ], + "type": "object" + } + ] + }, + "ListenerBind": { + "description": "Socket address the runtime listens on, written host:port with an IP address host ([addr]:port for IPv6).", + "maxLength": 128, + "minLength": 1, + "type": "string" + }, + "ListenerConfig": { + "additionalProperties": false, + "description": "The listener of a runtime that declares no TLS or exposure settings.", + "properties": { + "bind": { + "$ref": "#/$defs/ListenerBind" + } + }, + "required": [ + "bind" + ], + "type": "object" + }, + "ListenerNetworkExposure": { + "description": "The operator-declared private network placement of an HTTP listener.", + "enum": [ + "private-address", + "container-private" + ], + "type": "string" + }, + "OidcClientsConfig": { + "description": "The OAuth clients a runtime admits access tokens for, and the assertion\nauthorities each client may exchange a subject token from, written under\n`authentication.oidc` beside the issuer.", + "properties": { + "allowedClients": { + "default": [], + "description": "Client identifiers whose access tokens are admitted. A runtime decides\nwhether an empty list is acceptable in production.", + "items": { + "type": "string" + }, + "type": "array" + }, + "assertionIssuers": { + "additionalProperties": { + "items": { + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + "maxItems": 16, + "type": "array", + "uniqueItems": true + }, + "default": {}, + "description": "Assertion authorities each client may exchange a subject token from,\nkeyed by client identifier. An empty map applies no rule. Once a\nclient is listed, a token it exchanged is accepted only for one of\nthat client's declared authorities.", + "maxProperties": 64, + "propertyNames": { + "maxLength": 128, + "minLength": 1 + }, + "type": "object" + } + }, + "type": "object" + }, + "OidcIssuerConfig": { + "description": "The OIDC issuer a runtime accepts access tokens from: the exact `iss`\nvalue, the `aud` value a token must carry, and where the issuer's signing\nkeys come from, written under `authentication.oidc` beside the product's\nown token rules.", + "properties": { + "audience": { + "description": "The audience every accepted access token must carry in `aud`.", + "maxLength": 512, + "minLength": 1, + "type": "string" + }, + "issuer": { + "description": "Exact issuer accepted in access-token `iss` claims, an absolute\n`https` URL.", + "pattern": "^https?://", + "type": "string" + }, + "jwksSource": { + "$ref": "#/$defs/JwksSource", + "default": { + "kind": "discovery" + }, + "description": "Where the issuer's signing keys come from. Absent reads the issuer's\nOpenID Connect discovery document." + } + }, + "required": [ + "issuer", + "audience" + ], + "type": "object" + }, + "PackageConfig": { + "additionalProperties": false, + "description": "The package a runtime serves: `root` is the absolute package directory,\nand `expectedDigest`, when set, pins the package digest the runtime must\nfind there. The package digest is the digest of the package's\n`SHA256SUMS` file; see [`crate::package`].", + "properties": { + "expectedDigest": { + "description": "`sha256:` label of the package digest, the digest of the package's\n`SHA256SUMS` file. When set, the runtime refuses to start on any other\npackage.", + "pattern": "^sha256:[0-9a-f]{64}$", + "type": [ + "string", + "null" + ] + }, + "root": { + "description": "Absolute path of the package directory.", + "pattern": "^/", + "type": "string" + } + }, + "required": [ + "root" + ], + "type": "object" + }, + "PrivateListenerConfig": { + "additionalProperties": false, + "description": "The listener of a runtime that declares its TLS termination and network\nexposure.", + "properties": { + "bind": { + "$ref": "#/$defs/ListenerBind" + }, + "networkExposure": { + "$ref": "#/$defs/ListenerNetworkExposure", + "default": "private-address" + }, + "tlsTermination": { + "$ref": "#/$defs/TlsTermination" + } + }, + "required": [ + "bind", + "tlsTermination" + ], + "type": "object" + }, + "SecretProvidersConfig": { + "additionalProperties": false, + "anyOf": [ + { + "properties": { + "file": { + "$ref": "#/$defs/FileSecretProviderConfig" + } + }, + "required": [ + "file" + ] + }, + { + "properties": { + "environment": { + "$ref": "#/$defs/EnvironmentSecretProviderConfig" + } + }, + "required": [ + "environment" + ] + } + ], + "description": "The secret providers a runtime enables. A reference is resolved only by a\nprovider declared here: `secret:file/name` under `file.root`, and\n`secret:env/NAME` only when `environment: {}` is present.", + "properties": { + "environment": { + "anyOf": [ + { + "$ref": "#/$defs/EnvironmentSecretProviderConfig" + }, + { + "type": "null" + } + ], + "description": "Enables `secret:env/NAME` references, read from the process\nenvironment. Declared as an empty mapping: `environment: {}`." + }, + "file": { + "anyOf": [ + { + "$ref": "#/$defs/FileSecretProviderConfig" + }, + { + "type": "null" + } + ], + "description": "Enables `secret:file/name` references, read from files under `root`." + } + }, + "type": "object" + }, + "SecretReference": { + "description": "An exact secret reference: secret:file/name, resolved under secretProviders.file.root, or secret:env/NAME, resolved only when secretProviders.environment is declared.", + "pattern": "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$", + "type": "string" + }, + "TlsTermination": { + "description": "Declares the trusted transport boundary for a runtime's plaintext HTTP\nlistener. Production listeners require operator-controlled upstream TLS\ntermination; direct plaintext is limited to the explicit loopback-only\ndevelopment mode.", + "enum": [ + "operator-controlled-upstream", + "development-loopback" + ], + "type": "string" + } + }, + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Registry Stack shared runtime configuration blocks" +} diff --git a/products/platform/scripts/check-config-conformance.py b/products/platform/scripts/check-config-conformance.py new file mode 100755 index 0000000000..49356cf834 --- /dev/null +++ b/products/platform/scripts/check-config-conformance.py @@ -0,0 +1,857 @@ +#!/usr/bin/env python3 +"""Hold every runtime to the shared runtime configuration surface. + +Each row names one product runtime and the evidence that it conforms: + +- its generated runtime schema embeds every shared configuration block it + uses unchanged from the canonical platform schema, and any block it carries + under a shared name matches that schema too; +- a runtime without a generated schema holds each shared block as a field of + its runtime struct, typed with the `registry_platform_config` type itself, + and a hand-written schema carrying a shared block keeps every canonical + keyword unchanged and narrows it only with `allOf`; +- its non-test code reads `runtime.yaml` through `RuntimeConfigLoader` and no + source calls the legacy `expand_config_env_vars` expansion; +- named tests prove a `*Ref` field refuses `${VAR}` substitution, an authored + project file refuses an environment expression, and a mismatched package pin + reports the shared expected-and-found digest shape. The Rust test jobs run + those tests; this gate fails when one is renamed or removed. + +A row may exempt one of these only with a stated reason. A product adopting the +loader adds a row. Rows for the shared ctl verbs, `--format`, and exit classes +join when those surfaces land. + +With `--check-generated`, the gate also regenerates the canonical schema and +fails when the committed copy differs. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + + +ROOT = Path(__file__).resolve().parents[3] +CANONICAL_SCHEMA = "products/platform/generated/runtime-config-blocks.schema.json" +GENERATOR_OUTPUT = "products/platform/generated" +GENERATOR_ARGUMENTS: tuple[str, ...] = ( + "cargo", + "run", + "--locked", + "-p", + "registry-platform-config", + "--features", + "schema", + "--example", + "shared-blocks-schema", + "--", + "--output", +) +GENERATOR_COMMAND = " ".join((*GENERATOR_ARGUMENTS, GENERATOR_OUTPUT)) +LEGACY_EXPANSION = re.compile(r"\bexpand_config_env_vars\w*\b") +LOADER_USE = re.compile(r"\bRuntimeConfigLoader\s*::\s*new\s*\(") +PLATFORM_CRATE = "registry_platform_config" +# Keywords a hand-written copy of a shared block may add: each one only +# describes or narrows what the canonical block accepts. +NARROWING_KEYWORDS = frozenset({"$comment", "allOf", "description", "title"}) + + +@dataclass(frozen=True) +class TestRef: + path: str + name: str + + +@dataclass(frozen=True) +class Exemption: + reason: str + + +@dataclass(frozen=True) +class RustBlock: + """A runtime struct field that must hold a shared block type.""" + + path: str + struct: str + field: str + block: str + + +@dataclass(frozen=True) +class HandSchema: + """A shared block written by hand inside a product schema.""" + + path: str + pointer: tuple[str, ...] + block: str + + +@dataclass(frozen=True) +class Row: + product: str + loader_sources: tuple[str, ...] + runtime_schema: str | Exemption + shared_blocks: tuple[str, ...] + reference_refusal: TestRef | Exemption + authored_refusal: TestRef | Exemption + digest_mismatch: TestRef + rust_blocks: tuple[RustBlock, ...] = () + hand_schemas: tuple[HandSchema, ...] = () + + +ROWS: tuple[Row, ...] = ( + Row( + product="relay", + loader_sources=("crates/registry-relay-v2/src",), + runtime_schema="crates/registry-relayctl/schemas/authoring/runtime.schema.json", + shared_blocks=( + "EnvironmentSecretProviderConfig", + "FileSecretProviderConfig", + "JwksSource", + "ListenerBind", + "ListenerConfig", + "PackageConfig", + "SecretProvidersConfig", + ), + reference_refusal=TestRef( + "crates/registry-relay-v2/src/contract.rs", + "environment_substitution_never_reaches_a_secret_reference", + ), + authored_refusal=TestRef( + "crates/registry-relay-v2/src/contract.rs", + "an_authored_contract_carrying_an_environment_expression_is_refused", + ), + digest_mismatch=TestRef( + "crates/registry-relay-v2/tests/process_http.rs", + "built_relay_check_honors_a_package_digest_pin", + ), + ), + Row( + product="render", + loader_sources=("crates/registry-render/src",), + runtime_schema=Exemption("Render publishes no generated runtime schema"), + shared_blocks=(), + reference_refusal=TestRef( + "crates/registry-render/src/runtime.rs", + "environment_expressions_substitute_values_but_never_secret_references", + ), + authored_refusal=TestRef( + "crates/registry-render/src/manifest.rs", + "an_authored_manifest_carrying_an_environment_expression_is_refused", + ), + digest_mismatch=TestRef( + "crates/registry-render/tests/serve.rs", + "serve_startup_package_digest_mismatch_uses_common_expected_and_found_shape", + ), + rust_blocks=( + RustBlock( + "crates/registry-render/src/runtime.rs", + "RenderRuntime", + "package", + "PackageConfig", + ), + RustBlock( + "crates/registry-render/src/runtime.rs", + "RenderRuntime", + "secret_providers", + "SecretProvidersConfig", + ), + RustBlock( + "crates/registry-render/src/runtime.rs", + "ListenerRuntime", + "bind", + "ListenerBind", + ), + ), + ), + Row( + product="discovery", + loader_sources=("crates/registry-discovery/src",), + runtime_schema=Exemption("Discovery publishes no generated runtime schema"), + shared_blocks=(), + reference_refusal=Exemption("the Discovery runtime has no *Ref field"), + authored_refusal=Exemption( + "the Discovery runtime serves a built index and reads no authored " + "project file" + ), + digest_mismatch=TestRef( + "crates/registry-discovery/src/startup.rs", + "startup_verifies_package_and_refuses_expected_digest_mismatch_with_common_shape", + ), + rust_blocks=( + RustBlock( + "crates/registry-discovery/src/startup.rs", + "RuntimeConfig", + "listener", + "ListenerConfig", + ), + ), + hand_schemas=( + HandSchema( + "products/discovery/schemas/runtime.schema.json", + ("properties", "listener"), + "ListenerConfig", + ), + ), + ), + Row( + product="evidence", + loader_sources=("crates/registry-evidence/src",), + runtime_schema=Exemption( + "Evidence publishes the frozen hand-written " + "products/evidence/contracts/runtime.schema.yaml, held by its own " + "contract checks, not a generated runtime schema" + ), + shared_blocks=(), + reference_refusal=TestRef( + "crates/registry-evidence/src/config.rs", + "environment_substitution_fills_values_and_never_a_secret_reference", + ), + authored_refusal=TestRef( + "crates/registry-evidence/src/config.rs", + "an_authored_bundle_carrying_an_environment_expression_is_refused", + ), + digest_mismatch=TestRef( + "crates/registry-evidence/src/runtime_tests.rs", + "an_expected_package_digest_admits_only_the_bundle_it_names", + ), + rust_blocks=( + RustBlock( + "crates/registry-evidence/src/config.rs", + "RuntimeConfig", + "package", + "PackageConfig", + ), + RustBlock( + "crates/registry-evidence/src/config.rs", + "RuntimeConfig", + "secret_providers", + "SecretProvidersConfig", + ), + RustBlock( + "crates/registry-evidence/src/config.rs", + "ListenerConfig", + "bind", + "ListenerBind", + ), + RustBlock( + "crates/registry-evidence/src/config.rs", + "MetricsListenerConfig", + "bind", + "ListenerBind", + ), + RustBlock( + "crates/registry-evidence/src/config.rs", + "OidcAuthenticationConfig", + "provider", + "OidcIssuerConfig", + ), + RustBlock( + "crates/registry-evidence/src/config.rs", + "AuditConfig", + "key", + "AuditKeyConfig", + ), + ), + ), + Row( + product="breg", + loader_sources=("crates/registry-breg/src",), + runtime_schema="products/breg/generated/runtime/runtime.schema.json", + shared_blocks=( + "EnvironmentSecretProviderConfig", + "FileSecretProviderConfig", + "JwksSource", + "ListenerBind", + "SecretProvidersConfig", + "SecretReference", + ), + reference_refusal=TestRef( + "crates/registry-breg/tests/runtime_config.rs", + "a_substitution_inside_a_secret_reference_is_refused", + ), + authored_refusal=TestRef( + "crates/registry-breg/tests/compiler_contract.rs", + "an_authored_project_carrying_an_environment_expression_is_refused", + ), + digest_mismatch=TestRef( + "crates/registry-breg/tests/runtime_config.rs", + "shared_package_envelope_and_pin_are_checked_before_startup", + ), + rust_blocks=( + RustBlock( + "crates/registry-breg/src/runtime_config.rs", + "RawRuntimeConfig", + "secret_providers", + "SecretProvidersConfig", + ), + RustBlock( + "crates/registry-breg/src/runtime_config.rs", + "RawListenerConfig", + "bind", + "ListenerBind", + ), + RustBlock( + "crates/registry-breg/src/runtime_config.rs", + "RawMetricsListenerConfig", + "bind", + "ListenerBind", + ), + RustBlock( + "crates/registry-breg/src/runtime_config.rs", + "RawOidcVerifierConfig", + "provider", + "OidcIssuerConfig", + ), + RustBlock( + "crates/registry-breg/src/runtime_config.rs", + "RawAuditConfig", + "key", + "AuditKeyConfig", + ), + ), + ), + Row( + product="casework", + loader_sources=("crates/registry-casework/src",), + runtime_schema="products/casework/generated/runtime/runtime.schema.json", + shared_blocks=( + "DatabaseConfig", + "EnvironmentSecretProviderConfig", + "FileSecretProviderConfig", + "JwksSource", + "ListenerBind", + "ListenerNetworkExposure", + "PrivateListenerConfig", + "SecretProvidersConfig", + "SecretReference", + "TlsTermination", + ), + reference_refusal=TestRef( + "crates/registry-casework/src/config.rs", + "environment_expressions_substitute_values_but_never_secret_references", + ), + authored_refusal=TestRef( + "crates/registry-casework/src/config.rs", + "an_authored_project_carrying_an_environment_expression_is_refused", + ), + digest_mismatch=TestRef( + "crates/registry-casework/src/config.rs", + "a_package_digest_mismatch_is_refused_in_the_shared_shape", + ), + rust_blocks=( + RustBlock( + "crates/registry-casework/src/config.rs", + "OidcConfig", + "provider", + "OidcIssuerConfig", + ), + RustBlock( + "crates/registry-casework/src/config.rs", + "OidcConfig", + "clients", + "OidcClientsConfig", + ), + RustBlock( + "crates/registry-casework/src/config.rs", + "AuditConfig", + "key", + "AuditKeyConfig", + ), + ), + ), + Row( + product="scheduling", + loader_sources=("crates/registry-scheduling/src",), + runtime_schema="products/scheduling/generated/runtime/runtime.schema.json", + shared_blocks=( + "DatabaseConfig", + "EnvironmentSecretProviderConfig", + "FileSecretProviderConfig", + "JwksSource", + "ListenerBind", + "ListenerNetworkExposure", + "PackageConfig", + "PrivateListenerConfig", + "SecretProvidersConfig", + "SecretReference", + "TlsTermination", + ), + reference_refusal=TestRef( + "crates/registry-scheduling/src/config.rs", + "environment_expressions_substitute_values_but_never_secret_references", + ), + authored_refusal=TestRef( + "crates/registry-scheduling/src/config.rs", + "an_authored_policy_carrying_an_environment_expression_is_refused", + ), + digest_mismatch=TestRef( + "crates/registry-scheduling/src/config.rs", + "a_pinned_package_digest_must_match_the_verified_package", + ), + rust_blocks=( + RustBlock( + "crates/registry-scheduling/src/config.rs", + "OidcConfig", + "provider", + "OidcIssuerConfig", + ), + RustBlock( + "crates/registry-scheduling/src/config.rs", + "OidcConfig", + "clients", + "OidcClientsConfig", + ), + RustBlock( + "crates/registry-scheduling/src/config.rs", + "AuditConfig", + "key", + "AuditKeyConfig", + ), + ), + ), +) + +EXPECTED_PRODUCTS = frozenset( + {"relay", "render", "discovery", "evidence", "breg", "casework", "scheduling"} +) + + +class GeneratorFailed(Exception): + pass + + +def read_defs(path: Path) -> dict[str, object] | None: + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + defs = document.get("$defs") if isinstance(document, dict) else None + return defs if isinstance(defs, dict) else None + + +def rust_sources(root: Path, directories: tuple[str, ...]) -> list[Path]: + return sorted( + path for directory in directories for path in (root / directory).rglob("*.rs") + ) + + +RUST_LITERAL_OR_COMMENT = re.compile( + r"//[^\n]*" + r"|/\*.*?\*/" + r'|b?r(?P#*)".*?"(?P=hashes)' + r'|b?"(?:\\.|[^"\\])*"' + r"|b?'(?:\\.|[^'\\])'", + re.DOTALL, +) +TEST_ONLY_ITEM = re.compile(r"#\[cfg\(test\)\]((?:\s*#\[[^\]]*\])*)\s*") +MODULE_DECLARATION = re.compile( + r"(?:pub(?:\([^)]*\))?\s+)?mod\s+(?P\w+)\s*(?P[;{])" +) +PATH_ATTRIBUTE = re.compile(r"#\[path\s*=\s*\"(?P[^\"]+)\"\s*\]") + + +def blank(match: re.Match[str]) -> str: + return re.sub(r"[^\n]", " ", match.group(0)) + + +def code_only(text: str) -> str: + """Blank Rust comments and string and character literals, keeping offsets. + + Nested block comments are not modelled; the product sources carry none. + """ + return RUST_LITERAL_OR_COMMENT.sub(blank, text) + + +def matching_brace(code: str, opening: int) -> int: + depth = 0 + for index in range(opening, len(code)): + if code[index] == "{": + depth += 1 + elif code[index] == "}": + depth -= 1 + if depth == 0: + return index + return len(code) - 1 + + +def module_file(declaring: Path, name: str, path_attribute: str | None) -> list[Path]: + if path_attribute is not None: + return [declaring.parent / path_attribute] + if declaring.name in {"lib.rs", "main.rs", "mod.rs"}: + base = declaring.parent + else: + base = declaring.parent / declaring.stem + return [base / f"{name}.rs", base / name / "mod.rs"] + + +def strip_test_items(path: Path, text: str) -> tuple[str, list[Path]]: + """Blank every `#[cfg(test)]` item and name the files of test-only modules.""" + + code = code_only(text) + test_files: list[Path] = [] + position = 0 + while (found := TEST_ONLY_ITEM.search(code, position)) is not None: + rest = found.end() + declaration = MODULE_DECLARATION.match(code, rest) + if declaration is not None and declaration.group("end") == ";": + # `code` blanks string literals; the `#[path]` value is read from + # the same offsets of the original text. + attribute = PATH_ATTRIBUTE.search(text, found.start(1), found.end(1)) + test_files += module_file( + path, + declaration.group("name"), + attribute.group("path") if attribute else None, + ) + end = declaration.end() + else: + opening = code.find("{", rest) + semicolon = code.find(";", rest) + if opening == -1 or (semicolon != -1 and semicolon < opening): + end = semicolon + 1 if semicolon != -1 else len(code) + else: + end = matching_brace(code, opening) + 1 + code = code[: found.start()] + re.sub( + r"[^\n]", " ", code[found.start() : end] + ) + code[end:] + position = end + return code, test_files + + +def production_code(paths: list[Path]) -> dict[Path, str]: + """Return the non-test code of each source, dropping test-only module files.""" + + stripped: dict[Path, str] = {} + test_files: set[Path] = set() + for path in paths: + code, files = strip_test_items(path, path.read_text(encoding="utf-8")) + stripped[path] = code + test_files.update(file.resolve() for file in files) + return { + path: code for path, code in stripped.items() if path.resolve() not in test_files + } + + +def has_test(text: str, name: str) -> bool: + attribute = r"#\[[^\]]*\]\s*" + test = rf"#\[(?:tokio::)?test(?:\([^\]]*\))?\]\s*(?:{attribute})*(?:async\s+)?fn\s+{name}\s*\(" + return re.search(test, text) is not None + + +def check_exemption(row: Row, field: str, value: object) -> list[str]: + if isinstance(value, Exemption) and not value.reason.strip(): + return [f"{row.product}: the {field} exemption needs a reason"] + return [] + + +def check_schema(root: Path, row: Row, canonical: dict[str, object]) -> list[str]: + if isinstance(row.runtime_schema, Exemption): + if row.shared_blocks: + return [ + f"{row.product}: a product without a runtime schema declares no " + "shared blocks" + ] + return [] + problems = [ + f"{row.product}: shared block {name} is not in {CANONICAL_SCHEMA}" + for name in row.shared_blocks + if name not in canonical + ] + schema = row.runtime_schema + if not (root / schema).is_file(): + return problems + [f"{row.product}: runtime schema {schema} is missing"] + defs = read_defs(root / schema) + if defs is None: + return problems + [f"{row.product}: runtime schema {schema} has no $defs"] + for name in row.shared_blocks: + if name in canonical and name not in defs: + problems.append( + f"{row.product}: {schema} does not embed shared block {name}" + ) + for name in sorted(set(defs) & set(canonical)): + if defs[name] != canonical[name]: + problems.append( + f"{row.product}: {schema} re-declares shared block {name} instead " + "of embedding it unchanged" + ) + return problems + + +def check_loader(root: Path, row: Row) -> list[str]: + sources = rust_sources(root, row.loader_sources) + problems = [] + texts = {path: path.read_text(encoding="utf-8") for path in sources} + production = production_code(sources) + if not any(LOADER_USE.search(code) for code in production.values()): + problems.append( + f"{row.product}: no source under {', '.join(row.loader_sources)} reads " + "runtime.yaml through RuntimeConfigLoader" + ) + for path, text in texts.items(): + if LEGACY_EXPANSION.search(text): + problems.append( + f"{row.product}: {path.relative_to(root).as_posix()} calls " + "expand_config_env_vars; read runtime.yaml through " + "RuntimeConfigLoader instead" + ) + return problems + + +def struct_fields(code: str, struct: str) -> dict[str, str] | None: + declaration = re.search( + rf"\bstruct\s+{struct}\b[^{{;]*\{{", code + ) + if declaration is None: + return None + body = code[declaration.end() : matching_brace(code, declaration.end() - 1)] + body = re.sub(r"#\[[^\]]*\]", " ", body) + fields: dict[str, str] = {} + depth = 0 + current = "" + for character in body + ",": + if character in "<([": + depth += 1 + elif character in ">)]": + depth -= 1 + if character == "," and depth == 0: + field = re.fullmatch( + r"\s*(?:pub(?:\([^)]*\))?\s+)?(\w+)\s*:\s*(.+?)\s*", current, re.DOTALL + ) + if field is not None: + fields[field.group(1)] = re.sub(r"\s+", "", field.group(2)) + current = "" + else: + current += character + return fields + + +def imports_from_platform(code: str, block: str) -> bool: + for use in re.finditer(rf"\buse\s+{PLATFORM_CRATE}\s*::\s*([^;]*);", code): + if re.search(rf"(? list[str]: + problems: list[str] = [] + for entry in row.rust_blocks: + if entry.block not in canonical: + problems.append( + f"{row.product}: shared block {entry.block} is not in {CANONICAL_SCHEMA}" + ) + continue + path = root / entry.path + if not path.is_file(): + problems.append(f"{row.product}: runtime source {entry.path} is missing") + continue + code = production_code([path]).get(path, "") + fields = struct_fields(code, entry.struct) + if fields is None: + problems.append(f"{row.product}: {entry.path} has no struct {entry.struct}") + continue + written = fields.get(entry.field) + if written == f"{PLATFORM_CRATE}::{entry.block}": + continue + if written != entry.block: + problems.append( + f"{row.product}: {entry.path} {entry.struct}.{entry.field} is not " + f"typed {entry.block}" + ) + continue + if re.search(rf"\b(?:struct|enum|type|union)\s+{entry.block}\b", code): + problems.append( + f"{row.product}: {entry.path} declares its own {entry.block} " + "instead of using the shared block" + ) + if not imports_from_platform(code, entry.block): + problems.append( + f"{row.product}: {entry.path} does not import {entry.block} from " + f"{PLATFORM_CRATE}" + ) + return problems + + +def resolve(node: object, canonical: dict[str, object]) -> object: + while isinstance(node, dict) and set(node) == {"$ref"}: + reference = node["$ref"] + if not isinstance(reference, str) or not reference.startswith("#/$defs/"): + break + node = canonical.get(reference.removeprefix("#/$defs/")) + return node + + +def narrowing_drift( + hand: object, shared: object, canonical: dict[str, object], at: str +) -> str | None: + """Name the first way `hand` departs from the shared block, or None.""" + + shared = resolve(shared, canonical) + if not isinstance(hand, dict) or not isinstance(shared, dict): + return None if hand == shared else f"the schema differs at {at}" + for keyword, expected in shared.items(): + if keyword == "description": + continue + if keyword not in hand: + return f"{keyword} is missing at {at}" + if keyword == "properties" and isinstance(expected, dict): + written = hand[keyword] + if not isinstance(written, dict) or set(written) != set(expected): + return f"properties differ at {at}" + for name in sorted(expected): + inner = f"{at.rstrip('/')}/properties/{name}" + drift = narrowing_drift(written[name], expected[name], canonical, inner) + if drift is not None: + return drift + elif hand[keyword] != expected: + return f"{keyword} differs at {at}" + for keyword in sorted(set(hand) - set(shared)): + if keyword not in NARROWING_KEYWORDS: + return f"{keyword} is not a narrowing keyword at {at}" + return None + + +def check_hand_schemas(root: Path, row: Row, canonical: dict[str, object]) -> list[str]: + problems: list[str] = [] + for entry in row.hand_schemas: + pointer = "/" + "/".join(entry.pointer) + if entry.block not in canonical: + problems.append( + f"{row.product}: shared block {entry.block} is not in {CANONICAL_SCHEMA}" + ) + continue + try: + node: object = json.loads((root / entry.path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + problems.append(f"{row.product}: schema {entry.path} is missing or invalid") + continue + for step in entry.pointer: + node = node.get(step) if isinstance(node, dict) else None + if node is None: + problems.append(f"{row.product}: {entry.path} has no schema at {pointer}") + continue + drift = narrowing_drift(node, canonical[entry.block], canonical, "/") + if drift is not None: + problems.append( + f"{row.product}: {entry.path} at {pointer} does not match shared " + f"block {entry.block}: {drift}" + ) + return problems + + +def check_test(root: Path, row: Row, value: TestRef | Exemption, duty: str) -> list[str]: + if isinstance(value, Exemption): + return [] + path = root / value.path + if not path.is_file(): + return [f"{row.product}: test file {value.path} is missing"] + if not has_test(path.read_text(encoding="utf-8"), value.name): + return [f"{row.product}: {value.path} has no test named {value.name} ({duty})"] + return [] + + +def check(root: Path, rows: tuple[Row, ...] = ROWS) -> list[str]: + canonical = read_defs(root / CANONICAL_SCHEMA) + if canonical is None: + return [f"{CANONICAL_SCHEMA} is missing; run {GENERATOR_COMMAND}"] + problems: list[str] = [] + for row in rows: + problems += check_exemption(row, "runtime_schema", row.runtime_schema) + problems += check_exemption(row, "reference_refusal", row.reference_refusal) + problems += check_exemption(row, "authored_refusal", row.authored_refusal) + problems += check_schema(root, row, canonical) + problems += check_rust_blocks(root, row, canonical) + problems += check_hand_schemas(root, row, canonical) + problems += check_loader(root, row) + problems += check_test( + root, row, row.reference_refusal, "a *Ref field must refuse ${VAR}" + ) + problems += check_test( + root, + row, + row.authored_refusal, + "an authored project file must refuse ${VAR}", + ) + problems += check_test( + root, + row, + row.digest_mismatch, + "a package digest mismatch must report the shared expected-and-found shape", + ) + return problems + + +def check_inventory(rows: tuple[Row, ...] = ROWS) -> list[str]: + products = [row.product for row in rows] + problems = [] + duplicates = sorted({product for product in products if products.count(product) > 1}) + if duplicates: + problems.append(f"duplicate conformance rows: {', '.join(duplicates)}") + missing = sorted(EXPECTED_PRODUCTS - set(products)) + extra = sorted(set(products) - EXPECTED_PRODUCTS) + if missing: + problems.append(f"missing conformance rows: {', '.join(missing)}") + if extra: + problems.append(f"unexpected conformance rows: {', '.join(extra)}") + return problems + + +def run_generator(root: Path) -> Callable[[Path], None]: + def generate(output: Path) -> None: + command = [*GENERATOR_ARGUMENTS, str(output)] + completed = subprocess.run(command, cwd=root, check=False) + if completed.returncode != 0: + raise GeneratorFailed( + f"generator exited with status {completed.returncode}" + ) + + return generate + + +def check_canonical_freshness( + root: Path, generate: Callable[[Path], None] | None = None +) -> list[str]: + generate = generate or run_generator(root) + committed = root / CANONICAL_SCHEMA + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + try: + generate(output) + except GeneratorFailed as error: + return [f"{CANONICAL_SCHEMA} could not be regenerated: {error}"] + regenerated = output / committed.name + if not regenerated.is_file() or regenerated.read_bytes() != committed.read_bytes(): + return [f"{CANONICAL_SCHEMA} differs from its generator; run {GENERATOR_COMMAND}"] + return [] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0]) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument( + "--check-generated", + action="store_true", + help="also regenerate the canonical shared-blocks schema and compare", + ) + arguments = parser.parse_args(argv) + root = arguments.root.resolve() + problems = check_inventory() + check(root) + if arguments.check_generated and not problems: + problems = check_canonical_freshness(root) + if problems: + print("runtime configuration conformance failed:", file=sys.stderr) + for problem in problems: + print(f"- {problem}", file=sys.stderr) + return 1 + print(f"runtime configuration conformance holds for {len(ROWS)} products") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/products/platform/scripts/test_check_config_conformance.py b/products/platform/scripts/test_check_config_conformance.py new file mode 100644 index 0000000000..b3dee44b9e --- /dev/null +++ b/products/platform/scripts/test_check_config_conformance.py @@ -0,0 +1,552 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import dataclasses +import importlib.util +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "products" / "platform" / "scripts" / "check-config-conformance.py" + +spec = importlib.util.spec_from_file_location("check_config_conformance", SCRIPT) +assert spec is not None and spec.loader is not None +gate = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = gate +spec.loader.exec_module(gate) + +CANONICAL = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Registry Stack shared runtime configuration blocks", + "$defs": { + "ListenerBind": {"type": "string", "maxLength": 64}, + "ListenerConfig": { + "type": "object", + "properties": {"bind": {"$ref": "#/$defs/ListenerBind"}}, + "required": ["bind"], + }, + "PackageConfig": { + "type": "object", + "properties": {"root": {"type": "string"}}, + "required": ["root"], + }, + }, +} + +LOADER_SOURCE = """ +use registry_platform_config::RuntimeConfigLoader; + +pub fn load() { + let _ = RuntimeConfigLoader::new(ENVELOPE); +} + +#[cfg(test)] +mod tests { + #[test] + fn a_reference_refuses_substitution() {} + + #[test] + #[should_panic] + fn an_authored_file_refuses_substitution() {} + + #[test] + fn a_package_digest_mismatch_uses_the_shared_shape() {} +} +""" + + +def row(**changes: object) -> object: + base = gate.Row( + product="sample", + loader_sources=("crates/sample/src",), + runtime_schema="crates/sample/runtime.schema.json", + shared_blocks=("ListenerBind", "ListenerConfig"), + reference_refusal=gate.TestRef( + "crates/sample/src/lib.rs", "a_reference_refuses_substitution" + ), + authored_refusal=gate.TestRef( + "crates/sample/src/lib.rs", "an_authored_file_refuses_substitution" + ), + digest_mismatch=gate.TestRef( + "crates/sample/src/lib.rs", "a_package_digest_mismatch_uses_the_shared_shape" + ), + ) + return dataclasses.replace(base, **changes) + + +class ConfigConformanceFixtureTest(unittest.TestCase): + def setUp(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.write_json(gate.CANONICAL_SCHEMA, CANONICAL) + runtime = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "SampleRuntimeConfig", + "type": "object", + "$defs": { + name: CANONICAL["$defs"][name] + for name in ("ListenerBind", "ListenerConfig") + } + | {"AuditConfig": {"type": "object"}}, + } + self.write_json("crates/sample/runtime.schema.json", runtime) + self.write_text("crates/sample/src/lib.rs", LOADER_SOURCE) + + def write_json(self, relative: str, value: object) -> None: + self.write_text(relative, json.dumps(value, indent=2) + "\n") + + def write_text(self, relative: str, text: str) -> None: + path = self.root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + def edit_runtime_schema(self, edit) -> None: + path = self.root / "crates/sample/runtime.schema.json" + document = json.loads(path.read_text(encoding="utf-8")) + edit(document["$defs"]) + self.write_json("crates/sample/runtime.schema.json", document) + + def problems(self, *rows: object) -> list[str]: + return gate.check(self.root, rows or (row(),)) + + def assert_one_problem(self, expected: str, *rows: object) -> None: + problems = self.problems(*rows) + self.assertEqual(1, len(problems), problems) + self.assertIn(expected, problems[0]) + + def test_a_conforming_product_passes(self) -> None: + self.assertEqual([], self.problems()) + + def test_an_altered_shared_block_is_refused(self) -> None: + self.edit_runtime_schema( + lambda defs: defs["ListenerBind"].update({"maxLength": 4096}) + ) + self.assert_one_problem( + "sample: crates/sample/runtime.schema.json re-declares shared block " + "ListenerBind instead of embedding it unchanged" + ) + + def test_a_missing_shared_block_is_refused(self) -> None: + self.edit_runtime_schema(lambda defs: defs.pop("ListenerConfig")) + self.assert_one_problem( + "sample: crates/sample/runtime.schema.json does not embed shared " + "block ListenerConfig" + ) + + def test_an_undeclared_copy_of_a_shared_block_must_still_match(self) -> None: + self.edit_runtime_schema( + lambda defs: defs.update({"PackageConfig": {"type": "object"}}) + ) + self.assert_one_problem("re-declares shared block PackageConfig") + + def test_a_declared_block_the_platform_does_not_publish_is_refused(self) -> None: + self.assert_one_problem( + "sample: shared block AuditConfig is not in " + "products/platform/generated/runtime-config-blocks.schema.json", + row(shared_blocks=("ListenerBind", "ListenerConfig", "AuditConfig")), + ) + + def test_a_missing_runtime_schema_is_refused(self) -> None: + (self.root / "crates/sample/runtime.schema.json").unlink() + self.assert_one_problem( + "sample: runtime schema crates/sample/runtime.schema.json is missing" + ) + + def test_a_product_without_the_shared_loader_is_refused(self) -> None: + self.write_text( + "crates/sample/src/lib.rs", + LOADER_SOURCE.replace("RuntimeConfigLoader::new(ENVELOPE)", "parse()"), + ) + self.assert_one_problem( + "sample: no source under crates/sample/src reads runtime.yaml through " + "RuntimeConfigLoader" + ) + + def assert_loader_refused(self) -> None: + self.assert_one_problem( + "sample: no source under crates/sample/src reads runtime.yaml through " + "RuntimeConfigLoader" + ) + + def without_loader(self) -> str: + return LOADER_SOURCE.replace("RuntimeConfigLoader::new(ENVELOPE)", "parse()") + + def test_a_loader_call_inside_a_test_module_does_not_count(self) -> None: + self.write_text( + "crates/sample/src/lib.rs", + self.without_loader().replace( + "fn a_reference_refuses_substitution() {}", + "fn a_reference_refuses_substitution() {\n" + " let _ = RuntimeConfigLoader::new(ENVELOPE);\n }", + ), + ) + self.assert_loader_refused() + + def test_a_loader_call_in_a_comment_or_a_string_does_not_count(self) -> None: + self.write_text( + "crates/sample/src/lib.rs", + self.without_loader() + + "// RuntimeConfigLoader::new(ENVELOPE)\n" + + "/* RuntimeConfigLoader::new(ENVELOPE) */\n" + + 'const NOTE: &str = "RuntimeConfigLoader::new(ENVELOPE)";\n' + + 'const RAW: &str = r#"RuntimeConfigLoader::new(ENVELOPE)"#;\n', + ) + self.assert_loader_refused() + + def test_a_loader_call_in_a_test_only_module_file_does_not_count(self) -> None: + self.write_text( + "crates/sample/src/lib.rs", + self.without_loader() + + "#[cfg(test)]\nmod loader_tests;\n" + + '#[cfg(test)]\n#[path = "elsewhere/probe.rs"]\nmod probe;\n', + ) + call = "fn t() { let _ = RuntimeConfigLoader::new(ENVELOPE); }\n" + self.write_text("crates/sample/src/loader_tests.rs", call) + self.write_text("crates/sample/src/elsewhere/probe.rs", call) + self.assert_loader_refused() + + def test_a_loader_call_after_a_test_module_still_counts(self) -> None: + self.write_text( + "crates/sample/src/lib.rs", + self.without_loader() + + "\npub fn late() { let _ = RuntimeConfigLoader::new(ENVELOPE); }\n", + ) + self.assertEqual([], self.problems()) + + def test_a_product_still_calling_the_legacy_expansion_is_refused(self) -> None: + self.write_text( + "crates/sample/src/legacy.rs", + "fn read() { registry_platform_config::expand_config_env_vars(text) }\n", + ) + self.assert_one_problem( + "sample: crates/sample/src/legacy.rs calls expand_config_env_vars" + ) + + def test_a_missing_reference_refusal_test_is_refused(self) -> None: + self.write_text( + "crates/sample/src/lib.rs", + LOADER_SOURCE.replace("a_reference_refuses_substitution", "renamed"), + ) + self.assert_one_problem( + "sample: crates/sample/src/lib.rs has no test named " + "a_reference_refuses_substitution (a *Ref field must refuse ${VAR})" + ) + + def test_a_refusal_function_that_is_not_a_test_is_refused(self) -> None: + self.write_text( + "crates/sample/src/lib.rs", + LOADER_SOURCE.replace( + "#[test]\n #[should_panic]\n fn an_authored", + "fn an_authored", + ), + ) + self.assert_one_problem( + "has no test named an_authored_file_refuses_substitution " + "(an authored project file must refuse ${VAR})" + ) + + def test_a_missing_test_file_is_refused(self) -> None: + self.assert_one_problem( + "sample: test file crates/sample/src/missing.rs is missing", + row( + reference_refusal=gate.TestRef( + "crates/sample/src/missing.rs", "a_reference_refuses_substitution" + ) + ), + ) + + def test_a_missing_digest_mismatch_test_is_refused(self) -> None: + self.write_text( + "crates/sample/src/lib.rs", + LOADER_SOURCE.replace( + "a_package_digest_mismatch_uses_the_shared_shape", "renamed" + ), + ) + self.assert_one_problem( + "sample: crates/sample/src/lib.rs has no test named " + "a_package_digest_mismatch_uses_the_shared_shape " + "(a package digest mismatch must report the shared expected-and-found shape)" + ) + + def test_the_inventory_refuses_a_missing_product_row(self) -> None: + rows = tuple(row for row in gate.ROWS if row.product != "evidence") + self.assertEqual( + ["missing conformance rows: evidence"], gate.check_inventory(rows) + ) + + def test_an_exemption_needs_a_reason(self) -> None: + self.assert_one_problem( + "sample: the authored_refusal exemption needs a reason", + row(authored_refusal=gate.Exemption("")), + ) + + def test_an_exempt_schema_declares_no_shared_blocks(self) -> None: + self.assert_one_problem( + "sample: a product without a runtime schema declares no shared blocks", + row(runtime_schema=gate.Exemption("no generated runtime schema")), + ) + + def test_exemptions_with_reasons_pass(self) -> None: + self.assertEqual( + [], + self.problems( + row( + runtime_schema=gate.Exemption("no generated runtime schema"), + shared_blocks=(), + reference_refusal=gate.Exemption("no *Ref field"), + authored_refusal=gate.Exemption("no authored project file"), + ) + ), + ) + + def block_row(self, *blocks: object) -> object: + return row( + rust_blocks=blocks + or ( + gate.RustBlock( + "crates/sample/src/lib.rs", "SampleRuntime", "package", "PackageConfig" + ), + ) + ) + + def with_runtime_struct(self, declaration: str, imports: str = "PackageConfig") -> None: + self.write_text( + "crates/sample/src/lib.rs", + f"use registry_platform_config::{{{imports}, RuntimeConfigLoader}};\n" + + declaration + + LOADER_SOURCE.replace( + "use registry_platform_config::RuntimeConfigLoader;\n", "" + ), + ) + + RUNTIME_STRUCT = """ +#[derive(Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct SampleRuntime { + pub listener: ListenerRuntime, + /// The sealed package. + pub package: PackageConfig, + #[serde(default)] + pub limits: Limits, +} +""" + + def test_a_runtime_struct_holding_the_shared_type_passes(self) -> None: + self.with_runtime_struct(self.RUNTIME_STRUCT) + self.assertEqual([], self.problems(self.block_row())) + + def test_a_runtime_field_of_another_type_is_refused(self) -> None: + self.with_runtime_struct( + self.RUNTIME_STRUCT.replace( + "pub package: PackageConfig", "pub package: PackageRuntime" + ) + ) + self.assert_one_problem( + "sample: crates/sample/src/lib.rs SampleRuntime.package is not typed " + "PackageConfig", + self.block_row(), + ) + + def test_a_local_copy_of_a_shared_type_is_refused(self) -> None: + self.with_runtime_struct( + self.RUNTIME_STRUCT + "pub struct PackageConfig { pub root: String }\n", + imports="ListenerBind", + ) + problems = self.problems(self.block_row()) + self.assertIn( + "sample: crates/sample/src/lib.rs declares its own PackageConfig " + "instead of using the shared block", + problems, + ) + self.assertIn( + "sample: crates/sample/src/lib.rs does not import PackageConfig from " + "registry_platform_config", + problems, + ) + + def test_a_shared_type_named_by_its_full_path_passes(self) -> None: + self.with_runtime_struct( + self.RUNTIME_STRUCT.replace( + "pub package: PackageConfig", + "pub package: registry_platform_config::PackageConfig", + ), + imports="ListenerBind", + ) + self.assertEqual([], self.problems(self.block_row())) + + def test_a_missing_runtime_struct_is_refused(self) -> None: + self.assert_one_problem( + "sample: crates/sample/src/lib.rs has no struct SampleRuntime", + self.block_row(), + ) + + def test_a_runtime_struct_inside_a_test_module_does_not_count(self) -> None: + self.with_runtime_struct( + "#[cfg(test)]\nmod fixtures {\n" + self.RUNTIME_STRUCT + "}\n" + ) + self.assert_one_problem( + "sample: crates/sample/src/lib.rs has no struct SampleRuntime", + self.block_row(), + ) + + def test_a_rust_block_the_platform_does_not_publish_is_refused(self) -> None: + self.with_runtime_struct( + self.RUNTIME_STRUCT.replace("PackageConfig", "AuditConfig"), + imports="AuditConfig", + ) + self.assert_one_problem( + "sample: shared block AuditConfig is not in " + "products/platform/generated/runtime-config-blocks.schema.json", + self.block_row( + gate.RustBlock( + "crates/sample/src/lib.rs", "SampleRuntime", "package", "AuditConfig" + ) + ), + ) + + HAND_LISTENER = { + "type": "object", + "required": ["bind"], + "properties": { + "bind": { + "type": "string", + "maxLength": 64, + "description": "A socket address literal.", + "allOf": [{"pattern": ":[0-9]+$"}], + } + }, + } + + def hand_row(self) -> object: + return row( + hand_schemas=( + gate.HandSchema( + "crates/sample/hand.schema.json", + ("properties", "listener"), + "ListenerConfig", + ), + ) + ) + + def write_hand_schema(self, listener: object) -> None: + self.write_json( + "crates/sample/hand.schema.json", + {"type": "object", "properties": {"listener": listener}}, + ) + + def test_a_hand_written_block_that_only_narrows_passes(self) -> None: + self.write_hand_schema(self.HAND_LISTENER) + self.assertEqual([], self.problems(self.hand_row())) + + def assert_hand_drift(self, listener: object, detail: str) -> None: + self.write_hand_schema(listener) + self.assert_one_problem( + "sample: crates/sample/hand.schema.json at /properties/listener does " + f"not match shared block ListenerConfig: {detail}", + self.hand_row(), + ) + + def test_a_hand_written_block_with_a_changed_bound_is_refused(self) -> None: + listener = json.loads(json.dumps(self.HAND_LISTENER)) + listener["properties"]["bind"]["maxLength"] = 4096 + self.assert_hand_drift(listener, "maxLength differs at /properties/bind") + + def test_a_hand_written_block_with_an_extra_member_is_refused(self) -> None: + listener = json.loads(json.dumps(self.HAND_LISTENER)) + listener["properties"]["port"] = {"type": "integer"} + self.assert_hand_drift(listener, "properties differ at /") + + def test_a_hand_written_block_that_drops_a_requirement_is_refused(self) -> None: + listener = json.loads(json.dumps(self.HAND_LISTENER)) + listener["required"] = [] + self.assert_hand_drift(listener, "required differs at /") + + def test_a_hand_written_block_that_widens_is_refused(self) -> None: + listener = json.loads(json.dumps(self.HAND_LISTENER)) + listener["properties"]["bind"]["anyOf"] = [{"type": "integer"}] + self.assert_hand_drift( + listener, "anyOf is not a narrowing keyword at /properties/bind" + ) + + def test_a_missing_hand_written_block_is_refused(self) -> None: + self.write_json("crates/sample/hand.schema.json", {"type": "object"}) + self.assert_one_problem( + "sample: crates/sample/hand.schema.json has no schema at " + "/properties/listener", + self.hand_row(), + ) + + def test_a_missing_canonical_schema_is_refused(self) -> None: + (self.root / gate.CANONICAL_SCHEMA).unlink() + problems = self.problems() + self.assertEqual( + [f"{gate.CANONICAL_SCHEMA} is missing; run {gate.GENERATOR_COMMAND}"], + problems, + ) + + def fake_generator(self, rendered: str | None): + def generate(output: Path) -> None: + if rendered is None: + raise gate.GeneratorFailed("generator exited with status 101") + (output / Path(gate.CANONICAL_SCHEMA).name).write_text( + rendered, encoding="utf-8" + ) + + return generate + + def test_a_fresh_canonical_schema_passes(self) -> None: + committed = (self.root / gate.CANONICAL_SCHEMA).read_text(encoding="utf-8") + self.assertEqual( + [], + gate.check_canonical_freshness(self.root, self.fake_generator(committed)), + ) + + def test_a_stale_canonical_schema_is_refused(self) -> None: + self.assertEqual( + [ + f"{gate.CANONICAL_SCHEMA} differs from its generator; run " + f"{gate.GENERATOR_COMMAND}" + ], + gate.check_canonical_freshness(self.root, self.fake_generator("{}\n")), + ) + + def test_a_failing_generator_is_refused(self) -> None: + self.assertEqual( + [ + f"{gate.CANONICAL_SCHEMA} could not be regenerated: generator " + "exited with status 101" + ], + gate.check_canonical_freshness(self.root, self.fake_generator(None)), + ) + + def test_main_reports_problems_and_fails(self) -> None: + self.edit_runtime_schema(lambda defs: defs.pop("ListenerConfig")) + completed = subprocess.run( + (sys.executable, str(SCRIPT), "--root", str(self.root)), + check=False, + capture_output=True, + text=True, + ) + # The fixture root carries no product rows of its own, so every real + # row reports its missing inputs; the exit status is what matters. + self.assertEqual(1, completed.returncode) + self.assertIn("runtime configuration conformance failed", completed.stderr) + + +class ConfigConformanceRepositoryTest(unittest.TestCase): + def test_the_repository_conforms(self) -> None: + self.assertEqual([], gate.check(ROOT, gate.ROWS)) + + def test_every_row_names_a_distinct_product(self) -> None: + products = [entry.product for entry in gate.ROWS] + self.assertEqual(len(products), len(set(products))) + + +if __name__ == "__main__": + unittest.main() diff --git a/products/relay-v2/CHANGELOG.md b/products/relay-v2/CHANGELOG.md index 5cd9abb620..3b9f55d2b4 100644 --- a/products/relay-v2/CHANGELOG.md +++ b/products/relay-v2/CHANGELOG.md @@ -54,6 +54,114 @@ Migration: has no path to prove. 5. Reseal packages; the audit event schema artifact changed. +### BREAKING: sealed packages use the shared package format + +`relayctl package` writes the shared Registry Stack package format: a +`SHA256SUMS` file, one `sha256sum` line per file sorted by path, in place of +the `relay-package.json` manifest and its `relay.registrystack.org/package/v1alpha3` +version. The package digest, the `sha256:` digest of `SHA256SUMS`, replaces +`packageRevision`, and `package.expectedDigest` pins it. `--revision TEXT` +records one free-text line in a `REVISION` file the digest covers, and +`--dry-run` reports the digest without writing. The JSON report's +`details.manifest` is replaced by `details.package`, which states the package +digest, the contract revision, the source schema fingerprints, every file with +its digest and size, and the derived exposure of every generated artifact. + +At startup `relay` refuses a changed, missing, or extra package file by name, +and refuses a `package.root` that still holds `relay-package.json`, naming +`relayctl package` and never the directory. The package no longer stores +artifact visibility, media types, or operation bindings; the runtime derives +them from the compiled Registry, as it already did to check them. A failed +`relayctl package` write now removes the directory it created. + +Migration: rebuild every package with `relayctl package`, and replace a +pinned `package.expectedDigest` with the package digest the new report states. + +### BREAKING: read runtime.yaml through the shared configuration loader + +`relay` reads its deployment binding through the shared Registry Stack runtime +configuration loader, with the shared listener, package, secret-provider, and +OIDC blocks. There is no compatibility reader: every removed key is refused +with a diagnostic naming the field and its replacement, never its value. + +Before: + +```yaml +apiVersion: relay.registrystack.org/v2alpha1 +kind: RelayRuntime +server: {bind: "127.0.0.1:8080"} +packagePath: package +authentication: + issuer: + id: institutional-issuer + discoveryUrl: https://identity.example.invalid/.well-known/openid-configuration + audience: relay-registry + tokenTypes: [at+jwt] + algorithms: [ES256] +audit: {path: var/audit.jsonl} +``` + +After: + +```yaml +apiVersion: registry.registrystack.org/relay-runtime/v1alpha1 +kind: RelayRuntimeConfig +listener: {bind: "127.0.0.1:8080"} +package: {root: /srv/relay/package} +secretProviders: + file: {root: /run/secrets/relay} +authentication: + oidc: + issuer: https://identity.example.invalid + audience: relay-registry + tokenTypes: [at+jwt] + algorithms: [ES256] +audit: {path: var/audit.jsonl} +``` + +Migration: + +1. Replace the envelope with `apiVersion: + registry.registrystack.org/relay-runtime/v1alpha1` and `kind: + RelayRuntimeConfig`. +2. Move `server.bind` to `listener.bind`, and `packagePath` to `package.root` + as an absolute path. Optionally pin the package with + `package.expectedDigest`, its `sha256:` package digest; any other package + at that path is refused. +3. Declare `secretProviders`. A `secret:file/` reference now resolves under + `secretProviders.file.root` instead of the runtime file's directory; move + the secret files there. A `secret:env/` reference needs + `secretProviders.environment: {}`. +4. Keep the `audit` block in the shared destination shape described above; + `audit.sink` and `audit.integrityKeyRef` are refused with a diagnostic + naming `audit.path`. `cursor.integrityKeyRef` is unchanged. +5. Replace `authentication.issuer` with `authentication.oidc`. `issuer` is the + exact token `iss` value; `id`, `trustedIssuer`, `discoveryUrl`, and + `jwksUrl` are gone. The key source is `jwksSource`: the default `kind: + discovery` reads the issuer's own `/.well-known/openid-configuration`, and + `kind: uri` with `uri` binds one exact JWKS endpoint, which may sit on + another host. A discovery document served from a different origin than the + issuer is no longer configurable; use `kind: uri` with the JWKS URL it + named. `kind: static` is refused. Omit `authentication.oidc` when every + access rule is public. +6. `relay check` and `relay serve` take the required absolute + `--runtime-config `. The `--runtime` flag, the `RELAY_RUNTIME` + environment variable, and the `/etc/relay/runtime.yaml` default for `relay + check` are gone. The container image passes `--runtime-config + /etc/relay/runtime.yaml` in its default command. + +Values other than secret references may use `${VAR}` and `${VAR:-default}` +substitution; a field whose name ends in `Ref` refuses it. The runtime file +must be absolute, free of symbolic links, at most 1 MiB, and owned by root or +the service identity with no group- or world-writable ancestor other than a +root-owned sticky directory. `relayctl` and the editor check the runtime with +an empty environment, so a substitution without a default is reported there. +Substitution applies to `runtime.yaml` only: an environment expression in the +authored `registry.yaml` is refused as `contract.environment_expression` with +the field that holds it. +`relayctl init` writes a starter whose `package.root` reads +`RELAY_PACKAGE_ROOT` and defaults to `/srv/relay/package`. + ## v0.26.0 - 2026-09-03 ### BREAKING: adopt Registry Record profile v1 diff --git a/products/relay-v2/CONFIGURATION-EXAMPLES.md b/products/relay-v2/CONFIGURATION-EXAMPLES.md index f82d281223..42435090e8 100644 --- a/products/relay-v2/CONFIGURATION-EXAMPLES.md +++ b/products/relay-v2/CONFIGURATION-EXAMPLES.md @@ -16,7 +16,7 @@ portability tooling, not a Version one runtime input. The intended boundaries are firmer than the syntax: - `RegistryContract` is governed, versioned, compiled and sealed by `relayctl package`, verified at startup, and cannot be overridden by runtime configuration; -- `RelayRuntime` binds deployment-local paths, listeners, token issuers, and audit storage without changing resources, operations, disclosure, or semantics; +- `RelayRuntimeConfig` binds deployment-local paths, listeners, token issuers, and audit storage without changing resources, operations, disclosure, or semantics; - SQLite views and columns are source bindings, while resources and properties are the public model; - one contract describes one Registry; each resource is a Record type within it; - every resource declares required `datasetIdentifier` and `entityTypeIdentifier` values beside `id`; Relay never infers either value from the resource id, route, view, or semantic class; @@ -307,17 +307,20 @@ metadataVisibility: processing: operation-bound --- -apiVersion: relay.registrystack.org/v2alpha1 -kind: RelayRuntime -server: {bind: "127.0.0.1:8080"} -packagePath: /srv/relay/social-assistance-package +apiVersion: registry.registrystack.org/relay-runtime/v1alpha1 +kind: RelayRuntimeConfig +listener: {bind: "127.0.0.1:8080"} +package: {root: /srv/relay/social-assistance-package} +secretProviders: + file: {root: /run/secrets/relay} sources: assistance: {path: /srv/registries/social-assistance.sqlite} authentication: - issuer: - id: institutional-authorization-server - trustedIssuer: https://identity.example.invalid - discoveryUrl: https://identity-transport.example.invalid/.well-known/openid-configuration + oidc: + issuer: https://identity.example.invalid + jwksSource: + kind: uri + uri: https://identity-transport.example.invalid/jwks audience: relay-social-assistance tokenTypes: [at+jwt] algorithms: [ES256] @@ -330,17 +333,17 @@ limits: {requestTimeoutMilliseconds: 1500, concurrentQueries: 16} quotas: {requestsPerMinute: 120, burst: 20} ``` -`trustedIssuer` is the exact JWT `iss` value Relay accepts. `discoveryUrl` is -the operator-selected metadata transport and may use a different hostname; -the returned discovery document must still declare the exact trusted issuer. -Existing runtimes may omit `trustedIssuer` only when `discoveryUrl` is the -canonical issuer plus `/.well-known/openid-configuration`. As a controlled -alternative, set `trustedIssuer` with `jwksUrl` and omit `discoveryUrl`; Relay -then binds that exact key endpoint directly while preserving exact token issuer -validation. Defining both transports or neither fails startup. Run `relay -check --runtime ` before routing traffic to prove the sealed -package, source, audit, secret, and issuer key transport without binding the -listener. +`authentication.oidc.issuer` is the exact JWT `iss` value Relay accepts. +`jwksSource` says where the signing keys come from. The default, `kind: +discovery`, reads the issuer plus `/.well-known/openid-configuration`, and the +returned discovery document must declare the exact issuer. `kind: uri` binds +one exact key endpoint instead, which may use a different hostname, while token +issuer validation stays exact. `kind: static` is refused. A `secret:file/` +reference resolves under `secretProviders.file.root`, and a `secret:env/` +reference needs `secretProviders.environment: {}`. Run `relay check +--runtime-config /etc/relay/runtime.yaml` before routing traffic to prove the +sealed package, source, audit, secret, and issuer key transport without binding +the listener. `audit` takes the shape every Registry Stack product shares. `destination` is `file` (the default) or `stdout`. A `file` destination needs an absolute @@ -542,13 +545,14 @@ metadataVisibility: processing: public --- -apiVersion: relay.registrystack.org/v2alpha1 -kind: RelayRuntime -server: {bind: "127.0.0.1:8080"} -packagePath: /srv/relay/business-register-package +apiVersion: registry.registrystack.org/relay-runtime/v1alpha1 +kind: RelayRuntimeConfig +listener: {bind: "127.0.0.1:8080"} +package: {root: /srv/relay/business-register-package} +secretProviders: + file: {root: /run/secrets/relay} sources: companies: {path: /srv/registries/business-register.sqlite} -authentication: {issuer: null} audit: destination: file path: /var/lib/relay/audit/business-register.jsonl @@ -871,16 +875,17 @@ metadataVisibility: processing: operation-bound --- -apiVersion: relay.registrystack.org/v2alpha1 -kind: RelayRuntime -server: {bind: "127.0.0.1:8080"} -packagePath: /srv/relay/civil-events-package +apiVersion: registry.registrystack.org/relay-runtime/v1alpha1 +kind: RelayRuntimeConfig +listener: {bind: "127.0.0.1:8080"} +package: {root: /srv/relay/civil-events-package} +secretProviders: + file: {root: /run/secrets/relay} sources: events: {path: /srv/registries/civil-events.sqlite} authentication: - issuer: - id: civil-registry-authorization-server - discoveryUrl: https://identity.example.invalid/.well-known/openid-configuration + oidc: + issuer: https://identity.example.invalid audience: relay-civil-events tokenTypes: [at+jwt] algorithms: [ES256] @@ -1239,16 +1244,15 @@ audit audit.destination audit.path authentication -authentication.issuer -authentication.issuer.algorithms -authentication.issuer.algorithms[] -authentication.issuer.audience -authentication.issuer.discoveryUrl -authentication.issuer.id -authentication.issuer.jwksUrl -authentication.issuer.tokenTypes -authentication.issuer.tokenTypes[] -authentication.issuer.trustedIssuer +authentication.oidc +authentication.oidc.algorithms +authentication.oidc.algorithms[] +authentication.oidc.audience +authentication.oidc.issuer +authentication.oidc.jwksSource +authentication.oidc.jwksSource.kind +authentication.oidc.tokenTypes +authentication.oidc.tokenTypes[] cursor cursor.integrityKeyRef cursor.maximumAgeSeconds @@ -1256,12 +1260,15 @@ kind limits limits.concurrentQueries limits.requestTimeoutMilliseconds -packagePath +listener +listener.bind +package +package.root quotas quotas.burst quotas.requestsPerMinute -server -server.bind +secretProviders +secretProviders.environment shutdown shutdown.gracePeriodMilliseconds sources diff --git a/products/relay-v2/IMPLEMENTATION.md b/products/relay-v2/IMPLEMENTATION.md index bd93ca8bb2..45d2574afc 100644 --- a/products/relay-v2/IMPLEMENTATION.md +++ b/products/relay-v2/IMPLEMENTATION.md @@ -123,10 +123,11 @@ source requiredness and full SHACL cardinality, including every Point whether or not a particular access profile discloses it. Semantic generation includes the Point once as its property and never duplicates carrier metadata. -`RelayRuntime` is a separate strict deployment file. It binds listener, -`packagePath`, SQLite paths, at most one issuer and audience, secrets, cursor -key, audit destination, timeouts, concurrency, quotas, and -shutdown. +`RelayRuntimeConfig` is a separate strict deployment file read by the shared +platform runtime loader. It binds `listener.bind`, `package.root`, SQLite +paths, at most one `authentication.oidc` issuer and audience, the +`secretProviders` that resolve secret references, cursor key, audit destination, +timeouts, concurrency, quotas, and shutdown. It cannot add or weaken a resource, operation, disclosure, access rule, classification, semantic mapping, or metadata visibility decision. The audit block is mandatory and uses the shape every Registry Stack product @@ -141,7 +142,8 @@ only production packaging path. It creates a deterministic sealed directory with: ```text -relay-package.json +SHA256SUMS +REVISION (only with --revision) registry.yaml governed/... compiled/registry.json @@ -159,19 +161,21 @@ identifiers only for operations with a public access profile. It emits one distinct binding identity per exact semantic-class and operation-family pair, so independent resources cannot become a false combined capability. -`relay-package.json` is canonical JSON with package version -`relay.registrystack.org/package/v1alpha3`, containing -`packageRevision`, `contractRevision`, the expected SQLite schema fingerprint, -the generated-artifact inventory and operation bindings, and for every relative -regular file its path, size, SHA-256 digest, media type, visibility, and -generated/authored status. `compiled/registry.json` is the canonical compiled -runtime plan produced by the shared compiler. +The directory is in the shared Registry Stack package format: +`SHA256SUMS` lists the SHA-256 digest of every other file, and the package +digest is the `sha256:` digest of `SHA256SUMS`. `package.expectedDigest` pins +that digest, and `relayctl package --dry-run` reports it without writing. +`compiled/registry.json` is the canonical compiled runtime plan produced by the +shared compiler, and it carries the expected SQLite schema fingerprint and +observed schema of every source. The package stores no artifact inventory, +visibility, media type, or operation binding: the runtime derives all of them +from the compiled Registry, and the package report states them for review. Every operation-bound generated artifact has an explicit `accessBinding`: `{kind: access-profile, identifier: ...}` for a Record access profile or -`{kind: fixed-operation}` for a statistical structure. `PackageArtifact` -contains no `accessProfileIdentifier`; the separate -`operationArtifactBindings` entries retain that field for their existing -Record-operation binding contract. +`{kind: fixed-operation}` for a statistical structure. A reported artifact +contains no `accessProfileIdentifier`. +A directory that still carries the retired `relay-package.json` manifest is +refused with a message naming `relayctl package`. References cannot escape the directory and symlinks are rejected. The runtime file, sealed package tree, and their ancestry must be owned by root or the Relay service user and must not be writable by another account; only a @@ -181,18 +185,19 @@ deployment bindings; Relay captures a snapshot digest or explicitly reports an unversioned live source. Snapshot execution verifies the captured digest before and after every statement; operators still provide external immutability, preferably a read-only mount, because no process can exclude a privileged -change-and-restore entirely between those checks. `relay serve --runtime ` resolves the sealed -package only from the runtime's `packagePath`; it never accepts a mutable +change-and-restore entirely between those checks. `relay serve --runtime-config ` resolves the sealed +package only from the runtime's absolute `package.root`; it never accepts a mutable authoring project or loose contract file. The complete governed file closure is captured into memory with file count, size, path, symlink, and permission bounds before parsing. Canonical typed inputs produce `contractRevision`. Compilation and artifact generation are -atomic packaging operations. Startup verifies canonical compiled bytes, source -schema bindings, governed-file and artifact digests, and operation-artifact -bindings. It recompiles the captured inputs solely to require exact equality -with the packaged runtime plan, then activates the packaged artifacts without -regenerating them. There is no hot reload, partial activation, overlay, +atomic packaging operations. Startup verifies `SHA256SUMS` and refuses a +changed, missing, or extra file by name, re-reads every file it uses against its +verified digest, requires the file set to equal what the compiled Registry +names, and checks canonical compiled bytes and source schema bindings. It +recompiles the captured inputs and regenerates the artifacts solely to require +exact equality with the packaged runtime plan and artifact bytes. There is no hot reload, partial activation, overlay, fallback, or remote vocabulary fetch. Registry Manifest projection is deferred portability tooling. Source columns, @@ -257,7 +262,7 @@ Editor authoring remains another caller of the compiler, not another compiler. `registry.yaml`, holds the bounded governed closure in memory, and passes the current buffers to `registry-relay-v2::authoring`. It never observes SQLite or source values. `relayctl tooling editor` embeds reproducible JSON Schemas -derived from the strict `RegistryContract` and `RelayRuntime` Rust types and +derived from the strict `RegistryContract` and `RelayRuntimeConfig` Rust types and writes collision-safe project-local mappings for VS Code and Zed. ### Registry Record and response shapes diff --git a/products/relay-v2/README.md b/products/relay-v2/README.md index daca4e205c..613ec73b2e 100644 --- a/products/relay-v2/README.md +++ b/products/relay-v2/README.md @@ -70,12 +70,18 @@ tests do not require Docker. problem inventory used by `registry-relay-client`; it has no live deployment or fixture dependency. -For an assembled deployment, `relay check --runtime ` performs -the complete startup preparation without taking the listener socket. It -verifies the sealed package, observed SQLite source, audit writer, secrets, and -configured issuer discovery or JWKS transport. Exact token `iss` validation is -bound to `authentication.issuer.trustedIssuer` when that explicit field is -present, independent of the transport hostname. +For an assembled deployment, `relay check --runtime-config +/etc/relay/runtime.yaml` performs the complete startup preparation without +taking the listener socket. The path is required and absolute; there is no +default and no environment variable. It verifies the runtime envelope +(`registry.registrystack.org/relay-runtime/v1alpha1`, kind +`RelayRuntimeConfig`), the sealed package at `package.root` and its optional +`package.expectedDigest` pin, the observed SQLite source, the audit writer, +secrets through the declared `secretProviders`, and the issuer key source. +Exact token `iss` validation is bound to `authentication.oidc.issuer`, +independent of the `jwksSource` transport hostname. A removed key such as +`server`, `packagePath`, `audit.integrityKeyRef`, or `authentication.issuer` is +refused with a diagnostic naming its replacement. Adding `--require-audit-under ` proves that the configured audit file resolves at or below a directory the deployment declares persistent. diff --git a/products/relay-v2/acceptance/business-registry/runtime.yaml b/products/relay-v2/acceptance/business-registry/runtime.yaml index 358b1af6a3..a02fcdd0ca 100644 --- a/products/relay-v2/acceptance/business-registry/runtime.yaml +++ b/products/relay-v2/acceptance/business-registry/runtime.yaml @@ -1,15 +1,19 @@ -apiVersion: relay.registrystack.org/v2alpha1 -kind: RelayRuntime -server: +apiVersion: registry.registrystack.org/relay-runtime/v1alpha1 +kind: RelayRuntimeConfig +listener: bind: 127.0.0.1:18082 -packagePath: package +package: + root: /srv/relay/package +secretProviders: + environment: {} sources: companies: path: fixture.sqlite authentication: - issuer: - id: synthetic-business-issuer - discoveryUrl: https://identity.example.invalid/.well-known/openid-configuration + oidc: + issuer: https://identity.example.invalid + jwksSource: + kind: discovery audience: relay-business-registry tokenTypes: [at+jwt] algorithms: [ES256] diff --git a/products/relay-v2/acceptance/civil-event/runtime.yaml b/products/relay-v2/acceptance/civil-event/runtime.yaml index 71e1c7b9c4..736464e6ff 100644 --- a/products/relay-v2/acceptance/civil-event/runtime.yaml +++ b/products/relay-v2/acceptance/civil-event/runtime.yaml @@ -1,15 +1,19 @@ -apiVersion: relay.registrystack.org/v2alpha1 -kind: RelayRuntime -server: +apiVersion: registry.registrystack.org/relay-runtime/v1alpha1 +kind: RelayRuntimeConfig +listener: bind: 127.0.0.1:18083 -packagePath: package +package: + root: /srv/relay/package +secretProviders: + environment: {} sources: events: path: fixture.sqlite authentication: - issuer: - id: institutional-issuer - discoveryUrl: https://issuer.example.invalid/.well-known/openid-configuration + oidc: + issuer: https://issuer.example.invalid + jwksSource: + kind: discovery audience: relay-civil-events tokenTypes: [at+jwt] algorithms: [ES256] diff --git a/products/relay-v2/acceptance/labour-statistics/runtime.yaml b/products/relay-v2/acceptance/labour-statistics/runtime.yaml index 9206ba5d34..3a10affd9a 100644 --- a/products/relay-v2/acceptance/labour-statistics/runtime.yaml +++ b/products/relay-v2/acceptance/labour-statistics/runtime.yaml @@ -1,15 +1,19 @@ -apiVersion: relay.registrystack.org/v2alpha1 -kind: RelayRuntime -server: +apiVersion: registry.registrystack.org/relay-runtime/v1alpha1 +kind: RelayRuntimeConfig +listener: bind: 127.0.0.1:18084 -packagePath: package +package: + root: /srv/relay/package +secretProviders: + environment: {} sources: labour-statistics: path: fixture.sqlite authentication: - issuer: - id: synthetic-statistics-issuer - discoveryUrl: https://identity.example.invalid/.well-known/openid-configuration + oidc: + issuer: https://identity.example.invalid + jwksSource: + kind: discovery audience: relay-labour-statistics tokenTypes: [at+jwt] algorithms: [EdDSA] diff --git a/products/relay-v2/acceptance/social-assistance/runtime.yaml b/products/relay-v2/acceptance/social-assistance/runtime.yaml index ce33f7fe3a..2ee28fb1e3 100644 --- a/products/relay-v2/acceptance/social-assistance/runtime.yaml +++ b/products/relay-v2/acceptance/social-assistance/runtime.yaml @@ -1,15 +1,19 @@ -apiVersion: relay.registrystack.org/v2alpha1 -kind: RelayRuntime -server: +apiVersion: registry.registrystack.org/relay-runtime/v1alpha1 +kind: RelayRuntimeConfig +listener: bind: 127.0.0.1:18081 -packagePath: package +package: + root: /srv/relay/package +secretProviders: + environment: {} sources: assistance: path: fixture.sqlite authentication: - issuer: - id: synthetic-external-issuer - discoveryUrl: https://identity.example.invalid/.well-known/openid-configuration + oidc: + issuer: https://identity.example.invalid + jwksSource: + kind: discovery audience: relay-social-assistance tokenTypes: [at+jwt] algorithms: [ES256] diff --git a/products/relay-v2/contracts/generated-baselines.yaml b/products/relay-v2/contracts/generated-baselines.yaml index 1944e4e684..9d1aaae4e0 100644 --- a/products/relay-v2/contracts/generated-baselines.yaml +++ b/products/relay-v2/contracts/generated-baselines.yaml @@ -2,1511 +2,1397 @@ schemaVersion: relay.registrystack.org/generated-baselines/v1alpha1 product: relay-v2 projects: social-assistance: - packageRevision: sha256:23701f8acdb93dff81ea734087c1e9e2d2bad866b3f2f4ba4513f5f545c95dd8 + packageDigest: sha256:263cf7a0276b38c015211b3c2b847723f7ce49dbf1d229924ae275c45852e619 contractRevision: sha256:d05828ef947a78e3716e808b40fa60952c2179291c0d68402e3f9ce83b299454 sourceSchemaFingerprints: assistance: sha256:936a90a03d06be67a76226d6999a830c04f6604a3ff8b340a62fdd378d8c6d91 artifacts: - - accessBinding: - identifier: caseworker - kind: access-profile - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-capability + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-capability + path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker.capability.json mediaType: application/json + visibility: operation-bound operationIdentifier: assistance-enrolment.lookup.by-case-and-person - path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker.capability.json + accessBinding: + kind: access-profile + identifier: caseworker sha256: sha256:ab97f78557672e8fc84daa2f6befa95352e6d1bb832b591a5d6d2f3ab4ec6c13 - visibility: operation-bound - - accessBinding: null - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-classifications + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-classifications + path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker.classifications.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker.classifications.json + accessBinding: null sha256: sha256:57a918902073fdd17ca48974e6e7d153eed11a9f33409b39e193f6c5030f53f8 - visibility: operator-only - - accessBinding: - identifier: caseworker - kind: access-profile - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-context - mediaType: application/ld+json - operationIdentifier: assistance-enrolment.lookup.by-case-and-person + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-context path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker.context.jsonld - sha256: sha256:1dab07096684ecb2f75e7700edb3de4ccf297735f92321348dfa6f85c5912dd6 + mediaType: application/ld+json visibility: operation-bound - - accessBinding: - identifier: caseworker - kind: access-profile - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-processing - mediaType: application/json operationIdentifier: assistance-enrolment.lookup.by-case-and-person + accessBinding: + kind: access-profile + identifier: caseworker + sha256: sha256:1dab07096684ecb2f75e7700edb3de4ccf297735f92321348dfa6f85c5912dd6 + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-processing path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker.processing.json - sha256: sha256:af17652b596290134bb38c594b14f7dac2b9caa6b9bef1119511b2f399053e3e + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: caseworker - kind: access-profile - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-schema - mediaType: application/schema+json operationIdentifier: assistance-enrolment.lookup.by-case-and-person + accessBinding: + kind: access-profile + identifier: caseworker + sha256: sha256:af17652b596290134bb38c594b14f7dac2b9caa6b9bef1119511b2f399053e3e + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-schema path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker.schema.json - sha256: sha256:ede0df8f6febdb0c4db8f200d360c8d6a8d19cbf96a0c87b21c35d34a0a97bd9 + mediaType: application/schema+json visibility: operation-bound - - accessBinding: - identifier: caseworker - kind: access-profile - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-shacl - mediaType: text/turtle operationIdentifier: assistance-enrolment.lookup.by-case-and-person + accessBinding: + kind: access-profile + identifier: caseworker + sha256: sha256:ede0df8f6febdb0c4db8f200d360c8d6a8d19cbf96a0c87b21c35d34a0a97bd9 + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-shacl path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker.shacl.ttl - sha256: sha256:dd1a21bd1517943b1569efa6e74d1c08c9840df256f8382e440d4436ce30c179 + mediaType: text/turtle visibility: operation-bound - - accessBinding: - identifier: caseworker - kind: access-profile - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-vocabulary - mediaType: application/ld+json operationIdentifier: assistance-enrolment.lookup.by-case-and-person + accessBinding: + kind: access-profile + identifier: caseworker + sha256: sha256:dd1a21bd1517943b1569efa6e74d1c08c9840df256f8382e440d4436ce30c179 + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker-vocabulary path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-caseworker.vocabulary.jsonld - sha256: sha256:d894822eb794725509df894466e19f0e96caa99e8a612a358cb1dc25b71a1a86 + mediaType: application/ld+json visibility: operation-bound - - accessBinding: - identifier: limited + operationIdentifier: assistance-enrolment.lookup.by-case-and-person + accessBinding: kind: access-profile - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-capability + identifier: caseworker + sha256: sha256:d894822eb794725509df894466e19f0e96caa99e8a612a358cb1dc25b71a1a86 + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-capability + path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-limited.capability.json mediaType: application/json + visibility: operation-bound operationIdentifier: assistance-enrolment.lookup.by-case-and-person - path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-limited.capability.json + accessBinding: + kind: access-profile + identifier: limited sha256: sha256:1bef469fc0c563d0f277a870309cca7f7a1ccafb009bc4214a7067675a571dd6 - visibility: operation-bound - - accessBinding: null - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-classifications + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-classifications + path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-limited.classifications.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-limited.classifications.json + accessBinding: null sha256: sha256:cab6d216d1ce7e68ff149eb73971ad317977dbd1108c2b4b7e00e00b1f5db4de - visibility: operator-only - - accessBinding: - identifier: limited - kind: access-profile - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-context - mediaType: application/ld+json - operationIdentifier: assistance-enrolment.lookup.by-case-and-person + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-context path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-limited.context.jsonld - sha256: sha256:9f1929c170672728f3dac8e068dd132ecb0ecc86af8e85d7c56c0503ad5d1b3d + mediaType: application/ld+json visibility: operation-bound - - accessBinding: - identifier: limited - kind: access-profile - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-processing - mediaType: application/json operationIdentifier: assistance-enrolment.lookup.by-case-and-person + accessBinding: + kind: access-profile + identifier: limited + sha256: sha256:9f1929c170672728f3dac8e068dd132ecb0ecc86af8e85d7c56c0503ad5d1b3d + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-processing path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-limited.processing.json - sha256: sha256:3b16cd7620010e103eb2da975a9036d89ffc0d68ffc23b26374a4b540e17bdd0 + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: limited - kind: access-profile - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-schema - mediaType: application/schema+json operationIdentifier: assistance-enrolment.lookup.by-case-and-person + accessBinding: + kind: access-profile + identifier: limited + sha256: sha256:3b16cd7620010e103eb2da975a9036d89ffc0d68ffc23b26374a4b540e17bdd0 + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-schema path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-limited.schema.json - sha256: sha256:cd8fe63d85448dacb61ab52bfec85cdace0976f0f33f67edee639eff75b6ca60 + mediaType: application/schema+json visibility: operation-bound - - accessBinding: - identifier: limited - kind: access-profile - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-shacl - mediaType: text/turtle operationIdentifier: assistance-enrolment.lookup.by-case-and-person + accessBinding: + kind: access-profile + identifier: limited + sha256: sha256:cd8fe63d85448dacb61ab52bfec85cdace0976f0f33f67edee639eff75b6ca60 + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-shacl path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-limited.shacl.ttl - sha256: sha256:a7d40f9b07a9d179524694069ac36add9e001678cd8045c7aefaac1305b9372e + mediaType: text/turtle visibility: operation-bound - - accessBinding: - identifier: limited + operationIdentifier: assistance-enrolment.lookup.by-case-and-person + accessBinding: kind: access-profile - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-vocabulary + identifier: limited + sha256: sha256:a7d40f9b07a9d179524694069ac36add9e001678cd8045c7aefaac1305b9372e + - id: assistance-enrolment--lookup-by-case-and-person--access-profile-limited-vocabulary + path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-limited.vocabulary.jsonld mediaType: application/ld+json + visibility: operation-bound operationIdentifier: assistance-enrolment.lookup.by-case-and-person - path: generated/artifacts/assistance-enrolment--lookup-by-case-and-person--access-profile-limited.vocabulary.jsonld + accessBinding: + kind: access-profile + identifier: limited sha256: sha256:d856a45b101cce510ad7ff1d1773f0326e69cd4b52d5ea00fa6534f74d79a881 - visibility: operation-bound - - accessBinding: null - id: assistance-enrolment-classification + - id: assistance-enrolment-classification + path: generated/artifacts/assistance-enrolment.classifications.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/assistance-enrolment.classifications.json + accessBinding: null sha256: sha256:32707dfb3d94080914c914d5bded55741758dfd4e1f2eef49c89e4376042eace - visibility: operator-only - - accessBinding: null - id: assistance-enrolment-codelist-0 + - id: assistance-enrolment-codelist-0 + path: generated/artifacts/assistance-enrolment.codelist-0.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/assistance-enrolment.codelist-0.schema.json + accessBinding: null sha256: sha256:a836883fac30ae1cacbd657d7429ff08f9bfff1a3b8abea0bcc4fa5401a7f200 - visibility: operator-only - - accessBinding: null - id: assistance-enrolment-codelist-1 + - id: assistance-enrolment-codelist-1 + path: generated/artifacts/assistance-enrolment.codelist-1.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/assistance-enrolment.codelist-1.schema.json + accessBinding: null sha256: sha256:4dd49c40c44f8acbd56f319d4af5c9b48ffee24e5b0bd267f0c6f4833adc73d1 - visibility: operator-only - - accessBinding: null - id: assistance-enrolment-codelist-2 + - id: assistance-enrolment-codelist-2 + path: generated/artifacts/assistance-enrolment.codelist-2.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/assistance-enrolment.codelist-2.schema.json + accessBinding: null sha256: sha256:e42dfbcab45a66032d126e0f203523ae44a6bc034278f2ce222f96f1ff0a78f0 - visibility: operator-only - - accessBinding: null - id: assistance-enrolment-full-schema + - id: assistance-enrolment-full-schema + path: generated/artifacts/assistance-enrolment.full.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/assistance-enrolment.full.schema.json + accessBinding: null sha256: sha256:7cf7d6d573511e1e73625834b5ce3cbd852947fb60f55d5f0d7ce4777dc64b03 - visibility: operator-only - - accessBinding: null - id: assistance-enrolment-full-shacl + - id: assistance-enrolment-full-shacl + path: generated/artifacts/assistance-enrolment.full.shacl.ttl mediaType: text/turtle + visibility: operator-only operationIdentifier: null - path: generated/artifacts/assistance-enrolment.full.shacl.ttl + accessBinding: null sha256: sha256:cbc3c4e9f0313955b38c7451859639c84618d4c105bc8936be9d1e4e1efefae3 - visibility: operator-only - - accessBinding: null - id: assistance-enrolment-full-vocabulary + - id: assistance-enrolment-full-vocabulary + path: generated/artifacts/assistance-enrolment.full.vocabulary.jsonld mediaType: application/ld+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/assistance-enrolment.full.vocabulary.jsonld + accessBinding: null sha256: sha256:ce35a8374c9a8758f7eb42ed86f77503f2f371ba3da16d17eaa2df1d7a320f92 - visibility: operator-only - - accessBinding: null - id: assistance-enrolment-processing-full + - id: assistance-enrolment-processing-full + path: generated/artifacts/assistance-enrolment.processing.full.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/assistance-enrolment.processing.full.json + accessBinding: null sha256: sha256:b3806fac8892ef081c3d8e26ca475fb37c3d318302f593b25828c20110a1f7b5 - visibility: operator-only - - accessBinding: null - id: audit-event-schema + - id: audit-event-schema + path: generated/artifacts/audit-event.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/audit-event.schema.json + accessBinding: null sha256: sha256:41358576a5fcbf4a6182f22b2213f26dadd762e3a1c6d76ea1c0948106b7e6e2 - visibility: operator-only - - accessBinding: null - id: capability-inventory-full + - id: capability-inventory-full + path: generated/artifacts/capabilities.full.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/capabilities.full.json + accessBinding: null sha256: sha256:de54fd7af84cbec361d531837366d2c2094874cf1ca8b2c73da09b55e3129f7a - visibility: operator-only - - accessBinding: null - id: capability-inventory + - id: capability-inventory + path: generated/artifacts/capabilities.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/artifacts/capabilities.json + accessBinding: null sha256: sha256:d71f00d4194279a5206ac702196884750ca13f6fdc31d98d94913eec3fab0ea0 - visibility: public - - accessBinding: null - id: discovery-description + - id: discovery-description + path: generated/artifacts/discovery.jsonld mediaType: application/ld+json;profile="https://registrystack.org/discovery/profile/v1alpha1" + visibility: public operationIdentifier: null - path: generated/artifacts/discovery.jsonld + accessBinding: null sha256: sha256:dcacbf20520b93942b8132cdc629c19ca1f53a8402f555ed4c62ed958b956138 - visibility: public - - accessBinding: null - id: openapi-full + - id: openapi-full + path: generated/openapi.full.yaml mediaType: application/yaml + visibility: operator-only operationIdentifier: null - path: generated/openapi.full.yaml + accessBinding: null sha256: sha256:23bf20b6c4e11bfa568f9428519185b87ca1b605489315bd10605b5a62e21910 - visibility: operator-only - - accessBinding: null - id: openapi-public + - id: openapi-public + path: generated/openapi.public.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/openapi.public.json + accessBinding: null sha256: sha256:9f261308a11edf80b0b7aeb0e1f173f4f16b08db433944816fe12cfe49f5ced1 - visibility: public governedFiles: - - generated: false - mediaType: application/yaml - path: governed/codelists/enrolment-status.yaml + - path: governed/codelists/enrolment-status.yaml sha256: sha256:12b3004a2a947ebbf00769a24a0e59d72f226da2166514fa9b39c30d936844c4 - size: 94 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/codelists/programmes.yaml + bytes: 94 + - path: governed/codelists/programmes.yaml sha256: sha256:7255a70432da192c9c782c13751552868c424652184ca66c5adb5d4acee0ee45 - size: 85 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/codelists/record-lifecycle.yaml + bytes: 85 + - path: governed/codelists/record-lifecycle.yaml sha256: sha256:b77eab2bec905fdbb76824fc5ee717c65d4d2c4a77d67cf3a3bcbaf40040d1a4 - size: 93 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/classification-review-rationale.md + bytes: 93 + - path: governed/governance/classification-review-rationale.md sha256: sha256:377253745d4f0f85e1bbcb25ec470c93efafdf320ae01805bac798e78a8830f1 - size: 263 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/classification-review.yaml + bytes: 263 + - path: governed/governance/classification-review.yaml sha256: sha256:bfa457f3ec27a68bd02fd08aa697ef5f62587e08396d25eb2cd3b72fb5a99265 - size: 763 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/identifier-lifecycle.yaml + bytes: 763 + - path: governed/governance/identifier-lifecycle.yaml sha256: sha256:e78de15ae1dd7cf169c639483a41ee9943b064aa163c940f63ffbf6e4af9f6e2 - size: 269 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/legal-basis.yaml + bytes: 269 + - path: governed/governance/legal-basis.yaml sha256: sha256:090009737bd9a730e6cf4694e4182fed5778d300885ed48678c7c4628a5c95e5 - size: 248 - visibility: operator-only - - generated: false - mediaType: application/json - path: governed/reports/identification-report.json + bytes: 248 + - path: governed/reports/identification-report.json sha256: sha256:4570e2e7b4de293f8ef33ca2661c8b3a524c671558324ca7923f306c35bcd9b3 - size: 6673 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: registry.yaml + bytes: 6673 + - path: registry.yaml sha256: sha256:a5b0313dd8d9fbfe9b5b40e9109d4a335a675592d1fe417a63b9fde44007bf7f - size: 6630 - visibility: operator-only + bytes: 6630 business-registry: - packageRevision: sha256:c9e68ab5a0cb18a44c19ce420cf8381590d4d26ab640f35df7822aafa5e5075f + packageDigest: sha256:5cd990faa375f74422fa0fc17bffd2702af2f6ffe734f8f979ea236040c98712 contractRevision: sha256:e7bce610dec7dd1d5227e7931950fce675804efdf9537b8f31346c749732b5ae sourceSchemaFingerprints: companies: sha256:dd62b98578f0fa7341eeeaaac4b34da9b79405ae067dc06e5edb004c2d4a38fe artifacts: - - accessBinding: null - id: audit-event-schema + - id: audit-event-schema + path: generated/artifacts/audit-event.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/audit-event.schema.json + accessBinding: null sha256: sha256:41358576a5fcbf4a6182f22b2213f26dadd762e3a1c6d76ea1c0948106b7e6e2 - visibility: operator-only - - accessBinding: null - id: capability-inventory-full + - id: capability-inventory-full + path: generated/artifacts/capabilities.full.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/capabilities.full.json + accessBinding: null sha256: sha256:9008dcf5f27379c91f4d01cf4201db8309c9c95a82ad7d68280418e769b50fba - visibility: operator-only - - accessBinding: null - id: capability-inventory + - id: capability-inventory + path: generated/artifacts/capabilities.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/artifacts/capabilities.json + accessBinding: null sha256: sha256:e2473e2f90545eb2f059297b81ef13dd7eb3387b9db26ff8738ae697bcb3817a - visibility: public - - accessBinding: null - id: discovery-description + - id: discovery-description + path: generated/artifacts/discovery.jsonld mediaType: application/ld+json;profile="https://registrystack.org/discovery/profile/v1alpha1" + visibility: public operationIdentifier: null - path: generated/artifacts/discovery.jsonld + accessBinding: null sha256: sha256:bcfa5ca9b58e956952e0022912a4a174b245eaa989dac1757fa9c2e1fb0b7e50 - visibility: public - - accessBinding: null - id: registered-business--list--access-profile-public-register-classifications + - id: registered-business--list--access-profile-public-register-classifications + path: generated/artifacts/registered-business--list--access-profile-public-register.classifications.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-business--list--access-profile-public-register.classifications.json + accessBinding: null sha256: sha256:a242a993d24505958b52108869dfce092ec302eb025443a4df0d648cdf67911c - visibility: public - - accessBinding: null - id: registered-business--list--access-profile-public-register-context + - id: registered-business--list--access-profile-public-register-context + path: generated/artifacts/registered-business--list--access-profile-public-register.context.jsonld mediaType: application/ld+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-business--list--access-profile-public-register.context.jsonld + accessBinding: null sha256: sha256:84b847bd7d2d9d2c1763484590632c8aaa2c99176f1042d1a5ff7075e5151809 - visibility: public - - accessBinding: null - id: registered-business--list--access-profile-public-register-processing + - id: registered-business--list--access-profile-public-register-processing + path: generated/artifacts/registered-business--list--access-profile-public-register.processing.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-business--list--access-profile-public-register.processing.json + accessBinding: null sha256: sha256:5f87f5571e1ac60546c0e5da43d0e7396b2a65eafa33782b2ead0e221cd5b333 - visibility: public - - accessBinding: null - id: registered-business--list--access-profile-public-register-schema + - id: registered-business--list--access-profile-public-register-schema + path: generated/artifacts/registered-business--list--access-profile-public-register.schema.json mediaType: application/schema+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-business--list--access-profile-public-register.schema.json + accessBinding: null sha256: sha256:54a00d5406be89f91e687de570a790c473516a4f9caf0d26cbde985e0f55e78e - visibility: public - - accessBinding: null - id: registered-business--list--access-profile-public-register-shacl + - id: registered-business--list--access-profile-public-register-shacl + path: generated/artifacts/registered-business--list--access-profile-public-register.shacl.ttl mediaType: text/turtle + visibility: public operationIdentifier: null - path: generated/artifacts/registered-business--list--access-profile-public-register.shacl.ttl + accessBinding: null sha256: sha256:7a9a3c4648e09025b07cbf70e9282a6d8c3e48eb52ebbcf9b6baa8995052f24b - visibility: public - - accessBinding: null - id: registered-business--list--access-profile-public-register-vocabulary + - id: registered-business--list--access-profile-public-register-vocabulary + path: generated/artifacts/registered-business--list--access-profile-public-register.vocabulary.jsonld mediaType: application/ld+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-business--list--access-profile-public-register.vocabulary.jsonld + accessBinding: null sha256: sha256:24bcf44aa7b04353a8a23b2d80e5c4fe1cf6a60f0b03d0f0a0c48611631ee5d7 - visibility: public - - accessBinding: - identifier: registrar - kind: access-profile - id: registered-business--list--access-profile-registrar-capability - mediaType: application/json - operationIdentifier: registered-business.list + - id: registered-business--list--access-profile-registrar-capability path: generated/artifacts/registered-business--list--access-profile-registrar.capability.json - sha256: sha256:85fb0f4c355fff814567e444f0d5664011577e2c053c7fc6dad48e27be6d31a7 - visibility: operation-bound - - accessBinding: - identifier: registrar - kind: access-profile - id: registered-business--list--access-profile-registrar-classifications mediaType: application/json + visibility: operation-bound operationIdentifier: registered-business.list + accessBinding: + kind: access-profile + identifier: registrar + sha256: sha256:85fb0f4c355fff814567e444f0d5664011577e2c053c7fc6dad48e27be6d31a7 + - id: registered-business--list--access-profile-registrar-classifications path: generated/artifacts/registered-business--list--access-profile-registrar.classifications.json - sha256: sha256:f4e8deb74f2d80d3f5a5ed946112a8e93055dfc9752c8a90b55d3f882ef49ec9 + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: registrar - kind: access-profile - id: registered-business--list--access-profile-registrar-context - mediaType: application/ld+json operationIdentifier: registered-business.list + accessBinding: + kind: access-profile + identifier: registrar + sha256: sha256:f4e8deb74f2d80d3f5a5ed946112a8e93055dfc9752c8a90b55d3f882ef49ec9 + - id: registered-business--list--access-profile-registrar-context path: generated/artifacts/registered-business--list--access-profile-registrar.context.jsonld - sha256: sha256:0d03cb7c4b559406b6f402ee1c5f83ab1ec437a173f7de7cc5fda94bc7e4fcac + mediaType: application/ld+json visibility: operation-bound - - accessBinding: - identifier: registrar - kind: access-profile - id: registered-business--list--access-profile-registrar-processing - mediaType: application/json operationIdentifier: registered-business.list + accessBinding: + kind: access-profile + identifier: registrar + sha256: sha256:0d03cb7c4b559406b6f402ee1c5f83ab1ec437a173f7de7cc5fda94bc7e4fcac + - id: registered-business--list--access-profile-registrar-processing path: generated/artifacts/registered-business--list--access-profile-registrar.processing.json - sha256: sha256:2df97861c32c42672e87d2945d47871b736dfe97b6618243f3a1e1bb357166e5 + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: registrar - kind: access-profile - id: registered-business--list--access-profile-registrar-schema - mediaType: application/schema+json operationIdentifier: registered-business.list + accessBinding: + kind: access-profile + identifier: registrar + sha256: sha256:2df97861c32c42672e87d2945d47871b736dfe97b6618243f3a1e1bb357166e5 + - id: registered-business--list--access-profile-registrar-schema path: generated/artifacts/registered-business--list--access-profile-registrar.schema.json - sha256: sha256:a2edfd6085caaa71c9a6cd2bb31aa79d90b2f313468a5548ea8b3fb7c5de8c35 + mediaType: application/schema+json visibility: operation-bound - - accessBinding: - identifier: registrar - kind: access-profile - id: registered-business--list--access-profile-registrar-shacl - mediaType: text/turtle operationIdentifier: registered-business.list + accessBinding: + kind: access-profile + identifier: registrar + sha256: sha256:a2edfd6085caaa71c9a6cd2bb31aa79d90b2f313468a5548ea8b3fb7c5de8c35 + - id: registered-business--list--access-profile-registrar-shacl path: generated/artifacts/registered-business--list--access-profile-registrar.shacl.ttl - sha256: sha256:0ccfbcbfc3945a70bb7069526d3a46ce38ffcfa7582141b30d640c50b0bb9e99 + mediaType: text/turtle visibility: operation-bound - - accessBinding: - identifier: registrar + operationIdentifier: registered-business.list + accessBinding: kind: access-profile - id: registered-business--list--access-profile-registrar-vocabulary + identifier: registrar + sha256: sha256:0ccfbcbfc3945a70bb7069526d3a46ce38ffcfa7582141b30d640c50b0bb9e99 + - id: registered-business--list--access-profile-registrar-vocabulary + path: generated/artifacts/registered-business--list--access-profile-registrar.vocabulary.jsonld mediaType: application/ld+json + visibility: operation-bound operationIdentifier: registered-business.list - path: generated/artifacts/registered-business--list--access-profile-registrar.vocabulary.jsonld + accessBinding: + kind: access-profile + identifier: registrar sha256: sha256:1a1d5fec8194398211a8d8ea6618cef48b291b8d42814d94c5cba9af82d84b36 - visibility: operation-bound - - accessBinding: null - id: registered-business--read--access-profile-public-register-classifications + - id: registered-business--read--access-profile-public-register-classifications + path: generated/artifacts/registered-business--read--access-profile-public-register.classifications.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-business--read--access-profile-public-register.classifications.json + accessBinding: null sha256: sha256:e628697ef0efdac1f00119132f9d592ec29fa328d6a56cece6fb90c80b2a6c4b - visibility: public - - accessBinding: null - id: registered-business--read--access-profile-public-register-context + - id: registered-business--read--access-profile-public-register-context + path: generated/artifacts/registered-business--read--access-profile-public-register.context.jsonld mediaType: application/ld+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-business--read--access-profile-public-register.context.jsonld + accessBinding: null sha256: sha256:84b847bd7d2d9d2c1763484590632c8aaa2c99176f1042d1a5ff7075e5151809 - visibility: public - - accessBinding: null - id: registered-business--read--access-profile-public-register-processing + - id: registered-business--read--access-profile-public-register-processing + path: generated/artifacts/registered-business--read--access-profile-public-register.processing.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-business--read--access-profile-public-register.processing.json + accessBinding: null sha256: sha256:c438483b841471785c8393a20648facca8cc970114f894faaf3eca78eac03c6c - visibility: public - - accessBinding: null - id: registered-business--read--access-profile-public-register-schema + - id: registered-business--read--access-profile-public-register-schema + path: generated/artifacts/registered-business--read--access-profile-public-register.schema.json mediaType: application/schema+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-business--read--access-profile-public-register.schema.json + accessBinding: null sha256: sha256:ac5e366bbb779100cc2a61a3a9729c096386ab3bb6e757df2383b98c84976c20 - visibility: public - - accessBinding: null - id: registered-business--read--access-profile-public-register-shacl + - id: registered-business--read--access-profile-public-register-shacl + path: generated/artifacts/registered-business--read--access-profile-public-register.shacl.ttl mediaType: text/turtle + visibility: public operationIdentifier: null - path: generated/artifacts/registered-business--read--access-profile-public-register.shacl.ttl + accessBinding: null sha256: sha256:7a9a3c4648e09025b07cbf70e9282a6d8c3e48eb52ebbcf9b6baa8995052f24b - visibility: public - - accessBinding: null - id: registered-business--read--access-profile-public-register-vocabulary + - id: registered-business--read--access-profile-public-register-vocabulary + path: generated/artifacts/registered-business--read--access-profile-public-register.vocabulary.jsonld mediaType: application/ld+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-business--read--access-profile-public-register.vocabulary.jsonld + accessBinding: null sha256: sha256:24bcf44aa7b04353a8a23b2d80e5c4fe1cf6a60f0b03d0f0a0c48611631ee5d7 - visibility: public - - accessBinding: - identifier: registrar - kind: access-profile - id: registered-business--read--access-profile-registrar-capability - mediaType: application/json - operationIdentifier: registered-business.read + - id: registered-business--read--access-profile-registrar-capability path: generated/artifacts/registered-business--read--access-profile-registrar.capability.json - sha256: sha256:ba796c7376aac054e133e8cc31441831d16142f5f7c6b3aebb142631ed0ee3b7 - visibility: operation-bound - - accessBinding: - identifier: registrar - kind: access-profile - id: registered-business--read--access-profile-registrar-classifications mediaType: application/json + visibility: operation-bound operationIdentifier: registered-business.read + accessBinding: + kind: access-profile + identifier: registrar + sha256: sha256:ba796c7376aac054e133e8cc31441831d16142f5f7c6b3aebb142631ed0ee3b7 + - id: registered-business--read--access-profile-registrar-classifications path: generated/artifacts/registered-business--read--access-profile-registrar.classifications.json - sha256: sha256:a447eae6a497e23ab720dfd67767a9665b55e231157a45766460452aeb6a9ff0 + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: registrar - kind: access-profile - id: registered-business--read--access-profile-registrar-context - mediaType: application/ld+json operationIdentifier: registered-business.read + accessBinding: + kind: access-profile + identifier: registrar + sha256: sha256:a447eae6a497e23ab720dfd67767a9665b55e231157a45766460452aeb6a9ff0 + - id: registered-business--read--access-profile-registrar-context path: generated/artifacts/registered-business--read--access-profile-registrar.context.jsonld - sha256: sha256:0d03cb7c4b559406b6f402ee1c5f83ab1ec437a173f7de7cc5fda94bc7e4fcac + mediaType: application/ld+json visibility: operation-bound - - accessBinding: - identifier: registrar - kind: access-profile - id: registered-business--read--access-profile-registrar-processing - mediaType: application/json operationIdentifier: registered-business.read + accessBinding: + kind: access-profile + identifier: registrar + sha256: sha256:0d03cb7c4b559406b6f402ee1c5f83ab1ec437a173f7de7cc5fda94bc7e4fcac + - id: registered-business--read--access-profile-registrar-processing path: generated/artifacts/registered-business--read--access-profile-registrar.processing.json - sha256: sha256:cdfd9e044be2f18addbe824be6f4a0a07379364542577987833759bd48e038ee + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: registrar - kind: access-profile - id: registered-business--read--access-profile-registrar-schema - mediaType: application/schema+json operationIdentifier: registered-business.read + accessBinding: + kind: access-profile + identifier: registrar + sha256: sha256:cdfd9e044be2f18addbe824be6f4a0a07379364542577987833759bd48e038ee + - id: registered-business--read--access-profile-registrar-schema path: generated/artifacts/registered-business--read--access-profile-registrar.schema.json - sha256: sha256:8b52f1b14249af4b3a260288a5ce801d5a83d1741b7473d1b6f0f4f771b7488c + mediaType: application/schema+json visibility: operation-bound - - accessBinding: - identifier: registrar - kind: access-profile - id: registered-business--read--access-profile-registrar-shacl - mediaType: text/turtle operationIdentifier: registered-business.read + accessBinding: + kind: access-profile + identifier: registrar + sha256: sha256:8b52f1b14249af4b3a260288a5ce801d5a83d1741b7473d1b6f0f4f771b7488c + - id: registered-business--read--access-profile-registrar-shacl path: generated/artifacts/registered-business--read--access-profile-registrar.shacl.ttl - sha256: sha256:0ccfbcbfc3945a70bb7069526d3a46ce38ffcfa7582141b30d640c50b0bb9e99 + mediaType: text/turtle visibility: operation-bound - - accessBinding: - identifier: registrar + operationIdentifier: registered-business.read + accessBinding: kind: access-profile - id: registered-business--read--access-profile-registrar-vocabulary + identifier: registrar + sha256: sha256:0ccfbcbfc3945a70bb7069526d3a46ce38ffcfa7582141b30d640c50b0bb9e99 + - id: registered-business--read--access-profile-registrar-vocabulary + path: generated/artifacts/registered-business--read--access-profile-registrar.vocabulary.jsonld mediaType: application/ld+json + visibility: operation-bound operationIdentifier: registered-business.read - path: generated/artifacts/registered-business--read--access-profile-registrar.vocabulary.jsonld + accessBinding: + kind: access-profile + identifier: registrar sha256: sha256:1a1d5fec8194398211a8d8ea6618cef48b291b8d42814d94c5cba9af82d84b36 - visibility: operation-bound - - accessBinding: null - id: registered-business-classification + - id: registered-business-classification + path: generated/artifacts/registered-business.classifications.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-business.classifications.json + accessBinding: null sha256: sha256:0eba6b9824ab9b0e21482bf49cecc6db1401073f2103df2ca57fd93b1383915a - visibility: operator-only - - accessBinding: null - id: registered-business-codelist-0 + - id: registered-business-codelist-0 + path: generated/artifacts/registered-business.codelist-0.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-business.codelist-0.schema.json + accessBinding: null sha256: sha256:b5f27954974850cd56ec6e271a4f630ce749efc332e58b6a407ece3f943f3d20 - visibility: operator-only - - accessBinding: null - id: registered-business-codelist-1 + - id: registered-business-codelist-1 + path: generated/artifacts/registered-business.codelist-1.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-business.codelist-1.schema.json + accessBinding: null sha256: sha256:69064b6563a9376270b2d6535a338a5766071012817d25edb0351f8e0e65b76b - visibility: operator-only - - accessBinding: null - id: registered-business-codelist-2 + - id: registered-business-codelist-2 + path: generated/artifacts/registered-business.codelist-2.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-business.codelist-2.schema.json + accessBinding: null sha256: sha256:4df390c7d6dbf8dae80011b4ea93545b7f2688cc7337a0534f322a92530d3b96 - visibility: operator-only - - accessBinding: null - id: registered-business-codelist-3 + - id: registered-business-codelist-3 + path: generated/artifacts/registered-business.codelist-3.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-business.codelist-3.schema.json + accessBinding: null sha256: sha256:e42dfbcab45a66032d126e0f203523ae44a6bc034278f2ce222f96f1ff0a78f0 - visibility: operator-only - - accessBinding: null - id: registered-business-full-schema + - id: registered-business-full-schema + path: generated/artifacts/registered-business.full.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-business.full.schema.json + accessBinding: null sha256: sha256:4a8de6fa88e7e9b548fd8bc10f3b5f8273f8cdf9f59ea8fcba0742d67a970d04 - visibility: operator-only - - accessBinding: null - id: registered-business-full-shacl + - id: registered-business-full-shacl + path: generated/artifacts/registered-business.full.shacl.ttl mediaType: text/turtle + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-business.full.shacl.ttl + accessBinding: null sha256: sha256:8c13a0e4656738ab0ffd73f0d7a90ca59e332fec7f1d7657c9849a88b1b99353 - visibility: operator-only - - accessBinding: null - id: registered-business-full-vocabulary + - id: registered-business-full-vocabulary + path: generated/artifacts/registered-business.full.vocabulary.jsonld mediaType: application/ld+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-business.full.vocabulary.jsonld + accessBinding: null sha256: sha256:f57ec119ca7d4dc0534ee8e2c5f8756e336f0f90bd34e22fab18f731baffe181 - visibility: operator-only - - accessBinding: null - id: registered-business-processing-full + - id: registered-business-processing-full + path: generated/artifacts/registered-business.processing.full.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-business.processing.full.json + accessBinding: null sha256: sha256:9e14c3d53958f18e29ee021c74f6f8ea0ceacb0452d01f5f13f5ea7270006158 - visibility: operator-only - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--list--access-profile-registrar-premises-capability - mediaType: application/json - operationIdentifier: registered-premises.list + - id: registered-premises--list--access-profile-registrar-premises-capability path: generated/artifacts/registered-premises--list--access-profile-registrar-premises.capability.json - sha256: sha256:35928e0927b265a67a674189ae8329834fd0307bee2472739fad578e7138b57f - visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--list--access-profile-registrar-premises-classifications mediaType: application/json + visibility: operation-bound operationIdentifier: registered-premises.list + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:35928e0927b265a67a674189ae8329834fd0307bee2472739fad578e7138b57f + - id: registered-premises--list--access-profile-registrar-premises-classifications path: generated/artifacts/registered-premises--list--access-profile-registrar-premises.classifications.json - sha256: sha256:0106551e4daf2c835d5b38f4ac10fec04b65d114a979d4001e76d1711089ee25 + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--list--access-profile-registrar-premises-context - mediaType: application/ld+json operationIdentifier: registered-premises.list + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:0106551e4daf2c835d5b38f4ac10fec04b65d114a979d4001e76d1711089ee25 + - id: registered-premises--list--access-profile-registrar-premises-context path: generated/artifacts/registered-premises--list--access-profile-registrar-premises.context.jsonld - sha256: sha256:ab52b4962af18ab257eb8fd537da835628db8477ac8ea939c04b94e4cd7cac27 + mediaType: application/ld+json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--list--access-profile-registrar-premises-geojson-schema - mediaType: application/schema+json operationIdentifier: registered-premises.list + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:ab52b4962af18ab257eb8fd537da835628db8477ac8ea939c04b94e4cd7cac27 + - id: registered-premises--list--access-profile-registrar-premises-geojson-schema path: generated/artifacts/registered-premises--list--access-profile-registrar-premises.geojson.schema.json - sha256: sha256:17c3c49c52438d809b7924282f136f250a492bafc9c6a2c004fd8330d6fcbe23 + mediaType: application/schema+json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--list--access-profile-registrar-premises-processing - mediaType: application/json operationIdentifier: registered-premises.list + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:17c3c49c52438d809b7924282f136f250a492bafc9c6a2c004fd8330d6fcbe23 + - id: registered-premises--list--access-profile-registrar-premises-processing path: generated/artifacts/registered-premises--list--access-profile-registrar-premises.processing.json - sha256: sha256:a25a0dd195a49fd20617134557c5ae679326da397f4ee5a7a639e85412cf2cc5 + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--list--access-profile-registrar-premises-schema - mediaType: application/schema+json operationIdentifier: registered-premises.list + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:a25a0dd195a49fd20617134557c5ae679326da397f4ee5a7a639e85412cf2cc5 + - id: registered-premises--list--access-profile-registrar-premises-schema path: generated/artifacts/registered-premises--list--access-profile-registrar-premises.schema.json - sha256: sha256:7fe438c9eb2d6403781e6c21f0d08ae8ad6c57e1f2461c41969ff10cd2358dae + mediaType: application/schema+json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--list--access-profile-registrar-premises-shacl - mediaType: text/turtle operationIdentifier: registered-premises.list + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:7fe438c9eb2d6403781e6c21f0d08ae8ad6c57e1f2461c41969ff10cd2358dae + - id: registered-premises--list--access-profile-registrar-premises-shacl path: generated/artifacts/registered-premises--list--access-profile-registrar-premises.shacl.ttl - sha256: sha256:b38b9adb19075153a8dbe810fe847b2fc85e3c1dcdea50b9878b898bf1f15e47 + mediaType: text/turtle visibility: operation-bound - - accessBinding: - identifier: registrar-premises + operationIdentifier: registered-premises.list + accessBinding: kind: access-profile - id: registered-premises--list--access-profile-registrar-premises-vocabulary + identifier: registrar-premises + sha256: sha256:b38b9adb19075153a8dbe810fe847b2fc85e3c1dcdea50b9878b898bf1f15e47 + - id: registered-premises--list--access-profile-registrar-premises-vocabulary + path: generated/artifacts/registered-premises--list--access-profile-registrar-premises.vocabulary.jsonld mediaType: application/ld+json + visibility: operation-bound operationIdentifier: registered-premises.list - path: generated/artifacts/registered-premises--list--access-profile-registrar-premises.vocabulary.jsonld + accessBinding: + kind: access-profile + identifier: registrar-premises sha256: sha256:fda4c06e770961e39aecdcc006020ed14ded2f6194e03a40bc7393bfe44d0aee - visibility: operation-bound - - accessBinding: null - id: registered-premises--read--access-profile-public-premises-classifications + - id: registered-premises--read--access-profile-public-premises-classifications + path: generated/artifacts/registered-premises--read--access-profile-public-premises.classifications.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--read--access-profile-public-premises.classifications.json + accessBinding: null sha256: sha256:9becb3f908a3eb9a24c7cd5e728d485cc762e675464c04c35cf45569b9b1fea6 - visibility: public - - accessBinding: null - id: registered-premises--read--access-profile-public-premises-context + - id: registered-premises--read--access-profile-public-premises-context + path: generated/artifacts/registered-premises--read--access-profile-public-premises.context.jsonld mediaType: application/ld+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--read--access-profile-public-premises.context.jsonld + accessBinding: null sha256: sha256:195ebb46a655e151fcd5e7f6ee5c4b1b99f33da734ed856a50bf1dedd3b363bf - visibility: public - - accessBinding: null - id: registered-premises--read--access-profile-public-premises-geojson-schema + - id: registered-premises--read--access-profile-public-premises-geojson-schema + path: generated/artifacts/registered-premises--read--access-profile-public-premises.geojson.schema.json mediaType: application/schema+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--read--access-profile-public-premises.geojson.schema.json + accessBinding: null sha256: sha256:3eb8430996cd2febd9e5fa41ef3800b282754652c2addcc1e5b72cade19c619a - visibility: public - - accessBinding: null - id: registered-premises--read--access-profile-public-premises-processing + - id: registered-premises--read--access-profile-public-premises-processing + path: generated/artifacts/registered-premises--read--access-profile-public-premises.processing.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--read--access-profile-public-premises.processing.json + accessBinding: null sha256: sha256:8d8af9e03e0ee99008eea2d865678fb330db321f2f01c416de5f6401de3b38fd - visibility: public - - accessBinding: null - id: registered-premises--read--access-profile-public-premises-schema + - id: registered-premises--read--access-profile-public-premises-schema + path: generated/artifacts/registered-premises--read--access-profile-public-premises.schema.json mediaType: application/schema+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--read--access-profile-public-premises.schema.json + accessBinding: null sha256: sha256:7bee8e68da72100b12c83e7df783fea99085f0d142c139b1170dcb855c015c99 - visibility: public - - accessBinding: null - id: registered-premises--read--access-profile-public-premises-shacl + - id: registered-premises--read--access-profile-public-premises-shacl + path: generated/artifacts/registered-premises--read--access-profile-public-premises.shacl.ttl mediaType: text/turtle + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--read--access-profile-public-premises.shacl.ttl + accessBinding: null sha256: sha256:ba02552f0894dbbe49745cc2d24ee5f8a526c285363a1b4862a5e108cff8cf58 - visibility: public - - accessBinding: null - id: registered-premises--read--access-profile-public-premises-vocabulary + - id: registered-premises--read--access-profile-public-premises-vocabulary + path: generated/artifacts/registered-premises--read--access-profile-public-premises.vocabulary.jsonld mediaType: application/ld+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--read--access-profile-public-premises.vocabulary.jsonld + accessBinding: null sha256: sha256:5a2dc2d0c3ddd2dfca8e858e9149ee4948fe211d059b5adcafed78e8bd9e9029 - visibility: public - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--read--access-profile-registrar-premises-capability - mediaType: application/json - operationIdentifier: registered-premises.read + - id: registered-premises--read--access-profile-registrar-premises-capability path: generated/artifacts/registered-premises--read--access-profile-registrar-premises.capability.json - sha256: sha256:265f678b6f69f3ca18401c09cd2703f3d8b0fae8bb42297b55b1c90e19281d4d - visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--read--access-profile-registrar-premises-classifications mediaType: application/json + visibility: operation-bound operationIdentifier: registered-premises.read + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:265f678b6f69f3ca18401c09cd2703f3d8b0fae8bb42297b55b1c90e19281d4d + - id: registered-premises--read--access-profile-registrar-premises-classifications path: generated/artifacts/registered-premises--read--access-profile-registrar-premises.classifications.json - sha256: sha256:bf7bd7e61ca541bf62c660c9ed47c85ea57d99c89c499166847cb7814cd043a0 + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--read--access-profile-registrar-premises-context - mediaType: application/ld+json operationIdentifier: registered-premises.read + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:bf7bd7e61ca541bf62c660c9ed47c85ea57d99c89c499166847cb7814cd043a0 + - id: registered-premises--read--access-profile-registrar-premises-context path: generated/artifacts/registered-premises--read--access-profile-registrar-premises.context.jsonld - sha256: sha256:ab52b4962af18ab257eb8fd537da835628db8477ac8ea939c04b94e4cd7cac27 + mediaType: application/ld+json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--read--access-profile-registrar-premises-geojson-schema - mediaType: application/schema+json operationIdentifier: registered-premises.read + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:ab52b4962af18ab257eb8fd537da835628db8477ac8ea939c04b94e4cd7cac27 + - id: registered-premises--read--access-profile-registrar-premises-geojson-schema path: generated/artifacts/registered-premises--read--access-profile-registrar-premises.geojson.schema.json - sha256: sha256:b1f8f1ce635b49a46e52818bd2f5c4bec113060ab76f17db86b47420c08df36c + mediaType: application/schema+json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--read--access-profile-registrar-premises-processing - mediaType: application/json operationIdentifier: registered-premises.read + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:b1f8f1ce635b49a46e52818bd2f5c4bec113060ab76f17db86b47420c08df36c + - id: registered-premises--read--access-profile-registrar-premises-processing path: generated/artifacts/registered-premises--read--access-profile-registrar-premises.processing.json - sha256: sha256:220845cbffa466f53025e7e6b8e55fcd11589b8631b6ff3c39ce0a051d643880 + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--read--access-profile-registrar-premises-schema - mediaType: application/schema+json operationIdentifier: registered-premises.read + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:220845cbffa466f53025e7e6b8e55fcd11589b8631b6ff3c39ce0a051d643880 + - id: registered-premises--read--access-profile-registrar-premises-schema path: generated/artifacts/registered-premises--read--access-profile-registrar-premises.schema.json - sha256: sha256:ad0f8389d8e281646491535d9a4f346f665a4e88d635642b4130ded1142eaeed + mediaType: application/schema+json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--read--access-profile-registrar-premises-shacl - mediaType: text/turtle operationIdentifier: registered-premises.read + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:ad0f8389d8e281646491535d9a4f346f665a4e88d635642b4130ded1142eaeed + - id: registered-premises--read--access-profile-registrar-premises-shacl path: generated/artifacts/registered-premises--read--access-profile-registrar-premises.shacl.ttl - sha256: sha256:b38b9adb19075153a8dbe810fe847b2fc85e3c1dcdea50b9878b898bf1f15e47 + mediaType: text/turtle visibility: operation-bound - - accessBinding: - identifier: registrar-premises + operationIdentifier: registered-premises.read + accessBinding: kind: access-profile - id: registered-premises--read--access-profile-registrar-premises-vocabulary + identifier: registrar-premises + sha256: sha256:b38b9adb19075153a8dbe810fe847b2fc85e3c1dcdea50b9878b898bf1f15e47 + - id: registered-premises--read--access-profile-registrar-premises-vocabulary + path: generated/artifacts/registered-premises--read--access-profile-registrar-premises.vocabulary.jsonld mediaType: application/ld+json + visibility: operation-bound operationIdentifier: registered-premises.read - path: generated/artifacts/registered-premises--read--access-profile-registrar-premises.vocabulary.jsonld + accessBinding: + kind: access-profile + identifier: registrar-premises sha256: sha256:fda4c06e770961e39aecdcc006020ed14ded2f6194e03a40bc7393bfe44d0aee - visibility: operation-bound - - accessBinding: null - id: registered-premises--search-within-bbox--access-profile-public-premises-classifications + - id: registered-premises--search-within-bbox--access-profile-public-premises-classifications + path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.classifications.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.classifications.json + accessBinding: null sha256: sha256:c8665707ce24f1ddea2f88035f7ba07087d04521b9534a0281464aada67f7ee5 - visibility: public - - accessBinding: null - id: registered-premises--search-within-bbox--access-profile-public-premises-context + - id: registered-premises--search-within-bbox--access-profile-public-premises-context + path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.context.jsonld mediaType: application/ld+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.context.jsonld + accessBinding: null sha256: sha256:195ebb46a655e151fcd5e7f6ee5c4b1b99f33da734ed856a50bf1dedd3b363bf - visibility: public - - accessBinding: null - id: registered-premises--search-within-bbox--access-profile-public-premises-geojson-schema + - id: registered-premises--search-within-bbox--access-profile-public-premises-geojson-schema + path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.geojson.schema.json mediaType: application/schema+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.geojson.schema.json + accessBinding: null sha256: sha256:8fa5da5c0295a530e154e08d39561105eeb451ce84ce6bc612a4c8e8b4b09db8 - visibility: public - - accessBinding: null - id: registered-premises--search-within-bbox--access-profile-public-premises-processing + - id: registered-premises--search-within-bbox--access-profile-public-premises-processing + path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.processing.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.processing.json + accessBinding: null sha256: sha256:1346f2361d748c1af103a515ee09642cc588a3919d48351a66db4a42d9e093ec - visibility: public - - accessBinding: null - id: registered-premises--search-within-bbox--access-profile-public-premises-schema + - id: registered-premises--search-within-bbox--access-profile-public-premises-schema + path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.schema.json mediaType: application/schema+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.schema.json + accessBinding: null sha256: sha256:cea336ae0da003fcb225a651fbac68f5311ba4f1f4f5ae7ced752ce2075f7d79 - visibility: public - - accessBinding: null - id: registered-premises--search-within-bbox--access-profile-public-premises-shacl + - id: registered-premises--search-within-bbox--access-profile-public-premises-shacl + path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.shacl.ttl mediaType: text/turtle + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.shacl.ttl + accessBinding: null sha256: sha256:ba02552f0894dbbe49745cc2d24ee5f8a526c285363a1b4862a5e108cff8cf58 - visibility: public - - accessBinding: null - id: registered-premises--search-within-bbox--access-profile-public-premises-vocabulary + - id: registered-premises--search-within-bbox--access-profile-public-premises-vocabulary + path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.vocabulary.jsonld mediaType: application/ld+json + visibility: public operationIdentifier: null - path: generated/artifacts/registered-premises--search-within-bbox--access-profile-public-premises.vocabulary.jsonld + accessBinding: null sha256: sha256:5a2dc2d0c3ddd2dfca8e858e9149ee4948fe211d059b5adcafed78e8bd9e9029 - visibility: public - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--search-within-bbox--access-profile-registrar-premises-capability - mediaType: application/json - operationIdentifier: registered-premises.search.within-bbox + - id: registered-premises--search-within-bbox--access-profile-registrar-premises-capability path: generated/artifacts/registered-premises--search-within-bbox--access-profile-registrar-premises.capability.json - sha256: sha256:0ce7701018e46c871ce5b26a36a3539c22a6e3701115d90b064fdf8a5c26c545 - visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--search-within-bbox--access-profile-registrar-premises-classifications mediaType: application/json + visibility: operation-bound operationIdentifier: registered-premises.search.within-bbox + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:0ce7701018e46c871ce5b26a36a3539c22a6e3701115d90b064fdf8a5c26c545 + - id: registered-premises--search-within-bbox--access-profile-registrar-premises-classifications path: generated/artifacts/registered-premises--search-within-bbox--access-profile-registrar-premises.classifications.json - sha256: sha256:004de44703b7266e293af3d833cfe375b73b44c63e80df1907e85749c84cc7e5 + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--search-within-bbox--access-profile-registrar-premises-context - mediaType: application/ld+json operationIdentifier: registered-premises.search.within-bbox + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:004de44703b7266e293af3d833cfe375b73b44c63e80df1907e85749c84cc7e5 + - id: registered-premises--search-within-bbox--access-profile-registrar-premises-context path: generated/artifacts/registered-premises--search-within-bbox--access-profile-registrar-premises.context.jsonld - sha256: sha256:ab52b4962af18ab257eb8fd537da835628db8477ac8ea939c04b94e4cd7cac27 + mediaType: application/ld+json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--search-within-bbox--access-profile-registrar-premises-geojson-schema - mediaType: application/schema+json operationIdentifier: registered-premises.search.within-bbox + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:ab52b4962af18ab257eb8fd537da835628db8477ac8ea939c04b94e4cd7cac27 + - id: registered-premises--search-within-bbox--access-profile-registrar-premises-geojson-schema path: generated/artifacts/registered-premises--search-within-bbox--access-profile-registrar-premises.geojson.schema.json - sha256: sha256:ca73b4358f749f3e3b5a52eaba18cbf32bda03142a445dcd613324bb2041e5a0 + mediaType: application/schema+json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--search-within-bbox--access-profile-registrar-premises-processing - mediaType: application/json operationIdentifier: registered-premises.search.within-bbox + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:ca73b4358f749f3e3b5a52eaba18cbf32bda03142a445dcd613324bb2041e5a0 + - id: registered-premises--search-within-bbox--access-profile-registrar-premises-processing path: generated/artifacts/registered-premises--search-within-bbox--access-profile-registrar-premises.processing.json - sha256: sha256:0b77612ff0279d2f96aab1577d3a98b7131ebcbcdc662222cfa4a0d4b3ce248e + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--search-within-bbox--access-profile-registrar-premises-schema - mediaType: application/schema+json operationIdentifier: registered-premises.search.within-bbox + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:0b77612ff0279d2f96aab1577d3a98b7131ebcbcdc662222cfa4a0d4b3ce248e + - id: registered-premises--search-within-bbox--access-profile-registrar-premises-schema path: generated/artifacts/registered-premises--search-within-bbox--access-profile-registrar-premises.schema.json - sha256: sha256:def5969193fa9297263cb73777c399ee99311451dc3473f0ef251dd75d84bc87 + mediaType: application/schema+json visibility: operation-bound - - accessBinding: - identifier: registrar-premises - kind: access-profile - id: registered-premises--search-within-bbox--access-profile-registrar-premises-shacl - mediaType: text/turtle operationIdentifier: registered-premises.search.within-bbox + accessBinding: + kind: access-profile + identifier: registrar-premises + sha256: sha256:def5969193fa9297263cb73777c399ee99311451dc3473f0ef251dd75d84bc87 + - id: registered-premises--search-within-bbox--access-profile-registrar-premises-shacl path: generated/artifacts/registered-premises--search-within-bbox--access-profile-registrar-premises.shacl.ttl - sha256: sha256:b38b9adb19075153a8dbe810fe847b2fc85e3c1dcdea50b9878b898bf1f15e47 + mediaType: text/turtle visibility: operation-bound - - accessBinding: - identifier: registrar-premises + operationIdentifier: registered-premises.search.within-bbox + accessBinding: kind: access-profile - id: registered-premises--search-within-bbox--access-profile-registrar-premises-vocabulary + identifier: registrar-premises + sha256: sha256:b38b9adb19075153a8dbe810fe847b2fc85e3c1dcdea50b9878b898bf1f15e47 + - id: registered-premises--search-within-bbox--access-profile-registrar-premises-vocabulary + path: generated/artifacts/registered-premises--search-within-bbox--access-profile-registrar-premises.vocabulary.jsonld mediaType: application/ld+json + visibility: operation-bound operationIdentifier: registered-premises.search.within-bbox - path: generated/artifacts/registered-premises--search-within-bbox--access-profile-registrar-premises.vocabulary.jsonld + accessBinding: + kind: access-profile + identifier: registrar-premises sha256: sha256:fda4c06e770961e39aecdcc006020ed14ded2f6194e03a40bc7393bfe44d0aee - visibility: operation-bound - - accessBinding: null - id: registered-premises-classification + - id: registered-premises-classification + path: generated/artifacts/registered-premises.classifications.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-premises.classifications.json + accessBinding: null sha256: sha256:9744b16430fa0771dee1b2d1b3b0ef447697d72b8fa341957b7b103cb8cd9beb - visibility: operator-only - - accessBinding: null - id: registered-premises-codelist-0 + - id: registered-premises-codelist-0 + path: generated/artifacts/registered-premises.codelist-0.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-premises.codelist-0.schema.json + accessBinding: null sha256: sha256:e42dfbcab45a66032d126e0f203523ae44a6bc034278f2ce222f96f1ff0a78f0 - visibility: operator-only - - accessBinding: null - id: registered-premises-full-schema + - id: registered-premises-full-schema + path: generated/artifacts/registered-premises.full.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-premises.full.schema.json + accessBinding: null sha256: sha256:3b89b94cf6e1d5f4155363a3d78c47b35bf270c4246d1f0b40750f3b6f7f9cfb - visibility: operator-only - - accessBinding: null - id: registered-premises-full-shacl + - id: registered-premises-full-shacl + path: generated/artifacts/registered-premises.full.shacl.ttl mediaType: text/turtle + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-premises.full.shacl.ttl + accessBinding: null sha256: sha256:b94692815a7ffbaf2a09a7dc1e7b150a04f25687557da3707088732aede65c08 - visibility: operator-only - - accessBinding: null - id: registered-premises-full-vocabulary + - id: registered-premises-full-vocabulary + path: generated/artifacts/registered-premises.full.vocabulary.jsonld mediaType: application/ld+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-premises.full.vocabulary.jsonld + accessBinding: null sha256: sha256:fda4c06e770961e39aecdcc006020ed14ded2f6194e03a40bc7393bfe44d0aee - visibility: operator-only - - accessBinding: null - id: registered-premises-processing-full + - id: registered-premises-processing-full + path: generated/artifacts/registered-premises.processing.full.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/registered-premises.processing.full.json + accessBinding: null sha256: sha256:68d4d19f9cd242b1344adf7862fff9579b0a254c78146c48efc5961f5a452385 - visibility: operator-only - - accessBinding: null - id: openapi-full + - id: openapi-full + path: generated/openapi.full.yaml mediaType: application/yaml + visibility: operator-only operationIdentifier: null - path: generated/openapi.full.yaml + accessBinding: null sha256: sha256:ed305f59d27e11414ae94244785f5ff3fded5a1863011a2b3e9c31284c57270f - visibility: operator-only - - accessBinding: null - id: openapi-public + - id: openapi-public + path: generated/openapi.public.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/openapi.public.json + accessBinding: null sha256: sha256:543c0aecaba6b38769a7a872dedc5efdf49672aa16f730311f604fbd983cbd07 - visibility: public governedFiles: - - generated: false - mediaType: application/yaml - path: governed/codelists/business-status.yaml + - path: governed/codelists/business-status.yaml sha256: sha256:5ece98ce569218b515f0712ccb604a6bed50ba0aec100dc882d21b4ef5fbca27 - size: 97 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/codelists/jurisdictions.yaml + bytes: 97 + - path: governed/codelists/jurisdictions.yaml sha256: sha256:c3d4a8e7dcf74ab3034ce3ea816c476ab1201a6029dad03b188fd1493c175a89 - size: 77 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/codelists/legal-forms.yaml + bytes: 77 + - path: governed/codelists/legal-forms.yaml sha256: sha256:6e70156f5503d1e97f0598f550c2bafec6674987179e1d38e391bcd44bd50edd - size: 105 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/codelists/record-lifecycle.yaml + bytes: 105 + - path: governed/codelists/record-lifecycle.yaml sha256: sha256:f3f7e339409460ae587ec9ff0d290c08a28cc588060d9a265969f0eb809f9dff - size: 95 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/classification-review-rationale.md + bytes: 95 + - path: governed/governance/classification-review-rationale.md sha256: sha256:494960ac760eaba7466d09fb4a8fa1ce824ba3f2a557a5abdf9e3bbed8823755 - size: 526 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/classification-review.yaml + bytes: 526 + - path: governed/governance/classification-review.yaml sha256: sha256:3955cb3e5336840c28453fcf16bfaab6f32aa70119f95c0cd5290ef0a0d27dd1 - size: 423 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/identifier-lifecycle.yaml + bytes: 423 + - path: governed/governance/identifier-lifecycle.yaml sha256: sha256:4b9cf35d384254effc3e17502608195310dc208699b9b4ab93ae108f37d44c18 - size: 262 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/legal-basis.yaml + bytes: 262 + - path: governed/governance/legal-basis.yaml sha256: sha256:41c664bdb8b8737940c6def0b505e5b72fe70406132abaeef805b4c66b66573c - size: 241 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/semantics/semic-business-alignment.yaml + bytes: 241 + - path: governed/semantics/semic-business-alignment.yaml sha256: sha256:6a46a9be0a3d5b4a5650934c7e8ef73ad1803cb479981f7d235a1c17a335af52 - size: 668 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: registry.yaml + bytes: 668 + - path: registry.yaml sha256: sha256:f0aaf62ca111f6c123a1cc23971f60c4d237652a1c416ae25d97529f9947f687 - size: 11451 - visibility: operator-only + bytes: 11451 civil-event: - packageRevision: sha256:8a422b8eea607b864742408f1ceee1fb786bc215ce90942a186dd1dc35ad16e8 + packageDigest: sha256:8a0a9789089727a3169360b3a5927d9270c9208bae4c09ac4835d0e8591774b9 contractRevision: sha256:61632ff51324349d7fbb4c80738bf2d3a42cc5cb337882f13e2c67b492918a39 sourceSchemaFingerprints: events: sha256:7f770d64cb19ec54caca2aa56378b13a43cd5edc206ff44b5fecc99ee9e63759 artifacts: - - accessBinding: null - id: audit-event-schema + - id: audit-event-schema + path: generated/artifacts/audit-event.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/audit-event.schema.json + accessBinding: null sha256: sha256:41358576a5fcbf4a6182f22b2213f26dadd762e3a1c6d76ea1c0948106b7e6e2 - visibility: operator-only - - accessBinding: null - id: capability-inventory-full + - id: capability-inventory-full + path: generated/artifacts/capabilities.full.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/capabilities.full.json + accessBinding: null sha256: sha256:c80082188ded5cda0715c2712024d910a75d4e864652f77fac5b820425fbdb9a - visibility: operator-only - - accessBinding: null - id: capability-inventory + - id: capability-inventory + path: generated/artifacts/capabilities.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/artifacts/capabilities.json + accessBinding: null sha256: sha256:cf92efcdb7b5a0e51ed36f87fa75b808d65a03e21b7815b73525d660f267ad24 - visibility: public - - accessBinding: - identifier: registrar-verification - kind: access-profile - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-capability + - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-capability + path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-registrar-verification.capability.json mediaType: application/json + visibility: operation-bound operationIdentifier: civil-event.lookup.verify-registration - path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-registrar-verification.capability.json + accessBinding: + kind: access-profile + identifier: registrar-verification sha256: sha256:bd320a60c565f0d280646ea6c36c16cdf207c3d157bc89f205d3730c89013946 - visibility: operation-bound - - accessBinding: null - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-classifications + - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-classifications + path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-registrar-verification.classifications.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-registrar-verification.classifications.json + accessBinding: null sha256: sha256:308553521c3108baf781299bd10c25eec92f71f8eb0dcee1b5211dd953968d2a - visibility: operator-only - - accessBinding: - identifier: registrar-verification - kind: access-profile - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-context - mediaType: application/ld+json - operationIdentifier: civil-event.lookup.verify-registration + - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-context path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-registrar-verification.context.jsonld - sha256: sha256:493f9032276257e5f0cd95bfb92fb3c2356955ef3f789c6713cf75a2a9428fa0 + mediaType: application/ld+json visibility: operation-bound - - accessBinding: - identifier: registrar-verification - kind: access-profile - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-processing - mediaType: application/json operationIdentifier: civil-event.lookup.verify-registration + accessBinding: + kind: access-profile + identifier: registrar-verification + sha256: sha256:493f9032276257e5f0cd95bfb92fb3c2356955ef3f789c6713cf75a2a9428fa0 + - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-processing path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-registrar-verification.processing.json - sha256: sha256:21021162d3ede3099de37e1a142d35751fef060505ca482b5eae4657b2970e4c + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: registrar-verification - kind: access-profile - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-schema - mediaType: application/schema+json operationIdentifier: civil-event.lookup.verify-registration + accessBinding: + kind: access-profile + identifier: registrar-verification + sha256: sha256:21021162d3ede3099de37e1a142d35751fef060505ca482b5eae4657b2970e4c + - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-schema path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-registrar-verification.schema.json - sha256: sha256:6662cf6129954b25d0033a5a30c9fa6cb83c5abc3f6c76bbc518df5730d0702b + mediaType: application/schema+json visibility: operation-bound - - accessBinding: - identifier: registrar-verification - kind: access-profile - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-shacl - mediaType: text/turtle operationIdentifier: civil-event.lookup.verify-registration + accessBinding: + kind: access-profile + identifier: registrar-verification + sha256: sha256:6662cf6129954b25d0033a5a30c9fa6cb83c5abc3f6c76bbc518df5730d0702b + - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-shacl path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-registrar-verification.shacl.ttl - sha256: sha256:5827ad9c0ee40e27fe91ce98030ad4c9b4980fe2370c1eab6a6ccb1f098d375a + mediaType: text/turtle visibility: operation-bound - - accessBinding: - identifier: registrar-verification - kind: access-profile - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-vocabulary - mediaType: application/ld+json operationIdentifier: civil-event.lookup.verify-registration + accessBinding: + kind: access-profile + identifier: registrar-verification + sha256: sha256:5827ad9c0ee40e27fe91ce98030ad4c9b4980fe2370c1eab6a6ccb1f098d375a + - id: civil-event--lookup-verify-registration--access-profile-registrar-verification-vocabulary path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-registrar-verification.vocabulary.jsonld - sha256: sha256:fdcf02c1ff87421d65b707e8dd0de30432d2650b9c53914e55002218d4da1cb1 + mediaType: application/ld+json visibility: operation-bound - - accessBinding: - identifier: supervisory + operationIdentifier: civil-event.lookup.verify-registration + accessBinding: kind: access-profile - id: civil-event--lookup-verify-registration--access-profile-supervisory-capability + identifier: registrar-verification + sha256: sha256:fdcf02c1ff87421d65b707e8dd0de30432d2650b9c53914e55002218d4da1cb1 + - id: civil-event--lookup-verify-registration--access-profile-supervisory-capability + path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-supervisory.capability.json mediaType: application/json + visibility: operation-bound operationIdentifier: civil-event.lookup.verify-registration - path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-supervisory.capability.json + accessBinding: + kind: access-profile + identifier: supervisory sha256: sha256:6d8006991a29b5a7d416e48d5a204125de51cf27db5e8e2d84c25dea88218dad - visibility: operation-bound - - accessBinding: null - id: civil-event--lookup-verify-registration--access-profile-supervisory-classifications + - id: civil-event--lookup-verify-registration--access-profile-supervisory-classifications + path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-supervisory.classifications.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-supervisory.classifications.json + accessBinding: null sha256: sha256:89cfc3c62a0c1181618fb840c104ac864d006722161a7ff4b8a9af72ba3d3565 - visibility: operator-only - - accessBinding: - identifier: supervisory - kind: access-profile - id: civil-event--lookup-verify-registration--access-profile-supervisory-context - mediaType: application/ld+json - operationIdentifier: civil-event.lookup.verify-registration + - id: civil-event--lookup-verify-registration--access-profile-supervisory-context path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-supervisory.context.jsonld - sha256: sha256:d36d8bb5bb9fbf7ef9daaf4d91aaeefb94df8d97ae2db5b8ffff5c097d252b33 + mediaType: application/ld+json visibility: operation-bound - - accessBinding: - identifier: supervisory - kind: access-profile - id: civil-event--lookup-verify-registration--access-profile-supervisory-processing - mediaType: application/json operationIdentifier: civil-event.lookup.verify-registration + accessBinding: + kind: access-profile + identifier: supervisory + sha256: sha256:d36d8bb5bb9fbf7ef9daaf4d91aaeefb94df8d97ae2db5b8ffff5c097d252b33 + - id: civil-event--lookup-verify-registration--access-profile-supervisory-processing path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-supervisory.processing.json - sha256: sha256:b0f5dcd7b36e3a39c585a7e31ad16327332abb7c89ff4f982248cbfbeb490633 + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: supervisory - kind: access-profile - id: civil-event--lookup-verify-registration--access-profile-supervisory-schema - mediaType: application/schema+json operationIdentifier: civil-event.lookup.verify-registration + accessBinding: + kind: access-profile + identifier: supervisory + sha256: sha256:b0f5dcd7b36e3a39c585a7e31ad16327332abb7c89ff4f982248cbfbeb490633 + - id: civil-event--lookup-verify-registration--access-profile-supervisory-schema path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-supervisory.schema.json - sha256: sha256:7dd826ceec7cee6387b0df9b8509e4bed1aac0e11df6b523faee9b2a5a6eb8fc + mediaType: application/schema+json visibility: operation-bound - - accessBinding: - identifier: supervisory - kind: access-profile - id: civil-event--lookup-verify-registration--access-profile-supervisory-shacl - mediaType: text/turtle operationIdentifier: civil-event.lookup.verify-registration + accessBinding: + kind: access-profile + identifier: supervisory + sha256: sha256:7dd826ceec7cee6387b0df9b8509e4bed1aac0e11df6b523faee9b2a5a6eb8fc + - id: civil-event--lookup-verify-registration--access-profile-supervisory-shacl path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-supervisory.shacl.ttl - sha256: sha256:0198ff1898b2c460329f2286c76395ad43696734e11196cf5a39ab252e4a62ea + mediaType: text/turtle visibility: operation-bound - - accessBinding: - identifier: supervisory - kind: access-profile - id: civil-event--lookup-verify-registration--access-profile-supervisory-vocabulary - mediaType: application/ld+json operationIdentifier: civil-event.lookup.verify-registration + accessBinding: + kind: access-profile + identifier: supervisory + sha256: sha256:0198ff1898b2c460329f2286c76395ad43696734e11196cf5a39ab252e4a62ea + - id: civil-event--lookup-verify-registration--access-profile-supervisory-vocabulary path: generated/artifacts/civil-event--lookup-verify-registration--access-profile-supervisory.vocabulary.jsonld - sha256: sha256:f69da73736b4c0847cb66bb1524fb8f6182b7ff71c81eecc73f3588778d172d7 + mediaType: application/ld+json visibility: operation-bound - - accessBinding: - identifier: registrar + operationIdentifier: civil-event.lookup.verify-registration + accessBinding: kind: access-profile - id: civil-event--read--access-profile-registrar-capability + identifier: supervisory + sha256: sha256:f69da73736b4c0847cb66bb1524fb8f6182b7ff71c81eecc73f3588778d172d7 + - id: civil-event--read--access-profile-registrar-capability + path: generated/artifacts/civil-event--read--access-profile-registrar.capability.json mediaType: application/json + visibility: operation-bound operationIdentifier: civil-event.read - path: generated/artifacts/civil-event--read--access-profile-registrar.capability.json + accessBinding: + kind: access-profile + identifier: registrar sha256: sha256:0d3aedb3ca083f1c61f2b3eeecb5277fec483e033832aed89f0e95888a41f412 - visibility: operation-bound - - accessBinding: null - id: civil-event--read--access-profile-registrar-classifications + - id: civil-event--read--access-profile-registrar-classifications + path: generated/artifacts/civil-event--read--access-profile-registrar.classifications.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/civil-event--read--access-profile-registrar.classifications.json + accessBinding: null sha256: sha256:8dd41e9146bcb6972dd66ba897c35df25472584ac278dadea283a3e4006c055a - visibility: operator-only - - accessBinding: - identifier: registrar - kind: access-profile - id: civil-event--read--access-profile-registrar-context - mediaType: application/ld+json - operationIdentifier: civil-event.read + - id: civil-event--read--access-profile-registrar-context path: generated/artifacts/civil-event--read--access-profile-registrar.context.jsonld - sha256: sha256:35bcada2f88b65a70a952b348d893df55f839aec9146eadcada5e55d6151d1f3 + mediaType: application/ld+json visibility: operation-bound - - accessBinding: - identifier: registrar - kind: access-profile - id: civil-event--read--access-profile-registrar-processing - mediaType: application/json operationIdentifier: civil-event.read + accessBinding: + kind: access-profile + identifier: registrar + sha256: sha256:35bcada2f88b65a70a952b348d893df55f839aec9146eadcada5e55d6151d1f3 + - id: civil-event--read--access-profile-registrar-processing path: generated/artifacts/civil-event--read--access-profile-registrar.processing.json - sha256: sha256:fdb6099a53ca0f2127a5d45e8084947829de9d95de7ab49a8e32ff4abf6bfe9c + mediaType: application/json visibility: operation-bound - - accessBinding: - identifier: registrar - kind: access-profile - id: civil-event--read--access-profile-registrar-schema - mediaType: application/schema+json operationIdentifier: civil-event.read + accessBinding: + kind: access-profile + identifier: registrar + sha256: sha256:fdb6099a53ca0f2127a5d45e8084947829de9d95de7ab49a8e32ff4abf6bfe9c + - id: civil-event--read--access-profile-registrar-schema path: generated/artifacts/civil-event--read--access-profile-registrar.schema.json - sha256: sha256:0bfa1e08af652ffaf9c74a4c1960c316437b8f3647c8fba3bef9265ef798c06e + mediaType: application/schema+json visibility: operation-bound - - accessBinding: - identifier: registrar - kind: access-profile - id: civil-event--read--access-profile-registrar-shacl - mediaType: text/turtle operationIdentifier: civil-event.read + accessBinding: + kind: access-profile + identifier: registrar + sha256: sha256:0bfa1e08af652ffaf9c74a4c1960c316437b8f3647c8fba3bef9265ef798c06e + - id: civil-event--read--access-profile-registrar-shacl path: generated/artifacts/civil-event--read--access-profile-registrar.shacl.ttl - sha256: sha256:d76a102df435cabfb9aada54a4a257b3aeac37bd46db157f2a095e035ee5461d + mediaType: text/turtle visibility: operation-bound - - accessBinding: - identifier: registrar + operationIdentifier: civil-event.read + accessBinding: kind: access-profile - id: civil-event--read--access-profile-registrar-vocabulary + identifier: registrar + sha256: sha256:d76a102df435cabfb9aada54a4a257b3aeac37bd46db157f2a095e035ee5461d + - id: civil-event--read--access-profile-registrar-vocabulary + path: generated/artifacts/civil-event--read--access-profile-registrar.vocabulary.jsonld mediaType: application/ld+json + visibility: operation-bound operationIdentifier: civil-event.read - path: generated/artifacts/civil-event--read--access-profile-registrar.vocabulary.jsonld + accessBinding: + kind: access-profile + identifier: registrar sha256: sha256:6a8225b7efed28ae336c11cbeec58bc94eaf89dcd18d2a097bc76c470f33ab85 - visibility: operation-bound - - accessBinding: null - id: civil-event-classification + - id: civil-event-classification + path: generated/artifacts/civil-event.classifications.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/civil-event.classifications.json + accessBinding: null sha256: sha256:f5bbe318289cea6113fcf05873855ca972c0e0a1796d66f2b9ec28265cdaa25d - visibility: operator-only - - accessBinding: null - id: civil-event-codelist-0 + - id: civil-event-codelist-0 + path: generated/artifacts/civil-event.codelist-0.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/civil-event.codelist-0.schema.json + accessBinding: null sha256: sha256:cbd45c06b830956e657b9e930bdd9479278f42061c7333dd5694a57a5b2a0c73 - visibility: operator-only - - accessBinding: null - id: civil-event-codelist-1 + - id: civil-event-codelist-1 + path: generated/artifacts/civil-event.codelist-1.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/civil-event.codelist-1.schema.json + accessBinding: null sha256: sha256:e42dfbcab45a66032d126e0f203523ae44a6bc034278f2ce222f96f1ff0a78f0 - visibility: operator-only - - accessBinding: null - id: civil-event-codelist-2 + - id: civil-event-codelist-2 + path: generated/artifacts/civil-event.codelist-2.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/civil-event.codelist-2.schema.json + accessBinding: null sha256: sha256:c770e1867500e4c771718f0d412bc92be30d138fda628a85cff787f67ec9db09 - visibility: operator-only - - accessBinding: null - id: civil-event-codelist-3 + - id: civil-event-codelist-3 + path: generated/artifacts/civil-event.codelist-3.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/civil-event.codelist-3.schema.json + accessBinding: null sha256: sha256:3dd13f1498de4f4b16597ae4285412ef9e4b9859da058e45168a7de2e2252655 - visibility: operator-only - - accessBinding: null - id: civil-event-full-schema + - id: civil-event-full-schema + path: generated/artifacts/civil-event.full.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/civil-event.full.schema.json + accessBinding: null sha256: sha256:8ce53651000a4e825df4964ae4555d98b77f3b4aa15fa7eb0d97b8cdc9a0cb7d - visibility: operator-only - - accessBinding: null - id: civil-event-full-shacl + - id: civil-event-full-shacl + path: generated/artifacts/civil-event.full.shacl.ttl mediaType: text/turtle + visibility: operator-only operationIdentifier: null - path: generated/artifacts/civil-event.full.shacl.ttl + accessBinding: null sha256: sha256:cdf718da6547e9dc6dec21ada8ef74662d04636b6854215c3f0bed6f879cc560 - visibility: operator-only - - accessBinding: null - id: civil-event-full-vocabulary + - id: civil-event-full-vocabulary + path: generated/artifacts/civil-event.full.vocabulary.jsonld mediaType: application/ld+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/civil-event.full.vocabulary.jsonld + accessBinding: null sha256: sha256:437d021d8cd85c4e7847dc9df983c7375a8332efb0b5ac77ce7a5fda4fda3b58 - visibility: operator-only - - accessBinding: null - id: civil-event-processing-full + - id: civil-event-processing-full + path: generated/artifacts/civil-event.processing.full.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/civil-event.processing.full.json + accessBinding: null sha256: sha256:762086646e734b6a8248a6bb62675490edc9a303559dca7e08719925656fec40 - visibility: operator-only - - accessBinding: null - id: discovery-description + - id: discovery-description + path: generated/artifacts/discovery.jsonld mediaType: application/ld+json;profile="https://registrystack.org/discovery/profile/v1alpha1" + visibility: public operationIdentifier: null - path: generated/artifacts/discovery.jsonld + accessBinding: null sha256: sha256:57804ccece4aa71d7b882cbf4f8b28c7b4787c4ca41d76bbcd4e94cc8d26414b - visibility: public - - accessBinding: null - id: openapi-full + - id: openapi-full + path: generated/openapi.full.yaml mediaType: application/yaml + visibility: operator-only operationIdentifier: null - path: generated/openapi.full.yaml + accessBinding: null sha256: sha256:75f04195ce58fce80908834c4fe4c1b0f88e1136057a1b60712cd2d421cc4b8f - visibility: operator-only - - accessBinding: null - id: openapi-public + - id: openapi-public + path: generated/openapi.public.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/openapi.public.json + accessBinding: null sha256: sha256:9bbc84b3631b6410d07228e095dc9f44ba3862a5f8d7445ee323168730a3be85 - visibility: public governedFiles: - - generated: false - mediaType: application/yaml - path: governed/codelists/civil-event-selector-types.yaml + - path: governed/codelists/civil-event-selector-types.yaml sha256: sha256:b50078dbd85a1aef52a98394578e8d4b8e3132833f9e5d785b0f60854d0d53c6 - size: 92 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/codelists/civil-event-types.yaml + bytes: 92 + - path: governed/codelists/civil-event-types.yaml sha256: sha256:3a7f08db1c4b675c93f33cc85ac51506d67c276d400003a10a5669073225060f - size: 83 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/codelists/record-lifecycle.yaml + bytes: 83 + - path: governed/codelists/record-lifecycle.yaml sha256: sha256:b1cf74b2b256bc0702afb312eb75bc024ebd8196c234858208c55efbd25a13d7 - size: 95 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/codelists/registration-areas.yaml + bytes: 95 + - path: governed/codelists/registration-areas.yaml sha256: sha256:7f6c0cc66d81529315bf9c9e2c5e5691fe166ebb6136283ed6ed35cf12193e9a - size: 86 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/codelists/registration-status.yaml + bytes: 86 + - path: governed/codelists/registration-status.yaml sha256: sha256:7b26678d41f6705d3bb83d274b234834f3c98f5265a93151417fe91d499d2cf4 - size: 104 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/classification-review-rationale.md + bytes: 104 + - path: governed/governance/classification-review-rationale.md sha256: sha256:58606b2d7a0c69145ca9f1e951f2851701b75b0753afa86801f89acf51a20685 - size: 258 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/classification-review.yaml + bytes: 258 + - path: governed/governance/classification-review.yaml sha256: sha256:e07c84d64d973249f2ac729c29c0ef5e3d0f83b41f21d42138d55fbc04939d9c - size: 423 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/identifier-lifecycle.yaml + bytes: 423 + - path: governed/governance/identifier-lifecycle.yaml sha256: sha256:5f85e60331b58db6c85a9b54c7c9c5a6035045a4e557bd8ab83183958d15c9e4 - size: 253 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/legal-basis.yaml + bytes: 253 + - path: governed/governance/legal-basis.yaml sha256: sha256:7a20ef65053c949a3a78b3ea198752de5e9533566104e69cab6d0c60e804f1cf - size: 237 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/semantics/publicschema-event-alignment.yaml + bytes: 237 + - path: governed/semantics/publicschema-event-alignment.yaml sha256: sha256:c89af1aae44c66ce3ef8e6a1e6e6d9b063c50559d288b7ed290dff74422ea9d9 - size: 391 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: registry.yaml + bytes: 391 + - path: registry.yaml sha256: sha256:188777ffd96a101394ef5b628896831ba724e4bd9ec623df81a96b8587490675 - size: 8828 - visibility: operator-only + bytes: 8828 labour-statistics: - packageRevision: sha256:a9aa3523ef6a805480aad3230fb99090bd33164f53660bcd7593f0396387242e + packageDigest: sha256:39dba945bbd87492c7c32e030f8f9528e583ca088b8995bc88a76af17abac22a contractRevision: sha256:5fa07359d1278ada6adaf24c6f838e8dfb7ac9e5d5b6774683d71c66c3f2ee0d sourceSchemaFingerprints: labour-statistics: sha256:21890387ccf8d33a95a4ba38d085b6eac79e4522b8a8b34a10a4a637711b4034 artifacts: - - accessBinding: null - id: audit-event-schema + - id: audit-event-schema + path: generated/artifacts/audit-event.schema.json mediaType: application/schema+json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/audit-event.schema.json + accessBinding: null sha256: sha256:41358576a5fcbf4a6182f22b2213f26dadd762e3a1c6d76ea1c0948106b7e6e2 - visibility: operator-only - - accessBinding: null - id: capability-inventory-full + - id: capability-inventory-full + path: generated/artifacts/capabilities.full.json mediaType: application/json + visibility: operator-only operationIdentifier: null - path: generated/artifacts/capabilities.full.json + accessBinding: null sha256: sha256:2c90384770bde74a9148090f075f5781b80f3800aeea62a7e6226eb0eb189a99 - visibility: operator-only - - accessBinding: null - id: capability-inventory + - id: capability-inventory + path: generated/artifacts/capabilities.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/artifacts/capabilities.json + accessBinding: null sha256: sha256:733c513f641adcfa50106e08f54160e7f733512d67f272d73bf6e9d042ebf9e9 - visibility: public - - accessBinding: null - id: discovery-description + - id: discovery-description + path: generated/artifacts/discovery.jsonld mediaType: application/ld+json;profile="https://registrystack.org/discovery/profile/v1alpha1" + visibility: public operationIdentifier: null - path: generated/artifacts/discovery.jsonld + accessBinding: null sha256: sha256:612225f14fbcb191b4dd3abd2472e05176474b18773ec0343071712655373ba4 - visibility: public - - accessBinding: - kind: fixed-operation - id: labour-force-authority-sdmx-dataflow-structure - mediaType: application/vnd.sdmx.structure+json;version=2.1.0 - operationIdentifier: labour-force-authority.statistics.read + - id: labour-force-authority-sdmx-dataflow-structure path: generated/artifacts/labour-force-authority.sdmx.dataflow.json - sha256: sha256:7b200550e0979a6057ab874149707f3c988cbc7ef284d05355ee7f2b78959034 - visibility: operation-bound - - accessBinding: - kind: fixed-operation - id: labour-force-authority-sdmx-datastructure-structure mediaType: application/vnd.sdmx.structure+json;version=2.1.0 + visibility: operation-bound operationIdentifier: labour-force-authority.statistics.read + accessBinding: + kind: fixed-operation + sha256: sha256:7b200550e0979a6057ab874149707f3c988cbc7ef284d05355ee7f2b78959034 + - id: labour-force-authority-sdmx-datastructure-structure path: generated/artifacts/labour-force-authority.sdmx.datastructure.json - sha256: sha256:a7ec6833c0c722a540301de6a56d6a9cbe45dd13baca244f040d90c8b82a9fc5 + mediaType: application/vnd.sdmx.structure+json;version=2.1.0 visibility: operation-bound - - accessBinding: + operationIdentifier: labour-force-authority.statistics.read + accessBinding: kind: fixed-operation - id: labour-force-participation-sdmx-dataflow-structure - mediaType: application/vnd.sdmx.structure+json;version=2.1.0 - operationIdentifier: labour-force-participation.statistics.read + sha256: sha256:a7ec6833c0c722a540301de6a56d6a9cbe45dd13baca244f040d90c8b82a9fc5 + - id: labour-force-participation-sdmx-dataflow-structure path: generated/artifacts/labour-force-participation.sdmx.dataflow.json - sha256: sha256:1fb3ceabc947633a622beccd26e61856af883ac95e491f06fae58897cf8537ce + mediaType: application/vnd.sdmx.structure+json;version=2.1.0 visibility: public - - accessBinding: + operationIdentifier: labour-force-participation.statistics.read + accessBinding: kind: fixed-operation - id: labour-force-participation-sdmx-datastructure-structure + sha256: sha256:1fb3ceabc947633a622beccd26e61856af883ac95e491f06fae58897cf8537ce + - id: labour-force-participation-sdmx-datastructure-structure + path: generated/artifacts/labour-force-participation.sdmx.datastructure.json mediaType: application/vnd.sdmx.structure+json;version=2.1.0 + visibility: public operationIdentifier: labour-force-participation.statistics.read - path: generated/artifacts/labour-force-participation.sdmx.datastructure.json + accessBinding: + kind: fixed-operation sha256: sha256:3cf4b8413da33f44efd8cc5da65f3469903c6a5682041a589cd68d1bdeb58b0e - visibility: public - - accessBinding: null - id: openapi-full + - id: openapi-full + path: generated/openapi.full.yaml mediaType: application/yaml + visibility: operator-only operationIdentifier: null - path: generated/openapi.full.yaml + accessBinding: null sha256: sha256:3c7e37ea89674eb5d6512dd47a90afdad2b01660eaac09797541081832374f72 - visibility: operator-only - - accessBinding: null - id: openapi-public + - id: openapi-public + path: generated/openapi.public.json mediaType: application/json + visibility: public operationIdentifier: null - path: generated/openapi.public.json + accessBinding: null sha256: sha256:bd7c23117ffdb03e4f620005a40e8cfc9a6476af295ef891ff774aa5e1c0076f - visibility: public governedFiles: - - generated: false - mediaType: application/yaml - path: governed/codelists/areas.yaml + - path: governed/codelists/areas.yaml sha256: sha256:2b671910eed1c82dc5fe4e92dd66af8b65b1aff0e5bf170faa5d142c572fe954 - size: 85 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/codelists/sex.yaml + bytes: 85 + - path: governed/codelists/sex.yaml sha256: sha256:f5b1b7e4bf552939066482863eb6e231eb1b45a40af0af590b1a35a2001dafa8 - size: 67 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/codelists/units.yaml + bytes: 67 + - path: governed/codelists/units.yaml sha256: sha256:058110db16e7819646007df66c990708e131d5b8b884292f27938813ee8a8b7c - size: 72 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/classification-review-rationale.md + bytes: 72 + - path: governed/governance/classification-review-rationale.md sha256: sha256:2a6b99b26c1784ac37e2fa9a0c0f0feb4902cb49b5a0004cefd4184a170125dc - size: 394 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/classification-review.yaml + bytes: 394 + - path: governed/governance/classification-review.yaml sha256: sha256:1ccf9b7ba3f636fa4823c4045e9bb275dd25a385143f912d41b4a1a00d62b5c8 - size: 417 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/identifier-lifecycle.yaml + bytes: 417 + - path: governed/governance/identifier-lifecycle.yaml sha256: sha256:ff5e3188599db11ac8f4a3d6b5a98214d3506aca037d6621882433c7fa3a61e8 - size: 198 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: governed/governance/legal-basis.yaml + bytes: 198 + - path: governed/governance/legal-basis.yaml sha256: sha256:eba91ee1f22227fb6a4bd82bf6a80f2d71899ea137d5da3fd2e1a4a9ee483faa - size: 284 - visibility: operator-only - - generated: false - mediaType: application/yaml - path: registry.yaml + bytes: 284 + - path: registry.yaml sha256: sha256:0c5f25437b86c815ad6202e65a1a5602fdb52a80e2922258d061cc347627d1e8 - size: 6440 - visibility: operator-only + bytes: 6440 diff --git a/products/relay-v2/contracts/security-invariant-matrix.yaml b/products/relay-v2/contracts/security-invariant-matrix.yaml index b39550ede6..843f6780a8 100644 --- a/products/relay-v2/contracts/security-invariant-matrix.yaml +++ b/products/relay-v2/contracts/security-invariant-matrix.yaml @@ -20,7 +20,7 @@ invariants: - {path: crates/registry-relay-v2/tests/acceptance_http.rs, name: provider_discovery_description_route_serves_compiled_exact_bytes_without_authentication} - id: sec-contract-runtime-separation threat: Deployment configuration weakens governed disclosure or authorization. - enforcementPoint: RegistryContract and RelayRuntime closed-schema compilation before readiness. + enforcementPoint: RegistryContract and RelayRuntimeConfig closed-schema compilation before readiness. expected: Runtime configuration cannot add or alter resources, operations, disclosure, semantics, classification, access, or metadata visibility. evidence: config-validation negativeTest: runtime_rejects_governed_override @@ -28,12 +28,29 @@ invariants: - {path: crates/registry-relay-v2/src/contract.rs, name: runtime_rejects_governed_override} - id: sec-package-activation-integrity threat: A sealed package activates a compiled access or disclosure model that was not derived from its captured contract and governed files. - enforcementPoint: Package construction and startup independently reproduce the compiled Registry from the captured contract, observed schemas, and governed closure and require exact equality before activation. - expected: Same-identity semantic mismatch, file tampering, or inconsistent artifacts prevent packaging or readiness before source or listener activation. + enforcementPoint: Startup verifies the package's SHA256SUMS in the shared package format, re-reads every file against its verified digest, requires the file set to equal what the compiled Registry names, and independently reproduces the compiled Registry and every artifact from the captured contract, observed schemas, and governed closure, requiring exact equality before activation. + expected: A changed, missing, or extra file is refused by name with relayctl package and without the directory; a resealed package whose compiled Registry, artifacts, or file set its inputs do not reproduce, a retired relay-package.json, or a mismatched package.expectedDigest prevents readiness before source or listener activation. evidence: sealed-package-derivation-tests - negativeTest: sealed_package_reproduces_and_tampering_is_refused - tests: - - {path: crates/registry-relay-v2/src/package.rs, name: sealed_package_reproduces_and_tampering_is_refused} + negativeTest: a_changed_missing_or_extra_file_is_refused_by_name + tests: + - {path: crates/registry-relay-v2/src/package.rs, name: a_changed_missing_or_extra_file_is_refused_by_name} + - {path: crates/registry-relay-v2/src/package.rs, name: a_resealed_package_must_reproduce_its_compiled_registry_and_artifacts} + - {path: crates/registry-relay-v2/src/package.rs, name: a_resealed_package_must_hold_exactly_what_its_registry_names} + - {path: crates/registry-relay-v2/src/package.rs, name: a_directory_that_is_not_a_package_is_refused_with_the_command} + - {path: crates/registry-relay-v2/tests/process_http.rs, name: built_relay_check_honors_a_package_digest_pin} + - {path: crates/registry-relay-v2/tests/process_http.rs, name: built_relay_check_refuses_a_changed_package_file_by_name_without_a_pin} + - id: sec-runtime-secret-provider-boundary + threat: A deployment secret is read from an undeclared place, such as a file beside the runtime configuration or an environment variable the operator never enabled, or environment substitution rewrites a secret reference. + enforcementPoint: The shared runtime configuration loader and secret-provider block, checked before any secret is resolved. + expected: A secret:file reference resolves only under secretProviders.file.root, a secret:env reference resolves only when secretProviders.environment is declared, a Ref field refuses substitution, the runtime file must be absolute, bounded, and operator-owned, and every refusal names the field without echoing a value or path. + evidence: runtime-loader-and-secret-provider-tests + negativeTest: a_secret_reference_needs_its_provider_declared + tests: + - {path: crates/registry-relay-v2/src/contract.rs, name: a_secret_reference_needs_its_provider_declared} + - {path: crates/registry-relay-v2/src/contract.rs, name: environment_substitution_never_reaches_a_secret_reference} + - {path: crates/registry-relay-v2/src/contract.rs, name: refusals_name_the_field_and_never_the_value} + - {path: crates/registry-relay-v2/src/startup.rs, name: secrets_resolve_only_through_the_declared_providers} + - {path: crates/registry-relay-v2/src/startup.rs, name: a_runtime_below_a_writable_ancestor_is_refused_without_naming_the_path} - id: sec-one-registry-boundary threat: Source, operation, disclosure, quota, or audit state crosses resource boundaries within one compiled Registry. enforcementPoint: Resource-qualified compiled operations and operation-keyed runtime state. @@ -74,12 +91,14 @@ invariants: - {path: crates/registry-relay-v2/src/auth.rs, name: malformed_subject_cannot_fall_back_to_client_identifier} - id: sec-issuer-identity-transport-separation threat: A rollout or internal discovery hostname silently changes the JWT issuer Relay trusts, or two configured key transports create an ambiguous trust source. - enforcementPoint: Closed RelayRuntime issuer profile, exact discovery metadata comparison, and exact JWT issuer verification. - expected: Relay accepts exactly one discovery or direct JWKS transport, validates trustedIssuer as the exact token iss value independently of the transport authority, refuses discovery metadata declaring another issuer, and preserves legacy derivation only for the canonical discovery URL form. + enforcementPoint: Closed RelayRuntimeConfig authentication.oidc profile, one jwksSource, exact discovery metadata comparison, and exact JWT issuer verification. + expected: Relay accepts one issuer with exactly one key source, discovery derived from the issuer or one exact JWKS uri, validates authentication.oidc.issuer as the exact token iss value independently of the JWKS transport authority, refuses discovery metadata declaring another issuer, and refuses the static key source. evidence: issuer-transport-configuration-and-real-router-token-verification-tests negativeTest: real_jwt_path_uses_trusted_issuer_not_the_jwks_transport_host tests: - - {path: crates/registry-relay-v2/src/contract.rs, name: runtime_separates_trusted_issuer_from_one_key_transport} + - {path: crates/registry-relay-v2/src/contract.rs, name: uri_key_source_keeps_the_issuer_separate_from_the_key_transport} + - {path: crates/registry-relay-v2/src/contract.rs, name: discovery_key_source_derives_the_issuer_discovery_document} + - {path: crates/registry-relay-v2/src/contract.rs, name: static_key_source_is_refused} - {path: crates/registry-relay-v2/tests/acceptance_http.rs, name: real_jwt_path_uses_trusted_issuer_not_the_jwks_transport_host} - {path: crates/registry-platform-oidc/src/lib.rs, name: explicit_discovery_transport_preserves_the_canonical_issuer} - {path: crates/registry-platform-oidc/src/lib.rs, name: oidc_userinfo_jwt_requires_issuer_audience_and_matching_subject} diff --git a/products/relay-v2/scripts/test_adopter_workflow.py b/products/relay-v2/scripts/test_adopter_workflow.py index f768f32630..4ffe5f1e07 100755 --- a/products/relay-v2/scripts/test_adopter_workflow.py +++ b/products/relay-v2/scripts/test_adopter_workflow.py @@ -270,35 +270,48 @@ def validate_openapi(package: Path, artifacts: list[dict[str, Any]]) -> None: raise GateFailure("public OpenAPI capability paths do not match public discovery") -def validate_exposure_and_identity(package: Path, generated: Path) -> dict[str, Any]: - manifest = json.loads((package / "relay-package.json").read_text(encoding="utf-8")) - if manifest.get("packageVersion") != "relay.registrystack.org/package/v1alpha3": - raise GateFailure("sealed package has an unsupported manifest") - artifacts = manifest.get("artifacts") - operation_bindings = manifest.get("operationArtifactBindings") - files = manifest.get("files") - if ( - not isinstance(artifacts, list) - or not isinstance(operation_bindings, list) - or not isinstance(files, list) - ): - raise GateFailure("sealed package inventory is incomplete") +def package_sums(package: Path) -> dict[str, str]: + sums = (package / "SHA256SUMS").read_text(encoding="utf-8") + if not sums.endswith("\n"): + raise GateFailure("package SHA256SUMS does not end with a line feed") + listed: dict[str, str] = {} + for line in sums[:-1].split("\n"): + digest, separator, path = line.partition(" ") + if not separator or len(digest) != 64 or path in listed: + raise GateFailure("package SHA256SUMS is malformed") + listed[path] = f"sha256:{digest}" + if list(listed) != sorted(listed): + raise GateFailure("package SHA256SUMS is not sorted") + return listed + + +def validate_exposure_and_identity( + package: Path, generated: Path, summary: dict[str, Any] +) -> dict[str, Any]: + if summary.get("packageDigest") != file_sha256(package / "SHA256SUMS"): + raise GateFailure("package digest is not the digest of its SHA256SUMS") + on_disk = { + path.relative_to(package).as_posix() + for path in package.rglob("*") + if path.is_file() + } + listed = package_sums(package) + if on_disk != set(listed) | {"SHA256SUMS"}: + raise GateFailure("package files and SHA256SUMS disagree") + for path, digest in listed.items(): + if file_sha256(package / path) != digest: + raise GateFailure("package file bytes do not match SHA256SUMS") + artifacts = summary.get("artifacts") + files = summary.get("files") + if not isinstance(artifacts, list) or not isinstance(files, list): + raise GateFailure("package report inventory is incomplete") file_inventory = {entry["path"]: entry for entry in files} if len(file_inventory) != len(files): - raise GateFailure("sealed package contains duplicate file inventory paths") - compiled = file_inventory.get("compiled/registry.json") - if ( - not compiled - or not compiled.get("generated") - or compiled.get("visibility") != "operator-only" - ): - raise GateFailure("sealed package omits its operator-only compiled Registry") - for entry in files: - path = package / entry["path"] - if not path.is_file() or file_sha256(path) != entry.get("sha256"): - raise GateFailure("sealed package file bytes do not match their inventory") - if not entry.get("generated") and entry.get("visibility") != "operator-only": - raise GateFailure("an authored governed file is not operator-only") + raise GateFailure("package report contains duplicate file inventory paths") + if {path: entry["sha256"] for path, entry in file_inventory.items()} != listed: + raise GateFailure("package report and SHA256SUMS disagree") + if "compiled/registry.json" not in file_inventory: + raise GateFailure("package omits its compiled Registry") artifact_ids: set[str] = set() for artifact in artifacts: identifier = artifact.get("id") @@ -306,10 +319,8 @@ def validate_exposure_and_identity(package: Path, generated: Path) -> dict[str, if identifier in artifact_ids or path not in file_inventory: raise GateFailure("generated artifact inventory is not one-to-one") artifact_ids.add(identifier) - file_entry = file_inventory[path] - for key in ("mediaType", "visibility", "sha256"): - if artifact.get(key) != file_entry.get(key): - raise GateFailure("artifact exposure inventory disagrees with file inventory") + if artifact.get("sha256") != file_inventory[path]["sha256"]: + raise GateFailure("artifact exposure inventory disagrees with file inventory") visibility = artifact.get("visibility") operation = artifact.get("operationIdentifier") access_binding = artifact.get("accessBinding") @@ -344,16 +355,20 @@ def validate_exposure_and_identity(package: Path, generated: Path) -> dict[str, if by_id.get("openapi-public", {}).get("visibility") != "public": raise GateFailure("public OpenAPI is not explicitly public") validate_openapi(package, artifacts) - return manifest + return summary -def baseline(manifest: dict[str, Any]) -> dict[str, Any]: +def baseline(summary: dict[str, Any]) -> dict[str, Any]: return { - "packageRevision": manifest["packageRevision"], - "contractRevision": manifest["contractRevision"], - "sourceSchemaFingerprints": manifest["sourceSchemaFingerprints"], - "artifacts": manifest["artifacts"], - "governedFiles": [entry for entry in manifest["files"] if not entry["generated"]], + "packageDigest": summary["packageDigest"], + "contractRevision": summary["contractRevision"], + "sourceSchemaFingerprints": summary["sourceSchemaFingerprints"], + "artifacts": summary["artifacts"], + "governedFiles": [ + entry + for entry in summary["files"] + if not entry["path"].startswith(("compiled/", "generated/")) + ], } @@ -492,9 +507,9 @@ def accepted(arguments: list[str]) -> dict[str, Any]: exercise_nontrivial_diff(accepted, project_name, project, previous, root) package_report = accepted(["package", str(project), "--output", str(root / "package")]) - manifest = validate_exposure_and_identity(root / "package", root / "generated") - if package_report["details"]["manifest"] != manifest: - raise GateFailure(f"{project_name}: package report bytes and sealed manifest differ") + summary = validate_exposure_and_identity( + root / "package", root / "generated", package_report["details"]["package"] + ) drift = root / "schema-drift" shutil.copytree(project, drift) @@ -517,7 +532,7 @@ def accepted(arguments: list[str]) -> dict[str, Any]: key_paths = check["details"].get("configuration_key_paths") if not isinstance(key_paths, dict): raise GateFailure(f"{project_name}: shared check report omitted configuration key paths") - return reports + [refusal], outputs, {"manifest": manifest, "keyPaths": key_paths} + return reports + [refusal], outputs, {"package": summary, "keyPaths": key_paths} def documented_key_paths(text: str, marker: str) -> set[str]: @@ -573,7 +588,7 @@ def main() -> int: raise GateFailure(f"{project_name}: {error}") from error canaries = protected_canaries(PRODUCT_ROOT / "acceptance" / project_name) assert_value_free(outputs, canaries, project_name) - snapshots[project_name] = baseline(result["manifest"]) + snapshots[project_name] = baseline(result["package"]) for kind in key_paths: key_paths[kind].update(result["keyPaths"][kind]) diff --git a/products/relay-v2/scripts/validate_product.py b/products/relay-v2/scripts/validate_product.py index 18d78ea7b6..b0b7df27c1 100644 --- a/products/relay-v2/scripts/validate_product.py +++ b/products/relay-v2/scripts/validate_product.py @@ -52,6 +52,7 @@ "sec-provider-public-projection", "sec-contract-runtime-separation", "sec-package-activation-integrity", + "sec-runtime-secret-provider-boundary", "sec-one-registry-boundary", "sec-sqlite-read-only", "sec-sqlite-connection-recovery", diff --git a/products/render/ACCEPTANCE.md b/products/render/ACCEPTANCE.md index 8cb9e1733a..6096e41be2 100644 --- a/products/render/ACCEPTANCE.md +++ b/products/render/ACCEPTANCE.md @@ -3,24 +3,25 @@ Binds each Definition-of-Done area to its executable evidence. Run everything with `cargo test --locked -p registry-render` (CI runs the same suites on two OSes via `.github/workflows/render-golden.yml`). Updated for -the PR #1113 review round (2026-09-17) and immutable bundle snapshot -hardening (2026-09-19); see EVIDENCE.md for the change list. +the PR #1113 review round (2026-09-17), immutable bundle snapshot +hardening (2026-09-19), and the shared package migration; see EVIDENCE.md +for the change list. | DoD area | Executable evidence | |---|---| | Merge gate (library ≡ CLI) | `EVIDENCE.md` procedure + hashes (2026-09-17, all three bundles, byte-identical); `golden_hashes_match` keeps the library side pinned | | Determinism (fresh world/library, canonical injection, ident, evict, font order) | `golden.rs`: `golden_hashes_match`, `rendering_is_deterministic_across_fresh_worlds_and_processes`, `injected_bytes_are_the_hashed_canonical_bytes`, `issued_at_changes_bytes_and_is_the_only_knob` | -| Bundle format, sealing, immutable verified-byte consumption, tamper/unsealed refusals | `golden.rs`: `tampered_sealed_bundle_is_refused`, `sealed_template_and_package_bytes_are_bound_to_the_loaded_snapshot`, `unsealed_bundle_is_refused_for_serving`; unit: `bundle::tests::assembly_uses_captured_bytes`, `bundle::tests::accepted_manifest_path_spellings_render_from_snapshot`, `bundle::tests::bundle_root_and_ancestor_symlinks_are_refused_for_every_spelling`, `bundle::tests::sealed_load_checks_the_manifest_before_capturing_descendants`, `manifest::tests`, `problem::tests` | -| Path safety (world-enforced; virtual-path diagnostics) | `golden.rs`: `data_paths_cannot_escape_the_bundle`, `unvendored_package_import_fails_without_network`, `compile_diagnostics_report_virtual_paths_only`; unit: `world::tests::virtual_paths_reject_escape_attempts`, manifest schema-path validation tests; scaffold symlink-seal refusal | +| Package format, repeatability, optional revision, immutable verified-byte consumption, drift and missing-envelope refusals | `scaffold.rs`: `package_command_writes_repeatable_verified_envelopes_and_revision`, `package_refuses_a_broken_bundle_before_writing_output`, `retired_seal_command_names_the_package_replacement`; `golden.rs`: `tampered_package_is_refused`, `verified_template_and_typst_package_bytes_are_bound_to_the_loaded_snapshot`, `directory_without_sum_file_is_not_a_package_but_remains_authoring_source`; unit: `bundle::tests::package_load_refuses_bytes_replaced_after_shared_verification`, `bundle::tests::assembly_uses_captured_bytes`, `bundle::tests::accepted_manifest_path_spellings_render_from_snapshot`, `bundle::tests::bundle_root_and_ancestor_symlinks_are_refused_for_every_spelling`, `bundle::tests::source_load_checks_the_manifest_before_capturing_descendants`, `manifest::tests`, `problem::tests` | +| Path safety (world-enforced; virtual-path diagnostics) | `golden.rs`: `data_paths_cannot_escape_the_bundle`, `unvendored_package_import_fails_without_network`, `compile_diagnostics_report_virtual_paths_only`; unit: `world::tests::virtual_paths_reject_escape_attempts`, manifest schema-path validation tests; `scaffold::bundle_with_symlink_cannot_be_packaged` | | Assets (media type, size caps, virtual namespace) | `golden.rs`: `wrong_media_type_and_oversize_assets_are_refused`; merge gate on the card bundle proves the virtual `assets/` namespace renders byte-identically to a real file | | Data validation with JSON pointers | `golden.rs`: `schema_violations_carry_json_pointers`, `bad_locale_is_refused_with_a_pointer` | -| Labels + script coverage at check time | `registry-render check` on all three bundles (CI smoke via the golden suite's sealed loads); coverage logic in `check.rs::check_script_coverage`, exercised by the sealed Arabic bundles; per-locale label key sets must agree: `scaffold::check_names_a_locale_missing_a_label_key` | +| Labels + script coverage at check time | `registry-render check` on all three packaged example bundles; coverage logic in `check.rs::check_script_coverage`, exercised by the Arabic packages; per-locale label key sets must agree: `scaffold::check_names_a_locale_missing_a_label_key` | | Resource enforcement (kill at timeout, recycle, recover) | `serve.rs`: `pathological_renders_are_bounded_and_the_service_recovers` (504 problem, service healthy afterwards, both kills audited); the CLI shares the wall: `scaffold::compile_is_bounded_by_a_timeout` | | Auth (constant-time, indistinguishable 401s, audited, key-file normalization) | `serve.rs`: `unauthorized_requests_are_refused_and_audited`, `api_key_file_with_one_trailing_newline_is_trimmed`, `api_key_with_stray_whitespace_is_refused_at_startup` | | HTTP contract (headers, JSON variant, correlation, problems, 413, versions in /health) | `serve.rs`: `render_returns_pdf_with_hash_headers_matching_golden`, `json_variant_serves_openfn_clients`, `missing_issued_at_and_bad_data_are_named_problems`, `oversized_bodies_are_refused_after_auth_as_problems`, `serve_health_and_ready` | | Audit (request entry before the render, response entry before the response, failures audited, value-free) | `serve.rs`: `a_render_writes_a_request_entry_then_a_response_entry_sharing_correlation`, `a_refused_request_entry_prevents_the_render`, `a_refused_response_entry_withholds_the_pdf`, `audit_events_are_value_free` (canary + API-key scans), `unauthorized_requests_are_refused_and_audited`, `pathological_renders_are_bounded_and_the_service_recovers`; unit: `server::tests::the_request_entry_is_accepted_before_the_render_starts`, `server::tests::a_refused_request_entry_prevents_the_render`, `server::tests::a_refusal_before_the_render_is_one_response_entry`, `server::tests::a_refusal_after_the_render_starts_keeps_the_render_identity`, `server::tests::concurrent_calls_sharing_an_idempotency_key_pair_their_own_entries`, `runtime::tests::the_audit_block_takes_the_shared_destination_shape`, `runtime::tests::an_audit_block_outside_the_shared_shape_is_refused`; `scaffold.rs`: `check_proves_the_audit_file_resolves_under_the_root`, `check_refuses_to_prove_a_stdout_audit_destination` | -| Sealed-bundle-only serving (startup and per render) | `serve.rs`: `tampered_bundle_refuses_to_serve` (exit code = BundleTampered), `bundle_drift_after_serve_starts_is_refused_per_render` (tamper and unseal after startup) | -| Cross-machine byte stability; closure drift | `.github/workflows/render-golden.yml` runs the golden, serve, and scaffold suites on ubuntu-24.04 and macos-14 against the same `golden.json`; `golden_hashes_match` also pins each bundle's file closure and requires every non-virtual dep to be manifest-governed | +| Package-only serving, pinning, and startup/per-render verification | `serve.rs`: `serve_startup_package_digest_mismatch_uses_common_expected_and_found_shape`, `package_changed_missing_and_extra_files_refuse_startup_by_name`, `bundle_drift_after_serve_starts_is_refused_per_render`, `serve_health_and_ready`; the last test starts with a matching `package.expectedDigest`, and the ordinary serving tests omit the pin while still verifying the envelope | +| Cross-machine byte stability; closure drift | `.github/workflows/render-golden.yml` runs the golden, serve, and scaffold suites on ubuntu-24.04 and macos-14 against the same `golden.json`; `golden_hashes_match` pins each package's shared file closure and requires every non-virtual dependency to be package-governed | | DX (scaffold compiles offline, validate dry-run, errors) | Manual smoke 2026-09-17 recorded in EVIDENCE-style: `registry-render init` + first compile offline, warning-clean after the font fix; `registry-render validate` refuses bad data with exit 9 and pointers | | Domain neutrality | Bundle fixtures and label files carry all domain wording; production crate types are domain-free (reviewer-verified; a scan gate can follow the Relay pattern post-v1) | | OpenFn journey (walked 2026-09-17) | `integrations/openfn/JOURNEY.md`: real OpenFn CLI 1.40.1 + language-common 3.3.4 job (notify.js idiom) read back through a real breg v0.32.0 dev registry's `record-reader` profile (readableFields = the template data contract, `get`-only, row boundary), rendered via `Accept: application/json` to the golden `pdfSha256`, delivered, then a delivery-outage dead-letter and a recovery replay with the same `eventEffectId` delivered byte-identical PDFs (`cmp` clean); unknown record → 404 read-back refusal with nothing rendered; `registry-render audit-verify` after shutdown: 3 records, all correlated by the effect id (the walk predates the shared audit writer, which removed `audit-verify`) | @@ -34,7 +35,7 @@ hardening (2026-09-19); see EVIDENCE.md for the change list. contract (brief rows, `APP_DOCUMENTS_FILE`, host route, sha256 attach) but must be *walked* in the App Kit repository once the kit-side documents capability is built there — every render-side behavior that journey - exercises (loopback serve, sealed bundle, PDF bytes and hash headers, + exercises (loopback serve, verified package, PDF bytes and hash headers, deterministic redelivery) is covered above by the serve suite and the walked OpenFn journey. A 2026-09-17 read-only sizing of the kit worktree puts that capability at a multi-component change (config parser, render diff --git a/products/render/CONCEPT.md b/products/render/CONCEPT.md index 4d30956f60..9225016aeb 100644 --- a/products/render/CONCEPT.md +++ b/products/render/CONCEPT.md @@ -8,7 +8,7 @@ Date: 2026-09-17 Render publishes governed registry data onto physical media — paper and cards — with the same discipline the rest of Registry Stack applies to APIs. It is a pure rendering function (bundle + validated data in, -byte-stable PDF + hashes out), a sealed governed template bundle as its +byte-stable PDF + hashes out), a governed template package as its content unit, and a value-free audit log as its issuance record. No database, no outbound calls, no engine changes in any other product. @@ -32,10 +32,11 @@ an Evidence definition; Render mints nothing and does nothing at scan time. One crate (`crates/registry-render`), one binary (`registry-render`), following the -house product anatomy: `init`/`check`/`validate`/`seal`/`compile`/`serve`/ +house product anatomy: `init`/`check`/`validate`/`package`/`compile`/`serve`/ `healthcheck`; a strict runtime YAML -(`render.registrystack.org/v1alpha1`) for deployment-local bindings; the -`registry-platform-*` primitives (config secrets, httpsec layers and +(`registry.registrystack.org/render-runtime/v1alpha1`, read by the shared +runtime configuration loader) for deployment-local bindings; the +`registry-platform-*` primitives (config loader and secrets, httpsec layers and problems, the shared audit writer, authcommon key handling, buildinfo, canonical JSON) reused rather than reinvented. @@ -47,8 +48,8 @@ equality; the two-OS golden CI job keeps it proven. ## Trust rules -- Templates are governed input: reviewed, sealed (per-file hashes), and - verified at startup and by `check`. Not treated as hostile — sized for +- Templates are governed input: reviewed, built into the shared package + envelope, and verified at startup and by each worker. They are not treated as hostile — sized for accidents, with kill-at-timeout supervision for the pathological case. - Request data is less trusted than templates: strict JSON Schema validation with pointers, size and media-type caps on assets, inert diff --git a/products/render/DEFINITION-OF-DONE.md b/products/render/DEFINITION-OF-DONE.md index d20374c08e..3f90c2ff64 100644 --- a/products/render/DEFINITION-OF-DONE.md +++ b/products/render/DEFINITION-OF-DONE.md @@ -56,27 +56,27 @@ mechanisms (small fixed page, QR) and is not a fourth project. | Area | Done when | |---|---| -| Product boundary | One crate `registry-render` and one binary `registry-render` (`init`, `check`, `validate`, `seal`, `compile`, `serve`, `healthcheck`) implement the product. Render is a deterministic document renderer: not a document store, e-signature, credentials-issuance, HTML converter, or imposition engine. No `renderctl`, no client crates, no new artifact family. | +| Product boundary | One crate `registry-render` and one binary `registry-render` (`init`, `check`, `validate`, `package`, `compile`, `serve`, `healthcheck`) implement the product. Render is a deterministic document renderer: not a document store, e-signature, credentials-issuance, HTML converter, or imposition engine. No `renderctl`, no client crates, no new artifact family. | | Pure runtime | The render path performs no network I/O, no filesystem access outside the bundle and the virtual `assets/` namespace, and holds no mutable cross-request state. `serve` holds exactly one secret (the caller API key), via a `secret:file/…` ref; `compile` holds none. | -| Bundle format | A closed, versioned manifest (`render.registrystack.org/v1alpha1`, unknown fields rejected) defines document types (id, version, entry, schema, labels, pdfStandard) and per-file hashes. `registry-render seal` writes hashes; each load captures the bundle once and verifies the seal over that immutable snapshot before labels, schemas, fonts, templates, or packages consume it; `serve` refuses an unsealed or hash-mismatched bundle with a named problem; `compile`/`--watch` run unsealed with a one-line notice. | +| Package format | A closed, versioned manifest (`render.registrystack.org/v1alpha1`, unknown fields rejected) defines document types (id, version, entry, schema, labels, pdfStandard) without carrying its own identity. `registry-render package` validates raw authoring source and writes a new directory using the shared `SHA256SUMS` envelope plus optional `REVISION`. Runtime configuration uses `package.root` and may pin `package.expectedDigest`. Each runtime load verifies the shared envelope, captures the package once, binds the captured bytes to the recorded file digests, and uses those bytes for manifest, label, schema, font, template, vendored-package, and `read()` consumption. Missing, changed, and extra files fail by name. Retired manifest `hashes`, `seal`, and `check --seal` are refused with replacement guidance. `compile`, `validate`, and `check` accept raw authoring source or a verified current package; `serve` always requires a package. | | Template payload contract | `sys.inputs.data` is exactly the RFC 8785 canonicalization of `{data, assets}` whose sha256 is `dataSha256`, wrapped in the specified envelope (`data`, `assets`, `labels`, `locale`, `issuedAt`, `document`). The renderer version appears nowhere in the envelope, the PDF bytes, or the hashes. | | Fonts and scripts | The world contains only bundle fonts plus the binary's baseline set (typst-assets), deterministically ordered; host font discovery is never called. `registry-render check` fails when a document's label locales need a script not covered by bundle+baseline fonts. Missing-glyph and other Typst warnings are returned in `rendered.warnings`; `--strict` fails on them. | -| Determinism | For fixed (bundle hash, type, locale, canonical data+assets, issuedAt), bytes are identical: fresh `World` + `Library` per render, sorted font book, `comemo::evict()` per render, deterministic `PdfOptions::ident`. Golden tests pin expected sha256s for all three bundles and run on two OSes in CI; a dependency bump that changes golden hashes is a reviewed diff, never a silent pass. | +| Determinism | For fixed (package hash, type, locale, canonical data+assets, issuedAt), bytes are identical: fresh `World` + `Library` per render, sorted font book, `comemo::evict()` per render, deterministic `PdfOptions::ident`. Golden tests pin expected sha256s for all three bundles and run on two OSes in CI; a dependency bump that changes golden hashes is a reviewed diff, never a silent pass. | | Issued time | `issuedAt` is required on HTTP and fails with a problem document when absent; CLI requires `--issued-at` or an explicit `--now`. No code path renders with a wall-clock default. | | Assets | Request assets are base64, media-type checked (jpeg/png by magic bytes), and capped at code level (per-asset 2 MiB, per-request 8 MiB — a reviewed constant, not per-document schema declaration); decoded by the renderer and exposed only as virtual `assets/` files; `dataSha256` covers their exact bytes. Oversized or wrong-type assets fail validation with a data-path pointer. (Wording amended 2026-09-17 to match the implementation.) | -| Path safety | World-enforced: `read`/`image`/package resolution reject `..` and absolute paths, then require an exact key in the immutable bundle snapshot or the request-local `assets/` namespace. Symlinks are refused while sealing and loading. Negative tests cover traversal, absolute paths, symlinks, virtual-namespace escape, and path replacement after seal verification. | +| Path safety | World-enforced: `read`/`image`/package resolution reject `..` and absolute paths, then require an exact key in the immutable bundle snapshot or the request-local `assets/` namespace. Symlinks are refused while packaging and loading. Negative tests cover traversal, absolute paths, symlinks, virtual-namespace escape, and deterministic path replacement after shared verification. | | Resource enforcement | `serve` renders in a supervised worker process: killed at the configured timeout, memory-capped by rlimit, recycled after a panic (`catch_unwind` in-process backstop → 500 problem + audit), bounded concurrency (documented max, CPU-capped pool), graceful shutdown draining in-flight renders. A pathological-template test proves kill → problem → audit, repeatedly, without degrading the service. CLI uses bounded worker threads with the same timeout semantics. | -| Validation and errors | All failure classes — schema violations (with JSON pointers), bundle drift (with offending hash), compile errors (file/line), missing labels, unvendored imports, timeout, panic, oversize output — are RFC 9457 problem documents on HTTP and typed exit codes under CLI `--json`. No stack trace or raw Typst diagnostic escapes to a caller. `registry-render validate` dry-runs data against the schema without rendering. | +| Validation and errors | All failure classes — schema violations (with JSON pointers), package drift (with the offending file), package pin mismatch (with the common expected/found shape), compile errors (file/line), missing labels, unvendored imports, timeout, panic, oversize output — are RFC 9457 problem documents on HTTP and typed exit codes under CLI `--json`. No stack trace or raw Typst diagnostic escapes to a caller. `registry-render validate` dry-runs data against the schema without rendering. | | HTTP contract | `GET /v1/documents`; `POST /v1/render/{type}` with required `issuedAt`; `Accept: application/pdf`/`*/*`/absent returns PDF bytes + `X-Registry-Pdf-Sha256`/`X-Registry-Data-Sha256`/`X-Registry-Document-Version`; `Accept: application/json` returns `{pdfBase64, pdfSha256, dataSha256, documentVersion}`. `Idempotency-Key` is an opaque correlation id: echoed and audited, never dedupe. `/health`, `/ready` (audit readiness included), static `/openapi.json`. Listener defaults to loopback/private; no TLS code. | | Authentication | API key from a secret file, ≥32 bytes, constant-time compare (authcommon); malformed and missing keys get indistinguishable `401` problems; `401`s are audited. No OIDC in v1. | | Audit | The shared platform audit writer, `file` (owner-only, `fsync`ed, single-writer, rotated and retained) or `stdout`, one envelope per line with schema `render.registrystack.org/audit/v1`. A render writes a `request` entry accepted **before** the worker starts and a `response` entry accepted **before** the response, both failing closed; refusals before any render (validation, 401, 413) are one `response` entry; failures after the render starts (timeout, panic, drift) are the `response` entry; the pair shares one `correlation`; events carry type/version/bundle/renderer+Typst pin/hashes/caller/trace/correlation and no data values or asset bytes. `registry-render check --require-audit-under` proves a `file` destination resolves under a persistent root and refuses `stdout`. | | DX budget | `registry-render init` scaffolds a bundle (manifest, template, schema, label files, starter fonts, fixture) that compiles offline without edits; templates are plain Typst usable with upstream tooling; example bundles double as golden fixtures; the payload contract is one documented page; errors per the validation row are the default path, not an opt-in. | | Domain neutrality | Production code, manifests, CLI options, and schemas contain no deployment-, program-, or client-specific terms; Arabic/French, receipt, certificate, and card vocabulary appears only in example bundles, fixtures, and docs. | -| Operability | Startup runs the same verification as `check` before listening; graceful shutdown drains; structured value-free lifecycle logs (fixed dimensions: method class, route template, status, latency, trace id); no `/metrics` route in v1; `--version` reports `DISPLAY_VERSION` plus the Typst pin. | +| Operability | Startup verifies the shared package envelope and optional digest pin, then performs Render semantic validation over those exact captured bytes before listening. Every worker repeats package verification and the parent refuses a response whose package digest differs from startup. Graceful shutdown drains; structured value-free lifecycle logs use fixed dimensions (method class, route template, status, latency, trace id); no `/metrics` route in v1; `--version` reports `DISPLAY_VERSION` plus the Typst pin. | | App Kit journey | The kit's documents capability uses an `APP_DOCUMENTS_FILE` mapping to loopback destinations with credential files (taskDispatch precedent); the bundle lives under `project/deployment/render/bundle/`; the document field list and QR destination are `agreed` brief rows before the template is written. Done only when a staff user prints a receipt from a record, the PDF attaches to breg with matching sha256 fields, and the QR verify page resolves, closed by a kit-gate record of `COMPLETED with 0 flags` produced in the App Kit repository. **Status: not yet walked** — the kit-side documents capability does not exist yet; ACCEPTANCE.md records the deferral and sizing, and `integrations/app-kit/README.md` is the stack-side contract that capability will implement. | | OpenFn journey | A job in the kit's tested idiom (bridge envelope destructured, config validated, `parseAs: "json"`, status checked, minimized return) reads record data back through a least-privilege breg reader profile whose readable fields equal the template data contract, renders via `Accept: application/json`, and delivers the PDF; a dead-letter replay produces byte-identical output. Walked end to end without the App Kit in the picture. | | Dependency policy | cargo-deny passes with a reviewed, explained Typst-tree delta (licenses/sources); `Cargo.lock` pins typst/typst-pdf/typst-kit/typst-assets; no git or vendored source dependencies. | -| Verification evidence | Formatting, `cargo check/test/clippy -D warnings --locked`, dependency policy, golden-hash drift, closure drift (manifest hashes vs captured closure), value-free canary scans for audit and logs, and the two-OS golden job pass on one revision. Every security row above has a named threat, enforcement point, and executable negative test (security-invariant matrix in the product repo). | +| Verification evidence | Formatting, `cargo check/test/clippy -D warnings --locked`, dependency policy, golden-hash drift, package closure drift (`SHA256SUMS` vs captured closure), value-free canary scans for audit and logs, and the two-OS golden job pass on one revision. Every security row above has a named threat, enforcement point, and executable negative test (security-invariant matrix in the product repo). | ## Stop boundary (explicitly not in v1) @@ -95,8 +95,8 @@ CI must invoke: OSes; - the security-invariant matrix (`SECURITY-MATRIX.md`) mapping every security row to an enforcement point and an executable negative test - (path escape, asset abuse, timeout kill, panic recycle, unsealed serve, - tampered bundle, audit fail-closed ordering, 401 handling, value-free logs/audit canary + (path escape, asset abuse, timeout kill, panic recycle, missing package + envelope, package drift and pin mismatch, audit fail-closed ordering, 401 handling, value-free logs/audit canary scans); - the lib-mode equivalence record (merge-gate results, folded into the product docs) — see `EVIDENCE.md`; diff --git a/products/render/EVIDENCE.md b/products/render/EVIDENCE.md index 7bc7d9ba5b..b637f2653d 100644 --- a/products/render/EVIDENCE.md +++ b/products/render/EVIDENCE.md @@ -6,6 +6,40 @@ All hashes below were produced on macOS (aarch64) with the workspace lockfile in this tree; the two-OS CI golden job re-proves them on Linux for every change. +## Shared package migration (2026-09-26) + +Render now uses the Registry Stack package envelope instead of embedding +per-file hashes in `manifest.yaml`. `registry-render package` validates raw +authoring source, copies the exact validated files into a new directory, and +writes sorted `SHA256SUMS` plus optional `REVISION`. The three maintained +examples are current packages. Their package digests are: + +| Package | digest | +|---|---| +| receipt | `sha256:a4109d91fe340fd054c09f1b3b0d857332f964dd853265b13b014f248d7a2d38` | +| certificate | `sha256:66e42dd1232470c8153411d16a9099ddb50727e76df57153f63c799fcade2051` | +| beneficiary-card | `sha256:6089a24558875d34a015cd09e207d4b5fd6bec62716fbd5075576398343cd6ac` | + +Startup verifies `package.root`, applies optional `package.expectedDigest`, +captures the package through held descriptors, and binds every captured byte +and the captured sum file back to the shared verification result before any +Render validation or consumer construction. Every serve worker repeats that +process and the parent requires its digest to match startup. This covers lazy +Typst template imports, vendored package source, fonts, schemas, labels, and +`read()` calls rather than proving only startup metadata. + +Executable proof covers successful repeatable packaging with an optional +revision, raw authoring checks, current-package check/validate/compile, +missing/changed/extra file refusals by name, matching and mismatching digest +pins, no-pin serving, path replacement between shared verification and product +capture, and drift before a later render. The focused migration run passed 34 +unit tests, 14 golden tests, 19 scaffold/CLI tests, and 20 serve tests, followed +by clippy for all Render targets with warnings denied. + +The dated entries below preserve the evidence history of the retired manifest +seal. Where they use “seal” or old test names, this section and the current +acceptance map describe the replacement contract. + ## Merge gate: library mode ≡ Typst CLI (2026-09-17) The Definition of Done's merge gate: a library-mode render must reproduce the @@ -101,8 +135,8 @@ Findings folded back into the product (the gate's purpose): ## Golden record -`golden.json` pins, per bundle: PDF sha256, envelope (`dataSha256`) -sha256, warnings (empty), bundle hash. Issued time fixed at +`golden.json` pins, per bundle: PDF sha256, request envelope (`dataSha256`) +sha256, warnings (empty), and package hash. Issued time fixed at `2026-09-16T10:32:00Z`. Regenerate only via the documented command and only as a reviewed diff: @@ -121,7 +155,7 @@ every technical property the corpus exists to exercise: Arabic/RTL primary with French secondary, bidirectional text with embedded Latin, a ten-digit identifier pattern, a currency (now `XTS`, the ISO 4217 code reserved for testing), a region field, an embedded photo, and an offline QR. Content changed in all three -bundles, so all three were re-sealed and every pinned value moved. The document +bundles, so all three were rebuilt and every pinned value moved. The document versions were deliberately *not* bumped (receipt stays v3, certificate and beneficiary-card v1): nothing has been released, and a bump would imply an earlier version existed in the wild. @@ -137,15 +171,15 @@ the findings above. | certificate | `252d40678919cb8496950101efa7fa4984ccd6825655679cca6dead05c7696c0` | | beneficiary-card | `cf607a3b82a2027abfb4345377b1b84cd2e616da7376e18f29355ca790c3da2f` | -Envelope (`dataSha256`) and bundle hashes moved with them and live in +Envelope (`dataSha256`) and package hashes live in `golden.json`: receipt envelope -`8ab91deaf04f2ab9634da9fe45e63615f285c936659f8d839f413f647641b33a` / bundle -`96d199d20c94b39d4947168a84337277db1f01e86bb8907d0614d211855b0e26`; certificate +`8ab91deaf04f2ab9634da9fe45e63615f285c936659f8d839f413f647641b33a` / package +`a4109d91fe340fd054c09f1b3b0d857332f964dd853265b13b014f248d7a2d38`; certificate envelope `cccb32e6fdcf4db99a556ede3abedcc4f46b94575b20a4b5ce361edfd1ac555a` / -bundle `3bb68569252cd621f0b6992b329de04524c29d02995c14056892f177ac9e6dff`; +package `66e42dd1232470c8153411d16a9099ddb50727e76df57153f63c799fcade2051`; beneficiary-card envelope -`c595a3670a9b83327a8e16ef403b8253039d6d548b3388875930850d0f41fa18` / bundle -`4dd435c0bed4c80f4408dc87173b0f72887071ca78f734441bed979cabfe7d88`. The +`c595a3670a9b83327a8e16ef403b8253039d6d548b3388875930850d0f41fa18` / package +`6089a24558875d34a015cd09e207d4b5fd6bec62716fbd5075576398343cd6ac`. The per-render dependency closures are unchanged. Every warning list is empty. The Arabic strings in the rewritten fixtures are plausible modern standard @@ -354,20 +388,19 @@ seven more, each fixed with a failing test first except where noted: Test totals after the third pass: 71 (22 unit, 13 golden, 19 serve end-to-end, 17 scaffold/CLI), all green with `--locked`. -## Immutable bundle snapshot hardening (2026-09-19) +## Immutable bundle snapshot hardening (2026-09-19; package binding updated 2026-09-26) -Sealed loads now capture the complete bundle through held directory +Package loads capture the complete directory through held directory descriptors, opening every component in the configured root spelling and every descendant with `NOFOLLOW`, and refusing symlinks and non-regular files without a pathname reopen. The manifest is opened and parsed first through the secured root descriptor, so -a missing, malformed, or unsealed manifest is refused before any descendant is -read. Its exact bytes are retained in the snapshot. The manifest hash map is -verified over the captured bytes, and the same immutable snapshot supplies the +a missing or malformed manifest is refused before any descendant is read. Its +exact bytes are retained in the snapshot. The shared package verification is +bound to the captured file set and digests, and the same immutable snapshot supplies the manifest, labels, schemas, fonts, templates, package sources, and other -project/package files consumed by Typst. Sealing uses the same root traversal -and hashes the captured bytes, so it cannot accept a root or ancestor symlink -that loading would immediately refuse. +project/package files consumed by Typst. Authoring capture and package loading +use the same root traversal, so neither accepts a root or ancestor symlink. Executable proof covers both sides of the former verified/use gap: @@ -375,14 +408,17 @@ Executable proof covers both sides of the former verified/use gap: labels, schema, and font paths after capture; assembly still consumes the captured bytes. - `bundle::tests::bundle_root_and_ancestor_symlinks_are_refused_for_every_spelling` - covers a root symlink with a trailing slash and a symlinked ancestor for both - snapshot loading and sealing. -- `bundle::tests::sealed_load_checks_the_manifest_before_capturing_descendants` - places a refused symlink beside missing, malformed, and unsealed manifests; + covers a root symlink with a trailing slash and a symlinked ancestor during + snapshot loading. +- `bundle::tests::source_load_checks_the_manifest_before_capturing_descendants` + places a refused symlink beside missing and malformed manifests; each manifest result wins before descendant capture. -- `golden::sealed_template_and_package_bytes_are_bound_to_the_loaded_snapshot` - replaces the template, a package source, and a non-source file after sealed - load; every render remains byte-identical under the original bundle hash. +- `bundle::tests::package_load_refuses_bytes_replaced_after_shared_verification` + deterministically replaces a verified file before product capture and proves + the consumed-byte binding refuses it. +- `golden::verified_template_and_typst_package_bytes_are_bound_to_the_loaded_snapshot` + replaces the template, a package source, and a non-source file after package + load; every render remains byte-identical under the original package hash. - The existing per-render serve drift test still refuses drift present before a worker captures its snapshot. diff --git a/products/render/PAYLOAD.md b/products/render/PAYLOAD.md index eee847a02e..adea03a1fe 100644 --- a/products/render/PAYLOAD.md +++ b/products/render/PAYLOAD.md @@ -65,9 +65,12 @@ Rules worth internalizing: ## Authoring loop Edit templates freely with upstream tooling (tinymist/VS Code give live -preview if you point them at the bundle). `registry-render compile` runs -unsealed bundles with a notice; `registry-render seal` writes the manifest -hashes; `registry-render serve` requires the sealed bundle. +preview if you point them at the source bundle). `registry-render compile`, +`validate`, and `check` accept raw source during authoring. When it is ready, +`registry-render package --bundle --output ` validates +the source and writes the shared `SHA256SUMS` envelope plus an optional +`REVISION`. `registry-render serve` always verifies that package, including an +optional `package.expectedDigest` pin, before it listens. `registry-render check` verifies everything, including that every label character is drawable by some bundle or baseline font — a successful render that prints tofu is wrong on paper, so coverage diff --git a/products/render/README.md b/products/render/README.md index 4478997c10..4f12598698 100644 --- a/products/render/README.md +++ b/products/render/README.md @@ -5,7 +5,7 @@ receipts, notices, certificates, letters, ID/member cards (photo, QR, duplex), tags and labels — using [Typst](https://typst.app) embedded as a library. -> Give Render a sealed template bundle and validated record data. Render +> Give Render a verified template package and validated record data. Render > returns the identical PDF every time, with the hashes and audit events an > institution needs to stand behind the printed artifact. @@ -28,7 +28,8 @@ cargo run -p registry-render -- init ./my-bundle # write data.json with {reference, body, footer-note} cargo run -p registry-render -- compile --bundle ./my-bundle --type letter \ --data data.json --issued-at 2026-01-01T00:00:00Z --out letter.pdf -cargo run -p registry-render -- seal --bundle ./my-bundle # when ready +cargo run -p registry-render -- package --bundle ./my-bundle \ + --output ./my-package --revision "$(git rev-parse HEAD)" ``` The scaffold compiles offline out of the box: the binary embeds a baseline @@ -43,8 +44,10 @@ Latin font set. Scripts beyond Latin (Arabic, Hebrew, …) need a bundle font - [DEFINITION-OF-DONE.md](DEFINITION-OF-DONE.md) — the acceptance contract. - [EVIDENCE.md](EVIDENCE.md) — the merge gate (library ≡ Typst CLI) and the golden-hash record. -- `bundles/` — three coequal example bundles: a bilingual RTL receipt, a - PDF/A-4 certificate, an ID-1 duplex card with a photo. Copy them. +- `bundles/` — three immutable packaged examples: a bilingual RTL receipt, a + PDF/A-4 certificate, and an ID-1 duplex card with a photo. Use them with + `check`, `compile`, or `serve`; start editable source with + `registry-render init`. - `integrations/` — the OpenFn job and App Kit wiring sketches. ## Exit codes @@ -59,8 +62,8 @@ exits 101): |---|---|---| | 2 | `invalid-argument` | malformed CLI or request argument | | 3 | `manifest-invalid` | bundle manifest missing or structurally invalid | -| 4 | `bundle-tampered` | sealed bundle hashes do not match the manifest | -| 5 | `bundle-unsealed` | operation requires a sealed bundle | +| 4 | `bundle-tampered` | package files do not match the shared `SHA256SUMS` envelope | +| 5 | `bundle-unsealed` | runtime package lacks `SHA256SUMS` | | 6 | `unknown-document` | requested document type or locale not in the bundle | | 7 | `labels-invalid` | label table missing, invalid, or key sets diverge across locales | | 8 | `font-invalid` | bundle font cannot be loaded, or label script uncovered | @@ -81,7 +84,7 @@ exits 101): ## Guarantees, mechanistically -- **Byte-stable**: for fixed (bundle hash, type, locale, canonical data, +- **Byte-stable**: for fixed (package hash, type, locale, canonical data, `issuedAt`) the PDF bytes are fixed — fresh Typst world and library per render, deterministic font order, the RFC 8785 canonical envelope is exactly the string injected into the template and exactly the bytes @@ -90,8 +93,9 @@ exits 101): - **Path-safe**: a template's world contains exactly the bundle, the request's decoded assets (`assets/`), and vendored packages — enforced by lexical checks and exact lookups in an immutable bundle - snapshot, not by template discipline. Sealed loads verify that snapshot - once, and Typst consumes those exact verified bytes. + snapshot, not by template discipline. Package loads bind the shared + checksum envelope to that snapshot, and Typst consumes those exact + verified bytes. - **Resource-bounded**: serves render in a supervised worker process, killed at the timeout, memory-capped on Linux, recycled on panic. - **Auditable**: value-free events (hashes, versions, caller fingerprint, diff --git a/products/render/SECURITY-MATRIX.md b/products/render/SECURITY-MATRIX.md index 4db6c2cbcb..a5485e6da8 100644 --- a/products/render/SECURITY-MATRIX.md +++ b/products/render/SECURITY-MATRIX.md @@ -10,24 +10,24 @@ fixed or explicitly documented as residuals below), extended by the PR |---|---|---|---| | 1 | Untrusted data reaches a path-taking template function and reads files outside the bundle | `RenderWorld::snapshot_path` (world.rs): lexical rejection of `..`/absolute components followed by exact lookup in the immutable bundle snapshot; Typst additionally virtualizes absolute paths as root-relative before the world sees them | `golden::data_paths_cannot_escape_the_bundle`; `world::tests::virtual_paths_reject_escape_attempts` | | 2 | A template imports an unvendored package; render fetches from the network | There is no network code path in library mode; package resolution serves only `packages/` through the world | `golden::unvendored_package_import_fails_without_network` | -| 3 | Bundle content drifts after sealing (tamper, accidental edit, or a change between seal verification and Typst access) | Each load captures every bundle file once, verifies the sealed per-file sha256 map over those immutable bytes, then uses that same snapshot for the manifest, labels, schemas, fonts, templates, package sources, and other package/project files; serve does this per render and still requires the response's bundle hash to equal the startup hash | `bundle::tests::assembly_uses_captured_bytes`; `golden::tampered_sealed_bundle_is_refused`; `golden::sealed_template_and_package_bytes_are_bound_to_the_loaded_snapshot`; `serve::tampered_bundle_refuses_to_serve`; `serve::bundle_drift_after_serve_starts_is_refused_per_render` | -| 4 | An unsealed bundle is served, or an invalid/unsealed manifest causes unrelated bundle content to be loaded first | `Bundle::load_sealed` reads and parses the root manifest through the secured root descriptor, enforces the seal requirement, then captures descendants while retaining those exact manifest bytes; enforced at startup and in every worker | `bundle::tests::sealed_load_checks_the_manifest_before_capturing_descendants`; `golden::unsealed_bundle_is_refused_for_serving`; `serve::bundle_drift_after_serve_starts_is_refused_per_render` (seal stripped after startup → 400 per render) | +| 3 | Package content drifts after shared verification through tamper, accidental edit, or deterministic path replacement before Typst access | Shared verification checks `SHA256SUMS`; Render captures every file once and binds those exact bytes and the captured sum file back to the verified file digests before parsing the manifest, labels, schemas, fonts, templates, vendored package sources, or other project files. Serve repeats this in every worker and requires the worker's package digest to equal the startup digest. | `bundle::tests::package_load_refuses_bytes_replaced_after_shared_verification`; `golden::tampered_package_is_refused`; `golden::verified_template_and_typst_package_bytes_are_bound_to_the_loaded_snapshot`; `serve::package_changed_missing_and_extra_files_refuse_startup_by_name`; `serve::bundle_drift_after_serve_starts_is_refused_per_render` | +| 4 | A directory without the shared envelope is served, a digest pin is bypassed, or invalid manifest bytes cause unrelated content to be trusted | `runtime::load_package` verifies the shared package and optional `package.expectedDigest` before semantic assembly; `BundleSnapshot::load` reads and parses the root manifest first through a secured root descriptor; workers repeat package verification for every render | `bundle::tests::source_load_checks_the_manifest_before_capturing_descendants`; `golden::directory_without_sum_file_is_not_a_package_but_remains_authoring_source`; `serve::serve_startup_package_digest_mismatch_uses_common_expected_and_found_shape`; `serve::bundle_drift_after_serve_starts_is_refused_per_render` | | 5 | Caller without the API key reads documents or renders | Bearer authentication as a layer on `/v1/*` **before body buffering**; constant-time compare; ≥32-byte ASCII key; identical 401 bodies for missing/wrong/short keys; the key file gets exactly one trailing line ending trimmed and any other whitespace refuses startup (no silently mis-armed key with `/health` green) | `serve::unauthorized_requests_are_refused_and_audited` (two indistinguishable 401s); `serve::api_key_file_with_one_trailing_newline_is_trimmed`; `serve::api_key_with_stray_whitespace_is_refused_at_startup`; `/v1/documents` behind the same layer | | 6 | The append-only audit log is used as an unauthenticated write oracle (huge or hostile route params) | Route parameters are bounded (64 chars) and kebab-validated (`sanitize_document_type`) before any audit append, including pre-auth 401 events | code-reviewed; audit shape asserted in `serve::unauthorized_requests_are_refused_and_audited` | | 7 | Request data values or the API key leak into the audit log or logs | Audit events carry a closed, value-free field set; lifecycle logs use fixed dimensions (method class, route template, status, latency, trace id) | `serve::audit_events_are_value_free` (canary + key scans over the audit file) | | 8 | A render starts, or a document leaves, with no accepted audit record of it | The shared audit writer accepts a `file` entry only after `fsync` (owner-only file, single-writer lock) and a `stdout` entry only after it is written and flushed, then stops accepting after any failure. The `request` entry is accepted before the worker starts, else `503 audit-failed` and no render; the `response` entry is accepted before the document leaves, else `503 audit-failed` and no PDF or hash headers; `/ready` reports a stopped writer. Each call pairs its entries under a correlation the server draws, never the caller's `Idempotency-Key` (echoed only as `correlationId`), so calls sharing a key cannot be confused, and a call that ends before its outcome is written writes an `unfinished` response. Host-level tampering with the log is not defended here: the log carries no hash chain or signature (residual below) | `server::tests::the_request_entry_is_accepted_before_the_render_starts`; `server::tests::a_refused_request_entry_prevents_the_render`; `serve::a_refused_request_entry_prevents_the_render`; `serve::a_refused_response_entry_withholds_the_pdf`; `serve::a_render_writes_a_request_entry_then_a_response_entry_sharing_correlation` (file mode 0600); `server::tests::concurrent_calls_sharing_an_idempotency_key_pair_their_own_entries` | -| 9 | A render succeeds but is wrong on paper (missing glyphs) | Check-time label-script coverage over the actual label characters; render-time warnings; `--strict` fails on warnings | `scaffold::check_seal_refuses_to_seal_a_broken_bundle` (CJK label, no font → refusal, and no seal written) | +| 9 | A render succeeds but is wrong on paper (missing glyphs) | Check-time label-script coverage over the actual label characters; package validates before it writes any output; render-time warnings; `--strict` fails on warnings | `scaffold::package_refuses_a_broken_bundle_before_writing_output` (CJK label, no font, so the package directory is never created) | | 10 | Pathological template data exhausts CPU or memory | Serves render in a supervised worker process: killed at the configured timeout, `RLIMIT_AS` on Linux, recycled on panic; bounded concurrency; request body and output caps with hard ceilings | `serve::pathological_renders_are_bounded_and_the_service_recovers` (timeout or memory wall is audited; a fresh worker immediately serves a healthy render) | | 11 | Assets smuggle executables or balloons | base64-decoded, JPEG/PNG magic sniffed, per-asset 2 MiB / per-request 8 MiB caps, exact bytes covered by `dataSha256` | `golden::wrong_media_type_and_oversize_assets_are_refused` | | 12 | Deeply nested JSON parse DoS | `serde_json` default 128-depth recursion limit on top of the tower body-limit layer | platform-owned; body limit exercised by config ceiling (`runtime.rs` 64 MiB hard cap) | | 13 | Response header injection via echoed `Idempotency-Key` | `HeaderValue::from_str` rejects CR/LF/NUL; 128-char bound at read | code-reviewed; echo asserted in `serve::render_returns_pdf_with_hash_headers_matching_golden` | | 14 | Byte drift between renders, machines, or dependency upgrades | Fresh world+library per render, canonical (RFC 8785) envelope = injected = hashed bytes, deterministic font order, `ident: Auto` (content-derived), comemo eviction; lockfile pins Typst **and the deflate stack** | `golden::golden_hashes_match` + `rendering_is_deterministic…` on two OSes in CI (`.github/workflows/render-golden.yml`); merge-gate record in EVIDENCE.md | -| 15 | Sealed-bundle hash coverage misses nested files or symlinks | Hashes cover every file except the root manifest itself (nested `manifest.yaml` is governed content); the exact root manifest bytes identify the bundle and are included in the render snapshot; sealing and loading open every root spelling component and descendant through held directory descriptors with `NOFOLLOW`, accept only opened regular files, and seal the captured bytes | `scaffold::bundle_with_symlink_cannot_be_sealed`; `serve::bundle_drift_after_serve_starts_is_refused_per_render`; `bundle::tests::assembly_uses_captured_bytes`; `bundle::tests::bundle_root_and_ancestor_symlinks_are_refused_for_every_spelling` | +| 15 | Package checksum coverage misses nested files, extra files, or symlinks | The shared writer lists every product file and optional `REVISION` in sorted `SHA256SUMS`; verification refuses changed, missing, extra, symbolic-link, and special-file entries. Render opens every root spelling component and descendant through held directory descriptors with `NOFOLLOW` and accepts only opened regular files before binding the captured set to that shared verification. | `scaffold::bundle_with_symlink_cannot_be_packaged`; `serve::package_changed_missing_and_extra_files_refuse_startup_by_name`; `serve::bundle_drift_after_serve_starts_is_refused_per_render`; `bundle::tests::bundle_root_and_ancestor_symlinks_are_refused_for_every_spelling` | | 16 | 401 and 413 audit-append failures vanish silently | Failures are logged at error level even though the refusal itself stands | code-reviewed with the lifecycle-log middleware | | 17 | Compile diagnostics or bundle-load failures leak the host deployment (paths) or raw engine output | World file errors carry virtual, root-relative paths only; a redaction pass replaces any residual host bundle root with `` in problem details and warnings, and per-request bundle-load failures in the worker are redacted the same way before they cross the pipe | `golden::compile_diagnostics_report_virtual_paths_only`; `world::tests::not_found_reports_the_virtual_path_not_the_host_root`; `serve::bundle_drift_after_serve_starts_is_refused_per_render` (a refused symlink names `/escape`, never the host path) | | 18 | Oversized or unframed bodies DoS the service or dodge the audit trail | Body ceiling enforced **after authentication** (unauthenticated oversized bodies are 401s, never buffered); an authenticated over-ceiling body gets the audited `body-too-large` problem (413), and chunked transfer is refused up front with an audited problem — once a chunked stream trips the stream limit mid-body the connection is broken and no response can be delivered at all, so refusal must precede the first body byte; the tower stream limit stays as the last-ditch backstop (a mid-stream abort there closes the connection: the residual recorded below) | `serve::oversized_bodies_are_refused_after_auth_as_problems`; `serve::chunked_bodies_are_refused_upfront_as_problems` | | 19 | A panicking worker's stderr (paths, data fragments) reaches the operator log | Worker stderr is piped and drained, never inherited; problem documents are the diagnosis surface | code-reviewed with the supervise loop (`worker::supervise`); `serve::pathological_renders_are_bounded_and_the_service_recovers` exercises abnormal worker exits without asserting on the drained bytes, so no test covers the drain itself | -| 20 | Serve listens on a public or all-interfaces address | `runtime::validate_bind` refuses non-loopback/private/unspecified binds at startup; the whole `server:` section defaults to `127.0.0.1:8080` | `runtime::tests::loopback_private_and_link_local_binds_are_allowed`, `runtime::tests::public_and_unspecified_binds_are_refused`, `runtime::tests::bind_defaults_to_loopback` | +| 20 | Serve listens on a public or all-interfaces address | `runtime::validate_bind` refuses non-loopback/private/unspecified binds at startup; `listener.bind` is required, so no address is chosen implicitly | `runtime::tests::loopback_private_and_link_local_binds_are_allowed`, `runtime::tests::public_and_unspecified_binds_are_refused`, `runtime::tests::the_listener_bind_is_required` | | 21 | A caller-controlled value makes a refusal unbounded (response and log amplification) | Problem details are capped at 2048 characters with an explicit truncation marker before they reach the wire; the values that can grow (an undeclared locale, an asset name) are caller-supplied | `serve::problem_details_are_bounded`; `server::tests::a_long_detail_is_cut_on_a_character_boundary`; `server::tests::a_detail_at_the_cap_is_kept_verbatim` | | 22 | A shutdown grace that disarms the drain or overflows the bounded wait | `runtime::load` refuses anything outside 1 to 3600 seconds at startup, so zero cannot drop renders in flight and a huge value cannot panic the wait that adds the grace twice | `runtime::tests::shutdown_grace_outside_the_supported_range_is_refused`; `runtime::tests::shutdown_grace_at_the_range_ends_is_accepted` | diff --git a/products/render/bundles/card/SHA256SUMS b/products/render/bundles/card/SHA256SUMS new file mode 100644 index 0000000000..5c3e1cf71a --- /dev/null +++ b/products/render/bundles/card/SHA256SUMS @@ -0,0 +1,18 @@ +ea92354cde46561d82fa4edfd38038f0bec3c63f3d269f5479c322399bbfe175 fixtures/data.json +61e161e58f98a40ec8a0e566670ce6d0ecadf866e8385fe03ae1caa4a22bb015 fixtures/photo.b64 +04017b0ccdd1156dcde3bc386734c1b03165261762f16a025c8afc9088ae76e9 fonts/NotoNaskhArabic-Bold.ttf +6f0a92031367b2f5a2078fe9d24f3433122b61a0bad57c423aad8f3c39aa2e6e fonts/NotoNaskhArabic-Regular.ttf +1df075a380fc7cb898acf64c1f7b3b4dd780de3caa860178bf929de35817a913 fonts/NotoSans-Bold.ttf +478c558ea716033cd60c03438f628dfa75694dcf6b5f6d505a2f05fd2b4f3823 fonts/NotoSans-Regular.ttf +0dab92d0544f7b233403f14b84a663bdbfa746982eda629e7f4f9ffe1b036feb fonts/OFL.txt +84649172cb0efb97d8c553b61a811294a3ae580f86f5e82faf5ac137f5db8350 labels/ar.yaml +1a094b73077baef2cd829da5fd23ca6ae515276019f4fb2868a931b851922a40 labels/fr.yaml +efe2ace2855b61c3ec4697cc3adad98c94e3766eefd982db17548c9c9ae03f72 manifest.yaml +f0f6fec86b9678afa492dc078f81b8576bf25f0a1ca22be9f40e78b731fdf988 packages/preview/zebra/0.1.0/LICENSE +d420f30d5c82f6fdd1f8ecaf58f98eaf6320670ab0ce626976c15f853cb57b6a packages/preview/zebra/0.1.0/README.md +9fb67e4684d1d35b81bb895b5e637286ef56a28c9fbc3715e53af30aa9efb5d4 packages/preview/zebra/0.1.0/src/generic.typ +5e2eb89b3fa143cb7f6458153ff95f44f81e2179adba150c3558c75b40283b46 packages/preview/zebra/0.1.0/src/lib.typ +9c4c9501d950d13505263d52da39d844d1c312d3c5ccc436a8f35602fd834975 packages/preview/zebra/0.1.0/src/zebra.wasm +ece9c7e0e324a9566845a7e89dcda7b424d6c038b6ce8446d0b00b8f685c17af packages/preview/zebra/0.1.0/typst.toml +d5cd498618e88d992ca3566fdec368b918a890727e2dac9314be8ad7e88828d8 schemas/card.schema.json +e90ef86681ae8a1dcc4c5f2de849b963d7b14f780608aaa7a37b7181ecad1d5d templates/card.typ diff --git a/products/render/bundles/card/manifest.yaml b/products/render/bundles/card/manifest.yaml index cbb495feb2..fb06060b8c 100644 --- a/products/render/bundles/card/manifest.yaml +++ b/products/render/bundles/card/manifest.yaml @@ -9,21 +9,3 @@ documents: labels: - ar - fr -hashes: - fixtures/data.json: ea92354cde46561d82fa4edfd38038f0bec3c63f3d269f5479c322399bbfe175 - fixtures/photo.b64: 61e161e58f98a40ec8a0e566670ce6d0ecadf866e8385fe03ae1caa4a22bb015 - fonts/NotoNaskhArabic-Bold.ttf: 04017b0ccdd1156dcde3bc386734c1b03165261762f16a025c8afc9088ae76e9 - fonts/NotoNaskhArabic-Regular.ttf: 6f0a92031367b2f5a2078fe9d24f3433122b61a0bad57c423aad8f3c39aa2e6e - fonts/NotoSans-Bold.ttf: 1df075a380fc7cb898acf64c1f7b3b4dd780de3caa860178bf929de35817a913 - fonts/NotoSans-Regular.ttf: 478c558ea716033cd60c03438f628dfa75694dcf6b5f6d505a2f05fd2b4f3823 - fonts/OFL.txt: 0dab92d0544f7b233403f14b84a663bdbfa746982eda629e7f4f9ffe1b036feb - labels/ar.yaml: 84649172cb0efb97d8c553b61a811294a3ae580f86f5e82faf5ac137f5db8350 - labels/fr.yaml: 1a094b73077baef2cd829da5fd23ca6ae515276019f4fb2868a931b851922a40 - packages/preview/zebra/0.1.0/LICENSE: f0f6fec86b9678afa492dc078f81b8576bf25f0a1ca22be9f40e78b731fdf988 - packages/preview/zebra/0.1.0/README.md: d420f30d5c82f6fdd1f8ecaf58f98eaf6320670ab0ce626976c15f853cb57b6a - packages/preview/zebra/0.1.0/src/generic.typ: 9fb67e4684d1d35b81bb895b5e637286ef56a28c9fbc3715e53af30aa9efb5d4 - packages/preview/zebra/0.1.0/src/lib.typ: 5e2eb89b3fa143cb7f6458153ff95f44f81e2179adba150c3558c75b40283b46 - packages/preview/zebra/0.1.0/src/zebra.wasm: 9c4c9501d950d13505263d52da39d844d1c312d3c5ccc436a8f35602fd834975 - packages/preview/zebra/0.1.0/typst.toml: ece9c7e0e324a9566845a7e89dcda7b424d6c038b6ce8446d0b00b8f685c17af - schemas/card.schema.json: d5cd498618e88d992ca3566fdec368b918a890727e2dac9314be8ad7e88828d8 - templates/card.typ: e90ef86681ae8a1dcc4c5f2de849b963d7b14f780608aaa7a37b7181ecad1d5d diff --git a/products/render/bundles/certificate/SHA256SUMS b/products/render/bundles/certificate/SHA256SUMS new file mode 100644 index 0000000000..746c1b548f --- /dev/null +++ b/products/render/bundles/certificate/SHA256SUMS @@ -0,0 +1,11 @@ +34fb9e0e641ea5176a0ce156fbe5eebf34e2e4014c4b6f3e035e7a83db3462ba fixtures/data.json +a7c74d341a024e41428f041b1ae93d0686413923905195b7869b5c0d74792747 labels/en.yaml +c55501f16bca875960df7c17f485c574c5dd870146048e8dcfdb4e6d464b626b manifest.yaml +f0f6fec86b9678afa492dc078f81b8576bf25f0a1ca22be9f40e78b731fdf988 packages/preview/zebra/0.1.0/LICENSE +d420f30d5c82f6fdd1f8ecaf58f98eaf6320670ab0ce626976c15f853cb57b6a packages/preview/zebra/0.1.0/README.md +9fb67e4684d1d35b81bb895b5e637286ef56a28c9fbc3715e53af30aa9efb5d4 packages/preview/zebra/0.1.0/src/generic.typ +5e2eb89b3fa143cb7f6458153ff95f44f81e2179adba150c3558c75b40283b46 packages/preview/zebra/0.1.0/src/lib.typ +9c4c9501d950d13505263d52da39d844d1c312d3c5ccc436a8f35602fd834975 packages/preview/zebra/0.1.0/src/zebra.wasm +ece9c7e0e324a9566845a7e89dcda7b424d6c038b6ce8446d0b00b8f685c17af packages/preview/zebra/0.1.0/typst.toml +f27c0c341f2ad02a56b031b7be05891422e398377491f4af75668a16cba387a0 schemas/certificate.schema.json +0f75915946dd918318a2c13559a1619e080be205351662f62216da467d6a74e5 templates/certificate.typ diff --git a/products/render/bundles/certificate/manifest.yaml b/products/render/bundles/certificate/manifest.yaml index 03997a305e..ad9d613c80 100644 --- a/products/render/bundles/certificate/manifest.yaml +++ b/products/render/bundles/certificate/manifest.yaml @@ -9,14 +9,3 @@ documents: labels: - en pdfStandard: a-4 -hashes: - fixtures/data.json: 34fb9e0e641ea5176a0ce156fbe5eebf34e2e4014c4b6f3e035e7a83db3462ba - labels/en.yaml: a7c74d341a024e41428f041b1ae93d0686413923905195b7869b5c0d74792747 - packages/preview/zebra/0.1.0/LICENSE: f0f6fec86b9678afa492dc078f81b8576bf25f0a1ca22be9f40e78b731fdf988 - packages/preview/zebra/0.1.0/README.md: d420f30d5c82f6fdd1f8ecaf58f98eaf6320670ab0ce626976c15f853cb57b6a - packages/preview/zebra/0.1.0/src/generic.typ: 9fb67e4684d1d35b81bb895b5e637286ef56a28c9fbc3715e53af30aa9efb5d4 - packages/preview/zebra/0.1.0/src/lib.typ: 5e2eb89b3fa143cb7f6458153ff95f44f81e2179adba150c3558c75b40283b46 - packages/preview/zebra/0.1.0/src/zebra.wasm: 9c4c9501d950d13505263d52da39d844d1c312d3c5ccc436a8f35602fd834975 - packages/preview/zebra/0.1.0/typst.toml: ece9c7e0e324a9566845a7e89dcda7b424d6c038b6ce8446d0b00b8f685c17af - schemas/certificate.schema.json: f27c0c341f2ad02a56b031b7be05891422e398377491f4af75668a16cba387a0 - templates/certificate.typ: 0f75915946dd918318a2c13559a1619e080be205351662f62216da467d6a74e5 diff --git a/products/render/bundles/receipt/SHA256SUMS b/products/render/bundles/receipt/SHA256SUMS new file mode 100644 index 0000000000..51a3358620 --- /dev/null +++ b/products/render/bundles/receipt/SHA256SUMS @@ -0,0 +1,17 @@ +f1b2fbcff5975a17b3f9ff46d9e5a2de0bc88de4e6b3ff75fad42fe477ceb5f8 fixtures/data.json +04017b0ccdd1156dcde3bc386734c1b03165261762f16a025c8afc9088ae76e9 fonts/NotoNaskhArabic-Bold.ttf +6f0a92031367b2f5a2078fe9d24f3433122b61a0bad57c423aad8f3c39aa2e6e fonts/NotoNaskhArabic-Regular.ttf +1df075a380fc7cb898acf64c1f7b3b4dd780de3caa860178bf929de35817a913 fonts/NotoSans-Bold.ttf +478c558ea716033cd60c03438f628dfa75694dcf6b5f6d505a2f05fd2b4f3823 fonts/NotoSans-Regular.ttf +0dab92d0544f7b233403f14b84a663bdbfa746982eda629e7f4f9ffe1b036feb fonts/OFL.txt +e1d67aa8925038d59d78dd9f7501f40a081ae37019441572dfc73d072d774057 labels/ar.yaml +ddf9abcbb39afff44d39f459b7e6db0a1b990b51bcab928ca166cde2b8874b65 labels/fr.yaml +f4bb21a61015f8ffffebc9e6c3fe3f755214b77ada44306444531b44d31115c2 manifest.yaml +f0f6fec86b9678afa492dc078f81b8576bf25f0a1ca22be9f40e78b731fdf988 packages/preview/zebra/0.1.0/LICENSE +d420f30d5c82f6fdd1f8ecaf58f98eaf6320670ab0ce626976c15f853cb57b6a packages/preview/zebra/0.1.0/README.md +9fb67e4684d1d35b81bb895b5e637286ef56a28c9fbc3715e53af30aa9efb5d4 packages/preview/zebra/0.1.0/src/generic.typ +5e2eb89b3fa143cb7f6458153ff95f44f81e2179adba150c3558c75b40283b46 packages/preview/zebra/0.1.0/src/lib.typ +9c4c9501d950d13505263d52da39d844d1c312d3c5ccc436a8f35602fd834975 packages/preview/zebra/0.1.0/src/zebra.wasm +ece9c7e0e324a9566845a7e89dcda7b424d6c038b6ce8446d0b00b8f685c17af packages/preview/zebra/0.1.0/typst.toml +9d8388a0cb66fb8ef7dae6b60bd7337f06df02f557c339aadac02e85dedd6736 schemas/receipt.schema.json +1d112f888dc63e0f47173a149e1f21d1eccfdf5f191a1b88534f2e16cec1dafa templates/receipt.typ diff --git a/products/render/bundles/receipt/manifest.yaml b/products/render/bundles/receipt/manifest.yaml index 4a452aefaf..57b2b44a09 100644 --- a/products/render/bundles/receipt/manifest.yaml +++ b/products/render/bundles/receipt/manifest.yaml @@ -9,20 +9,3 @@ documents: labels: - ar - fr -hashes: - fixtures/data.json: f1b2fbcff5975a17b3f9ff46d9e5a2de0bc88de4e6b3ff75fad42fe477ceb5f8 - fonts/NotoNaskhArabic-Bold.ttf: 04017b0ccdd1156dcde3bc386734c1b03165261762f16a025c8afc9088ae76e9 - fonts/NotoNaskhArabic-Regular.ttf: 6f0a92031367b2f5a2078fe9d24f3433122b61a0bad57c423aad8f3c39aa2e6e - fonts/NotoSans-Bold.ttf: 1df075a380fc7cb898acf64c1f7b3b4dd780de3caa860178bf929de35817a913 - fonts/NotoSans-Regular.ttf: 478c558ea716033cd60c03438f628dfa75694dcf6b5f6d505a2f05fd2b4f3823 - fonts/OFL.txt: 0dab92d0544f7b233403f14b84a663bdbfa746982eda629e7f4f9ffe1b036feb - labels/ar.yaml: e1d67aa8925038d59d78dd9f7501f40a081ae37019441572dfc73d072d774057 - labels/fr.yaml: ddf9abcbb39afff44d39f459b7e6db0a1b990b51bcab928ca166cde2b8874b65 - packages/preview/zebra/0.1.0/LICENSE: f0f6fec86b9678afa492dc078f81b8576bf25f0a1ca22be9f40e78b731fdf988 - packages/preview/zebra/0.1.0/README.md: d420f30d5c82f6fdd1f8ecaf58f98eaf6320670ab0ce626976c15f853cb57b6a - packages/preview/zebra/0.1.0/src/generic.typ: 9fb67e4684d1d35b81bb895b5e637286ef56a28c9fbc3715e53af30aa9efb5d4 - packages/preview/zebra/0.1.0/src/lib.typ: 5e2eb89b3fa143cb7f6458153ff95f44f81e2179adba150c3558c75b40283b46 - packages/preview/zebra/0.1.0/src/zebra.wasm: 9c4c9501d950d13505263d52da39d844d1c312d3c5ccc436a8f35602fd834975 - packages/preview/zebra/0.1.0/typst.toml: ece9c7e0e324a9566845a7e89dcda7b424d6c038b6ce8446d0b00b8f685c17af - schemas/receipt.schema.json: 9d8388a0cb66fb8ef7dae6b60bd7337f06df02f557c339aadac02e85dedd6736 - templates/receipt.typ: 1d112f888dc63e0f47173a149e1f21d1eccfdf5f191a1b88534f2e16cec1dafa diff --git a/products/render/golden.json b/products/render/golden.json index 598dc1eb23..3016ce7aac 100644 --- a/products/render/golden.json +++ b/products/render/golden.json @@ -7,7 +7,7 @@ "pdfSha256": "3f31d7fdc1492471259754a9acc9435c6c6475c9aa08f30a9ddcb04c8cab1519", "dataSha256": "8ab91deaf04f2ab9634da9fe45e63615f285c936659f8d839f413f647641b33a", "warnings": [], - "bundleHash": "96d199d20c94b39d4947168a84337277db1f01e86bb8907d0614d211855b0e26", + "bundleHash": "a4109d91fe340fd054c09f1b3b0d857332f964dd853265b13b014f248d7a2d38", "deps": [ "@preview/zebra:0.1.0/src/generic.typ", "@preview/zebra:0.1.0/src/lib.typ", @@ -21,7 +21,7 @@ "pdfSha256": "252d40678919cb8496950101efa7fa4984ccd6825655679cca6dead05c7696c0", "dataSha256": "cccb32e6fdcf4db99a556ede3abedcc4f46b94575b20a4b5ce361edfd1ac555a", "warnings": [], - "bundleHash": "3bb68569252cd621f0b6992b329de04524c29d02995c14056892f177ac9e6dff", + "bundleHash": "66e42dd1232470c8153411d16a9099ddb50727e76df57153f63c799fcade2051", "deps": [ "@preview/zebra:0.1.0/src/generic.typ", "@preview/zebra:0.1.0/src/lib.typ", @@ -35,7 +35,7 @@ "pdfSha256": "cf607a3b82a2027abfb4345377b1b84cd2e616da7376e18f29355ca790c3da2f", "dataSha256": "c595a3670a9b83327a8e16ef403b8253039d6d548b3388875930850d0f41fa18", "warnings": [], - "bundleHash": "4dd435c0bed4c80f4408dc87173b0f72887071ca78f734441bed979cabfe7d88", + "bundleHash": "6089a24558875d34a015cd09e207d4b5fd6bec62716fbd5075576398343cd6ac", "deps": [ "@preview/zebra:0.1.0/src/generic.typ", "@preview/zebra:0.1.0/src/lib.typ", diff --git a/products/render/integrations/app-kit/README.md b/products/render/integrations/app-kit/README.md index 7023edf4c9..c63e11ce53 100644 --- a/products/render/integrations/app-kit/README.md +++ b/products/render/integrations/app-kit/README.md @@ -16,7 +16,7 @@ brief must carry `agreed` rows for: - the reliance/verification wording printed next to the QR; - where the QR points (the host's public origin, the `registry-public-check` answer contract on the verify page); -- who signs off each bundle version (`registry-render seal` + review). +- who signs off each package version (`registry-render package` + review). ## Config (host, `taskDispatch` precedent) @@ -37,7 +37,7 @@ from files at load, mirroring the kit's task-dispatch configuration: } ``` -The bundle lives under `project/deployment/render/bundle/` so the kit's +The package lives under `project/deployment/render/package/` so the kit's writes-stay-in-`project/` boundary holds; `deployment/local.py` supervises `registry-render serve` beside breg and casework. diff --git a/products/render/integrations/openfn/JOURNEY.md b/products/render/integrations/openfn/JOURNEY.md index 4347fa5877..d3e8bc7bd1 100644 --- a/products/render/integrations/openfn/JOURNEY.md +++ b/products/render/integrations/openfn/JOURNEY.md @@ -19,7 +19,7 @@ re-walking it reproduces them against the current pinned values. | --- | --- | | OpenFn CLI | 1.40.1 (`openfn execute`, kit-style flags: `--no-autoinstall --no-expand-adaptors --no-cache-steps`) | | Adaptor | `@openfn/language-common` 3.3.4 (`util.request`, `parseAs: "json"`) | -| registry-render | this repo's binary, `registry-render serve` on loopback: sealed receipt bundle, API key + audit key in owner-only files | +| registry-render | this repo's binary, `registry-render serve` on loopback: verified receipt package, API key in an owner-only file, audit to an owner-only file | | breg | v0.32.0 dev stack (`bregctl dev`: PostgreSQL + ThunderID issuer + registry, all loopback) | | job | `receipt-job.js` (this directory), notify.js idiom | diff --git a/products/render/integrations/openfn/README.md b/products/render/integrations/openfn/README.md index 80e413d537..bcf7020a38 100644 --- a/products/render/integrations/openfn/README.md +++ b/products/render/integrations/openfn/README.md @@ -8,9 +8,9 @@ dead-letter replay. Deployment shape: -1. Run the binary with a runtime file (sealed bundle, loopback bind, API +1. Run the binary with a runtime file (verified package, loopback bind, API key in an owner-only file, audit file or `stdout` destination): - `registry-render serve --runtime /etc/registry-render/runtime.yaml` + `registry-render serve --runtime-config /etc/registry-render/runtime.yaml` 2. Put the API key *value* in the job's private configuration (kit precedent: `notification.json`), beside the breg reader token and the delivery adaptor's key. diff --git a/products/scheduling/CHANGELOG.md b/products/scheduling/CHANGELOG.md index bbbe22f122..27a2ed82e3 100644 --- a/products/scheduling/CHANGELOG.md +++ b/products/scheduling/CHANGELOG.md @@ -58,6 +58,51 @@ `unfinished` with reason `records.replace-unacknowledged`, since it may have taken effect. +- BREAKING: read `runtime.yaml` through the shared Registry Stack runtime + configuration loader. The file is capped at 1 MiB, and the runtime + configuration path and every configured path are refused when they pass + through a symbolic link. +- BREAKING: remove `authentication.oidc.jwksUri`. Declare + `authentication.oidc.jwksSource` with `kind: uri` and `uri` instead; the + removed key is refused with a diagnostic naming its replacement. +- BREAKING: `listener.bind` is required; it no longer defaults to + `127.0.0.1:8105`. +- BREAKING: an environment expression such as `${VAR}` in the authored + `scheduling.yaml`, or in a records or fixture document the authoring tooling + reads, is refused with the path of the field that holds it. +- Accept `${VAR}`, `${VAR:-default}`, and `${VAR:?message}` in string values of + `runtime.yaml`, never in a `*Ref` field or beneath one, nor under + `secretProviders`. +- BREAKING: a Scheduling package is the shared Registry Stack package format. + `schedulingctl package PROJECT --output DIRECTORY` writes `scheduling.yaml` + and `SHA256SUMS`, one `sha256sum` line per file sorted by path, into a new + directory, in place of `scheduling.package.json` beside the project; + `--dry-run` reports the digest without writing, and `--revision TEXT` + records one free-text line in a `REVISION` file the digest covers. The + report names `packageDigest`, the SHA-256 digest of `SHA256SUMS`, and no + longer carries the manifest's `policyDigest`; the semantic policy digest + `explain` reports, the store records, and hooks carry is unchanged. +- BREAKING: the runtime verifies `package.root` as a package at every start, + in every listener mode, with or without `package.expectedDigest`. A + changed, missing, or extra file, a directory without `SHA256SUMS` such as an + authored project, and a directory that still holds `scheduling.package.json` + are each refused by name, naming `schedulingctl package`. A + `development-loopback` runtime no longer serves an authored project; the + demo packages its project copy before it starts the runtime. +- Add the optional `package.expectedDigest` pin, compared with the package + digest at startup. A mismatch is refused in the shape every Registry Stack + runtime shares: + `package.expectedDigest is but the package at package.root is `. +- BREAKING: `authentication.oidc.issuer` must be an absolute `https` URL + without credentials, query, or fragment, or a loopback `http` URL under + `development-loopback`, and `authentication.oidc.audience` is at most 512 + characters without control characters. Both are checked by the shared OIDC + issuer block, the same one the other runtimes use. +- A refused `authentication.oidc.assertionIssuers` map is now reported at that + field rather than at `authentication.oidc`. Its bounds are unchanged. +- `audit.hashKeyRef` must be an exact secret reference when the document is + read, not only when its provider is checked. + ## v0.34.0 - 2026-09-25 - Registry Scheduling has no user-visible changes in this release. diff --git a/products/scheduling/RUNTIME-CONFIG.md b/products/scheduling/RUNTIME-CONFIG.md index 60a6074316..a9472a36d5 100644 --- a/products/scheduling/RUNTIME-CONFIG.md +++ b/products/scheduling/RUNTIME-CONFIG.md @@ -2,8 +2,21 @@ Scheduling reads one versioned operator document selected with `scheduling --runtime-config ABSOLUTE_FILE serve` or `migrate`. The selected -file path and every operated resource path are absolute. Local development -tooling may resolve paths before it writes the file. +file path and every operated resource path are absolute, and none may pass +through a symbolic link. Local development tooling may resolve paths before it +writes the file. The file is read through the shared Registry Stack runtime +configuration loader: it must be a YAML mapping of at most 1 MiB, and unknown +keys are refused with the path of the offending field. + +String values in `runtime.yaml` may take a deployment value from the +environment when the runtime starts: `${VAR}` requires `VAR`, `${VAR:-default}` +falls back to `default`, and `${VAR:?message}` refuses to start with `message` +when `VAR` is unset. Substitution never applies to a field whose name ends in +`Ref` or `Refs`, or to any value beneath one, because a secret reference must +be written literally and resolved by a declared provider. It never applies to +the authored `scheduling.yaml` or to the records and fixture documents the +authoring tooling reads either: an environment expression there is refused +with the path of the field that holds it. The closed envelope is: @@ -13,14 +26,24 @@ kind: SchedulingRuntimeConfig ``` `package.root` selects one directory. The runtime always loads -`package.root/scheduling.yaml`; no second project selector can override it. -With `listener.tlsTermination: operator-controlled-upstream`, the directory -must also contain a matching `scheduling.package.json`. Development loopback -may select an authored project directory without that manifest. - -`listener` is required. `listener.bind` is one numeric socket address, -including bracketed IPv6 forms, and defaults to `127.0.0.1:8105` when omitted -from the listener block. `listener.tlsTermination` is required. Use +`package.root/scheduling.yaml`; no second project selector can override it. In +every listener mode, with or without `package.expectedDigest`, the directory +must be a package `schedulingctl package --output` wrote: a `SHA256SUMS` file +listing exactly `scheduling.yaml`, whose bytes match the listed digest. A +changed, missing, or extra file is refused by name, and so is a directory +without `SHA256SUMS`, such as an authored project; each refusal names +`schedulingctl package`. A directory that holds the retired +`scheduling.package.json` is refused the same way. + +`package.expectedDigest` optionally pins the package the runtime must serve: +`sha256:` followed by 64 lowercase hexadecimal digits, the `packageDigest` +`schedulingctl package` prints, which is the SHA-256 digest of the package's +`SHA256SUMS`. Any other package is a startup refusal that names the expected +digest and the one found. + +`listener` is required. `listener.bind` is required and is one numeric +socket address, including bracketed IPv6 forms. `listener.tlsTermination` is +required. Use `operator-controlled-upstream` behind an operator-managed TLS edge or `development-loopback` for direct local development, which the runtime refuses on any non-loopback bind. `listener.networkExposure` defaults to @@ -53,10 +76,13 @@ because its database URL is absent is not database verification, and the escape never turns a deployed runtime into a plaintext client. `authentication.oidc` requires `issuer` and `audience`. `jwksSource` defaults -to discovery and can instead select a static `documentRef`, which does no -rotation of its own: rolling a key means replacing the referenced document and -restarting Scheduling. `jwksUri` overrides the discovery document's JWKS -address. `scopeClaim` defaults to `registry_scopes` for compatibility with +to `kind: discovery`. `kind: uri` with `uri` fetches the key set from a fixed +HTTPS address instead of the one discovery names; plain `http` is accepted only +for a loopback host under development loopback. `kind: static` with +`documentRef` reads a pinned key set, which does no rotation of its own: +rolling a key means replacing the referenced document and restarting +Scheduling. The removed `jwksUri` key is refused with a diagnostic naming +`jwksSource` `kind: uri` as its replacement. `scopeClaim` defaults to `registry_scopes` for compatibility with existing deployments; stock ThunderID emits `scope`, so the maintained example and `schedulingctl init` set that explicit override. `readsScope` defaults to `scheduling-read` and `explainScope` to `scheduling-explain`; the two must diff --git a/products/scheduling/SECURITY-REVIEW-NOTES.md b/products/scheduling/SECURITY-REVIEW-NOTES.md index 27f9ffc303..2b780794d8 100644 --- a/products/scheduling/SECURITY-REVIEW-NOTES.md +++ b/products/scheduling/SECURITY-REVIEW-NOTES.md @@ -297,16 +297,19 @@ keeping only the shape word. *Tests:* `typed_parse_path_does_not_echo_the_rejected_value` (SCHEDULING-SEC-13). -**H. The package manifest is a different document from the policy it -identifies.** *Threat:* one `apiVersion` and `kind` pair named both -documents, so neither reader could refuse the other's document on its -declared names, and the package identity digest folded in a pair that did -not describe it. *Default:* the manifest declares its own pair, which the -identity digest binds; a document carrying the authored policy's names -never verifies as a package identity. *Tests:* -`a_manifest_wearing_the_authored_policy_names_is_refused` with -`a_package_manifest_is_a_different_document_from_the_policy_it_identifies`, -`crates/registry-scheduling/src/config.rs`. +**H. The runtime serves only a verified shared package.** *Threat:* a +runtime that reads an authored project directly, or a package whose files +were edited after review, serves a policy nobody packaged. *Default:* at +every start, in every listener mode and with or without a pin, the runtime +verifies `package.root` against its `SHA256SUMS`: a changed, missing, or +extra file, an absent `SHA256SUMS`, or the retired `scheduling.package.json` +is refused by name before the policy is parsed, and the package digest is +the SHA-256 digest of `SHA256SUMS`. The semantic policy digest the store and +hooks record is unchanged and no longer doubles as the package identity. +*Tests:* `every_listener_mode_verifies_the_package_without_a_pin`, +`crates/registry-scheduling/src/config.rs`, and +`package_writes_a_package_the_runtime_verifies_and_refuses_replacement`, +`crates/registry-schedulingctl/src/lib.rs`. **I. Client tolerance is response-only.** *Threat:* relaxing client-side parsing so a client outlives a deployment that adds a member would, if diff --git a/products/scheduling/contracts/security-invariant-matrix.yaml b/products/scheduling/contracts/security-invariant-matrix.yaml index e214ecdfbd..980a371b6b 100644 --- a/products/scheduling/contracts/security-invariant-matrix.yaml +++ b/products/scheduling/contracts/security-invariant-matrix.yaml @@ -287,15 +287,15 @@ invariants: An unverified, drifted, or failing policy package is served, so the deployment answers under terms nobody reviewed. enforcementPoint: >- - Startup package verification in production, the authoring exception - confined to a loopback listener, and the authoring checks re-run before - the policy reaches the runtime. + Startup verification of the shared package against its SHA256SUMS in + every listener mode, and the authoring checks re-run before the policy + reaches the runtime. refusal: >- Refuse to start rather than serve an unverified package or a policy that carries findings. negativeTest: path: crates/registry-scheduling/src/config.rs - name: production_requires_a_verified_package_while_loopback_accepts_authoring + name: every_listener_mode_verifies_the_package_without_a_pin - id: SCHEDULING-SEC-16 state: enforced threat: >- diff --git a/products/scheduling/contracts/security-test-traceability.yaml b/products/scheduling/contracts/security-test-traceability.yaml index a2eb2efe51..bb33aacbb2 100644 --- a/products/scheduling/contracts/security-test-traceability.yaml +++ b/products/scheduling/contracts/security-test-traceability.yaml @@ -141,7 +141,7 @@ entries: - {path: crates/registry-schedulingctl/tests/records_apply_postgres.rs, name: records_apply_replaces_facts_wholesale_and_audits_each_write} - id: SCHEDULING-SEC-15 tests: - - {path: crates/registry-scheduling/src/config.rs, name: production_requires_a_verified_package_while_loopback_accepts_authoring} + - {path: crates/registry-scheduling/src/config.rs, name: every_listener_mode_verifies_the_package_without_a_pin} - {path: crates/registry-scheduling/src/config.rs, name: a_refused_authored_policy_names_the_member_and_the_cause} - {path: crates/registry-scheduling-core/src/policy.rs, name: the_policy_digest_is_stable_and_moves_with_content} - {path: crates/registry-scheduling/tests/postgres_commitments.rs, name: adoption_binds_one_identity_and_refuses_a_second} diff --git a/products/scheduling/demo/run.sh b/products/scheduling/demo/run.sh index 5ac353fa85..c707f6a26b 100755 --- a/products/scheduling/demo/run.sh +++ b/products/scheduling/demo/run.sh @@ -11,7 +11,7 @@ set -euo pipefail # AT-05 (hold expiry), AT-06 (lost-confirmation idempotent replay), and # AT-19 (the daylight-saving fold grid). See README.md beside this script. -demo_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +demo_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P) root=$(cd -- "$demo_dir/../../.." && pwd) run_dir="$demo_dir/.run" support="$demo_dir/support/demo.py" @@ -165,7 +165,7 @@ fi project="$run_dir/project" "$schedulingctl" check --deny-findings "$project" >/dev/null "$schedulingctl" test "$project" >/dev/null -"$schedulingctl" package "$project" >/dev/null +"$schedulingctl" package "$project" --output "$run_dir/package" >/dev/null "$scheduling" --runtime-config "$run_dir/runtime.yaml" migrate "$schedulingctl" records apply "$run_dir/runtime.yaml" "$run_dir/records.yaml" >/dev/null diff --git a/products/scheduling/demo/support/demo.py b/products/scheduling/demo/support/demo.py index 7ea3ea7dc2..11df32f013 100644 --- a/products/scheduling/demo/support/demo.py +++ b/products/scheduling/demo/support/demo.py @@ -384,7 +384,7 @@ def generate_database_tls(root: Path) -> None: def runtime_config( - project: Path, + package_root: Path, secrets_root: Path, audit_path: Path, port: int, @@ -399,7 +399,7 @@ def runtime_config( apiVersion: registry.registrystack.org/scheduling-runtime/v1alpha1 kind: SchedulingRuntimeConfig package: - root: {project} + root: {package_root} listener: bind: 127.0.0.1:{port} tlsTermination: development-loopback @@ -435,7 +435,9 @@ def prepare( database_root_ca: Path | None, ) -> None: """Lay out the run directory: project copy, key material, records, and - the runtime configuration, all owner-only.""" + the runtime configuration, all owner-only. The runtime serves the package + `schedulingctl package` writes from the project copy into `package`, so + the configuration names that directory before it exists.""" project = root / "project" shutil.copytree(example, project) policy_path = project / "scheduling.yaml" @@ -491,7 +493,11 @@ def prepare( (root / "records.yaml").write_text(DEMO_RECORDS) (root / "runtime.yaml").write_text( runtime_config( - project.resolve(), secrets_root.resolve(), audit_path.resolve(), port, trust_root + root.resolve() / "package", + secrets_root.resolve(), + audit_path.resolve(), + port, + trust_root, ) ) diff --git a/products/scheduling/demo/support/test_demo.py b/products/scheduling/demo/support/test_demo.py index 6031a39c42..dcbb9c243a 100644 --- a/products/scheduling/demo/support/test_demo.py +++ b/products/scheduling/demo/support/test_demo.py @@ -138,12 +138,12 @@ def test_the_records_document_carries_one_station_per_pool(self): class RuntimeConfigTest(unittest.TestCase): def test_the_runtime_config_pins_every_required_block(self): config = demo.runtime_config( - Path("/run/project"), Path("/run/secrets"), Path("/run/audit"), 8105, None + Path("/run/package"), Path("/run/secrets"), Path("/run/audit"), 8105, None ) for required in ( "apiVersion: registry.registrystack.org/scheduling-runtime/v1alpha1", "kind: SchedulingRuntimeConfig", - "root: /run/project", + "root: /run/package", "bind: 127.0.0.1:8105", "tlsTermination: development-loopback", "root: /run/secrets", @@ -161,7 +161,7 @@ def test_the_runtime_config_pins_every_required_block(self): def test_the_database_trust_root_is_named_when_present(self): config = demo.runtime_config( - Path("/run/project"), Path("/run/secrets"), Path("/run/audit"), 8105, + Path("/run/package"), Path("/run/secrets"), Path("/run/audit"), 8105, Path("/run/secrets/db-root-ca"), ) self.assertIn( @@ -171,7 +171,7 @@ def test_the_database_trust_root_is_named_when_present(self): def test_the_audit_path_names_the_audit_file(self): config = demo.runtime_config( - Path("/run/project"), Path("/run/secrets"), Path("/run/audit/audit.jsonl"), 8105, None + Path("/run/package"), Path("/run/secrets"), Path("/run/audit/audit.jsonl"), 8105, None ) self.assertIn("path: /run/audit/audit.jsonl", config) diff --git a/products/scheduling/examples/standalone-arrival-window/runtime.example.yaml b/products/scheduling/examples/standalone-arrival-window/runtime.example.yaml index c379a26487..40a4c73ea0 100644 --- a/products/scheduling/examples/standalone-arrival-window/runtime.example.yaml +++ b/products/scheduling/examples/standalone-arrival-window/runtime.example.yaml @@ -4,17 +4,28 @@ # location, and point the secret references at secrets the configured provider # can resolve. Scheduling reads the authored policy at # package.root/scheduling.yaml and refuses to start when that file is missing -# or does not pass its checks. +# or does not pass its checks. Every configured path must be absolute and must +# not pass through a symbolic link. +# +# String values in this file may use ${VAR}, ${VAR:-default}, or ${VAR:?message} +# to take a deployment value from the environment when the runtime starts. +# Substitution never applies to a *Ref field, whose value must be written as a +# literal secret reference, nor under secretProviders, nor to the authored +# scheduling.yaml. apiVersion: registry.registrystack.org/scheduling-runtime/v1alpha1 kind: SchedulingRuntimeConfig package: - # The selected package always contains the policy at scheduling.yaml; no - # second selector can override it. Under - # listener.tlsTermination: operator-controlled-upstream the directory must - # also contain the scheduling.package.json manifest `schedulingctl package` - # writes; development loopback may select an unpackaged project directory. + # The package `schedulingctl package --output` writes: scheduling.yaml and + # the SHA256SUMS file that lists it. The runtime verifies it at every start, + # in every listener mode, and refuses a changed, missing, or extra file; no + # second selector can override the policy it holds. root: /srv/registry-scheduling/package + # Pin the package the runtime must serve: the packageDigest + # `schedulingctl package` prints, the SHA-256 digest of SHA256SUMS. Any + # other package is a startup refusal. + # expectedDigest: sha256:<64 lowercase hex digits> listener: + # Required: the IP address and port the runtime listens on. bind: 127.0.0.1:8105 tlsTermination: development-loopback networkExposure: private-address @@ -81,6 +92,8 @@ authentication: # jwksSource: # kind: static # documentRef: secret:file/jwks.json + # `kind: uri` with `uri: https://...` fetches the key set from a fixed + # address instead of the one discovery names. audit: # One single-writer destination. `file`, the default, appends to the # absolute path, rotates at rotateBytes (default 100 MiB), and deletes diff --git a/products/scheduling/examples/standalone-exact-time/runtime.example.yaml b/products/scheduling/examples/standalone-exact-time/runtime.example.yaml index c379a26487..40a4c73ea0 100644 --- a/products/scheduling/examples/standalone-exact-time/runtime.example.yaml +++ b/products/scheduling/examples/standalone-exact-time/runtime.example.yaml @@ -4,17 +4,28 @@ # location, and point the secret references at secrets the configured provider # can resolve. Scheduling reads the authored policy at # package.root/scheduling.yaml and refuses to start when that file is missing -# or does not pass its checks. +# or does not pass its checks. Every configured path must be absolute and must +# not pass through a symbolic link. +# +# String values in this file may use ${VAR}, ${VAR:-default}, or ${VAR:?message} +# to take a deployment value from the environment when the runtime starts. +# Substitution never applies to a *Ref field, whose value must be written as a +# literal secret reference, nor under secretProviders, nor to the authored +# scheduling.yaml. apiVersion: registry.registrystack.org/scheduling-runtime/v1alpha1 kind: SchedulingRuntimeConfig package: - # The selected package always contains the policy at scheduling.yaml; no - # second selector can override it. Under - # listener.tlsTermination: operator-controlled-upstream the directory must - # also contain the scheduling.package.json manifest `schedulingctl package` - # writes; development loopback may select an unpackaged project directory. + # The package `schedulingctl package --output` writes: scheduling.yaml and + # the SHA256SUMS file that lists it. The runtime verifies it at every start, + # in every listener mode, and refuses a changed, missing, or extra file; no + # second selector can override the policy it holds. root: /srv/registry-scheduling/package + # Pin the package the runtime must serve: the packageDigest + # `schedulingctl package` prints, the SHA-256 digest of SHA256SUMS. Any + # other package is a startup refusal. + # expectedDigest: sha256:<64 lowercase hex digits> listener: + # Required: the IP address and port the runtime listens on. bind: 127.0.0.1:8105 tlsTermination: development-loopback networkExposure: private-address @@ -81,6 +92,8 @@ authentication: # jwksSource: # kind: static # documentRef: secret:file/jwks.json + # `kind: uri` with `uri: https://...` fetches the key set from a fixed + # address instead of the one discovery names. audit: # One single-writer destination. `file`, the default, appends to the # absolute path, rotates at rotateBytes (default 100 MiB), and deletes diff --git a/products/scheduling/generated/runtime/runtime.schema.json b/products/scheduling/generated/runtime/runtime.schema.json index 2df06c6b24..978dab70ed 100644 --- a/products/scheduling/generated/runtime/runtime.schema.json +++ b/products/scheduling/generated/runtime/runtime.schema.json @@ -2,6 +2,7 @@ "$defs": { "AuditConfig": { "additionalProperties": false, + "description": "The audit journal: where it is written and the key its hashes use.", "else": { "properties": { "path": { @@ -29,8 +30,8 @@ "description": "Where audit entries go: a rotated `file` (the default) or `stdout`." }, "hashKeyRef": { - "pattern": "^secret:(?:env|file)/", - "type": "string" + "$ref": "#/$defs/SecretReference", + "description": "Secret reference to the audit hash key." }, "path": { "default": null, @@ -140,21 +141,26 @@ }, "DatabaseConfig": { "additionalProperties": false, + "description": "The PostgreSQL connection a stateful runtime uses. Both URLs are secret\nreferences and may name the same secret.", "properties": { "migrationUrlRef": { + "description": "Secret reference to the migration connection URL.", "pattern": "^secret:(?:env|file)/", "type": "string" }, "runtimeUrlRef": { + "description": "Secret reference to the least-privileged runtime connection URL.", "pattern": "^secret:(?:env|file)/", "type": "string" }, "testOnlyPlaintext": { "default": false, + "description": "Allow a plaintext connection. Refused outside test builds.", "type": "boolean" }, "trustedRootCertificateRef": { "default": null, + "description": "Secret reference to a PEM root certificate the connection trusts.", "pattern": "^secret:(?:env|file)/", "type": [ "string", @@ -194,12 +200,15 @@ }, "EnvironmentSecretProviderConfig": { "additionalProperties": false, + "description": "The environment secret provider. It takes no settings.", "type": "object" }, "FileSecretProviderConfig": { "additionalProperties": false, + "description": "The file secret provider.", "properties": { "root": { + "description": "Absolute directory holding one file per secret. Each file must be a\nregular file owned by the runtime user, mode 0400 or 0600, with exactly\none hard link.", "pattern": "^/", "type": "string" } @@ -219,6 +228,7 @@ "type": "integer" }, "hmacSha256KeyRef": { + "pattern": "^secret:(?:env|file)/", "type": "string" }, "maximumAttempts": { @@ -238,27 +248,71 @@ ], "type": "object" }, - "ListenerConfig": { - "additionalProperties": false, - "properties": { - "bind": { - "default": "127.0.0.1:8105", - "type": "string" + "JwksSource": { + "description": "Where a runtime obtains the OIDC issuer's signing keys.", + "oneOf": [ + { + "additionalProperties": false, + "description": "Read `jwks_uri` from the issuer's OpenID Connect discovery document.", + "properties": { + "kind": { + "const": "discovery", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" }, - "networkExposure": { - "$ref": "#/$defs/ListenerNetworkExposure" + { + "additionalProperties": false, + "description": "Fetch the key set from this absolute `https` URI, skipping discovery.", + "properties": { + "kind": { + "const": "uri", + "type": "string" + }, + "uri": { + "pattern": "^https?://", + "type": "string" + } + }, + "required": [ + "kind", + "uri" + ], + "type": "object" }, - "tlsTermination": { - "$ref": "#/$defs/TlsTermination" + { + "additionalProperties": false, + "description": "Read the key set from a secret, for deployments without network access\nto the issuer.", + "properties": { + "documentRef": { + "pattern": "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$", + "type": "string" + }, + "kind": { + "const": "static", + "type": "string" + } + }, + "required": [ + "kind", + "documentRef" + ], + "type": "object" } - }, - "required": [ - "tlsTermination" - ], - "type": "object" + ] + }, + "ListenerBind": { + "description": "Socket address the runtime listens on, written host:port with an IP address host ([addr]:port for IPv6).", + "maxLength": 128, + "minLength": 1, + "type": "string" }, "ListenerNetworkExposure": { - "description": "The operator-declared private network placement of the HTTP listener.", + "description": "The operator-declared private network placement of an HTTP listener.", "enum": [ "private-address", "container-private" @@ -267,9 +321,11 @@ }, "OidcConfig": { "additionalProperties": false, + "description": "The access tokens this runtime accepts: the issuer and its keys, the\nclients admitted, and the scopes each route family requires.", "properties": { "allowedClients": { "default": [], + "description": "Client identifiers whose access tokens are admitted. A runtime decides\nwhether an empty list is acceptable in production.", "items": { "type": "string" }, @@ -287,7 +343,7 @@ "uniqueItems": true }, "default": {}, - "description": "The assertion authorities each client may exchange a subject token\nfrom, keyed by client identifier.\n\nA deployment that performs no token exchange leaves this empty. Once a\nclient is listed, a token it exchanged is accepted only for one of that\nclient's declared authorities, so an assertion minted by an unrelated\nauthority the issuer happens to federate cannot become a booking\ncredential here.", + "description": "Assertion authorities each client may exchange a subject token from,\nkeyed by client identifier. An empty map applies no rule. Once a\nclient is listed, a token it exchanged is accepted only for one of\nthat client's declared authorities.", "maxProperties": 64, "propertyNames": { "maxLength": 128, @@ -296,6 +352,9 @@ "type": "object" }, "audience": { + "description": "The audience every accepted access token must carry in `aud`.", + "maxLength": 512, + "minLength": 1, "type": "string" }, "explainScope": { @@ -304,17 +363,16 @@ "type": "string" }, "issuer": { + "description": "Exact issuer accepted in access-token `iss` claims, an absolute\n`https` URL.", + "pattern": "^https?://", "type": "string" }, "jwksSource": { - "$ref": "#/$defs/OidcJwksSource" - }, - "jwksUri": { - "default": null, - "type": [ - "string", - "null" - ] + "$ref": "#/$defs/JwksSource", + "default": { + "kind": "discovery" + }, + "description": "Where the issuer's signing keys come from. Absent reads the issuer's\nOpenID Connect discovery document." }, "readsScope": { "default": "scheduling-read", @@ -332,40 +390,49 @@ ], "type": "object" }, - "OidcJwksSource": { - "oneOf": [ - { - "additionalProperties": false, - "properties": { - "kind": { - "const": "discovery", - "type": "string" - } - }, - "required": [ - "kind" - ], - "type": "object" + "PackageConfig": { + "additionalProperties": false, + "description": "The package a runtime serves: `root` is the absolute package directory,\nand `expectedDigest`, when set, pins the package digest the runtime must\nfind there. The package digest is the digest of the package's\n`SHA256SUMS` file; see [`crate::package`].", + "properties": { + "expectedDigest": { + "description": "`sha256:` label of the package digest, the digest of the package's\n`SHA256SUMS` file. When set, the runtime refuses to start on any other\npackage.", + "pattern": "^sha256:[0-9a-f]{64}$", + "type": [ + "string", + "null" + ] }, - { - "additionalProperties": false, - "properties": { - "documentRef": { - "pattern": "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$", - "type": "string" - }, - "kind": { - "const": "static", - "type": "string" - } - }, - "required": [ - "kind", - "documentRef" - ], - "type": "object" + "root": { + "description": "Absolute path of the package directory.", + "pattern": "^/", + "type": "string" } - ] + }, + "required": [ + "root" + ], + "type": "object" + }, + "PrivateListenerConfig": { + "additionalProperties": false, + "description": "The listener of a runtime that declares its TLS termination and network\nexposure.", + "properties": { + "bind": { + "$ref": "#/$defs/ListenerBind" + }, + "networkExposure": { + "$ref": "#/$defs/ListenerNetworkExposure", + "default": "private-address" + }, + "tlsTermination": { + "$ref": "#/$defs/TlsTermination" + } + }, + "required": [ + "bind", + "tlsTermination" + ], + "type": "object" }, "ReminderDestinationConfig": { "additionalProperties": false, @@ -410,20 +477,6 @@ }, "type": "object" }, - "RuntimePackageConfig": { - "additionalProperties": false, - "description": "The root of an authored scheduling project, holding `scheduling.yaml` and\nits optional package manifest.", - "properties": { - "root": { - "pattern": "^/", - "type": "string" - } - }, - "required": [ - "root" - ], - "type": "object" - }, "SecretProvidersConfig": { "additionalProperties": false, "anyOf": [ @@ -448,6 +501,7 @@ ] } ], + "description": "The secret providers a runtime enables. A reference is resolved only by a\nprovider declared here: `secret:file/name` under `file.root`, and\n`secret:env/NAME` only when `environment: {}` is present.", "properties": { "environment": { "anyOf": [ @@ -457,7 +511,8 @@ { "type": "null" } - ] + ], + "description": "Enables `secret:env/NAME` references, read from the process\nenvironment. Declared as an empty mapping: `environment: {}`." }, "file": { "anyOf": [ @@ -467,13 +522,19 @@ { "type": "null" } - ] + ], + "description": "Enables `secret:file/name` references, read from files under `root`." } }, "type": "object" }, + "SecretReference": { + "description": "An exact secret reference: secret:file/name, resolved under secretProviders.file.root, or secret:env/NAME, resolved only when secretProviders.environment is declared.", + "pattern": "^(?:secret:env/[A-Z][A-Z0-9_]{0,127}|secret:file/[a-z][a-z0-9._-]{0,127})$", + "type": "string" + }, "TlsTermination": { - "description": "Declares the trusted transport boundary for the runtime's plaintext HTTP\nlistener. Production listeners require operator-controlled upstream TLS\ntermination; direct plaintext is limited to the explicit loopback-only\ndevelopment mode.", + "description": "Declares the trusted transport boundary for a runtime's plaintext HTTP\nlistener. Production listeners require operator-controlled upstream TLS\ntermination; direct plaintext is limited to the explicit loopback-only\ndevelopment mode.", "enum": [ "operator-controlled-upstream", "development-loopback" @@ -603,10 +664,10 @@ "type": "string" }, "listener": { - "$ref": "#/$defs/ListenerConfig" + "$ref": "#/$defs/PrivateListenerConfig" }, "package": { - "$ref": "#/$defs/RuntimePackageConfig" + "$ref": "#/$defs/PackageConfig" }, "retention": { "$ref": "#/$defs/RetentionConfig" diff --git a/products/scheduling/scripts/check-checkpoint.sh b/products/scheduling/scripts/check-checkpoint.sh index 658da39792..67d744c97b 100755 --- a/products/scheduling/scripts/check-checkpoint.sh +++ b/products/scheduling/scripts/check-checkpoint.sh @@ -115,9 +115,30 @@ if [ "$plain_explain" = "$banded_explain" ]; then exit 1 fi -# The package journey on a fresh project: the manifest the runtime verifies. -"$schedulingctl_bin" init "$work/package" --template standalone-exact-time >/dev/null -"$schedulingctl_bin" package "$work/package" >/dev/null +# The package journey on a fresh project: the shared package the runtime +# verifies, planned and then written to the same digest. +"$schedulingctl_bin" init "$work/package-project" --template standalone-exact-time >/dev/null +planned=$("$schedulingctl_bin" package "$work/package-project" --dry-run --format json | + python3 -c 'import json, sys; print(json.load(sys.stdin)["packageDigest"])') +written=$("$schedulingctl_bin" package "$work/package-project" --output "$work/package" --format json | + python3 -c 'import json, sys; print(json.load(sys.stdin)["packageDigest"])') +if [ "$planned" != "$written" ]; then + echo "the planned package digest $planned differs from the written $written" >&2 + exit 1 +fi +if [ ! -f "$work/package/SHA256SUMS" ] || [ ! -f "$work/package/scheduling.yaml" ]; then + echo 'schedulingctl package wrote no SHA256SUMS or scheduling.yaml' >&2 + exit 1 +fi +sums_digest="sha256:$(python3 -c 'import hashlib, sys; print(hashlib.sha256(open(sys.argv[1], "rb").read()).hexdigest())' "$work/package/SHA256SUMS")" +if [ "$written" != "$sums_digest" ]; then + echo "the package digest $written is not the SHA-256 digest of SHA256SUMS" >&2 + exit 1 +fi +if [ -e "$work/package/scheduling.package.json" ]; then + echo 'schedulingctl package wrote the retired scheduling.package.json' >&2 + exit 1 +fi # The committed examples are the starter templates' output, so neither can # drift from what an adopter initializes, including its live records document. diff --git a/release/docker/Dockerfile.breg b/release/docker/Dockerfile.breg index 0a3dc38b38..bfb94c8165 100644 --- a/release/docker/Dockerfile.breg +++ b/release/docker/Dockerfile.breg @@ -42,4 +42,4 @@ EXPOSE 8080 # Base Registry Engine serves GET /health and GET /healthz. Distroless has no shell # or HTTP client, and the runtime deliberately has no healthcheck subcommand. ENTRYPOINT ["/usr/local/bin/breg"] -CMD ["--config", "/etc/breg/runtime.yaml"] +CMD ["--runtime-config", "/etc/breg/runtime.yaml"] diff --git a/release/docker/Dockerfile.discovery b/release/docker/Dockerfile.discovery index c553047b22..b6de9561ce 100644 --- a/release/docker/Dockerfile.discovery +++ b/release/docker/Dockerfile.discovery @@ -9,10 +9,10 @@ ADD --checksum=sha256:967aa62605721081c3eb2a17650611a792aa802d76a6511d1840242623 ADD --checksum=sha256:8784eda966b189c777a384dac5ce009e8fc9b52d006926c5a013e7fa8aa688cc https://snapshot.debian.org/archive/debian/20260913T000000Z/pool/main/g/glibc/libc6_2.41-12+deb13u4_arm64.deb /workspace/runtime-packages/libc6_2.41-12+deb13u4_arm64.deb # Discovery serves one immutable index and writes nothing, so the image -# pre-owns no managed state or audit directory. The runtime file and the index -# it names are startup-only deployment artifacts mounted read-only together -# under /etc/registry-discovery: the index path is resolved relative to the -# runtime file's own directory. +# pre-owns no managed state or audit directory. Mount runtime.yaml read-only +# under /etc/registry-discovery and mount the complete package directory that +# its absolute package.root names. The runtime verifies SHA256SUMS before it +# consumes discovery-index.json. RUN --mount=type=bind,source=dist/image-bin,target=/workspace/image-bin \ --mount=type=bind,source=LICENSE,target=/workspace/LICENSE \ --mount=type=bind,source=release/scripts/install-runtime-libc6.sh,target=/workspace/install-runtime-libc6.sh,readonly \ @@ -36,7 +36,7 @@ EXPOSE 8080 # Discovery serves GET /health for the platform's HTTP probe. The Distroless # image has no shell or HTTP client, and Discovery has no healthcheck -# subcommand. It also reads no environment variable, so the command is what -# binds the runtime file. +# subcommand. The command binds the runtime file; that file may use the shared +# loader's explicit environment substitutions. ENTRYPOINT ["/usr/local/bin/discovery"] -CMD ["--runtime", "/etc/registry-discovery/runtime.yaml"] +CMD ["--runtime-config", "/etc/registry-discovery/runtime.yaml"] diff --git a/release/docker/Dockerfile.evidence b/release/docker/Dockerfile.evidence index 9a5e8fe2b1..1df5c786a0 100644 --- a/release/docker/Dockerfile.evidence +++ b/release/docker/Dockerfile.evidence @@ -34,11 +34,9 @@ COPY --from=runtime-root /workspace/runtime-root/ / WORKDIR /var/lib/registry-evidence -ENV REGISTRY_EVIDENCE_RUNTIME=/etc/registry-evidence/runtime.yaml - EXPOSE 8080 # Evidence serves GET /health for the platform's HTTP probe. The Distroless # image has no shell or HTTP client, and Evidence has no healthcheck subcommand. ENTRYPOINT ["/usr/local/bin/evidence"] -CMD ["serve"] +CMD ["serve", "--runtime-config", "/etc/registry-evidence/runtime.yaml"] diff --git a/release/docker/Dockerfile.relay b/release/docker/Dockerfile.relay index 8699ee6d4a..65504c24fc 100644 --- a/release/docker/Dockerfile.relay +++ b/release/docker/Dockerfile.relay @@ -47,4 +47,4 @@ ENV RELAY_HEALTHCHECK_URL=http://127.0.0.1:8080/health HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 CMD ["/usr/local/bin/relay", "healthcheck"] ENTRYPOINT ["/usr/local/bin/relay"] -CMD ["serve", "--runtime", "/etc/relay/runtime.yaml"] +CMD ["serve", "--runtime-config", "/etc/relay/runtime.yaml"] diff --git a/release/scripts/check-debian13-images.py b/release/scripts/check-debian13-images.py index c5b7527b65..5b3ce91b1b 100755 --- a/release/scripts/check-debian13-images.py +++ b/release/scripts/check-debian13-images.py @@ -149,27 +149,26 @@ ENV RELAY_HEALTHCHECK_URL=http://127.0.0.1:8080/health HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 CMD ["/usr/local/bin/relay", "healthcheck"] ENTRYPOINT ["/usr/local/bin/relay"] -CMD ["serve", "--runtime", "/etc/relay/runtime.yaml"] +CMD ["serve", "--runtime-config", "/etc/relay/runtime.yaml"] """ # Each entry pins the runtime instructions that bind one HTTP-probed service to -# its configuration. Discovery reads no environment variable, so it declares no +# its configuration. A service that reads no environment variable declares no # `environment` and binds its runtime file through the command instead. HTTP_PROBE_DOCKERFILES = { Path("release/docker/Dockerfile.discovery"): { "binary": "discovery", "entrypoint": 'ENTRYPOINT ["/usr/local/bin/discovery"]', - "command": 'CMD ["--runtime", "/etc/registry-discovery/runtime.yaml"]', + "command": 'CMD ["--runtime-config", "/etc/registry-discovery/runtime.yaml"]', }, Path("release/docker/Dockerfile.evidence"): { "binary": "evidence", - "environment": "ENV REGISTRY_EVIDENCE_RUNTIME=/etc/registry-evidence/runtime.yaml", "entrypoint": 'ENTRYPOINT ["/usr/local/bin/evidence"]', - "command": 'CMD ["serve"]', + "command": 'CMD ["serve", "--runtime-config", "/etc/registry-evidence/runtime.yaml"]', }, Path("release/docker/Dockerfile.breg"): { "binary": "breg", "entrypoint": 'ENTRYPOINT ["/usr/local/bin/breg"]', - "command": 'CMD ["--config", "/etc/breg/runtime.yaml"]', + "command": 'CMD ["--runtime-config", "/etc/breg/runtime.yaml"]', }, Path("release/docker/Dockerfile.casework"): { "binary": "casework", @@ -608,7 +607,7 @@ def check_repository(root: Path = ROOT) -> list[str]: ) require( runtime_stage(text), - 'CMD ["serve", "--runtime", "/etc/relay/runtime.yaml"]', + 'CMD ["serve", "--runtime-config", "/etc/relay/runtime.yaml"]', relative, "absolute Relay V2 runtime configuration binding", failures, diff --git a/release/scripts/check-gates-inventory.py b/release/scripts/check-gates-inventory.py index 3c0ac3e5ec..ff298e9e8f 100644 --- a/release/scripts/check-gates-inventory.py +++ b/release/scripts/check-gates-inventory.py @@ -144,6 +144,19 @@ "Platform hygiene alignment", "run: products/platform/scripts/check-hygiene-alignment.sh", ), + ("Runtime configuration conformance job", "config-conformance:"), + ( + "Runtime configuration conformance path filter", + "config_conformance: ${{ steps.filter.outputs.config_conformance }}", + ), + ( + "Runtime configuration conformance gate", + "run: products/platform/scripts/check-config-conformance.py --check-generated", + ), + ( + "Runtime configuration conformance gate tests", + "run: python3 -m unittest products/platform/scripts/test_check_config_conformance.py", + ), ("Secret scan job", "secrets:"), ("Gitleaks version pin", 'GITLEAKS_VERSION: "8.30.1"'), ("Gitleaks archive checksum", "GITLEAKS_LINUX_X64_SHA256:"), diff --git a/release/scripts/rehearse-upgrade.py b/release/scripts/rehearse-upgrade.py index 02671eb7ee..a03f5a0e56 100644 --- a/release/scripts/rehearse-upgrade.py +++ b/release/scripts/rehearse-upgrade.py @@ -346,18 +346,6 @@ def wait_for_casework_audit(postgres: Postgres) -> None: raise RehearsalError("the previous Casework release did not drain its audit outbox") -def upgrade_evidence_audit_configuration(bundle: dict[str, Any], runtime: dict[str, Any]) -> None: - audit = bundle["audit"] - if "hashSecretRef" in audit: - bundle["audit"] = {"hashKeyRef": audit["hashSecretRef"], - "hashKeyVersion": audit["hashKeyVersion"]} - if "auditStorage" in runtime: - storage = runtime.pop("auditStorage") - runtime["audit"] = {"path": storage["path"]} - if "maximumFileBytes" in storage: - runtime["audit"]["rotateBytes"] = storage["maximumFileBytes"] - - def breg_view_differences(before: dict[str, Any], after: dict[str, Any]) -> list[str]: # An ETag binds the active package revision, which changes on rebuild. # Record identifiers, domain data, and stored revisions must still match. @@ -1002,7 +990,8 @@ def rehearse_breg(work: Path, keys: Keys, postgres: Postgres, old: Side, new: Si old.run_json("bregctl", "--format", "json", "apply", "--runtime-config", str(breg.runtime), "--package", str(package), "--initial") ready = f"http://127.0.0.1:{breg.port}/ready" - service = Service(old, "breg", ["--config", str(breg.runtime)], work / "breg-old.log", ready) + service = Service(old, "breg", breg_arguments(breg_reads_runtime_config(old), breg.runtime), + work / "breg-old.log", ready) try: seeded = breg.seed("before") before_views = breg.views(seeded) @@ -1031,7 +1020,7 @@ def rehearse_breg(work: Path, keys: Keys, postgres: Postgres, old: Side, new: Si breg.write_runtime(breg.runtime, "registry", successor, successor_revision, registry["package"]["sequence"], breg.port) new.run_json("bregctl", "--format", "json", "verify", "--runtime-config", str(breg.runtime)) - service = Service(new, "breg", ["--config", str(breg.runtime)], + service = Service(new, "breg", breg_arguments(breg_reads_runtime_config(new), breg.runtime), work / "breg-successor.log", ready) try: successor_views = breg.views(seeded) @@ -1292,6 +1281,93 @@ def rehearse_casework(work: Path, keys: Keys, postgres: Postgres, old: Side, new EVIDENCE_KID = "upgrade-rehearsal-evidence-issuer" EVIDENCE_REQUIREMENT = "urn:example:requirement:record-status:v1" EVIDENCE_PURPOSE = "record-status-check" +EVIDENCE_RUNTIME_API_VERSION = "registry.registrystack.org/evidence-runtime/v1alpha1" +EVIDENCE_RUNTIME_KIND = "EvidenceRuntimeConfig" + + +def migrate_evidence_governance(document: dict[str, Any]) -> dict[str, Any]: + """Rewrite bundle-shaped governance written before the access-token rules + moved under `authentication.oidc`, `audit.hashSecretRef` became + `audit.hashKeyRef`, and `audit.format` and `audit.failClosed` were + removed. A current document is returned unchanged.""" + + migrated = json.loads(json.dumps(document)) + authentication = migrated.get("authentication") + if isinstance(authentication, dict) and "oidc" not in authentication: + rules = {key: value for key, value in authentication.items() if key != "kind"} + audiences = rules.pop("audiences", None) + if audiences is not None: + if len(audiences) != 1: + raise RehearsalError("Evidence governance must name exactly one audience " + "to move it under authentication.oidc") + rules["audience"] = audiences[0] + jwks_uri = rules.pop("jwksUri", None) + if jwks_uri is not None: + rules["jwksSource"] = {"kind": "uri", "uri": jwks_uri} + migrated["authentication"] = {"oidc": rules} + audit = migrated.get("audit") + if isinstance(audit, dict) and ({"hashSecretRef", "format", "failClosed"} & audit.keys()): + migrated["audit"] = {"hashKeyRef": audit.get("hashKeyRef", audit.get("hashSecretRef")), + "hashKeyVersion": audit["hashKeyVersion"]} + return migrated + + +def evidence_bind(host: str, port: int) -> str: + return f"[{host}]:{port}" if ":" in host else f"{host}:{port}" + + +def migrate_evidence_runtime(document: dict[str, Any]) -> dict[str, Any]: + """Rewrite an Evidence runtime file written before the apiVersion/kind + envelope, `package.root`, `host:port` bind addresses, and the `audit` + destination block that replaced `auditStorage`. A current document is + returned unchanged.""" + + if "apiVersion" in document and "auditStorage" not in document: + return json.loads(json.dumps(document)) + migrated: dict[str, Any] = {"apiVersion": EVIDENCE_RUNTIME_API_VERSION, + "kind": EVIDENCE_RUNTIME_KIND} + for key, value in document.items(): + if key == "version": + continue + if key == "auditStorage": + migrated["audit"] = {"path": value["path"]} + if "maximumFileBytes" in value: + migrated["audit"]["rotateBytes"] = value["maximumFileBytes"] + elif key == "bundleDirectory": + migrated["package"] = {"root": value} + elif key in ("listener", "metricsListener") and isinstance(value, dict): + listener = {} + for name, setting in value.items(): + if name == "bindHost": + listener["bind"] = evidence_bind(setting, value["port"]) + elif name != "port": + listener[name] = setting + migrated[key] = listener + else: + migrated[key] = json.loads(json.dumps(value)) + return migrated + + +def breg_arguments(reads_runtime_config: bool, runtime: Path) -> list[str]: + """Name the runtime file the way one side's `breg` binary reads it.""" + + return ["--runtime-config" if reads_runtime_config else "--config", str(runtime)] + + +def breg_reads_runtime_config(side: Side) -> bool: + return "--runtime-config" in side.run("breg", "--help").stdout + + +def evidence_arguments(reads_runtime_config: bool, runtime: Path, subcommand: str) -> list[str]: + """Name the runtime file the way one side's `evidence` binary reads it.""" + + if reads_runtime_config: + return [subcommand, "--runtime-config", str(runtime)] + return ["--runtime", str(runtime), subcommand] + + +def reads_runtime_config(side: Side) -> bool: + return "--runtime-config" in side.run("evidence", "check", "--help").stdout class Evidence: @@ -1315,21 +1391,58 @@ def author(self, side: Side) -> None: str(self.target)) governance_path = self.target / "governance.yaml" governance = load_yaml(governance_path) - governance["authentication"]["issuer"] = self.issuer - governance["authentication"]["jwksUri"] = f"{self.issuer}/oauth2/jwks" + authentication = governance["authentication"] + if "oidc" in authentication: + authentication["oidc"]["issuer"] = self.issuer + authentication["oidc"]["jwksSource"] = {"kind": "uri", + "uri": f"{self.issuer}/oauth2/jwks"} + else: + authentication["issuer"] = self.issuer + authentication["jwksUri"] = f"{self.issuer}/oauth2/jwks" dump_yaml(governance_path, governance) self.governance = governance side.run("evidencectl", "build", "--project", str(self.project), "--target", str(self.target), "--output", str(self.candidate)) runtime = load_yaml(self.candidate / "runtime.yaml") - runtime["bundleDirectory"] = str(self.candidate / "bundle") + if "package" in runtime: + runtime["package"]["root"] = str(self.candidate / "bundle") + runtime["listener"]["bind"] = f"127.0.0.1:{self.port}" + else: + runtime["bundleDirectory"] = str(self.candidate / "bundle") + runtime["listener"]["port"] = self.port runtime["auditStorage" if "auditStorage" in runtime else "audit"]["path"] = str(self.audit / "evidence.jsonl") - runtime["listener"]["port"] = self.port runtime["sourceExtracts"] = {"record-status-extract": {"path": str(self.extract())}} dump_yaml(self.runtime, runtime) # Evidence refuses a deployment input it could rewrite. self.runtime.chmod(0o444) + def upgrade(self, side: Side) -> None: + """Carry the deployment into the configuration grammar this side reads. + + This is the documented upgrade step for a release that renames + Evidence configuration keys: rewrite the target's governance and + runtime, rebuild the candidate with this side's evidencectl, and point + the operative runtime at it. The audit chain, secrets, and keys stay + where they are. + """ + + governance_path = self.target / "governance.yaml" + governance = load_yaml(governance_path) + if not reads_runtime_config(side) or "oidc" in governance["authentication"]: + return + self.governance = migrate_evidence_governance(governance) + dump_yaml(governance_path, self.governance) + target_runtime = self.target / "runtime.yaml" + dump_yaml(target_runtime, migrate_evidence_runtime(load_yaml(target_runtime))) + upgraded = self.work / "candidate-upgraded" + side.run("evidencectl", "build", "--project", str(self.project), "--target", + str(self.target), "--output", str(upgraded)) + runtime = migrate_evidence_runtime(load_yaml(self.runtime)) + runtime["package"]["root"] = str(upgraded / "bundle") + self.runtime.chmod(0o600) + dump_yaml(self.runtime, runtime) + self.runtime.chmod(0o444) + def extract(self) -> Path: """Publish the starter's synthetic extract as a fresh read-only SQLite file.""" @@ -1350,8 +1463,10 @@ def extract(self) -> Path: def request(self) -> tuple[int, Any]: authentication = self.governance["authentication"] + authentication = authentication.get("oidc", authentication) + audience = authentication.get("audience") or authentication["audiences"][0] profile = next(iter(self.governance["authorityProfiles"].values())) - claims = {"iss": self.issuer, "aud": authentication["audiences"][0], + claims = {"iss": self.issuer, "aud": audience, "sub": "upgrade-rehearsal-caller", "client_id": "upgrade-rehearsal-caller", "scope": " ".join(authentication["requiredScopes"]), "registry_actor_kind": "service", @@ -1369,26 +1484,23 @@ def request(self) -> tuple[int, Any]: "values": {"record_reference": "REC-0001"}}}]}) return status, body - def upgrade_audit_configuration(self) -> None: - bundle_path = self.candidate / "bundle" / "evidence.yaml" - bundle = load_yaml(bundle_path) - runtime = load_yaml(self.runtime) - upgrade_evidence_audit_configuration(bundle, runtime) - for path, document in ((bundle_path, bundle), (self.runtime, runtime)): - path.chmod(0o600) - dump_yaml(path, document) - path.chmod(0o444) - def rehearse_evidence(work: Path, keys: Keys, old: Side, new: Side, report: dict[str, Any]) -> None: evidence = Evidence(work, keys) try: evidence.author(old) - runtime = ["--runtime", str(evidence.runtime)] - old.run("evidence", *runtime, "check") + old_reads = reads_runtime_config(old) + + def on_old(subcommand: str) -> list[str]: + return evidence_arguments(old_reads, evidence.runtime, subcommand) + + def on_new(subcommand: str) -> list[str]: + return evidence_arguments(True, evidence.runtime, subcommand) + + old.run("evidence", *on_old("check")) ready = f"http://127.0.0.1:{evidence.port}/ready" - service = Service(old, "evidence", [*runtime, "serve"], work / "evidence-old.log", ready) + service = Service(old, "evidence", on_old("serve"), work / "evidence-old.log", ready) try: status, before = evidence.request() expect_status("previous release evidence request", status, before, 200) @@ -1399,10 +1511,10 @@ def rehearse_evidence(work: Path, keys: Keys, old: Side, new: Side, records_before = audit_record_count(evidence.audit, "evidence.jsonl") archive = work / "audit-archive" archive_audit_files(evidence.audit, "evidence.jsonl", archive) - evidence.upgrade_audit_configuration() - new.run("evidence", *runtime, "check") - service = Service(new, "evidence", [*runtime, "serve"], work / "evidence-new.log", ready) + evidence.upgrade(new) + new.run("evidence", *on_new("check")) + service = Service(new, "evidence", on_new("serve"), work / "evidence-new.log", ready) try: status, after = evidence.request() expect_status("upgraded evidence request", status, after, 200) diff --git a/release/scripts/test_check_debian13_images.py b/release/scripts/test_check_debian13_images.py index 7258d5ff88..6d9bdc5a93 100644 --- a/release/scripts/test_check_debian13_images.py +++ b/release/scripts/test_check_debian13_images.py @@ -358,8 +358,8 @@ def test_adopter_images_keep_libc_root_owned_and_normalize_metadata(self) -> Non ) def test_http_probed_images_bind_fixed_config_and_entrypoint(self) -> None: - # Discovery reads no environment variable, so its configuration binding - # is the command; the others bind it through the environment. + # A service that reads no environment variable binds its configuration + # through the command; one that declares an environment binds it there. wrong = { "environment": "ENV WRONG_CONFIG=/tmp/config.yaml", "command": 'CMD ["--runtime", "/tmp/runtime.yaml"]', @@ -426,7 +426,7 @@ def test_relay_v2_image_binds_the_runtime_configuration(self) -> None: dockerfile = root / "release/docker/Dockerfile.relay" dockerfile.write_text( dockerfile.read_text(encoding="utf-8").replace( - 'CMD ["serve", "--runtime", "/etc/relay/runtime.yaml"]', + 'CMD ["serve", "--runtime-config", "/etc/relay/runtime.yaml"]', 'CMD ["serve"]', ), encoding="utf-8", diff --git a/release/scripts/test_check_gates_inventory.py b/release/scripts/test_check_gates_inventory.py index 28605052dd..16ba296c8e 100644 --- a/release/scripts/test_check_gates_inventory.py +++ b/release/scripts/test_check_gates_inventory.py @@ -1415,6 +1415,16 @@ def test_missing_platform_fuzz_bound_is_reported(self) -> None: ), ) + def test_missing_config_conformance_gate_is_reported(self) -> None: + text = self.workflow.replace( + "run: products/platform/scripts/check-config-conformance.py --check-generated", + "run: products/platform/scripts/check-config-conformance.py", + ) + self.assertIn( + "Runtime configuration conformance gate", + self.module.missing_gates(text), + ) + def test_missing_platform_fuzz_runner_is_reported(self) -> None: text = self.workflow.replace( "run: products/platform/scripts/run-fuzz-smoke.sh", diff --git a/release/scripts/test_registry_release.py b/release/scripts/test_registry_release.py index 562cb598e9..b6aafe15f4 100755 --- a/release/scripts/test_registry_release.py +++ b/release/scripts/test_registry_release.py @@ -1846,6 +1846,7 @@ def test_required_rust_context_aggregates_path_gated_shards(self) -> None: "casework-postgres", "scheduling-contracts", "scheduling-postgres", + "config-conformance", }, set(rust_result["needs"]), ) diff --git a/release/scripts/test_rehearse_upgrade.py b/release/scripts/test_rehearse_upgrade.py index 65ae678dec..b072cfcabf 100644 --- a/release/scripts/test_rehearse_upgrade.py +++ b/release/scripts/test_rehearse_upgrade.py @@ -194,6 +194,72 @@ def test_a_focused_run_needs_only_the_named_products_binaries(self) -> None: MODULE.check_binaries(side, "0.33.0", MODULE.BINARIES) +class EvidenceGrammarTest(unittest.TestCase): + OLD_GOVERNANCE = { + "version": 1, + "authentication": { + "kind": "oidc-access-token", + "issuer": "http://127.0.0.1:9000", + "audiences": ["urn:example:evidence"], + "jwksUri": "http://127.0.0.1:9000/oauth2/jwks", + "requiredScopes": ["evidence"], + }, + "audit": {"format": "keyed-jsonl", "hashSecretRef": "secret:file/audit", + "hashKeyVersion": 1, "failClosed": True}, + } + OLD_RUNTIME = { + "version": 1, + "bundleDirectory": "/srv/candidate/bundle", + "listener": {"bindHost": "127.0.0.1", "port": 8080, "tlsTermination": "x"}, + "metricsListener": {"bindHost": "::1", "port": 9090}, + "secretProviders": {"file": {"root": "/srv/secrets"}}, + "auditStorage": {"path": "/audit/evidence.jsonl", "maximumFileBytes": 1048576}, + } + + def test_an_earlier_governance_moves_to_the_oidc_block_and_the_current_audit_key(self) -> None: + migrated = MODULE.migrate_evidence_governance(self.OLD_GOVERNANCE) + self.assertEqual(migrated["authentication"], {"oidc": { + "issuer": "http://127.0.0.1:9000", + "audience": "urn:example:evidence", + "jwksSource": {"kind": "uri", "uri": "http://127.0.0.1:9000/oauth2/jwks"}, + "requiredScopes": ["evidence"], + }}) + self.assertEqual(migrated["audit"], {"hashKeyRef": "secret:file/audit", + "hashKeyVersion": 1}) + self.assertEqual(self.OLD_GOVERNANCE["authentication"]["kind"], "oidc-access-token") + self.assertEqual(MODULE.migrate_evidence_governance(migrated), migrated) + + def test_an_earlier_governance_with_several_audiences_is_refused(self) -> None: + governance = {"authentication": {**self.OLD_GOVERNANCE["authentication"], + "audiences": ["a", "b"]}} + with self.assertRaisesRegex(Error, "one audience"): + MODULE.migrate_evidence_governance(governance) + + def test_an_earlier_runtime_gains_the_envelope_package_bind_addresses_and_audit_block(self) -> None: + migrated = MODULE.migrate_evidence_runtime(self.OLD_RUNTIME) + self.assertEqual(migrated, { + "apiVersion": "registry.registrystack.org/evidence-runtime/v1alpha1", + "kind": "EvidenceRuntimeConfig", + "package": {"root": "/srv/candidate/bundle"}, + "listener": {"bind": "127.0.0.1:8080", "tlsTermination": "x"}, + "metricsListener": {"bind": "[::1]:9090"}, + "secretProviders": {"file": {"root": "/srv/secrets"}}, + "audit": {"path": "/audit/evidence.jsonl", "rotateBytes": 1048576}, + }) + self.assertEqual(MODULE.migrate_evidence_runtime(migrated), migrated) + + def test_the_runtime_file_is_named_the_way_each_side_reads_it(self) -> None: + runtime = Path("/srv/runtime.yaml") + self.assertEqual(MODULE.evidence_arguments(True, runtime, "check"), + ["check", "--runtime-config", "/srv/runtime.yaml"]) + self.assertEqual(MODULE.evidence_arguments(False, runtime, "check"), + ["--runtime", "/srv/runtime.yaml", "check"]) + self.assertEqual(MODULE.breg_arguments(True, runtime), + ["--runtime-config", "/srv/runtime.yaml"]) + self.assertEqual(MODULE.breg_arguments(False, runtime), + ["--config", "/srv/runtime.yaml"]) + + class StateComparisonTest(unittest.TestCase): def test_a_table_that_lost_rows_or_vanished_is_a_loss(self) -> None: before = {"public.a": 3, "public.b": 2, "public.c": 0} @@ -271,16 +337,6 @@ def test_retired_audit_tables_are_preserved_before_exclusion(self) -> None: with self.assertRaisesRegex(Error, "archive"): MODULE.archive_audit_tables(postgres, "registry", before, Path(temporary) / "bad") - def test_evidence_audit_upgrade_preserves_key_and_removes_legacy_settings(self) -> None: - bundle = {"audit": {"format": "jsonl", "failClosed": True, - "hashSecretRef": "secret:file/audit", "hashKeyVersion": 7}} - runtime = {"auditStorage": {"path": "/audit/evidence.jsonl", "maximumFileBytes": 1048576}} - MODULE.upgrade_evidence_audit_configuration(bundle, runtime) - self.assertEqual(bundle["audit"], {"hashKeyRef": "secret:file/audit", "hashKeyVersion": 7}) - self.assertEqual(runtime, {"audit": {"path": "/audit/evidence.jsonl", "rotateBytes": 1048576}}) - MODULE.upgrade_evidence_audit_configuration(bundle, runtime) - self.assertEqual(bundle["audit"]["hashKeyVersion"], 7) - def test_outbox_drain_waits_and_refuses_a_timeout(self) -> None: postgres = unittest.mock.Mock() postgres.sql.side_effect = ["t", "2", "0"] diff --git a/release/scripts/test_release_workflow_structure.py b/release/scripts/test_release_workflow_structure.py index 63f5927553..8d1afb4d13 100644 --- a/release/scripts/test_release_workflow_structure.py +++ b/release/scripts/test_release_workflow_structure.py @@ -758,7 +758,7 @@ def test_release_embeds_evidencectl_tag_and_publishes_latest_alias(self) -> None assemble, ) - def test_candidate_checks_the_casework_package_manifest_it_emits(self) -> None: + def test_candidate_checks_the_casework_package_sum_file_it_emits(self) -> None: _, document = workflow("release-candidate.yml") assemble = step_run( document, @@ -766,7 +766,7 @@ def test_candidate_checks_the_casework_package_manifest_it_emits(self) -> None: "Assemble public payload and validate version-appropriate install inputs", ) self.assertIn( - 'test -f "${casework_package}/casework.package.json"', + 'test -f "${casework_package}/SHA256SUMS"', assemble, ) self.assertNotIn('test -f "${casework_package}/package.json"', assemble)