From fde49d33a378fd5e1e00c89792d0ad226ee2aca9 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sun, 20 Sep 2026 18:13:32 +0700 Subject: [PATCH] fix(drive): chained and composite joins leave out a referenced document that is not in state A by-id join (the chained document query, and a composite query's by-id sub-query) refused the whole result when a derived id had no document: corrupted state on the server, an invalid proof in the verifier. The rule leaned on `refersTo: permanentDocument` targets never leaving state. A reference is validated when it is written and never again, and `permanentDocument` only keeps the target's owner from deleting it, so the query layer should not assume the target is still there: one removed post would fail every page holding a like of it, for every client. Both assembly functions (`assemble_chained_outer_documents` and the composite `assemble_documents`) now leave such an id out. They are shared by the server and the verifier, so the unproven response, the proof and the verification change together. A duplicated outer document, and one no proven join value references, are still refused. The omission is proven, not trusted: the verifier re-derives the outer query from the PROVEN join values, every derived id is a queried key, and grovedb refuses a proof without the coverage to show a queried key present or absent. #4852 pinned that a prover withholding an existing document is refused inside grovedb, before assembly. Join sources must still declare `permanentDocument`. Co-Authored-By: Claude Fable 5.1 --- .../platform/v0/objective-c/Platform.pbobjc.h | 12 +- .../protos/platform/v0/platform.proto | 12 +- packages/js-evo-sdk/README.md | 4 +- .../document_query/v1/dispatch/chained.rs | 125 ++++++++++++++---- .../src/proof/chained_document.rs | 17 ++- .../v0/tests/chained_query_e2e_tests.rs | 87 +++++++++--- .../v0/tests/composite_query_e2e_tests.rs | 107 ++++++++++----- .../src/query/chained_document_query/mod.rs | 52 ++++---- .../src/query/composite_document_query/mod.rs | 55 ++++---- .../verify_chained_documents_proof/mod.rs | 15 ++- .../verify_chained_documents_proof/v0/mod.rs | 8 +- .../verify_composite_documents_proof/mod.rs | 9 +- .../wasm-sdk/src/queries/chained_document.rs | 13 +- .../src/queries/composite_document.rs | 16 ++- 14 files changed, 356 insertions(+), 176 deletions(-) diff --git a/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h b/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h index f21e484bfb5..be02084d7f7 100644 --- a/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h +++ b/packages/dapi-grpc/clients/platform/v0/objective-c/Platform.pbobjc.h @@ -4938,8 +4938,8 @@ GPB_FINAL @interface GetDocumentsRequest_GetDocumentsRequestV1_SubQuery : GPBMes * caps the rows the lookup returns in total, in walk order, like * an ordinary IN query's limit (at most 100). Lookups already * bounded by their values (a unique index, or an indexOnly - * terminal with every prefix fixed), by-id joins (completeness is - * set equality) and counts take none. + * terminal with every prefix fixed), by-id joins (every derived + * id is fetched) and counts take none. **/ @property(nonatomic, readwrite) uint32_t limit; @@ -5722,8 +5722,9 @@ GPB_FINAL @interface GetDocumentsResponse_GetDocumentsResponseV1_ResultData : GP * semi-join, in inner order (the last inner projection's * join-property value is the pagination cursor; outer * documents are ordered by first appearance of their id - * among the inner projections, deduplicated). Routed when - * the request's `chained` message is present. + * among the inner projections, deduplicated; a join value + * whose document is no longer in state has none). Routed + * when the request's `chained` message is present. **/ @property(nonatomic, readwrite, strong, null_resettable) GetDocumentsResponse_GetDocumentsResponseV1_ChainedDocuments *chained; @@ -5807,7 +5808,8 @@ GPB_FINAL @interface GetDocumentsResponse_GetDocumentsResponseV1_CompositeDocume /** * DOCUMENTS: a by-id join in first-appearance order of the - * derived ids; a lookup or sibling in query order. + * derived ids (one whose document is no longer in state is + * left out); a lookup or sibling in query order. **/ @property(nonatomic, readwrite, strong, null_resettable) GetDocumentsResponse_GetDocumentsResponseV1_Documents *documents; diff --git a/packages/dapi-grpc/protos/platform/v0/platform.proto b/packages/dapi-grpc/protos/platform/v0/platform.proto index b0da5aad0c7..b9c31f865d1 100644 --- a/packages/dapi-grpc/protos/platform/v0/platform.proto +++ b/packages/dapi-grpc/protos/platform/v0/platform.proto @@ -1656,8 +1656,8 @@ message GetDocumentsRequest { // caps the rows the lookup returns in total, in walk order, like // an ordinary IN query's limit (at most 100). Lookups already // bounded by their values (a unique index, or an indexOnly - // terminal with every prefix fixed), by-id joins (completeness is - // set equality) and counts take none. + // terminal with every prefix fixed), by-id joins (every derived + // id is fetched) and counts take none. optional uint32 limit = 5; enum Kind { // The matching documents. @@ -2042,8 +2042,9 @@ message GetDocumentsResponse { // semi-join, in inner order (the last inner projection's // join-property value is the pagination cursor; outer // documents are ordered by first appearance of their id - // among the inner projections, deduplicated). Routed when - // the request's `chained` message is present. + // among the inner projections, deduplicated; a join value + // whose document is no longer in state has none). Routed + // when the request's `chained` message is present. ChainedDocuments chained = 6; // Composite-mode result: the page plus one result per // sub-query, in request order. Routed when the request @@ -2067,7 +2068,8 @@ message GetDocumentsResponse { message SubQueryResult { oneof result { // DOCUMENTS: a by-id join in first-appearance order of the - // derived ids; a lookup or sibling in query order. + // derived ids (one whose document is no longer in state is + // left out); a lookup or sibling in query order. Documents documents = 1; // COUNT: one entry per derived value that has a count tree // (a value with no entry counts zero), keyed by the diff --git a/packages/js-evo-sdk/README.md b/packages/js-evo-sdk/README.md index 58e123c0912..fb5b8895685 100644 --- a/packages/js-evo-sdk/README.md +++ b/packages/js-evo-sdk/README.md @@ -256,7 +256,7 @@ try { ## Chained queries (provable semi-join) -A `refersTo: permanentDocument` declaration also lights up the read side: a **chained query** answers `SELECT * FROM post WHERE $id IN (SELECT postId FROM like WHERE $ownerId = me)` in one verified round trip. The node returns the inner indexOnly page and the referenced documents under ONE merged proof — a single quorum-signed state root by construction — and the SDK re-derives the outer query itself and checks it against the *proven* inner values — the node cannot substitute, omit, or inject joined documents (a missing referenced document fails verification outright, since `permanentDocument` references cannot dangle). +A `refersTo: permanentDocument` declaration also lights up the read side: a **chained query** answers `SELECT * FROM post WHERE $id IN (SELECT postId FROM like WHERE $ownerId = me)` in one verified round trip. The node returns the inner indexOnly page and the referenced documents under ONE merged proof — a single quorum-signed state root by construction — and the SDK re-derives the outer query itself and checks it against the *proven* inner values — the node cannot substitute, omit, or inject joined documents. A referenced document that was removed after the inner document was written is proven absent and simply has no entry in `outerDocuments`, so match the two halves by id, not by position. ```ts // The posts I liked, newest page first by postId. @@ -292,7 +292,7 @@ The inner query must target an indexOnly document type and resolve to an index c A **composite query** answers a page and everything a UI needs to render it in ONE verified round trip: the page documents, plus one to ten sub-queries whose `IN` clause the node derives from the proven page (or from an earlier `documents` sub-query). The request never names the derived values. Four sub-query shapes exist: -- a **by-id join** (`bind.field: '$id'`): the documents a page property refers to (the property must declare `refersTo: permanentDocument` targeting the sub-query's type, so a missing document fails verification); +- a **by-id join** (`bind.field: '$id'`): the documents a page property refers to (the property must declare `refersTo: permanentDocument` targeting the sub-query's type; a referenced document removed since is proven absent and left out); - an **indexed lookup** (`bind.field` an indexed property or `$ownerId`): documents keyed by a page value, in this or any other contract, with a `limit` on the rows it returns in total unless the index already bounds them (a unique index, or an indexOnly terminal with every prefix fixed); - a **count** (`kind: 'counts'`): one count per page value from a `countable` index covering the fixed clauses plus the bound field; - a **sibling** (no `bind`): an independent documents query proven under the same root. diff --git a/packages/rs-drive-abci/src/query/document_query/v1/dispatch/chained.rs b/packages/rs-drive-abci/src/query/document_query/v1/dispatch/chained.rs index d08c20cf4f4..8b37602a306 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/dispatch/chained.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/dispatch/chained.rs @@ -346,6 +346,43 @@ mod tests { } } + /// The chained query a client rebuilds to verify + /// [`chained_request`]'s proof. + fn client_side_chained_query<'a>( + contract: &'a dpp::prelude::DataContract, + version: &PlatformVersion, + ) -> DriveDocumentQuery<'a> { + let inner = DriveDocumentQuery { + contract, + document_type: contract + .document_type_for_name("like") + .expect("like doctype"), + internal_clauses: drive::query::InternalClauses::extract_from_clauses( + vec![drive::query::WhereClause { + field: "$ownerId".to_string(), + operator: drive::query::WhereOperator::Equal, + value: Value::Identifier(OWNER_1), + }], + version, + ) + .expect("clauses extract"), + offset: None, + limit: Some(10), + order_by: Default::default(), + start_at: None, + start_at_included: true, + block_time_ms: None, + resolved_time_ranges: vec![], + sub_queries: vec![], + }; + inner.with_by_id_join( + "postId", + contract + .document_type_for_name("post") + .expect("post doctype"), + ) + } + #[test] fn should_return_both_halves_without_proof() { let (platform, state, version, contract) = setup_yappr_state(); @@ -406,40 +443,74 @@ mod tests { // Client-side composition: rebuild the same chained query and // verify the single merged proof. + let chained = client_side_chained_query(&contract, version); + let (_root_hash, verified) = chained + .verify_chained_documents_proof(proof.grovedb_proof.as_slice(), version) + .expect("chained proof verifies — the proof alone carries everything"); + assert_eq!(verified.outer_documents.len(), 2); + assert_eq!( + verified + .outer_documents + .iter() + .map(|p| p.id().to_buffer()) + .collect::>(), + vec![POST_A, POST_B] + ); + } + + /// A like whose post is not in state (removed after the like was + /// written) does not fail the page on either wire mode: the like + /// stays in the inner half and the post is left out of the outer + /// half, proven absent. + #[test] + fn should_leave_out_a_liked_post_that_is_not_in_state() { + const MISSING_POST: [u8; 32] = [0xC3; 32]; + let (platform, state, version, contract) = setup_yappr_state(); let like_type = contract .document_type_for_name("like") .expect("like doctype"); - let inner = DriveDocumentQuery { - contract: &contract, - document_type: like_type, - internal_clauses: drive::query::InternalClauses::extract_from_clauses( - vec![drive::query::WhereClause { - field: "$ownerId".to_string(), - operator: drive::query::WhereOperator::Equal, - value: Value::Identifier(OWNER_1), - }], + let mut like = like_type.random_document(Some(3), version).expect("like"); + let mut props = std::collections::BTreeMap::new(); + props.insert("hashtag".to_string(), Value::Text("dash".to_string())); + props.insert("postId".to_string(), Value::Identifier(MISSING_POST)); + like.set_properties(props); + like.set_owner_id(Identifier::from(OWNER_1)); + store_document(&platform.platform, &contract, like_type, &like, version); + + let result = platform + .platform + .query_documents_v1( + chained_request(false, contract.id().to_vec()), + &state, version, ) - .expect("clauses extract"), - offset: None, - limit: Some(10), - order_by: Default::default(), - start_at: None, - start_at_included: true, - block_time_ms: None, - resolved_time_ranges: vec![], - sub_queries: vec![], + .expect("query executes"); + assert!(result.errors.is_empty(), "errors: {:?}", result.errors); + let Some(ResponseResult::Data(data)) = result.data.expect("response data").result else { + panic!("expected a data result"); }; - let chained = inner.with_by_id_join( - "postId", - contract - .document_type_for_name("post") - .expect("post doctype"), - ); - let (_root_hash, verified) = chained + let Some(result_data::Variant::Chained(chained)) = data.variant else { + panic!("expected the chained variant"); + }; + assert_eq!(chained.inner_documents.len(), 3); + assert_eq!(chained.outer_documents.len(), 2); + + let result = platform + .platform + .query_documents_v1( + chained_request(true, contract.id().to_vec()), + &state, + version, + ) + .expect("query executes"); + assert!(result.errors.is_empty(), "errors: {:?}", result.errors); + let Some(ResponseResult::Proof(proof)) = result.data.expect("response data").result else { + panic!("expected a proof result"); + }; + let (_root_hash, verified) = client_side_chained_query(&contract, version) .verify_chained_documents_proof(proof.grovedb_proof.as_slice(), version) - .expect("chained proof verifies — the proof alone carries everything"); - assert_eq!(verified.outer_documents.len(), 2); + .expect("the proof verifies with the missing post proven absent"); + assert_eq!(verified.inner_documents.len(), 3); assert_eq!( verified .outer_documents diff --git a/packages/rs-drive-proof-verifier/src/proof/chained_document.rs b/packages/rs-drive-proof-verifier/src/proof/chained_document.rs index bad9bb2412f..a9d6080f426 100644 --- a/packages/rs-drive-proof-verifier/src/proof/chained_document.rs +++ b/packages/rs-drive-proof-verifier/src/proof/chained_document.rs @@ -7,10 +7,12 @@ //! lifts the inner limit into a per-instance branch limit). The //! verifier ([`DriveDocumentQuery::verify_chained_documents_proof`]) //! reconstructs the merged query from the response's UNTRUSTED -//! join-value hint, verifies in one pass, and requires the proven -//! outer documents to match the PROVEN inner join values exactly — a -//! missing referenced document is an invalid proof (`refersTo: -//! permanentDocument` targets cannot dangle) — and this module's +//! join-value hint and verifies in one pass. Every PROVEN inner join +//! value is a queried outer `$id` the proof must show present or +//! absent: one proven absent (the referenced document was removed +//! after the inner document was written) has no outer document, and an +//! outer document no proven join value references is an invalid proof. +//! This module's //! [`FromProof`] impl composes that with the tenderdash signature //! binding of the single root. //! @@ -41,7 +43,10 @@ pub struct ChainedDocuments { /// property carries the pagination cursor. pub inner_documents: Vec, /// The joined outer documents, ordered by first appearance of their - /// id among the inner projections (deduplicated). + /// id among the inner projections (deduplicated). A join value whose + /// document is proven absent (removed after the inner document was + /// written) has no entry here, so this can be shorter than the + /// distinct join values; match the halves by id, not by position. pub outer_documents: Vec, } @@ -50,7 +55,7 @@ pub struct ChainedDocuments { /// /// The merk-level composition (bootstrap subset pass on the inner /// query, merged-query re-derivation, authoritative full verification, -/// exact set equality against the PROVEN join values) lives in rs-drive's +/// assembly against the PROVEN join values) lives in rs-drive's /// [`DriveDocumentQuery::verify_chained_documents_proof`]; this /// wrapper adds the [`verify_tenderdash_proof`] binding — the root hash /// the proof commits to is only an attested fact once it is tied to the diff --git a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/chained_query_e2e_tests.rs b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/chained_query_e2e_tests.rs index 08325d1e16b..e286d30f40e 100644 --- a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/chained_query_e2e_tests.rs +++ b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/chained_query_e2e_tests.rs @@ -344,23 +344,64 @@ fn should_reject_invalid_chained_shapes() { ); } -/// A like whose referenced post is missing is corrupted state at the -/// drive level (consensus validates references on write): the chained -/// execution refuses to return a partial join. +/// A like whose referenced post is not in state (it was removed after +/// the like was written) does not fail the page: the inner half keeps +/// the like, the outer half leaves the post out, and the server and the +/// verifier agree. #[test] -fn should_refuse_a_dangling_reference() { +fn should_leave_out_a_referenced_post_that_is_not_in_state() { let (drive, contract) = setup_likes(); let pv = platform_version(); - // A like referencing POST_A — which was never inserted. - let like = build_like(&contract, "dash", POST_A, OWNER_1, 1); - insert_like(&drive, &contract, &like, true).expect("insert like"); + // Likes of POST_A, POST_B and POST_C; POST_B was never inserted. + insert_post(&drive, &contract, POST_A, "dash", "post a", 10); + insert_post(&drive, &contract, POST_C, "dash", "post c", 12); + for (post, seed) in [(POST_A, 1u64), (POST_B, 2), (POST_C, 3)] { + let like = build_like(&contract, "dash", post, OWNER_1, seed); + insert_like(&drive, &contract, &like, true).expect("insert like"); + } let chained = chained_posts_i_liked(&contract, OWNER_1, None, Some(10)); - let refused = drive.query_chained_documents(&chained, None, None, pv); - assert!( - matches!(refused, Err(Error::Proof(_))), - "a dangling reference must refuse the join, got {refused:?}" + let outcome = drive + .query_chained_documents(&chained, None, None, pv) + .expect("a missing referenced post does not fail the join"); + assert_eq!(outcome.result.inner_documents.len(), 3); + assert_eq!( + outcome + .result + .outer_documents + .iter() + .map(|d| d.id().to_buffer()) + .collect::>(), + vec![POST_A, POST_C], ); + + let (proof, _) = drive + .query_chained_documents_with_proof(&chained, pv) + .expect("chained proof generates"); + let (_root, verified) = chained + .verify_chained_documents_proof(proof.as_slice(), pv) + .expect("the proof verifies with the missing post proven absent"); + assert_eq!(verified.inner_documents.len(), 3); + assert_eq!(verified.outer_documents, outcome.result.outer_documents); + + // No referenced post in state at all: every like, no post. + let (drive, contract) = setup_likes(); + let like = build_like(&contract, "dash", POST_A, OWNER_1, 1); + insert_like(&drive, &contract, &like, true).expect("insert like"); + let chained = chained_posts_i_liked(&contract, OWNER_1, None, Some(10)); + let outcome = drive + .query_chained_documents(&chained, None, None, pv) + .expect("chained query executes"); + assert_eq!(outcome.result.inner_documents.len(), 1); + assert!(outcome.result.outer_documents.is_empty()); + let (proof, _) = drive + .query_chained_documents_with_proof(&chained, pv) + .expect("chained proof generates"); + let (_root, verified) = chained + .verify_chained_documents_proof(proof.as_slice(), pv) + .expect("the proof verifies"); + assert_eq!(verified.inner_documents.len(), 1); + assert!(verified.outer_documents.is_empty()); } /// A proof covering only the inner half — exactly what a node that @@ -450,12 +491,12 @@ fn grove_verify_outer_half( .collect()) } -/// The soundness the "a removed referenced document is an absence, not -/// an invalid proof" relaxation rests on, half one: when a referenced +/// The soundness "a referenced document that is not in state is left +/// out, not an invalid proof" rests on, half one: when a referenced /// post is NOT in state, the honest merged proof still satisfies /// grovedb's verification of the full derived query, so the absence of -/// that `$id` is itself proven. Today only the exact-set assembly -/// refuses the result. +/// that `$id` is itself proven, and the verifier returns the outer half +/// without it. #[test] fn should_prove_the_absence_of_a_missing_referenced_post() { let pv = platform_version(); @@ -489,10 +530,18 @@ fn should_prove_the_absence_of_a_missing_referenced_post() { .collect(); assert_eq!(present, expected, "missing {missing:?}"); - let refused = chained.verify_chained_documents_proof(proof.as_slice(), pv); - assert!( - matches!(refused, Err(Error::Proof(_))), - "the exact-set assembly is what refuses a dangling join today, got {refused:?}" + let (_root, verified) = chained + .verify_chained_documents_proof(proof.as_slice(), pv) + .expect("the verifier leaves the proven-absent posts out"); + assert_eq!(verified.inner_documents.len(), POSTS.len()); + assert_eq!( + verified + .outer_documents + .iter() + .map(|d| d.id().to_buffer()) + .collect::>(), + expected, + "missing {missing:?}" ); } } diff --git a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/composite_query_e2e_tests.rs b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/composite_query_e2e_tests.rs index b7ebdbd6417..902679a4a32 100644 --- a/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/composite_query_e2e_tests.rs +++ b/packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/composite_query_e2e_tests.rs @@ -940,44 +940,85 @@ fn should_refuse_composite_queries_on_plain_surfaces() { ); } -/// A by-id join whose derived id has no document is an invalid proof -/// (and corrupted state on the server): a permanentDocument reference -/// cannot dangle. +/// A by-id join whose derived id has no document (the quoted post was +/// removed after the quoting one was written) does not fail the page: +/// the join leaves it out, and a later binding derives from the quoted +/// posts that ARE there. The server and the verifier agree. #[test] -fn should_refuse_a_dangling_reference() { - let (drive, feed, _dashpay) = setup(); +fn should_leave_out_a_joined_document_that_is_not_in_state() { + let (drive, feed, dashpay) = setup(); + seed_feed(&drive, &feed, &dashpay); + // A fourth `dash` post, by owner 1, quoting a post that is not there. insert_post( &drive, &feed, - POST_A, + [0xF6; 32], OWNER_1, "dash", Some(MISSING_POST), - 1, + 5, ); let pv = platform_version(); - let query = page_by_hashtag(&feed, "dash", Some(10)).with_sub_queries(vec![bound( - &feed, - "post", - SubQueryKind::Documents, - BindingSource::Page, - "quotedPostId", - "$id", - None, - )]); + let query = feed_query(&feed, &dashpay, None); - let refused = drive.query_composite_documents(&query, None, None, pv); - assert!( - matches!(refused, Err(Error::Proof(_))), - "expected the missing-document refusal, got {refused:?}" - ); - let (proof, _) = drive + let materialized = drive + .query_composite_documents(&query, None, None, pv) + .expect("a missing quoted post does not fail the composition") + .result; + let (proof, _page) = drive .query_composite_documents_with_proof(&query, pv) - .expect("the proof itself generates"); - assert!( - query.verify_composite_documents_proof(&proof, pv).is_err(), - "the verifier must refuse a dangling reference" + .expect("composite proves"); + let (_root, verified) = query + .verify_composite_documents_proof(&proof, pv) + .expect("the proof verifies with the missing quoted post proven absent"); + + assert_eq!( + ids(&materialized.page_documents), + vec![POST_A, POST_B, POST_C, [0xF6; 32]] + ); + assert_eq!( + ids(materialized.sub_results[QUOTED_POSTS].documents()), + vec![POST_D], + "D is there, the other quoted post is not" ); + assert_eq!( + owner_ids(materialized.sub_results[QUOTED_AUTHOR_PROFILES].documents()), + vec![OWNER_3], + "derived from the quoted posts that are in state" + ); + assert_eq!(verified.page_documents, materialized.page_documents); + assert_eq!(verified.sub_results, materialized.sub_results); + + // The only quoted post missing: an empty join, and nothing derived + // from it. + let (drive, feed, dashpay) = setup(); + insert_post( + &drive, + &feed, + POST_A, + OWNER_1, + "dash", + Some(MISSING_POST), + 1, + ); + let query = feed_query(&feed, &dashpay, None); + let materialized = drive + .query_composite_documents(&query, None, None, pv) + .expect("composite executes") + .result; + assert!(materialized.sub_results[QUOTED_POSTS] + .documents() + .is_empty()); + assert!(materialized.sub_results[QUOTED_AUTHOR_PROFILES] + .documents() + .is_empty()); + let (proof, _page) = drive + .query_composite_documents_with_proof(&query, pv) + .expect("composite proves"); + let (_root, verified) = query + .verify_composite_documents_proof(&proof, pv) + .expect("the proof verifies"); + assert_eq!(verified.sub_results, materialized.sub_results); } /// When the page is itself a by-ids fetch and a join targets the same @@ -1945,7 +1986,7 @@ fn quoted_posts_join(feed: &DataContract) -> DriveDocumentQuery<'_> { /// The by-id join's counterpart of the chained soundness pair, half /// one: with a quoted post NOT in state, the honest merged proof still /// satisfies grovedb's verification of the full derived query, so the -/// absence is proven and only the assembly refuses the result today. +/// absence is proven, and the verifier returns the join without it. #[test] fn should_prove_the_absence_of_a_missing_joined_document() { let pv = platform_version(); @@ -1991,10 +2032,14 @@ fn should_prove_the_absence_of_a_missing_joined_document() { .collect(); assert_eq!(present, expected, "missing {missing:?}"); - let refused = query.verify_composite_documents_proof(&proof, pv); - assert!( - matches!(refused, Err(Error::Proof(_))), - "the assembly is what refuses a dangling join today, got {refused:?}" + let (_root, verified) = query + .verify_composite_documents_proof(&proof, pv) + .expect("the verifier leaves the proven-absent quoted posts out"); + assert_eq!(verified.page_documents.len(), QUOTING.len()); + assert_eq!( + ids(verified.sub_results[0].documents()), + expected, + "missing {missing:?}" ); } } diff --git a/packages/rs-drive/src/query/chained_document_query/mod.rs b/packages/rs-drive/src/query/chained_document_query/mod.rs index 3c1c69d2321..4c2586f0e9e 100644 --- a/packages/rs-drive/src/query/chained_document_query/mod.rs +++ b/packages/rs-drive/src/query/chained_document_query/mod.rs @@ -29,10 +29,15 @@ //! ([`DriveDocumentQuery::chained_join_values`] → //! [`DriveDocumentQuery::derive_chained_outer_query`], the same functions //! the server executes), so a server cannot substitute, omit, or inject -//! outer documents. Because the join property's `refersTo` targets a -//! `permanentDocument` type (non-deletable, enforced at write time), -//! every proven join value MUST resolve to a document — a missing outer -//! document is an invalid proof, not an absence. +//! outer documents. A join value may have NO outer document: a +//! `refersTo` target is validated when the inner document is written +//! and never again, and `permanentDocument` only keeps the target's +//! owner from deleting it, so this layer does not assume the target is +//! still in state. Such a join value is left out of the outer half, and +//! the omission is proven, not trusted: the merged query carries every +//! derived `$id`, and grovedb refuses a proof that lacks the coverage to +//! show a queried key present or absent, so a prover cannot pass an +//! existing document off as removed. //! //! Guardrails (v1): the inner query must resolve to an indexOnly index //! that carries the join property (as terminal or prefix property, so @@ -193,11 +198,10 @@ impl<'a> DriveDocumentQuery<'a> { } // The join property must be a same-contract permanentDocument - // reference targeting the outer type. `refersTo` writes are - // existence-validated and permanentDocument targets can never be - // deleted, so every proven join value MUST resolve — which is - // what lets the verifier treat a missing outer document as an - // invalid proof instead of needing absence proofs. + // reference targeting the outer type: the declaration is what + // names the outer type, and `refersTo` writes are + // existence-validated, so a join value with no outer document + // is a target removed since (see the module docs). let Some(join_document_property) = self.document_type.flattened_properties().get(join_property) else { @@ -237,8 +241,8 @@ impl<'a> DriveDocumentQuery<'a> { _ => { return Err(unsupported(format!( "chained query join property \"{}\" must carry a `refersTo: \ - permanentDocument` declaration: only a permanent-document reference \ - guarantees every proven join value resolves to an outer document", + permanentDocument` declaration: it is what names the outer document \ + type the proven join values are fetched from", join_property, ))); } @@ -305,8 +309,8 @@ impl<'a> DriveDocumentQuery<'a> { /// The derived outer query: a pure by-ids fetch of the join values /// from the outer type's primary storage. No clauses, no limit, no - /// cursor — completeness is set-equality against `join_values`, - /// checked by the verifier. + /// cursor — completeness is grovedb's: every join value is a queried + /// key, proven present or absent. pub fn derive_chained_outer_query( &self, join_values: &[Identifier], @@ -348,10 +352,12 @@ impl<'a> DriveDocumentQuery<'a> { } /// Reorders the outer documents (returned in key order by the by-ids - /// query) into first-appearance join order, and enforces EXACT set - /// equality between the proven outer ids and the derived join - /// values — both directions. Shared by the server (where a mismatch - /// is corrupted state: permanentDocument references cannot dangle) + /// query) into first-appearance join order. A join value with no + /// outer document is left out: the referenced document was removed + /// after the inner document was written, and the by-ids query that + /// found nothing under its `$id` is the very query the proof covers. + /// An outer document carried twice, or one no join value references, + /// is refused. Shared by the server (where that is corrupted state) /// and the verifier (where it is an invalid proof). pub fn assemble_chained_outer_documents( &self, @@ -373,15 +379,9 @@ impl<'a> DriveDocumentQuery<'a> { } let mut ordered = Vec::with_capacity(join_values.len()); for join_value in join_values { - let document = by_id.remove(join_value).ok_or_else(|| { - Error::Proof(crate::error::proof::ProofError::CorruptedProof(format!( - "chained outer results are missing referenced document {}: a \ - permanentDocument reference cannot dangle, so the outer half does \ - not prove the derived query", - join_value - ))) - })?; - ordered.push(document); + if let Some(document) = by_id.remove(join_value) { + ordered.push(document); + } } if let Some((extra_id, _)) = by_id.into_iter().next() { return Err(Error::Proof( diff --git a/packages/rs-drive/src/query/composite_document_query/mod.rs b/packages/rs-drive/src/query/composite_document_query/mod.rs index 6f910485752..b9f26c35c0e 100644 --- a/packages/rs-drive/src/query/composite_document_query/mod.rs +++ b/packages/rs-drive/src/query/composite_document_query/mod.rs @@ -28,10 +28,8 @@ //! every sub-query itself with the SAME builders the server ran, merges //! the same way, and verifies the whole composition in one authoritative //! pass; then it recomputes the derived values from the proven page and -//! refuses any divergence from the bootstrap, any result outside a -//! derived value set, and (for by-id joins on `refersTo: -//! permanentDocument` properties, which cannot dangle) any missing -//! referenced document. A node that ignores the sub-queries serves a +//! refuses any divergence from the bootstrap and any result outside a +//! derived value set. A node that ignores the sub-queries serves a //! page-only proof, which cannot satisfy the merged query whenever a //! sub-query derived anything — the composition fails closed. //! @@ -39,9 +37,12 @@ //! //! - **Documents by id** (`bind.field == "$id"`): the classic join. The //! source property must declare `refersTo: permanentDocument` targeting -//! the sub-query's type, so every derived id MUST resolve — the result -//! is the referenced documents in first-appearance order, set-equal to -//! the derived ids. +//! the sub-query's type — the result is the referenced documents in +//! first-appearance order of the derived ids. A derived id with no +//! document is left out: a reference is validated when it is written +//! and never again, so the target may have been removed since, and +//! the absence is proven (every derived id is a queried key grovedb +//! must show present or absent). //! - **Documents by an indexed property** (`bind.field` is `$ownerId` or //! an indexed property): a lookup, `WHERE AND //! IN `, with an explicit limit unless the values @@ -161,7 +162,7 @@ pub struct DriveSubQuery<'a> { /// Documents or counts. pub kind: SubQueryKind, /// The fixed clauses (everything but the derived `IN`), typed. - /// Must be empty for a by-id join, which resolves every derived id. + /// Must be empty for a by-id join, which fetches every derived id. pub where_clauses: Vec, /// Ordering; documents only. Every component of the merged proof /// walks in the page's direction, so a documents sub-query must agree @@ -570,8 +571,8 @@ impl<'a> DriveDocumentQuery<'a> { } if sub_query.limit.is_some() { return Err(label( - "a by-id join takes no limit: every derived id must resolve, so \ - completeness is set equality, not a page", + "a by-id join takes no limit: every derived id is fetched, so \ + completeness is per id, not a page", )); } if !sub_query.order_by.is_empty() { @@ -580,9 +581,9 @@ impl<'a> DriveDocumentQuery<'a> { first appearance", )); } - // Only a permanentDocument reference guarantees every - // derived id resolves, which is what lets a missing - // document be an invalid proof instead of an absence. + // The permanentDocument declaration is what names the + // type the derived ids are fetched from, and makes each + // one an id that resolved when it was written. match source_property_type { Some(DocumentPropertyType::IdentifierWithReference( DocumentPropertyReferenceTarget::PermanentDocument { @@ -607,8 +608,8 @@ impl<'a> DriveDocumentQuery<'a> { _ => { return Err(label(&format!( "a by-id join needs a source property declaring `refersTo: \ - permanentDocument` (\"{}\" does not): only a permanent-document \ - reference guarantees every derived id resolves", + permanentDocument` (\"{}\" does not): it is what names the \ + document type the derived ids are fetched from", binding.source_property, ))); } @@ -994,8 +995,7 @@ impl<'a> DriveDocumentQuery<'a> { if sub_query.is_by_id_join() { if !sub_query.where_clauses.is_empty() { return Err(unsupported( - "a by-id join takes no fixed clauses: every derived id must resolve" - .to_string(), + "a by-id join takes no fixed clauses: every derived id is fetched".to_string(), )); } return Ok(DriveDocumentQuery { @@ -1416,9 +1416,12 @@ impl<'a> DriveDocumentQuery<'a> { /// Assembles one documents sub-query's result from its decoded /// documents, keeping only the ones its derived values admit and, for - /// a by-id join, enforcing exact set equality in first-appearance - /// order. Shared by the server (where a violation is corrupted state) - /// and the verifier (where it is an invalid proof). + /// a by-id join, putting them in the derived ids' first-appearance + /// order. A derived id with no document is left out: the referenced + /// document was removed after the referring one was written, and the + /// fetch that found nothing under it is the query the proof covers. + /// Shared by the server (where a violation is corrupted state) and + /// the verifier (where it is an invalid proof). fn assemble_documents( &self, sub_query: &DriveSubQuery<'a>, @@ -1446,15 +1449,9 @@ impl<'a> DriveDocumentQuery<'a> { } let mut ordered = Vec::with_capacity(values.len()); for value in values { - let document = by_id.remove(value).ok_or_else(|| { - corrupted_proof(format!( - "composite join results are missing referenced document {}: a \ - permanentDocument reference cannot dangle, so the proof does not \ - cover the derived query", - value - )) - })?; - ordered.push(document.clone()); + if let Some(document) = by_id.remove(value) { + ordered.push(document.clone()); + } } return Ok(ordered); } diff --git a/packages/rs-drive/src/verify/chained_document/verify_chained_documents_proof/mod.rs b/packages/rs-drive/src/verify/chained_document/verify_chained_documents_proof/mod.rs index a1f28a9c7e6..9da6a6e4c57 100644 --- a/packages/rs-drive/src/verify/chained_document/verify_chained_documents_proof/mod.rs +++ b/packages/rs-drive/src/verify/chained_document/verify_chained_documents_proof/mod.rs @@ -20,13 +20,14 @@ impl DriveDocumentQuery<'_> { /// it from its materialization), the merged query is rebuilt, and /// the AUTHORITATIVE full pass verifies the whole composition — /// grovedb enforces the inner page's lifted per-instance limit and - /// range completeness — with the proven outer documents required - /// to match the proven inner join values exactly. A missing - /// referenced document is an invalid proof (`refersTo: - /// permanentDocument` targets cannot dangle), and so is an extra - /// one; a proof covering only the inner half (an old node serving - /// the plain query) fails the full pass whenever the inner page is - /// non-empty. + /// range completeness, and every derived outer `$id` is a queried + /// key it must show present or absent. A join value whose outer + /// document is proven absent (the referenced document was removed + /// after the inner document was written) is left out of the outer + /// half; an outer document no proven join value references is an + /// invalid proof, and a proof covering only the inner half (an old + /// node serving the plain query) fails the full pass whenever the + /// inner page is non-empty. /// /// One proof means one root by construction; the caller combines /// the returned root hash with the surrounding tenderdash diff --git a/packages/rs-drive/src/verify/chained_document/verify_chained_documents_proof/v0/mod.rs b/packages/rs-drive/src/verify/chained_document/verify_chained_documents_proof/v0/mod.rs index a99ecb7f67e..a0433689dff 100644 --- a/packages/rs-drive/src/verify/chained_document/verify_chained_documents_proof/v0/mod.rs +++ b/packages/rs-drive/src/verify/chained_document/verify_chained_documents_proof/v0/mod.rs @@ -122,9 +122,11 @@ impl DriveDocumentQuery<'_> { } // The full pass's PROVEN join values are authoritative — the - // bootstrap candidates were only for reconstructing the query — - // and the exact-set assembly refuses any divergence between - // them and the proven outer documents, in either direction. + // bootstrap candidates were only for reconstructing the query. + // A join value the full pass proved no document under is left + // out by the assembly (grovedb already refused a proof without + // the coverage to show it absent); an outer document no proven + // join value references is refused. let join_values = self.chained_join_values(&inner_documents)?; let outer_documents = self.assemble_chained_outer_documents(&join_values, outer_documents)?; diff --git a/packages/rs-drive/src/verify/composite_document/verify_composite_documents_proof/mod.rs b/packages/rs-drive/src/verify/composite_document/verify_composite_documents_proof/mod.rs index 8fbd96e6baa..3747e33764d 100644 --- a/packages/rs-drive/src/verify/composite_document/verify_composite_documents_proof/mod.rs +++ b/packages/rs-drive/src/verify/composite_document/verify_composite_documents_proof/mod.rs @@ -21,10 +21,11 @@ impl DriveDocumentQuery<'_> { /// enforces every component's per-instance limit and range /// completeness. The proven results are then routed back to their /// components: an entry no derivation asked for is an invalid proof, - /// so is a by-id join missing a referenced document (a - /// `permanentDocument` reference cannot dangle), and so is any - /// divergence between the values the proven page derives and the - /// candidates the query was built from. A proof covering only the + /// and so is any divergence between the values the proven page + /// derives and the candidates the query was built from. A by-id + /// join's derived id that is proven absent (the referenced document + /// was removed after the referring one was written) is left out of + /// that join's result. A proof covering only the /// page (an old node serving the plain query) fails the full pass /// whenever a sub-query derived anything. /// diff --git a/packages/wasm-sdk/src/queries/chained_document.rs b/packages/wasm-sdk/src/queries/chained_document.rs index d9007796b93..0ccc06ba813 100644 --- a/packages/wasm-sdk/src/queries/chained_document.rs +++ b/packages/wasm-sdk/src/queries/chained_document.rs @@ -78,7 +78,10 @@ interface ChainedDocumentsResult { innerDocuments: Document[]; /** * The joined outer documents, ordered by first appearance of their - * id among the inner projections (deduplicated). + * id among the inner projections (deduplicated). A referenced + * document that was removed after the inner document was written is + * proven absent and has no entry here: match the halves by id, not by + * position. */ outerDocuments: Document[]; } @@ -174,10 +177,10 @@ impl WasmSdk { /// both verified halves. /// /// The composition is always proof-verified: one merged grovedb - /// proof commits to one quorum-signed root, and the proven outer - /// documents must match the proven inner join values exactly (a - /// missing referenced document is a verification error, not an - /// absence). + /// proof commits to one quorum-signed root, and every proven inner + /// join value is an outer `$id` the proof must show present or + /// absent (a referenced document removed since is proven absent and + /// left out; the node cannot pass an existing one off as removed). #[wasm_bindgen( js_name = "getChainedDocuments", unchecked_return_type = "ChainedDocumentsResult" diff --git a/packages/wasm-sdk/src/queries/composite_document.rs b/packages/wasm-sdk/src/queries/composite_document.rs index 95621d456e7..95d5eecb30f 100644 --- a/packages/wasm-sdk/src/queries/composite_document.rs +++ b/packages/wasm-sdk/src/queries/composite_document.rs @@ -62,9 +62,10 @@ export interface CompositeBind { /** * The sub-query field receiving the `IN` clause. `$id` makes this a * by-id JOIN (the source property must declare `refersTo: - * permanentDocument` targeting the sub-query's document type, so a - * missing document is a verification error); otherwise `$ownerId` or an - * indexed property (a LOOKUP, where absence is a proven fact). + * permanentDocument` targeting the sub-query's document type; a + * referenced document removed since is proven absent and left out); + * otherwise `$ownerId` or an indexed property (a LOOKUP, where absence + * is a proven fact too). */ field: string; } @@ -119,7 +120,8 @@ export interface CompositeDocumentsQuery { /** * A verified `documents` sub-result: a by-id join in first-appearance - * order of the derived ids among the source documents; a lookup or + * order of the derived ids among the source documents (a derived id + * whose document is no longer in state is left out); a lookup or * sibling in query order. */ export interface CompositeDocumentsSubResult { @@ -423,9 +425,9 @@ impl WasmSdk { /// /// The composition is always proof-verified: one merged grovedb /// proof commits to one quorum-signed root, every sub-query is - /// re-derived from the proven page, and a by-id join whose - /// referenced document is missing is a verification error, not an - /// absence. + /// re-derived from the proven page, and a by-id join's referenced + /// document that was removed since is proven absent and left out + /// (the node cannot pass an existing one off as removed). #[wasm_bindgen( js_name = "getCompositeDocuments", unchecked_return_type = "CompositeDocumentsResult"