fix(drive): chained and composite joins leave out a referenced document that is not in state

packages/dapi-grpc/protos/platform/v0/platform.proto

@@ -1656,8 +1656,8 @@ message GetDocumentsRequest {
       // caps the rows the lookup returns in total, in walk order, like
       // an ordinary IN query's limit (at most 100). Lookups already
       // bounded by their values (a unique index, or an indexOnly
-      // terminal with every prefix fixed), by-id joins (completeness is
-      // set equality) and counts take none.
+      // terminal with every prefix fixed), by-id joins (every derived
+      // id is fetched) and counts take none.
       optional uint32 limit = 5;
       enum Kind {
         // The matching documents.
@@ -2042,8 +2042,9 @@ message GetDocumentsResponse {
         // semi-join, in inner order (the last inner projection's
         // join-property value is the pagination cursor; outer
         // documents are ordered by first appearance of their id
-        // among the inner projections, deduplicated). Routed when
-        // the request's `chained` message is present.
+        // among the inner projections, deduplicated; a join value
+        // whose document is no longer in state has none). Routed
+        // when the request's `chained` message is present.
         ChainedDocuments chained = 6;
         // Composite-mode result: the page plus one result per
         // sub-query, in request order. Routed when the request
@@ -2067,7 +2068,8 @@ message GetDocumentsResponse {
       message SubQueryResult {
         oneof result {
           // DOCUMENTS: a by-id join in first-appearance order of the
-          // derived ids; a lookup or sibling in query order.
+          // derived ids (one whose document is no longer in state is
+          // left out); a lookup or sibling in query order.
           Documents documents = 1;
           // COUNT: one entry per derived value that has a count tree
           // (a value with no entry counts zero), keyed by the

packages/js-evo-sdk/README.md

@@ -256,7 +256,7 @@ try {
 
 ## Chained queries (provable semi-join)
 
-A `refersTo: permanentDocument` declaration also lights up the read side: a **chained query** answers `SELECT * FROM post WHERE $id IN (SELECT postId FROM like WHERE $ownerId = me)` in one verified round trip. The node returns the inner indexOnly page and the referenced documents under ONE merged proof — a single quorum-signed state root by construction — and the SDK re-derives the outer query itself and checks it against the *proven* inner values — the node cannot substitute, omit, or inject joined documents (a missing referenced document fails verification outright, since `permanentDocument` references cannot dangle).
+A `refersTo: permanentDocument` declaration also lights up the read side: a **chained query** answers `SELECT * FROM post WHERE $id IN (SELECT postId FROM like WHERE $ownerId = me)` in one verified round trip. The node returns the inner indexOnly page and the referenced documents under ONE merged proof — a single quorum-signed state root by construction — and the SDK re-derives the outer query itself and checks it against the *proven* inner values — the node cannot substitute, omit, or inject joined documents. A referenced document that was removed after the inner document was written is proven absent and simply has no entry in `outerDocuments`, so match the two halves by id, not by position.
 
 ```ts
 // The posts I liked, newest page first by postId.
@@ -292,7 +292,7 @@ The inner query must target an indexOnly document type and resolve to an index c
 
 A **composite query** answers a page and everything a UI needs to render it in ONE verified round trip: the page documents, plus one to ten sub-queries whose `IN` clause the node derives from the proven page (or from an earlier `documents` sub-query). The request never names the derived values. Four sub-query shapes exist:
 
-- a **by-id join** (`bind.field: '$id'`): the documents a page property refers to (the property must declare `refersTo: permanentDocument` targeting the sub-query's type, so a missing document fails verification);
+- a **by-id join** (`bind.field: '$id'`): the documents a page property refers to (the property must declare `refersTo: permanentDocument` targeting the sub-query's type; a referenced document removed since is proven absent and left out);
 - an **indexed lookup** (`bind.field` an indexed property or `$ownerId`): documents keyed by a page value, in this or any other contract, with a `limit` on the rows it returns in total unless the index already bounds them (a unique index, or an indexOnly terminal with every prefix fixed);
 - a **count** (`kind: 'counts'`): one count per page value from a `countable` index covering the fixed clauses plus the bound field;
 - a **sibling** (no `bind`): an independent documents query proven under the same root.

packages/rs-drive-abci/src/query/document_query/v1/dispatch/chained.rs

@@ -346,6 +346,43 @@ mod tests {
         }
     }
 
+    /// The chained query a client rebuilds to verify
+    /// [`chained_request`]'s proof.
+    fn client_side_chained_query<'a>(
+        contract: &'a dpp::prelude::DataContract,
+        version: &PlatformVersion,
+    ) -> DriveDocumentQuery<'a> {
+        let inner = DriveDocumentQuery {
+            contract,
+            document_type: contract
+                .document_type_for_name("like")
+                .expect("like doctype"),
+            internal_clauses: drive::query::InternalClauses::extract_from_clauses(
+                vec![drive::query::WhereClause {
+                    field: "$ownerId".to_string(),
+                    operator: drive::query::WhereOperator::Equal,
+                    value: Value::Identifier(OWNER_1),
+                }],
+                version,
+            )
+            .expect("clauses extract"),
+            offset: None,
+            limit: Some(10),
+            order_by: Default::default(),
+            start_at: None,
+            start_at_included: true,
+            block_time_ms: None,
+            resolved_time_ranges: vec![],
+            sub_queries: vec![],
+        };
+        inner.with_by_id_join(
+            "postId",
+            contract
+                .document_type_for_name("post")
+                .expect("post doctype"),
+        )
+    }
+
     #[test]
     fn should_return_both_halves_without_proof() {
         let (platform, state, version, contract) = setup_yappr_state();
@@ -406,40 +443,74 @@ mod tests {
 
         // Client-side composition: rebuild the same chained query and
         // verify the single merged proof.
+        let chained = client_side_chained_query(&contract, version);
+        let (_root_hash, verified) = chained
+            .verify_chained_documents_proof(proof.grovedb_proof.as_slice(), version)
+            .expect("chained proof verifies — the proof alone carries everything");
+        assert_eq!(verified.outer_documents.len(), 2);
+        assert_eq!(
+            verified
+                .outer_documents
+                .iter()
+                .map(|p| p.id().to_buffer())
+                .collect::<Vec<_>>(),
+            vec![POST_A, POST_B]
+        );
+    }
+
+    /// A like whose post is not in state (removed after the like was
+    /// written) does not fail the page on either wire mode: the like
+    /// stays in the inner half and the post is left out of the outer
+    /// half, proven absent.
+    #[test]
+    fn should_leave_out_a_liked_post_that_is_not_in_state() {
+        const MISSING_POST: [u8; 32] = [0xC3; 32];
+        let (platform, state, version, contract) = setup_yappr_state();
         let like_type = contract
             .document_type_for_name("like")
             .expect("like doctype");
-        let inner = DriveDocumentQuery {
-            contract: &contract,
-            document_type: like_type,
-            internal_clauses: drive::query::InternalClauses::extract_from_clauses(
-                vec![drive::query::WhereClause {
-                    field: "$ownerId".to_string(),
-                    operator: drive::query::WhereOperator::Equal,
-                    value: Value::Identifier(OWNER_1),
-                }],
+        let mut like = like_type.random_document(Some(3), version).expect("like");
+        let mut props = std::collections::BTreeMap::new();
+        props.insert("hashtag".to_string(), Value::Text("dash".to_string()));
+        props.insert("postId".to_string(), Value::Identifier(MISSING_POST));
+        like.set_properties(props);
+        like.set_owner_id(Identifier::from(OWNER_1));
+        store_document(&platform.platform, &contract, like_type, &like, version);
+
+        let result = platform
+            .platform
+            .query_documents_v1(
+                chained_request(false, contract.id().to_vec()),
+                &state,
                 version,
             )
-            .expect("clauses extract"),
-            offset: None,
-            limit: Some(10),
-            order_by: Default::default(),
-            start_at: None,
-            start_at_included: true,
-            block_time_ms: None,
-            resolved_time_ranges: vec![],
-            sub_queries: vec![],
+            .expect("query executes");
+        assert!(result.errors.is_empty(), "errors: {:?}", result.errors);
+        let Some(ResponseResult::Data(data)) = result.data.expect("response data").result else {
+            panic!("expected a data result");
         };
-        let chained = inner.with_by_id_join(
-            "postId",
-            contract
-                .document_type_for_name("post")
-                .expect("post doctype"),
-        );
-        let (_root_hash, verified) = chained
+        let Some(result_data::Variant::Chained(chained)) = data.variant else {
+            panic!("expected the chained variant");
+        };
+        assert_eq!(chained.inner_documents.len(), 3);
+        assert_eq!(chained.outer_documents.len(), 2);
+
+        let result = platform
+            .platform
+            .query_documents_v1(
+                chained_request(true, contract.id().to_vec()),
+                &state,
+                version,
+            )
+            .expect("query executes");
+        assert!(result.errors.is_empty(), "errors: {:?}", result.errors);
+        let Some(ResponseResult::Proof(proof)) = result.data.expect("response data").result else {
+            panic!("expected a proof result");
+        };
+        let (_root_hash, verified) = client_side_chained_query(&contract, version)
             .verify_chained_documents_proof(proof.grovedb_proof.as_slice(), version)
-            .expect("chained proof verifies — the proof alone carries everything");
-        assert_eq!(verified.outer_documents.len(), 2);
+            .expect("the proof verifies with the missing post proven absent");
+        assert_eq!(verified.inner_documents.len(), 3);
         assert_eq!(
             verified
                 .outer_documents

packages/rs-drive-proof-verifier/src/proof/chained_document.rs

@@ -7,10 +7,12 @@
 //! lifts the inner limit into a per-instance branch limit). The
 //! verifier ([`DriveDocumentQuery::verify_chained_documents_proof`])
 //! reconstructs the merged query from the response's UNTRUSTED
-//! join-value hint, verifies in one pass, and requires the proven
-//! outer documents to match the PROVEN inner join values exactly — a
-//! missing referenced document is an invalid proof (`refersTo:
-//! permanentDocument` targets cannot dangle) — and this module's
+//! join-value hint and verifies in one pass. Every PROVEN inner join
+//! value is a queried outer `$id` the proof must show present or
+//! absent: one proven absent (the referenced document was removed
+//! after the inner document was written) has no outer document, and an
+//! outer document no proven join value references is an invalid proof.
+//! This module's
 //! [`FromProof`] impl composes that with the tenderdash signature
 //! binding of the single root.
 //!
@@ -41,7 +43,10 @@ pub struct ChainedDocuments {
     /// property carries the pagination cursor.
     pub inner_documents: Vec<Document>,
     /// The joined outer documents, ordered by first appearance of their
-    /// id among the inner projections (deduplicated).
+    /// id among the inner projections (deduplicated). A join value whose
+    /// document is proven absent (removed after the inner document was
+    /// written) has no entry here, so this can be shorter than the
+    /// distinct join values; match the halves by id, not by position.
     pub outer_documents: Vec<Document>,
 }
 
@@ -50,7 +55,7 @@ pub struct ChainedDocuments {
 ///
 /// The merk-level composition (bootstrap subset pass on the inner
 /// query, merged-query re-derivation, authoritative full verification,
-/// exact set equality against the PROVEN join values) lives in rs-drive's
+/// assembly against the PROVEN join values) lives in rs-drive's
 /// [`DriveDocumentQuery::verify_chained_documents_proof`]; this
 /// wrapper adds the [`verify_tenderdash_proof`] binding — the root hash
 /// the proof commits to is only an attested fact once it is tied to the

packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/chained_query_e2e_tests.rs

@@ -344,23 +344,64 @@ fn should_reject_invalid_chained_shapes() {
     );
 }
 
-/// A like whose referenced post is missing is corrupted state at the
-/// drive level (consensus validates references on write): the chained
-/// execution refuses to return a partial join.
+/// A like whose referenced post is not in state (it was removed after
+/// the like was written) does not fail the page: the inner half keeps
+/// the like, the outer half leaves the post out, and the server and the
+/// verifier agree.
 #[test]
-fn should_refuse_a_dangling_reference() {
+fn should_leave_out_a_referenced_post_that_is_not_in_state() {
     let (drive, contract) = setup_likes();
     let pv = platform_version();
-    // A like referencing POST_A — which was never inserted.
-    let like = build_like(&contract, "dash", POST_A, OWNER_1, 1);
-    insert_like(&drive, &contract, &like, true).expect("insert like");
+    // Likes of POST_A, POST_B and POST_C; POST_B was never inserted.
+    insert_post(&drive, &contract, POST_A, "dash", "post a", 10);
+    insert_post(&drive, &contract, POST_C, "dash", "post c", 12);
+    for (post, seed) in [(POST_A, 1u64), (POST_B, 2), (POST_C, 3)] {
+        let like = build_like(&contract, "dash", post, OWNER_1, seed);
+        insert_like(&drive, &contract, &like, true).expect("insert like");
+    }
 
     let chained = chained_posts_i_liked(&contract, OWNER_1, None, Some(10));
-    let refused = drive.query_chained_documents(&chained, None, None, pv);
-    assert!(
-        matches!(refused, Err(Error::Proof(_))),
-        "a dangling reference must refuse the join, got {refused:?}"
+    let outcome = drive
+        .query_chained_documents(&chained, None, None, pv)
+        .expect("a missing referenced post does not fail the join");
+    assert_eq!(outcome.result.inner_documents.len(), 3);
+    assert_eq!(
+        outcome
+            .result
+            .outer_documents
+            .iter()
+            .map(|d| d.id().to_buffer())
+            .collect::<Vec<_>>(),
+        vec![POST_A, POST_C],
     );
+
+    let (proof, _) = drive
+        .query_chained_documents_with_proof(&chained, pv)
+        .expect("chained proof generates");
+    let (_root, verified) = chained
+        .verify_chained_documents_proof(proof.as_slice(), pv)
+        .expect("the proof verifies with the missing post proven absent");
+    assert_eq!(verified.inner_documents.len(), 3);
+    assert_eq!(verified.outer_documents, outcome.result.outer_documents);
+
+    // No referenced post in state at all: every like, no post.
+    let (drive, contract) = setup_likes();
+    let like = build_like(&contract, "dash", POST_A, OWNER_1, 1);
+    insert_like(&drive, &contract, &like, true).expect("insert like");
+    let chained = chained_posts_i_liked(&contract, OWNER_1, None, Some(10));
+    let outcome = drive
+        .query_chained_documents(&chained, None, None, pv)
+        .expect("chained query executes");
+    assert_eq!(outcome.result.inner_documents.len(), 1);
+    assert!(outcome.result.outer_documents.is_empty());
+    let (proof, _) = drive
+        .query_chained_documents_with_proof(&chained, pv)
+        .expect("chained proof generates");
+    let (_root, verified) = chained
+        .verify_chained_documents_proof(proof.as_slice(), pv)
+        .expect("the proof verifies");
+    assert_eq!(verified.inner_documents.len(), 1);
+    assert!(verified.outer_documents.is_empty());
 }
 
 /// A proof covering only the inner half — exactly what a node that
@@ -450,12 +491,12 @@ fn grove_verify_outer_half(
         .collect())
 }
 
-/// The soundness the "a removed referenced document is an absence, not
-/// an invalid proof" relaxation rests on, half one: when a referenced
+/// The soundness "a referenced document that is not in state is left
+/// out, not an invalid proof" rests on, half one: when a referenced
 /// post is NOT in state, the honest merged proof still satisfies
 /// grovedb's verification of the full derived query, so the absence of
-/// that `$id` is itself proven. Today only the exact-set assembly
-/// refuses the result.
+/// that `$id` is itself proven, and the verifier returns the outer half
+/// without it.
 #[test]
 fn should_prove_the_absence_of_a_missing_referenced_post() {
     let pv = platform_version();
@@ -489,10 +530,18 @@ fn should_prove_the_absence_of_a_missing_referenced_post() {
             .collect();
         assert_eq!(present, expected, "missing {missing:?}");
 
-        let refused = chained.verify_chained_documents_proof(proof.as_slice(), pv);
-        assert!(
-            matches!(refused, Err(Error::Proof(_))),
-            "the exact-set assembly is what refuses a dangling join today, got {refused:?}"
+        let (_root, verified) = chained
+            .verify_chained_documents_proof(proof.as_slice(), pv)
+            .expect("the verifier leaves the proven-absent posts out");
+        assert_eq!(verified.inner_documents.len(), POSTS.len());
+        assert_eq!(
+            verified
+                .outer_documents
+                .iter()
+                .map(|d| d.id().to_buffer())
+                .collect::<Vec<_>>(),
+            expected,
+            "missing {missing:?}"
         );
     }
 }