feat(platform)!: require fee history for storage refunds and credit their recorded owners

book/src/fees/overview.md

@@ -49,8 +49,9 @@ in `FeeStorageVersion`:
 | `storage_seek_cost` | 2,000 | Cost of a single disk seek |
 
 Storage fees are **refundable**: when data is deleted, a portion of the original
-storage fee is returned to the identity that paid it (see [Refunds](#refunds)
-below).
+storage fee becomes a refund for the owner recorded in the stored bytes' storage
+flags, which is not always the identity that paid the fee (see
+[Refunds](#refunds) below).
 
 The documents of a type that declares a `ttl` (protocol version 14) are the exception: they
 carry no storage flags and refund nothing, their bytes are priced for the time they live, and
@@ -468,6 +469,38 @@ out to proposers.
 There is a **dust limit**: refunds below 32 bytes worth of storage credits are
 discarded to prevent micro-refund spam.
 
+### Fee history and refund ownership (protocol version 15 onward)
+
+A refund is priced with the fee history of the block that removes the bytes:
+the `previous_fee_versions` map platform state carries, which the epoch change
+hook extends whenever the fee version number changes. `Drive::calculate_fee`
+v1 (`DRIVE_VERSION_V10`) consults that history for every owner-attributed
+storage removal, on every fee version number, and returns an internal error
+when a caller passes none. Earlier generations priced fee version number 1
+against an empty history, so a caller that forgot the history silently
+refunded at the first generation's storage rates; from protocol version 15
+that omission halts instead of mispricing. Every shipped schedule shares fee
+version number 1 and the same storage rates, so the credits themselves are
+unchanged for every shipped input.
+
+Refunds follow the recorded owner in the element's storage flags.
+`Drive::credit_storage_refunds_to_owners_operations` credits each owner that
+has a balance element without consulting any key or permission, so a frozen
+but existing owner still receives its bookkeeping refund. Two shares of a
+refund never reach a balance and are reported for the caller instead: the
+part that clears an owner's negative credit (identity debt, which lives
+outside the credit sum trees) and the part whose owner has no balance element
+(the native stand-in for a wiped owner). The caller moves both into the
+current epoch's processing pool with a single pool write and records every
+refund against its storage epoch in the pending epoch refunds, so the credit
+conservation check stays balanced. This primitive is the settlement step
+block lifecycle paths that remove owner-attributed bytes are meant to use in
+the block that removes them; at protocol version 15 the vote poll end cleanup
+does not yet price or settle its refunds, and wiring it up is a separate
+change. The protocol 12 schema migration, which shrank stored contracts
+without refunding the stripped bytes, ran once at that activation and is the
+recorded historical exception; it replays exactly as executed.
+
 ## Epoch-Based Fee Distribution
 
 Fees do not go directly to the block proposer. Instead, they accumulate in
@@ -541,6 +574,15 @@ Fee versions are stored in the `FEE_VERSIONS` array and looked up by number. The
 `uses_version_fee_multiplier_permille` field allows a global scaling factor
 (permille = divide by 1000; a value of 1000 means no change).
 
+`fee_version_number` keys the persisted fee history that refunds are priced
+against. A schedule that changes storage rates needs a new number, because the
+refund code resolves the schedule for an epoch through the history and (in
+generations before protocol version 15) shortcut number 1 to the first
+generation's rates. `FEE_VERSION1` and `FEE_VERSION2` share number 1 because
+only a non-storage group changed between them; a test in `rs-drive`'s fee
+operation module pins that every shipped schedule keeps the first generation's
+storage rates.
+
 ## Key Source Files
 
 | File | Contents |

packages/rs-drive-abci/src/query/document_query/v1/dispatch/chained.rs

@@ -291,11 +291,14 @@ mod tests {
     use dpp::dashcore::Network;
     use dpp::data_contract::document_type::random_document::CreateRandomDocument;
     use dpp::document::{Document, DocumentV0Getters, DocumentV0Setters};
+    use dpp::fee::default_costs::CachedEpochIndexFeeVersions;
     use dpp::identifier::Identifier;
     use dpp::platform_value::Value;
     use dpp::tests::json_document::json_document_to_contract;
+    use dpp::version::fee::FeeVersion;
     use drive::drive::contract::moderation::types::ContractDocumentRemovalEntry;
     use drive::util::test_helpers::with_likes_read_whole_through_by_liker;
+    use std::collections::BTreeMap;
 
     const YAPPR_CONTRACT_PATH: &str =
         "../rs-drive/tests/supporting_files/contract/yappr-likes/yappr-likes-contract.json";
@@ -555,6 +558,9 @@ mod tests {
     fn should_report_a_deleted_post_of_a_deletable_document_join() {
         let (platform, state, version, contract) =
             setup_yappr_state_at(YAPPR_DELETABLE_POSTS_CONTRACT_PATH);
+        // The post is owner-flagged, so pricing its removal needs the fee history of the
+        // removing block, as every production caller passes.
+        let fee_history: CachedEpochIndexFeeVersions = BTreeMap::from([(0, FeeVersion::first())]);
         platform
             .drive
             .delete_document_for_contract(
@@ -565,7 +571,7 @@ mod tests {
                 true,
                 None,
                 version,
-                None,
+                Some(&fee_history),
             )
             .expect("expected to delete the post");
 

packages/rs-drive/src/drive/contract/apply/apply_contract_with_serialization/mod.rs

@@ -14,6 +14,7 @@ use std::borrow::Cow;
 use std::collections::HashMap;
 
 mod v0;
+mod v1;
 
 /// Drive contract application methods.
 impl Drive {
@@ -71,9 +72,18 @@ impl Drive {
                 transaction,
                 platform_version,
             ),
+            1 => self.apply_contract_with_serialization_v1(
+                contract,
+                contract_serialization,
+                block_info,
+                apply,
+                storage_flags,
+                transaction,
+                platform_version,
+            ),
             version => Err(Error::Drive(DriveError::UnknownVersionMismatch {
                 method: "apply_contract_with_serialization".to_string(),
-                known_versions: vec![0],
+                known_versions: vec![0, 1],
                 received: version,
             })),
         }
@@ -127,7 +137,9 @@ impl Drive {
             .apply
             .apply_contract_with_serialization
         {
-            0 => self.apply_contract_with_serialization_operations_v0(
+            // Generation 1 of the wrapper changes only when its owned transaction commits;
+            // the operation builder is generation 0's.
+            0 | 1 => self.apply_contract_with_serialization_operations_v0(
                 contract,
                 contract_serialization,
                 block_info,
@@ -138,7 +150,7 @@ impl Drive {
             ),
             version => Err(Error::Drive(DriveError::UnknownVersionMismatch {
                 method: "apply_contract_with_serialization_operations".to_string(),
-                known_versions: vec![0],
+                known_versions: vec![0, 1],
                 received: version,
             })),
         }

packages/rs-drive/src/drive/contract/apply/apply_contract_with_serialization/v1/mod.rs

@@ -0,0 +1,80 @@
+use crate::drive::Drive;
+use crate::error::Error;
+use crate::fees::op::LowLevelDriveOperation;
+use crate::fees::op::LowLevelDriveOperation::CalculatedCostOperation;
+use crate::util::storage_flags::StorageFlags;
+use dpp::block::block_info::BlockInfo;
+use dpp::fee::fee_result::FeeResult;
+use dpp::prelude::DataContract;
+use dpp::version::PlatformVersion;
+use grovedb::batch::KeyInfoPath;
+use grovedb::{EstimatedLayerInformation, TransactionArg};
+use std::borrow::Cow;
+use std::collections::HashMap;
+
+impl Drive {
+    /// Generation 1 differs from the previous one in one thing: when the caller supplies no
+    /// transaction and the operations are applied, the write and its pricing share one
+    /// owned transaction that is committed only after `Drive::calculate_fee` succeeded.
+    /// Earlier generations applied the batch (committing it on its own without a caller
+    /// transaction) and priced it afterwards, so from protocol version 15, where pricing an
+    /// owner-attributed storage removal without the fee history is an error, a call passing
+    /// no history persisted the write and then failed. It now fails before anything is
+    /// written. With a caller transaction nothing is committed by Drive in either
+    /// generation.
+    #[inline(always)]
+    #[allow(clippy::too_many_arguments)]
+    pub(super) fn apply_contract_with_serialization_v1(
+        &self,
+        contract: &DataContract,
+        contract_serialization: Vec<u8>,
+        block_info: BlockInfo,
+        apply: bool,
+        storage_flags: Option<Cow<StorageFlags>>,
+        transaction: TransactionArg,
+        platform_version: &PlatformVersion,
+    ) -> Result<FeeResult, Error> {
+        let owned_transaction =
+            (apply && transaction.is_none()).then(|| self.grove.start_transaction());
+        let transaction = owned_transaction.as_ref().or(transaction);
+        let mut cost_operations = vec![];
+        let mut estimated_costs_only_with_layer_info = if apply {
+            None::<HashMap<KeyInfoPath, EstimatedLayerInformation>>
+        } else {
+            Some(HashMap::new())
+        };
+        let batch_operations = self.apply_contract_with_serialization_operations(
+            contract,
+            contract_serialization,
+            &block_info,
+            &mut estimated_costs_only_with_layer_info,
+            storage_flags,
+            transaction,
+            platform_version,
+        )?;
+        let fetch_cost = LowLevelDriveOperation::combine_cost_operations(&batch_operations);
+        self.apply_batch_low_level_drive_operations(
+            estimated_costs_only_with_layer_info,
+            transaction,
+            batch_operations,
+            &mut cost_operations,
+            &platform_version.drive,
+        )?;
+        cost_operations.push(CalculatedCostOperation(fetch_cost));
+
+        // A pricing error drops the owned transaction with everything it wrote.
+        let fees = Drive::calculate_fee(
+            None,
+            Some(cost_operations),
+            &block_info.epoch,
+            self.config.epochs_per_era,
+            platform_version,
+            None,
+        )?;
+        if let Some(owned_transaction) = owned_transaction {
+            self.commit_transaction(owned_transaction, &platform_version.drive)?;
+        }
+
+        Ok(fees)
+    }
+}

packages/rs-drive/src/drive/contract/insert/insert_contract/v0/tests/index_only_e2e_tests.rs

@@ -3817,3 +3817,151 @@ fn untagged_like_skips_the_hashtag_index_and_prices_lower() {
 
     assert_grovedb_is_consistent(&drive);
 }
+
+/// Deleting an owner-flagged like frees the owner's bytes, so pricing it without the fee
+/// history is rejected from protocol version 15. The wrapper owns its transaction when the
+/// caller passes none, so the rejected delete leaves every index entry and the root hash as
+/// they were; with the history it refunds the owner and commits, and protocol version 14
+/// still deletes without one.
+#[test]
+fn should_leave_an_index_only_document_in_place_when_the_wrapper_cannot_price_its_removal() {
+    use crate::error::drive::DriveError;
+    use crate::error::Error;
+    use dpp::fee::default_costs::CachedEpochIndexFeeVersions;
+    use dpp::version::fee::FeeVersion;
+    use std::borrow::Cow;
+    use std::collections::BTreeMap;
+
+    let insert_flagged_like = |drive: &Drive, contract: &DataContract, like: &Document, pv| {
+        let document_type = contract
+            .document_type_for_name(DOCTYPE)
+            .expect("like doctype exists");
+        drive
+            .add_document_for_contract(
+                DocumentAndContractInfo {
+                    owned_document_info: OwnedDocumentInfo {
+                        document_info: DocumentRefInfo((
+                            like,
+                            Some(Cow::Owned(StorageFlags::SingleEpochOwned(0, OWNER_1))),
+                        )),
+                        owner_id: None,
+                    },
+                    contract,
+                    document_type,
+                },
+                false,
+                BlockInfo::default(),
+                true,
+                None,
+                pv,
+                None,
+            )
+            .expect("insert like with owned flags");
+    };
+    let root_hash = |drive: &Drive, pv: &PlatformVersion| {
+        drive
+            .grove
+            .root_hash(None, &pv.drive.grove_version)
+            .unwrap()
+            .expect("expected a root hash")
+    };
+
+    let pv = platform_version();
+    let (drive, contract) = setup_likes();
+    let document_type = contract
+        .document_type_for_name(DOCTYPE)
+        .expect("like doctype exists");
+    let like = build_like(&contract, "dash", POST_A, OWNER_1, 1);
+    insert_flagged_like(&drive, &contract, &like, pv);
+    let before = root_hash(&drive, pv);
+    let mut by_post_level = doctype_path(&contract);
+    by_post_level.push(b"postId".to_vec());
+
+    let result = drive.delete_index_only_document_for_contract(
+        like.clone(),
+        &contract,
+        document_type,
+        BlockInfo::default(),
+        true,
+        None,
+        pv,
+        None,
+    );
+    assert!(
+        matches!(
+            result,
+            Err(Error::Drive(DriveError::CorruptedCodeExecution(_)))
+        ),
+        "deleting owner-flagged entries without a fee history must be rejected, got {:?}",
+        result
+    );
+    assert_eq!(
+        root_hash(&drive, pv),
+        before,
+        "a rejected delete must not persist"
+    );
+    assert!(
+        read_grove_element(&drive, &by_post_level, &POST_A).is_some(),
+        "the like's group is still there"
+    );
+
+    let history: CachedEpochIndexFeeVersions = BTreeMap::from([(0, FeeVersion::first())]);
+    let fee_result = drive
+        .delete_index_only_document_for_contract(
+            like,
+            &contract,
+            document_type,
+            BlockInfo::default(),
+            true,
+            None,
+            pv,
+            Some(&history),
+        )
+        .expect("expected to delete the like with the fee history");
+    assert!(fee_result
+        .fee_refunds
+        .calculate_refunds_amount_for_identity(Identifier::from(OWNER_1))
+        .is_some());
+    assert!(
+        read_grove_element(&drive, &by_post_level, &POST_A).is_none(),
+        "the delete was committed"
+    );
+    assert_grovedb_is_consistent(&drive);
+
+    // Protocol version 14 prices the shipped shortcut without a history and commits.
+    let frozen = PlatformVersion::get(14).expect("protocol version 14");
+    let drive = setup_drive_with_initial_state_structure(Some(frozen));
+    let contract = json_document_to_contract(
+        "tests/supporting_files/contract/yappr-likes/yappr-likes-contract.json",
+        false,
+        frozen,
+    )
+    .expect("expected to parse the yappr-likes contract");
+    drive
+        .apply_contract(
+            &contract,
+            BlockInfo::default(),
+            true,
+            StorageFlags::optional_default_as_cow(),
+            None,
+            frozen,
+        )
+        .expect("expected to apply the yappr-likes contract");
+    let document_type = contract
+        .document_type_for_name(DOCTYPE)
+        .expect("like doctype exists");
+    let like = build_like(&contract, "dash", POST_A, OWNER_1, 1);
+    insert_flagged_like(&drive, &contract, &like, frozen);
+    drive
+        .delete_index_only_document_for_contract(
+            like,
+            &contract,
+            document_type,
+            BlockInfo::default(),
+            true,
+            None,
+            frozen,
+            None,
+        )
+        .expect("protocol version 14 prices the shipped shortcut without a history");
+}