diff --git a/Cargo.lock b/Cargo.lock index 13dc6cc8..ae94c3d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2203,6 +2203,16 @@ dependencies = [ "siphasher", ] +[[package]] +name = "faster-hex" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04839bdf9d8c10f66806fad16b852fc72aab80873aebc3cb69d85b4fa41543ed" +dependencies = [ + "autocfg", + "heapless 0.8.0", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -2680,6 +2690,16 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.5.0" @@ -4563,6 +4583,7 @@ dependencies = [ "p3-field", "p3-matrix", "p3-maybe-rayon", + "p3-merkle-tree", "p3-security", "p3-util", "rand 0.10.2", @@ -5042,7 +5063,7 @@ dependencies = [ "cobs", "embedded-io 0.4.0", "embedded-io 0.6.1", - "heapless", + "heapless 0.7.17", "serde", ] @@ -5526,6 +5547,7 @@ dependencies = [ "bs58", "cbor4ii 0.3.3", "clap", + "faster-hex", "futures", "hex", "libc", diff --git a/Cargo.toml b/Cargo.toml index 1f59c06b..9c7d2fc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,9 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" bincode = "1.3" hex = "0.4" +# #152: `rand_sendTransaction`'s hex body (2.9 MB of hex a bundle transaction) decoded with a +# SIMD/table decoder; `hex` took 18 ms of a 92 ms admission. MIT, no dependencies at this feature set. +faster-hex = { version = "0.10.1", default-features = false } # The scan heads `rand_getScanLeaves` serves: 1 208 bytes a leaf as base64 rather than hex, a third # less on the wire. Already in the lock through reqwest/hyper; pinned to that version. base64 = "=0.22.1" @@ -97,7 +100,10 @@ opt-level = 2 # Plonky3 0.7.0 with one fix in two crates (upstream issue Plonky3 #2363, fix PR #2368): HidingFriPcs and # MerkleTreeHidingMmcs held a spin lock around their RNG across rayon work, and a # worker waiting on a stolen job could steal another `commit` and spin on its own lock for ever -# (vendor/p3-fri/src/hiding_pcs.rs, "RandProtocol patch"). Everything else is the crates.io source. +# (vendor/p3-fri/src/hiding_pcs.rs, "RandProtocol patch"). v0.8.3 (#151) adds a second change to the +# same two crates: their random columns and Merkle salts are counter-mode ChaCha20 streams, one per +# chunk of rows, filled in parallel (vendor/p3-merkle-tree/src/counter_rng.rs; the proof format and +# the verifier are unchanged). Everything else is the crates.io source. # `vendor/p3-batch-stark` is a third: p3-batch-stark 0.7.0 with a quotient-layout switch # (`QuotientLayout`), the circuits fork copied verbatim from circuits 5ff7676; circuits' # `vendor/PROVENANCE.md` and `recursion/docs/05-quotient-layout.md` are its record. diff --git a/crates/randprotocol-node/Cargo.toml b/crates/randprotocol-node/Cargo.toml index ec5b8aee..98862532 100644 --- a/crates/randprotocol-node/Cargo.toml +++ b/crates/randprotocol-node/Cargo.toml @@ -31,6 +31,7 @@ serde = { workspace = true } serde_json = { workspace = true } bincode = { workspace = true } hex = { workspace = true } +faster-hex = { workspace = true } base64 = { workspace = true } thiserror = { workspace = true } anyhow = { workspace = true } diff --git a/crates/randprotocol-node/src/admission.rs b/crates/randprotocol-node/src/admission.rs index 33308b45..1d8b7524 100644 --- a/crates/randprotocol-node/src/admission.rs +++ b/crates/randprotocol-node/src/admission.rs @@ -1038,6 +1038,33 @@ pub fn faucet_minters(gs: &randprotocol_core::genesis::GenesisState) -> std::col } } +/// #153: why this node cannot serve `rand_mint`, or `None` when it can — the faucet is on (genesis +/// `faucet`), this node runs as a validator, and its key is one of the chain's faucet minters +/// ([`faucet_minters`]: `staking.faucet_minters` where the genesis lists them, else the genesis +/// validators). What `rand_status` / `rand_getEmission` report as `faucet` (true exactly when this +/// is `None`), and what `rand_mint` checks before it builds anything — so a validator bonded after +/// genesis, whose key the pool and the ledger would refuse as a minter, says so up front instead of +/// advertising a faucet whose every mint fails at admission. +pub fn faucet_refusal(gs: &randprotocol_core::genesis::GenesisState, is_validator: bool, key: &randprotocol_core::Address) -> Option { + if !gs.faucet { + return Some("faucet is disabled on this chain".into()); + } + if !is_validator { + return Some("faucet mints are signed by validators; ask a validator node".into()); + } + let minters = faucet_minters(gs); + if minters.contains(key) { + return None; + } + let source = if gs.staking.as_ref().is_some_and(|s| s.faucet_minters.is_some()) { "staking.faucet_minters" } else { "the genesis validators" }; + let list: Vec = minters.iter().map(|a| a.to_base58()).collect(); + Some(format!( + "this node's key {key} is not a faucet minter on this chain; the minters are {source} ({}): {}; ask a node run by one of them", + list.len(), + list.join(", ") + )) +} + /// A faucet `Mint` whose minter has a row in the validator register but is not in `minters` /// ([`faucet_minters`]): `TxError::MinterNotAllowed`. `None` for everything else — including a /// minter with no row at all, which is the ledger's own `MinterNotValidator`, left to `validate` @@ -2200,6 +2227,42 @@ mod tests { assert!(pool.insert(honest, &l, &StubExecutor).is_ok()); } + /// #153: `faucet` in `rand_status` means "this node can mint here" — the faucet on, a validator, + /// and a key among the chain's faucet minters — and every other case names its reason, the + /// non-minter's naming the minter set. + #[test] + fn the_faucet_is_served_only_by_a_validator_whose_key_is_a_faucet_minter() { + use crate::storage::fixtures; + use randprotocol_core::genesis::{FaucetMinter, StakingConfig}; + use randprotocol_core::UNITS_PER_RAND; + let (a, b, later) = (fixtures::key(1), fixtures::key(2), fixtures::key(9)); + let mut g = fixtures::genesis_file_of(1, &[&a, &b], Vec::new(), 10); + g.faucet = true; + let gs = g.build(&randprotocol_core::confidential::StubExecutor).unwrap(); + // A genesis validator mints; a key bonded later (the guardian hosts of issue #153) does not. + assert_eq!(faucet_refusal(&gs, true, &a.address()), None); + let why = faucet_refusal(&gs, true, &later.address()).expect("a post-genesis key is no minter"); + assert!(why.contains(&later.address().to_base58()) && why.contains("is not a faucet minter"), "{why}"); + assert!(why.contains("the genesis validators (2)") && why.contains(&a.address().to_base58()) && why.contains(&b.address().to_base58()), "{why}"); + // Not a validator, or no faucet: those reasons first, as before. + assert_eq!(faucet_refusal(&gs, false, &a.address()).as_deref(), Some("faucet mints are signed by validators; ask a validator node")); + g.faucet = false; + let off = g.build(&randprotocol_core::confidential::StubExecutor).unwrap(); + assert_eq!(faucet_refusal(&off, true, &a.address()).as_deref(), Some("faucet is disabled on this chain")); + // A genesis that lists its minters: the list decides, and the message names it. + g.faucet = true; + g.staking = Some(StakingConfig { + faucet_budget_per_epoch: 100 * UNITS_PER_RAND, + bond_activation_epochs: 2, + faucet_minters: Some(vec![FaucetMinter(later.address())]), + ..Default::default() + }); + let listed = g.build(&randprotocol_core::confidential::StubExecutor).unwrap(); + assert_eq!(faucet_refusal(&listed, true, &later.address()), None); + let why = faucet_refusal(&listed, true, &a.address()).unwrap(); + assert!(why.contains("staking.faucet_minters (1)") && why.contains(&later.address().to_base58()), "{why}"); + } + /// Where a genesis lists `staking.faucet_minters` the pool's minters are that list, not the /// genesis validators: a listed non-genesis key is admitted, an unlisted genesis key is not. #[test] diff --git a/crates/randprotocol-node/src/node.rs b/crates/randprotocol-node/src/node.rs index cc851f57..7d8a5920 100644 --- a/crates/randprotocol-node/src/node.rs +++ b/crates/randprotocol-node/src/node.rs @@ -2665,7 +2665,8 @@ pub async fn start_with(cfg: NodeConfig, rpc_options: RpcOptions, net_options: N let status = Arc::new(RwLock::new(NodeStatus { is_validator: hs.is_validator(), active_validator: hs.current_set().contains(&key.address()), - faucet: gs.faucet, + // #153: "this node can mint here", not just "the genesis turns the faucet on". + faucet: admission::faucet_refusal(&gs, hs.is_validator(), &key.address()).is_none(), confidential: gs.confidential, testnet: gs.testnet, fri_profile: gs.fri_profile.clone(), @@ -3827,12 +3828,14 @@ impl Node { /// preserving and no reason to keep an outgoing-viewing record, so the `to_sender` half of /// the envelope is addressed to a key that is dropped on the next line and never recoverable. /// Only `to` can open the note, which is the whole intent. + /// + /// #153: and only a key among the chain's faucet minters (`admission::faucet_refusal`, the + /// predicate `rand_status.faucet` reports) — refused here, naming the minter set, before + /// anything is built, rather than by the pool after. async fn mint(&mut self, to: ShieldedAddress, amount: u64) -> std::result::Result { - if !self.gs.faucet { - return Err("faucet is disabled on this chain".into()); - } - if !self.hs.is_validator() { - return Err("faucet mints are signed by validators; ask a validator node".into()); + let key = Keypair::from_seed(self.cfg.seed).expect("seed validated at startup"); + if let Some(why) = admission::faucet_refusal(&self.gs, self.hs.is_validator(), &key.address()) { + return Err(why); } if amount > FAUCET_MAX_UNITS { return Err(format!("mint of {amount} exceeds the faucet cap of {FAUCET_MAX_UNITS}")); @@ -3848,7 +3851,6 @@ impl Node { admission::FAUCET_MINT_PER_SEC )); } - let key = Keypair::from_seed(self.cfg.seed).expect("seed validated at startup"); let height = self.hs.tip_ledger().height(); let envelope_bytes = self.hs.tip_ledger().envelope_bytes(); let domain = *self.hs.tip_ledger().binding_domain(); @@ -7607,6 +7609,39 @@ mod tests { (node, seen_rx, applied_rx) } + /// #153: a validator whose key is outside the chain's faucet minters — a guardian host bonded + /// after genesis — refuses `rand_mint` up front, naming the minter set, and nothing reaches the + /// pool. + #[tokio::test] + async fn a_validator_outside_the_faucet_minters_refuses_rand_mint_up_front() { + use randprotocol_core::genesis::{FaucetMinter, StakingConfig}; + let mint_on = |minters: Vec| async move { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::open(dir.path()).unwrap(); + let mut g = crate::storage::fixtures::genesis_file_of(1, &[&key(1)], Vec::new(), 10); + g.faucet = true; + g.staking = Some(StakingConfig { + faucet_budget_per_epoch: 1_000 * randprotocol_core::UNITS_PER_RAND, + bond_activation_epochs: 2, + faucet_minters: Some(minters.into_iter().map(FaucetMinter).collect()), + ..Default::default() + }); + let gs = g.build(&StubExecutor).unwrap(); + storage.init_genesis(&gs).unwrap(); + let hs = resume_consensus(&storage, &gs, Some(key(1)), Duration::from_secs(1), Duration::from_secs(8), Arc::new(StubExecutor)).unwrap(); + let (mut node, _seen) = bare_node(storage, gs, hs); + let r = node.mint(crate::storage::fixtures::payout(9), 1_000).await; + (r, node.mempool.len()) + }; + let (r, pooled) = mint_on(vec![key(9).address()]).await; + let why = r.unwrap_err(); + assert!(why.contains(&format!("this node's key {} is not a faucet minter", key(1).address())), "{why}"); + assert!(why.contains("staking.faucet_minters (1)") && why.contains(&key(9).address().to_base58()), "{why}"); + assert_eq!(pooled, 0, "refused before a mint was built or pooled"); + // (A listed key gets past this check — `admission::faucet_refusal`'s own test; building the + // mint itself needs the real executor, `a_faucet_mint_is_sealed_in_the_chains_declared_envelope_format`.) + } + /// One connected peer claiming `height`. fn claiming_peer(node: &mut Node, height: u64) -> PeerId { let p = PeerId::random(); diff --git a/crates/randprotocol-node/src/rpc.rs b/crates/randprotocol-node/src/rpc.rs index dbc5009b..dbbf4696 100644 --- a/crates/randprotocol-node/src/rpc.rs +++ b/crates/randprotocol-node/src/rpc.rs @@ -274,6 +274,9 @@ pub struct NodeStatus { /// validator that has bonded in but whose epoch has not arrived — or one that has unbonded /// below the minimum — is `is_validator` without being `active_validator`. pub active_validator: bool, + /// #153: this node can serve `rand_mint` — the genesis turns the faucet on, this node runs as a + /// validator, and its key is one of the chain's faucet minters (`admission::faucet_refusal`). + /// Before v0.8.3 it was the genesis flag alone, `true` on validators whose every mint failed. pub faucet: bool, pub confidential: bool, /// Audit v6, STAKE-2: the genesis says `testnet: true` — the marker that lets a faucet sit @@ -1598,6 +1601,20 @@ fn parse_shielded(params: &Value, idx: usize) -> Result Option> { + let src = s.strip_prefix("0x").unwrap_or(s).as_bytes(); + if !src.len().is_multiple_of(2) { + return None; + } + let mut out = vec![0u8; src.len() / 2]; + faster_hex::hex_decode(src, &mut out).ok()?; + Some(out) +} + /// A 32-byte value as hex, with or without `0x`: a bridge token address or emitter. fn parse_bytes32(params: &Value, idx: usize, name: &str) -> Result<[u8; 32], RpcError> { let s: String = param(params, idx, name)?; @@ -2765,8 +2782,7 @@ async fn dispatch(st: &RpcState, req: &Request) -> Result { } "rand_sendTransaction" => { let raw: String = param(p, 0, "tx")?; - let bytes = hex::decode(raw.strip_prefix("0x").unwrap_or(&raw)) - .map_err(|_| RpcError::invalid_params("tx must be hex"))?; + let bytes = decode_hex_body(&raw).ok_or_else(|| RpcError::invalid_params("tx must be hex"))?; let tx = Transaction::decode(&bytes).map_err(|e| RpcError::invalid_params(format!("tx decode: {e}")))?; // The body limit admits more than a block can carry — two `max_proof_bytes` proofs // are a legal `Call` by the per-part caps and can still be over `max_block_bytes` — so @@ -5861,6 +5877,40 @@ pub(crate) mod tests { assert_eq!(call(&st, "rand_sendTransaction", json!(["00"])).await.unwrap_err().code, -32602); } + /// #152: `rand_sendTransaction`'s faster-hex decode gives exactly `hex::decode`'s verdicts — + /// the same bytes for every valid body (either case, mixed, with or without `0x`), a refusal + /// for every odd length and every non-hex character, at every position a SIMD lane boundary + /// could hide one in. + #[test] + fn the_hex_body_decodes_as_hex_does() { + use rand::{Rng, SeedableRng}; + let mut rng = rand::rngs::StdRng::seed_from_u64(152); + for len in [0usize, 1, 2, 7, 15, 16, 31, 32, 33, 63, 64, 65, 127, 1000, 4097] { + let bytes: Vec = (0..len).map(|_| rng.gen()).collect(); + let lower = hex::encode(&bytes); + let upper = lower.to_uppercase(); + let mixed: String = lower.chars().enumerate().map(|(i, c)| if i % 3 == 0 { c.to_ascii_uppercase() } else { c }).collect(); + for body in [lower.clone(), upper, mixed, format!("0x{lower}")] { + assert_eq!(decode_hex_body(&body), Some(bytes.clone()), "{len} bytes"); + assert_eq!(decode_hex_body(&body).ok_or(()), hex::decode(body.strip_prefix("0x").unwrap_or(&body)).map_err(|_| ())); + } + if len > 0 { + assert_eq!(decode_hex_body(&lower[1..]), None, "odd length"); + for pos in [0, lower.len() / 2, lower.len() - 1] { + for bad in ['g', 'G', 'x', ' ', '/', ':', '@', '`', '\u{7f}'] { + let mut b = lower.clone().into_bytes(); + b[pos] = bad as u8; + let b = String::from_utf8(b).unwrap(); + assert_eq!(decode_hex_body(&b), None, "{bad:?} at {pos} of {len} bytes"); + assert!(hex::decode(&b).is_err()); + } + } + } + } + assert_eq!(decode_hex_body("0x"), Some(vec![])); + assert_eq!(decode_hex_body("0X00"), None, "only a lowercase 0x prefix, as before"); + } + #[tokio::test] async fn status_reports_note_and_nullifier_counts() { let (_d, st, _gs) = chain(); diff --git a/crates/randprotocol-zkvm/src/executor.rs b/crates/randprotocol-zkvm/src/executor.rs index 164eb48d..d409036e 100644 --- a/crates/randprotocol-zkvm/src/executor.rs +++ b/crates/randprotocol-zkvm/src/executor.rs @@ -369,7 +369,7 @@ pub const MIN_PRIVATE_TABLE_LOG_HEIGHT: u8 = 7; /// Reads the header only; nothing is verified. Next to the call caps `verify_call` enforces, but /// policy, not one of them (see the constant's doc comment). pub fn call_private_table_under_floor(proof: &[u8]) -> Option<(&'static str, u8)> { - let proof = decode_canonical(proof).ok()?; + let proof = decode_canonical_shared(proof).ok()?; let min = MIN_PRIVATE_TABLE_LOG_HEIGHT; if proof.input_log_height < min { return Some(("input", proof.input_log_height)); @@ -389,7 +389,7 @@ pub fn call_private_table_under_floor(proof: &[u8]) -> Option<(&'static str, u8) /// a forged header can only get its own transaction refused (PST-1: /// `program_state::panicked`). pub fn call_proof_outputs(proof: &[u8]) -> Option<[u32; 8]> { - let proof = decode_canonical(proof).ok()?; + let proof = decode_canonical_shared(proof).ok()?; let mut out = [0u32; 8]; for (i, o) in out.iter_mut().enumerate() { *o = u32::try_from(*proof.public_values.get(pv::OUT0 + i)?).ok()?; @@ -663,7 +663,7 @@ pub fn honest_fri_arities(proof: &Proof) -> Option> { /// [`non_canonical`] over proof bytes: `None` for bytes that do not decode canonically — those /// are refused as `MalformedProof` wherever they are verified. pub fn non_canonical_proof(bytes: &[u8]) -> Option { - non_canonical(&decode_canonical(bytes).ok()?) + non_canonical(&*decode_canonical_shared(bytes).ok()?) } /// The largest `input_log_height` a call at `tier` can honestly declare — the input table's @@ -966,15 +966,11 @@ impl ZkExecutor { public_log_height: u8, exact: bool, pinned_tier: usize, - ) -> Result { - self.check_decoded( - decode_canonical(proof)?, - program_log_height, - input_log_height, - public_log_height, - exact, - pinned_tier, - ) + ) -> Result, ConfidentialError> { + // #152: the memoised decode — one decode per proof per admission, shared by every check. + let proof = decode_canonical_shared(proof)?; + self.check_decoded_ref(&proof, program_log_height, input_log_height, public_log_height, exact, pinned_tier)?; + Ok(proof) } /// [`Self::decode_and_check`] on a proof already decoded (by [`decode_canonical`]): for @@ -988,6 +984,20 @@ impl ZkExecutor { exact: bool, pinned_tier: usize, ) -> Result { + self.check_decoded_ref(&proof, program_log_height, input_log_height, public_log_height, exact, pinned_tier)?; + Ok(proof) + } + + /// [`Self::check_decoded`] by reference (#152: on a memoised decode, which is shared). + fn check_decoded_ref( + &self, + proof: &Proof, + program_log_height: u8, + input_log_height: u8, + public_log_height: u8, + exact: bool, + pinned_tier: usize, + ) -> Result<(), ConfidentialError> { // Constraint set 9: a modq table is the multisig auth guest's alone, and that guest has // its own path (`decode_and_check_msauth`). A call, hardened call, invoke or state proof // (`exact` false) declaring one is refused here, before anything is sized from the header @@ -1082,7 +1092,7 @@ impl ZkExecutor { if proof.public_values[pv::IN0..pv::IN0 + 8].iter().any(|v| *v > u32::MAX as u64) { return Err(ConfidentialError::InvalidProof("H_IN word not a u32".into())); } - Ok(proof) + Ok(()) } /// The chain's bundle guest: since chain 14 the hidden-asset guest @@ -1383,8 +1393,8 @@ impl ZkExecutor { /// header (all eight words — a junk header costs comparisons, never a tier-18 key build), the /// machine's own ranges, the degree-bits vector of the pinned shape (which is also the /// instance-count check), and the public-value layout. - fn check_msauth_decoded(&self, proof: Proof) -> Result { - if let Some(field) = Self::msauth_header_mismatch(&proof) { + fn check_msauth_decoded(&self, proof: &Proof) -> Result<(), ConfidentialError> { + if let Some(field) = Self::msauth_header_mismatch(proof) { return Err(ConfidentialError::InvalidProof(format!("not the multisig auth guest's shape: {field}"))); } let s = Self::msauth_heights(); @@ -1421,12 +1431,15 @@ impl ZkExecutor { if proof.public_values[pv::IN0..pv::IN0 + 8].iter().any(|v| *v > u32::MAX as u64) { return Err(ConfidentialError::InvalidProof("H_IN word not a u32".into())); } - Ok(proof) + Ok(()) } - /// [`Self::check_msauth_decoded`] over proof bytes (the canonical decode first). - fn decode_and_check_msauth(&self, proof: &[u8]) -> Result { - self.check_msauth_decoded(decode_canonical(proof)?) + /// [`Self::check_msauth_decoded`] over proof bytes (the canonical decode first, memoised: + /// #152). + fn decode_and_check_msauth(&self, proof: &[u8]) -> Result, ConfidentialError> { + let p = decode_canonical_shared(proof)?; + self.check_msauth_decoded(&p)?; + Ok(p) } /// The `c` a multisig auth proof publishes, after the structural checks at the pinned shape. @@ -1600,12 +1613,14 @@ impl ZkExecutor { proof: &[u8], ) -> Result<(randprotocol_core::confidential::AuthShape, Word8, Option), ConfidentialError> { use randprotocol_core::confidential::AuthShape; - let p = decode_canonical(proof)?; - let (shape, p) = if Self::is_single_key_header(&p) { + let p = decode_canonical_shared(proof)?; + let shape = if Self::is_single_key_header(&p) { let (plh, ilh, pubh) = Self::auth_heights(); - (AuthShape::SingleKey, self.check_decoded(p, plh, ilh, pubh, true, AUTH_TIER)?) + self.check_decoded_ref(&p, plh, ilh, pubh, true, AUTH_TIER)?; + AuthShape::SingleKey } else { - (AuthShape::Multisig, self.check_msauth_decoded(p)?) + self.check_msauth_decoded(&p)?; + AuthShape::Multisig }; Ok((shape, published_digest(&p.public_values)?, Some(p.public_values[pv::GAS]))) } @@ -2198,6 +2213,55 @@ pub fn decode_canonical(bytes: &[u8]) -> Result { Ok(proof) } +/// How many canonical decodes [`decode_canonical_shared`] keeps: a bundle and an auth proof for +/// each of a few admissions in flight at once (a node verifies `cores − 2` transactions +/// concurrently). Each entry is a copy of the bytes and the decoded proof, ~3 MB for a tier-14 +/// bundle proof, so the memo holds at most ~50 MB. +pub const DECODE_MEMO_ENTRIES: usize = 16; + +type DecodeMemo = std::collections::VecDeque<(std::sync::Arc<[u8]>, std::sync::Arc)>; +static DECODE_MEMO: std::sync::Mutex = std::sync::Mutex::new(std::collections::VecDeque::new()); +static DECODES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// #152: [`decode_canonical`], memoised on the exact bytes. One admission asks for the same proof +/// four times — the canonical-proof rule, `check_auth_fields`' header, the digest and gas reads, +/// and the verify — and each used to decode (and re-encode, the canonical check) its 1.4–1.5 MB +/// afresh: ~10 ms of a 73 ms warm admission. The ledger asks through `&dyn ConfidentialExecutor` +/// with the bytes, as before; this is where those calls meet, so the first decodes and the rest +/// share it. A pure function of the bytes (a hit compares them in full), so no verdict can +/// differ from an unmemoised decode; refusals are not kept. Least recently inserted out. +pub fn decode_canonical_shared(bytes: &[u8]) -> Result, ConfidentialError> { + { + let memo = DECODE_MEMO.lock().unwrap_or_else(|e| e.into_inner()); + if let Some((_, p)) = memo.iter().find(|(b, _)| **b == *bytes) { + return Ok(p.clone()); + } + } + DECODES.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let proof = std::sync::Arc::new(decode_canonical(bytes)?); + let mut memo = DECODE_MEMO.lock().unwrap_or_else(|e| e.into_inner()); + if !memo.iter().any(|(b, _)| **b == *bytes) { + if memo.len() >= DECODE_MEMO_ENTRIES { + memo.pop_front(); + } + memo.push_back((std::sync::Arc::from(bytes), proof.clone())); + } + Ok(proof) +} + +/// A test and bench instrument: how many canonical decodes [`decode_canonical_shared`] ran +/// (misses) in this process. +#[doc(hidden)] +pub fn shared_decodes() -> usize { + DECODES.load(std::sync::atomic::Ordering::Relaxed) +} + +/// A test and bench instrument: empty the decode memo (a bench timing a cold admission). +#[doc(hidden)] +pub fn clear_decode_memo() { + DECODE_MEMO.lock().unwrap_or_else(|e| e.into_inner()).clear(); +} + impl ConfidentialExecutor for ZkExecutor { fn check_program(&self, base_pc: u32, words: &[u32]) -> Result, ConfidentialError> { if !base_pc.is_multiple_of(4) { @@ -2572,7 +2636,7 @@ impl ConfidentialExecutor for ZkExecutor { /// Constraint set 8: `pv::GAS` of the canonically decoded proof — a decode, no verification /// (the trait's contract: trusted only once `verify_bundle` accepted the same bytes). fn bundle_gas_limit(&self, proof: &[u8]) -> Result, ConfidentialError> { - let proof = decode_canonical(proof)?; + let proof = decode_canonical_shared(proof)?; if proof.public_values.len() != pv::NUM { return Err(ConfidentialError::MalformedProof); } diff --git a/crates/randprotocol-zkvm/src/key_derivation_v2.rs b/crates/randprotocol-zkvm/src/key_derivation_v2.rs index a5f6ad9b..68803dd1 100644 --- a/crates/randprotocol-zkvm/src/key_derivation_v2.rs +++ b/crates/randprotocol-zkvm/src/key_derivation_v2.rs @@ -52,6 +52,7 @@ use crate::machine::Val; use core::convert::Infallible; use p3_field::{PrimeCharacteristicRing, PrimeField64}; +use p3_merkle_tree::CounterModeRng; use rand_core::{SeedableRng, TryCryptoRng, TryRng}; /// `true` from constraint set 7 (chain 16): every verifier key is salted from this module's stream. @@ -75,6 +76,10 @@ pub struct KeyRngV2 { /// Squeezed words not yet handed out, consumed from `pos`. out: [u32; 4], pos: usize, + /// Keyed by a secret seed ([`SeedableRng::from_seed`]) rather than a public label: only then + /// does it hand out counter-mode keys ([`CounterModeRng`], fullnode #151). A label stream is a + /// verifier key's, public and pinned, and keeps its serial output bit for bit. + seeded: bool, } /// `[byte length, bytes packed four to a word, little-endian, zero-padded]` — step 1. @@ -88,7 +93,7 @@ fn push_bytes(words: &mut Vec, bytes: &[u8]) { } impl KeyRngV2 { - fn from_parts(marker: &[u8], bytes: &[u8]) -> Self { + fn from_parts(marker: &[u8], bytes: &[u8], seeded: bool) -> Self { let mut words = Vec::new(); push_bytes(&mut words, DOMAIN); push_bytes(&mut words, marker); @@ -102,12 +107,12 @@ impl KeyRngV2 { state = crate::hash::permute_state(state); } // `pos == 4`: the first draw squeezes. - Self { state, out: [0; 4], pos: 4 } + Self { state, out: [0; 4], pos: 4, seeded } } /// The stream for `label`: a pure function of `DOMAIN`, `label` and the committed Poseidon2 /// table. - pub fn from_label(label: &[u8]) -> Self { Self::from_parts(b"label", label) } + pub fn from_label(label: &[u8]) -> Self { Self::from_parts(b"label", label, false) } fn squeeze(&mut self) { self.state = crate::hash::permute_state(self.state); @@ -154,7 +159,18 @@ impl SeedableRng for KeyRngV2 { /// *provided* `seed_from_u64` expands a `u64` through `rand_core`'s own PCG32 — a stream this /// module exists to stop depending on — so a consensus-facing caller uses `from_label` (or /// `from_seed` with bytes it specifies), never `seed_from_u64`. - fn from_seed(seed: [u8; 32]) -> Self { Self::from_parts(b"seed", &seed) } + fn from_seed(seed: [u8; 32]) -> Self { Self::from_parts(b"seed", &seed, true) } +} + +/// Fullnode #151 (v0.8.3): a seed-keyed stream (a proving config's clone, keyed by OS entropy) +/// hands the hiding PCS and MMCS one fresh 256-bit key per call, squeezed from this sponge, and +/// they expand it in parallel with ChaCha20 (`p3_merkle_tree::counter_rng`). A label stream — a +/// verifier key's, which is public and pinned by consensus — hands out none and is drawn from +/// serially exactly as before, so no verifier key moves. +impl CounterModeRng for KeyRngV2 { + fn counter_mode_key(&mut self) -> Option<[u8; 32]> { + self.seeded.then(|| p3_merkle_tree::counter_rng::draw_key(self)) + } } /// The `(mmcs_rng, pcs_rng)` pair behind `machine::key_config`, from [`MMCS_LABEL`] and @@ -197,6 +213,17 @@ impl TryRng for SaltRng { /// is (its own `TryCryptoRng` note). impl TryCryptoRng for SaltRng {} +/// Fullnode #151: OS entropy (`Fresh`) and a seed-keyed sponge hand out counter-mode keys; a +/// label stream does not ([`KeyRngV2`]'s impl). +impl CounterModeRng for SaltRng { + fn counter_mode_key(&mut self) -> Option<[u8; 32]> { + match self { + SaltRng::Fresh(r) => r.counter_mode_key(), + SaltRng::Key(r) => r.counter_mode_key(), + } + } +} + impl SeedableRng for SaltRng { type Seed = [u8; 32]; /// Always a [`KeyRngV2`] — never `StdRng`. This is what p3's `Clone` for the hiding MMCS and diff --git a/crates/randprotocol-zkvm/src/machine.rs b/crates/randprotocol-zkvm/src/machine.rs index 5dd071b4..c8cc394b 100644 --- a/crates/randprotocol-zkvm/src/machine.rs +++ b/crates/randprotocol-zkvm/src/machine.rs @@ -43,7 +43,7 @@ pub type Config = StarkConfig; #[cfg_attr(not(any(feature = "reference-backend", feature = "cuda", feature = "mock-cuda")), allow(dead_code))] pub(crate) const PERM_SEED: u64 = crate::poseidon2_constants::PERM_SEED; // "RandZK" -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum FriProfile { /// 16 queries, 4 PoW bits — for `cargo test`. 3·16+4 = 52 conjectured bits; not a /// production target, only fast enough for the suite. @@ -228,6 +228,14 @@ mod cuda_cfg { } } +/// A test instrument (fullnode #151): a `Config` over explicit generators — a seeded pair makes a +/// proof reproducible (`tests/counter_rng.rs`). Never a production path: `make_config` (fresh +/// entropy) proves, `key_config` (label streams) keys. +#[doc(hidden)] +pub fn config_with_rngs(profile: FriProfile, mmcs_rng: SaltRng, pcs_rng: SaltRng) -> Config { + build_config(profile, mmcs_rng, pcs_rng) +} + pub fn make_config(profile: FriProfile) -> Config { // Fresh entropy per proof, taken from the OS: this is the config actually used to prove, // so main-trace and quotient commitments stay hiding. @@ -1724,6 +1732,84 @@ fn common_from_bytes(bytes: &[u8]) -> Result, InstallKeyError Ok(CommonData { preprocessed, lookups: wire.lookups }) } +/// Fullnode #151 (v0.8.3): the proving side's preprocessed data — `ProverData`, the +/// preprocessed commitment and the Merkle tree `prove_batch` opens it from — cached once per +/// process for every [`Machine`] (the wallet and `rand-prover` build a `Machine` per proof). +/// +/// **Why caching is sound.** `prove_traces` builds it as `ProverData::from_airs_and_degrees( +/// &key_config(profile), airs, degrees)`, and every input is fixed by the cache key: `key_config` +/// salts the preprocessed commit from `key_derivation_v2`'s *label* streams (deterministic and +/// public — not the proving config's OS entropy; the hiding PCS pads preprocessed columns with +/// zero columns, not random ones, and `key_derivation_v2::KeyRngV2` hands those streams no +/// counter-mode key), the AIRs are `chips_with_modq(tier, keccak, sha256, modq)`, and the degrees +/// are the declared heights. So the build is a pure function of `(profile, tier, keccak, sha256, +/// modq, degrees)`: two builds are identical, and identical to the commitment `verifier_key` +/// recomputes (`tests/prover_data_cache.rs` checks both, and that proofs from the cached data +/// verify). The preprocessed columns are public (the verifier key), so sharing them between +/// proofs reveals nothing a proof's own randomness hides. +/// +/// Bounded: [`PROVER_DATA_CACHE_CAPACITY`] shapes, least recently used out, and only at tiers up +/// to [`PROVER_DATA_CACHE_MAX_TIER`] — a tier-18 preprocessed tree is GBs, built once per rare +/// multisig proof, and must not stay resident in a wallet after it. Concurrent first provers of +/// one shape wait for one build (a `OnceLock` slot, as `KeyCache`). +pub const PROVER_DATA_CACHE_CAPACITY: usize = 4; +/// The highest tier whose `ProverData` the process-wide cache keeps (the bundle guest is 14, the +/// auth guest 10). +pub const PROVER_DATA_CACHE_MAX_TIER: usize = 16; + +/// `(profile, tier, keccak, sha256, modq, extended degree bits per instance)`. +type ProverDataKey = (FriProfile, usize, u8, u8, u8, Vec); +type ProverDataSlot = Arc>>>; + +#[derive(Default)] +struct ProverDataCache { + map: HashMap, + tick: u64, +} + +static PROVER_DATA_CACHE: std::sync::OnceLock> = std::sync::OnceLock::new(); +static PROVER_DATA_BUILDS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// A test instrument: how many preprocessed builds `prove_traces` ran in this process (cache +/// misses, and every build past [`PROVER_DATA_CACHE_MAX_TIER`]). +#[doc(hidden)] +pub fn prover_data_builds() -> usize { PROVER_DATA_BUILDS.load(std::sync::atomic::Ordering::SeqCst) } + +/// The `ProverData` for one proving shape, from the process-wide cache or built (see +/// [`PROVER_DATA_CACHE_CAPACITY`]). +fn cached_prover_data(profile: FriProfile, tier: Tier, keccak: u8, sha256: u8, modq: u8, airs: &[Chip], degrees: Vec) -> Arc> { + let build = |degrees: &[usize]| { + PROVER_DATA_BUILDS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Arc::new(ProverData::from_airs_and_degrees(&key_config(profile), airs, degrees)) + }; + if tier.0 > PROVER_DATA_CACHE_MAX_TIER { + return build(°rees); + } + let key: ProverDataKey = (profile, tier.0, keccak, sha256, modq, degrees); + let slot = { + let mut cache = PROVER_DATA_CACHE.get_or_init(Default::default).lock().unwrap_or_else(|e| e.into_inner()); + cache.tick += 1; + let now = cache.tick; + match cache.map.get_mut(&key) { + Some((slot, used)) => { + *used = now; + slot.clone() + } + None => { + while cache.map.len() >= PROVER_DATA_CACHE_CAPACITY { + let Some(lru) = cache.map.iter().min_by_key(|(_, (_, used))| *used).map(|(k, _)| k.clone()) else { break }; + cache.map.remove(&lru); + } + let slot = ProverDataSlot::default(); + cache.map.insert(key.clone(), (slot.clone(), now)); + slot + } + } + }; + // Built outside the cache lock: a prover of another shape is never held up by this build. + slot.get_or_init(|| build(&key.5)).clone() +} + pub struct Machine { pub config: Config, pub profile: FriProfile, keys: Mutex, key_builds: std::sync::atomic::AtomicUsize, build_gate: std::sync::OnceLock } /// A wrapper every verifier-key build of a machine runs inside ([`Machine::set_key_build_gate`]): @@ -2096,8 +2182,11 @@ impl Machine { // `prover_data` only need to be *structurally* compatible (same hash/compress/Dft/FRI // parameters, which `key_config` and `make_config` share via `build_config`); their // RNG state can differ freely. - let key_cfg = key_config(self.profile); - let prover_data = ProverData::from_airs_and_degrees(&key_cfg, &airs, &self.log_ext_degrees_with_modq(tier, traces.program_log_height, traces.input_log_height, traces.keccak_log_height, traces.sha256_log_height, traces.modq_log_height, traces.public_log_height, traces.mem_log_height)); + // + // Fullnode #151: that build is a pure function of the shape, so it comes from the + // process-wide cache (`cached_prover_data`) instead of being rebuilt for every proof. + let degrees = self.log_ext_degrees_with_modq(tier, traces.program_log_height, traces.input_log_height, traces.keccak_log_height, traces.sha256_log_height, traces.modq_log_height, traces.public_log_height, traces.mem_log_height); + let prover_data = cached_prover_data(self.profile, tier, traces.keccak_log_height, traces.sha256_log_height, traces.modq_log_height, &airs, degrees); let batch = prove_batch(&self.config, &instances, &prover_data); Proof { tier, program_log_height: traces.program_log_height, input_log_height: traces.input_log_height, keccak_log_height: traces.keccak_log_height, sha256_log_height: traces.sha256_log_height, modq_log_height: traces.modq_log_height, public_log_height: traces.public_log_height, mem_log_height: traces.mem_log_height, public_values: traces.public_values.iter().map(|x| x.as_canonical_u64()).collect(), batch } } diff --git a/crates/randprotocol-zkvm/tests/counter_rng.rs b/crates/randprotocol-zkvm/tests/counter_rng.rs new file mode 100644 index 00000000..1e0154fd --- /dev/null +++ b/crates/randprotocol-zkvm/tests/counter_rng.rs @@ -0,0 +1,176 @@ +//! Fullnode #151 (v0.8.3): the hiding PCS's random columns and the hiding MMCS's Merkle salts are +//! counter-mode ChaCha20 streams (`p3_merkle_tree::counter_rng`), one per (lane, chunk of rows), +//! keyed by a 256-bit secret the configuration's generator draws once per call — generated in +//! parallel instead of serially behind a lock. +//! +//! What this file pins: +//! - the vendored module: chunk `c` is exactly stream `(lane, c)`, in order, so the thread count +//! cannot change a value; values are reproducible per key and lane and differ across them; no +//! salt repeats across chunks or lanes; the values spread evenly over the field; +//! - which generators hand out keys: OS entropy and a seed-keyed sponge do, a verifier key's label +//! stream does not (and is not advanced by being asked), so no verifier key moves; +//! - a whole proof under seeded generators, fixed blinds and a fixed `H_IN` salt is byte-for-byte +//! reproducible, another seed gives other bytes, and both verify; +//! - a proof from this prover is accepted by the verifier path v0.8.0 nodes run (`ZkExecutor`'s +//! `non_canonical_proof` / `auth_header` / `verify_auth`, the same code: #151 touches no +//! verifier function), and its wire encoding round-trips canonically. + +use p3_field::{PrimeCharacteristicRing, PrimeField64}; +use p3_merkle_tree::counter_rng::{self, random_matrix, CounterModeRng, CHUNK_ROWS}; +use rand::{RngExt, SeedableRng}; +use randprotocol_core::confidential::ConfidentialExecutor; +use randprotocol_zkvm::emulator::execute; +use randprotocol_zkvm::executor::{decode_canonical, prove_auth, ZkExecutor}; +use randprotocol_zkvm::key_derivation_v2::{self as v2, KeyRngV2, SaltRng}; +use randprotocol_zkvm::machine::{build_traces_salted_with, config_with_rngs, Backend, FriProfile, Machine, ProveOptions, Tier, Val, TIERS}; +use randprotocol_zkvm::notes::SpendKey; +use randprotocol_zkvm::tables::blind::Blind; +use randprotocol_zkvm::guests; +use std::collections::BTreeSet; + +const KEY: [u8; 32] = [7; 32]; + +/// The chunking is a fixed function of the height: the parallel fill equals the chunks drawn one +/// after another, each from its own stream. +#[test] +fn the_values_are_the_chunk_streams_in_order() { + let (h, w) = (3 * CHUNK_ROWS + 17, 5); + let m = random_matrix::(h, w, &KEY, 9); + let mut want = Vec::new(); + for c in 0..h.div_ceil(CHUNK_ROWS) { + let rows = CHUNK_ROWS.min(h - c * CHUNK_ROWS); + let mut rng = counter_rng::stream(&KEY, 9, c); + want.extend((0..rows * w).map(|_| rng.random::())); + } + assert_eq!(m.values, want); + let mut filled = vec![Val::ZERO; h * w]; + counter_rng::fill(&mut filled, w, &KEY, 9); + assert_eq!(filled, want, "fill and random_matrix agree"); +} + +/// Same key, lane and shape: the same values. Another lane or another key: other values. +#[test] +fn it_is_deterministic_per_key_and_lane() { + let a = random_matrix::(2 * CHUNK_ROWS, 4, &KEY, 0); + assert_eq!(a, random_matrix::(2 * CHUNK_ROWS, 4, &KEY, 0)); + assert_ne!(a, random_matrix::(2 * CHUNK_ROWS, 4, &KEY, 1)); + assert_ne!(a, random_matrix::(2 * CHUNK_ROWS, 4, &[8; 32], 0)); +} + +/// No salt repeats across 64 chunks of two lanes (overlapping streams would repeat whole runs; +/// for independent uniform Goldilocks⁴ salts the chance of any repeat here is below 2^-200). +#[test] +fn no_salt_repeats_across_chunks_or_lanes() { + let h = 32 * CHUNK_ROWS; + let mut seen = BTreeSet::new(); + for lane in 0..2 { + let m = random_matrix::(h, 4, &KEY, lane); + for row in m.values.as_chunks::<4>().0 { + let salt: [u64; 4] = row.map(|x| x.as_canonical_u64()); + assert!(seen.insert(salt), "a salt repeated"); + } + } + assert_eq!(seen.len(), 2 * h); + // And no single element repeats either (2^17 draws from a 2^64 field: a repeat is ~2^-30). + let all: BTreeSet = seen.iter().flatten().copied().collect(); + assert_eq!(all.len(), 2 * h * 4); +} + +/// Statistical sanity: each of 16 equal buckets of `[0, p)` holds 1/16 of 2^16 samples within five +/// standard deviations, and the top bit is set about half the time. +#[test] +fn the_values_are_spread_over_the_field() { + let m = random_matrix::(1 << 14, 4, &KEY, 3); + let n = m.values.len() as f64; + let mut buckets = [0usize; 16]; + for x in &m.values { + buckets[((x.as_canonical_u64() as u128 * 16) / Val::ORDER_U64 as u128) as usize] += 1; + } + let (mean, sd) = (n / 16.0, (n * (1.0 / 16.0) * (15.0 / 16.0)).sqrt()); + for b in buckets { + assert!((b as f64 - mean).abs() < 5.0 * sd, "bucket {b} vs mean {mean}"); + } + let high = m.values.iter().filter(|x| x.as_canonical_u64() >> 63 == 1).count() as f64; + assert!((high - n / 2.0).abs() < 5.0 * (n / 4.0).sqrt()); +} + +/// OS entropy and a seed-keyed sponge hand out fresh keys; a label stream hands out none and is +/// not advanced by the question — so a verifier key's salts are the serial stream, unchanged. +#[test] +fn only_secret_generators_hand_out_counter_mode_keys() { + let mut fresh = SaltRng::fresh(); + let (a, b) = (fresh.counter_mode_key().unwrap(), fresh.counter_mode_key().unwrap()); + assert_ne!(a, b); + let mut seeded = SaltRng::from_seed([3; 32]); + let k = seeded.counter_mode_key().unwrap(); + assert_eq!(Some(k), SaltRng::from_seed([3; 32]).counter_mode_key(), "seeded: reproducible"); + assert_ne!(Some(k), seeded.counter_mode_key()); + + let mut label = KeyRngV2::from_label(v2::MMCS_LABEL); + assert_eq!(label.counter_mode_key(), None); + let mut untouched = KeyRngV2::from_label(v2::MMCS_LABEL); + assert_eq!(label.random::(), untouched.random::(), "asking does not advance the stream"); + assert_eq!(SaltRng::key(v2::MMCS_LABEL).counter_mode_key(), None); +} + +/// Fixed, balanced LogUp blinds (`tables::blind::fresh`'s shape, from constants). +fn fixed_blinds(n: usize) -> Vec { + let r: Vec<[Val; 2]> = (0..n).map(|i| [Val::from_u64(1 + i as u64), Val::from_u64(1000 + i as u64)]).collect(); + (0..n) + .map(|i| { + let prev = r[(i + n - 1) % n]; + [r[i][0], r[i][1], prev[0], prev[1]] + }) + .collect() +} + +/// A proof under seeded generators: the main/permutation/quotient random columns, the +/// randomisation polynomial and every Merkle salt are counter-mode streams from those seeds. +fn seeded_proof(seed: u8) -> (Machine, randprotocol_zkvm::isa::Program, randprotocol_zkvm::machine::Proof) { + let mut m = Machine::new(FriProfile::Test); + m.config = config_with_rngs( + FriProfile::Test, + SaltRng::Fresh(rand::rngs::StdRng::seed_from_u64(u64::from(seed))), + SaltRng::from_seed([seed; 32]), + ); + let p = guests::balance_check(1000); + let inputs = [400, 250, 300, 75]; + let exec = execute(&p, &inputs, &[], Tier(*TIERS.last().unwrap()).max_cycles()).unwrap(); + let tier = Tier(TIERS[0]); + let traces = build_traces_salted_with(&p, &inputs, &[], [1, 2, 3, 4], &exec, tier, ProveOptions::default()).unwrap(); + let blinds = fixed_blinds(traces.as_slice().len()); + let proof = m.prove_traces_with_blinds(&p, &traces, tier, &blinds); + (m, p, proof) +} + +/// A seeded run stays deterministic: same seeds, same bytes (whatever the thread count — the +/// chunking is fixed); another seed, other bytes; both verify. +#[test] +fn a_seeded_proof_is_reproducible_and_verifies() { + let (m, p, a) = seeded_proof(1); + let (_, _, b) = seeded_proof(1); + let (_, _, c) = seeded_proof(2); + assert_eq!(a.to_bytes(), b.to_bytes(), "same seeds: the same proof"); + assert_ne!(a.to_bytes(), c.to_bytes(), "another seed: other randomness"); + m.verify(&p.digest(), &a).unwrap(); + m.verify(&p.digest(), &c).unwrap(); +} + +/// A proof from this prover is accepted by the verifier path every v0.8.0 node runs at admission +/// (the same functions, unchanged by #151): the canonical-proof rule, the auth header's structural +/// checks, and the STARK verify against the binding — and its encoding is canonical. +#[test] +fn a_new_proof_passes_the_unchanged_v080_verifier_path() { + const BINDING: [u32; 8] = [0x1111_1111, 2, 3, 4, 5, 6, 7, 0xffff_ffff]; + let (proof, c, tier) = prove_auth(FriProfile::Test, &SpendKey([7; 8]), &[9; 8], &BINDING, Backend::Cpu).unwrap(); + assert_eq!(tier, 10); + assert_eq!(decode_canonical(&proof).unwrap().to_bytes(), proof, "wire format: canonical round trip"); + let ex = ZkExecutor::new(FriProfile::Test); + assert_eq!(ex.non_canonical_proof(&proof), None); + let (_shape, published, _gas) = ex.auth_header(&proof).unwrap(); + assert_eq!(published, c); + assert_eq!(ex.verify_auth(&ZkExecutor::hc_auth(), &proof, &BINDING), Ok(c)); + let mut other = BINDING; + other[0] ^= 1; + assert!(ex.verify_auth(&ZkExecutor::hc_auth(), &proof, &other).is_err(), "and still bound to its binding"); +} diff --git a/crates/randprotocol-zkvm/tests/decode_memo.rs b/crates/randprotocol-zkvm/tests/decode_memo.rs new file mode 100644 index 00000000..00495a01 --- /dev/null +++ b/crates/randprotocol-zkvm/tests/decode_memo.rs @@ -0,0 +1,50 @@ +//! Fullnode #152 (v0.8.3): one admission decodes each proof once. The ledger's checks — the +//! canonical-proof rule, `check_auth_fields`' header, the digest and gas reads, the verify — each +//! hand the executor the same bytes; `executor::decode_canonical_shared` decodes the first time and +//! answers the rest from a memo keyed by the exact bytes. +//! +//! Its own test binary: the decode counter is process-wide. + +use randprotocol_core::confidential::ConfidentialExecutor; +use randprotocol_zkvm::executor::{clear_decode_memo, decode_canonical, decode_canonical_shared, prove_auth, shared_decodes, ZkExecutor}; +use randprotocol_zkvm::machine::{Backend, FriProfile}; +use randprotocol_zkvm::notes::SpendKey; + +const BINDING: [u32; 8] = [0x1111_1111, 2, 3, 4, 5, 6, 7, 0xffff_ffff]; + +#[test] +fn one_admission_decodes_each_proof_once_with_the_same_verdicts() { + let (proof, c, _) = prove_auth(FriProfile::Test, &SpendKey([7; 8]), &[9; 8], &BINDING, Backend::Cpu).unwrap(); + let ex = ZkExecutor::new(FriProfile::Test); + clear_decode_memo(); + let before = shared_decodes(); + + // The admission sequence `Ledger::validate` runs on an auth proof. + assert_eq!(ex.non_canonical_proof(&proof), None); + let (_shape, published, gas) = ex.auth_header(&proof).unwrap(); + assert_eq!(published, c); + assert_eq!(ex.auth_proof_digest(&proof), Ok(c)); + assert_eq!(ex.auth_gas_limit(&proof), Ok(gas)); + assert_eq!(ex.verify_auth(&ZkExecutor::hc_auth(), &proof, &BINDING), Ok(c)); + assert_eq!(shared_decodes(), before + 1, "five reads of one proof, one decode"); + + // The memoised decode is the decode. + assert_eq!(decode_canonical_shared(&proof).unwrap().to_bytes(), decode_canonical(&proof).unwrap().to_bytes()); + + // Other bytes are other entries — never a hit on a prefix or a near miss — and a refusal is + // the refusal it always was (and is not memoised). + let mut padded = proof.clone(); + padded.push(0); + assert!(decode_canonical_shared(&padded).is_err()); + assert!(decode_canonical_shared(&padded).is_err()); + assert_eq!(ex.non_canonical_proof(&padded), None, "undecodable: the ledger's MalformedProof, not a canonical-rule name"); + assert!(ex.verify_auth(&ZkExecutor::hc_auth(), &padded, &BINDING).is_err()); + let mut flipped = proof.clone(); + let last = flipped.len() - 1; + flipped[last] ^= 1; + assert!(ex.verify_auth(&ZkExecutor::hc_auth(), &flipped, &BINDING).is_err(), "a one-byte change is not a hit"); + // And the honest proof is still answered from the memo after all that. + let n = shared_decodes(); + assert_eq!(ex.verify_auth(&ZkExecutor::hc_auth(), &proof, &BINDING), Ok(c)); + assert_eq!(shared_decodes(), n); +} diff --git a/crates/randprotocol-zkvm/tests/prover_data_cache.rs b/crates/randprotocol-zkvm/tests/prover_data_cache.rs new file mode 100644 index 00000000..b5f094f3 --- /dev/null +++ b/crates/randprotocol-zkvm/tests/prover_data_cache.rs @@ -0,0 +1,37 @@ +//! Fullnode #151 (v0.8.3): `prove_traces` takes the preprocessed `ProverData` from a process-wide +//! cache keyed by the proving shape, instead of rebuilding it (LDE + Merkle tree) for every proof. +//! +//! Its own test binary, so the build counter (`machine::prover_data_builds`) sees no other test's +//! proofs. + +use randprotocol_zkvm::guests; +use randprotocol_zkvm::machine::{prover_data_builds, FriProfile, Machine, PROVER_DATA_CACHE_MAX_TIER}; + +/// Two proofs of one shape — from two different `Machine`s, as the wallet and `rand-prover` make +/// one per proof — build the preprocessed data once, and both proofs verify on a third machine that +/// recomputes the verifier key from scratch (so the cached commitment is the verifier key's: the +/// build is a pure function of the shape). +#[test] +fn proofs_of_one_shape_share_one_preprocessed_build_and_verify() { + assert!(PROVER_DATA_CACHE_MAX_TIER >= 14, "the bundle guest's tier is cached"); + let p = guests::balance_check(1000); + let inputs = [400, 250, 300, 75]; + let before = prover_data_builds(); + let (a, _) = Machine::new(FriProfile::Test).prove(&p, &inputs, &[], None).unwrap(); + assert_eq!(prover_data_builds(), before + 1, "the first proof builds"); + let (b, _) = Machine::new(FriProfile::Test).prove(&p, &inputs, &[], None).unwrap(); + assert_eq!(prover_data_builds(), before + 1, "the second proof of the shape is a cache hit"); + assert_ne!(a.to_bytes(), b.to_bytes(), "the proofs' own randomness is still fresh"); + + // The cached preprocessed commitment is exactly the one a verifier recomputes: the prover's + // transcript observes the cached commitment and the verifier's observes the one it rebuilds, + // so a difference would change every challenge and fail the verify. + let v = Machine::new(FriProfile::Test); + v.verify(&p.digest(), &a).unwrap(); + v.verify(&p.digest(), &b).unwrap(); + + // Another profile is another shape: its own build. + let (c, _) = Machine::new(FriProfile::Production).prove(&p, &inputs, &[], None).unwrap(); + assert_eq!(prover_data_builds(), before + 2); + Machine::new(FriProfile::Production).verify(&p.digest(), &c).unwrap(); +} diff --git a/deploy/sync-zkvm.sh b/deploy/sync-zkvm.sh index 4f75e471..01d7d2bd 100755 --- a/deploy/sync-zkvm.sh +++ b/deploy/sync-zkvm.sh @@ -257,6 +257,7 @@ rsync -a --delete --exclude executor.rs --exclude shielded.rs --exclude call_env --exclude viewing.rs --exclude bundle.rs --exclude hidden_bundle.rs \ --exclude hidden_cheating.rs --exclude hidden_trace_forgery.rs --exclude guest_provenance.rs --exclude 'auth*.rs' --exclude evm_rt.rs --exclude sbpf_rt.rs \ --exclude perps_vectors.rs --exclude modq_cheating.rs \ + --exclude counter_rng.rs --exclude prover_data_cache.rs --exclude decode_memo.rs \ "$SRC/tests/" "$DST/tests/" [ -f "$DST/src/guests.rs" ] || cp "$SRC/src/guests.rs" "$DST/src/guests.rs" # M4.1/M4.2: vendor the compiled guest binaries the vendored `tests/e2e.rs` and the local @@ -536,6 +537,17 @@ PY grep -q 'pub fn export_cached_verifier_key_with_modq' "$DST/src/machine.rs" \ && echo "anchors: export_cached ok" \ || { echo "sync-zkvm.sh: install_verifier_key moved; update the export_cached patch" >&2; exit 1; } +# Fullnode #151 (v0.8.3): the prover's random columns and Merkle salts are counter-mode streams +# (`vendor/p3-merkle-tree/src/counter_rng.rs`), which `key_derivation_v2::{KeyRngV2, SaltRng}` opt +# into (`impl CounterModeRng`), and `prove_traces` takes its `ProverData` from a process-wide cache +# (`machine::cached_prover_data`). Both were made on this side first and are not yet in circuits' +# `research/`; a sync from a circuits commit without them would silently put the serial RNG and +# the per-proof preprocessed rebuild back (and not compile: `HidingFriPcs` needs `SaltRng: +# CounterModeRng`). So the sync stops until circuits carries them. `tests/counter_rng.rs` and +# `tests/prover_data_cache.rs` are this side's and are kept by the rsync above. +grep -q 'impl CounterModeRng for SaltRng' "$DST/src/key_derivation_v2.rs" && grep -q 'fn cached_prover_data' "$DST/src/machine.rs" \ + && echo "anchors: #151 counter-mode RNG + ProverData cache ok" \ + || { echo "sync-zkvm.sh: circuits' research/ lacks fullnode #151 (key_derivation_v2's CounterModeRng impls, machine.rs's cached_prover_data); port them upstream first" >&2; exit 1; } # M3.4/M4.1: `hash.rs`'s program-digest header and `tables/cpu.rs`'s in-circuit copy of the same # constant both read `crate::notes::domain::HC` upstream; M4.1 added a second such pair for the # input digest, `crate::notes::domain::IN`. Constraint set 6 added a third: `crate::notes:: diff --git a/docs/block-space.md b/docs/block-space.md index 939666f5..56141496 100644 --- a/docs/block-space.md +++ b/docs/block-space.md @@ -17,6 +17,20 @@ Measured on the research crate (`docs/03-privacy.md` there), `fib` guest, one ma | verify, uncached, tier 12 | 809 ms | 838 ms | | verify, warm key | ~16 ms | ~16 ms | +Those two verify rows are the research crate's tier-12 `fib` proof. A chain-21 transfer carries a +tier-14 bundle proof and a tier-10 auth proof (2.87–2.90 MB together), and admitting one costs more +(2026-10-28, Apple M4 Max, n = 38 chain-21 transactions, the node's own executor without its verdict +cache; fullnode issue #152): + +| admission of one chain-21 transfer | v0.8.0 – v0.8.2 | v0.8.3 | +|---|---|---| +| verify, warm keys (header checks + bundle STARK + auth STARK), p50 | 73 ms (72.2 re-measured) | 62 ms | +| verify, cold (first after start: both keys built), p50 | 1.6 s | 1.56 s | +| decode of the `rand_sendTransaction` body (hex + bincode), p50 | 18.5 ms | 1.2 ms | + +v0.8.3 decodes each proof once per admission instead of four times and decodes the hex body with +`faster-hex`; the two STARK verifies themselves (~31 ms and ~27 ms) are unchanged. + A shielded bundle (`bundle` guest, tier 14) is not yet measured at 80 queries; each tier above 12 adds two FRI layers, so expect **about 1.3 MB**. Everything else in a transfer is small: two nullifiers and two commitments (128 bytes), two envelopes (ML-KEM-768 ciphertext 1 088 bytes plus @@ -68,8 +82,9 @@ works the arithmetic). Fees are flat floors (`docs/fees.md`): 0.001 RAND per bundle regardless of size, because a validator's cost is one nearly constant STARK verification. That was true at 27 queries and -remains true at 80: verification moved from 809 ms to 838 ms uncached, 16 ms warm. The floor -prices **verification**, not **bytes**. +remains true at 80: verification moved from 809 ms to 838 ms uncached, 16 ms warm (the tier-12 +`fib` figures; a chain-21 transfer's two proofs take 62 ms warm and 1.56 s cold as of v0.8.3, §1). +The floor prices **verification**, not **bytes**. Bytes are the scarce thing now. Three slots per block with a flat 0.001 RAND floor and a first-come mempool means demand above 1.5 transfers per second is rationed by arrival order, diff --git a/docs/compute-optimization.md b/docs/compute-optimization.md index 61700127..064b22df 100644 --- a/docs/compute-optimization.md +++ b/docs/compute-optimization.md @@ -30,8 +30,8 @@ prices block space and proving as two separate goods. | block caps | 20 MiB per block, 4 MiB per proof, 2 000 transactions | `docs/node-hardware.md` §0 | | **transfers per block** | **7** (20 MiB / 2.85 MB); 3 on the 4 MiB chains | derived | | **throughput** | **~6 tx/s** | derived | -| bundle verify, validator | 838 ms cold, ~16 ms warm; auth proof similar | `docs/node-hardware.md` §2.1 | -| validator work per block | 14 warm verifications ≈ 0.25 s of one core | derived | +| transfer verify, validator (bundle + auth proof) | 62 ms warm, 1.56 s cold (v0.8.3; 73 ms / 1.6 s before), M4 Max, n = 38 chain-21 transactions; the earlier 838 ms / 16 ms was a tier-12 research proof | `docs/node-hardware.md` §2.1, fullnode #152 | +| validator work per block | 7 warm transfer verifications ≈ 0.43 s of one core (≈ 0.51 s before v0.8.3) | derived | | validator RAM | ~650 MiB resident | `docs/node-hardware.md` §2.1 | | mempool admission | 4 blocking verification workers, queue 64 deep | `docs/architecture.md` §8 | | wallet proof, tier 14 | 107.5 s on 1 thread, 13.9 s on 16 (M4 Max); 6.2 GB | `docs/node-hardware.md` §6 | @@ -43,7 +43,7 @@ prices block space and proving as two separate goods. ### 1.2 What bounds throughput, in order -1. **Bytes, not compute.** A validator verifies a transaction in 16 ms but stores and gossips +1. **Bytes, not compute.** A validator verifies a transaction in 62 ms but stores and gossips 2.85 MB of it. The block cap is what limits the chain to seven transfers, and the cap cannot simply rise: at 100 transfers a block every validator would ingest 285 MB every 1.2 s. 2. **Recursion is not yet buyable.** Aggregation (`docs/aggregation.md`) is the designed remedy: @@ -71,7 +71,7 @@ guarantees neither lane can starve the other. ────────────────────── ────────────────────────────────── wallet ──bundle+proofs──▶ any node ──gossip──▶ leader wallet ──bundle+proofs──▶ aggregator ingress │ │ - │ verify each proof (CPU, 16 ms); dedupe + │ verify each proof (CPU, 62 ms); dedupe │ nullifiers; batch B bundles → leaf proof; │ 2-to-1 tree → one aggregate (GPU) │ │ @@ -378,8 +378,8 @@ whole chain is. The root aggregate stays a hash-based STARK. An aggregator runs the bundle mempool the validators run today, with the same cheap-first admission order (`docs/architecture.md` §8), the same nullifier and commitment conflict checks, -and CPU verification of every inner proof — 16 ms warm, so a 32-core host admits ~2 000 bundles a -second, more than any one operator will prove. A wallet submits with `rand_sendTransaction` to an +and CPU verification of every inner proof — 62 ms warm for a transfer's two proofs, so a 32-core +host admits ~500 transfers a second, more than any one operator will prove. A wallet submits with `rand_sendTransaction` to an aggregator's RPC, or to any node, which forwards to the `rand-bundles` gossip topic that aggregators subscribe to and validators do not. The wallet learns which lane it is in from `rand_getLimits` (§6.2) and the receipt names the covering aggregate. diff --git a/docs/node-hardware.md b/docs/node-hardware.md index 438010f2..58d5ffcd 100644 --- a/docs/node-hardware.md +++ b/docs/node-hardware.md @@ -54,7 +54,7 @@ on an aggregation chain, in aggregators. | disk, pruning validator | block tables 3.3–4.4 GB with one day kept (≤ 65.5k blocks), where they were 19 GB unpruned: chain 14, 18 validators, 2026-09-25. Not re-measured with chain 18's 26-signature certificates | `AGENTS.md`, "v0.5.7" | | disk, idle growth, archive | about 60 KB a block (the certificate inside the header), ≈ 3.6 GB/day at 1.4 s blocks: chain 14, 18 validators. Not re-measured on chain 18 | `AGENTS.md`, "v0.5.5" | | startup | a restart of the same build: seconds (one ledger load and the head's state root; the startup checkpoint). The first start of a new build: the full check, which took 239 s at 59 183 blocks (chain 12, unpruned) and up to 30 min at 259 k blocks on 48 GB droplets. A pruned node's check is structural, not a replay | `deploy/README.md` (v0.3 update, step 2); `AGENTS.md`, "v0.5.7" | -| bundle verify | 838 ms cold, about 16 ms warm | `docs/superpowers/specs/2026-09-15-block-aggregation.md` §1 | +| transfer verify (bundle + auth proof, admission) | v0.8.3: **62 ms warm, 1.56 s cold** (p50; the cold one builds both verifier keys). v0.8.0–v0.8.2: 73 ms warm, 1.6 s cold. Warm stages: bundle STARK ~31 ms, auth STARK ~27 ms, header/digest/canonical checks ~4 ms (~10 ms before v0.8.3, when each check re-decoded the 1.4–1.5 MB proofs). Plus the `rand_sendTransaction` body decode: 1.2 ms (18.5 ms before v0.8.3). Apple M4 Max, n = 38 chain-21 transactions (2.87–2.90 MB), 2026-10-28. The earlier "838 ms cold, about 16 ms warm" was a tier-12 research proof | fullnode issue #152; `nettest/data/verify_bench.jsonl` | | call verify | about 233 ms first (uncached) verify at tier 10, production profile (constraint set 5) | `docs/confidential.md` | | network | TCP 30303 inbound on public nodes | `docs/deploy.md` | | block apply, 2 000 bundles, 80 M nullifiers | 56.1 ms at block 10 000 (incremental nullifier root on; the replica's apply on a clone of the tip) | this page §7 | diff --git a/docs/rpc.md b/docs/rpc.md index 82d335e3..0f8ea8d2 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -170,6 +170,14 @@ the normal mempool, so the mint goes through consensus and every node applies it no validator key and answers `faucet mints are signed by validators; ask a validator node`. Poll `rand_getTransaction` for the commit. +Only a validator whose key is one of the chain's faucet minters can mint: the genesis +`staking.faucet_minters` where it lists them, otherwise the genesis validators. Any other validator +— one bonded after genesis, such as chain 21's guardian hosts — refuses up front with +`this node's key
is not a faucet minter on this chain; the minters are ` +(v0.8.3, issue #153; before, it built the mint and the pool refused it). `faucet` in `rand_status` +and `rand_getEmission` is `true` exactly when this node would serve `rand_mint`: the faucet is on, +the node runs as a validator, and its key is a faucet minter. + On a chain whose genesis sets `staking.faucet_recipients` (chain 15) only the listed wallets can be paid: a mint to any other address is refused at admission with `the faucet may not mint to this recipient (not in staking.faucet_recipients)` (`docs/staking.md` §2). @@ -1983,6 +1991,15 @@ the proof's published digest against the one it computed before it submits anyth What changed for clients, in one place. Newest first. +### v0.8.3 — `faucet` means "this node can mint here" (node-only; issue #153) + +- `rand_status.faucet` and `rand_getEmission.faucet` are `true` only when the genesis turns the + faucet on, the node runs as a validator **and** its key is one of the chain's faucet minters. + Before, they echoed the genesis flag, so a validator bonded after genesis advertised a faucet + whose every mint failed at admission. +- `rand_mint` on such a validator refuses before building anything, naming the minter set + (`this node's key … is not a faucet minter on this chain; the minters are …`). + ### v0.8.2 — perps fields in `rand_getPerps` and `rand_getSupply` (node-only on chain 21; the rules they report are genesis-gated) - `rand_getPerps`: `bound_payouts`, `initial_collateral`, `collateral_in`, `collateral_out`, diff --git a/vendor/p3-fri/Cargo.toml b/vendor/p3-fri/Cargo.toml index f073690a..6e98e880 100644 --- a/vendor/p3-fri/Cargo.toml +++ b/vendor/p3-fri/Cargo.toml @@ -71,6 +71,10 @@ version = "0.7.0" [dependencies.p3-matrix] version = "0.7.0" +# RandProtocol patch (#151): `p3_merkle_tree::counter_rng`, shared with the hiding MMCS. +[dependencies.p3-merkle-tree] +version = "0.7.0" + [dependencies.p3-maybe-rayon] version = "0.7.0" @@ -82,7 +86,8 @@ version = "0.7.0" [dependencies.rand] version = "0.10.1" -features = ["std_rng"] +# RandProtocol patch (#151): `chacha` for `counter_rng`'s ChaCha20 streams. +features = ["std_rng", "chacha"] default-features = false [dependencies.serde] diff --git a/vendor/p3-fri/src/hiding_pcs.rs b/vendor/p3-fri/src/hiding_pcs.rs index a1d9d004..13346c4f 100644 --- a/vendor/p3-fri/src/hiding_pcs.rs +++ b/vendor/p3-fri/src/hiding_pcs.rs @@ -5,12 +5,15 @@ use p3_challenger::{CanObserve, FieldChallenger, GrindingChallenger}; use p3_commit::{Mmcs, OpenedValues, Pcs, PolynomialSpace}; use p3_dft::TwoAdicSubgroupDft; use p3_field::coset::TwoAdicMultiplicativeCoset; -use p3_field::{ExtensionField, TwoAdicField, batch_multiplicative_inverse}; +use p3_field::{ExtensionField, Field, TwoAdicField, batch_multiplicative_inverse}; use p3_matrix::Matrix; use p3_matrix::bitrev::{BitReversalPerm, BitReversibleMatrix}; use p3_matrix::dense::{DenseMatrix, RowMajorMatrix, RowMajorMatrixCow}; use p3_matrix::horizontally_truncated::HorizontallyTruncated; use p3_matrix::row_index_mapped::RowIndexMappedView; +use p3_maybe_rayon::prelude::*; +use p3_merkle_tree::CounterModeRng; +use p3_merkle_tree::counter_rng::{self, CHUNK_ROWS}; use rand::distr::{Distribution, StandardUniform}; use rand::{CryptoRng, RngExt, SeedableRng}; use spin::Mutex; @@ -46,6 +49,53 @@ where } } +impl HidingFriPcs +where + R: CounterModeRng, +{ + /// RandProtocol patch (fullnode #151, v0.8.3): one call's 256-bit counter-mode key + /// (`p3_merkle_tree::counter_rng`), drawn under the lock and used outside it; `None` keeps the + /// serial path (a generator whose own stream must not move). + fn counter_key(&self) -> Option<[u8; 32]> { + self.rng.lock().counter_mode_key() + } +} + +/// RandProtocol patch (fullnode #151, v0.8.3): `RowMajorMatrix::with_random_cols` with the new +/// columns drawn from counter-mode streams (`p3_merkle_tree::counter_rng`, lane `lane`) and the +/// rows copied and filled in parallel, a chunk of `CHUNK_ROWS` rows per stream. Same layout as +/// `with_random_cols`: each row is the old row followed by `num_cols` uniform elements. +fn with_random_cols_counter( + mat: &RowMajorMatrix, + num_cols: usize, + key: &[u8; 32], + lane: u32, +) -> RowMajorMatrix +where + StandardUniform: Distribution, +{ + let old_w = mat.width(); + let new_w = old_w + num_cols; + let h = mat.height(); + let mut values = F::zero_vec(new_w * h); + if new_w > 0 { + values + .par_chunks_mut(CHUNK_ROWS * new_w) + .enumerate() + .for_each(|(c, dst)| { + let mut rng = counter_rng::stream(key, lane, c); + let src = &mat.values[c * CHUNK_ROWS * old_w..]; + for (r, row) in dst.chunks_exact_mut(new_w).enumerate() { + row[..old_w].copy_from_slice(&src[r * old_w..(r + 1) * old_w]); + for v in &mut row[old_w..] { + *v = rng.random(); + } + } + }); + } + RowMajorMatrix::new(values, new_w) +} + impl Clone for HidingFriPcs where Val: Clone, @@ -91,7 +141,7 @@ where Challenge: TwoAdicField + ExtensionField, Challenger: FieldChallenger + CanObserve + GrindingChallenger, - R: CryptoRng + SeedableRng + Send + Sync, + R: CryptoRng + SeedableRng + CounterModeRng + Send + Sync, { type Domain = TwoAdicMultiplicativeCoset; type Commitment = InputMmcs::Commitment; @@ -122,11 +172,15 @@ where &self, evaluations: impl IntoIterator)>, ) -> (Self::Commitment, Self::ProverData) { + // RandProtocol patch (fullnode #151, v0.8.3): one counter-mode key per commit, matrix `m` + // on lane `m`, its random columns filled in parallel (`with_random_cols_counter`). + let key = self.counter_key(); let randomized_evaluations: Vec<(Self::Domain, RowMajorMatrix)> = info_span!("randomize polys").in_scope(|| { evaluations .into_iter() - .map(|(domain, mat)| { + .enumerate() + .map(|(m, (domain, mat))| { let mat_width = mat.width(); // Let `w` and `h` be the width and height of the original matrix. The randomized matrix should have height `2h` and width `w + num_random_codewords`. // To generate it, we add `w + 2 * num_random_codewords` columns to the original matrix, then reshape it by setting the width to `w + num_random_codewords`. @@ -137,11 +191,11 @@ where // stolen half would steal another instance's `commit` and spin on its own // lock for ever (seen on a prover host: three workers spinning for hours). // So the lock is held only to fork a child RNG, never across rayon work. - let mut child = self.fork_rng(); - let mut random_evaluation = mat.with_random_cols( - mat_width + 2 * self.num_random_codewords, - &mut child, - ); + let num_cols = mat_width + 2 * self.num_random_codewords; + let mut random_evaluation = match &key { + Some(key) => with_random_cols_counter(&mat, num_cols, key, lane(m)), + None => mat.with_random_cols(num_cols, &mut self.fork_rng()), + }; random_evaluation.width = mat_width + self.num_random_codewords; (domain, random_evaluation) @@ -204,20 +258,48 @@ where .map(|i| cis[i] * last_chunk_ci_inv) .collect_vec(); - let mut rng = self.fork_rng(); // RandProtocol patch: see `commit` - let randomized_evaluations: Vec> = evaluations - .into_iter() - .map(|mat| mat.with_random_cols(self.num_random_codewords, &mut rng)) - .collect(); + // RandProtocol patch (fullnode #151, v0.8.3): with a counter-mode key, chunk `i`'s random + // columns are lane `i` and the random polynomial `t_j` is lane `n + j` (`n` chunks), each + // filled in parallel. Without one, the serial draw below, unchanged. + let n = evaluations.len(); + let (randomized_evaluations, mut all_random_values): (Vec>, Vec) = + match self.counter_key() { + Some(key) => { + let randomized_evaluations: Vec> = evaluations + .iter() + .enumerate() + .map(|(i, mat)| with_random_cols_counter(mat, self.num_random_codewords, &key, lane(i))) + .collect(); + let h = randomized_evaluations[0].height(); + let w = randomized_evaluations[0].width(); + let mut all = Val::zero_vec(n * h * w); + if h * w > 0 { + for (j, block) in all.chunks_exact_mut(h * w).take(n - 1).enumerate() { + counter_rng::fill(block, w, &key, lane(n + j)); + } + } + (randomized_evaluations, all) + } + None => { + let mut rng = self.fork_rng(); // RandProtocol patch: see `commit` + let randomized_evaluations: Vec> = evaluations + .into_iter() + .map(|mat| mat.with_random_cols(self.num_random_codewords, &mut rng)) + .collect(); + let h = randomized_evaluations[0].height(); + let w = randomized_evaluations[0].width(); + let all = (0..(randomized_evaluations.len() - 1) * h * w) + .map(|_| rng.random()) + .chain(core::iter::repeat_n(Val::ZERO, h * w)) + .collect::>(); + (randomized_evaluations, all) + } + }; // Add random values to the LDE evaluations as described in https://eprint.iacr.org/2024/1037.pdf. // If we have `d` chunks, let q'_i(X) = q_i(X) + v_H_i(X) * t_i(X) where t(X) is random, for 1 <= i < d. // q'_d(X) = q_d(X) - v_H_d(X) c_i \sum t_i(X) where c_i is a Lagrange normalization constant. let h = randomized_evaluations[0].height(); let w = randomized_evaluations[0].width(); - let mut all_random_values = (0..(randomized_evaluations.len() - 1) * h * w) - .map(|_| rng.random()) - .chain(core::iter::repeat_n(Val::ZERO, h * w)) - .collect::>(); // Set the random values for the final chunk accordingly for j in 0..last_chunk { @@ -461,14 +543,21 @@ where &self, ext_trace_domains: impl IntoIterator, ) -> Option<(Self::Commitment, Self::ProverData)> { + // RandProtocol patch (fullnode #151, v0.8.3): domain `i` on counter-mode lane `i`. + let key = self.counter_key(); + let width = self.num_random_codewords + Challenge::DIMENSION; let random_input_vals = ext_trace_domains .into_iter() - .map(|domain| { - let m = DenseMatrix::rand( - &mut self.fork_rng(), // RandProtocol patch: see `commit` - domain.size(), - self.num_random_codewords + Challenge::DIMENSION, - ); + .enumerate() + .map(|(i, domain)| { + let m = match &key { + Some(key) => counter_rng::random_matrix(domain.size(), width, key, lane(i)), + None => DenseMatrix::rand( + &mut self.fork_rng(), // RandProtocol patch: see `commit` + domain.size(), + width, + ), + }; (domain, m) }) @@ -494,6 +583,11 @@ where } } +/// A matrix index as a counter-mode lane (`p3_merkle_tree::counter_rng`). +fn lane(i: usize) -> u32 { + u32::try_from(i).expect("fewer than 2^32 matrices in one call") +} + /// Compute the normalizing constants for the Langrange selectors of the provided domains. /// See Section 4.2 of for more details. fn get_zp_cis(qc_domains: &[D]) -> Vec> { diff --git a/vendor/p3-merkle-tree/Cargo.toml b/vendor/p3-merkle-tree/Cargo.toml index 3bb0c1c0..c77cc94f 100644 --- a/vendor/p3-merkle-tree/Cargo.toml +++ b/vendor/p3-merkle-tree/Cargo.toml @@ -73,7 +73,8 @@ version = "0.7.0" [dependencies.rand] version = "0.10.1" -features = ["std_rng"] +# RandProtocol patch (#151): `chacha` for `counter_rng`'s ChaCha20 streams. +features = ["std_rng", "chacha"] default-features = false [dependencies.serde] diff --git a/vendor/p3-merkle-tree/src/counter_rng.rs b/vendor/p3-merkle-tree/src/counter_rng.rs new file mode 100644 index 00000000..c623f385 --- /dev/null +++ b/vendor/p3-merkle-tree/src/counter_rng.rs @@ -0,0 +1,130 @@ +//! RandProtocol patch (fullnode #151, v0.8.3): counter-mode randomness for the hiding PCS and +//! the hiding MMCS, generated in parallel. +//! +//! Both hiding layers only need their random values to be **uniform over the field and secret**; +//! nothing in the protocol (prover or verifier) depends on them being drawn one after another +//! from a single sequential stream. Upstream Plonky3 nevertheless drew every random column and +//! every Merkle salt from one generator behind a lock, serially: on a tier-14 bundle proof that +//! was ~7.5 s of single-threaded work, 41% of the proof. +//! +//! The construction here: per call (one `commit`, one quotient randomisation, …) the caller's +//! generator `R` supplies one fresh 256-bit secret key ([`CounterModeRng::counter_mode_key`]). +//! The values are then ChaCha20 keyed by that secret, with the 64-bit ChaCha nonce ("stream") +//! set to `lane << 32 | chunk`: `lane` names the matrix (or block) within the call, `chunk` a run +//! of [`CHUNK_ROWS`] consecutive rows. Each chunk is filled by its own stream, so chunks are +//! filled in parallel; distinct `(lane, chunk)` pairs are distinct ChaCha nonces under one key, +//! so the streams never overlap (each nonce has 2^64 blocks of its own, far beyond any chunk). +//! +//! **Security argument.** ChaCha20 is a PRF in (key, nonce, block counter). Keyed by a uniformly +//! random secret key that the caller's cryptographic generator drew, its outputs over distinct +//! (nonce, counter) inputs are computationally indistinguishable from independent uniform words +//! — the same assumption the previous generators already rested on (`rand`'s `StdRng` is +//! ChaCha12; the Poseidon2 sponge of `KeyRngV2::from_seed` is a keyed permutation). Field +//! elements are sampled from those words exactly as before: `StandardUniform`'s rejection +//! sampling for the field (`p3-goldilocks`: draw a `u64`, keep it if it is below the modulus), +//! one stream per chunk, so every value is uniform over the field and independent of the others. +//! +//! **Determinism.** The chunking is a fixed function of the matrix height ([`CHUNK_ROWS`]), never +//! of the thread count, so a run seeded deterministically produces the same values on one thread +//! and on sixty-four. +//! +//! **What does not change.** The values are still committed exactly as before (same widths, same +//! layout, same salts per leaf), so the proof's wire format and the verifier are untouched: the +//! verifier never sees *how* a random value was drawn. A generator that returns `None` from +//! [`CounterModeRng::counter_mode_key`] keeps the old serial path bit for bit — the verifier-key +//! salt streams (`KeyRngV2::from_label`), which are public, consensus-pinned, and must not move. + +use alloc::vec::Vec; + +use p3_matrix::dense::RowMajorMatrix; +use p3_maybe_rayon::prelude::*; +use rand::distr::{Distribution, StandardUniform}; +use rand::rngs::{ChaCha20Rng, StdRng}; +use rand::{CryptoRng, Rng, RngExt, SeedableRng}; + +/// Rows per counter-mode chunk: the unit of parallel work and of stream assignment. Fixed, so the +/// values do not depend on the thread count. +pub const CHUNK_ROWS: usize = 1 << 10; + +/// A cryptographic generator that may hand out a fresh secret key for counter-mode generation. +pub trait CounterModeRng: CryptoRng { + /// A fresh 256-bit secret key drawn from this generator (advancing it), or `None` when this + /// generator's own serial stream must be used unchanged — a deterministic, public stream + /// whose exact output is pinned elsewhere (a verifier key's salts). + fn counter_mode_key(&mut self) -> Option<[u8; 32]>; +} + +/// Draws 32 bytes from `rng`: the key a secret-seeded generator hands out. +pub fn draw_key(rng: &mut R) -> [u8; 32] { + let mut key = [0u8; 32]; + rng.fill_bytes(&mut key); + key +} + +impl CounterModeRng for StdRng { + fn counter_mode_key(&mut self) -> Option<[u8; 32]> { + Some(draw_key(self)) + } +} + +impl CounterModeRng for ChaCha20Rng { + fn counter_mode_key(&mut self) -> Option<[u8; 32]> { + Some(draw_key(self)) + } +} + +/// The stream for `(lane, chunk)` under `key`: ChaCha20, nonce `lane << 32 | chunk`, from block 0. +pub fn stream(key: &[u8; 32], lane: u32, chunk: usize) -> ChaCha20Rng { + let chunk = u32::try_from(chunk).expect("fewer than 2^32 chunks of CHUNK_ROWS rows"); + let mut rng = ChaCha20Rng::from_seed(*key); + rng.set_stream((u64::from(lane) << 32) | u64::from(chunk)); + rng +} + +/// Fills `values` — row-major, `width` per row — with uniform elements, chunk `c` (rows +/// `c·CHUNK_ROWS ..`) from [`stream`]`(key, lane, c)`, chunks in parallel. +pub fn fill(values: &mut [T], width: usize, key: &[u8; 32], lane: u32) +where + StandardUniform: Distribution, +{ + if width == 0 || values.is_empty() { + return; + } + values + .par_chunks_mut(CHUNK_ROWS * width) + .enumerate() + .for_each(|(c, chunk)| { + let mut rng = stream(key, lane, c); + for v in chunk { + *v = rng.random(); + } + }); +} + +/// A `height × width` matrix of uniform elements — the counter-mode twin of +/// `RowMajorMatrix::rand` — chunk `c` from [`stream`]`(key, lane, c)`, chunks in parallel. +pub fn random_matrix( + height: usize, + width: usize, + key: &[u8; 32], + lane: u32, +) -> RowMajorMatrix +where + StandardUniform: Distribution, +{ + let values: Vec = if width == 0 { + Vec::new() + } else { + (0..height.div_ceil(CHUNK_ROWS)) + .into_par_iter() + .flat_map_iter(|c| { + let rows = CHUNK_ROWS.min(height - c * CHUNK_ROWS); + let mut rng = stream(key, lane, c); + (0..rows * width).map(move |_| rng.random()) + }) + .collect() + }; + RowMajorMatrix::new(values, width) +} + +// Tests: `randprotocol-zkvm/tests/counter_rng.rs` (this vendored copy carries no dev-dependencies). diff --git a/vendor/p3-merkle-tree/src/hiding_mmcs.rs b/vendor/p3-merkle-tree/src/hiding_mmcs.rs index d5034ca7..66b3d4f2 100644 --- a/vendor/p3-merkle-tree/src/hiding_mmcs.rs +++ b/vendor/p3-merkle-tree/src/hiding_mmcs.rs @@ -14,6 +14,7 @@ use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use spin::Mutex; +use crate::counter_rng::{CounterModeRng, random_matrix}; use crate::mmcs::check_widths; use crate::pruning::PrunedMerklePaths; use crate::{MerkleCap, MerkleTree, MerkleTreeError, MerkleTreeMmcs}; @@ -107,7 +108,7 @@ where C: PseudoCompressionFunction<[PW::Value; DIGEST_ELEMS], N> + PseudoCompressionFunction<[PW; DIGEST_ELEMS], N> + Sync, - R: CryptoRng + SeedableRng + Send, + R: CryptoRng + SeedableRng + CounterModeRng + Send, PW::Value: Eq + Clone, [PW::Value; DIGEST_ELEMS]: Serialize + for<'de> Deserialize<'de>, StandardUniform: Distribution, @@ -134,19 +135,36 @@ where &self, inputs: Vec, ) -> (Self::Commitment, Self::ProverData) { - // RandProtocol patch (2026-10-01, upstream Plonky3 #2363 / PR #2368): draw every salt under - // the lock, then release it before hashing. The tree build is parallel, and a waiting rayon - // worker may run a queued commit on this same MMCS, which would spin for ever on a lock its - // own thread still holds. - let salted_inputs = { - let mut rng = self.rng.lock(); - inputs + // RandProtocol patch (2026-10-01, upstream Plonky3 #2363 / PR #2368): the lock is never held + // across rayon work. The tree build is parallel, and a waiting rayon worker may run a + // queued commit on this same MMCS, which would spin for ever on a lock its own thread + // still holds. + // + // RandProtocol patch (fullnode #151, v0.8.3): the salts are counter-mode streams + // (`counter_rng`), generated in parallel outside the lock — the lock is taken only to draw + // this commit's 256-bit key. Salt matrix `m` is lane `m`. A generator that hands out no + // key (a verifier key's public label stream) keeps the serial draw, bit for bit. + let key = self.rng.lock().counter_mode_key(); + let salted_inputs = match key { + Some(key) => inputs .into_iter() - .map(|mat| { - let salts = RowMajorMatrix::rand(&mut *rng, mat.height(), SALT_ELEMS); + .enumerate() + .map(|(m, mat)| { + let lane = u32::try_from(m).expect("fewer than 2^32 matrices in one commit"); + let salts = random_matrix(mat.height(), SALT_ELEMS, &key, lane); HorizontalPair::new(mat, salts) }) - .collect() + .collect(), + None => { + let mut rng = self.rng.lock(); + inputs + .into_iter() + .map(|mat| { + let salts = RowMajorMatrix::rand(&mut *rng, mat.height(), SALT_ELEMS); + HorizontalPair::new(mat, salts) + }) + .collect() + } }; self.inner.commit(salted_inputs) } diff --git a/vendor/p3-merkle-tree/src/lib.rs b/vendor/p3-merkle-tree/src/lib.rs index 0e0f09f8..60b526f3 100644 --- a/vendor/p3-merkle-tree/src/lib.rs +++ b/vendor/p3-merkle-tree/src/lib.rs @@ -3,11 +3,13 @@ extern crate alloc; +pub mod counter_rng; mod hiding_mmcs; mod merkle_tree; mod mmcs; mod pruning; +pub use counter_rng::CounterModeRng; pub use hiding_mmcs::*; pub use merkle_tree::MerkleTree; pub use mmcs::{MerkleTreeError, MerkleTreeMmcs, PrunedBatchOpening, PrunedProofError};