From 70aa6b13546e844736918d2c32528e553440fb23 Mon Sep 17 00:00:00 2001 From: Dendi Suhubdy <4477337+dendisuhubdy@users.noreply.github.com> Date: Wed, 28 Oct 2026 07:57:05 +0800 Subject: [PATCH] v0.8.3: bound admission memory; ack only after hand-off (#157) One byte budget (AdmissionBudget, a tokio semaphore counted in bytes) over the memory admission holds before a transaction is pooled: - RPC: every POST body is charged its Content-Length (else the body cap) before axum reads it, held until the response is built, so a rand_sendTransaction stays charged while it waits for its verdict. A charge that does not fit is answered at once with -32005 (HTTP 503, Retry-After: 1); the body is drained chunk by chunk, never buffered, so the client reads the error instead of a reset. - Verify queue: each queued transaction (gossip or RPC) carries a charge of its bincode length, released when its verdict has been handled. Gossip past the budget is reported Ignore (never Reject); RPC gets MempoolError::AdmissionBusy, answered -32005. Default: 1/16 of the host's RAM within 64-256 MiB (about 117 MiB on the fleet's 2 GB droplets), never under one largest submission (body cap plus block cap). --admission-budget-mb overrides; a value under that floor is refused at startup. rand_status gains admission_bytes and admission_budget_bytes. Ack semantics were already "verified, pooled and handed to gossip" (the RPC reply is sent from Node::on_verdict after insert_verified and broadcast); now documented in docs/rpc.md. The unit templates set MemoryHigh=85%, and docs/node-hardware.md makes 4 GB the validator minimum. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GFs5h141Ct6RTGGrCTQ1vy --- CHANGELOG.md | 8 +- crates/randprotocol-node/src/admission.rs | 196 ++++++++++++++++++- crates/randprotocol-node/src/main.rs | 13 +- crates/randprotocol-node/src/mempool.rs | 6 + crates/randprotocol-node/src/node.rs | 109 ++++++++++- crates/randprotocol-node/src/rpc.rs | 162 +++++++++++++++ crates/randprotocol-node/tests/common/mod.rs | 1 + crates/randprotocol-prover/src/memory.rs | 9 + deploy/cutover-droplet-rand.sh | 4 + deploy/vps-setup.sh | 4 + docs/cli.md | 1 + docs/node-hardware.md | 8 +- docs/rpc.md | 31 ++- 13 files changed, 537 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2036eb45..7befa6b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,13 @@ stopped at v0.4 until 2026-09-30, when the entries v0.5 to v0.6.7 were written f ## Unreleased -Nothing yet. +- **Admission memory is bounded** (issue #157): one byte budget (`--admission-budget-mb`; default + 1/16 of RAM within 64–256 MiB, never under one largest transaction) over RPC request bodies and + the verify queue, gossiped and submitted. Past it `rand_sendTransaction` answers `-32005` + (HTTP 503, `Retry-After: 1`) without reading the request into memory, and gossip sheds with + `Ignore`. `rand_status` gains `admission_bytes` and `admission_budget_bytes`. The hash + `rand_sendTransaction` returns still means verified, pooled and handed to gossip, now + documented. The unit templates set `MemoryHigh=85%`; 4 GB is the validator minimum. ## v0.8.2 — 2026-10-27 (chain 21, node-only; the genesis-gated rules wait for the next cut) diff --git a/crates/randprotocol-node/src/admission.rs b/crates/randprotocol-node/src/admission.rs index 33308b45..ee00249c 100644 --- a/crates/randprotocol-node/src/admission.rs +++ b/crates/randprotocol-node/src/admission.rs @@ -618,6 +618,137 @@ impl VerifyLimits { } } +/// Issue #157: one byte budget over the memory admission holds before a transaction is pooled. +/// +/// The 2026-10-27 overnight run OOM-killed 2 GB validators 24 times in 8 h. Each kill followed a +/// burst of eight ~2.9 MB transactions: the RPC buffered the 5.7 MB hex bodies, and gossip queued +/// the decoded transactions for verification, 64 deep beside 4 in flight, with nothing counting +/// bytes. Each of those holders now takes an [`AdmissionCharge`] from this one budget and gives it +/// back when it is done: +/// +/// - the RPC: a POST's declared body length (`Content-Length`, else the body cap), held from +/// before the body is read until the response is built (`rpc::charge_body`). A +/// `rand_sendTransaction` waits for its verdict inside that window, so its hex body stays +/// charged throughout; +/// - the verify queue: [`charge_of`] the decoded transaction, from when it is queued (gossip or +/// RPC) until its verdict has been handled (`Verdict::charge`). +/// +/// It is never waited on. A charge that does not fit is refused at once: the RPC answers `-32005` +/// ("busy, retry"), and gossip reports `Ignore`, so the forwarder is not penalised and an honest +/// peer's copy reaches this node again later. A byte count, not an RSS measurement: what it bounds +/// is what bursts of large transactions add on top of a node's resting footprint. +#[derive(Clone, Debug)] +pub struct AdmissionBudget { + sem: std::sync::Arc, + limit: usize, +} + +/// Bytes held against an [`AdmissionBudget`]; dropping it gives them back. +#[derive(Debug)] +pub struct AdmissionCharge { + _permit: tokio::sync::OwnedSemaphorePermit, +} + +impl AdmissionCharge { + /// A charge of nothing against a budget of its own, for tests that build a [`Verdict`] by + /// hand. + #[cfg(test)] + pub(crate) fn none() -> AdmissionCharge { + AdmissionBudget::new(0).try_charge(0).expect("a zero charge always fits") + } +} + +/// A charge the budget could not take now: `wanted` bytes with `available` of `limit` free. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AdmissionBusy { + pub wanted: usize, + pub available: usize, + pub limit: usize, +} + +impl std::fmt::Display for AdmissionBusy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "busy: this node's admission memory is full ({} of {} MiB in use, {} KiB wanted); retry shortly", + (self.limit - self.available) >> 20, + self.limit >> 20, + self.wanted.div_ceil(1024) + ) + } +} + +impl AdmissionBudget { + /// The default's ceiling: 256 MiB, on any host with about 4 GB or more. + pub const DEFAULT_MAX: usize = 256 << 20; + /// The default's floor before [`AdmissionBudget::default_for`] raises it to one largest + /// submission. + pub const DEFAULT_MIN: usize = 64 << 20; + /// The default is this fraction of the host's RAM: 1/16, so ~117 MiB on the fleet's 1 967 MB + /// droplets, which sit at ~1.4 GB resident with ~250 MB to spare. Counted bytes are not all of + /// the memory they cost (a decoded transaction's allocations, verify scratch), so the budget + /// stays well under the headroom. + pub const RAM_DIVISOR: u64 = 16; + + pub fn new(limit: usize) -> AdmissionBudget { + let limit = limit.min(tokio::sync::Semaphore::MAX_PERMITS); + AdmissionBudget { sem: std::sync::Arc::new(tokio::sync::Semaphore::new(limit)), limit } + } + + /// The default budget on a host with `total_ram` bytes: `total_ram / 16` within + /// [`DEFAULT_MIN`](Self::DEFAULT_MIN)..=[`DEFAULT_MAX`](Self::DEFAULT_MAX), and never below + /// `floor` — one largest submission ([`AdmissionBudget::floor_for`]), so the budget can + /// always admit the biggest transaction the chain allows once it is otherwise idle. `0` + /// (RAM unknown) takes the ceiling. + pub fn default_for(total_ram: u64, floor: usize) -> usize { + let share = match total_ram { + 0 => Self::DEFAULT_MAX, + n => usize::try_from(n / Self::RAM_DIVISOR).unwrap_or(usize::MAX).clamp(Self::DEFAULT_MIN, Self::DEFAULT_MAX), + }; + share.max(floor) + } + + /// One largest `rand_sendTransaction` in flight: its body at the RPC's cap plus the decoded + /// transaction at the block cap. A budget under this could refuse a valid transaction for + /// ever, so `--admission-budget-mb` below it is refused at startup. + pub fn floor_for(max_body_bytes: usize, max_block_bytes: usize) -> usize { + max_body_bytes.saturating_add(max_block_bytes) + } + + /// Take `bytes` now, or say why not. Never waits. A charge larger than the whole budget is + /// always refused (it could never fit). + pub fn try_charge(&self, bytes: usize) -> Result { + let busy = || AdmissionBusy { wanted: bytes, available: self.available(), limit: self.limit }; + let n = u32::try_from(bytes).map_err(|_| busy())?; + if bytes > self.limit { + return Err(busy()); + } + match self.sem.clone().try_acquire_many_owned(n) { + Ok(permit) => Ok(AdmissionCharge { _permit: permit }), + Err(_) => Err(busy()), + } + } + + pub fn limit(&self) -> usize { + self.limit + } + + pub fn available(&self) -> usize { + self.sem.available_permits() + } + + /// Bytes charged right now. + pub fn in_use(&self) -> usize { + self.limit - self.available() + } +} + +/// What a decoded transaction is charged while it waits for verification: its bincode length, +/// computed without encoding it. +pub fn charge_of(tx: &Transaction) -> usize { + bincode::serialized_size(tx).map_or(usize::MAX, |n| usize::try_from(n).unwrap_or(usize::MAX)) +} + /// What this node has decided to tell gossipsub about one delivered message. /// /// libp2p's own [`MessageAcceptance`] derives `Debug` and nothing else — no `Clone`, no `PartialEq` @@ -666,6 +797,9 @@ pub struct Verdict { /// found the state-proof key-build gate taken ([`crate::node::pool_verify_without_waiting`]). /// The answer is `StateProofBusy`, retryable: `result` is unused, nothing is cached. pub busy: bool, + /// Issue #157: the transaction's bytes against the [`AdmissionBudget`], taken when it was + /// queued and given back when this verdict is dropped, at the end of `Node::on_verdict`. + pub charge: AdmissionCharge, } /// How many refused state proofs to remember (RPL-3 final fix wave, I1). Only proofs against the @@ -1198,7 +1332,8 @@ pub fn acceptance_for_pool(e: &MempoolError, hash: Hash, refused: &mut RefusedCa | MempoolError::PerpBackpressure { .. } | MempoolError::PerpHeightBackpressure { .. } | MempoolError::PerpCancelUnknownOrder(_) - | MempoolError::StateProofBusy => Acceptance::Ignore, + | MempoolError::StateProofBusy + | MempoolError::AdmissionBusy(_) => Acceptance::Ignore, } } @@ -2288,4 +2423,63 @@ mod tests { } assert!(!l.allow(&mut b, t0 + Duration::from_secs(3600)), "the burst is the ceiling"); } + + /// Issue #157: the budget takes charges up to its limit and no further, refuses at once + /// (never waits), and gets each charge's bytes back when the charge is dropped. + #[test] + fn the_admission_budget_refuses_past_its_limit_and_releases_on_drop() { + let budget = AdmissionBudget::new(10_000); + let a = budget.try_charge(6_000).expect("fits"); + let b = budget.try_charge(4_000).expect("fits exactly"); + assert_eq!((budget.in_use(), budget.available()), (10_000, 0)); + let busy = budget.try_charge(1).unwrap_err(); + assert_eq!(busy, AdmissionBusy { wanted: 1, available: 0, limit: 10_000 }); + assert!(busy.to_string().starts_with("busy:") && busy.to_string().contains("retry"), "{busy}"); + drop(a); + assert_eq!(budget.in_use(), 4_000, "the first charge's bytes are back"); + assert!(budget.try_charge(6_001).is_err(), "one byte too many is still refused"); + let c = budget.try_charge(6_000).expect("fits again"); + // A clone is the same budget, not a second one. + let shared = budget.clone(); + assert!(shared.try_charge(1).is_err()); + drop(b); + drop(c); + assert_eq!((budget.in_use(), shared.in_use()), (0, 0)); + // Larger than the whole budget: refused even when idle, since it could never fit. + assert_eq!(budget.try_charge(10_001).unwrap_err().available, 10_000); + assert!(budget.try_charge(usize::MAX).is_err()); + // A zero charge always fits. + let full = budget.try_charge(10_000).unwrap(); + assert!(budget.try_charge(0).is_ok()); + drop(full); + assert_eq!(budget.in_use(), 0); + } + + /// Issue #157: the default is 1/16 of the host's RAM within 64–256 MiB, and never under one + /// largest submission. + #[test] + fn the_default_admission_budget_follows_the_host() { + const MIB: usize = 1 << 20; + let floor = AdmissionBudget::floor_for(8_867_840, 4 * MIB); + assert_eq!(floor, 8_867_840 + 4 * MIB); + // The fleet's 2 GB droplets (1 967 MB): ~117 MiB. + assert_eq!(AdmissionBudget::default_for(1_967 * 1_000_000, floor), 1_967 * 1_000_000 / 16); + assert_eq!(AdmissionBudget::default_for(4 << 30, floor), 256 * MIB, "4 GiB: the ceiling"); + assert_eq!(AdmissionBudget::default_for(64 << 30, floor), 256 * MIB); + assert_eq!(AdmissionBudget::default_for(512 << 20, floor), 64 * MIB, "the floor of the range"); + assert_eq!(AdmissionBudget::default_for(0, floor), 256 * MIB, "RAM unknown"); + // A chain whose largest submission is past the share: the submission wins. + assert_eq!(AdmissionBudget::default_for(1 << 30, 300 * MIB), 300 * MIB); + } + + /// Issue #157: a transaction is charged its bincode length, computed without encoding it. + #[test] + fn a_transaction_is_charged_its_encoded_length() { + let tx = Transaction { + chain_id: 7, + bundle: None, + action: randprotocol_core::Action::Deploy { base_pc: 0, words: vec![0x13; 1_000], public: vec![] }, + }; + assert_eq!(charge_of(&tx), tx.encoded_len()); + } } diff --git a/crates/randprotocol-node/src/main.rs b/crates/randprotocol-node/src/main.rs index 2a0fa85d..05dba4f6 100644 --- a/crates/randprotocol-node/src/main.rs +++ b/crates/randprotocol-node/src/main.rs @@ -845,6 +845,13 @@ enum Cmd { /// Proof-verification workers (default: the cores minus two, at least four — spec 2026-10-05 §6). #[arg(long, value_name = "N")] verify_workers: Option, + /// Admission memory budget in MiB (issue #157): one byte budget over RPC request bodies + /// and transactions waiting for verification, gossiped or submitted. Past it, + /// `rand_sendTransaction` answers -32005 (busy, retry) and gossiped transactions are shed + /// unpenalised. Default: 1/16 of the host's RAM within 64–256 MiB, and never under one + /// largest transaction (body cap plus block cap); a value under that is refused. + #[arg(long, value_name = "MIB", value_parser = clap::value_parser!(u64).range(1..))] + admission_budget_mb: Option, /// Apply incoming proposals on the consensus loop, as before v0.8.0, instead of on a /// blocking worker (spec 2026-10-11 off-loop apply §4.5): the kill switch. The by-hash /// fetch answer binding is a separate fix and stays on. @@ -2997,6 +3004,7 @@ async fn main() -> Result<()> { prover_cpu, prover_threads, verify_workers, + admission_budget_mb, apply_proposals_inline, prover_skip_memory_check, prover_allow_origin, @@ -3047,7 +3055,10 @@ async fn main() -> Result<()> { Some(std::sync::Arc::::from(token)) } }; - let rpc_options = node::RpcOptions { public_addr: public_rpc, viewing_token }; + let admission_budget = admission_budget_mb + .map(|mb| usize::try_from(mb).ok().and_then(|mb| mb.checked_mul(1 << 20)).ok_or_else(|| anyhow::anyhow!("--admission-budget-mb {mb} is too large"))) + .transpose()?; + let rpc_options = node::RpcOptions { public_addr: public_rpc, viewing_token, admission_budget }; let handle = node::start_with(NodeConfig { viewing_open: rpc_viewing_open, datadir, diff --git a/crates/randprotocol-node/src/mempool.rs b/crates/randprotocol-node/src/mempool.rs index 5c293018..f444f601 100644 --- a/crates/randprotocol-node/src/mempool.rs +++ b/crates/randprotocol-node/src/mempool.rs @@ -92,6 +92,11 @@ pub enum MempoolError { /// the node has a key for always passes. #[error("this node is at its limit of verifier-key builds for new state-proof shapes; retry later")] StateProofBusy, + /// Issue #157: the admission byte budget ([`crate::admission::AdmissionBudget`]) cannot take + /// this transaction now. Retryable, never cached, no peer penalised; `rand_sendTransaction` + /// answers it as `-32005`. + #[error("{0}")] + AdmissionBusy(crate::admission::AdmissionBusy), } /// The refusal a `Warming` deferral carries, as `rand_sendTransaction` reports it. @@ -129,6 +134,7 @@ impl MempoolError { | MempoolError::PerpHeightBackpressure { .. } | MempoolError::PerpCancelUnknownOrder(_) | MempoolError::StateProofBusy + | MempoolError::AdmissionBusy(_) ) } } diff --git a/crates/randprotocol-node/src/node.rs b/crates/randprotocol-node/src/node.rs index cc851f57..4350d940 100644 --- a/crates/randprotocol-node/src/node.rs +++ b/crates/randprotocol-node/src/node.rs @@ -2,7 +2,7 @@ //! network handle; turns consensus `Action`s into I/O and network events into //! consensus input. -use crate::admission::{self, GossipOutcome, Verdict, VerifySource, VerifyLimits}; +use crate::admission::{self, AdmissionBudget, AdmissionCharge, GossipOutcome, Verdict, VerifySource, VerifyLimits}; use crate::compact; use crate::mempool::{Mempool, MempoolError}; use crate::network::{ @@ -1385,11 +1385,14 @@ struct Node { /// Verifications on blocking workers right now, capped at `verify_limits.in_flight`. verify_in_flight: usize, /// Transactions waiting for one of those slots, capped at `verify_limits.queue` by - /// [`GossipOutcome::for_transaction`]. - verify_queue: VecDeque<(Transaction, VerifySource)>, + /// [`GossipOutcome::for_transaction`], each with its bytes charged against `admission`. + verify_queue: VecDeque<(Transaction, VerifySource, AdmissionCharge)>, /// The verification concurrency and queue bounds, sized to the host once at startup /// (spec 2026-10-05 §6; `--verify-workers` overrides). verify_limits: VerifyLimits, + /// Issue #157: the byte budget the verify queue shares with the RPC's request bodies + /// (`RpcState::admission`, the same semaphore). + admission: AdmissionBudget, /// The sender every verification task answers on; its receiver is an arm of the loop's /// `select!`. verdicts_tx: mpsc::Sender, @@ -2493,6 +2496,11 @@ pub struct RpcOptions { /// A bearer token the viewing-key methods require on the operator's listener /// (`--rpc-viewing-token-file`). `None` keeps the loopback rule. pub viewing_token: Option>, + /// Issue #157: the admission byte budget in bytes (`--admission-budget-mb`), shared by the + /// RPC's request bodies and the verify queue (gossip and RPC); `None` sizes it to the host + /// ([`AdmissionBudget::default_for`]). Here rather than in [`NodeConfig`] so the many literal + /// `NodeConfig`s stay untouched. + pub admission_budget: Option, } /// What [`start_with`] adds to the network beyond [`NodeConfig`] (audit v6, NET-1): its own @@ -2696,7 +2704,27 @@ pub async fn start_with(cfg: NodeConfig, rpc_options: RpcOptions, net_options: N tracing::warn!("this chain's genesis carries its own gas section; --gas-price/--byte-price are ignored"); } let rpc_limits = rpc::ChainLimits::of(&gs.ledger).with_gas_policy(cfg.gas_policy); + // Issue #157: one byte budget over admission memory — the RPC's request bodies and the verify + // queue — sized from the host's RAM unless `--admission-budget-mb` says otherwise, and never + // under one largest submission. + let admission = { + let floor = AdmissionBudget::floor_for(rpc_limits.rpc_max_body_bytes(), rpc_limits.max_block_bytes); + let bytes = match rpc_options.admission_budget { + Some(b) if b < floor => anyhow::bail!( + "--admission-budget-mb {} is under one largest transaction on this chain ({} MiB: the {}-byte request-body cap plus the {}-byte block cap)", + b >> 20, + floor.div_ceil(1 << 20), + rpc_limits.rpc_max_body_bytes(), + rpc_limits.max_block_bytes + ), + Some(b) => b, + None => AdmissionBudget::default_for(randprotocol_prover::memory::total_bytes(), floor), + }; + AdmissionBudget::new(bytes) + }; + tracing::info!("admission budget: {} MiB (request bodies and the verify queue)", admission.limit() >> 20); let rpc_state = RpcState { + admission: admission.clone(), limiter: Arc::new(crate::rpc::RpcLimiter::default()), viewing_open: cfg.viewing_open, public: false, @@ -2927,6 +2955,7 @@ pub async fn start_with(cfg: NodeConfig, rpc_options: RpcOptions, net_options: N snapshot: None, verify_in_flight: 0, verify_limits, + admission, verify_queue: VecDeque::new(), verdicts_tx, applied_tx, @@ -3136,6 +3165,8 @@ impl Node { // questions for an operator. s.refused_cache = self.refused.len(); s.verify_queue = self.verify_queue.len(); + s.admission_bytes = self.admission.in_use(); + s.admission_budget_bytes = self.admission.limit(); // Read without touching the registry's lock: this runs on the node loop every pass, and a // scan holds its own key's lock for as long as it takes (audit v3, VK-1). s.viewing_keys = self.viewing_count.load(std::sync::atomic::Ordering::Relaxed); @@ -3163,7 +3194,7 @@ impl Node { /// goes straight back to the `select!`. The verdict returns on its own arm. fn pump_verify(&mut self) { while self.verify_in_flight < self.verify_limits.in_flight { - let Some((tx, source)) = self.verify_queue.pop_front() else { break }; + let Some((tx, source, charge)) = self.verify_queue.pop_front() else { break }; let ledger = self.snapshot(); let storage = self.storage.clone(); let profile = core_profile(&self.gs.fri_profile); @@ -3188,7 +3219,7 @@ impl Node { }; // The loop is the only receiver and outlives every task it spawned, so a send // failure means the node is already shutting down. - let _ = out.blocking_send(Verdict { tx, result, source, proof_key, busy }); + let _ = out.blocking_send(Verdict { tx, result, source, proof_key, busy, charge }); }); } } @@ -3309,11 +3340,17 @@ impl Node { let a = admission::acceptance_for_pool(&e, hash, &mut self.refused); self.note_refusal(hash); self.report(id, GossipOutcome::Report(a)).await; - } else { + } else if let Ok(charge) = self.admission.try_charge(admission::charge_of(&tx)) { self.mempool.state_proof_queued(&tx, self.hs.tip_ledger(), self.executor.as_ref()); self.mempool.verify_queued(&tx); - self.verify_queue.push_back((tx, VerifySource::Gossip(id))); + self.verify_queue.push_back((tx, VerifySource::Gossip(id), charge)); self.pump_verify(); + } else { + // Issue #157: the admission budget is spent. Shed like a full queue — `Ignore`, so the + // forwarder is not penalised and a later copy is judged afresh — rather than queueing + // bytes this host may not have. A park waiting for it is still filled below. + tracing::debug!(%hash, in_use = self.admission.in_use(), "admission budget full; gossiped transaction shed"); + self.report(id, GossipOutcome::Report(admission::Acceptance::Ignore)).await; } match for_park { Some(tx) => self.fill_parks(tx).await, @@ -3413,9 +3450,18 @@ impl Node { let _ = reply.send(Err(e)); return; } + // Issue #157: the decoded transaction's bytes, beside the request body the RPC already + // charged. Refused now (`-32005`, retry) rather than queued past the budget. + let charge = match self.admission.try_charge(admission::charge_of(&tx)) { + Ok(c) => c, + Err(busy) => { + let _ = reply.send(Err(MempoolError::AdmissionBusy(busy))); + return; + } + }; self.mempool.state_proof_queued(&tx, self.hs.tip_ledger(), self.executor.as_ref()); self.mempool.verify_queued(&tx); - self.verify_queue.push_back((tx, VerifySource::Rpc(reply))); + self.verify_queue.push_back((tx, VerifySource::Rpc(reply), charge)); self.pump_verify(); } @@ -7595,6 +7641,7 @@ mod tests { snapshot: None, verify_in_flight: 0, verify_limits: VerifyLimits::for_cores(1), + admission: AdmissionBudget::new(AdmissionBudget::DEFAULT_MAX), verify_queue: VecDeque::new(), verdicts_tx, applied_tx, @@ -7643,7 +7690,7 @@ mod tests { assert!(matches!(refused, randprotocol_core::TxError::Perps(PerpError::ProofRefused(_))), "{refused:?}"); let key = admission::state_proof_key(&junk, &tip, &StubExecutor).expect("a state proof over a closed window"); let (first, _) = oneshot::channel(); - node.on_verdict(Verdict { tx: junk.clone(), result: Err(refused.clone()), source: VerifySource::Rpc(first), proof_key: Some(key), busy: false }) + node.on_verdict(Verdict { tx: junk.clone(), result: Err(refused.clone()), source: VerifySource::Rpc(first), proof_key: Some(key), busy: false, charge: AdmissionCharge::none() }) .await .unwrap(); @@ -7780,7 +7827,7 @@ mod tests { // The busy verdict: the RPC caller hears `StateProofBusy`; nothing is cached. let (reply, answer) = oneshot::channel(); node.verify_in_flight = 1; - node.on_verdict(Verdict { tx: cold.clone(), result: Ok(()), source: VerifySource::Rpc(reply), proof_key: None, busy: true }).await.unwrap(); + node.on_verdict(Verdict { tx: cold.clone(), result: Ok(()), source: VerifySource::Rpc(reply), proof_key: None, busy: true, charge: AdmissionCharge::none() }).await.unwrap(); let busy = answer.await.unwrap().unwrap_err(); assert_eq!(busy, MempoolError::StateProofBusy); assert!(busy.is_retryable()); @@ -8720,6 +8767,48 @@ mod tests { assert_eq!(reports(&drain(&mut seen).await).len(), 1, "the refused one reported now, the queued one on its verdict"); } + /// Issue #157: the verify queue holds a transaction only with its bytes charged against the + /// admission budget. Past the budget a gossiped transaction is shed `Ignore` (never `Reject`) + /// and an RPC submission is answered `AdmissionBusy` at once; the charge comes back when the + /// verdict has been handled, and the next transaction is queued. No worker runs (no verify + /// slots), so nothing here depends on timing. + #[tokio::test] + async fn the_verify_queue_is_bounded_by_the_admission_budget() { + let (_d, storage, gs, hs) = replica_past_a_boundary(); + let (mut node, mut seen) = bare_node(storage, gs, hs); + let ledger = node.hs.tip_ledger().clone(); + let (first, second, third) = (next_tx(&ledger, 1), next_tx(&ledger, 2), next_tx(&ledger, 3)); + node.admission = AdmissionBudget::new(admission::charge_of(&first)); + node.verify_limits = VerifyLimits { in_flight: 0, queue: 64 }; + + node.on_gossiped_tx(first.clone(), gossip_id()).await.unwrap(); + assert_eq!(node.verify_queue.len(), 1, "the first fits the budget exactly"); + assert_eq!(node.admission.in_use(), node.admission.limit()); + + node.on_gossiped_tx(second.clone(), gossip_id()).await.unwrap(); + assert_eq!(node.verify_queue.len(), 1, "the second is not queued"); + assert_eq!(reports(&drain(&mut seen).await), vec![libp2p::gossipsub::MessageAcceptance::Ignore], "shed, its forwarder not penalised"); + assert!(node.refused.get(&second.hash()).is_none(), "and nothing cached against it"); + + let (reply, rx) = oneshot::channel(); + node.submit_tx(third.clone(), reply).await; + let refused = rx.await.unwrap().unwrap_err(); + assert!(matches!(refused, MempoolError::AdmissionBusy(_)) && refused.is_retryable(), "{refused}"); + assert_eq!(node.verify_queue.len(), 1); + + // The first one's verdict: handled, then its charge is back. + let (tx, source, charge) = node.verify_queue.pop_front().unwrap(); + node.on_verdict(Verdict { tx, result: Ok(()), source, proof_key: None, busy: false, charge }).await.unwrap(); + assert!(node.mempool.contains(&first.hash()), "pooled on its verdict"); + assert_eq!(node.admission.in_use(), 0, "released once the verdict is handled"); + + // The budget has room again: the RPC submission now queues (its size fits the budget). + let room = node.admission.limit() >= admission::charge_of(&third); + let (reply, _rx) = oneshot::channel(); + node.submit_tx(third.clone(), reply).await; + assert_eq!(node.verify_queue.len(), usize::from(room)); + } + /// A connected forwarder whose transaction allowance is spent, under a limiter that does not /// refill, so every delivery it forwards in the test is the limiter's refusal. fn over_its_allowance(node: &mut Node) -> PeerId { diff --git a/crates/randprotocol-node/src/rpc.rs b/crates/randprotocol-node/src/rpc.rs index dbc5009b..6753c54e 100644 --- a/crates/randprotocol-node/src/rpc.rs +++ b/crates/randprotocol-node/src/rpc.rs @@ -262,6 +262,12 @@ pub struct NodeStatus { /// Normally 0: a queue that sits near the cap means this node is shedding gossiped /// transactions, and it is unrelated to the sync counters above. pub verify_queue: usize, + /// Issue #157: bytes charged against the admission budget right now — request bodies being + /// served plus transactions waiting for or under verification — and the budget itself + /// (`--admission-budget-mb`). Near the budget, `rand_sendTransaction` answers `-32005` and + /// gossiped transactions are shed with `Ignore`. + pub admission_bytes: usize, + pub admission_budget_bytes: usize, pub mempool_size: usize, /// Viewing keys this node holds for node-side scanning (`rand_importViewingKey`), against /// `viewing::MAX_VIEWING_KEYS`. In memory only, so it reads zero after every restart. Nonzero @@ -647,6 +653,9 @@ pub struct RpcState { /// with it, being on loopback is no longer enough — any process on the host can be on /// loopback — and a caller anywhere that presents it is served. pub viewing_token: Option>, + /// Issue #157: the node's admission byte budget, shared by both listeners and the verify + /// queue. Every POST body is charged against it before it is read ([`charge_body`]). + pub admission: crate::admission::AdmissionBudget, } impl RpcState { @@ -743,6 +752,10 @@ fn id_member<'de, D: serde::Deserializer<'de>>(d: D) -> Result, D: Value::deserialize(d).map(Some) } +/// JSON-RPC `-32005`, "busy, retry" (issue #157): the code the sale proxy already used for "slow +/// down", which the wallet retries with back-off (`randprotocol_client::is_rate_limited`). +pub const BUSY_CODE: i64 = -32005; + #[derive(Debug)] struct RpcError { code: i64, @@ -768,6 +781,16 @@ impl RpcError { fn internal(m: impl std::fmt::Display) -> RpcError { RpcError { code: -32603, message: m.to_string(), data: None } } + /// Code `-32005`: busy, retry (issue #157) — the admission byte budget cannot take this + /// request now. Nothing was queued; the same request may succeed in a moment. `data` carries + /// the numbers. + fn busy(b: crate::admission::AdmissionBusy) -> RpcError { + RpcError { + code: BUSY_CODE, + message: b.to_string(), + data: Some(json!({ "wanted": b.wanted, "available": b.available, "limit": b.limit })), + } + } /// Code `-32010`: the height is below this node's retention floor (history pruning spec §4). fn pruned(h: u64, floor: u64) -> RpcError { RpcError { @@ -1122,6 +1145,8 @@ pub async fn serve(addr: SocketAddr, state: RpcState) -> anyhow::Result<(SocketA // upgrade is a GET with no body, so the layer costs it nothing; frames are bounded by // `ws::WS_MAX_FRAME_BYTES` instead. .layer(axum::extract::DefaultBodyLimit::max(max_body_bytes)) + // Issue #157: every POST body charged against the admission budget before it is read. + .layer(axum::middleware::from_fn_with_state(state.clone(), charge_body)) .with_state(state); let listener = tokio::net::TcpListener::bind(addr).await?; let bound = listener.local_addr()?; @@ -1136,6 +1161,59 @@ pub async fn serve(addr: SocketAddr, state: RpcState) -> anyhow::Result<(SocketA Ok((bound, task)) } +/// Issue #157: charge a POST's body against the admission budget before axum reads it, and hold +/// the charge until the response is built — for `rand_sendTransaction`, until the verdict, so a +/// burst of large submissions is bounded while it waits rather than buffered without limit. The +/// declared `Content-Length` is what is charged (capped at the body limit, which refuses anything +/// longer); a body without one is charged the whole limit. A charge that does not fit is answered +/// at once with `-32005` and HTTP 503 + `Retry-After: 1`. A GET (the WebSocket upgrade) has no +/// body and is not charged. +async fn charge_body( + State(st): State, + req: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + use axum::response::IntoResponse; + if req.method() != axum::http::Method::POST { + return next.run(req).await; + } + let declared = req + .headers() + .get(axum::http::header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + .map_or(st.max_body_bytes, |n| n.min(st.max_body_bytes)); + match st.admission.try_charge(declared) { + Ok(charge) => { + let response = next.run(req).await; + drop(charge); + response + } + Err(busy) => { + // The body is read and dropped a chunk at a time — never buffered — before the answer + // goes out: a server that answers without reading leaves hyper to close the connection + // under a client still writing, which then sees a reset instead of this `-32005`. At + // most the body limit is drained; past it the connection closes as it would at the + // limit anyway. + use futures::StreamExt; + let mut body = req.into_body().into_data_stream(); + let mut drained = 0usize; + while drained <= st.max_body_bytes { + match body.next().await { + Some(Ok(chunk)) => drained += chunk.len(), + _ => break, + } + } + ( + StatusCode::SERVICE_UNAVAILABLE, + [(axum::http::header::RETRY_AFTER, "1")], + Json(error_value(Value::Null, RpcError::busy(busy))), + ) + .into_response() + } + } +} + /// One error response object. Built from an `RpcError` so every `-32600` in this file comes from /// `RpcError::invalid_request` — including the oversized-body one `rejection_error` already makes. /// The most requests one client address may make in a burst, and the rate it refills at @@ -2785,8 +2863,11 @@ async fn dispatch(st: &RpcState, req: &Request) -> Result { .send(NodeCommand::SubmitTx { tx, reply }) .await .map_err(|_| RpcError::internal("node loop closed"))?; + // Issue #157: the hash comes back only from `Node::on_verdict`, after the transaction + // verified, was pooled and was handed to gossip (`docs/rpc.md`, "What a hash means"). match rx.await.map_err(|_| RpcError::internal("node loop dropped reply"))? { Ok(h) => Ok(json!(h.to_hex())), + Err(MempoolError::AdmissionBusy(b)) => Err(RpcError::busy(b)), Err(e) => Err(RpcError::rejected(e.to_string())), } } @@ -4487,6 +4568,7 @@ pub(crate) mod tests { refusals: tokio::sync::broadcast::channel(HEAD_CHANNEL).0, ws_conns: Arc::new(std::sync::atomic::AtomicUsize::new(0)), viewing: Arc::new(RwLock::new(crate::viewing::Registry::default())), + admission: crate::admission::AdmissionBudget::new(crate::admission::AdmissionBudget::DEFAULT_MAX), } } @@ -8911,6 +8993,86 @@ pub(crate) mod tests { task.abort(); } + /// Issue #157: eight concurrent ~1.6 MB `rand_sendTransaction`s against a budget of three + /// bodies. Exactly three reach the node loop, which holds their replies (as a node does while + /// it verifies); the other five are answered `-32005` (HTTP 503, `Retry-After`) at once, the + /// budget never over its limit. Released, the three get their hashes and the budget is empty + /// again; a retry then goes through. Deterministic: every step waits on a message, none on a + /// clock — the held three cannot answer before the test replies, so whatever answers first + /// is a refusal. + #[tokio::test] + async fn concurrent_large_submissions_past_the_admission_budget_are_told_to_retry() { + const N: usize = 8; + const HELD: usize = 3; + let gs = genesis_with(7, vec![alloc_note(20, 1_000)]); + let (_dir, st) = state_for(&gs); + let tx = |i: u32| Transaction { + chain_id: gs.chain_id, + bundle: None, + action: Action::Deploy { base_pc: i, words: vec![0x13; 200_000], public: vec![] }, + }; + let body = |i: u32| serde_json::to_vec(&json!({ "jsonrpc": "2.0", "id": i, "method": "rand_sendTransaction", "params": [hex::encode(tx(i).encode())] })).unwrap(); + let len = body(0).len(); + assert!(len > 1_500_000 && (0..N as u32).all(|i| body(i).len() == len), "equal, large bodies: {len} B"); + let budget = crate::admission::AdmissionBudget::new(HELD * len + len / 2); + // A stand-in node loop that answers nothing until the test says so. + let (node_tx, mut node_rx) = mpsc::channel(N); + let st = RpcState { admission: budget.clone(), node: node_tx, ..st }; + let (addr, task) = serve("127.0.0.1:0".parse().unwrap(), st).await.unwrap(); + + let (done_tx, mut done) = mpsc::unbounded_channel(); + for i in 0..N as u32 { + let (raw, done_tx) = (body(i), done_tx.clone()); + // Every outcome is sent, a transport error included, so a failure fails the test + // rather than leaving it waiting for an answer that never comes. + tokio::spawn(async move { + let answer = async { + let resp = reqwest::Client::new().post(format!("http://{addr}")).header("content-type", "application/json").body(raw).send().await?; + let status = resp.status(); + let retry_after = resp.headers().get("retry-after").and_then(|v| v.to_str().ok()).map(str::to_string); + Ok::<_, reqwest::Error>((status, retry_after, resp.json::().await?)) + }; + let _ = done_tx.send(answer.await.map_err(|e| e.to_string())); + }); + } + + let mut held = Vec::new(); + for _ in 0..HELD { + match node_rx.recv().await.expect("a submission reaches the node") { + NodeCommand::SubmitTx { tx, reply } => held.push((tx.hash(), reply)), + _ => panic!("only submissions are sent"), + } + } + for _ in 0..N - HELD { + let (status, retry_after, v) = done.recv().await.unwrap().expect("the refused request is answered, not reset"); + assert_eq!(status, reqwest::StatusCode::SERVICE_UNAVAILABLE, "{v}"); + assert_eq!(retry_after.as_deref(), Some("1")); + assert_eq!(v["error"]["code"], json!(BUSY_CODE), "{v}"); + assert!(v["error"]["message"].as_str().unwrap().starts_with("busy:"), "{v}"); + assert_eq!(v["error"]["data"]["limit"], json!(budget.limit())); + } + assert_eq!(budget.in_use(), HELD * len, "three bodies charged, never more than the budget"); + assert!(node_rx.try_recv().is_err(), "no fourth submission reached the node"); + + for (hash, reply) in held.drain(..) { + reply.send(Ok(hash)).unwrap(); + } + for _ in 0..HELD { + let (status, _, v) = done.recv().await.unwrap().expect("answered"); + assert_eq!(status, reqwest::StatusCode::OK); + assert!(v["result"].as_str().is_some_and(|h| h.len() == 64), "{v}"); + } + assert_eq!(budget.in_use(), 0, "every charge is back once its response is out"); + + // With room again, the same request a refused caller retries reaches the node. + let raw = body(7); + let retry = tokio::spawn(async move { reqwest::Client::new().post(format!("http://{addr}")).header("content-type", "application/json").body(raw).send().await.unwrap().status() }); + let NodeCommand::SubmitTx { tx: got, reply } = node_rx.recv().await.unwrap() else { panic!("a submission") }; + reply.send(Ok(got.hash())).unwrap(); + assert_eq!(retry.await.unwrap(), reqwest::StatusCode::OK); + task.abort(); + } + /// A body within the limit still reaches the dispatcher, and a malformed one answers JSON too. #[tokio::test] async fn a_malformed_body_answers_json_rather_than_plain_text() { diff --git a/crates/randprotocol-node/tests/common/mod.rs b/crates/randprotocol-node/tests/common/mod.rs index 4f5fae96..54fea224 100644 --- a/crates/randprotocol-node/tests/common/mod.rs +++ b/crates/randprotocol-node/tests/common/mod.rs @@ -239,6 +239,7 @@ async fn serve_genesis_seeded_with( refusals: broadcast::channel(capacity).0, ws_conns: Arc::new(AtomicUsize::new(0)), viewing: Arc::new(RwLock::new(randprotocol_node::viewing::Registry::default())), + admission: randprotocol_node::admission::AdmissionBudget::new(randprotocol_node::admission::AdmissionBudget::DEFAULT_MAX), }; let ws_conns = state.ws_conns.clone(); let status = state.status.clone(); diff --git a/crates/randprotocol-prover/src/memory.rs b/crates/randprotocol-prover/src/memory.rs index 54c792e5..494250ee 100644 --- a/crates/randprotocol-prover/src/memory.rs +++ b/crates/randprotocol-prover/src/memory.rs @@ -16,6 +16,15 @@ pub fn available_bytes() -> u64 { sys.available_memory() } +/// The machine's total memory in bytes (`0` if the OS does not say): what the node sizes its +/// admission byte budget from (issue #157), where the available figure at startup — before the +/// node has grown to its working size — would overstate the room. +pub fn total_bytes() -> u64 { + let mut sys = sysinfo::System::new(); + sys.refresh_memory(); + sys.total_memory() +} + fn gb(b: u64) -> f64 { b as f64 / 1e9 } /// `Err` names both numbers when `max_parallel` proofs would not fit in available memory. diff --git a/deploy/cutover-droplet-rand.sh b/deploy/cutover-droplet-rand.sh index e179e5fa..44dd4548 100755 --- a/deploy/cutover-droplet-rand.sh +++ b/deploy/cutover-droplet-rand.sh @@ -55,6 +55,10 @@ After=network-online.target [Service] Environment=RUST_LOG=info,libp2p=warn,libp2p_mdns=off ExecStart=/usr/local/bin/rand-node run --datadir \$DATA --key /root/fullnode/deploy/node-$NAME.key.json --validator --listen /ip4/0.0.0.0/tcp/30303 --rpc 127.0.0.1:8545 --no-mdns$BOOT_ARGS +# Issue #157: reclaim and throttle above 85% of RAM rather than let the OOM killer take the node +# mid-burst (it did 24 times in 8 h on the 2 GB validators). No MemoryMax: a cap kills, which is +# what this avoids. Pair it with swap; 4 GB is the validator minimum (docs/node-hardware.md). +MemoryHigh=85% Restart=always RestartSec=3 [Install] diff --git a/deploy/vps-setup.sh b/deploy/vps-setup.sh index e00c885d..a6d7a4c8 100755 --- a/deploy/vps-setup.sh +++ b/deploy/vps-setup.sh @@ -48,6 +48,10 @@ After=network-online.target [Service] Environment=RUST_LOG=info,libp2p=warn,libp2p_mdns=off ExecStart=/usr/local/bin/rand-node run --datadir $DATA --key $KEYFILE $VALIDATOR_FLAG --listen /ip4/0.0.0.0/tcp/30303 --rpc 127.0.0.1:8545 $PUBLIC_RPC_ARGS --no-mdns $BOOT_ARGS +# Issue #157: reclaim and throttle above 85% of RAM rather than let the OOM killer take the node +# mid-burst (it did 24 times in 8 h on the 2 GB validators). No MemoryMax: a cap kills, which is +# what this avoids. Pair it with swap; 4 GB is the validator minimum (docs/node-hardware.md). +MemoryHigh=85% Restart=always RestartSec=3 [Install] diff --git a/docs/cli.md b/docs/cli.md index 23d758a1..8f546d8b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -408,6 +408,7 @@ one the cut announced; a mismatch on one node is almost always an old binary. | `--prover-cpu` | off | prove on the CPU even when a GPU is visible to a CUDA build | | `--prover-threads ` | the cores minus one, at most 8 | CPU threads the hosted prover's proofs use (`RAYON_NUM_THREADS` when set and no flag) | | `--verify-workers ` | the cores minus two, at least four | the proof-verification workers of admission and the consensus loop's lookups; the queue and in-flight limits follow it (spec 2026-10-05 §6, `docs/compute-optimization.md` §3.3) | +| `--admission-budget-mb ` | 1/16 of the host's RAM, within 64–256 MiB, never under one largest transaction | the admission byte budget (issue #157): RPC request bodies plus transactions waiting for verification. Past it `rand_sendTransaction` answers `-32005` (busy, retry) and gossiped transactions are shed with `Ignore`; a value under one largest transaction (body cap plus block cap) is refused at startup (`docs/rpc.md`) | | `--apply-proposals-inline` | off | restore the inline apply: incoming proposals are applied on the consensus loop, as before v0.8.0's off-loop apply; by default the apply runs on a blocking thread (at most 4 at once, up to 8 waiting behind a parent, at most 2 per proposer), with the prechecks, `on_proposal`'s checks and the insert still on the loop (`docs/compute-optimization.md` §3.3). The by-hash fetch answer binding (an answer that is not the block asked for is dropped) is a separate fix and stays on with this flag | | `--prover-skip-memory-check` | off | skip the free-memory gate | | `--prover-fee ` | none | the fee every job sent to the hosted prover must pay, in RAND (display units, up to 9 decimals): one RAND output to `--prover-fee-address` inside the bundle proved; needs `--prover-fee-address`, and only a v3 (split-authorisation) witness can carry it (`docs/prover.md` §3.5) | diff --git a/docs/node-hardware.md b/docs/node-hardware.md index 438010f2..b3844c6a 100644 --- a/docs/node-hardware.md +++ b/docs/node-hardware.md @@ -81,7 +81,13 @@ Sources: `deploy/nodes.env`, `deploy/README.md`. Rules that follow from these numbers: -- **2 GB of RAM runs a validator** on this chain today. The fleet's `s-1vcpu-2gb` droplets do it. +- **4 GB of RAM is the validator minimum** (issue #157). A 2 GB droplet runs a validator at rest + (~1.4 GB resident), but the 2026-10-27 overnight run OOM-killed the 2 GB validators 24 times in + 8 h, each after a burst of eight ~2.9 MB transactions; the one 4 GB node in a burst survived + it. v0.8.3 bounds admission memory (`--admission-budget-mb`, 1/16 of RAM by default, ~117 MiB + on a 2 GB host), and the unit templates (`deploy/vps-setup.sh`, `deploy/cutover-droplet-rand.sh`) + set `MemoryHigh=85%` so the kernel reclaims and throttles before it kills. A 2 GB host that has + to stay on needs both, plus swap (the fleet's have 1 GB). - **Disk, not RAM, is the limit.** Keep at least 20 GB free for builds (`docs/block-space.md` §5) and delete a retired chain's data dir after a cut-over (`deploy/retire-chain-dirs.sh`). A validator runs `--prune-history 24h`; a node refuses to start under 1 GB free and diff --git a/docs/rpc.md b/docs/rpc.md index 82d335e3..3527b05b 100644 --- a/docs/rpc.md +++ b/docs/rpc.md @@ -158,6 +158,30 @@ On a genesis with `hc_auth_multisig` the first term counts **three** proofs at t (`3 × max_proof_bytes`: bundle, multisig auth, call), since the multisig auth proof is not the single-key guest's small fixed shape (`rpc::rpc_max_body_bytes`). +**What a hash means** (issue #157). The hash comes back only after the transaction has passed +every check above, the proofs included, has been pooled at the current tip, and has been handed +to this node's gossip for publishing (`Node::on_verdict`). This has been so since admission moved +off the loop; nothing acknowledges a transaction that is only queued. The hand-off is to the +node's own network task, so a node killed in the moment after it answers can still lose the +transaction before the first peer has it. The OOM kills of 2026-10-27 that lost 18 acknowledged +transactions were that window; the admission budget below is what closes them. Acceptance is +still not commitment. + +**Busy, retry: `-32005`** (issue #157). Every POST body is charged against the node's admission +byte budget before it is read, and held until the answer; a transaction is charged again, by its +decoded size, while it waits for verification. A request the budget cannot take now is answered at +once with HTTP 503, `Retry-After: 1` and + +```json +{ "jsonrpc": "2.0", "id": null, "error": { "code": -32005, + "message": "busy: this node's admission memory is full (120 of 123 MiB in use, 5627 KiB wanted); retry shortly", + "data": { "wanted": 5761934, "available": 2516582, "limit": 128974848 } } } +``` + +Nothing was queued: send the same request again in a moment. `rand` does (it backs off on +`-32005` five times, about 31 s). A gossiped transaction that does not fit is reported `Ignore`, so +its forwarder is never penalised. + ### `rand_mint` (testnet faucet) Params: `[address]` or `[address, amount]`, where `address` is a `rand1…` shielded address and `amount` is a string of units, at most `100000000000` (100 RAND; the default). Result: the mint @@ -1065,7 +1089,7 @@ Params: `[]`. Result: "syncing": false, "sync_target": 1998, "sync_inflight_age_ms": null, "sync_failures": 0, "sync_late_batches": 0, "compact_fetched": 0, "peer_count": 5, "connected_peers": 5, "reserved_peers": 5, "ws_clients": 3, "refused_cache": 0, - "verify_queue": 0, "mempool_size": 0, + "verify_queue": 0, "admission_bytes": 0, "admission_budget_bytes": 123000000, "mempool_size": 0, "is_validator": true, "active_validator": true, "faucet": true, "confidential": true, "testnet": false, "fri_profile": "production", "programs": 2, "viewing_keys": 0, "notes": 41, "nullifiers": 12, "tree_root": "6b1d…c4", "hc_bundle": "f07a…19", "hc_auth": null, "hc_auth_multisig": null, @@ -1119,6 +1143,11 @@ otherwise looks identical to a node that is behind and working: off the consensus loop. 64 deep at most; a queue that stays full means verifications are arriving faster than ~20 ms apiece drains them, and what does not fit is shed — an honest peer re-gossips on its next heartbeat — rather than queued unboundedly. +- `admission_bytes` / `admission_budget_bytes` (issue #157) — bytes charged right now against the + admission budget, and the budget (`--admission-budget-mb`; default 1/16 of the host's RAM within + 64–256 MiB, never under one largest transaction). The charges are every POST body being served + and every transaction waiting for or under verification, gossiped or submitted. At the budget, + `rand_sendTransaction` answers `-32005` and gossiped transactions are shed with `Ignore`. `is_validator` says this node holds a validator key; `active_validator` says that key is in the set running the current epoch (spec §8) — a validator that has bonded in