diff --git a/Cargo.toml b/Cargo.toml index 11177ef..a782859 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -117,9 +117,14 @@ harness = false name = "product_zipper" harness = false +[[bench]] +name = "catamorphism" +harness = false + [[bench]] name = "sla" harness = false +required-features = ["viz"] [[bench]] name = "multiplicities" diff --git a/benches/act_paths.rs b/benches/act_paths.rs index b95691b..13b1a58 100644 --- a/benches/act_paths.rs +++ b/benches/act_paths.rs @@ -26,7 +26,7 @@ use divan::{Bencher, Divan, counter::ItemsCount}; use pathmap::PathMap; use pathmap::arena_compact::{ACTOutputStream, ArenaCompactTree}; use pathmap::paths_serialization::{for_each_deserialized_path, serialize_paths}; -use pathmap::zipper::ZipperMoving; +use pathmap::morphisms::CatamorphismCachedIterative; use rand::{Rng, SeedableRng, rngs::StdRng}; use std::path::Path; diff --git a/benches/binary_keys.rs b/benches/binary_keys.rs index e1804cd..b886de6 100644 --- a/benches/binary_keys.rs +++ b/benches/binary_keys.rs @@ -115,7 +115,7 @@ fn binary_descend_until_max_bytes(bencher: Bencher, n: u64) { }); } -#[divan::bench(args = [125, 250, 500, 1000, 2000, 4000])] +#[divan::bench(args = [125, 250, 500, 1000, 2000, 4000, 100000])] fn binary_val_count_bench(bencher: Bencher, n: u64) { let keys = make_keys(n as usize, 1); diff --git a/benches/catamorphism.rs b/benches/catamorphism.rs new file mode 100644 index 0000000..34e1eae --- /dev/null +++ b/benches/catamorphism.rs @@ -0,0 +1,140 @@ +use divan::{Divan, Bencher, black_box}; +use core::convert::Infallible; +use pathmap::alloc::GlobalAlloc; +use pathmap::morphisms::CatamorphismCached; +use pathmap::utils::ByteMask; +use pathmap::utils::ints::gen_int_range; +use pathmap::PathMap; + +fn main() { + // Run registered benchmarks. + let divan = Divan::from_args() + .sample_count(4000); + + divan.main(); +} + +fn build_map(count: u64) -> PathMap<()> { + // Dense range of u64 keys encoded as paths; sized to keep benches fast and stable. + gen_int_range::<(), 8, u64>(0, count, 1, ()) +} + +const MAP_COUNT: u64 = 20_000_000; + +// A complete binary trie keeps every internal node as a two-entry LineListNode. +const BINARY_TREE_DEPTH: usize = 18; +const BINARY_TREE_LEAF_COUNT: usize = 1 << BINARY_TREE_DEPTH; + +fn build_binary_tree_map() -> PathMap<()> { + let mut map = PathMap::new(); + for leaf in 0..BINARY_TREE_LEAF_COUNT { + let mut path = [0u8; BINARY_TREE_DEPTH]; + for (level, byte) in path.iter_mut().enumerate() { + *byte = ((leaf >> (BINARY_TREE_DEPTH - level - 1)) & 1) as u8; + } + map.insert(path, ()); + } + map +} + +#[divan::bench()] +fn factored_cata_jumping_val_count(bencher: Bencher) { + let map = build_map(MAP_COUNT); + let mut sink = 0usize; + bencher.bench_local(|| { + let rz = map.read_zipper(); + *black_box(&mut sink) = CatamorphismCached::<(), GlobalAlloc>::factored_cata_jumping::<_, _, Infallible, _, _, _, false>(&rz, + |_| Ok(0usize), + |_mask, w: usize, total| { *total += w; Ok(()) }, + |_mask, v, total, _| Ok((v.is_some() as usize) + total.unwrap_or(0)), + ).unwrap(); + }); + assert_eq!(sink, MAP_COUNT as usize); +} + +#[divan::bench()] +fn factored_cata_binary_tree_leaf_count(bencher: Bencher) { + let map = build_binary_tree_map(); + let mut sink = 0usize; + bencher.bench_local(|| { + let rz = map.read_zipper(); + *black_box(&mut sink) = CatamorphismCached::<(), GlobalAlloc> + ::factored_cata_jumping::<_, _, Infallible, _, _, _, false>(&rz, + |_| Ok(0usize), + |_mask, child_count: usize, total| { + *total += child_count; + Ok(()) + }, + |_mask, value, total, _| Ok((value.is_some() as usize) + total.unwrap_or(0)), + ) + .unwrap(); + }); + assert_eq!(sink, BINARY_TREE_LEAF_COUNT); +} + +#[divan::bench()] +fn cached_jumping_cata_val_count(bencher: Bencher) { + let map = build_map(MAP_COUNT); + let mut sink = 0usize; + bencher.bench_local(|| { + let rz = map.read_zipper(); + *black_box(&mut sink) = CatamorphismCached::<(), GlobalAlloc>::cata_jumping_cached(&rz, |_mask: &ByteMask, children: &mut [usize], val, _sub_path| { + let mut sum: usize = children.iter().sum(); + if val.is_some() { + sum += 1; + } + sum + }); + }); + assert_eq!(sink, MAP_COUNT as usize); +} + +#[divan::bench()] +fn factored_cata_jumping_total_len(bencher: Bencher) { + let map = build_map(MAP_COUNT); + let mut sink = (0usize, 0usize); + bencher.bench_local(|| { + let rz = map.read_zipper(); + *black_box(&mut sink) = CatamorphismCached::<(), GlobalAlloc>::factored_cata_jumping::<_, _, Infallible, _, _, _, true>(&rz, + |_| Ok((0usize, 0usize)), + |_mask: &ByteMask, w: (usize, usize), acc: &mut (usize, usize)| { + acc.0 += w.0; + // Every folded child hangs below exactly one branch byte. `prefix` accounts + // for compressed runs separately in `summarize_f` below. + acc.1 += w.1 + w.0; + Ok(()) + }, + |_mask: &ByteMask, val, acc, prefix| { + let (count, total_len) = acc.unwrap_or((0, 0)); + let count = count + val.is_some() as usize; + Ok((count, total_len + count * prefix.len())) + }, + ).unwrap(); + }); + assert_eq!(sink, (MAP_COUNT as usize, MAP_COUNT as usize * 8)); +} + +#[divan::bench()] +fn cached_jumping_cata_total_len(bencher: Bencher) { + let map = build_map(MAP_COUNT); + let mut sink = (0usize, 0usize); + bencher.bench_local(|| { + let rz = map.read_zipper(); + *black_box(&mut sink) = CatamorphismCached::<(), GlobalAlloc>::cata_jumping_cached(&rz, |mask: &ByteMask, children: &mut [(usize, usize)], val, sub_path| { + let mut count = 0usize; + let mut total_len = 0usize; + let prefix_len = sub_path.len(); + if val.is_some() { + count += 1; + total_len += prefix_len; + } + for (_byte, child) in mask.iter().zip(children.iter_mut()) { + count += child.0; + // The child is below one mask byte as well as this callback's prefix. + total_len += child.1 + child.0 * (prefix_len + 1); + } + (count, total_len) + }); + }); + assert_eq!(sink, (MAP_COUNT as usize, MAP_COUNT as usize * 8)); +} diff --git a/benches/cities.rs b/benches/cities.rs index 231cb6e..9c27c9a 100644 --- a/benches/cities.rs +++ b/benches/cities.rs @@ -171,10 +171,7 @@ fn cities_val_count(bencher: Bencher) { #[cfg(feature="arena_compact")] #[divan::bench()] fn cities_val_count_act(bencher: Bencher) { - use pathmap::{ - arena_compact::ArenaCompactTree, - zipper::ZipperMoving, - }; + use pathmap::{morphisms::CatamorphismCachedIterative, arena_compact::ArenaCompactTree}; let pairs = read_data(); let mut map = PathMap::new(); let mut unique_count = 0; diff --git a/benches/multiplicities.rs b/benches/multiplicities.rs index 4bdd753..99ed9d4 100644 --- a/benches/multiplicities.rs +++ b/benches/multiplicities.rs @@ -1,5 +1,5 @@ use pathmap::*; -use pathmap::zipper::{ZipperMoving, ZipperWriting}; +use pathmap::zipper::{CatamorphismCached, ZipperWriting}; fn main() { const SILLY_LARGE_COUNTS: bool = false; diff --git a/benches/oeis.rs b/benches/oeis.rs index afb485f..a179b79 100644 --- a/benches/oeis.rs +++ b/benches/oeis.rs @@ -1,7 +1,6 @@ use std::io::Read; -use std::usize; use pathmap::PathMap; -use pathmap::zipper::{Zipper, ZipperValues, ZipperMoving, ZipperPath, ZipperWriting, ZipperCreation}; +use pathmap::zipper::{Zipper, ZipperValues, ZipperMoving, ZipperPath, ZipperWriting, ZipperCreation, CatamorphismCached}; use num::BigInt; use divan::{Divan, Bencher, black_box}; diff --git a/benches/product_zipper.rs b/benches/product_zipper.rs index fcac057..c128e66 100644 --- a/benches/product_zipper.rs +++ b/benches/product_zipper.rs @@ -89,7 +89,7 @@ fn val_count_cata(_bm: &ByteMask, vals: &mut[usize], _val: Option<&V>, _path: fn introspecting_pathmap_pathmap(bencher: Bencher) { use pathmap::{ PathMap, - morphisms::Catamorphism, + morphisms::CatamorphismSideEffecting, zipper::{ProductZipper}, }; let mut sink = 0; @@ -106,7 +106,7 @@ fn introspecting_pathmap_pathmap(bencher: Bencher) { fn generic_pathmap_pathmap(bencher: Bencher) { use pathmap::{ PathMap, - morphisms::Catamorphism, + morphisms::CatamorphismSideEffecting, zipper::{ProductZipperG}, }; let mut sink = 0; @@ -125,7 +125,7 @@ fn generic_act_act(bencher: Bencher) { use pathmap::{ PathMap, arena_compact::{ArenaCompactTree}, - morphisms::Catamorphism, + morphisms::CatamorphismSideEffecting, zipper::{ProductZipperG}, }; let mut sink = 0; @@ -146,7 +146,7 @@ fn generic_pathmap_act(bencher: Bencher) { use pathmap::{ PathMap, arena_compact::{ArenaCompactTree}, - morphisms::Catamorphism, + morphisms::CatamorphismSideEffecting, zipper::{ProductZipperG}, }; let mut sink = 0; diff --git a/benches/shakespeare.rs b/benches/shakespeare.rs index 2040ba5..67f754d 100644 --- a/benches/shakespeare.rs +++ b/benches/shakespeare.rs @@ -171,10 +171,7 @@ fn shakespeare_sentences_val_count(bencher: Bencher) { #[cfg(feature="arena_compact")] #[divan::bench()] fn shakespeare_sentences_val_count_act(bencher: Bencher) { - use pathmap::{ - arena_compact::ArenaCompactTree, - zipper::ZipperMoving, - }; + use pathmap::{morphisms::CatamorphismCachedIterative, arena_compact::ArenaCompactTree}; let strings = read_data(false); let mut map = PathMap::new(); let mut unique_count = 0; diff --git a/benches/sla.rs b/benches/sla.rs index b6f0432..02b3763 100644 --- a/benches/sla.rs +++ b/benches/sla.rs @@ -1,4 +1,3 @@ -#![allow(unused)] use std::hash::{Hasher, Hash}; use std::time::Instant; use num_traits::Zero; @@ -7,7 +6,7 @@ use rand::prelude::StdRng; use rand::{Rng, SeedableRng}; use rand_distr::Distribution; use pathmap::*; -use pathmap::morphisms::Catamorphism; +use pathmap::morphisms::CatamorphismCached; use pathmap::ring::{AlgebraicResult, Lattice}; use pathmap::utils::{BitMask, ByteMask, ints::{indices_to_weave, indices_to_bob}}; use pathmap::zipper::{ReadZipperUntracked, WriteZipperUntracked, Zipper, ZipperMoving, ZipperValues, ZipperWriting}; @@ -643,4 +642,4 @@ fn main() { tipover_attention_bob(); tipover_attention_weave(0.02); -} \ No newline at end of file +} diff --git a/benches/sparse_keys.rs b/benches/sparse_keys.rs index e010d2f..d8c905f 100644 --- a/benches/sparse_keys.rs +++ b/benches/sparse_keys.rs @@ -134,7 +134,7 @@ fn sparse_descend_until_max_bytes(bencher: Bencher, n: u64) { }); } -#[divan::bench(args = [125, 250, 500, 1000, 2000, 4000])] +#[divan::bench(args = [125, 250, 500, 1000, 2000, 4000, 20_000, 100_000])] fn sparse_val_count_bench(bencher: Bencher, n: u64) { let mut r = StdRng::seed_from_u64(1); diff --git a/benches/superdense_keys.rs b/benches/superdense_keys.rs index 0371789..f53c386 100644 --- a/benches/superdense_keys.rs +++ b/benches/superdense_keys.rs @@ -311,7 +311,7 @@ fn from_prefix_key(k: Vec) -> u64 { u64::from_le_bytes(buf) & (!0u64 >> shift) } -#[divan::bench(sample_size = 1, args = [100, 200, 400, 800, 1600, 3200, 20_000])] +#[divan::bench(sample_size = 1, args = [100, 200, 400, 800, 1600, 3200, 20_000, 100_000])] fn superdense_val_count_bench(bencher: Bencher, n: u64) { let mut map: PathMap = PathMap::new(); @@ -326,12 +326,9 @@ fn superdense_val_count_bench(bencher: Bencher, n: u64) { } #[cfg(feature="arena_compact")] -#[divan::bench(sample_size = 1, args = [100, 200, 400, 800, 1600, 3200, 20_000])] +#[divan::bench(sample_size = 1, args = [100, 200, 400, 800, 1600, 3200, 20_000, 100_000])] fn superdense_val_count_bench_act(bencher: Bencher, n: u64) { - use pathmap::{ - arena_compact::ArenaCompactTree, - zipper::ZipperMoving, - }; + use pathmap::{morphisms::CatamorphismCachedIterative, arena_compact::ArenaCompactTree}; let mut map: PathMap = PathMap::new(); for i in 0..n { map.set_val_at(prefix_key(&i), i); } let act = ArenaCompactTree::from_zipper(map.read_zipper(), |&v| v); diff --git a/pathmap-derive/src/lib.rs b/pathmap-derive/src/lib.rs index 5111c6d..05d0a78 100644 --- a/pathmap-derive/src/lib.rs +++ b/pathmap-derive/src/lib.rs @@ -490,12 +490,6 @@ fn derive_poly_zipper_with_traits( } } - fn val_count(&self) -> usize { - match self { - #(#variant_arms => inner.val_count(),)* - } - } - fn descend_to>(&mut self, k: K) { match self { #(#variant_arms => inner.descend_to(k),)* @@ -653,6 +647,12 @@ fn derive_poly_zipper_with_traits( impl #impl_generics pathmap::zipper::ZipperPathBuffer for #enum_name #ty_generics #zipper_path_buffer_where { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { + match self { + #(#variant_arms => unsafe { inner.path_assert_len(len) },)* + } + } + unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { match self { #(#variant_arms => unsafe { inner.origin_path_assert_len(len) },)* diff --git a/src/arena_compact.rs b/src/arena_compact.rs index 9ee4a49..fe70d3f 100644 --- a/src/arena_compact.rs +++ b/src/arena_compact.rs @@ -85,7 +85,7 @@ use crate::alloc::{GlobalAlloc, global_alloc}; use crate::timed_span::{TimingEntries::*, COUNTERS, timed_span}; use crate::{ PathMap, - morphisms::Catamorphism, + morphisms::CatamorphismSideEffecting, utils::{BitMask, ByteMask, find_prefix_overlap}, zipper::{ Zipper, ZipperValues, ZipperForking, ZipperAbsolutePath, ZipperIteration, @@ -896,7 +896,7 @@ impl ArenaCompactTree> { pub fn from_zipper(zipper: Z, map: M) -> Self where V: Clone + Send + Sync + Unpin, - Z: Catamorphism, + Z: CatamorphismSideEffecting, M: Fn(&V) -> u64, { build_arena_tree(zipper, map) @@ -1066,7 +1066,7 @@ impl ArenaCompactTree { ) -> Result where V: Clone + Send + Sync + Unpin, - Z: Catamorphism, + Z: CatamorphismSideEffecting, F: Fn(&V) -> u64, P: AsRef { @@ -1146,7 +1146,7 @@ impl NodeBranch { fn build_arena_tree(zipper: Z, map_val: F) -> ArenaCompactTree> where V: Clone + Send + Sync + Unpin, - Z: Catamorphism, + Z: CatamorphismSideEffecting, F: Fn(&V) -> u64, { let mut arena = ArenaCompactTree::new(); @@ -1354,10 +1354,9 @@ struct CachedFrame { /// contiguous — and the copy keeps pointing at the original children. So a /// repeated subtrie costs one node, not a subtrie. /// -/// The traversal itself is the jumping catamorphism, unrolled (see -/// `morphisms::into_cata_cached_body`, which this follows closely). It is +/// The traversal itself is an unrolled jumping catamorphism. It is /// spelled out here rather than delegating to -/// [`Catamorphism::into_cata_jumping_cached`] for two reasons: +/// [`CatamorphismCached::cata_jumping_cached`] for two reasons: /// - the cached cata only consults its cache one byte below a fork, whereas we /// also consult it at the fork we land on after jumping over a chain of /// bytes. That is where a subtrie grafted under a multi-byte path shows up, @@ -1368,7 +1367,7 @@ struct CachedFrame { /// - the cached cata's algebra is an `Fn`, so writing to the arena from it /// would need interior mutability. /// -/// TODO: GOAT: introduce an abstraction/modify `into_cata_jumping_cached`, +/// TODO: GOAT: introduce an abstraction/modify `cata_jumping_cached`, /// to address the problems listed above. The suggested API is to have /// `FnMut` variant for the caching catamorphism. /// @@ -1576,7 +1575,7 @@ fn dump_arena_tree( ) -> Result, std::io::Error> where V: Clone + Send + Sync + Unpin, - Z: Catamorphism, + Z: CatamorphismSideEffecting, F: Fn(&V) -> u64, P: AsRef, { @@ -2518,6 +2517,10 @@ where Storage: AsRef<[u8]> impl<'tree, Storage, Value> ZipperPathBuffer for ACTZipper<'tree, Storage, Value> where Storage: AsRef<[u8]> { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { + assert!(len <= self.path.capacity() - self.origin_depth); + unsafe{ core::slice::from_raw_parts(self.path.as_ptr().add(self.origin_depth), len) } + } unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { // Safety: we're not creating a slice larger than capacity assert!(self.path.capacity() >= len); @@ -2915,23 +2918,6 @@ where Storage: AsRef<[u8]> self.invalid = 0; } - /// Returns the total number of values contained at and below the zipper's focus, including the focus itself - /// - /// WARNING: This is not a cheap method. It may have an order-N cost - fn val_count(&self) -> usize { - timed_span!(ValueCount, COUNTERS); - let mut zipper = self.clone(); - zipper.reset(); - let mut count = 0; - if zipper.is_val() { - count += 1; - } - while zipper.to_next_val() { - count += 1; - } - count - } - /// Moves the zipper deeper into the trie, to the `key` specified relative to the current zipper focus /// /// Returns `true` if the zipper points to an existing path within the tree, otherwise `false`. The @@ -3315,7 +3301,8 @@ where mod tests { use super::{ArenaCompactTree, ACTZipper}; use crate::{ - morphisms::Catamorphism, PathMap, zipper::{zipper_iteration_tests, zipper_moving_tests, ZipperIteration, ZipperMoving, ZipperPath, ZipperValues} + morphisms::CatamorphismSideEffecting, + PathMap, zipper::{zipper_iteration_tests, zipper_moving_tests, ZipperIteration, ZipperMoving, ZipperPath, ZipperValues} }; zipper_moving_tests::zipper_moving_tests!(arena_compact_zipper, @@ -3378,6 +3365,16 @@ mod tests { } } + crate::morphisms::cached_catamorphism_tests::cached_catamorphism_tests!( + act_zipper, + |keys: &[&[u8]]| { + let map = keys.iter().enumerate().map(|(idx, path)| (*path, idx as u64)).collect::>(); + ArenaCompactTree::from_zipper(map.read_zipper(), |&value| value) + }, + |tree: &mut ArenaCompactTree>| tree.read_zipper_u64(), + CatamorphismCachedIterative + ); + /// Build `map` both ways and check the results describe the same trie. /// /// When there is nothing to re-use the two builders must agree byte for diff --git a/src/dense_byte_node.rs b/src/dense_byte_node.rs index 5ffdb0f..ed4aeb2 100644 --- a/src/dense_byte_node.rs +++ b/src/dense_byte_node.rs @@ -1,7 +1,6 @@ use core::fmt::{Debug, Formatter}; use core::ptr; -use std::collections::HashMap; use std::hint::unreachable_unchecked; use crate::alloc::Allocator; @@ -10,7 +9,9 @@ use crate::utils::ByteMask; use crate::utils::BitMask; use crate::trie_node::*; +use crate::gxhash::HashMap; use crate::line_list_node::LineListNode; +use crate::morphisms::summarize_run; //NOTE: This: `core::array::from_fn(|i| i as u8);` ought to work, but https://github.com/rust-lang/rust/issues/109341 const ALL_BYTES: [u8; 256] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255]; @@ -365,6 +366,10 @@ impl> ByteNode /// Iterates the entries in `self`, calling `func` for each entry /// The arguments to `func` are: `func(self, key_byte, n)`, where `n` is the number of times /// `func` has been called prior. This corresponds to index of the `CoFree` in the `values` vec + /// + /// PERF GOAT: this method is useful if you know you need the corresponding path byte. If, however, the + /// byte might be unneeded, it's better to let self.values govern the loop, because using the mask + /// in the loop means the bitwise ops to manipulate the mask are always required. #[inline] fn for_each_item(&self, mut func: F) { let mut n = 0; @@ -381,6 +386,46 @@ impl> ByteNode } } } + + #[inline(always)] + pub(crate) fn node_recursive_cata(&self, passed_in_val: Option<&V>, start_f: StartF, fold_child_f: FoldChildF, finalize_f: FinalizeF, cache: &mut HashMap) -> Result + where + W: Clone, + StartF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + FinalizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8]) -> Result, + { + let mask = &self.mask; + let mut ws = start_f(mask)?; + for cf in self.values.iter() { + let path = &[]; + + //Do the recursive calling + //PERF NOTE: The reason we have four code paths around the call to `branch_f` instead of just doing + // `let w = cf.rec().map(|rec| recursive_cata(...))` and then calling branch_f with the appropriate + // pair of `Option<&V>` and `Option` is that the compiler won't optimize the implemntation of + // `branch_f` around whether the options are none or not. That means we often pay for two dependent + // branches instead of one, and the difference was 25% to the val_count benchmark. Breaking out the calls + // like this gives the optimizer a site to specialize for each permutation + match (cf.rec(), cf.val()) { + (Some(rec), Some(val)) => { + let w = recursive_cata_cached::<_, _, _, _, _, _, _, _, COMPUTE_PATH>(rec, Some(val), start_f, fold_child_f, finalize_f, cache)?; + fold_child_f(mask, w, &mut ws)?; + }, + (Some(rec), None) => { + let w = recursive_cata_cached::<_, _, _, _, _, _, _, _, COMPUTE_PATH>(rec, None, start_f, fold_child_f, finalize_f, cache)?; + fold_child_f(mask, summarize_run::<_, _, _, _, _, _, _, COMPUTE_PATH>(None, Some(w), path, start_f, fold_child_f, finalize_f)?, &mut ws)?; + }, + (None, Some(val)) => { + fold_child_f(mask, summarize_run::<_, _, _, _, _, _, _, COMPUTE_PATH>(Some(val), None, path, start_f, fold_child_f, finalize_f)?, &mut ws)?; + }, + (None, None) => { + fold_child_f(mask, summarize_run::<_, _, _, _, _, _, _, COMPUTE_PATH>(None, None, path, start_f, fold_child_f, finalize_f)?, &mut ws)?; + }, + } + } + finalize_f(mask, passed_in_val, Some(ws), &[]) + } } impl> ByteNode where Self: TrieNodeDowncast { @@ -1012,34 +1057,13 @@ impl> TrieNode let k = k as usize; (next_token, &ALL_BYTES[k..=k], cf.rec(), cf.val()) } - fn node_val_count(&self, cache: &mut HashMap) -> usize { - //Discussion: These two implementations do the same thing but with a slightly different ordering of - // the operations. In `all_dense_nodes`, the "Branchy" impl wins. But in a mixed-node setting, the - // IMPL B is the winner. My suspicion is that the ListNode's heavily branching structure leads to - // underutilization elsewhere in the CPU so we get better instruction parallelism with IMPL B. - - //IMPL A "Branchy" - // let mut result = 0; - // for cf in self.values.iter() { - // if cf.value.is_some() { - // result += 1; - // } - // match &cf.rec { - // Some(rec) => result += rec.borrow().node_subtree_len(), - // None => {} - // } - // } - // result - - //IMPL B "Arithmetic" - return self.values.iter().rfold(0, |t, cf| { - t + cf.has_val() as usize + cf.rec().map(|r| val_count_below_node(r, cache)).unwrap_or(0) - }); - } - fn node_goat_val_count(&self) -> usize { - return self.values.iter().rfold(0, |t, cf| { - t + cf.has_val() as usize - }); + #[inline] + fn node_val_count(&self) -> usize { + let mut result = 0; + for cf in self.values.iter() { + result += cf.has_val() as usize + } + result } fn node_child_iter_start(&self) -> (u64, Option<&TrieNodeODRc>) { for (pos, cf) in self.values.iter().enumerate() { diff --git a/src/dependent_zipper.rs b/src/dependent_zipper.rs index fe328c7..2dc413e 100644 --- a/src/dependent_zipper.rs +++ b/src/dependent_zipper.rs @@ -219,6 +219,7 @@ impl<'trie, PrimaryZ, SecondaryZ, V, C, F : Clone + for <'a> FnOnce(C, &'a [u8], PrimaryZ: ZipperMoving + ZipperPath + ZipperPathBuffer, SecondaryZ: ZipperMoving + ZipperPathBuffer, { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { unsafe{ self.primary.path_assert_len(len) } } unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { unsafe{ self.primary.origin_path_assert_len(len) } } fn prepare_buffers(&mut self) { self.primary.prepare_buffers() } fn reserve_buffers(&mut self, path_len: usize, stack_depth: usize) { self.primary.reserve_buffers(path_len, stack_depth) } @@ -347,9 +348,6 @@ impl<'trie, PrimaryZ, SecondaryZ, V, C, F : Clone + for <'a> FnOnce(C, &'a [u8], self.secondary.clear(); self.primary.reset(); } - fn val_count(&self) -> usize { - unimplemented!("method will probably get removed") - } fn descend_to_existing>(&mut self, path: K) -> usize { let mut path = path.as_ref(); let mut descended = 0; diff --git a/src/empty_node.rs b/src/empty_node.rs index 722952a..3477cba 100644 --- a/src/empty_node.rs +++ b/src/empty_node.rs @@ -1,6 +1,5 @@ use core::fmt::Debug; -use std::collections::HashMap; use crate::alloc::Allocator; use crate::trie_node::*; @@ -67,10 +66,7 @@ impl TrieNode for EmptyNode { fn next_items(&self, _token: IterToken) -> (IterToken, &[u8], Option<&TrieNodeODRc>, Option<&V>) { (NODE_ITER_FINISHED, &[], None, None) } - fn node_val_count(&self, _cache: &mut HashMap) -> usize { - 0 - } - fn node_goat_val_count(&self) -> usize { + fn node_val_count(&self) -> usize { 0 } fn node_child_iter_start(&self) -> (u64, Option<&TrieNodeODRc>) { diff --git a/src/empty_zipper.rs b/src/empty_zipper.rs index 16860fc..66225d8 100644 --- a/src/empty_zipper.rs +++ b/src/empty_zipper.rs @@ -36,7 +36,6 @@ impl ZipperMoving for EmptyZipper { #[inline] fn focus_byte(&self) -> Option { self.path.last().cloned() } fn reset(&mut self) { self.path.truncate(self.path_start_idx) } - fn val_count(&self) -> usize { 0 } fn descend_to>(&mut self, k: K) { self.path.extend_from_slice(k.as_ref()); } @@ -118,6 +117,10 @@ impl<'a, V: Clone + Send + Sync> ZipperReadOnlyConditionalIteration<'a, V> for E } impl ZipperPathBuffer for EmptyZipper { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { + assert!(len <= self.path.capacity() - self.path_start_idx); + unsafe{ core::slice::from_raw_parts(self.path.as_ptr().add(self.path_start_idx), len) } + } unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { assert!(len <= self.path.capacity()); unsafe{ core::slice::from_raw_parts(self.path.as_ptr(), len) } diff --git a/src/experimental.rs b/src/experimental.rs index f39c2ca..fa38217 100644 --- a/src/experimental.rs +++ b/src/experimental.rs @@ -31,6 +31,10 @@ impl Zipper for FullZipper { } impl ZipperPathBuffer for FullZipper { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { + assert!(len <= self.path.capacity()); + unsafe{ core::slice::from_raw_parts(self.path.as_ptr(), len) } + } unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { assert!(len <= self.path.capacity()); unsafe{ core::slice::from_raw_parts(self.path.as_ptr(), len) } @@ -49,7 +53,6 @@ impl ZipperMoving for FullZipper { #[inline] fn focus_byte(&self) -> Option { self.path.last().cloned() } fn reset(&mut self) { self.path.clear() } - fn val_count(&self) -> usize { usize::MAX/2 } // usize::MAX is a dangerous default for overflow fn descend_to>(&mut self, k: K) { self.path.extend_from_slice(k.as_ref()); } @@ -122,7 +125,6 @@ impl ZipperMoving for NullZipper { #[inline] fn focus_byte(&self) -> Option { None } fn reset(&mut self) {} - fn val_count(&self) -> usize { 0 } fn descend_to>(&mut self, _k: K) {} fn descend_to_byte(&mut self, _k: u8) {} fn descend_indexed_byte(&mut self, _idx: usize) -> Option { None } diff --git a/src/experimental/serialization.rs b/src/experimental/serialization.rs index 423fe93..6b7ddfa 100644 --- a/src/experimental/serialization.rs +++ b/src/experimental/serialization.rs @@ -2,7 +2,7 @@ use std::{any::type_name, hash::Hasher, io::{BufRead, BufReader, BufWriter, Read, Seek, Write}, path::PathBuf}; -use crate::{morphisms::Catamorphism, PathMap, zipper::{ZipperMoving, ZipperWriting}}; +use crate::{morphisms::CatamorphismSideEffecting, PathMap, zipper::{ZipperMoving, ZipperWriting}}; use crate::TrieValue; extern crate alloc; use alloc::collections::BTreeMap; @@ -83,7 +83,7 @@ pub const META_DATA_FILENAME : &'static str = "meta.json"; /// Filename of the zero compressesed data file at the `out_dir_path` formal parameter in [`write_trie`] pub const ZERO_COMPRESSED_HEX_DATA_FILENAME : &'static str = "zero_compressed_hex.data"; -pub fn write_trie ,V: TrieValue>( +pub fn write_trie ,V: TrieValue>( memo : impl AsRef, cata : C, serialize_value : impl for<'read, 'encode> Fn(&'read V, &'encode mut Vec)->ValueSlice<'read, 'encode>, diff --git a/src/experimental/tree_serialization.rs b/src/experimental/tree_serialization.rs index 936d35e..111754b 100644 --- a/src/experimental/tree_serialization.rs +++ b/src/experimental/tree_serialization.rs @@ -3,12 +3,12 @@ use std::io::Write; use std::ptr::slice_from_raw_parts; use crate::alloc::Allocator; use crate::{zipper, TrieValue}; -use crate::morphisms::{Catamorphism, new_map_from_ana_jumping}; +use crate::morphisms::{CatamorphismSideEffecting, new_map_from_ana_jumping}; use crate::utils::{BitMask, ByteMask}; use crate::write_zipper::ZipperWriting; /// WIP -pub fn serialize_fork, F: FnMut(usize, &[u8], &V) -> ()>(rz: RZ, target: &mut Vec, _fv: F) -> std::io::Result { +pub fn serialize_fork, F: FnMut(usize, &[u8], &V) -> ()>(rz: RZ, target: &mut Vec, _fv: F) -> std::io::Result { unsafe { thread_local! { static WRITTEN: UnsafeCell = UnsafeCell::new(0) @@ -57,7 +57,7 @@ pub fn deserialize_fork LineListNode { } } #[inline] - unsafe fn child_in_slot(&self) -> &TrieNodeODRc { + pub(crate) unsafe fn child_in_slot(&self) -> &TrieNodeODRc { match SLOT { 0 => unsafe{ &*self.val_or_child0.child }, 1 => unsafe{ &*self.val_or_child1.child }, @@ -430,7 +431,7 @@ impl LineListNode { } } #[inline] - unsafe fn val_in_slot(&self) -> &V { + pub(crate) unsafe fn val_in_slot(&self) -> &V { match SLOT { 0 => unsafe{ &**self.val_or_child0.val }, 1 => unsafe{ &**self.val_or_child1.val }, @@ -1998,26 +1999,7 @@ impl TrieNode for LineListNode } } #[inline] - fn node_val_count(&self, cache: &mut HashMap) -> usize { - let mut result = 0; - if self.is_used_value_0() { - result += 1; - } - if self.is_used_value_1() { - result += 1; - } - if self.is_used_child_0() { - let child_node = unsafe{ self.child_in_slot::<0>() }; - result += val_count_below_node(child_node, cache); - } - if self.is_used_child_1() { - let child_node = unsafe{ self.child_in_slot::<1>() }; - result += val_count_below_node(child_node, cache); - } - result - } - #[inline] - fn node_goat_val_count(&self) -> usize { + fn node_val_count(&self) -> usize { //Here are 3 alternative implementations. They're basically the same in perf, with a slight edge to the // inline bitwise arithmetic version. @@ -2759,6 +2741,225 @@ impl LineListNode { } } } + + #[inline(always)] + pub(crate) fn node_recursive_cata(&self, passed_in_val: Option<&V>, start_f: StartF, fold_child_f: FoldChildF, finalize_f: FinalizeF, cache: &mut HashMap) -> Result + where + W: Clone, + StartF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + FinalizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8]) -> Result, + { + macro_rules! summarize { + ($val:expr, $downstream:expr, $prefix:expr) => { + summarize_run::<_, _, _, _, _, _, _, COMPUTE_PATH>($val, $downstream, $prefix, start_f, fold_child_f, finalize_f) + }; + } + //Pair node can have the following permutations: (Slot0, Slot1) + // + // - Case 1 (Empty, Empty) + // Run only `finalize_f` on default `Acc` + // - Case 2 (Child, Empty) + // Recursively call on child, then run only `collapse_f` on the result, specifying the path. + // When a value was passed in at the first byte of that path, split out that byte as a + // value-bearing branch before collapsing the remaining run. + // - Case 3 (Child, Val), 1-byte key, same key byte + // Recursively call on child, then run only `collapse_f` on the result, specifying the value and the 1-byte path + // - Case 4 (Child, Val), different key bytes + // Recursively call on child, run `branch_f(collapse_f())` on the result. Then run `branch_f(collapse_f())` again + // with the value's path. And finally, run `finalize_f()` + // - Case 5 (Child, Child) + // Recursively call on child0, run `branch_f(collapse_f())` on the result. Do the same for child1. Finally, run + // `finalize_f()` + // - Case 6 (Val, Empty) + // Run only `collapse_f` on the val + // - Case 7 (Val, Val), common first byte (meaning slot0 is a 1-byte path) + // run `collapse_f` on the slot1 val, specifying the path, then run `collapse_f` again on the slot0 val, specifying + // the common prefix byte + // - Case 8 (Val, Val), different first bytes + // Run `branch_f(collapse_f())` on each val, then `finalize_f` at the end + // - Case 9 (Val, Child), 1-byte key, same key byte (We could eliminate this case by requiring a canonical ordering for identical one-byte keys, but currently we don't) + // See "Case 3 (Child, Val), 1-byte key, same key byte" + // - Case 10 (Val, Child), different key bytes + // See "Case 4 (Child, Val), different key bytes" + // + //GOAT, It would be nice to refactor the pair_node in order to express each of these permutations as a unique value for the 4 header bits, so we could take the appropriate code path without looking at any path bytes + + match self.header >> 12 { + //Case 1 (Empty, Empty) + 0 => finalize_f(&ByteMask::EMPTY, passed_in_val, None, &[]), + //Case 2 (Child, Empty) = (1 << 3) + (1 << 1) | (1 << 3) + (1 << 1) + 1 + 10 | 11 => { + let child_node = unsafe{ self.child_in_slot::<0>() }; + let child_w = recursive_cata_cached::<_, _, _, _, _, _, _, _, COMPUTE_PATH>(child_node, None, start_f, fold_child_f, finalize_f, cache)?; + let path = unsafe{ self.key_unchecked::<0>() }; + if passed_in_val.is_none() { + summarize!(None, Some(child_w), path) + } else { + let child_w = summarize!(None, Some(child_w), &path[1..])?; + let mask = ByteMask::from(path[0]); + let mut acc = start_f(&mask)?; + fold_child_f(&mask, child_w, &mut acc)?; + finalize_f(&mask, passed_in_val, Some(acc), &[]) + } + }, + //(Child, Val) = (1 << 3) + (1 << 2) + (1 << 1) + 14 => { + let child_node = unsafe{ self.child_in_slot::<0>() }; + let key0 = unsafe{ self.key_unchecked::<0>() }; + let key1 = unsafe{ self.key_unchecked::<1>() }; + let (key0_byte, key1_byte) = unsafe{ (*key0.get_unchecked(0), *key1.get_unchecked(0)) }; + if key0_byte == key1_byte { + //Case 3 + debug_assert_eq!(key0.len(), 1); + debug_assert_eq!(key1.len(), 1); + let val = unsafe { self.val_in_slot::<1>() }; + let child_w = recursive_cata_cached::<_, _, _, _, _, _, _, _, COMPUTE_PATH>(child_node, Some(val), start_f, fold_child_f, finalize_f, cache)?; + summarize!(passed_in_val, Some(child_w), key0) + } else { + //Case 4 + let child_w = recursive_cata_cached::<_, _, _, _, _, _, _, _, COMPUTE_PATH>(child_node, None, start_f, fold_child_f, finalize_f, cache)?; + let path = &key0[1..]; + let mask = ByteMask::from((key0_byte, key1_byte)); + let mut acc = start_f(&mask)?; + fold_child_f(&mask, summarize!(None, Some(child_w), path)?, &mut acc)?; + + let val = unsafe { self.val_in_slot::<1>() }; + let path = &key1[1..]; + fold_child_f(&mask, summarize!(Some(val), None, path)?, &mut acc)?; + + finalize_f(&mask, passed_in_val, Some(acc), &[]) + } + }, + //Case 5 (Child, Child) = (1 << 3) + (1 << 2) + (1 << 1) + 1 + 15 => { + let key0 = unsafe{ self.key_unchecked::<0>() }; + let key1 = unsafe{ self.key_unchecked::<1>() }; + let (key0_byte, key1_byte) = unsafe{ (*key0.get_unchecked(0), *key1.get_unchecked(0)) }; + let path0 = &key0[1..]; + let path1 = &key1[1..]; + let mask = ByteMask::from((key0_byte, key1_byte)); + let mut acc = start_f(&mask)?; + let child_node = unsafe{ self.child_in_slot::<0>() }; + let child_w = recursive_cata_cached::<_, _, _, _, _, _, _, _, COMPUTE_PATH>(child_node, None, start_f, fold_child_f, finalize_f, cache)?; + fold_child_f(&mask, summarize!(None, Some(child_w), path0)?, &mut acc)?; + + let child_node = unsafe{ self.child_in_slot::<1>() }; + let child_w = recursive_cata_cached::<_, _, _, _, _, _, _, _, COMPUTE_PATH>(child_node, None, start_f, fold_child_f, finalize_f, cache)?; + fold_child_f(&mask, summarize!(None, Some(child_w), path1)?, &mut acc)?; + + finalize_f(&mask, passed_in_val, Some(acc), &[]) + }, + //Case 6 (Val, Empty) = (1 << 3) | (1 << 3) + 1 + 8 | 9 => { + let val = unsafe { self.val_in_slot::<0>() }; + let path = unsafe{ self.key_unchecked::<0>() }; + if passed_in_val.is_none() { + summarize!(None, Some(summarize!(Some(val), None, path)?), &[]) + } else { + let child_w = summarize!(Some(val), None, &path[1..])?; + let mask = ByteMask::from(path[0]); + let mut acc = start_f(&mask)?; + fold_child_f(&mask, child_w, &mut acc)?; + finalize_f(&mask, passed_in_val, Some(acc), &[]) + } + }, + //(Val, Val) = (1 << 3) + (1 << 2) + 12 => { + let key0 = unsafe{ self.key_unchecked::<0>() }; + let key1 = unsafe{ self.key_unchecked::<1>() }; + let (key0_byte, key1_byte) = unsafe{ (*key0.get_unchecked(0), *key1.get_unchecked(0)) }; + if key0_byte == key1_byte { + //Case 7 (Val, Val), common first byte (meaning slot0 is a 1-byte path) + debug_assert_eq!(key0.len(), 1); + debug_assert!(key1.len() > 1); + let val = unsafe { self.val_in_slot::<1>() }; + let child_w = summarize!(Some(val), None, &key1[2..])?; + let mask = ByteMask::from(key1[1]); + let mut acc = start_f(&mask)?; + fold_child_f(&mask, child_w, &mut acc)?; + + let val = unsafe { self.val_in_slot::<0>() }; + let w = finalize_f(&mask, Some(val), Some(acc), &[])?; + summarize!(passed_in_val, Some(w), &key1[0..1]) + } else { + //Case 8 (Val, Val), different first bytes + let path0 = &key0[1..]; + let path1 = &key1[1..]; + let mask = ByteMask::from((key0_byte, key1_byte)); + let mut acc = start_f(&mask)?; + let val = unsafe{ self.val_in_slot::<0>() }; + fold_child_f(&mask, summarize!(Some(val), None, path0)?, &mut acc)?; + + let val = unsafe{ self.val_in_slot::<1>() }; + fold_child_f(&mask, summarize!(Some(val), None, path1)?, &mut acc)?; + + finalize_f(&mask, passed_in_val, Some(acc), &[]) + } + }, + //(Val, Child) = (1 << 3) + (1 << 2) + 1 + 13 => { + let child_node = unsafe{ self.child_in_slot::<1>() }; + let key0 = unsafe{ self.key_unchecked::<0>() }; + let key1 = unsafe{ self.key_unchecked::<1>() }; + let (key0_byte, key1_byte) = unsafe{ (*key0.get_unchecked(0), *key1.get_unchecked(0)) }; + if key0_byte == key1_byte { + //Case 9 (Val, Child), same key byte. The value is at the common first byte; + // the child may carry a further compressed suffix below that byte. + debug_assert_eq!(key0.len(), 1); + let val = unsafe { self.val_in_slot::<0>() }; + let child_w = if key1.len() == 1 { + recursive_cata_cached::<_, _, _, _, _, _, _, _, COMPUTE_PATH>(child_node, Some(val), start_f, fold_child_f, finalize_f, cache)? + } else { + let child_w = recursive_cata_cached::<_, _, _, _, _, _, _, _, COMPUTE_PATH>(child_node, None, start_f, fold_child_f, finalize_f, cache)?; + summarize!(Some(val), Some(child_w), &key1[1..])? + }; + summarize!(passed_in_val, Some(child_w), key0) + } else { + //Case 10 (Val, Child), different key bytes + let mask = ByteMask::from((key0_byte, key1_byte)); + let mut acc = start_f(&mask)?; + + let val = unsafe { self.val_in_slot::<0>() }; + let path = &key0[1..]; + fold_child_f(&mask, summarize!(Some(val), None, path)?, &mut acc)?; + + let child_w = recursive_cata_cached::<_, _, _, _, _, _, _, _, COMPUTE_PATH>(child_node, None, start_f, fold_child_f, finalize_f, cache)?; + let path = &key1[1..]; + fold_child_f(&mask, summarize!(None, Some(child_w), path)?, &mut acc)?; + + finalize_f(&mask, passed_in_val, Some(acc), &[]) + } + }, + _ => { unsafe { unreachable_unchecked() } } + } + + // let mut ws = Some(Acc::default()); + + // if self.is_used_value_0() { + // let downstream = collapse_f(Some(unsafe { self.val_in_slot::<0>() }), None, &[]); + // branch_f(&ByteMask::new(), downstream, unsafe { ws.as_mut().unwrap_unchecked() }); + // } + // if self.is_used_value_1() { + // let downstream = collapse_f(Some(unsafe { self.val_in_slot::<1>() }), None, &[]); + // branch_f(&ByteMask::new(), downstream, unsafe { ws.as_mut().unwrap_unchecked() }); + // } + // if self.is_used_child_0() { + // let child_node = unsafe{ self.child_in_slot::<0>() }; + // let w = recursive_cata::<_, _, _, _, _, _, _, COMPUTE_PATH>(child_node, collapse_f, branch_f, finalize_f); + // let downstream = collapse_f(None, Some(w), &[]); + // branch_f(&ByteMask::new(), downstream, unsafe { ws.as_mut().unwrap_unchecked() }); + + // } + // if self.is_used_child_1() { + // let child_node = unsafe{ self.child_in_slot::<1>() }; + // let w = recursive_cata::<_, _, _, _, _, _, _, COMPUTE_PATH>(child_node, collapse_f, branch_f, finalize_f); + // let downstream = collapse_f(None, Some(w), &[]); + // branch_f(&ByteMask::new(), downstream, unsafe { ws.as_mut().unwrap_unchecked() }); + // } + + // finalize_f(&ByteMask::new(), unsafe { std::mem::take(&mut ws).unwrap_unchecked() }) + } } impl TrieNodeDowncast for LineListNode { diff --git a/src/morphisms.rs b/src/morphisms.rs index 2852a2c..a2a34a6 100644 --- a/src/morphisms.rs +++ b/src/morphisms.rs @@ -12,8 +12,8 @@ //! ### Catamorphism //! //! Process a trie from the leaves towards the root. This algorithm proceeds in a depth-first order, -//! working from the leaves upward calling a closure on each path in the trie. A summary "accumulator" -//! type `W` is used to represent the information about the trie and carry it upwards to the next invocation +//! working from the leaves upward calling a closure on each path in the trie. A summary type `W` is +//! used to represent the information about the sub-trie and carry it upwards to the next invocation //! of the closure. //! //! The word "catamorphism" comes from the Greek for "down", because the root is considered the bottom @@ -23,10 +23,10 @@ //! //! **NOTE**: The traversal order, while depth-first, is subtly different from the order of //! [`ZipperIteration::to_next_val`](crate::zipper::ZipperIteration::to_next_val) and -//! [`ZipperMoving::to_next_step`](crate::zipper::ZipperMoving::to_next_step). The -//! zipper methods visit values occurring along a path first before descending to the -//! branches below, while the `cata` methods visit the deepest values first, before -//! returning to higher levels to aggregate information from deeper in the trie. +//! [`ZipperMoving::to_next_step`](crate::zipper::ZipperMoving::to_next_step). The zipper methods +//! visit values occurring along a path first before descending to the branches below, while the `cata` +//! methods visit the deepest values first, before returning to higher levels to aggregate information +//! from deeper in the trie. //! //! ### Anamorphism //! @@ -42,7 +42,7 @@ //! //! In most cases, jumping morphisms will perform substantially better than stepping morphisms, so you should use //! them when it is convenient. It is always possible to re-express an `alg` for a stepping morphism in terms of -//! an `alg` for a jumping morphism, however sometimes the logic become uglier and / or the performance benefit +//! an `alg` for a jumping morphism, however sometimes the logic becomes uglier and / or the performance benefit //! is negligible. Therefore, the stepping methods can be thought of as a convenience API. //! //! ### Side-Effecting vs Cached Iteration @@ -52,12 +52,12 @@ //! //! | side_effect | cached | //! |-------------------------------------------------|---------------------------------------------| -//! | Visits the entire trie | Short-circuits shared branches | +//! | Visits every path in the trie | Short-circuits shared subtries | //! | Always re-computes subtrie info (e.g. `W`) | Reuses shared subtrie info | //! | Guaranteed and deterministic `alg` exec order | Unpredictable callback `alg` exec order | //! | `alg` may capture and modify environment ([`FnMut`](https://doc.rust-lang.org/std/ops/trait.FnMut.html)) | `alg` must be a pure [`Fn`](https://doc.rust-lang.org/std/ops/trait.Fn.html) | //! | Owned `W` type passed between `alg` invocations | `W` type must implement [`Clone`](https://doc.rust-lang.org/std/clone/trait.Clone.html) | -//! | `alg` is aware of the entire `path` | `alg` only sees path byte, or sub-path slice | +//! | `alg` is aware of the entire `path` | `alg` only sees path byte, or prefix slice | //! //! All else being equal, `cached` methods are likely to be more efficient because structural sharing //! occurs frequently in pathmap tries. At the extreme, `side_effect` methods may produce a combinitoric @@ -71,17 +71,18 @@ use std::ptr::slice_from_raw_parts; use reusing_vec::ReusingQueue; use crate::utils::*; -use crate::alloc::Allocator; +use crate::alloc::{Allocator, GlobalAlloc}; use crate::PathMap; use crate::trie_node::TrieNodeODRc; +use crate::trie_node::recursive_cata_cached; use crate::zipper; use crate::zipper::*; use crate::gxhash::{self, HashMap, HashMapExt}; -/// Provides methods to perform a catamorphism on types that can reference or contain a trie -pub trait Catamorphism { - /// Applies a "stepping" catamorphism to the trie descending from the zipper's root, running the `alg_f` at every +/// Provides methods to perform side-effecting catamorphisms appropriate for serialization and full-path operations +pub trait CatamorphismSideEffecting { + /// Applies a "stepping" catamorphism to the subtrie descending from the zipper's focus, running the `alg_f` at every /// step (at every byte) /// /// ## Arguments to `alg_f`: @@ -96,12 +97,9 @@ pub trait Catamorphism { /// - `value`: A value associated with a given path in the trie, or `None` if the trie has no value at /// that path. /// - /// - `path`: The [`origin_path`](ZipperAbsolutePath::origin_path) for the invocation. The `alg_f` will - /// be run exactly once for each unique path in the trie. + /// - `path`: The absolute [`origin_path`](ZipperAbsolutePath::origin_path) for the invocation. The `alg_f` will + /// be run exactly once for each path in the subtrie rooted at the initial focus. /// - /// ## Behavior - /// - /// The focus position of the zipper will be ignored and it will be immediately reset to the root. fn into_cata_side_effect(self, mut alg_f: AlgF) -> W where AlgF: FnMut(&ByteMask, &mut [W], Option<&V>, &[u8]) -> W, @@ -114,11 +112,11 @@ pub trait Catamorphism { /// Allows the closure to return an error, stopping traversal immediately /// - /// See [Catamorphism::into_cata_side_effect] + /// See [CatamorphismSideEffecting::into_cata_side_effect] fn into_cata_side_effect_fallible(self, alg_f: AlgF) -> Result where AlgF: FnMut(&ByteMask, &mut [W], Option<&V>, &[u8]) -> Result; - /// Applies a "jumping" catamorphism to the trie + /// Applies a "jumping" catamorphism to the subtrie descending from the zipper's focus. /// /// A "jumping" catamorphism is a form of catamorphism where the `alg_f` "jumps over" (isn't called for) /// path bytes in the trie where there isn't either a `value` or a branch where `children.len() > 1`. @@ -129,7 +127,7 @@ pub trait Catamorphism { /// - `jumped_byte_cnt`: The number of bytes before the `alg_f` will be called again. The "jumped" substring /// is equal to `path[path.len()-jumped_byte_cnt..]` /// - /// See [into_cata_side_effect](Catamorphism::into_cata_side_effect) for explanation of other arguments and + /// See [into_cata_side_effect](CatamorphismSideEffecting::into_cata_side_effect) for explanation of other arguments and /// behavior fn into_cata_jumping_side_effect(self, mut alg_f: AlgF) -> W where @@ -143,121 +141,351 @@ pub trait Catamorphism { /// Allows the closure to return an error, stopping traversal immediately /// - /// See [Catamorphism::into_cata_jumping_side_effect] + /// See [CatamorphismSideEffecting::into_cata_jumping_side_effect] fn into_cata_jumping_side_effect_fallible(self, alg_f: AlgF) -> Result where AlgF: FnMut(&ByteMask, &mut [W], usize, Option<&V>, &[u8]) -> Result; +} - /// Applies a **cached**, **stepping**, catamorphism to the trie descending from the zipper's - /// root, running the `alg_f` at every step (at every byte) - /// - /// This method may re-use previous calculations of `W`, if the value for a shared subtrie has - /// been previously computed. - /// - /// ## Arguments to `alg_f`: - /// `(child_mask: &`[`ByteMask`]`, children: &mut [W], val: Option<&V>` - /// - /// - `child_mask`: A [`ByteMask`] indicating the corresponding byte for each downstream branche in - /// `children`. - /// - /// - `children`: A slice containing all the `W` values from previous invocations of `alg_f` for - /// downstream branches. - /// - /// - `value`: A value associated with a given path in the trie, or `None` if the trie has no value at - /// that path. - /// - /// ## Behavior - /// - /// The focus position of the zipper will be ignored and it will be immediately reset to the root. - fn into_cata_cached(self, alg_f: AlgF) -> W - where - W: Clone, - AlgF: Fn(&ByteMask, &mut [W], Option<&V>) -> W, - Self: Sized - { - self.into_cata_cached_fallible(|mask, children, val| -> Result { - Ok(alg_f(mask, children, val)) - }).unwrap() - } +macro_rules! define_cached_cata_trait { + ($(#[$meta:meta])* $trait_name:ident [$($generics:tt)*]) => { + $(#[$meta])* + pub trait $trait_name<$($generics)*> { + /// Applies a **cached**, **stepping**, catamorphism to the subtrie descending from the + /// zipper's current focus, running `alg_f` at every step (every byte). + /// + /// This method may reuse previously calculated `W` values when a shared subtrie has + /// already been computed. + /// + /// ## Arguments to `alg_f` + /// + /// `(child_mask: &`[`ByteMask`]`, children: &mut [W], value: Option<&V>)` + /// + /// - `child_mask` indicates the corresponding byte for every downstream branch in + /// `children`. + /// - `children` contains the `W` values produced for downstream branches. + /// - `value` is the value associated with this path, or `None` when there is none. + /// + fn cata_cached(&self, alg_f: AlgF) -> W + where + W: Clone, + AlgF: Fn(&ByteMask, &mut [W], Option<&V>) -> W + { + self.cata_cached_fallible(|mask, children, val| -> Result { + Ok(alg_f(mask, children, val)) + }).unwrap() + } - /// Allows the closure to return an error, stopping traversal immediately - /// - /// See [Catamorphism::into_cata_cached] - fn into_cata_cached_fallible(self, alg_f: AlgF) -> Result - where - W: Clone, - AlgF: Fn(&ByteMask, &mut [W], Option<&V>) -> Result; + /// Allows the closure to return an error, stopping traversal immediately. + /// + /// See [`Self::cata_cached`] for the closure arguments and traversal behavior. + fn cata_cached_fallible(&self, alg_f: AlgF) -> Result + where + W: Clone, + AlgF: Fn(&ByteMask, &mut [W], Option<&V>) -> Result + { + self.cata_jumping_cached_fallible(|mask, children, val, prefix| { + let mut w = alg_f(mask, children, val)?; + for &byte in prefix.iter().rev() { + let child_mask = ByteMask::from(byte); + w = alg_f(&child_mask, core::slice::from_mut(&mut w), None)?; + } + Ok(w) + }) + } - /// Applies a "jumping" catamorphism to the trie - /// - /// A "jumping" catamorphism is a form of catamorphism where the `alg_f` "jumps over" (isn't called for) - /// path bytes in the trie where there isn't either a `value` or a branch where `children.len() > 1`. - /// - /// This method may re-use previous calculations of `W`, if the value for a shared subtrie has - /// been previously computed. + /// Applies a **cached**, **jumping** catamorphism to the subtrie descending from the + /// zipper's current focus. + /// + /// A jumping catamorphism may omit calls to `alg_f` for path bytes that have neither + /// a value nor a branch with more than one child, passing those bytes as `prefix` + /// instead. Implementations may also fall back to stepping through some or all such + /// bytes, in which case `prefix` is shorter or empty. + /// + /// This method may reuse previously calculated `W` values when a shared subtrie has + /// already been computed. + /// + /// ## Arguments to `alg_f` + /// + /// `(child_mask: &`[`ByteMask`]`, children: &mut [W], value: Option<&V>, prefix: &[u8])` + /// + /// `prefix` is the sequence of bytes collapsed into this `alg_f` call. For example, + /// an implementation that collapses every non-branching run in this trie: + /// + /// ```text + /// ─── c ─── o ─── m ─┬─ b ─── o → "combo" + /// ├─ e ─── t → "comet" + /// └─ f ─── o ─── r ─── t → "comfort" + /// ``` + /// + /// This implementation calls `alg_f` four times: + /// + /// 1. `alg_f(ByteMask::EMPTY, &[], Some(&()), b"o")` + /// 2. `alg_f(ByteMask::EMPTY, &[], Some(&()), b"t")` + /// 3. `alg_f(ByteMask::EMPTY, &[], Some(&()), b"ort")` + /// 4. `alg_f(ByteMask::from_iter([b'b', b'e', b'f']), &[..], None, b"com")` + /// + /// See [`Self::cata_cached`] for the other arguments and traversal behavior. + fn cata_jumping_cached(&self, alg_f: AlgF) -> W + where + W: Clone, + AlgF: Fn(&ByteMask, &mut [W], Option<&V>, &[u8]) -> W + { + self.cata_jumping_cached_fallible(|mask, children, val, prefix| -> Result { + Ok(alg_f(mask, children, val, prefix)) + }).unwrap() + } + + /// Allows the closure to return an error, stopping traversal immediately. + /// + /// See [`Self::cata_jumping_cached`] for the closure arguments and traversal behavior. + fn cata_jumping_cached_fallible(&self, alg_f: AlgF) -> Result + where + W: Clone, + AlgF: Fn(&ByteMask, &mut [W], Option<&V>, &[u8]) -> Result + { + let children = std::cell::RefCell::new(CataChildren::::new()); + let children = &children; + let alg_f = &alg_f; + + self.factored_cata_jumping::( + move |mask| { + debug_assert!(children.try_borrow_mut().is_ok()); + Ok(unsafe { &mut *children.as_ptr() }.new_acc(mask.count_bits())) + }, + move |_mask, child, acc| { + debug_assert!(children.try_borrow_mut().is_ok()); + unsafe { &mut *children.as_ptr() }.push(acc, child); + Ok(()) + }, + move |mask, value, acc, prefix| match acc { + Some(acc) => { + debug_assert!(children.try_borrow_mut().is_ok()); + unsafe { &mut *children.as_ptr() }.summarize(acc, mask.count_bits(), |children| { + alg_f(mask, children, value, prefix) + }) + }, + None => alg_f(mask, &mut [], value, prefix), + }, + ) + } + + /// Hashes the logical trie and all of its values. + fn hash(&self) -> u128 + where + V: std::hash::Hash, + { + self.hash_with(|v| { + let mut hasher = gxhash::GxHasher::with_seed(0); + v.hash(&mut hasher); + hasher.finish_u128() + }) + } + + /// Hashes the logical trie using the provided function to hash values. + fn hash_with(&self, val_hash: F) -> u128 + where + F: Fn(&V) -> u128, + { + self.cata_cached(|bm, hs, mv| { + let mut hasher = gxhash::GxHasher::with_seed(0b0100001010101101111110010110100110000010011000100100100111110111i64); + hasher.write(unsafe { slice_from_raw_parts(bm.0.as_ptr() as *const u8, 32).as_ref().unwrap_unchecked() }); + hasher.write(unsafe { slice_from_raw_parts(hs.as_ptr() as *const u8, 16 * hs.len()).as_ref().unwrap_unchecked() }); + if let Some(v) = mv { hasher.write_u128(val_hash(v)) }; + hasher.finish_u128() + }) + } + + /// Returns the total number of values contained at and below the zipper's focus, including the + /// focus itself + fn val_count(&self) -> usize { + self.factored_cata_jumping::<_, _, Infallible, _, _, _, false>( + |_| Ok(0usize), + |_mask, w: usize, total| { *total += w; Ok(()) }, + |_mask, v, total, _| Ok((v.is_some() as usize) + total.unwrap_or(0)), + ).unwrap_or(0) + } + + /// A low-level cached catamorphism API that decomposes the algebra into multiple + /// functions and can avoid redundant path computation. + /// + /// ## Closures + /// + /// `NewAccF` creates an accumulator for a logical trie node with one or more downstream + /// child branches: `fn(child_mask: &ByteMask) -> Result`. + /// + /// `FoldChildF` folds one downstream child's `W` into that accumulator. It is called + /// once per downstream branch, in the same order as the bits in `child_mask`. Every + /// call for a logical trie node receives its complete child mask, so use call + /// order to associate a child result with its byte: + /// `fn(child_mask: &ByteMask, downstream: W, accumulator: &mut Acc) -> Result<(), Err>`. + /// + /// `SummarizeF` produces the `W` for a logical trie node and a non-branching path + /// `prefix` above it. Its result must summarize the subtrie from the start of `prefix`, + /// including `value` and downstream children. + /// + /// - `child_mask` describes the node's immediate child bytes. + /// - `accumulator` contains results folded from child branches, or is `None` when no + /// accumulator was needed. + /// - `prefix` is a collapsed non-branching path above the traversal node. It never + /// includes a path position that is part of a `child_mask` for this or another + /// `summarize_f` call. It may be empty when an implementation falls back to stepping. + /// + /// The `SummarizeF` signature is + /// `fn(child_mask: &ByteMask, value: Option<&V>, accumulator: Option, prefix: &[u8]) -> Result`. + /// + /// ## Behavior + /// + /// * Errors from any callback stop traversal immediately and are returned to the caller. + /// * Every call to `new_acc_f` will lead to a matching call to `summarize_f` where ownership + /// of the `Acc` object is given to the `SummarizeF` callback, unless an error halts the catamorphism. + /// * `prefix` is **only an optimization** for path bytes with a single child. Your + /// algebra must not rely on a given path byte being represented as a prefix, instead of as + /// a sequence of `new_acc_f`, then `fold_child_f`. + /// + /// With `COMPUTE_PATH = false`, path runs are not materialized and `prefix` is always + /// empty. Use that only when the algebra does not depend on path bytes, only on values + /// and/or path endpoints. + fn factored_cata_jumping( + &self, + new_acc_f: NewAccF, + fold_child_f: FoldChildF, + summarize_f: SummarizeF, + ) -> Result + where + W: Clone, + NewAccF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + SummarizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8]) -> Result; + + /// A **stepping** catamorphism based on a similar factored algebra to [`Self::factored_cata_jumping`] + /// + /// Use this when the cata must evaluate once per path byte, including bytes in + /// non-branching runs. Unlike the jumping version, `summarize_f` has no `prefix`: it + /// is called once for every path byte. The callback roles and child-mask ordering are + /// otherwise the same as for [`Self::factored_cata_jumping`]. + fn factored_cata( + &self, + new_acc_f: NewAccF, + fold_child_f: FoldChildF, + summarize_f: SummarizeF, + ) -> Result + where + W: Clone, + NewAccF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + SummarizeF: Copy + Fn(&ByteMask, Option<&V>, Option) -> Result, + Self: Sized, + { + self.factored_cata_jumping::<_, _, _, _, _, _, true>( + new_acc_f, + fold_child_f, + |mask, val, acc, prefix| { + let mut w = summarize_f(mask, val, acc)?; + for byte in prefix.iter().rev() { + let mask = ByteMask::from(*byte); + let mut acc = new_acc_f(&mask)?; + fold_child_f(&mask, w, &mut acc)?; + w = summarize_f(&mask, None, Some(acc))?; + } + Ok(w) + }, + ) + } + } + }; +} + +define_cached_cata_trait! { + /// Cached catamorphisms evaluated with recursive traversal. /// - /// ## Arguments to `alg_f`: - /// `(child_mask: &`[`ByteMask`]`, children: &mut [W], value: Option<&V>, sub_path: &[u8]` + /// This is the normal choice: it is substantially faster (about 10x) than + /// [`CatamorphismCachedIterative`] for typical tries and does not require cloning or creating + /// a zipper. It uses the call stack, so prefer [`CatamorphismCachedIterative`] when paths can + /// be deeply nested so there may be a risk of stack overflow. /// - /// - `sub_path`: A slice of path bytes for which the `alf_f` will not be called. Consider the - /// trie below: + /// [`CatamorphismCachedIterative`] deliberately has the same method names and signatures so it + /// is perfectly interchangeable. Import one trait for ordinary method syntax. When both + /// strategies are needed in one scope, call the desired trait with fully qualified syntax. + CatamorphismCached [V, A = GlobalAlloc] +} + +define_cached_cata_trait! { + /// Cached catamorphisms evaluated with iterative zipper traversal. /// - /// ```txt - /// ─── c ─── o ─── m ─┬─ b ─── o → "combo" - /// ├─ e ─── t → "comet" - /// └─ f ─── o ─── r ─── t → "comfort" - /// ``` - /// The `alg_f` would be called 4 times for this trie. - /// 1. `alg_f(ByteMask::EMPTY, &[], Some(&()), b"o")` - /// 2. `alg_f(ByteMask::EMPTY, &[], Some(&()), b"t")` - /// 3. `alg_f(ByteMask::EMPTY, &[], Some(&()), b"ort")` - /// 4. `alg_f(ByteMask::from_iter([b'b', b'e', b'f']), &[..], None, b"com")` + /// Prefer this trait when the trie can contain unbounded or adversarially deep paths and stack + /// safety matters more than throughput. It traverses a zipper, so an implementation must be + /// able to create or clone a movable zipper; for ordinary bounded-depth tries, + /// [`CatamorphismCached`] is faster. /// - /// See [into_cata_cached](Catamorphism::into_cata_cached) for explanation of other arguments and behavior - fn into_cata_jumping_cached(self, alg_f: AlgF) -> W - where - W: Clone, - AlgF: Fn(&ByteMask, &mut [W], Option<&V>, &[u8]) -> W, - Self: Sized - { - self.into_cata_jumping_cached_fallible(|mask, children, val, sub_path| -> Result { - Ok(alg_f(mask, children, val, sub_path)) - }).unwrap() + /// This trait intentionally has the same method names and signatures as [`CatamorphismCached`]. + /// Import one trait for ordinary method syntax. When both strategies are needed in one scope, + /// call the desired trait with fully qualified syntax. such as + /// `CatamorphismCachedIterative::cata_cached(&zipper, algebra)`. + CatamorphismCachedIterative [V] +} + +/// Shared child-result storage used to adapt the factored cached-cata API to a single-function algebra. +struct CataChildren { + children: Vec, + #[cfg(debug_assertions)] + allocations: Vec<(usize, usize)>, + #[cfg(debug_assertions)] + next_allocation: usize, +} + +struct CataChildrenAcc { + start: usize, + #[cfg(debug_assertions)] + allocation: usize, +} + +impl CataChildren { + #[inline] + fn new() -> Self { + Self { + children: Vec::new(), + #[cfg(debug_assertions)] + allocations: Vec::new(), + #[cfg(debug_assertions)] + next_allocation: 0, + } } - /// Allows the closure to return an error, stopping traversal immediately - /// - /// See [Catamorphism::into_cata_jumping_cached] - fn into_cata_jumping_cached_fallible(self, alg_f: AlgF) -> Result - where - W: Clone, - AlgF: Fn(&ByteMask, &mut [W], Option<&V>, &[u8]) -> Result; + #[inline] + fn new_acc(&mut self, capacity: usize) -> CataChildrenAcc { + let start = self.children.len(); + self.children.reserve(capacity); + #[cfg(debug_assertions)] + let allocation = { + let allocation = self.next_allocation; + self.next_allocation += 1; + self.allocations.push((start, allocation)); + allocation + }; + CataChildrenAcc { + start, + #[cfg(debug_assertions)] + allocation, + } + } - /// Hash the logical `PathMap` and all its values - fn hash(self) -> u128 - where - Self: Sized, - V: std::hash::Hash - { - self.hash_with(|v| { - let mut hasher = gxhash::GxHasher::with_seed(0); - v.hash(&mut hasher); - hasher.finish_u128() - }) + #[inline(always)] + fn push(&mut self, _acc: &CataChildrenAcc, child: W) { + #[cfg(debug_assertions)] + debug_assert_eq!(self.allocations.last(), Some(&(_acc.start, _acc.allocation))); + self.children.push(child); } - /// Hash the logical `PathMap`, using the provided function to hash values - fn hash_with(self, val_hash: F) -> u128 - where - Self: Sized, - F: Fn(&V) -> u128 - { - self.into_cata_cached(|bm, hs, mv| { - let mut hasher = gxhash::GxHasher::with_seed(0b0100001010101101111110010110100110000010011000100100100111110111i64); - hasher.write(unsafe { slice_from_raw_parts(bm.0.as_ptr() as *const u8, 32).as_ref().unwrap_unchecked() }); - hasher.write(unsafe { slice_from_raw_parts(hs.as_ptr() as *const u8, 16*hs.len()).as_ref().unwrap_unchecked() }); - if let Some(v) = mv { hasher.write_u128(val_hash(v)) }; - hasher.finish_u128() - }) + #[inline] + fn summarize(&mut self, acc: CataChildrenAcc, child_count: usize, summarize_f: impl FnOnce(&mut [W]) -> R) -> R { + #[cfg(debug_assertions)] + debug_assert_eq!(self.allocations.pop(), Some((acc.start, acc.allocation))); + debug_assert!(acc.start <= self.children.len()); + debug_assert_eq!(self.children.len() - acc.start, child_count); + + let result = summarize_f(&mut self.children[acc.start..]); + self.children.truncate(acc.start); + result } } @@ -323,8 +551,8 @@ impl SplitCata { /// A compatibility shim to provide a 4-function "jumping" catamorphism API /// -/// - `jump_f`: `FnMut(sub_path: &[u8], w: W, path: &[u8]) -> W` -/// Elevates a result `w` descending from the relative path, `sub_path` to the current position at `path` +/// - `jump_f`: `FnMut(prefix: &[u8], w: W, path: &[u8]) -> W` +/// Elevates a result `w` descending from the relative path, `prefix` to the current position at `path` /// /// See [`SplitCata`] for a description of additional args #[deprecated] @@ -383,7 +611,7 @@ impl SplitCataJumping { } } -impl<'a, Z, V: 'a> Catamorphism for Z where Z: Zipper + ZipperReadOnlyConditionalValues<'a, V> + ZipperConcrete + ZipperAbsolutePath + ZipperPathBuffer { +impl<'a, Z, V: 'a> CatamorphismSideEffecting for Z where Z: Zipper + ZipperReadOnlyConditionalValues<'a, V> + ZipperAbsolutePath + ZipperPathBuffer { fn into_cata_side_effect_fallible(self, mut alg_f: AlgF) -> Result where AlgF: FnMut(&ByteMask, &mut [W], Option<&V>, &[u8]) -> Result, { @@ -399,27 +627,9 @@ impl<'a, Z, V: 'a> Catamorphism for Z where Z: Zipper + ZipperReadOnlyConditi alg_f(mask, children, jump_len, val, path) }) } - fn into_cata_cached_fallible(self, alg_f: AlgF) -> Result - where - W: Clone, - AlgF: Fn(&ByteMask, &mut [W], Option<&V>) -> Result - { - into_cata_cached_body::(self, |mask, children, val, sub_path, _debug_path, _z| { - debug_assert_eq!(sub_path.len(), 0); - alg_f(mask, children, val) - }) - } - fn into_cata_jumping_cached_fallible(self, alg_f: AlgF) -> Result - where - W: Clone, - AlgF: Fn(&ByteMask, &mut [W], Option<&V>, &[u8]) -> Result - { - into_cata_cached_body::(self, - |mask, children, val, sub_path, _debug_path, _z| alg_f(mask, children, val, sub_path)) - } } -impl Catamorphism for PathMap { +impl CatamorphismSideEffecting for PathMap { fn into_cata_side_effect_fallible(self, alg_f: AlgF) -> Result where AlgF: FnMut(&ByteMask, &mut [W], Option<&V>, &[u8]) -> Result { @@ -432,21 +642,116 @@ impl Catamorph let rz = self.into_read_zipper(&[]); rz.into_cata_jumping_side_effect_fallible(alg_f) } - fn into_cata_cached_fallible(self, alg_f: AlgF) -> Result - where - W: Clone, - AlgF: Fn(&ByteMask, &mut [W], Option<&V>) -> Result +} + +impl CatamorphismCached for Z where Z: Zipper + ZipperConcrete + ZipperInfallibleSubtries { + fn factored_cata_jumping(&self, new_acc_f: NewAccF, fold_child_f: FoldChildF, summarize_f: SummarizeF) -> Result + where + W: Clone, + NewAccF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + SummarizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8]) -> Result, { - let rz = self.into_read_zipper(&[]); - rz.into_cata_cached_fallible(alg_f) + let focus = self.get_focus(); + let mut cache = HashMap::new(); + let w = if let Some(node) = focus.0.borrow() { + recursive_cata_cached::<_, _, Acc, _, Err, _, _, _, COMPUTE_PATH>(node, self.val(), new_acc_f, fold_child_f, summarize_f, &mut cache)? + } else { + match focus.into_option() { + Some(node) => recursive_cata_cached::<_, _, Acc, _, Err, _, _, _, COMPUTE_PATH>(&node, self.val(), new_acc_f, fold_child_f, summarize_f, &mut cache)?, + None => return summarize_f(&ByteMask::EMPTY, self.val(), None, &[]), + } + }; + Ok(w) } - fn into_cata_jumping_cached_fallible(self, alg_f: AlgF) -> Result - where - W: Clone, - AlgF: Fn(&ByteMask, &mut [W], Option<&V>, &[u8]) -> Result +} + +impl<'a, Z, V: 'a> CatamorphismCachedIterative for Z where Z: Clone + Zipper + ZipperReadOnlyConditionalValues<'a, V> + ZipperConcrete + ZipperPathBuffer { + fn factored_cata_jumping(&self, new_acc_f: NewAccF, fold_child_f: FoldChildF, summarize_f: SummarizeF) -> Result + where + W: Clone, + NewAccF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + SummarizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8]) -> Result, { - let rz = self.into_read_zipper(&[]); - rz.into_cata_jumping_cached_fallible(alg_f) + summarize_cached_body::<_, V, Acc, _, _, _, _, _, COMPUTE_PATH>( + self.clone(), + new_acc_f, + fold_child_f, + summarize_f, + ) + } +} + +impl CatamorphismCached for PathMap { + fn factored_cata_jumping(&self, new_acc_f: NewAccF, fold_child_f: FoldChildF, summarize_f: SummarizeF) -> Result + where + W: Clone, + NewAccF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + SummarizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8]) -> Result, + { + let w = match self.root() { + Some(node) => { + let mut cache = HashMap::new(); + recursive_cata_cached::<_, _, Acc, _, Err, _, _, _, COMPUTE_PATH>(node, self.root_val(), new_acc_f, fold_child_f, summarize_f, &mut cache) + }, + None => summarize_f(&ByteMask::EMPTY, None, None, &[]), + }; + w + } +} + +impl CatamorphismCachedIterative for PathMap { + fn factored_cata_jumping(&self, new_acc_f: NewAccF, fold_child_f: FoldChildF, summarize_f: SummarizeF) -> Result + where + W: Clone, + NewAccF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + SummarizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8]) -> Result, + { + summarize_cached_body::<_, V, Acc, _, _, _, _, _, COMPUTE_PATH>( + self.read_zipper(), + new_acc_f, + fold_child_f, + summarize_f, + ) + } +} + + +/// Helper function to summarize one path run (section of non-branching path bytes) +// +//NOTE: #[inline(always)] here leads to a huge bloating of the size of the a stack frame in the recursive cata +// (about 2.5x bloat. but #[inline(never)] costs about 25% performance. Letting the compiler do its thing seems +// to be the sweet spot. +pub(crate) fn summarize_run( + val: Option<&V>, + downstream: Option, + prefix: &[u8], + start_f: StartF, + fold_child_f: FoldChildF, + finalize_f: FinalizeF, +) -> Result +where + StartF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + FinalizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8]) -> Result, +{ + match (val, downstream, prefix) { + (None, Some(w), []) => Ok(w), + (val, Some(w), prefix) => { + let mask = if !prefix.is_empty() { ByteMask::from(*prefix.last().unwrap()) } else { ByteMask::EMPTY }; + let mut acc = start_f(&mask)?; + fold_child_f(&mask, w, &mut acc)?; + let prefix = if COMPUTE_PATH { + if prefix.is_empty() { prefix } else { &prefix[..prefix.len() - 1] } + } else { + &[] + }; + finalize_f(&mask, val, Some(acc), prefix) + }, + (val, None, prefix) => finalize_f(&ByteMask::EMPTY, val, None, if COMPUTE_PATH { prefix } else { &[] }), } } @@ -457,16 +762,16 @@ fn cata_side_effect_body<'a, Z, V: 'a, W, Err, AlgF, const JUMPING: bool>(mut z: AlgF: FnMut(&ByteMask, &mut [W], usize, Option<&V>, &[u8], &Z) -> Result { //`stack` holds a "frame" at each forking point above the zipper position. No frames exist for values - let mut stack = Vec::::with_capacity(12); + let mut stack = Vec::::with_capacity(12); let mut children = Vec::::new(); let mut frame_idx = 0; - z.reset(); + let focus_depth = z.depth(); z.prepare_buffers(); - //Push a stack frame for the root, and start on the first branch off the root - stack.push(StackFrame::from(&z)); + // Push a stack frame for the initial focus, and start on its first branch. + stack.push(SideEffectStackFrame::new(&z)); if z.descend_first_byte().is_none() { - //Empty trie is a special case + // A leaf focus is a special case. return alg_f(&ByteMask::EMPTY, &mut [], 0, z.val(), z.origin_path(), &z) } @@ -482,7 +787,15 @@ fn cata_side_effect_body<'a, Z, V: 'a, W, Err, AlgF, const JUMPING: bool>(mut z: if is_leaf { //Ascend back to the last fork point from this leaf - let cur_w = ascend_to_fork::(&mut z, &mut alg_f, &mut [])?; + let cur_w = match ascend_to_focus_or_fork::( + &mut z, + focus_depth, + &mut alg_f, + &mut [], + )? { + AscendResult::Parent(w) => w, + AscendResult::Focus(w) => return Ok(w), + }; children.push(cur_w); stack[frame_idx].child_idx += 1; @@ -508,7 +821,15 @@ fn cata_side_effect_body<'a, Z, V: 'a, W, Err, AlgF, const JUMPING: bool>(mut z: debug_assert_eq!(stack[frame_idx].child_idx, stack[frame_idx].child_cnt); let child_start = children.len() - stack[frame_idx].child_cnt as usize; let children2 = &mut children[child_start..]; - let cur_w = ascend_to_fork::(&mut z, &mut alg_f, children2)?; + let cur_w = match ascend_to_focus_or_fork::( + &mut z, + focus_depth, + &mut alg_f, + children2, + )? { + AscendResult::Parent(w) => w, + AscendResult::Focus(w) => return Ok(w), + }; children.truncate(child_start); frame_idx -= 1; @@ -522,8 +843,16 @@ fn cata_side_effect_body<'a, Z, V: 'a, W, Err, AlgF, const JUMPING: bool>(mut z: let descended = z.descend_indexed_byte(stack[frame_idx].child_idx as usize); debug_assert!(descended.is_some()); } else { - //Push a new stack frame for this branch - Stack::push_state_raw(&mut stack, &mut frame_idx, &z); + // Push a new stack frame for this branch, reusing a frame left by a completed + // sibling branch when possible. + frame_idx += 1; + assert!(frame_idx <= stack.len(), "stack invariant: frame index <= length"); + let frame = SideEffectStackFrame::new(&z); + if frame_idx == stack.len() { + stack.push(frame); + } else { + stack[frame_idx] = frame; + } //Descend the first child branch let descended = z.descend_first_byte(); @@ -532,153 +861,126 @@ fn cata_side_effect_body<'a, Z, V: 'a, W, Err, AlgF, const JUMPING: bool>(mut z: } } +/// Ascends from a completed child to either its parent fork or the initial cata focus. +/// +/// A jumping ascent may otherwise pass a valueless unary focus in one movement, so the focus +/// boundary is checked explicitly rather than relying on `at_root()`. #[inline(always)] -fn ascend_to_fork<'a, Z, V: 'a, W, Err, AlgF, const JUMPING: bool>(z: &mut Z, - alg_f: &mut AlgF, children: &mut [W] -) -> Result - where +fn ascend_to_focus_or_fork<'a, Z, V: 'a, W, Err, AlgF, const JUMPING: bool>( + z: &mut Z, + focus_depth: usize, + alg_f: &mut AlgF, + children: &mut [W], +) -> Result, Err> +where Z: Zipper + ZipperReadOnlyConditionalValues<'a, V> + ZipperAbsolutePath + ZipperPathBuffer, - AlgF: FnMut(&ByteMask, &mut [W], usize, Option<&V>, &[u8], &Z) -> Result + AlgF: FnMut(&ByteMask, &mut [W], usize, Option<&V>, &[u8], &Z) -> Result, { - let z_witness = z.witness(); - let mut w; + let witness = z.witness(); let mut child_mask = ByteMask::from(z.child_mask()); let mut children = &mut children[..]; + if JUMPING { - //This loop runs until we got to a fork or the root. We will take a spin through the loop - // for each value we encounter along the way while ascending + let mut w; loop { + let old_depth = z.depth(); let old_path_len = z.origin_path().len(); - let old_val = z.get_val_with_witness(&z_witness); + let old_value = z.get_val_with_witness(&witness); let ascended = z.ascend_until(); debug_assert!(ascended > 0); - let origin_path = unsafe{ z.origin_path_assert_len(old_path_len) }; + // SAFETY: `ascend_until` only shortens the path, so the former path remains initialized + // in the prepared buffer until the next movement. + let origin_path = unsafe { z.origin_path_assert_len(old_path_len) }; + + if z.depth() < focus_depth { + // A valueless unary focus is not an `ascend_until` stopping point. Its jumping + // cata is just its child's result, but the preceding callback must report the + // jump relative to this focus rather than to the zipper's root. + debug_assert!(old_depth > focus_depth); + w = alg_f( + &child_mask, + children, + old_depth - focus_depth, + old_value, + origin_path, + z, + )?; + return Ok(AscendResult::Focus(w)) + } + let jump_len = if z.child_count() != 1 || z.is_val() { - old_path_len - (z.origin_path().len()+1) + ascended - 1 } else { - old_path_len - z.origin_path().len() + ascended }; - - w = alg_f(&child_mask, children, jump_len, old_val, origin_path, &z)?; + w = alg_f(&child_mask, children, jump_len, old_value, origin_path, z)?; + + if z.depth() == focus_depth && z.child_count() == 1 { + // A valued unary focus stops `ascend_until`; complete that focus here rather + // than continuing to its parent. A valueless focus is a jumping passthrough. + return if z.is_val() { + let byte = origin_path[old_path_len - jump_len - 1]; + let child_mask = ByteMask::from(byte); + let mut child = [w]; + alg_f(&child_mask, &mut child, 0, z.val(), z.origin_path(), z) + .map(AscendResult::Focus) + } else { + Ok(AscendResult::Focus(w)) + } + } if z.child_count() != 1 || z.at_root() { - return Ok(w) + return Ok(AscendResult::Parent(w)) } children = core::array::from_mut(&mut w); - - // SAFETY: We will never over-read the path buffer because we only get here after we ascended - let byte = *unsafe{ z.origin_path_assert_len(old_path_len-jump_len) }.last().unwrap(); - child_mask = ByteMask::EMPTY; - child_mask.set_bit(byte); + let byte = origin_path[old_path_len - jump_len - 1]; + child_mask = ByteMask::from(byte); } } else { - //This loop runs at each byte step as we ascend + let mut w; loop { let origin_path = z.origin_path(); let byte = origin_path.last().copied().unwrap_or(0); - let val = z.val(); - w = alg_f(&child_mask, children, 0, val, origin_path, &z)?; + let value = z.val(); + w = alg_f(&child_mask, children, 0, value, origin_path, z)?; let ascended = z.ascend_byte(); debug_assert!(ascended); + if z.depth() == focus_depth && z.child_count() == 1 { + let child_mask = ByteMask::from(byte); + let mut child = [w]; + return alg_f(&child_mask, &mut child, 0, z.val(), z.origin_path(), z) + .map(AscendResult::Focus) + } + if z.child_count() != 1 || z.at_root() { - return Ok(w) + return Ok(AscendResult::Parent(w)) } children = core::array::from_mut(&mut w); - child_mask = ByteMask::EMPTY; - child_mask.set_bit(byte); + child_mask = ByteMask::from(byte); } } } -/// Internal structure to hold temporary info used inside morphism apply methods -struct StackFrame { +/// A frame for the side-effecting cata's explicit traversal stack. +struct SideEffectStackFrame { child_idx: u16, child_cnt: u16, - child_addr: Option, -} - -impl StackFrame { - /// Allocates a new StackFrame - fn from(zipper: &Z) -> Self - where Z: Zipper, - { - let mut stack_frame = StackFrame { - child_cnt: 0, - child_idx: 0, - child_addr: None, - }; - stack_frame.reset(zipper); - stack_frame - } - - /// Resets a StackFrame to the state needed to iterate a new forking point - fn reset(&mut self, zipper: &Z) - where Z: Zipper, - { - self.child_cnt = zipper.child_count() as u16; - self.child_idx = 0; - } -} - -struct Stack { - stack: Vec, - position: usize, } -impl Stack { - pub fn new() -> Self { - Self { - stack: Vec::with_capacity(12), - position: !0, - } - } - /// Return the reference to the top stack frame - #[inline] - pub fn last_mut(&mut self) -> Option<&mut StackFrame> { - let idx = self.position; - self.stack.get_mut(idx) - } - - /// Return the reference to the top stack frame - /// and decrease stack pointer. Doesn't free the stack frame. +impl SideEffectStackFrame { #[inline] - pub fn pop_mut(&mut self) -> Option<&mut StackFrame> { - if self.position == !0 { - return None; - } - let idx = self.position; - self.position = self.position.wrapping_sub(1); - self.stack.get_mut(idx) - } - - /// Push stack state for current zipper position - /// - /// This function re-uses allocations for stack frames, - /// to avoid allocator thrashing. - pub fn push_state(&mut self, z: &Z) - where Z: Zipper + ZipperPath, - { - Self::push_state_raw(&mut self.stack, &mut self.position, z); - } - - pub fn push_state_raw<'a, Z>( - stack: &mut Vec, - position: &mut usize, - zipper: &Z) - where Z: Zipper + ZipperPath, + fn new(zipper: &Z) -> Self + where + Z: Zipper, { - *position = position.wrapping_add(1); - assert!(*position <= stack.len(), - "stack invariant: position <= len"); - if *position == stack.len() { - stack.push(StackFrame::from(zipper)); - } else { - stack[*position].reset(zipper); + Self { + child_idx: 0, + child_cnt: zipper.child_count() as u16, } } } @@ -759,108 +1061,339 @@ impl CacheStrategy for DoCache { fn clone(w: &W) -> W { w.clone() } } -/// Internal implementation behind all cached catas -/// -/// AlgF args: (child_mask, children, value, sub_path, debug_path, zipper) -pub(crate) fn into_cata_cached_body<'a, Z, V: 'a, W, E, AlgF, Cache, const JUMPING: bool, const DEBUG_PATH: bool>( - mut zipper: Z, mut alg_f: AlgF +/// Stack frame used in iterative (zipper-based) implementation of Summarization trait +struct SummarizeStackFrame { + child_idx: u16, + child_cnt: u16, + child_addr: Option, + accumulator: Acc, +} + +impl SummarizeStackFrame { + #[inline] + fn new(child_cnt: usize, accumulator: Acc) -> Self { + Self { + child_idx: 0, + child_cnt: child_cnt as u16, + child_addr: None, + accumulator, + } + } +} + +/// Result of ascending from a completed child. +enum AscendResult { + Parent(W), + Focus(W), +} + +#[inline(always)] +fn no_debug_path(_zipper: &Z, _depth: usize) -> &[u8] { + &[] +} + +/// Ascend from a leaf or completed fork, summarizing each value and non-branching path run on the +/// way to the parent fork. +#[inline(always)] +fn summarize_ascend_to_fork<'a, Z, V: 'a, Acc, W, E, DebugPathF, NewAccF, FoldChildF, SummarizeF, const COMPUTE_PATH: bool, const DEBUG_PATH: bool>( + zipper: &mut Z, + focus_depth: usize, + mut accumulator: Option, + debug_path_f: DebugPathF, + new_acc_f: NewAccF, + fold_child_f: FoldChildF, + summarize_f: SummarizeF, +) -> Result, E> +where + Z: Zipper + ZipperReadOnlyConditionalValues<'a, V> + ZipperPathBuffer, + DebugPathF: Copy + for<'z> Fn(&'z Z, usize) -> &'z [u8], + NewAccF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), E>, + SummarizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8], &[u8]) -> Result, +{ + let witness = zipper.witness(); + let mut child_mask = ByteMask::from(zipper.child_mask()); + + loop { + let old_depth = zipper.depth(); + let old_value = zipper.get_val_with_witness(&witness); + if old_depth == focus_depth { + let debug_path = if DEBUG_PATH { debug_path_f(zipper, old_depth) } else { &[] }; + return summarize_f(&child_mask, old_value, accumulator, &[], debug_path) + .map(AscendResult::Focus); + } + + let ascended = zipper.ascend_until(); + debug_assert!(ascended > 0); + let depth = zipper.depth(); + debug_assert_eq!(old_depth - depth, ascended); + + // SAFETY: `ascend_until` only shortens the logical path; the bytes it removed remain + // initialized in the zipper's prepared path buffer until the next zipper movement. + let path = unsafe { zipper.path_assert_len(old_depth) }; + let debug_path = if DEBUG_PATH { debug_path_f(zipper, old_depth) } else { &[] }; + + // `ascend_until` can pass the initial focus when that focus is a valueless unary + // position in a compressed run. That position is the traversal root, so preserve the + // entire suffix below it and finish there rather than continuing toward the zipper root. + if depth < focus_depth { + debug_assert!(focus_depth < old_depth); + return summarize_f( + &child_mask, + old_value, + accumulator, + if COMPUTE_PATH { &path[focus_depth..old_depth] } else { &[] }, + debug_path, + ).map(AscendResult::Focus); + } + + let jump_len = if zipper.child_count() != 1 || zipper.is_val() { + ascended - 1 + } else { + ascended + }; + let prefix = if COMPUTE_PATH { + &path[old_depth - jump_len..] + } else { + &[] + }; + + let w = summarize_f(&child_mask, old_value, accumulator, prefix, debug_path)?; + + if zipper.child_count() != 1 || zipper.at_root() { + return Ok(AscendResult::Parent(w)) + } + + debug_assert!(old_depth > jump_len); + let byte = path[old_depth - jump_len - 1]; + child_mask = ByteMask::from(byte); + let mut next_accumulator = new_acc_f(&child_mask)?; + fold_child_f(&child_mask, w, &mut next_accumulator)?; + accumulator = Some(next_accumulator); + } +} + +/// Iterative cached traversal behind [`CatamorphismCached::factored_cata_jumping`]. +fn summarize_cached_body<'a, Z, V: 'a, Acc, W, E, NewAccF, FoldChildF, SummarizeF, const COMPUTE_PATH: bool>( + zipper: Z, + new_acc_f: NewAccF, + fold_child_f: FoldChildF, + summarize_f: SummarizeF, ) -> Result - where - Cache: CacheStrategy, - Z: Zipper + ZipperReadOnlyConditionalValues<'a, V> + ZipperConcrete + ZipperAbsolutePath + ZipperPathBuffer, - AlgF: FnMut(&ByteMask, &mut [W], Option<&V>, &[u8], &[u8], &Z) -> Result +where + W: Clone, + Z: Zipper + ZipperReadOnlyConditionalValues<'a, V> + ZipperConcrete + ZipperPathBuffer, + NewAccF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), E>, + SummarizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8]) -> Result, +{ + summarize_cached_body_with_debug::( + zipper, + no_debug_path::, + new_acc_f, + fold_child_f, + move |mask, value, accumulator, prefix, _debug_path| { + summarize_f(mask, value, accumulator, prefix) + }, + ) +} + +/// Shared iterative cached traversal used by the ordinary and debug cata adapters. +fn summarize_cached_body_with_debug<'a, Z, V: 'a, Acc, W, E, DebugPathF, NewAccF, FoldChildF, SummarizeF, const COMPUTE_PATH: bool, const DEBUG_PATH: bool>( + mut zipper: Z, + debug_path_f: DebugPathF, + new_acc_f: NewAccF, + fold_child_f: FoldChildF, + summarize_f: SummarizeF, +) -> Result +where + W: Clone, + Z: Zipper + ZipperReadOnlyConditionalValues<'a, V> + ZipperConcrete + ZipperPathBuffer, + DebugPathF: Copy + for<'z> Fn(&'z Z, usize) -> &'z [u8], + NewAccF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), E>, + SummarizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8], &[u8]) -> Result, { - zipper.reset(); + let focus_depth = zipper.depth(); zipper.prepare_buffers(); - let mut stack = Stack::new(); - let mut children = Vec::::new(); + let root_child_cnt = zipper.child_count(); + if root_child_cnt == 0 { + let debug_path = if DEBUG_PATH { debug_path_f(&zipper, focus_depth) } else { &[] }; + return summarize_f(&ByteMask::EMPTY, zipper.val(), None, &[], debug_path) + } + + let passthrough_root = root_child_cnt == 1 && !zipper.is_val(); + let mut stack = Vec::>::with_capacity(12); + if passthrough_root { + // A unary root without a value passes its child's W through unchanged, so it has no Acc. + zipper.descend_indexed_byte(0); + while zipper.child_count() < 2 { + if !zipper.descend_until() { + return summarize_ascend_to_fork::( + &mut zipper, + focus_depth, + None, + debug_path_f, + new_acc_f, + fold_child_f, + summarize_f, + ).map(|result| match result { + AscendResult::Parent(w) | AscendResult::Focus(w) => w, + }) + } + } + let accumulator = new_acc_f(&ByteMask::from(zipper.child_mask()))?; + stack.push(SummarizeStackFrame::new(zipper.child_count(), accumulator)); + } else { + let accumulator = new_acc_f(&ByteMask::from(zipper.child_mask()))?; + stack.push(SummarizeStackFrame::new(zipper.child_count(), accumulator)); + } + let mut cache = HashMap::::new(); - stack.push_state(&zipper); 'outer: loop { let frame_mut = stack.last_mut() - .expect("into_cata stack is emptied before we returned to root"); - // This branch represents the body of the for loop. + .expect("summarization stack is emptied before we returned to root"); + if frame_mut.child_idx < frame_mut.child_cnt { - let descended = zipper.descend_indexed_byte(frame_mut.child_idx as usize); - debug_assert!(descended.is_some()); + zipper.descend_indexed_byte(frame_mut.child_idx as usize); frame_mut.child_idx += 1; frame_mut.child_addr = zipper.shared_node_id(); - // Read and reuse value from cache, if exists - if let Some(cache) = Cache::get(&cache, frame_mut.child_addr) { - // DO NOT modify the W from cache - children.push(cache); + if let Some(cached) = DoCache::get(&cache, frame_mut.child_addr) { zipper.ascend_byte(); + let child_mask = ByteMask::from(zipper.child_mask()); + fold_child_f(&child_mask, cached, &mut frame_mut.accumulator)?; continue 'outer; } - // Descend until leaf or branch let mut is_leaf = false; - 'descend: while zipper.child_count() < 2 { + while zipper.child_count() < 2 { if !zipper.descend_until() { is_leaf = true; - break 'descend; + break; } } if is_leaf { - // If we encounter a leaf, ascend immediately. - // This branch will preserve the current stack frame. - let cur_w = ascend_to_fork::( - &mut zipper, &mut |mask, children, jump, val, path, z| { - alg_f(mask, children, val, &path[path.len()-jump..], path, z) - }, &mut [])?; - // Put value to cache (1) - Cache::insert(&mut cache, frame_mut.child_addr, &cur_w); - children.push(cur_w); + let cur_w = match summarize_ascend_to_fork::( + &mut zipper, + focus_depth, + None, + debug_path_f, + new_acc_f, + fold_child_f, + summarize_f, + )? { + AscendResult::Parent(w) => w, + AscendResult::Focus(w) => return Ok(w), + }; + DoCache::insert(&mut cache, frame_mut.child_addr, &cur_w); + let child_mask = ByteMask::from(zipper.child_mask()); + fold_child_f(&child_mask, cur_w, &mut frame_mut.accumulator)?; continue 'outer; } - // Enter one recursion step - stack.push_state(&zipper); + let accumulator = new_acc_f(&ByteMask::from(zipper.child_mask()))?; + stack.push(SummarizeStackFrame::new(zipper.child_count(), accumulator)); continue 'outer; } - // This branch represents the rest of the function after the loop - let frame_idx = stack.position; - let StackFrame { child_cnt, .. } = stack.pop_mut() - .expect("we just checked that stack is not empty, pop must return Some"); - let child_start = children.len() - *child_cnt as usize; - let children2 = &mut children[child_start..]; - - if frame_idx == 0 { - // Final branch - debug_assert!(zipper.at_root(), "must be at root when cata is done"); - let value = zipper.val(); - let child_mask = ByteMask::from(zipper.child_mask()); - return if JUMPING && *child_cnt == 1 && value.is_none() { - Ok(children.pop().unwrap()) + let frame = stack.pop() + .expect("we just checked that the summarization stack is not empty"); + + if stack.is_empty() { + return if passthrough_root { + summarize_ascend_to_fork::( + &mut zipper, + focus_depth, + Some(frame.accumulator), + debug_path_f, + new_acc_f, + fold_child_f, + summarize_f, + ).map(|result| match result { + AscendResult::Parent(w) | AscendResult::Focus(w) => w, + }) } else { - let debug_path = if DEBUG_PATH { - zipper.origin_path() - } else { - &[] - }; - alg_f(&child_mask, children2, value, &[], debug_path, &zipper) + debug_assert_eq!(zipper.depth(), focus_depth, "must be at the initial focus when summarization is done"); + let child_mask = ByteMask::from(zipper.child_mask()); + let debug_path = if DEBUG_PATH { debug_path_f(&zipper, focus_depth) } else { &[] }; + summarize_f(&child_mask, zipper.val(), Some(frame.accumulator), &[], debug_path) }; } - let cur_w = ascend_to_fork::( - &mut zipper, &mut |mask, children, jump, val, path, z| { - alg_f(mask, children, val, &path[path.len()-jump..], path, z) - }, children2)?; - children.truncate(child_start); + let cur_w = match summarize_ascend_to_fork::( + &mut zipper, + focus_depth, + Some(frame.accumulator), + debug_path_f, + new_acc_f, + fold_child_f, + summarize_f, + )? { + AscendResult::Parent(w) => w, + AscendResult::Focus(w) => return Ok(w), + }; - // Exit one recursion step let frame_mut = stack.last_mut() - .expect("when we're not at root, expect parent stack"); - // Put value to cache (2) after recursion - Cache::insert(&mut cache, frame_mut.child_addr, &cur_w); - children.push(cur_w); + .expect("when we're not at root, expect a parent summarization stack frame"); + DoCache::insert(&mut cache, frame_mut.child_addr, &cur_w); + let child_mask = ByteMask::from(zipper.child_mask()); + fold_child_f(&child_mask, cur_w, &mut frame_mut.accumulator)?; } } +#[inline(always)] +fn debug_origin_path<'a, Z>(zipper: &'a Z, depth: usize) -> &'a [u8] +where + Z: Zipper + ZipperAbsolutePath + ZipperPathBuffer, +{ + let root_prefix_len = zipper.origin_path().len() - zipper.depth(); + // SAFETY: the caller supplies a depth that the zipper occupied earlier in this traversal; + // `prepare_buffers` keeps that path initialized until the next movement. + unsafe { zipper.origin_path_assert_len(root_prefix_len + depth) } +} + +/// Debug-only adapter over the iterative cached cata. The extra path is an absolute path borrowed +/// directly from the zipper buffer and must not influence the cached algebra's result. +pub(crate) fn cata_jumping_cached_debug_body<'a, Z, V: 'a, W, E, AlgF>( + zipper: Z, + alg_f: AlgF, +) -> Result +where + W: Clone, + Z: Zipper + ZipperReadOnlyConditionalValues<'a, V> + ZipperConcrete + ZipperAbsolutePath + ZipperPathBuffer, + AlgF: Fn(&ByteMask, &mut [W], Option<&V>, &[u8], &[u8]) -> Result, +{ + let children = std::cell::RefCell::new(CataChildren::::new()); + let children = &children; + let alg_f = &alg_f; + + summarize_cached_body_with_debug::( + zipper, + debug_origin_path::, + move |mask| { + debug_assert!(children.try_borrow_mut().is_ok()); + Ok(unsafe { &mut *children.as_ptr() }.new_acc(mask.count_bits())) + }, + move |_mask, child, accumulator| { + debug_assert!(children.try_borrow_mut().is_ok()); + unsafe { &mut *children.as_ptr() }.push(accumulator, child); + Ok(()) + }, + move |mask, value, accumulator, prefix, debug_path| match accumulator { + Some(accumulator) => { + debug_assert!(children.try_borrow_mut().is_ok()); + unsafe { &mut *children.as_ptr() }.summarize(accumulator, mask.count_bits(), |children| { + alg_f(mask, children, value, prefix, debug_path) + }) + }, + None => alg_f(mask, &mut [], value, prefix, debug_path), + }, + ) +} + // This is a naive implementation of caching/jumping cata // The code is left in for reference/readability, since the unrolled version // is very hard to read. It took several days to debug the unrolled version. @@ -1307,17 +1840,909 @@ impl TrieBuilder { // } // } +/// Shared cached-cata conformance tests for zipper implementations. #[cfg(test)] -mod tests { - use std::ops::Range; +pub(crate) mod cached_catamorphism_tests { + use core::convert::Infallible; + + use crate::alloc::GlobalAlloc; + use crate::utils::{ByteMask, ByteMaskIter}; + use crate::write_zipper::{WriteZipperOwned, ZipperWriting}; + use crate::zipper::{Zipper, ZipperIteration, ZipperMoving, ZipperPath}; + use crate::PathMap; + + pub const CACHED_CATA_TEST_KEYS: &[&[u8]] = &[ + b"arrow", b"bow", b"cannon", b"roman", b"romane", b"romanus", b"romulus", + b"rubens", b"ruber", b"rubicon", b"rubicundus", b"rom'i", + ]; + pub const CACHED_CATA_FOLD_ORDER_KEYS: &[&[u8]] = &[b"a", b"b"]; + pub const CACHED_CATA_PASSTHROUGH_KEYS: &[&[u8]] = &[b"abc"]; + + /// The branch bytes a node owes its `fold_child` calls. + /// + /// The contract is one fold per bit, in ascending mask-bit order. The reconstruction + /// probe uses this to make an invalid callback sequence fail at the first bad fold. + pub(crate) struct BranchBytes(ByteMaskIter); + + impl BranchBytes { + fn of(child_mask: &ByteMask) -> Self { + Self(child_mask.iter()) + } + + fn take(&mut self) -> u8 { + self.0.next().expect( + "cached-cata contract violation: more fold_child calls than child-mask bits", + ) + } + + fn finish(mut self) { + assert!( + self.0.next().is_none(), + "cached-cata contract violation: fewer fold_child calls than child-mask bits", + ); + } + } + + /// Accumulator for the cached-cata reconstruction probe. + /// + /// Every branch owns one write zipper. Child maps are grafted one mask-selected byte below + /// that zipper; `recon_summarize` then places the prefix and optional value around them. + pub(crate) struct ReconAcc { + branch_bytes: BranchBytes, + wz: WriteZipperOwned<()>, + } + + pub(crate) fn recon_start(child_mask: &ByteMask) -> Result { + Ok(ReconAcc { + branch_bytes: BranchBytes::of(child_mask), + wz: PathMap::new().into_write_zipper(b""), + }) + } + + pub(crate) fn recon_fold( + _child_mask: &ByteMask, + child: PathMap<()>, + accumulator: &mut ReconAcc, + ) -> Result<(), Infallible> { + let branch_byte = accumulator.branch_bytes.take(); + accumulator.wz.descend_to_byte(branch_byte); + accumulator.wz.graft_map(child); + accumulator.wz.ascend_byte(); + Ok(()) + } + + pub(crate) fn recon_summarize( + _child_mask: &ByteMask, + value: Option<&()>, + children: Option, + prefix: &[u8], + ) -> Result, Infallible> { + match children { + Some(ReconAcc { branch_bytes, mut wz }) => { + branch_bytes.finish(); + if !prefix.is_empty() { + wz.insert_prefix(prefix); + } + if value.is_some() { + wz.descend_to(prefix); + wz.set_val(()); + } + Ok(wz.into_map()) + } + None => { + let mut map = PathMap::new(); + if value.is_some() { + map.set_val_at(prefix, ()); + } + Ok(map) + } + } + } + + /// Reconstructs a logical trie through a named cached-cata implementation. + /// + /// This is intentionally a macro because the two engines are traits with the same methods, + /// rather than values of a common engine type. + macro_rules! reconstruct_trie { + (CatamorphismCached, $subject:expr) => {{ + <_ as $crate::morphisms::CatamorphismCached<(), $crate::alloc::GlobalAlloc>> + ::factored_cata_jumping::<_, _, core::convert::Infallible, _, _, _, true>( + $subject, + $crate::morphisms::cached_catamorphism_tests::recon_start, + $crate::morphisms::cached_catamorphism_tests::recon_fold, + $crate::morphisms::cached_catamorphism_tests::recon_summarize, + ) + .unwrap() + }}; + (CatamorphismCachedIterative, $subject:expr) => {{ + <_ as $crate::morphisms::CatamorphismCachedIterative<()>> + ::factored_cata_jumping::<_, _, core::convert::Infallible, _, _, _, true>( + $subject, + $crate::morphisms::cached_catamorphism_tests::recon_start, + $crate::morphisms::cached_catamorphism_tests::recon_fold, + $crate::morphisms::cached_catamorphism_tests::recon_summarize, + ) + .unwrap() + }}; + } + #[allow(unused_imports)] // Exported for conformance tests in other modules. + pub(crate) use reconstruct_trie; + + /// Compares value paths without materializing either map's complete path list. + #[track_caller] + pub(crate) fn assert_same_paths( + got: &PathMap<()>, + expected: &PathMap<()> + ) { + let mut got = got.read_zipper(); + let mut expected = expected.read_zipper(); + + assert_eq!(got.is_val(), expected.is_val()); + if got.is_val() { + assert_eq!(got.path(), expected.path()); + } + + loop { + let got_has_value = got.to_next_val(); + let expected_has_value = expected.to_next_val(); + assert_eq!(got_has_value, expected_has_value, ); + if !got_has_value { + break; + } + assert_eq!(got.path(), expected.path()); + } + } + + /// Runs both cached-cata engines through the reconstruction probe and checks each result + /// against the expected logical trie as well as against one another. + #[track_caller] + pub(crate) fn assert_reconstructs_like(subject: &Z, expected: &PathMap<()>) + where + Z: crate::morphisms::CatamorphismCached<(), GlobalAlloc> + + crate::morphisms::CatamorphismCachedIterative<()>, + { + let iterative = reconstruct_trie!(CatamorphismCachedIterative, subject); + assert_same_paths(&iterative, expected); + + let recursive = reconstruct_trie!(CatamorphismCached, subject); + assert_same_paths(&recursive, expected); + assert_same_paths(&recursive, &iterative); + } + + fn map_from_keys(keys: &[&[u8]]) -> PathMap<()> { + keys.iter().copied().map(|path| (path, ())).collect() + } + + fn map_from_owned_keys(keys: &[Vec]) -> PathMap<()> { + keys.iter().map(|path| (path.as_slice(), ())).collect() + } + + /// Checks a logical key set in both insertion orders. This deliberately does not inspect + /// the resulting storage layout: a valid cached cata must honor the same observable contract + /// regardless of how the implementation stores these keys. + #[track_caller] + fn assert_logical_key_case_roundtrips(keys: &[Vec]) { + let map = map_from_owned_keys(keys); + assert_reconstructs_like(&map, &map); + + let mut reversed = keys.to_vec(); + reversed.reverse(); + let map = map_from_owned_keys(&reversed); + assert_reconstructs_like(&map, &map); + } + + /// Finds prefixes where the logical input keys have at least two different following bytes. + /// These are derived from the test data alone, not from the trie's concrete representation. + fn logical_branch_prefixes(keys: &[Vec]) -> Vec> { + let mut prefixes = Vec::new(); + for key in keys { + for len in 0..key.len() { + let prefix = &key[..len]; + let first_child = keys.iter() + .filter(|candidate| candidate.starts_with(prefix)) + .filter_map(|candidate| candidate.get(len)) + .next(); + let is_branch = first_child.is_some_and(|first| { + keys.iter() + .filter(|candidate| candidate.starts_with(prefix)) + .filter_map(|candidate| candidate.get(len)) + .any(|child| child != first) + }); + if is_branch && !prefixes.iter().any(|existing| existing == prefix) { + prefixes.push(prefix.to_vec()); + } + } + } + prefixes + } + + /// Runs the focused-zipper portion of the differential harness. Both the requested focus + /// and the expected subtrie come from the logical source keys, keeping this test independent + /// of the storage representation. + fn assert_logical_focus_roundtrips(keys: &[Vec], focus: &[u8]) { + let map = map_from_owned_keys(keys); + let expected_keys: Vec> = keys.iter() + .filter_map(|key| key.strip_prefix(focus).map(ToOwned::to_owned)) + .collect(); + let expected = map_from_owned_keys(&expected_keys); + + let mut zipper = map.read_zipper(); + zipper.descend_to(focus); + assert_eq!(zipper.path(), focus); + assert_reconstructs_like(&zipper, &expected); + } + + fn assert_random_case( + seed: &[u8; 32], + round: usize, + focus: Option<&[u8]>, + test: impl FnOnce(), + ) { + if std::panic::catch_unwind(std::panic::AssertUnwindSafe(test)).is_err() { + panic!( + "random cached-cata regression: seed={seed:?}, round={round}, focus={focus:?}", + ); + } + } + + #[test] + fn reconstruction_harness_sanity() { + let map = map_from_keys(&[b"a1", b"a2", b"b"]); + + assert_reconstructs_like(&map, &map); + } + + /// A value and a child with different first bytes must be folded in ascending byte order. + /// + /// The keys are deliberately chosen only through the public map API; their physical node + /// representation is not part of this test's setup or assertion. + #[test] + fn recursive_cata_regression_val_child_fold_order() { + let map = map_from_keys(&[b"a", b"b1", b"b2"]); + + assert_reconstructs_like(&map, &map); + } + + /// A short value sharing its first byte with a longer value must stay at its exact path. + #[test] + fn recursive_cata_regression_shared_byte_value_position() { + let map = map_from_keys(&[b"a", b"abc"]); + + assert_reconstructs_like(&map, &map); + } + + /// A value passed through a non-branching run must not cause its downstream child to be + /// folded through an empty mask. + #[test] + fn recursive_cata_regression_passed_value_child_mask() { + let map = map_from_keys(&[b"x", b"xy", b"xyz", b"xa1", b"xa2", b"q"]); + + assert_reconstructs_like(&map, &map); + } + + /// A focus within a compressed path must include every value below it, including any value + /// held at the focus. Reconstructed paths are relative to that focus. + #[test] + fn recursive_cata_regression_mid_node_focus() { + let map = map_from_keys(&[b"abc1", b"abc2"]); + let mut zipper = map.read_zipper(); + zipper.descend_to(b"ab"); + assert_eq!(zipper.path(), b"ab"); + assert_reconstructs_like(&zipper, &map_from_keys(&[b"c1", b"c2"])); + + let map = map_from_keys(&[b"ab", b"abcd"]); + let mut zipper = map.read_zipper(); + zipper.descend_to(b"ab"); + assert_eq!(zipper.path(), b"ab"); + assert_reconstructs_like(&zipper, &map_from_keys(&[b"", b"cd"])); + + let map = map_from_keys(&[b"abc"]); + let mut zipper = map.read_zipper(); + zipper.descend_to(b"abc"); + assert_eq!(zipper.path(), b"abc"); + assert_reconstructs_like(&zipper, &map_from_keys(&[b""])); + } + + /// Systematically exercises logical value/branch/prefix combinations that may be represented + /// compactly in different ways. The corpus speaks only in paths and values; it neither + /// assumes nor inspects a particular concrete node representation. + #[test] + fn recursive_cata_logical_prefix_and_branch_corpus() { + let mut cases: Vec>> = vec![ + vec![], + vec![b"".to_vec()], + vec![b"abcd".to_vec()], + vec![b"a".to_vec(), b"ab".to_vec(), b"abc".to_vec()], + vec![b"a".to_vec(), b"b".to_vec()], + vec![b"a1".to_vec(), b"a2".to_vec(), b"b1".to_vec(), b"b2".to_vec()], + vec![b"a".to_vec(), b"ab1".to_vec(), b"ab2".to_vec()], + vec![b"".to_vec(), b"q1".to_vec(), b"q2".to_vec()], + // A value mid-run with both a deeper child and a sibling branch. + vec![b"accab".to_vec(), b"ac".to_vec(), b"accac".to_vec(), b"abacc".to_vec()], + ]; + + for tail_len in 1..=4 { + let tail = &b"cdef"[..tail_len]; + + let mut shared_value_tail = b"a".to_vec(); + shared_value_tail.extend_from_slice(tail); + cases.push(vec![b"a".to_vec(), shared_value_tail]); + + let mut high_child_1 = b"b1".to_vec(); + high_child_1.extend_from_slice(tail); + let mut high_child_2 = b"b2".to_vec(); + high_child_2.extend_from_slice(tail); + cases.push(vec![b"a".to_vec(), high_child_1, high_child_2]); + + let mut low_child_1 = b"a1".to_vec(); + low_child_1.extend_from_slice(tail); + let mut low_child_2 = b"a2".to_vec(); + low_child_2.extend_from_slice(tail); + cases.push(vec![low_child_1, low_child_2, b"b".to_vec()]); + + let mut same_byte_child_1 = b"ab1".to_vec(); + same_byte_child_1.extend_from_slice(tail); + let mut same_byte_child_2 = b"ab2".to_vec(); + same_byte_child_2.extend_from_slice(tail); + cases.push(vec![b"a".to_vec(), same_byte_child_1, same_byte_child_2]); + } + + for keys in cases { + assert_logical_key_case_roundtrips(&keys); + } + } + + /// A 256-way logical branch exercises storage configurations that use wide byte-indexed + /// nodes without making the test depend on any particular node representation. Miri uses + /// representative mask-word-boundary bytes instead of the full branch fanout. + #[test] + fn recursive_cata_wide_logical_branch() { + #[cfg(miri)] + let keys: Vec> = [0, 1, 63, 64, 127, 128, 191, 192, 254, 255] + .into_iter() + .map(|byte| vec![byte, b'a', byte]) + .collect(); + #[cfg(not(miri))] + let keys: Vec> = (0u8..=u8::MAX) + .map(|byte| vec![byte, b'a', byte]) + .collect(); + + assert_logical_key_case_roundtrips(&keys); + #[cfg(miri)] + let focus_bytes = [0, 64, 128, 255]; + #[cfg(not(miri))] + let focus_bytes = [0, 63, 64, 127, 128, 191, 192, 255]; + for byte in focus_bytes { + assert_logical_focus_roundtrips(&keys, &[byte]); + } + } + + /// Differential coverage over seeded logical maps and root/value/branch/proper-prefix + /// focuses. The seed is fixed so any failure is reproducible without observing concrete + /// node layout. + #[test] + fn recursive_cata_randomized_maps_and_foci() { + const SEED: [u8; 32] = [31; 32]; + + #[cfg(miri)] + const ROUNDS: usize = 1; + #[cfg(not(miri))] + const ROUNDS: usize = 64; + + #[cfg(miri)] + const KEYS_PER_ROUND: usize = 12; + #[cfg(not(miri))] + const KEYS_PER_ROUND: usize = 48; + + #[cfg(miri)] + const FOCI_PER_ROUND: usize = 3; + #[cfg(not(miri))] + const FOCI_PER_ROUND: usize = 8; + + use rand::{Rng, SeedableRng}; + use rand::rngs::StdRng; + + let mut rng = StdRng::from_seed(SEED); + for round in 0..ROUNDS { + let keys: Vec> = (0..KEYS_PER_ROUND) + .map(|_| { + let len = rng.random_range(0..=6); + (0..len) + .map(|_| b'a' + rng.random_range(0..3)) + .collect() + }) + .collect(); + let branch_prefixes = logical_branch_prefixes(&keys); + + assert_random_case(&SEED, round, None, || { + assert_logical_key_case_roundtrips(&keys); + }); + assert_random_case(&SEED, round, Some(b""), || { + assert_logical_focus_roundtrips(&keys, b""); + }); + + for focus_idx in 0..FOCI_PER_ROUND { + let key = &keys[rng.random_range(0..keys.len())]; + let focus = match focus_idx % 3 { + // An exact value path. + 0 => key.clone(), + // A proper prefix, which may fall inside a compressed run. + 1 if !key.is_empty() => { + key[..rng.random_range(0..key.len())].to_vec() + } + // A logical branch prefix, or root if the generated map has none. + _ if branch_prefixes.is_empty() => Vec::new(), + _ => branch_prefixes[rng.random_range(0..branch_prefixes.len())].clone(), + }; + assert_random_case(&SEED, round, Some(&focus), || { + assert_logical_focus_roundtrips(&keys, &focus); + }); + } + } + } + + macro_rules! define_cached_catamorphism_test_suite { + ($suite_name:ident, $cata_trait:ident, [$($cata_args:ty),+]) => { + pub(crate) mod $suite_name { + use super::{Infallible, ZipperMoving, ZipperPath}; + use crate::morphisms::$cata_trait; + use crate::utils::BitMask; + + pub fn factored_cata_propagates_callback_errors(zipper: Z) + where + Z: crate::morphisms::$cata_trait<$($cata_args),*>, + { + let error = $cata_trait::<$($cata_args),*> + ::factored_cata_jumping::<(), (), &'static str, _, _, _, false>( + &zipper, + |_| Err("new"), + |_mask, _child, _acc| Ok(()), + |_mask, _value, _acc, _prefix| Ok(()), + ); + assert_eq!(error, Err("new")); + + let error = $cata_trait::<$($cata_args),*> + ::factored_cata_jumping::<(), (), &'static str, _, _, _, false>( + &zipper, + |_| Ok(()), + |_mask, _child, _acc| Err("fold"), + |_mask, _value, _acc, _prefix| Ok(()), + ); + assert_eq!(error, Err("fold")); + + let error = $cata_trait::<$($cata_args),*> + ::factored_cata_jumping::<(), (), &'static str, _, _, _, false>( + &zipper, + |_| Ok(()), + |_mask, _child, _acc| Ok(()), + |_mask, _value, _acc, _prefix| Err("summarize"), + ); + assert_eq!(error, Err("summarize")); + } + + pub fn leaf_count_stepping(zipper: Z) + where + Z: crate::morphisms::$cata_trait<$($cata_args),*>, + { + let count = $cata_trait::<$($cata_args),*>::cata_cached( + &zipper, + |_mask, children: &mut [usize], value| { + if children.is_empty() { + assert!(value.is_some()); + 1 + } else { + children.iter().sum() + } + }, + ); + assert_eq!(count, 11); + } + + pub fn leaf_count_jumping(zipper: Z) + where + Z: crate::morphisms::$cata_trait<$($cata_args),*>, + { + let count = $cata_trait::<$($cata_args),*>::cata_jumping_cached( + &zipper, + |_mask, children: &mut [usize], value, _prefix| { + if children.is_empty() { + assert!(value.is_some()); + 1 + } else { + children.iter().sum() + } + }, + ); + assert_eq!(count, 11); + } + + pub fn leaf_count_factored_jumping(zipper: Z) + where + Z: crate::morphisms::$cata_trait<$($cata_args),*>, + { + let count = $cata_trait::<$($cata_args),*> + ::factored_cata_jumping::( + &zipper, + |_| Ok(0), + |_mask, child, total| { *total += child; Ok(()) }, + |_mask, value, total, _prefix| match total { + Some(total) => Ok(total), + None => { + assert!(value.is_some()); + Ok(1) + } + }, + ) + .unwrap(); + assert_eq!(count, 11); + } + + pub fn leaf_count_factored_stepping(zipper: Z) + where + Z: crate::morphisms::$cata_trait<$($cata_args),*>, + { + let count = $cata_trait::<$($cata_args),*> + ::factored_cata::( + &zipper, + |_| Ok(0), + |_mask, child, total| { *total += child; Ok(()) }, + |_mask, value, total| match total { + Some(total) => Ok(total), + None => { + assert!(value.is_some()); + Ok(1) + } + }, + ) + .unwrap(); + assert_eq!(count, 11); + } + + /// A cached cata starts at the zipper's focus: it includes a value held there and all of its + /// descendants, but not values elsewhere in the trie. + pub fn cata_from_value_focus(mut zipper: Z) + where + Z: ZipperMoving + ZipperPath + crate::morphisms::$cata_trait<$($cata_args),*>, + { + zipper.descend_to(b"roman"); + let count = $cata_trait::<$($cata_args),*>::cata_cached( + &zipper, + |_mask, children: &mut [usize], value| { + value.is_some() as usize + children.iter().sum::() + }, + ); + + assert_eq!(count, 3); + assert_eq!(zipper.path(), b"roman"); + } + + /// The focus may be within a compressed run rather than at an explicit logical branch. + /// Traversal still covers precisely the subtrie below that point. + pub fn cata_from_mid_run_focus(mut zipper: Z) + where + Z: ZipperMoving + ZipperPath + crate::morphisms::$cata_trait<$($cata_args),*>, + { + zipper.descend_to(b"roma"); + let count = $cata_trait::<$($cata_args),*>::cata_jumping_cached( + &zipper, + |_mask, children: &mut [usize], value, _prefix| { + value.is_some() as usize + children.iter().sum::() + }, + ); + + assert_eq!(count, 3); + assert_eq!(zipper.path(), b"roma"); + } + + pub fn longest_path_jumping(zipper: Z) + where + Z: crate::morphisms::$cata_trait<$($cata_args),*>, + { + let longest = $cata_trait::<$($cata_args),*>::cata_jumping_cached( + &zipper, + |mask, children: &mut [Vec], _value, prefix| { + let mut longest = mask.iter().zip(children.iter_mut()) + .max_by_key(|(_byte, rest)| rest.len()) + .map_or_else(Vec::new, |(byte, rest)| { + let mut path = core::mem::take(rest); + path.insert(0, byte); + path + }); + let mut path = prefix.to_vec(); + path.append(&mut longest); + path + }, + ); + assert_eq!(longest, b"rubicundus"); + } + + pub fn longest_path_factored_jumping(zipper: Z) + where + Z: crate::morphisms::$cata_trait<$($cata_args),*>, + { + let longest = $cata_trait::<$($cata_args),*> + ::factored_cata_jumping::>, Vec, Infallible, _, _, _, true>( + &zipper, + |_| Ok(Vec::new()), + |_mask, child, children| { children.push(child); Ok(()) }, + |mask, _value, children, prefix| { + let mut longest = children.map_or_else(Vec::new, |children| { + if mask.is_empty_mask() { + children.into_iter().max_by_key(|rest| rest.len()).unwrap_or_default() + } else { + mask.iter().zip(children).max_by_key(|(_byte, rest)| rest.len()) + .map_or_else(Vec::new, |(byte, mut rest)| { + rest.insert(0, byte); + rest + }) + } + }); + let mut path = prefix.to_vec(); + path.append(&mut longest); + Ok(path) + }, + ) + .unwrap(); + assert_eq!(longest, b"rubicundus"); + } + + pub fn branch_values_stepping(zipper: Z) + where + Z: crate::morphisms::$cata_trait<$($cata_args),*>, + { + let values = $cata_trait::<$($cata_args),*>::cata_cached( + &zipper, + |_mask, children: &mut [Vec], value| { + if children.is_empty() { + Vec::new() + } else if let Some(value) = value { + vec![*value] + } else { + let mut values = children.first_mut().map_or_else(Vec::new, core::mem::take); + for child in &mut children[1..] { + values.append(child); + } + values + } + }, + ); + assert_eq!(values, vec![3]); + } + + pub fn factored_cata_folds_each_child_immediately(zipper: Z) + where + Z: crate::morphisms::$cata_trait<$($cata_args),*>, + { + use std::cell::RefCell; + + let events = RefCell::new(Vec::new()); + let result = $cata_trait::<$($cata_args),*> + ::factored_cata_jumping::, u64, Infallible, _, _, _, false>( + &zipper, + |_mask| { + events.borrow_mut().push("new"); + Ok(Vec::new()) + }, + |_mask, child, accumulator| { + events.borrow_mut().push(if child == 0 { "fold 0" } else { "fold 1" }); + accumulator.push(child); + Ok(()) + }, + |_mask, value, accumulator, _prefix| { + match value { + Some(0) => events.borrow_mut().push("summarize 0"), + Some(1) => events.borrow_mut().push("summarize 1"), + _ => events.borrow_mut().push("summarize root"), + } + Ok(value.copied().unwrap_or_else(|| accumulator.unwrap().into_iter().sum())) + }, + ) + .unwrap(); + assert_eq!(result, 1); + assert_eq!(events.into_inner(), ["new", "summarize 0", "fold 0", "summarize 1", "fold 1", "summarize root"]); + } + + pub fn factored_cata_passthrough_root(zipper: Z) + where + Z: crate::morphisms::$cata_trait<$($cata_args),*>, + { + let result = $cata_trait::<$($cata_args),*> + ::factored_cata_jumping::)>, Vec, Infallible, _, _, _, true>( + &zipper, + |_| Ok(Vec::new()), + |mask, child, children| { + let byte = mask.iter().nth(children.len()) + .expect("fold_child called more times than child-mask bits"); + children.push((byte, child)); + Ok(()) + }, + |_mask, value, children, prefix| { + let mut path = prefix.to_vec(); + match (value, children) { + (Some(&0), None) => {}, + (None, Some(mut children)) => { + assert_eq!(children.len(), 1); + let (byte, child) = children.pop().unwrap(); + path.push(byte); + path.extend(child); + }, + _ => panic!("unexpected callback shape for a single value path"), + } + Ok(path) + }, + ) + .unwrap(); + assert_eq!(result, b"abc"); + } + + /// Internal helper that gives the zipper constructor the lifetime of the test store. + pub fn run_test<'a, Z, Store>( + store: &'a mut Store, + make_z: impl Fn(&'a mut Store) -> Z, + test: impl Fn(Z), + ) + where + Z: 'a + crate::morphisms::$cata_trait<$($cata_args),*>, + { + test(make_z(store)); + } + + } + }; + } + + define_cached_catamorphism_test_suite!(recursive, CatamorphismCached, [u64, crate::alloc::GlobalAlloc]); + define_cached_catamorphism_test_suite!(iterative, CatamorphismCachedIterative, [u64]); + + macro_rules! cached_catamorphism_case { + ($z_name:ident, $implementation:ident, $suite:ident, $read_keys:expr, $make_z:expr, $keys:ident, $test:ident) => { + paste::paste! { + #[test] + fn [<$z_name _ $implementation _ $test>]() { + let mut temp_store = ($read_keys)(crate::morphisms::cached_catamorphism_tests::$keys); + crate::morphisms::cached_catamorphism_tests::$suite::run_test( + &mut temp_store, + $make_z, + crate::morphisms::cached_catamorphism_tests::$suite::$test, + ); + } + } + }; + } + pub(crate) use cached_catamorphism_case; + + macro_rules! cached_catamorphism_tests { + ($z_name:ident, $read_keys:expr, $make_z:expr, CatamorphismCached) => { + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_tests!($z_name, $read_keys, $make_z, recursive, recursive); + }; + ($z_name:ident, $read_keys:expr, $make_z:expr, CatamorphismCachedIterative) => { + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_tests!($z_name, $read_keys, $make_z, iterative, iterative); + }; + ($z_name:ident, $read_keys:expr, $make_z:expr, $implementation:ident, $suite:ident) => { + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_case!($z_name, $implementation, $suite, $read_keys, $make_z, CACHED_CATA_TEST_KEYS, leaf_count_stepping); + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_case!($z_name, $implementation, $suite, $read_keys, $make_z, CACHED_CATA_TEST_KEYS, leaf_count_jumping); + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_case!($z_name, $implementation, $suite, $read_keys, $make_z, CACHED_CATA_TEST_KEYS, leaf_count_factored_jumping); + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_case!($z_name, $implementation, $suite, $read_keys, $make_z, CACHED_CATA_TEST_KEYS, leaf_count_factored_stepping); + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_case!($z_name, $implementation, $suite, $read_keys, $make_z, CACHED_CATA_TEST_KEYS, cata_from_value_focus); + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_case!($z_name, $implementation, $suite, $read_keys, $make_z, CACHED_CATA_TEST_KEYS, cata_from_mid_run_focus); + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_case!($z_name, $implementation, $suite, $read_keys, $make_z, CACHED_CATA_TEST_KEYS, longest_path_jumping); + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_case!($z_name, $implementation, $suite, $read_keys, $make_z, CACHED_CATA_TEST_KEYS, longest_path_factored_jumping); + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_case!($z_name, $implementation, $suite, $read_keys, $make_z, CACHED_CATA_TEST_KEYS, branch_values_stepping); + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_case!($z_name, $implementation, $suite, $read_keys, $make_z, CACHED_CATA_FOLD_ORDER_KEYS, factored_cata_folds_each_child_immediately); + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_case!($z_name, $implementation, $suite, $read_keys, $make_z, CACHED_CATA_PASSTHROUGH_KEYS, factored_cata_passthrough_root); + $crate::morphisms::cached_catamorphism_tests::cached_catamorphism_case!($z_name, $implementation, $suite, $read_keys, $make_z, CACHED_CATA_TEST_KEYS, factored_cata_propagates_callback_errors); + }; + } + pub(crate) use cached_catamorphism_tests; +} + +#[cfg(test)] +mod tests { + use std::ops::Range; use crate::PathMap; use crate::utils::BitMask; use super::*; + trait TestRecursiveCata: Sized { + fn recursive_cata_cached(&self, alg_f: AlgF) -> W + where + Self: CatamorphismCached, + W: Clone, + AlgF: Fn(&ByteMask, &mut [W], Option<&V>) -> W, + { + CatamorphismCached::::cata_cached(self, alg_f) + } + + fn recursive_cata_jumping_cached(&self, alg_f: AlgF) -> W + where + Self: CatamorphismCached, + W: Clone, + AlgF: Fn(&ByteMask, &mut [W], Option<&V>, &[u8]) -> W, + { + CatamorphismCached::::cata_jumping_cached(self, alg_f) + } + + fn recursive_factored_cata_jumping(&self, new_acc_f: NewAccF, fold_child_f: FoldChildF, summarize_f: SummarizeF) -> Result + where + Self: CatamorphismCached, + W: Clone, + NewAccF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + SummarizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8]) -> Result, + { + CatamorphismCached::::factored_cata_jumping::(self, new_acc_f, fold_child_f, summarize_f) + } + + fn recursive_factored_cata(&self, new_acc_f: NewAccF, fold_child_f: FoldChildF, summarize_f: SummarizeF) -> Result + where + Self: CatamorphismCached, + W: Clone, + NewAccF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + SummarizeF: Copy + Fn(&ByteMask, Option<&V>, Option) -> Result, + { + CatamorphismCached::::factored_cata(self, new_acc_f, fold_child_f, summarize_f) + } + } + + impl TestRecursiveCata for Z {} + + #[test] + fn recursive_and_iterative_cached_catas_agree() { + let map: PathMap = [ + (b"abc".as_slice(), 1), + (b"abd".as_slice(), 2), + (b"ax".as_slice(), 3), + ] + .into_iter() + .collect(); + + let alg = |_mask: &ByteMask, children: &mut [usize], value: Option<&usize>| { + children.iter().sum::() + value.copied().unwrap_or(0) + }; + let recursive_zipper = map.read_zipper(); + let recursive = CatamorphismCached::::cata_cached( + &recursive_zipper, + alg, + ); + let iterative_zipper = map.read_zipper(); + let iterative = CatamorphismCachedIterative::::cata_cached( + &iterative_zipper, + alg, + ); + + assert_eq!(recursive, iterative); + assert_eq!(recursive, 6); + } + + #[test] + fn cached_cata_facade_borrows_its_receiver() { + let map: PathMap = [(b"a".as_slice(), 42)].into_iter().collect(); + let stepping = CatamorphismCached::cata_cached(&map, |_mask, children: &mut [u64], value| { + children.iter().sum::() + value.copied().unwrap_or_default() + }); + let jumping = CatamorphismCached::cata_jumping_cached(&map, |_mask, children: &mut [u64], value, _prefix| { + children.iter().sum::() + value.copied().unwrap_or_default() + }); + let fallible = CatamorphismCached::cata_cached_fallible(&map, |_mask, children: &mut [u64], value| { + Ok::<_, Infallible>(children.iter().sum::() + value.copied().unwrap_or_default()) + }); + let jumping_fallible = CatamorphismCached::cata_jumping_cached_fallible(&map, |_mask, children: &mut [u64], value, _prefix| { + Ok::<_, Infallible>(children.iter().sum::() + value.copied().unwrap_or_default()) + }); + + assert_eq!((stepping, jumping, fallible, jumping_fallible), (42, 42, Ok(42), Ok(42))); + assert_eq!(CatamorphismCached::hash(&map), CatamorphismCached::hash(&map)); + assert_eq!( + CatamorphismCached::hash_with(&map, |value| *value as u128), + CatamorphismCached::hash_with(&map, |value| *value as u128), + ); + } + fn check_side_effect_catas<'a, W, V, Z, AlgF, Assert>( zipper: Z, mut f_side: AlgF, mut assert: Assert) where - Z: Clone + Catamorphism, W: Clone, + Z: Clone + CatamorphismSideEffecting, W: Clone, AlgF: FnMut(&ByteMask, &mut [W], usize, Option<&V>, &[u8]) -> W, Assert: FnMut(W, &str), { @@ -1329,32 +2754,32 @@ mod tests { assert(output, "into_cata_jumping_side_effect"); } - fn check_pure_catas<'a, W, V, Z, AlgFP, Assert>( + fn check_pure_catas<'a, W, V: Clone + Send + Sync, Z, AlgFP, Assert>( zipper: Z, f_pure: AlgFP, mut assert: Assert) where - Z: Clone + Catamorphism, W: Clone, + Z: Clone + CatamorphismCached, W: Clone, AlgFP: Fn(&ByteMask, &mut [W], Option<&V>, &[u8]) -> W, Assert: FnMut(W, &str), { - let output = zipper.clone().into_cata_cached( + let output = zipper.clone().recursive_cata_cached( |bm, ch, v| f_pure(bm, ch, v, &[])); - assert(output, "into_cata_cached"); - let output = zipper.clone().into_cata_jumping_cached( - |bm, ch, v, sub_path| f_pure(bm, ch, v, sub_path)); - assert(output, "into_cata_jumping_cached"); + assert(output, "cata_cached"); + let output = zipper.clone().recursive_cata_jumping_cached( + |bm, ch, v, prefix| f_pure(bm, ch, v, prefix)); + assert(output, "cata_jumping_cached"); } - fn check_all_catas<'a, W, V, Z, AlgF, Assert>( + fn check_all_catas<'a, W, V: Clone + Send + Sync, Z, AlgF, Assert>( zipper: Z, alg_f: AlgF, mut assert: Assert) where - Z: Clone + Catamorphism, W: Clone, + Z: Clone + CatamorphismSideEffecting + CatamorphismCached, W: Clone, AlgF: Fn(&ByteMask, &mut [W], Option<&V>) -> W, Assert: FnMut(W, &str), { check_side_effect_catas(zipper.clone(), |mask, children, _jmp, val, _path| { alg_f(mask, children, val) }, &mut assert); - check_pure_catas(zipper.clone(), |mask, children, val, _sub_path| { + check_pure_catas(zipper.clone(), |mask, children, val, _prefix| { alg_f(mask, children, val) }, &mut assert); } @@ -1405,19 +2830,20 @@ mod tests { } (val.is_some(), sum) }; - let output = map.read_zipper().into_cata_cached(pure_alg_stepping); + let zipper = map.read_zipper(); + let output = CatamorphismCached::<(), GlobalAlloc>::cata_cached(&zipper, pure_alg_stepping); assert_eq!(output.1, expected_sum); //The pure jumping cata is a variant on the above, but we also need to care about - // the sub_path we jump over. So we either count the value associated with the last - // byte of the sub-path, or with the next parent path byte. + // the prefix we jump over. So we either count the value associated with the last + // byte of the prefix, or with the next parent path byte. // //This code works fine for both stepping and jumping, but is a little more complicated // than the stepping-only version - let pure_alg = |child_mask: &ByteMask, children: &mut [(bool, u32)], val: Option<&()>, sub_path: &[u8]| { + let pure_alg = |child_mask: &ByteMask, children: &mut [(bool, u32)], val: Option<&()>, prefix: &[u8]| { let mut sum = 0; if val.is_some() { - if let Some(path_byte) = sub_path.last() { + if let Some(path_byte) = prefix.last() { sum += (*path_byte as char).to_digit(10).unwrap(); } } @@ -1427,7 +2853,7 @@ mod tests { } sum += *downstream_sum; } - (val.is_some() && sub_path.len()==0, sum) + (val.is_some() && prefix.len()==0, sum) }; //Test both stepping and jumping cached catas @@ -1474,11 +2900,11 @@ mod tests { assert_eq!(std::str::from_utf8(longest.as_slice()).unwrap(), "rubicundus")); //================================================================================= - // PureLongestPath - Finds the longest path in the trie by concatenating sub-paths; + // PureLongestPath - Finds the longest path in the trie by concatenating prefix paths; // This is necessary for pure catas because the same subtrie may share multiple base paths - fn longest_partial_path(child_mask: &ByteMask, children: &mut[Vec], sub_path: &[u8]) -> Vec { + fn longest_partial_path(child_mask: &ByteMask, children: &mut[Vec], prefix: &[u8]) -> Vec { if children.len() == 0 { - sub_path.to_vec() + prefix.to_vec() } else { let mut longest_downstream_path = child_mask.iter() .zip(children.iter_mut()).max_by_key(|(_byte, path_rest)| path_rest.len()) @@ -1487,7 +2913,7 @@ mod tests { path_rest.insert(0, byte); path_rest }); - let mut path = sub_path.to_vec(); + let mut path = prefix.to_vec(); path.append(&mut longest_downstream_path); path } @@ -1766,8 +3192,8 @@ mod tests { // println!("alg: \"{}\"", String::from_utf8_lossy(_path)); alg_cnt += 1; }, - |_sub_path, _, _path| { - // println!("jump: over \"{}\" to \"{}\"", String::from_utf8_lossy(_sub_path), String::from_utf8_lossy(_path)); + |_prefix, _, _path| { + // println!("jump: over \"{}\" to \"{}\"", String::from_utf8_lossy(_prefix), String::from_utf8_lossy(_path)); jump_cnt += 1; } )); @@ -1807,8 +3233,8 @@ mod tests { // println!("alg: {_path:?}, mask: {_mask:?}"); alg_cnt += 1; }, - |_sub_path, _, _path| { - // println!("jump: over {_sub_path:?} to {_path:?}"); + |_prefix, _, _path| { + // println!("jump: over {_prefix:?} to {_path:?}"); jump_cnt += 1; } )); @@ -1847,9 +3273,9 @@ mod tests { // println!("alg: {path:?}"); assert_eq!(path, &[]); }, - |sub_path, _, path| { - // println!("jump: over {sub_path:?} to {path:?}"); - assert_eq!(sub_path, &[98]); + |prefix, _, path| { + // println!("jump: over {prefix:?} to {path:?}"); + assert_eq!(prefix, &[98]); assert_eq!(path, &[97]); } )) @@ -1971,7 +3397,7 @@ mod tests { // println!("tree: {:#?}", visit(&mut make_map().read_zipper())); use core::sync::atomic::{AtomicU64, Ordering::*}; let calls_cached = AtomicU64::new(0); - let tree_cached: Rc::> = make_map().into_cata_cached( + let tree_cached: Rc::> = make_map().recursive_cata_cached( |_bm, children, value| { calls_cached.fetch_add(1, Relaxed); Rc::new(Node::new(value, children)) @@ -1989,6 +3415,523 @@ mod tests { eprintln!("calls_cached: {calls_cached}\ncalls_side: {calls_side}"); } + /// Adapted from morphisms::cata_test1 for recursive_cata (jumping). + #[test] + fn recursive_cata_jumping_sum_digits() { + let tests = [ + (vec![], 0), + (vec!["1"], 1), + (vec!["1", "2"], 3), + (vec!["1", "2", "3", "4", "5", "6"], 21), + (vec!["a1", "a2"], 3), + (vec!["a1", "a2", "a3", "a4", "a5", "a6"], 21), + (vec!["12345"], 5), + (vec!["1", "12", "123", "1234", "12345"], 15), + (vec!["123", "123456", "123789"], 18), + (vec!["12", "123", "123456", "123789"], 20), + (vec!["1", "2", "123", "123765", "1234", "12345", "12349"], 29), + ]; + + #[derive(Default)] + struct SumAcc { + idx: usize, + sum: u32, + } + + for (keys, expected_sum) in tests { + let map: PathMap<()> = keys.into_iter().map(|v| (v, ())).collect(); + let sum = map.recursive_factored_cata_jumping::<_, _, Infallible, _, _, _, true>( + |_| Ok(SumAcc::default()), + |mask: &ByteMask, w: (bool, u32), acc: &mut SumAcc| { + if let Some(byte) = mask.indexed_bit::(acc.idx) { + acc.idx += 1; + if w.0 { + acc.sum += (byte as char).to_digit(10).unwrap(); + } + } + acc.sum += w.1; + Ok(()) + }, + |_mask, val, acc, prefix| { + let mut sum = acc.map(|acc| acc.sum).unwrap_or(0); + if val.is_some() { + if let Some(byte) = prefix.last() { + sum += (*byte as char).to_digit(10).unwrap(); + } + } + Ok((val.is_some() && prefix.is_empty(), sum)) + }, + ).unwrap().1; + assert_eq!(sum, expected_sum); + } + } + + /// Adapted from morphisms::cata_test1 for recursive_cata_stepping (non-jumping). + #[test] + fn recursive_cata_stepping_sum_digits() { + let tests = [ + (vec![], 0), + (vec!["1"], 1), + (vec!["1", "2"], 3), + (vec!["1", "2", "3", "4", "5", "6"], 21), + (vec!["a1", "a2"], 3), + (vec!["a1", "a2", "a3", "a4", "a5", "a6"], 21), + (vec!["12345"], 5), + (vec!["1", "12", "123", "1234", "12345"], 15), + (vec!["123", "123456", "123789"], 18), + (vec!["12", "123", "123456", "123789"], 20), + (vec!["1", "2", "123", "123765", "1234", "12345", "12349"], 29), + ]; + + #[derive(Default)] + struct SumAcc { + idx: usize, + sum: u32, + } + + for (keys, expected_sum) in tests { + let map: PathMap<()> = keys.into_iter().map(|v| (v, ())).collect(); + let sum = map.recursive_factored_cata::( + |_| Ok(SumAcc::default()), + |mask: &ByteMask, w: (bool, u32), acc: &mut SumAcc| { + if let Some(byte) = mask.iter().nth(acc.idx) { + acc.idx += 1; + if w.0 { + acc.sum += (byte as char).to_digit(10).unwrap(); + } + } + acc.sum += w.1; + Ok(()) + }, + |_mask, val, acc| { + Ok((val.is_some(), acc.map(|acc| acc.sum).unwrap_or(0))) + }, + ).unwrap().1; + assert_eq!(sum, expected_sum); + } + } + + /// Ports the leaf-count portion of `cata_test2` to the summarization API and + /// compares both summarization traversals with the existing cached catas. + #[test] + fn recursive_cata_leaf_count_matches_cached_catas() { + let mut map = PathMap::new(); + let words = ["arrow", "bow", "cannon", "roman", "romane", "romanus", "romulus", "rubens", "ruber", "rubicon", "rubicundus", "rom'i"]; + words.iter().enumerate().for_each(|(i, word)| { map.set_val_at(word.as_bytes(), i); }); + + let cached_stepping = map.read_zipper().recursive_cata_cached(|_mask, children: &mut [usize], val| { + if children.is_empty() { + assert!(val.is_some()); + 1 + } else { + children.iter().sum() + } + }); + let cached_jumping = map.read_zipper().recursive_cata_jumping_cached(|_mask, children: &mut [usize], val, _prefix| { + if children.is_empty() { + assert!(val.is_some()); + 1 + } else { + children.iter().sum() + } + }); + + let jumping = map.recursive_factored_cata_jumping::( + |_| Ok(0), + |_mask, child, total| { *total += child; Ok(()) }, + |_mask, val, children, _prefix| match children { + Some(total) => Ok(total), + None => { + assert!(val.is_some()); + Ok(1) + }, + }, + ); + let stepping = map.recursive_factored_cata::( + |_| Ok(0), + |_mask, child, total| { *total += child; Ok(()) }, + |_mask, val, children| match children { + Some(total) => Ok(total), + None => { + assert!(val.is_some()); + Ok(1) + }, + }, + ); + + assert_eq!(cached_stepping, 11); + assert_eq!(cached_jumping, cached_stepping); + assert_eq!(jumping.unwrap(), cached_jumping); + assert_eq!(stepping.unwrap(), cached_stepping); + } + + /// Ports the pure longest-path calculation from `cata_test2`. The stepping + /// version deliberately uses the fold ordinal to associate each child with + /// the corresponding bit in the shared mask. + #[test] + fn recursive_cata_longest_path_matches_cached_cata() { + let mut map = PathMap::new(); + let words = ["arrow", "bow", "cannon", "roman", "romane", "romanus", "romulus", "rubens", "ruber", "rubicon", "rubicundus", "rom'i"]; + words.iter().enumerate().for_each(|(i, word)| { map.set_val_at(word.as_bytes(), i); }); + + let cached = map.read_zipper().recursive_cata_jumping_cached(|mask, children: &mut [Vec], _val, prefix| { + let mut longest = mask.iter().zip(children.iter_mut()) + .max_by_key(|(_byte, rest)| rest.len()) + .map_or_else(Vec::new, |(byte, rest)| { + let mut path = std::mem::take(rest); + path.insert(0, byte); + path + }); + let mut path = prefix.to_vec(); + path.append(&mut longest); + path + }); + + // This uses allocation for readability; performance-sensitive code can fold a longest path directly. + let jumping = map.recursive_factored_cata_jumping::>, Vec, Infallible, _, _, _, true>( + |_| Ok(Vec::new()), + |_mask, child, children| { children.push(child); Ok(()) }, + |mask, _val, children, prefix| { + let mut longest = children.map_or_else(Vec::new, |children| { + if mask.is_empty_mask() { + children.into_iter().max_by_key(|rest| rest.len()).unwrap_or_default() + } else { + mask.iter().zip(children) + .max_by_key(|(_byte, rest)| rest.len()) + .map_or_else(Vec::new, |(byte, mut rest)| { + rest.insert(0, byte); + rest + }) + } + }); + let mut path = prefix.to_vec(); + path.append(&mut longest); + Ok(path) + }, + ); + let stepping = map.recursive_factored_cata::<(usize, Vec), Vec, Infallible, _, _, _>( + |_| Ok((0, Vec::new())), + |mask, child, state| { + let mut path = Vec::with_capacity(child.len() + 1); + if let Some(byte) = mask.indexed_bit::(state.0) { + state.0 += 1; + path.push(byte); + } + path.extend(child); + if path.len() > state.1.len() { + state.1 = path; + } + Ok(()) + }, + |_mask, _val, state| Ok(state.map_or_else(Vec::new, |(_, path)| path)), + ); + let adapted = map.recursive_cata_jumping_cached(|mask, children: &mut [Vec], _val, prefix| { + let mut longest = mask.iter().zip(children.iter_mut()) + .max_by_key(|(_byte, rest)| rest.len()) + .map_or_else(Vec::new, |(byte, rest)| { + let mut path = std::mem::take(rest); + path.insert(0, byte); + path + }); + let mut path = prefix.to_vec(); + path.append(&mut longest); + path + }); + + assert_eq!(std::str::from_utf8(&cached).unwrap(), "rubicundus"); + assert_eq!(adapted, cached); + assert_eq!(jumping.unwrap(), cached); + assert_eq!(stepping.unwrap(), cached); + } + + /// Ports the branch-value portion of `cata_test2` to both summarization + /// traversals. Values with no downstream results are deliberately omitted. + #[test] + fn recursive_cata_branch_values_matches_cached_catas() { + let mut map = PathMap::new(); + let words = ["arrow", "bow", "cannon", "roman", "romane", "romanus", "romulus", "rubens", "ruber", "rubicon", "rubicundus", "rom'i"]; + words.iter().enumerate().for_each(|(i, word)| { map.set_val_at(word.as_bytes(), i); }); + + let cached_stepping = map.read_zipper().recursive_cata_cached(|_mask, children: &mut [Vec], val| { + if children.is_empty() { + Vec::new() + } else if let Some(val) = val { + vec![*val] + } else { + let mut values = children.first_mut().map_or_else(Vec::new, std::mem::take); + for child in &mut children[1..] { + values.append(child); + } + values + } + }); + let cached_jumping = map.read_zipper().recursive_cata_jumping_cached(|_mask, children: &mut [Vec], val, _prefix| { + if children.is_empty() { + Vec::new() + } else if let Some(val) = val { + vec![*val] + } else { + let mut values = children.first_mut().map_or_else(Vec::new, std::mem::take); + for child in &mut children[1..] { + values.append(child); + } + values + } + }); + + let jumping = map.recursive_factored_cata_jumping::, Vec, Infallible, _, _, _, false>( + |_| Ok(Vec::new()), + |_mask, child, values| { values.extend(child); Ok(()) }, + |_mask, val, children, _prefix| match children { + None => Ok(Vec::new()), + Some(values) => Ok(val.map_or(values, |val| vec![*val])), + }, + ); + let stepping = map.recursive_factored_cata::, Vec, Infallible, _, _, _>( + |_| Ok(Vec::new()), + |_mask, child, values| { values.extend(child); Ok(()) }, + |_mask, val, children| match children { + None => Ok(Vec::new()), + Some(values) => Ok(val.map_or(values, |val| vec![*val])), + }, + ); + + assert_eq!(cached_stepping, vec![3]); + assert_eq!(cached_jumping, cached_stepping); + assert_eq!(jumping.unwrap(), cached_jumping); + assert_eq!(stepping.unwrap(), cached_stepping); + } + + /// Parallel port of `cata_test_cached`: the input deliberately contains + /// shared subtries, so this exercises `CatamorphismCached`'s factored traversal. + #[test] + fn recursive_cata_cached_dag_matches_cached_cata() { + fn make_map() -> PathMap { + let mut map: PathMap = PathMap::from_iter([([0], 0)]); + for _level in 0..3 { + let previous = map.read_zipper(); + let next = PathMap::new_from_ana(false, |quit, _val, children, _path| { + if quit { return; } + for byte in 0..=2 { + children.graft_at_byte(byte, &previous); + } + }); + drop(previous); + map = next; + } + map + } + + use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; + use std::rc::Rc; + + #[derive(Clone, Debug, PartialEq)] + struct Node { + value: Option, + children: Vec>>, + } + impl Node { + fn new(value: Option<&V>, children: Option>>>) -> Self { + Self { value: value.cloned(), children: children.unwrap_or_default() } + } + } + + let cached_calls = AtomicU64::new(0); + let cached: Rc> = make_map().recursive_cata_cached(|_mask, children, value| { + cached_calls.fetch_add(1, Relaxed); + Rc::new(Node { value: value.cloned(), children: children.to_vec() }) + }); + + let jumping_calls = AtomicU64::new(0); + let jumping: Rc> = make_map().read_zipper().recursive_cata_jumping_cached(|_mask, children, value, _prefix| { + jumping_calls.fetch_add(1, Relaxed); + Rc::new(Node { value: value.cloned(), children: children.to_vec() }) + }); + + let adapted_calls = AtomicU64::new(0); + let adapted: Rc> = make_map().recursive_cata_jumping_cached(|_mask, children, value, _prefix| { + adapted_calls.fetch_add(1, Relaxed); + Rc::new(Node { value: value.cloned(), children: children.to_vec() }) + }); + + assert_eq!(adapted, jumping); + assert_eq!(adapted_calls.load(Relaxed), jumping_calls.load(Relaxed)); + + let summarization_calls = AtomicU64::new(0); + let summarized = make_map().recursive_factored_cata::>>, Rc>, Infallible, _, _, _>( + |_| Ok(Vec::new()), + |_mask, child, children| { children.push(child); Ok(()) }, + |_mask, value, children| { + summarization_calls.fetch_add(1, Relaxed); + Ok(Rc::new(Node::new(value, children))) + }, + ); + + assert_eq!(summarized.unwrap(), cached); + assert_eq!(summarization_calls.load(Relaxed), cached_calls.load(Relaxed)); + + let iterative_calls = AtomicU64::new(0); + let iterative_map = make_map(); + let iterative = iterative_map.read_zipper().recursive_factored_cata::>>, Rc>, Infallible, _, _, _>( + |_| Ok(Vec::new()), + |_mask, child, children| { children.push(child); Ok(()) }, + |_mask, value, children| { + iterative_calls.fetch_add(1, Relaxed); + Ok(Rc::new(Node::new(value, children))) + }, + ); + + assert_eq!(iterative.unwrap(), cached); + assert_eq!(iterative_calls.load(Relaxed), cached_calls.load(Relaxed)); + } + + #[test] + fn recursive_cata_on_zipper_uses_its_focus_as_root() { + let map: PathMap<()> = [ + (b"a".as_slice(), ()), + (b"ab".as_slice(), ()), + (b"ac".as_slice(), ()), + (b"z".as_slice(), ()), + ] + .into_iter() + .collect(); + let mut zipper = map.read_zipper(); + zipper.descend_to(b"a"); + + let count = zipper.recursive_factored_cata_jumping::( + |_| Ok(0), + |_mask, child, total| { *total += child; Ok(()) }, + |_mask, value, children, _prefix| { + Ok(value.is_some() as usize + children.unwrap_or(0)) + }, + ); + + assert_eq!(count.unwrap(), 3); + assert_eq!(zipper.path(), b"a"); + } + + #[test] + fn side_effecting_cata_uses_its_focus_as_root() { + let map: PathMap<()> = [ + (b"a".as_slice(), ()), + (b"ab".as_slice(), ()), + (b"ac".as_slice(), ()), + (b"z".as_slice(), ()), + ] + .into_iter() + .collect(); + + let mut stepping_zipper = map.read_zipper(); + stepping_zipper.descend_to(b"a"); + let mut stepping_paths = Vec::new(); + let stepping = stepping_zipper.into_cata_side_effect(|_mask, children: &mut [usize], value, path| { + stepping_paths.push(path.to_vec()); + value.is_some() as usize + children.iter().sum::() + }); + + assert_eq!(stepping, 3); + assert!(stepping_paths.iter().all(|path| path.starts_with(b"a"))); + assert!(stepping_paths.contains(&b"a".to_vec())); + + let mut jumping_zipper = map.read_zipper(); + jumping_zipper.descend_to(b"a"); + let jumping = jumping_zipper.into_cata_jumping_side_effect(|_mask, children: &mut [usize], _jump, value, _path| { + value.is_some() as usize + children.iter().sum::() + }); + + assert_eq!(jumping, 3); + + // A valueless unary focus may lie in a compressed path, so the jumping traversal must + // not ascend past it while looking for the next logical callback point. + let unary_map: PathMap<()> = [(b"abc".as_slice(), ()), (b"z".as_slice(), ())] + .into_iter() + .collect(); + let mut unary_zipper = unary_map.read_zipper(); + unary_zipper.descend_to(b"a"); + let unary = unary_zipper.into_cata_jumping_side_effect(|_mask, children: &mut [usize], _jump, value, _path| { + value.is_some() as usize + children.iter().sum::() + }); + + assert_eq!(unary, 1); + } + + /// A bounded deep-path smoke test for the recursive cata. + /// + /// The implementation uses the Rust call stack once per physical node. `all_dense_nodes` + /// creates more physical nodes for a path, so it needs a smaller safe bound; intentionally + /// testing beyond that bound can abort the whole test process instead of reporting a normal + /// assertion failure. + #[test] + fn recursive_cata_deep_path_smoke() { + #[cfg(feature = "all_dense_nodes")] + const PATH_LEN: usize = 200; + #[cfg(not(feature = "all_dense_nodes"))] + const PATH_LEN: usize = 8_000; + + let mut map = PathMap::<()>::new(); + let path = vec![b'a'; PATH_LEN]; + map.set_val_at(&path, ()); + + let count = map.recursive_factored_cata_jumping::<_, _, Infallible, _, _, _, false>( + |_| Ok(0usize), + |_mask, w: usize, total| { *total += w; Ok(()) }, + |_mask, v, total, _| Ok((v.is_some() as usize) + total.unwrap_or(0)), + ); + assert_eq!(count.unwrap(), 1); + } + + #[test] + fn recursive_cata_propagates_callback_errors() { + let map: PathMap<()> = [(b"a".as_slice(), ()), (b"b".as_slice(), ())].into_iter().collect(); + + let error = map.recursive_factored_cata_jumping::<(), (), &'static str, _, _, _, false>( + |_| Err("start"), + |_mask, _child, _acc| Ok(()), + |_mask, _value, _acc, _prefix| Ok(()), + ); + assert_eq!(error, Err("start")); + + let error = map.recursive_factored_cata_jumping::<(), (), &'static str, _, _, _, false>( + |_| Ok(()), + |_mask, _child, _acc| Err("fold"), + |_mask, _value, _acc, _prefix| Ok(()), + ); + assert_eq!(error, Err("fold")); + + let error = map.recursive_factored_cata_jumping::<(), (), &'static str, _, _, _, false>( + |_| Ok(()), + |_mask, _child, _acc| Ok(()), + |_mask, _value, _acc, _prefix| Err("summarize"), + ); + assert_eq!(error, Err("summarize")); + } + + #[test] + fn recursive_cata_stops_after_summarize_error() { + use core::sync::atomic::{AtomicUsize, Ordering::Relaxed}; + + let map: PathMap<()> = [ + (b"a".as_slice(), ()), + (b"b".as_slice(), ()), + (b"c".as_slice(), ()), + ] + .into_iter() + .collect(); + let summarize_calls = AtomicUsize::new(0); + + let error = map.recursive_factored_cata_jumping::<(), (), &'static str, _, _, _, false>( + |_| Ok(()), + |_mask, _child, _acc| Ok(()), + |_mask, _value, _acc, _prefix| { + summarize_calls.fetch_add(1, Relaxed); + Err("summarize") + }, + ); + + assert_eq!(error, Err("summarize")); + assert_eq!(summarize_calls.load(Relaxed), 1); + } + /// Generate some basic tries using the [TrieBuilder::push_byte] API #[test] fn ana_test1() { diff --git a/src/overlay_zipper.rs b/src/overlay_zipper.rs index 96b847e..9aa6dec 100644 --- a/src/overlay_zipper.rs +++ b/src/overlay_zipper.rs @@ -160,10 +160,6 @@ impl ZipperMoving self.b.reset(); } - fn val_count(&self) -> usize { - todo!() - } - fn descend_to>(&mut self, path: P) { let path = path.as_ref(); self.a.descend_to(path); diff --git a/src/path_tracker.rs b/src/path_tracker.rs index e86ca66..1a2056b 100644 --- a/src/path_tracker.rs +++ b/src/path_tracker.rs @@ -74,7 +74,6 @@ impl ZipperMoving for PathTracker { None } } - fn val_count(&self) -> usize { self.zipper.val_count() } fn descend_to>(&mut self, path: K) { let path = path.as_ref(); self.path.extend_from_slice(path); @@ -206,6 +205,10 @@ impl<'a, Z: ZipperReadOnlyConditionalValues<'a, V>, V: Clone + Send + Sync> Zipp } impl ZipperPathBuffer for PathTracker { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { + assert!(len <= self.path.capacity() - self.origin_len); + unsafe { core::slice::from_raw_parts(self.path.as_ptr().add(self.origin_len), len) } + } unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { assert!(len <= self.path.capacity()); let ptr = self.path.as_ptr(); diff --git a/src/prefix_zipper.rs b/src/prefix_zipper.rs index e935359..f94623d 100644 --- a/src/prefix_zipper.rs +++ b/src/prefix_zipper.rs @@ -296,6 +296,10 @@ impl<'prefix, 'source, Z, V> ZipperReadOnlyConditionalValues<'source, V> impl<'prefix, Z> ZipperPathBuffer for PrefixZipper<'prefix, Z> where Z: ZipperMoving { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { + assert!(len <= self.path.capacity() - self.origin_depth); + unsafe{ core::slice::from_raw_parts(self.path.as_ptr().add(self.origin_depth), len) } + } unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { assert!(self.path.capacity() >= len); unsafe{ core::slice::from_raw_parts(self.path.as_ptr(), len) } @@ -385,10 +389,6 @@ impl<'prefix, Z> ZipperMoving for PrefixZipper<'prefix, Z> self.set_valid(0); } - fn val_count(&self) -> usize { - self.source.val_count() - } - fn descend_to_existing>(&mut self, patho: K) -> usize { if self.position.is_invalid() { return 0; diff --git a/src/product_zipper.rs b/src/product_zipper.rs index 83bb67f..d93f5fa 100644 --- a/src/product_zipper.rs +++ b/src/product_zipper.rs @@ -174,10 +174,6 @@ impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> Zipper self.factor_paths.clear(); self.z.reset() } - fn val_count(&self) -> usize { - debug_assert!(self.focus_factor() == self.factor_count() - 1); - self.z.val_count() - } fn descend_to_existing>(&mut self, k: K) -> usize { let k = k.as_ref(); let mut descended = 0; @@ -364,6 +360,7 @@ impl ZipperConcrete for ProductZip } impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> ZipperPathBuffer for ProductZipper<'_, 'trie, V, A> { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { unsafe{ self.z.path_assert_len(len) } } unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { unsafe{ self.z.origin_path_assert_len(len) } } fn prepare_buffers(&mut self) { self.z.prepare_buffers() } fn reserve_buffers(&mut self, path_len: usize, stack_depth: usize) { self.z.reserve_buffers(path_len, stack_depth) } @@ -552,6 +549,7 @@ impl<'trie, PrimaryZ, SecondaryZ, V> ZipperPathBuffer PrimaryZ: ZipperMoving + ZipperPath + ZipperPathBuffer, SecondaryZ: ZipperMoving + ZipperPath + ZipperPathBuffer, { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { unsafe{ self.primary.path_assert_len(len) } } unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { unsafe{ self.primary.origin_path_assert_len(len) } } fn prepare_buffers(&mut self) { self.primary.prepare_buffers() } fn reserve_buffers(&mut self, path_len: usize, stack_depth: usize) { self.primary.reserve_buffers(path_len, stack_depth) } @@ -682,10 +680,6 @@ impl<'trie, PrimaryZ, SecondaryZ, V> ZipperMoving for ProductZipperG<'trie, Prim } self.primary.reset(); } - #[inline] - fn val_count(&self) -> usize { - unimplemented!("method will probably get removed") - } fn descend_to_existing>(&mut self, path: K) -> usize { let mut path = path.as_ref(); let mut descended = 0; @@ -925,7 +919,7 @@ mod tests { use crate::utils::ByteMask; use crate::zipper::*; use crate::PathMap; - use crate::morphisms::Catamorphism; + use crate::morphisms::CatamorphismSideEffecting; macro_rules! impl_product_zipper_tests { ($mod:ident, $ProductZipper:ident, $convert:ident) => { diff --git a/src/random.rs b/src/random.rs index b3fbea1..ab1a89a 100644 --- a/src/random.rs +++ b/src/random.rs @@ -8,7 +8,7 @@ use std::marker::PhantomData; use rand::distr::Uniform; use crate::TrieValue; use crate::utils::{BitMask, ByteMask}; -use crate::zipper::{ReadZipperUntracked, Zipper, ZipperPath, ZipperReadOnlyIteration, ZipperMoving, ZipperReadOnlyValues}; +use crate::zipper::{ReadZipperUntracked, Zipper, ZipperPath, ZipperReadOnlyIteration, ZipperMoving, ZipperReadOnlyValues, CatamorphismCached}; // Re-export generic combinators pub use distr_combinators::*; @@ -140,14 +140,15 @@ pub struct FairTriePath { } impl Distribution<(Vec, Option)> for FairTriePath { fn sample(&self, rng: &mut R) -> (Vec, Option) { - use crate::morphisms::Catamorphism; + //TODO: There has to be a more efficient way to implement this than making two passes through the whole trie + use crate::morphisms::{CatamorphismSideEffecting, CatamorphismCached}; // it's much cheaper to draw many samples at once, but the current Distribution API is broken - let size = Catamorphism::into_cata_cached(self.source.clone(), |_: &ByteMask, ws: &mut [usize], _mv: Option<&T>| { + let size = self.source.cata_cached(|_: &ByteMask, ws: &mut [usize], _mv: Option<&T>| { ws.iter().sum::() + 1 }); let target = rng.random_range(0..size); let mut i = 0; - Catamorphism::into_cata_side_effect_fallible(self.source.clone(), |_: &ByteMask, _, mv: Option<&T>, path: &[u8]| { + self.source.clone().into_cata_side_effect_fallible(|_: &ByteMask, _, mv: Option<&T>, path: &[u8]| { if i == target { Err((path.to_vec(), mv.cloned())) } else { i += 1; Ok(()) } }).unwrap_err() } diff --git a/src/tiny_node.rs b/src/tiny_node.rs index 5cdb357..c79b940 100644 --- a/src/tiny_node.rs +++ b/src/tiny_node.rs @@ -9,12 +9,12 @@ use core::mem::MaybeUninit; use core::fmt::{Debug, Formatter}; -use std::collections::HashMap; use fast_slice_utils::{find_prefix_overlap, starts_with}; use crate::utils::ByteMask; use crate::alloc::Allocator; use crate::trie_node::*; +use crate::gxhash::HashMap; use crate::ring::*; /// A borrowed reference to a payload with a key stored elsewhere, contained in 16 Bytes @@ -121,6 +121,16 @@ impl<'a, V: Clone + Send + Sync, A: Allocator> TinyRefNode<'a, V, A> { fn key(&self) -> &[u8] { unsafe{ core::slice::from_raw_parts(self.key_bytes.as_ptr().cast(), self.key_len()) } } + + pub(crate) fn node_recursive_cata(&self, passed_in_val: Option<&V>, start_f: StartF, fold_child_f: FoldChildF, finalize_f: FinalizeF, cache: &mut HashMap) -> Result + where + W: Clone, + StartF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + FinalizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8]) -> Result, + { + self.into_full().unwrap().node_recursive_cata::<_, _, _, _, _, _, COMPUTE_PATH>(passed_in_val, start_f, fold_child_f, finalize_f, cache) + } } impl<'a, V: Clone + Send + Sync, A: Allocator> TrieNode for TinyRefNode<'a, V, A> { @@ -217,12 +227,8 @@ impl<'a, V: Clone + Send + Sync, A: Allocator> TrieNode for TinyRefNode<'a fn new_iter_token(&self) -> IterToken { unreachable!() } fn iter_token_for_path(&self, _key: &[u8]) -> IterToken { unreachable!() } fn next_items(&self, _token: IterToken) -> (IterToken, &'a[u8], Option<&TrieNodeODRc>, Option<&V>) { unreachable!() } - fn node_val_count(&self, cache: &mut HashMap) -> usize { - let temp_node = self.into_full().unwrap(); - temp_node.node_val_count(cache) - } - fn node_goat_val_count(&self) -> usize { - self.into_full().unwrap().node_goat_val_count() + fn node_val_count(&self) -> usize { + self.into_full().unwrap().node_val_count() } fn node_child_iter_start(&self) -> (u64, Option<&TrieNodeODRc>) { if self.is_used_child() { @@ -343,4 +349,4 @@ mod tests { //Confirm TinyRefNode is 16 bytes assert_eq!(std::mem::size_of::>(), 16); } -} \ No newline at end of file +} diff --git a/src/trie_map.rs b/src/trie_map.rs index 3343fa5..4eb7fa3 100644 --- a/src/trie_map.rs +++ b/src/trie_map.rs @@ -1,6 +1,7 @@ use core::cell::UnsafeCell; +use core::convert::Infallible; use crate::alloc::{Allocator, GlobalAlloc, global_alloc}; -use crate::morphisms::{new_map_from_ana_in, TrieBuilder}; +use crate::morphisms::{new_map_from_ana_in, CatamorphismCached, TrieBuilder}; use crate::trie_node::*; use crate::zipper::*; use crate::merkleization::{MerkleizeResult, merkleize_impl}; @@ -498,24 +499,18 @@ impl PathMap { /// /// WARNING: This is not a cheap method. It may have an order-N cost pub fn val_count(&self) -> usize { - let root_val = unsafe{ &*self.root_val.get() }.is_some() as usize; match self.root() { - Some(root) => val_count_below_root(root.as_tagged()) + root_val, - None => root_val - } - } - - /// GOAT, temporary method to do side-by-side comparison between abstracted val_count and bespoke version - pub fn goat_val_count(&self) -> usize { - let root_val = unsafe{ &*self.root_val.get() }.is_some() as usize; - match self.root() { - Some(root) => { - traverse_physical(root, - |node, ctx: usize| { ctx + node.node_goat_val_count() }, - |ctx, child_ctx| { ctx + child_ctx }, - ) + root_val + Some(_root) => { + match self.factored_cata_jumping::<_, _, Infallible, _, _, _, false>( + |_| Ok(0usize), + |_mask, w: usize, total| { *total += w; Ok(()) }, + |_mask, v, total, _| Ok((v.is_some() as usize) + total.unwrap_or(0)), + ) { + Ok(count) => count, + Err(never) => match never {}, + } }, - None => root_val + None => unsafe{ &*self.root_val.get() }.is_some() as usize } } diff --git a/src/trie_node.rs b/src/trie_node.rs index 4cb2076..4598b30 100644 --- a/src/trie_node.rs +++ b/src/trie_node.rs @@ -2,7 +2,6 @@ use core::hint::unreachable_unchecked; use core::mem::ManuallyDrop; use core::ptr::NonNull; -use std::collections::HashMap; use dyn_clone::*; use local_or_heap::LocalOrHeap; use arrayvec::ArrayVec; @@ -14,6 +13,8 @@ use crate::ring::*; use crate::tiny_node::TinyRefNode; use crate::line_list_node::LineListNode; +use crate::gxhash::HashMap; + #[cfg(feature = "bridge_nodes")] use crate::bridge_node::BridgeNode; @@ -214,15 +215,9 @@ pub(crate) trait TrieNode: TrieNodeDowncas /// - `value` that exists at the path, or `None` fn next_items(&self, token: IterToken) -> (IterToken, &[u8], Option<&TrieNodeODRc>, Option<&V>); - /// Returns the total number of leaves contained within the whole subtree defined by the node - /// GOAT, this should be deprecated - fn node_val_count(&self, cache: &mut HashMap) -> usize; - /// Returns the number of values contained within the node itself, irrespective of the positions within /// the node; does not include onward links - /// - /// GOAT, this should replace node_val_count - fn node_goat_val_count(&self) -> usize; + fn node_val_count(&self) -> usize; /// Returns the first downstream child of a node, and a token that can be used to access subsequent children /// @@ -1242,23 +1237,12 @@ mod tagged_node_ref { } #[inline] - pub fn node_val_count(&self, cache: &mut HashMap) -> usize { + pub fn node_val_count(&self) -> usize { match self { - Self::DenseByteNode(node) => node.node_val_count(cache), - Self::LineListNode(node) => node.node_val_count(cache), - Self::CellByteNode(node) => node.node_val_count(cache), - Self::TinyRefNode(node) => node.node_val_count(cache), - Self::EmptyNode => 0, - } - } - - #[inline] - pub fn node_goat_val_count(&self) -> usize { - match self { - Self::DenseByteNode(node) => node.node_goat_val_count(), - Self::LineListNode(node) => node.node_goat_val_count(), - Self::CellByteNode(node) => node.node_goat_val_count(), - Self::TinyRefNode(node) => node.node_goat_val_count(), + Self::DenseByteNode(node) => node.node_val_count(), + Self::LineListNode(node) => node.node_val_count(), + Self::CellByteNode(node) => node.node_val_count(), + Self::TinyRefNode(node) => node.node_val_count(), Self::EmptyNode => 0, } } @@ -2385,89 +2369,66 @@ mod tagged_node_ref { } } -/// Returns the count of values in the subtrie descending from the node, caching shared subtries -pub(crate) fn val_count_below_root(node: TaggedNodeRef) -> usize { - let mut cache = std::collections::HashMap::new(); - node.node_val_count(&mut cache) -} - -pub(crate) fn val_count_below_node(node: &TrieNodeODRc, cache: &mut HashMap) -> usize { - if node.is_empty() { - return 0 - } - if node.refcount() > 1 { - let hash = node.shared_node_id(); - match cache.get(&hash) { - Some(cached) => *cached, - None => { - let val = node.as_tagged().node_val_count(cache); - cache.insert(hash, val); - val - }, - } - } else { - node.as_tagged().node_val_count(cache) - } -} - -/// Recursively traverses a trie descending from `node`, visiting every physical non-empty node once -pub(crate) fn traverse_physical(node: &TrieNodeODRc, node_f: NodeF, fold_f: FoldF) -> Ctx - where +/// Internal implementation of `CatamorphismCached::factored_cata_jumping` +pub(crate) fn recursive_cata_cached( + node: &TrieNodeODRc, + passed_in_val: Option<&V>, + start_f: StartF, + fold_child_f: FoldChildF, + finalize_f: FinalizeF, + cache: &mut HashMap, +) -> Result +where V: Clone + Send + Sync, A: Allocator, - Ctx: Clone + Default, - NodeF: Fn(TaggedNodeRef, Ctx) -> Ctx + Copy, - FoldF: Fn(Ctx, Ctx) -> Ctx + Copy + W: Clone, + StartF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + FinalizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8]) -> Result, { - let mut cache = std::collections::HashMap::new(); - traverse_physical_internal(node, node_f, fold_f, &mut cache) -} - -fn traverse_physical_internal(node: &TrieNodeODRc, node_f: NodeF, fold_f: FoldF, cache: &mut HashMap) -> Ctx - where - V: Clone + Send + Sync, - A: Allocator, - Ctx: Clone + Default, - NodeF: Fn(TaggedNodeRef, Ctx) -> Ctx + Copy, - FoldF: Fn(Ctx, Ctx) -> Ctx + Copy -{ - if node.is_empty() { - return Ctx::default() - } - - if node.refcount() > 1 { + // NOTE: A caller-supplied value can make this trie-node boundary fall within a factored_cata callback, + // therefore the W is not cacheable based on node ID alone. + // FUTURE: When the node contract associates values at the root of nodes with the node and not with the parent, + // then the `passed_in_val.is_none()` check can be removed + if passed_in_val.is_none() && !node.is_empty() && node.refcount() > 1 { let hash = node.shared_node_id(); match cache.get(&hash) { - Some(cached) => cached.clone(), + Some(cached) => Ok(cached.clone()), None => { - let ctx = traverse_physical_children_internal(node.as_tagged(), node_f, fold_f, cache); - cache.insert(hash, ctx.clone()); - ctx + let w = recursive_cata_dispatch::<_, _, _, _, _, _, _, _, COMPUTE_PATH>(node, passed_in_val, start_f, fold_child_f, finalize_f, cache)?; + cache.insert(hash, w.clone()); + Ok(w) }, } } else { - traverse_physical_children_internal(node.as_tagged(), node_f, fold_f, cache) + recursive_cata_dispatch::<_, _, _, _, _, _, _, _, COMPUTE_PATH>(node, passed_in_val, start_f, fold_child_f, finalize_f, cache) } } -fn traverse_physical_children_internal(node: TaggedNodeRef, node_f: NodeF, fold_f: FoldF, cache: &mut HashMap) -> Ctx - where +#[inline(always)] +fn recursive_cata_dispatch( + node: &TrieNodeODRc, + passed_in_val: Option<&V>, + start_f: StartF, + fold_child_f: FoldChildF, + finalize_f: FinalizeF, + cache: &mut HashMap, +) -> Result +where V: Clone + Send + Sync, A: Allocator, - Ctx: Clone + Default, - NodeF: Fn(TaggedNodeRef, Ctx) -> Ctx + Copy, - FoldF: Fn(Ctx, Ctx) -> Ctx + Copy + W: Clone, + StartF: Copy + Fn(&ByteMask) -> Result, + FoldChildF: Copy + Fn(&ByteMask, W, &mut Acc) -> Result<(), Err>, + FinalizeF: Copy + Fn(&ByteMask, Option<&V>, Option, &[u8]) -> Result, { - let mut ctx = Ctx::default(); - - let (mut tok, mut child) = node.node_child_iter_start(); - while let Some(child_node) = child { - let child_ctx = traverse_physical_internal(child_node, node_f, fold_f, cache); - ctx = fold_f(ctx, child_ctx); - (tok, child) = node.node_child_iter_next(tok); + match node.as_tagged() { + TaggedNodeRef::DenseByteNode(node) => { node.node_recursive_cata::<_, _, _, _, _, _, COMPUTE_PATH>(passed_in_val, start_f, fold_child_f, finalize_f, cache) } + TaggedNodeRef::LineListNode(node) => { node.node_recursive_cata::<_, _, _, _, _, _, COMPUTE_PATH>(passed_in_val, start_f, fold_child_f, finalize_f, cache) } + TaggedNodeRef::CellByteNode(node) => { node.node_recursive_cata::<_, _, _, _, _, _, COMPUTE_PATH>(passed_in_val, start_f, fold_child_f, finalize_f, cache) } + TaggedNodeRef::TinyRefNode(node) => { node.node_recursive_cata::<_, _, _, _, _, _, COMPUTE_PATH>(passed_in_val, start_f, fold_child_f, finalize_f, cache) } + TaggedNodeRef::EmptyNode => { finalize_f(&ByteMask::EMPTY, passed_in_val, None, &[]) } } - - node_f(node, ctx) } /// Internal function to walk a mut TrieNodeODRc ref along a path @@ -2521,6 +2482,7 @@ pub(crate) fn make_cell_node(node: &mut Tr // module come from the visibility of the trait it is derived on. In this case, `TrieNode` //Credit to QuineDot for his ideas on this pattern here: https://users.rust-lang.org/t/inferred-lifetime-for-dyn-trait/112116/7 pub(crate) use opaque_dyn_rc_trie_node::TrieNodeODRc; + #[cfg(not(feature = "slim_ptrs"))] mod opaque_dyn_rc_trie_node { use std::sync::Arc; @@ -3011,6 +2973,7 @@ mod opaque_dyn_rc_trie_node { pub(crate) fn new_empty() -> Self { Self { ptr: SlimNodePtr::new_empty(), alloc: MaybeUninit::uninit() } } + #[inline(always)] pub(crate) fn is_empty(&self) -> bool { self.tag() == EMPTY_NODE_TAG } @@ -3285,5 +3248,4 @@ mod tests { node_ref.make_unique(); drop(cloned); } - } diff --git a/src/utils/debug/diff_zipper.rs b/src/utils/debug/diff_zipper.rs index 9808df1..127fb8b 100644 --- a/src/utils/debug/diff_zipper.rs +++ b/src/utils/debug/diff_zipper.rs @@ -71,12 +71,6 @@ impl ZipperMoving for DiffZi println!("DiffZipper: reset") } } - fn val_count(&self) -> usize { - let a = self.a.val_count(); - let b = self.b.val_count(); - assert_eq!(a, b); - a - } fn descend_to>(&mut self, path: P) { let path = path.as_ref(); self.a.descend_to(path); @@ -230,6 +224,12 @@ impl ZipperAbsol impl ZipperPathBuffer for DiffZipper { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { + let a = unsafe{ self.a.path_assert_len(len) }; + let b = unsafe{ self.b.path_assert_len(len) }; + assert_eq!(a, b); + a + } unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { let a = unsafe{ self.a.origin_path_assert_len(len) }; let b = unsafe{ self.b.origin_path_assert_len(len) }; diff --git a/src/utils/debug/morphism_debug.rs b/src/utils/debug/morphism_debug.rs index 550f2ac..388c2e4 100644 --- a/src/utils/debug/morphism_debug.rs +++ b/src/utils/debug/morphism_debug.rs @@ -4,44 +4,135 @@ use crate::utils::ByteMask; use crate::alloc::Allocator; use crate::PathMap; use crate::zipper::*; -use crate::morphisms::{into_cata_cached_body, DoCache}; +use crate::morphisms::cata_jumping_cached_debug_body; /// Debug extension trait for catamorphisms /// /// This trait provides debug-only catamorphism methods that may expose additional /// information useful for debugging and development. pub trait CatamorphismDebug { - /// A version of [`into_cata_jumping_cached`](crate::morphisms::Catamorphism::into_cata_jumping_cached) where - /// the full path is available to the closure; **For debugging purposes only** + /// A debug-only version of [`cata_jumping_cached`](crate::morphisms::CatamorphismCached::cata_jumping_cached) + /// where the full absolute path is available to the closure. /// /// Using data from the full path for your algorithm **will** lead to incorrect behavior. - /// You must either adapt your algorithm not to require full path data or use the one of - /// the `_side_effect` methods. - fn into_cata_jumping_cached_fallible_debug(self, alg_f: AlgF) -> Result + /// You must either adapt your algorithm not to require full path data or use one of the + /// methods in [`crate::morphisms::CatamorphismSideEffecting`]. + fn cata_jumping_cached_debug(&self, alg_f: AlgF) -> W + where + W: Clone, + AlgF: Fn(&ByteMask, &mut [W], Option<&V>, &[u8], &[u8]) -> W, + Self: Sized, + { + self.cata_jumping_cached_fallible_debug(|mask, children, value, prefix, path| { + Ok::<_, core::convert::Infallible>(alg_f(mask, children, value, prefix, path)) + }) + .unwrap() + } + + /// Fallible form of [`Self::cata_jumping_cached_debug`]. + fn cata_jumping_cached_fallible_debug(&self, alg_f: AlgF) -> Result where W: Clone, AlgF: Fn(&ByteMask, &mut [W], Option<&V>, &[u8], &[u8]) -> Result; } -impl<'a, Z, V: 'a> CatamorphismDebug for Z where Z: Zipper + ZipperReadOnlyConditionalValues<'a, V> + ZipperConcrete + ZipperAbsolutePath + ZipperPathBuffer { - fn into_cata_jumping_cached_fallible_debug(self, alg_f: AlgF) -> Result +impl<'a, Z, V: 'a> CatamorphismDebug for Z where Z: Clone + Zipper + ZipperReadOnlyConditionalValues<'a, V> + ZipperConcrete + ZipperAbsolutePath + ZipperPathBuffer { + fn cata_jumping_cached_fallible_debug(&self, alg_f: AlgF) -> Result where W: Clone, AlgF: Fn(&ByteMask, &mut [W], Option<&V>, &[u8], &[u8]) -> Result { - into_cata_cached_body::(self, |mask, children, val, sub_path, debug_path, _z| { - alg_f(mask, children, val, sub_path, debug_path) - }) + cata_jumping_cached_debug_body(self.clone(), alg_f) } } impl CatamorphismDebug for PathMap { - fn into_cata_jumping_cached_fallible_debug(self, alg_f: AlgF) -> Result + fn cata_jumping_cached_fallible_debug(&self, alg_f: AlgF) -> Result where W: Clone, AlgF: Fn(&ByteMask, &mut [W], Option<&V>, &[u8], &[u8]) -> Result { - let rz = self.into_read_zipper(&[]); - rz.into_cata_jumping_cached_fallible_debug(alg_f) + self.read_zipper().cata_jumping_cached_fallible_debug(alg_f) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::morphisms::CatamorphismCached; + + #[test] + fn debug_cached_cata_uses_the_zipper_focus_and_absolute_paths() { + let map: PathMap<()> = [ + (b"a".as_slice(), ()), + (b"ab".as_slice(), ()), + (b"ac".as_slice(), ()), + (b"z".as_slice(), ()), + ] + .into_iter() + .collect(); + let mut zipper = map.read_zipper(); + zipper.descend_to(b"a"); + let paths = std::cell::RefCell::new(Vec::new()); + + let count = zipper.cata_jumping_cached_debug(|_mask, children: &mut [usize], value, _prefix, path| { + paths.borrow_mut().push(path.to_vec()); + value.is_some() as usize + children.iter().sum::() + }); + + assert_eq!(count, 3); + assert!(paths.borrow().iter().all(|path| path.starts_with(b"a"))); + assert!(paths.borrow().contains(&b"a".to_vec())); + assert_eq!(zipper.path(), b"a"); + } + + #[test] + fn debug_cached_cata_does_not_ascend_past_a_unary_focus() { + let map: PathMap<()> = [(b"abc".as_slice(), ()), (b"z".as_slice(), ())] + .into_iter() + .collect(); + let mut zipper = map.read_zipper(); + zipper.descend_to(b"a"); + let paths = std::cell::RefCell::new(Vec::new()); + + let count = zipper.cata_jumping_cached_debug(|_mask, children: &mut [usize], value, _prefix, path| { + paths.borrow_mut().push(path.to_vec()); + value.is_some() as usize + children.iter().sum::() + }); + + assert_eq!(count, 1); + assert!(paths.borrow().iter().all(|path| path.starts_with(b"a"))); + assert_eq!(zipper.path(), b"a"); + } + + #[test] + fn debug_cached_cata_short_circuits_shared_subtries() { + let child: PathMap<()> = [(b"c".as_slice(), ()), (b"d".as_slice(), ())] + .into_iter() + .collect(); + let mut map = PathMap::new(); + let mut writer = map.write_zipper(); + for path in [b"a".as_slice(), b"b".as_slice()] { + writer.reset(); + writer.descend_to(path); + writer.graft_map(child.clone()); + } + drop(writer); + + let cached_calls = std::sync::atomic::AtomicUsize::new(0); + let cached = map.cata_jumping_cached(|_mask, children: &mut [usize], value, _prefix| { + cached_calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + value.is_some() as usize + children.iter().sum::() + }); + + let debug_calls = std::sync::atomic::AtomicUsize::new(0); + let debug = map.cata_jumping_cached_debug(|_mask, children: &mut [usize], value, _prefix, _path| { + debug_calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + value.is_some() as usize + children.iter().sum::() + }); + + assert_eq!(cached, 4); + assert_eq!(debug, cached); + assert_eq!(debug_calls.load(std::sync::atomic::Ordering::Relaxed), cached_calls.load(std::sync::atomic::Ordering::Relaxed)); } } diff --git a/src/utils/mod.rs b/src/utils/mod.rs index c100e1f..cb558ea 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -388,6 +388,26 @@ impl From for ByteMask { } } +impl From<(u8, u8)> for ByteMask { + #[inline] + fn from(byte_pair: (u8, u8)) -> Self { + let mut new_mask = Self::new(); + new_mask.set_bit(byte_pair.0); + new_mask.set_bit(byte_pair.1); + new_mask + } +} + +impl From<[u8; 2]> for ByteMask { + #[inline] + fn from(byte_pair: [u8; 2]) -> Self { + let mut new_mask = Self::new(); + new_mask.set_bit(byte_pair[0]); + new_mask.set_bit(byte_pair[1]); + new_mask + } +} + impl From> for ByteMask { #[inline] fn from(range: Range) -> Self { diff --git a/src/write_zipper.rs b/src/write_zipper.rs index 1ca87c8..93bce9d 100644 --- a/src/write_zipper.rs +++ b/src/write_zipper.rs @@ -420,12 +420,18 @@ impl<'a, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> ZipperInfallibleSubt fn try_borrow_focus(&self) -> Option> { self.z.try_borrow_focus() } } +impl ZipperConcrete for WriteZipperTracked<'_, '_, V, A> { + #[inline] + fn shared_node_id(&self) -> Option { None } + #[inline] + fn is_shared(&self) -> bool { false } +} + impl<'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> ZipperMoving for WriteZipperTracked<'a, 'path, V, A> { #[inline] fn depth(&self) -> usize { self.z.depth() } fn at_root(&self) -> bool { self.z.at_root() } #[inline] fn focus_byte(&self) -> Option { self.z.focus_byte() } fn reset(&mut self) { self.z.reset() } - fn val_count(&self) -> usize { self.z.val_count() } fn descend_to>(&mut self, k: K) { self.z.descend_to(k) } fn descend_to_byte(&mut self, k: u8) { self.z.descend_to_byte(k) } fn descend_indexed_byte(&mut self, child_idx: usize) -> Option { self.z.descend_indexed_byte(child_idx) } @@ -444,6 +450,7 @@ impl<'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> ZipperPath fo } impl<'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> ZipperPathBuffer for WriteZipperTracked<'a, 'path, V, A> { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { unsafe{ self.z.path_assert_len(len) } } unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { unsafe{ self.z.origin_path_assert_len(len) } } fn prepare_buffers(&mut self) { self.z.prepare_buffers() } fn reserve_buffers(&mut self, path_len: usize, stack_depth: usize) { self.z.reserve_buffers(path_len, stack_depth) } @@ -585,12 +592,18 @@ impl<'a, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> ZipperInfallibleSubt fn try_borrow_focus(&self) -> Option> { self.z.try_borrow_focus() } } +impl ZipperConcrete for WriteZipperUntracked<'_, '_, V, A> { + #[inline] + fn shared_node_id(&self) -> Option { None } + #[inline] + fn is_shared(&self) -> bool { false } +} + impl<'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> ZipperMoving for WriteZipperUntracked<'a, 'path, V, A> { #[inline] fn depth(&self) -> usize { self.z.depth() } fn at_root(&self) -> bool { self.z.at_root() } #[inline] fn focus_byte(&self) -> Option { self.z.focus_byte() } fn reset(&mut self) { self.z.reset() } - fn val_count(&self) -> usize { self.z.val_count() } fn descend_to>(&mut self, k: K) { self.z.descend_to(k) } fn descend_to_byte(&mut self, k: u8) { self.z.descend_to_byte(k) } fn descend_indexed_byte(&mut self, child_idx: usize) -> Option { self.z.descend_indexed_byte(child_idx) } @@ -609,6 +622,7 @@ impl<'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> ZipperPath fo } impl<'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> ZipperPathBuffer for WriteZipperUntracked<'a, 'path, V, A> { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { unsafe{ self.z.path_assert_len(len) } } unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { unsafe{ self.z.origin_path_assert_len(len) } } fn prepare_buffers(&mut self) { self.z.prepare_buffers() } fn reserve_buffers(&mut self, path_len: usize, stack_depth: usize) { self.z.reserve_buffers(path_len, stack_depth) } @@ -732,6 +746,7 @@ impl Clone for WriteZipp impl Zipper for WriteZipperOwned { zipper_impl_lens!(Zipper self => self.z); } impl ZipperValues for WriteZipperOwned { zipper_impl_lens!(ZipperValues self => self.z); } impl ZipperInfallibleSubtries for WriteZipperOwned { zipper_impl_lens!(ZipperInfallibleSubtries self => self.z); } +impl ZipperConcrete for WriteZipperOwned { zipper_impl_lens!(ZipperConcrete self => self.z); } impl ZipperMoving for WriteZipperOwned { zipper_impl_lens!(ZipperMoving self => self.z); } impl ZipperPath for WriteZipperOwned { zipper_impl_lens!(ZipperPath self => self.z); } impl ZipperPathBuffer for WriteZipperOwned { zipper_impl_lens!(ZipperPathBuffer self => self.z); } @@ -981,6 +996,14 @@ impl<'trie, V: Clone + Send + Sync + Unpin, A: Allocator + 'trie> Zipper for Wri } } +impl ZipperConcrete for WriteZipperCore<'_, '_, V, A> { + #[inline] + fn shared_node_id(&self) -> Option { None } + + #[inline] + fn is_shared(&self) -> bool { false } +} + impl<'trie, V: Clone + Send + Sync + Unpin, A: Allocator + 'trie> ZipperForking for WriteZipperCore<'trie, '_, V, A> { type ReadZipperT<'a> = crate::zipper::read_zipper_core::ReadZipperCore<'a, 'a, V, A> where Self: 'a; fn fork_read_zipper<'a>(&'a self) -> Self::ReadZipperT<'a> { @@ -1034,15 +1057,6 @@ impl<'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> ZipperMoving self.key.prefix_idx.clear(); } - fn val_count(&self) -> usize { - let root_val = self.is_val() as usize; - let focus = self.get_focus(); - if focus.is_none() { - root_val - } else { - val_count_below_root(focus.as_tagged()) + root_val - } - } fn descend_to>(&mut self, k: K) { let key = k.as_ref(); self.key.prepare_buffers(); @@ -1132,6 +1146,16 @@ impl<'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> ZipperAbsolut } impl<'a, 'path, V: Clone + Send + Sync + Unpin, A: Allocator + 'a> ZipperPathBuffer for WriteZipperCore<'a, 'path, V, A> { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { + let start = self.key.origin_path.len(); + if self.key.prefix_buf.capacity() > 0 { + assert!(len <= self.key.prefix_buf.capacity() - start); + unsafe{ core::slice::from_raw_parts(self.key.prefix_buf.as_ptr().add(start), len) } + } else { + assert_eq!(len, 0); + &[] + } + } unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { if self.key.prefix_buf.capacity() > 0 { assert!(len <= self.key.prefix_buf.capacity()); diff --git a/src/zipper.rs b/src/zipper.rs index 99d2ac1..90ef138 100644 --- a/src/zipper.rs +++ b/src/zipper.rs @@ -28,6 +28,7 @@ pub use crate::poly_zipper::{PolyZipper, PolyZipperExplicit}; pub use crate::dependent_zipper::DependentProductZipperG; use crate::zipper_tracking::*; +pub use crate::morphisms::CatamorphismCached; //re-exported so `use pathmap::zipper::*` gives the caller access to `val_count`, etc /// The most fundamantal interface for a zipper, compatible with all zipper types pub trait Zipper { @@ -167,12 +168,6 @@ pub trait ZipperMoving: Zipper { } } - /// Returns the total number of values contained at and below the zipper's focus, including the focus itself - /// - /// WARNING: This is not a cheap method. It may have an order-N cost - //GOAT! This doesn't belong here. Should be a function that uses a non-side-effect catamorphism - fn val_count(&self) -> usize; - /// Moves the zipper deeper into the trie, to the `key` specified relative to the current zipper focus fn descend_to>(&mut self, k: K); @@ -1147,6 +1142,12 @@ pub trait ZipperConcrete { /// Provides more direct control over a [ZipperMoving] zipper's path buffer pub trait ZipperPathBuffer: ZipperMoving { + /// Internal method to get the relative path buffer, beyond its current logical length. + /// + /// Panics if `len` exceeds that buffer's capacity. The returned bytes beyond [`ZipperPath::path`] + /// are only valid when the caller knows they were initialized by earlier zipper movement. + unsafe fn path_assert_len(&self, len: usize) -> &[u8]; + /// Internal method to get the path, beyond its length. Panics if `len` > the path's capacity, or /// if the zipper is relative and doesn't have an `origin_path` /// @@ -1229,7 +1230,6 @@ macro_rules! zipper_impl_lens { fn at_root(&$s) -> bool { $e.at_root() } #[inline] fn focus_byte(&$s) -> Option { $e.focus_byte() } fn reset(&mut $s) { $e.reset() } - fn val_count(&$s) -> usize { $e.val_count() } fn descend_to>(&mut $s, k: K) { $e.descend_to(k) } fn descend_to_check>(&mut $s, k: K) -> bool { $e.descend_to_check(k) } fn descend_to_existing>(&mut $s, k: K) -> usize { $e.descend_to_existing(k) } @@ -1278,6 +1278,7 @@ macro_rules! zipper_impl_lens { #[inline] fn is_shared(&$s) -> bool { $e.is_shared() } }; (ZipperPathBuffer $s: ident => $e:expr) => { + unsafe fn path_assert_len(&$s, len: usize) -> &[u8] { unsafe{ $e.path_assert_len(len) } } unsafe fn origin_path_assert_len(&$s, len: usize) -> &[u8] { unsafe{ $e.origin_path_assert_len(len) } } fn prepare_buffers(&mut $s) { $e.prepare_buffers() } fn reserve_buffers(&mut $s, path_len: usize, stack_depth: usize) { $e.reserve_buffers(path_len, stack_depth) } @@ -1979,20 +1980,6 @@ pub(crate) mod read_zipper_core { self.prefix_buf.truncate(self.origin_path.len()); } - fn val_count(&self) -> usize { - timed_span!(ValueCount, COUNTERS); - let root_val = self.is_val() as usize; - if self.node_key().len() == 0 { - val_count_below_root(*self.focus_node) + root_val - } else { - let focus = self.get_focus(); - if focus.0.is_none() { - root_val - } else { - val_count_below_root(focus.0.as_tagged()) + root_val - } - } - } fn descend_to>(&mut self, k: K) { timed_span!(DescendTo, COUNTERS); let k = k.as_ref(); @@ -2415,6 +2402,16 @@ pub(crate) mod read_zipper_core { } impl<'trie, V: Clone + Send + Sync + Unpin + 'trie, A: Allocator + 'trie> ZipperPathBuffer for ReadZipperCore<'trie, '_, V, A> { + unsafe fn path_assert_len(&self, len: usize) -> &[u8] { + let start = self.origin_path.len(); + if self.prefix_buf.capacity() > 0 { + assert!(len <= self.prefix_buf.capacity() - start); + unsafe{ core::slice::from_raw_parts(self.prefix_buf.as_ptr().add(start), len) } + } else { + assert_eq!(len, 0); + &[] + } + } unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] { if self.prefix_buf.capacity() > 0 { assert!(len <= self.prefix_buf.capacity()); @@ -4999,6 +4996,20 @@ mod tests { use crate::{alloc::global_alloc, PathMap}; use super::*; + crate::morphisms::cached_catamorphism_tests::cached_catamorphism_tests!( + read_zipper, + |keys: &[&[u8]]| keys.iter().enumerate().map(|(idx, path)| (*path, idx as u64)).collect::>(), + |map: &mut PathMap| map.read_zipper(), + CatamorphismCached + ); + + crate::morphisms::cached_catamorphism_tests::cached_catamorphism_tests!( + read_zipper, + |keys: &[&[u8]]| keys.iter().enumerate().map(|(idx, path)| (*path, idx as u64)).collect::>(), + |map: &mut PathMap| map.read_zipper(), + CatamorphismCachedIterative + ); + /// Drives a [TruncatingObserver] with the supplied movements, returning what reached the /// downstream observer along with the observer's final overshoot fn run_truncating(limit: usize, movements: &[&[u8]]) -> (Vec, usize) { @@ -5133,7 +5144,7 @@ mod tests { assert_eq!(moved, false); assert_eq!(pz.path(), b""); assert_eq!(observed, b""); - } +} super::zipper_moving_tests::zipper_moving_tests!(read_zipper, |keys: &[&[u8]]| {