Add node-based catamorphism POC

Cargo.toml

@@ -117,9 +117,14 @@ harness = false
 name = "product_zipper"
 harness = false
 
+[[bench]]
+name = "catamorphism"
+harness = false
+
 [[bench]]
 name = "sla"
 harness = false
+required-features = ["viz"]
 
 [[bench]]
 name = "multiplicities"

benches/act_paths.rs

@@ -26,7 +26,7 @@ use divan::{Bencher, Divan, counter::ItemsCount};
 use pathmap::PathMap;
 use pathmap::arena_compact::{ACTOutputStream, ArenaCompactTree};
 use pathmap::paths_serialization::{for_each_deserialized_path, serialize_paths};
-use pathmap::zipper::ZipperMoving;
+use pathmap::morphisms::CatamorphismCachedIterative;
 use rand::{Rng, SeedableRng, rngs::StdRng};
 use std::path::Path;
 

benches/binary_keys.rs

@@ -115,7 +115,7 @@ fn binary_descend_until_max_bytes(bencher: Bencher, n: u64) {
     });
 }
 
-#[divan::bench(args = [125, 250, 500, 1000, 2000, 4000])]
+#[divan::bench(args = [125, 250, 500, 1000, 2000, 4000, 100000])]
 fn binary_val_count_bench(bencher: Bencher, n: u64) {
 
     let keys = make_keys(n as usize, 1);

benches/catamorphism.rs

@@ -0,0 +1,140 @@
+use divan::{Divan, Bencher, black_box};
+use core::convert::Infallible;
+use pathmap::alloc::GlobalAlloc;
+use pathmap::morphisms::CatamorphismCached;
+use pathmap::utils::ByteMask;
+use pathmap::utils::ints::gen_int_range;
+use pathmap::PathMap;
+
+fn main() {
+    // Run registered benchmarks.
+    let divan = Divan::from_args()
+        .sample_count(4000);
+
+    divan.main();
+}
+
+fn build_map(count: u64) -> PathMap<()> {
+    // Dense range of u64 keys encoded as paths; sized to keep benches fast and stable.
+    gen_int_range::<(), 8, u64>(0, count, 1, ())
+}
+
+const MAP_COUNT: u64 = 20_000_000;
+
+// A complete binary trie keeps every internal node as a two-entry LineListNode.
+const BINARY_TREE_DEPTH: usize = 18;
+const BINARY_TREE_LEAF_COUNT: usize = 1 << BINARY_TREE_DEPTH;
+
+fn build_binary_tree_map() -> PathMap<()> {
+    let mut map = PathMap::new();
+    for leaf in 0..BINARY_TREE_LEAF_COUNT {
+        let mut path = [0u8; BINARY_TREE_DEPTH];
+        for (level, byte) in path.iter_mut().enumerate() {
+            *byte = ((leaf >> (BINARY_TREE_DEPTH - level - 1)) & 1) as u8;
+        }
+        map.insert(path, ());
+    }
+    map
+}
+
+#[divan::bench()]
+fn factored_cata_jumping_val_count(bencher: Bencher) {
+    let map = build_map(MAP_COUNT);
+    let mut sink = 0usize;
+    bencher.bench_local(|| {
+        let rz = map.read_zipper();
+        *black_box(&mut sink) = CatamorphismCached::<(), GlobalAlloc>::factored_cata_jumping::<_, _, Infallible, _, _, _, false>(&rz,
+            |_| Ok(0usize),
+            |_mask, w: usize, total| { *total += w; Ok(()) },
+            |_mask, v, total, _| Ok((v.is_some() as usize) + total.unwrap_or(0)),
+        ).unwrap();
+    });
+    assert_eq!(sink, MAP_COUNT as usize);
+}
+
+#[divan::bench()]
+fn factored_cata_binary_tree_leaf_count(bencher: Bencher) {
+    let map = build_binary_tree_map();
+    let mut sink = 0usize;
+    bencher.bench_local(|| {
+        let rz = map.read_zipper();
+        *black_box(&mut sink) = CatamorphismCached::<(), GlobalAlloc>
+            ::factored_cata_jumping::<_, _, Infallible, _, _, _, false>(&rz,
+                |_| Ok(0usize),
+                |_mask, child_count: usize, total| {
+                    *total += child_count;
+                    Ok(())
+                },
+                |_mask, value, total, _| Ok((value.is_some() as usize) + total.unwrap_or(0)),
+            )
+            .unwrap();
+    });
+    assert_eq!(sink, BINARY_TREE_LEAF_COUNT);
+}
+
+#[divan::bench()]
+fn cached_jumping_cata_val_count(bencher: Bencher) {
+    let map = build_map(MAP_COUNT);
+    let mut sink = 0usize;
+    bencher.bench_local(|| {
+        let rz = map.read_zipper();
+        *black_box(&mut sink) = CatamorphismCached::<(), GlobalAlloc>::cata_jumping_cached(&rz, |_mask: &ByteMask, children: &mut [usize], val, _sub_path| {
+            let mut sum: usize = children.iter().sum();
+            if val.is_some() {
+                sum += 1;
+            }
+            sum
+        });
+    });
+    assert_eq!(sink, MAP_COUNT as usize);
+}
+
+#[divan::bench()]
+fn factored_cata_jumping_total_len(bencher: Bencher) {
+    let map = build_map(MAP_COUNT);
+    let mut sink = (0usize, 0usize);
+    bencher.bench_local(|| {
+        let rz = map.read_zipper();
+        *black_box(&mut sink) = CatamorphismCached::<(), GlobalAlloc>::factored_cata_jumping::<_, _, Infallible, _, _, _, true>(&rz,
+            |_| Ok((0usize, 0usize)),
+            |_mask: &ByteMask, w: (usize, usize), acc: &mut (usize, usize)| {
+                acc.0 += w.0;
+                // Every folded child hangs below exactly one branch byte. `prefix` accounts
+                // for compressed runs separately in `summarize_f` below.
+                acc.1 += w.1 + w.0;
+                Ok(())
+            },
+            |_mask: &ByteMask, val, acc, prefix| {
+                let (count, total_len) = acc.unwrap_or((0, 0));
+                let count = count + val.is_some() as usize;
+                Ok((count, total_len + count * prefix.len()))
+            },
+        ).unwrap();
+    });
+    assert_eq!(sink, (MAP_COUNT as usize, MAP_COUNT as usize * 8));
+}
+
+#[divan::bench()]
+fn cached_jumping_cata_total_len(bencher: Bencher) {
+    let map = build_map(MAP_COUNT);
+    let mut sink = (0usize, 0usize);
+    bencher.bench_local(|| {
+        let rz = map.read_zipper();
+        *black_box(&mut sink) = CatamorphismCached::<(), GlobalAlloc>::cata_jumping_cached(&rz, |mask: &ByteMask, children: &mut [(usize, usize)], val, sub_path| {
+            let mut count = 0usize;
+            let mut total_len = 0usize;
+            let prefix_len = sub_path.len();
+            if val.is_some() {
+                count += 1;
+                total_len += prefix_len;
+            }
+            for (_byte, child) in mask.iter().zip(children.iter_mut()) {
+                count += child.0;
+                // The child is below one mask byte as well as this callback's prefix.
+                total_len += child.1 + child.0 * (prefix_len + 1);
+            }
+            (count, total_len)
+        });
+    });
+    assert_eq!(sink, (MAP_COUNT as usize, MAP_COUNT as usize * 8));
+}

benches/cities.rs

@@ -171,10 +171,7 @@ fn cities_val_count(bencher: Bencher) {
 #[cfg(feature="arena_compact")]
 #[divan::bench()]
 fn cities_val_count_act(bencher: Bencher) {
-    use pathmap::{
-        arena_compact::ArenaCompactTree,
-        zipper::ZipperMoving,
-    };
+    use pathmap::{morphisms::CatamorphismCachedIterative, arena_compact::ArenaCompactTree};
     let pairs = read_data();
     let mut map = PathMap::new();
     let mut unique_count = 0;

benches/multiplicities.rs

@@ -1,5 +1,5 @@
 use pathmap::*;
-use pathmap::zipper::{ZipperMoving, ZipperWriting};
+use pathmap::zipper::{CatamorphismCached, ZipperWriting};
 
 fn main() {
     const SILLY_LARGE_COUNTS: bool = false;

benches/oeis.rs

@@ -1,7 +1,6 @@
 use std::io::Read;
-use std::usize;
 use pathmap::PathMap;
-use pathmap::zipper::{Zipper, ZipperValues, ZipperMoving, ZipperPath, ZipperWriting, ZipperCreation};
+use pathmap::zipper::{Zipper, ZipperValues, ZipperMoving, ZipperPath, ZipperWriting, ZipperCreation, CatamorphismCached};
 use num::BigInt;
 use divan::{Divan, Bencher, black_box};
 

benches/product_zipper.rs

@@ -89,7 +89,7 @@ fn val_count_cata<V>(_bm: &ByteMask, vals: &mut[usize], _val: Option<&V>, _path:
 fn introspecting_pathmap_pathmap(bencher: Bencher) {
     use pathmap::{
         PathMap,
-        morphisms::Catamorphism,
+        morphisms::CatamorphismSideEffecting,
         zipper::{ProductZipper},
     };
     let mut sink = 0;
@@ -106,7 +106,7 @@ fn introspecting_pathmap_pathmap(bencher: Bencher) {
 fn generic_pathmap_pathmap(bencher: Bencher) {
     use pathmap::{
         PathMap,
-        morphisms::Catamorphism,
+        morphisms::CatamorphismSideEffecting,
         zipper::{ProductZipperG},
     };
     let mut sink = 0;
@@ -125,7 +125,7 @@ fn generic_act_act(bencher: Bencher) {
     use pathmap::{
         PathMap,
         arena_compact::{ArenaCompactTree},
-        morphisms::Catamorphism,
+        morphisms::CatamorphismSideEffecting,
         zipper::{ProductZipperG},
     };
     let mut sink = 0;
@@ -146,7 +146,7 @@ fn generic_pathmap_act(bencher: Bencher) {
     use pathmap::{
         PathMap,
         arena_compact::{ArenaCompactTree},
-        morphisms::Catamorphism,
+        morphisms::CatamorphismSideEffecting,
         zipper::{ProductZipperG},
     };
     let mut sink = 0;

benches/shakespeare.rs

@@ -171,10 +171,7 @@ fn shakespeare_sentences_val_count(bencher: Bencher) {
 #[cfg(feature="arena_compact")]
 #[divan::bench()]
 fn shakespeare_sentences_val_count_act(bencher: Bencher) {
-    use pathmap::{
-        arena_compact::ArenaCompactTree,
-        zipper::ZipperMoving,
-    };
+    use pathmap::{morphisms::CatamorphismCachedIterative, arena_compact::ArenaCompactTree};
     let strings = read_data(false);
     let mut map = PathMap::new();
     let mut unique_count = 0;

benches/sla.rs

@@ -1,4 +1,3 @@
-#![allow(unused)]
 use std::hash::{Hasher, Hash};
 use std::time::Instant;
 use num_traits::Zero;
@@ -7,7 +6,7 @@ use rand::prelude::StdRng;
 use rand::{Rng, SeedableRng};
 use rand_distr::Distribution;
 use pathmap::*;
-use pathmap::morphisms::Catamorphism;
+use pathmap::morphisms::CatamorphismCached;
 use pathmap::ring::{AlgebraicResult, Lattice};
 use pathmap::utils::{BitMask, ByteMask, ints::{indices_to_weave, indices_to_bob}};
 use pathmap::zipper::{ReadZipperUntracked, WriteZipperUntracked, Zipper, ZipperMoving, ZipperValues, ZipperWriting};
@@ -643,4 +642,4 @@ fn main() {
 
     tipover_attention_bob();
     tipover_attention_weave(0.02);
-}
+}

benches/sparse_keys.rs

@@ -134,7 +134,7 @@ fn sparse_descend_until_max_bytes(bencher: Bencher, n: u64) {
     });
 }
 
-#[divan::bench(args = [125, 250, 500, 1000, 2000, 4000])]
+#[divan::bench(args = [125, 250, 500, 1000, 2000, 4000, 20_000, 100_000])]
 fn sparse_val_count_bench(bencher: Bencher, n: u64) {
 
     let mut r = StdRng::seed_from_u64(1);

benches/superdense_keys.rs

@@ -311,7 +311,7 @@ fn from_prefix_key(k: Vec<u8>) -> u64 {
     u64::from_le_bytes(buf) & (!0u64 >> shift)
 }
 
-#[divan::bench(sample_size = 1, args = [100, 200, 400, 800, 1600, 3200, 20_000])]
+#[divan::bench(sample_size = 1, args = [100, 200, 400, 800, 1600, 3200, 20_000, 100_000])]
 fn superdense_val_count_bench(bencher: Bencher, n: u64) {
 
     let mut map: PathMap<u64> = PathMap::new();
@@ -326,12 +326,9 @@ fn superdense_val_count_bench(bencher: Bencher, n: u64) {
 }
 
 #[cfg(feature="arena_compact")]
-#[divan::bench(sample_size = 1, args = [100, 200, 400, 800, 1600, 3200, 20_000])]
+#[divan::bench(sample_size = 1, args = [100, 200, 400, 800, 1600, 3200, 20_000, 100_000])]
 fn superdense_val_count_bench_act(bencher: Bencher, n: u64) {
-    use pathmap::{
-        arena_compact::ArenaCompactTree,
-        zipper::ZipperMoving,
-    };
+    use pathmap::{morphisms::CatamorphismCachedIterative, arena_compact::ArenaCompactTree};
     let mut map: PathMap<u64> = PathMap::new();
     for i in 0..n { map.set_val_at(prefix_key(&i), i); }
     let act = ArenaCompactTree::from_zipper(map.read_zipper(), |&v| v);

pathmap-derive/src/lib.rs

@@ -490,12 +490,6 @@ fn derive_poly_zipper_with_traits(
                     }
                 }
 
-                fn val_count(&self) -> usize {
-                    match self {
-                        #(#variant_arms => inner.val_count(),)*
-                    }
-                }
-
                 fn descend_to<K: AsRef<[u8]>>(&mut self, k: K) {
                     match self {
                         #(#variant_arms => inner.descend_to(k),)*
@@ -653,6 +647,12 @@ fn derive_poly_zipper_with_traits(
             impl #impl_generics pathmap::zipper::ZipperPathBuffer for #enum_name #ty_generics
             #zipper_path_buffer_where
             {
+                unsafe fn path_assert_len(&self, len: usize) -> &[u8] {
+                    match self {
+                        #(#variant_arms => unsafe { inner.path_assert_len(len) },)*
+                    }
+                }
+
                 unsafe fn origin_path_assert_len(&self, len: usize) -> &[u8] {
                     match self {
                         #(#variant_arms => unsafe { inner.origin_path_assert_len(len) },)*