From 85506469407671983ad6384fd63ebcaa782f5b29 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 07:12:59 +0000 Subject: [PATCH 01/19] dt64: MapTable::locate and LocalMap::extends for multi-tile cubes locate finds a global id's (tile ordinal, local) across the listed tiles and refuses an id held by two tiles or none. extends says whether a map only appended to an older one: prepare keeps the supplied order, so an appended id never renumbers an existing position. Tests live in tests/dr.rs, since dr.rs must not name its parent module. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/lance-graph-dt64/src/dr.rs | 53 +++++++++++++++++ crates/lance-graph-dt64/tests/dr.rs | 91 ++++++++++++++++++++++++++++- 2 files changed, 143 insertions(+), 1 deletion(-) diff --git a/crates/lance-graph-dt64/src/dr.rs b/crates/lance-graph-dt64/src/dr.rs index 0c46caa69..2ba34b2bb 100644 --- a/crates/lance-graph-dt64/src/dr.rs +++ b/crates/lance-graph-dt64/src/dr.rs @@ -599,6 +599,59 @@ impl MapTable { self.frames.check(frame).map_err(MapRead::Stale)?; self.tiles.get(&tile).ok_or(MapRead::NoSuchTile(tile)) } + + /// Find which of the listed `tiles` holds `global` under `frame`. + /// Returns `(ordinal, local)`: the index of the holding tile in `tiles` + /// and the id's local position in that tile's map. Every listed tile is + /// read, so an id held by two of them is refused rather than resolved to + /// the first. + pub fn locate( + &self, + frame: &Frame, + tiles: &[TileId], + global: u64, + ) -> Result<(u32, u8), LocateRefusal> { + let mut found: Option<(u32, u8)> = None; + for (ordinal, &tile) in tiles.iter().enumerate() { + let map = self.map(frame, tile).map_err(LocateRefusal::Read)?; + // A lookup that finds nothing is not an error here; only the + // overall absence is. + if let Ok(local) = map.local(global) { + if found.is_some() { + return Err(LocateRefusal::Ambiguous); + } + let ordinal = u32::try_from(ordinal).map_err(|_| LocateRefusal::Ambiguous)?; + found = Some((ordinal, local)); + } + } + found.ok_or(LocateRefusal::Unmapped) + } +} + +/// Why [`MapTable::locate`] refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LocateRefusal { + /// A listed tile could not be read under the frame (stale frame, or no + /// map for that tile). + Read(MapRead), + /// The id is held by more than one listed tile. + Ambiguous, + /// No listed tile holds the id. + Unmapped, +} + +impl LocalMap { + /// True iff `older` is a pure prefix of `self`: same width, and every + /// position of `older` keeps its global id in `self`, with `self` only + /// appending. [`LocalMap::prepare`] does not sort the local order (only + /// its lookup index is sorted), so an appended id never renumbers an + /// existing position, whatever its value. A reorder, a removal or a + /// width change makes this false. + pub fn extends(&self, older: &LocalMap) -> bool { + self.width == older.width + && self.globals.len() >= older.globals.len() + && self.globals[..older.globals.len()] == older.globals[..] + } } /// The HHTL16 Cartesian address of cell `(x, y)` of one tile of `width`: the diff --git a/crates/lance-graph-dt64/tests/dr.rs b/crates/lance-graph-dt64/tests/dr.rs index 0e1da49fb..3168817ee 100644 --- a/crates/lance-graph-dt64/tests/dr.rs +++ b/crates/lance-graph-dt64/tests/dr.rs @@ -220,7 +220,11 @@ fn the_coordinates_revision_is_a_function_of_the_held_maps() { assert_ne!(base, at(vec![(1, &[10, 20])]), "tile removed"); // Same maps in the same order under another tile name: only the tile id // in the digest tells these apart. - assert_ne!(base, at(vec![(1, &[10, 20]), (3, &[30, 40])]), "tile renamed"); + assert_ne!( + base, + at(vec![(1, &[10, 20]), (3, &[30, 40])]), + "tile renamed" + ); } #[test] @@ -240,3 +244,88 @@ fn a_removed_tile_is_gone_and_another_tables_frame_is_foreign() { MapRead::Stale(FrameRefusal::ForeignFrame) ); } + +// ------------------------------------------- locate / extends (D-CODEGRAPH-0 wide closure) + +mod locate { + use lance_graph_dt64::dr::{LocalMap, LocateRefusal, MapRead, MapTable, TileId, Width}; + + fn table(tiles: Vec<(TileId, Vec)>) -> MapTable { + let mut t = MapTable::new(1, 1); + let changes = tiles + .into_iter() + .map(|(id, g)| (id, LocalMap::prepare(Width::W64, &g).ok())) + .collect(); + t.advance(1, 1, changes); + t + } + + /// Fails if the ordinal is the tile id instead of the index in the list, + /// or the local is not the position within the holding tile. + #[test] + fn locate_finds_ordinal_and_local_across_two_tiles() { + let t = table(vec![(7, vec![10, 20, 30]), (3, vec![40, 50])]); + let f = t.seal(); + assert_eq!(t.locate(&f, &[7, 3], 20), Ok((0, 1))); + assert_eq!(t.locate(&f, &[7, 3], 50), Ok((1, 1))); + // List order defines the ordinal. + assert_eq!(t.locate(&f, &[3, 7], 50), Ok((0, 1))); + } + + /// Fails if the first hit wins instead of the overlap being refused. + #[test] + fn locate_refuses_an_id_in_two_tiles() { + let t = table(vec![(1, vec![5, 6]), (2, vec![6, 7])]); + let f = t.seal(); + assert_eq!(t.locate(&f, &[1, 2], 6), Err(LocateRefusal::Ambiguous)); + // Non-vacuous: a listing that excludes the second tile resolves. + assert_eq!(t.locate(&f, &[1], 6), Ok((0, 1))); + } + + /// Fails if an absent id resolves to anything, or a stale frame is read. + #[test] + fn locate_refuses_an_absent_id() { + let t = table(vec![(1, vec![5, 6]), (2, vec![8])]); + let f = t.seal(); + assert_eq!(t.locate(&f, &[1, 2], 99), Err(LocateRefusal::Unmapped)); + assert_eq!( + t.locate(&f, &[1, 9], 5), + Err(LocateRefusal::Read(MapRead::NoSuchTile(9))) + ); + } + + fn map(width: Width, g: &[u64]) -> LocalMap { + LocalMap::prepare(width, g).expect("fixture") + } + + /// Fails if appending (even an id smaller than the existing ones) is not + /// accepted, or if equality is not accepted as a trivial extension. + #[test] + fn extends_true_for_pure_prefix_extension() { + let old = map(Width::W64, &[30, 20]); + assert!(map(Width::W64, &[30, 20, 10]).extends(&old)); + assert!(old.extends(&old)); + } + + /// Fails if a reorder of the same ids counts as an extension. + #[test] + fn extends_false_for_reorder() { + let old = map(Width::W64, &[1, 2]); + assert!(!map(Width::W64, &[2, 1, 3]).extends(&old)); + } + + /// Fails if dropping an id counts as an extension. + #[test] + fn extends_false_for_removal() { + let old = map(Width::W64, &[1, 2, 3]); + assert!(!map(Width::W64, &[1, 2]).extends(&old)); + assert!(!map(Width::W64, &[1, 3]).extends(&old)); + } + + /// Fails if the width is ignored. + #[test] + fn extends_false_for_width_change() { + let old = map(Width::W64, &[1, 2]); + assert!(!map(Width::W256, &[1, 2, 3]).extends(&old)); + } +} From 08fc06214092aaf6891c16eb69753f5bb2546931 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 07:15:41 +0000 Subject: [PATCH 02/19] xlog: wide closure kernel with block presence, delta log, explicit refold, refresh closure_wide runs the same semi-naive fixpoint over any number of positions, one row at a time; rows keep only nonzero 256-position blocks, with a multi-word presence mask per row (no tile cap). DeltaLog appends ADD / RETIRE edges; refold absorbs them only when called; refresh recomputes only the rows that reached a retired edge's source and seeds additions incrementally. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/lance-graph-xlog/src/lib.rs | 1 + crates/lance-graph-xlog/src/wide.rs | 745 ++++++++++++++++++++++++++++ 2 files changed, 746 insertions(+) create mode 100644 crates/lance-graph-xlog/src/wide.rs diff --git a/crates/lance-graph-xlog/src/lib.rs b/crates/lance-graph-xlog/src/lib.rs index 240756dc6..7d8a28d79 100644 --- a/crates/lance-graph-xlog/src/lib.rs +++ b/crates/lance-graph-xlog/src/lib.rs @@ -50,6 +50,7 @@ use ndarray::simd::{checked_binary_i64, mask_and, I64ArithError, I64Op}; pub mod cube; pub mod lens; +pub mod wide; /// The bounds of the admitted chain: `X == left`, `Z == right` when set. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] diff --git a/crates/lance-graph-xlog/src/wide.rs b/crates/lance-graph-xlog/src/wide.rs new file mode 100644 index 000000000..c03c43f8f --- /dev/null +++ b/crates/lance-graph-xlog/src/wide.rs @@ -0,0 +1,745 @@ +//! Wide transitive closure: more than 64 positions, block presence, and a +//! sparse delta log that is merged on read. +//! +//! A *position* is a wide coordinate `tile_ord * 256 + local`. One closure row +//! is split into 256-position *blocks* (one dt64 tile, four `u64` words). A +//! block is allocated only when some bit in it is set, so a row that reaches +//! two tiles of a hundred stores two blocks, and the presence words say which. +//! +//! Everything here is a free function over parallel `src` / `dst` lanes so the +//! kernel can be tested without a `Cube`. +//! +//! Invariants worth knowing before editing: +//! +//! * Positions never renumber. A delta (and the old closure it is applied to) +//! names positions by value, so renumbering one position would make every +//! older row and every logged edge point at the wrong symbol. New symbols +//! are appended at the end (`n_new >= old.n()`); a vanished symbol keeps its +//! position and simply loses its edges. +//! * The closure is a derived read. It is never persisted as a delta; it is +//! re-derived for the rows a change can reach. +//! * Deletions cannot be merged into a closure row: a removed edge may or may +//! not have been the only path to a target. So every row that could reach a +//! retired source is recomputed, and every other row is kept bit for bit. +//! Insertions are monotone, so they are merged exactly by seeding. + +use std::collections::BTreeSet; + +use ndarray::simd::{mask_andnot, mask_any, mask_or_assign}; + +/// Positions in one block: one dt64 tile. +pub const BLOCK_POSITIONS: usize = 256; + +/// `u64` words in one block. +pub const BLOCK_WORDS: usize = 4; + +/// Largest supported position count. Row offsets are `u32` block slots, so +/// `n * ceil(n / 256)` must fit in 32 bits; this bound guarantees it. +pub const MAX_POSITIONS: u32 = (1 << 20) - 256; + +/// Closure (or edge) rows with block presence: a block is allocated only if +/// some word of it is nonzero. An absent block reads as zero. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WideRows { + n: u32, + blocks: u32, + /// `n * pw` words, `pw = ceil(blocks / 64)`; bit `b` of a row's presence + /// words says block `b` is allocated for that row. + presence: Vec, + /// Per row: the slot (in blocks) where its packed blocks start in `words`. + offset: Vec, + /// The present blocks only, four words each, row after row, block order. + words: Vec, +} + +/// Presence words per row for `blocks` blocks. +fn pw_of(blocks: u32) -> usize { + (blocks as usize).div_ceil(64) +} + +/// Calls `f` with the index of every set bit, lowest first. +fn for_each_bit(words: &[u64], mut f: impl FnMut(usize)) { + for (wi, &w) in words.iter().enumerate() { + let mut bits = w; + while bits != 0 { + f(wi * 64 + bits.trailing_zeros() as usize); + bits &= bits - 1; + } + } +} + +impl WideRows { + /// An empty builder for `n` rows of `blocks` blocks. + fn builder(n: u32, blocks: u32) -> Self { + Self { + n, + blocks, + presence: Vec::with_capacity(n as usize * pw_of(blocks)), + offset: Vec::with_capacity(n as usize), + words: Vec::new(), + } + } + + /// Appends the next row from a dense scratch of `blocks * 4` words, + /// allocating exactly the blocks that have a set bit. + fn push_dense_row(&mut self, dense: &[u64]) { + let pw = pw_of(self.blocks); + let start = self.presence.len(); + self.presence.resize(start + pw, 0); + // Capacity is checked by `check_dims`, so the slot count fits `u32`. + self.offset + .push(u32::try_from(self.words.len() / BLOCK_WORDS).unwrap_or(u32::MAX)); + for b in 0..self.blocks as usize { + let Some(chunk) = dense.get(b * BLOCK_WORDS..(b + 1) * BLOCK_WORDS) else { + break; + }; + if chunk.iter().any(|&w| w != 0) { + if let Some(p) = self.presence.get_mut(start + b / 64) { + *p |= 1 << (b % 64); + } + self.words.extend_from_slice(chunk); + } + } + } + + /// Positions (rows and columns). + pub fn n(&self) -> u32 { + self.n + } + + /// Blocks per row: `ceil(n / 256)`. + pub fn blocks(&self) -> u32 { + self.blocks + } + + /// Blocks actually stored, over all rows. + pub fn blocks_allocated(&self) -> u64 { + (self.words.len() / BLOCK_WORDS) as u64 + } + + /// Whether position `y` is in row `x`. Tests the presence bit, then ranks + /// it with a popcount to find the stored block; there is no search. + pub fn contains(&self, x: u32, y: u32) -> bool { + if x >= self.n || y >= self.n { + return false; + } + let pw = pw_of(self.blocks); + let b = y as usize / BLOCK_POSITIONS; + let (j, bit) = (b / 64, b % 64); + let Some(row) = self.presence.get(x as usize * pw..(x as usize + 1) * pw) else { + return false; + }; + let Some(&word) = row.get(j) else { + return false; + }; + if word >> bit & 1 == 0 { + return false; + } + let below: u32 = row.iter().take(j).map(|w| w.count_ones()).sum::() + + (word & ((1u64 << bit) - 1)).count_ones(); + let Some(&start) = self.offset.get(x as usize) else { + return false; + }; + let slot = start as usize + below as usize; + let within = (y as usize % BLOCK_POSITIONS) / 64; + self.words + .get(slot * BLOCK_WORDS + within) + .is_some_and(|w| w >> (y % 64) & 1 == 1) + } + + /// Writes row `x` as dense words into `out` (`ceil(n / 64)` words are + /// meaningful; a longer `out` is zero-extended, a shorter one truncated). + /// Absent blocks and rows `x >= n` read as zero. + pub fn row_into(&self, x: u32, out: &mut [u64]) { + out.fill(0); + if x >= self.n { + return; + } + let pw = pw_of(self.blocks); + let Some(row) = self.presence.get(x as usize * pw..(x as usize + 1) * pw) else { + return; + }; + let Some(&start) = self.offset.get(x as usize) else { + return; + }; + let mut slot = start as usize; + for_each_bit(row, |b| { + if let Some(src) = self.words.get(slot * BLOCK_WORDS..(slot + 1) * BLOCK_WORDS) { + for (k, &w) in src.iter().enumerate() { + if let Some(d) = out.get_mut(b * BLOCK_WORDS + k) { + *d = w; + } + } + } + slot += 1; + }); + } + + /// ORs row `y` into `dst` (dense, at least `blocks * 4` words), touching + /// only the blocks row `y` has; absent blocks cost nothing. + fn or_row_into(&self, y: u32, dst: &mut [u64]) { + if y >= self.n { + return; + } + let pw = pw_of(self.blocks); + let Some(row) = self.presence.get(y as usize * pw..(y as usize + 1) * pw) else { + return; + }; + let Some(&start) = self.offset.get(y as usize) else { + return; + }; + let mut slot = start as usize; + for_each_bit(row, |b| { + if let (Some(s), Some(d)) = ( + self.words.get(slot * BLOCK_WORDS..(slot + 1) * BLOCK_WORDS), + dst.get_mut(b * BLOCK_WORDS..(b + 1) * BLOCK_WORDS), + ) { + mask_or_assign(d, s); + } + slot += 1; + }); + } +} + +/// What one wide closure or refresh did. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct WideStats { + /// Most composition rounds any one row needed, including the last, which + /// found nothing new. + pub rounds: u32, + /// Rows of `edge` (or of a reused closure) or'ed into a frontier: one per + /// pair that entered a delta. For a full build this equals `|edge⁺|`. + pub rows_composed: u64, + /// Blocks stored in the result. + pub blocks_allocated: u64, + /// Blocks a dense layout would store: `n * blocks`. + pub blocks_reserved: u64, + /// Rows recomputed because a retired edge could have changed them. Zero + /// for a refresh that only added edges. + pub rows_recomputed: u64, +} + +/// Why a wide closure or refresh refused. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WideRefusal { + /// A row's delta did not empty within one round per position, which a + /// correct semi-naive closure cannot do. + FixpointBound, + /// A position at or beyond the allowed range (`n`), or `n` itself beyond + /// [`MAX_POSITIONS`] (then `n` carries that bound). + PositionOutOfRange { + /// The offending position. + pos: u32, + /// The exclusive bound it violated. + n: u32, + }, + /// The `src` and `dst` lanes differ in length. + LengthMismatch, +} + +/// Checks `n` against [`MAX_POSITIONS`]; returns the blocks per row. +fn check_dims(n: u32) -> Result { + if n > MAX_POSITIONS { + return Err(WideRefusal::PositionOutOfRange { + pos: n, + n: MAX_POSITIONS, + }); + } + Ok(n.div_ceil(BLOCK_POSITIONS as u32)) +} + +/// Checks the lanes have equal length and every position is below `n`. +fn validate(n: u32, src: &[u32], dst: &[u32]) -> Result<(), WideRefusal> { + if src.len() != dst.len() { + return Err(WideRefusal::LengthMismatch); + } + for &p in src.iter().chain(dst) { + if p >= n { + return Err(WideRefusal::PositionOutOfRange { pos: p, n }); + } + } + Ok(()) +} + +/// Per-row working words, each `blocks * 4` long and reused across rows. +struct Scratch { + full: Vec, + delta: Vec, + next: Vec, +} + +impl Scratch { + fn new(blocks: u32) -> Self { + let w = blocks as usize * BLOCK_WORDS; + Self { + full: vec![0; w], + delta: vec![0; w], + next: vec![0; w], + } + } +} + +/// Rows of an older closure that stay valid: position `y` is reusable when it +/// existed then and no retired edge can reach it, so its old row is final. +struct Reuse<'a> { + old: &'a WideRows, + affected: &'a [bool], +} + +impl Reuse<'_> { + fn closed(&self, y: u32) -> bool { + y < self.old.n() && !self.affected.get(y as usize).copied().unwrap_or(true) + } +} + +/// Semi-naive fixpoint on one row, from `s.full` / `s.delta` as seeded: +/// `delta' = (delta ∘ edge) andnot full; full |= delta'`. +/// +/// With `reuse`, a delta bit `y` whose old row is still valid is not +/// composed through `edge`: its whole old row is or'ed into `full` instead. +/// That is exact because everything an unaffected `y` reaches is itself +/// unaffected (an affected member would make `y` affected), so none of those +/// positions needs further expansion. +fn run_fixpoint( + edge: &WideRows, + n: u32, + s: &mut Scratch, + reuse: Option<&Reuse<'_>>, + stats: &mut WideStats, +) -> Result<(), WideRefusal> { + let Scratch { full, delta, next } = s; + let mut rounds = 0u32; + while mask_any(delta) { + // A path visits each position once, so a delta is empty by round n+1. + if rounds > n { + return Err(WideRefusal::FixpointBound); + } + rounds += 1; + next.fill(0); + let mut composed = 0u64; + for_each_bit(delta, |y| { + composed += 1; + let y = u32::try_from(y).unwrap_or(u32::MAX); + match reuse { + Some(r) if r.closed(y) => r.old.or_row_into(y, full), + _ => edge.or_row_into(y, next), + } + }); + stats.rows_composed += composed; + mask_andnot(next, full, delta); + mask_or_assign(full, delta); + } + stats.rounds = stats.rounds.max(rounds); + Ok(()) +} + +/// Packs `(src, dst)` lanes into edge rows. Duplicate edges set the same bit, +/// so they cannot change the result. +fn build_edges(n: u32, blocks: u32, src: &[u32], dst: &[u32]) -> Result { + validate(n, src, dst)?; + let nu = n as usize; + // Counting sort of destinations by source. + let mut start = vec![0usize; nu + 1]; + for &s in src { + if let Some(c) = start.get_mut(s as usize + 1) { + *c += 1; + } + } + for i in 0..nu { + let prev = start.get(i).copied().unwrap_or(0); + if let Some(c) = start.get_mut(i + 1) { + *c += prev; + } + } + let mut cursor = start.clone(); + let mut sorted = vec![0u32; src.len()]; + for (&s, &d) in src.iter().zip(dst) { + if let Some(c) = cursor.get_mut(s as usize) { + if let Some(slot) = sorted.get_mut(*c) { + *slot = d; + } + *c += 1; + } + } + let mut out = WideRows::builder(n, blocks); + let mut dense = vec![0u64; blocks as usize * BLOCK_WORDS]; + for x in 0..nu { + let (lo, hi) = ( + start.get(x).copied().unwrap_or(0), + start.get(x + 1).copied().unwrap_or(0), + ); + dense.fill(0); + for &d in sorted.get(lo..hi).unwrap_or(&[]) { + if let Some(w) = dense.get_mut(d as usize / 64) { + *w |= 1 << (d % 64); + } + } + out.push_dense_row(&dense); + } + Ok(out) +} + +/// Fills the stats fields that describe the result. +fn finish(mut stats: WideStats, rows: &WideRows) -> WideStats { + stats.blocks_allocated = rows.blocks_allocated(); + stats.blocks_reserved = u64::from(rows.n()) * u64::from(rows.blocks()); + stats +} + +/// `edge⁺` over `n` positions from parallel `src` / `dst` lanes (wide +/// coordinates), semi-naively, one row at a time. +/// +/// Rows are independent, so only one row of working words is live at once; +/// a block no row reaches is never allocated. Duplicate edges do not change +/// the result. `rows_composed` equals `|edge⁺|`. +pub fn closure_wide( + n: u32, + src: &[u32], + dst: &[u32], +) -> Result<(WideRows, WideStats), WideRefusal> { + let blocks = check_dims(n)?; + let edge = build_edges(n, blocks, src, dst)?; + let mut stats = WideStats::default(); + let mut s = Scratch::new(blocks); + let mut out = WideRows::builder(n, blocks); + for x in 0..n { + edge.row_into(x, &mut s.full); + s.delta.copy_from_slice(&s.full); + run_fixpoint(&edge, n, &mut s, None, &mut stats)?; + out.push_dense_row(&s.full); + } + let stats = finish(stats, &out); + Ok((out, stats)) +} + +/// Append-only delta lanes. Op `1` is ADD, `0` is RETIRE; each append gets a +/// strictly larger version than the one before. Nothing is updated in place. +#[derive(Clone, Debug, Default)] +pub struct DeltaLog { + src: Vec, + dst: Vec, + version: Vec, + op: Vec, +} + +impl DeltaLog { + /// An empty log. + pub fn new() -> Self { + Self::default() + } + + fn append(&mut self, src: u32, dst: u32, op: u8) { + let v = self.version.last().map_or(1, |&v| v + 1); + self.src.push(src); + self.dst.push(dst); + self.version.push(v); + self.op.push(op); + } + + /// Appends an ADD of edge `src -> dst`. + pub fn add(&mut self, src: u32, dst: u32) { + self.append(src, dst, 1); + } + + /// Appends a RETIRE of edge `src -> dst`. + pub fn retire(&mut self, src: u32, dst: u32) { + self.append(src, dst, 0); + } + + /// Number of logged operations. + pub fn len(&self) -> usize { + self.src.len() + } + + /// Version of the newest operation, or `0` for an empty log. + pub fn last_version(&self) -> u64 { + self.version.last().copied().unwrap_or(0) + } + + /// Whether nothing has been logged. + pub fn is_empty(&self) -> bool { + self.src.is_empty() + } +} + +fn key(s: u32, d: u32) -> u64 { + u64::from(s) << 32 | u64::from(d) +} + +fn lanes(keys: impl Iterator) -> (Vec, Vec) { + keys.map(|k| ((k >> 32) as u32, (k & 0xffff_ffff) as u32)) + .unzip() +} + +/// The edge set after applying the log in append (= version) order; the +/// newest op per `(src, dst)` wins. +fn fold_keys(base: &BTreeSet, log: &DeltaLog) -> BTreeSet { + let mut eff = base.clone(); + for ((&s, &d), &op) in log.src.iter().zip(&log.dst).zip(&log.op) { + if op == 1 { + eff.insert(key(s, d)); + } else { + eff.remove(&key(s, d)); + } + } + eff +} + +/// Folds `base` and the log into the effective edge set, as sorted `src` / +/// `dst` lanes. This is the explicit re-fold: the caller decides when the +/// resident base is replaced by its result and the log emptied; nothing +/// triggers it automatically. Lanes of unequal length are zipped to the +/// shorter. +pub fn refold(base_src: &[u32], base_dst: &[u32], log: &DeltaLog) -> (Vec, Vec) { + let base: BTreeSet = base_src + .iter() + .zip(base_dst) + .map(|(&s, &d)| key(s, d)) + .collect(); + lanes(fold_keys(&base, log).into_iter()) +} + +/// Copies `rows` into a builder of `n` rows (`n >= rows.n()`), padding with +/// empty rows; blocks per row are recomputed for `n`. +fn widen(rows: &WideRows, n: u32, blocks: u32) -> WideRows { + let mut out = WideRows::builder(n, blocks); + let mut dense = vec![0u64; blocks as usize * BLOCK_WORDS]; + for x in 0..n { + rows.row_into(x, &mut dense); + out.push_dense_row(&dense); + } + out +} + +/// A mask of `positions` over `blocks * 4` words. +fn position_mask(blocks: u32, positions: impl Iterator) -> Vec { + let mut m = vec![0u64; blocks as usize * BLOCK_WORDS]; + for p in positions { + if let Some(w) = m.get_mut(p as usize / 64) { + *w |= 1 << (p % 64); + } + } + m +} + +/// Brings `old` (the closure of the base edges) up to date with `log` without +/// a full rebuild, for `n_new >= old.n()` positions. +/// +/// The effective edge set is `base` folded with the log. Two phases, both +/// exact: +/// +/// 1. Retired edges (in `base`, not effective). The affected rows are +/// `A = { x : x == a or a ∈ R⁺(x) }` for every retired source `a`; they +/// are recomputed against the edges that survive, reusing `R⁺(y)` for any +/// `y ∉ A`. Every other row is kept bit for bit. `rows_recomputed` is +/// `|A|`. +/// 2. Added edges (effective, not in `base`). Monotone, so each row that can +/// reach an added source is seeded with that source's new targets and run +/// through the same fixpoint; nothing is recomputed and `rows_recomputed` +/// stays zero. +/// +/// An edge added and then retired within the log cancels and changes nothing. +/// Base edges must lie below `old.n()`; log edges below `n_new`. +pub fn refresh( + old: &WideRows, + base_src: &[u32], + base_dst: &[u32], + log: &DeltaLog, + n_new: u32, +) -> Result<(WideRows, WideStats), WideRefusal> { + if n_new < old.n() { + // Positions never renumber, so a closure cannot shrink. + return Err(WideRefusal::PositionOutOfRange { + pos: old.n(), + n: n_new, + }); + } + let blocks = check_dims(n_new)?; + validate(old.n(), base_src, base_dst)?; + validate(n_new, &log.src, &log.dst)?; + + let base: BTreeSet = base_src + .iter() + .zip(base_dst) + .map(|(&s, &d)| key(s, d)) + .collect(); + let eff = fold_keys(&base, log); + let retired: Vec = base.difference(&eff).copied().collect(); + let added: Vec = eff.difference(&base).copied().collect(); + + let mut stats = WideStats::default(); + let mut s = Scratch::new(blocks); + + // Phase 1: retired edges. + let phase1; + let r1: &WideRows = if retired.is_empty() { + if n_new == old.n() { + old + } else { + phase1 = widen(old, n_new, blocks); + &phase1 + } + } else { + let (rs, _) = lanes(retired.iter().copied()); + let retired_mask = position_mask(blocks, rs.iter().copied()); + let mut affected = vec![false; n_new as usize]; + for x in 0..n_new { + let hit = if retired_mask + .get(x as usize / 64) + .is_some_and(|w| w >> (x % 64) & 1 == 1) + { + true + } else { + old.row_into(x, &mut s.next); + s.next.iter().zip(&retired_mask).any(|(&r, &m)| r & m != 0) + }; + if let Some(a) = affected.get_mut(x as usize) { + *a = hit; + } + } + let (e1s, e1d) = lanes(base.intersection(&eff).copied()); + let e1 = build_edges(n_new, blocks, &e1s, &e1d)?; + let reuse = Reuse { + old, + affected: &affected, + }; + let mut out = WideRows::builder(n_new, blocks); + for x in 0..n_new { + if affected.get(x as usize).copied().unwrap_or(false) { + e1.row_into(x, &mut s.full); + s.delta.copy_from_slice(&s.full); + run_fixpoint(&e1, n_new, &mut s, Some(&reuse), &mut stats)?; + stats.rows_recomputed += 1; + } else { + old.row_into(x, &mut s.full); + } + out.push_dense_row(&s.full); + } + phase1 = out; + &phase1 + }; + + // Phase 2: added edges. + if added.is_empty() { + let result = if std::ptr::eq(r1, old) { + old.clone() + } else { + r1.clone() + }; + let stats = finish(stats, &result); + return Ok((result, stats)); + } + let (as_, ad) = lanes(added.iter().copied()); + let added_edges = build_edges(n_new, blocks, &as_, &ad)?; + let (es, ed) = lanes(eff.iter().copied()); + let eff_edges = build_edges(n_new, blocks, &es, &ed)?; + let added_mask = position_mask(blocks, as_.iter().copied()); + let mut out = WideRows::builder(n_new, blocks); + for x in 0..n_new { + r1.row_into(x, &mut s.full); + // Sources a path from x can start an added edge at: x and R⁺(x). + for (t, (&f, &m)) in s.next.iter_mut().zip(s.full.iter().zip(&added_mask)) { + *t = f & m; + } + if added_mask + .get(x as usize / 64) + .is_some_and(|w| w >> (x % 64) & 1 == 1) + { + if let Some(t) = s.next.get_mut(x as usize / 64) { + *t |= 1 << (x % 64); + } + } + if mask_any(&s.next) { + let Scratch { full, delta, next } = &mut s; + delta.fill(0); + for_each_bit(next, |src| { + added_edges.or_row_into(u32::try_from(src).unwrap_or(u32::MAX), delta); + }); + mask_andnot(delta, full, next); + delta.copy_from_slice(next); + mask_or_assign(full, delta); + run_fixpoint(&eff_edges, n_new, &mut s, None, &mut stats)?; + } + out.push_dense_row(&s.full); + } + let stats = finish(stats, &out); + Ok((out, stats)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Naive reachability by DFS from every node, written independently of + /// the kernel. + fn naive(n: usize, edges: &[(u32, u32)]) -> Vec> { + let mut adj = vec![Vec::new(); n]; + for &(a, b) in edges { + adj[a as usize].push(b as usize); + } + let mut out = vec![vec![false; n]; n]; + for x in 0..n { + let mut stack: Vec = adj[x].clone(); + while let Some(y) = stack.pop() { + if !out[x][y] { + out[x][y] = true; + stack.extend(adj[y].iter().copied()); + } + } + } + out + } + + fn build(n: u32, edges: &[(u32, u32)]) -> (WideRows, WideStats) { + let s: Vec = edges.iter().map(|e| e.0).collect(); + let d: Vec = edges.iter().map(|e| e.1).collect(); + closure_wide(n, &s, &d).expect("closure") + } + + fn assert_matches(rows: &WideRows, n: usize, edges: &[(u32, u32)]) -> u64 { + let reference = naive(n, edges); + let mut pairs = 0u64; + assert_eq!(reference.len(), n); + for (x, row) in reference.iter().enumerate() { + for (y, &want) in row.iter().enumerate() { + assert_eq!(rows.contains(x as u32, y as u32), want, "({x},{y})"); + pairs += u64::from(want); + } + } + pairs + } + + #[test] + fn chain_of_five() { + let edges: Vec<(u32, u32)> = (0..4).map(|i| (i, i + 1)).collect(); + let (rows, stats) = build(5, &edges); + let pairs = assert_matches(&rows, 5, &edges); + assert_eq!(pairs, 10); + assert_eq!(stats.rows_composed, pairs); + assert_eq!(stats.blocks_reserved, 5); + } + + #[test] + fn cycle_reaches_itself() { + let edges = [(0, 1), (1, 2), (2, 3), (3, 0), (0, 1)]; + let (rows, stats) = build(4, &edges); + let pairs = assert_matches(&rows, 4, &edges); + assert_eq!(pairs, 16); + assert_eq!(stats.rows_composed, 16); + assert!(rows.contains(2, 2)); + } + + #[test] + fn chain_crossing_a_block_boundary() { + let edges: Vec<(u32, u32)> = (0..299).map(|i| (i, i + 1)).collect(); + let (rows, stats) = build(300, &edges); + let pairs = assert_matches(&rows, 300, &edges); + assert_eq!(pairs, 300 * 299 / 2); + assert_eq!(stats.rows_composed, pairs); + // Row 0 reaches 1..=299: both blocks. Row 290 reaches only block 1. + let mut dense = vec![0u64; 5]; + rows.row_into(290, &mut dense); + assert_eq!(dense.iter().map(|w| w.count_ones()).sum::(), 9); + assert!(stats.blocks_allocated < stats.blocks_reserved); + } +} From 8bda67816394aff0c8d77e0ca5f147de5e496c61 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 07:21:23 +0000 Subject: [PATCH 03/19] xlog: multi-tile cube, wide closure, per-relation delta log, rebase; dt64 Axis::for_survivors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cube::new_tiled places ids at tile_ordinal * 256 + local through MapTable::locate (D-DYNBAG-4's global -> (tile, local) map). closure_wide folds base + delta log and never refuses for width. add_edge / retire_edge append to a per-relation log; refresh_closure recomputes only affected rows; refold runs only when called. rebase adopts a new frame only if every tile's map extends the old one: a resize is a new frame, never in place (D-DYNBAG §4.3). Axis::for_survivors is §4.1's width rule. The narrow single-tile path is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/lance-graph-dt64/src/dr.rs | 22 ++ crates/lance-graph-xlog/src/cube.rs | 334 ++++++++++++++++++++++++++-- 2 files changed, 341 insertions(+), 15 deletions(-) diff --git a/crates/lance-graph-dt64/src/dr.rs b/crates/lance-graph-dt64/src/dr.rs index 2ba34b2bb..e3036e0c5 100644 --- a/crates/lance-graph-dt64/src/dr.rs +++ b/crates/lance-graph-dt64/src/dr.rs @@ -59,6 +59,28 @@ impl Axis { self.tile.positions() * self.tiles as usize } + /// The axis a selection of `survivors` positions needs (D-DYNBAG §4.1): + /// up to 64 fit one DR64 tile; 65 to 256 one DR256 tile; more take + /// `ceil(n / 256)` DR256 tiles; none takes one DR64 tile. The width is a + /// cost decision: a DR64 register is 512 B and a DR256 one 8 KiB, so the + /// narrow tile is used whenever the selection fits it. `tiles` saturates + /// at `u32::MAX` rather than overflow. + pub const fn for_survivors(survivors: usize) -> Axis { + if survivors <= 64 { + Axis::new(Width::W64, 1) + } else if survivors <= 256 { + Axis::new(Width::W256, 1) + } else { + let tiles = survivors.div_ceil(256); + let tiles = if tiles > u32::MAX as usize { + u32::MAX + } else { + tiles as u32 + }; + Axis::new(Width::W256, tiles) + } + } + /// The tile a position falls in. pub const fn tile_of(self, position: usize) -> usize { position / self.tile.positions() diff --git a/crates/lance-graph-xlog/src/cube.rs b/crates/lance-graph-xlog/src/cube.rs index 55d4f4417..95a06d918 100644 --- a/crates/lance-graph-xlog/src/cube.rs +++ b/crates/lance-graph-xlog/src/cube.rs @@ -37,11 +37,12 @@ use std::cell::Cell; use std::sync::atomic::{AtomicU64, Ordering}; -use lance_graph_dt64::dr::{Frame, LocalMap, MapRead, MapRefusal, MapTable, TileId}; +use lance_graph_dt64::dr::{Frame, LocalMap, LocateRefusal, MapRead, MapRefusal, MapTable, TileId}; use lance_graph_mask_risc::{LaneRef, Out}; use lance_graph_quack::{Agg, Cmp, Col, Filter, GroupAddr, GroupAgg, Mask, Query}; use ndarray::simd::{checked_binary_i64, mask_and, mask_andnot, mask_any, mask_or_assign, I64Op}; +use crate::wide::{self, DeltaLog, WideRefusal, WideRows, WideStats, BLOCK_POSITIONS}; use crate::{all_live, run_lanes, NativeRefusal}; /// A relation's index in the order given to [`Cube::new`]. @@ -62,7 +63,7 @@ enum Node { Bind { rel: RelId, col: u8, - value: Option, + value: Option, }, /// `left.lcol == right.rcol`. Join { @@ -100,6 +101,20 @@ pub enum CubeRefusal { /// The fixpoint did not empty its delta within one round per position, /// which a correct semi-naive closure cannot do. FixpointBound, + /// An id could not be placed in a tiled cube's tile list (not held by any + /// tile, held by two, or a tile unreadable under the frame). + Locate(LocateRefusal), + /// The wide closure or refresh refused. + Wide(WideRefusal), + /// [`Cube::rebase`] was offered a frame under which this tile's map is not + /// a pure extension of the one the cube was built on: some position would + /// renumber, so every lane and logged edge would point at the wrong + /// symbol. Carries the tile. + Renumbered(TileId), + /// The terminal reads one tile's map (`materialize_rows`) and the cube + /// spans several; use [`Cube::closure_wide`] or read the lanes by + /// coordinate. + Tiled, } /// What the cube has done since it was built. @@ -157,7 +172,7 @@ struct Rel { struct Side { rel: RelId, /// `(free column, value)`; `Some(None)`: bound to an unmapped id. - bound: Option<(u8, Option)>, + bound: Option<(u8, Option)>, } /// A side with the column it joins on. @@ -170,8 +185,21 @@ pub struct Cube { id: u64, frame: Frame, tile: TileId, + /// Every tile the cube spans, in coordinate order: tile `i` owns wide + /// coordinates `i * 256 .. (i + 1) * 256`. One entry for [`Cube::new`]. + tiles: Vec, + /// Built by [`Cube::new_tiled`]: lanes hold wide coordinates + /// (`tile_ordinal * 256 + local`) instead of one tile's locals. + tiled: bool, + /// The maps as they were when the cube was built or last rebased, one per + /// entry of `tiles`. A [`MapTable`] drops a map when it advances, so the + /// old assignment is gone by the time [`Cube::rebase`] must compare it; + /// the cube keeps its own copy of the ids (a few bytes per position). + snapshots: Vec, n: usize, rels: Vec, + /// One append-only delta log per relation, parallel to `rels`. + logs: Vec, nodes: Vec, stats: Cell, } @@ -186,7 +214,7 @@ impl Cube { relations: &[&[(u32, u32)]], ) -> Result { let map = table.map(frame, tile).map_err(CubeRefusal::Map)?; - let rels = relations + let rels: Vec = relations .iter() .map(|tuples| rel(tuples, map)) .collect::>()?; @@ -194,7 +222,65 @@ impl Cube { id: NEXT_CUBE.fetch_add(1, Ordering::Relaxed), frame: frame.clone(), tile, + tiles: vec![tile], + tiled: false, + snapshots: vec![snapshot(map)?], n: map.len(), + logs: vec![DeltaLog::new(); rels.len()], + rels, + nodes: Vec::new(), + stats: Cell::new(CubeStats::default()), + }) + } + + /// Build a cube over several tiles. Every id is looked up once with + /// [`MapTable::locate`]; its wide coordinate is `tile_ordinal * 256 + + /// local`, where the ordinal is the tile's index in `tiles`. The 256-wide + /// block is used whatever the tile's own width, so a DR64 tile simply + /// occupies the first 64 positions of its block and coordinates stay + /// stable if the tile later widens. + /// + /// `n` is `256 * tiles.len()`: those positions are reserved, and an + /// unused one costs nothing in a wide row (an absent block reads as + /// zero). An id held by no tile is refused as [`CubeRefusal::Locate`], + /// so a selection that outgrows its tiles is a new frame with more + /// tiles, never an in-place resize. A tiled cube is read by + /// [`Cube::closure_wide`]; the single-tile terminals that decode a local + /// through one map refuse it. + pub fn new_tiled( + table: &MapTable, + frame: &Frame, + tiles: &[TileId], + relations: &[&[(u32, u32)]], + ) -> Result { + let place = |id: u32| -> Result { + let (ordinal, local) = table + .locate(frame, tiles, u64::from(id)) + .map_err(CubeRefusal::Locate)?; + Ok(ordinal * BLOCK_POSITIONS as u32 + u32::from(local)) + }; + let rels: Vec = relations + .iter() + .map(|tuples| rel_with(tuples, &place)) + .collect::>()?; + let snapshots = tiles + .iter() + .map(|&t| { + table + .map(frame, t) + .map_err(CubeRefusal::Map) + .and_then(snapshot) + }) + .collect::>()?; + Ok(Self { + id: NEXT_CUBE.fetch_add(1, Ordering::Relaxed), + frame: frame.clone(), + tile: tiles.first().copied().unwrap_or(0), + tiles: tiles.to_vec(), + tiled: true, + snapshots, + n: BLOCK_POSITIONS.saturating_mul(tiles.len()), + logs: vec![DeltaLog::new(); rels.len()], rels, nodes: Vec::new(), stats: Cell::new(CubeStats::default()), @@ -210,9 +296,13 @@ impl Cube { table: &MapTable, tuples: &[(u32, u32)], ) -> Result { + if self.tiled { + return Err(CubeRefusal::Tiled); + } let map = self.map(table)?; let r = rel(tuples, map)?; self.rels.push(r); + self.logs.push(DeltaLog::new()); let mut s = self.stats.get(); s.relations_added += 1; self.stats.set(s); @@ -259,8 +349,12 @@ impl Cube { return Err(CubeRefusal::NotASide); }; check_col(col)?; - let map = self.map(table)?; - let value = map.local(u64::from(value)).ok(); + // A value no tile holds binds to nothing; any other refusal is real. + let value = match self.coord(table, value) { + Ok(c) => Some(c), + Err(CubeRefusal::Domain(_) | CubeRefusal::Locate(LocateRefusal::Unmapped)) => None, + Err(e) => return Err(e), + }; Ok(self.push(Node::Bind { rel, col, value })) } @@ -316,6 +410,35 @@ impl Cube { table.map(&self.frame, self.tile).map_err(CubeRefusal::Map) } + /// Re-checks the cube's frame against every tile it spans. For a + /// single-tile cube this is exactly [`Cube::map`]'s check. + fn check_frame(&self, table: &MapTable) -> Result<(), CubeRefusal> { + if !self.tiled { + return self.map(table).map(|_| ()); + } + for &t in &self.tiles { + table.map(&self.frame, t).map_err(CubeRefusal::Map)?; + } + Ok(()) + } + + /// A global id's coordinate in this cube: the local position for a + /// single-tile cube, the wide coordinate for a tiled one. Counts one map + /// lookup per tile read. + fn coord(&self, table: &MapTable, id: u32) -> Result { + if self.tiled { + let (ordinal, local) = table + .locate(&self.frame, &self.tiles, u64::from(id)) + .map_err(CubeRefusal::Locate)?; + return Ok(ordinal * BLOCK_POSITIONS as u32 + u32::from(local)); + } + let l = self + .map(table)? + .local(u64::from(id)) + .map_err(CubeRefusal::Domain)?; + Ok(u32::from(l)) + } + fn run(&self, side: Side, query: &Query, out: Out<'_>) -> Result<(), CubeRefusal> { let r = &self.rels[side.rel as usize]; let lanes = [LaneRef::U32(&r.cols[0]), LaneRef::U32(&r.cols[1])]; @@ -331,9 +454,7 @@ impl Cube { match side.bound { None => Some(Filter::plane(Mask(0))), Some((_, None)) => None, - Some((col, Some(v))) => { - Some(Filter::cmp(Col(u16::from(col)), Cmp::EqU32(u32::from(v)))) - } + Some((col, Some(v))) => Some(Filter::cmp(Col(u16::from(col)), Cmp::EqU32(v))), } } @@ -379,7 +500,7 @@ impl Cube { /// position of the shared map. pub fn support(&self, table: &MapTable, join: View) -> Result, CubeRefusal> { let ((l, lc), (r, rc)) = self.sides(join)?; - self.map(table)?; + self.check_frame(table)?; let a = self.reached(l, lc)?; let b = self.reached(r, rc)?; let mut out = vec![0u64; a.len()]; @@ -391,7 +512,7 @@ impl Cube { /// `count` counts body bindings. A count past `i64` is refused. pub fn bag_count(&self, table: &MapTable, join: View) -> Result, CubeRefusal> { let ((l, lc), (r, rc)) = self.sides(join)?; - self.map(table)?; + self.check_frame(table)?; let a = self.degrees(l, lc)?; let b = self.degrees(r, rc)?; let mut out = vec![0i64; self.n]; @@ -413,7 +534,7 @@ impl Cube { .rels .get(rel as usize) .ok_or(CubeRefusal::UnknownRel(rel))?; - self.map(table)?; + self.check_frame(table)?; if self.n > 64 { return Err(CubeRefusal::ClosureWidth(self.n)); } @@ -447,6 +568,168 @@ impl Cube { Ok((Closure { rows: full }, stats)) } + /// `edge⁺` of relation `rel` over wide coordinates, with no limit on the + /// number of positions beyond [`wide::MAX_POSITIONS`]. Rows keep block + /// presence, so a position range nothing reaches costs nothing. + /// + /// The edges used are the relation's *effective* edges: its base lanes + /// folded with the relation's delta log ([`Cube::add_edge`], + /// [`Cube::retire_edge`]). The fold is done on the fly and the base is not + /// changed; replacing it is the explicit [`Cube::refold`]. The narrow + /// [`Cube::closure`] still reads the base lanes only. + pub fn closure_wide( + &self, + table: &MapTable, + rel: RelId, + ) -> Result<(WideRows, WideStats), CubeRefusal> { + let r = self + .rels + .get(rel as usize) + .ok_or(CubeRefusal::UnknownRel(rel))?; + self.check_frame(table)?; + let n = u32::try_from(self.n).unwrap_or(u32::MAX); + let log = self.log(rel)?; + if log.is_empty() { + return wide::closure_wide(n, &r.cols[0], &r.cols[1]).map_err(CubeRefusal::Wide); + } + let (src, dst) = wide::refold(&r.cols[0], &r.cols[1], log); + wide::closure_wide(n, &src, &dst).map_err(CubeRefusal::Wide) + } + + fn log(&self, rel: RelId) -> Result<&DeltaLog, CubeRefusal> { + self.logs + .get(rel as usize) + .ok_or(CubeRefusal::UnknownRel(rel)) + } + + /// Logs an ADD of edge `src -> dst` (global ids) on relation `rel`. The + /// ids are placed exactly as at build, so an id outside the cube's map is + /// refused: a new symbol is a rebase to a new frame ([`Cube::rebase`]), + /// never an in-place resize, because positions must not renumber under + /// edges already stored. The resident lanes are not touched; the log is + /// merged on read. + pub fn add_edge( + &mut self, + table: &MapTable, + rel: RelId, + src: u32, + dst: u32, + ) -> Result<(), CubeRefusal> { + let (s, d) = self.log_coords(table, rel, src, dst)?; + if let Some(l) = self.logs.get_mut(rel as usize) { + l.add(s, d); + } + Ok(()) + } + + /// Logs a RETIRE of edge `src -> dst` (global ids) on relation `rel`; see + /// [`Cube::add_edge`] for the placement rule. Retiring an edge that is + /// not present is a no-op when folded. A closure refresh recomputes every + /// row that could reach the retired source, because a closure row cannot + /// be patched for a deletion: the removed edge may or may not have been + /// the only path to a target. + pub fn retire_edge( + &mut self, + table: &MapTable, + rel: RelId, + src: u32, + dst: u32, + ) -> Result<(), CubeRefusal> { + let (s, d) = self.log_coords(table, rel, src, dst)?; + if let Some(l) = self.logs.get_mut(rel as usize) { + l.retire(s, d); + } + Ok(()) + } + + fn log_coords( + &self, + table: &MapTable, + rel: RelId, + src: u32, + dst: u32, + ) -> Result<(u32, u32), CubeRefusal> { + self.log(rel)?; + Ok((self.coord(table, src)?, self.coord(table, dst)?)) + } + + /// Logged operations on relation `rel` not yet folded into its base. + /// An unknown relation has none. + pub fn delta_len(&self, rel: RelId) -> usize { + self.logs.get(rel as usize).map_or(0, DeltaLog::len) + } + + /// Brings `old` up to date with relation `rel`'s delta log without a full + /// rebuild ([`wide::refresh`]). **Precondition:** `old` is the closure of + /// the relation's *base* lanes (for example the result of + /// [`Cube::closure_wide`] taken while the log was empty, or of the last + /// [`Cube::refold`]); given any other closure the result is not defined. + pub fn refresh_closure( + &self, + table: &MapTable, + rel: RelId, + old: &WideRows, + ) -> Result<(WideRows, WideStats), CubeRefusal> { + let r = self + .rels + .get(rel as usize) + .ok_or(CubeRefusal::UnknownRel(rel))?; + self.check_frame(table)?; + let n = u32::try_from(self.n).unwrap_or(u32::MAX); + wide::refresh(old, &r.cols[0], &r.cols[1], self.log(rel)?, n).map_err(CubeRefusal::Wide) + } + + /// Replaces relation `rel`'s base lanes with the base folded with its + /// log, and empties the log. Explicit only: nothing in the cube calls + /// this on a threshold, so the caller decides when a long log is worth + /// the rewrite. The old closure of the base is invalid afterwards. + pub fn refold(&mut self, rel: RelId) -> Result<(), CubeRefusal> { + let log = self.log(rel)?; + let r = self + .rels + .get(rel as usize) + .ok_or(CubeRefusal::UnknownRel(rel))?; + let (src, dst) = wide::refold(&r.cols[0], &r.cols[1], log); + let n = src.len(); + if let Some(r) = self.rels.get_mut(rel as usize) { + r.n = n; + r.live = all_live(n); + r.cols = [src, dst]; + } + if let Some(l) = self.logs.get_mut(rel as usize) { + *l = DeltaLog::new(); + } + Ok(()) + } + + /// Adopts `frame`, sealed after the table advanced to add symbols. Allowed + /// only if, for every tile the cube holds, the new map is a pure + /// extension of the one the cube was built (or last rebased) on + /// ([`LocalMap::extends`]): positions never renumber, so every stored + /// coordinate and logged edge keeps naming the same symbol. Otherwise + /// [`CubeRefusal::Renumbered`] names the tile and the cube is unchanged. + /// + /// A single-tile cube's `n` is its map's length, so it grows with the + /// map; a tiled cube's positions were reserved at build and stay fixed. + pub fn rebase(&mut self, table: &MapTable, frame: &Frame) -> Result<(), CubeRefusal> { + let mut next = Vec::with_capacity(self.tiles.len()); + for (&t, old) in self.tiles.iter().zip(&self.snapshots) { + let map = table.map(frame, t).map_err(CubeRefusal::Map)?; + if !map.extends(old) { + return Err(CubeRefusal::Renumbered(t)); + } + next.push(snapshot(map)?); + } + if !self.tiled { + if let Some(m) = next.first() { + self.n = m.len(); + } + } + self.snapshots = next; + self.frame = frame.clone(); + Ok(()) + } + /// The joined tuples `(left free, join value, right free)` as global ids: /// the one place this module builds rows, for a caller that downloads the /// relation. Restricted to the join's support first, so rows with no @@ -457,6 +740,9 @@ impl Cube { join: View, ) -> Result, CubeRefusal> { let ((l, lc), (r, rc)) = self.sides(join)?; + if self.tiled { + return Err(CubeRefusal::Tiled); + } let map = self.map(table)?; let keep = &self.support(table, join)?; let rows = |side: Side, jc: u8| { @@ -466,7 +752,7 @@ impl Cube { let bound = side.bound.map(|(_, v)| v); (0..j.len()) .filter(move |&i| keep[j[i] as usize / 64] >> (j[i] % 64) & 1 == 1) - .filter(move |&i| bound.is_none_or(|v| v == Some(f[i] as u8))) + .filter(move |&i| bound.is_none_or(|v| v == Some(f[i]))) .map(move |i| (j[i], f[i])) }; let mut out = Vec::new(); @@ -493,15 +779,33 @@ fn check_col(col: u8) -> Result<(), CubeRefusal> { } } +/// A copy of a tile's map for later comparison by [`Cube::rebase`]. +fn snapshot(map: &LocalMap) -> Result { + let globals: Vec = (0..map.len()) + .filter_map(|l| u8::try_from(l).ok().and_then(|l| map.global(l))) + .collect(); + LocalMap::prepare(map.width(), &globals).map_err(CubeRefusal::Domain) +} + fn rel(tuples: &[(u32, u32)], map: &LocalMap) -> Result { + rel_with(tuples, &|id| { + let l = map.local(u64::from(id)).map_err(CubeRefusal::Domain)?; + Ok(u32::from(l)) + }) +} + +/// Builds a relation's lanes, placing every id with `place`. +fn rel_with( + tuples: &[(u32, u32)], + place: &dyn Fn(u32) -> Result, +) -> Result { let mut cols = [ Vec::with_capacity(tuples.len()), Vec::with_capacity(tuples.len()), ]; for &(a, b) in tuples { for (c, id) in [(0, a), (1, b)] { - let l = map.local(u64::from(id)).map_err(CubeRefusal::Domain)?; - cols[c].push(u32::from(l)); + cols[c].push(place(id)?); } } Ok(Rel { From a31ac167a31d455e976e1196ff67e7b0a8630524 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 07:24:13 +0000 Subject: [PATCH 04/19] codegraph: open stores of any size on the multi-tile cube; impact reads whole rows Pos is u32; the tile layout comes from dt64's Axis::for_survivors, one tile keeps the single-tile path bit for bit, more tiles use Cube::new_tiled. Closures are WideRows from closure_wide. impact copied only the first word of a closure row, which was wrong once a graph has more than 64 positions; it now reads the whole row. TooMany and ClosureWidth are gone; their two tests are re-pinned to the new truth (a 70-function chain closes across words; 300 symbols open across two tiles). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/lance-graph-codegraph/src/diag.rs | 4 +- crates/lance-graph-codegraph/src/graph.rs | 82 ++++++++++++-------- crates/lance-graph-codegraph/src/query.rs | 29 +++---- crates/lance-graph-codegraph/tests/guards.rs | 43 +++++----- 4 files changed, 88 insertions(+), 70 deletions(-) diff --git a/crates/lance-graph-codegraph/src/diag.rs b/crates/lance-graph-codegraph/src/diag.rs index 7124dffd0..f41249c31 100644 --- a/crates/lance-graph-codegraph/src/diag.rs +++ b/crates/lance-graph-codegraph/src/diag.rs @@ -139,10 +139,10 @@ impl Codegraph { else { continue; }; - let callee = usize::from(callee); + let callee = callee as usize; if !has_bit(self.fns(), callee) && has_bit(self.stored_owner(), callee) { set_bit(&mut dangling_rows, r); - set_bit(&mut dangling_callers, usize::from(caller)); + set_bit(&mut dangling_callers, caller as usize); } } _ => {} diff --git a/crates/lance-graph-codegraph/src/graph.rs b/crates/lance-graph-codegraph/src/graph.rs index 7400fd81b..c75e8f715 100644 --- a/crates/lance-graph-codegraph/src/graph.rs +++ b/crates/lance-graph-codegraph/src/graph.rs @@ -1,17 +1,19 @@ -//! The codegraph: artifact rows interned into one tile of symbol positions -//! and folded into six relations of an xlog [`Cube`]. +//! The codegraph: artifact rows interned into symbol positions (one 64- or +//! 256-position tile, or several 256-position tiles) and folded into six +//! relations of an xlog [`Cube`]. use std::cell::OnceCell; use std::collections::HashMap; use std::path::Path; -use lance_graph_dt64::dr::{LocalMap, MapRefusal, MapTable, TileId, Width}; -use lance_graph_xlog::cube::{Closure, ClosureStats, Cube, CubeRefusal, RelId}; +use lance_graph_dt64::dr::{Axis, LocalMap, MapRefusal, MapTable, TileId, Width}; +use lance_graph_xlog::cube::{Cube, CubeRefusal, RelId}; +use lance_graph_xlog::wide::{WideRows, WideStats}; use crate::artifact::{self, Lanes, NO_SPAN}; -/// A symbol's local position in the tile (at most 256 symbols). -pub type Pos = u8; +/// A symbol's position: its interning index, which is also its cube coordinate. +pub type Pos = u32; /// Why a directory could not become a [`Codegraph`]. #[derive(Debug)] @@ -34,8 +36,6 @@ pub enum IntakeRefusal { /// The offending line. line: usize, }, - /// More than 256 symbols: one tile cannot hold them. Carries the count. - TooMany(usize), /// The tile map was refused. Map(MapRefusal), /// The cube was refused. @@ -64,7 +64,8 @@ pub const RCONF: RelId = 4; /// `RPOSS`: the possible reverse edges, including dispatch and `?dynamic`. pub const RPOSS: RelId = 5; -const TILE: TileId = 0; +/// Positions per full tile of a multi-tile store. +const TILE_POSITIONS: usize = 256; const NONE: u32 = u32::MAX; const DYNAMIC: &str = "?dynamic"; @@ -92,15 +93,15 @@ pub struct Codegraph { table: MapTable, cube: Cube, masks: Masks, - rconf: OnceCell<(Closure, ClosureStats)>, - rposs: OnceCell<(Closure, ClosureStats)>, + rconf: OnceCell<(WideRows, WideStats)>, + rposs: OnceCell<(WideRows, WideStats)>, } -fn intern(names: &mut Vec, index: &mut HashMap, s: &str) -> u32 { +fn intern(names: &mut Vec, index: &mut HashMap, s: &str) -> Pos { if let Some(&p) = index.get(s) { return p; } - let p = names.len() as u32; + let p = names.len() as Pos; names.push(s.to_string()); index.insert(s.to_string(), p); p @@ -111,7 +112,7 @@ fn dispatch_target(object: &str) -> &str { object.split(" via ").next().unwrap_or(object) } -fn set(words: &mut [u64], p: u32) { +fn set(words: &mut [u64], p: Pos) { words[p as usize / 64] |= 1 << (p % 64); } @@ -124,7 +125,7 @@ impl Codegraph { // Pass 1: intern symbols in first-seen row order; remember each row's // subject and object ids in lanes (NONE when the row has no symbol). let mut names: Vec = Vec::new(); - let mut idx: HashMap = HashMap::new(); + let mut idx: HashMap = HashMap::new(); let mut sid = vec![NONE; rows]; let mut oid = vec![NONE; rows]; let mut any_dynamic = false; @@ -159,11 +160,6 @@ impl Codegraph { } let dyn_id = any_dynamic.then(|| intern(&mut names, &mut idx, DYNAMIC)); let n = names.len(); - let width = match n { - 0..=64 => Width::W64, - 65..=256 => Width::W256, - _ => return Err(IntakeRefusal::TooMany(n)), - }; let words = n.div_ceil(64); // Pass 2: relations and masks. @@ -215,19 +211,42 @@ impl Codegraph { for (p, name) in names.iter().enumerate() { let owner = name.split("::").next().unwrap_or(name); if crates.contains_key(owner) { - set(&mut m.stored_owner, p as u32); + set(&mut m.stored_owner, p as Pos); } } - let ids: Vec = (0..n as u64).collect(); - let map = LocalMap::prepare(width, &ids).map_err(IntakeRefusal::Map)?; + // Tile t holds the global ids [t*256, min(n, (t+1)*256)). Every tile + // but the last is full, so tile t's local j sits at cube coordinate + // t*256 + j == the global id == the symbol's position. The masks and + // `symbol()` rely on coordinate == position. + let axis = Axis::for_survivors(n); + let mut changes = Vec::new(); + let mut tile_ids: Vec = Vec::new(); + for t in 0..axis.tiles { + let lo = t as usize * TILE_POSITIONS; + let hi = n.min(lo + TILE_POSITIONS); + debug_assert!( + axis.tiles == 1 || t + 1 == axis.tiles || hi - lo == TILE_POSITIONS, + "only the last tile may be partly filled" + ); + let ids: Vec = (lo as u64..hi as u64).collect(); + let map = LocalMap::prepare(axis.tile, &ids).map_err(IntakeRefusal::Map)?; + changes.push((t, Some(map))); + tile_ids.push(t); + } + debug_assert!(axis.tiles == 1 || axis.tile == Width::W256); let mut table = MapTable::new(1, 1); - table.advance(1, 1, vec![(TILE, Some(map))]); + table.advance(1, 1, changes); let frame = table.seal(); let slices: Vec<&[(u32, u32)]> = rel.iter().map(Vec::as_slice).collect(); - let cube = Cube::new(&table, &frame, TILE, &slices).map_err(IntakeRefusal::Cube)?; + let cube = if axis.tiles == 1 { + Cube::new(&table, &frame, 0, &slices) + } else { + Cube::new_tiled(&table, &frame, &tile_ids, &slices) + } + .map_err(IntakeRefusal::Cube)?; - let index = idx.into_iter().map(|(k, v)| (k, v as Pos)).collect(); + let index = idx.into_iter().map(|(k, v)| (k, v)).collect(); Ok(Self { lanes, names, @@ -248,7 +267,7 @@ impl Codegraph { /// The symbol at position `p`. pub fn symbol(&self, p: Pos) -> Option<&str> { - self.names.get(usize::from(p)).map(String::as_str) + self.names.get(p as usize).map(String::as_str) } /// Mask words per position mask: `n.div_ceil(64)`. @@ -261,7 +280,7 @@ impl Codegraph { let mut w = vec![0u64; self.words()]; for s in symbols { if let Some(p) = self.pos(s) { - set(&mut w, u32::from(p)); + set(&mut w, p); } } w @@ -365,9 +384,8 @@ impl Codegraph { } /// `edge⁺` of [`RCONF`] or [`RPOSS`], computed once and cached. Any other - /// relation is `UnknownRel`; more than 64 positions is `ClosureWidth` - /// (the cube's limit). - pub fn closure(&self, rel: RelId) -> Result<&(Closure, ClosureStats), CubeRefusal> { + /// relation is `UnknownRel`. Any number of positions closes. + pub fn closure(&self, rel: RelId) -> Result<&(WideRows, WideStats), CubeRefusal> { let cell = match rel { RCONF => &self.rconf, RPOSS => &self.rposs, @@ -376,7 +394,7 @@ impl Codegraph { if let Some(c) = cell.get() { return Ok(c); } - let computed = self.cube.closure(&self.table, rel)?; + let computed = self.cube.closure_wide(&self.table, rel)?; Ok(cell.get_or_init(|| computed)) } } diff --git a/crates/lance-graph-codegraph/src/query.rs b/crates/lance-graph-codegraph/src/query.rs index 29150e1a1..ae94ce7b6 100644 --- a/crates/lance-graph-codegraph/src/query.rs +++ b/crates/lance-graph-codegraph/src/query.rs @@ -10,6 +10,7 @@ use std::collections::HashSet; use lance_graph_xlog::cube::{CubeRefusal, RelId}; +use lance_graph_xlog::wide::WideRows; use ndarray::simd::{mask_and, mask_andnot_assign, mask_or_assign}; use crate::graph::{Codegraph, Pos, CALLS, IMPL, MAY, NODE, RCONF, RPOSS}; @@ -50,22 +51,16 @@ pub struct Sites { /// Why a query was refused. #[derive(Debug)] pub enum QueryRefusal { - /// The closure refused: more than 64 positions. The answer is UNKNOWN, - /// never "unaffected". - ClosureWidth(usize), /// Any other cube refusal. Cube(CubeRefusal), } fn refuse(e: CubeRefusal) -> QueryRefusal { - match e { - CubeRefusal::ClosureWidth(n) => QueryRefusal::ClosureWidth(n), - other => QueryRefusal::Cube(other), - } + QueryRefusal::Cube(e) } fn has(mask: &[u64], p: Pos) -> bool { - mask.get(usize::from(p) / 64) + mask.get(p as usize / 64) .is_some_and(|w| w >> (p % 64) & 1 == 1) } @@ -91,7 +86,7 @@ impl Codegraph { let (bound_col, join_col) = if into { (1, 0) } else { (0, 1) }; let (cube, table) = self.cube_and_table_mut(); let scan = cube.scan(rel)?; - let bound = cube.bind(table, scan, bound_col, u32::from(v))?; + let bound = cube.bind(table, scan, bound_col, v)?; let node = cube.scan(NODE)?; let join = cube.join(bound, join_col, node, 0)?; let mask = cube.support(table, join)?; @@ -167,7 +162,7 @@ impl Codegraph { let mut unknown = vec![0u64; w]; if let Some(d) = self.pos("?dynamic") { if self.symbol(f).is_some() && has(self.dynamic_callers(), f) { - unknown[usize::from(d) / 64] |= 1 << (d % 64); + unknown[d as usize / 64] |= 1 << (d % 64); } } Ok(Calls { @@ -232,17 +227,17 @@ impl Codegraph { /// confirmed (dispatch edges). Unknown: when `g`, or anything it reaches /// in `RPOSS⁺`, is an impl or referenced and a `?dynamic` position exists, /// the dynamic callers (`RPOSS⁺` of `?dynamic`) not already graded. All - /// masks are restricted to functions. A closure wider than 64 positions - /// is refused as `ClosureWidth`; an out-of-range `g` yields empty masks. + /// masks are restricted to functions. Closures of any width are read; an + /// out-of-range `g` yields empty masks. pub fn impact(&self, g: Pos) -> Result { let w = self.words(); let cc = &self.closure(RCONF).map_err(refuse)?.0; let cp = &self.closure(RPOSS).map_err(refuse)?.0; - let row = |c: &lance_graph_xlog::cube::Closure, p: Pos| -> Vec { + // `row_into` zero-extends or truncates to the buffer, so a `words()` + // long buffer holds exactly the interned positions' bits. + let row = |c: &WideRows, p: Pos| -> Vec { let mut m = vec![0u64; w]; - if let (Some(r), Some(slot)) = (c.rows().get(usize::from(p)), m.first_mut()) { - *slot = *r; - } + c.row_into(p, &mut m); m }; if self.symbol(g).is_none() { @@ -265,7 +260,7 @@ impl Codegraph { let mut unknown = vec![0u64; w]; if let Some(d) = self.pos("?dynamic") { let mut reach = reach_poss; - reach[usize::from(g) / 64] |= 1 << (g % 64); + reach[g as usize / 64] |= 1 << (g % 64); let mut dyn_reachable = self.impl_subjects().to_vec(); mask_or_assign(&mut dyn_reachable, self.referenced()); if intersects(&reach, &dyn_reachable) { diff --git a/crates/lance-graph-codegraph/tests/guards.rs b/crates/lance-graph-codegraph/tests/guards.rs index f7f59f529..4f97b15c6 100644 --- a/crates/lance-graph-codegraph/tests/guards.rs +++ b/crates/lance-graph-codegraph/tests/guards.rs @@ -5,8 +5,7 @@ use std::path::PathBuf; use lance_graph_codegraph::diag::{CrateKind, Run}; use lance_graph_codegraph::graph::IntakeRefusal; -use lance_graph_codegraph::query::QueryRefusal; -use lance_graph_codegraph::{Codegraph, Pos}; +use lance_graph_codegraph::Codegraph; const HEADER: &str = "#version 2\n#schema predicate\tsubject\tobject\tgrade\tfile\tstart\tend\n"; @@ -27,10 +26,13 @@ fn pop(mask: &[u64]) -> u32 { mask.iter().map(|w| w.count_ones()).sum() } -/// Fails if a 70-symbol chain is refused at open, or if impact does not -/// refuse with ClosureWidth(70) (the answer must be UNKNOWN, not empty). +/// Re-pinned: this used to assert `ClosureWidth(70)` because the cube's +/// closure held one `u64` per row. Closures are now wide, so a 70-symbol +/// chain (two words per row) must close across the word boundary. The chain +/// is f0 -> f1 -> ... -> f69, so the callers of f69 transitively are f0..f68. +/// Fails if any of those 69 bits is lost, including those in the second word. #[test] -fn closure_wider_than_64_is_refused() { +fn a_70_function_chain_closes_across_words() { let mut text = String::from(HEADER); for i in 0..70 { text += &row("DEFINES", &format!("chain::f{i}"), &format!("chain::f{i}")); @@ -44,27 +46,31 @@ fn closure_wider_than_64_is_refused() { } text += "CRATE\tchain\tCOMPLETE\tCOMPILER\t-\t-\t-\n"; let dir = write("chain", &text); - let g = Codegraph::open(&dir).expect("70 symbols fit a tile"); - let f0 = g.pos("chain::f0").unwrap(); - let r = g.impact(f0); - assert!(matches!(r, Err(QueryRefusal::ClosureWidth(70))), "{r:?}"); + let g = Codegraph::open(&dir).expect("70 symbols open"); + let f69 = g.pos("chain::f69").unwrap(); + let r = g.impact(f69).expect("wide closure never refuses"); + assert_eq!(pop(&r.confirmed), 69); + let others: Vec = (0..69).map(|i| format!("chain::f{i}")).collect(); + let refs: Vec<&str> = others.iter().map(String::as_str).collect(); + assert_eq!(r.confirmed, g.mask_of(&refs)); let _ = std::fs::remove_dir_all(&dir); } -/// Fails if 300 symbols are accepted into one 256-position tile. +/// Re-pinned: this used to assert `TooMany(300)`. A store of any size opens +/// now: 300 symbols take two 256-position tiles, and coordinates equal +/// positions. Fails if open is refused or a position past the first tile +/// does not round-trip. #[test] -fn too_many_symbols_is_refused() { +fn three_hundred_symbols_open_across_two_tiles() { let mut text = String::from(HEADER); for i in 0..300 { text += &row("DEFINES", &format!("big::s{i}"), &format!("big::s{i}")); } let dir = write("many", &text); - let r = Codegraph::open(&dir); - assert!( - matches!(r, Err(IntakeRefusal::TooMany(300))), - "{:?}", - r.err() - ); + let g = Codegraph::open(&dir).expect("300 symbols open"); + assert!(g.symbol(299).is_some()); + assert_eq!(g.pos("big::s299"), Some(299)); + assert_eq!(g.symbol(300), None); let _ = std::fs::remove_dir_all(&dir); } @@ -125,8 +131,7 @@ fn permuted_columns_read_identically() { assert_eq!(a.rows(), b.rows()); assert_eq!(a.words(), b.words()); assert!(pop(a.fns()) > 0 && pop(a.called()) > 0 && pop(a.may_targets()) > 0); - for pos in 0..=u8::MAX { - let pos: Pos = pos; + for pos in 0..=255_u32 { assert_eq!(a.symbol(pos), b.symbol(pos)); } assert_eq!(a.fns(), b.fns()); From c11984b2865433f1b91d1c37774ba91512b8ed89 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 07:24:24 +0000 Subject: [PATCH 05/19] codegraph: drop an identity map (clippy) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/lance-graph-codegraph/src/graph.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/lance-graph-codegraph/src/graph.rs b/crates/lance-graph-codegraph/src/graph.rs index c75e8f715..a5e0e0d90 100644 --- a/crates/lance-graph-codegraph/src/graph.rs +++ b/crates/lance-graph-codegraph/src/graph.rs @@ -246,7 +246,7 @@ impl Codegraph { } .map_err(IntakeRefusal::Cube)?; - let index = idx.into_iter().map(|(k, v)| (k, v)).collect(); + let index = idx.into_iter().collect(); Ok(Self { lanes, names, From 8be0615418fe6129384be467e41bccc828a78cfa Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 07:25:38 +0000 Subject: [PATCH 06/19] wide closure falsifiers; codegraph queries trim cube masks to the symbol width xlog tests/wide_closure.rs: narrow vs wide bit-identical up to 64 positions; a 694-position three-tile graph with cross-tile cycles against a DFS written in the test; absent blocks allocate nothing (with its anti-vacuity twin); refresh equals a rebuild after every step, with rows_recomputed == |A|; refold is explicit and preserves the closure; rebase accepts appends and refuses a reorder; growth is a new frame. codegraph: on a multi-tile store the cube's support mask covers every reserved position (12 words for 694 symbols) and callers panicked on the width mismatch; partners now trims to the symbol width. Found by running on mask-risc's real store; queries_work_across_tiles pins it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- .../examples/store_timing.rs | 80 +++ crates/lance-graph-codegraph/src/query.rs | 8 +- crates/lance-graph-codegraph/tests/guards.rs | 38 ++ crates/lance-graph-xlog/tests/wide_closure.rs | 524 ++++++++++++++++++ 4 files changed, 649 insertions(+), 1 deletion(-) create mode 100644 crates/lance-graph-codegraph/examples/store_timing.rs create mode 100644 crates/lance-graph-xlog/tests/wide_closure.rs diff --git a/crates/lance-graph-codegraph/examples/store_timing.rs b/crates/lance-graph-codegraph/examples/store_timing.rs new file mode 100644 index 000000000..b0fbd6edb --- /dev/null +++ b/crates/lance-graph-codegraph/examples/store_timing.rs @@ -0,0 +1,80 @@ +//! Per-call timing over any ruff_charon store directory (default: the rev1 +//! fixture), in nanoseconds; open and the first closure builds are cold and +//! reported in microseconds, not gated. Usage: `store_timing [dir] [symbol]`. + +use std::hint::black_box; +use std::path::PathBuf; +use std::time::Instant; + +use lance_graph_codegraph::graph::{RCONF, RPOSS}; +use lance_graph_codegraph::Codegraph; + +fn main() { + let mut args = std::env::args().skip(1); + let dir = args.next().map_or_else( + || PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data/rev1"), + PathBuf::from, + ); + let t = Instant::now(); + let mut g = Codegraph::open(&dir).expect("store opens"); + println!( + "open: {} us (cold, reported, not gated)", + t.elapsed().as_micros() + ); + let n = (0u32..).take_while(|&p| g.symbol(p).is_some()).count(); + println!("symbols: {n}, mask words: {}", g.words()); + let probe = args.next().and_then(|s| g.pos(&s)).unwrap_or_else(|| { + // The function with the most confirmed callers is a dense probe. + (0..n as u32) + .max_by_key(|&p| { + g.called() + .get(p as usize / 64) + .map_or(0, |w| w >> (p % 64) & 1) + }) + .unwrap_or(0) + }); + println!("probe: {}", g.symbol(probe).unwrap_or("?")); + for (name, rel) in [("RCONF", RCONF), ("RPOSS", RPOSS)] { + let t = Instant::now(); + let (_, stats) = g.closure(rel).expect("closure"); + println!( + "closure {name}: {} us (cold, reported, not gated); rounds {}, pairs {}, blocks {}/{}", + t.elapsed().as_micros(), + stats.rounds, + stats.rows_composed, + stats.blocks_allocated, + stats.blocks_reserved + ); + } + const N: u32 = 10_000; + let time = |label: &str, f: &mut dyn FnMut()| { + for _ in 0..1000 { + f(); + } + let t = Instant::now(); + for _ in 0..N { + f(); + } + println!( + "{label}: {:.1} ns/call", + t.elapsed().as_nanos() as f64 / f64::from(N) + ); + }; + time("impact", &mut || { + black_box(g.impact(probe).expect("impact")); + }); + time("references", &mut || { + black_box(g.references(probe)); + }); + time("callers", &mut || { + black_box(g.callers(probe).expect("callers")); + }); + let i = g.impact(probe).expect("impact"); + let pop = |m: &[u64]| m.iter().map(|w| w.count_ones()).sum::(); + println!( + "impact sizes: confirmed {}, possible {}, unknown {}", + pop(&i.confirmed), + pop(&i.possible), + pop(&i.unknown) + ); +} diff --git a/crates/lance-graph-codegraph/src/query.rs b/crates/lance-graph-codegraph/src/query.rs index ae94ce7b6..1498c4ab5 100644 --- a/crates/lance-graph-codegraph/src/query.rs +++ b/crates/lance-graph-codegraph/src/query.rs @@ -84,12 +84,18 @@ impl Codegraph { return Ok((vec![0; self.words()], 0)); } let (bound_col, join_col) = if into { (1, 0) } else { (0, 1) }; + let words = self.words(); let (cube, table) = self.cube_and_table_mut(); let scan = cube.scan(rel)?; let bound = cube.bind(table, scan, bound_col, v)?; let node = cube.scan(NODE)?; let join = cube.join(bound, join_col, node, 0)?; - let mask = cube.support(table, join)?; + let mut mask = cube.support(table, join)?; + // A multi-tile cube answers over every reserved position (256 per + // tile); the graph's masks cover only the interned symbols. Positions + // past the last symbol are never set (coordinate == position), so the + // tail words carry nothing and are dropped. + mask.resize(words, 0); let counts = cube.bag_count(table, join)?; let sites = counts.iter().fold(0u64, |a, &c| { a.saturating_add(u64::try_from(c).unwrap_or(0)) diff --git a/crates/lance-graph-codegraph/tests/guards.rs b/crates/lance-graph-codegraph/tests/guards.rs index 4f97b15c6..bee4a9ffc 100644 --- a/crates/lance-graph-codegraph/tests/guards.rs +++ b/crates/lance-graph-codegraph/tests/guards.rs @@ -190,3 +190,41 @@ fn an_uncalled_private_impl_method_is_not_dead() { ); let _ = std::fs::remove_dir_all(&d); } + +/// Fails if a query on a store spanning several tiles mixes mask widths: the +/// cube answers over every reserved position (256 per tile), the graph over +/// its symbols only. Found by running callers on mask-risc's 694-symbol store, +/// which panicked; no fixture had more than one tile and a call. +#[test] +fn queries_work_across_tiles() { + let mut text = String::from(HEADER); + for i in 0..300 { + text += &row("DEFINES", &format!("big::s{i}"), &format!("big::s{i}")); + } + // A chain s0 -> s1 -> ... -> s299 crosses the tile boundary at 256. + for i in 0..299 { + text += &row( + "CALLS_RESOLVED", + &format!("big::s{i}"), + &format!("big::s{}", i + 1), + ); + } + let dir = write("cross_tile_calls", &text); + let mut g = Codegraph::open(&dir).expect("opens"); + let last = g.pos("big::s299").expect("s299"); + let callers = g.callers(last).expect("callers"); + assert_eq!(callers.graded.confirmed, g.mask_of(&["big::s298"])); + assert_eq!(callers.sites, 1); + let callees = g + .callees(g.pos("big::s255").expect("s255")) + .expect("callees"); + assert_eq!(callees.graded.confirmed, g.mask_of(&["big::s256"])); + let impact = g.impact(last).expect("impact"); + assert_eq!( + pop(&impact.confirmed), + 299, + "every earlier link reaches s299" + ); + assert_eq!(g.references(last).confirmed, 1); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/lance-graph-xlog/tests/wide_closure.rs b/crates/lance-graph-xlog/tests/wide_closure.rs new file mode 100644 index 000000000..47472d5de --- /dev/null +++ b/crates/lance-graph-xlog/tests/wide_closure.rs @@ -0,0 +1,524 @@ +//! Wide closure (`Cube::closure_wide`, `refresh_closure`, `refold`, `rebase`) +//! against an independent per-node DFS over an adjacency list built here. +//! The reference never calls the code under test. Fixtures come from a tiny +//! deterministic LCG, so nothing depends on `rand` or on files outside the +//! repository. + +use std::collections::BTreeSet; +use std::ops::Range; + +use lance_graph_dt64::dr::{Frame, LocalMap, LocateRefusal, MapTable, TileId, Width}; +use lance_graph_xlog::cube::{Cube, CubeRefusal}; +use lance_graph_xlog::wide::{WideRows, WideStats}; + +/// Positions in one block (one dt64 tile). +const BLOCK: usize = 256; + +/// The three-tile fixture: tile ids, the global ids each holds, and the +/// reserved wide positions (`256 * 3`). +const TILES: [TileId; 3] = [11, 22, 33]; +const N_RES: usize = 768; +/// Ids actually in use in the three-tile fixture: 0..256, 256..512, 512..600. +const N_USED: u32 = 600; + +/// 64-bit LCG (Knuth's MMIX constants); high bits only. +struct Lcg(u64); + +impl Lcg { + fn next(&mut self) -> u64 { + self.0 = self + .0 + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + self.0 >> 33 + } + + /// Uniform-ish value in `0..k` (`k > 0`). + fn below(&mut self, k: u32) -> u32 { + (self.next() % u64::from(k)) as u32 + } +} + +/// A table whose tiles are `W256` maps holding the given global ids in +/// ascending order, so a global id `g` in tile ordinal `k` has local position +/// `g - 256 * k` and wide coordinate `g`. +fn make_table(tiles: &[(TileId, Range)]) -> (MapTable, Frame) { + let mut t = MapTable::new(1, 1); + let changes: Vec<(TileId, Option)> = tiles + .iter() + .map(|(id, r)| { + let globals: Vec = r.clone().collect(); + (*id, Some(LocalMap::prepare(Width::W256, &globals).unwrap())) + }) + .collect(); + t.advance(1, 1, changes); + let f = t.seal(); + (t, f) +} + +fn three_tile_table() -> (MapTable, Frame) { + make_table(&[ + (TILES[0], 0..256), + (TILES[1], 256..512), + (TILES[2], 512..u64::from(N_USED)), + ]) +} + +/// Reference reachability by DFS from every node over an adjacency list: +/// `out[x][y]` is true when `y` is reachable from `x` in one or more steps. +fn reference(n: usize, edges: &[(u32, u32)]) -> Vec> { + let mut adj = vec![Vec::new(); n]; + for &(a, b) in edges { + adj[a as usize].push(b as usize); + } + let mut out = vec![vec![false; n]; n]; + for x in 0..n { + let mut stack: Vec = adj[x].clone(); + while let Some(y) = stack.pop() { + if !out[x][y] { + out[x][y] = true; + stack.extend(adj[y].iter().copied()); + } + } + } + out +} + +/// Row `x` as dense words (`blocks * 4`). +fn dense(rows: &WideRows, x: usize) -> Vec { + let mut v = vec![0u64; rows.blocks() as usize * 4]; + rows.row_into(x as u32, &mut v); + v +} + +fn bit(row: &[u64], y: usize) -> bool { + row.get(y / 64).is_some_and(|w| w >> (y % 64) & 1 == 1) +} + +/// Every row of `a` equals the same row of `b`, read through `row_into`. +fn assert_rows_eq(a: &WideRows, b: &WideRows, what: &str) { + assert_eq!(a.n(), b.n(), "{what}: n"); + assert_eq!(a.blocks(), b.blocks(), "{what}: blocks"); + for x in 0..a.n() as usize { + assert_eq!(dense(a, x), dense(b, x), "{what}: row {x}"); + } +} + +/// Every `(x, y)` of the reserved range agrees with the reference (positions +/// beyond it must read as unreachable). Returns the number of reachable +/// pairs. +fn assert_matches_ref(rows: &WideRows, reference: &[Vec], what: &str) -> u64 { + assert_eq!(rows.n() as usize, reference.len(), "{what}: n"); + let mut pairs = 0u64; + for (x, ref_row) in reference.iter().enumerate() { + let d = dense(rows, x); + for (y, &want) in ref_row.iter().enumerate() { + assert_eq!(bit(&d, y), want, "{what}: ({x},{y})"); + pairs += u64::from(want); + } + } + pairs +} + +/// Blocks a sparse layout must store: one per `(row, block)` holding a +/// reachable position. +fn blocks_in_use(reference: &[Vec]) -> u64 { + let mut count = 0u64; + for row in reference { + for chunk in row.chunks(BLOCK) { + count += u64::from(chunk.iter().any(|&b| b)); + } + } + count +} + +fn random_edges(l: &mut Lcg, n_used: u32, k: usize) -> BTreeSet<(u32, u32)> { + let mut s = BTreeSet::new(); + while s.len() < k { + s.insert((l.below(n_used), l.below(n_used))); + } + s +} + +fn vec_of(s: &BTreeSet<(u32, u32)>) -> Vec<(u32, u32)> { + s.iter().copied().collect() +} + +fn tiled_cube( + table: &MapTable, + frame: &Frame, + tiles: &[TileId], + edges: &BTreeSet<(u32, u32)>, +) -> Cube { + let v = vec_of(edges); + Cube::new_tiled(table, frame, tiles, &[v.as_slice()]).unwrap() +} + +/// With at most 64 positions in one `W64`-sized block, the wide kernel and +/// the narrow `Cube::closure` give the same bits, word for word, and both +/// agree with the reference. +/// +/// Disable it catches: dropping the `andnot` in the wide kernel +/// (`mask_andnot(next, full, delta)` in `run_fixpoint`), so a delta +/// re-enters positions already in `full` (rows_composed and, on cycles, the +/// fixpoint bound diverge from the narrow kernel). +#[test] +fn wide_matches_narrow_bit_for_bit_up_to_64() { + let mut l = Lcg(0x5eed_0001); + for trial in 0..20u32 { + let n = if trial == 19 { + 64 + } else { + 1 + l.below(64) as usize + }; + let edges: Vec<(u32, u32)> = (0..2 * n) + .map(|_| (l.below(n as u32), l.below(n as u32))) + .collect(); + let (table, frame) = make_table(&[(7, 0..n as u64)]); + let cube = Cube::new(&table, &frame, 7, &[edges.as_slice()]).unwrap(); + let (narrow, _) = cube.closure(&table, 0).unwrap(); + let (wide, _) = cube.closure_wide(&table, 0).unwrap(); + let reference = reference(n, &edges); + assert_eq!(wide.n() as usize, n, "trial {trial}"); + assert_eq!(reference.len(), n); + for (x, ref_row) in reference.iter().enumerate() { + let d = dense(&wide, x); + assert_eq!(narrow.rows()[x], d[0], "trial {trial} row {x} word 0"); + assert!(d[1..].iter().all(|&w| w == 0), "trial {trial} row {x} tail"); + for (y, &want) in ref_row.iter().enumerate() { + let w = wide.contains(x as u32, y as u32); + assert_eq!(narrow.contains(x as u8, y as u8), w, "t{trial} ({x},{y})"); + assert_eq!(want, w, "t{trial} ref ({x},{y})"); + } + } + } +} + +/// 694 positions across three `W256` tiles (256 + 256 + 182), edges that form +/// a ring through all three tiles plus random chords, and a sparser random +/// graph without the ring: the wide closure equals the reference for every +/// `(x, y)` of the 768 reserved positions, and `rows_composed` equals the +/// number of reachable pairs. +/// +/// Disable it catches: packing only block 0 of each row (every cross-tile +/// pair goes missing), or composing through only the first block of a delta. +#[test] +fn three_tile_graph_with_cross_tile_cycles_matches_bfs() { + let mut l = Lcg(0x5eed_0002); + let ids: u32 = 694; + let (table, frame) = make_table(&[(1, 0..256), (2, 256..512), (3, 512..694)]); + for (name, ring, chords) in [("ring+chords", true, 1500usize), ("sparse", false, 800)] { + let mut edges: BTreeSet<(u32, u32)> = random_edges(&mut l, ids, chords); + if ring { + edges.extend((0..ids).map(|i| (i, (i + 1) % ids))); + } + let cube = tiled_cube(&table, &frame, &[1, 2, 3], &edges); + let (rows, stats) = cube.closure_wide(&table, 0).unwrap(); + let reference = reference(N_RES, &vec_of(&edges)); + let pairs = assert_matches_ref(&rows, &reference, name); + assert_eq!(stats.rows_composed, pairs, "{name}: rows_composed"); + assert_eq!(stats.blocks_reserved, (N_RES * 3) as u64, "{name}"); + assert_eq!(stats.blocks_allocated, blocks_in_use(&reference), "{name}"); + // The ring makes every id reach every id, across all tiles. + if ring { + assert_eq!(pairs, u64::from(ids) * u64::from(ids), "{name}: all pairs"); + } else { + assert!(pairs > 0 && pairs < u64::from(ids) * u64::from(ids), "{name}"); + } + } +} + +/// Block presence: edges that stay inside one tile allocate only that tile's +/// block, and the cross blocks cost nothing. The twin adds one edge from tile +/// 0 to tile 1 and the count must grow to what the reference says. +/// +/// Disable it catches: allocating every block of a row (the first count +/// becomes `rows * 2`), and, for the twin, a presence bit that is never set +/// (the count would not grow). +#[test] +fn absent_blocks_allocate_nothing() { + let mut l = Lcg(0x5eed_0003); + let (table, frame) = make_table(&[(1, 0..256), (2, 256..512)]); + let mut edges = BTreeSet::new(); + while edges.len() < 300 { + edges.insert((l.below(256), l.below(256))); + } + while edges.len() < 600 { + edges.insert((256 + l.below(256), 256 + l.below(256))); + } + let cube = tiled_cube(&table, &frame, &[1, 2], &edges); + let (rows, stats) = cube.closure_wide(&table, 0).unwrap(); + let reference_a = reference(512, &vec_of(&edges)); + assert_matches_ref(&rows, &reference_a, "in-tile"); + let rows_with_reach = reference_a + .iter() + .filter(|r| r.iter().any(|&b| b)) + .count() as u64; + assert!(rows_with_reach > 0, "fixture must reach something"); + // Each reaching row stores exactly one block (its own tile's); no cross. + assert_eq!(stats.blocks_allocated, rows_with_reach); + assert_eq!(stats.blocks_allocated, blocks_in_use(&reference_a)); + assert_eq!(stats.blocks_reserved, 512 * 2); + for x in 0..512 { + let d = dense(&rows, x); + let cross = if x < 256 { 256..512 } else { 0..256 }; + assert!(cross.into_iter().all(|y| !bit(&d, y)), "row {x} crossed"); + } + + // Anti-vacuity: one cross-tile edge must make blocks appear. + let mut with_cross = edges.clone(); + with_cross.insert((7, 300)); + let cube = tiled_cube(&table, &frame, &[1, 2], &with_cross); + let (rows_b, stats_b) = cube.closure_wide(&table, 0).unwrap(); + let reference_b = reference(512, &vec_of(&with_cross)); + assert_matches_ref(&rows_b, &reference_b, "cross"); + assert!(stats_b.blocks_allocated > stats.blocks_allocated); + assert_eq!(stats_b.blocks_allocated, blocks_in_use(&reference_b)); +} + +/// One checked step of the refresh script: the refreshed closure equals the +/// closure of a fresh cube built from the effective edge set (every row, via +/// `row_into`) and the reference; `rows_recomputed` equals `|A|`, the rows +/// that are a retired source or reach one in the base closure. +#[allow(clippy::too_many_arguments)] +fn check_step( + label: &str, + cube: &Cube, + table: &MapTable, + frame: &Frame, + old: &WideRows, + base: &BTreeSet<(u32, u32)>, + ref_base: &[Vec], + eff: &BTreeSet<(u32, u32)>, +) -> WideStats { + let (new, stats) = cube.refresh_closure(table, 0, old).unwrap(); + let fresh = tiled_cube(table, frame, &TILES, eff); + let (rebuilt, _) = fresh.closure_wide(table, 0).unwrap(); + assert_rows_eq(&new, &rebuilt, label); + assert_matches_ref(&new, &reference(N_RES, &vec_of(eff)), label); + let retired_src: BTreeSet = base.difference(eff).map(|e| e.0 as usize).collect(); + let expected = (0..N_RES) + .filter(|x| retired_src.contains(x) || retired_src.iter().any(|&a| ref_base[*x][a])) + .count() as u64; + assert_eq!(stats.rows_recomputed, expected, "{label}: rows_recomputed"); + stats +} + +/// A scripted sequence on one cube, checked after every step: add only, +/// retire only, mixed, and an add followed by a retire of the same new edge. +/// An add-only step recomputes no row (can stay silent); a retire step +/// recomputes some rows but not all of them (not vacuous). +/// +/// Disable it catches: skipping the affected-rows recompute on retire and +/// reusing the old rows (the retire steps then differ from the rebuild and +/// `rows_recomputed` is 0), or recomputing every row on every refresh (the +/// add-only `== 0` and the `|A|` equality fail). +#[test] +fn refresh_equals_rebuild_after_every_step() { + let mut l = Lcg(0x5eed_0004); + let (table, frame) = three_tile_table(); + let base = random_edges(&mut l, N_USED, 650); + let base_v = vec_of(&base); + let ref_base = reference(N_RES, &base_v); + let mut cube = tiled_cube(&table, &frame, &TILES, &base); + let (old, _) = cube.closure_wide(&table, 0).unwrap(); + let mut eff = base.clone(); + + // Step 1: add only. + for _ in 0..40 { + let e = (l.below(N_USED), l.below(N_USED)); + cube.add_edge(&table, 0, e.0, e.1).unwrap(); + eff.insert(e); + } + assert!(eff.len() > base.len(), "step 1 must add something new"); + let s1 = check_step("add-only", &cube, &table, &frame, &old, &base, &ref_base, &eff); + assert_eq!(s1.rows_recomputed, 0, "an add-only step recomputes no row"); + + // Step 2: retire only, from the base. + let mut retired = 0; + for _ in 0..15 { + let e = base_v[l.below(base_v.len() as u32) as usize]; + cube.retire_edge(&table, 0, e.0, e.1).unwrap(); + retired += usize::from(eff.remove(&e)); + } + assert!(retired > 0, "step 2 must retire a base edge"); + let s2 = check_step("retire-only", &cube, &table, &frame, &old, &base, &ref_base, &eff); + assert!(s2.rows_recomputed > 0, "a retire recomputes the rows it can reach"); + assert!(s2.rows_recomputed < N_RES as u64, "but not every row"); + + // Step 3: mixed. + for _ in 0..10 { + let e = (l.below(N_USED), l.below(N_USED)); + cube.add_edge(&table, 0, e.0, e.1).unwrap(); + eff.insert(e); + let cur = vec_of(&eff); + let r = cur[l.below(cur.len() as u32) as usize]; + cube.retire_edge(&table, 0, r.0, r.1).unwrap(); + eff.remove(&r); + } + let s3 = check_step("mixed", &cube, &table, &frame, &old, &base, &ref_base, &eff); + assert!(s3.rows_recomputed < N_RES as u64); + + // Step 4: add then retire the same new edge: nothing changes. + let fresh_edge = loop { + let e = (l.below(N_USED), l.below(N_USED)); + if !eff.contains(&e) && !base.contains(&e) { + break e; + } + }; + cube.add_edge(&table, 0, fresh_edge.0, fresh_edge.1).unwrap(); + cube.retire_edge(&table, 0, fresh_edge.0, fresh_edge.1).unwrap(); + let s4 = check_step("cancel", &cube, &table, &frame, &old, &base, &ref_base, &eff); + assert_eq!(s4.rows_recomputed, s3.rows_recomputed, "a cancelled edge adds no work"); + + // Cancel on its own: a log of only add-then-retire leaves the closure + // bit for bit and recomputes nothing. + let mut quiet = tiled_cube(&table, &frame, &TILES, &base); + quiet.add_edge(&table, 0, fresh_edge.0, fresh_edge.1).unwrap(); + quiet.retire_edge(&table, 0, fresh_edge.0, fresh_edge.1).unwrap(); + assert_eq!(quiet.delta_len(0), 2); + let (same, s) = quiet.refresh_closure(&table, 0, &old).unwrap(); + assert_rows_eq(&same, &old, "cancel-only"); + assert_eq!(s.rows_recomputed, 0); +} + +/// The log is merged on read and folded only when asked: `closure_wide` +/// leaves `delta_len` alone, `refold` empties it, and the closure is the +/// same before and after. +/// +/// Disable it catches: a `refold` that forgets to empty the log (delta_len +/// stays > 0) or one that drops the retired edges (rows differ), and a +/// `closure_wide` that folds on its own (delta_len would hit 0 early). +#[test] +fn refold_is_explicit_and_preserves_the_closure() { + let mut l = Lcg(0x5eed_0005); + let (table, frame) = three_tile_table(); + let base = random_edges(&mut l, N_USED, 650); + let base_v = vec_of(&base); + let mut cube = tiled_cube(&table, &frame, &TILES, &base); + let mut eff = base.clone(); + for _ in 0..20 { + let e = (l.below(N_USED), l.below(N_USED)); + cube.add_edge(&table, 0, e.0, e.1).unwrap(); + eff.insert(e); + } + for _ in 0..10 { + let e = base_v[l.below(base_v.len() as u32) as usize]; + cube.retire_edge(&table, 0, e.0, e.1).unwrap(); + eff.remove(&e); + } + assert_eq!(cube.delta_len(0), 30); + let (before, _) = cube.closure_wide(&table, 0).unwrap(); + assert_eq!(cube.delta_len(0), 30, "reading must not fold"); + assert_matches_ref(&before, &reference(N_RES, &vec_of(&eff)), "before refold"); + + cube.refold(0).unwrap(); + assert_eq!(cube.delta_len(0), 0, "refold empties the log"); + let (after, _) = cube.closure_wide(&table, 0).unwrap(); + assert_rows_eq(&before, &after, "refold"); + assert_matches_ref(&after, &reference(N_RES, &vec_of(&eff)), "after refold"); +} + +/// A pure extension of a tile map (ids appended under a new frame) is +/// accepted by `rebase`, after which an appended id can be used. A reordered +/// map is refused as `Renumbered(tile)` and the cube keeps answering under +/// its previous frame. +/// +/// Disable it catches: a `rebase` that skips the `extends` check (the +/// reorder is accepted), or one that adopts the frame before checking (the +/// cube is left on the refused frame and the last assertions fail). +#[test] +fn rebase_accepts_prefix_extension_and_refuses_reorder() { + let globals_a: Vec = (0..100).collect(); + let globals_b: Vec = (100..200).collect(); + let (mut table, frame_a) = make_table(&[(10, 0..100), (20, 100..200)]); + let edges = [(0u32, 1u32), (1, 150), (150, 199), (199, 0), (120, 121)]; + let mut cube = Cube::new_tiled(&table, &frame_a, &[10, 20], &[edges.as_slice()]).unwrap(); + let (rows_a, _) = cube.closure_wide(&table, 0).unwrap(); + + // Extension: tile 20 gains ids 5000 and 5001 at the end. + let mut extended = globals_b.clone(); + extended.extend([5000, 5001]); + table.advance( + 2, + 2, + vec![(20, Some(LocalMap::prepare(Width::W256, &extended).unwrap()))], + ); + let frame_b = table.seal(); + assert_eq!(cube.rebase(&table, &frame_b), Ok(())); + let (rows_b, _) = cube.closure_wide(&table, 0).unwrap(); + assert_rows_eq(&rows_a, &rows_b, "existing positions keep their rows"); + // 5000 is local 100 of tile ordinal 1: wide coordinate 356. 199 is 355. + cube.add_edge(&table, 0, 199, 5000).unwrap(); + let (rows_c, _) = cube.closure_wide(&table, 0).unwrap(); + assert!(bit(&dense(&rows_c, 355), 356), "appended id is addressable"); + + // Reorder: a second table whose tile 20 swaps its first two ids. + let mut swapped = extended.clone(); + swapped.swap(0, 1); + let mut other = MapTable::new(1, 1); + other.advance( + 1, + 1, + vec![ + (10, Some(LocalMap::prepare(Width::W256, &globals_a).unwrap())), + (20, Some(LocalMap::prepare(Width::W256, &swapped).unwrap())), + ], + ); + let frame_c = other.seal(); + assert_eq!( + cube.rebase(&other, &frame_c), + Err(CubeRefusal::Renumbered(20)) + ); + // Still on frame_b: the same table answers, and the logged edge stays. + let (rows_d, _) = cube.closure_wide(&table, 0).unwrap(); + assert_rows_eq(&rows_c, &rows_d, "refused rebase leaves the cube alone"); + assert_eq!(cube.delta_len(0), 1); +} + +/// An id that no tile of the cube holds is refused by `add_edge` and +/// `retire_edge`, on either endpoint, and nothing is logged: growth is a new +/// frame (see `rebase`), never an in-place resize. +/// +/// Disable it catches: placing an unknown id at a default coordinate (the +/// call succeeds and the log grows), or logging the source before the +/// destination is checked (delta_len becomes 1 after a refused call). +#[test] +fn growth_is_a_new_frame() { + let (table, frame) = make_table(&[(10, 0..100), (20, 100..200)]); + let edges = [(0u32, 1u32), (1, 150)]; + let mut tiled = Cube::new_tiled(&table, &frame, &[10, 20], &[edges.as_slice()]).unwrap(); + let (before, _) = tiled.closure_wide(&table, 0).unwrap(); + for (a, b) in [(5000, 1), (1, 5000), (5000, 5001)] { + assert_eq!( + tiled.add_edge(&table, 0, a, b), + Err(CubeRefusal::Locate(LocateRefusal::Unmapped)), + "add ({a},{b})" + ); + assert_eq!( + tiled.retire_edge(&table, 0, a, b), + Err(CubeRefusal::Locate(LocateRefusal::Unmapped)), + "retire ({a},{b})" + ); + } + assert_eq!(tiled.delta_len(0), 0); + let (after, _) = tiled.closure_wide(&table, 0).unwrap(); + assert_rows_eq(&before, &after, "tiled"); + + // The single-tile cube refuses the same way, through its one map. + let (table1, frame1) = make_table(&[(7, 0..100)]); + let e1 = [(0u32, 1u32)]; + let mut single = Cube::new(&table1, &frame1, 7, &[e1.as_slice()]).unwrap(); + assert!(matches!( + single.add_edge(&table1, 0, 1, 5000), + Err(CubeRefusal::Domain(_)) + )); + assert!(matches!( + single.retire_edge(&table1, 0, 5000, 1), + Err(CubeRefusal::Domain(_)) + )); + assert_eq!(single.delta_len(0), 0); + // Anti-vacuity: a known pair is accepted by the same call. + single.add_edge(&table1, 0, 1, 2).unwrap(); + assert_eq!(single.delta_len(0), 1); +} From 9a4247f2ff931669431ba9b712101c73cadb7afe Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 07:33:50 +0000 Subject: [PATCH 07/19] xlog: every cube read sees base + delta log (merge on read) One accessor returns a relation's effective lanes: the base when its log is empty, otherwise a cached fold of base and log, dropped on every add_edge, retire_edge, refold and rebase. Terminals (support, bag_count), the narrow closure, closure_wide and materialize_rows read through it, so views no longer see stale edges until an explicit refold. Reads never modify the base or the log; refold stays the only thing that replaces the base. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/lance-graph-xlog/src/cube.rs | 110 +++++++++++++----- crates/lance-graph-xlog/tests/wide_closure.rs | 58 ++++++--- 2 files changed, 122 insertions(+), 46 deletions(-) diff --git a/crates/lance-graph-xlog/src/cube.rs b/crates/lance-graph-xlog/src/cube.rs index 95a06d918..f85e92839 100644 --- a/crates/lance-graph-xlog/src/cube.rs +++ b/crates/lance-graph-xlog/src/cube.rs @@ -34,7 +34,7 @@ //! resident lanes (D-STAUNEN-5 P1). The delta difference is `mask_andnot` //! inside the cube; nothing is added to the cube and no tuple is built. -use std::cell::Cell; +use std::cell::{Cell, OnceCell}; use std::sync::atomic::{AtomicU64, Ordering}; use lance_graph_dt64::dr::{Frame, LocalMap, LocateRefusal, MapRead, MapRefusal, MapTable, TileId}; @@ -200,6 +200,11 @@ pub struct Cube { rels: Vec, /// One append-only delta log per relation, parallel to `rels`. logs: Vec, + /// Per relation, the base lanes folded with the relation's log, built on + /// first read when the log is non-empty and dropped whenever the base or + /// the log changes. A derived read: nothing persistent is written, and it + /// is recomputed by folding. Parallel to `rels`. + effective: Vec>, nodes: Vec, stats: Cell, } @@ -227,6 +232,7 @@ impl Cube { snapshots: vec![snapshot(map)?], n: map.len(), logs: vec![DeltaLog::new(); rels.len()], + effective: empty_effective(rels.len()), rels, nodes: Vec::new(), stats: Cell::new(CubeStats::default()), @@ -281,6 +287,7 @@ impl Cube { snapshots, n: BLOCK_POSITIONS.saturating_mul(tiles.len()), logs: vec![DeltaLog::new(); rels.len()], + effective: empty_effective(rels.len()), rels, nodes: Vec::new(), stats: Cell::new(CubeStats::default()), @@ -303,6 +310,7 @@ impl Cube { let r = rel(tuples, map)?; self.rels.push(r); self.logs.push(DeltaLog::new()); + self.effective.push(OnceCell::new()); let mut s = self.stats.get(); s.relations_added += 1; self.stats.set(s); @@ -439,8 +447,38 @@ impl Cube { Ok(u32::from(l)) } + /// The lanes every read goes through: relation `rel`'s base folded with + /// its delta log ("sparse write, merge on read"). With an empty log these + /// are the base lanes themselves, no copy. Otherwise they are the cached + /// fold, built on first use. A read never modifies the base or the log; + /// only [`Cube::refold`] replaces the base. + fn lanes(&self, rel: RelId) -> Result<&Rel, CubeRefusal> { + let i = rel as usize; + let base = self.rels.get(i).ok_or(CubeRefusal::UnknownRel(rel))?; + let log = self.log(rel)?; + if log.is_empty() { + return Ok(base); + } + let cell = self.effective.get(i).ok_or(CubeRefusal::UnknownRel(rel))?; + Ok(cell.get_or_init(|| { + let (src, dst) = wide::refold(&base.cols[0], &base.cols[1], log); + Rel { + n: src.len(), + live: all_live(src.len()), + cols: [src, dst], + } + })) + } + + /// Drops relation `rel`'s cached effective lanes. + fn invalidate(&mut self, rel: RelId) { + if let Some(c) = self.effective.get_mut(rel as usize) { + c.take(); + } + } + fn run(&self, side: Side, query: &Query, out: Out<'_>) -> Result<(), CubeRefusal> { - let r = &self.rels[side.rel as usize]; + let r = self.lanes(side.rel)?; let lanes = [LaneRef::U32(&r.cols[0]), LaneRef::U32(&r.cols[1])]; let mut s = self.stats.get(); s.programs += 1; @@ -530,10 +568,7 @@ impl Cube { table: &MapTable, rel: RelId, ) -> Result<(Closure, ClosureStats), CubeRefusal> { - let r = self - .rels - .get(rel as usize) - .ok_or(CubeRefusal::UnknownRel(rel))?; + let r = self.lanes(rel)?; self.check_frame(table)?; if self.n > 64 { return Err(CubeRefusal::ClosureWidth(self.n)); @@ -574,26 +609,19 @@ impl Cube { /// /// The edges used are the relation's *effective* edges: its base lanes /// folded with the relation's delta log ([`Cube::add_edge`], - /// [`Cube::retire_edge`]). The fold is done on the fly and the base is not - /// changed; replacing it is the explicit [`Cube::refold`]. The narrow - /// [`Cube::closure`] still reads the base lanes only. + /// [`Cube::retire_edge`]), read through the same accessor as every other + /// read, so the narrow [`Cube::closure`] and the terminals agree with it. + /// The base and the log are not changed by a read; replacing the base is + /// the explicit [`Cube::refold`]. pub fn closure_wide( &self, table: &MapTable, rel: RelId, ) -> Result<(WideRows, WideStats), CubeRefusal> { - let r = self - .rels - .get(rel as usize) - .ok_or(CubeRefusal::UnknownRel(rel))?; + let r = self.lanes(rel)?; self.check_frame(table)?; let n = u32::try_from(self.n).unwrap_or(u32::MAX); - let log = self.log(rel)?; - if log.is_empty() { - return wide::closure_wide(n, &r.cols[0], &r.cols[1]).map_err(CubeRefusal::Wide); - } - let (src, dst) = wide::refold(&r.cols[0], &r.cols[1], log); - wide::closure_wide(n, &src, &dst).map_err(CubeRefusal::Wide) + wide::closure_wide(n, &r.cols[0], &r.cols[1]).map_err(CubeRefusal::Wide) } fn log(&self, rel: RelId) -> Result<&DeltaLog, CubeRefusal> { @@ -619,6 +647,7 @@ impl Cube { if let Some(l) = self.logs.get_mut(rel as usize) { l.add(s, d); } + self.invalidate(rel); Ok(()) } @@ -639,6 +668,7 @@ impl Cube { if let Some(l) = self.logs.get_mut(rel as usize) { l.retire(s, d); } + self.invalidate(rel); Ok(()) } @@ -699,6 +729,7 @@ impl Cube { if let Some(l) = self.logs.get_mut(rel as usize) { *l = DeltaLog::new(); } + self.invalidate(rel); Ok(()) } @@ -727,6 +758,10 @@ impl Cube { } self.snapshots = next; self.frame = frame.clone(); + // `n` may have grown; cached folds were built under the old one. + for c in &mut self.effective { + c.take(); + } Ok(()) } @@ -745,19 +780,11 @@ impl Cube { } let map = self.map(table)?; let keep = &self.support(table, join)?; - let rows = |side: Side, jc: u8| { - let rel = &self.rels[side.rel as usize]; - let fc = 1 - jc as usize; - let (j, f) = (&rel.cols[jc as usize], &rel.cols[fc]); - let bound = side.bound.map(|(_, v)| v); - (0..j.len()) - .filter(move |&i| keep[j[i] as usize / 64] >> (j[i] % 64) & 1 == 1) - .filter(move |&i| bound.is_none_or(|v| v == Some(f[i]))) - .map(move |i| (j[i], f[i])) - }; + let (rel_l, rel_r) = (self.lanes(l.rel)?, self.lanes(r.rel)?); + let rows = side_rows; let mut out = Vec::new(); - for (jl, fl) in rows(l, lc) { - for (jr, fr) in rows(r, rc) { + for (jl, fl) in rows(keep, l, rel_l, lc) { + for (jr, fr) in rows(keep, r, rel_r, rc) { if jl == jr { let g = |x: u32| map.global(x as u8).expect("local position from intake"); out.push((g(fl), g(jl), g(fr))); @@ -771,6 +798,27 @@ impl Cube { } } +/// The `(join value, free value)` pairs of one side's effective lanes whose +/// join value is in `keep` and which satisfy the side's bound. +fn side_rows<'a>( + keep: &'a [u64], + side: Side, + rel: &'a Rel, + jc: u8, +) -> impl Iterator + 'a { + let fc = 1 - jc as usize; + let (j, f) = (&rel.cols[jc as usize], &rel.cols[fc]); + let bound = side.bound.map(|(_, v)| v); + (0..j.len()) + .filter(move |&i| keep[j[i] as usize / 64] >> (j[i] % 64) & 1 == 1) + .filter(move |&i| bound.is_none_or(|v| v == Some(f[i]))) + .map(move |i| (j[i], f[i])) +} + +fn empty_effective(n: usize) -> Vec> { + (0..n).map(|_| OnceCell::new()).collect() +} + fn check_col(col: u8) -> Result<(), CubeRefusal> { if col > 1 { Err(CubeRefusal::NoSuchColumn(col)) diff --git a/crates/lance-graph-xlog/tests/wide_closure.rs b/crates/lance-graph-xlog/tests/wide_closure.rs index 47472d5de..4fbe5b5c9 100644 --- a/crates/lance-graph-xlog/tests/wide_closure.rs +++ b/crates/lance-graph-xlog/tests/wide_closure.rs @@ -223,7 +223,10 @@ fn three_tile_graph_with_cross_tile_cycles_matches_bfs() { if ring { assert_eq!(pairs, u64::from(ids) * u64::from(ids), "{name}: all pairs"); } else { - assert!(pairs > 0 && pairs < u64::from(ids) * u64::from(ids), "{name}"); + assert!( + pairs > 0 && pairs < u64::from(ids) * u64::from(ids), + "{name}" + ); } } } @@ -250,10 +253,7 @@ fn absent_blocks_allocate_nothing() { let (rows, stats) = cube.closure_wide(&table, 0).unwrap(); let reference_a = reference(512, &vec_of(&edges)); assert_matches_ref(&rows, &reference_a, "in-tile"); - let rows_with_reach = reference_a - .iter() - .filter(|r| r.iter().any(|&b| b)) - .count() as u64; + let rows_with_reach = reference_a.iter().filter(|r| r.iter().any(|&b| b)).count() as u64; assert!(rows_with_reach > 0, "fixture must reach something"); // Each reaching row stores exactly one block (its own tile's); no cross. assert_eq!(stats.blocks_allocated, rows_with_reach); @@ -331,7 +331,9 @@ fn refresh_equals_rebuild_after_every_step() { eff.insert(e); } assert!(eff.len() > base.len(), "step 1 must add something new"); - let s1 = check_step("add-only", &cube, &table, &frame, &old, &base, &ref_base, &eff); + let s1 = check_step( + "add-only", &cube, &table, &frame, &old, &base, &ref_base, &eff, + ); assert_eq!(s1.rows_recomputed, 0, "an add-only step recomputes no row"); // Step 2: retire only, from the base. @@ -342,8 +344,20 @@ fn refresh_equals_rebuild_after_every_step() { retired += usize::from(eff.remove(&e)); } assert!(retired > 0, "step 2 must retire a base edge"); - let s2 = check_step("retire-only", &cube, &table, &frame, &old, &base, &ref_base, &eff); - assert!(s2.rows_recomputed > 0, "a retire recomputes the rows it can reach"); + let s2 = check_step( + "retire-only", + &cube, + &table, + &frame, + &old, + &base, + &ref_base, + &eff, + ); + assert!( + s2.rows_recomputed > 0, + "a retire recomputes the rows it can reach" + ); assert!(s2.rows_recomputed < N_RES as u64, "but not every row"); // Step 3: mixed. @@ -366,16 +380,27 @@ fn refresh_equals_rebuild_after_every_step() { break e; } }; - cube.add_edge(&table, 0, fresh_edge.0, fresh_edge.1).unwrap(); - cube.retire_edge(&table, 0, fresh_edge.0, fresh_edge.1).unwrap(); - let s4 = check_step("cancel", &cube, &table, &frame, &old, &base, &ref_base, &eff); - assert_eq!(s4.rows_recomputed, s3.rows_recomputed, "a cancelled edge adds no work"); + cube.add_edge(&table, 0, fresh_edge.0, fresh_edge.1) + .unwrap(); + cube.retire_edge(&table, 0, fresh_edge.0, fresh_edge.1) + .unwrap(); + let s4 = check_step( + "cancel", &cube, &table, &frame, &old, &base, &ref_base, &eff, + ); + assert_eq!( + s4.rows_recomputed, s3.rows_recomputed, + "a cancelled edge adds no work" + ); // Cancel on its own: a log of only add-then-retire leaves the closure // bit for bit and recomputes nothing. let mut quiet = tiled_cube(&table, &frame, &TILES, &base); - quiet.add_edge(&table, 0, fresh_edge.0, fresh_edge.1).unwrap(); - quiet.retire_edge(&table, 0, fresh_edge.0, fresh_edge.1).unwrap(); + quiet + .add_edge(&table, 0, fresh_edge.0, fresh_edge.1) + .unwrap(); + quiet + .retire_edge(&table, 0, fresh_edge.0, fresh_edge.1) + .unwrap(); assert_eq!(quiet.delta_len(0), 2); let (same, s) = quiet.refresh_closure(&table, 0, &old).unwrap(); assert_rows_eq(&same, &old, "cancel-only"); @@ -461,7 +486,10 @@ fn rebase_accepts_prefix_extension_and_refuses_reorder() { 1, 1, vec![ - (10, Some(LocalMap::prepare(Width::W256, &globals_a).unwrap())), + ( + 10, + Some(LocalMap::prepare(Width::W256, &globals_a).unwrap()), + ), (20, Some(LocalMap::prepare(Width::W256, &swapped).unwrap())), ], ); From 3c9c95bd8e12635314ff9019bf5560143ad445df Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 07:35:14 +0000 Subject: [PATCH 08/19] codegraph: refresh_crate (sparse write, merge on read); xlog delta log is a multiset Codegraph::refresh_crate re-reads one crate's rows and updates the resident graph without rebuilding: rows carry their crate, positions never renumber, new symbols extend the last tile under a new frame (growth past the reservation is NeedsReopen), edge changes go to the cube's delta log grouped by relation and in ascending (src tile, src, dst) order, cached closures are refreshed. refold() absorbs the logs only when called. A call graph is a bag: two call sites of one pair are two edges. The delta log fold counts occurrences (ADD +1, RETIRE -1, floored at 0) and the refresh diffs multisets; the closure sees an edge only while its count is above 0. Found because a refresh to rev2 counted one call site of area_of where a clean open counts two. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/lance-graph-codegraph/src/artifact.rs | 58 ++- crates/lance-graph-codegraph/src/graph.rs | 434 ++++++++++++++---- crates/lance-graph-codegraph/src/lib.rs | 2 +- crates/lance-graph-codegraph/tests/refresh.rs | 235 ++++++++++ crates/lance-graph-xlog/src/wide.rs | 82 +++- 5 files changed, 695 insertions(+), 116 deletions(-) create mode 100644 crates/lance-graph-codegraph/tests/refresh.rs diff --git a/crates/lance-graph-codegraph/src/artifact.rs b/crates/lance-graph-codegraph/src/artifact.rs index b88b104ad..0dfc88f14 100644 --- a/crates/lance-graph-codegraph/src/artifact.rs +++ b/crates/lance-graph-codegraph/src/artifact.rs @@ -44,6 +44,49 @@ pub struct Lanes { pub start: Vec, /// Span end, [`NO_SPAN`] when `-`. pub end: Vec, + /// The crate file (`.rows.tsv` stem) each row was read from. + pub krate: Vec, +} + +impl Lanes { + /// A copy holding only the rows not read from `krate`, in order. + pub fn without_crate(&self, krate: &str) -> Lanes { + let mut out = Lanes::default(); + for i in (0..self.predicate.len()).filter(|&i| self.krate[i] != krate) { + out.predicate.push(self.predicate[i].clone()); + out.subject.push(self.subject[i].clone()); + out.object.push(self.object[i].clone()); + out.grade.push(self.grade[i].clone()); + out.file.push(self.file[i].clone()); + out.start.push(self.start[i]); + out.end.push(self.end[i]); + out.krate.push(self.krate[i].clone()); + } + out + } + + /// Append every row of `other` after this lane set's rows. + pub fn append(&mut self, mut other: Lanes) { + self.predicate.append(&mut other.predicate); + self.subject.append(&mut other.subject); + self.object.append(&mut other.object); + self.grade.append(&mut other.grade); + self.file.append(&mut other.file); + self.start.append(&mut other.start); + self.end.append(&mut other.end); + self.krate.append(&mut other.krate); + } +} + +const SUFFIX: &str = ".rows.tsv"; + +/// Read the single file `.rows.tsv` of `dir`; its rows carry `krate`. +pub fn read_crate(dir: &Path, krate: &str) -> Result { + let text = + fs::read_to_string(dir.join(format!("{krate}{SUFFIX}"))).map_err(IntakeRefusal::Io)?; + let mut lanes = Lanes::default(); + read_text_as(&text, krate, &mut lanes)?; + Ok(lanes) } /// Read every `*.rows.tsv` in `dir`, sorted by file name so the row order is @@ -64,7 +107,12 @@ pub fn read_dir(dir: &Path) -> Result { let mut lanes = Lanes::default(); for p in &paths { let text = fs::read_to_string(p).map_err(IntakeRefusal::Io)?; - read_text(&text, &mut lanes)?; + let stem = p + .file_name() + .and_then(|n| n.to_str()) + .and_then(|n| n.strip_suffix(SUFFIX)) + .unwrap_or(""); + read_text_as(&text, stem, &mut lanes)?; } Ok(lanes) } @@ -77,8 +125,13 @@ fn number(field: &str) -> Option { } } -/// Append one file's rows to `lanes`. +/// Append one file's rows to `lanes`, with an empty crate name. pub fn read_text(text: &str, lanes: &mut Lanes) -> Result<(), IntakeRefusal> { + read_text_as(text, "", lanes) +} + +/// Append one file's rows to `lanes`, attributing each to `krate`. +pub fn read_text_as(text: &str, krate: &str, lanes: &mut Lanes) -> Result<(), IntakeRefusal> { let mut lines = text.lines(); let found = lines .next() @@ -123,6 +176,7 @@ pub fn read_text(text: &str, lanes: &mut Lanes) -> Result<(), IntakeRefusal> { lanes.file.push(f[at[4]].to_string()); lanes.start.push(start); lanes.end.push(end); + lanes.krate.push(krate.to_string()); } Ok(()) } diff --git a/crates/lance-graph-codegraph/src/graph.rs b/crates/lance-graph-codegraph/src/graph.rs index a5e0e0d90..8ae95bdc7 100644 --- a/crates/lance-graph-codegraph/src/graph.rs +++ b/crates/lance-graph-codegraph/src/graph.rs @@ -90,6 +90,8 @@ pub struct Codegraph { names: Vec, index: HashMap, crates: HashMap, + /// The reservation made at open; refresh grows only inside it. + axis: Axis, table: MapTable, cube: Cube, masks: Masks, @@ -116,104 +118,172 @@ fn set(words: &mut [u64], p: Pos) { words[p as usize / 64] |= 1 << (p % 64); } +/// Row subject/object ids after interning; `NONE` when a row has no symbol. +struct Resolved { + sid: Vec, + oid: Vec, + dyn_id: Option, +} + +/// Intern every row's symbols into `names`/`idx` in row order. Symbols already +/// interned keep their position, so running this over a lane set that gained +/// or lost rows only ever appends. +fn resolve(lanes: &Lanes, names: &mut Vec, idx: &mut HashMap) -> Resolved { + let rows = lanes.predicate.len(); + let mut sid = vec![NONE; rows]; + let mut oid = vec![NONE; rows]; + let mut any_dynamic = false; + for i in 0..rows { + let (s, o) = (lanes.subject[i].as_str(), lanes.object[i].as_str()); + match lanes.predicate[i].as_str() { + "DEFINES" | "CALLS_RESOLVED" | "PUBLIC" | "IMPLEMENTS" | "SYNTHETIC" | "REFERENCES" => { + sid[i] = intern(names, idx, s); + oid[i] = intern(names, idx, o); + } + "MAY_DISPATCH" => { + sid[i] = intern(names, idx, s); + oid[i] = intern(names, idx, dispatch_target(o)); + } + // The caller is a symbol; the object ("dynamic") is not. + "UNRESOLVED_CALL" => { + sid[i] = intern(names, idx, s); + any_dynamic = true; + } + _ => {} + } + } + let dyn_id = any_dynamic.then(|| intern(names, idx, DYNAMIC)); + Resolved { sid, oid, dyn_id } +} + +fn crate_statuses(lanes: &Lanes) -> HashMap { + let mut crates = HashMap::new(); + for i in 0..lanes.predicate.len() { + if lanes.predicate[i] == "CRATE" { + let st = if lanes.object[i] == "COMPLETE" { + RunStatus::Complete + } else { + RunStatus::Partial + }; + crates.insert(lanes.subject[i].clone(), st); + } + } + crates +} + +/// The five edge relations (everything but `NODE`, which is left empty) of a +/// lane set. Intake and refresh both build edges here, so they agree. +fn edges_of(lanes: &Lanes, r: &Resolved) -> [Vec<(u32, u32)>; 6] { + let mut rel: [Vec<(u32, u32)>; 6] = Default::default(); + for i in 0..lanes.predicate.len() { + let (s, o) = (r.sid[i], r.oid[i]); + match lanes.predicate[i].as_str() { + "CALLS_RESOLVED" => { + rel[CALLS as usize].push((s, o)); + rel[RCONF as usize].push((o, s)); + rel[RPOSS as usize].push((o, s)); + } + "IMPLEMENTS" => { + rel[IMPL as usize].push((s, o)); + rel[RPOSS as usize].push((s, o)); + } + "MAY_DISPATCH" => { + rel[MAY as usize].push((s, o)); + rel[RPOSS as usize].push((o, s)); + } + "UNRESOLVED_CALL" => { + if let Some(d) = r.dyn_id { + rel[RPOSS as usize].push((d, s)); + } + } + _ => {} + } + } + rel +} + +/// Position masks of a lane set over `names`. +fn masks_of( + lanes: &Lanes, + r: &Resolved, + names: &[String], + crates: &HashMap, +) -> Masks { + let words = names.len().div_ceil(64); + let mut m = Masks { + fns: vec![0; words], + public: vec![0; words], + impl_subjects: vec![0; words], + synthetic: vec![0; words], + referenced: vec![0; words], + called: vec![0; words], + may_targets: vec![0; words], + dynamic_callers: vec![0; words], + stored_owner: vec![0; words], + }; + for i in 0..lanes.predicate.len() { + let (s, o) = (r.sid[i], r.oid[i]); + match lanes.predicate[i].as_str() { + "DEFINES" => set(&mut m.fns, s), + "PUBLIC" => set(&mut m.public, s), + "SYNTHETIC" => set(&mut m.synthetic, s), + "REFERENCES" => set(&mut m.referenced, o), + "CALLS_RESOLVED" => set(&mut m.called, o), + "IMPLEMENTS" => set(&mut m.impl_subjects, s), + "MAY_DISPATCH" => set(&mut m.may_targets, o), + "UNRESOLVED_CALL" => set(&mut m.dynamic_callers, s), + _ => {} + } + } + for (p, name) in names.iter().enumerate() { + let owner = name.split("::").next().unwrap_or(name); + if crates.contains_key(owner) { + set(&mut m.stored_owner, p as Pos); + } + } + m +} + +/// What one [`Codegraph::refresh_crate`] did. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct RefreshStats { + /// Edges logged as ADD, over the five edge relations (not `NODE`). + pub edges_added: u64, + /// Edges logged as RETIRE, over the five edge relations. + pub edges_retired: u64, + /// Symbols interned past the last position. + pub symbols_appended: u64, + /// Closure rows recomputed (retired edges); 0 when no closure was cached. + pub rows_recomputed: u64, +} + +/// Why a refresh was refused. +#[derive(Debug)] +pub enum RefreshRefusal { + /// The crate's file could not be read or parsed. + Intake(IntakeRefusal), + /// The cube refused an edge, a rebase, or a closure refresh. + Cube(CubeRefusal), + /// The new symbols do not fit the reserved tiles; open the directory + /// afresh. Nothing was changed. + NeedsReopen { + /// Why growth in place is impossible. + reason: &'static str, + }, +} + impl Codegraph { /// Read every `*.rows.tsv` in `dir` and build the graph. pub fn open(dir: &Path) -> Result { let lanes = artifact::read_dir(dir)?; - let rows = lanes.predicate.len(); - - // Pass 1: intern symbols in first-seen row order; remember each row's - // subject and object ids in lanes (NONE when the row has no symbol). let mut names: Vec = Vec::new(); let mut idx: HashMap = HashMap::new(); - let mut sid = vec![NONE; rows]; - let mut oid = vec![NONE; rows]; - let mut any_dynamic = false; - let mut crates = HashMap::new(); - for i in 0..rows { - let (s, o) = (lanes.subject[i].as_str(), lanes.object[i].as_str()); - match lanes.predicate[i].as_str() { - "DEFINES" | "CALLS_RESOLVED" | "PUBLIC" | "IMPLEMENTS" | "SYNTHETIC" - | "REFERENCES" => { - sid[i] = intern(&mut names, &mut idx, s); - oid[i] = intern(&mut names, &mut idx, o); - } - "MAY_DISPATCH" => { - sid[i] = intern(&mut names, &mut idx, s); - oid[i] = intern(&mut names, &mut idx, dispatch_target(o)); - } - // The caller is a symbol; the object ("dynamic") is not. - "UNRESOLVED_CALL" => { - sid[i] = intern(&mut names, &mut idx, s); - any_dynamic = true; - } - "CRATE" => { - let st = if o == "COMPLETE" { - RunStatus::Complete - } else { - RunStatus::Partial - }; - crates.insert(s.to_string(), st); - } - _ => {} - } - } - let dyn_id = any_dynamic.then(|| intern(&mut names, &mut idx, DYNAMIC)); + let res = resolve(&lanes, &mut names, &mut idx); + let crates = crate_statuses(&lanes); let n = names.len(); - let words = n.div_ceil(64); - - // Pass 2: relations and masks. - let mut rel: [Vec<(u32, u32)>; 6] = Default::default(); - let mut m = Masks { - fns: vec![0; words], - public: vec![0; words], - impl_subjects: vec![0; words], - synthetic: vec![0; words], - referenced: vec![0; words], - called: vec![0; words], - may_targets: vec![0; words], - dynamic_callers: vec![0; words], - stored_owner: vec![0; words], - }; + let mut rel = edges_of(&lanes, &res); rel[NODE as usize] = (0..n as u32).map(|p| (p, p)).collect(); - for i in 0..rows { - let (s, o) = (sid[i], oid[i]); - match lanes.predicate[i].as_str() { - "DEFINES" => set(&mut m.fns, s), - "PUBLIC" => set(&mut m.public, s), - "SYNTHETIC" => set(&mut m.synthetic, s), - "REFERENCES" => set(&mut m.referenced, o), - "CALLS_RESOLVED" => { - set(&mut m.called, o); - rel[CALLS as usize].push((s, o)); - rel[RCONF as usize].push((o, s)); - rel[RPOSS as usize].push((o, s)); - } - "IMPLEMENTS" => { - set(&mut m.impl_subjects, s); - rel[IMPL as usize].push((s, o)); - rel[RPOSS as usize].push((s, o)); - } - "MAY_DISPATCH" => { - set(&mut m.may_targets, o); - rel[MAY as usize].push((s, o)); - rel[RPOSS as usize].push((o, s)); - } - "UNRESOLVED_CALL" => { - set(&mut m.dynamic_callers, s); - if let Some(d) = dyn_id { - rel[RPOSS as usize].push((d, s)); - } - } - _ => {} - } - } - for (p, name) in names.iter().enumerate() { - let owner = name.split("::").next().unwrap_or(name); - if crates.contains_key(owner) { - set(&mut m.stored_owner, p as Pos); - } - } + let m = masks_of(&lanes, &res, &names, &crates); // Tile t holds the global ids [t*256, min(n, (t+1)*256)). Every tile // but the last is full, so tile t's local j sits at cube coordinate @@ -246,12 +316,12 @@ impl Codegraph { } .map_err(IntakeRefusal::Cube)?; - let index = idx.into_iter().collect(); Ok(Self { lanes, names, - index, + index: idx, crates, + axis, table, cube, masks: m, @@ -260,6 +330,186 @@ impl Codegraph { }) } + /// Re-read `.rows.tsv` from `dir` and bring the resident graph up + /// to date without rebuilding it ("sparse write, merge on read"). + /// + /// * Rows are attributed to their crate file by the `krate` lane + /// ([`Lanes::krate`]); the crate's old rows are dropped from the lanes + /// and the new file's rows appended. + /// * Positions never renumber. A vanished symbol keeps its position (and + /// its `NODE` edge) with no rows; new symbols are appended. Growth must + /// fit the positions reserved at open (`Axis::for_survivors(n).reserved()`): + /// a new tile, or a W64 tile that would become W256, is + /// [`RefreshRefusal::NeedsReopen`] and changes nothing. Growth inside + /// the reservation extends the last tile's map under a new frame + /// (`MapTable::advance` + `seal`) and `Cube::rebase`s onto it. + /// * Edges: the whole graph's edges before and after are compared per + /// relation as multisets (a call graph is a bag: two call sites of one + /// pair are two edges), so an edge a *different* crate still produces + /// is never retired and a second call site is never lost. The difference is + /// logged with `retire_edge` / `add_edge`; the resident lanes are + /// untouched. **Apply order** (so the next tile/row is predictable): + /// relation by relation in id order; within a relation, retirements + /// then additions, each ascending by `(src tile, src, dst)` (tile = + /// src / 256, so plain `(src, dst)` order); `NODE` adds for appended + /// symbols ascending by id; the appended ids' global-to-local lookups + /// (inside `add_edge` and the map `prepare`) are therefore ascending. + /// * A cached `RCONF`/`RPOSS` closure is brought up to date with + /// `Cube::refresh_closure` only when that relation's log was empty + /// before this call (its precondition is a closure of the *base* + /// lanes; a cached closure already includes the log). Otherwise the + /// cache is dropped and rebuilt on next read by `closure_wide`, which + /// folds base + log. No cache means nothing to refresh. + /// * Masks and crate statuses are recomputed from the lanes. + /// + /// Refusals before the first cube mutation leave the graph unchanged; a + /// `Cube` refusal afterwards (not expected) may leave it partly updated. + pub fn refresh_crate( + &mut self, + dir: &Path, + krate: &str, + ) -> Result { + let fresh = artifact::read_crate(dir, krate).map_err(RefreshRefusal::Intake)?; + let mut new_lanes = self.lanes.without_crate(krate); + new_lanes.append(fresh); + + let mut names = self.names.clone(); + let mut idx = self.index.clone(); + let old_res = resolve(&self.lanes, &mut names, &mut idx); + let new_res = resolve(&new_lanes, &mut names, &mut idx); + let (n_old, n_new) = (self.names.len(), names.len()); + if n_new > self.axis.reserved() { + return Err(RefreshRefusal::NeedsReopen { + reason: "new symbols exceed the positions reserved at open (a new tile, or a W64 tile becoming W256)", + }); + } + + let old_e = edges_of(&self.lanes, &old_res); + let new_e = edges_of(&new_lanes, &new_res); + let mut retires: Vec> = Vec::new(); + let mut adds: Vec> = Vec::new(); + for r in 0..6 { + // Multiset diff: a call graph is a bag, two rows for one + // (src, dst) are two call sites. BTreeMap iterates ascending. + let mut o: std::collections::BTreeMap<(u32, u32), usize> = + std::collections::BTreeMap::new(); + let mut n: std::collections::BTreeMap<(u32, u32), usize> = + std::collections::BTreeMap::new(); + for &e in &old_e[r] { + *o.entry(e).or_insert(0) += 1; + } + for &e in &new_e[r] { + *n.entry(e).or_insert(0) += 1; + } + let mut gone: Vec<(u32, u32)> = Vec::new(); + let mut came: Vec<(u32, u32)> = Vec::new(); + for (&e, &oc) in &o { + let nc = n.get(&e).copied().unwrap_or(0); + for _ in nc..oc { + gone.push(e); + } + } + for (&e, &nc) in &n { + let oc = o.get(&e).copied().unwrap_or(0); + for _ in oc..nc { + came.push(e); + } + } + retires.push(gone); + adds.push(came); + } + let crates = crate_statuses(&new_lanes); + let masks = masks_of(&new_lanes, &new_res, &names, &crates); + + let pending = [ + self.cube.delta_len(RCONF) > 0, + self.cube.delta_len(RPOSS) > 0, + ]; + let mut stats = RefreshStats { + symbols_appended: (n_new - n_old) as u64, + ..RefreshStats::default() + }; + + // Extend the last tile's map (ids ascending) and adopt the new frame. + if n_new > n_old { + let last = self.axis.tiles.saturating_sub(1); + let lo = last as u64 * TILE_POSITIONS as u64; + let ids: Vec = (lo..n_new as u64).collect(); + let map = LocalMap::prepare(self.axis.tile, &ids) + .map_err(|e| RefreshRefusal::Intake(IntakeRefusal::Map(e)))?; + let rev = self.table.revisions(); + self.table + .advance(rev.source + 1, rev.relation_map, vec![(last, Some(map))]); + let frame = self.table.seal(); + self.cube + .rebase(&self.table, &frame) + .map_err(RefreshRefusal::Cube)?; + } + + for r in 0..6usize { + let rel = r as RelId; + if rel == NODE { + for p in n_old as u32..n_new as u32 { + self.cube + .add_edge(&self.table, NODE, p, p) + .map_err(RefreshRefusal::Cube)?; + } + continue; + } + for &(s, d) in &retires[r] { + self.cube + .retire_edge(&self.table, rel, s, d) + .map_err(RefreshRefusal::Cube)?; + stats.edges_retired += 1; + } + for &(s, d) in &adds[r] { + self.cube + .add_edge(&self.table, rel, s, d) + .map_err(RefreshRefusal::Cube)?; + stats.edges_added += 1; + } + } + + let changed = stats.edges_added + stats.edges_retired > 0 || n_new > n_old; + for (rel, pend) in [(RCONF, pending[0]), (RPOSS, pending[1])] { + let cell = if rel == RCONF { + &mut self.rconf + } else { + &mut self.rposs + }; + if !changed { + continue; + } + if let Some(old) = cell.take() { + if !pend { + let (rows, st) = self + .cube + .refresh_closure(&self.table, rel, &old.0) + .map_err(RefreshRefusal::Cube)?; + stats.rows_recomputed += st.rows_recomputed; + let _ = cell.set((rows, st)); + } + } + } + + self.lanes = new_lanes; + self.names = names; + self.index = idx; + self.crates = crates; + self.masks = masks; + Ok(stats) + } + + /// Absorb every relation's delta log into its base lanes (explicit only; + /// nothing calls this on a threshold). Cached closures stay valid: the + /// base now equals the effective edges they were computed from. + pub fn refold(&mut self) -> Result<(), CubeRefusal> { + for rel in 0..6 { + self.cube.refold(rel)?; + } + Ok(()) + } + /// The position of `symbol`, if interned. pub fn pos(&self, symbol: &str) -> Option { self.index.get(symbol).copied() diff --git a/crates/lance-graph-codegraph/src/lib.rs b/crates/lance-graph-codegraph/src/lib.rs index aa97fb89c..bcab215cb 100644 --- a/crates/lance-graph-codegraph/src/lib.rs +++ b/crates/lance-graph-codegraph/src/lib.rs @@ -8,4 +8,4 @@ pub mod diag; pub mod graph; pub mod query; -pub use graph::{Codegraph, IntakeRefusal, Pos, RunStatus}; +pub use graph::{Codegraph, IntakeRefusal, Pos, RefreshRefusal, RefreshStats, RunStatus}; diff --git a/crates/lance-graph-codegraph/tests/refresh.rs b/crates/lance-graph-codegraph/tests/refresh.rs new file mode 100644 index 000000000..c468fe497 --- /dev/null +++ b/crates/lance-graph-codegraph/tests/refresh.rs @@ -0,0 +1,235 @@ +//! Refresh tests: `refresh_crate` / `refold` against clean opens of the +//! committed fixture stores (copied to a temp dir; `tests/data` is never +//! written). + +use std::fs; +use std::path::{Path, PathBuf}; + +use lance_graph_codegraph::diag::CrateKind; +use lance_graph_codegraph::{Codegraph, Pos, RefreshRefusal}; + +fn data(store: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data") + .join(store) +} + +/// A fresh temp dir holding a copy of `store`'s files. +fn copy_store(store: &str, tag: &str) -> PathBuf { + // Tests run in parallel and several share a tag, so every call gets its + // own directory: one test deleting another's files mid-read made a + // `rows.tsv` look empty (`Version { found: 0 }`). + static NEXT: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!("cg_refresh_{}_{tag}_{n}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("temp dir"); + copy_files(&data(store), &dir); + dir +} + +fn copy_files(from: &Path, to: &Path) { + for e in fs::read_dir(from).expect("read store") { + let p = e.expect("entry").path(); + fs::copy(&p, to.join(p.file_name().expect("name"))).expect("copy"); + } +} + +fn names(g: &Codegraph, mask: &[u64]) -> Vec { + let mut v: Vec = (0..mask.len() * 64) + .filter(|&p| mask[p / 64] >> (p % 64) & 1 == 1) + .filter_map(|p| g.symbol(p as Pos).map(str::to_string)) + .collect(); + v.sort(); + v +} + +fn all_names(g: &Codegraph) -> Vec { + (0..g.words() * 64) + .filter_map(|p| g.symbol(p as Pos).map(str::to_string)) + .collect() +} + +fn refreshed_rev2() -> Codegraph { + let dir = copy_store("rev1", "eq"); + let mut g = Codegraph::open(&dir).expect("open rev1"); + let r2 = data("rev2"); + for k in ["cg_lib", "cg_app"] { + fs::copy( + r2.join(format!("{k}.rows.tsv")), + dir.join(format!("{k}.rows.tsv")), + ) + .expect("copy rev2"); + g.refresh_crate(&dir, k).expect("refresh"); + } + g +} + +/// Red if refresh leaves any callers/callees/impact answer, or the +/// DeadFunction set, different from a clean open of rev2 (disable: skip the +/// `add_edge`/`retire_edge` calls in `refresh_crate`). +#[test] +fn refresh_equals_clean_open() { + let mut g = refreshed_rev2(); + let mut clean = Codegraph::open(&data("rev2")).expect("open rev2"); + for name in all_names(&clean) { + let (pc, pg) = ( + clean.pos(&name).expect("clean"), + g.pos(&name).expect("refreshed"), + ); + let (cc, cg) = (clean.callers(pc).expect("q"), g.callers(pg).expect("q")); + let (ec, eg) = (clean.callees(pc).expect("q"), g.callees(pg).expect("q")); + for (a, b) in [ + (&cc.graded.confirmed, &cg.graded.confirmed), + (&cc.graded.possible, &cg.graded.possible), + (&cc.graded.unknown, &cg.graded.unknown), + ] { + assert_eq!(names(&clean, a), names(&g, b), "callers of {name}"); + } + assert_eq!(cc.sites, cg.sites, "callers sites of {name}"); + for (a, b) in [ + (&ec.graded.confirmed, &eg.graded.confirmed), + (&ec.graded.possible, &eg.graded.possible), + (&ec.graded.unknown, &eg.graded.unknown), + ] { + assert_eq!(names(&clean, a), names(&g, b), "callees of {name}"); + } + let (ic, ig) = (clean.impact(pc).expect("q"), g.impact(pg).expect("q")); + for (a, b) in [ + (&ic.confirmed, &ig.confirmed), + (&ic.possible, &ig.possible), + (&ic.unknown, &ig.unknown), + ] { + assert_eq!(names(&clean, a), names(&g, b), "impact of {name}"); + } + } + let (dc, dg) = ( + &clean.diagnostics(CrateKind::Lib)[0], + &g.diagnostics(CrateKind::Lib)[0], + ); + assert_eq!(names(&clean, &dc.positions), names(&g, &dg.positions)); +} + +/// Red if the dependent crate's stale call is dropped or duplicated (disable: +/// retire edges of every crate, not only the refreshed one's difference). +#[test] +fn skipping_a_dependent_leaves_one_dangling_call() { + let dir = copy_store("rev1", "dangle"); + let mut g = Codegraph::open(&dir).expect("open"); + fs::copy( + data("rev2").join("cg_lib.rows.tsv"), + dir.join("cg_lib.rows.tsv"), + ) + .expect("copy"); + g.refresh_crate(&dir, "cg_lib").expect("refresh"); + let f = &g.diagnostics(CrateKind::Lib)[2]; + assert_eq!(f.rows.iter().map(|w| w.count_ones()).sum::(), 1); + let row = f + .rows + .iter() + .enumerate() + .find_map(|(w, &b)| (b != 0).then(|| w * 64 + b.trailing_zeros() as usize)) + .expect("one row"); + assert_eq!(g.row_subject(row), "cg_app::run"); + assert_eq!(g.row_object(row), "cg_lib::area_dyn"); +} + +/// Red if refresh reassigns positions (disable: rebuild names from scratch in +/// `refresh_crate`). +#[test] +fn a_refresh_does_not_renumber() { + let dir = copy_store("rev1", "pos"); + let mut g = Codegraph::open(&dir).expect("open"); + let before: Vec<(String, Pos)> = all_names(&g) + .into_iter() + .map(|n| { + let p = g.pos(&n).expect("pos"); + (n, p) + }) + .collect(); + fs::copy( + data("rev2").join("cg_lib.rows.tsv"), + dir.join("cg_lib.rows.tsv"), + ) + .expect("copy"); + g.refresh_crate(&dir, "cg_lib").expect("refresh"); + for (n, p) in before { + assert_eq!(g.pos(&n), Some(p), "{n}"); + assert_eq!(g.symbol(p), Some(n.as_str())); + } +} + +/// Red if the cached closure is rebuilt from scratch instead of refreshed +/// (disable: drop the cache unconditionally). Anti-vacuity: rev2 retires +/// edges, so some rows must be recomputed. +#[test] +fn refresh_recomputes_less_than_everything() { + let dir = copy_store("rev1", "less"); + let mut g = Codegraph::open(&dir).expect("open"); + g.closure(lance_graph_codegraph::graph::RCONF) + .expect("prime"); + g.closure(lance_graph_codegraph::graph::RPOSS) + .expect("prime"); + fs::copy( + data("rev2").join("cg_lib.rows.tsv"), + dir.join("cg_lib.rows.tsv"), + ) + .expect("copy"); + let st = g.refresh_crate(&dir, "cg_lib").expect("refresh"); + assert!(st.edges_retired > 0); + assert!(st.rows_recomputed > 0, "{st:?}"); + assert!(st.rows_recomputed < all_names(&g).len() as u64, "{st:?}"); +} + +/// Red if refold changes any answer (disable: have refold drop the log +/// without folding it into the base). +#[test] +fn refold_keeps_answers() { + let mut g = refreshed_rev2(); + let ps: Vec = all_names(&g).iter().filter_map(|n| g.pos(n)).collect(); + let snap = |g: &mut Codegraph| -> Vec<(Vec, Vec, Vec)> { + ps.iter() + .map(|&p| { + ( + g.callers(p).expect("q").graded.confirmed, + g.callees(p).expect("q").graded.confirmed, + g.impact(p).expect("q").confirmed, + ) + }) + .collect() + }; + let before = snap(&mut g); + g.refold().expect("refold"); + assert_eq!(before, snap(&mut g)); +} + +fn rows_file(count: usize) -> String { + let mut s = + String::from("#version 2\n#schema predicate\tsubject\tobject\tgrade\tfile\tstart\tend\n"); + for i in 0..count { + s.push_str(&format!( + "DEFINES\tbig::f{i}\tbig::f{i}\tCOMPILER\tbig/src/lib.rs\t{i}\t{}\n", + i + 1 + )); + } + s +} + +/// Red if growth past the reserved tile is accepted (disable: remove the +/// capacity check in `refresh_crate`). Also checks nothing changed. +#[test] +fn growth_past_the_reservation_needs_reopen() { + let dir = std::env::temp_dir().join(format!("cg_refresh_{}_big", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("dir"); + fs::write(dir.join("big.rows.tsv"), rows_file(64)).expect("write"); + let mut g = Codegraph::open(&dir).expect("open"); + fs::write(dir.join("big.rows.tsv"), rows_file(65)).expect("write"); + let rows = g.rows(); + match g.refresh_crate(&dir, "big") { + Err(RefreshRefusal::NeedsReopen { .. }) => {} + other => panic!("expected NeedsReopen, got {other:?}"), + } + assert_eq!(g.rows(), rows); + assert_eq!(g.pos("big::f64"), None); +} diff --git a/crates/lance-graph-xlog/src/wide.rs b/crates/lance-graph-xlog/src/wide.rs index c03c43f8f..4233ece77 100644 --- a/crates/lance-graph-xlog/src/wide.rs +++ b/crates/lance-graph-xlog/src/wide.rs @@ -23,7 +23,7 @@ //! retired source is recomputed, and every other row is kept bit for bit. //! Insertions are monotone, so they are merged exactly by seeding. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use ndarray::simd::{mask_andnot, mask_any, mask_or_assign}; @@ -471,32 +471,49 @@ fn lanes(keys: impl Iterator) -> (Vec, Vec) { .unzip() } -/// The edge set after applying the log in append (= version) order; the -/// newest op per `(src, dst)` wins. -fn fold_keys(base: &BTreeSet, log: &DeltaLog) -> BTreeSet { - let mut eff = base.clone(); +/// The edge multiset after applying the log in append (= version) order: +/// occurrences per `(src, dst)` in the base (duplicates kept), ADD = +1, +/// RETIRE = -1 floored at 0 (retiring an absent edge is a no-op). Entries +/// with count 0 are removed. +fn fold_counts(base_src: &[u32], base_dst: &[u32], log: &DeltaLog) -> BTreeMap { + let mut counts: BTreeMap = BTreeMap::new(); + for (&s, &d) in base_src.iter().zip(base_dst) { + *counts.entry(key(s, d)).or_insert(0) += 1; + } for ((&s, &d), &op) in log.src.iter().zip(&log.dst).zip(&log.op) { + let k = key(s, d); if op == 1 { - eff.insert(key(s, d)); - } else { - eff.remove(&key(s, d)); + *counts.entry(k).or_insert(0) += 1; + } else if let Some(c) = counts.get_mut(&k) { + *c -= 1; + if *c == 0 { + counts.remove(&k); + } } } - eff + counts } -/// Folds `base` and the log into the effective edge set, as sorted `src` / -/// `dst` lanes. This is the explicit re-fold: the caller decides when the -/// resident base is replaced by its result and the log emptied; nothing -/// triggers it automatically. Lanes of unequal length are zipped to the -/// shorter. +/// The set of edges present (count > 0) in the folded multiset. +fn fold_keys(base_src: &[u32], base_dst: &[u32], log: &DeltaLog) -> BTreeSet { + fold_counts(base_src, base_dst, log).into_keys().collect() +} + +/// Folds `base` and the log into the effective edge multiset, as `src` / +/// `dst` lanes: ascending `(src, dst)`, each edge repeated once per +/// occurrence. A call graph is a bag, so duplicates are preserved (two call +/// sites stay two). The log is applied in append order: ADD is +1, RETIRE is +/// -1 floored at 0 (retiring an absent edge is a no-op). This is the explicit +/// re-fold: the caller decides when the resident base is replaced by its +/// result and the log emptied; nothing triggers it automatically. Lanes of +/// unequal length are zipped to the shorter. pub fn refold(base_src: &[u32], base_dst: &[u32], log: &DeltaLog) -> (Vec, Vec) { - let base: BTreeSet = base_src - .iter() - .zip(base_dst) - .map(|(&s, &d)| key(s, d)) - .collect(); - lanes(fold_keys(&base, log).into_iter()) + let counts = fold_counts(base_src, base_dst, log); + lanes( + counts + .into_iter() + .flat_map(|(k, c)| std::iter::repeat_n(k, c as usize)), + ) } /// Copies `rows` into a builder of `n` rows (`n >= rows.n()`), padding with @@ -563,7 +580,10 @@ pub fn refresh( .zip(base_dst) .map(|(&s, &d)| key(s, d)) .collect(); - let eff = fold_keys(&base, log); + // Set-level view of the multiset fold: an edge is present iff its count + // is > 0. A count dropping 2 -> 1 retires nothing; only a drop to 0 + // retires, only a rise from 0 adds. + let eff = fold_keys(base_src, base_dst, log); let retired: Vec = base.difference(&eff).copied().collect(); let added: Vec = eff.difference(&base).copied().collect(); @@ -670,6 +690,26 @@ pub fn refresh( mod tests { use super::*; + /// The edge log is a multiset: duplicates in the base survive, retire + /// removes one occurrence, over-retire floors at zero, add-then-retire + /// cancels. + #[test] + fn refold_is_a_multiset() { + let (bs, bd) = (vec![1, 1], vec![2, 2]); + let mut log = DeltaLog::new(); + assert_eq!(refold(&bs, &bd, &log), (vec![1, 1], vec![2, 2])); + log.retire(1, 2); + assert_eq!(refold(&bs, &bd, &log), (vec![1], vec![2])); + log.retire(1, 2); + assert_eq!(refold(&bs, &bd, &log), (Vec::new(), Vec::new())); + log.retire(1, 2); // absent: no-op, floored at 0 + assert_eq!(refold(&bs, &bd, &log), (Vec::new(), Vec::new())); + let mut log = DeltaLog::new(); + log.add(5, 6); + log.retire(5, 6); + assert_eq!(refold(&bs, &bd, &log), (vec![1, 1], vec![2, 2])); + } + /// Naive reachability by DFS from every node, written independently of /// the kernel. fn naive(n: usize, edges: &[(u32, u32)]) -> Vec> { From 03f7798a68bb6ede7758ea4961381f0770d4eb6b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 07:36:56 +0000 Subject: [PATCH 09/19] codegraph: store_timing measures refresh_crate and the row read Measured on mask-risc's store (17,155 rows), release: refresh_crate of an unchanged crate 24,589-36,259 us, of which reading the rows file is 14,078-19,708 us. A millisecond result is a failed gate: the reader builds one String per field (about 120,000 allocations) and refresh re-interns and diffs through string-keyed maps. Recorded here before the fix. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- .../examples/store_timing.rs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/crates/lance-graph-codegraph/examples/store_timing.rs b/crates/lance-graph-codegraph/examples/store_timing.rs index b0fbd6edb..e2291a606 100644 --- a/crates/lance-graph-codegraph/examples/store_timing.rs +++ b/crates/lance-graph-codegraph/examples/store_timing.rs @@ -69,6 +69,33 @@ fn main() { time("callers", &mut || { black_box(g.callers(probe).expect("callers")); }); + // Refresh every crate in the store from its own unchanged file: the + // whole intake-side diff runs and finds nothing to log. + let crates: Vec = std::fs::read_dir(&dir) + .expect("store dir") + .filter_map(|e| e.ok()) + .filter_map(|e| { + e.file_name() + .to_str() + .and_then(|n| n.strip_suffix(".rows.tsv")) + .map(str::to_string) + }) + .collect(); + for k in &crates { + let t = Instant::now(); + let lanes = lance_graph_codegraph::artifact::read_crate(&dir, k).expect("read"); + println!( + " of which reading {k}.rows.tsv ({} rows): {} us", + lanes.predicate.len(), + t.elapsed().as_micros() + ); + let t = Instant::now(); + let stats = g.refresh_crate(&dir, k).expect("refresh"); + println!( + "refresh_crate {k} (unchanged): {} us; {stats:?}", + t.elapsed().as_micros() + ); + } let i = g.impact(probe).expect("impact"); let pop = |m: &[u64]| m.iter().map(|w| w.count_ones()).sum::(); println!( From d5b64799da19f4c011d6c3bf40ae8f04dcd9d6d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 08:02:50 +0000 Subject: [PATCH 10/19] codegraph: interned id lanes, digest-skip for unchanged crates, id-based references Rows are stored as u32 ids into one interner plus a predicate code; parsing allocates only for names not seen before. refresh_crate skips a crate whose rows file digest is unchanged (no delta) and diffs changed crates with sorted u64 keys. references compares ids, not strings. Measured on mask-risc's store (17,155 rows), release: reading the rows file 14-20 ms -> 6.8-10.3 ms; refresh of an unchanged crate 25-36 ms -> 1.5-1.7 ms; references 122 us -> 45 us; impact 286 ns. Still a failed gate: the remaining cost is reading and hashing the 2.5 MB text file, which only a format change on the ruff_charon side removes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- .../examples/store_timing.rs | 5 +- crates/lance-graph-codegraph/src/artifact.rs | 419 ++++++++++++-- crates/lance-graph-codegraph/src/diag.rs | 26 +- crates/lance-graph-codegraph/src/graph.rs | 514 ++++++++++++------ crates/lance-graph-codegraph/src/query.rs | 27 +- 5 files changed, 747 insertions(+), 244 deletions(-) diff --git a/crates/lance-graph-codegraph/examples/store_timing.rs b/crates/lance-graph-codegraph/examples/store_timing.rs index e2291a606..3f3b6e92e 100644 --- a/crates/lance-graph-codegraph/examples/store_timing.rs +++ b/crates/lance-graph-codegraph/examples/store_timing.rs @@ -83,10 +83,11 @@ fn main() { .collect(); for k in &crates { let t = Instant::now(); - let lanes = lance_graph_codegraph::artifact::read_crate(&dir, k).expect("read"); + let mut syms = lance_graph_codegraph::artifact::Symbols::default(); + let lanes = lance_graph_codegraph::artifact::read_crate(&dir, k, &mut syms).expect("read"); println!( " of which reading {k}.rows.tsv ({} rows): {} us", - lanes.predicate.len(), + lanes.len(), t.elapsed().as_micros() ); let t = Instant::now(); diff --git a/crates/lance-graph-codegraph/src/artifact.rs b/crates/lance-graph-codegraph/src/artifact.rs index 0dfc88f14..c9a02d33a 100644 --- a/crates/lance-graph-codegraph/src/artifact.rs +++ b/crates/lance-graph-codegraph/src/artifact.rs @@ -1,10 +1,19 @@ //! The reader for `*.rows.tsv` artifacts (version 2). //! -//! Lines stream straight into column lanes: one push per field per line, no -//! per-row struct. Columns are located by name from the `#schema` line, so a -//! file with permuted columns reads identically. +//! Lines stream straight into integer lanes: one push per field per line, no +//! per-row struct and no per-field `String`. Every distinct text (symbol, +//! grade, file, crate name) is stored once in a shared [`Symbols`] interner +//! and a row holds only `u32` ids, so reading a 17k-row file allocates for +//! the distinct names only, not ~120k times (one `String` per field per +//! row). Millisecond reads were the symptom; the workspace rule is that +//! answers come from folds over integer lanes, never from owned per-row text. +//! +//! Columns are located by name from the `#schema` line, so a file with +//! permuted columns reads identically. +use std::collections::HashMap; use std::fs; +use std::hash::{BuildHasherDefault, Hasher}; use std::path::Path; use crate::graph::IntakeRefusal; @@ -26,48 +35,267 @@ const REQUIRED: [&str; 7] = [ /// Marks an absent (`-`) span bound in [`Lanes::start`] / [`Lanes::end`]. pub const NO_SPAN: u32 = u32::MAX; -/// All rows of a directory as parallel column lanes; index `i` of every lane -/// is row `i`. Invariant: all lanes have equal length. +/// "No id" in an id lane (for example [`Lanes::may_target`] of a non-MAY row). +pub const NO_ID: u32 = u32::MAX; + +/// Predicate code of `DEFINES`. +pub const P_DEFINES: u8 = 0; +/// Predicate code of `CALLS_RESOLVED`. +pub const P_CALLS: u8 = 1; +/// Predicate code of `MAY_DISPATCH`. +pub const P_MAY: u8 = 2; +/// Predicate code of `UNRESOLVED_CALL`. +pub const P_UNRESOLVED: u8 = 3; +/// Predicate code of `PUBLIC`. +pub const P_PUBLIC: u8 = 4; +/// Predicate code of `IMPLEMENTS`. +pub const P_IMPLEMENTS: u8 = 5; +/// Predicate code of `SYNTHETIC`. +pub const P_SYNTHETIC: u8 = 6; +/// Predicate code of `REFERENCES`. +pub const P_REFERENCES: u8 = 7; +/// Predicate code of `CRATE`. +pub const P_CRATE: u8 = 8; +/// Any other predicate; its text is kept in [`Lanes`]'s side table so nothing +/// is lost. +pub const P_OTHER: u8 = 9; + +/// Canonical text of each predicate code below [`P_OTHER`]. +const PRED_NAMES: [&str; 9] = [ + "DEFINES", + "CALLS_RESOLVED", + "MAY_DISPATCH", + "UNRESOLVED_CALL", + "PUBLIC", + "IMPLEMENTS", + "SYNTHETIC", + "REFERENCES", + "CRATE", +]; + +/// The code of a predicate text. +pub fn pred_code(s: &str) -> u8 { + match s { + "DEFINES" => P_DEFINES, + "CALLS_RESOLVED" => P_CALLS, + "MAY_DISPATCH" => P_MAY, + "UNRESOLVED_CALL" => P_UNRESOLVED, + "PUBLIC" => P_PUBLIC, + "IMPLEMENTS" => P_IMPLEMENTS, + "SYNTHETIC" => P_SYNTHETIC, + "REFERENCES" => P_REFERENCES, + "CRATE" => P_CRATE, + _ => P_OTHER, + } +} + +/// The per-crate file digest. Equal digests mean the file is unchanged, which +/// is no delta, so a refresh can skip it entirely. It folds eight bytes per +/// step (the same mix as [`FastHasher`]) with the length in the last step: +/// byte-at-a-time FNV-1a over a 2.5 MB rows file was most of an unchanged +/// refresh's cost. A change detector, not a cryptographic hash. +pub fn digest(bytes: &[u8]) -> u64 { + let mut h = FastHasher(0xcbf2_9ce4_8422_2325); + h.write(bytes); + h.write_u64(bytes.len() as u64); + h.finish() +} + +/// A multiply-rotate hasher folding eight bytes per step. The interner is +/// keyed by short symbol paths and is hit about once per row; SipHash made +/// those lookups the dominant cost. Not DoS-resistant, which is fine for a +/// local artifact reader. +#[derive(Default)] +pub struct FastHasher(u64); + +const K: u64 = 0x517c_c1b7_2722_0a95; + +impl Hasher for FastHasher { + fn write(&mut self, bytes: &[u8]) { + let (chunks, rest) = bytes.as_chunks::<8>(); + for &w in chunks { + self.0 = (self.0.rotate_left(5) ^ u64::from_le_bytes(w)).wrapping_mul(K); + } + if !rest.is_empty() { + let mut w = [0u8; 8]; + w[..rest.len()].copy_from_slice(rest); + // The length is folded in so "a" and "a\0" differ. + let tail = u64::from_le_bytes(w) ^ ((rest.len() as u64) << 56); + self.0 = (self.0.rotate_left(5) ^ tail).wrapping_mul(K); + } + } + + fn write_u8(&mut self, b: u8) { + self.0 = (self.0.rotate_left(5) ^ u64::from(b)).wrapping_mul(K); + } + + fn finish(&self) -> u64 { + // hashbrown reads the low bits for the slot and the top bits for the + // control byte; a multiply leaves the low bits weak. + self.0.rotate_left(26) + } +} + +/// Build-hasher for the crate's interner maps. +pub type FastBuild = BuildHasherDefault; + +/// One string interner shared by the whole graph: ids are dense `u32`s in +/// first-seen order. A new name is the only allocation (two small boxes). +#[derive(Debug, Default)] +pub struct Symbols { + names: Vec>, + index: HashMap, u32, FastBuild>, +} + +impl Symbols { + /// The id of `s`, interning it on first sight. + pub fn intern(&mut self, s: &str) -> u32 { + if let Some(&id) = self.index.get(s) { + return id; + } + let id = self.names.len() as u32; + self.names.push(s.into()); + self.index.insert(s.into(), id); + id + } + + /// The id of `s` if it was interned. + pub fn get(&self, s: &str) -> Option { + self.index.get(s).copied() + } + + /// The text of `id`; empty for an id never issued. + pub fn name(&self, id: u32) -> &str { + self.names.get(id as usize).map_or("", |b| &**b) + } + + /// Distinct names interned. + pub fn len(&self) -> usize { + self.names.len() + } + + /// Whether nothing is interned. + pub fn is_empty(&self) -> bool { + self.names.is_empty() + } +} + +/// All rows of a directory as parallel integer lanes; index `i` of every lane +/// is row `i`. Invariant: all per-row lanes have equal length. Texts are ids +/// into the [`Symbols`] the rows were read with. #[derive(Debug, Default)] pub struct Lanes { - /// Relation name of each row. - pub predicate: Vec, - /// Subject symbol of each row. - pub subject: Vec, - /// Object text of each row. - pub object: Vec, - /// Grade (provenance) of each row. - pub grade: Vec, - /// Source file of each row, `-` when none. - pub file: Vec, + /// Predicate code of each row (`P_*`). + pub pred: Vec, + /// Subject text id of each row. + pub subject: Vec, + /// Object text id of each row, verbatim from the artifact. + pub object: Vec, + /// Grade (provenance) text id of each row. + pub grade: Vec, + /// Source file text id of each row, the id of `-` when none. + pub file: Vec, /// Span start, [`NO_SPAN`] when `-`. pub start: Vec, /// Span end, [`NO_SPAN`] when `-`. pub end: Vec, - /// The crate file (`.rows.tsv` stem) each row was read from. - pub krate: Vec, + /// The crate (`.rows.tsv` stem) text id each row was read from. + pub krate: Vec, + /// For `MAY_DISPATCH` rows the id of the `Trait::method` part of the + /// object (before " via "); [`NO_ID`] for every other row. Precomputed so + /// no query splits a string. + pub may_target: Vec, + /// `(row, predicate text id)` of each [`P_OTHER`] row, ascending by row. + other_pred: Vec<(u32, u32)>, + /// `(crate text id, FNV-1a 64 of the file it was read from)`. + digests: Vec<(u32, u64)>, } impl Lanes { - /// A copy holding only the rows not read from `krate`, in order. - pub fn without_crate(&self, krate: &str) -> Lanes { - let mut out = Lanes::default(); - for i in (0..self.predicate.len()).filter(|&i| self.krate[i] != krate) { - out.predicate.push(self.predicate[i].clone()); - out.subject.push(self.subject[i].clone()); - out.object.push(self.object[i].clone()); - out.grade.push(self.grade[i].clone()); - out.file.push(self.file[i].clone()); - out.start.push(self.start[i]); - out.end.push(self.end[i]); - out.krate.push(self.krate[i].clone()); + /// Rows held. + pub fn len(&self) -> usize { + self.pred.len() + } + + /// Whether no rows are held. + pub fn is_empty(&self) -> bool { + self.pred.is_empty() + } + + /// The predicate text of `row`; the original text for [`P_OTHER`] rows. + pub fn predicate_name<'a>(&self, row: usize, symbols: &'a Symbols) -> &'a str { + let code = self.pred[row]; + if let Some(n) = PRED_NAMES.get(code as usize) { + return n; + } + u32::try_from(row) + .ok() + .and_then(|r| self.other_pred.binary_search_by_key(&r, |&(r, _)| r).ok()) + .map_or("", |k| symbols.name(self.other_pred[k].1)) + } + + /// The recorded file digest of `krate`, if its file was read. + pub fn digest_of(&self, krate: u32) -> Option { + self.digests.iter().find(|d| d.0 == krate).map(|d| d.1) + } + + fn set_digest(&mut self, krate: u32, d: u64) { + match self.digests.iter_mut().find(|e| e.0 == krate) { + Some(e) => e.1 = d, + None => self.digests.push((krate, d)), + } + } + + /// Drop every row read from `krate`, in place and keeping the order of + /// the rest. The crate's digest is kept until [`Lanes::append`] replaces it. + pub fn drop_crate(&mut self, krate: u32) { + let n = self.pred.len(); + let mut w = 0usize; + let mut op = 0usize; + let mut other = Vec::with_capacity(self.other_pred.len()); + for i in 0..n { + let is_other = self.pred[i] == P_OTHER; + let entry = if is_other { + let e = self.other_pred.get(op).copied(); + op += 1; + e + } else { + None + }; + if self.krate[i] == krate { + continue; + } + if let Some((_, id)) = entry { + other.push((w as u32, id)); + } + self.pred[w] = self.pred[i]; + self.subject[w] = self.subject[i]; + self.object[w] = self.object[i]; + self.grade[w] = self.grade[i]; + self.file[w] = self.file[i]; + self.start[w] = self.start[i]; + self.end[w] = self.end[i]; + self.krate[w] = self.krate[i]; + self.may_target[w] = self.may_target[i]; + w += 1; } - out + self.pred.truncate(w); + self.subject.truncate(w); + self.object.truncate(w); + self.grade.truncate(w); + self.file.truncate(w); + self.start.truncate(w); + self.end.truncate(w); + self.krate.truncate(w); + self.may_target.truncate(w); + self.other_pred = other; } - /// Append every row of `other` after this lane set's rows. + /// Append every row of `other` after this lane set's rows, and adopt its + /// file digests. pub fn append(&mut self, mut other: Lanes) { - self.predicate.append(&mut other.predicate); + let base = self.pred.len() as u32; + self.pred.append(&mut other.pred); self.subject.append(&mut other.subject); self.object.append(&mut other.object); self.grade.append(&mut other.grade); @@ -75,30 +303,40 @@ impl Lanes { self.start.append(&mut other.start); self.end.append(&mut other.end); self.krate.append(&mut other.krate); + self.may_target.append(&mut other.may_target); + self.other_pred + .extend(other.other_pred.iter().map(|&(r, id)| (r + base, id))); + for (k, d) in other.digests { + self.set_digest(k, d); + } } } const SUFFIX: &str = ".rows.tsv"; +/// The text of the single file `.rows.tsv` of `dir`. +pub fn read_crate_text(dir: &Path, krate: &str) -> Result { + fs::read_to_string(dir.join(format!("{krate}{SUFFIX}"))).map_err(IntakeRefusal::Io) +} + /// Read the single file `.rows.tsv` of `dir`; its rows carry `krate`. -pub fn read_crate(dir: &Path, krate: &str) -> Result { - let text = - fs::read_to_string(dir.join(format!("{krate}{SUFFIX}"))).map_err(IntakeRefusal::Io)?; +pub fn read_crate(dir: &Path, krate: &str, symbols: &mut Symbols) -> Result { + let text = read_crate_text(dir, krate)?; let mut lanes = Lanes::default(); - read_text_as(&text, krate, &mut lanes)?; + read_text_as(&text, krate, symbols, &mut lanes)?; Ok(lanes) } /// Read every `*.rows.tsv` in `dir`, sorted by file name so the row order is /// deterministic. -pub fn read_dir(dir: &Path) -> Result { +pub fn read_dir(dir: &Path, symbols: &mut Symbols) -> Result { let mut paths = Vec::new(); for entry in fs::read_dir(dir).map_err(IntakeRefusal::Io)? { let path = entry.map_err(IntakeRefusal::Io)?.path(); let is_rows = path .file_name() .and_then(|n| n.to_str()) - .is_some_and(|n| n.ends_with(".rows.tsv")); + .is_some_and(|n| n.ends_with(SUFFIX)); if is_rows { paths.push(path); } @@ -112,7 +350,7 @@ pub fn read_dir(dir: &Path) -> Result { .and_then(|n| n.to_str()) .and_then(|n| n.strip_suffix(SUFFIX)) .unwrap_or(""); - read_text_as(&text, stem, &mut lanes)?; + read_text_as(&text, stem, symbols, &mut lanes)?; } Ok(lanes) } @@ -125,13 +363,61 @@ fn number(field: &str) -> Option { } } +/// The `Trait::method` part of a MAY_DISPATCH object. +pub fn dispatch_target(object: &str) -> &str { + object.find(" via ").map_or(object, |i| &object[..i]) +} + +/// A tiny recency cache in front of the interner for fields that repeat from +/// row to row (a function's rows are adjacent, a file's rows are adjacent, +/// grades take a handful of values): a hit costs a short compare instead of a +/// hash. A miss interns and stores round-robin. +struct Memo<'a, const N: usize> { + slots: [(&'a str, u32); N], + filled: usize, + next: usize, +} + +impl<'a, const N: usize> Memo<'a, N> { + fn new() -> Self { + Self { + slots: [("", 0); N], + filled: 0, + next: 0, + } + } + + fn id(&mut self, s: &'a str, sym: &mut Symbols) -> u32 { + for &(t, id) in &self.slots[..self.filled] { + if t == s { + return id; + } + } + let id = sym.intern(s); + self.slots[self.next] = (s, id); + self.next = (self.next + 1) % N; + self.filled = (self.filled + 1).min(N); + id + } +} + /// Append one file's rows to `lanes`, with an empty crate name. -pub fn read_text(text: &str, lanes: &mut Lanes) -> Result<(), IntakeRefusal> { - read_text_as(text, "", lanes) +pub fn read_text( + text: &str, + symbols: &mut Symbols, + lanes: &mut Lanes, +) -> Result<(), IntakeRefusal> { + read_text_as(text, "", symbols, lanes) } -/// Append one file's rows to `lanes`, attributing each to `krate`. -pub fn read_text_as(text: &str, krate: &str, lanes: &mut Lanes) -> Result<(), IntakeRefusal> { +/// Append one file's rows to `lanes`, attributing each to `krate`, and record +/// the file's digest under `krate`. +pub fn read_text_as( + text: &str, + krate: &str, + symbols: &mut Symbols, + lanes: &mut Lanes, +) -> Result<(), IntakeRefusal> { let mut lines = text.lines(); let found = lines .next() @@ -157,26 +443,53 @@ pub fn read_text_as(text: &str, krate: &str, lanes: &mut Lanes) -> Result<(), In missing: want.to_string(), })?; } + let kid = symbols.intern(krate); + let mut fields: Vec<&str> = Vec::with_capacity(names.len()); + let mut subjects: Memo<4> = Memo::new(); + let mut files: Memo<2> = Memo::new(); + let mut grades: Memo<8> = Memo::new(); for (i, line) in lines.enumerate() { // Lines 1 and 2 were the header; data starts at line 3. let lineno = i + 3; if line.is_empty() { continue; } - let f: Vec<&str> = line.split('\t').collect(); - if f.len() != names.len() { + fields.clear(); + fields.extend(line.split('\t')); + if fields.len() != names.len() { return Err(IntakeRefusal::Arity { line: lineno }); } - let start = number(f[at[5]]).ok_or(IntakeRefusal::Arity { line: lineno })?; - let end = number(f[at[6]]).ok_or(IntakeRefusal::Arity { line: lineno })?; - lanes.predicate.push(f[at[0]].to_string()); - lanes.subject.push(f[at[1]].to_string()); - lanes.object.push(f[at[2]].to_string()); - lanes.grade.push(f[at[3]].to_string()); - lanes.file.push(f[at[4]].to_string()); + let start = number(fields[at[5]]).ok_or(IntakeRefusal::Arity { line: lineno })?; + let end = number(fields[at[6]]).ok_or(IntakeRefusal::Arity { line: lineno })?; + let pred_text = fields[at[0]]; + let code = pred_code(pred_text); + let object_text = fields[at[2]]; + let subject = subjects.id(fields[at[1]], symbols); + let object = symbols.intern(object_text); + let may = if code == P_MAY { + let t = dispatch_target(object_text); + if t.len() == object_text.len() { + object + } else { + symbols.intern(t) + } + } else { + NO_ID + }; + if code == P_OTHER { + let id = symbols.intern(pred_text); + lanes.other_pred.push((lanes.pred.len() as u32, id)); + } + lanes.pred.push(code); + lanes.subject.push(subject); + lanes.object.push(object); + lanes.grade.push(grades.id(fields[at[3]], symbols)); + lanes.file.push(files.id(fields[at[4]], symbols)); lanes.start.push(start); lanes.end.push(end); - lanes.krate.push(krate.to_string()); + lanes.krate.push(kid); + lanes.may_target.push(may); } + lanes.set_digest(kid, digest(text.as_bytes())); Ok(()) } diff --git a/crates/lance-graph-codegraph/src/diag.rs b/crates/lance-graph-codegraph/src/diag.rs index f41249c31..fc996c077 100644 --- a/crates/lance-graph-codegraph/src/diag.rs +++ b/crates/lance-graph-codegraph/src/diag.rs @@ -9,7 +9,8 @@ use ndarray::simd::mask_andnot; use ndarray::simd::mask_andnot_assign; -use crate::graph::{Codegraph, RunStatus}; +use crate::artifact::{P_CALLS, P_UNRESOLVED}; +use crate::graph::Codegraph; /// Which diagnostic a [`Finding`] belongs to. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -118,24 +119,21 @@ impl Codegraph { let words = self.words(); let row_words = self.rows().div_ceil(64); - // One lane scan per diagnostic, into masks (not a tuple decode): - // CRATE rows for the run status, UNRESOLVED_CALL rows, and the - // CALLS_RESOLVED rows that dangle. - let mut partial = false; + // One scan over the integer lanes, into masks (not a tuple decode): + // UNRESOLVED_CALL rows and the CALLS_RESOLVED rows that dangle. The + // run status is the CRATE map (any non-COMPLETE crate), which is what + // a per-CRATE-row lookup of the same map computed. + let l = self.lanes(); + let partial = self.any_partial(); let mut unresolved_rows = vec![0u64; row_words]; let mut dangling_rows = vec![0u64; row_words]; let mut dangling_callers = vec![0u64; words]; for r in 0..self.rows() { - match self.row_predicate(r) { - "CRATE" => { - if self.crate_status(self.row_subject(r)) == Some(RunStatus::Partial) { - partial = true; - } - } - "UNRESOLVED_CALL" => set_bit(&mut unresolved_rows, r), - "CALLS_RESOLVED" => { + match l.pred[r] { + P_UNRESOLVED => set_bit(&mut unresolved_rows, r), + P_CALLS => { let (Some(caller), Some(callee)) = - (self.pos(self.row_subject(r)), self.pos(self.row_object(r))) + (self.pos_of_id(l.subject[r]), self.pos_of_id(l.object[r])) else { continue; }; diff --git a/crates/lance-graph-codegraph/src/graph.rs b/crates/lance-graph-codegraph/src/graph.rs index 8ae95bdc7..9e00d54dc 100644 --- a/crates/lance-graph-codegraph/src/graph.rs +++ b/crates/lance-graph-codegraph/src/graph.rs @@ -10,7 +10,10 @@ use lance_graph_dt64::dr::{Axis, LocalMap, MapRefusal, MapTable, TileId, Width}; use lance_graph_xlog::cube::{Cube, CubeRefusal, RelId}; use lance_graph_xlog::wide::{WideRows, WideStats}; -use crate::artifact::{self, Lanes, NO_SPAN}; +use crate::artifact::{ + self, Lanes, Symbols, NO_SPAN, P_CALLS, P_CRATE, P_DEFINES, P_IMPLEMENTS, P_MAY, P_PUBLIC, + P_REFERENCES, P_SYNTHETIC, P_UNRESOLVED, +}; /// A symbol's position: its interning index, which is also its cube coordinate. pub type Pos = u32; @@ -83,13 +86,88 @@ struct Masks { stored_owner: Vec, } -/// The queryable codegraph. Owns the artifact lanes, the symbol table, the -/// tile map and the cube; relations were dropped after the cube took them. +/// Symbol positions over the shared interner. A position is the symbol's +/// first-seen index among the row symbols (the cube coordinate); the interner +/// also holds texts that are not symbols (grades, files, full MAY objects), +/// so the two numberings are kept apart: `of_id` maps an interner id to its +/// position (`NONE` when the text is not a symbol), `id_of` is the inverse. +/// Query time does one interner lookup and one array read, never a name map. +#[derive(Default)] +struct Positions { + of_id: Vec, + id_of: Vec, + /// Interner id of the first `::` segment of each position's name (the + /// owning crate), computed once when the position is assigned so the + /// `stored_owner` mask never splits strings. + owner: Vec, + last_owner: u32, +} + +impl Positions { + fn len(&self) -> usize { + self.id_of.len() + } + + /// The position of interner id `id`, `None` when it is not a symbol. + fn at(&self, id: u32) -> Option { + self.of_id.get(id as usize).copied().filter(|&p| p != NONE) + } + + /// Assign `id` the next position unless it already has one. + fn assign(&mut self, id: u32, symbols: &mut Symbols) -> Pos { + if self.of_id.len() <= id as usize { + self.of_id.resize(symbols.len().max(id as usize + 1), NONE); + } + let cur = self.of_id[id as usize]; + if cur != NONE { + return cur; + } + let p = self.id_of.len() as Pos; + self.of_id[id as usize] = p; + self.id_of.push(id); + let owner = { + let name = symbols.name(id); + match name.find("::") { + None => id, + Some(cut) => { + let pre = &name[..cut]; + if self.last_owner != NONE && symbols.name(self.last_owner) == pre { + self.last_owner + } else if let Some(o) = symbols.get(pre) { + o + } else { + let owned = pre.to_string(); + symbols.intern(&owned) + } + } + } + }; + self.last_owner = owner; + self.owner.push(owner); + p + } + + /// Undo every assignment from position `n` on (a refused refresh). + fn truncate(&mut self, n: usize) { + for &id in &self.id_of[n..] { + if let Some(slot) = self.of_id.get_mut(id as usize) { + *slot = NONE; + } + } + self.id_of.truncate(n); + self.owner.truncate(n); + } +} + +/// The queryable codegraph. Owns the artifact lanes, the symbol interner and +/// position table, the tile map and the cube; relations were dropped after +/// the cube took them. pub struct Codegraph { lanes: Lanes, - names: Vec, - index: HashMap, - crates: HashMap, + symbols: Symbols, + pos: Positions, + /// Status by crate-name interner id. + crates: HashMap, /// The reservation made at open; refresh grows only inside it. axis: Axis, table: MapTable, @@ -99,101 +177,103 @@ pub struct Codegraph { rposs: OnceCell<(WideRows, WideStats)>, } -fn intern(names: &mut Vec, index: &mut HashMap, s: &str) -> Pos { - if let Some(&p) = index.get(s) { - return p; - } - let p = names.len() as Pos; - names.push(s.to_string()); - index.insert(s.to_string(), p); - p -} - -/// The `Trait::method` part of a MAY_DISPATCH object. -fn dispatch_target(object: &str) -> &str { - object.split(" via ").next().unwrap_or(object) -} - fn set(words: &mut [u64], p: Pos) { - words[p as usize / 64] |= 1 << (p % 64); -} - -/// Row subject/object ids after interning; `NONE` when a row has no symbol. -struct Resolved { - sid: Vec, - oid: Vec, - dyn_id: Option, + if let Some(w) = words.get_mut(p as usize / 64) { + *w |= 1 << (p % 64); + } } -/// Intern every row's symbols into `names`/`idx` in row order. Symbols already -/// interned keep their position, so running this over a lane set that gained -/// or lost rows only ever appends. -fn resolve(lanes: &Lanes, names: &mut Vec, idx: &mut HashMap) -> Resolved { - let rows = lanes.predicate.len(); - let mut sid = vec![NONE; rows]; - let mut oid = vec![NONE; rows]; +/// Assign positions to the symbols of `rows` in row order. Symbols already +/// positioned keep theirs, so running this over a lane set that gained rows +/// only ever appends. Returns whether a row had an `UNRESOLVED_CALL` (the +/// caller then adds the `?dynamic` position last). +fn resolve( + lanes: &Lanes, + rows: std::ops::Range, + symbols: &mut Symbols, + pos: &mut Positions, +) -> bool { let mut any_dynamic = false; - for i in 0..rows { - let (s, o) = (lanes.subject[i].as_str(), lanes.object[i].as_str()); - match lanes.predicate[i].as_str() { - "DEFINES" | "CALLS_RESOLVED" | "PUBLIC" | "IMPLEMENTS" | "SYNTHETIC" | "REFERENCES" => { - sid[i] = intern(names, idx, s); - oid[i] = intern(names, idx, o); + for i in rows { + let s = lanes.subject[i]; + match lanes.pred[i] { + P_DEFINES | P_CALLS | P_PUBLIC | P_IMPLEMENTS | P_SYNTHETIC | P_REFERENCES => { + pos.assign(s, symbols); + pos.assign(lanes.object[i], symbols); } - "MAY_DISPATCH" => { - sid[i] = intern(names, idx, s); - oid[i] = intern(names, idx, dispatch_target(o)); + P_MAY => { + pos.assign(s, symbols); + pos.assign(lanes.may_target[i], symbols); } // The caller is a symbol; the object ("dynamic") is not. - "UNRESOLVED_CALL" => { - sid[i] = intern(names, idx, s); + P_UNRESOLVED => { + pos.assign(s, symbols); any_dynamic = true; } _ => {} } } - let dyn_id = any_dynamic.then(|| intern(names, idx, DYNAMIC)); - Resolved { sid, oid, dyn_id } + any_dynamic +} + +/// Add the `?dynamic` position (idempotent). +fn add_dynamic(symbols: &mut Symbols, pos: &mut Positions) { + let d = symbols.intern(DYNAMIC); + pos.assign(d, symbols); +} + +fn dynamic_pos(symbols: &Symbols, pos: &Positions) -> Option { + symbols.get(DYNAMIC).and_then(|id| pos.at(id)) } -fn crate_statuses(lanes: &Lanes) -> HashMap { +fn crate_statuses(lanes: &Lanes, symbols: &Symbols) -> HashMap { + let complete = symbols.get("COMPLETE"); let mut crates = HashMap::new(); - for i in 0..lanes.predicate.len() { - if lanes.predicate[i] == "CRATE" { - let st = if lanes.object[i] == "COMPLETE" { + for i in 0..lanes.len() { + if lanes.pred[i] == P_CRATE { + let st = if Some(lanes.object[i]) == complete { RunStatus::Complete } else { RunStatus::Partial }; - crates.insert(lanes.subject[i].clone(), st); + crates.insert(lanes.subject[i], st); } } crates } -/// The five edge relations (everything but `NODE`, which is left empty) of a -/// lane set. Intake and refresh both build edges here, so they agree. -fn edges_of(lanes: &Lanes, r: &Resolved) -> [Vec<(u32, u32)>; 6] { +/// The five edge relations (everything but `NODE`, which is left empty) of +/// the given rows, in row order. Intake and refresh both build edges here, so +/// they agree. Every symbol of these rows must already be positioned. +fn edges_of( + lanes: &Lanes, + rows: impl Iterator, + pos: &Positions, + dyn_pos: Option, +) -> [Vec<(u32, u32)>; 6] { let mut rel: [Vec<(u32, u32)>; 6] = Default::default(); - for i in 0..lanes.predicate.len() { - let (s, o) = (r.sid[i], r.oid[i]); - match lanes.predicate[i].as_str() { - "CALLS_RESOLVED" => { + let p = |id: u32| pos.at(id).unwrap_or(NONE); + for i in rows { + match lanes.pred[i] { + P_CALLS => { + let (s, o) = (p(lanes.subject[i]), p(lanes.object[i])); rel[CALLS as usize].push((s, o)); rel[RCONF as usize].push((o, s)); rel[RPOSS as usize].push((o, s)); } - "IMPLEMENTS" => { + P_IMPLEMENTS => { + let (s, o) = (p(lanes.subject[i]), p(lanes.object[i])); rel[IMPL as usize].push((s, o)); rel[RPOSS as usize].push((s, o)); } - "MAY_DISPATCH" => { + P_MAY => { + let (s, o) = (p(lanes.subject[i]), p(lanes.may_target[i])); rel[MAY as usize].push((s, o)); rel[RPOSS as usize].push((o, s)); } - "UNRESOLVED_CALL" => { - if let Some(d) = r.dyn_id { - rel[RPOSS as usize].push((d, s)); + P_UNRESOLVED => { + if let Some(d) = dyn_pos { + rel[RPOSS as usize].push((d, p(lanes.subject[i]))); } } _ => {} @@ -202,14 +282,14 @@ fn edges_of(lanes: &Lanes, r: &Resolved) -> [Vec<(u32, u32)>; 6] { rel } -/// Position masks of a lane set over `names`. +/// Position masks of a lane set. fn masks_of( lanes: &Lanes, - r: &Resolved, - names: &[String], - crates: &HashMap, + pos: &Positions, + n_ids: usize, + crates: &HashMap, ) -> Masks { - let words = names.len().div_ceil(64); + let words = pos.len().div_ceil(64); let mut m = Masks { fns: vec![0; words], public: vec![0; words], @@ -221,29 +301,88 @@ fn masks_of( dynamic_callers: vec![0; words], stored_owner: vec![0; words], }; - for i in 0..lanes.predicate.len() { - let (s, o) = (r.sid[i], r.oid[i]); - match lanes.predicate[i].as_str() { - "DEFINES" => set(&mut m.fns, s), - "PUBLIC" => set(&mut m.public, s), - "SYNTHETIC" => set(&mut m.synthetic, s), - "REFERENCES" => set(&mut m.referenced, o), - "CALLS_RESOLVED" => set(&mut m.called, o), - "IMPLEMENTS" => set(&mut m.impl_subjects, s), - "MAY_DISPATCH" => set(&mut m.may_targets, o), - "UNRESOLVED_CALL" => set(&mut m.dynamic_callers, s), + let p = |id: u32| pos.at(id).unwrap_or(NONE); + for i in 0..lanes.len() { + let (s, o) = (lanes.subject[i], lanes.object[i]); + match lanes.pred[i] { + P_DEFINES => set(&mut m.fns, p(s)), + P_PUBLIC => set(&mut m.public, p(s)), + P_SYNTHETIC => set(&mut m.synthetic, p(s)), + P_REFERENCES => set(&mut m.referenced, p(o)), + P_CALLS => set(&mut m.called, p(o)), + P_IMPLEMENTS => set(&mut m.impl_subjects, p(s)), + P_MAY => set(&mut m.may_targets, p(lanes.may_target[i])), + P_UNRESOLVED => set(&mut m.dynamic_callers, p(s)), _ => {} } } - for (p, name) in names.iter().enumerate() { - let owner = name.split("::").next().unwrap_or(name); - if crates.contains_key(owner) { - set(&mut m.stored_owner, p as Pos); + let mut is_crate = vec![false; n_ids]; + for &c in crates.keys() { + if let Some(f) = is_crate.get_mut(c as usize) { + *f = true; + } + } + for (q, &owner) in pos.owner.iter().enumerate() { + if is_crate.get(owner as usize).copied().unwrap_or(false) { + set(&mut m.stored_owner, q as Pos); } } m } +/// Sorted edge keys `(src << 32) | dst` of one relation. +fn keys(edges: &[(u32, u32)]) -> Vec { + let mut k: Vec = edges + .iter() + .map(|&(s, d)| (u64::from(s) << 32) | u64::from(d)) + .collect(); + k.sort_unstable(); + k +} + +fn run_len(v: &[u64], i: usize) -> usize { + let x = v[i]; + v[i..].iter().take_while(|&&y| y == x).count() +} + +/// Multiset difference of two sorted key lists in one merge pass: each key +/// is logged `old - new` times into `gone` or `new - old` times into `came`, +/// ascending. A call graph is a bag: two rows for one `(src, dst)` are two +/// call sites. +fn diff_sorted(old: &[u64], new: &[u64], gone: &mut Vec, came: &mut Vec) { + let (mut i, mut j) = (0usize, 0usize); + while i < old.len() || j < new.len() { + let order = match (old.get(i), new.get(j)) { + (Some(a), Some(b)) => a.cmp(b), + (Some(_), None) => std::cmp::Ordering::Less, + _ => std::cmp::Ordering::Greater, + }; + match order { + std::cmp::Ordering::Equal => { + let (ro, rn) = (run_len(old, i), run_len(new, j)); + let key = old[i]; + if ro > rn { + gone.extend(std::iter::repeat_n(key, ro - rn)); + } else { + came.extend(std::iter::repeat_n(key, rn - ro)); + } + i += ro; + j += rn; + } + std::cmp::Ordering::Less => { + let r = run_len(old, i); + gone.extend(std::iter::repeat_n(old[i], r)); + i += r; + } + std::cmp::Ordering::Greater => { + let r = run_len(new, j); + came.extend(std::iter::repeat_n(new[j], r)); + j += r; + } + } + } +} + /// What one [`Codegraph::refresh_crate`] did. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct RefreshStats { @@ -275,15 +414,20 @@ pub enum RefreshRefusal { impl Codegraph { /// Read every `*.rows.tsv` in `dir` and build the graph. pub fn open(dir: &Path) -> Result { - let lanes = artifact::read_dir(dir)?; - let mut names: Vec = Vec::new(); - let mut idx: HashMap = HashMap::new(); - let res = resolve(&lanes, &mut names, &mut idx); - let crates = crate_statuses(&lanes); - let n = names.len(); - let mut rel = edges_of(&lanes, &res); + let mut symbols = Symbols::default(); + let lanes = artifact::read_dir(dir, &mut symbols)?; + let mut pos = Positions { + last_owner: NONE, + ..Positions::default() + }; + if resolve(&lanes, 0..lanes.len(), &mut symbols, &mut pos) { + add_dynamic(&mut symbols, &mut pos); + } + let crates = crate_statuses(&lanes, &symbols); + let n = pos.len(); + let mut rel = edges_of(&lanes, 0..lanes.len(), &pos, dynamic_pos(&symbols, &pos)); rel[NODE as usize] = (0..n as u32).map(|p| (p, p)).collect(); - let m = masks_of(&lanes, &res, &names, &crates); + let m = masks_of(&lanes, &pos, symbols.len(), &crates); // Tile t holds the global ids [t*256, min(n, (t+1)*256)). Every tile // but the last is full, so tile t's local j sits at cube coordinate @@ -318,8 +462,8 @@ impl Codegraph { Ok(Self { lanes, - names, - index: idx, + symbols, + pos, crates, axis, table, @@ -333,9 +477,16 @@ impl Codegraph { /// Re-read `.rows.tsv` from `dir` and bring the resident graph up /// to date without rebuilding it ("sparse write, merge on read"). /// + /// * **Digest skip.** The FNV-1a digest of every crate file is recorded + /// when it is read (at open, at each successful refresh). A file whose + /// digest equals the recorded one is unchanged, which is no delta: + /// the call returns `RefreshStats::default()` at once, without parsing + /// and without moving the crate's rows. (Before this, an unchanged + /// crate cost 25-36 ms to find nothing.) /// * Rows are attributed to their crate file by the `krate` lane /// ([`Lanes::krate`]); the crate's old rows are dropped from the lanes - /// and the new file's rows appended. + /// and the new file's rows appended. Only that one file is parsed, into + /// the graph's shared interner (allocating for new names only). /// * Positions never renumber. A vanished symbol keeps its position (and /// its `NODE` edge) with no rows; new symbols are appended. Growth must /// fit the positions reserved at open (`Axis::for_survivors(n).reserved()`): @@ -343,10 +494,13 @@ impl Codegraph { /// [`RefreshRefusal::NeedsReopen`] and changes nothing. Growth inside /// the reservation extends the last tile's map under a new frame /// (`MapTable::advance` + `seal`) and `Cube::rebase`s onto it. - /// * Edges: the whole graph's edges before and after are compared per - /// relation as multisets (a call graph is a bag: two call sites of one - /// pair are two edges), so an edge a *different* crate still produces - /// is never retired and a second call site is never lost. The difference is + /// * Edges: the crate's old edges and its new edges are compared per + /// relation as multisets of sorted `(src << 32) | dst` keys in one merge + /// pass (a call graph is a bag: two call sites of one pair are two + /// edges). Every other crate's rows are in both the old and the new + /// graph, so this equals the whole-graph difference: an edge a + /// *different* crate still produces is never retired and a second call + /// site is never lost. No string is hashed or compared. The difference is /// logged with `retire_edge` / `add_edge`; the resident lanes are /// untouched. **Apply order** (so the next tile/row is predictable): /// relation by relation in id order; within a relation, retirements @@ -369,57 +523,63 @@ impl Codegraph { dir: &Path, krate: &str, ) -> Result { - let fresh = artifact::read_crate(dir, krate).map_err(RefreshRefusal::Intake)?; - let mut new_lanes = self.lanes.without_crate(krate); - new_lanes.append(fresh); - - let mut names = self.names.clone(); - let mut idx = self.index.clone(); - let old_res = resolve(&self.lanes, &mut names, &mut idx); - let new_res = resolve(&new_lanes, &mut names, &mut idx); - let (n_old, n_new) = (self.names.len(), names.len()); + let text = artifact::read_crate_text(dir, krate).map_err(RefreshRefusal::Intake)?; + // An unchanged file is no delta. The digest recorded when the crate + // was last read (at open or at a successful refresh) decides, before + // any parsing, so the common "nothing happened" refresh costs one + // read and one pass over the bytes. + let file_digest = artifact::digest(text.as_bytes()); + if let Some(k) = self.symbols.get(krate) { + if self.lanes.digest_of(k) == Some(file_digest) { + return Ok(RefreshStats::default()); + } + } + let mut fresh = Lanes::default(); + artifact::read_text_as(&text, krate, &mut self.symbols, &mut fresh) + .map_err(RefreshRefusal::Intake)?; + let kid = self.symbols.intern(krate); + + // Edges of the crate's old rows, before any new position exists. Every + // other crate's rows are in both the old and the new lane set, so the + // whole-graph multiset difference equals this crate's old rows against + // its new rows. + let old_dyn = dynamic_pos(&self.symbols, &self.pos); + let old_e = edges_of( + &self.lanes, + (0..self.lanes.len()).filter(|&i| self.lanes.krate[i] == kid), + &self.pos, + old_dyn, + ); + + // New symbols come only from the fresh rows (all other rows are + // already positioned), appended in fresh-row order, `?dynamic` last: + // the same order a rebuilt lane set would intern in. + let n_old = self.pos.len(); + if resolve(&fresh, 0..fresh.len(), &mut self.symbols, &mut self.pos) { + add_dynamic(&mut self.symbols, &mut self.pos); + } + let n_new = self.pos.len(); if n_new > self.axis.reserved() { + self.pos.truncate(n_old); return Err(RefreshRefusal::NeedsReopen { reason: "new symbols exceed the positions reserved at open (a new tile, or a W64 tile becoming W256)", }); } - let old_e = edges_of(&self.lanes, &old_res); - let new_e = edges_of(&new_lanes, &new_res); - let mut retires: Vec> = Vec::new(); - let mut adds: Vec> = Vec::new(); + let new_e = edges_of( + &fresh, + 0..fresh.len(), + &self.pos, + dynamic_pos(&self.symbols, &self.pos), + ); + let mut retires: Vec> = Vec::new(); + let mut adds: Vec> = Vec::new(); for r in 0..6 { - // Multiset diff: a call graph is a bag, two rows for one - // (src, dst) are two call sites. BTreeMap iterates ascending. - let mut o: std::collections::BTreeMap<(u32, u32), usize> = - std::collections::BTreeMap::new(); - let mut n: std::collections::BTreeMap<(u32, u32), usize> = - std::collections::BTreeMap::new(); - for &e in &old_e[r] { - *o.entry(e).or_insert(0) += 1; - } - for &e in &new_e[r] { - *n.entry(e).or_insert(0) += 1; - } - let mut gone: Vec<(u32, u32)> = Vec::new(); - let mut came: Vec<(u32, u32)> = Vec::new(); - for (&e, &oc) in &o { - let nc = n.get(&e).copied().unwrap_or(0); - for _ in nc..oc { - gone.push(e); - } - } - for (&e, &nc) in &n { - let oc = o.get(&e).copied().unwrap_or(0); - for _ in oc..nc { - came.push(e); - } - } + let (mut gone, mut came) = (Vec::new(), Vec::new()); + diff_sorted(&keys(&old_e[r]), &keys(&new_e[r]), &mut gone, &mut came); retires.push(gone); adds.push(came); } - let crates = crate_statuses(&new_lanes); - let masks = masks_of(&new_lanes, &new_res, &names, &crates); let pending = [ self.cube.delta_len(RCONF) > 0, @@ -435,8 +595,13 @@ impl Codegraph { let last = self.axis.tiles.saturating_sub(1); let lo = last as u64 * TILE_POSITIONS as u64; let ids: Vec = (lo..n_new as u64).collect(); - let map = LocalMap::prepare(self.axis.tile, &ids) - .map_err(|e| RefreshRefusal::Intake(IntakeRefusal::Map(e)))?; + let map = match LocalMap::prepare(self.axis.tile, &ids) { + Ok(m) => m, + Err(e) => { + self.pos.truncate(n_old); + return Err(RefreshRefusal::Intake(IntakeRefusal::Map(e))); + } + }; let rev = self.table.revisions(); self.table .advance(rev.source + 1, rev.relation_map, vec![(last, Some(map))]); @@ -456,13 +621,15 @@ impl Codegraph { } continue; } - for &(s, d) in &retires[r] { + for &key in &retires[r] { + let (s, d) = ((key >> 32) as u32, key as u32); self.cube .retire_edge(&self.table, rel, s, d) .map_err(RefreshRefusal::Cube)?; stats.edges_retired += 1; } - for &(s, d) in &adds[r] { + for &key in &adds[r] { + let (s, d) = ((key >> 32) as u32, key as u32); self.cube .add_edge(&self.table, rel, s, d) .map_err(RefreshRefusal::Cube)?; @@ -492,11 +659,12 @@ impl Codegraph { } } - self.lanes = new_lanes; - self.names = names; - self.index = idx; - self.crates = crates; - self.masks = masks; + // Drop the crate's old rows, then append the fresh ones (the crate + // moves to the end of the lanes) and adopt the new file digest. + self.lanes.drop_crate(kid); + self.lanes.append(fresh); + self.crates = crate_statuses(&self.lanes, &self.symbols); + self.masks = masks_of(&self.lanes, &self.pos, self.symbols.len(), &self.crates); Ok(stats) } @@ -510,19 +678,40 @@ impl Codegraph { Ok(()) } - /// The position of `symbol`, if interned. + /// The position of `symbol`, if it is a symbol of the graph: one interner + /// lookup, then one array read. pub fn pos(&self, symbol: &str) -> Option { - self.index.get(symbol).copied() + self.symbols.get(symbol).and_then(|id| self.pos.at(id)) } /// The symbol at position `p`. pub fn symbol(&self, p: Pos) -> Option<&str> { - self.names.get(p as usize).map(String::as_str) + self.id_of(p).map(|id| self.symbols.name(id)) + } + + /// Interner id of the symbol at position `p`. + pub(crate) fn id_of(&self, p: Pos) -> Option { + self.pos.id_of.get(p as usize).copied() + } + + /// Position of interner id `id`, when that text is a symbol. + pub(crate) fn pos_of_id(&self, id: u32) -> Option { + self.pos.at(id) + } + + /// The artifact lanes, for the crate's own folds. + pub(crate) fn lanes(&self) -> &Lanes { + &self.lanes + } + + /// Whether any stored crate's CRATE row is not `COMPLETE`. + pub(crate) fn any_partial(&self) -> bool { + self.crates.values().any(|&s| s == RunStatus::Partial) } /// Mask words per position mask: `n.div_ceil(64)`. pub fn words(&self) -> usize { - self.names.len().div_ceil(64) + self.pos.len().div_ceil(64) } /// A mask with the bits of `symbols` set; unknown symbols are ignored. @@ -538,40 +727,41 @@ impl Codegraph { /// The extractor status of `krate`, from its CRATE row. pub fn crate_status(&self, krate: &str) -> Option { - self.crates.get(krate).copied() + let id = self.symbols.get(krate)?; + self.crates.get(&id).copied() } /// Data rows read. pub fn rows(&self) -> usize { - self.lanes.predicate.len() + self.lanes.len() } /// Predicate of `row`; panics only on an out-of-range row, like indexing. pub fn row_predicate(&self, row: usize) -> &str { - &self.lanes.predicate[row] + self.lanes.predicate_name(row, &self.symbols) } /// Subject of `row`. pub fn row_subject(&self, row: usize) -> &str { - &self.lanes.subject[row] + self.symbols.name(self.lanes.subject[row]) } /// Object of `row`, verbatim from the artifact. pub fn row_object(&self, row: usize) -> &str { - &self.lanes.object[row] + self.symbols.name(self.lanes.object[row]) } /// `(file, start, end)` of `row`, `None` when the row has no site (`-`). pub fn site(&self, row: usize) -> Option<(&str, u32, u32)> { let (f, s, e) = ( - self.lanes.file.get(row)?, + self.symbols.name(*self.lanes.file.get(row)?), self.lanes.start[row], self.lanes.end[row], ); if f == "-" || s == NO_SPAN || e == NO_SPAN { None } else { - Some((f.as_str(), s, e)) + Some((f, s, e)) } } diff --git a/crates/lance-graph-codegraph/src/query.rs b/crates/lance-graph-codegraph/src/query.rs index 1498c4ab5..145ad58cc 100644 --- a/crates/lance-graph-codegraph/src/query.rs +++ b/crates/lance-graph-codegraph/src/query.rs @@ -7,12 +7,11 @@ //! (`UNRESOLVED_CALL`, the `?dynamic` position) or a function-pointer //! reference could reach it; it is never reported as "unaffected". -use std::collections::HashSet; - use lance_graph_xlog::cube::{CubeRefusal, RelId}; use lance_graph_xlog::wide::WideRows; use ndarray::simd::{mask_and, mask_andnot_assign, mask_or_assign}; +use crate::artifact::{P_CALLS, P_IMPLEMENTS, P_MAY, P_REFERENCES}; use crate::graph::{Codegraph, Pos, CALLS, IMPL, MAY, NODE, RCONF, RPOSS}; /// An answer in three grades, each a position mask of `words()` words. @@ -190,28 +189,30 @@ impl Codegraph { let rows = self.rows(); let mut mask = vec![0u64; rows.div_ceil(64)]; let (mut confirmed, mut possible) = (0u64, 0u64); - let Some(name) = self.symbol(g) else { + let Some(gid) = self.id_of(g) else { return Sites { rows: mask, confirmed, possible, }; }; - // The Trait::method nodes `g` implements, read from the same lanes. - let impls: HashSet<&str> = (0..rows) - .filter(|&i| self.row_predicate(i) == "IMPLEMENTS" && self.row_subject(i) == name) - .map(|i| self.row_object(i)) + // One fold over the integer lanes, no string compare: the + // `Trait::method` node ids `g` implements come from IMPLEMENTS rows by + // id, and each MAY_DISPATCH row carries its precomputed target id. + let l = self.lanes(); + let mut impls: Vec = (0..rows) + .filter(|&i| l.pred[i] == P_IMPLEMENTS && l.subject[i] == gid) + .map(|i| l.object[i]) .collect(); + impls.sort_unstable(); + impls.dedup(); for i in 0..rows { - let object = self.row_object(i); - let hit = match self.row_predicate(i) { - "CALLS_RESOLVED" | "REFERENCES" if object == name => { + let hit = match l.pred[i] { + P_CALLS | P_REFERENCES if l.object[i] == gid => { confirmed += 1; true } - "MAY_DISPATCH" - if impls.contains(object.split(" via ").next().unwrap_or(object)) => - { + P_MAY if impls.binary_search(&l.may_target[i]).is_ok() => { possible += 1; true } From b2f45f151c394dada13c05c57972a8701ec8b674 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 08:41:34 +0000 Subject: [PATCH 11/19] dt64: versioned, append-only DR index; xlog and codegraph read through it Operator ruling: versioned, replayable immutability is the DR global -> local index itself; reading an older version is the same lookup with the version as part of the key. dt64::dr: each tile's mapping is an append-only log keyed by RouteVersion. An extension logs a longer length (no copy), a replacement adds a generation, a removal logs an event; nothing is updated in place. MapTable and RouteLog share one RouteVersion sequence through a VersionClock. map(frame, tile) reads the layout the frame was sealed at; map_at(version, tile) reads any version by number; a frame from another table or a future version is refused. Reads return a borrowed TileMap that respects the version's length and allocate nothing. xlog: Cube::rebase drops its private per-tile snapshot and reads the old layout through the index; a cube keeps answering after the table advances. Re-pinned to the new rule (old assertion kept as a comment): dt64 tests/dr.rs (two), xlog tests/cube.rs, xlog-relcomp-probe an_old_map_frame_replays_its_own_layout (method admission is still gated by the bind store's route versions, not the tile map). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/lance-graph-dt64/src/dr.rs | 412 +++++++++++++++--- crates/lance-graph-dt64/tests/dr.rs | 266 ++++++++++- crates/lance-graph-xlog/src/cube.rs | 50 +-- crates/lance-graph-xlog/src/lib.rs | 62 ++- crates/lance-graph-xlog/tests/cube.rs | 37 +- .../lance-graph-xlog/tests/versioned_map.rs | 106 +++++ crates/lance-graph-xlog/tests/wide_closure.rs | 26 +- .../xlog-relcomp-probe/tests/dto_binding.rs | 32 +- 8 files changed, 846 insertions(+), 145 deletions(-) create mode 100644 crates/lance-graph-xlog/tests/versioned_map.rs diff --git a/crates/lance-graph-dt64/src/dr.rs b/crates/lance-graph-dt64/src/dr.rs index e3036e0c5..a2f330361 100644 --- a/crates/lance-graph-dt64/src/dr.rs +++ b/crates/lance-graph-dt64/src/dr.rs @@ -21,6 +21,7 @@ use std::cell::Cell; use std::collections::{BTreeMap, BTreeSet}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Weak}; use lance_graph_contract::class_view::ClassId; @@ -154,6 +155,38 @@ pub const MAX_RESERVED: usize = 1 << 16; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct RouteVersion(pub u64); +/// The one counter behind [`RouteVersion`]. A [`RouteLog`], a [`FrameStore`] +/// and a [`MapTable`] built from clones of the same clock advance ONE +/// sequence: a version then names a single point of the DR log, and a frame's +/// version answers `RouteLog::routed_at` as well as `MapTable::map_at`. +/// +/// `clone` SHARES the counter (that is how two structures are joined); +/// [`VersionClock::fork`] copies its present value into an independent one. +/// The counter only ever increases; nothing rewinds it. +#[derive(Debug, Default, Clone)] +pub struct VersionClock(Arc); + +impl VersionClock { + pub fn new() -> VersionClock { + VersionClock::default() + } + + /// The last version issued (`RouteVersion(0)` before any). + pub fn current(&self) -> RouteVersion { + RouteVersion(self.0.load(Ordering::SeqCst)) + } + + /// Issue the next version. + pub fn tick(&self) -> RouteVersion { + RouteVersion(self.0.fetch_add(1, Ordering::SeqCst) + 1) + } + + /// An independent clock starting at this one's present value. + pub fn fork(&self) -> VersionClock { + VersionClock(Arc::new(AtomicU64::new(self.0.load(Ordering::SeqCst)))) + } +} + /// Why the route log refused. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RouteRefusal { @@ -173,12 +206,23 @@ pub enum RouteRefusal { /// DR64 direct routing: the append-only log from a DR position to the /// classid bound there. Only routing is versioned (D-DT64-6 §3); what a /// classid means is the binding side's, and does not change. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Default)] pub struct RouteLog { /// Per position, the route events in version order: `(version, class)`, /// `None` for a retract. routes: BTreeMap)>>, - version: RouteVersion, + clock: VersionClock, +} + +/// A cloned log gets a forked clock: copying a log must not make the copy's +/// routes advance the original's version sequence. +impl Clone for RouteLog { + fn clone(&self) -> RouteLog { + RouteLog { + routes: self.routes.clone(), + clock: self.clock.fork(), + } + } } impl RouteLog { @@ -186,9 +230,23 @@ impl RouteLog { RouteLog::default() } - /// The current end of the log. + /// A log that advances `clock`, shared with whoever else holds a clone of + /// it (typically a [`MapTable`]). + pub fn with_clock(clock: VersionClock) -> RouteLog { + RouteLog { + routes: BTreeMap::new(), + clock, + } + } + + /// The clock this log advances (a clone that shares the counter). + pub fn clock(&self) -> VersionClock { + self.clock.clone() + } + + /// The current end of the (possibly shared) version sequence. pub fn current_version(&self) -> RouteVersion { - self.version + self.clock.current() } /// Route `position` to `class`, whose identity reservation is `identity`; @@ -217,12 +275,12 @@ impl RouteLog { } fn push(&mut self, position: u16, class: Option) -> RouteVersion { - self.version = RouteVersion(self.version.0 + 1); + let version = self.clock.tick(); self.routes .entry(position) .or_default() - .push((self.version, class)); - self.version + .push((version, class)); + version } /// The classid `position` routed to at `version`, `None` if it was @@ -232,10 +290,11 @@ impl RouteLog { version: RouteVersion, position: u16, ) -> Result, RouteRefusal> { - if version > self.version { + let current = self.clock.current(); + if version > current { return Err(RouteRefusal::FutureVersion { asked: version, - current: self.version, + current, }); } let Some(events) = self.routes.get(&position) else { @@ -312,28 +371,60 @@ pub enum FrameRefusal { pub struct FrameStore { identity: Arc<()>, current: Arc, + clock: VersionClock, + /// The version the current seal was issued at. + version: RouteVersion, } impl FrameStore { pub fn new(revisions: FrameRevisions) -> Self { + Self::with_clock(revisions, VersionClock::new()) + } + + /// A store whose advances tick `clock`. + pub fn with_clock(revisions: FrameRevisions, clock: VersionClock) -> Self { + let version = clock.current(); Self { identity: Arc::new(()), current: Arc::new(revisions), + clock, + version, } } - /// Seal the current frame. + /// Seal the current frame, stamped with the version it was issued at. pub fn seal(&self) -> Frame { Frame { store: Arc::downgrade(&self.identity), seal: Arc::downgrade(&self.current), at: *self.current, + version: self.version, } } - /// Move to new revisions; every frame sealed before is stale from now on. - pub fn advance(&mut self, revisions: FrameRevisions) { + /// Move to new revisions and a new version; `check` refuses every frame + /// sealed before (a version-keyed reader such as [`MapTable::map`] still + /// answers them, at their own version). Returns the new version. + pub fn advance(&mut self, revisions: FrameRevisions) -> RouteVersion { + let version = self.clock.tick(); + self.advance_at(revisions, version); + version + } + + fn advance_at(&mut self, revisions: FrameRevisions, version: RouteVersion) { self.current = Arc::new(revisions); + self.version = version; + } + + /// The version `frame` was issued at, if this store issued it. A frame of + /// another store is `ForeignFrame` even at equal revisions and versions. + /// Unlike [`FrameStore::check`] this does not require the frame to be + /// current: it is the key for reading history. + pub fn issued(&self, frame: &Frame) -> Result { + if !Weak::ptr_eq(&frame.store, &Arc::downgrade(&self.identity)) { + return Err(FrameRefusal::ForeignFrame); + } + Ok(frame.version) } pub fn revisions(&self) -> FrameRevisions { @@ -371,9 +462,15 @@ pub struct Frame { store: Weak<()>, seal: Weak, at: FrameRevisions, + version: RouteVersion, } impl Frame { + /// The version this frame was issued at: the key it reads history under. + pub fn version(&self) -> RouteVersion { + self.version + } + /// The revisions this frame was sealed at (for reporting, never identity). pub fn revisions(&self) -> FrameRevisions { self.at @@ -517,15 +614,20 @@ impl LocalMap { /// as its `coordinates` revision. A digest, not an identity: two maps /// over the same ids in the same order are the same assignment. pub fn digest(&self) -> u64 { - let mut h: u64 = 0xcbf2_9ce4_8422_2325; - for &g in &self.globals { - for b in g.to_le_bytes() { - h ^= u64::from(b); - h = h.wrapping_mul(0x0000_0100_0000_01b3); - } + fnv_ids(&self.globals) + } +} + +/// FNV-1a over a local → global table, in local order. +fn fnv_ids(globals: &[u64]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for &g in globals { + for b in g.to_le_bytes() { + h ^= u64::from(b); + h = h.wrapping_mul(0x0000_0100_0000_01b3); } - h } + h } /// A tile's name in a [`MapTable`], chosen by the caller. @@ -534,45 +636,205 @@ pub type TileId = u32; /// Why a [`MapTable`] read was refused. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MapRead { - /// The frame is not the table's current seal. + /// The frame was not issued by this table's store (`ForeignFrame`). A + /// frame of THIS table is never stale: it reads the layout it was sealed + /// under. Stale(FrameRefusal), - /// The current frame holds no map for this tile. + /// The table holds no map for this tile at the frame's (or asked) version. NoSuchTile(TileId), + /// A numeric version past the end of the shared version sequence. + FutureVersion { + asked: RouteVersion, + current: RouteVersion, + }, +} + +/// A tile's history: append-only. Nothing here is updated in place or +/// compacted, so any past version stays answerable. +/// +/// - an EXTENSION (the new map keeps every old position and only appends) is a +/// longer length at a new version over the SAME generation: the generation +/// is replaced by its superset (the old prefix is bit-identical inside it, +/// so no information is lost) and no copy is kept; +/// - a REPLACEMENT (reorder, removal of an id, width change, or a re-add after +/// a removal) is a new generation; +/// - a REMOVAL is an event to `None`. +#[derive(Debug, Default)] +struct TileLog { + gens: Vec, + /// `(version, Some((generation, len)))`, `None` = removed; version order. + events: Vec<(RouteVersion, Option<(usize, usize)>)>, +} + +impl TileLog { + fn append(&mut self, version: RouteVersion, m: LocalMap) { + let len = m.len(); + let live = match self.events.last() { + Some((_, Some((g, _)))) => Some(*g), + _ => None, + }; + if let Some(g) = live { + if let Some(slot) = self.gens.get_mut(g) { + if m.extends(slot) { + m.lookups.set(slot.lookups.get()); + *slot = m; + self.events.push((version, Some((g, len)))); + return; + } + } + } + let g = self.gens.len(); + self.gens.push(m); + self.events.push((version, Some((g, len)))); + } + + fn remove(&mut self, version: RouteVersion) { + if matches!(self.events.last(), Some((_, Some(_)))) { + self.events.push((version, None)); + } + } + + /// The layout in force at `version`: the last event at or before it. + fn at(&self, version: RouteVersion) -> Option> { + let seen = self.events.partition_point(|(v, _)| *v <= version); + let (g, len) = self.events.get(seen.checked_sub(1)?)?.1?; + Some(TileMap { + map: self.gens.get(g)?, + len, + }) + } +} + +/// A borrowed view of one tile's map at one version: a generation plus a +/// length. It owns no ids and allocates nothing. `local` and `global` respect +/// `len`, so an id appended after the version is absent at it. +#[derive(Clone, Copy, Debug)] +pub struct TileMap<'t> { + map: &'t LocalMap, + len: usize, } -/// The versioned global → local table (D-DT64-7 O5): every tile's -/// [`LocalMap`] under one [`FrameStore`]. Maps change only through -/// [`MapTable::advance`], which sets the `coordinates` revision from the maps -/// it holds, and a map is read only under the frame it was sealed under. +impl<'t> TileMap<'t> { + pub fn width(&self) -> Width { + self.map.width + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// global → local. Counted (on the underlying generation). A position at + /// or past `len` was appended later, so it is `Unmapped` here. + pub fn local(&self, global: u64) -> Result { + let local = self.map.local(global)?; + if usize::from(local) < self.len { + Ok(local) + } else { + Err(MapRefusal::Unmapped) + } + } + + /// local → global, `None` at or past `len`. + pub fn global(&self, local: u8) -> Option { + if usize::from(local) < self.len { + self.map.global(local) + } else { + None + } + } + + /// The local → global table at this version, in local order. + pub fn globals(&self) -> &'t [u64] { + self.map.globals.get(..self.len).unwrap_or(&[]) + } + + /// How many lookups the underlying generation has served. + pub fn lookups(&self) -> u64 { + self.map.lookups() + } + + /// FNV-1a over the local → global table at this version. + pub fn digest(&self) -> u64 { + fnv_ids(self.globals()) + } + + /// True iff `older` is a pure prefix of `self`: same width, and every + /// position of `older` keeps its id here, with `self` only appending. + pub fn extends(&self, older: TileMap<'_>) -> bool { + self.map.width == older.map.width + && self.len >= older.len + && self.globals().get(..older.len) == Some(older.globals()) + } +} + +/// The versioned global → local table (D-DT64-7 O5): every tile's history of +/// [`LocalMap`]s under one [`FrameStore`]. Maps change only through +/// [`MapTable::advance`], which appends: an extension is a length at a +/// version, a replacement a new generation, a removal an event; nothing is +/// updated in place or compacted. A frame carries the version it was sealed +/// at ([`RouteVersion`], one per `advance`, shared with a [`RouteLog`] when +/// they share a [`VersionClock`]) and reads the layout in force then, so an +/// old frame replays the old layout instead of being refused. The +/// `coordinates` revision is still set from the CURRENT state, for reporting +/// and for [`FrameStore::check`]'s currency test; it cannot key history (a +/// digest collides and is not monotone), the version does. #[derive(Debug)] pub struct MapTable { frames: FrameStore, - tiles: BTreeMap, + tiles: BTreeMap, } impl MapTable { - /// An empty table at the given source and relation-map revisions. + /// An empty table at the given source and relation-map revisions, with + /// its own version clock. pub fn new(source: u64, relation_map: u64) -> Self { + Self::with_clock(source, relation_map, VersionClock::new()) + } + + /// An empty table whose advances tick `clock` (shared with a + /// [`RouteLog`] to make both one DR log). + pub fn with_clock(source: u64, relation_map: u64, clock: VersionClock) -> Self { let tiles = BTreeMap::new(); let coordinates = Self::coordinates(&tiles); Self { - frames: FrameStore::new(FrameRevisions { - source, - relation_map, - coordinates, - }), + frames: FrameStore::with_clock( + FrameRevisions { + source, + relation_map, + coordinates, + }, + clock, + ), tiles, } } - /// FNV-1a over `(tile, map digest)` in tile order. - fn coordinates(tiles: &BTreeMap) -> u64 { + /// The clock this table advances (a clone that shares the counter). + pub fn clock(&self) -> VersionClock { + self.frames.clock.clone() + } + + /// The end of the (possibly shared) version sequence. + pub fn current_version(&self) -> RouteVersion { + self.frames.clock.current() + } + + /// FNV-1a over `(tile, map digest)` in tile order, over the CURRENT state + /// (removed tiles are absent). + fn coordinates(tiles: &BTreeMap) -> u64 { let mut h: u64 = 0xcbf2_9ce4_8422_2325; - for (tile, map) in tiles { + for (tile, log) in tiles { + let Some(view) = log.at(RouteVersion(u64::MAX)) else { + continue; + }; let bytes = tile .to_le_bytes() .into_iter() - .chain(map.digest().to_le_bytes()); + .chain(view.digest().to_le_bytes()); for b in bytes { h ^= u64::from(b); h = h.wrapping_mul(0x0000_0100_0000_01b3); @@ -581,30 +843,39 @@ impl MapTable { h } - /// Insert (`Some`), replace (`Some`) or remove (`None`) tile maps, then - /// re-seal. Every frame sealed before is stale afterwards, even when - /// nothing changed. + /// Insert or replace (`Some`) or remove (`None`) tile maps, then re-seal + /// at a new version. Past versions are untouched: `check` still calls + /// every earlier frame non-current, but `map` reads it at its own + /// version. An advance that changes nothing adds no event and no + /// generation, yet still takes a new version (a new, distinct frame). + /// Returns that version. pub fn advance( &mut self, source: u64, relation_map: u64, changes: Vec<(TileId, Option)>, - ) { + ) -> RouteVersion { + let version = self.frames.clock.tick(); for (tile, map) in changes { match map { - Some(m) => { - self.tiles.insert(tile, m); - } + Some(m) => self.tiles.entry(tile).or_default().append(version, m), None => { - self.tiles.remove(&tile); + if let Some(log) = self.tiles.get_mut(&tile) { + log.remove(version); + } } } } - self.frames.advance(FrameRevisions { - source, - relation_map, - coordinates: Self::coordinates(&self.tiles), - }); + let coordinates = Self::coordinates(&self.tiles); + self.frames.advance_at( + FrameRevisions { + source, + relation_map, + coordinates, + }, + version, + ); + version } /// Seal the current frame. @@ -616,10 +887,45 @@ impl MapTable { self.frames.revisions() } - /// The map of `tile`, if `frame` is the current seal. - pub fn map(&self, frame: &Frame, tile: TileId) -> Result<&LocalMap, MapRead> { - self.frames.check(frame).map_err(MapRead::Stale)?; - self.tiles.get(&tile).ok_or(MapRead::NoSuchTile(tile)) + /// The map of `tile` as of `frame`'s version. A frame of this table + /// always resolves (old frames replay their layout); a frame of another + /// table is `Stale(ForeignFrame)`; a tile absent then is `NoSuchTile`. + /// Borrows; allocates nothing. + pub fn map(&self, frame: &Frame, tile: TileId) -> Result, MapRead> { + let version = self.frames.issued(frame).map_err(MapRead::Stale)?; + self.view(version, tile) + } + + /// The map of `tile` as of a numeric `version` (for replay where only the + /// number survives). A version past the end of the sequence is + /// `FutureVersion`. + pub fn map_at(&self, version: RouteVersion, tile: TileId) -> Result, MapRead> { + let current = self.frames.clock.current(); + if version > current { + return Err(MapRead::FutureVersion { + asked: version, + current, + }); + } + self.view(version, tile) + } + + fn view(&self, version: RouteVersion, tile: TileId) -> Result, MapRead> { + self.tiles + .get(&tile) + .and_then(|log| log.at(version)) + .ok_or(MapRead::NoSuchTile(tile)) + } + + /// How many generations `tile` has stored (diagnostic: an extension does + /// not add one). + pub fn generation_count(&self, tile: TileId) -> usize { + self.tiles.get(&tile).map_or(0, |l| l.gens.len()) + } + + /// How many events `tile` has logged (diagnostic). + pub fn event_count(&self, tile: TileId) -> usize { + self.tiles.get(&tile).map_or(0, |l| l.events.len()) } /// Find which of the listed `tiles` holds `global` under `frame`. diff --git a/crates/lance-graph-dt64/tests/dr.rs b/crates/lance-graph-dt64/tests/dr.rs index 3168817ee..8df6081c8 100644 --- a/crates/lance-graph-dt64/tests/dr.rs +++ b/crates/lance-graph-dt64/tests/dr.rs @@ -168,22 +168,29 @@ fn map(ids: &[u64]) -> LocalMap { LocalMap::prepare(Width::W64, ids).unwrap() } +/// RE-PINNED (D-DR-VMAP). Was: an old frame was refused with +/// `Stale(CoordinatesChanged)` after the table advanced. Now the table is an +/// append-only log and a frame carries its version, so the old frame still +/// reads the layout it was sealed under, and the new frame reads the new one. #[test] -fn a_map_is_read_only_under_the_frame_it_was_sealed_under() { +fn a_map_is_read_under_the_version_its_frame_was_sealed_at() { let mut t = MapTable::new(7, 11); t.advance(7, 11, vec![(1, Some(map(&[10, 20, 30])))]); let old = t.seal(); assert_eq!(t.map(&old, 1).unwrap().local(20), Ok(1)); t.advance(7, 11, vec![(1, Some(map(&[30, 20, 10])))]); - assert_eq!( - t.map(&old, 1).unwrap_err(), - MapRead::Stale(FrameRefusal::CoordinatesChanged) - ); + // Old assertion, now false: Stale(CoordinatesChanged). + assert_eq!(t.map(&old, 1).unwrap().local(20), Ok(1)); + assert_eq!(t.map(&old, 1).unwrap().local(30), Ok(2), "old layout"); let now = t.seal(); - assert_eq!(t.map(&now, 1).unwrap().local(30), Ok(0)); + assert_eq!(t.map(&now, 1).unwrap().local(30), Ok(0), "new layout"); assert_eq!(t.map(&now, 2).unwrap_err(), MapRead::NoSuchTile(2)); } +/// RE-PINNED (D-DR-VMAP). Was: the old frame was refused with +/// `Stale(FrameResealed)`. A reseal is still a distinct frame (identity), but +/// it is no longer a refusal: the old frame reads, `FrameStore::check` still +/// calls it non-current. #[test] fn an_advance_that_changes_nothing_still_reseals() { let mut t = MapTable::new(7, 11); @@ -192,10 +199,12 @@ fn an_advance_that_changes_nothing_still_reseals() { let before = t.revisions(); t.advance(7, 11, vec![]); assert_eq!(t.revisions(), before, "nothing moved"); - assert_eq!( - t.map(&old, 1).unwrap_err(), - MapRead::Stale(FrameRefusal::FrameResealed) - ); + let new = t.seal(); + assert!(!old.same_issue(&new), "a reseal is a new frame"); + assert!(new.version() > old.version()); + // Old assertion, now false: Stale(FrameResealed). + assert_eq!(t.map(&old, 1).unwrap().local(20), Ok(1)); + assert_eq!(t.map(&new, 1).unwrap().local(20), Ok(1)); } #[test] @@ -329,3 +338,240 @@ mod locate { assert!(!map(Width::W256, &[1, 2, 3]).extends(&old)); } } + +// ------------------------------------------- D-DR-VMAP falsifiers F1-F6 + +mod vmap { + use super::map; + use lance_graph_dt64::dr::{ + Axis, FrameRefusal, MapRead, MapRefusal, MapTable, RouteLog, RouteVersion, VersionClock, + Width, + }; + use std::alloc::{GlobalAlloc, Layout, System}; + use std::cell::Cell; + + struct Counting; + + thread_local! { + // Const-initialised and drop-free, so touching it inside the + // allocator never allocates. Per thread: parallel tests cannot + // pollute the count. + static ALLOCS: Cell = const { Cell::new(0) }; + } + + fn bump() { + let _ = ALLOCS.try_with(|c| c.set(c.get() + 1)); + } + + // SAFETY: forwards every call unchanged to the system allocator. + unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + bump(); + System.alloc(l) + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + System.dealloc(p, l) + } + unsafe fn realloc(&self, p: *mut u8, l: Layout, n: usize) -> *mut u8 { + bump(); + System.realloc(p, l, n) + } + } + + #[global_allocator] + static A: Counting = Counting; + + /// F1 replay. Fails (disable: `map` ignores `frame.version` and reads the + /// last event) because f1 would then see the v2 layout. + #[test] + fn f1_every_id_replays_under_its_own_version() { + let mut t = MapTable::new(1, 1); + t.advance(1, 1, vec![(1, Some(map(&[10, 20, 30])))]); + let f1 = t.seal(); + t.advance(1, 1, vec![(1, Some(map(&[30, 20, 10])))]); + let f2 = t.seal(); + let v1 = [(10, 0), (20, 1), (30, 2)]; + let v2 = [(10, 2), (20, 1), (30, 0)]; + for (id, l) in v1 { + assert_eq!(t.map(&f1, 1).unwrap().local(id), Ok(l)); + } + for (id, l) in v2 { + assert_eq!(t.map(&f2, 1).unwrap().local(id), Ok(l)); + } + // Anti-vacuity: the two versions disagree on 10 and 30. + assert_ne!(v1[0].1, v2[0].1); + assert_ne!(v1[2].1, v2[2].1); + // The numeric read agrees with the frame read. + assert_eq!(t.map_at(f1.version(), 1).unwrap().local(10), Ok(0)); + assert_eq!(t.map_at(f2.version(), 1).unwrap().local(10), Ok(2)); + } + + /// F2 extension is a length at a version. Fails (disable: drop the + /// `>= len` guard in `TileMap::local`) because 30 would resolve at v1; + /// fails if an extension mints a generation. + #[test] + fn f2_an_extension_is_a_length_not_a_copy() { + let mut t = MapTable::new(1, 1); + t.advance(1, 1, vec![(1, Some(map(&[10, 20])))]); + let f1 = t.seal(); + t.advance(1, 1, vec![(1, Some(map(&[10, 20, 30])))]); + let f2 = t.seal(); + let at1 = t.map(&f1, 1).unwrap(); + assert_eq!(at1.len(), 2); + assert_eq!(at1.local(30), Err(MapRefusal::Unmapped)); + assert_eq!(at1.global(2), None); + assert_eq!(at1.local(20), Ok(1)); + let at2 = t.map(&f2, 1).unwrap(); + assert_eq!(at2.len(), 3); + assert_eq!(at2.local(30), Ok(2)); + assert_eq!(at2.global(2), Some(30)); + assert_eq!(t.generation_count(1), 1, "no generation for an extension"); + assert_eq!(t.event_count(1), 2, "one event per version"); + assert!(at2.extends(at1) && !at1.extends(at2)); + } + + /// F3 history survives a replacement and a removal. Fails (disable: + /// `remove` deletes the tile instead of logging `None`) because f1 and f2 + /// would stop resolving. + #[test] + fn f3_a_removal_is_an_event_not_an_erasure() { + let mut t = MapTable::new(1, 1); + t.advance(1, 1, vec![(1, Some(map(&[10, 20, 30])))]); + let f1 = t.seal(); + t.advance(1, 1, vec![(1, Some(map(&[30, 20, 10])))]); + let f2 = t.seal(); + t.advance(1, 1, vec![(1, None)]); + let f3 = t.seal(); + assert_eq!(t.map(&f1, 1).unwrap().local(10), Ok(0)); + assert_eq!(t.map(&f2, 1).unwrap().local(10), Ok(2)); + assert_eq!(t.map(&f3, 1).unwrap_err(), MapRead::NoSuchTile(1)); + assert_eq!(t.generation_count(1), 2, "replacement kept both layouts"); + // Re-adding after a removal is a new generation, and f3 stays gone. + t.advance(1, 1, vec![(1, Some(map(&[10, 20, 30])))]); + assert_eq!(t.generation_count(1), 3); + assert_eq!(t.map(&f3, 1).unwrap_err(), MapRead::NoSuchTile(1)); + } + + /// F4 a foreign frame is refused even at equal revisions and equal + /// version. Fails (disable: `issued` skips the store identity check) + /// because the foreign frame would read. + #[test] + fn f4_a_foreign_frame_is_refused_at_equal_version() { + let mut a = MapTable::new(1, 1); + let mut b = MapTable::new(1, 1); + a.advance(1, 1, vec![(1, Some(map(&[10])))]); + b.advance(1, 1, vec![(1, Some(map(&[10])))]); + assert_eq!(a.revisions(), b.revisions()); + let fb = b.seal(); + assert_eq!(fb.version(), a.seal().version(), "equal versions"); + assert_eq!( + a.map(&fb, 1).unwrap_err(), + MapRead::Stale(FrameRefusal::ForeignFrame) + ); + assert!(b.map(&fb, 1).is_ok(), "its own table reads it"); + } + + /// F5 a lookup allocates nothing. Fails (disable: `map` returns an owned + /// prefix copy) because the counter would move. + #[test] + fn f5_map_and_local_allocate_nothing() { + let mut t = MapTable::new(1, 1); + t.advance(1, 1, vec![(1, Some(map(&[10, 20, 30])))]); + let f1 = t.seal(); + t.advance(1, 1, vec![(1, Some(map(&[10, 20, 30, 40])))]); + let f2 = t.seal(); + let before = ALLOCS.with(Cell::get); + let mut hits = 0u32; + for i in 0..10_000u64 { + let frame = if i % 2 == 0 { &f1 } else { &f2 }; + if let Ok(view) = t.map(frame, 1) { + if view.local(20 + (i % 2) * 20).is_ok() { + hits += 1; + } + } + } + let after = ALLOCS.with(Cell::get); + assert_eq!(after - before, 0, "lookups allocated"); + // Anti-vacuity: the loop really resolved ids (20 under both, 40 only + // under f2), so a skipped body cannot pass. + assert_eq!(hits, 10_000 / 2 + 10_000 / 2); + // And the counter can move: building a map allocates. + let b2 = ALLOCS.with(Cell::get); + let _ = map(&[1, 2, 3]); + assert!(ALLOCS.with(Cell::get) > b2); + } + + /// F6 silence twin: an advance with no changes adds no event and no + /// generation, yet takes a version. Fails (disable: push an event per + /// tile on every advance) because the counts would move. + #[test] + fn f6_an_empty_advance_adds_a_version_and_no_history() { + let mut t = MapTable::new(1, 1); + t.advance(1, 1, vec![(1, Some(map(&[10, 20]))), (2, Some(map(&[5])))]); + let f_old = t.seal(); + let (g, e) = (t.generation_count(1), t.event_count(1)); + let (g2, e2) = (t.generation_count(2), t.event_count(2)); + let v = t.advance(1, 1, vec![]); + let f_new = t.seal(); + assert_eq!((t.generation_count(1), t.event_count(1)), (g, e)); + assert_eq!((t.generation_count(2), t.event_count(2)), (g2, e2)); + assert_eq!(f_new.version(), v); + assert!(v > f_old.version()); + assert!(!f_old.same_issue(&f_new)); + for tile in [1, 2] { + assert_eq!( + t.map(&f_old, tile).unwrap().digest(), + t.map(&f_new, tile).unwrap().digest() + ); + } + } + + /// Q1: a RouteLog and a MapTable on one clock advance ONE sequence, so a + /// frame's version also answers `routed_at`. Fails if the two keep + /// separate counters (versions would collide / not interleave). + #[test] + fn a_shared_clock_makes_one_version_sequence() { + let clock = VersionClock::new(); + let mut log = RouteLog::with_clock(clock.clone()); + let mut t = MapTable::with_clock(1, 1, clock.clone()); + let id = Axis::new(Width::W64, 1); + let r1 = log.route(3, 9, id).unwrap(); + let v1 = t.advance(1, 1, vec![(1, Some(map(&[10])))]); + let r2 = log.route(3, 8, id).unwrap(); + assert_eq!([r1.0, v1.0, r2.0], [1, 2, 3], "one interleaved sequence"); + let f = t.seal(); + assert_eq!(log.routed_at(f.version(), 3), Ok(Some(9))); + assert_eq!(log.routed_at(r2, 3), Ok(Some(8))); + assert_eq!(log.current_version(), t.current_version()); + // Separate clocks do not interleave (silence twin). + let mut solo = RouteLog::new(); + assert_eq!(solo.route(3, 9, id).unwrap(), RouteVersion(1)); + // A clone of a log forks the clock. + let mut copy = log.clone(); + copy.route(4, 1, id).unwrap(); + assert_eq!(log.current_version(), clock.current()); + assert!(copy.current_version() > log.current_version()); + } + + /// Q2: a version past the end is a typed refusal, not a clamp. + #[test] + fn map_at_refuses_a_version_past_the_end() { + let mut t = MapTable::new(1, 1); + t.advance(1, 1, vec![(1, Some(map(&[10])))]); + let cur = t.current_version(); + assert!(t.map_at(cur, 1).is_ok()); + let future = RouteVersion(cur.0 + 1); + assert_eq!( + t.map_at(future, 1).unwrap_err(), + MapRead::FutureVersion { + asked: future, + current: cur + } + ); + assert_eq!( + t.map_at(RouteVersion(0), 1).unwrap_err(), + MapRead::NoSuchTile(1), + "before the tile existed" + ); + } +} diff --git a/crates/lance-graph-xlog/src/cube.rs b/crates/lance-graph-xlog/src/cube.rs index f85e92839..ed44ed874 100644 --- a/crates/lance-graph-xlog/src/cube.rs +++ b/crates/lance-graph-xlog/src/cube.rs @@ -37,7 +37,7 @@ use std::cell::{Cell, OnceCell}; use std::sync::atomic::{AtomicU64, Ordering}; -use lance_graph_dt64::dr::{Frame, LocalMap, LocateRefusal, MapRead, MapRefusal, MapTable, TileId}; +use lance_graph_dt64::dr::{Frame, LocateRefusal, MapRead, MapRefusal, MapTable, TileId, TileMap}; use lance_graph_mask_risc::{LaneRef, Out}; use lance_graph_quack::{Agg, Cmp, Col, Filter, GroupAddr, GroupAgg, Mask, Query}; use ndarray::simd::{checked_binary_i64, mask_and, mask_andnot, mask_any, mask_or_assign, I64Op}; @@ -191,11 +191,6 @@ pub struct Cube { /// Built by [`Cube::new_tiled`]: lanes hold wide coordinates /// (`tile_ordinal * 256 + local`) instead of one tile's locals. tiled: bool, - /// The maps as they were when the cube was built or last rebased, one per - /// entry of `tiles`. A [`MapTable`] drops a map when it advances, so the - /// old assignment is gone by the time [`Cube::rebase`] must compare it; - /// the cube keeps its own copy of the ids (a few bytes per position). - snapshots: Vec, n: usize, rels: Vec, /// One append-only delta log per relation, parallel to `rels`. @@ -229,7 +224,6 @@ impl Cube { tile, tiles: vec![tile], tiled: false, - snapshots: vec![snapshot(map)?], n: map.len(), logs: vec![DeltaLog::new(); rels.len()], effective: empty_effective(rels.len()), @@ -269,22 +263,15 @@ impl Cube { .iter() .map(|tuples| rel_with(tuples, &place)) .collect::>()?; - let snapshots = tiles - .iter() - .map(|&t| { - table - .map(frame, t) - .map_err(CubeRefusal::Map) - .and_then(snapshot) - }) - .collect::>()?; + for &t in tiles { + table.map(frame, t).map_err(CubeRefusal::Map)?; + } Ok(Self { id: NEXT_CUBE.fetch_add(1, Ordering::Relaxed), frame: frame.clone(), tile: tiles.first().copied().unwrap_or(0), tiles: tiles.to_vec(), tiled: true, - snapshots, n: BLOCK_POSITIONS.saturating_mul(tiles.len()), logs: vec![DeltaLog::new(); rels.len()], effective: empty_effective(rels.len()), @@ -414,7 +401,7 @@ impl Cube { )) } - fn map<'t>(&self, table: &'t MapTable) -> Result<&'t LocalMap, CubeRefusal> { + fn map<'t>(&self, table: &'t MapTable) -> Result, CubeRefusal> { table.map(&self.frame, self.tile).map_err(CubeRefusal::Map) } @@ -736,27 +723,30 @@ impl Cube { /// Adopts `frame`, sealed after the table advanced to add symbols. Allowed /// only if, for every tile the cube holds, the new map is a pure /// extension of the one the cube was built (or last rebased) on - /// ([`LocalMap::extends`]): positions never renumber, so every stored + /// ([`TileMap::extends`]): positions never renumber, so every stored /// coordinate and logged edge keeps naming the same symbol. Otherwise /// [`CubeRefusal::Renumbered`] names the tile and the cube is unchanged. /// /// A single-tile cube's `n` is its map's length, so it grows with the /// map; a tiled cube's positions were reserved at build and stay fixed. pub fn rebase(&mut self, table: &MapTable, frame: &Frame) -> Result<(), CubeRefusal> { - let mut next = Vec::with_capacity(self.tiles.len()); - for (&t, old) in self.tiles.iter().zip(&self.snapshots) { + // The table is append-only: the cube's own frame still reads the + // layout it was built on, so the old side of the comparison is read + // through the index and the cube keeps no copy of it. + let mut first_len = None; + for &t in &self.tiles { + let old = table.map(&self.frame, t).map_err(CubeRefusal::Map)?; let map = table.map(frame, t).map_err(CubeRefusal::Map)?; if !map.extends(old) { return Err(CubeRefusal::Renumbered(t)); } - next.push(snapshot(map)?); + first_len.get_or_insert(map.len()); } if !self.tiled { - if let Some(m) = next.first() { - self.n = m.len(); + if let Some(len) = first_len { + self.n = len; } } - self.snapshots = next; self.frame = frame.clone(); // `n` may have grown; cached folds were built under the old one. for c in &mut self.effective { @@ -827,15 +817,7 @@ fn check_col(col: u8) -> Result<(), CubeRefusal> { } } -/// A copy of a tile's map for later comparison by [`Cube::rebase`]. -fn snapshot(map: &LocalMap) -> Result { - let globals: Vec = (0..map.len()) - .filter_map(|l| u8::try_from(l).ok().and_then(|l| map.global(l))) - .collect(); - LocalMap::prepare(map.width(), &globals).map_err(CubeRefusal::Domain) -} - -fn rel(tuples: &[(u32, u32)], map: &LocalMap) -> Result { +fn rel(tuples: &[(u32, u32)], map: TileMap<'_>) -> Result { rel_with(tuples, &|id| { let l = map.local(u64::from(id)).map_err(CubeRefusal::Domain)?; Ok(u32::from(l)) diff --git a/crates/lance-graph-xlog/src/lib.rs b/crates/lance-graph-xlog/src/lib.rs index 7d8a28d79..e2ef54043 100644 --- a/crates/lance-graph-xlog/src/lib.rs +++ b/crates/lance-graph-xlog/src/lib.rs @@ -39,7 +39,7 @@ //! back to ids. None is a computation over the population. use lance_graph_contract::set_rules::Population; -use lance_graph_dt64::dr::{Frame, LocalMap, MapRead, MapRefusal, MapTable, TileId}; +use lance_graph_dt64::dr::{Frame, LocalMap, MapRead, MapRefusal, MapTable, TileId, TileMap}; use lance_graph_dt64::{AlgebraRef, MethodBinding}; use lance_graph_mask_risc::{execute_into, ExecError, Foreign, LaneRef, Out, Planes, Scratch}; use lance_graph_quack::{ @@ -52,6 +52,46 @@ pub mod cube; pub mod lens; pub mod wide; +/// The local ↔ global id lookups the Y-domain functions need. Implemented by +/// a standalone [`LocalMap`] and by the versioned [`TileMap`] a `MapTable` +/// hands out, so one set of queries serves both. +pub trait MidMap { + /// Local position of a global id. + fn local(&self, global: u64) -> Result; + /// Global id at a local position, if held. + fn global(&self, local: u8) -> Option; + /// Number of positions. + fn len(&self) -> usize; + /// True iff no positions. + fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl MidMap for LocalMap { + fn local(&self, global: u64) -> Result { + LocalMap::local(self, global) + } + fn global(&self, local: u8) -> Option { + LocalMap::global(self, local) + } + fn len(&self) -> usize { + LocalMap::len(self) + } +} + +impl MidMap for TileMap<'_> { + fn local(&self, global: u64) -> Result { + TileMap::local(self, global) + } + fn global(&self, local: u8) -> Option { + TileMap::global(self, local) + } + fn len(&self) -> usize { + TileMap::len(self) + } +} + /// The bounds of the admitted chain: `X == left`, `Z == right` when set. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct Bounds { @@ -118,7 +158,7 @@ fn side( tuples: &[(u32, u32)], y_of: fn(&(u32, u32)) -> u32, free_of: fn(&(u32, u32)) -> u32, - mid: &LocalMap, + mid: &impl MidMap, ) -> Result { let y_local = tuples .iter() @@ -136,7 +176,7 @@ fn side( fn sides( left: &[(u32, u32)], right: &[(u32, u32)], - mid: &LocalMap, + mid: &impl MidMap, ) -> Result<(Side, Side), NativeRefusal> { // r(X, Y): Y is column 1, X free. s(Y, Z): Y is column 0, Z free. Ok(( @@ -216,7 +256,7 @@ pub fn support( q: Bounds, left: &[(u32, u32)], right: &[(u32, u32)], - mid: &LocalMap, + mid: &impl MidMap, ) -> Result { let (l, r) = sides(left, right, mid)?; support_of(&l, &r, q, mid.len()) @@ -241,7 +281,7 @@ pub fn bag_count( q: Bounds, left: &[(u32, u32)], right: &[(u32, u32)], - mid: &LocalMap, + mid: &impl MidMap, ) -> Result, NativeRefusal> { let (l, r) = sides(left, right, mid)?; bag_count_of(&l, &r, q, mid.len()) @@ -278,7 +318,7 @@ impl Intake { right: &[(u32, u32)], ) -> Result { let mid = table.map(frame, tile).map_err(BoundRefusal::Map)?; - let (l, r) = sides(left, right, mid).map_err(BoundRefusal::Native)?; + let (l, r) = sides(left, right, &mid).map_err(BoundRefusal::Native)?; Ok(Self { frame: frame.clone(), tile, @@ -288,8 +328,10 @@ impl Intake { }) } - /// The Y map the lanes were built against, if its frame is still current. - pub fn map<'t>(&self, table: &'t MapTable) -> Result<&'t LocalMap, BoundRefusal> { + /// The Y map the lanes were built against, read at the intake's frame + /// (an old frame reads its own layout, so this only fails for a foreign + /// frame or a tile the table never held). + pub fn map<'t>(&self, table: &'t MapTable) -> Result, BoundRefusal> { table.map(&self.frame, self.tile).map_err(BoundRefusal::Map) } @@ -361,7 +403,7 @@ pub fn bag_count_admitted( } /// Global ids of the set bits of `mask`, in local order. -pub fn ids(mid: &LocalMap, mask: &[u64]) -> Vec { +pub fn ids(mid: &impl MidMap, mask: &[u64]) -> Vec { (0..mid.len()) .filter(|&i| mask[i / 64] >> (i % 64) & 1 == 1) .filter_map(|i| mid.global(i as u8)) @@ -370,7 +412,7 @@ pub fn ids(mid: &LocalMap, mask: &[u64]) -> Vec { /// `(global id, count)` for every Y with at least one derivation, in local /// order. -pub fn nonzero(mid: &LocalMap, counts: &[i64]) -> Vec<(u64, i64)> { +pub fn nonzero(mid: &impl MidMap, counts: &[i64]) -> Vec<(u64, i64)> { counts .iter() .enumerate() diff --git a/crates/lance-graph-xlog/tests/cube.rs b/crates/lance-graph-xlog/tests/cube.rs index 4a86f6054..54e26532b 100644 --- a/crates/lance-graph-xlog/tests/cube.rs +++ b/crates/lance-graph-xlog/tests/cube.rs @@ -103,13 +103,13 @@ fn the_cube_answers_what_the_free_functions_answer() { let free = support(q, &R, &S, &y_map).unwrap(); assert_eq!( - ids(map, &cube.support(&table, j).unwrap()), + ids(&map, &cube.support(&table, j).unwrap()), ids(&y_map, &free.answer), "{q:?}" ); let free = bag_count(q, &R, &S, &y_map).unwrap(); assert_eq!( - nonzero(map, &cube.bag_count(&table, j).unwrap()), + nonzero(&map, &cube.bag_count(&table, j).unwrap()), nonzero(&y_map, &free), "{q:?}" ); @@ -135,7 +135,7 @@ fn any_pair_joins_on_any_columns_without_a_new_lookup() { let j = cube.join(l, lc, r, rc).unwrap(); let want = oracle(rels[a as usize], lc, None, rels[b as usize], rc, None); let got = cube.bag_count(&table, j).unwrap(); - let got: BTreeMap = nonzero(map, &got).into_iter().collect(); + let got: BTreeMap = nonzero(&map, &got).into_iter().collect(); assert_eq!(got, counts(&want), "rel {a}.{lc} = rel {b}.{rc}"); let mut rows = cube.materialize_rows(&table, j).unwrap(); rows.sort_unstable(); @@ -214,13 +214,13 @@ fn differential_against_enumeration() { let j = cube.join(l, lc, r, rc).unwrap(); let want = oracle(&a, lc, lb, &b, rc, rb); nonempty += usize::from(!want.is_empty()); - let got: BTreeMap = nonzero(map, &cube.bag_count(&table, j).unwrap()) + let got: BTreeMap = nonzero(&map, &cube.bag_count(&table, j).unwrap()) .into_iter() .collect(); assert_eq!(got, counts(&want)); let sup: Vec = counts(&want).into_keys().collect(); assert_eq!( - ids(map, &cube.support(&table, j).unwrap()) + ids(&map, &cube.support(&table, j).unwrap()) .into_iter() .collect::>(), sup.into_iter().collect() @@ -251,13 +251,26 @@ fn a_stale_frame_a_foreign_view_and_a_bound_join_column_are_refused() { ); assert_eq!(cube.support(&table, l), Err(CubeRefusal::NotAJoin)); + // Previously: after `table.advance(1, 1, vec![])` both reads were refused + // as `MapRead::Stale(_)`. The table is now a versioned append-only index: + // a frame of this table reads the layout it was sealed under, so the + // cube keeps answering, identically, after the table moves on. `Stale` + // is left for a frame this table never issued (see below). + let (sup_before, bag_before) = ( + cube.support(&table, j).unwrap(), + cube.bag_count(&table, j).unwrap(), + ); table.advance(1, 1, vec![]); + assert_eq!(cube.support(&table, j), Ok(sup_before)); + assert_eq!(cube.bag_count(&table, j), Ok(bag_before)); + + let alien = table_over(&rels); assert!(matches!( - cube.support(&table, j), + cube.support(&alien, j), Err(CubeRefusal::Map(MapRead::Stale(_))) )); assert!(matches!( - cube.bag_count(&table, j), + cube.bag_count(&alien, j), Err(CubeRefusal::Map(MapRead::Stale(_))) )); } @@ -297,7 +310,7 @@ fn an_added_relation_joins_like_a_base_one() { let r = cube.scan(0).unwrap(); let j = cube.join(l, 0, r, 1).unwrap(); let map = table.map(&table.seal(), TILE).unwrap(); - let got: BTreeMap = nonzero(map, &cube.bag_count(&table, j).unwrap()) + let got: BTreeMap = nonzero(&map, &cube.bag_count(&table, j).unwrap()) .into_iter() .collect(); assert_eq!(got, counts(&oracle(&derived, 0, None, &R, 1, None))); @@ -306,9 +319,15 @@ fn an_added_relation_joins_like_a_base_one() { cube.add_relation(&table, &[(1, 1000)]), Err(CubeRefusal::Domain(_)) )); + // Previously: after `table.advance(1, 1, vec![])` a derived relation was + // refused as `MapRead::Stale(_)`. The cube's frame now reads its own + // layout after the table advances, so the same relation is accepted; a + // foreign table is what is refused. table.advance(1, 1, vec![]); + assert!(cube.add_relation(&table, &derived).is_ok()); + let alien = table_over(&rels); assert!(matches!( - cube.add_relation(&table, &derived), + cube.add_relation(&alien, &derived), Err(CubeRefusal::Map(MapRead::Stale(_))) )); } diff --git a/crates/lance-graph-xlog/tests/versioned_map.rs b/crates/lance-graph-xlog/tests/versioned_map.rs new file mode 100644 index 000000000..2b55790de --- /dev/null +++ b/crates/lance-graph-xlog/tests/versioned_map.rs @@ -0,0 +1,106 @@ +//! F7: a cube keeps answering after the map table advances. +//! +//! The table is a versioned append-only index, so a cube's own frame reads the +//! layout it was built under. Without `rebase` the cube answers exactly as +//! before; after a `rebase` onto an extending frame it answers over the +//! extended map. +//! +//! Disable it catches: restoring current-frame-only reads in dt64's +//! `MapTable::map` (a frame older than the table's state is refused as +//! stale) turns `an_old_frame_keeps_answering_without_rebase` red. + +use lance_graph_dt64::dr::{LocalMap, MapTable, Width}; +use lance_graph_xlog::cube::Cube; +use lance_graph_xlog::wide::WideRows; + +const TILE: u32 = 7; +const EDGES: [(u32, u32); 4] = [(1, 2), (2, 3), (3, 4), (4, 1)]; +const R: [(u32, u32); 3] = [(1, 2), (2, 3), (3, 4)]; +const S: [(u32, u32); 3] = [(2, 3), (3, 4), (4, 5)]; + +fn base_table() -> MapTable { + let mut t = MapTable::new(1, 1); + t.advance( + 1, + 1, + vec![( + TILE, + Some(LocalMap::prepare(Width::W256, &[1, 2, 3, 4, 5]).unwrap()), + )], + ); + t +} + +fn extend(t: &mut MapTable) { + t.advance( + 2, + 2, + vec![( + TILE, + Some(LocalMap::prepare(Width::W256, &[1, 2, 3, 4, 5, 6, 7]).unwrap()), + )], + ); +} + +#[test] +fn an_old_frame_keeps_answering_without_rebase() { + let mut table = base_table(); + let f1 = table.seal(); + let rels: [&[(u32, u32)]; 2] = [&R, &S]; + let mut cube = Cube::new(&table, &f1, TILE, &rels).unwrap(); + let l = cube.scan(0).unwrap(); + let r = cube.scan(1).unwrap(); + let j = cube.join(l, 1, r, 0).unwrap(); + let support = cube.support(&table, j).unwrap(); + let bag = cube.bag_count(&table, j).unwrap(); + assert!(support.iter().any(|w| *w != 0), "fixture must be non-empty"); + + extend(&mut table); + assert_eq!(cube.support(&table, j).unwrap(), support); + assert_eq!(cube.bag_count(&table, j).unwrap(), bag); +} + +fn dense(rows: &WideRows, x: u32) -> Vec { + let mut v = vec![0u64; rows.blocks() as usize * 4]; + rows.row_into(x, &mut v); + v +} + +#[test] +fn an_old_frame_keeps_answering_the_wide_closure_without_rebase() { + let mut table = base_table(); + let f1 = table.seal(); + let cube = Cube::new_tiled(&table, &f1, &[TILE], &[&EDGES]).unwrap(); + let (before, _) = cube.closure_wide(&table, 0).unwrap(); + let want: Vec> = (0..before.n()).map(|x| dense(&before, x)).collect(); + assert!(want.iter().any(|r| r.iter().any(|w| *w != 0))); + + extend(&mut table); + let (after, _) = cube.closure_wide(&table, 0).unwrap(); + assert_eq!(after.n(), before.n()); + for x in 0..after.n() { + assert_eq!(dense(&after, x), want[x as usize], "row {x}"); + } +} + +#[test] +fn rebase_onto_an_extension_answers_over_the_extended_map() { + let mut table = base_table(); + let f1 = table.seal(); + let rels: [&[(u32, u32)]; 2] = [&R, &S]; + let mut cube = Cube::new(&table, &f1, TILE, &rels).unwrap(); + let l = cube.scan(0).unwrap(); + let r = cube.scan(1).unwrap(); + let j = cube.join(l, 1, r, 0).unwrap(); + let before = cube.support(&table, j).unwrap(); + + extend(&mut table); + let f2 = table.seal(); + cube.rebase(&table, &f2).unwrap(); + let after = cube.support(&table, j).unwrap(); + // Same word count (both maps fit one word), the same positions set: the + // extension appended ids 6 and 7 without renumbering. + assert_eq!(before, after); + // The new ids are addressable now: a derived relation over id 7 is accepted. + assert!(cube.add_relation(&table, &[(6, 7)]).is_ok()); +} diff --git a/crates/lance-graph-xlog/tests/wide_closure.rs b/crates/lance-graph-xlog/tests/wide_closure.rs index 4fbe5b5c9..4a60c70af 100644 --- a/crates/lance-graph-xlog/tests/wide_closure.rs +++ b/crates/lance-graph-xlog/tests/wide_closure.rs @@ -454,7 +454,6 @@ fn refold_is_explicit_and_preserves_the_closure() { /// cube is left on the refused frame and the last assertions fail). #[test] fn rebase_accepts_prefix_extension_and_refuses_reorder() { - let globals_a: Vec = (0..100).collect(); let globals_b: Vec = (100..200).collect(); let (mut table, frame_a) = make_table(&[(10, 0..100), (20, 100..200)]); let edges = [(0u32, 1u32), (1, 150), (150, 199), (199, 0), (120, 121)]; @@ -478,24 +477,21 @@ fn rebase_accepts_prefix_extension_and_refuses_reorder() { let (rows_c, _) = cube.closure_wide(&table, 0).unwrap(); assert!(bit(&dense(&rows_c, 355), 356), "appended id is addressable"); - // Reorder: a second table whose tile 20 swaps its first two ids. + // Reorder: the SAME table advances again, tile 20 swapping its first two + // ids. (Previously a second, unrelated table was used and the cube's + // private snapshot supplied the old layout; the cube now reads its old + // layout through the table's index, so the comparison must stay within + // one table. A foreign table's frame is refused as `Map(Stale)`.) let mut swapped = extended.clone(); swapped.swap(0, 1); - let mut other = MapTable::new(1, 1); - other.advance( - 1, - 1, - vec![ - ( - 10, - Some(LocalMap::prepare(Width::W256, &globals_a).unwrap()), - ), - (20, Some(LocalMap::prepare(Width::W256, &swapped).unwrap())), - ], + table.advance( + 3, + 3, + vec![(20, Some(LocalMap::prepare(Width::W256, &swapped).unwrap()))], ); - let frame_c = other.seal(); + let frame_c = table.seal(); assert_eq!( - cube.rebase(&other, &frame_c), + cube.rebase(&table, &frame_c), Err(CubeRefusal::Renumbered(20)) ); // Still on frame_b: the same table answers, and the logged edge stays. diff --git a/crates/xlog-relcomp-probe/tests/dto_binding.rs b/crates/xlog-relcomp-probe/tests/dto_binding.rs index e13013adf..051334753 100644 --- a/crates/xlog-relcomp-probe/tests/dto_binding.rs +++ b/crates/xlog-relcomp-probe/tests/dto_binding.rs @@ -421,7 +421,7 @@ fn executing_the_admitted_bag_method_is_refused_in_the_lab_register() { // ---------------- D-DYNBAG-0 step 2: the admitted method drives execution -use lance_graph_dt64::dr::{LocalMap, MapRead, MapTable, Width}; +use lance_graph_dt64::dr::{LocalMap, MapTable, Width}; use xlog_relcomp_probe::ChainQuery; use xlog_relcomp_probe::native::{BoundRefusal, Intake, Word, bag_count_admitted, nonzero}; @@ -479,7 +479,7 @@ fn the_admitted_method_drives_the_count() { let table = y_table(&r, &s); let intake = Intake::new(&table, &table.seal(), Y_TILE, &r, &s).unwrap(); let counts = bag_count_admitted(b.method, word_of, &intake, &table, chain().bounds()).unwrap(); - let got: Vec<(u64, i64)> = nonzero(intake.map(&table).unwrap(), &counts); + let got: Vec<(u64, i64)> = nonzero(&intake.map(&table).unwrap(), &counts); let want: Vec<(u64, i64)> = b .expected .iter() @@ -500,29 +500,33 @@ fn a_retracted_route_leaves_nothing_to_run() { assert!(bind(&store, &frames, &views, &routed).is_err()); } +/// Re-pinned for the versioned DR index (2026-11-02, operator: "versioned +/// replayable immutability ... the index"). This test used to assert that a +/// frame sealed before an `advance` is refused (`MapRead::Stale( +/// FrameResealed)`) by `bag_count_admitted`, `Intake::support` and +/// `Intake::new`. Now an old frame is a replay key: it reads the tile layout +/// it was sealed at, so the answers under it are the same before and after +/// the table advances. Method admission is unaffected: it is gated by the +/// bind store's route versions, not by the tile map. Red if dt64 goes back to +/// refusing every frame but the current one. #[test] -fn a_stale_map_frame_is_refused_before_anything_runs() { +fn an_old_map_frame_replays_its_own_layout() { let (r, s) = fixture(); let mut store = bind_store(); let method = admitted(&mut store, &r, &s).clone(); let mut table = y_table(&r, &s); let old = table.seal(); let intake = Intake::new(&table, &old, Y_TILE, &r, &s).unwrap(); + let bag_before = bag_count_admitted(&method, word_of, &intake, &table, chain().bounds()); + let support_before = intake.support(&table, chain().bounds()); + assert!(bag_before.is_ok() && support_before.is_ok()); table.advance(1, 1, vec![]); - let stale = Err(BoundRefusal::Map(MapRead::Stale( - FrameRefusal::FrameResealed, - ))); assert_eq!( bag_count_admitted(&method, word_of, &intake, &table, chain().bounds()), - stale - ); - assert_eq!(intake.support(&table, chain().bounds()).err(), stale.err()); - assert_eq!( - Intake::new(&table, &old, Y_TILE, &r, &s).err(), - Some(BoundRefusal::Map(MapRead::Stale( - FrameRefusal::FrameResealed - ))) + bag_before ); + assert_eq!(intake.support(&table, chain().bounds()), support_before); + assert!(Intake::new(&table, &old, Y_TILE, &r, &s).is_ok()); } /// Gate 3 (MISSING-5): Y is looked up once per tuple at intake; queries read From dbe4505767df8c7035052e3a81b725feab45ff6b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 08:46:50 +0000 Subject: [PATCH 12/19] board + plans: D-CODEGRAPH-0 wide closure, DR versioned index (D-DYNBAG-4), binary lanes Plans: d-codegraph-0-wide-closure-v1, d-dynbag-4-dr-versioned-mapping-v1, d-codegraph-0-binary-lanes-v1. STATUS_BOARD: D-DYNBAG-4 in PR (partial), D-CODEGRAPH-0-C2 in PR, D-CODEGRAPH-0-BIN queued. Gap-map entry: the wave's rulings, results, measured numbers, the failed refresh gate and what fixes it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- .claude/board/INTEGRATION_PLANS.md | 5 + .claude/board/STATUS_BOARD.md | 4 +- .../2026-11-02-d-codegraph-0-gap-map.md | 13 + .../plans/d-codegraph-0-binary-lanes-v1.md | 222 ++++++++++++++++++ .../plans/d-codegraph-0-wide-closure-v1.md | 219 +++++++++++++++++ .../d-dynbag-4-dr-versioned-mapping-v1.md | 179 ++++++++++++++ 6 files changed, 641 insertions(+), 1 deletion(-) create mode 100644 .claude/plans/d-codegraph-0-binary-lanes-v1.md create mode 100644 .claude/plans/d-codegraph-0-wide-closure-v1.md create mode 100644 .claude/plans/d-dynbag-4-dr-versioned-mapping-v1.md diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index b8a5d491b..35c3f1282 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -1,3 +1,8 @@ +## 2026-11-02 — D-CODEGRAPH-0 wide closure, DR versioned index, binary lanes → `.claude/plans/d-codegraph-0-wide-closure-v1.md`, `.claude/plans/d-dynbag-4-dr-versioned-mapping-v1.md`, `.claude/plans/d-codegraph-0-binary-lanes-v1.md` + +**Status:** wide closure and the DR versioned index IN PR; binary lanes PLANNED. **Board:** [STATUS_BOARD](STATUS_BOARD.md) D-CODEGRAPH-0-C2, D-CODEGRAPH-0-BIN, D-DYNBAG-4; [entry](entries/2026-11-02-d-codegraph-0-gap-map.md). Operator rulings: wider call chain with presence bits for thin provisioning and sparse write merged on read; multi-word presence, explicit refold only; versioned, replayable immutability is the DR index itself (one RouteVersion counter, old versions readable by number); counting is Quack's (mask-risc, ndarray); codebooks up to 512 entries use codec9; strings only at intake, labels through codebooks or a KV table; predictive prefetch through `ndarray::simd::prefetch_read_*`, measured before claimed. +**Confidence:** wide closure and the DR index measured (tests, disable runs, mask-risc store); binary lanes plan only. + ## 2026-11-02 — D-CODEGRAPH-0: a resident Codegraph, Charon → ruff → lance-graph + crai-rs → `.claude/plans/d-codegraph-0-resident-codegraph-v1.md` **Status:** PROPOSED; slice A (ruff Charon intake) started. **Board:** [entry](entries/2026-11-02-d-codegraph-0-gap-map.md), [STATUS_BOARD](STATUS_BOARD.md) D-CODEGRAPH-0. Kythe semantic facts + R2IL executable semantics + DT64/DR binding. Gap map G1–G9: no Charon intake in ruff, no Kythe producer in Rust, no `ref/call` edges, no Rust-body → R2IL lowering, no resident or incremental store, closure capped at 64 per tile, body anchor open, ruff sink unmerged. Slices A (intake) → B (resident, incremental) → C (queries, diagnostics, bodies) → D (MCP) → E (token measurement). diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 071031b8c..e2c16eee9 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -41,7 +41,7 @@ Plan: `.claude/plans/d-dynbag-holographic-bags-v1.md`. Plan only. | D-DYNBAG-1 | ℕ-compose probe: `matmul_i8_to_i32` over two DR64² tiles for `p(X, Z)`; B projection = `relcomp::compose` | In PR: exact vs both oracles (300 cases), projection = compose; MEASURED ~21 µs/tile on the AVX-512 VNNI tier (no AMX) vs 50 ns–2.4 µs for B (plan §7.6) | D-DYNBAG-0 | | D-DYNBAG-2 | semiring parameter K (B, ℕ) in relalg `Rel` + Quack surface; typed refusal per (op, K) | Queued | D-DYNBAG-1 | | D-DYNBAG-3 | missing ℕ words backend-first (i64 masked select, min, truncated minus, compare-to-mask, masked sum; maybe bit-packed product) | Queued | D-DYNBAG-2 | -| D-DYNBAG-4 | DR{dynamic}: width from survivor count, multi-tile map, cross-tile fold/compose | Queued | DT64 `dr` owner | +| D-DYNBAG-4 | DR{dynamic}: width from survivor count, multi-tile map, cross-tile fold/compose | In PR (partial): `Axis::for_survivors` (width rule), `MapTable::locate` (global → (tile, local)), versioned append-only index (`map(frame)` replays the frame's layout, `map_at(RouteVersion)`, shared `VersionClock`), xlog `Cube::new_tiled` + `closure_wide` (cross-tile closure). Plans `d-codegraph-0-wide-closure-v1.md`, `d-dynbag-4-dr-versioned-mapping-v1.md`. Open: cross-tile ℕ-compose (§7.4) | DT64 `dr` owner | | D-DYNBAG-5 | `sql-quack-probe` modelled on `lance-graph-cypher-quack` | Queued | parser decision | | D-DYNBAG-6 | scoreboard vs DuckDB / PostgreSQL, ns units, losses reported | Queued | D-DYNBAG-5 | | D-DYNBAG-7 | second Datalog frontend (Scallop) through the same path | Queued | D-DYNBAG-2 | @@ -240,6 +240,8 @@ Plan: [d-codegraph-0-resident-codegraph-v1.md](../plans/d-codegraph-0-resident-c | **D-CODEGRAPH-0-A** | ruff `ruff_charon`: LLBC → Kythe facts with a witness status per call, for one real crate | SHIPPED: ruff #129 (merged `9913e90946`), lance-graph #1918 (merged `c11e4262e`): calls byte-identical to `llbc_calls.py` on the fixture and `mask-risc`; receipt conserved (16,149); Kythe `/xrefs` 3/3 call pins | rust-analyzer oracle (`sag-ra-probe GRAPH=1`) and rustc on `fixtures/codegraph`; `POSSIBLE` never counted as a confirmed caller | | **D-CODEGRAPH-0-B** | resident rows via the ruff sink; per-file digests; incremental invalidation | LANDED (offline form): ruff #130 (merged `95be2e480b`; offline store in `ruff_charon`; `RefinedTruthSink` not reused, typed to R2IL). Incremental == clean on the fixture, byte-identical; skipped dependent leaves exactly one dangling edge. Follow-up fixes in ruff #131 (open). Lance-graph SoA residency still open | clean import == incremental import | | **D-CODEGRAPH-0-C** | callers / callees / impact / diagnostics; tiled closure; body → R2IL design check | SHIPPED (fixture scale): ruff #132 (merged `ed8fd4194f`, rows v2) + lance-graph #1921 (merged `4ab93bf3e`), `crates/lance-graph-codegraph` (queries as confirmed / possible / unknown masks over the xlog cube; dead function, unresolved call, dangling call; unused import NOT_RUN). 27 tests, 12 disable runs red. Tiled closure deferred: refusals `ClosureWidth` (> 64) and `TooMany` (> 256) are surfaced as typed UNKNOWN | each guard disable-verified; unused import proven only without macros, traits, globs, re-exports | +| **D-CODEGRAPH-0-C2** | wide closure (multi-word rows, block presence), multi-tile cube, merge-on-read refresh, interned lanes | IN PR: lance-graph branch `claude/clone-repositories-71a5sw`. Impact over any size; mask-risc store (695 symbols, 3 tiles): impact 286 ns, callers 29 µs, references 45 µs; refresh of an unchanged crate 1.5–1.7 ms = FAILED gate (reading the 2.5 MB text file), fixed by D-CODEGRAPH-0-BIN | narrow vs wide bit-identical ≤ 64; BFS reference on 694 positions; refresh == clean open; every guard disable-verified | +| **D-CODEGRAPH-0-BIN** | ruff_charon binary row lanes + header digest; codec9 small codebooks; Quack counting; no strings in the hot path | QUEUED: plan `d-codegraph-0-binary-lanes-v1.md` (operator: graph-wide u32 ids, TSV kept as debug `open_tsv`). Ruff writer PR first, then the codegraph reader | unchanged refresh ≤ 20 µs; changed-crate open ≤ 1 ms; binary == TSV row for row | | **D-CODEGRAPH-0-D** | local stdio MCP over C | QUEUED | only after B and C gates | | **D-CODEGRAPH-0-E** | token measurement with and without the Codegraph | QUEUED | same revision, questions, model; nothing claimed before measured | diff --git a/.claude/board/entries/2026-11-02-d-codegraph-0-gap-map.md b/.claude/board/entries/2026-11-02-d-codegraph-0-gap-map.md index 4e85cc541..82af8cdfc 100644 --- a/.claude/board/entries/2026-11-02-d-codegraph-0-gap-map.md +++ b/.claude/board/entries/2026-11-02-d-codegraph-0-gap-map.md @@ -102,3 +102,16 @@ Rows v2 in ruff adds the facts the queries need: `PUBLIC`, `IMPLEMENTS` (impl me Twelve disable runs each turn their own test red. The one that first stayed green (dropping the impl-method exclusion from the dead-function check) is now covered by a synthetic store: Charon marks every impl method of a public trait public, so the fixtures never reach that case. Open: tiled closure for more than 64 positions and multi-tile cubes for more than 256 (mask-risc has 459 functions, so every cube query on it is refused today, as a typed UNKNOWN); import facts for unused imports; body → R2IL (G4, G7). + +## Wide closure, merge on read, DR versioned index (branch `claude/clone-repositories-71a5sw`) + +Operator rulings: wider call chain, presence bits for thin provisioning, sparse write merged on read; multi-word presence; refold only when called; versioned, replayable immutability is the DR index (one RouteVersion counter, old versions readable by number). + +- dt64: `MapTable::locate` (global → (tile, local)), `LocalMap::extends`, `Axis::for_survivors`. The tile index is append-only: an extension is a length at a version, a replacement a new generation, a removal an event. An old frame reads its own layout; `map_at(version)` reads any version; a foreign frame or a future version is refused; lookups allocate nothing. +- xlog: `closure_wide` over any number of positions, one row at a time, keeping only nonzero 256-position blocks with a multi-word presence mask; `Cube::new_tiled`; a per-relation delta log (multiset: ADD +1, RETIRE −1); every read sees base + log; `refresh` recomputes only rows that reached a retired edge's source; `refold` only when called; `rebase` accepts a frame only if every tile extends the old one. +- codegraph: stores of any size; `Pos` is u32; `impact` reads whole closure rows (it copied only the first word, wrong past 64 positions); `refresh_crate` re-reads one crate and logs the edge changes (positions never renumber; growth past the reservation is NeedsReopen); rows are interned id lanes; an unchanged crate is skipped by digest. +- Bugs found by measurement: `callers` panicked on a multi-tile store (cube mask width vs symbol width); refresh lost a duplicate call site because the fold was a set (a call graph is a bag). +- Measured on mask-risc's store (695 symbols, 3 tiles), release: impact 286 ns, callers 29 µs, references 45 µs; closures 257 / 337 µs cold; RCONF keeps 614 of 2,304 blocks, RPOSS 951. Refresh of an unchanged crate 1.5–1.7 ms: a failed gate, caused by reading and hashing the 2.5 MB text rows file; the fix is D-CODEGRAPH-0-BIN (binary lanes with a header digest). +- Every new guard was disable-verified red; the re-pinned tests keep their old assertion as a comment. + +Open: binary lanes (plan `d-codegraph-0-binary-lanes-v1.md`); counting through Quack instead of xlog's `fold_counts`; predictive prefetch, measured before claimed; cross-tile ℕ-compose (D-DYNBAG §7.4). diff --git a/.claude/plans/d-codegraph-0-binary-lanes-v1.md b/.claude/plans/d-codegraph-0-binary-lanes-v1.md new file mode 100644 index 000000000..d7a683781 --- /dev/null +++ b/.claude/plans/d-codegraph-0-binary-lanes-v1.md @@ -0,0 +1,222 @@ +# Plan: ruff_charon binary row lanes + header digest (2026-11-02) + +**D-id:** D-CODEGRAPH-0-BIN ([STATUS_BOARD](../board/STATUS_BOARD.md)). Operator answers after planning: queries keep graph-wide u32 ids (one id-translation pass per changed crate); the TSV reader stays as a debug `open_tsv`. + +DECISION (operator 2026-11-02): each crate becomes fixed-width id lanes + a symbol +table, content digest in the header. Unchanged crate = header read; changed crate = +no text parsing; TSV stays as a debug export. Budget: unchanged refresh µs, batch µs; +a ms result is a failed gate. + +## 1. Findings (what happens today) +Writer — ruff_charon/src/store.rs (scratchpad ruffwt checkout): +- :31-44 `ROWS_VERSION=2`, 7 named columns; files.tsv (path,fnv64), residuals.tsv. +- :217-264 `import`: `relations()` -> one `writeln!` per row of TEXT (pred, subject, + object, GRADE, file, start, end); trailing `CRATE COMPLETE|PARTIAL` row; + `replace` (exclusive temp + fsync + rename, :168-208) for rows, residuals, files + (files last = "a crate whose digests are present has its rows", :259). +- :92-148 `read_table`: per line a BTreeMap (one alloc per field); + `rows()` :289 rebuilds `Row` (owned Strings). `stale` :352 = source-file fnv64 + vs files.tsv (unrelated to the rows digest). `dangling` :375 = string filters. +Reader — lance-graph-codegraph/src/artifact.rs: +- :415-495 `read_text_as`: `text.lines()`, `split('\t')` per line, `number()` parse, + `Memo` + `Symbols::intern` per field, `dispatch_target` string split for MAY rows, + `P_OTHER` side table. This is the 6.8-10.3 ms. +- :97 `digest` = 8-byte multiply-rotate fold + length, over the whole TSV text. +- graph.rs:521-536 `refresh_crate`: `read_crate_text` (whole 2.5 MB file) then + `digest(text)` then compare `lanes.digest_of(k)`: the 1.5-1.7 ms unchanged path is + read + hash of the whole file. graph.rs:416 `open` -> `artifact::read_dir`. +- Precedent for in-place reads: canonical_node.rs:1929 `node_rows_from_le_bytes` + (one `unsafe` cast, sound only because every field is a byte array — no native- + endian integers). artifact.rs:115 already uses safe `as_chunks::<8>()`. +- Counting today is hand-rolled: xlog/src/wide.rs:478 `fold_counts` (BTreeMap + multiset, ADD +1, RETIRE -1 floored at 0, in log order). +- Codebook surfaces read: contract codebook.rs:37-135 (`Codebook` u8, <=256, owns + `HashMap`; `FamilyCodebookRegistry` family u16 -> Codebook); dt64 + lib.rs:29-68 + 433-468 (labels resolved late from the ClassView; `field_view` + yields `FieldRow{position:u8,label}` = schema FIELDS, not a symbol table). + +## 2. Format `.rows.bin` (all integers little-endian) +Rule (operator): lanes and every fold carry ONLY ids; names are labels resolved +late, never carried through a query. Names live in their own section(s), never +interleaved with lanes. + +Fixed header, 64 bytes (the only bytes an unchanged check reads): + 0 [u8;8] magic b"CHRNROWS" + 8 u16 major (=1) 10 u16 minor (=0) + 12 u32 flags bit0 = crate PARTIAL (mirrors the CRATE row, which stays a row) + 16 u64 digest (definition below) + 24 u64 file_len (whole file; truncation check before any section read) + 32 u32 n_rows 36 u32 n_symbols 40 u32 n_files 44 u32 n_sections + 48 [u8;16] reserved, must be 0 +Section directory at 64: n_sections x 16 B {tag u32, elem_width u16, enc u16, +offset u32, len u32}; offsets 8-aligned, ascending, non-overlapping, inside +file_len; for a lane `len == n_rows*elem_width` (presence planes: ceil(n/64)*8). +Sections (tag -> content): + PRED_MAG u8 lane + PRED_PRES u64 plane codec9 codeword = predicate codebook id + PRED_CB KV (id -> UTF-8) the predicate labels, <=511 entries + GRADE_MAG/GRADE_PRES + GRADE_CB codec9, grade codebook (handful) + FILE_MAG/FILE_PRES + FILE_CB codec9 when n_files <= 511 (enc=CODEC9); + else FILE u32 lane (enc=U32). "-" (no + site) = codeword 0 = NEVER_TOUCHED, not + a string entry. + SUBJ, OBJ, MAYT u32 lanes crate-local symbol ids (MAYT: NO_ID=u32::MAX for + non-MAY rows; the writer precomputes the `Trait::method` id so + no reader splits strings) + START, END u32 lanes NO_SPAN = u32::MAX; writer refuses usize > u32::MAX-1 + SYM_OFF + SYM_BLOB KV table: (n_symbols+1) u32 offsets into one UTF-8 blob; + label(i) = blob[off[i]..off[i+1]] borrowed, validated once + (`str::from_utf8(blob)` + `is_char_boundary` per offset). +Codec9 mapping (operator: codebooks <=512 use codec9, -255..255, presence bit +flipped): codeword = `EpiLane9::codeword()` (9 bits, presence in bit 8); codeword +0 (= -0, NEVER_TOUCHED) = absent; codeword c in 1..=511 = codebook entry c. Batch +form = u8 magnitude lane + presence plane, 64 lanes per u64 (codec9 batch shape). +u32 stays where ids exceed 511: symbol ids (subject/object/may_target; mask-risc +has 17,155 rows and thousands of names), spans, and files of a crate with >511. +Why not a contract/DT64 surface for symbols: `Codebook` is u8 (<=256) and owns +Strings (a second copy of the label); a FamilyCodebookRegistry split would cut +names into arbitrary 256-buckets with no meaning; DT64 `field_view` labels ClassView +fields (positions <=255), not data symbols. So symbols use the KV fallback section. +Predicate/grade codebooks DO fit: their labels populate a contract `Codebook` +(or are resolved through it) at the graph, exactly as late labels. + +Digest: the reader's existing `artifact::digest` fold (artifact.rs:97; 8 bytes per +step, length folded) over bytes [32 .. file_len) — every byte after the digest +and length fields, so it covers counts, directory, codebooks, lanes. Both repos +implement it from the spec (no cross-repo dep: codegraph does not depend on +ruff_charon and must not start); a committed golden vector (fixed bytes -> fixed +u64) is asserted in both test suites. The writer output is deterministic (rows in +`relations()` order, symbols in first-seen order), so equal content = equal digest. +Versioning: reader refuses magic mismatch, major != 1, minor > known, any unknown +tag with bit31 set ("must understand"), a missing required section, reserved != 0, +`file_len != actual size`, digest mismatch, any lane id >= its table size. +Interner (ruling 4, no strings in the hot path: never `Symbols::intern` at +open/refresh): writer adds SYM_HASH (u64 lane, `digest(name bytes)`) +and SYM_OWNER (u32 lane: local id of the name's crate prefix before "::", which +the writer interns into the table) per symbol. The graph-wide interner becomes +`u64 name-hash -> global id` plus, per global id, a label locator +`(crate slot, local id)` into that crate's retained file buffer. On a hash hit the +borrowed blob slices are compared byte-for-byte (no allocation) to refuse a +collision. Owner, `?dynamic` (a reserved global id, not an interned string) and +completeness (header flag bit0, replacing graph.rs:230 `symbols.get("COMPLETE")`) +are all ids/flags; resolve (graph.rs:128-142 `name.find("::")`) reads SYM_OWNER. + +## 3. Zero-copy reading +- One `fs::read` of the file into one `Vec` per changed crate (the kernel copy + is the intake edge). Lanes are read in place through safe lenses: + `bytes[off..off+len].as_chunks::<4>()` + `u32::from_le_bytes` at use (the same + idiom as artifact.rs:115); codec9 lanes as `&[u8]` + `&[u64]`-by-chunks. NO + `unsafe` cast: the only precedent (canonical_node.rs:1929) is sound because its + fields are byte arrays; a u32 cast would be native-endian. 8-aligned offsets + keep a later aligned view possible without a format change. +- What is copied, honestly: the gather into the graph's resident `Vec` lanes + (local -> global ids). That is a translation, not a duplicate of the file's + meaning; true in-place borrowing of SUBJ/OBJ needs crate-local ids to stay the + query currency (operator question 1). PRED/GRADE/FILE codec9 lanes and START/END + can be appended as raw LE bytes (no translation needed). +- The retained per-crate buffer is the label source (KV section); refresh swaps + the buffer; a retired buffer is kept while any global id's locator points into + it (positions never renumber), dropped at reopen. +- `Lanes` keeps no `other_pred` text side table: predicates are codebook ids. + +## 4. Changes +Writer (ruff_charon store.rs): +- `import` builds per crate: codebooks (predicate, grade, file), symbol table + (first-seen order, plus owner prefixes and MAY targets), lanes; serialises + `.rows.bin` via `replace` (same temp+fsync+rename; order rows.bin, + residuals, files — files last unchanged). +- `rows.tsv` only when `Store::with_tsv_export(true)` / env `CHARON_TSV=1`. + `Store::rows` reads rows.bin and builds `Row` (debug/test API, strings allowed + there); `all_rows`, `dangling`, `crates` (files.tsv) keep their signatures. + `stale` unchanged (source-file digests in files.tsv are a separate concern). +- `ROWS_VERSION` stays 2 for the TSV export; binary has its own major/minor. +Reader (lance-graph-codegraph): +- new `artifact_bin.rs`: `Header::read(&mut File)` (64 bytes, `read_exact`), + `CrateFile::parse(Vec) -> Result<_, IntakeRefusal>` (all refusals of §2 as + new `IntakeRefusal` variants: Magic, Major, Digest, Truncated, Section, IdRange). +- `open`: `*.rows.bin`; TSV reader kept only behind `#[cfg(test)]`/a debug fn + (operator question 2). +- `refresh_crate`: read the 64-byte header; `digest_of(kid) == header.digest` -> + return `RefreshStats::default()` without reading the body. Else read, verify, + gather, then the existing edge-diff path. +- Strings at the edge only: `pos(&str)` in, `symbol(pos)`/`label` out. diag.rs + already folds row masks (no `row_*` call found; keep it so). query.rs:82,169,250 + use `symbol(x).is_none()` as an existence test -> replace with an id-range test + (`has_pos`). `row_predicate/row_subject/row_object` (graph.rs:740-751) become + `#[doc(hidden)]` display/test helpers, documented "not for query paths"; 5 test + call sites stay. +Counting (ruling 1: Quack's concern, executed by mask-risc on ndarray): +- `xlog/src/wide.rs:478 fold_counts` (BTreeMap) is replaced by two Quack + `Agg::GroupReduce` calls (quack lib.rs:1224/1263) over a dense edge-ordinal key + lane: `GroupAgg::Count` over base rows, `GroupAgg::SumI32` over the delta log's + signed lane; effective count = Count + Sum, present = > 0. Owner: xlog calls + Quack (as `bag_count`, xlog lib.rs:240, already does); codegraph never counts. +- The delta log's op lane becomes codec9 (+1 add, -1 retire, -0 = untouched). +- Two named gaps, not hand-rolled around: (a) the dense edge ordinal (rank of + `(src<<32)|dst` among the distinct keys) is not a count — it comes from the + sorted merge refresh_crate already performs; (b) `SumI32` needs an i32 column, + and a codec9 lane is mag+presence: Quack needs a codec9-signed column reader + (GroupFold over sign+magnitude) or one decode to i32 per batch. STOP and land + that in Quack first if (b) is missing. +- Semantics change to verify: fold_counts floors RETIRE at 0 in LOG ORDER; a sum + is order-free. They agree iff the log never retires more than is present at + that point; refresh_crate computes retirements against the old multiset, so it + holds there — pinned by a falsifier, not assumed. + +## 5. Falsifiers (each with its disable run, red-then-green; commit before disabling) +- F1 round trip: writer emits bin + TSV for the same crate; reader resolves every + row of both to labels; equal row for row (and `Store::rows(bin)==rows(tsv)`). + Disable: swap SUBJ/OBJ lanes in the writer -> red. Second fixture with a + P_OTHER-class predicate, a `-` site and a MAY row (shapes differ, assert so). +- F2 header-only unchanged refresh: keep the 64-byte header, overwrite the body + with garbage -> refresh returns default stats (body provably unread). Disable: + refresh reads + verifies the whole file -> Digest refusal -> red. Also count + bytes via a reader that records `read` lengths: == 64. +- F3 refusals: flipped body byte (Digest), truncated file (Truncated), major+1, + minor+1, missing SUBJ section, lane id >= n_symbols, reserved != 0, offset past + file_len. Each a separate case; disable each check -> its case red. +- F4 no strings in open/refresh: a counting global allocator over `open` + + changed `refresh_crate` on a fixture; allocations bounded by lane/buffer count, + independent of n_symbols (double the symbols -> same count). Disable: intern + labels -> red. +- F5 hash collision guard: two names forced to one SYM_HASH (test-only hasher + hook) -> refused, not merged. Silence twin: distinct names never refused. +- F6 codec9 boundaries: codebooks of 511 (codec9) and 512 (refused / U32 enc for + files); codeword 0 reads as absent, never as entry 0. +- F7 counting parity: GroupReduce(Count)+GroupReduce(SumI32) == old fold_counts + on random logs that satisfy the no-over-retire precondition; a log that + over-retires is shown to DIFFER (documents the precondition). Disable: drop the + Sum term -> red. +- F8 cross-repo golden: committed tiny `*.rows.bin` + `*.rows.tsv` written by the + ruff writer, read in codegraph tests; digest golden vector in both repos. +- Timing (release, mask-risc store, pre-registered, N>=40, report median+spread): + unchanged refresh per crate <= 20 µs; changed crate open (17,155 rows) target + <= 1 ms with the gather + hash map as the expected cost. If it lands >= 1 ms + that is a failed gate: STOP and report where, do not ship. TSV numbers + (6.8-10.3 ms / 1.5-1.7 ms) are the baseline quoted beside, not re-derived. + +## 6. Steps and PR order +PR A (ruff, writer first): +- SONNET (ruff worker): `crates/ruff_charon/src/store.rs` + new `store_bin.rs` + only: format writer, codebooks, SYM_HASH/SYM_OWNER, TSV export flag, `rows()` + via bin, F1/F3(writer side)/F6/F8 fixture generation. No cargo. +- HAIKU (quote-harvester): verbatim header/section constants from store_bin.rs + for the reader brief. +- ORCHESTRATOR: spec review, cargo test/clippy -p ruff_charon, disable runs, PR. +PR B (lance-graph, after A merges; fixtures copied from A): +- SONNET (codegraph worker): `lance-graph-codegraph/src/artifact_bin.rs`, + `artifact.rs` (TSV behind test), `graph.rs` (open/refresh/resolve/accessors), + `query.rs` (has_pos). No cargo. +- SONNET (xlog worker, disjoint file): `lance-graph-xlog/src/wide.rs` fold -> + Quack GroupReduce, codec9 op lane — only after the Quack codec9-column gap (4b) + is answered; if missing, a separate PR in lance-graph-quack precedes it. +- HAIKU (census-counter): every `symbols.name`/`.get(&str)`/`intern` call site + remaining on open/refresh/query paths after PR B (must be edge-only). +- ORCHESTRATOR: gates (-p scoped), F2/F4/F7 disable runs, timing gate, board + entry (`.claude/board/entries/` + index), PR. + +## 7. Operator questions +1. Interner currency: keep graph-wide u32 ids (one gather per changed crate, + F4-bounded) or make (crate, local id) the query currency so SUBJ/OBJ lanes are + borrowed in place with no gather (bigger change to positions/cube)? +2. TSV in codegraph: remove the TSV reader from `open` entirely (bin only, TSV + read only in tests), or keep a debug `open_tsv`? diff --git a/.claude/plans/d-codegraph-0-wide-closure-v1.md b/.claude/plans/d-codegraph-0-wide-closure-v1.md new file mode 100644 index 000000000..2c91b7d1f --- /dev/null +++ b/.claude/plans/d-codegraph-0-wide-closure-v1.md @@ -0,0 +1,219 @@ +# Wide call-chain closure: multi-word rows, block presence, merge on read + +D-CODEGRAPH-0 slice C follow-up. Operator decision 2026-11-02: "Wider call +chain. Presence bit for thin provisioning and sparse write merge on read." +Planning only; nothing edited, nothing compiled. + +## 1. Findings from the source + +- F1 `xlog/src/cube.rs:417-419` refuses `self.n > 64` with + `ClosureWidth(n)`. The kernel holds one `u64` per row (`edge`, `full`, + `delta`, `next`, :420-426), and `edge[a] |= 1 << b` (:422) is the only + reason for the limit. The semi-naive loop itself (:429-446) is already + width-agnostic except for the per-row word. +- F2 `Closure { rows: Vec }` (:118-121); `contains(x: u8, y: u8)` hard- + codes `y < 64` (:133). Both are part of the 64-only shape. +- F3 `ClosureStats::rows_composed == |edge⁺|` (:142-145) is a measurable + invariant of semi-naive evaluation. It is a free falsifier for the wide + kernel. +- F4 The `Cube` is single-tile: one `tile: TileId` (:172), built from + `table.map(frame, tile)` (:188). `Rel.cols` are already `Vec` (:151), + so a lane can hold a wide position without a format change. +- F5 `LocalMap::prepare` refuses `> width.positions()` (dt64 `dr.rs:443`) + and stores `u8` locals (:446-447). `MapTable` already keeps many tiles in + a `BTreeMap` (:528). Multi-tile coordinates exist; only + the Cube cannot span them. +- F6 `MapTable::advance` makes every earlier frame stale "even when nothing + changed" (:562-564). So appending a tile for new symbols stales the cube, + and an incremental refresh needs a defined way to move to the new frame. +- F7 Thin provisioning is already doctrine in dt64: "a reservation costs + nothing; only the identity tiles a route actually touches are claimed … + nothing is ever allocated here" (`dr.rs:70-72`). The contract's presence + plane says the same at lane level: "A sparse write leaves an absent lane + unwritten, so it keeps the value read before" (`lane_codec.rs:65-66`). + HHTL: a child mask is "the parent's OR its own delta" (`hhtl.rs:34-37`). +- F8 Split tunnel (`split-tunnel-sparse-append-resident-fold.md`): rule 1 + sparse records `(version, owner, op, …)` with `op ∈ {BORN, SET, DELETE}` + (:51-55); rule 3 never update in place, never compact the PERSISTED log + (:73-79); rule 5 committed records stay resident (:85-88); rule 7 derived + reads write nothing and are recomputed by folding (:95-99). The capstone's + thought layers are "sparse write, merge on read", higher layer wins a + shared lane (capstone :32-33, :49-50). +- F9 Codegraph refuses `n > 256` as `TooMany` (`graph.rs:162-166`), + `Pos = u8` (:14), and builds a dense identity map `ids = 0..n` (:222). + Symbol ids are first-seen intern order, so global id → tile is + `g / 256` by construction. +- F10 BUG today, hidden by F1: `impact`'s `row` closure copies ONLY the + first word of a closure row into a `w`-word mask (`query.rs:241-247`). + With a 64-only closure that is correct; with any wider closure it would + silently drop every position ≥ 64. It must change in the same PR. +- F11 Existing pins that move: `codegraph/tests/guards.rs:50` + (`ClosureWidth(70)`), `:64` (`TooMany(300)`), and `xlog/tests/ + staunen5.rs:174` (`ClosureWidth(65)`). They are re-pinned, not deleted. +- F12 Measured store (coordinator, 2026-11-02; not re-derived): + `scratchpad/mrstore`, ruff_charon rows v2 from `repro/mask_risc.llbc`, + receipt 17154 conserved. DEFINES 459, CALLS_RESOLVED 15684, IMPLEMENTS + 278, REFERENCES 484, PUBLIC 185, SYNTHETIC 58, MAY_DISPATCH 2, + UNRESOLVED_CALL 4. Interned symbols 694 → 3 tiles of 256. Callees by + owner: mask_risc 194, core 132, ndarray 68, alloc 16. + +## 2. Design + +**Interpretation, checked against the source.** (1) Wider chain = closure +over N > 64 positions AND across tiles: confirmed necessary, F1 + F4 + F9. +(2) Presence bits = per closure row, one bit per 256-position BLOCK (one +dt64 tile); an absent block reads as zero and is never allocated: matches +F7. (3) Sparse write, merge on read = refresh appends relation deltas and a +read folds base ∪ deltas: matches F8, with one correction below — the +CLOSURE is a derived read (rule 7), so it is never persisted as a delta; it +is re-derived for the affected rows only. + +**Challenge, stated plainly.** At mask-risc's 694 positions a dense row is +11 words (88 B; 694 rows ≈ 61 KB). Block presence buys almost nothing there. +It pays at workspace scale (tens of thousands of symbols, most callers +reaching few tiles). So the plan makes dense the all-present special case of +the block layout, not a separate path, and measures both on mask-risc. + +### 2a. Data layout — lands in `lance-graph-xlog` (new `src/wide.rs`) + +xlog owns relations and their closure; nothing here is a graph store. + +``` +WideRows (one per closure / per edge matrix) + n: u32 positions (wide coordinate = tile_ord*256 + local) + blocks: u32 ceil(n / 256) + presence: Vec per row: ceil(blocks/64) words; bit b = block b allocated + offset: Vec per row: start of its packed blocks in `words` + words: Vec present blocks only, 4 words each, block order +``` + +- A block is 4 words = 256 bits = one tile, so presence aligns with dt64's + "claimed tile" (F7). Reading bit y of row x: test presence bit y/256, then + popcount(presence below it) → block slot; no search. +- Writes during the fixpoint use a per-row scratch of `blocks*4` words + (dense, reused across rows), then pack: a block is allocated iff any word + is nonzero. Counter `ClosureStats::blocks_allocated` (and + `blocks_reserved = n_rows * blocks`) is what the presence falsifier reads. +- **64-position fast path is kept verbatim.** `Cube::closure` keeps the + existing code for `n <= 64` and returns the narrow `Closure` unchanged + (rows(), contains). The wide kernel is used for `n > 64` and can be forced + on narrow input in tests to prove bit identity. + +**Algorithm:** the same semi-naive loop, multi-word: `delta' = (delta ∘ edge) +andnot full; full |= delta'`, per row, with `mask_or_assign` / +`mask_andnot` / `mask_any` from `ndarray::simd` over the row's 4-word blocks. +Composition ORs edge row y into the scratch only over y's PRESENT blocks. +Cost: `rows_composed == |edge⁺|` still holds (each pair enters a delta once, +F3); each composition costs 4·popcount(presence(edge[y])) words, so total +≈ `4 · Σ_{(x,y)∈edge⁺} |blocks(edge[y])|` word ORs plus one andnot pass over +present blocks per row per round. Worst case dense is `O(N³/64)`; the round +bound stays `rounds <= n + 1` (`FixpointBound`). + +### 2b. Cross-tile — block matrix, one fixpoint (not per-tile + frontier) + +A position is `(tile_ord, local)` folded into one wide coordinate +`tile_ord * 256 + local` (u32, fits the existing `Rel.cols`, F4). The cube +holds an ordered `tiles: Vec`; `tile_ord` is the index in that list. + +Chosen: ONE semi-naive fixpoint over the block matrix. Rejected: per-tile +closure plus a boundary frontier — cross-tile cycles force iterating between +tiles until a global fixpoint anyway, so it is the same computation with a +second loop and a second termination argument. The one case where per-tile +wins (tiles nearly disconnected) is exactly what block presence already makes +free: an unreached tile is an absent block and costs nothing. + +Lookup at build (cold, once): new dt64 `MapTable::locate(frame, tiles, g)` +tries each listed tile's `LocalMap::local` and refuses an id found in two +tiles (`MapRefusal::Duplicate`) or none. Cost O(T·log 256) per id, build +only. Codegraph chunks its dense ids `0..n` into tiles of 256 (F9), so its +tile is `g/256`; `locate` stays general for callers whose ids are not dense. + +**Ordering note (cheap, measurable):** intern crate-local symbols first and +external owners (core/ndarray/alloc, F12) after, grouped by owner, so tiles +align with owners and a crate-local reach touches fewer blocks. Measured, not +assumed: report blocks_allocated for both intern orders on mask-risc. + +### 2c. Sparse write, merge on read + +- **Delta = relation edges, keyed `(rel, src_pos, dst_pos)`, op ∈ {ADD, + RETIRE}, stamped `version`.** One crate re-import = RETIRE every edge whose + provenance row belongs to that crate's old rows + ADD the new rows' edges. + Symbols: new symbols get NEW positions appended to the last tile or a newly + claimed tile; a vanished symbol keeps its position and is masked out by a + `retired` symbol mask. Positions are never renumbered (base ordinal, never + claim order; capstone §2), which is what keeps old lanes valid. +- **Where:** resident in xlog, `Cube`'s `DeltaLog` (append-only per-relation + u32 lanes `src, dst, version, op`), next to the base lanes. Persistence of + crate rows stays ruff_charon's store (it already writes per-crate + `*.rows.tsv`); this plan adds no Lance write and no persisted closure. +- **Read fold for an edge row:** `edge'(x) = base(x) ∪ adds(x) andnot + retires(x)`, deltas applied in append (= version) order, newest wins per + key, no sort (split-tunnel rule 3). +- **Closure under insertions only** is exactly incremental: seed + `delta(x) = (new targets of a)` for every x with `a ∈ R⁺(x) ∪ {x}` and run + the same fixpoint; monotone, exact. +- **Closure under deletions cannot be merged incrementally** (a removed edge + may or may not have been the only path). So: affected rows A = { x : x = a + or bit a set in R⁺(x) } for every source `a` of a RETIRED edge — the union + of column a over the closure, read as rows of the REVERSE closure when it + is held (codegraph holds RCONF/RPOSS and their forwards are cheap) else a + word test per row. Rows outside A keep their row bit-for-bit. Rows in A are + recomputed by the same fixpoint over A, using `R⁺(y)` directly for any + `y ∉ A` (their closure is unchanged), so cost scales with |A|, not N. +- **Compaction/checkpoint:** the RESIDENT base may be re-folded (base ∪ + deltas → new base, delta log emptied) at any time: it is a derived read + (rule 7), nothing persistent changes. The persisted rows are never + compacted (rule 3). Trigger: delta log longer than base edges / 8 — a + policy pin, measured, not a law. +- **Frame:** `MapTable::advance` stales the cube (F6). New dt64 + `LocalMap::extends(&self, older: &LocalMap) -> bool` (older's `globals` is + a prefix) lets `Cube::rebase(table, new_frame)` accept a new frame iff every + tile it holds is unchanged or prefix-extended, and refuse otherwise. That + is the dt64-legal statement of "positions never renumber". + +Ownership: dt64 = coordinates/frames; xlog = relations, closure, deltas; codegraph = intake, symbols, crate refresh. + +## 3. API changes + +- dt64: `MapTable::locate(&self, frame: &Frame, tiles: &[TileId], global: u64) -> Result<(u32, u8), MapRefusal>` (tile ordinal, local); `LocalMap::extends(&self, older: &LocalMap) -> bool`. Additive. +- xlog: `Cube::new_tiled(table, frame, tiles: &[TileId], relations) -> Result` (`Cube::new` = `new_tiled(.., &[tile], ..)`, unchanged); `Cube::closure_wide(&self, table, rel) -> Result<(WideClosure, ClosureStats), CubeRefusal>`; `WideClosure::{contains(x: u32, y: u32) -> bool, row_into(x: u32, out: &mut [u64]), presence(x) -> &[u64]}`; `Cube::apply_delta(&mut self, rel, adds: &[(u32,u32)], retires: &[(u32,u32)], version: u64)`; `Cube::refresh_closure(&self, table, rel, prev: &WideClosure) -> Result<(WideClosure, ClosureStats), CubeRefusal>`; `Cube::rebase(&mut self, table, frame) -> Result<(), CubeRefusal>`. `ClosureStats` gains `blocks_allocated`, `blocks_reserved`, `rows_recomputed` (additive fields; struct is `Default`). +- Kept: `Cube::closure` and `Closure` for `n <= 64`, bit-for-bit. `ClosureWidth` now refused only by `closure` (narrow), never by `closure_wide`; new `CubeRefusal::TileCount(usize)` above 64 tiles (presence word cap, 16,384 positions) — a policy pin. +- codegraph (BREAKING, named): `Pos = u32`; `IntakeRefusal::TooMany` only above the tile cap; `closure()` returns the wide form; `impact` reads whole rows via `row_into` (fixes F10); new `Codegraph::refresh_crate(&mut self, dir: &Path, krate: &str)`. + +## 4. Steps + +1. HAIKU `census-counter`: every caller of `Closure::{rows,contains}`, `Pos`, `ClosureWidth`, `TooMany` across `crates/` (with command). HAIKU `quote-harvester`: ruff_charon `rows` invocation + crate-name arg from `ruff/crates/ruff_charon/src/main.rs:1-90`. +2. SONNET W1, file `lance-graph-dt64/src/dr.rs` only: `locate` + `extends` + unit tests (two-tile duplicate refused; absent refused; prefix vs reorder for `extends`). +3. SONNET W2, file `lance-graph-xlog/src/wide.rs` (new) only: `WideRows`, pack/scratch, the multi-word semi-naive kernel as a free fn over `(n, &[u32] src, &[u32] dst)`, stats counters, insertion-only and affected-rows recompute. Spec = §2a/§2c verbatim. No cube.rs edits. +4. SONNET W3 (after W2 lands), `lance-graph-xlog/src/cube.rs` + `lib.rs`: `new_tiled`, `closure_wide`, `DeltaLog`, `apply_delta`, `refresh_closure`, `rebase`; narrow path untouched. +5. SONNET W4 (after W3), `lance-graph-codegraph/src/{graph,query}.rs`: tile chunking, `Pos` widen, impact via `row_into`, `refresh_crate`, intern order option. +6. SONNET W5, test files only: `xlog/tests/wide_closure.rs`, `codegraph/tests/wide.rs` (synthetic fixtures generated in-test, no /tmp). +7. ORCHESTRATOR: `CARGO_PROFILE_DEV_DEBUG=0` compile/test per crate once, re-pin F11 tests, disable runs (§5), mask-risc measurement (§6), board entry + commit. Workers never run cargo or git. + +## 5. Falsifiers and disable runs + +| falsifier | disable that must turn it red | +|---|---| +| narrow vs wide bit-identical on random graphs n ≤ 64 (wide forced) | wide kernel drops the andnot (`full` not subtracted) | +| 694-position synthetic graph (3 tiles, cross-tile cycles): `closure_wide` == per-node BFS over an adjacency list built IN THE TEST | pack only block 0 of each row | +| `rows_composed == |edge⁺|` from the BFS reference | count every OR including absent blocks | +| absent blocks allocate nothing: two disconnected tiles → `blocks_allocated` == present count, cross blocks 0 (anti-vacuity: the connected twin allocates them) | allocate every block | +| merge-on-read == full rebuild after EVERY step of a refresh sequence (add-only, retire-only, mixed, symbol appended in new tile) | skip the affected-rows recompute on retire | +| rows outside A unchanged and `rows_recomputed == |A|` (can-stay-silent: an add-only step recomputes 0 rows) | treat every row as affected | +| `rebase` refuses a reordered tile, accepts a prefix extension | `extends` returns true | +| impact over > 64 positions sees a caller at position ≥ 256 | restore the first-word-only `row` (F10) | + +A disable is evidence only if its test binary built and printed `test result` red. + +## 6. Timing gates and the mask-risc measurement + +Store exists: `scratchpad/mrstore` (F12; 694 symbols, 3 tiles). Re-producible by `ruff_charon rows repro/mask_risc.llbc ` per §4 step 1 quote; the store and LLBC are session scratch, so committed tests use synthetic fixtures only. Extend `codegraph/examples/timing.rs` with `CODEGRAPH_DIR`. + +- Gated, ns: `WideClosure::contains`, `impact` per call (3 masks, 11 words/row) — gate ≤ 1 µs per call, release. +- Gated, µs: `closure_wide` for RCONF and RPOSS on mask-risc, and `refresh_crate` for one crate — must be µs; any ms is a failed gate (P0). Refresh must beat full rebuild. +- Reported, not gated: open (cold intake), `blocks_allocated / blocks_reserved` for both intern orders, `rounds`, `|edge⁺|`. + +## 7. Open questions for the operator + +1. Tile cap for the presence word: 64 tiles (16,384 positions) as a pin, or a multi-word presence from the start? +2. Should a refreshed resident base (re-fold) be allowed to run implicitly by the delta-length trigger, or only on an explicit call? diff --git a/.claude/plans/d-dynbag-4-dr-versioned-mapping-v1.md b/.claude/plans/d-dynbag-4-dr-versioned-mapping-v1.md new file mode 100644 index 000000000..507078b3d --- /dev/null +++ b/.claude/plans/d-dynbag-4-dr-versioned-mapping-v1.md @@ -0,0 +1,179 @@ +# D-DR-VMAP — the DR global→local index, versioned, immutable, replayable + +STATUS: PLAN. Ruling: the versioned mapping IS `dt64::dr`'s index (old version = +same lookup, version in the key); D-DYNBAG §4.3 (plan ll.120-123). + +## 1. Findings (read, file:line) + +- `RouteLog` already IS the pattern (dr.rs:177-182): `routes: BTreeMap)>>`, append-only `push` (dr.rs:219-226), + `routed_at` = `partition_point(v <= version)` (dr.rs:245), `FutureVersion` + the only refusal (dr.rs:235). Nothing overwritten except the counter. +- `RouteVersion(pub u64)` (dr.rs:155): append-only log position, +1 per event; Ord, Copy. +- `FrameRevisions` (dr.rs:290-295) = (source, relation_map, coordinates). + `coordinates` is an FNV digest (dr.rs:519-529, 567-581): NOT monotone, two + equal assignments collide, and `FrameResealed` (dr.rs:305) says equal + revisions are still a different frame. => unfit as a version key. +- `FrameStore` keeps only `current: Arc`; `advance` drops it + (dr.rs:335-337). `Frame` = Weak store + Weak seal + copy `at` (dr.rs:370-374); + `check` = current-seal-only (dr.rs:346-363). +- HISTORY OVERWRITTEN HERE: `MapTable.tiles: BTreeMap` + (dr.rs:550); `advance` `insert` replaces / `remove` drops (dr.rs:593-601); + `map` refuses any non-current frame via `frames.check` (dr.rs:620-623). +- `LocalMap::prepare` keeps supplied order, sorts only the lookup index + (dr.rs:463-481); `extends` = same width + older globals a prefix (dr.rs:668-678). + So a prefix of a map is a valid map: `local(g)` on the full index, accepted iff + the returned position < prefix length. `lookups: Cell` (dr.rs:459). +- Consumer workaround: xlog `Cube.snapshots: Vec` (cube.rs:194-198) + exists only because "a MapTable drops a map when it advances"; built by + `snapshot()` which COPIES every id + re-prepares (cube.rs:830-836), at + cube.rs:232, 272-280, 752; consumed by `rebase` (cube.rs:745-767). +- Cube reads always use its own frame: cube.rs:216, 270, 410, 420, 717; xlog + lib.rs:280, 293. Today any table advance makes every such read `Stale`, which + is what forces `rebase`. +- codegraph `refresh_crate`: `advance(rev.source+1, …, [(last, Some(map))])` + then `seal` then `cube.rebase` (graph.rs:438-446); build at graph.rs:308-310. +- ClassView/WideFieldMask in dt64: only on the binding side — `resolve_at` + (lib.rs:795-806) = `routed_at(version)` then `resolve` (ClassView basis check); + `Resolved::field_view(&WideFieldMask)` (lib.rs:460). dr.rs imports neither + (dr.rs:26-27 import ClassId, Morton8x8 only). + +## 2. Design — `MapTable` becomes a log, shaped exactly like `RouteLog` + +**Version key: `RouteVersion`, reused, not invented.** It is already the type of +"a position in an append-only DR log, +1 per event". `FrameRevisions` stays on the +frame for REPORTING only (as its own doc says, dr.rs:377); it cannot key history +(digest, collisions, `FrameResealed`). Each `MapTable::advance` bumps the table's +`RouteVersion` by one, including an advance that changes nothing (keeps +dr.rs:584-586's "a new frame even when nothing changed": new version, same maps). + +**Frame carries its version.** `Frame` gains a private `version: RouteVersion`; +`FrameStore` gains `version: RouteVersion`, bumped in `advance`, stamped in +`seal`. Still no public constructor (dr.rs:366-368 holds): a version is only ever +received from a store, never typed in. New `FrameStore::issued(&Frame) -> +Result`: `ForeignFrame` unless the store Weak matches, +else `Ok(frame.version)`. `FrameStore::check` (current-only) is UNCHANGED — +`Guards::check` (dr.rs:421) keeps its currency semantics. + +**Append-only per-tile layout** (replaces dr.rs:550): +``` +struct TileLog { + gens: Vec, // one per generation + events: Vec<(RouteVersion, Option<(u32, u16)>)>, // (gen, len); None = removed +} +tiles: BTreeMap, version: RouteVersion (in FrameStore) +``` +`advance(changes)`, per `(tile, Some(m))`: +- **extension** (`m.extends(gens[last])`, i.e. same gen still live): replace + `gens[last]` with `m` (it is a superset; the old prefix is bit-identical inside + it, so no information is lost) and push `(v, Some((last, m.len())))`. An + extension is a LENGTH at a version. +- **replacement** (anything else): push `m` as a new gen, push `(v, Some((g, len)))`. +- **remove** `(tile, None)`: push `(v, None)`. Nothing is ever deleted. +- unchanged tiles get no event; their last event still answers. +Never update a past event, never compact (split-tunnel rule 3, ll.71-77). + +**Read** — `map(&Frame, TileId) -> Result, MapRead>`: +`issued(frame)?` → `tiles.get(tile)` → `events.partition_point(v <= frame.version)` +→ last event → `TileMap { map: &gens[g], len }`; `None`/no event → `NoSuchTile`. +`TileMap<'t>` is `Copy`, borrows, holds no ids: `local(g)` = `map.local(g)` then +`Unmapped` if `>= len`; `global(l)` = `None` if `l >= len`; `len()`, `width()`, +`digest()` over the prefix, `extends(other: TileMap)` (same gen ⇒ `len >=`; +else prefix slice compare). `locate` (dr.rs:630) reads through `TileMap` unchanged. + +**What "stale" means afterwards.** A frame of THIS table always resolves — at its +own version, to the layout it was sealed under (replay). Refused: a frame of +another table (`ForeignFrame`, dr.rs:300), a tile absent at that version +(`NoSuchTile`). `MapRead::Stale` keeps the name but carries only `ForeignFrame`; +`SourceChanged/MapChanged/CoordinatesChanged/FrameResealed` stay for +`FrameStore::check`/`Guards`. `FutureVersion` is unreachable (store-issued +frames are ≤ current) — keep a `debug_assert`, not a refusal. + +**Cost (to be MEASURED, ns, release):** `map` = 1 `Weak::ptr_eq` + BTreeMap get ++ `partition_point` over that tile's events (log₂ k) ⇒ expected tens of ns; +`TileMap::local` = existing ≤8-step binary search + one compare. Zero allocation +per lookup. `advance` = per change one `extends` (≤256 compares) + amortized +`Vec::push`, plus the existing digest. Memory: one ≤256×17 B LocalMap per +GENERATION (extensions do not add one) + 16 B per event. +(Extension carries the old gen's `lookups` count over; re-add after remove = new gen.) + +## 3. ClassView + WideFieldMask — where they enter (and where not) + +The TILE-MAP half does not need them, and they are not forced in: a tile map is +identity → position, no fields. They enter one layer up, already in code: +- version → classid: `BindStore::routed_at` / `resolve_at(view, version, pos, + schema_rev)` (lib.rs:768, 795-806), ClassView basis check inside `resolve` + (lib.rs:773-791; trait at contract class_view.rs:1057, `field_count` :1097). +- what a version CARRIES = the projection `Resolved::field_view(&WideFieldMask)` + (lib.rs:460; `WideFieldMask` class_view.rs:243, `from_positions` :310, + `full_for` :360). Same version key (RouteVersion) on both halves makes + "the classid at v, read through its ClassView, masked by a WideFieldMask, over + the tile layout at v" one coordinate — IF both logs share one counter (Q1). + +## 4. Consumer changes + +- **xlog `Cube`:** delete `snapshots` (cube.rs:194-198), `snapshot()` + (cube.rs:830-836) and its three call sites (cube.rs:232, 272-280, 752). + `rebase(table, frame)` becomes: for each tile, `old = table.map(&self.frame, + t)?`, `new = table.map(frame, t)?`, `new.extends(old)` else `Renumbered(t)`; + then adopt `frame` and grow `n`. Reads at `self.frame` (cube.rs:216, 270, 410, + 420, 717; lib.rs:280, 293) now SUCCEED after a table advance — rebase becomes + "adopt a newer layout", not "unblock reads". +- **xlog signatures:** `&LocalMap` → `TileMap<'_>` in `rel` (cube.rs:838) and + lib.rs:121, 139, 219, 244, 364, 373 (`ids`, `nonzero`). Mechanical. +- **codegraph:** graph.rs:308-310, 438-446 unchanged (`advance`/`seal`/`rebase` keep signatures). + +**Tests whose semantics change (re-pin, never delete; old assertion as comment):** +- dt64 tests/dr.rs:172 `a_map_is_read_only_under_the_frame_it_was_sealed_under` + (asserts `Stale(CoordinatesChanged)` at :180) → old frame reads OLD map. +- dt64 tests/dr.rs:188 `an_advance_that_changes_nothing_still_reseals` + (`Stale(FrameResealed)` :197) → old frame still reads; frames are distinct + issues (`same_issue` false) — "reseals" survives as identity, not refusal. +- UNCHANGED: dt64 tests/dr.rs:231 (`ForeignFrame` :244). +- xlog tests/cube.rs:236 `a_stale_frame_…_are_refused` (:257, :261) and + tests/cube.rs:284 `an_added_relation_joins_like_a_base_one` (:312) — the + `Stale` arms after `advance(1,1,vec![])` flip to success. +- xlog tests/wide_closure.rs:456 `rebase_accepts_prefix_extension_and_refuses_reorder` + (:499 `Renumbered(20)`) — UNCHANGED (must stay green; it now proves the index). + +## 5. Falsifiers (dt64 tests/dr.rs) + disable-run table + +| # | test | disable (commit first, then break, then checkout) | +|---|---|---| +| F1 | replay: build v1 `[10,20,30]`, replace at v2 `[30,20,10]`; for EVERY id, `map(f1).local == v1 local` and `map(f2).local == v2 local`, and they differ for 10/30 | `map` ignores `frame.version` (reads last event) | +| F2 | extension: v1 `[10,20]`, v2 `[10,20,30]`; `map(f1).local(30) == Unmapped`, `.len()==2`, `.global(2)==None`; `map(f2).local(30)==2`; gens.len()==1 (anti-vacuity: the extension did not mint a gen) | drop the `>= len` guard in `TileMap::local` | +| F3 | replacement keeps both: after F1 plus a remove at v3, f1/f2 still read, f3 → `NoSuchTile` | `remove` deletes `tiles[t]` instead of pushing `None` | +| F4 | foreign: frame of table B on table A → `Stale(ForeignFrame)` even at equal revisions and equal version | `issued` skips the store `Weak::ptr_eq` | +| F5 | no allocation: counting `#[global_allocator]` in a dedicated test binary; 10⁴ `map`+`local` calls allocate 0 bytes | make `map` return an owned prefix copy | +| F6 | silence twin: an `advance` with no changes adds NO event and no gen, yet bumps the version (f_old and f_new read identical maps, `same_issue` false) | push an event per tile on every advance (gens/events counts move) | +| F7 | xlog: cube built at f1 still `closure`/`materialize_rows` correctly after an advance, with no rebase | re-add the current-only check in `map` | + +Release bench: `map`+`local` at k = 1/64/4096 events per tile, in ns (ms = failed gate). + +## 6. Steps + +1. HAIKU `quote-harvester`: every `table.map(` / `&LocalMap` / `.snapshots` + site in xlog + codegraph src AND tests, verbatim with lines (closed space: + the three crates' `src/` + `tests/`). Orchestrator re-reads two. +2. SONNET-A (owns dt64 `src/dr.rs` + `tests/dr.rs` only): §2 verbatim, re-pin + the two dt64 tests of §4, add F1–F6. No cargo. +3. SONNET-B (owns `lance-graph-xlog/src/*` + `tests/*` only), after A lands: + delete snapshots, rewrite `rebase`, `&LocalMap`→`TileMap`, re-pin the two + cube tests, add F7. No cargo. +4. ORCHESTRATOR: codegraph import touch-up (if any); then ONE gate: + `CARGO_PROFILE_DEV_DEBUG=0 cargo test -p lance-graph-dt64 -p lance-graph-xlog + -p lance-graph-codegraph` + clippy `-D warnings` + fmt; run every disable in + §5 (commit → break → read `test result:` line → checkout); bench in release; + board: STATUS_BOARD row + entry file + `entries_index.py --write`; amend + D-DYNBAG §4.2/§4.3 to cite `TileLog`; commit. + +## 7. Operator questions (≤2) + +Q1. One counter or two? Should `MapTable` advances and `BindStore`'s `RouteLog` + routes share ONE `RouteVersion` sequence (one DR log: a frame's version then + also answers `routed_at`/`resolve_at`, §3), or keep separate counters of the + same type (plan default: separate, because no current owner holds both)? +Q2. Read by number? Plan exposes old versions ONLY through store-issued frames + (no `map_at(RouteVersion, tile)`), so a version cannot be forged. Is a + numeric read wanted for replay-from-Lance (where only the number survives + a restart), or does replay re-issue frames by re-running `advance`? From 29a559b646483142512f1065103f114f48959c00 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 09:39:14 +0000 Subject: [PATCH 13/19] board: architecture review Codegraph x DR3 x DT64 x Ruff (graded, with next-step proposal) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- ...graph-dr3-dt64-ruff-architecture-review.md | 168 ++++++++++++++++++ .claude/board/entries/README.md | 3 +- 2 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md diff --git a/.claude/board/entries/2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md b/.claude/board/entries/2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md new file mode 100644 index 000000000..997dce732 --- /dev/null +++ b/.claude/board/entries/2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md @@ -0,0 +1,168 @@ +# Architektur-Review: Codegraph × DR³ × DT64 × Ruff (2026-11-02) + +**Status:** MEASURED (read-only Review) · **Bezug:** D-CODEGRAPH-0, D-DYNBAG-4, PR #1923 +**Stand:** lance-graph `claude/clone-repositories-71a5sw` (= PR #1923, offen); ruff `origin/main` ed8fd4194f (#129–#132 gemergt). +Grade: VERIFIED / PARTIAL / MISSING / CONTRADICTED. Pfade relativ zu `crates/`, sofern nicht anders genannt. +Jeder Zeilenbeleg wurde gelesen; Grep-Treffer zählen nur als Fundort. Ein „0 Treffer“ nennt seinen geschlossenen Suchraum. + +## 1 — Ein kanonischer Codegraph? + +- **1.1 Normalisierte Repräsentation — PARTIAL.** Nur die Def-/Call-Ebene ist normalisiert. `Lanes` (codegraph `src/artifact.rs:187-211`) halten Prädikat + u32-Interner-IDs. Die Prädikate sind DEFINES, CALLS_RESOLVED, MAY_DISPATCH, UNRESOLVED_CALL, PUBLIC, IMPLEMENTS, SYNTHETIC, REFERENCES und CRATE (`artifact.rs:63-75`). Typen, Generics, Effekte und Commands fehlen. Die Identität ist der qualifizierte Namens-String, weder Charon-`FunDeclId` noch Kythe-Ticket noch classid (ruff_charon `src/store.rs:31-54`). +- **1.2 Verlust beim Import — VERIFIED.** `FunFacts` tragen id, Callee-Klasse, refs, deref-Writes und asserts (ruff_charon `src/lib.rs:141-154`). Persistiert werden nur predicate/subject/object/grade/file/start/end (`store.rs:31-39`). Verloren gehen: FunDeclId, Generics, Places, Effekte (nur im Debugtext `lib.rs:336-342`), Typen, Macro-Herkunft, `use`-Items und `kythe_target` (`lib.rs:352-360`). +- **1.3 Kythe und R2IL auf dieselben DR-Koordinaten — MISSING.** + - Codegraph hängt nicht von r2il ab (codegraph `Cargo.toml:11-16`), und `src/` erwähnt keinen BindStore. + - Ein Kythe-Knoten lässt sich verlustfrei auf eine DR-Position legen. + - Ein R2IL-Wort ist dagegen eine `MethodBinding` (dt64 `src/lib.rs:544`), keine Codegraph-Position. + - Beides auf eine Koordinate zu legen, würde Identität (Name) und Verhalten (Wort) zusammenwerfen. Sie bleiben getrennte Achsen, die über eine Kante verbunden werden. +- **1.4 Mehrere Stores mit gleicher Information — PARTIAL.** + - Faktische Autorität ist der ruff_charon-TSV-Store: Codegraph liest nur `*.rows.tsv` (graph.rs:415, artifact.rs:318). + - Codegraph-Lanes und Cube sind eine Projektion. + - Der ruff_r2il-OfflineSink ist verwaist; kein `impl RefinedTruthSink` in `crates/`. + - sag-ra-probe dient als Orakel. + - Die tract-harvest Registry (`registry.rs:198`, `dr3.rs:146` `HashMap`) ist eine **dritte, String-gekeyte Methodenautorität**. + - Der dt64 BindStore ist eine vierte Autorität, die keine Codegraph-Symbole kennt. +- **1.5 Gate gegen eine zweite Source of Truth — MISSING.** + - Codegraph fehlt in `.claude/v3/COMPONENT-MAP.md` (0 Treffer). + - **Die Codegraph-Tests laufen in keiner CI.** `rust-test.yml` führt dt64 (:150) und xlog (:152) aus, aber nicht codegraph. Geschlossener Suchraum: alle Workflow-Dateien. + +## 2 — F### und DT64 als gemeinsame Methodenautorität? + +- **2.1 Auflösung über DT64 — PARTIAL.** + - Reserviert und real: + - `HARDCODED_BEHAVIOUR = 0xFFFF` (contract `ogar_codebook.rs:583`); + - `is_behaviour_lens` mit der Grenze ≥ 0xF000 (`class_view.rs:1053`). + - DT64 löst Identität und Stelligkeit auf, nie Verhalten. + - `declare_algebra` speichert nur `arity: Vec` (dt64 `lib.rs:631-655`). + - Doc: „decides nothing about what a word does“ (`lib.rs:68-73`). + - Ausgeführt wird in der Linse selbst. Die F000-Einträge in `POLYFILL` (r2il `polyfill.rs:39`) verzweigen in den Opcode-`match` von `FoldDialect` (r2il `lib.rs:1129` `INT_ADD`, `:1247` `GROUP_SUM`). Das ist so gewollt, aber DT64 ist damit nur **Identitäts- und Admissions-Autorität**, keine Ausführungsautorität. +- **2.2 Nutzung der Bindings:** + + | Komponente | Grad | Beleg | + |---|---|---| + | R2IL F000 | VERIFIED | `binding.rs` admit :289, Permit :344 | + | XLOG F100 | PARTIAL | Binding nur als Form-Gate; Bedeutung liefert der Aufrufer über das lokale `enum Word` (xlog `lib.rs:351-360`, „MISSING-1“) | + | ONNX | MISSING (bewusst) | `lib.rs:11-13` | + | egglog-r2il-probe | MISSING | ruft `FoldDialect` direkt | + | r2il-mask-abi-probe | MISSING | — | + | ogar-r2il / ogar-loco | MISSING | eigene `FnIndex` über `Vocabulary` | + | xlog-relcomp-probe | PARTIAL | — | + + xlog-relcomp-probe nutzt **`dr_math_binding_probe::MethodSignature`** mit eigener `method_ordinal: u16` und eigenen `FrameStore`/`Guards`/`Refusal`. Das ist eine zweite Methodenidentität ohne dt64-Abhängigkeit. +- **2.3 Dispatch außerhalb DT64 — VERIFIED, mehrfach:** + - `FoldDialect` `match`; + - r2il `procs.rs` mit Opcodes, die nicht in `POLYFILL` stehen (`procs.rs:15-20` gegen `polyfill.rs:28-32`); + - XLOG `Word`; + - die Prädikat-Codetabelle im Codegraph (`artifact.rs:66-84`); + - ONNX-Funktionsnamen; + - `bind_dr64`. +- **2.4 Identität, Signatur, Körper und Implementierung getrennt — PARTIAL.** In DT64 ist das sauber getrennt: `MethodId` ≠ `name` (`lib.rs:95`, `:200-202`), `SigRev`, geordnete `effects`, `Guard`, Alias am Linsenplatz (`polyfill.rs:43-55`). Es gibt vier Kollapsstellen: + 1. `arity` ohne Einheit: Stack-Operanden bei F000, Spalten bei F100 (xlog `lens.rs:21-27`). + 2. `AlgebraRef.word: u8` ist zugleich Identität und Position. Die Ordnung wird doppelt geführt (POLYFILL und `FnIndex`) und nur per Test verbunden. + 3. Bei `ActionDef` ist `predicate: &str` selbst die Identität (contract `action.rs:79-93`). + 4. `MethodSignature.method_ordinal` (siehe 2.2). +- **2.5 Kann Ruff Adapter, Compiler-Ops und Kommandos über Bindings auflösen? — MISSING.** ruff_charon kennt nur Rust-Callees (`lib.rs:521-524`). Eine Zeile „Callee = (Linse, Wort)“ gibt es nicht. Deshalb meinen `users_of` (dt64 `lib.rs:715`) und `callers` (codegraph `query.rs:125`) verschiedene Mengen. +- **2.6 Dynamik, UNKNOWN, Guards, Effekte, Quellzeugen — PARTIAL.** + - VERIFIED: Effekt-Reihenfolge (`lib.rs:216-221`), Guard-Witness (`:167-178`), benannte Refusals, `Graded{confirmed,possible,unknown}`. + - Der Quellzeuge steht nicht im `MethodBinding`. Er ist nur über die Lab-`Attachment` in dt64-admission-probe verbunden. + +## 3 — Lazy Hydration ohne Hot-Path-Kosten? + +- **3.1 Resident nur Identität und Relationen — PARTIAL.** + - Keine Bodies, kein Quelltext und keine IR resident: VERIFIED (graph.rs:165-178). + - **MethodRefs: MISSING.** In `codegraph/src` gibt es 0 Treffer für `MethodRef|BindStore|RouteVersion|map_at`. + - Versionen werden nur über Change-Digests geführt (`artifact.rs:97`, „nicht kryptografisch“). +- **3.2 Bodies und IR on demand — PARTIAL.** Die Quellen existieren: LLBC-JSON mit `files[].contents` und strukturierten Bodies. `lance-graph-hydrate` ist der **falsche Kandidat**: Es kopiert Artefakte S3→Volume→Lance, ein Item-Lookup ist das nicht (CONTRADICTED als Item-Hydration). Ein On-demand-API im Codegraph fehlt. +- **3.3 Seam ohne Re-Parse — MISSING bis auf Quelltext-Slices.** Der einzige Seam ist `(file, start, end)`. **FunDeclId wird nicht persistiert.** Wer einen Body will, muss das ganze LLBC neu laden und per Namen suchen. Bei Impl-Methoden ist das kollisionsanfällig (`lib.rs:73-90`). +- **3.4 Zugehörigkeit zu Quelle, Compiler, Frame und Version — PARTIAL.** + - Quell-Digest (`files.tsv`, fnv64) und `ROWS_VERSION` sind VERIFIED. + - **Charon-/LLBC-Version: MISSING.** Charon ist nur als Path-Dependency gepinnt (`Cargo.toml:17-18`). + - Es gibt zwei Hashes (fnv64 und FastHasher), keinen gemeinsamen Content-Key. +- **3.5 Wiederverwendung — VERIFIED innerhalb des Codegraph.** Digest-Skip, Multiset-Delta, `refresh_closure` und `OnceCell` funktionieren. Aber: + - Das Gate ist **gerissen**: Ein unveränderter Refresh kostet 1,5–1,7 ms, weil die TSV gelesen und gehasht wird. + - `with_dependents` (`store.rs:398-411`) ist ungenutzt. + +## 4 — MLIR und Rust als SoC-konforme Sichten? + +- **4.1 Versionsmodelle je Compiler — PARTIAL.** Charon ist ein Fork mit gepinnter Nightly, aber seine Version steht nicht in den Zeilen. ruff_r2il liest Maschinencode; laut OGAR ist das „NOT on this path“ (ogar-r2il `lib.rs:27-33`). Die R2IL-Arity-Tabelle (82 Einträge) ist nur per Drift-Test an r2sleigh gebunden. +- **4.2 Versionierte Projektionen derselben Identität — PARTIAL.** Die Bausteine sind da: `RouteVersion`, `VersionClock`, `map_at` und `FrameRevisions` (dt64 `dr.rs:153-167`, `:349`, `:902`). Es fehlt eine Achse „Sicht / Frontend / Version“ an der Codegraph-Identität. Heute versioniert DR nur das Routing. +- **4.3 MLIR über den R2IL-Polyfill — MISSING im Code, Pfad plausibel.** Geschlossener Suchraum: alle Crate-Verzeichnisse und `Cargo.toml` in lance-graph und OGAR, Muster `mlir|dialegg|melior|llvm`: 0 Treffer. Ohne neue Architektur ginge es so: ein MLIR-Op-Name wird eine weitere `AliasLanguage` am POLYFILL-Platz (`polyfill.rs:58-65`), und ein Op ohne Wort bekommt eine eigene F###-Linse. Regionen und Kontrollfluss haben kein Gegenstück; `PROC_DEF/PROC_CALL` ist OPEN (`procs.rs:7-9`). +- **4.4 Was egglog/DialEgg brauchen — PARTIAL.** + - Vorhanden: `RuleId{law, domain}`, eine Zulassungstabelle, ein i128-Orakel und `ReplayCapsule` mit benannten Refusals. + - Es fehlen Effekte, Typen, Guards und Herkunft. + - **egglog floatet auf `branch = "main"`** (egglog-r2il-probe `Cargo.toml:25`). Ein `ProofChanged` kann deshalb ein Werkzeugwechsel sein, ohne dass sich die Semantik geändert hat. +- **4.5 Fehlende Information nicht als Gleichheit lesen — PARTIAL.** Die Wächter existieren: Receipt-Erhaltung, Refusals, K3- und Guard-Refusal sowie Overflow-Refusal. Es bleiben sechs Lücken: + 1. `Arg` ist untypisiert. + 2. `arity` hat keine Einheit. + 3. Das Binding trägt keinen Quellzeugen. + 4. Frontend- und egglog-Version fehlen. + 5. egglog sieht keine Effekte. + 6. Pfade außerhalb DT64 bleiben für `users_of` unsichtbar. Ein leeres `users_of` heißt deshalb nicht, dass es keine Nutzer gibt. + +## 5 — Dependency-Graph, MCP, Mermaid-Indizes? + +- **5.1 Calls, References, Implementations, Dependencies — PARTIAL.** `callers`, `callees`, `references` und `impact` sind VERIFIED und geben Masken zurück (query.rs:125/157/188/239). Es fehlen eine Vorwärts-Closure und **Crate-Dependencies**: kein Prädikat, und die CRATE-Zeile ist nur ein Vollständigkeitsstatus. +- **5.2 MCP — MISSING.** `Cargo.toml` und `*.rs` enthalten kein `mcp`, `rmcp` oder `jsonrpc`. Slice D ist laut Plan bewusst als letztes geplant. Für einen Adapter ist eine Hürde zu beachten: `callers` und `callees` nehmen `&mut self`. +- **5.3 `.index.md` + Mermaid — MISSING.** Die Präzedenz existiert: `entries_index.py`, `supersession_index.py` und die CI-Regeneration mit Diff. Ein Generator muss nach Namen sortieren, weil Positionen von der Lesereihenfolge abhängen. +- **5.4–5.6 — Empfehlung, kein Code.** Ein Index pro Crate, Mermaid nur auf Crate- oder Modul-Ebene. Alles, was größen- oder zustandsabhängig ist, holt ein Agent per Query: Bodies, Masken, Closures, Diagnosen, Residuals. Die CI prüft gegen die committeten Rows, nie per Neukompilat. + +## 6 — Was beweist PR #1923? + +- **6.1 Implementiert und getestet — VERIFIED:** + - dt64 `MapTable::locate`, `map_at`, `TileMap` und `VersionClock` mit F1–F6 (`tests/dr.rs:342-577`); + - xlog `closure_wide`, `new_tiled`, Multiset-DeltaLog, merge-on-read sowie refold, refresh und rebase; + - Codegraph `refresh_crate`, Digest-Skip und interned Lanes. + + Gemessen und **gescheitert**: der unveränderte Refresh mit 1,5–1,7 ms. +- **6.2 Alte und neue Bindings, spät eintreffende IDs — VERIFIED für dt64/xlog.** F2: ID 30 ist bei v1 `Unmapped` und bei v2 `Ok(2)`, ohne neue Generation. `extends` verlangt ein Präfix, `rebase` refusiert ein Reorder. Für den Codegraph ist das nur PARTIAL, weil dessen Tests ohne CI laufen. +- **6.3 Kann ein gescheiterter Refresh den Zustand teilweise ändern? — CONTRADICTED.** Der Code dokumentiert das sogar selbst (graph.rs:519-520): + - **Cube-Refusal:** `pos` ist schon erweitert, `advance`, `seal` und `rebase` sind gelaufen (:598-604), und `add_edge`/`retire_edge` werden einzeln mit `?` angewandt (:607-630). Lanes, Masken und Digest folgen erst danach (:658-662). Ein Retry zählt bereits geloggte ADDs im Multiset ein zweites Mal. + - **NeedsReopen („nothing was changed“):** `pos.truncate` setzt nur die Positionen zurück. Die Interner-Einträge aus `read_text_as`/`intern` (:538-540) und `last_owner` (:151-159) bleiben stehen. Der Test prüft die Länge des Interners nicht (refresh.rs:233-234). +- **6.4 Lokalität des Tile-Routings — PARTIAL.** `MapTable::locate` liest alle Tiles linear (dr.rs:936-954), obwohl der Codegraph die Tile als `g/256` kennt (graph.rs:71, 596). Prefetch ist nicht implementiert. +- **6.5 Noch String/TSV — VERIFIED.** Transport, Digest, Symbolidentität, Grade, `?dynamic`, `may_target`, der Pfad und das String-gekeyte Query-API. +- **6.6 Falsifikatoren an den Grenzen — PARTIAL.** dt64, xlog und r2il↔BindStore sind in der CI. Ruff-Rows↔Codegraph und Codegraph↔xlog sind **nicht** in der CI. Keine Falsifikatoren gibt es für Kythe↔DR, R2IL↔Codegraph, Symbol↔classid und die Atomizität. + +## 7 — Kleinster nächster Schritt + +Drei Abweichungen, nach Risiko geordnet. Keine führt einen neuen Store oder eine neue Architektur ein. Alles verwendet vorhandene Teile. + +### P1 — PR #1923 korrekt machen (Defekt, keine Entscheidung nötig) +- **Änderung:** + - `refresh_crate` erst vollständig vorbereiten: Interner-Einträge, Positionen, Edge-Delta und Cube-Prüfung laufen auf Kopien bzw. Staging. Danach wird in einem Schritt angewandt. + - Alternativ wird der vorhandene NeedsReopen-Pfad auch für Cube-Refusals genutzt. Die Interner-Länge und `last_owner` werden dabei zurückgesetzt. + - Dazu kommt eine CI-Zeile `cargo test --manifest-path crates/lance-graph-codegraph/Cargo.toml` in `rust-test.yml` neben :150/:152. +- **Falsifikatoren** (jeder mit Disable-Lauf, rot→grün): + 1. Ein injizierter Cube-Refusal lässt Interner-Länge, `pos`, Lanes, Masken und Digest exakt unverändert. Ein Retry ist byte-gleich zum Clean-Open, ohne doppelte ADD-Multiplizität. + 2. NeedsReopen lässt die Interner-Länge unverändert. Das erweitert den bestehenden Test `refresh.rs:233`. + 3. Die CI-Zeile fällt, wenn ein Codegraph-Test kaputt ist (einmal mit gebrochenem Test verifiziert). + +### P2 — Identitätsnaht Position ↔ Charon ↔ DT64 (braucht eine Entscheidung zum Zeilenformat) +- **Änderung:** + - Die Zeilen (bzw. der Header des Binärformats aus D-CODEGRAPH-0-BIN) bekommen drei Felder: die Charon-`FunDeclId`, die Charon/LLBC-Version und einen LLBC-Digest. + - Ein Aufruf eines Linsenworts bekommt eine eigene Zeile mit Callee = `AlgebraRef` statt eines Namens. Das ist ein neues Prädikat in der bestehenden Codetabelle, kein neuer Store. +- **Falsifikatoren:** + 1. Ein Fixture holt den Body einer Impl-Methode über die ID, ohne Namenssuche. Zwei Impl-Methoden mit gleichem `path_name` müssen verschiedene Bodies liefern. + 2. Auf einem Fixture mit drei Aufrufern eines F000-Worts gilt `users_of(word) == callers(word)`. Ein Aufrufer, der um das Binding herumgeht, muss als Differenz sichtbar werden. + 3. Ein Wechsel der Charon-Version allein macht den Store `stale`, auch wenn die Quelle unverändert ist. + +### P3 — Parallele Methodenautoritäten benennen und absperren (braucht eine Entscheidung zur Zuständigkeit) +- **Bestand:** + - BindStore (Autorität); + - `dr_math_binding_probe::MethodSignature`, tract `Registry`, XLOG `Word`, ogar-loco `FnIndex`; + - der POLYFILL-Positionsabgleich, der nur per Test geführt wird. +- **Änderung, ohne neue Architektur:** + - Ein Census-Test zählt jedes ausführbare Wort in `FoldDialect` und `procs` sowie jedes XLOG-`Word` und verlangt dafür eine BindStore-Deklaration. Ein Wort ohne Deklaration steht in einer exakten Ratchet-Liste, die in beide Richtungen fällt (neu oder veraltet). + - egglog wird auf eine `rev` gepinnt, damit `ProofChanged` eine Bedeutung hat. +- **Falsifikatoren:** + 1. Ein neuer Opcode ohne Deklaration lässt den Census rot werden. + 2. Ein Eintrag, der nicht mehr gilt, lässt die Ratchet rot werden. + 3. Ein egglog-Wechsel ohne Pin-Bump wird vom Build abgelehnt. + +### Ausdrücklich nicht jetzt +- **MCP, `.index.md`, Mermaid:** Erst nach P1/P2. Ein Index über instabile, String-gekeyte Identitäten würde nur Drift erzeugen. +- **MLIR:** Erst wenn ein konkreter Op fehlt. Dann als Alias oder F###-Linse, nie als neue Ausführungsschicht. +- **Prefetch:** Erst nach dem direkten Tile-Index (6.4), und erst gemessen. +- **D-CODEGRAPH-0-BIN** (Binärformat, Zählen über Quack) bleibt der Fix für das gerissene Refresh-Gate. Er sollte das Feldset aus P2 in den Header aufnehmen, damit das Format nicht zweimal migriert wird. + +## Offene Entscheidungen +- **P2:** Trägt das Binärformat FunDeclId, Charon-Version und LLBC-Digest im Header oder je Zeile? Bekommen Linsenwort-Calls ein eigenes Prädikat? +- **P3:** Werden `MethodSignature` (dr-math-binding-probe) und die tract-`Registry` auf BindStore migriert, oder als Lab-Autoritäten ausdrücklich ausgenommen? diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 98d9b3ffd..2d0c7e726 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,11 +25,12 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -545 entries, 2026-08-06 .. 2026-11-02. +546 entries, 2026-08-06 .. 2026-11-02. | date | entry id | finding | file | |---|---|---|---| | 2026-11-02 | `D-CODEGRAPH-0` | | [2026-11-02-d-codegraph-0-gap-map.md](2026-11-02-d-codegraph-0-gap-map.md) | +| 2026-11-02 | `codegraph-dr3-dt64-ruff-architecture-review` | | [2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md](2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md) | | 2026-11-01 | `D-XLOG-NATIVE-7` | | [2026-11-01-xlog-resident-hypercube-context.md](2026-11-01-xlog-resident-hypercube-context.md) | | 2026-11-01 | `D-XLOG-NATIVE-8` | | [2026-11-01-xlog-behaviour-lens-f100.md](2026-11-01-xlog-behaviour-lens-f100.md) | | 2026-11-01 | `D-DT64-7-5` | | [2026-11-01-maptable-versioned-local-map.md](2026-11-01-maptable-versioned-local-map.md) | From fbd133b208208b20d83c944f868cd456737c8f3d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 10:44:34 +0000 Subject: [PATCH 14/19] codegraph: atomic refresh_crate (prepare, validate, commit); xlog Cube::stage/commit, binds keep their global id refresh_crate now stages the interner (SymbolStage), the positions (PosStage), the grown tile map (not handed to the table), the edge batch (Cube::stage, checked without changing the cube) and the closure refresh (Cube::stage_closure). A refusal at any of these changes nothing: no interner entry, position, last_owner, table version, frame, log entry, lane, mask or digest. The commit (table advance + seal, Cube::commit, absorb, closures, lanes) has no error path. Cube::rebase checks before it changes anything. Node::Bind keeps the global id it binds, and an extending rebase or commit places a value that was unmapped when the view was built (codex P2 on #1923). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/lance-graph-codegraph/src/artifact.rs | 113 +++++- crates/lance-graph-codegraph/src/graph.rs | 377 +++++++++++++------ crates/lance-graph-dt64/src/dr.rs | 11 + crates/lance-graph-xlog/src/cube.rs | 274 +++++++++++++- 4 files changed, 639 insertions(+), 136 deletions(-) diff --git a/crates/lance-graph-codegraph/src/artifact.rs b/crates/lance-graph-codegraph/src/artifact.rs index c9a02d33a..1c7df34b7 100644 --- a/crates/lance-graph-codegraph/src/artifact.rs +++ b/crates/lance-graph-codegraph/src/artifact.rs @@ -178,6 +178,115 @@ impl Symbols { pub fn is_empty(&self) -> bool { self.names.is_empty() } + + /// Append the names a [`SymbolStage`] over this interner allocated, in + /// order: they receive exactly the ids the stage handed out, because the + /// interner did not change while the stage borrowed it. Cannot fail. + pub fn absorb(&mut self, staged: StagedNames) { + debug_assert_eq!( + staged.base_len, + self.names.len(), + "interner changed under the stage" + ); + for name in staged.names { + let id = self.names.len() as u32; + self.index.insert(name.clone(), id); + self.names.push(name); + } + } +} + +/// What reading rows needs of an interner. [`Symbols`] writes into itself; +/// a [`SymbolStage`] leaves its base untouched and collects new names. +pub trait Interner { + /// The id of `s`, interning it on first sight. + fn intern(&mut self, s: &str) -> u32; + /// The id of `s` if it was interned. + fn get(&self, s: &str) -> Option; + /// The text of `id`; empty for an id never issued. + fn name(&self, id: u32) -> &str; + /// Distinct names interned. + fn len(&self) -> usize; +} + +impl Interner for Symbols { + fn intern(&mut self, s: &str) -> u32 { + Symbols::intern(self, s) + } + fn get(&self, s: &str) -> Option { + Symbols::get(self, s) + } + fn name(&self, id: u32) -> &str { + Symbols::name(self, id) + } + fn len(&self) -> usize { + Symbols::len(self) + } +} + +/// An interner over a borrowed [`Symbols`] that never writes to it: a name +/// the base knows keeps its id, a new name gets the next id past the base +/// and is held here until [`Symbols::absorb`] (a refresh that commits) or +/// dropped (a refresh that refuses, leaving the base exactly as it was). +pub struct SymbolStage<'a> { + base: &'a Symbols, + names: Vec>, + index: HashMap, u32, FastBuild>, +} + +/// The names a dropped-borrow [`SymbolStage`] allocated, for [`Symbols::absorb`]. +pub struct StagedNames { + base_len: usize, + names: Vec>, +} + +impl<'a> SymbolStage<'a> { + /// A stage over `base`. + pub fn new(base: &'a Symbols) -> Self { + Self { + base, + names: Vec::new(), + index: HashMap::default(), + } + } + + /// Names allocated so far. + pub fn added(&self) -> usize { + self.names.len() + } + + /// The allocated names, releasing the borrow of the base. + pub fn into_names(self) -> StagedNames { + StagedNames { + base_len: self.base.len(), + names: self.names, + } + } +} + +impl Interner for SymbolStage<'_> { + fn intern(&mut self, s: &str) -> u32 { + if let Some(id) = self.get(s) { + return id; + } + let id = (self.base.len() + self.names.len()) as u32; + self.names.push(s.into()); + self.index.insert(s.into(), id); + id + } + fn get(&self, s: &str) -> Option { + self.base.get(s).or_else(|| self.index.get(s).copied()) + } + fn name(&self, id: u32) -> &str { + let id = id as usize; + match id.checked_sub(self.base.len()) { + None => self.base.name(id as u32), + Some(k) => self.names.get(k).map_or("", |b| &**b), + } + } + fn len(&self) -> usize { + self.base.len() + self.names.len() + } } /// All rows of a directory as parallel integer lanes; index `i` of every lane @@ -387,7 +496,7 @@ impl<'a, const N: usize> Memo<'a, N> { } } - fn id(&mut self, s: &'a str, sym: &mut Symbols) -> u32 { + fn id(&mut self, s: &'a str, sym: &mut impl Interner) -> u32 { for &(t, id) in &self.slots[..self.filled] { if t == s { return id; @@ -415,7 +524,7 @@ pub fn read_text( pub fn read_text_as( text: &str, krate: &str, - symbols: &mut Symbols, + symbols: &mut impl Interner, lanes: &mut Lanes, ) -> Result<(), IntakeRefusal> { let mut lines = text.lines(); diff --git a/crates/lance-graph-codegraph/src/graph.rs b/crates/lance-graph-codegraph/src/graph.rs index 9e00d54dc..0358e2108 100644 --- a/crates/lance-graph-codegraph/src/graph.rs +++ b/crates/lance-graph-codegraph/src/graph.rs @@ -7,12 +7,12 @@ use std::collections::HashMap; use std::path::Path; use lance_graph_dt64::dr::{Axis, LocalMap, MapRefusal, MapTable, TileId, Width}; -use lance_graph_xlog::cube::{Cube, CubeRefusal, RelId}; +use lance_graph_xlog::cube::{Cube, CubeRefusal, EdgeOp, RelId, StagedEdges}; use lance_graph_xlog::wide::{WideRows, WideStats}; use crate::artifact::{ - self, Lanes, Symbols, NO_SPAN, P_CALLS, P_CRATE, P_DEFINES, P_IMPLEMENTS, P_MAY, P_PUBLIC, - P_REFERENCES, P_SYNTHETIC, P_UNRESOLVED, + self, Interner, Lanes, SymbolStage, Symbols, NO_SPAN, P_CALLS, P_CRATE, P_DEFINES, + P_IMPLEMENTS, P_MAY, P_PUBLIC, P_REFERENCES, P_SYNTHETIC, P_UNRESOLVED, }; /// A symbol's position: its interning index, which is also its cube coordinate. @@ -73,7 +73,7 @@ const NONE: u32 = u32::MAX; const DYNAMIC: &str = "?dynamic"; /// Position masks, one bit per position, `words()` words each. -#[derive(Default)] +#[derive(Debug, Default)] struct Masks { fns: Vec, public: Vec, @@ -92,7 +92,10 @@ struct Masks { /// so the two numberings are kept apart: `of_id` maps an interner id to its /// position (`NONE` when the text is not a symbol), `id_of` is the inverse. /// Query time does one interner lookup and one array read, never a name map. -#[derive(Default)] +/// +/// Positions are only ever assigned through a [`PosStage`] and adopted with +/// [`Positions::absorb`], so a refused refresh leaves this table untouched. +#[derive(Debug, Default)] struct Positions { of_id: Vec, id_of: Vec, @@ -103,28 +106,70 @@ struct Positions { last_owner: u32, } -impl Positions { +/// Reading a position: the committed table, or a stage over it. +trait PosAt { + /// The position of interner id `id`, `None` when it is not a symbol. + fn at(&self, id: u32) -> Option; + /// Positions assigned. + fn len(&self) -> usize; +} + +impl PosAt for Positions { + fn at(&self, id: u32) -> Option { + self.of_id.get(id as usize).copied().filter(|&p| p != NONE) + } fn len(&self) -> usize { self.id_of.len() } +} - /// The position of interner id `id`, `None` when it is not a symbol. +/// New positions over a borrowed [`Positions`], which is never written: the +/// stage holds the appended `(id, owner)` pairs and its own `last_owner` +/// until [`Positions::absorb`]. +struct PosStage<'a> { + base: &'a Positions, + ids: Vec, + owners: Vec, + of_new: HashMap, + last_owner: u32, +} + +/// What a [`PosStage`] assigned, without the borrow. +struct StagedPositions { + base_len: usize, + ids: Vec, + owners: Vec, + last_owner: u32, +} + +impl PosAt for PosStage<'_> { fn at(&self, id: u32) -> Option { - self.of_id.get(id as usize).copied().filter(|&p| p != NONE) + self.base.at(id).or_else(|| self.of_new.get(&id).copied()) + } + fn len(&self) -> usize { + self.base.len() + self.ids.len() } +} - /// Assign `id` the next position unless it already has one. - fn assign(&mut self, id: u32, symbols: &mut Symbols) -> Pos { - if self.of_id.len() <= id as usize { - self.of_id.resize(symbols.len().max(id as usize + 1), NONE); +impl<'a> PosStage<'a> { + fn new(base: &'a Positions) -> Self { + Self { + base, + ids: Vec::new(), + owners: Vec::new(), + of_new: HashMap::new(), + last_owner: base.last_owner, } - let cur = self.of_id[id as usize]; - if cur != NONE { - return cur; + } + + /// Assign `id` the next position unless it already has one. + fn assign(&mut self, id: u32, symbols: &mut impl Interner) -> Pos { + if let Some(p) = self.at(id) { + return p; } - let p = self.id_of.len() as Pos; - self.of_id[id as usize] = p; - self.id_of.push(id); + let p = self.len() as Pos; + self.of_new.insert(id, p); + self.ids.push(id); let owner = { let name = symbols.name(id); match name.find("::") { @@ -143,19 +188,44 @@ impl Positions { } }; self.last_owner = owner; - self.owner.push(owner); + self.owners.push(owner); p } - /// Undo every assignment from position `n` on (a refused refresh). - fn truncate(&mut self, n: usize) { - for &id in &self.id_of[n..] { - if let Some(slot) = self.of_id.get_mut(id as usize) { - *slot = NONE; + fn finish(self) -> StagedPositions { + StagedPositions { + base_len: self.base.len(), + ids: self.ids, + owners: self.owners, + last_owner: self.last_owner, + } + } +} + +impl Positions { + fn new() -> Self { + Self { + last_owner: NONE, + ..Self::default() + } + } + + /// Adopt what a stage over this table assigned. Cannot fail. + fn absorb(&mut self, staged: StagedPositions) { + debug_assert_eq!( + staged.base_len, + self.id_of.len(), + "positions changed under the stage" + ); + for id in staged.ids { + if self.of_id.len() <= id as usize { + self.of_id.resize(id as usize + 1, NONE); } + self.of_id[id as usize] = self.id_of.len() as Pos; + self.id_of.push(id); } - self.id_of.truncate(n); - self.owner.truncate(n); + self.owner.extend(staged.owners); + self.last_owner = staged.last_owner; } } @@ -175,6 +245,20 @@ pub struct Codegraph { masks: Masks, rconf: OnceCell<(WideRows, WideStats)>, rposs: OnceCell<(WideRows, WideStats)>, + /// Test-only: make the next refresh's validation refuse at a chosen + /// point, so the falsifiers can show a refusal changes nothing. + #[cfg(test)] + inject: Option, +} + +/// Where a test makes a refresh's validation refuse. +#[cfg(test)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Inject { + /// As if `Cube::stage` refused the batch. + Stage, + /// As if `Cube::stage_closure` refused. + Closure, } fn set(words: &mut [u64], p: Pos) { @@ -190,8 +274,8 @@ fn set(words: &mut [u64], p: Pos) { fn resolve( lanes: &Lanes, rows: std::ops::Range, - symbols: &mut Symbols, - pos: &mut Positions, + symbols: &mut impl Interner, + pos: &mut PosStage<'_>, ) -> bool { let mut any_dynamic = false; for i in rows { @@ -217,12 +301,12 @@ fn resolve( } /// Add the `?dynamic` position (idempotent). -fn add_dynamic(symbols: &mut Symbols, pos: &mut Positions) { +fn add_dynamic(symbols: &mut impl Interner, pos: &mut PosStage<'_>) { let d = symbols.intern(DYNAMIC); pos.assign(d, symbols); } -fn dynamic_pos(symbols: &Symbols, pos: &Positions) -> Option { +fn dynamic_pos(symbols: &impl Interner, pos: &impl PosAt) -> Option { symbols.get(DYNAMIC).and_then(|id| pos.at(id)) } @@ -248,7 +332,7 @@ fn crate_statuses(lanes: &Lanes, symbols: &Symbols) -> HashMap { fn edges_of( lanes: &Lanes, rows: impl Iterator, - pos: &Positions, + pos: &impl PosAt, dyn_pos: Option, ) -> [Vec<(u32, u32)>; 6] { let mut rel: [Vec<(u32, u32)>; 6] = Default::default(); @@ -416,13 +500,15 @@ impl Codegraph { pub fn open(dir: &Path) -> Result { let mut symbols = Symbols::default(); let lanes = artifact::read_dir(dir, &mut symbols)?; - let mut pos = Positions { - last_owner: NONE, - ..Positions::default() + let mut pos = Positions::new(); + let staged = { + let mut stage = PosStage::new(&pos); + if resolve(&lanes, 0..lanes.len(), &mut symbols, &mut stage) { + add_dynamic(&mut symbols, &mut stage); + } + stage.finish() }; - if resolve(&lanes, 0..lanes.len(), &mut symbols, &mut pos) { - add_dynamic(&mut symbols, &mut pos); - } + pos.absorb(staged); let crates = crate_statuses(&lanes, &symbols); let n = pos.len(); let mut rel = edges_of(&lanes, 0..lanes.len(), &pos, dynamic_pos(&symbols, &pos)); @@ -471,6 +557,8 @@ impl Codegraph { masks: m, rconf: OnceCell::new(), rposs: OnceCell::new(), + #[cfg(test)] + inject: None, }) } @@ -516,8 +604,18 @@ impl Codegraph { /// folds base + log. No cache means nothing to refresh. /// * Masks and crate statuses are recomputed from the lanes. /// - /// Refusals before the first cube mutation leave the graph unchanged; a - /// `Cube` refusal afterwards (not expected) may leave it partly updated. + /// **Atomic: prepare → validate → commit.** Every step that can refuse + /// (reading the file, interning, assigning positions, the reservation, + /// the grown tile map, placing every logged edge, the closure refresh) + /// runs against staged state: a [`SymbolStage`] and a [`PosStage`] over + /// the graph's interner and positions, a grown [`LocalMap`] the table has + /// not yet been given, and a batch [`Cube::stage`] checked without + /// changing the cube. A refusal therefore changes nothing: not the + /// interner, the positions, `last_owner`, the table (no version is + /// issued and no frame sealed), the cube, its logs, the lanes, the masks + /// or the digests. Only then does the commit run, and it has no error + /// path: the table advances and seals, the cube adopts the frame and the + /// batch, the stages are absorbed, the closures and lanes replaced. pub fn refresh_crate( &mut self, dir: &Path, @@ -534,16 +632,19 @@ impl Codegraph { return Ok(RefreshStats::default()); } } + + // ---- prepare: everything below borrows the graph immutably ---- + let mut syms = SymbolStage::new(&self.symbols); let mut fresh = Lanes::default(); - artifact::read_text_as(&text, krate, &mut self.symbols, &mut fresh) + artifact::read_text_as(&text, krate, &mut syms, &mut fresh) .map_err(RefreshRefusal::Intake)?; - let kid = self.symbols.intern(krate); + let kid = syms.intern(krate); // Edges of the crate's old rows, before any new position exists. Every // other crate's rows are in both the old and the new lane set, so the // whole-graph multiset difference equals this crate's old rows against // its new rows. - let old_dyn = dynamic_pos(&self.symbols, &self.pos); + let old_dyn = dynamic_pos(&syms, &self.pos); let old_e = edges_of( &self.lanes, (0..self.lanes.len()).filter(|&i| self.lanes.krate[i] == kid), @@ -555,104 +656,111 @@ impl Codegraph { // already positioned), appended in fresh-row order, `?dynamic` last: // the same order a rebuilt lane set would intern in. let n_old = self.pos.len(); - if resolve(&fresh, 0..fresh.len(), &mut self.symbols, &mut self.pos) { - add_dynamic(&mut self.symbols, &mut self.pos); + let mut pos = PosStage::new(&self.pos); + if resolve(&fresh, 0..fresh.len(), &mut syms, &mut pos) { + add_dynamic(&mut syms, &mut pos); } - let n_new = self.pos.len(); + let n_new = pos.len(); if n_new > self.axis.reserved() { - self.pos.truncate(n_old); return Err(RefreshRefusal::NeedsReopen { reason: "new symbols exceed the positions reserved at open (a new tile, or a W64 tile becoming W256)", }); } - let new_e = edges_of( - &fresh, - 0..fresh.len(), - &self.pos, - dynamic_pos(&self.symbols, &self.pos), - ); - let mut retires: Vec> = Vec::new(); - let mut adds: Vec> = Vec::new(); - for r in 0..6 { - let (mut gone, mut came) = (Vec::new(), Vec::new()); - diff_sorted(&keys(&old_e[r]), &keys(&new_e[r]), &mut gone, &mut came); - retires.push(gone); - adds.push(came); - } - - let pending = [ - self.cube.delta_len(RCONF) > 0, - self.cube.delta_len(RPOSS) > 0, - ]; + let new_e = edges_of(&fresh, 0..fresh.len(), &pos, dynamic_pos(&syms, &pos)); + // The batch in apply order: relation by relation in id order; NODE + // adds for the appended positions; otherwise retirements then + // additions, each ascending by (src, dst). + let mut ops: Vec<(RelId, EdgeOp, u32, u32)> = Vec::new(); let mut stats = RefreshStats { symbols_appended: (n_new - n_old) as u64, ..RefreshStats::default() }; - - // Extend the last tile's map (ids ascending) and adopt the new frame. - if n_new > n_old { - let last = self.axis.tiles.saturating_sub(1); - let lo = last as u64 * TILE_POSITIONS as u64; - let ids: Vec = (lo..n_new as u64).collect(); - let map = match LocalMap::prepare(self.axis.tile, &ids) { - Ok(m) => m, - Err(e) => { - self.pos.truncate(n_old); - return Err(RefreshRefusal::Intake(IntakeRefusal::Map(e))); - } - }; - let rev = self.table.revisions(); - self.table - .advance(rev.source + 1, rev.relation_map, vec![(last, Some(map))]); - let frame = self.table.seal(); - self.cube - .rebase(&self.table, &frame) - .map_err(RefreshRefusal::Cube)?; - } - for r in 0..6usize { let rel = r as RelId; if rel == NODE { - for p in n_old as u32..n_new as u32 { - self.cube - .add_edge(&self.table, NODE, p, p) - .map_err(RefreshRefusal::Cube)?; - } + ops.extend((n_old as u32..n_new as u32).map(|p| (NODE, EdgeOp::Add, p, p))); continue; } - for &key in &retires[r] { - let (s, d) = ((key >> 32) as u32, key as u32); - self.cube - .retire_edge(&self.table, rel, s, d) - .map_err(RefreshRefusal::Cube)?; - stats.edges_retired += 1; - } - for &key in &adds[r] { - let (s, d) = ((key >> 32) as u32, key as u32); - self.cube - .add_edge(&self.table, rel, s, d) - .map_err(RefreshRefusal::Cube)?; - stats.edges_added += 1; - } + let (mut gone, mut came) = (Vec::new(), Vec::new()); + diff_sorted(&keys(&old_e[r]), &keys(&new_e[r]), &mut gone, &mut came); + stats.edges_retired += gone.len() as u64; + stats.edges_added += came.len() as u64; + let split = |k: u64| ((k >> 32) as u32, k as u32); + ops.extend( + gone.into_iter() + .map(split) + .map(|(s, d)| (rel, EdgeOp::Retire, s, d)), + ); + ops.extend( + came.into_iter() + .map(split) + .map(|(s, d)| (rel, EdgeOp::Add, s, d)), + ); } - let changed = stats.edges_added + stats.edges_retired > 0 || n_new > n_old; - for (rel, pend) in [(RCONF, pending[0]), (RPOSS, pending[1])] { - let cell = if rel == RCONF { - &mut self.rconf - } else { - &mut self.rposs - }; - if !changed { - continue; + // The last tile's grown map (ids ascending), not yet in the table. + let last = self.axis.tiles.saturating_sub(1); + let grown = if n_new > n_old { + let lo = last as u64 * TILE_POSITIONS as u64; + let ids: Vec = (lo..n_new as u64).collect(); + Some( + LocalMap::prepare(self.axis.tile, &ids) + .map_err(|e| RefreshRefusal::Intake(IntakeRefusal::Map(e)))?, + ) + } else { + None + }; + + // ---- validate: the cube checks the batch without changing ---- + let staged = self + .stage_batch(grown.as_ref().map(|m| (last, m)), &ops) + .map_err(RefreshRefusal::Cube)?; + let changed = !staged.is_empty(); + // A cached closure is refreshed only when its relation's log was + // empty (its precondition is a closure of the base); otherwise it is + // dropped and rebuilt on next read. Computed now, so a refusal here + // also changes nothing. + let mut closures: [Option>; 2] = [None, None]; + if changed { + for (slot, rel) in [(0usize, RCONF), (1, RPOSS)] { + let cell = if rel == RCONF { + &self.rconf + } else { + &self.rposs + }; + if let Some(old) = cell.get() { + closures[slot] = Some(if self.cube.delta_len(rel) == 0 { + Some( + self.stage_closure(rel, &old.0, &staged) + .map_err(RefreshRefusal::Cube)?, + ) + } else { + None + }); + } + } + } + let names = syms.into_names(); + let positions = pos.finish(); + + // ---- commit: no error path below ---- + self.symbols.absorb(names); + self.pos.absorb(positions); + let frame = match grown { + Some(map) => { + let rev = self.table.revisions(); + self.table + .advance(rev.source + 1, rev.relation_map, vec![(last, Some(map))]); + self.table.seal() } - if let Some(old) = cell.take() { - if !pend { - let (rows, st) = self - .cube - .refresh_closure(&self.table, rel, &old.0) - .map_err(RefreshRefusal::Cube)?; + None => self.cube.frame().clone(), + }; + self.cube.commit(&self.table, &frame, staged); + for (slot, cell) in [(0usize, &mut self.rconf), (1, &mut self.rposs)] { + if let Some(next) = closures[slot].take() { + cell.take(); + if let Some((rows, st)) = next { stats.rows_recomputed += st.rows_recomputed; let _ = cell.set((rows, st)); } @@ -668,6 +776,33 @@ impl Codegraph { Ok(stats) } + /// [`Cube::stage`], or the refusal a test injected in its place. + fn stage_batch( + &self, + grown: Option<(TileId, &LocalMap)>, + ops: &[(RelId, EdgeOp, u32, u32)], + ) -> Result { + #[cfg(test)] + if self.inject == Some(Inject::Stage) { + return Err(CubeRefusal::Renumbered(0)); + } + self.cube.stage(&self.table, grown, ops) + } + + /// [`Cube::stage_closure`], or the refusal a test injected in its place. + fn stage_closure( + &self, + rel: RelId, + old: &WideRows, + staged: &StagedEdges, + ) -> Result<(WideRows, WideStats), CubeRefusal> { + #[cfg(test)] + if self.inject == Some(Inject::Closure) { + return Err(CubeRefusal::FixpointBound); + } + self.cube.stage_closure(rel, old, staged) + } + /// Absorb every relation's delta log into its base lanes (explicit only; /// nothing calls this on a threshold). Cached closures stay valid: the /// base now equals the effective edges they were computed from. diff --git a/crates/lance-graph-dt64/src/dr.rs b/crates/lance-graph-dt64/src/dr.rs index a2f330361..c1af50857 100644 --- a/crates/lance-graph-dt64/src/dr.rs +++ b/crates/lance-graph-dt64/src/dr.rs @@ -578,6 +578,17 @@ impl LocalMap { }) } + /// This map, whole, as a [`TileMap`]: what a table would read for it + /// once it is appended. Lets a caller validate a map it has not yet + /// handed to [`MapTable::advance`] (an extension check, a lookup) + /// without advancing the table or issuing a version. + pub fn view(&self) -> TileMap<'_> { + TileMap { + map: self, + len: self.globals.len(), + } + } + pub fn width(&self) -> Width { self.width } diff --git a/crates/lance-graph-xlog/src/cube.rs b/crates/lance-graph-xlog/src/cube.rs index ed44ed874..5c9fb82c9 100644 --- a/crates/lance-graph-xlog/src/cube.rs +++ b/crates/lance-graph-xlog/src/cube.rs @@ -37,7 +37,9 @@ use std::cell::{Cell, OnceCell}; use std::sync::atomic::{AtomicU64, Ordering}; -use lance_graph_dt64::dr::{Frame, LocateRefusal, MapRead, MapRefusal, MapTable, TileId, TileMap}; +use lance_graph_dt64::dr::{ + Frame, LocalMap, LocateRefusal, MapRead, MapRefusal, MapTable, TileId, TileMap, +}; use lance_graph_mask_risc::{LaneRef, Out}; use lance_graph_quack::{Agg, Cmp, Col, Filter, GroupAddr, GroupAgg, Mask, Query}; use ndarray::simd::{checked_binary_i64, mask_and, mask_andnot, mask_any, mask_or_assign, I64Op}; @@ -55,15 +57,53 @@ pub struct View { node: u32, } +/// One logged edge operation of a [`Cube::stage`] batch. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EdgeOp { + /// [`Cube::add_edge`]. + Add, + /// [`Cube::retire_edge`]. + Retire, +} + +/// A batch of edge operations [`Cube::stage`] has checked, placed at their +/// coordinates and not yet applied. Holds no borrow, so the caller may +/// advance the table between staging and [`Cube::commit`]. +#[derive(Debug)] +pub struct StagedEdges { + cube: u64, + /// The grown tile and its staged map length. + grown: Option<(TileId, usize)>, + n: usize, + ops: Vec<(RelId, EdgeOp, u32, u32)>, + rebind: Vec<(usize, u32)>, +} + +impl StagedEdges { + /// Operations in the batch. + pub fn len(&self) -> usize { + self.ops.len() + } + + /// Whether the batch holds no operation. + pub fn is_empty(&self) -> bool { + self.ops.is_empty() + } +} + #[derive(Clone, Copy, Debug)] enum Node { Scan(RelId), - /// `col == value`. `None`: the value is not in the map, so no row - /// matches. + /// `col == global`. `value` is `global`'s coordinate under the cube's + /// frame, `None` while no tile holds it (no row matches). The global id + /// is kept so an extending rebase can place a value that was unmapped + /// when the view was built: the view means "column equals this id", and + /// keeps meaning it under every later frame. Bind { rel: RelId, col: u8, value: Option, + global: u32, }, /// `left.lcol == right.rcol`. Join { @@ -345,12 +385,18 @@ impl Cube { }; check_col(col)?; // A value no tile holds binds to nothing; any other refusal is real. + let global = value; let value = match self.coord(table, value) { Ok(c) => Some(c), Err(CubeRefusal::Domain(_) | CubeRefusal::Locate(LocateRefusal::Unmapped)) => None, Err(e) => return Err(e), }; - Ok(self.push(Node::Bind { rel, col, value })) + Ok(self.push(Node::Bind { + rel, + col, + value, + global, + })) } /// `left.lcol == right.rcol`, where each side is a scan or a bind. @@ -376,7 +422,9 @@ impl Cube { fn side(&self, v: View, join_col: u8) -> Result { match self.node(v)? { Node::Scan(rel) => Ok(Side { rel, bound: None }), - Node::Bind { rel, col, value } if col != join_col => Ok(Side { + Node::Bind { + rel, col, value, .. + } if col != join_col => Ok(Side { rel, bound: Some((col, value)), }), @@ -434,6 +482,74 @@ impl Cube { Ok(u32::from(l)) } + /// Every tile the cube spans, read under `frame`, in coordinate order. + fn views_under<'t>( + &self, + table: &'t MapTable, + frame: &Frame, + ) -> Result>, CubeRefusal> { + self.tiles + .iter() + .map(|&t| table.map(frame, t).map_err(CubeRefusal::Map)) + .collect() + } + + /// [`Cube::coord`] over explicit tile views (in `self.tiles` order) + /// instead of the table under the cube's frame: the same placement and + /// the same refusals, so a staged layout places ids exactly as the + /// committed one will. + fn coord_in(&self, views: &[TileMap<'_>], id: u32) -> Result { + let g = u64::from(id); + if self.tiled { + let mut found = None; + for (ordinal, v) in views.iter().enumerate() { + if let Ok(local) = v.local(g) { + if found.is_some() { + return Err(CubeRefusal::Locate(LocateRefusal::Ambiguous)); + } + let ordinal = u32::try_from(ordinal) + .map_err(|_| CubeRefusal::Locate(LocateRefusal::Ambiguous))?; + found = Some(ordinal * BLOCK_POSITIONS as u32 + u32::from(local)); + } + } + return found.ok_or(CubeRefusal::Locate(LocateRefusal::Unmapped)); + } + let view = views + .first() + .ok_or(CubeRefusal::Map(MapRead::NoSuchTile(self.tile)))?; + view.local(g).map(u32::from).map_err(CubeRefusal::Domain) + } + + /// The binds that were unmapped and are placed under `views`: `(node, + /// coordinate)`. A bind that was mapped keeps its coordinate, because a + /// layout is only adopted when it extends the old one (no renumbering). + fn rebind(&self, views: &[TileMap<'_>]) -> Result, CubeRefusal> { + let mut out = Vec::new(); + for (i, node) in self.nodes.iter().enumerate() { + if let Node::Bind { + value: None, + global, + .. + } = *node + { + match self.coord_in(views, global) { + Ok(c) => out.push((i, c)), + Err(CubeRefusal::Domain(_) | CubeRefusal::Locate(LocateRefusal::Unmapped)) => {} + Err(e) => return Err(e), + } + } + } + Ok(out) + } + + fn apply_rebind(&mut self, rebind: &[(usize, u32)]) { + for &(i, c) in rebind { + if let Some(Node::Bind { value, .. }) = self.nodes.get_mut(i) { + *value = Some(c); + } + } + } + /// The lanes every read goes through: relation `rel`'s base folded with /// its delta log ("sparse write, merge on read"). With an empty log these /// are the base lanes themselves, no copy. Otherwise they are the cached @@ -670,6 +786,16 @@ impl Cube { Ok((self.coord(table, src)?, self.coord(table, dst)?)) } + /// Relation `rel`'s delta log (read-only), `None` for an unknown relation. + pub fn delta_log(&self, rel: RelId) -> Option<&DeltaLog> { + self.logs.get(rel as usize) + } + + /// The frame the cube reads its tile maps under. + pub fn frame(&self) -> &Frame { + &self.frame + } + /// Logged operations on relation `rel` not yet folded into its base. /// An unknown relation has none. pub fn delta_len(&self, rel: RelId) -> usize { @@ -720,6 +846,127 @@ impl Cube { Ok(()) } + /// Checks a batch of edge operations, optionally under one grown tile map, + /// against the cube **without changing it or the table**: the grown map + /// must extend the tile's current map (as [`Cube::rebase`] requires), and + /// every id must be placed under the resulting layout (as + /// [`Cube::add_edge`] requires). The map is not handed to the table, so + /// no version is issued and no frame is sealed. The returned batch is + /// applied by [`Cube::commit`], which cannot refuse. + pub fn stage( + &self, + table: &MapTable, + grown: Option<(TileId, &LocalMap)>, + ops: &[(RelId, EdgeOp, u32, u32)], + ) -> Result { + let mut views = self.views_under(table, &self.frame)?; + if let Some((t, map)) = grown { + let i = self + .tiles + .iter() + .position(|&x| x == t) + .ok_or(CubeRefusal::Map(MapRead::NoSuchTile(t)))?; + let candidate = map.view(); + if !candidate.extends(views[i]) { + return Err(CubeRefusal::Renumbered(t)); + } + views[i] = candidate; + } + let n = if self.tiled { + self.n + } else { + views.first().map_or(self.n, TileMap::len) + }; + let mut placed = Vec::with_capacity(ops.len()); + for &(rel, op, src, dst) in ops { + self.log(rel)?; + placed.push(( + rel, + op, + self.coord_in(&views, src)?, + self.coord_in(&views, dst)?, + )); + } + let rebind = if grown.is_some() { + self.rebind(&views)? + } else { + Vec::new() + }; + Ok(StagedEdges { + cube: self.id, + grown: grown.map(|(t, m)| (t, m.len())), + n, + ops: placed, + rebind, + }) + } + + /// The closure of relation `rel` as it will be once `staged` is + /// committed: [`Cube::refresh_closure`] over the relation's log followed + /// by the staged operations, at the staged size. Changes nothing, so a + /// caller can compute it before committing. Same precondition: `old` is + /// the closure of the relation's base lanes. + pub fn stage_closure( + &self, + rel: RelId, + old: &WideRows, + staged: &StagedEdges, + ) -> Result<(WideRows, WideStats), CubeRefusal> { + let r = self + .rels + .get(rel as usize) + .ok_or(CubeRefusal::UnknownRel(rel))?; + let mut log = self.log(rel)?.clone(); + for &(r2, op, s, d) in &staged.ops { + if r2 == rel { + match op { + EdgeOp::Add => log.add(s, d), + EdgeOp::Retire => log.retire(s, d), + } + } + } + let n = u32::try_from(staged.n).unwrap_or(u32::MAX); + wide::refresh(old, &r.cols[0], &r.cols[1], &log, n).map_err(CubeRefusal::Wide) + } + + /// Applies a batch [`Cube::stage`] accepted. Cannot refuse: every check + /// ran in `stage`. `frame` must be the frame sealed after the table + /// advanced with exactly the staged tile map (the cube's own frame when + /// nothing grew); the operations are then logged in the order given, the + /// size and any newly placeable binds adopted, and the affected cached + /// folds dropped. A batch staged by another cube is a caller bug and is + /// not applied. + pub fn commit(&mut self, table: &MapTable, frame: &Frame, staged: StagedEdges) { + debug_assert_eq!(staged.cube, self.id, "a batch staged by another cube"); + if staged.cube != self.id { + return; + } + debug_assert!( + staged + .grown + .is_none_or(|(t, len)| table.map(frame, t).is_ok_and(|m| m.len() == len)), + "the frame does not read the staged map" + ); + let _ = table; + if staged.grown.is_some() { + self.n = staged.n; + self.apply_rebind(&staged.rebind); + self.frame = frame.clone(); + for c in &mut self.effective { + c.take(); + } + } + for &(rel, op, s, d) in &staged.ops { + if let Some(l) = self.logs.get_mut(rel as usize) { + match op { + EdgeOp::Add => l.add(s, d), + EdgeOp::Retire => l.retire(s, d), + } + } + self.invalidate(rel); + } + } + /// Adopts `frame`, sealed after the table advanced to add symbols. Allowed /// only if, for every tile the cube holds, the new map is a pure /// extension of the one the cube was built (or last rebased) on @@ -733,20 +980,21 @@ impl Cube { // The table is append-only: the cube's own frame still reads the // layout it was built on, so the old side of the comparison is read // through the index and the cube keeps no copy of it. - let mut first_len = None; - for &t in &self.tiles { - let old = table.map(&self.frame, t).map_err(CubeRefusal::Map)?; - let map = table.map(frame, t).map_err(CubeRefusal::Map)?; - if !map.extends(old) { + // Everything that can refuse is checked before anything changes. + let old = self.views_under(table, &self.frame)?; + let new = self.views_under(table, frame)?; + for ((&t, o), m) in self.tiles.iter().zip(&old).zip(&new) { + if !m.extends(*o) { return Err(CubeRefusal::Renumbered(t)); } - first_len.get_or_insert(map.len()); } + let rebind = self.rebind(&new)?; if !self.tiled { - if let Some(len) = first_len { - self.n = len; + if let Some(m) = new.first() { + self.n = m.len(); } } + self.apply_rebind(&rebind); self.frame = frame.clone(); // `n` may have grown; cached folds were built under the old one. for c in &mut self.effective { From c4bab690407520abcf92180d86a65a32cad5cfaa Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 10:45:55 +0000 Subject: [PATCH 15/19] codegraph, xlog: falsifiers for atomic refresh and bind re-resolution Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/lance-graph-codegraph/src/graph.rs | 194 ++++++++++++++++++ .../lance-graph-xlog/tests/versioned_map.rs | 120 ++++++++++- 2 files changed, 313 insertions(+), 1 deletion(-) diff --git a/crates/lance-graph-codegraph/src/graph.rs b/crates/lance-graph-codegraph/src/graph.rs index 0358e2108..2a9412734 100644 --- a/crates/lance-graph-codegraph/src/graph.rs +++ b/crates/lance-graph-codegraph/src/graph.rs @@ -973,3 +973,197 @@ impl Codegraph { Ok(cell.get_or_init(|| computed)) } } + +/// P1 falsifiers: a refused refresh changes nothing, a retry equals the +/// refresh that never failed, and multiplicities survive. They live here +/// (not in `tests/`) because the refusals are injected through the +/// test-only [`Inject`] hook and the snapshot reads private state. +#[cfg(test)] +mod atomic { + use super::*; + use std::fs; + use std::path::PathBuf; + use std::sync::atomic::{AtomicU32, Ordering}; + + /// Everything a refresh may touch, in a comparable form: interner, + /// positions (incl. `last_owner`), lanes and digests, crate statuses, + /// table version, revisions and per-tile history, the cube's frame and + /// every delta log, masks, and the cached closures. + fn snapshot(g: &Codegraph) -> String { + let names: Vec<&str> = (0..g.symbols.len() as u32) + .map(|i| g.symbols.name(i)) + .collect(); + let mut crates: Vec<(u32, RunStatus)> = g.crates.iter().map(|(&k, &v)| (k, v)).collect(); + crates.sort_by_key(|c| c.0); + let tiles: Vec<(usize, usize)> = (0..g.axis.tiles) + .map(|t| (g.table.generation_count(t), g.table.event_count(t))) + .collect(); + let logs: Vec<_> = (0..6).map(|r| g.cube.delta_log(r).cloned()).collect(); + format!( + "{names:?}\n{:?}\n{:?}\n{crates:?}\n{:?} {:?} {tiles:?}\n{:?}\n{logs:?}\n{:?}\n{:?}\n{:?}", + g.pos, + g.lanes, + g.table.current_version(), + g.table.revisions(), + g.cube.frame(), + g.masks, + g.rconf.get(), + g.rposs.get(), + ) + } + + fn dir(tag: &str) -> PathBuf { + static NEXT: AtomicU32 = AtomicU32::new(0); + let n = NEXT.fetch_add(1, Ordering::Relaxed); + let d = std::env::temp_dir().join(format!("cg_atomic_{}_{tag}_{n}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(&d).expect("temp dir"); + d + } + + const HEAD: &str = "#version 2\n#schema predicate\tsubject\tobject\tgrade\tfile\tstart\tend\n"; + + /// v1: `m::a` calls `m::b` at two sites. v2: three sites, and a new + /// symbol `m::c` that also calls `m::b` (growth inside the reservation). + fn rows(v: u8) -> String { + let mut s = String::from(HEAD); + let mut def = |f: &str, at: u32| { + s.push_str(&format!( + "DEFINES\t{f}\t{f}\tCOMPILER\tm/src/lib.rs\t{at}\t{}\n", + at + 1 + )); + }; + def("m::a", 0); + def("m::b", 10); + if v == 2 { + def("m::c", 20); + } + let sites: &[(&str, u32)] = if v == 1 { + &[("m::a", 2), ("m::a", 4)] + } else { + &[("m::a", 2), ("m::a", 4), ("m::a", 6), ("m::c", 22)] + }; + for &(f, at) in sites { + s.push_str(&format!( + "CALLS_RESOLVED\t{f}\tm::b\tCOMPILER\tm/src/lib.rs\t{at}\t{}\n", + at + 1 + )); + } + s + } + + /// A graph opened at v1 with both closures cached (so the refresh has a + /// closure to stage), and the directory now holding v2. + fn primed(tag: &str) -> (PathBuf, Codegraph) { + let d = dir(tag); + fs::write(d.join("m.rows.tsv"), rows(1)).expect("write"); + let g = Codegraph::open(&d).expect("open"); + g.closure(RCONF).expect("rconf"); + g.closure(RPOSS).expect("rposs"); + fs::write(d.join("m.rows.tsv"), rows(2)).expect("write"); + (d, g) + } + + /// The snapshot can fire: a successful refresh changes it. Without this + /// the "unchanged" assertions below could pass on a blind snapshot. + #[test] + fn a_successful_refresh_changes_the_snapshot() { + let (d, mut g) = primed("fires"); + let before = snapshot(&g); + let st = g.refresh_crate(&d, "m").expect("refresh"); + assert!(st.symbols_appended > 0 && st.edges_added > 0, "{st:?}"); + assert_ne!(snapshot(&g), before); + } + + /// Red if a refused reservation leaves anything behind: the interner + /// entries of the new names, a position, `last_owner` (disable: absorb + /// the staged names and positions before the reservation check). + #[test] + fn a_refused_reservation_changes_nothing() { + let d = dir("reopen"); + let big = |n: usize| { + let mut s = String::from(HEAD); + for i in 0..n { + s.push_str(&format!( + "DEFINES\tbig::f{i}\tbig::f{i}\tCOMPILER\tbig/src/lib.rs\t{i}\t{}\n", + i + 1 + )); + } + s + }; + fs::write(d.join("big.rows.tsv"), big(64)).expect("write"); + let mut g = Codegraph::open(&d).expect("open"); + fs::write(d.join("big.rows.tsv"), big(65)).expect("write"); + // Anti-vacuity: the refused file carries a name the interner lacks. + assert_eq!(g.symbols.get("big::f64"), None); + let before = snapshot(&g); + match g.refresh_crate(&d, "big") { + Err(RefreshRefusal::NeedsReopen { .. }) => {} + other => panic!("expected NeedsReopen, got {other:?}"), + } + assert_eq!(g.symbols.get("big::f64"), None, "interner leak"); + assert_eq!(snapshot(&g), before); + } + + /// Red if a refusal from `Cube::stage` leaves the table advanced, a + /// frame sealed or anything else changed (disable: advance and seal the + /// table before staging the batch). + #[test] + fn an_injected_stage_refusal_changes_nothing() { + let (d, mut g) = primed("stage"); + let before = snapshot(&g); + let v = g.table.current_version(); + g.inject = Some(Inject::Stage); + match g.refresh_crate(&d, "m") { + Err(RefreshRefusal::Cube(_)) => {} + other => panic!("expected the injected Cube refusal, got {other:?}"), + } + assert_eq!(g.table.current_version(), v, "a version was issued"); + assert_eq!(snapshot(&g), before); + } + + /// Red if a refusal while staging a closure refresh comes after any + /// commit step (disable: absorb the staged names and positions before + /// the closure block). + #[test] + fn an_injected_closure_refusal_changes_nothing() { + let (d, mut g) = primed("closure"); + // Anti-vacuity: both closures are cached over empty logs, so the + // refresh does stage them. + assert!(g.rconf.get().is_some() && g.cube.delta_len(RCONF) == 0); + let before = snapshot(&g); + g.inject = Some(Inject::Closure); + assert!(matches!( + g.refresh_crate(&d, "m"), + Err(RefreshRefusal::Cube(_)) + )); + assert_eq!(snapshot(&g), before); + } + + /// Red if a retry after a refusal differs from the refresh that never + /// failed: a second version, a doubled ADD, a doubled name (disable: + /// advance and seal the table before staging). And red if the bag is + /// lost: `m::b` has four call sites, as in a clean open of v2 (disable: + /// deduplicate the keys before `diff_sorted`). + #[test] + fn a_retry_after_a_refusal_equals_the_refresh_that_never_failed() { + let (d, mut g) = primed("retry"); + let (_, mut twin) = primed("twin"); + twin.refresh_crate(&d, "m").expect("twin refresh"); + for inject in [Inject::Stage, Inject::Closure] { + g.inject = Some(inject); + assert!(g.refresh_crate(&d, "m").is_err()); + } + g.inject = None; + g.refresh_crate(&d, "m").expect("retry"); + assert_eq!(snapshot(&g), snapshot(&twin)); + + let clean_dir = dir("clean"); + fs::write(clean_dir.join("m.rows.tsv"), rows(2)).expect("write"); + let mut clean = Codegraph::open(&clean_dir).expect("open clean"); + let (pb, cb) = (g.pos("m::b").expect("b"), clean.pos("m::b").expect("b")); + let (got, want) = (g.callers(pb).expect("q"), clean.callers(cb).expect("q")); + assert_eq!(want.sites, 4, "fixture: four call sites of m::b"); + assert_eq!(got.sites, want.sites, "bag multiplicity"); + } +} diff --git a/crates/lance-graph-xlog/tests/versioned_map.rs b/crates/lance-graph-xlog/tests/versioned_map.rs index 2b55790de..66dcdfebc 100644 --- a/crates/lance-graph-xlog/tests/versioned_map.rs +++ b/crates/lance-graph-xlog/tests/versioned_map.rs @@ -10,7 +10,7 @@ //! stale) turns `an_old_frame_keeps_answering_without_rebase` red. use lance_graph_dt64::dr::{LocalMap, MapTable, Width}; -use lance_graph_xlog::cube::Cube; +use lance_graph_xlog::cube::{Cube, CubeRefusal, EdgeOp}; use lance_graph_xlog::wide::WideRows; const TILE: u32 = 7; @@ -104,3 +104,121 @@ fn rebase_onto_an_extension_answers_over_the_extended_map() { // The new ids are addressable now: a derived relation over id 7 is accepted. assert!(cube.add_relation(&table, &[(6, 7)]).is_ok()); } + +/// One relation `[(1, 2)]` over the single tile `0` mapping `[1, 2]`, and a +/// view `R ⋈ (R with column 0 bound to 3)` on column 1. Id 3 is unmapped, so +/// the bound side stores no coordinate. +fn bound_to_unmapped() -> (MapTable, Cube, lance_graph_xlog::cube::View) { + let mut table = MapTable::new(1, 1); + table.advance( + 1, + 1, + vec![(0, Some(LocalMap::prepare(Width::W64, &[1, 2]).unwrap()))], + ); + let frame = table.seal(); + let edges = [(1u32, 2u32)]; + let mut cube = Cube::new(&table, &frame, 0, &[edges.as_slice()]).unwrap(); + let scan = cube.scan(0).unwrap(); + let bound = cube.bind(&table, scan, 0, 3).unwrap(); + let scan2 = cube.scan(0).unwrap(); + let join = cube.join(bound, 1, scan2, 1).unwrap(); + (table, cube, join) +} + +/// The map extended to `[1, 2, 3]` under a new frame. +fn extend_to_3(table: &mut MapTable) -> lance_graph_dt64::dr::Frame { + table.advance( + 2, + 1, + vec![(0, Some(LocalMap::prepare(Width::W64, &[1, 2, 3]).unwrap()))], + ); + table.seal() +} + +/// Codex P2 on #1923. Red if a view bound to an unmapped id keeps answering +/// "nothing" after an extending rebase places that id (disable: drop the +/// `apply_rebind` call in `Cube::rebase`). The old view must answer exactly +/// what a view built after the rebase answers; the bind means "column 0 is +/// id 3" under every frame. +#[test] +fn a_bind_to_an_unmapped_id_is_placed_by_an_extending_rebase() { + let (mut table, mut cube, old_join) = bound_to_unmapped(); + // Before: id 3 is unmapped, so the bound side reaches nothing. + assert!(cube + .support(&table, old_join) + .unwrap() + .iter() + .all(|&w| w == 0)); + let f2 = extend_to_3(&mut table); + cube.rebase(&table, &f2).unwrap(); + cube.add_edge(&table, 0, 3, 2).unwrap(); + let scan = cube.scan(0).unwrap(); + let bound = cube.bind(&table, scan, 0, 3).unwrap(); + let scan2 = cube.scan(0).unwrap(); + let new_join = cube.join(bound, 1, scan2, 1).unwrap(); + let fresh = cube.support(&table, new_join).unwrap(); + // Anti-vacuity: the fresh view does see the new edge (id 2 is local 1). + assert_eq!(fresh[0] >> 1 & 1, 1, "the fresh view reaches id 2"); + assert_eq!(cube.support(&table, old_join).unwrap(), fresh); + assert_eq!( + cube.bag_count(&table, old_join).unwrap(), + cube.bag_count(&table, new_join).unwrap() + ); +} + +/// The same through `Cube::stage` + `Cube::commit` (the refresh path). Red if +/// `commit` does not apply the staged rebinds (disable: drop the +/// `apply_rebind` call in `Cube::commit`). +#[test] +fn a_bind_to_an_unmapped_id_is_placed_by_a_committed_batch() { + let (mut table, mut cube, old_join) = bound_to_unmapped(); + let grown = LocalMap::prepare(Width::W64, &[1, 2, 3]).unwrap(); + let staged = cube + .stage(&table, Some((0, &grown)), &[(0, EdgeOp::Add, 3, 2)]) + .unwrap(); + let f2 = extend_to_3(&mut table); + cube.commit(&table, &f2, staged); + let s = cube.support(&table, old_join).unwrap(); + assert_eq!(s[0] >> 1 & 1, 1, "the old view reaches id 2 through (3, 2)"); +} + +/// A mapped bind keeps its meaning: binding to id 1 before the rebase and +/// after it answers the same (twin of the two tests above: re-resolution +/// touches only binds that were unmapped). +#[test] +fn a_bind_to_a_mapped_id_keeps_its_answer_across_a_rebase() { + let (mut table, mut cube, _) = bound_to_unmapped(); + let scan = cube.scan(0).unwrap(); + let bound = cube.bind(&table, scan, 0, 1).unwrap(); + let scan2 = cube.scan(0).unwrap(); + let join = cube.join(bound, 1, scan2, 1).unwrap(); + let before = cube.support(&table, join).unwrap(); + assert_eq!(before[0] >> 1 & 1, 1, "bound to 1 reaches 2"); + let f2 = extend_to_3(&mut table); + cube.rebase(&table, &f2).unwrap(); + assert_eq!(cube.support(&table, join).unwrap(), before); +} + +/// Red if `Cube::stage` changes the cube or the table (disable: let `stage` +/// log the edges it places). Staging issues no version and logs nothing; +/// only `commit` does. +#[test] +fn staging_changes_nothing() { + let (table, cube, _) = bound_to_unmapped(); + let v = table.current_version(); + let grown = LocalMap::prepare(Width::W64, &[1, 2, 3]).unwrap(); + let staged = cube + .stage(&table, Some((0, &grown)), &[(0, EdgeOp::Add, 3, 2)]) + .unwrap(); + assert_eq!(staged.len(), 1); + assert_eq!(table.current_version(), v); + assert_eq!(cube.delta_len(0), 0); + // A non-extending map is refused at staging, before anything moves. + let swapped = LocalMap::prepare(Width::W64, &[2, 1, 3]).unwrap(); + assert!(matches!( + cube.stage(&table, Some((0, &swapped)), &[]), + Err(CubeRefusal::Renumbered(0)) + )); + // An id outside the staged layout is refused at staging too. + assert!(cube.stage(&table, None, &[(0, EdgeOp::Add, 9, 1)]).is_err()); +} From 0fe4c13032c2b848dec514b199ae9b6459d411fb Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 10:47:48 +0000 Subject: [PATCH 16/19] ci: run the codegraph suites; xlog staging test names the guard it pins Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- .github/workflows/rust-test.yml | 8 ++++++++ crates/lance-graph-codegraph/src/artifact.rs | 4 ++++ crates/lance-graph-xlog/tests/versioned_map.rs | 6 +++--- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 37ed8b095..8c0dfe880 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -150,6 +150,14 @@ jobs: run: cargo test --manifest-path crates/lance-graph-dt64/Cargo.toml - name: Run xlog tests run: cargo test --manifest-path crates/lance-graph-xlog/Cargo.toml + # lance-graph-codegraph is workspace-EXCLUDED (own [workspace], as + # sag-ra-probe), so no line above reaches it. Its suites pin the atomic + # refresh (a refused refresh changes nothing; a retry equals the refresh + # that never failed, call-site multiplicity included), the guards, and + # the hand-checked oracle answers over the committed fixtures. Until + # this line they ran in no workflow. + - name: Run codegraph tests (workspace-excluded) + run: cargo test --manifest-path crates/lance-graph-codegraph/Cargo.toml # weather-poc is workspace-EXCLUDED. A new workflow introduced by the # same pull request cannot gate that pull request because `pull_request` # workflow selection comes from the base branch. Run both surfaces here diff --git a/crates/lance-graph-codegraph/src/artifact.rs b/crates/lance-graph-codegraph/src/artifact.rs index 1c7df34b7..31fa3f882 100644 --- a/crates/lance-graph-codegraph/src/artifact.rs +++ b/crates/lance-graph-codegraph/src/artifact.rs @@ -207,6 +207,10 @@ pub trait Interner { fn name(&self, id: u32) -> &str; /// Distinct names interned. fn len(&self) -> usize; + /// Whether nothing is interned. + fn is_empty(&self) -> bool { + self.len() == 0 + } } impl Interner for Symbols { diff --git a/crates/lance-graph-xlog/tests/versioned_map.rs b/crates/lance-graph-xlog/tests/versioned_map.rs index 66dcdfebc..97eecbf33 100644 --- a/crates/lance-graph-xlog/tests/versioned_map.rs +++ b/crates/lance-graph-xlog/tests/versioned_map.rs @@ -199,9 +199,9 @@ fn a_bind_to_a_mapped_id_keeps_its_answer_across_a_rebase() { assert_eq!(cube.support(&table, join).unwrap(), before); } -/// Red if `Cube::stage` changes the cube or the table (disable: let `stage` -/// log the edges it places). Staging issues no version and logs nothing; -/// only `commit` does. +/// Staging issues no version and logs nothing; only `commit` does (`stage` +/// takes `&self`, so that half is held by the type). Red if a non-extending +/// map is staged (disable: drop the `extends` check in `Cube::stage`). #[test] fn staging_changes_nothing() { let (table, cube, _) = bound_to_unmapped(); From b6931d9d70652ef333955d59316f769dd9b0581f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 10:51:52 +0000 Subject: [PATCH 17/19] board + plans: operator decisions on the codegraph review (P1 done, P2 BIN amendment, P3 DT64 authority census); pin egglog Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- .claude/board/INTEGRATION_PLANS.md | 5 ++ .claude/board/STATUS_BOARD.md | 5 +- ...-02-codegraph-review-operator-decisions.md | 55 +++++++++++++ .claude/board/entries/README.md | 3 +- .../plans/d-codegraph-0-binary-lanes-v1.md | 80 +++++++++++++++++++ .../d-dt64-method-authority-census-v1.md | 57 +++++++++++++ crates/egglog-r2il-probe/Cargo.toml | 6 +- 7 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 .claude/board/entries/2026-11-02-codegraph-review-operator-decisions.md create mode 100644 .claude/plans/d-dt64-method-authority-census-v1.md diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index 35c3f1282..9d17dbc69 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -1,3 +1,8 @@ +## 2026-11-02 — D-DT64-AUTH: DT64 as the one method authority, census + ratchet → `.claude/plans/d-dt64-method-authority-census-v1.md` + +**Status:** PROPOSED, plan only (operator decision P3 of the architecture review). **Board:** [STATUS_BOARD](STATUS_BOARD.md) D-DT64-AUTH; [review](entries/2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md); [decisions](entries/2026-11-02-codegraph-review-operator-decisions.md). Same day: `d-codegraph-0-binary-lanes-v1.md` gained Amendment A (P2: provenance header, `(artifact, FunDeclId)`, `INVOKES_WORD`). +**Confidence:** plan; inputs located in code, census not yet written. + ## 2026-11-02 — D-CODEGRAPH-0 wide closure, DR versioned index, binary lanes → `.claude/plans/d-codegraph-0-wide-closure-v1.md`, `.claude/plans/d-dynbag-4-dr-versioned-mapping-v1.md`, `.claude/plans/d-codegraph-0-binary-lanes-v1.md` **Status:** wide closure and the DR versioned index IN PR; binary lanes PLANNED. **Board:** [STATUS_BOARD](STATUS_BOARD.md) D-CODEGRAPH-0-C2, D-CODEGRAPH-0-BIN, D-DYNBAG-4; [entry](entries/2026-11-02-d-codegraph-0-gap-map.md). Operator rulings: wider call chain with presence bits for thin provisioning and sparse write merged on read; multi-word presence, explicit refold only; versioned, replayable immutability is the DR index itself (one RouteVersion counter, old versions readable by number); counting is Quack's (mask-risc, ndarray); codebooks up to 512 entries use codec9; strings only at intake, labels through codebooks or a KV table; predictive prefetch through `ndarray::simd::prefetch_read_*`, measured before claimed. diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index e2c16eee9..a70a366b5 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -240,8 +240,9 @@ Plan: [d-codegraph-0-resident-codegraph-v1.md](../plans/d-codegraph-0-resident-c | **D-CODEGRAPH-0-A** | ruff `ruff_charon`: LLBC → Kythe facts with a witness status per call, for one real crate | SHIPPED: ruff #129 (merged `9913e90946`), lance-graph #1918 (merged `c11e4262e`): calls byte-identical to `llbc_calls.py` on the fixture and `mask-risc`; receipt conserved (16,149); Kythe `/xrefs` 3/3 call pins | rust-analyzer oracle (`sag-ra-probe GRAPH=1`) and rustc on `fixtures/codegraph`; `POSSIBLE` never counted as a confirmed caller | | **D-CODEGRAPH-0-B** | resident rows via the ruff sink; per-file digests; incremental invalidation | LANDED (offline form): ruff #130 (merged `95be2e480b`; offline store in `ruff_charon`; `RefinedTruthSink` not reused, typed to R2IL). Incremental == clean on the fixture, byte-identical; skipped dependent leaves exactly one dangling edge. Follow-up fixes in ruff #131 (open). Lance-graph SoA residency still open | clean import == incremental import | | **D-CODEGRAPH-0-C** | callers / callees / impact / diagnostics; tiled closure; body → R2IL design check | SHIPPED (fixture scale): ruff #132 (merged `ed8fd4194f`, rows v2) + lance-graph #1921 (merged `4ab93bf3e`), `crates/lance-graph-codegraph` (queries as confirmed / possible / unknown masks over the xlog cube; dead function, unresolved call, dangling call; unused import NOT_RUN). 27 tests, 12 disable runs red. Tiled closure deferred: refusals `ClosureWidth` (> 64) and `TooMany` (> 256) are surfaced as typed UNKNOWN | each guard disable-verified; unused import proven only without macros, traits, globs, re-exports | -| **D-CODEGRAPH-0-C2** | wide closure (multi-word rows, block presence), multi-tile cube, merge-on-read refresh, interned lanes | IN PR: lance-graph branch `claude/clone-repositories-71a5sw`. Impact over any size; mask-risc store (695 symbols, 3 tiles): impact 286 ns, callers 29 µs, references 45 µs; refresh of an unchanged crate 1.5–1.7 ms = FAILED gate (reading the 2.5 MB text file), fixed by D-CODEGRAPH-0-BIN | narrow vs wide bit-identical ≤ 64; BFS reference on 694 positions; refresh == clean open; every guard disable-verified | -| **D-CODEGRAPH-0-BIN** | ruff_charon binary row lanes + header digest; codec9 small codebooks; Quack counting; no strings in the hot path | QUEUED: plan `d-codegraph-0-binary-lanes-v1.md` (operator: graph-wide u32 ids, TSV kept as debug `open_tsv`). Ruff writer PR first, then the codegraph reader | unchanged refresh ≤ 20 µs; changed-crate open ≤ 1 ms; binary == TSV row for row | +| **D-CODEGRAPH-0-C2** | wide closure (multi-word rows, block presence), multi-tile cube, merge-on-read refresh, interned lanes | IN PR: lance-graph branch `claude/clone-repositories-71a5sw`. Impact over any size; mask-risc store (695 symbols, 3 tiles): impact 286 ns, callers 29 µs, references 45 µs; refresh of an unchanged crate 1.5–1.7 ms = FAILED gate (reading the 2.5 MB text file), fixed by D-CODEGRAPH-0-BIN. `refresh_crate` atomic (prepare → validate → commit: staged interner, positions, tile map, `Cube::stage`, closure; commit has no error path); codegraph suites now in `rust-test.yml` | narrow vs wide bit-identical ≤ 64; BFS reference on 694 positions; refresh == clean open; a refused refresh changes nothing (NeedsReopen, injected stage and closure refusals; no version issued); retry == the refresh that never failed, call-site multiplicity included; 7 disable runs red | +| **D-CODEGRAPH-0-BIN** | ruff_charon binary row lanes + header digest; codec9 small codebooks; Quack counting; no strings in the hot path; provenance header + per-row `(artifact, FunDeclId)`; `INVOKES_WORD` to a registered DT64 word | QUEUED: plan `d-codegraph-0-binary-lanes-v1.md` (operator: graph-wide u32 ids, TSV kept as debug `open_tsv`; Amendment A §8: PROV section, FUNDECL/ARTREF lanes, `INVOKES_WORD`, F9–F12). Ruff writer PR first, then the codegraph reader. OPEN: source of the "function realises word" association | unchanged refresh ≤ 20 µs; changed-crate open ≤ 1 ms; binary == TSV row for row | +| **D-DT64-AUTH** | DT64 / F### as the one method authority: census of production words with a two-way exception ratchet; parallel identities (`MethodSignature`, tract `Registry`, XLOG `Word`, ogar-loco `FnIndex`) stay as listed `LAB` adapters | QUEUED: plan `d-dt64-method-authority-census-v1.md` (operator decision 2026-11-02); egglog pinned to a commit in `egglog-r2il-probe` | undeclared production word → red; stale exception → red; current tree green | | **D-CODEGRAPH-0-D** | local stdio MCP over C | QUEUED | only after B and C gates | | **D-CODEGRAPH-0-E** | token measurement with and without the Codegraph | QUEUED | same revision, questions, model; nothing claimed before measured | diff --git a/.claude/board/entries/2026-11-02-codegraph-review-operator-decisions.md b/.claude/board/entries/2026-11-02-codegraph-review-operator-decisions.md new file mode 100644 index 000000000..be7e6ae65 --- /dev/null +++ b/.claude/board/entries/2026-11-02-codegraph-review-operator-decisions.md @@ -0,0 +1,55 @@ +# Codegraph review: operator decisions and P1 result (2026-11-02) + +**Status:** P1 DONE (in PR #1923, not merged); P2, P3 DECIDED, prepared as plans. +**Source:** [architecture review](2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md) §7. + +## Principle (DECISION, operator 2026-11-02) +One canonical normalised codegraph over DR³; one method authority, DT64. +Ruff/Charon, Rust/MIR, MLIR and other frontends may keep versioned IR +artifacts and caches, never a second independent semantic store. Lazy +hydration only outside the hot path. MCP, `.index.md`, Mermaid and external +call queries are later deterministic projections of the same graph, never a +new source of truth. No automatic merge. + +## P1 — atomic refresh (DONE, measured) +- `refresh_crate` is prepare → validate → commit + (`lance-graph-codegraph/src/graph.rs`). Staged: interner (`SymbolStage`, + `artifact.rs`), positions incl. `last_owner` (`PosStage`), the grown tile + map (not handed to the table), the edge batch (`Cube::stage`, `&self`), the + closure refresh (`Cube::stage_closure`). Commit: `Symbols::absorb`, + `Positions::absorb`, `MapTable::advance` + `seal`, `Cube::commit`, closure + cells, lanes, masks; no error path. `Cube::rebase` also checks before it + changes anything. +- Codex P2 on #1923 closed: `Node::Bind` keeps its global id; an extending + rebase or commit places a bind that was unmapped. A mapped bind keeps its + coordinate (positions never renumber), so a view keeps its meaning. +- Falsifiers: NeedsReopen, injected stage refusal, injected closure refusal + each leave a full-state snapshot unchanged (interner, positions, + `last_owner`, lanes, digests, statuses, table version/revisions/tile + history, cube frame and every log, masks, cached closures); no version is + issued; retry == the refresh that never failed, and `m::b` keeps 4 call + sites as in a clean open; the snapshot can fire. xlog: bind placed by + rebase and by commit; mapped bind unchanged; staging issues no version and + refuses a non-extending map. +- 7 disable runs, each red on its own test only: absorb before the + reservation check; advance before staging; absorb before the closure + block; dedup keys before the multiset diff; drop rebind in `rebase`; drop + rebind in `commit`; drop the `extends` check in `stage`. +- CI: `rust-test.yml` now runs `crates/lance-graph-codegraph` (workspace + excluded; ran in no workflow before). + +## P2 — header plus per-row FunDeclId (DECIDED, plan amended) +`d-codegraph-0-binary-lanes-v1.md` §8: PROV section (charon version and +commit, LLBC schema version, LLBC digest, toolchain, target + build config, +source revision; UNKNOWN never equal to a value), FUNDECL + ARTREF lanes +(identity = `(artifact, FunDeclId)`), typed objects, `INVOKES_WORD` to a +registered DT64 reference (refused when undeclared), F9–F12. +OPEN: where "function realises word W" comes from (lens-declared table or a +Charon-read attribute; never a name match). + +## P3 — DT64 the method authority (DECIDED, plan written) +`d-dt64-method-authority-census-v1.md`: adapters may stay as listed `LAB` +exceptions with a removal path; production words must be declared; the +census fails on an undeclared word and on a stale exception. egglog pinned +to `5deaa5637a306c48c998f88d5d08792d2dbfd037` in `egglog-r2il-probe` +(was `branch = "main"`); its 7 suites pass at the pin. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 2d0c7e726..421c52faa 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,11 +25,12 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -546 entries, 2026-08-06 .. 2026-11-02. +547 entries, 2026-08-06 .. 2026-11-02. | date | entry id | finding | file | |---|---|---|---| | 2026-11-02 | `D-CODEGRAPH-0` | | [2026-11-02-d-codegraph-0-gap-map.md](2026-11-02-d-codegraph-0-gap-map.md) | +| 2026-11-02 | `codegraph-review-operator-decisions` | | [2026-11-02-codegraph-review-operator-decisions.md](2026-11-02-codegraph-review-operator-decisions.md) | | 2026-11-02 | `codegraph-dr3-dt64-ruff-architecture-review` | | [2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md](2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md) | | 2026-11-01 | `D-XLOG-NATIVE-7` | | [2026-11-01-xlog-resident-hypercube-context.md](2026-11-01-xlog-resident-hypercube-context.md) | | 2026-11-01 | `D-XLOG-NATIVE-8` | | [2026-11-01-xlog-behaviour-lens-f100.md](2026-11-01-xlog-behaviour-lens-f100.md) | diff --git a/.claude/plans/d-codegraph-0-binary-lanes-v1.md b/.claude/plans/d-codegraph-0-binary-lanes-v1.md index d7a683781..9a389c1f6 100644 --- a/.claude/plans/d-codegraph-0-binary-lanes-v1.md +++ b/.claude/plans/d-codegraph-0-binary-lanes-v1.md @@ -220,3 +220,83 @@ PR B (lance-graph, after A merges; fixtures copied from A): borrowed in place with no gather (bigger change to positions/cube)? 2. TSV in codegraph: remove the TSV reader from `open` entirely (bin only, TSV read only in tests), or keep a debug `open_tsv`? + +## 8. Amendment A (2026-11-02): provenance header + per-row function identity + +DECISION (operator, 2026-11-02, binding; supersedes nothing above, extends §2): +the format carries the artifact context in a header section and a function +identity per row, so one migration covers both. SCOPE: `.rows.bin` +and the codegraph reader. REVISIT WHEN: a second frontend (Rust MIR, MLIR) +writes rows. + +**Canonical identity is never `FunDeclId` alone.** A Charon `FunDeclId` is an +index into ONE LLBC file's `fun_decls`; two LLBC artifacts reuse the same +numbers. The identity of a function fact is `(artifact, FunDeclId)`, where +`artifact` names the LLBC file and its provenance below. + +### 8.1 Header: a `PROV` section (covered by the digest) +Not in the fixed 64 bytes: the unchanged check stays a header read (F2), and +the digest over [32 .. file_len) covers this section, so any provenance +change is a changed crate. One record per LLBC artifact the file was built +from (normally one): + +| field | source (read, never assumed) | +|---|---| +| charon version | `CrateData::charon_version` (charon `export.rs:21`; charon-ml compares it for equality, so it is the de-facto LLBC schema key) | +| charon commit | the git HEAD of the charon checkout the ruff_charon build links (`ruff_charon/Cargo.toml` path dependency); captured at build, not at run | +| LLBC schema version | today the same value as `charon_version` (no separate schema field exists in `CrateData`); a separate field is written so a future split does not change the layout | +| LLBC content digest | `artifact::digest` over the LLBC file bytes the writer read | +| Rust toolchain | the `rustc -vV` commit-hash line of the toolchain Charon ran under | +| target + build config | `TranslatedCrate::target_information` keys (target triples) and the relevant `TranslatedCrate::options` (`CliOpts`: features, cfg, flags), serialised in a fixed field order | +| source revision | git HEAD of the analysed source tree plus a dirty flag; `UNKNOWN` when not a git tree, never a guess | + +A writer that cannot fill a field writes `UNKNOWN` for it; the reader keeps +UNKNOWN distinct from every value (two UNKNOWN records are not "equal +provenance"). + +### 8.2 Per-row lanes +- `FUNDECL` u32 lane: the Charon `FunDeclId` of the function the fact belongs + to, `NO_ID` when no function owns it (a CRATE row, a file-level fact). +- `ARTREF` lane (codec9, elided when the PROV section has one record = + implicit 0): which PROV record the row's `FunDeclId` indexes. +- Source reference: the existing `FILE` + `START`/`END` lanes. +- Predicate: the existing predicate codebook id. Typed arguments: an + `OBJ_KIND` codec9 lane says what `OBJ` holds (symbol id, or a word + reference, below); no object is ever a string to be parsed. +- Evidence class: the existing `GRADE` codebook (COMPILER / … ), renamed in + docs to "evidence class"; same lane. + +### 8.3 `INVOKES_WORD` +A call whose callee is a behaviour-lens word gets its own predicate, +`INVOKES_WORD`, with `OBJ_KIND = WORD` and the object a registered DT64 +reference: lens classid (u32 lane `WORD_LENS`) and word (u8, `WORD_IDX`), +i.e. an `AlgebraRef`; a `MethodRef` form is reserved (`OBJ_KIND = METHOD`). +- Registered, not named: the codegraph reader checks each reference against + the DT64 `BindStore` (`declare_algebra` / the lens basis) and refuses an + undeclared one as a named intake refusal. No second method catalogue, no + name convention. +- OPEN, decided in its own reviewable step: where the association "this Rust + function realises word W" comes from. Candidates: (a) the lens crate + declares it next to the word (the POLYFILL table, read at build); (b) an + attribute on the realising function, read by Charon. A name match is not a + candidate. +- Repeated call sites stay distinct: each row keeps its own span, and the + edge fold is a multiset (the P1 retry falsifier pins multiplicity). + +### 8.4 Falsifiers added to §5 +- F9 provenance: changing only the charon commit (or the toolchain, or the + target) makes an unchanged-source crate a changed crate; an `UNKNOWN` + field never compares equal to a known one. +- F10 identity: two LLBC artifacts with overlapping `FunDeclId` numbers in + one crate file resolve to different functions; dropping `ARTREF` merges + them (disable run). +- F11 `INVOKES_WORD` registration: an undeclared `(lens, word)` is refused at + intake (disable: skip the check → accepted, red). +- F12 one fact set, two answers: `users_of(W)` (DT64: the method bindings + whose effects reference W) and `callers(W)` (codegraph: the functions with + an `INVOKES_WORD` row to W) answer different questions, so they are + compared through the facts that join them: each binding's realising + function. On one committed fixture, the realising functions of + `users_of(W)` equal `callers(W)`. A second fixture adds a Rust caller of W + with no binding; it must appear in exactly the difference (anti-vacuity: + the difference is non-empty there and empty in the first). diff --git a/.claude/plans/d-dt64-method-authority-census-v1.md b/.claude/plans/d-dt64-method-authority-census-v1.md new file mode 100644 index 000000000..a6055613f --- /dev/null +++ b/.claude/plans/d-dt64-method-authority-census-v1.md @@ -0,0 +1,57 @@ +# Plan: DT64 as the one method authority — census and ratchet (2026-11-02) + +**D-id:** D-DT64-AUTH ([STATUS_BOARD](../board/STATUS_BOARD.md)). +**Source:** architecture review +[entry](../board/entries/2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md) §2, §7 P3. + +DECISION (operator, 2026-11-02): DT64 / `0xFFFF::0xF###` is the canonical +method authority. SCOPE: every executable method call a production surface +exposes. BASIS: the review found four parallel method identities +(`dr_math_binding_probe::MethodSignature`, the tract-harvest `Registry`, +XLOG `enum Word`, ogar-loco `FnIndex`) and dispatch paths DT64 never sees +(`FoldDialect` `match`, r2il `procs`). REVISIT WHEN: a second production +lens executes words outside R2IL. + +## 1. The rule +The four may stay as lab, frontend or execution adapters while: +1. none of them is a global method identity or an admission authority; +2. every production-exposed executable method call is bound and admitted + through DT64 / F### (`BindStore::admit`); +3. internal opcodes stay implementation detail (a `FoldDialect` arm is not a + method; the word that reaches it is); +4. each deviation is listed in the census with an explicit `LAB` status and a + removal path; +5. a new undeclared production word, and a stale exception, both fail the + census test. + +No full migration now; nothing here moves an adapter. + +## 2. Census (one test, one committed list) +- Inputs, each read from the code that owns it, never restated: the R2IL + polyfill words (`lance-graph-r2il/src/polyfill.rs` `POLYFILL`), the XLOG + words (`lance-graph-xlog` `enum Word` and `lens.rs` declarations), the + `BindStore` declarations each lens makes (`declare_algebra` / + `polyfill_arities`). +- Check: every production word has a DT64 declaration at its lens position; + every DT64 declaration names a word that exists. +- Exceptions: `.claude/board/dt64-authority-exceptions.toml` (or a const + table in the test; decided in the implementing PR), one row per deviation: + surface, status `LAB`, why, removal path. Exact match in both directions: + an undeclared word not listed fails; a listed exception that no longer + deviates fails. +- Home: a test in `lance-graph-r2il` or `lance-graph-dt64`, whichever already + depends on both inputs without a new edge (decided by reading the + dependency graph in the implementing PR, not here). + +## 3. Falsifiers +- C1: adding a polyfill word without a DT64 declaration turns the census red + (disable: drop the declaration loop → green with the undeclared word, red + restored on revert). +- C2: listing an exception that no longer deviates turns it red. +- C3 silence: on the current tree, with the four adapters listed as `LAB`, + the census is green (the ratchet is not a guard that fires on everything). + +## 4. Not in this plan +Migrating `MethodSignature`, the tract `Registry`, `Word` or `FnIndex`; +executing anything through DT64; the `INVOKES_WORD` association (that is +D-CODEGRAPH-0-BIN §8.3). diff --git a/crates/egglog-r2il-probe/Cargo.toml b/crates/egglog-r2il-probe/Cargo.toml index 93e03211c..8ed3b4c8c 100644 --- a/crates/egglog-r2il-probe/Cargo.toml +++ b/crates/egglog-r2il-probe/Cargo.toml @@ -22,7 +22,11 @@ description = "Stage B probe: egglog as a cold, bounded, proof-producing compile [dependencies] # Public API only (EGraph::new_with_proofs, parse_and_run_program, prove-variants). -egglog = { git = "https://github.com/AdaWorldAPI/egglog", branch = "main" } +# Pinned to a commit (DECISION 2026-11-02): with a floating branch a +# `ReplayRefusal::ProofChanged` could be an egglog change rather than a change +# in the program, and the replay capsule records no egglog version. Moving +# the pin is a deliberate edit, re-run with the capsule recheck. +egglog = { git = "https://github.com/AdaWorldAPI/egglog", rev = "5deaa5637a306c48c998f88d5d08792d2dbfd037" } # `set_rules::Arith`, the law-domain vocabulary. The catalogue is NOT copied. lance-graph-contract = { path = "../lance-graph-contract" } # FoldDialect, Val, INT_MULT, FoldError. From 1babb9a02992c1c707746473403ab51b40e5a1ee Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 10:56:18 +0000 Subject: [PATCH 18/19] pin egglog to one commit in every crate that links it; fix the plan row that cited the floating line Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- .claude/board/STATUS_BOARD.md | 2 +- .../2026-11-02-codegraph-review-operator-decisions.md | 7 +++++-- .claude/plans/d-onnx-relalg-cap-capstone-1750-v1.md | 2 +- crates/dr-math-binding-probe/Cargo.toml | 4 +++- crates/xlog-relcomp-probe/Cargo.toml | 4 +++- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index a70a366b5..9c42384e9 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -242,7 +242,7 @@ Plan: [d-codegraph-0-resident-codegraph-v1.md](../plans/d-codegraph-0-resident-c | **D-CODEGRAPH-0-C** | callers / callees / impact / diagnostics; tiled closure; body → R2IL design check | SHIPPED (fixture scale): ruff #132 (merged `ed8fd4194f`, rows v2) + lance-graph #1921 (merged `4ab93bf3e`), `crates/lance-graph-codegraph` (queries as confirmed / possible / unknown masks over the xlog cube; dead function, unresolved call, dangling call; unused import NOT_RUN). 27 tests, 12 disable runs red. Tiled closure deferred: refusals `ClosureWidth` (> 64) and `TooMany` (> 256) are surfaced as typed UNKNOWN | each guard disable-verified; unused import proven only without macros, traits, globs, re-exports | | **D-CODEGRAPH-0-C2** | wide closure (multi-word rows, block presence), multi-tile cube, merge-on-read refresh, interned lanes | IN PR: lance-graph branch `claude/clone-repositories-71a5sw`. Impact over any size; mask-risc store (695 symbols, 3 tiles): impact 286 ns, callers 29 µs, references 45 µs; refresh of an unchanged crate 1.5–1.7 ms = FAILED gate (reading the 2.5 MB text file), fixed by D-CODEGRAPH-0-BIN. `refresh_crate` atomic (prepare → validate → commit: staged interner, positions, tile map, `Cube::stage`, closure; commit has no error path); codegraph suites now in `rust-test.yml` | narrow vs wide bit-identical ≤ 64; BFS reference on 694 positions; refresh == clean open; a refused refresh changes nothing (NeedsReopen, injected stage and closure refusals; no version issued); retry == the refresh that never failed, call-site multiplicity included; 7 disable runs red | | **D-CODEGRAPH-0-BIN** | ruff_charon binary row lanes + header digest; codec9 small codebooks; Quack counting; no strings in the hot path; provenance header + per-row `(artifact, FunDeclId)`; `INVOKES_WORD` to a registered DT64 word | QUEUED: plan `d-codegraph-0-binary-lanes-v1.md` (operator: graph-wide u32 ids, TSV kept as debug `open_tsv`; Amendment A §8: PROV section, FUNDECL/ARTREF lanes, `INVOKES_WORD`, F9–F12). Ruff writer PR first, then the codegraph reader. OPEN: source of the "function realises word" association | unchanged refresh ≤ 20 µs; changed-crate open ≤ 1 ms; binary == TSV row for row | -| **D-DT64-AUTH** | DT64 / F### as the one method authority: census of production words with a two-way exception ratchet; parallel identities (`MethodSignature`, tract `Registry`, XLOG `Word`, ogar-loco `FnIndex`) stay as listed `LAB` adapters | QUEUED: plan `d-dt64-method-authority-census-v1.md` (operator decision 2026-11-02); egglog pinned to a commit in `egglog-r2il-probe` | undeclared production word → red; stale exception → red; current tree green | +| **D-DT64-AUTH** | DT64 / F### as the one method authority: census of production words with a two-way exception ratchet; parallel identities (`MethodSignature`, tract `Registry`, XLOG `Word`, ogar-loco `FnIndex`) stay as listed `LAB` adapters | QUEUED: plan `d-dt64-method-authority-census-v1.md` (operator decision 2026-11-02); egglog pinned to one commit in `egglog-r2il-probe`, `dr-math-binding-probe`, `xlog-relcomp-probe` | undeclared production word → red; stale exception → red; current tree green | | **D-CODEGRAPH-0-D** | local stdio MCP over C | QUEUED | only after B and C gates | | **D-CODEGRAPH-0-E** | token measurement with and without the Codegraph | QUEUED | same revision, questions, model; nothing claimed before measured | diff --git a/.claude/board/entries/2026-11-02-codegraph-review-operator-decisions.md b/.claude/board/entries/2026-11-02-codegraph-review-operator-decisions.md index be7e6ae65..236d7cf5b 100644 --- a/.claude/board/entries/2026-11-02-codegraph-review-operator-decisions.md +++ b/.claude/board/entries/2026-11-02-codegraph-review-operator-decisions.md @@ -51,5 +51,8 @@ Charon-read attribute; never a name match). `d-dt64-method-authority-census-v1.md`: adapters may stay as listed `LAB` exceptions with a removal path; production words must be declared; the census fails on an undeclared word and on a stale exception. egglog pinned -to `5deaa5637a306c48c998f88d5d08792d2dbfd037` in `egglog-r2il-probe` -(was `branch = "main"`); its 7 suites pass at the pin. +to `5deaa5637a306c48c998f88d5d08792d2dbfd037` in every crate that links it +(`egglog-r2il-probe`, `dr-math-binding-probe`, `xlog-relcomp-probe`; all three +floated on the fork's main branch, and `xlog-relcomp-probe` links the other +two, so one `rev` everywhere or cargo resolves two copies). All three suites +pass at the pin. diff --git a/.claude/plans/d-onnx-relalg-cap-capstone-1750-v1.md b/.claude/plans/d-onnx-relalg-cap-capstone-1750-v1.md index 9943d77c7..7d64f5381 100644 --- a/.claude/plans/d-onnx-relalg-cap-capstone-1750-v1.md +++ b/.claude/plans/d-onnx-relalg-cap-capstone-1750-v1.md @@ -489,7 +489,7 @@ None of these are done by S0. Classes: SAFE NOW (edit with this docs PR), SEPARA | `PR_ARC_INVENTORY.md` has no entries for #1726–#1739; its top entry is #1725 (catch-up comment: "#1660–#1725 had no entries"). | worktree read, line 1-3 | SEPARATE PR (board catch-up) | | `STATUS_BOARD.md` row D-TGA-5D (`:389`) says "In PR" although #1739 merged (merge `9a458cae6`). | worktree read | SEPARATE PR (board) | | STATUS_BOARD has no D-DRMC / D-QR rows on main; #1742 adds a D-DRMC section. | consolidated F7 | BLOCKED on #1742 | -| `egglog` is a git dependency on `branch = "main"` with no `rev` (`crates/dr-math-binding-probe/Cargo.toml:15`, `crates/egglog-r2il-probe/Cargo.toml:25`). The repo tracks no lockfile, so the dependency floats. The fork rule is satisfied; reproducibility is not. | worktree read | SEPARATE PR; choosing a `rev` is an operator decision | +| `egglog` was a git dependency on the fork's main branch with no `rev` (in `dr-math-binding-probe`, `egglog-r2il-probe` and `xlog-relcomp-probe`). The repo tracks no lockfile, so the dependency floated. The fork rule was satisfied; reproducibility was not. | worktree read | DONE 2026-11-02 (operator decision): all three pinned to one `rev` in PR #1923 | | D9 `TripletGraph::to_onnx()` "with custom ops for triples" conflicts with portable export (S5). | `STATUS_BOARD.md:2329`; opus-c item 13b | BLOCKED on OP-8 | | Citation fix: "no shared relational AST" is at `INTEGRATION_PLANS.md:16`, not `ISSUES.md:16`. | worktree read | SAFE NOW (this document uses the correct citation) | | Fact-base fix: `mish_f32`/`swish_f32` exist in ndarray. | `activations_ext.rs:143,168` | SAFE NOW (stated in section 6) | diff --git a/crates/dr-math-binding-probe/Cargo.toml b/crates/dr-math-binding-probe/Cargo.toml index 7237dbcf1..64be8a7aa 100644 --- a/crates/dr-math-binding-probe/Cargo.toml +++ b/crates/dr-math-binding-probe/Cargo.toml @@ -14,7 +14,9 @@ lance-graph-contract = { path = "../lance-graph-contract" } # parse_and_run_program, `prove`, the read-only `egglog::proof` module). # egglog checks every proof it returns with its own checker; that check is # egglog's, not an independent one. -egglog = { git = "https://github.com/AdaWorldAPI/egglog", branch = "main" } +# Pinned to one commit, the same in every crate that links egglog +# (DECISION 2026-11-02; a branch and a rev are two sources to cargo). +egglog = { git = "https://github.com/AdaWorldAPI/egglog", rev = "5deaa5637a306c48c998f88d5d08792d2dbfd037" } # Compiled identity of a certified lowering (relcomp::lowering_digest). sha2 = "0.10" diff --git a/crates/xlog-relcomp-probe/Cargo.toml b/crates/xlog-relcomp-probe/Cargo.toml index 0042c7588..4e7a781f7 100644 --- a/crates/xlog-relcomp-probe/Cargo.toml +++ b/crates/xlog-relcomp-probe/Cargo.toml @@ -45,7 +45,9 @@ causal-edge = { path = "../causal-edge" } # egglog, on the coordinate dr-math-binding-probe already uses, for the # shared-relation experiment (tests/shared_relation.rs): equality with proofs # over the same DR64 register the closure reads. -egglog = { git = "https://github.com/AdaWorldAPI/egglog", branch = "main" } +# Pinned to one commit, the same in every crate that links egglog +# (DECISION 2026-11-02; a branch and a rev are two sources to cargo). +egglog = { git = "https://github.com/AdaWorldAPI/egglog", rev = "5deaa5637a306c48c998f88d5d08792d2dbfd037" } [patch."https://github.com/AdaWorldAPI/lance-graph"] lance-graph-contract = { path = "../lance-graph-contract" } From 83ac4b3a8918704aa8f7f60c8ac45b3b1acf817f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 11:03:16 +0000 Subject: [PATCH 19/19] board: P4 review, Thought-Git x Codegraph x DT64 (graded, identity table, smallest adapter, falsifiers; no code) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- ...02-p4-thought-git-codegraph-dt64-review.md | 160 ++++++++++++++++++ .claude/board/entries/README.md | 3 +- 2 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 .claude/board/entries/2026-11-02-p4-thought-git-codegraph-dt64-review.md diff --git a/.claude/board/entries/2026-11-02-p4-thought-git-codegraph-dt64-review.md b/.claude/board/entries/2026-11-02-p4-thought-git-codegraph-dt64-review.md new file mode 100644 index 000000000..1628cc049 --- /dev/null +++ b/.claude/board/entries/2026-11-02-p4-thought-git-codegraph-dt64-review.md @@ -0,0 +1,160 @@ +# P4: Thought-Git × Codegraph × DT64 — Prüfung und kleinster Adapter (2026-11-02) + +**Status:** MEASURED (read-only Review), keine Implementierung. +**Bezug:** D-TGA, D-TGB, EXO-148, D-CODEGRAPH-0, D-DYNBAG-4, D-DT64-AUTH, PR #1923. +**Grade:** VERIFIED / PARTIAL / MISSING / CONTRADICTED. Jeder Beleg wurde gelesen; Harvest-Treffer dienten nur als Fundort. + +**Quellen:** +- `crates/thought-git-probe` (6 Quelldateien, 58 Tests); +- die Pläne `thought-git-api-v1.md`, `thought-git-basics-v1.md`, `d-dt64-4-edge-thoughtgit-hypothesis-time-v1.md` und `d-rmap-separate-tables-thought-palettes-v1.md` §7; +- STATUS_BOARD-Zeilen D-TGA-0..6, D-TGB-0..10 und die EXO-148-Zeilen; +- `ISSUES.md` `ISS-DURABLEREF-THOUGHTCOMMITID`; +- dt64 `lib.rs`/`dr.rs`, `lance-graph/src/graph/cycle_sink.rs`, `lance-graph-planner/src/persist_sink.rs`; +- die Codegraph-/xlog-Änderungen aus #1923. + +## 1. Was existiert + +| Baustein | Stand | Beleg | +|---|---|---| +| Slot-Pool 0 + 1..=63 mit Generation, `Stale` beim alten Handle, FIFO- oder Least-Used-Eviction mit Squash entlang der Kette | VERIFIED (Lab, D-TGB-1) | `thought-git-probe/src/slots.rs` (`Slot{index, generation}`, `SlotPool::branch/free`) | +| From-scratch-Orakel: nur `Commit` landet; ein Given wird nie überschrieben (Contradiction); uneinige Landungen speichern nichts (Divergence); absent ≠ 0 | VERIFIED (Lab, D-TGB-0) | `src/oracle.rs` `next_version` | +| Squash-Gate und geschützte Leser | VERIFIED (Lab, D-TGA-5/5D) | `src/squash.rs` | +| Receipt statt Historie; Replay ohne Slot; Fold-Zeugen; `CommitWitness` mit mehreren Thoughts pro Zyklus in einer Version (EXO-55) | VERIFIED (Lab, D-TGB-4) | `src/receipt.rs` (`Receipt`, `replay`, `VersionStore::commit_cycle`, `MetadataTable`) | +| Produktive Persistenzkante: ein Batch pro Zyklus; `Fenced` gegen `base_version`; Idempotenz über `(cycle, batch_hash)`; `HashConflict` schlägt geschlossen fehl | VERIFIED (Code) | `persist_sink.rs` `WalSink::commit_cycle`, `CommitOutcome`, `CommitError`; `cycle_sink.rs:170` `LanceCycleWriter` | +| DT64: Verhalten unveränderlich (D5); eine Revision ist eine abgeleitete Classid plus Re-Route; nur das Routing ist versioniert; Zulassung prüft `MethodRef{class, method, sig_rev, guard}` | VERIFIED (Code) | dt64 `lib.rs` Moduldoku, `register_derived`, `route`, `routed_at`, `admit` | +| Codegraph aus #1923 | VERIFIED (Code) | xlog `Cube::stage`/`commit`, codegraph `refresh_crate` | + +Zum Codegraph aus #1923: +- versionierter DR-Index (`MapTable`, `map_at`, `RouteVersion`); +- Sparse-Delta-Log als Multiset mit Merge-on-Read; +- atomarer Refresh über `SymbolStage`/`PosStage`/`Cube::stage`; +- Positionen werden nie umnummeriert. + +| Fehlt | Stand | Beleg | +|---|---|---| +| `DurableRef`, `ThoughtCommitId` (EXO-148) | MISSING | `ISSUES.md` `ISS-DURABLEREF-THOUGHTCOMMITID`, OPEN; im Probe ist `BaseRef{dataset, branch, version}` der Platzhalter (`receipt.rs:53-60`) | +| D-TGA-1 (Thought-Typ), -2 (Lese-Ebene R), -3 (Rebase mit DRed), -4 (land), -6 (MergeQueue) | MISSING | STATUS_BOARD Queued | +| D-TGB-2 (w_slot-Leser migrieren), -3 (Commit über `LanceCycleWriter`), -5 (resident folds), -7 (wiederverwendbare Crate `lance-graph-thought-git`) | MISSING | STATUS_BOARD Queued; `lance-graph-thought-git` gibt es nur als Plan (D-RFT, D-QSP, D-TGB-7) | +| Für dauerhaftes land/replay gilt: „No durable implementation until EXO-148 is resolved and operator authorizes“ | VERIFIED (Plan-Regel) | `d-dt64-4-…-v1.md:85` | + +## 2. Identitäten: sechs Regeln, nicht mischen + +| Identität | Ist | Regel | Darf nie | +|---|---|---|---| +| `RouteVersion` | Zähler `dt64::dr::VersionClock` (logisch, im Speicher; Route-Log + MapTable) | monoton, geteilt, nur Routing und Tile-Maps | als Lance-Version oder Thought-Identität gelesen werden | +| `LanceVersion` / `DatasetVersion` | Lance-Manifestversion; `contract::temporal_pov::VersionClock` bildet sie auf Zeilenbereiche ab | dauerhaft, pro Dataset | ohne Dataset und Branch verglichen werden (`d-rmap` §7.6.2) | +| `DurableRef{dataset, branch, version}` | MISSING (EXO-148) | wie oben, dataset- und branch-qualifiziert | durch eine nackte Version ersetzt werden | +| `ThoughtCommitId` | MISSING (EXO-148) | nie wiederverwendet, auch nicht nach discard/squash (`d-rmap` §7.6.1) | aus dem Slot abgeleitet werden | +| W-Slot (CE64 Bits 53..58) | Lane-Label, ein Register, keine Identität und kein Zeiger | nur zusammen mit Basisversion (für 0) oder Slot-Generation (für 1..63) lesbar | allein als Schlüssel dienen | +| Slot-Generation | `Slot.generation: u32` im Probe | lokaler ABA-Schutz des Pools, bedeutungslos nach dem Seal | über den Pool hinaus verglichen werden | + +Dazu gehören zwei Kennungen, die keine Thought-Identitäten sind: +- **Quell-Git-SHA:** Herkunft der Codegraph-Eingabe (P2-PROV, „source revision“). Sie ist Inhalt, keine Ordnung. +- **`(CycleId, batch_hash)`:** die Commit-Identität von `LanceCycleWriter`. + +**Verbindungen** laufen nur über explizite Datensätze, nie über Gleichsetzung: +- **Receipt:** Basis-`DurableRef`, Programm, Seeds, Slot-Generation. +- **`CommitWitness`:** Zyklus, Kohorte `thought → (Ausgabe-Fold, Outcome)`. +- **`MetaKey`:** Daten-`DurableRef`, Thought. + +> **Benennungsfalle (VERIFIED):** Zwei verschiedene Typen heißen `VersionClock`. +> - `dt64::dr::VersionClock` (`dr.rs:167`) ist ein atomarer logischer Zähler. +> - `contract::temporal_pov::VersionClock` (`temporal_pov.rs:252`) bildet Lance-Version auf Zeilenzahl ab. +> +> Eine Adapterschicht muss den Pfad voll ausschreiben. + +## 3. Antworten auf die zehn Fragen + +1. **Ein kanonischer Codegraph, 63 Layer nur als sparse Änderungen: PARTIAL.** + - Die Form passt: Der Codegraph ist bereits Basis plus sparse Delta-Log mit Merge-on-Read, und ein Thought ist laut Capstone eine sparse Lane. + - Es fehlen drei Dinge: + - **G2:** Neue Symbole erhalten Positionen `base.len()+k` (`PosStage::assign`). Zwei Branches, die beide Symbole hinzufügen, vergeben dieselben Positionen an verschiedene Namen. + - **G7:** Ein Delta-Log pro Relation gehört heute W0; für Branch-Logs gibt es keinen Lese-Fold (Basis + W0-Log + Branch-Log). + - **G8:** Siehe Frage 7. +2. **Ruff/Charon/MLIR/egglog prüfen Varianten ohne W0 zu ändern: PARTIAL.** + - `Cube::stage` und der Stage-Teil von `refresh_crate` prüfen eine Variante vollständig, ohne Cube, Tabelle oder Interner zu ändern (P1-Falsifikatoren). + - Damit ist das Prüfen gelöst, das Halten aber nicht: `StagedEdges` trägt keine Basisidentität (**G3**). Deshalb darf es nur aufrufgebunden sein und nie als Branch-Träger dienen. + - MLIR fehlt ganz (Review vom 2026-11-02). egglog ist jetzt gepinnt, aber seine Kapsel ist nicht dauerhaft. +3. **Alle Branches nutzen dieselben F###-Methoden mit expliziten Revisionen: PARTIAL.** + - Die Revisionsmechanik ist richtig: abgeleitete Classid plus Route; `sig_rev` in `MethodRef`. + - **G4:** `BindStore::route` schreibt ins globale Route-Log und tickt die geteilte `RouteVersion`. Eine branch-lokale Methodenrevision wäre damit sofort global sichtbar. + - Die Registrierung einer abgeleiteten Classid ist additiv und unschädlich: Unveränderliches Verhalten, solange nicht geroutet, ist unerreichbar. Die Route dagegen ist Zustand und gehört in den Branch. +4. **Identitäten verbinden ohne zu mischen: siehe §2.** VERIFIED für die vier existierenden, MISSING für `DurableRef`/`ThoughtCommitId`. +5. **branch/diff/squash/discard/replay im Speicher, nur land schreibt: PARTIAL.** + - Im Probe sind branch, squash, discard und replay VERIFIED. + - diff und rebase fehlen (D-TGA-1/3), land fehlt (D-TGA-4). + - Für den Codegraph ist discard trivial, weil W0 erst beim Commit berührt wird. +6. **Mehrere Thoughts pro Zyklus über `LanceCycleWriter` zu einer Version, mit Receipts: PARTIAL.** + - Der Probe beweist das Muster: `commit_cycle` mit Kohorte und `MetadataTable` mit Seal (`a_landing_needs_the_write_paths_commit_witness`). + - Die produktive Kante hat dieselbe Form plus Fence. + - Die Verdrahtung D-TGB-3 fehlt. + - **G6:** Das Probe-`commit_cycle` prüft `prior` nicht gegen die Basis; das tut nur das produktive `Fenced`. Ein Adapter darf deshalb nur die produktive Kante benutzen. +7. **Verworfene, widersprüchliche oder veraltete Branches übertragen nie still Fakten: PARTIAL.** + - Abgedeckt: + - Orakel: Contradiction und Divergence speichern nichts; + - Squash-Gate und geschützte Leser; + - `Stale`-Handles; + - Fence bei veralteter Basis; + - `HashConflict`; + - Receipt-Refusals. + - Es fehlen: + - Rule 1 „up to date before landing“ (D-TGA-4); + - DRed-Rebase (D-TGA-3); + - negationsbewusste MergeQueue (D-TGA-6). + - **G8 (neu):** Die Probe-Zellen haben Wertsemantik (überschreiben, Divergenz bei Uneinigkeit). Codegraph-Kanten sind ein Bag, also Multiplizitäten mit Summensemantik. Zwei Branches, die dieselbe Quelländerung ableiten, würden beim Summieren doppelt zählen. Squash- und Merge-Regeln für Bag-Zellen sind nicht festgelegt. +8. **MCP und Mermaid zeigen standardmäßig nur W0: MISSING** (es gibt beides noch nicht). Die Regel lässt sich ohne neue Architektur festlegen: + - Jede Projektion nimmt eine explizite Referenz; ohne Angabe ist das W0 bei seiner aktuellen `DurableRef`, nie `QueryReference::default()` (`d-rmap` §7.2). + - Eine Branch-Sicht trägt Thought-Identität, Basis und Slot-Generation im Kopf und ist als spekulativ markiert. +9. **Vorbedingungen vor der Integration:** + - EXO-148 (`DurableRef`, nie wiederverwendete `ThoughtCommitId`) für jedes dauerhafte land oder replay; + - D-TGA-1 und D-TGA-4 (Rule 1); + - D-TGA-3 oder die Ausnahme unten; + - D-TGB-2 (w_slot-Leser); + - D-TGB-3 (Commit über `LanceCycleWriter`); + - D-TGB-5 (resident folds); + - D-TGB-7: die gemeinsame Crate, deren erster Konsument der Codegraph sein sollte. Ein codegraph-eigener Branch-Mechanismus wäre ein zweiter. + - Bereits erfüllt sind D-TGB-0/1/4 und D-TGA-5/5D (Lab). +10. **Unabhängiger Falsifikator für „W0 nach land = zulässiges Delta, Replay stellt wieder her“: MISSING.** Machbar mit vorhandenen Teilen, siehe §5. + +## 4. Kleinster Adapter (Vorschlag, keine Implementierung) + +Ein Codegraph-Thought ist ein Thought der künftigen gemeinsamen Thought-Git-Crate. Seine Fakten sind Quellvarianten, keine Positionen. + +- **Was ein Branch hält:** seine Eingabe (die geänderten `rows`, später Binär-Lanes plus PROV), sein Receipt (Basis, Frontend und PROV als Programm, Slot-Generation) und eine Route-Overlay-Tabelle für Methodenrevisionen (Position → abgeleitete Classid). Keine Positionen, kein `StagedEdges`, kein eigener Graph. +- **Lesen im Branch:** W0 wird gefaltet; das Branch-Delta wird bei Bedarf per Stage berechnet und nie gespeichert. Bei Bedarf kommt dazu ein sparse Kanten-Log nach G7, erst wenn eine Messung es verlangt. +- **land:** Die Branch-Eingabe läuft erneut durch den Stage-Teil von `refresh_crate` gegen das aktuelle W0. Danach folgen Commit, Route-Overlay per `BindStore::route` und ein `LanceCycleWriter`-Batch für alle Landungen des Zyklus mit `CommitWitness`-Kohorte. + - Das löst G2: Positionen vergibt nur W0, zum Zeitpunkt des Landens. + - Es löst G3: Kein Stage überlebt einen W0-Wechsel. + - Es erfüllt Rule 1 per Konstruktion, weil die Landung neu berechnet statt angewandt wird. + - Für Codegraph-Fakten ist das eine echte Neuberechnung aus der Quelle, kein semi-naiver Re-Run. Die DRed-Forderung aus Rule 2 entfällt deshalb für diese Faktklasse. Diese Ausnahme muss als Entscheidung festgehalten werden, sie darf nicht stillschweigend gelten. +- **discard:** Branch-Eingabe und Overlay fallen weg; W0, `RouteVersion` und Interner bleiben unverändert (die P1-Snapshots beweisen das bereits für Stage). +- **squash:** Die Eingaben werden vereinigt, dann folgt das Squash-Gate. Bag-Zellen brauchen zuerst die Regel aus G8. +- **Was nicht dazukommt:** kein neuer Store, keine Graphkopie, keine neue Methodenautorität, kein neuer Versionszähler. + +## 5. Falsifikatoren (vorgeschlagen) + +- **F-LAND (unabhängig):** + - Nach dem Landen von Branch B auf W0@v ist der Codegraph-Snapshot namensgleich mit einem frischen `Codegraph::open` der Quellmenge „W0-Eingaben mit Bs Eingaben ersetzt“. Verglichen werden alle Abfragen inklusive der Aufrufstellen-Multiplizität; das ist das Orakel aus `refresh_equals_clean_open`. + - Ein zweiter Branch C über derselben Basis wird danach gelandet. Erwartet wird: Er wird gegen v+1 neu berechnet, und seine neuen Symbole bekommen andere Positionen als in seinem Stage gegen v (macht G2 sichtbar). + - Disable: Cs altes Stage anwenden. Dann kollidieren die Positionen, und der Test wird rot. +- **F-REPLAY:** + - Aus Bs Receipt allein (Basis-Platzhalter, Programm, PROV) wird die Eingabe neu erzeugt und auf eine frische Kopie von W0@v gelandet. Der Snapshot muss gleich dem aus F-LAND sein. + - Disable: PROV aus dem Receipt entfernen. Dann muss das Replay ablehnen. +- **F-DISCARD:** Nach branch, stage und discard ist der P1-Snapshot (Interner, Positionen, Tabellenversion, Logs, `RouteVersion`) unverändert. +- **F-ROUTE:** Eine branch-lokale Methodenrevision lässt `BindStore::current_version()` bis zum Landen unverändert. Danach ergibt `routed_at(v_alt)` die alte Classid und `routed_at(v_neu)` die neue. +- **F-FENCE:** Ein Batch gegen eine veraltete Basis ergibt `Fenced`, und es wird nichts geschrieben (produktive Kante; im Probe fehlt das, G6). + +## 6. Bewiesene Lücken, nach Priorität + +1. **G2** (Positionskollision) und **G3** (`StagedEdges` ohne Basis): durch Lesen des Codes belegt, durch F-LAND messbar. +2. **G4** (globale Route): durch Lesen von `BindStore::route` belegt. +3. **G8** (Wert- vs. Bag-Semantik): durch Lesen von `next_version` gegen das Delta-Log belegt. +4. **G6** (Fence im Probe): durch Lesen belegt. +5. **G5** (zwei `VersionClock`): Benennungsrisiko, kein Defekt. +6. **G1** (EXO-148): bekannt, blockiert jede dauerhafte Behauptung. + +## 7. Erster Schritt, wenn freigegeben + +Kein Produktionscode. Ein Lab-Test im Codegraph (Fixtures wie in `mod atomic`) setzt F-LAND und F-DISCARD mit zwei Quellvarianten über derselben Basis um. Ein Disable muss G2 sichtbar machen. Der Test nutzt `refresh_crate` und Snapshots aus #1923 und keine neue Struktur. + +Erst danach wird entschieden, wo die Branch-Eingabe lebt: in der D-TGB-7-Crate, nicht im Codegraph. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 421c52faa..26d5435d2 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,10 +25,11 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -547 entries, 2026-08-06 .. 2026-11-02. +548 entries, 2026-08-06 .. 2026-11-02. | date | entry id | finding | file | |---|---|---|---| +| 2026-11-02 | `p4-thought-git-codegraph-dt64-review` | | [2026-11-02-p4-thought-git-codegraph-dt64-review.md](2026-11-02-p4-thought-git-codegraph-dt64-review.md) | | 2026-11-02 | `D-CODEGRAPH-0` | | [2026-11-02-d-codegraph-0-gap-map.md](2026-11-02-d-codegraph-0-gap-map.md) | | 2026-11-02 | `codegraph-review-operator-decisions` | | [2026-11-02-codegraph-review-operator-decisions.md](2026-11-02-codegraph-review-operator-decisions.md) | | 2026-11-02 | `codegraph-dr3-dt64-ruff-architecture-review` | | [2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md](2026-11-02-codegraph-dr3-dt64-ruff-architecture-review.md) |