From 1a021cf64a8c0049b4ecc49096b2c95224e6c313 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 16:13:41 +0000 Subject: [PATCH 1/2] ruff_charon: binary rows format, writer and verifying reader (D-CODEGRAPH-0-BIN) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit store_bin.rs: `.rows.bin` per lance-graph plan d-codegraph-0-binary-lanes §2/§8 - 64-byte header (magic, major/minor, PARTIAL flag, digest over [32..file_len), file_len, counts), an aligned section directory, codec9 codebook lanes (PRED, GRADE, FILE <=511 else u32, OBJ_KIND), u32 id lanes (SUBJ, OBJ, MAYT, START, END, FUNDECL), a symbol table with SYM_HASH and SYM_OWNER, and a PROV section where UNKNOWN is distinct from every value. The reader refuses each malformed shape with its own error. Not yet wired into Store. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/ruff_charon/src/lib.rs | 1 + crates/ruff_charon/src/store_bin.rs | 1225 +++++++++++++++++++++++++ crates/ruff_charon/tests/store_bin.rs | 477 ++++++++++ 3 files changed, 1703 insertions(+) create mode 100644 crates/ruff_charon/src/store_bin.rs create mode 100644 crates/ruff_charon/tests/store_bin.rs diff --git a/crates/ruff_charon/src/lib.rs b/crates/ruff_charon/src/lib.rs index 31ad4ef0de7c91..5ed7a4a22d2831 100644 --- a/crates/ruff_charon/src/lib.rs +++ b/crates/ruff_charon/src/lib.rs @@ -30,6 +30,7 @@ //! is inferred from names. pub mod store; +pub mod store_bin; use std::fmt::Write as _; use std::path::Path; diff --git a/crates/ruff_charon/src/store_bin.rs b/crates/ruff_charon/src/store_bin.rs new file mode 100644 index 00000000000000..7a4f22059d9377 --- /dev/null +++ b/crates/ruff_charon/src/store_bin.rs @@ -0,0 +1,1225 @@ +//! The binary rows format `.rows.bin`: a pure writer and a verifying +//! reader for the lanes of lance-graph's +//! `.claude/plans/d-codegraph-0-binary-lanes-v1.md` §2 and §8. +//! +//! The format is columnar: one lane per field, each a section found through a +//! directory, every integer little-endian. Ids into the codebooks use the +//! `EpiLane9` codeword split (ndarray `src/hpc/codec9.rs`): a codeword `c` in +//! `1..=511` is a magnitude byte `c & 0xFF` plus a presence bit `c >> 8`, and +//! codeword 0 means absent. The header digest covers everything after the +//! first 32 bytes, so a reader refuses a damaged file before it decodes a lane. +//! +//! This module only encodes and decodes bytes. It does not touch the TSV store. +//! [`read`] rebuilds strings and is the debug/test path; the Codegraph reader in +//! lance-graph is separate and zero-copy. + +use std::collections::{BTreeMap, HashMap}; +use std::fmt; + +/// File magic, bytes 0..8. +pub const MAGIC: [u8; 8] = *b"CHRNROWS"; +/// Format major version. A reader refuses any other. +pub const MAJOR: u16 = 1; +/// Format minor version. A reader refuses a file with a larger minor. +pub const MINOR: u16 = 0; +/// Length of the fixed header. +pub const HEADER_LEN: usize = 64; +/// An absent id, span, owner or function declaration. +pub const NO_ID: u32 = u32::MAX; +/// Header flag bit 0: the crate description was partial (Charon reported errors). +pub const FLAG_PARTIAL: u32 = 1; +/// Codebook entries addressable by a codec9 lane (codewords `1..=511`). +pub const CODEC9_MAX: usize = 511; + +/// Section tags. Bit 31 means must-understand; every tag here has it set. +pub mod tag { + /// Predicate magnitude lane. + pub const PRED_MAG: u32 = 0x8000_0001; + /// Predicate presence plane. + pub const PRED_PRES: u32 = 0x8000_0002; + /// Predicate codebook. + pub const PRED_CB: u32 = 0x8000_0003; + /// Grade magnitude lane. + pub const GRADE_MAG: u32 = 0x8000_0004; + /// Grade presence plane. + pub const GRADE_PRES: u32 = 0x8000_0005; + /// Grade codebook. + pub const GRADE_CB: u32 = 0x8000_0006; + /// File lane: codec9 magnitude bytes, or raw `u32` when there are more than 511 files. + pub const FILE: u32 = 0x8000_0007; + /// File presence plane, present only when `FILE` is codec9. + pub const FILE_PRES: u32 = 0x8000_0008; + /// File codebook. + pub const FILE_CB: u32 = 0x8000_0009; + /// Subject symbol ids. + pub const SUBJ: u32 = 0x8000_000A; + /// Object symbol ids. + pub const OBJ: u32 = 0x8000_000B; + /// Dispatch-target symbol ids (`MAY_DISPATCH` rows only). + pub const MAYT: u32 = 0x8000_000C; + /// Span starts. + pub const START: u32 = 0x8000_000D; + /// Span ends. + pub const END: u32 = 0x8000_000E; + /// Symbol blob offsets, `n_symbols + 1` entries. + pub const SYM_OFF: u32 = 0x8000_000F; + /// Symbol label bytes. + pub const SYM_BLOB: u32 = 0x8000_0010; + /// Per-symbol digest of the label. + pub const SYM_HASH: u32 = 0x8000_0011; + /// Per-symbol owner id. + pub const SYM_OWNER: u32 = 0x8000_0012; + /// Per-row Charon function declaration index. + pub const FUNDECL: u32 = 0x8000_0013; + /// Object-kind magnitude lane. + pub const OBJ_KIND_MAG: u32 = 0x8000_0014; + /// Object-kind presence plane. + pub const OBJ_KIND_PRES: u32 = 0x8000_0015; + /// Provenance record. + pub const PROV: u32 = 0x8000_0016; + /// Every tag a version-1 reader understands, in ascending order. + pub const ALL: [u32; 22] = [ + PRED_MAG, + PRED_PRES, + PRED_CB, + GRADE_MAG, + GRADE_PRES, + GRADE_CB, + FILE, + FILE_PRES, + FILE_CB, + SUBJ, + OBJ, + MAYT, + START, + END, + SYM_OFF, + SYM_BLOB, + SYM_HASH, + SYM_OWNER, + FUNDECL, + OBJ_KIND_MAG, + OBJ_KIND_PRES, + PROV, + ]; +} + +const ENC_RAW: u16 = 0; +const ENC_CODEC9: u16 = 1; +const ENC_PRES: u16 = 2; +const ENC_KV: u16 = 3; + +const KIND_SYMBOL: usize = 1; +const KIND_MAX: usize = 3; +const DIR_ENTRY_LEN: usize = 16; +const PROV_FIELDS: usize = 7; +const MAY_DISPATCH: &str = "MAY_DISPATCH"; + +// --------------------------------------------------------------------------- +// digest +// --------------------------------------------------------------------------- + +const K: u64 = 0x517c_c1b7_2722_0a95; +const SEED: u64 = 0xcbf2_9ce4_8422_2325; + +fn step(h: u64, w: u64) -> u64 { + (h.rotate_left(5) ^ w).wrapping_mul(K) +} + +fn len_u64(n: usize) -> u64 { + u64::try_from(n).unwrap_or(u64::MAX) +} + +fn fold(mut h: u64, bytes: &[u8]) -> u64 { + let (chunks, rest) = bytes.as_chunks::<8>(); + for c in chunks { + h = step(h, u64::from_le_bytes(*c)); + } + if !rest.is_empty() { + let mut w = [0u8; 8]; + w[..rest.len()].copy_from_slice(rest); + // The tail length is folded in so "a" and "a\0" differ. + h = step(h, u64::from_le_bytes(w) ^ (len_u64(rest.len()) << 56)); + } + h +} + +/// The change-detection digest, identical to lance-graph's +/// `lance-graph-codegraph` `artifact::digest`: eight bytes per step, the byte +/// length folded in last. Not cryptographic. +#[must_use] +pub fn digest(bytes: &[u8]) -> u64 { + let h = fold(SEED, bytes); + let h = fold(h, &len_u64(bytes.len()).to_le_bytes()); + h.rotate_left(26) +} + +// --------------------------------------------------------------------------- +// types +// --------------------------------------------------------------------------- + +/// One row, in TSV terms. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BinRow { + /// Relation name, e.g. `CALLS_RESOLVED`. + pub predicate: String, + /// Subject label. + pub subject: String, + /// Object label. + pub object: String, + /// Grade of the evidence. + pub grade: String, + /// `(file, start, end)`, or `None` when the row has no source site. + pub site: Option<(String, usize, usize)>, + /// Charon `FunDeclId` index of the function the row was found in; `None` is [`NO_ID`]. + pub fun: Option, +} + +/// One provenance record. `None` is UNKNOWN and is distinct from every value, +/// including the string `"UNKNOWN"`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Prov { + /// Charon version. + pub charon_version: Option, + /// Charon commit. + pub charon_commit: Option, + /// LLBC schema version. + pub llbc_schema_version: Option, + /// Digest of the LLBC file. + pub llbc_digest: Option, + /// rustc commit. + pub rustc_commit: Option, + /// Target configuration. + pub target_config: Option, + /// Revision of the analysed source. + pub source_revision: Option, +} + +impl Prov { + fn fields(&self) -> [&Option; PROV_FIELDS] { + [ + &self.charon_version, + &self.charon_commit, + &self.llbc_schema_version, + &self.llbc_digest, + &self.rustc_commit, + &self.target_config, + &self.source_revision, + ] + } +} + +/// A file read back by [`read`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Decoded { + /// The rows, in file order. + pub rows: Vec, + /// The partial flag from the header. + pub partial: bool, + /// The provenance records. + pub prov: Vec, + /// The header digest, already verified against the body. + pub digest: u64, +} + +/// Why [`write`] refused its input. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WriteError { + /// More symbols than a `u32` id can name. + TooManySymbols, + /// A span offset that does not fit below [`NO_ID`]. + SpanTooLarge { + /// The offending offset. + value: usize, + }, + /// More rows than a `u32` row count can hold. + TooManyRows, + /// A codebook or blob that cannot be addressed. + CodebookTooLarge { + /// Which section. + section: &'static str, + /// Entry count, or byte length for a blob. + len: usize, + }, + /// No provenance record was given. + NoProv, + /// A function declaration index equal to [`NO_ID`]. + FunDeclTooLarge { + /// The offending index. + value: u32, + }, +} + +/// Why [`read`] refused a file; one variant per refusal. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReadError { + /// Wrong magic. + Magic, + /// A major version this reader does not know. + Major(u16), + /// A minor version newer than this reader. + Minor(u16), + /// Non-zero reserved header bytes. + Reserved, + /// The file is shorter or longer than its header says. + Truncated { + /// Length recorded in the header (0 when the header itself is cut). + header: u64, + /// Actual byte length. + actual: usize, + }, + /// The body does not match the header digest. + Digest { + /// Digest recorded in the header. + header: u64, + /// Digest of the body. + actual: u64, + }, + /// The section directory does not fit in the file. + Directory, + /// A directory entry that is misaligned, out of order, overlapping, outside the file or duplicated. + Section { + /// The entry's tag. + tag: u32, + }, + /// An unknown section with the must-understand bit. + UnknownRequired { + /// The tag. + tag: u32, + }, + /// A required section is absent. + MissingSection { + /// The tag. + tag: u32, + }, + /// A section whose width, encoding or length is wrong. + LaneLength { + /// The tag. + tag: u32, + }, + /// An id at or past the end of the table it names. + IdRange { + /// The lane. + tag: u32, + /// The row (or symbol) index. + row: usize, + }, + /// A string that is not valid UTF-8, or an offset inside a character. + Utf8 { + /// The section. + tag: u32, + }, + /// A codeword that is not allowed in its lane. + Codeword { + /// The lane. + tag: u32, + /// The row. + row: usize, + }, +} + +impl fmt::Display for WriteError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::TooManySymbols => write!(f, "more symbols than a u32 id can name"), + Self::SpanTooLarge { value } => write!(f, "span offset {value} does not fit a u32"), + Self::TooManyRows => write!(f, "more rows than a u32 row count can hold"), + Self::CodebookTooLarge { section, len } => { + write!(f, "{section} is too large to address ({len})") + } + Self::NoProv => write!(f, "no provenance record"), + Self::FunDeclTooLarge { value } => { + write!( + f, + "function declaration index {value} collides with the absent id" + ) + } + } + } +} + +impl std::error::Error for WriteError {} + +impl fmt::Display for ReadError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Magic => write!(f, "bad magic"), + Self::Major(v) => write!(f, "unsupported major version {v}"), + Self::Minor(v) => write!(f, "unsupported minor version {v}"), + Self::Reserved => write!(f, "reserved header bytes are not zero"), + Self::Truncated { header, actual } => { + write!(f, "file is {actual} bytes, header says {header}") + } + Self::Digest { header, actual } => { + write!(f, "digest mismatch: header {header:#x}, body {actual:#x}") + } + Self::Directory => write!(f, "section directory does not fit in the file"), + Self::Section { tag } => write!(f, "bad directory entry for section {tag:#x}"), + Self::UnknownRequired { tag } => write!(f, "unknown required section {tag:#x}"), + Self::MissingSection { tag } => write!(f, "missing section {tag:#x}"), + Self::LaneLength { tag } => write!(f, "section {tag:#x} has the wrong shape"), + Self::IdRange { tag, row } => write!(f, "id out of range in {tag:#x} at {row}"), + Self::Utf8 { tag } => write!(f, "invalid UTF-8 in section {tag:#x}"), + Self::Codeword { tag, row } => write!(f, "bad codeword in {tag:#x} at {row}"), + } + } +} + +impl std::error::Error for ReadError {} + +// --------------------------------------------------------------------------- +// writer +// --------------------------------------------------------------------------- + +/// The dispatch target of a `MAY_DISPATCH` object, derived exactly as +/// lance-graph's `artifact::dispatch_target` does: the `Trait::method` part +/// before `" via "`, or the whole object when there is none. +fn dispatch_target(object: &str) -> &str { + object.find(" via ").map_or(object, |i| &object[..i]) +} + +struct Symbols { + labels: Vec, + index: HashMap, +} + +impl Symbols { + fn intern(&mut self, s: &str) -> Result { + if let Some(&id) = self.index.get(s) { + return Ok(id); + } + let id = u32::try_from(self.labels.len()) + .ok() + .filter(|&i| i < NO_ID) + .ok_or(WriteError::TooManySymbols)?; + self.labels.push(s.to_owned()); + self.index.insert(s.to_owned(), id); + Ok(id) + } +} + +/// A first-seen string table; the entry's 0-based index plus one is its codeword. +#[derive(Default)] +struct Book { + entries: Vec, + index: HashMap, +} + +impl Book { + fn id(&mut self, s: &str) -> usize { + if let Some(&i) = self.index.get(s) { + return i; + } + let i = self.entries.len(); + self.entries.push(s.to_owned()); + self.index.insert(s.to_owned(), i); + i + } +} + +/// A codec9 id lane: magnitude bytes plus a presence plane. +struct Lane9 { + mag: Vec, + pres: Vec, +} + +impl Lane9 { + fn new(n: usize) -> Self { + Self { + mag: vec![0; n], + pres: vec![0; n.div_ceil(64)], + } + } + + /// Store codeword `c` (`0..=511`) at `row`. The caller guarantees `c <= 511` + /// and `row` is in range. + fn set(&mut self, row: usize, c: usize) { + if let Some(m) = self.mag.get_mut(row) { + *m = u8::try_from(c & 0xFF).unwrap_or(0); + } + if c >> 8 & 1 == 1 + && let Some(w) = self.pres.get_mut(row / 64) + { + *w |= 1 << (row % 64); + } + } +} + +struct Section { + tag: u32, + width: u16, + enc: u16, + data: Vec, +} + +fn pad8(v: &mut Vec) { + while !v.len().is_multiple_of(8) { + v.push(0); + } +} + +fn u32_lane(vals: &[u32]) -> Vec { + vals.iter().flat_map(|v| v.to_le_bytes()).collect() +} + +fn u64_lane(vals: &[u64]) -> Vec { + vals.iter().flat_map(|v| v.to_le_bytes()).collect() +} + +fn too_large(section: &'static str, len: usize) -> WriteError { + WriteError::CodebookTooLarge { section, len } +} + +/// `u32 n`, `n + 1` offsets, the UTF-8 blob, zero-padded to 8. +fn kv_codebook(section: &'static str, entries: &[String]) -> Result, WriteError> { + let n = u32::try_from(entries.len()).map_err(|_| too_large(section, entries.len()))?; + let mut out = n.to_le_bytes().to_vec(); + let mut blob: Vec = Vec::new(); + out.extend_from_slice(&0u32.to_le_bytes()); + for e in entries { + blob.extend_from_slice(e.as_bytes()); + let off = u32::try_from(blob.len()).map_err(|_| too_large(section, blob.len()))?; + out.extend_from_slice(&off.to_le_bytes()); + } + out.extend_from_slice(&blob); + pad8(&mut out); + Ok(out) +} + +fn prov_section(prov: &[Prov]) -> Result, WriteError> { + let n = u32::try_from(prov.len()).map_err(|_| too_large("PROV", prov.len()))?; + let mut out = n.to_le_bytes().to_vec(); + for rec in prov { + for field in rec.fields() { + match field { + None => out.extend_from_slice(&NO_ID.to_le_bytes()), + Some(s) => { + let len = u32::try_from(s.len()) + .ok() + .filter(|&l| l < NO_ID) + .ok_or_else(|| too_large("PROV", s.len()))?; + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(s.as_bytes()); + } + } + } + } + pad8(&mut out); + Ok(out) +} + +fn span_u32(value: usize) -> Result { + u32::try_from(value) + .ok() + .filter(|&v| v < NO_ID) + .ok_or(WriteError::SpanTooLarge { value }) +} + +fn lane9_sections(mag: u32, pres: u32, lane: &Lane9, out: &mut Vec
) { + out.push(Section { + tag: mag, + width: 1, + enc: ENC_CODEC9, + data: lane.mag.clone(), + }); + out.push(Section { + tag: pres, + width: 8, + enc: ENC_PRES, + data: u64_lane(&lane.pres), + }); +} + +fn raw_u32(tag: u32, vals: &[u32]) -> Section { + Section { + tag, + width: 4, + enc: ENC_RAW, + data: u32_lane(vals), + } +} + +/// Encode `rows` as a `.rows.bin` file. The same input always gives the +/// same bytes: every table is built in first-seen order. +/// +/// # Errors +/// A [`WriteError`] when a count, offset or codebook cannot be addressed, or +/// when `prov` is empty. +#[expect(clippy::too_many_lines, reason = "one pass that builds every lane")] +pub fn write(rows: &[BinRow], partial: bool, prov: &[Prov]) -> Result, WriteError> { + if prov.is_empty() { + return Err(WriteError::NoProv); + } + let n = rows.len(); + let n_rows = u32::try_from(n) + .ok() + .filter(|&v| v < NO_ID) + .ok_or(WriteError::TooManyRows)?; + + let mut syms = Symbols { + labels: Vec::new(), + index: HashMap::new(), + }; + let (mut preds, mut grades, mut files) = (Book::default(), Book::default(), Book::default()); + // Codebook ids per row; the file lane is built after the file count is known. + let (mut pred_ids, mut grade_ids, mut file_ids) = (Vec::new(), Vec::new(), Vec::new()); + let (mut subj, mut obj, mut mayt) = (Vec::new(), Vec::new(), Vec::new()); + let (mut start, mut end, mut fundecl) = (Vec::new(), Vec::new(), Vec::new()); + + for r in rows { + pred_ids.push(preds.id(&r.predicate)); + grade_ids.push(grades.id(&r.grade)); + subj.push(syms.intern(&r.subject)?); + obj.push(syms.intern(&r.object)?); + mayt.push(if r.predicate == MAY_DISPATCH { + syms.intern(dispatch_target(&r.object))? + } else { + NO_ID + }); + if let Some((file, s, e)) = &r.site { + file_ids.push(Some(files.id(file))); + start.push(span_u32(*s)?); + end.push(span_u32(*e)?); + } else { + file_ids.push(None); + start.push(NO_ID); + end.push(NO_ID); + } + fundecl.push(match r.fun { + Some(v) if v == NO_ID => return Err(WriteError::FunDeclTooLarge { value: v }), + Some(v) => v, + None => NO_ID, + }); + } + + // Owners are appended to the same table, so the loop runs to a fixpoint. + let mut owners: Vec = Vec::new(); + let mut i = 0; + while i < syms.labels.len() { + let prefix = syms.labels[i] + .find("::") + .map(|p| syms.labels[i][..p].to_owned()); + owners.push(match prefix { + Some(p) => syms.intern(&p)?, + None => NO_ID, + }); + i += 1; + } + + let n_symbols = u32::try_from(syms.labels.len()).map_err(|_| WriteError::TooManySymbols)?; + let n_files = u32::try_from(files.entries.len()) + .map_err(|_| too_large("FILE_CB", files.entries.len()))?; + for (name, book) in [("PRED_CB", &preds), ("GRADE_CB", &grades)] { + if book.entries.len() > CODEC9_MAX { + return Err(too_large(name, book.entries.len())); + } + } + + let mut sections: Vec
= Vec::new(); + + let (mut pl, mut gl) = (Lane9::new(n), Lane9::new(n)); + for row in 0..n { + pl.set(row, pred_ids[row] + 1); + gl.set(row, grade_ids[row] + 1); + } + lane9_sections(tag::PRED_MAG, tag::PRED_PRES, &pl, &mut sections); + lane9_sections(tag::GRADE_MAG, tag::GRADE_PRES, &gl, &mut sections); + sections.push(Section { + tag: tag::PRED_CB, + width: 1, + enc: ENC_KV, + data: kv_codebook("PRED_CB", &preds.entries)?, + }); + sections.push(Section { + tag: tag::GRADE_CB, + width: 1, + enc: ENC_KV, + data: kv_codebook("GRADE_CB", &grades.entries)?, + }); + + if files.entries.len() <= CODEC9_MAX { + let mut fl = Lane9::new(n); + for (row, f) in file_ids.iter().enumerate() { + fl.set(row, f.map_or(0, |id| id + 1)); + } + lane9_sections(tag::FILE, tag::FILE_PRES, &fl, &mut sections); + } else { + let raw: Vec = file_ids + .iter() + .map(|f| f.and_then(|id| u32::try_from(id).ok()).unwrap_or(NO_ID)) + .collect(); + sections.push(raw_u32(tag::FILE, &raw)); + } + sections.push(Section { + tag: tag::FILE_CB, + width: 1, + enc: ENC_KV, + data: kv_codebook("FILE_CB", &files.entries)?, + }); + + sections.push(raw_u32(tag::SUBJ, &subj)); + sections.push(raw_u32(tag::OBJ, &obj)); + sections.push(raw_u32(tag::MAYT, &mayt)); + sections.push(raw_u32(tag::START, &start)); + sections.push(raw_u32(tag::END, &end)); + + let mut blob: Vec = Vec::new(); + let mut offs: Vec = vec![0]; + for l in &syms.labels { + blob.extend_from_slice(l.as_bytes()); + offs.push(u32::try_from(blob.len()).map_err(|_| too_large("SYM_BLOB", blob.len()))?); + } + let hashes: Vec = syms.labels.iter().map(|l| digest(l.as_bytes())).collect(); + sections.push(raw_u32(tag::SYM_OFF, &offs)); + sections.push(Section { + tag: tag::SYM_BLOB, + width: 1, + enc: ENC_RAW, + data: blob, + }); + sections.push(Section { + tag: tag::SYM_HASH, + width: 8, + enc: ENC_RAW, + data: u64_lane(&hashes), + }); + sections.push(raw_u32(tag::SYM_OWNER, &owners)); + sections.push(raw_u32(tag::FUNDECL, &fundecl)); + + // Every row of this chunk names a symbol; WORD and METHOD are reserved. + let mut kinds = Lane9::new(n); + for row in 0..n { + kinds.set(row, KIND_SYMBOL); + } + lane9_sections(tag::OBJ_KIND_MAG, tag::OBJ_KIND_PRES, &kinds, &mut sections); + sections.push(Section { + tag: tag::PROV, + width: 1, + enc: ENC_KV, + data: prov_section(prov)?, + }); + + Ok(assemble( + [n_rows, n_symbols, n_files], + u32::from(partial) * FLAG_PARTIAL, + sections, + )) +} + +/// Lay out header, directory and sections (each starting on a multiple of 8). +fn assemble(counts: [u32; 3], flags: u32, mut sections: Vec
) -> Vec { + sections.sort_by_key(|s| s.tag); + let n_sections = u32::try_from(sections.len()).unwrap_or(u32::MAX); + let mut dir = Vec::with_capacity(sections.len() * DIR_ENTRY_LEN); + let mut body: Vec = Vec::new(); + let base = HEADER_LEN + sections.len() * DIR_ENTRY_LEN; + for s in §ions { + pad8(&mut body); + let offset = u32::try_from(base + body.len()).unwrap_or(u32::MAX); + let len = u32::try_from(s.data.len()).unwrap_or(u32::MAX); + dir.extend_from_slice(&s.tag.to_le_bytes()); + dir.extend_from_slice(&s.width.to_le_bytes()); + dir.extend_from_slice(&s.enc.to_le_bytes()); + dir.extend_from_slice(&offset.to_le_bytes()); + dir.extend_from_slice(&len.to_le_bytes()); + body.extend_from_slice(&s.data); + } + pad8(&mut body); + + let mut file = vec![0u8; HEADER_LEN]; + file[0..8].copy_from_slice(&MAGIC); + file[8..10].copy_from_slice(&MAJOR.to_le_bytes()); + file[10..12].copy_from_slice(&MINOR.to_le_bytes()); + file[12..16].copy_from_slice(&flags.to_le_bytes()); + file[24..32].copy_from_slice(&len_u64(base + body.len()).to_le_bytes()); + file[32..36].copy_from_slice(&counts[0].to_le_bytes()); + file[36..40].copy_from_slice(&counts[1].to_le_bytes()); + file[40..44].copy_from_slice(&counts[2].to_le_bytes()); + file[44..48].copy_from_slice(&n_sections.to_le_bytes()); + file.extend_from_slice(&dir); + file.extend_from_slice(&body); + let d = digest(&file[32..]); + file[16..24].copy_from_slice(&d.to_le_bytes()); + file +} + +// --------------------------------------------------------------------------- +// reader +// --------------------------------------------------------------------------- + +fn le16(b: &[u8], o: usize) -> u16 { + b.get(o..o + 2) + .and_then(|s| s.first_chunk::<2>()) + .map_or(0, |c| u16::from_le_bytes(*c)) +} + +fn le32(b: &[u8], o: usize) -> u32 { + b.get(o..o + 4) + .and_then(|s| s.first_chunk::<4>()) + .map_or(0, |c| u32::from_le_bytes(*c)) +} + +fn le64(b: &[u8], o: usize) -> u64 { + b.get(o..o + 8) + .and_then(|s| s.first_chunk::<8>()) + .map_or(0, |c| u64::from_le_bytes(*c)) +} + +/// Check the magic and major version of a header and return its digest field. +/// `None` when either is wrong. +#[must_use] +pub fn header_digest(header: &[u8; HEADER_LEN]) -> Option { + (header[0..8] == MAGIC && le16(header, 8) == MAJOR).then(|| le64(header, 16)) +} + +struct Sec<'a> { + width: u16, + enc: u16, + data: &'a [u8], +} + +type Secs<'a> = BTreeMap>; + +fn find<'s, 'a>(secs: &'s Secs<'a>, tag: u32) -> Result<&'s Sec<'a>, ReadError> { + secs.get(&tag).ok_or(ReadError::MissingSection { tag }) +} + +/// A fixed-shape section: exact element width, encoding and byte length. +fn lane<'a>( + secs: &Secs<'a>, + tag: u32, + width: u16, + enc: u16, + len: usize, +) -> Result<&'a [u8], ReadError> { + let s = find(secs, tag)?; + if s.width != width || s.enc != enc || s.data.len() != len { + return Err(ReadError::LaneLength { tag }); + } + Ok(s.data) +} + +fn u32s(b: &[u8]) -> Vec { + b.as_chunks::<4>() + .0 + .iter() + .map(|c| u32::from_le_bytes(*c)) + .collect() +} + +fn u64s(b: &[u8]) -> Vec { + b.as_chunks::<8>() + .0 + .iter() + .map(|c| u64::from_le_bytes(*c)) + .collect() +} + +fn lane_len(n: usize, width: usize, tag: u32) -> Result { + n.checked_mul(width).ok_or(ReadError::LaneLength { tag }) +} + +/// Only zero bytes, and fewer than 8 of them, may follow the last field. +fn only_padding(rest: &[u8], tag: u32) -> Result<(), ReadError> { + if rest.len() < 8 && rest.iter().all(|&b| b == 0) { + Ok(()) + } else { + Err(ReadError::LaneLength { tag }) + } +} + +fn decode_kv(tag: u32, data: &[u8]) -> Result, ReadError> { + let bad = ReadError::LaneLength { tag }; + if !data.len().is_multiple_of(8) { + return Err(bad); + } + let n = usize::try_from(le32(data, 0)).map_err(|_| bad.clone())?; + let table = n + .checked_add(1) + .and_then(|k| k.checked_mul(4)) + .and_then(|k| k.checked_add(4)) + .filter(|&k| k <= data.len()) + .ok_or_else(|| bad.clone())?; + let offs = u32s(&data[4..table]); + let blob = &data[table..]; + let blob_len = offs + .last() + .and_then(|&o| usize::try_from(o).ok()) + .ok_or_else(|| bad.clone())?; + if offs.first() != Some(&0) || blob_len > blob.len() || offs.windows(2).any(|w| w[0] > w[1]) { + return Err(bad); + } + only_padding(&blob[blob_len..], tag)?; + let mut out = Vec::with_capacity(n); + for w in offs.windows(2) { + let (a, b) = (w[0] as usize, w[1] as usize); + let s = std::str::from_utf8(&blob[a..b]).map_err(|_| ReadError::Utf8 { tag })?; + out.push(s.to_owned()); + } + Ok(out) +} + +fn decode_prov(data: &[u8]) -> Result, ReadError> { + let tag = tag::PROV; + let bad = ReadError::LaneLength { tag }; + if !data.len().is_multiple_of(8) || data.len() < 4 { + return Err(bad); + } + let n = le32(data, 0); + if n == 0 { + return Err(bad); + } + let mut pos = 4usize; + let mut out = Vec::new(); + for _ in 0..n { + let mut f: Vec> = Vec::with_capacity(PROV_FIELDS); + for _ in 0..PROV_FIELDS { + let len = data + .get(pos..pos + 4) + .map(|_| le32(data, pos)) + .ok_or_else(|| bad.clone())?; + pos += 4; + if len == NO_ID { + f.push(None); + continue; + } + let end = usize::try_from(len) + .ok() + .and_then(|l| pos.checked_add(l)) + .filter(|&e| e <= data.len()) + .ok_or_else(|| bad.clone())?; + let s = std::str::from_utf8(&data[pos..end]).map_err(|_| ReadError::Utf8 { tag })?; + f.push(Some(s.to_owned())); + pos = end; + } + let mut it = f.into_iter(); + let mut next = || it.next().flatten(); + out.push(Prov { + charon_version: next(), + charon_commit: next(), + llbc_schema_version: next(), + llbc_digest: next(), + rustc_commit: next(), + target_config: next(), + source_revision: next(), + }); + } + only_padding(&data[pos..], tag)?; + Ok(out) +} + +fn codeword(mag: &[u8], pres: &[u64], row: usize) -> usize { + let m = usize::from(mag.get(row).copied().unwrap_or(0)); + let p = pres.get(row / 64).map_or(0, |w| (w >> (row % 64)) & 1); + m | (usize::try_from(p).unwrap_or(0) << 8) +} + +/// Read a codec9 (magnitude, presence) lane pair of `n` rows. +fn lane9(secs: &Secs<'_>, mag: u32, pres: u32, n: usize) -> Result<(Vec, Vec), ReadError> { + let m = lane(secs, mag, 1, ENC_CODEC9, n)?.to_vec(); + let p = u64s(lane( + secs, + pres, + 8, + ENC_PRES, + lane_len(n.div_ceil(64), 8, pres)?, + )?); + Ok((m, p)) +} + +/// Look up codeword `c` of a codebook lane: 0 is refused (`Codeword`), past the book is `IdRange`. +fn book_entry(book: &[String], c: usize, tag: u32, row: usize) -> Result<&str, ReadError> { + if c == 0 { + return Err(ReadError::Codeword { tag, row }); + } + book.get(c - 1) + .map(String::as_str) + .ok_or(ReadError::IdRange { tag, row }) +} + +fn sym_ref(syms: &[&str], id: u32, tag: u32, row: usize) -> Result { + usize::try_from(id) + .ok() + .and_then(|i| syms.get(i)) + .map(|s| (*s).to_owned()) + .ok_or(ReadError::IdRange { tag, row }) +} + +/// Parse the header and directory; returns the counts, flags and sections. +fn parse_directory(bytes: &[u8]) -> Result<(u64, u32, [usize; 3], Secs<'_>), ReadError> { + let Some(h) = bytes.first_chunk::() else { + return Err(ReadError::Truncated { + header: 0, + actual: bytes.len(), + }); + }; + if h[0..8] != MAGIC { + return Err(ReadError::Magic); + } + let major = le16(h, 8); + if major != MAJOR { + return Err(ReadError::Major(major)); + } + let minor = le16(h, 10); + if minor > MINOR { + return Err(ReadError::Minor(minor)); + } + if h[48..64].iter().any(|&b| b != 0) { + return Err(ReadError::Reserved); + } + let file_len = le64(h, 24); + if file_len != len_u64(bytes.len()) { + return Err(ReadError::Truncated { + header: file_len, + actual: bytes.len(), + }); + } + let header_digest = le64(h, 16); + let actual = digest(&bytes[32..]); + if actual != header_digest { + return Err(ReadError::Digest { + header: header_digest, + actual, + }); + } + + let to_usize = |v: u32| usize::try_from(v).map_err(|_| ReadError::Directory); + let n_sections = to_usize(le32(h, 44))?; + let dir_end = n_sections + .checked_mul(DIR_ENTRY_LEN) + .and_then(|d| d.checked_add(HEADER_LEN)) + .filter(|&e| e <= bytes.len()) + .ok_or(ReadError::Directory)?; + + let mut secs: Secs<'_> = BTreeMap::new(); + let mut prev_end = dir_end; + for i in 0..n_sections { + let at = HEADER_LEN + i * DIR_ENTRY_LEN; + let t = le32(bytes, at); + let offset = to_usize(le32(bytes, at + 8))?; + let len = to_usize(le32(bytes, at + 12))?; + let end = offset.checked_add(len).filter(|&e| e <= bytes.len()); + let Some(end) = end.filter(|_| offset % 8 == 0 && offset >= prev_end) else { + return Err(ReadError::Section { tag: t }); + }; + prev_end = end; + let sec = Sec { + width: le16(bytes, at + 4), + enc: le16(bytes, at + 6), + data: &bytes[offset..end], + }; + if secs.insert(t, sec).is_some() { + return Err(ReadError::Section { tag: t }); + } + } + for &t in secs.keys() { + if t & 0x8000_0000 != 0 && !tag::ALL.contains(&t) { + return Err(ReadError::UnknownRequired { tag: t }); + } + } + let counts = [le32(h, 32), le32(h, 36), le32(h, 40)]; + let counts = counts.map(|c| usize::try_from(c).unwrap_or(usize::MAX)); + Ok((header_digest, le32(h, 12), counts, secs)) +} + +/// Decode and verify a `.rows.bin` file. Checks run in a fixed order and +/// each refusal is its own [`ReadError`]: header, length, digest, directory, +/// section shapes, then the contents of every lane. +/// +/// # Errors +/// The first [`ReadError`] found. +#[expect(clippy::too_many_lines, reason = "one pass that validates every lane")] +pub fn read(bytes: &[u8]) -> Result { + let (dig, flags, [n, n_syms, n_files], secs) = parse_directory(bytes)?; + for t in tag::ALL { + if t != tag::FILE_PRES && !secs.contains_key(&t) { + return Err(ReadError::MissingSection { tag: t }); + } + } + + let (pm, pp) = lane9(&secs, tag::PRED_MAG, tag::PRED_PRES, n)?; + let (gm, gp) = lane9(&secs, tag::GRADE_MAG, tag::GRADE_PRES, n)?; + let (km, kp) = lane9(&secs, tag::OBJ_KIND_MAG, tag::OBJ_KIND_PRES, n)?; + let preds = decode_kv(tag::PRED_CB, find(&secs, tag::PRED_CB)?.data)?; + let grades = decode_kv(tag::GRADE_CB, find(&secs, tag::GRADE_CB)?.data)?; + let files = decode_kv(tag::FILE_CB, find(&secs, tag::FILE_CB)?.data)?; + for (t, book) in [(tag::PRED_CB, &preds), (tag::GRADE_CB, &grades)] { + if find(&secs, t)?.enc != ENC_KV || book.len() > CODEC9_MAX { + return Err(ReadError::LaneLength { tag: t }); + } + } + if find(&secs, tag::FILE_CB)?.enc != ENC_KV || files.len() != n_files { + return Err(ReadError::LaneLength { tag: tag::FILE_CB }); + } + + // Files: codec9 with a presence plane up to 511 entries, raw u32 beyond. + let file9 = if n_files <= CODEC9_MAX { + Some(lane9(&secs, tag::FILE, tag::FILE_PRES, n)?) + } else { + if secs.contains_key(&tag::FILE_PRES) { + return Err(ReadError::LaneLength { + tag: tag::FILE_PRES, + }); + } + None + }; + let file_raw = match file9 { + Some(_) => Vec::new(), + None => u32s(lane( + &secs, + tag::FILE, + 4, + ENC_RAW, + lane_len(n, 4, tag::FILE)?, + )?), + }; + + let u32_lane_of = |t: u32| -> Result, ReadError> { + Ok(u32s(lane(&secs, t, 4, ENC_RAW, lane_len(n, 4, t)?)?)) + }; + let (subj, obj, mayt) = ( + u32_lane_of(tag::SUBJ)?, + u32_lane_of(tag::OBJ)?, + u32_lane_of(tag::MAYT)?, + ); + let (start, end, fundecl) = ( + u32_lane_of(tag::START)?, + u32_lane_of(tag::END)?, + u32_lane_of(tag::FUNDECL)?, + ); + + let off_len = lane_len( + n_syms + .checked_add(1) + .ok_or(ReadError::LaneLength { tag: tag::SYM_OFF })?, + 4, + tag::SYM_OFF, + )?; + let offs = u32s(lane(&secs, tag::SYM_OFF, 4, ENC_RAW, off_len)?); + let blob_sec = find(&secs, tag::SYM_BLOB)?; + if blob_sec.width != 1 || blob_sec.enc != ENC_RAW { + return Err(ReadError::LaneLength { tag: tag::SYM_BLOB }); + } + let blob_str = + std::str::from_utf8(blob_sec.data).map_err(|_| ReadError::Utf8 { tag: tag::SYM_BLOB })?; + let ok_offsets = offs.first() == Some(&0) + && offs.last().and_then(|&o| usize::try_from(o).ok()) == Some(blob_str.len()) + && offs.windows(2).all(|w| w[0] <= w[1]); + if !ok_offsets { + return Err(ReadError::LaneLength { tag: tag::SYM_OFF }); + } + let mut labels: Vec<&str> = Vec::with_capacity(n_syms); + for w in offs.windows(2) { + let (a, b) = (w[0] as usize, w[1] as usize); + if !blob_str.is_char_boundary(a) || !blob_str.is_char_boundary(b) { + return Err(ReadError::Utf8 { tag: tag::SYM_BLOB }); + } + labels.push(&blob_str[a..b]); + } + lane( + &secs, + tag::SYM_HASH, + 8, + ENC_RAW, + lane_len(n_syms, 8, tag::SYM_HASH)?, + )?; + let owners = u32s(lane( + &secs, + tag::SYM_OWNER, + 4, + ENC_RAW, + lane_len(n_syms, 4, tag::SYM_OWNER)?, + )?); + for (i, &o) in owners.iter().enumerate() { + if o != NO_ID && usize::try_from(o).is_ok_and(|o| o >= n_syms) { + return Err(ReadError::IdRange { + tag: tag::SYM_OWNER, + row: i, + }); + } + } + let prov_sec = find(&secs, tag::PROV)?; + if prov_sec.width != 1 || prov_sec.enc != ENC_KV { + return Err(ReadError::LaneLength { tag: tag::PROV }); + } + let prov = decode_prov(prov_sec.data)?; + + let mut rows = Vec::with_capacity(n); + for row in 0..n { + let kind = codeword(&km, &kp, row); + if kind == 0 || kind > KIND_MAX { + return Err(ReadError::Codeword { + tag: tag::OBJ_KIND_MAG, + row, + }); + } + let predicate = book_entry(&preds, codeword(&pm, &pp, row), tag::PRED_MAG, row)?.to_owned(); + let grade = book_entry(&grades, codeword(&gm, &gp, row), tag::GRADE_MAG, row)?.to_owned(); + let subject = sym_ref(&labels, subj[row], tag::SUBJ, row)?; + let object = sym_ref(&labels, obj[row], tag::OBJ, row)?; + if mayt[row] != NO_ID { + sym_ref(&labels, mayt[row], tag::MAYT, row)?; + } + let file: Option<&str> = match &file9 { + Some((m, p)) => match codeword(m, p, row) { + 0 => None, + c => Some(book_entry(&files, c, tag::FILE, row)?), + }, + None => match file_raw[row] { + NO_ID => None, + v => Some( + usize::try_from(v) + .ok() + .and_then(|i| files.get(i)) + .map(String::as_str) + .ok_or(ReadError::IdRange { + tag: tag::FILE, + row, + })?, + ), + }, + }; + let site = match file { + Some(f) => { + let span = |v: u32, t: u32| { + if v == NO_ID { + Err(ReadError::IdRange { tag: t, row }) + } else { + usize::try_from(v).map_err(|_| ReadError::IdRange { tag: t, row }) + } + }; + Some(( + f.to_owned(), + span(start[row], tag::START)?, + span(end[row], tag::END)?, + )) + } + None => { + if start[row] != NO_ID { + return Err(ReadError::IdRange { + tag: tag::START, + row, + }); + } + if end[row] != NO_ID { + return Err(ReadError::IdRange { tag: tag::END, row }); + } + None + } + }; + rows.push(BinRow { + predicate, + subject, + object, + grade, + site, + fun: (fundecl[row] != NO_ID).then_some(fundecl[row]), + }); + } + Ok(Decoded { + rows, + partial: flags & FLAG_PARTIAL != 0, + prov, + digest: dig, + }) +} diff --git a/crates/ruff_charon/tests/store_bin.rs b/crates/ruff_charon/tests/store_bin.rs new file mode 100644 index 00000000000000..bdebc1a224bb49 --- /dev/null +++ b/crates/ruff_charon/tests/store_bin.rs @@ -0,0 +1,477 @@ +//! Tests for the binary rows format: the digest vectors, the round trip, and +//! one refusal test per `ReadError` the reader promises. + +use ruff_charon::store_bin::{ + BinRow, CODEC9_MAX, HEADER_LEN, NO_ID, Prov, ReadError, WriteError, digest, read, tag, write, +}; + +// ---- helpers --------------------------------------------------------------- + +fn row( + pred: &str, + subj: &str, + obj: &str, + site: Option<(&str, usize, usize)>, + fun: Option, +) -> BinRow { + BinRow { + predicate: pred.into(), + subject: subj.into(), + object: obj.into(), + grade: "CHARON_RESOLVED".into(), + site: site.map(|(f, s, e)| (f.into(), s, e)), + fun, + } +} + +fn prov() -> Prov { + Prov { + charon_version: Some("0.1.2".into()), + charon_commit: Some("abc123".into()), + llbc_schema_version: Some("7".into()), + llbc_digest: None, + rustc_commit: Some("def456".into()), + target_config: None, + source_revision: Some("rev".into()), + } +} + +fn sample() -> Vec { + vec![ + row( + "DEFINES", + "krate::a::f", + "krate::a", + Some(("src/a.rs", 0, 10)), + Some(0), + ), + row( + "CALLS_RESOLVED", + "krate::a::f", + "krate::b::g", + Some(("src/a.rs", 4, 8)), + Some(0), + ), + row( + "MAY_DISPATCH", + "krate::b::g", + "Trait::run via dyn", + Some(("src/b.rs", 1, 2)), + Some(1), + ), + row("UNRESOLVED_CALL", "krate::b::g", "", None, None), + row( + "DEFINES", + "krate::b::\u{00fc}ber", + "krate::b", + Some(("src/b.rs", 3, 9)), + Some(2), + ), + ] +} + +fn rd32(b: &[u8], o: usize) -> u32 { + u32::from_le_bytes(b[o..o + 4].try_into().unwrap()) +} + +fn wr32(b: &mut [u8], o: usize, v: u32) { + b[o..o + 4].copy_from_slice(&v.to_le_bytes()); +} + +/// Recompute the digest field so a test can isolate a check that runs after it. +fn reseal(b: &mut [u8]) { + let d = digest(&b[32..]); + b[16..24].copy_from_slice(&d.to_le_bytes()); +} + +/// Offset of the directory entry for `t`. +fn entry(b: &[u8], t: u32) -> usize { + let n = rd32(b, 44) as usize; + (0..n) + .map(|i| HEADER_LEN + 16 * i) + .find(|&at| rd32(b, at) == t) + .expect("tag in directory") +} + +/// The bytes of section `t`. +fn section(b: &[u8], t: u32) -> &[u8] { + let at = entry(b, t); + let (off, len) = (rd32(b, at + 8) as usize, rd32(b, at + 12) as usize); + &b[off..off + len] +} + +fn files_rows(n: usize) -> Vec { + let mut rows: Vec = (0..n) + .map(|i| { + row( + "DEFINES", + &format!("k::f{i}"), + "k", + Some((&format!("src/f{i}.rs"), i, i + 1)), + None, + ) + }) + .collect(); + rows.push(row("UNRESOLVED_CALL", "k::z", "", None, None)); + rows +} + +// ---- digest ---------------------------------------------------------------- + +/// Red if the mix, the tail handling or the length fold differs from lance-graph's. +#[test] +fn digest_matches_the_golden_vectors() { + assert_eq!(digest(b""), 0xd0dd_3f66_b426_3ceb); + assert_eq!(digest(b"a"), 0x9fc7_5a5c_9bce_51be); + assert_eq!(digest(b"CHRNROWS digest golden v1"), 0xfe0a_b233_a400_27ce); +} + +// ---- round trip ------------------------------------------------------------ + +/// Red if any row field, the partial flag or the provenance does not survive. +#[test] +fn a_round_trip_returns_every_row() { + let rows = sample(); + assert!(rows.len() >= 5); + assert!(rows.iter().any(|r| r.predicate == "MAY_DISPATCH")); + assert!(rows.iter().any(|r| r.site.is_none())); + assert!(rows.iter().any(|r| r.fun.is_none())); + assert!(rows.iter().any(|r| !r.subject.is_ascii())); + let p = vec![prov()]; + let bytes = write(&rows, true, &p).unwrap(); + let d = read(&bytes).unwrap(); + assert_eq!(d.rows, rows); + assert!(d.partial); + assert_eq!(d.prov, p); + let bytes = write(&rows, false, &p).unwrap(); + assert!(!read(&bytes).unwrap().partial); +} + +/// Red if any table is built in hash order. +#[test] +fn writing_is_deterministic() { + let a = write(&sample(), false, &[prov()]).unwrap(); + let b = write(&sample(), false, &[prov()]).unwrap(); + assert_eq!(a, b); +} + +/// Red if the writer accepts an empty provenance list. +#[test] +fn an_empty_prov_list_is_refused() { + assert_eq!(write(&sample(), false, &[]), Err(WriteError::NoProv)); +} + +/// Red if a function index equal to the absent id is written. +#[test] +fn a_fundecl_equal_to_the_absent_id_is_refused() { + let rows = vec![row("DEFINES", "k::a", "k", None, Some(NO_ID))]; + assert_eq!( + write(&rows, false, &[prov()]), + Err(WriteError::FunDeclTooLarge { value: NO_ID }) + ); +} + +// ---- F3: one refusal per variant ------------------------------------------- + +fn good() -> Vec { + write(&sample(), false, &[prov()]).unwrap() +} + +/// Red if a flipped body byte is not caught by the digest. +#[test] +fn a_flipped_body_byte_is_a_digest_error() { + let mut b = good(); + let last = b.len() - 9; + b[last] ^= 0x55; + assert!(matches!(read(&b), Err(ReadError::Digest { .. }))); +} + +/// Red if a cut file is not caught by the length check. +#[test] +fn a_truncated_file_is_refused() { + let b = good(); + assert!(matches!( + read(&b[..b.len() - 8]), + Err(ReadError::Truncated { .. }) + )); + assert!(matches!(read(&b[..10]), Err(ReadError::Truncated { .. }))); +} + +/// Red if the magic is not checked. +#[test] +fn a_wrong_magic_is_refused() { + let mut b = good(); + b[0] = b'X'; + assert_eq!(read(&b), Err(ReadError::Magic)); +} + +/// Red if a newer major version is read. +#[test] +fn a_newer_major_is_refused() { + let mut b = good(); + b[8] += 1; + assert_eq!(read(&b), Err(ReadError::Major(2))); +} + +/// Red if a newer minor version is read. +#[test] +fn a_newer_minor_is_refused() { + let mut b = good(); + b[10] += 1; + assert_eq!(read(&b), Err(ReadError::Minor(1))); +} + +/// Red if reserved header bytes may carry data. +#[test] +fn a_reserved_byte_is_refused() { + let mut b = good(); + b[50] = 1; + reseal(&mut b); + assert_eq!(read(&b), Err(ReadError::Reserved)); +} + +/// Red if a missing required section is not named. +#[test] +fn a_missing_section_is_named() { + let mut b = good(); + let n = rd32(&b, 44) as usize; + let at = entry(&b, tag::SUBJ); + let end = HEADER_LEN + 16 * n; + b.copy_within(at + 16..end, at); + b[end - 16..end].fill(0); + wr32(&mut b, 44, (n - 1) as u32); + reseal(&mut b); + assert_eq!(read(&b), Err(ReadError::MissingSection { tag: tag::SUBJ })); +} + +/// Red if a symbol id at the table size is dereferenced. +#[test] +fn a_symbol_id_past_the_table_is_refused() { + let mut b = good(); + let n_syms = rd32(&b, 36); + let at = entry(&b, tag::SUBJ); + let off = rd32(&b, at + 8) as usize; + wr32(&mut b, off, n_syms); + reseal(&mut b); + assert_eq!( + read(&b), + Err(ReadError::IdRange { + tag: tag::SUBJ, + row: 0 + }) + ); +} + +/// Red if a section may start outside the file. +#[test] +fn a_section_offset_past_the_file_is_refused() { + let mut b = good(); + let at = entry(&b, tag::END); + let len = b.len() as u32; + wr32(&mut b, at + 8, len + 8); + reseal(&mut b); + assert_eq!(read(&b), Err(ReadError::Section { tag: tag::END })); +} + +/// Red if a section starts off an 8-byte boundary. +#[test] +fn a_misaligned_section_is_refused() { + let mut b = good(); + let at = entry(&b, tag::END); + let off = rd32(&b, at + 8); + wr32(&mut b, at + 8, off + 4); + reseal(&mut b); + assert_eq!(read(&b), Err(ReadError::Section { tag: tag::END })); +} + +/// Red if an unknown must-understand section is skipped. +#[test] +fn an_unknown_required_tag_is_refused() { + let mut b = good(); + let at = entry(&b, tag::END); + wr32(&mut b, at, 0x8000_00FF); + reseal(&mut b); + assert_eq!( + read(&b), + Err(ReadError::UnknownRequired { tag: 0x8000_00FF }) + ); +} + +/// Red if an unknown optional section (bit 31 clear) is refused rather than skipped. +#[test] +fn an_unknown_optional_tag_is_skipped() { + let mut b = good(); + let n = rd32(&b, 44) as usize; + let dir_end = HEADER_LEN + 16 * n; + // Make room for one more directory entry; every section moves by 16. + b.splice(dir_end..dir_end, [0u8; 16]); + for i in 0..n { + let at = HEADER_LEN + 16 * i; + let off = rd32(&b, at + 8); + wr32(&mut b, at + 8, off + 16); + } + let new_len = b.len() as u32; + b[24..32].copy_from_slice(&u64::from(new_len).to_le_bytes()); + wr32(&mut b, 44, (n + 1) as u32); + wr32(&mut b, dir_end, 0x0000_00FF); + wr32(&mut b, dir_end + 8, new_len); + reseal(&mut b); + assert_eq!(read(&b).unwrap().rows, sample()); +} + +/// Red if a lane of the wrong length is accepted. +#[test] +fn a_short_lane_is_refused() { + let mut b = good(); + let at = entry(&b, tag::START); + wr32(&mut b, at + 12, 4); + reseal(&mut b); + assert_eq!(read(&b), Err(ReadError::LaneLength { tag: tag::START })); +} + +/// Red if invalid UTF-8 in a symbol label is accepted. +#[test] +fn invalid_utf8_in_a_symbol_is_refused() { + let mut b = good(); + let at = entry(&b, tag::SYM_BLOB); + let off = rd32(&b, at + 8) as usize; + b[off] = 0xFF; + reseal(&mut b); + assert_eq!(read(&b), Err(ReadError::Utf8 { tag: tag::SYM_BLOB })); +} + +/// Red if an object-kind codeword outside 1..=3 is accepted. +#[test] +fn a_bad_object_kind_codeword_is_refused() { + let mut b = good(); + let at = entry(&b, tag::OBJ_KIND_MAG); + let off = rd32(&b, at + 8) as usize; + b[off + 2] = 9; + reseal(&mut b); + assert_eq!( + read(&b), + Err(ReadError::Codeword { + tag: tag::OBJ_KIND_MAG, + row: 2 + }) + ); +} + +// ---- F6: codec9 ------------------------------------------------------------- + +/// Red if the file lane stops being codec9 at 511 files, or loses a row. +#[test] +fn five_hundred_eleven_files_use_codec9_and_round_trip() { + let rows = files_rows(CODEC9_MAX); + let b = write(&rows, false, &[prov()]).unwrap(); + let at = entry(&b, tag::FILE); + assert_eq!(u16::from_le_bytes([b[at + 6], b[at + 7]]), 1, "enc CODEC9"); + assert_eq!(u16::from_le_bytes([b[at + 4], b[at + 5]]), 1, "u8 lane"); + let d = read(&b).unwrap(); + assert_eq!(d.rows, rows); + assert!( + d.rows.last().unwrap().site.is_none(), + "codeword 0 is not file 0" + ); +} + +/// Red if the 512th file does not switch the lane to raw u32, or loses a row. +#[test] +fn five_hundred_twelve_files_use_a_raw_lane_and_round_trip() { + let rows = files_rows(CODEC9_MAX + 1); + let b = write(&rows, false, &[prov()]).unwrap(); + let at = entry(&b, tag::FILE); + assert_eq!(u16::from_le_bytes([b[at + 6], b[at + 7]]), 0, "enc RAW"); + assert_eq!(u16::from_le_bytes([b[at + 4], b[at + 5]]), 4, "u32 lane"); + assert!( + (0..rd32(&b, 44) as usize).all(|i| rd32(&b, HEADER_LEN + 16 * i) != tag::FILE_PRES), + "no FILE_PRES with a raw lane" + ); + let d = read(&b).unwrap(); + assert_eq!(d.rows, rows); + assert!(d.rows.last().unwrap().site.is_none(), "NO_ID is not file 0"); +} + +/// Red if codewords 255 and 256 are not split into magnitude byte and presence bit. +#[test] +fn the_codeword_split_is_magnitude_and_presence() { + let rows: Vec = (0..300) + .map(|i| row(&format!("P{i}"), "k::a", "k", None, None)) + .collect(); + let b = write(&rows, false, &[prov()]).unwrap(); + let mag = section(&b, tag::PRED_MAG); + let pres = section(&b, tag::PRED_PRES); + let word = |w: usize| u64::from_le_bytes(pres[w * 8..w * 8 + 8].try_into().unwrap()); + // Row 254 has codeword 255, row 255 has codeword 256, row 256 has 257. + assert_eq!(mag[254], 255); + assert_eq!(word(3) >> 62 & 1, 0); + assert_eq!(mag[255], 0); + assert_eq!(word(3) >> 63 & 1, 1); + assert_eq!(mag[256], 1); + assert_eq!(word(4) & 1, 1); + assert_eq!(read(&b).unwrap().rows, rows); +} + +/// Red if a predicate codebook past 511 entries is written. +#[test] +fn five_hundred_twelve_predicates_are_refused() { + let rows: Vec = (0..=CODEC9_MAX) + .map(|i| row(&format!("P{i}"), "k::a", "k", None, None)) + .collect(); + assert!(matches!( + write(&rows, false, &[prov()]), + Err(WriteError::CodebookTooLarge { + section: "PRED_CB", + len: 512 + }) + )); + assert!(write(&rows[..CODEC9_MAX], false, &[prov()]).is_ok()); +} + +// ---- F9: provenance ---------------------------------------------------------- + +/// Red if the provenance is not covered by the digest. +#[test] +fn a_different_charon_commit_changes_the_digest() { + let mut p2 = prov(); + p2.charon_commit = Some("abc124".into()); + let a = write(&sample(), false, &[prov()]).unwrap(); + let b = write(&sample(), false, &[p2]).unwrap(); + assert_ne!(a[16..24], b[16..24]); +} + +/// Red if UNKNOWN (None) and the string "UNKNOWN" are written or read the same. +#[test] +fn unknown_is_distinct_from_the_string_unknown() { + let mut p_none = prov(); + p_none.rustc_commit = None; + let mut p_str = prov(); + p_str.rustc_commit = Some("UNKNOWN".into()); + let a = write(&sample(), false, &[p_none.clone()]).unwrap(); + let b = write(&sample(), false, &[p_str.clone()]).unwrap(); + assert_ne!(a, b); + assert_eq!(read(&a).unwrap().prov, vec![p_none]); + assert_eq!(read(&b).unwrap().prov, vec![p_str]); +} + +// ---- F10 (writer side) --------------------------------------------------------- + +/// Red if the function id is lost, or merged with the subject. +#[test] +fn fundecl_is_per_row_and_independent_of_the_subject() { + let rows = vec![ + row("CALLS_RESOLVED", "k::a", "k::x", None, Some(7)), + row("CALLS_RESOLVED", "k::b", "k::x", None, Some(7)), + row("CALLS_RESOLVED", "k::b", "k::y", None, None), + ]; + let b = write(&rows, false, &[prov()]).unwrap(); + let lane: Vec = section(&b, tag::FUNDECL) + .chunks(4) + .map(|c| u32::from_le_bytes(c.try_into().unwrap())) + .collect(); + assert_eq!(lane, vec![7, 7, NO_ID]); + let d = read(&b).unwrap(); + assert_eq!(d.rows, rows); + assert_ne!(d.rows[0].subject, d.rows[1].subject); + assert_eq!(d.rows[0].fun, d.rows[1].fun); +} From 63c74712fe47f02307373ed925407453fc40a83e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 16:17:32 +0000 Subject: [PATCH 2/2] ruff_charon: write rows.bin on import, read it back, record provenance and FunDeclId `Store::import` writes `.rows.bin` first, then the TSV (on by default until the lance-graph reader reads the binary format), residuals and files. `rows()` reads the binary file and falls back to `rows.tsv` for a crate stored before the format existed; `rows_tsv()` keeps the text path. Each file carries one provenance record: Charon version, the Charon commit captured at build time, the LLBC digest, target triples and options, and rustc commit and source revision when the caller supplies them (`import_with`); anything not known is UNKNOWN. Every relation row carries the FunDeclId of the function it came from. A golden `cg_lib.rows.bin` pins the bytes for the lance-graph reader. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01X6y3drwKSE2zSgoexheLFX --- crates/ruff_charon/build.rs | 36 +++ crates/ruff_charon/src/lib.rs | 24 ++ crates/ruff_charon/src/store.rs | 181 ++++++++++++++- .../ruff_charon/tests/data/golden/README.md | 3 + .../tests/data/golden/cg_lib.rows.bin | Bin 0 -> 3976 bytes .../tests/data/golden/cg_lib.rows.tsv | 31 +++ crates/ruff_charon/tests/store.rs | 8 +- crates/ruff_charon/tests/store_bin_store.rs | 208 ++++++++++++++++++ 8 files changed, 483 insertions(+), 8 deletions(-) create mode 100644 crates/ruff_charon/build.rs create mode 100644 crates/ruff_charon/tests/data/golden/README.md create mode 100644 crates/ruff_charon/tests/data/golden/cg_lib.rows.bin create mode 100644 crates/ruff_charon/tests/data/golden/cg_lib.rows.tsv create mode 100644 crates/ruff_charon/tests/store_bin_store.rs diff --git a/crates/ruff_charon/build.rs b/crates/ruff_charon/build.rs new file mode 100644 index 00000000000000..9eed49b01bd9a8 --- /dev/null +++ b/crates/ruff_charon/build.rs @@ -0,0 +1,36 @@ +//! Records the commit of the Charon checkout this crate is built against +//! (the path dependency `../../../charon/charon`), so `rows.bin` provenance can +//! name it. Any failure yields an empty value, which the store reads as UNKNOWN; +//! this script never fails the build. + +use std::path::PathBuf; +use std::process::Command; + +#[expect( + clippy::disallowed_methods, + reason = "a build script reads the real filesystem; the ty System abstraction does not apply" +)] +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + let manifest = std::env::var_os("CARGO_MANIFEST_DIR").map(PathBuf::from); + let repo = manifest.map(|m| m.join("../../../charon")); + let mut commit = String::new(); + if let Some(repo) = repo { + for rel in [".git/HEAD", ".git/refs/heads"] { + let p = repo.join(rel); + if p.exists() { + println!("cargo:rerun-if-changed={}", p.display()); + } + } + if let Ok(out) = Command::new("git") + .arg("-C") + .arg(&repo) + .args(["rev-parse", "HEAD"]) + .output() + && out.status.success() + { + commit = String::from_utf8_lossy(&out.stdout).trim().to_owned(); + } + } + println!("cargo:rustc-env=RUFF_CHARON_CHARON_COMMIT={commit}"); +} diff --git a/crates/ruff_charon/src/lib.rs b/crates/ruff_charon/src/lib.rs index 5ed7a4a22d2831..aaebf8bb9b17aa 100644 --- a/crates/ruff_charon/src/lib.rs +++ b/crates/ruff_charon/src/lib.rs @@ -35,6 +35,8 @@ pub mod store_bin; use std::fmt::Write as _; use std::path::Path; +use anyhow::Context as _; + use charon_lib::ast::llbc_ast::{Statement, StatementKind}; use charon_lib::ast::visitor::AstVisitable; use charon_lib::ast::*; @@ -48,14 +50,27 @@ pub struct Llbc { /// Charon reported errors: the crate description is partial. pub has_errors: bool, pub krate: TranslatedCrate, + /// The version of Charon that wrote the file (`CrateData::charon_version`). + pub charon_version: String, + /// [`store_bin::digest`] over the bytes of the LLBC file. + pub llbc_digest: u64, } /// Read an LLBC file written by `charon ... --dest-file`. pub fn load(path: &Path) -> anyhow::Result { + // Charon offers no deserialize-from-bytes API (only `deserialize_from_file`, + // which also needs `serde_stacker` for deep ASTs), so the file is read twice: + // once for the digest, once by Charon. + let bytes = std::fs::read(path) + .with_context(|| format!("Failed to read llbc file {}", path.display()))?; + let llbc_digest = store_bin::digest(&bytes); + drop(bytes); let data = CrateData::deserialize_from_file(path, SerializationFormat::Json)?; Ok(Llbc { has_errors: data.has_errors, krate: data.translated, + charon_version: data.charon_version.0, + llbc_digest, }) } @@ -360,6 +375,10 @@ pub struct Relation { /// What a Kythe `ref/call` points at when it is not `object`: the plain /// trait name for `MAY_DISPATCH`, which Kythe reads as the declaration. pub kythe_target: Option, + /// The Charon `FunDeclId` of the function the row was harvested from. It + /// indexes the `fun_decls` of ONE LLBC file, so it is never an identity on + /// its own: identity is `(artifact, FunDeclId)`. + pub fun: Option, } /// Something Charon gave that no relation represents, with the reason. @@ -401,6 +420,7 @@ pub fn relations( object: f.qualified.clone(), site: f.site.clone(), kythe_target: None, + fun: Some(f.id), }); let decl = krate.fun_decls.get(f.id); let mut marker = |predicate: &'static str, object: String| { @@ -411,6 +431,7 @@ pub fn relations( object, site: f.site.clone(), kythe_target: None, + fun: Some(f.id), }); }; if let Some(d) = decl { @@ -450,6 +471,7 @@ pub fn relations( object, site: f.site.clone(), kythe_target: None, + fun: Some(f.id), }), None => residual.push(Residual { reason: "TRAIT_METHOD_NOT_FOUND", @@ -530,6 +552,7 @@ pub fn relations( object, site: c.site, kythe_target, + fun: Some(f.id), }); } for r in f.refs { @@ -548,6 +571,7 @@ pub fn relations( object, site: r.site, kythe_target: None, + fun: Some(f.id), }), None => residual.push(Residual { reason: "REFERENCE_NOT_NAMED", diff --git a/crates/ruff_charon/src/store.rs b/crates/ruff_charon/src/store.rs index 14b254c7a11db4..e3e8f807104400 100644 --- a/crates/ruff_charon/src/store.rs +++ b/crates/ruff_charon/src/store.rs @@ -25,6 +25,7 @@ use std::io::{self, BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; +use crate::store_bin::{self, BinRow, Prov}; use crate::{GRADE, Llbc, relations}; /// The version of the rows artifact this crate writes and reads. @@ -147,16 +148,53 @@ fn read_table( Ok(rows) } +/// Provenance the store cannot know. The caller supplies it, or it is recorded +/// as UNKNOWN (`None`); it is never guessed. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SourceProv { + /// The commit of the rustc that Charon ran on. + pub rustc_commit: Option, + /// The revision of the analysed source tree. + pub source_revision: Option, +} + /// A directory of per-crate artifacts. #[derive(Debug, Clone)] pub struct Store { dir: PathBuf, + tsv_export: bool, +} + +fn invalid(m: impl ToString) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, m.to_string()) +} + +/// The Charon commit this crate was built against (`build.rs`); empty or +/// absent is UNKNOWN. +fn built_charon_commit() -> Option { + option_env!("RUFF_CHARON_CHARON_COMMIT") + .filter(|c| !c.is_empty()) + .map(str::to_owned) } impl Store { + /// A store that writes `rows.bin` and, for now, `rows.tsv` as well. #[must_use] pub fn new(dir: impl Into) -> Self { - Self { dir: dir.into() } + Self { + dir: dir.into(), + tsv_export: true, + } + } + + /// Whether `import` also writes `.rows.tsv`. On by default in this + /// release (transitional): the lance-graph codegraph reader still reads + /// `rows.tsv` until its binary reader lands, at which point the default + /// flips to off. `rows.bin` is always written. + #[must_use] + pub fn with_tsv_export(mut self, on: bool) -> Self { + self.tsv_export = on; + self } fn path(&self, crate_name: &str, suffix: &str) -> PathBuf { @@ -213,8 +251,43 @@ impl Store { /// /// # Errors /// I/O errors, or `InvalidData` when the receipt does not conserve. - #[expect(clippy::disallowed_methods, reason = "see `replace`")] pub fn import(&self, llbc: &Llbc, crate_name: &str) -> io::Result { + self.import_with(llbc, crate_name, &SourceProv::default()) + } + + /// [`Store::import`] with the provenance only the caller knows. + /// + /// # Errors + /// I/O errors, or `InvalidData` when the receipt does not conserve or the + /// rows cannot be encoded. + pub fn import_with( + &self, + llbc: &Llbc, + crate_name: &str, + src: &SourceProv, + ) -> io::Result { + self.import_inner(llbc, crate_name, src, built_charon_commit()) + } + + /// [`Store::import`] with a fixed provenance (no Charon commit, no rustc + /// commit, no source revision), so the bytes written do not depend on the + /// machine that built this crate. Only for the golden-file test. + /// + /// # Errors + /// As [`Store::import`]. + #[doc(hidden)] + pub fn import_for_golden(&self, llbc: &Llbc, crate_name: &str) -> io::Result { + self.import_inner(llbc, crate_name, &SourceProv::default(), None) + } + + #[expect(clippy::disallowed_methods, reason = "see `replace`")] + fn import_inner( + &self, + llbc: &Llbc, + crate_name: &str, + src: &SourceProv, + charon_commit: Option, + ) -> io::Result { let (rows, residual, receipt) = relations(&llbc.krate, crate_name); if !receipt.conserved() { return Err(io::Error::new( @@ -246,6 +319,36 @@ impl Store { "COMPLETE" }; writeln!(out, "CRATE\t{crate_name}\t{completeness}\t{GRADE}\t-\t-\t-")?; + // The same rows, in the same order, for the binary file. + let mut bin_rows = Vec::with_capacity(rows.len() + 1); + for r in &rows { + let fun = r + .fun + .map(|id| u32::try_from(id.index()).map_err(invalid)) + .transpose()?; + bin_rows.push(BinRow { + predicate: r.predicate.to_owned(), + subject: r.subject.clone(), + object: r.object.clone(), + grade: GRADE.to_owned(), + site: r.site.as_ref().map(|s| (s.file.clone(), s.start, s.end)), + fun, + }); + } + bin_rows.push(BinRow { + predicate: "CRATE".to_owned(), + subject: crate_name.to_owned(), + object: completeness.to_owned(), + grade: GRADE.to_owned(), + site: None, + fun: None, + }); + let bin = store_bin::write( + &bin_rows, + llbc.has_errors, + &[Self::prov(llbc, src, charon_commit)], + ) + .map_err(invalid)?; let mut res = Vec::new(); header(&mut res, RESIDUALS_VERSION, RESIDUALS_SCHEMA)?; for r in &residual { @@ -257,12 +360,55 @@ impl Store { writeln!(files, "{p}\t{d:016x}")?; } // Files last: a crate whose digests are present has its rows. - Self::replace(&self.path(crate_name, "rows.tsv"), &out)?; + Self::replace(&self.path(crate_name, "rows.bin"), &bin)?; + if self.tsv_export { + Self::replace(&self.path(crate_name, "rows.tsv"), &out)?; + } Self::replace(&self.path(crate_name, "residuals.tsv"), &res)?; Self::replace(&self.path(crate_name, "files.tsv"), &files)?; Ok(receipt) } + /// The provenance record of one import. `llbc_schema_version` repeats the + /// Charon version: no separate schema version exists today. `target_config` + /// is the sorted target triples, then `;`, then the Charon options as JSON + /// (serde's fixed field order). + fn prov(llbc: &Llbc, src: &SourceProv, charon_commit: Option) -> Prov { + let mut triples: Vec<&str> = llbc + .krate + .target_information + .keys() + .map(String::as_str) + .collect(); + triples.sort_unstable(); + let target_config = serde_json::to_string(&llbc.krate.options) + .ok() + .map(|opts| format!("{};{opts}", triples.join(","))); + Prov { + charon_version: Some(llbc.charon_version.clone()), + charon_commit, + llbc_schema_version: Some(llbc.charon_version.clone()), + llbc_digest: Some(format!("{:016x}", llbc.llbc_digest)), + rustc_commit: src.rustc_commit.clone(), + target_config, + source_revision: src.source_revision.clone(), + } + } + + fn decode_bin(&self, crate_name: &str) -> io::Result { + let path = self.path(crate_name, "rows.bin"); + let bytes = fs::read(&path)?; + store_bin::read(&bytes).map_err(|e| invalid(format!("{}: {e}", path.display()))) + } + + /// The provenance records of one crate's `rows.bin`. + /// + /// # Errors + /// I/O or format errors. + pub fn provenance(&self, crate_name: &str) -> io::Result> { + Ok(self.decode_bin(crate_name)?.prov) + } + /// The crates in the store, sorted. /// /// # Errors @@ -282,11 +428,38 @@ impl Store { Ok(out) } - /// One crate's rows, read by schema names. + /// One crate's rows, read from `rows.bin`, or from `rows.tsv` when the + /// crate was stored before the binary format existed. /// /// # Errors /// I/O or format errors. + #[expect(clippy::disallowed_methods, reason = "see `replace`")] pub fn rows(&self, crate_name: &str) -> io::Result> { + // A store written before the binary format has only `rows.tsv`; it + // stays readable until its crate is imported again. + if !self.path(crate_name, "rows.bin").exists() { + return self.rows_tsv(crate_name); + } + Ok(self + .decode_bin(crate_name)? + .rows + .into_iter() + .map(|r| Row { + predicate: r.predicate, + subject: r.subject, + object: r.object, + grade: r.grade, + site: r.site, + }) + .collect()) + } + + /// One crate's rows, read from `rows.tsv` by schema names (debug and test + /// path; the file exists only while the TSV export is on). + /// + /// # Errors + /// I/O or format errors. + pub fn rows_tsv(&self, crate_name: &str) -> io::Result> { let table = read_table( &self.path(crate_name, "rows.tsv"), ROWS_VERSION, diff --git a/crates/ruff_charon/tests/data/golden/README.md b/crates/ruff_charon/tests/data/golden/README.md new file mode 100644 index 00000000000000..b5f0451ff0cf0b --- /dev/null +++ b/crates/ruff_charon/tests/data/golden/README.md @@ -0,0 +1,3 @@ +# Golden rows files + +`cg_lib.rows.bin` and `cg_lib.rows.tsv` are the store's output for `tests/data/cg_lib.llbc`, imported with `Store::import_for_golden` (no Charon commit, rustc commit or source revision in the provenance record, so the bytes do not depend on the machine). `tests/store_bin_store.rs` writes them when run with `CHARON_BLESS=1` and otherwise requires a fresh import to match them byte for byte. lance-graph's codegraph tests copy these files (PR B) to check their reader against this writer. They change only with a deliberate re-bless after a deliberate format or fixture change. diff --git a/crates/ruff_charon/tests/data/golden/cg_lib.rows.bin b/crates/ruff_charon/tests/data/golden/cg_lib.rows.bin new file mode 100644 index 0000000000000000000000000000000000000000..1e222d01acc59fda4db73ab589fabc2590be44f9 GIT binary patch literal 3976 zcmcInU2I&%72aJZ4lV!XN3E1J!4fSJ+3XVHzl&62oJBU;IIh>wB7`Qh_s;GPzJHlJ z*IpAvq$(TIEvOp8Ds`OYK*`{Pf0^Dc^F3~maOg?o4?>XSw`cN4!AX8Jlkb6l5qw7` ze=?JYOrrmECjYri{(0~o_I(r^eGEN(mB?QJp98-p<5LhKd)I=Kzs`Wu`L6>ff1Lxz z5BV7DBc*pi&cGSv%1F7gtz6l@y_~w+}#UHjn0I&gvV_aus9OK)3KKB@J!U>p@=XCg@uNcq5MsNa8s`4 zpmPuqq6Emp5&`)~IX494T*-i9LqMJ-$gUY!+Q?7FCY{;XBs&z_|HUpjj^vC@;+yRv zkCXPFtdWcv|G#@O{PhgqLH~G$H_-3NK>7MLU}uIo^aQ#?gy(=)3iuPoyWu>82QVg( z4g#GCU2;e1o<0hk%kWF|_hq1*qQty2gF;VuHp6S^%g}3Z9()Ra7FYOiS zcVRz@gwm|TrMvkHCy3klBDubnm1>E7sfTAmC0Dr?EcUVe+@?9D^g@>*FF z(O19u=jI#Z-z~lQyfLz+WIx+U=fs`hEr`dBs$Q*N<;IJ{G%f+a9wz+>ONEF+V`y9l@M9bfwL;4E%g8RKk`K)>IHi z`K%|^nB_yu-Z2$N+GbpJv3lSAd&e|}LT4mek5mw7BkKh|5>16Y`Jfo+Doipj&V?gr z!i_^}riurg$qo~J8Rb@TNyarUa68sePn#)x6UOb_r1Wh!b~2eDipt!j?aFqkzChC4-v$1YvD&0o5*qZ!Ig zU`r@>m|n+=o}Q+Md(1qpR-hW@C7nMDVX}mc1PEV%*bE};1U91UdBQJf^SJD(-O9sQ{#M?`U#t4$fBww!f@M*T`eS;ys+L)p%eZA=HO6Z+Oc%C9HcbTy(@gCk?n#-TE%wJGkh(rf<A-o_i{=9?XAQUROYFo=CZ_lPv7xknM($?R4T;pvU ztT`gfj5Jp%NeaG<{Wjdmq!u1ql4#c6H|3(;=rU`i)@6z?uFc`rElK8WRFQjbOdfIgDT10B&s|@ zX+jWDF!4si$EK>O*k@}}_n0SrN#!u#IWjF*kWWS{GDK=B6|C7S`7DwFRTed79sdo$ z@ugP?AoNffNd92SzrdtV`=H$wCzS}4g9?E+zYt|eqY8wmjLc1(&bs&;gZo+(>~L%g g5%-zE?gmvPVKlwQl|Pi(cpV~ PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/data") + .join(name) +} + +fn llbc(name: &str) -> Llbc { + load(&data(name)).expect("llbc") +} + +/// A fresh directory per test. +fn temp(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("ruff_charon_bin_{name}_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("temp dir"); + dir +} + +/// Every fixture with the crate name its rows are harvested under. +const FIXTURES: &[(&str, &str)] = &[ + ("cg_lib.llbc", "cg_lib"), + ("cg_app.llbc", "cg_app"), + ("cg_lib_rev2.llbc", "cg_lib"), + ("cg_app_rev2.llbc", "cg_app"), + ("fnref.llbc", "fnref"), + ("trprobe.llbc", "trprobe"), +]; + +/// Red when `rows.bin` and `rows.tsv` disagree on any row of any fixture, or +/// when the fixtures stop carrying the shapes that make the comparison mean +/// something (dispatch rows, rows without a site, predicates beyond calls). +#[test] +fn bin_and_tsv_agree_row_for_row() { + let mut all = Vec::new(); + for (file, krate) in FIXTURES { + let store = Store::new(temp(&format!("agree_{file}"))); + store.import(&llbc(file), krate).expect("import"); + let bin = store.rows(krate).expect("bin rows"); + let tsv = store.rows_tsv(krate).expect("tsv rows"); + assert!(!bin.is_empty(), "{file}: no rows"); + assert_eq!(bin, tsv, "{file}: bin and tsv differ"); + all.extend(bin); + } + assert!(all.iter().any(|r| r.predicate == "MAY_DISPATCH")); + assert!(all.iter().any(|r| r.site.is_none())); + assert!(all.iter().any(|r| r.site.is_some())); + assert!( + all.iter() + .any(|r| !matches!(r.predicate.as_str(), "DEFINES" | "CALLS_RESOLVED")), + "no predicate beyond DEFINES and CALLS_RESOLVED in any fixture" + ); +} + +/// Red when `with_tsv_export(false)` still writes a TSV, or when `rows()` then +/// needs one. +#[test] +fn tsv_export_off_writes_no_tsv() { + let dir = temp("tsv_off"); + let store = Store::new(&dir).with_tsv_export(false); + store + .import(&llbc("cg_lib.llbc"), "cg_lib") + .expect("import"); + let tsv: Vec<_> = std::fs::read_dir(&dir) + .expect("dir") + .filter_map(Result::ok) + .filter(|e| e.file_name().to_string_lossy().ends_with(".rows.tsv")) + .collect(); + assert!(tsv.is_empty(), "a TSV was written: {tsv:?}"); + assert!(dir.join("cg_lib.rows.bin").exists()); + assert!(!store.rows("cg_lib").expect("rows").is_empty()); +} + +/// Red when a function row loses its `FunDeclId`, the CRATE row gains one, or +/// a DEFINES row names a different function than the one it was harvested from. +#[test] +fn every_function_row_carries_its_fundecl() { + let dir = temp("fundecl"); + let l = llbc("cg_lib.llbc"); + Store::new(&dir).import(&l, "cg_lib").expect("import"); + let decoded = + store_bin::read(&std::fs::read(dir.join("cg_lib.rows.bin")).expect("bin")).expect("read"); + let facts = fun_facts(&l.krate, "cg_lib"); + let mut defines = 0; + for r in &decoded.rows { + if r.predicate == "CRATE" { + assert_eq!(r.fun, None, "the CRATE row has no function"); + } else { + assert!(r.fun.is_some(), "{r:?} has no FunDeclId"); + } + if r.predicate == "DEFINES" { + defines += 1; + let f = facts + .iter() + .find(|f| f.qualified == r.subject) + .expect("a function for the subject"); + assert_eq!(r.fun, u32::try_from(f.id.index()).ok(), "{r:?}"); + } + } + assert!(defines > 0 && defines == facts.len()); + assert_eq!( + decoded + .rows + .iter() + .filter(|r| r.predicate == "CRATE") + .count(), + 1 + ); +} + +fn header(path: &Path) -> u64 { + let bytes = std::fs::read(path).expect("bin"); + let h = bytes + .first_chunk::<{ store_bin::HEADER_LEN }>() + .expect("header"); + store_bin::header_digest(h).expect("magic and major") +} + +/// Red when the provenance record is missing a field, takes a wrong value, or +/// when a change of `source_revision` alone does not change the file. +#[test] +fn provenance_reaches_the_file() { + let l = llbc("cg_lib.llbc"); + let (d1, d2) = (temp("prov_a"), temp("prov_b")); + Store::new(&d1).import(&l, "cg_lib").expect("import"); + let prov = Store::new(&d1).provenance("cg_lib").expect("prov"); + assert_eq!(prov.len(), 1); + let p = &prov[0]; + assert_eq!(p.charon_version.as_deref(), Some(l.charon_version.as_str())); + assert_eq!( + p.llbc_schema_version.as_deref(), + Some(l.charon_version.as_str()) + ); + assert_eq!( + p.llbc_digest.as_deref(), + Some(format!("{:016x}", l.llbc_digest).as_str()) + ); + assert_eq!((&p.rustc_commit, &p.source_revision), (&None, &None)); + assert!(p.target_config.is_some()); + + let src = SourceProv { + rustc_commit: Some("abc123".into()), + source_revision: Some("rev-7".into()), + }; + Store::new(&d2) + .import_with(&l, "cg_lib", &src) + .expect("import"); + let p2 = &Store::new(&d2).provenance("cg_lib").expect("prov")[0]; + assert_eq!(p2.rustc_commit.as_deref(), Some("abc123")); + assert_eq!(p2.source_revision.as_deref(), Some("rev-7")); + assert_ne!( + header(&d1.join("cg_lib.rows.bin")), + header(&d2.join("cg_lib.rows.bin")), + "provenance must be covered by the file digest" + ); + + let d3 = temp("prov_c"); + let only_rev = SourceProv { + rustc_commit: Some("abc123".into()), + source_revision: Some("rev-8".into()), + }; + Store::new(&d3) + .import_with(&l, "cg_lib", &only_rev) + .expect("import"); + assert_ne!( + header(&d2.join("cg_lib.rows.bin")), + header(&d3.join("cg_lib.rows.bin")), + "a different source_revision alone must change the digest" + ); +} + +/// Red when the bytes written for `cg_lib` differ from the blessed golden +/// files. `CHARON_BLESS=1` rewrites them. +#[test] +fn golden_bytes_are_stable() { + let dir = temp("golden"); + Store::new(&dir) + .import_for_golden(&llbc("cg_lib.llbc"), "cg_lib") + .expect("import"); + let golden = data("golden"); + let files = ["cg_lib.rows.bin", "cg_lib.rows.tsv"]; + if std::env::var_os("CHARON_BLESS").is_some_and(|v| v == "1") { + std::fs::create_dir_all(&golden).expect("golden dir"); + for f in files { + std::fs::copy(dir.join(f), golden.join(f)).expect("bless"); + } + return; + } + for f in files { + let want = std::fs::read(golden.join(f)).unwrap_or_else(|_| { + panic!("golden file {f} is missing: run with CHARON_BLESS=1 to create the golden files") + }); + let got = std::fs::read(dir.join(f)).expect("fresh file"); + assert!(got == want, "{f} differs from the golden file"); + } +}