From e806bc385315047b74f1b2d8e88b06c2329c093e Mon Sep 17 00:00:00 2001 From: josoriom Date: Tue, 22 Sep 2026 15:42:53 +0800 Subject: [PATCH 1/2] refactor!: improve api --- Cargo.lock | 18 +- Cargo.toml | 8 +- README.md | 168 ++-- crates/cli/Cargo.toml | 6 +- crates/cli/src/main.rs | 47 +- crates/cli/src/utilities/header.rs | 16 +- crates/cli/tests/f18_update_stamps_version.rs | 6 +- crates/parser/Cargo.toml | 2 +- crates/parser/examples/bench_window.rs | 4 +- crates/parser/src/api/convert.rs | 96 ++ crates/parser/src/api/mod.rs | 32 + crates/parser/src/api/options.rs | 18 + crates/parser/src/api/reader.rs | 1 + crates/parser/src/api/writer.rs | 117 +++ crates/parser/src/convert.rs | 191 ---- crates/parser/src/format.rs | 5 + crates/parser/src/ion/array_kind.rs | 1 + .../parser/src/ion/decoder/decode/arrays.rs | 831 +++++++++++++++++- crates/parser/src/ion/decoder/decode/mod.rs | 38 +- .../parser/src/ion/decoder/decode/spectra.rs | 40 +- crates/parser/src/ion/decoder/decode/tests.rs | 214 ++--- .../parser/src/ion/decoder/decode/to_mzml.rs | 16 +- .../parser/src/ion/decoder/decode/windows.rs | 58 +- .../src/ion/decoder/utilities/block_reader.rs | 70 +- .../src/ion/decoder/utilities/common.rs | 13 +- .../ion/decoder/utilities/spectrum_source.rs | 8 + crates/parser/src/ion/encoder/encode.rs | 37 +- crates/parser/src/ion/encoder/ion_writer.rs | 530 +++++++---- .../src/ion/encoder/utilities/block_writer.rs | 194 ++-- .../utilities/meta_collector/grouper.rs | 14 +- .../encoder/utilities/meta_collector/mod.rs | 22 +- .../encoder/utilities/meta_collector/tests.rs | 13 +- .../parser/src/ion/encoder/utilities/mod.rs | 2 +- .../src/ion/encoder/utilities/output.rs | 12 +- crates/parser/src/ion/error.rs | 5 + crates/parser/src/ion/filter_summary.rs | 2 + crates/parser/src/ion/header.rs | 33 +- crates/parser/src/ion/mod.rs | 47 +- crates/parser/src/ion/range.rs | 3 +- crates/parser/src/ion/scan.rs | 4 +- crates/parser/src/lib.rs | 32 +- crates/parser/src/mzml/schema.rs | 3 +- crates/parser/src/source.rs | 1 + crates/parser/src/utilities/test.rs | 4 +- crates/parser/tests/api_contract.rs | 661 -------------- crates/parser/tests/api_surface.rs | 200 +++++ crates/parser/tests/array_roundtrips.rs | 551 ------------ crates/parser/tests/binary_encoder.rs | 2 +- crates/parser/tests/common/mod.rs | 23 +- crates/parser/tests/concurrent_encode.rs | 15 +- crates/parser/tests/convert.rs | 126 ++- crates/parser/tests/decode_rejection.rs | 11 +- crates/parser/tests/empty_inputs.rs | 11 +- crates/parser/tests/file_encoder.rs | 57 +- crates/parser/tests/ion_stream_writer.rs | 15 +- crates/parser/tests/ion_writer_differ.rs | 11 +- crates/parser/tests/mz_order_rejection.rs | 106 ++- crates/parser/tests/roundtrip_ion.rs | 2 +- 58 files changed, 2494 insertions(+), 2279 deletions(-) create mode 100644 crates/parser/src/api/convert.rs create mode 100644 crates/parser/src/api/mod.rs create mode 100644 crates/parser/src/api/options.rs create mode 100644 crates/parser/src/api/reader.rs create mode 100644 crates/parser/src/api/writer.rs delete mode 100644 crates/parser/src/convert.rs create mode 100644 crates/parser/src/format.rs create mode 100644 crates/parser/src/source.rs delete mode 100644 crates/parser/tests/api_contract.rs create mode 100644 crates/parser/tests/api_surface.rs delete mode 100644 crates/parser/tests/array_roundtrips.rs diff --git a/Cargo.lock b/Cargo.lock index 0d1b5e8c..b435bc47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,9 +149,9 @@ checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" [[package]] name = "cc" -version = "1.4.6" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3eb0f42d6c360dc3f8a821f6bf2fdea7f72bfd36b3076eb0e6d1e9e0752fff4" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" dependencies = [ "find-msvc-tools", "shlex", @@ -205,7 +205,7 @@ checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "cli" -version = "0.1.3" +version = "0.2.0" dependencies = [ "assert_cmd", "clap", @@ -236,9 +236,9 @@ checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" [[package]] name = "cosmoz" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6e26bc8e36dd197e16c207f762eb2a9ecea9a346115dd24b728d40c869aa2d7" +checksum = "6297cbeee3a0ec235f52cb17f7968749f63013a4c8c9892360afbc2a776c731b" [[package]] name = "cpufeatures" @@ -339,9 +339,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" [[package]] name = "float-cmp" @@ -425,7 +425,7 @@ dependencies = [ [[package]] name = "ionic-rs" -version = "0.1.3" +version = "0.2.0" dependencies = [ "base64", "bytemuck", @@ -957,7 +957,7 @@ checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "xtask" -version = "0.1.3" +version = "0.2.0" dependencies = [ "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index c7f1e62d..477d1bf7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["crates/parser", "crates/cli", "crates/xtask"] resolver = "3" [workspace.package] -version = "0.1.3" +version = "0.2.0" license = "MIT" authors = [ "Javier Osorio M ", @@ -22,8 +22,8 @@ once_cell = "1.21.4" quick-xml = "0.40.1" serde = { version = "1", features = ["derive"] } serde_json = "1.0" -clap = "4.6.1" -cosmoz = { version = "0.1.1", features = ["std", "compression"] } -regex = "1.12.4" +clap = "4.6.7" +cosmoz = { version = "0.2.0", features = ["std", "compression"] } +regex = "1.13.1" rayon = "1.12.0" itoa = "1.0" diff --git a/README.md b/README.md index a0efc0a9..a6e5b376 100644 --- a/README.md +++ b/README.md @@ -46,88 +46,144 @@ A command-line tool for converting mzML files to Ionic. See the [CLI](crates/cli ## Usage -### Convert - -`ionic::convert` takes a path or a byte buffer. With `output` set it writes the file and returns `None`. With `output` empty it returns the result as `Some(bytes)`. `kind` defaults to `ConvertKind::Auto`, which reads the direction from the `.mzML` or `.ion` extension, and from the file signature for a buffer or a file without one of those extensions. +| Item | Available | What it does | +|---|---|---| +| `ionic::read` / `ionic::write` | std | Whole `.ion` file to/from an `MzML`, in one call. | +| `ionic::convert` | everywhere | Convert an in-memory buffer between mzML and Ionic. | +| `ionic::convert_file` | std | Convert between two paths, streaming to disk. | +| `IonReader::open` | std | Open a `.ion` file by path (memory-mapped). | +| `IonReader::from_bytes` | everywhere | Open a `.ion` file already in memory. | +| `IonReader::new` | everywhere | Open from any `ionic::source::ReadBytes` (partial/remote reads). | +| `IonWriter::create` | std | Write a new `.ion` file by path. | +| `IonWriter::to` | everywhere | Write into any `ionic::source::WriteBytes` sink, such as a `Vec`. | +| `ionic::mzml` | everywhere | mzML types and parser/serializer: `MzML`, `Spectrum`, `Chromatogram`, `NumericArray`, `parse_mzml`, `bin_to_mzml`, ... | +| `ionic::source` | everywhere | Partial/remote-read building blocks: `ReadBytes`, `WriteBytes`, `ByteRange`, `CallbackSource`, `header_ranges`, `merge_ranges`. | +| `ionic::format` | everywhere | File-format constants used by tooling: `CURRENT_VERSION`, `HEADER_SIZE`, `FILE_SIGNATURE`, `is_supported`, ... | + +"std" means the item needs a real filesystem and is not available on `wasm32-unknown-unknown`; "everywhere" means it also works there. + +### Read, one call ```rust -use ionic::{ConvertKind, ConvertOptions, WriteOptions}; - -ionic::convert(Path::new("run.mzML"), ConvertOptions { output: Some("run.ion".into()), ..Default::default() })?; -ionic::convert(Path::new("run.ion"), ConvertOptions { output: Some("run.mzML".into()), ..Default::default() })?; - -let ion_bytes = ionic::convert(Path::new("run.mzML"), ConvertOptions::default())?.unwrap(); -let mzml_bytes = ionic::convert(&ion_bytes, ConvertOptions::default())?.unwrap(); - -ionic::convert( - &mzml_bytes, - ConvertOptions { - output: Some("run.ion".into()), - kind: ConvertKind::MzmlToIon, - write: WriteOptions { compression_level: 0, ..Default::default() }, - ..Default::default() - }, -)?; +fn main() -> Result<(), Box> { + let mzml = ionic::read("run.ion")?; + let count = mzml.run.spectrum_list.map_or(0, |list| list.spectra.len()); + println!("{count} spectra"); + Ok(()) +} ``` -### Read an .ion file +### Reader with options ```rust use ionic::{ArrayKind, IonReader, ReadOptions}; -let mut reader = IonReader::open_file(Path::new("run.ion"), ReadOptions::default())?; -let mz = reader.get_spectrum_array(0, ArrayKind::Mz)?; -let intensity = reader.get_spectrum_array(0, ArrayKind::Intensity)?; +fn main() -> Result<(), Box> { + let options = ReadOptions { + parallel: false, + ..ReadOptions::default() + }; + let mut reader = IonReader::open("run.ion", &options)?; + let mz = reader.array(0, ArrayKind::Mz)?; + let intensity = reader.array(0, ArrayKind::Intensity)?; + println!("{} points", mz.len().min(intensity.len())); + Ok(()) +} ``` -### Reading options — ReadOptions +`ReadOptions::default()`: + +| Field | Default | +|---|---| +| `max_cached_bytes` | `256 * 1024 * 1024` (256 MiB decoded-block cache) | +| `verify_checksums` | `true` | +| `parallel` | `true` | +| `decompression_limit` | `DecompressionLimit::default()` (2 GiB uncompressed cap) | + +### Write, one call ```rust -pub struct ReadOptions { - pub max_cached_bytes: usize, // decoded-block cache cap; default 256 MiB - pub verify_checksums: bool, // check CRCs + layout on open; default true - pub parallel: bool, // decode blocks in parallel; default true - pub decompression_limit: DecompressionLimit, // zip-bomb guard; type from ionic::ion:: +use ionic::WriteOptions; + +fn main() -> Result<(), Box> { + let mzml = ionic::read("run.ion")?; + ionic::write("out.ion", &mzml, &WriteOptions::default())?; + Ok(()) } ``` -### Write an .ion file +### Streaming writer -Memory +The metadata passed to `create`/`to` carries run-level information; spectra and chromatograms +come from `write_spectrum`/`write_chromatogram` (or `write_stream`), not from the metadata +argument. `finish` is required — `Drop` does not finish the file. ```rust -use ionic::{IonWriter, MemoryReader, MzML, Spectrum, WriteOptions}; - -let mzml = MzML::from_spectra(vec![Spectrum::new("scan=1", vec![100.0, 200.0], vec![10.0, 20.0])]); -let mut source = MemoryReader::new(mzml); -let mut out: Vec = Vec::new(); -let mut writer = IonWriter::create(&mut out, WriteOptions::default())?; -writer.write_stream(&mut source)?; +use ionic::{IonReader, IonWriter, ReadOptions, WriteOptions}; + +fn main() -> Result<(), Box> { + let mut reader = IonReader::open("run.ion", &ReadOptions::default())?; + let metadata = reader.metadata()?; + + let mut writer = IonWriter::create("out.ion", &metadata, &WriteOptions::default())?; + for index in 0..reader.spectrum_count() as usize { + writer.write_spectrum(&reader.spectrum(index)?)?; + } + writer.finish()?; + Ok(()) +} ``` -File path +`WriteOptions::default()`: + +| Field | Default | +|---|---| +| `compression_level` | `12` (0 = off, 1..=22 zstd) | +| `force_f32` | `false` | +| `block_size` | `1024 * 1024` (1 MiB target uncompressed block) | +| `parallel` | `true` | +| `section_storage` | `SectionStorage::Disk` | +| `mz_window` | `20.0` | + +### Convert ```rust -use ionic::{FileWriter, IonWriter, MemoryReader, MzML, Spectrum, WriteOptions}; - -let mzml = MzML::from_spectra(vec![Spectrum::new("scan=1", vec![100.0, 200.0], vec![10.0, 20.0])]); -let mut source = MemoryReader::new(mzml); -let mut output = FileWriter::open_path(Path::new("out.ion"))?; // or FileWriter::open("out.ion") -let mut writer = IonWriter::create(&mut output, WriteOptions::default())?; -writer.write_stream(&mut source)?; -output.flush()?; +use ionic::ConvertOptions; + +fn main() -> Result<(), Box> { + ionic::convert_file("run.mzML", "run.ion", &ConvertOptions::default())?; + + let ion_bytes = std::fs::read("run.ion")?; + let mzml_bytes = ionic::convert(&ion_bytes, &ConvertOptions::default())?; + assert!(mzml_bytes.starts_with(b" Result<(), Box> { + let mut reader = IonReader::open("run.ion", &ReadOptions::default())?; + let ranges = reader.byte_ranges(0, Range { from: 200.0, to: 400.0 })?; + for range in ranges { + println!("fetch bytes {}..{}", range.offset, range.offset + range.length); + } + Ok(()) } ``` diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 92f88567..970f0fb5 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -9,11 +9,11 @@ publish = false serde = { workspace = true } serde_json = { workspace = true } clap = { workspace = true, features = ["derive"] } -ionic = { package = "ionic-rs", path = "../parser", version = "0.1.2" } +ionic = { package = "ionic-rs", path = "../parser", version = "0.2.0" } regex = { workspace = true } rayon = { workspace = true } -mimalloc = { version = "0.1.48" } -crc32fast = "1.5.0" +mimalloc = { version = "0.1.52" } +crc32fast = "1.5.2" memmap2 = "0.9" cosmoz = { workspace = true } diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index be354064..06061d33 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -15,11 +15,8 @@ use clap::{ builder::styling::{AnsiColor, Color, Style, Styles}, }; use ionic::{ - ConvertKind, ConvertOptions, - ion::{ - CURRENT_VERSION, DEFAULT_MZ_WINDOW, HEADER_SIZE, IonReader, ReadOptions, SectionStorage, - WriteOptions, get_version_from_header, update_header_version, - }, + ConvertKind, ConvertOptions, IonReader, ReadOptions, SectionStorage, WriteOptions, + format::{CURRENT_VERSION, DEFAULT_MZ_WINDOW, HEADER_SIZE, set_version, version_of}, mzml::{parse_mzml::parse_mzml, structs::*}, }; use mimalloc::MiMalloc; @@ -387,13 +384,20 @@ fn cat_chromatogram(file_path: &Path, index_1based: u32, with_arrays: bool) -> R print_json_compact(&chromatogram) } +fn ion_result_to_option(result: ionic::IonResult) -> Result, String> { + match result { + Ok(value) => Ok(Some(value)), + Err(ionic::IonError::OutOfRange { .. }) => Ok(None), + Err(error) => Err(error.to_string()), + } +} + fn load_spectrum_at(file_path: &Path, index: usize) -> Result, String> { match file_ext_lower(file_path).as_str() { "ion" => { - let mut ion = IonReader::open_file(file_path, ReadOptions::default()) - .map_err(|e| format!("IonReader::open_file failed: {e}"))?; - ion.spectrum(index) - .map_err(|e| format!("get_spectrum failed: {e}")) + let mut ion = IonReader::open(file_path, &ReadOptions::default()) + .map_err(|e| format!("IonReader::open failed: {e}"))?; + ion_result_to_option(ion.spectrum(index)) } "mzml" => { let bytes = fs::read(file_path).map_err(|e| format!("read failed: {e}"))?; @@ -409,10 +413,9 @@ fn load_spectrum_at(file_path: &Path, index: usize) -> Result, fn load_chromatogram_at(file_path: &Path, index: usize) -> Result, String> { match file_ext_lower(file_path).as_str() { "ion" => { - let mut ion = IonReader::open_file(file_path, ReadOptions::default()) - .map_err(|e| format!("IonReader::open_file failed: {e}"))?; - ion.chromatogram(index) - .map_err(|e| format!("get_chromatogram failed: {e}")) + let mut ion = IonReader::open(file_path, &ReadOptions::default()) + .map_err(|e| format!("IonReader::open failed: {e}"))?; + ion_result_to_option(ion.chromatogram(index)) } "mzml" => { let bytes = fs::read(file_path).map_err(|e| format!("read failed: {e}"))?; @@ -471,8 +474,8 @@ fn read_mzml_or_ion(file_path: &Path) -> Result { let ext = file_ext_lower(file_path); if ext == "ion" { - let ion = IonReader::open_file(file_path, ReadOptions::default()) - .map_err(|e| format!("IonReader::open_file failed: {e}"))?; + let ion = IonReader::open(file_path, &ReadOptions::default()) + .map_err(|e| format!("IonReader::open failed: {e}"))?; return ion.metadata().map_err(|e| format!("metadata failed: {e}")); } if ext == "mzml" { @@ -493,12 +496,11 @@ fn write_mzml_as_ion( sweep_orphans(output_path)?; let temp_output = TempOutput::new(output_path)?; let options = ConvertOptions { - output: Some(temp_output.path().to_path_buf()), kind: ConvertKind::MzmlToIon, write: config, ..Default::default() }; - ionic::convert(input_path, options).map_err(|error| error.to_string())?; + ionic::convert_file(input_path, temp_output.path(), &options).map_err(|error| error.to_string())?; temp_output.move_to(output_path) } @@ -516,7 +518,7 @@ fn update_version_in_file(path: &Path) -> Result { let mut header = [0u8; HEADER_SIZE]; file.read_exact(&mut header) .map_err(|error| format!("read header failed: {error}"))?; - let changed = update_header_version(&mut header).map_err(|error| error.to_string())?; + let changed = set_version(&mut header).map_err(|error| error.to_string())?; if !changed { return Ok(Outcome::Skipped); } @@ -550,7 +552,7 @@ fn ion_file_has_current_version(path: &Path) -> bool { if file.read_exact(&mut header).is_err() { return false; } - get_version_from_header(&header) == Some(CURRENT_VERSION) + version_of(&header) == Some(CURRENT_VERSION) } #[derive(Debug, Clone)] @@ -1069,7 +1071,6 @@ fn convert(cmd: ConvertArgs) -> Result<(), String> { }, |in_path, out_path| { let options = ConvertOptions { - output: Some(out_path.to_path_buf()), kind: ConvertKind::IonToMzml, read: ReadOptions { parallel: matches!(encoding, Encoding::WithinFileParallel), @@ -1077,7 +1078,7 @@ fn convert(cmd: ConvertArgs) -> Result<(), String> { }, ..Default::default() }; - ionic::convert(in_path, options) + ionic::convert_file(in_path, out_path, &options) .map(|_| Outcome::Written) .map_err(|e| format!("convert failed: {e}")) }, @@ -1163,9 +1164,9 @@ fn mzml_output_matches_source( if output_len == 0 { return false; } - let Ok(source) = IonReader::open_file( + let Ok(source) = IonReader::open( source_path, - ReadOptions { + &ReadOptions { parallel, ..ReadOptions::default() }, diff --git a/crates/cli/src/utilities/header.rs b/crates/cli/src/utilities/header.rs index a233d734..fda02a87 100644 --- a/crates/cli/src/utilities/header.rs +++ b/crates/cli/src/utilities/header.rs @@ -1,10 +1,9 @@ use std::{fs::File, io::IsTerminal, path::Path, sync::OnceLock}; -use cosmoz::{DecompressOptions, Decoder, decompress_into}; +use cosmoz::{DecompressOptions, Decoder}; -use ionic::ion::{ - CODEC_NONE, CODEC_ZSTD, FILE_SIGNATURE, FILE_TRAILER, HEADER_SIZE, get_version_from_header, - is_supported, +use ionic::format::{ + CODEC_NONE, CODEC_ZSTD, FILE_SIGNATURE, FILE_TRAILER, HEADER_SIZE, is_supported, version_of, }; static COLOR_ENABLED: OnceLock = OnceLock::new(); @@ -100,7 +99,8 @@ fn decompress_section( decoder: &mut Decoder, ) -> Result, String> { let mut plain = vec![0u8; plain_len]; - let written = decompress_into(stored, &mut plain, &DecompressOptions::default(), decoder) + let written = decoder + .decompress_into(stored, &mut plain) .map_err(|e| format!("decompress failed: {e:?}"))?; plain.truncate(written); Ok(plain) @@ -290,7 +290,7 @@ impl<'a> HeaderView<'a> { fn new(bytes: &'a [u8]) -> Self { let header = &bytes[..HEADER_SIZE]; let sections = build_sections(header, u64_at(header, 400)); - let mut decoder = Decoder::new(); + let mut decoder = Decoder::new(&DecompressOptions::default()); let spec_window_dir = open_window_directory(bytes, header, 32, 40, 384, &mut decoder); let chrom_window_dir = open_window_directory(bytes, header, 96, 104, 392, &mut decoder); let fixed = resolve_fixed_sections(bytes, header, &mut decoder); @@ -583,7 +583,7 @@ fn print_summary(view: &HeaderView<'_>) { &format!("\"{sig}\""), Some(view.signature_ok()), ); - let (version_text, version_ok) = match get_version_from_header(view.bytes) { + let (version_text, version_ok) = match version_of(view.bytes) { Some(v) => (v.to_string(), Some(is_supported(v))), None => ("?".to_string(), Some(false)), }; @@ -727,7 +727,7 @@ fn integrity_check_results(view: &HeaderView<'_>) -> [(&'static str, bool); 23] ("22 E global_meta CRC-32", view.crc_ok(192, 200, 1016)), ( "23 format version supported", - get_version_from_header(view.bytes) + version_of(view.bytes) .map(is_supported) .unwrap_or(false), ), diff --git a/crates/cli/tests/f18_update_stamps_version.rs b/crates/cli/tests/f18_update_stamps_version.rs index d709970b..6b3722a8 100644 --- a/crates/cli/tests/f18_update_stamps_version.rs +++ b/crates/cli/tests/f18_update_stamps_version.rs @@ -56,7 +56,7 @@ fn update_in_place_stamps_version_then_skips() { let (ok, stdout, _) = run_update(&["-i".as_ref(), dir.path().as_ref()]); assert!(ok, "got: {stdout}"); assert!(stdout.contains("ok=1"), "got: {stdout}"); - assert_eq!(version_of(&file), ionic::ion::CURRENT_VERSION); + assert_eq!(version_of(&file), ionic::format::CURRENT_VERSION); assert!(header_crc_matches(&file)); assert_eq!(fs::metadata(&file).unwrap().len(), original_len); assert!(check_passes(&file)); @@ -114,7 +114,7 @@ fn update_to_output_path_leaves_source_untouched() { assert!(ok, "got: {stdout}"); assert!(stdout.contains("ok=1"), "got: {stdout}"); assert_eq!(version_of(&source), 0); - assert_eq!(version_of(&target), ionic::ion::CURRENT_VERSION); + assert_eq!(version_of(&target), ionic::format::CURRENT_VERSION); assert!(check_passes(&target)); let (ok, stdout, _) = run_update(&args); @@ -126,7 +126,7 @@ fn update_to_output_path_leaves_source_untouched() { let (ok, stdout, _) = run_update(&args); assert!(ok, "got: {stdout}"); assert!(stdout.contains("ok=1"), "got: {stdout}"); - assert_eq!(version_of(&target), ionic::ion::CURRENT_VERSION); + assert_eq!(version_of(&target), ionic::format::CURRENT_VERSION); } #[test] diff --git a/crates/parser/Cargo.toml b/crates/parser/Cargo.toml index 262ec698..6449430b 100644 --- a/crates/parser/Cargo.toml +++ b/crates/parser/Cargo.toml @@ -27,7 +27,7 @@ serde_json = { workspace = true } hashbrown = "0.17.1" itoa = { workspace = true } cosmoz = { workspace = true } -crc32fast = "1.5.0" +crc32fast = "1.5.2" sha1 = "0.11" bytemuck = "1" diff --git a/crates/parser/examples/bench_window.rs b/crates/parser/examples/bench_window.rs index eefb1df1..284b8d86 100644 --- a/crates/parser/examples/bench_window.rs +++ b/crates/parser/examples/bench_window.rs @@ -1,6 +1,6 @@ use std::{path::Path, time::Instant}; -use ionic::ion::{IonReader, Range, ReadOptions}; +use ionic::{IonReader, Range, ReadOptions}; fn main() { let mut args = std::env::args().skip(1); @@ -20,7 +20,7 @@ fn main() { } fn run(path: &str, sample_count: usize) -> Result<(), String> { - let mut ion = IonReader::open_file(Path::new(path), ReadOptions::default()) + let mut ion = IonReader::open(Path::new(path), &ReadOptions::default()) .map_err(|error| format!("cannot open {path}: {error}"))?; let total_spectra = ion.spectrum_count() as usize; diff --git a/crates/parser/src/api/convert.rs b/crates/parser/src/api/convert.rs new file mode 100644 index 00000000..e99557f7 --- /dev/null +++ b/crates/parser/src/api/convert.rs @@ -0,0 +1,96 @@ +use crate::{ + ion::{IonError, IonResult, encoder::ion_writer::write_mzml_to_ion, format::FILE_SIGNATURE}, + mzml::{bin_to_mzml, parse_mzml}, +}; + +use super::options::{ConvertKind, ConvertOptions}; + +fn resolve_kind(bytes: &[u8], kind: ConvertKind) -> ConvertKind { + if kind != ConvertKind::Auto { + return kind; + } + if bytes.starts_with(&FILE_SIGNATURE) { + ConvertKind::IonToMzml + } else { + ConvertKind::MzmlToIon + } +} + +pub fn convert(input: &[u8], options: &ConvertOptions) -> IonResult> { + if resolve_kind(input, options.kind) == ConvertKind::IonToMzml { + let mut reader = crate::ion::decoder::decode::IonReader::from_bytes(input, &options.read)?; + let mzml = reader.to_mzml()?; + return Ok(bin_to_mzml(&mzml)?); + } + if input.starts_with(&FILE_SIGNATURE) { + return Err(IonError::from("input is already an ion file")); + } + let mzml = parse_mzml(input)?; + let mut output = Vec::new(); + write_mzml_to_ion(&mzml, &options.write, &mut output)?; + Ok(output) +} + +#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] +pub fn convert_file( + input: impl AsRef, + output: impl AsRef, + options: &ConvertOptions, +) -> IonResult<()> { + use std::io::Read; + + use crate::{ + ion::{encoder::ion_writer::IonWriter, encoder::utilities::FileWriter}, + mzml::{MzmlReader, structs::MzML}, + }; + + let input = input.as_ref(); + let output = output.as_ref(); + + let start = { + let mut start = Vec::with_capacity(FILE_SIGNATURE.len()); + std::fs::File::open(input)? + .take(FILE_SIGNATURE.len() as u64) + .read_to_end(&mut start)?; + start + }; + let is_ion = start.starts_with(&FILE_SIGNATURE); + + let kind = if options.kind != ConvertKind::Auto { + options.kind + } else if let Some(kind) = kind_from_extension(input) { + kind + } else if is_ion { + ConvertKind::IonToMzml + } else { + ConvertKind::MzmlToIon + }; + + if kind == ConvertKind::IonToMzml { + let mut reader = crate::ion::decoder::decode::IonReader::open(input, &options.read)?; + let mzml = reader.to_mzml()?; + let xml = bin_to_mzml(&mzml)?; + std::fs::write(output, &xml)?; + return Ok(()); + } + + if is_ion { + return Err(IonError::from("input is already an ion file")); + } + + let mut file = FileWriter::open_path(output)?; + let mut writer = IonWriter::begin(&mut file, &MzML::default(), &options.write)?; + writer.write_stream(&mut file, &mut MzmlReader::open(input)?)?; + file.flush()?; + Ok(()) +} + +#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] +fn kind_from_extension(path: &std::path::Path) -> Option { + let extension = path.extension()?.to_str()?.to_ascii_lowercase(); + match extension.as_str() { + "mzml" => Some(ConvertKind::MzmlToIon), + "ion" => Some(ConvertKind::IonToMzml), + _ => None, + } +} diff --git a/crates/parser/src/api/mod.rs b/crates/parser/src/api/mod.rs new file mode 100644 index 00000000..1bb01f3f --- /dev/null +++ b/crates/parser/src/api/mod.rs @@ -0,0 +1,32 @@ +mod convert; +mod options; +mod reader; +mod writer; + +pub use convert::convert; +#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] +pub use convert::convert_file; +pub use options::{ConvertKind, ConvertOptions, ReadOptions, WriteOptions}; +pub use reader::{IonReader, ScanQuery}; +pub use writer::IonWriter; + +#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] +use crate::{ion::IonResult, mzml::structs::MzML}; + +#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] +pub fn read(path: impl AsRef) -> IonResult { + IonReader::open(path, &ReadOptions::default())?.to_mzml() +} + +#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] +pub fn write( + path: impl AsRef, + mzml: &MzML, + options: &WriteOptions, +) -> IonResult<()> { + use crate::ion::encoder::{ion_writer::write_mzml_to_ion, utilities::FileWriter}; + + let mut file = FileWriter::open_path(path.as_ref())?; + write_mzml_to_ion(mzml, options, &mut file)?; + file.flush() +} diff --git a/crates/parser/src/api/options.rs b/crates/parser/src/api/options.rs new file mode 100644 index 00000000..d2db8a4a --- /dev/null +++ b/crates/parser/src/api/options.rs @@ -0,0 +1,18 @@ +pub use crate::ion::decoder::decode::ReadOptions; +pub use crate::ion::encoder::encode::WriteOptions; + +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ConvertKind { + #[default] + Auto, + MzmlToIon, + IonToMzml, +} + +#[derive(Debug, Clone, Default)] +pub struct ConvertOptions { + pub kind: ConvertKind, + pub read: ReadOptions, + pub write: WriteOptions, +} diff --git a/crates/parser/src/api/reader.rs b/crates/parser/src/api/reader.rs new file mode 100644 index 00000000..61c1420f --- /dev/null +++ b/crates/parser/src/api/reader.rs @@ -0,0 +1 @@ +pub use crate::ion::decoder::decode::{IonReader, ScanQuery}; diff --git a/crates/parser/src/api/writer.rs b/crates/parser/src/api/writer.rs new file mode 100644 index 00000000..21442c40 --- /dev/null +++ b/crates/parser/src/api/writer.rs @@ -0,0 +1,117 @@ +#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] +use std::path::Path; + +use crate::{ + ion::{ + IonResult, + encoder::{ion_writer::IonWriter as LowWriter, scan_stream::ScanStream, utilities::WriteBytes}, + }, + mzml::structs::{Chromatogram, MzML, Spectrum}, +}; + +#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] +use crate::ion::encoder::utilities::FileWriter; + +use super::options::WriteOptions; + +enum Sink<'out> { + Borrowed(&'out mut dyn WriteBytes), + #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] + Owned(FileWriter), +} + +impl WriteBytes for Sink<'_> { + fn write(&mut self, bytes: &[u8]) -> IonResult<()> { + match self { + Sink::Borrowed(output) => output.write(bytes), + #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] + Sink::Owned(file) => file.write(bytes), + } + } + + fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()> { + match self { + Sink::Borrowed(output) => output.patch(at, bytes), + #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] + Sink::Owned(file) => file.patch(at, bytes), + } + } + + fn position(&mut self) -> IonResult { + match self { + Sink::Borrowed(output) => output.position(), + #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] + Sink::Owned(file) => file.position(), + } + } +} + +fn strip_lists(metadata: &MzML) -> MzML { + let mut metadata = metadata.clone(); + metadata.run.spectrum_list = None; + metadata.run.chromatogram_list = None; + metadata +} + +pub struct IonWriter<'out> { + sink: Sink<'out>, + low: LowWriter, + metadata: MzML, +} + +#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] +impl IonWriter<'static> { + pub fn create( + path: impl AsRef, + metadata: &MzML, + options: &WriteOptions, + ) -> IonResult { + let file = FileWriter::open_path(path.as_ref())?; + let mut sink = Sink::Owned(file); + let low = LowWriter::begin(&mut sink, metadata, options)?; + Ok(Self { + sink, + low, + metadata: strip_lists(metadata), + }) + } +} + +impl<'out> IonWriter<'out> { + pub fn to( + output: &'out mut dyn WriteBytes, + metadata: &MzML, + options: &WriteOptions, + ) -> IonResult { + let mut sink = Sink::Borrowed(output); + let low = LowWriter::begin(&mut sink, metadata, options)?; + Ok(Self { + sink, + low, + metadata: strip_lists(metadata), + }) + } + + pub fn write_spectrum(&mut self, spectrum: &Spectrum) -> IonResult<()> { + self.low.push_spectrum(&mut self.sink, spectrum) + } + + pub fn write_chromatogram(&mut self, chromatogram: &Chromatogram) -> IonResult<()> { + self.low.push_chromatogram(&mut self.sink, chromatogram) + } + + pub fn write_stream(&mut self, scans: &mut dyn ScanStream) -> IonResult<()> { + self.low.write_stream(&mut self.sink, scans) + } + + pub fn finish(mut self) -> IonResult<()> { + self.low.finish(&mut self.sink, &self.metadata)?; + + #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] + if let Sink::Owned(file) = &mut self.sink { + file.flush()?; + } + + Ok(()) + } +} diff --git a/crates/parser/src/convert.rs b/crates/parser/src/convert.rs deleted file mode 100644 index 4da310d4..00000000 --- a/crates/parser/src/convert.rs +++ /dev/null @@ -1,191 +0,0 @@ -use std::{fs, path::PathBuf}; -#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] -use std::{fs::File, io::Read, path::Path}; - -use crate::{ - ion::{ - FILE_SIGNATURE, IonError, IonReader, IonResult, ReadOptions, WriteBytes, WriteOptions, - write_mzml_to_ion, - }, - mzml::{bin_to_mzml, parse_mzml}, -}; -#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] -use crate::{ - ion::{FileWriter, IonWriter}, - mzml::MzmlReader, -}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum ConvertKind { - #[default] - Auto, - MzmlToIon, - IonToMzml, -} - -pub enum Input<'a> { - Bytes(&'a [u8]), - #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - Path(PathBuf), -} - -#[derive(Debug, Clone, Default)] -pub struct ConvertOptions { - pub output: Option, - pub kind: ConvertKind, - pub read: ReadOptions, - pub write: WriteOptions, -} - -impl<'a> From<&'a [u8]> for Input<'a> { - fn from(bytes: &'a [u8]) -> Self { - Self::Bytes(bytes) - } -} - -impl<'a> From<&'a Vec> for Input<'a> { - fn from(bytes: &'a Vec) -> Self { - Self::Bytes(bytes) - } -} - -#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] -impl From<&Path> for Input<'_> { - fn from(path: &Path) -> Self { - Self::Path(path.to_path_buf()) - } -} - -#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] -impl From for Input<'_> { - fn from(path: PathBuf) -> Self { - Self::Path(path) - } -} - -#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] -impl From<&PathBuf> for Input<'_> { - fn from(path: &PathBuf) -> Self { - Self::Path(path.clone()) - } -} - -pub fn convert<'a>( - input: impl Into>, - options: ConvertOptions, -) -> IonResult>> { - let input = input.into(); - if get_kind(&input, options.kind)? == ConvertKind::IonToMzml { - return ion_to_mzml(&input, options); - } - mzml_to_ion(&input, options) -} - -fn get_kind(input: &Input<'_>, kind: ConvertKind) -> IonResult { - if kind != ConvertKind::Auto { - return Ok(kind); - } - #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - if let Input::Path(path) = input - && let Some(kind) = kind_from_extension(path) - { - return Ok(kind); - } - Ok(if input_is_ion(input)? { - ConvertKind::IonToMzml - } else { - ConvertKind::MzmlToIon - }) -} - -#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] -fn kind_from_extension(path: &Path) -> Option { - let extension = path.extension()?.to_str()?.to_ascii_lowercase(); - match extension.as_str() { - "mzml" => Some(ConvertKind::MzmlToIon), - "ion" => Some(ConvertKind::IonToMzml), - _ => None, - } -} - -fn input_is_ion(input: &Input<'_>) -> IonResult { - match input { - Input::Bytes(bytes) => Ok(bytes.starts_with(&FILE_SIGNATURE)), - #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - Input::Path(path) => Ok(read_file_start(path)?.starts_with(&FILE_SIGNATURE)), - } -} - -#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] -fn read_file_start(path: &Path) -> IonResult> { - let file = File::open(path).map_err(|error| { - IonError::from(format!( - "cannot open input file '{}': {error}", - path.display() - )) - })?; - let mut start = Vec::with_capacity(FILE_SIGNATURE.len()); - file.take(FILE_SIGNATURE.len() as u64) - .read_to_end(&mut start)?; - Ok(start) -} - -fn mzml_to_ion(input: &Input<'_>, options: ConvertOptions) -> IonResult>> { - if input_is_ion(input)? { - return Err(IonError::from("input is already an ion file")); - } - #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - let write = options.write; - #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] - let write = WriteOptions { - compression_level: 1, - ..options.write - }; - match options.output { - None => { - let mut bytes = Vec::new(); - write_ion(input, write, &mut bytes)?; - Ok(Some(bytes)) - } - #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - Some(path) => { - let mut file = FileWriter::open_path(&path)?; - write_ion(input, write, &mut file)?; - file.flush()?; - Ok(None) - } - #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] - Some(_) => Err(IonError::from( - "file output is not available in browser wasm", - )), - } -} - -fn write_ion(input: &Input<'_>, write: WriteOptions, output: &mut dyn WriteBytes) -> IonResult<()> { - match input { - Input::Bytes(bytes) => write_mzml_to_ion(&parse_mzml(bytes)?, write, output), - #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - Input::Path(path) => { - IonWriter::create(output, write)?.write_stream(&mut MzmlReader::open(path)?) - } - } -} - -fn ion_to_mzml(input: &Input<'_>, options: ConvertOptions) -> IonResult>> { - let mzml = open_ion(input, options.read)?.to_mzml()?; - let xml = bin_to_mzml(&mzml)?; - drop(mzml); - let Some(path) = options.output else { - return Ok(Some(xml)); - }; - fs::write(path, &xml)?; - Ok(None) -} - -fn open_ion(input: &Input<'_>, read: ReadOptions) -> IonResult { - match input { - Input::Bytes(bytes) => IonReader::open(bytes, read), - #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - Input::Path(path) => IonReader::open_file(path, read), - } -} diff --git a/crates/parser/src/format.rs b/crates/parser/src/format.rs new file mode 100644 index 00000000..eed4468c --- /dev/null +++ b/crates/parser/src/format.rs @@ -0,0 +1,5 @@ +pub use crate::ion::{ + CODEC_NONE, CODEC_ZSTD, CURRENT_VERSION, DEFAULT_MZ_WINDOW, FILE_SIGNATURE, FILE_TRAILER, + HEADER_FORMAT_VERSION_OFFSET, HEADER_SIZE, MAX_SUPPORTED_VERSION, MIN_SUPPORTED_VERSION, + allow_version, is_supported, set_version, version_of, +}; diff --git a/crates/parser/src/ion/array_kind.rs b/crates/parser/src/ion/array_kind.rs index 7e14687c..f53822ff 100644 --- a/crates/parser/src/ion/array_kind.rs +++ b/crates/parser/src/ion/array_kind.rs @@ -1,5 +1,6 @@ use crate::accessions::{INTENSITY_ARRAY, MZ_ARRAY, TIME_ARRAY}; +#[non_exhaustive] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ArrayKind { Mz, diff --git a/crates/parser/src/ion/decoder/decode/arrays.rs b/crates/parser/src/ion/decoder/decode/arrays.rs index aed9a96d..aa4b3f8d 100644 --- a/crates/parser/src/ion/decoder/decode/arrays.rs +++ b/crates/parser/src/ion/decoder/decode/arrays.rs @@ -1,7 +1,7 @@ use super::*; #[derive(Debug, Clone, Copy, Default)] -pub struct ArrayAddress { +pub(crate) struct ArrayAddress { pub(crate) block_id: u32, pub(crate) element_offset: u64, pub(crate) element_count: u64, @@ -13,16 +13,17 @@ pub struct ArrayAddress { pub(crate) array_cv_code: u8, } +#[cfg(test)] impl ArrayAddress { - pub fn array_type(&self) -> u32 { + pub(crate) fn array_type(&self) -> u32 { self.array_type } - pub fn dtype(&self) -> u8 { + pub(crate) fn dtype(&self) -> u8 { self.dtype } - pub fn array_filter(&self) -> u8 { + pub(crate) fn array_filter(&self) -> u8 { self.array_filter } } @@ -367,6 +368,7 @@ pub(crate) fn decode_into( Ok(()) } +#[cfg(test)] fn collect_entry_array_addresses( entry_bytes: &[u8], address_bytes: &[u8], @@ -388,6 +390,7 @@ fn collect_entry_array_addresses( Some(refs) } +#[cfg(test)] pub(crate) fn read_scan_arrays( container: &mut dyn ContainerAccess, entry_bytes: &[u8], @@ -439,6 +442,7 @@ pub(crate) fn read_scan_arrays( } impl IonReader { + #[allow(private_interfaces)] pub fn spectrum_array_addresses(&self, index: usize) -> Option> { if index >= self.header.spectrum_count as usize { return None; @@ -447,6 +451,7 @@ impl IonReader { .map(ArrayAddressList::into_vec) } + #[allow(private_interfaces)] pub fn chromatogram_array_addresses(&self, index: usize) -> Option> { if index >= self.header.chrom_count as usize { return None; @@ -459,6 +464,7 @@ impl IonReader { .map(ArrayAddressList::into_vec) } + #[allow(private_interfaces)] pub fn read_spectrum_values( &mut self, array_address: &ArrayAddress, @@ -475,6 +481,7 @@ impl IonReader { decode_into(out, raw, array_address.dtype, array_address.array_filter) } + #[allow(private_interfaces)] pub fn read_spectrum_array(&mut self, array_address: &ArrayAddress) -> IonResult { let (element_offset, count, stride) = address_read_params(array_address); let raw = self.spec_container.get_array_bytes_from_block( @@ -488,6 +495,7 @@ impl IonReader { super::to_mzml::decoded_bytes_to_binary_data(&values, array_address.dtype) } + #[allow(private_interfaces)] pub fn read_chromatogram_values( &mut self, array_address: &ArrayAddress, @@ -508,6 +516,7 @@ impl IonReader { decode_into(out, raw, array_address.dtype, array_address.array_filter) } + #[allow(private_interfaces)] pub fn read_chromatogram_array( &mut self, array_address: &ArrayAddress, @@ -546,14 +555,25 @@ impl IonReader { Ok(()) } - pub fn get_spectrum_array( + pub fn array(&mut self, spectrum_index: usize, array_type: impl Into) -> IonResult> { + let mut values = Vec::new(); + self.array_into(spectrum_index, array_type, &mut values)?; + Ok(values) + } + + pub fn array_into( &mut self, spectrum_index: usize, array_type: impl Into, - ) -> IonResult> { + out: &mut Vec, + ) -> IonResult<()> { + out.clear(); let array_type = array_type.into(); if spectrum_index >= self.header.spectrum_count as usize { - return Err("spectrum index out of range".into()); + return Err(IonError::OutOfRange { + index: spectrum_index, + count: self.header.spectrum_count, + }); } let Some(array_addresses) = read_array_addresses_from_buffers( @@ -561,7 +581,7 @@ impl IonReader { &self.spec_array_addresses, spectrum_index, ) else { - return Ok(Vec::new()); + return Ok(()); }; let groups = group_arrays(array_addresses.as_slice())?; @@ -570,13 +590,11 @@ impl IonReader { if group.array_type != array_type { continue; } - - let mut values = Vec::new(); - self.read_group_values(&group, &mut values)?; - return Ok(values); + self.read_group_values(&group, out)?; + return Ok(()); } - Ok(Vec::new()) + Ok(()) } } @@ -668,4 +686,791 @@ mod tests { .expect("raw filter must still work for any dtype"); assert_eq!(ok.as_ref(), raw.as_slice()); } + + mod array_roundtrips { + use crate::{ + ArrayKind, IonReader, Range, ReadOptions, ScanQuery, Select, WriteOptions, + format::{CURRENT_VERSION, HEADER_FORMAT_VERSION_OFFSET}, + ion::encoder::ion_writer::write_mzml_to_ion, + mzml::structs::{ + BinaryDataArray, BinaryDataArrayList, Chromatogram, CvParam, FileContent, + FileDescription, MzML, NumericArray, NumericType, Run, Scan, ScanList, + SourceFileList, Spectrum, SpectrumList, + }, + }; + use proptest::prelude::*; + + fn synthetic_ms_cv(accession: &str, value: Option<&str>) -> CvParam { + CvParam { + cv_ref: Some("MS".to_string()), + accession: Some(accession.to_string()), + name: accession.to_string(), + value: value.map(ToString::to_string), + ..Default::default() + } + } + + fn precision_accession(numeric_type: NumericType) -> &'static str { + match numeric_type { + NumericType::Float64 => "MS:1000523", + NumericType::Float32 => "MS:1000521", + NumericType::Float16 => "MS:1000520", + NumericType::Int64 => "MS:1000522", + NumericType::Int32 => "MS:1000519", + NumericType::Int16 => "MS:1000518", + } + } + + fn synthetic_binary_data_array( + role_accession: &str, + numeric_type: NumericType, + binary: NumericArray, + declared_length: Option, + ) -> BinaryDataArray { + BinaryDataArray { + array_length: declared_length, + cv_params: vec![ + synthetic_ms_cv(role_accession, None), + synthetic_ms_cv(precision_accession(numeric_type), None), + synthetic_ms_cv("MS:1000576", None), + ], + numeric_type: Some(numeric_type), + binary: Some(binary), + ..Default::default() + } + } + + fn minimal_file_description() -> FileDescription { + FileDescription { + file_content: FileContent::default(), + source_file_list: SourceFileList { + count: Some(0), + source_file: Vec::new(), + }, + contacts: Vec::new(), + } + } + + fn mzml_with_single_array( + numeric_type: NumericType, + binary: NumericArray, + len: usize, + ) -> MzML { + MzML { + file_description: Some(minimal_file_description()), + run: Run { + id: format!("array-test-{numeric_type:?}"), + spectrum_list: Some(SpectrumList { + count: Some(1), + spectra: vec![Spectrum { + id: "scan=1".to_string(), + index: Some(0), + default_array_length: Some(len), + binary_data_array_list: Some(BinaryDataArrayList { + count: Some(1), + binary_data_arrays: vec![synthetic_binary_data_array( + "MS:1000515", + numeric_type, + binary, + Some(len), + )], + }), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }, + ..Default::default() + } + } + + fn build_mzml(spectra: Vec, chromatograms: Vec) -> MzML { + MzML { + file_description: Some(minimal_file_description()), + run: Run { + id: "test-run".to_string(), + spectrum_list: if spectra.is_empty() { + None + } else { + Some(SpectrumList { + count: Some(spectra.len()), + spectra, + ..Default::default() + }) + }, + chromatogram_list: if chromatograms.is_empty() { + None + } else { + Some(crate::mzml::structs::ChromatogramList { + count: Some(chromatograms.len()), + chromatograms, + ..Default::default() + }) + }, + ..Default::default() + }, + ..Default::default() + } + } + + fn make_spectrum_f64(id: &str, mz: Vec, intensity: Vec) -> Spectrum { + let len = mz.len(); + Spectrum { + id: id.to_string(), + index: Some(0), + default_array_length: Some(len), + binary_data_array_list: Some(BinaryDataArrayList { + count: Some(2), + binary_data_arrays: vec![ + synthetic_binary_data_array( + "MS:1000514", + NumericType::Float64, + NumericArray::F64(mz), + Some(len), + ), + synthetic_binary_data_array( + "MS:1000515", + NumericType::Float64, + NumericArray::F64(intensity), + Some(len), + ), + ], + }), + ..Default::default() + } + } + + fn make_chromatogram_f64(id: &str, time: Vec, intensity: Vec) -> Chromatogram { + let len = time.len(); + Chromatogram { + id: id.to_string(), + index: Some(0), + default_array_length: Some(len), + binary_data_array_list: Some(BinaryDataArrayList { + count: Some(2), + binary_data_arrays: vec![ + synthetic_binary_data_array( + "MS:1000595", + NumericType::Float64, + NumericArray::F64(time), + Some(len), + ), + synthetic_binary_data_array( + "MS:1000515", + NumericType::Float64, + NumericArray::F64(intensity), + Some(len), + ), + ], + }), + ..Default::default() + } + } + + fn encode_to_ion(mzml: &MzML, compression_level: u8, force_f32: bool) -> Vec { + let mut out = Vec::new(); + write_mzml_to_ion( + mzml, + &WriteOptions { + compression_level, + force_f32, + ..Default::default() + }, + &mut out, + ) + .expect("encode should succeed"); + out + } + + fn decode_ion(bytes: &[u8]) -> crate::IonResult { + let mut decoder = IonReader::from_bytes(bytes, &ReadOptions::default())?; + decoder.to_mzml() + } + + fn roundtrip(mzml: &MzML) -> MzML { + decode_ion(&encode_to_ion(mzml, 0, false)).expect("decode should succeed") + } + + fn first_spectrum_binary(mzml: &MzML) -> Option<&NumericArray> { + mzml.run + .spectrum_list + .as_ref()? + .spectra + .first()? + .binary_data_array_list + .as_ref()? + .binary_data_arrays + .first()? + .binary + .as_ref() + } + + fn first_chrom_array_values_by_accession(c: &Chromatogram, accession: &str) -> Vec { + let bda = c + .binary_data_array_list + .as_ref() + .expect("binary data array list present") + .binary_data_arrays + .iter() + .find(|a| { + a.cv_params + .iter() + .any(|p| p.accession.as_deref() == Some(accession)) + }) + .expect("array with accession present"); + bda.binary.as_ref().expect("binary payload present").to_f64() + } + + #[test] + fn roundtrip_f64_array() { + let values = vec![1.0_f64, -2.5, 0.0, f64::MAX, f64::MIN, std::f64::consts::PI]; + let len = values.len(); + let mzml = + mzml_with_single_array(NumericType::Float64, NumericArray::F64(values.clone()), len); + let out = roundtrip(&mzml); + let bin = first_spectrum_binary(&out).expect("should have binary data"); + let got = bin.to_f64(); + assert_eq!(got, values); + } + + #[test] + fn roundtrip_f32_array() { + let values = vec![1.0_f32, -2.5, 0.0, f32::MAX, f32::MIN, std::f32::consts::PI]; + let len = values.len(); + let mzml = + mzml_with_single_array(NumericType::Float32, NumericArray::F32(values.clone()), len); + let out = roundtrip(&mzml); + let bin = first_spectrum_binary(&out).expect("should have binary data"); + let got = bin.to_f64(); + let expected: Vec = values.iter().map(|v| *v as f64).collect(); + assert_eq!(got, expected); + } + + #[test] + fn array_filter_label_matches_applied_transform() { + let raw_filter = 0u8; + let f64_dtype = 1u8; + let f32_dtype = 2u8; + + let f32_values = vec![10.0_f32, 11.0, 12.0, 13.0]; + let f32_len = f32_values.len(); + let f32_mzml = + mzml_with_single_array(NumericType::Float32, NumericArray::F32(f32_values), f32_len); + + let f64_values = vec![100.0_f64, 100.5, 101.0, 101.5]; + let f64_len = f64_values.len(); + let f64_mzml = + mzml_with_single_array(NumericType::Float64, NumericArray::F64(f64_values), f64_len); + + let f32_bytes = encode_to_ion(&f32_mzml, 9, false); + let f32_decoder = + IonReader::from_bytes(&f32_bytes, &ReadOptions::default()).expect("open f32 ion"); + let f32_refs = f32_decoder + .spectrum_array_addresses(0) + .expect("f32 array refs"); + + let f64_bytes = encode_to_ion(&f64_mzml, 9, false); + let f64_decoder = + IonReader::from_bytes(&f64_bytes, &ReadOptions::default()).expect("open f64 ion"); + let f64_refs = f64_decoder + .spectrum_array_addresses(0) + .expect("f64 array refs"); + + assert!( + f32_refs + .iter() + .any(|a| a.dtype() == f32_dtype && a.array_filter() == raw_filter), + "f32 intensity must be tagged raw" + ); + assert!( + f64_refs + .iter() + .any(|a| a.dtype() == f64_dtype && a.array_filter() == raw_filter), + "f64 intensity must be tagged raw (intensity is never delta-shuffled)" + ); + } + + #[test] + fn roundtrip_i64_array() { + let values = vec![0_i64, 1, -1, i64::MAX, i64::MIN, 42]; + let len = values.len(); + let mzml = + mzml_with_single_array(NumericType::Int64, NumericArray::I64(values.clone()), len); + let out = roundtrip(&mzml); + let bin = first_spectrum_binary(&out).expect("should have binary data"); + match bin { + NumericArray::I64(got) => assert_eq!(got, &values), + other => { + let got = other.to_f64(); + let expected: Vec = values.iter().map(|v| *v as f64).collect(); + assert_eq!( + got, expected, + "i64 roundtrip values differ (via f64 conversion)" + ); + } + } + } + + #[test] + fn roundtrip_i32_array() { + let values = vec![0_i32, 1, -1, i32::MAX, i32::MIN, 42]; + let len = values.len(); + let mzml = + mzml_with_single_array(NumericType::Int32, NumericArray::I32(values.clone()), len); + let out = roundtrip(&mzml); + let bin = first_spectrum_binary(&out).expect("should have binary data"); + match bin { + NumericArray::I32(got) => assert_eq!(got, &values), + other => { + let got = other.to_f64(); + let expected: Vec = values.iter().map(|v| *v as f64).collect(); + assert_eq!(got, expected, "i32 roundtrip values differ"); + } + } + } + + #[test] + fn roundtrip_i16_array() { + let values = vec![0_i16, 1, -1, i16::MAX, i16::MIN, 42]; + let len = values.len(); + let mzml = + mzml_with_single_array(NumericType::Int16, NumericArray::I16(values.clone()), len); + let out = roundtrip(&mzml); + let bin = first_spectrum_binary(&out).expect("should have binary data"); + match bin { + NumericArray::I16(got) => assert_eq!(got, &values), + other => { + let got = other.to_f64(); + let expected: Vec = values.iter().map(|v| *v as f64).collect(); + assert_eq!(got, expected, "i16 roundtrip values differ"); + } + } + } + + #[test] + fn roundtrip_single_element_per_type() { + let cases: Vec<(NumericType, NumericArray)> = vec![ + (NumericType::Float64, NumericArray::F64(vec![42.0])), + (NumericType::Float32, NumericArray::F32(vec![42.0])), + (NumericType::Int64, NumericArray::I64(vec![42])), + (NumericType::Int32, NumericArray::I32(vec![42])), + (NumericType::Int16, NumericArray::I16(vec![42])), + ]; + + for (nt, bin) in cases { + let mzml = mzml_with_single_array(nt, bin.clone(), 1); + let out = roundtrip(&mzml); + let got = first_spectrum_binary(&out).expect("should have binary data"); + let expected_f64 = bin.to_f64(); + let got_f64 = got.to_f64(); + assert_eq!( + got_f64, expected_f64, + "single-element roundtrip failed for {nt:?}" + ); + } + } + + #[test] + fn roundtrip_at_compression_levels() { + let values = vec![100.0_f64, 200.0, 300.0, 400.0, 500.0]; + let len = values.len(); + let mzml = + mzml_with_single_array(NumericType::Float64, NumericArray::F64(values.clone()), len); + + for level in [0, 3, 10, 22] { + let mut buf = Vec::new(); + write_mzml_to_ion( + &mzml, + &WriteOptions { + compression_level: level, + force_f32: false, + ..Default::default() + }, + &mut buf, + ) + .unwrap_or_else(|e| panic!("encode at level {level} failed: {e}")); + let mut decoder = IonReader::from_bytes(&buf, &ReadOptions::default()) + .unwrap_or_else(|e| panic!("decoder open at level {level} failed: {e}")); + let decoded = decoder + .to_mzml() + .unwrap_or_else(|e| panic!("to_mzml at level {level} failed: {e}")); + let got = first_spectrum_binary(&decoded).expect("should have binary data"); + assert_eq!( + got.to_f64(), + values, + "values differ at compression level {level}" + ); + } + } + + #[test] + fn roundtrip_force_f32_downcasts() { + let values = vec![100.0_f64, 200.0, 300.0]; + let len = values.len(); + let mzml = + mzml_with_single_array(NumericType::Float64, NumericArray::F64(values.clone()), len); + + let mut buf = Vec::new(); + write_mzml_to_ion( + &mzml, + &WriteOptions { + compression_level: 0, + force_f32: true, + ..Default::default() + }, + &mut buf, + ) + .expect("encode with force_f32"); + let mut decoder = + IonReader::from_bytes(&buf, &ReadOptions::default()).expect("decoder open"); + let decoded = decoder.to_mzml().expect("to_mzml"); + + let bin = first_spectrum_binary(&decoded).expect("should have binary data"); + let got = bin.to_f64(); + for (i, (g, e)) in got.iter().zip(values.iter()).enumerate() { + let expected_f32 = *e as f32 as f64; + assert!( + (g - expected_f32).abs() < 1e-6, + "index {i}: force_f32 value mismatch: {g} vs {expected_f32}" + ); + } + } + + fn encode_decode_mz_compressed(mz: Vec) -> Vec { + let len = mz.len(); + let mzml = mzml_with_single_array(NumericType::Float64, NumericArray::F64(mz), len); + let buf = encode_to_ion(&mzml, 3, false); + let decoded = decode_ion(&buf).unwrap(); + first_spectrum_binary(&decoded).unwrap().to_f64() + } + + #[test] + fn delta_mz_single_element_is_bit_exact() { + let input = vec![503.42f64]; + let got = encode_decode_mz_compressed(input.clone()); + assert_eq!(got[0].to_bits(), input[0].to_bits()); + } + + #[test] + fn delta_mz_two_elements_are_bit_exact() { + let input = vec![100.0f64, 200.5]; + let got = encode_decode_mz_compressed(input.clone()); + for (a, b) in got.iter().zip(input.iter()) { + assert_eq!(a.to_bits(), b.to_bits()); + } + } + + #[test] + fn delta_mz_monotonic_array_is_bit_exact() { + let input: Vec = (0..10_000).map(|i| 100.0 + i as f64 * 0.01).collect(); + let got = encode_decode_mz_compressed(input.clone()); + for (a, b) in got.iter().zip(input.iter()) { + assert_eq!(a.to_bits(), b.to_bits()); + } + } + + #[test] + fn delta_mz_special_values_are_bit_exact() { + let input = vec![f64::NAN, f64::INFINITY, f64::NEG_INFINITY, -0.0f64, 0.0f64]; + let got = encode_decode_mz_compressed(input.clone()); + for (a, b) in got.iter().zip(input.iter()) { + assert_eq!(a.to_bits(), b.to_bits()); + } + } + + #[test] + fn delta_mz_via_for_each_scan_is_bit_exact() { + let input: Vec = (0..500).map(|i| 100.0 + i as f64 * 0.05).collect(); + let intensity: Vec = vec![1.0; input.len()]; + let mut spectrum = make_spectrum_f64("scan=1", input.clone(), intensity); + spectrum.scan_list = Some(ScanList { + count: Some(1), + scans: vec![Scan { + cv_params: vec![CvParam { + cv_ref: Some("MS".to_string()), + accession: Some("MS:1000016".to_string()), + name: "scan start time".to_string(), + value: Some("1.0".to_string()), + unit_accession: Some("UO:0000031".to_string()), + ..Default::default() + }], + ..Default::default() + }], + ..Default::default() + }); + let mzml = build_mzml(vec![spectrum], vec![]); + let buf = encode_to_ion(&mzml, 3, false); + let mut ion = IonReader::from_bytes(&buf, &ReadOptions::default()).unwrap(); + let mut got_mz: Vec = Vec::new(); + let query = ScanQuery { + mz: Range { + from: 0.0, + to: f64::MAX, + }, + select: Select::All, + ms_level: None, + }; + ion.scans_in(&query, |window| { + got_mz = window.mz.to_vec(); + }) + .unwrap(); + assert_eq!(got_mz.len(), input.len()); + for (a, b) in got_mz.iter().zip(input.iter()) { + assert_eq!(a.to_bits(), b.to_bits()); + } + } + + #[test] + fn delta_on_mz_raw_on_intensity() { + let mz: Vec = (0..100).map(|i| 100.0 + i as f64).collect(); + let intensity: Vec = (0..100).map(|i| (i * 10) as f64).collect(); + let mzml = build_mzml( + vec![make_spectrum_f64("scan=1", mz.clone(), intensity.clone())], + vec![], + ); + let buf = encode_to_ion(&mzml, 3, false); + let mut decoder = IonReader::from_bytes(&buf, &ReadOptions::default()).unwrap(); + let refs = decoder.spectrum_array_addresses(0).unwrap(); + let mz_address = refs.iter().find(|r| r.array_type() == 1_000_514).unwrap(); + let int_ref = refs.iter().find(|r| r.array_type() == 1_000_515).unwrap(); + assert_eq!( + mz_address.array_filter(), + 2, + "m/z must use DeltaShuffle filter" + ); + assert_eq!( + int_ref.array_filter(), + 0, + "intensity must use raw filter (intensity is never delta-shuffled)" + ); + let mut got_mz = Vec::new(); + decoder + .read_spectrum_values(mz_address, &mut got_mz) + .unwrap(); + let mut got_int = Vec::new(); + decoder.read_spectrum_values(int_ref, &mut got_int).unwrap(); + for (a, b) in got_mz.iter().zip(mz.iter()) { + assert_eq!(a.to_bits(), b.to_bits()); + } + for (a, b) in got_int.iter().zip(intensity.iter()) { + assert_eq!(a.to_bits(), b.to_bits()); + } + } + + #[test] + fn format_version_always_matches_current() { + let values = vec![1.0_f64, 2.0, 3.0, 4.0, 5.0]; + let len = values.len(); + let mzml = + mzml_with_single_array(NumericType::Float64, NumericArray::F64(values.clone()), len); + for level in [0u8, 3, 22] { + let buf = encode_to_ion(&mzml, level, false); + let format_version = u16::from_le_bytes( + buf[HEADER_FORMAT_VERSION_OFFSET..HEADER_FORMAT_VERSION_OFFSET + 2] + .try_into() + .unwrap(), + ); + assert_eq!( + format_version, CURRENT_VERSION, + "format_version must match CURRENT_VERSION at compression level {level}" + ); + } + let out = roundtrip(&mzml); + let bin = first_spectrum_binary(&out).expect("binary data"); + for (a, b) in bin.to_f64().iter().zip(values.iter()) { + assert_eq!(a.to_bits(), b.to_bits()); + } + } + + #[test] + fn delta_not_applied_without_compression() { + let mz: Vec = (0..10).map(|i| 100.0 + i as f64).collect(); + let intensity = vec![1.0_f64; mz.len()]; + let mzml = build_mzml( + vec![make_spectrum_f64("scan=1", mz.clone(), intensity)], + vec![], + ); + let buf = encode_to_ion(&mzml, 0, false); + let mut decoder = IonReader::from_bytes(&buf, &ReadOptions::default()).unwrap(); + let refs = decoder.spectrum_array_addresses(0).unwrap(); + let mz_address = refs.iter().find(|r| r.array_type() == 1_000_514).unwrap(); + assert_eq!(mz_address.array_filter(), 0, "no delta without compression"); + let mut got = Vec::new(); + decoder.read_spectrum_values(mz_address, &mut got).unwrap(); + for (a, b) in got.iter().zip(mz.iter()) { + assert_eq!(a.to_bits(), b.to_bits()); + } + } + + #[test] + fn delta_shuffle_applied_to_time_array() { + let time: Vec = (0..100).map(|i| i as f64 * 0.1).collect(); + let intensity: Vec = vec![1.0; time.len()]; + let mzml = build_mzml( + vec![], + vec![make_chromatogram_f64("tic", time.clone(), intensity)], + ); + let buf = encode_to_ion(&mzml, 3, false); + let mut decoder = IonReader::from_bytes(&buf, &ReadOptions::default()).unwrap(); + let refs = decoder.chromatogram_array_addresses(0).unwrap(); + let time_ref = refs.iter().find(|r| r.array_type() == 1_000_595).unwrap(); + assert_eq!( + time_ref.array_filter(), + 2, + "time must use DeltaShuffle filter" + ); + let mut got = Vec::new(); + decoder + .read_chromatogram_values(time_ref, &mut got) + .unwrap(); + for (a, b) in got.iter().zip(time.iter()) { + assert_eq!(a.to_bits(), b.to_bits()); + } + } + + #[test] + fn read_chromatogram_array_keeps_native_f32_width_33() { + let time: Vec = (0..64).map(|i| i as f64 * 0.5).collect(); + let intensity: Vec = (0..64).map(|i| i as f64).collect(); + let mzml = build_mzml(vec![], vec![make_chromatogram_f64("tic", time, intensity)]); + let buf = encode_to_ion(&mzml, 3, true); + let mut decoder = IonReader::from_bytes(&buf, &ReadOptions::default()).unwrap(); + let refs = decoder.chromatogram_array_addresses(0).unwrap(); + let intensity_ref = refs.iter().find(|r| r.array_type() == 1_000_515).unwrap(); + let native = decoder.read_chromatogram_array(intensity_ref).unwrap(); + assert!( + matches!(native, NumericArray::F32(_)), + "read_chromatogram_array must keep the stored native f32 width" + ); + let mut widened = Vec::new(); + decoder + .read_chromatogram_values(intensity_ref, &mut widened) + .unwrap(); + assert_eq!( + widened.len(), + 64, + "read_chromatogram_values must still widen to f64" + ); + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(32))] + + #[test] + fn proptest_arbitrary_f64_roundtrip( + values in prop::collection::vec(prop::num::f64::ANY, 1..64) + ) { + let len = values.len(); + let mzml = mzml_with_single_array( + NumericType::Float64, + NumericArray::F64(values.clone()), + len, + ); + let out = roundtrip(&mzml); + let bin = first_spectrum_binary(&out).expect("should have binary data"); + let got = bin.to_f64(); + prop_assert_eq!(got.len(), values.len()); + for (i, (g, e)) in got.iter().zip(values.iter()).enumerate() { + if e.is_nan() { + prop_assert!(g.is_nan(), "index {}: expected NaN", i); + } else { + prop_assert_eq!(g.to_bits(), e.to_bits(), "index {}: bit mismatch", i); + } + } + } + + #[test] + fn proptest_delta2vbyte_mz_roundtrip( + mz in prop::collection::vec(0.0f64..100_000.0, 3..256) + ) { + let mut sorted_mz = mz.clone(); + sorted_mz.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let intensity = vec![1.0f64; sorted_mz.len()]; + let mzml = build_mzml( + vec![make_spectrum_f64("scan=1", sorted_mz.clone(), intensity)], + vec![], + ); + let buf = encode_to_ion(&mzml, 3, false); + let got = decode_ion(&buf).unwrap(); + let bin = first_spectrum_binary(&got).expect("should have binary data"); + let decoded = bin.to_f64(); + prop_assert_eq!(decoded.len(), sorted_mz.len()); + for (i, (g, e)) in decoded.iter().zip(sorted_mz.iter()).enumerate() { + if e.is_nan() { + prop_assert!(g.is_nan(), "index {}: expected NaN", i); + } else { + prop_assert_eq!(g.to_bits(), e.to_bits(), "index {}: bit mismatch", i); + } + } + } + + #[test] + fn proptest_chimp_time_roundtrip( + time in prop::collection::vec(prop::num::f64::ANY, 2..256) + ) { + let intensity = vec![1.0f64; time.len()]; + let mzml = build_mzml( + vec![], + vec![make_chromatogram_f64("tic", time.clone(), intensity)], + ); + let buf = encode_to_ion(&mzml, 3, false); + let got = decode_ion(&buf).unwrap(); + let chrom_list = got.run.chromatogram_list.unwrap(); + let decoded = first_chrom_array_values_by_accession(&chrom_list.chromatograms[0], "MS:1000595"); + prop_assert_eq!(decoded.len(), time.len()); + for (i, (g, e)) in decoded.iter().zip(time.iter()).enumerate() { + if e.is_nan() { + prop_assert!(g.is_nan(), "index {}: expected NaN", i); + } else { + prop_assert_eq!(g.to_bits(), e.to_bits(), "index {}: bit mismatch", i); + } + } + } + + #[test] + fn proptest_arbitrary_i32_roundtrip( + values in prop::collection::vec(prop::num::i32::ANY, 1..64) + ) { + let len = values.len(); + let mzml = mzml_with_single_array( + NumericType::Int32, + NumericArray::I32(values.clone()), + len, + ); + let out = roundtrip(&mzml); + let bin = first_spectrum_binary(&out).expect("should have binary data"); + match bin { + NumericArray::I32(got) => prop_assert_eq!(got, &values), + other => { + let got = other.to_f64(); + let expected: Vec = values.iter().map(|v| *v as f64).collect(); + prop_assert_eq!(got, expected); + } + } + } + + #[test] + fn proptest_arbitrary_i16_roundtrip( + values in prop::collection::vec(prop::num::i16::ANY, 1..64) + ) { + let len = values.len(); + let mzml = mzml_with_single_array( + NumericType::Int16, + NumericArray::I16(values.clone()), + len, + ); + let out = roundtrip(&mzml); + let bin = first_spectrum_binary(&out).expect("should have binary data"); + match bin { + NumericArray::I16(got) => prop_assert_eq!(got, &values), + other => { + let got = other.to_f64(); + let expected: Vec = values.iter().map(|v| *v as f64).collect(); + prop_assert_eq!(got, expected); + } + } + } + } + } } diff --git a/crates/parser/src/ion/decoder/decode/mod.rs b/crates/parser/src/ion/decoder/decode/mod.rs index bf347d79..836c0f25 100644 --- a/crates/parser/src/ion/decoder/decode/mod.rs +++ b/crates/parser/src/ion/decoder/decode/mod.rs @@ -2,6 +2,8 @@ use std::sync::Arc; #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] use crate::ion::decoder::utilities::byte_source::FileSource; +#[cfg(test)] +use crate::ion::decoder::utilities::spectrum_source::ScanSource; use crate::{ accessions::{ FLOAT_16BIT, FLOAT_32BIT, FLOAT_64BIT, INT_16BIT, INT_32BIT, INT_64BIT, format_accession, @@ -41,7 +43,7 @@ use crate::{ parse_header, parse_instrument_list, parse_referenceable_param_group_list, parse_sample_list, parse_scan_settings_list, parse_software_list, parse_spectrum, parse_spectrum_list, - spectrum_source::{ScanSource, ScanSummary, TimeUnit, f16_bits_to_f64}, + spectrum_source::{ScanSummary, TimeUnit, f16_bits_to_f64}, window_directory::WindowDirectory, }, }, @@ -111,15 +113,19 @@ mod spectra; mod to_mzml; mod windows; -pub use arrays::ArrayAddress; +pub(crate) use arrays::ArrayAddress; pub(crate) use arrays::{ ArrayGroup, address_read_params, dtype_stride, group_arrays, parse_array_address, - read_array_addresses_from_buffers, read_group_decoded_bytes, read_scan_arrays, - unfilter_array_bytes, + read_array_addresses_from_buffers, read_group_decoded_bytes, unfilter_array_bytes, }; +#[cfg(test)] +pub(crate) use arrays::read_scan_arrays; pub(crate) use to_mzml::attach_logical_array; pub use to_mzml::{Metadatum, MetadatumValue}; -pub use windows::{DataXY, ItemKind, ItemSlice, Pixel, Query, Select, Window}; +pub use windows::{DataXY, ScanQuery, Select, Window}; +#[cfg(test)] +pub(crate) use windows::{ItemKind, ItemSlice}; +pub(crate) use windows::Pixel; #[derive(Debug, Clone)] pub struct ReadOptions { @@ -152,6 +158,7 @@ pub struct IonReader { pub(crate) header: Header, pub(crate) source: Arc, pub(crate) spec_window_directory: WindowDirectoryCache, + #[allow(dead_code)] pub(crate) chrom_window_directory: WindowDirectoryCache, pub(crate) spec_summary_buf: Arc<[u8]>, pub(crate) chrom_summary_buf: Arc<[u8]>, @@ -164,7 +171,9 @@ pub struct IonReader { pub(crate) chrom_container: Option>, pub(crate) spec_meta_reader: MetaGroupReader, pub(crate) chrom_meta_reader: MetaGroupReader, + #[allow(dead_code)] pub(crate) mz_values: Vec, + #[allow(dead_code)] pub(crate) int_values: Vec, pub(crate) parallel: bool, pub(crate) decompression_limit: DecompressionLimit, @@ -172,20 +181,21 @@ pub struct IonReader { } impl IonReader { - pub fn open(bytes: &[u8], config: ReadOptions) -> IonResult { + pub fn from_bytes(bytes: &[u8], config: &ReadOptions) -> IonResult { let source = Arc::new(BytesSource::new(Arc::from(bytes))) as Arc; - Self::open_source(source, config) + Self::new(source, config) } #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - pub fn open_file(path: &std::path::Path, config: ReadOptions) -> IonResult { + pub fn open(path: impl AsRef, config: &ReadOptions) -> IonResult { + let path = path.as_ref(); let file = std::fs::File::open(path).map_err(|e| IonError::from(e.to_string()))?; let map = unsafe { memmap2::Mmap::map(&file).map_err(|e| IonError::from(e.to_string()))? }; let source = Arc::new(FileSource::new(map)) as Arc; - Self::open_source(source, config) + Self::new(source, config) } - pub fn open_source(source: Arc, config: ReadOptions) -> IonResult { + pub fn new(source: Arc, config: &ReadOptions) -> IonResult { let header_buf = source.read(ByteRange { offset: 0, length: 1024, @@ -500,6 +510,7 @@ impl IonReader { } } + #[cfg(test)] pub(crate) fn ensure_chrom_window_directory(&mut self) { if !matches!(self.chrom_window_directory, WindowDirectoryCache::Unloaded) { return; @@ -547,6 +558,7 @@ impl IonReader { .map_err(|error| IonError::MalformedSpectrumBounds(error.to_string())) } + #[cfg(test)] fn load_chrom_window_directory(&self) -> IonResult { if self.header.len_chrom_window_directory == 0 { return Err(IonError::MissingChromatogramBounds); @@ -595,7 +607,7 @@ impl IonReader { } } -pub fn coalesce_byte_ranges(ranges: &mut Vec, gap: u64) { +pub fn merge_ranges(ranges: &mut Vec, gap: u64) { ranges.retain(|range| range.length > 0); ranges.sort_unstable_by_key(|range| (range.offset, range.length)); let mut kept = 0; @@ -619,9 +631,9 @@ pub fn coalesce_byte_ranges(ranges: &mut Vec, gap: u64) { ranges.truncate(kept); } -pub fn open_ranges(header_bytes: &[u8]) -> IonResult> { +pub fn header_ranges(header_bytes: &[u8]) -> IonResult> { if header_bytes.len() < 1024 { - return Err("open_ranges: needs at least 1024 header bytes".into()); + return Err("header_ranges: needs at least 1024 header bytes".into()); } let header = parse_header(&header_bytes[..1024])?; open_byte_ranges(&header) diff --git a/crates/parser/src/ion/decoder/decode/spectra.rs b/crates/parser/src/ion/decoder/decode/spectra.rs index 7947e647..ec5e8b1b 100644 --- a/crates/parser/src/ion/decoder/decode/spectra.rs +++ b/crates/parser/src/ion/decoder/decode/spectra.rs @@ -90,6 +90,7 @@ pub(crate) fn build_one_chromatogram( )) } +#[cfg(test)] pub(crate) struct ScanIterator<'a, 'd> { pub(crate) summary_chunks: std::slice::ChunksExact<'a, u8>, pub(crate) entry_chunks: std::slice::ChunksExact<'a, u8>, @@ -102,6 +103,7 @@ pub(crate) struct ScanIterator<'a, 'd> { pub(crate) ms_level: u8, } +#[cfg(test)] impl<'a, 'd> ScanIterator<'a, 'd> { #[allow(clippy::too_many_arguments)] pub(crate) fn new( @@ -229,17 +231,6 @@ impl IonReader { .collect()) } - pub fn chromatogram_summary(&self, index: usize) -> Option { - let b = slice_summary( - &self.chrom_summary_buf, - 0, - index, - CHROM_SUMMARY_SIZE, - self.header.chrom_count, - )?; - Some(parse_chrom_summary(b)) - } - pub fn chromatogram_summaries(&self) -> IonResult> { let count = usize::try_from(self.header.chrom_count) .map_err(|_| IonError::from("chrom summary: out of bounds"))?; @@ -295,13 +286,18 @@ impl IonReader { self.chrom_meta_reader.read_item(index as u64) } - pub fn spectrum(&mut self, index: usize) -> IonResult> { + pub fn spectrum(&mut self, index: usize) -> IonResult { if index >= self.header.spectrum_count as usize { - return Ok(None); + return Err(IonError::OutOfRange { + index, + count: self.header.spectrum_count, + }); } let rows = self.spec_meta_reader.read_item(index as u64)?; let Some(mut spectrum) = build_one_spectrum(&rows, index) else { - return Ok(None); + return Err(IonError::from(format!( + "spectrum {index}: metadata is missing the spectrum tag" + ))); }; if let Some(array_addresses) = read_array_addresses_from_buffers( @@ -325,16 +321,21 @@ impl IonReader { } bd_list.count = Some(bd_list.binary_data_arrays.len()); } - Ok(Some(spectrum)) + Ok(spectrum) } - pub fn chromatogram(&mut self, index: usize) -> IonResult> { + pub fn chromatogram(&mut self, index: usize) -> IonResult { if index >= self.header.chrom_count as usize { - return Ok(None); + return Err(IonError::OutOfRange { + index, + count: self.header.chrom_count, + }); } let rows = self.chrom_meta_reader.read_item(index as u64)?; let Some(mut chromatogram) = build_one_chromatogram(&rows, index) else { - return Ok(None); + return Err(IonError::from(format!( + "chromatogram {index}: metadata is missing the chromatogram tag" + ))); }; if let (Some(array_addresses), Some(container)) = ( @@ -361,10 +362,11 @@ impl IonReader { } bd_list.count = Some(bd_list.binary_data_arrays.len()); } - Ok(Some(chromatogram)) + Ok(chromatogram) } } +#[cfg(test)] impl ScanSource for IonReader { fn for_each_summary(&mut self, callback: &mut dyn FnMut(usize, ScanSummary)) { for (index, chunk) in self diff --git a/crates/parser/src/ion/decoder/decode/tests.rs b/crates/parser/src/ion/decoder/decode/tests.rs index 3ddc0eed..dc6766fe 100644 --- a/crates/parser/src/ion/decoder/decode/tests.rs +++ b/crates/parser/src/ion/decoder/decode/tests.rs @@ -68,7 +68,7 @@ fn group_arrays_errors_on_type_mismatch_in_continuation() { #[test] fn new_reader_opens_old_fixture() { - let reader = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let reader = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); assert!(matches!( reader.spec_window_directory, WindowDirectoryCache::Unloaded @@ -77,7 +77,7 @@ fn new_reader_opens_old_fixture() { #[test] fn to_mzml_preserves_declared_spectrum_count() { - let mut reader = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let mut reader = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let mzml = reader.to_mzml().unwrap(); let list = mzml.run.spectrum_list.expect("spectrum list"); assert_eq!(list.count, Some(3476)); @@ -86,7 +86,7 @@ fn to_mzml_preserves_declared_spectrum_count() { #[test] fn to_mzml_keeps_data_type_cv_param_in_place() { - let mut reader = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let mut reader = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let mzml = reader.to_mzml().unwrap(); let spectra = mzml.run.spectrum_list.expect("spectrum list").spectra; let array = &spectra[0] @@ -99,11 +99,11 @@ fn to_mzml_keeps_data_type_cv_param_in_place() { #[test] fn get_spectrum_lazy_matches_full_conversion() { - let mut reader = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let mut reader = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let full = reader.to_mzml().unwrap(); let full_spectra = full.run.spectrum_list.expect("spectrum list").spectra; for (index, full_spectrum) in full_spectra.iter().enumerate() { - let lazy = reader.spectrum(index).unwrap().expect("spectrum present"); + let lazy = reader.spectrum(index).expect("spectrum present"); assert_eq!( format!("{lazy:?}"), format!("{full_spectrum:?}"), @@ -114,7 +114,7 @@ fn get_spectrum_lazy_matches_full_conversion() { #[test] fn metadata_at_matches_filtered_full_read() { - let mut reader = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let mut reader = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let all_spectra = reader.spectrum_metadata().unwrap(); for index in 0..reader.spectrum_count() as usize { @@ -141,19 +141,19 @@ fn metadata_at_matches_filtered_full_read() { #[test] fn open_parses_header() { - let d = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let d = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); assert!(d.spectrum_count() > 0); } #[test] fn summary_returns_none_out_of_bounds() { - let d = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let d = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); assert!(d.spectrum_summary(d.spectrum_count() as usize).is_none()); } #[test] fn summary_has_valid_rt() { - let d = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let d = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let r = d.spectrum_summary(0).unwrap(); assert!(r.rt.is_finite() && r.rt >= 0.0); assert!(r.ms_level >= 1); @@ -161,7 +161,7 @@ fn summary_has_valid_rt() { #[test] fn array_addresses_contain_mz_and_intensity() { - let d = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let d = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let refs = d.spectrum_array_addresses(0).unwrap(); assert!(refs.iter().any(|a| a.array_type == ACC_MZ)); assert!(refs.iter().any(|a| a.array_type == ACC_INT)); @@ -169,7 +169,7 @@ fn array_addresses_contain_mz_and_intensity() { #[test] fn read_array_produces_mz_values() { - let mut d = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let mut d = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let refs = d.spectrum_array_addresses(0).unwrap(); let mz_address = refs.iter().find(|a| a.array_type == ACC_MZ).unwrap(); @@ -182,7 +182,7 @@ fn read_array_produces_mz_values() { #[test] fn for_each_scan_yields_matching_scans() { - let mut d = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let mut d = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let mut count = 0usize; d.for_each_in_range(0.0, f64::MAX, 0, |summary, mz, int| { assert!(summary.rt.is_finite()); @@ -195,7 +195,7 @@ fn for_each_scan_yields_matching_scans() { #[test] fn for_each_scan_filters_by_ms_level() { - let mut d = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let mut d = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let mut count = 0usize; d.for_each_in_range(0.0, f64::MAX, 1, |_, _, _| { count += 1; @@ -209,25 +209,25 @@ fn for_each_scan_filters_by_ms_level() { fn reencoded_fixture() -> Vec { use crate::ion::encoder::{encode::WriteOptions, ion_writer::write_mzml_to_ion}; let mzml = { - let mut reader = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let mut reader = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); reader.to_mzml().unwrap() }; let mut out = Vec::new(); - write_mzml_to_ion(&mzml, WriteOptions::default(), &mut out).unwrap(); + write_mzml_to_ion(&mzml, &WriteOptions::default(), &mut out).unwrap(); out } #[test] fn scans_in_visits_every_scan_and_matches_read_window() { let bytes = reencoded_fixture(); - let mut reader = IonReader::open(&bytes, ReadOptions::default()).unwrap(); + let mut reader = IonReader::from_bytes(&bytes, &ReadOptions::default()).unwrap(); let mz = Range { from: 0.0, to: f64::MAX, }; let mut seen: Vec<(usize, Vec, Vec)> = Vec::new(); reader - .scans_in(mz, Select::All, None, &mut |window| { + .scans_in(&ScanQuery { mz, select: Select::All, ms_level: None }, |window| { seen.push((window.index, window.mz.to_vec(), window.intensity.to_vec())); }) .unwrap(); @@ -242,14 +242,14 @@ fn scans_in_visits_every_scan_and_matches_read_window() { #[test] fn scans_in_ms_level_filter_selects_only_matching_scans() { let bytes = reencoded_fixture(); - let mut reader = IonReader::open(&bytes, ReadOptions::default()).unwrap(); + let mut reader = IonReader::from_bytes(&bytes, &ReadOptions::default()).unwrap(); let mz = Range { from: 0.0, to: f64::MAX, }; let mut count = 0usize; reader - .scans_in(mz, Select::All, Some(1), &mut |_window| { + .scans_in(&ScanQuery { mz, select: Select::All, ms_level: Some(1) }, |_window| { count += 1; }) .unwrap(); @@ -266,7 +266,7 @@ fn scans_in_ms_level_filter_selects_only_matching_scans() { #[test] fn spec_array_addresses_store_cv_code_in_32_byte_records() { let bytes = reencoded_fixture(); - let reader = IonReader::open(&bytes, ReadOptions::default()).unwrap(); + let reader = IonReader::from_bytes(&bytes, &ReadOptions::default()).unwrap(); let table = &reader.spec_array_addresses; assert!( !table.is_empty(), @@ -289,7 +289,7 @@ fn spec_array_addresses_store_cv_code_in_32_byte_records() { #[test] fn to_mzml_produces_valid_structure() { - let mut d = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let mut d = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let mzml = d.to_mzml().unwrap(); let sl = mzml.run.spectrum_list.as_ref().unwrap(); assert!(!sl.spectra.is_empty()); @@ -306,13 +306,13 @@ fn to_mzml_produces_valid_structure() { #[test] fn global_metadata_returns_entries() { - let d = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let d = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); assert!(!d.global_metadata().unwrap().is_empty()); } #[test] fn spectrum_metadata_returns_entries() { - let d = IonReader::open(BYTES, ReadOptions::default()).unwrap(); + let d = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); assert!(!d.spectrum_metadata().unwrap().is_empty()); } @@ -324,7 +324,7 @@ fn custom_config_opens_successfully() { parallel: true, decompression_limit: DecompressionLimit::default(), }; - let d = IonReader::open(BYTES, config).unwrap(); + let d = IonReader::from_bytes(BYTES, &config).unwrap(); assert!(d.spectrum_count() > 0); } @@ -360,10 +360,10 @@ fn dtype_stride_maps_all_types() { #[test] fn open_bytes_gives_same_result_as_open() { let bytes_arc: Arc<[u8]> = Arc::from(BYTES); - let mut d1 = IonReader::open(BYTES, ReadOptions::default()).unwrap(); - let mut d2 = IonReader::open_source( + let mut d1 = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); + let mut d2 = IonReader::new( Arc::new(BytesSource::new(bytes_arc)), - ReadOptions::default(), + &ReadOptions::default(), ) .unwrap(); assert_eq!(d1.spectrum_count(), d2.spectrum_count()); @@ -377,7 +377,7 @@ fn open_source_uses_provided_source() { use crate::ion::decoder::utilities::byte_source::{BytesSource, ReadBytes}; let bytes_arc: Arc<[u8]> = Arc::from(BYTES); let source = Arc::new(BytesSource::new(bytes_arc.clone())) as Arc; - let mut d = IonReader::open_source(source, ReadOptions::default()).unwrap(); + let mut d = IonReader::new(source, &ReadOptions::default()).unwrap(); assert!(d.spectrum_count() > 0); let mzml = d.to_mzml().unwrap(); assert!(!mzml.run.spectrum_list.unwrap().spectra.is_empty()); @@ -469,7 +469,7 @@ fn mixed_normal_and_oversized_spectra_preserve_order_and_data() { let mut encoded = Vec::new(); write_mzml_to_ion( &mzml_in, - WriteOptions { + &WriteOptions { compression_level: 3, force_f32: false, block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, @@ -481,7 +481,7 @@ fn mixed_normal_and_oversized_spectra_preserve_order_and_data() { ) .unwrap(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let mzml_out = decoder.to_mzml().unwrap(); let out_spectra = mzml_out.run.spectrum_list.unwrap().spectra; @@ -588,7 +588,7 @@ fn oversized_array_roundtrips_with_compression_and_parallel() { let mut encoded = Vec::new(); write_mzml_to_ion( &mzml_in, - WriteOptions { + &WriteOptions { compression_level: 3, force_f32: false, block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, @@ -600,7 +600,7 @@ fn oversized_array_roundtrips_with_compression_and_parallel() { ) .unwrap(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let mzml_out = decoder.to_mzml().unwrap(); let spectra = mzml_out.run.spectrum_list.unwrap().spectra; @@ -702,7 +702,7 @@ fn oversized_array_roundtrips_through_encode_decode() { let mut encoded = Vec::new(); write_mzml_to_ion( &mzml_in, - WriteOptions { + &WriteOptions { compression_level: 0, force_f32: false, block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, @@ -714,7 +714,7 @@ fn oversized_array_roundtrips_through_encode_decode() { ) .unwrap(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let mzml_out = decoder.to_mzml().unwrap(); let spectra = mzml_out.run.spectrum_list.unwrap().spectra; @@ -808,7 +808,7 @@ fn encode_one_spectrum_windowed(mz: Vec, int: Vec, mz_window: f64) -> let mut encoded = Vec::new(); write_mzml_to_ion( &mzml_in, - WriteOptions { + &WriteOptions { compression_level: 3, force_f32: false, block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, @@ -858,7 +858,7 @@ fn encode_one_chromatogram_windowed(time: Vec, int: Vec, time_window: let mut encoded = Vec::new(); write_mzml_to_ion( &mzml_in, - WriteOptions { + &WriteOptions { compression_level: 3, force_f32: false, block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, @@ -911,7 +911,7 @@ fn to_mzml_keeps_all_spectra_across_metadata_group_boundaries() { let mut encoded = Vec::new(); write_mzml_to_ion( &mzml_in, - WriteOptions { + &WriteOptions { compression_level: 3, force_f32: false, block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, @@ -921,7 +921,7 @@ fn to_mzml_keeps_all_spectra_across_metadata_group_boundaries() { ) .unwrap(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); assert_eq!(decoder.header.spectrum_count, spectrum_count as u64); let out = decoder.to_mzml().unwrap(); @@ -978,7 +978,7 @@ fn to_mzml_keeps_all_chromatograms_across_metadata_group_boundaries() { let mut encoded = Vec::new(); write_mzml_to_ion( &mzml_in, - WriteOptions { + &WriteOptions { compression_level: 3, force_f32: false, block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, @@ -988,7 +988,7 @@ fn to_mzml_keeps_all_chromatograms_across_metadata_group_boundaries() { ) .unwrap(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); assert_eq!(decoder.header.chrom_count, chromatogram_count as u64); let out = decoder.to_mzml().unwrap(); @@ -1013,7 +1013,7 @@ fn split_mz_array_roundtrips_through_to_mzml() { let encoded = encode_one_spectrum_windowed(mz.clone(), int.clone(), 10.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); assert!( decoder.header.spec_block_count >= 4, "splitting must produce several window blocks, got {}", @@ -1057,8 +1057,8 @@ fn split_mz_array_roundtrips_through_get_spectrum() { let encoded = encode_one_spectrum_windowed(mz.clone(), int.clone(), 10.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); - let spectrum = decoder.spectrum(0).unwrap().unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); + let spectrum = decoder.spectrum(0).unwrap(); let arrays = &spectrum .binary_data_array_list .as_ref() @@ -1102,9 +1102,9 @@ fn read_spectrum_logical_array_joins_split_segments() { let encoded = encode_one_spectrum_windowed(mz.clone(), int.clone(), 10.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let mz_out = decoder - .get_spectrum_array(0, crate::accessions::MZ_ARRAY) + .array(0, crate::accessions::MZ_ARRAY) .unwrap(); assert_eq!(mz_out, mz); } @@ -1116,7 +1116,7 @@ fn centroided_small_arrays_are_encoded_correctly() { let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - IonReader::open(&encoded, ReadOptions::default()).unwrap(); + IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); } #[test] @@ -1127,10 +1127,10 @@ fn split_mz_array_roundtrips_with_disk_staged_bounds() { let encoded = encode_one_spectrum_windowed(mz.clone(), int.clone(), 10.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let mz_out = decoder - .get_spectrum_array(0, crate::accessions::MZ_ARRAY) + .array(0, crate::accessions::MZ_ARRAY) .unwrap(); assert_eq!(mz_out, mz); } @@ -1140,7 +1140,7 @@ fn read_window_handles_fractional_mz_window() { let mz: Vec = (0..2000).map(|i| 100.0 + i as f64 * 0.01).collect(); let int: Vec = (0..2000).map(|i| (i * 3) as f64).collect(); let encoded = encode_one_spectrum_windowed(mz.clone(), int.clone(), 2.5); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); for (low, high) in [(103.0, 107.0), (100.0, 120.0), (115.5, 116.5)] { let got = decoder @@ -1177,7 +1177,7 @@ fn window_fast_path_matches_brute_force() { let int: Vec = (0..n).map(|i| i as f64).collect(); let encoded = encode_one_spectrum_windowed(mz.clone(), int.clone(), 10.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let windows = [ (120.0, 130.0), @@ -1226,7 +1226,7 @@ fn window_errors_when_bounds_missing() { let int: Vec = (0..n).map(|i| i as f64).collect(); let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); decoder.spec_window_directory = WindowDirectoryCache::Missing; assert_eq!( @@ -1247,7 +1247,7 @@ fn window_on_unsplit_array_uses_fallback_and_is_correct() { let int: Vec = (0..10).map(|i| (i * 7) as f64).collect(); let encoded = encode_one_spectrum_windowed(mz.clone(), int.clone(), 0.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let got = decoder .read_window( @@ -1269,7 +1269,7 @@ fn window_out_of_range_index_errors() { let int: Vec = (0..10).map(|i| i as f64).collect(); let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); assert!( decoder .read_window( @@ -1291,9 +1291,9 @@ fn reader_reads_mz_range() { let encoded = encode_one_spectrum_windowed(mz.clone(), int.clone(), 10.0); let bytes_arc: Arc<[u8]> = Arc::from(encoded.as_slice()); - let mut reader = IonReader::open_source( + let mut reader = IonReader::new( Arc::new(BytesSource::new(bytes_arc)), - ReadOptions::default(), + &ReadOptions::default(), ) .unwrap(); let got = reader @@ -1317,7 +1317,7 @@ fn a1_window_directory_pairs_windows_with_mz_and_intensity_refs() { let int: Vec = (0..n).map(|i| i as f64).collect(); let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let _ = decoder .read_window( 0, @@ -1367,13 +1367,13 @@ fn generic_window_matches_read_mz_range() { let int: Vec = (0..n).map(|i| i as f64).collect(); let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let generic = decoder .read_spectrum_window(0, crate::accessions::MZ_ARRAY, ACC_INT, 120.0, 130.0) .unwrap(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let mz_range = decoder .read_window( 0, @@ -1394,7 +1394,7 @@ fn generic_window_missing_accession_returns_empty() { let int: Vec = (0..10).map(|i| i as f64).collect(); let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let got = decoder .read_spectrum_window(0, 99_999_999u32, ACC_INT, 1.0, 2.0) @@ -1425,7 +1425,7 @@ fn directory_crc_roundtrips() { let computed = crc32fast::hash(&encoded[start..end]); assert_eq!(computed, header.spec_directory_crc32); - assert!(IonReader::open(&encoded, ReadOptions::default()).is_ok()); + assert!(IonReader::from_bytes(&encoded, &ReadOptions::default()).is_ok()); } #[test] @@ -1439,7 +1439,7 @@ fn flipped_directory_offset_is_caught_before_any_read() { let (start, _end) = spec_directory_range(&header); encoded[start] ^= 0xFF; - let result = IonReader::open(&encoded, ReadOptions::default()); + let result = IonReader::from_bytes(&encoded, &ReadOptions::default()); assert!(result.is_err()); let message = format!("{}", result.err().unwrap()); assert!(message.contains("directory checksum mismatch")); @@ -1460,7 +1460,7 @@ fn verify_off_skips_directory_check() { verify_checksums: false, ..ReadOptions::default() }; - assert!(IonReader::open(&encoded, config).is_ok()); + assert!(IonReader::from_bytes(&encoded, &config).is_ok()); } #[test] @@ -1472,7 +1472,7 @@ fn empty_container_directory_crc_is_consistent() { let header = parse_header(&encoded[..1024]).unwrap(); assert_eq!(header.chrom_block_count, 0); assert_eq!(header.chrom_directory_crc32, crc32fast::hash(&[])); - assert!(IonReader::open(&encoded, ReadOptions::default()).is_ok()); + assert!(IonReader::from_bytes(&encoded, &ReadOptions::default()).is_ok()); } #[test] @@ -1498,7 +1498,7 @@ fn candidate_items_filters_by_axis_accession() { let int: Vec = (0..n).map(|i| i as f64).collect(); let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let mz_candidates = decoder .candidate_items( @@ -1529,7 +1529,7 @@ fn candidate_items_returns_empty_when_window_directory_missing() { let int: Vec = (0..n).map(|i| i as f64).collect(); let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); decoder.spec_window_directory = WindowDirectoryCache::Missing; @@ -1563,7 +1563,7 @@ fn candidate_items_empty_on_a1_crc_failure() { if a1_offset > 0 { encoded[a1_offset] ^= 0xFF; - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let candidates = decoder .candidate_items( ItemKind::Spectrum, @@ -1602,7 +1602,7 @@ fn ensure_chrom_window_directory_distinguishes_missing_from_bad_checksum() { encoded[b1_offset] ^= 0xFF; - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); decoder.ensure_chrom_window_directory(); assert!( @@ -1614,7 +1614,7 @@ fn ensure_chrom_window_directory_distinguishes_missing_from_bad_checksum() { ); let spectrum_only = encode_one_spectrum_windowed(vec![100.0, 200.0], vec![1.0, 2.0], 10.0); - let mut missing_decoder = IonReader::open(&spectrum_only, ReadOptions::default()).unwrap(); + let mut missing_decoder = IonReader::from_bytes(&spectrum_only, &ReadOptions::default()).unwrap(); missing_decoder.ensure_chrom_window_directory(); assert!( @@ -1656,7 +1656,7 @@ fn candidate_items_for_chrom_axis_without_bounds() { let int: Vec = (0..n).map(|i| i as f64).collect(); let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let candidates = decoder .candidate_items( @@ -1739,14 +1739,14 @@ fn block_ranges_for(decoder: &IonReader, block_ids: &[u32]) -> Vec { #[test] fn require_bounds_passes_on_file_with_a1() { let (_, _, encoded) = split_file_with_a1(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); assert!(decoder.require_bounds().is_ok()); } #[test] fn require_bounds_errors_when_a1_missing() { let (_, _, encoded) = split_file_with_a1(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); decoder.spec_window_directory = WindowDirectoryCache::Missing; assert_eq!( decoder.require_bounds(), @@ -1763,7 +1763,7 @@ fn require_bounds_errors_on_bad_checksum() { assert!(a1_offset > 0); encoded[a1_offset] ^= 0xFF; - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); assert_eq!( decoder.require_bounds(), Err(IonError::BadSpectrumBoundsChecksum) @@ -1773,7 +1773,7 @@ fn require_bounds_errors_on_bad_checksum() { #[test] fn require_bounds_errors_on_malformed_rows() { let (_, _, encoded) = split_file_with_a1(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); decoder.spec_window_directory = WindowDirectoryCache::Malformed("bad rows".to_string()); assert_eq!( decoder.require_bounds(), @@ -1784,7 +1784,7 @@ fn require_bounds_errors_on_malformed_rows() { #[test] fn read_mz_range_matches_brute_force() { let (mz, int, encoded) = split_file_with_a1(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); for (low, high) in [(120.0, 130.0), (100.0, 149.999), (130.5, 130.5)] { let got = decoder @@ -1813,7 +1813,7 @@ fn read_mz_range_matches_brute_force() { #[test] fn read_mz_range_errors_when_a1_missing() { let (_, _, encoded) = split_file_with_a1(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); decoder.spec_window_directory = WindowDirectoryCache::Missing; assert_eq!( decoder.read_window( @@ -1830,7 +1830,7 @@ fn read_mz_range_errors_when_a1_missing() { #[test] fn read_mz_range_errors_when_window_directory_malformed() { let (_, _, encoded) = split_file_with_a1(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); decoder.spec_window_directory = WindowDirectoryCache::Malformed("bad window directory".to_string()); @@ -1847,7 +1847,7 @@ fn read_mz_range_errors_when_window_directory_malformed() { #[test] fn read_mz_range_errors_on_low_above_high() { let (_, _, encoded) = split_file_with_a1(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); assert!( decoder .read_window( @@ -1864,7 +1864,7 @@ fn read_mz_range_errors_on_low_above_high() { #[test] fn read_mz_range_errors_on_non_finite_bounds() { let (_, _, encoded) = split_file_with_a1(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); assert!( decoder .read_window( @@ -1892,7 +1892,7 @@ fn read_mz_range_errors_on_non_finite_bounds() { #[test] fn plan_mz_range_returns_mz_and_intensity_blocks() { let (_, _, encoded) = split_file_with_a1(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let plan = decoder .byte_ranges( @@ -1928,7 +1928,7 @@ fn plan_mz_range_returns_mz_and_intensity_blocks() { #[test] fn plan_and_read_mz_range_use_same_segments() { let (_, _, encoded) = split_file_with_a1(); - let mut decoder = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let plan = decoder .byte_ranges( @@ -1956,7 +1956,7 @@ fn plan_open_ranges_includes_spec_a1() { let header = parse_header(&encoded[..1024]).unwrap(); assert!(header.len_spec_window_directory > 0); - let ranges = open_ranges(&encoded[..1024]).unwrap(); + let ranges = header_ranges(&encoded[..1024]).unwrap(); assert!(ranges.contains(&ByteRange { offset: header.off_spec_window_directory, length: header.len_spec_window_directory, @@ -2063,7 +2063,7 @@ fn encode_spectra_over_windows(rts: &[f64], mz_window: f64) -> Vec { let mut encoded = Vec::new(); write_mzml_to_ion( &mzml_in, - WriteOptions { + &WriteOptions { compression_level: 3, force_f32: false, block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, @@ -2078,7 +2078,7 @@ fn encode_spectra_over_windows(rts: &[f64], mz_window: f64) -> Vec { } fn open_reader(encoded: &[u8]) -> IonReader { - IonReader::open(encoded, ReadOptions::default()).unwrap() + IonReader::from_bytes(encoded, &ReadOptions::default()).unwrap() } #[test] @@ -2298,7 +2298,7 @@ fn coalesce_merges_adjacent_and_respects_gap() { length: 100, }, ]; - coalesce_byte_ranges(&mut overlapping, 0); + merge_ranges(&mut overlapping, 0); assert_eq!( overlapping, vec![ByteRange { @@ -2317,7 +2317,7 @@ fn coalesce_merges_adjacent_and_respects_gap() { length: 100, }, ]; - coalesce_byte_ranges(&mut touching, 0); + merge_ranges(&mut touching, 0); assert_eq!( touching, vec![ByteRange { @@ -2336,7 +2336,7 @@ fn coalesce_merges_adjacent_and_respects_gap() { length: 60, }, ]; - coalesce_byte_ranges(&mut inside_gap, 40); + merge_ranges(&mut inside_gap, 40); assert_eq!( inside_gap, vec![ByteRange { @@ -2355,7 +2355,7 @@ fn coalesce_merges_adjacent_and_respects_gap() { length: 60, }, ]; - coalesce_byte_ranges(&mut beyond_gap, 40); + merge_ranges(&mut beyond_gap, 40); assert_eq!(beyond_gap.len(), 2); let mut contained = vec![ @@ -2368,7 +2368,7 @@ fn coalesce_merges_adjacent_and_respects_gap() { length: 50, }, ]; - coalesce_byte_ranges(&mut contained, 0); + merge_ranges(&mut contained, 0); assert_eq!( contained, vec![ByteRange { @@ -2387,7 +2387,7 @@ fn coalesce_merges_adjacent_and_respects_gap() { length: 10, }, ]; - coalesce_byte_ranges(&mut with_empty, 0); + merge_ranges(&mut with_empty, 0); assert_eq!( with_empty, vec![ByteRange { @@ -2410,7 +2410,7 @@ fn coalesce_merges_adjacent_and_respects_gap() { length: 50, }, ]; - coalesce_byte_ranges(&mut unsorted, 0); + merge_ranges(&mut unsorted, 0); assert_eq!( unsorted, vec![ @@ -2426,7 +2426,7 @@ fn coalesce_merges_adjacent_and_respects_gap() { ); let mut nothing: Vec = Vec::new(); - coalesce_byte_ranges(&mut nothing, 16); + merge_ranges(&mut nothing, 16); assert!(nothing.is_empty()); } @@ -2434,11 +2434,11 @@ fn coalesce_merges_adjacent_and_respects_gap() { fn coalesced_open_ranges_cover_every_open_range() { let (_, _, encoded) = split_file_with_a1(); let header = parse_header(&encoded[..1024]).unwrap(); - let planned = open_ranges(&encoded[..1024]).unwrap(); + let planned = header_ranges(&encoded[..1024]).unwrap(); let alignment_gap = 8; let mut coalesced = planned.clone(); - coalesce_byte_ranges(&mut coalesced, alignment_gap); + merge_ranges(&mut coalesced, alignment_gap); for range in &planned { let range_end = range.offset + range.length; @@ -2466,7 +2466,7 @@ fn plan_open_ranges_includes_container_directories() { let header = parse_header(&encoded[..1024]).unwrap(); let (start, end) = spec_directory_range(&header); - let ranges = open_ranges(&encoded[..1024]).unwrap(); + let ranges = header_ranges(&encoded[..1024]).unwrap(); assert!(ranges.contains(&ByteRange { offset: start as u64, length: (end - start) as u64, @@ -2478,7 +2478,7 @@ fn reader_plans_and_reads_mz_range() { let (mz, int, encoded) = split_file_with_a1(); let bytes: Arc<[u8]> = Arc::from(encoded.as_slice()); let mut reader = - IonReader::open_source(Arc::new(BytesSource::new(bytes)), ReadOptions::default()).unwrap(); + IonReader::new(Arc::new(BytesSource::new(bytes)), &ReadOptions::default()).unwrap(); reader.require_bounds().unwrap(); @@ -2551,7 +2551,7 @@ fn roundtrip_compressed_sections_are_smaller_and_lossless() { let mut encoded = Vec::new(); write_mzml_to_ion( &mzml_in, - WriteOptions { + &WriteOptions { compression_level: 22, ..WriteOptions::default() }, @@ -2563,7 +2563,7 @@ fn roundtrip_compressed_sections_are_smaller_and_lossless() { assert_eq!(header.compression_codec, CODEC_ZSTD); assert_eq!(header.spectrum_count, spectrum_count as u64); - let mut reader = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut reader = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let address_count = reader.spec_array_addresses.len() as u64 / ARRAY_ADDRESS_BYTES as u64; assert_eq!(address_count, spectrum_count as u64 * 2); @@ -2671,7 +2671,7 @@ fn compression_disabled_keeps_sections_raw() { let mut encoded = Vec::new(); write_mzml_to_ion( &mzml_in, - WriteOptions { + &WriteOptions { compression_level: 0, ..WriteOptions::default() }, @@ -2686,7 +2686,7 @@ fn compression_disabled_keeps_sections_raw() { spectrum_count as u64 * SPEC_SUMMARY_SIZE as u64 ); - let mut reader = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut reader = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let mzml_out = reader.to_mzml().unwrap(); assert_eq!( mzml_out.run.spectrum_list.unwrap().spectra.len(), @@ -2721,7 +2721,7 @@ fn compressed_roundtrip_edge_cases() { }; let mut encoded = Vec::new(); - write_mzml_to_ion(&mzml_in, compressed_options, &mut encoded).unwrap(); + write_mzml_to_ion(&mzml_in, &compressed_options, &mut encoded).unwrap(); let header = parse_header(&encoded[..1024]).unwrap(); assert_eq!(header.compression_codec, CODEC_ZSTD); @@ -2729,7 +2729,7 @@ fn compressed_roundtrip_edge_cases() { assert_eq!(header.len_chrom_entries, 0); assert_eq!(header.len_chrom_array_addresses, 0); - let mut reader = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut reader = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let mzml_out = reader.to_mzml().unwrap(); assert_eq!(mzml_out.run.spectrum_list.unwrap().spectra.len(), 1); assert_eq!(reader.chromatogram_count(), 0); @@ -2739,7 +2739,7 @@ fn compressed_roundtrip_edge_cases() { let mzml_in = MzML::default(); let mut encoded = Vec::new(); - write_mzml_to_ion(&mzml_in, compressed_options, &mut encoded).unwrap(); + write_mzml_to_ion(&mzml_in, &compressed_options, &mut encoded).unwrap(); let header = parse_header(&encoded[..1024]).unwrap(); assert_eq!(header.compression_codec, CODEC_ZSTD); @@ -2750,7 +2750,7 @@ fn compressed_roundtrip_edge_cases() { assert_eq!(header.len_chrom_entries, 0); assert_eq!(header.len_chrom_array_addresses, 0); - let reader = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let reader = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); assert_eq!(reader.spectrum_count(), 0); assert_eq!(reader.chromatogram_count(), 0); } @@ -2776,13 +2776,13 @@ fn compressed_roundtrip_edge_cases() { }; let mut encoded = Vec::new(); - write_mzml_to_ion(&mzml_in, compressed_options, &mut encoded).unwrap(); + write_mzml_to_ion(&mzml_in, &compressed_options, &mut encoded).unwrap(); let header = parse_header(&encoded[..1024]).unwrap(); assert_eq!(header.compression_codec, CODEC_ZSTD); assert_eq!(header.len_spec_array_addresses, 0); - let mut reader = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut reader = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let mzml_out = reader.to_mzml().unwrap(); assert_eq!(mzml_out.run.spectrum_list.unwrap().spectra.len(), 1); } @@ -2819,9 +2819,9 @@ fn ms_level_round_trips_without_cvparam_3() { }; let mut encoded = Vec::new(); - write_mzml_to_ion(&mzml_in, WriteOptions::default(), &mut encoded).unwrap(); + write_mzml_to_ion(&mzml_in, &WriteOptions::default(), &mut encoded).unwrap(); - let mut reader = IonReader::open(&encoded, ReadOptions::default()).unwrap(); + let mut reader = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let mzml_out = reader.to_mzml().unwrap(); let decoded_spectrum = mzml_out .run diff --git a/crates/parser/src/ion/decoder/decode/to_mzml.rs b/crates/parser/src/ion/decoder/decode/to_mzml.rs index 0166ab22..e774c0e7 100644 --- a/crates/parser/src/ion/decoder/decode/to_mzml.rs +++ b/crates/parser/src/ion/decoder/decode/to_mzml.rs @@ -6,6 +6,7 @@ use rayon::prelude::*; use super::*; use crate::ion::decoder::utilities::byte_source::SourceBytes; +#[non_exhaustive] #[derive(Debug, Clone, PartialEq)] pub enum MetadatumValue { Number(f64), @@ -13,15 +14,16 @@ pub enum MetadatumValue { Empty, } +#[non_exhaustive] #[derive(Debug, Clone, PartialEq)] pub struct Metadatum { - pub(crate) item_index: u32, - pub(crate) id: u32, - pub(crate) parent_id: u32, - pub(crate) tag_id: TagId, - pub(crate) accession: Option, - pub(crate) unit_accession: Option, - pub(crate) value: MetadatumValue, + pub item_index: u32, + pub id: u32, + pub parent_id: u32, + pub tag_id: TagId, + pub accession: Option, + pub unit_accession: Option, + pub value: MetadatumValue, } pub(crate) struct MzmlConverter<'d> { diff --git a/crates/parser/src/ion/decoder/decode/windows.rs b/crates/parser/src/ion/decoder/decode/windows.rs index baaebea2..f5b2f71c 100644 --- a/crates/parser/src/ion/decoder/decode/windows.rs +++ b/crates/parser/src/ion/decoder/decode/windows.rs @@ -15,6 +15,7 @@ fn window_span(width: f64, from: f64, to: f64, window_count: usize) -> Option<(u )) } +#[non_exhaustive] #[derive(Debug, Clone, PartialEq)] pub struct DataXY { pub x: NumericArray, @@ -42,13 +43,6 @@ impl NumericArray { } } - pub fn as_f64(&self) -> Option<&[f64]> { - match self { - NumericArray::F64(values) => Some(values), - _ => None, - } - } - pub(crate) fn extend_f64(&self, out: &mut Vec) { match self { NumericArray::F64(values) => out.extend_from_slice(values), @@ -62,25 +56,30 @@ impl NumericArray { } #[derive(Debug, Clone, Copy, PartialEq)] -pub struct Pixel { - pub x: Range, - pub y: Range, - pub z: Range, +pub(crate) struct Pixel { + pub(crate) x: Range, + pub(crate) y: Range, + pub(crate) z: Range, } -#[derive(Debug, Clone, Copy, PartialEq)] +#[non_exhaustive] +#[allow(private_interfaces)] +#[derive(Debug, Clone, Copy, PartialEq, Default)] pub enum Select { + #[default] All, Rt(Range), Area(Pixel), } -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct Query { +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub struct ScanQuery { pub mz: Range, - pub rt: Range, + pub select: Select, + pub ms_level: Option, } +#[non_exhaustive] pub struct Window<'a> { pub index: usize, pub summary: &'a ScanSummary, @@ -121,17 +120,19 @@ fn scan_summary_from_record(record: &SpectrumSummary) -> ScanSummary { } } +#[cfg(test)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ItemKind { +pub(crate) enum ItemKind { Spectrum, Chromatogram, } +#[cfg(test)] #[derive(Debug, Clone, Copy)] -pub struct ItemSlice { - pub item_index: u64, - pub array_address_index: u64, - pub intensity_address_index: u64, +pub(crate) struct ItemSlice { + pub(crate) item_index: u64, + pub(crate) array_address_index: u64, + pub(crate) intensity_address_index: u64, } #[derive(Debug, Clone, Copy)] @@ -397,7 +398,7 @@ impl IonReader { for block_id in block_ids { ranges.push(self.spec_block_byte_range(block_id)?); } - coalesce_byte_ranges(&mut ranges, gap); + merge_ranges(&mut ranges, gap); Ok(ranges) } @@ -454,10 +455,8 @@ impl IonReader { pub fn scans_in( &mut self, - mz: Range, - select: Select, - ms_level: Option, - visit: &mut dyn FnMut(&Window), + query: &ScanQuery, + mut visit: impl FnMut(&Window), ) -> IonResult<()> { self.require_bounds()?; let count = self.header.spectrum_count as usize; @@ -467,13 +466,13 @@ impl IonReader { let Some(record) = self.spectrum_summary(index) else { continue; }; - if let Some(level) = ms_level + if let Some(level) = query.ms_level && record.ms_level != level { continue; } let summary = scan_summary_from_record(&record); - if !scan_is_selected(&select, &summary) { + if !scan_is_selected(&query.select, &summary) { continue; } let Some(refs) = self.spectrum_array_addresses(index) else { @@ -486,7 +485,7 @@ impl IonReader { if !has_mz || !has_intensity { continue; } - let data = self.read_window(index, mz)?; + let data = self.read_window(index, query.mz)?; mz_out.clear(); data.x.extend_f64(&mut mz_out); intensity_out.clear(); @@ -577,7 +576,8 @@ impl IonReader { Ok(out) } - pub fn candidate_items( + #[cfg(test)] + pub(crate) fn candidate_items( &mut self, target: ItemKind, axis_accession: u32, diff --git a/crates/parser/src/ion/decoder/utilities/block_reader.rs b/crates/parser/src/ion/decoder/utilities/block_reader.rs index e800a3a8..05020ec7 100644 --- a/crates/parser/src/ion/decoder/utilities/block_reader.rs +++ b/crates/parser/src/ion/decoder/utilities/block_reader.rs @@ -1,4 +1,6 @@ -use std::sync::Arc; +use std::{cell::RefCell, sync::Arc}; + +use cosmoz::{DecompressOptions, Decoder}; use crate::ion::{ ByteRange, IonError, IonResult, @@ -7,13 +9,65 @@ use crate::ion::{ format::CODEC_NONE, packing::PackingId, utilities::{ - common::{decompress_zstd, read_u32_le_at, read_u64_le_at}, + common::{read_u32_le_at, read_u64_le_at}, decompression_limit::DecompressionLimit, }, }; const LRU_NONE: usize = usize::MAX; +thread_local! { + static BLOCK_DECODE_SCRATCH: RefCell<(Decoder, Vec)> = + RefCell::new((Decoder::new(&DecompressOptions::default()), Vec::new())); + static UNSHUFFLE_SCRATCH: RefCell> = const { RefCell::new(Vec::new()) }; +} + +fn unshuffle_into_owned( + processor: &dyn BlockProcessor, + source: &[u8], + stride: Stride, + uncompressed_len: usize, +) -> Vec { + UNSHUFFLE_SCRATCH.with(|scratch| { + let mut scratch = scratch.borrow_mut(); + if scratch.len() < uncompressed_len { + scratch.resize(uncompressed_len, 0); + } + processor.unshuffle(source, &mut scratch[..uncompressed_len], stride.as_usize()); + scratch[..uncompressed_len].to_vec() + }) +} + +fn decompress_zstd_block( + compressed: &[u8], + target_len: usize, + budget: DecompressionLimit, +) -> IonResult> { + if target_len == 0 { + return Ok(Vec::new()); + } + budget.validate(compressed.len(), target_len)?; + + BLOCK_DECODE_SCRATCH.with(|scratch| { + let mut scratch = scratch.borrow_mut(); + let (decoder, buf) = &mut *scratch; + if buf.len() < target_len { + buf.resize(target_len, 0); + } + let actual = decoder + .decompress_into(compressed, &mut buf[..target_len]) + .map_err(|err| IonError::from(format!("zstd decode failed: {err:?}")))?; + + if actual != target_len { + return Err( + format!("zstd: bad decoded size (got={actual}, expected={target_len})").into(), + ); + } + + Ok(buf[..target_len].to_vec()) + }) +} + pub(crate) trait BlockProcessor { fn decompress( &self, @@ -47,7 +101,7 @@ impl BlockProcessor for DefaultBlockProcessor { target_len: usize, budget: DecompressionLimit, ) -> IonResult> { - decompress_zstd(compressed, target_len, budget) + decompress_zstd_block(compressed, target_len, budget) } #[inline] @@ -216,9 +270,8 @@ impl BlockReader

{ if !needs_unshuffle { return Ok(payload); } - let mut scratch = vec![0u8; uncompressed_len]; - self.processor - .unshuffle(&payload, &mut scratch, stride.as_usize()); + let scratch = + unshuffle_into_owned(&self.processor, &payload, stride, uncompressed_len); return Ok(SourceBytes::Owned(scratch)); } @@ -230,9 +283,8 @@ impl BlockReader

{ return Ok(SourceBytes::Owned(decompressed)); } - let mut scratch = vec![0u8; uncompressed_len]; - self.processor - .unshuffle(&decompressed, &mut scratch, stride.as_usize()); + let scratch = + unshuffle_into_owned(&self.processor, &decompressed, stride, uncompressed_len); Ok(SourceBytes::Owned(scratch)) } diff --git a/crates/parser/src/ion/decoder/utilities/common.rs b/crates/parser/src/ion/decoder/utilities/common.rs index 0b65fc91..afdaa4c2 100644 --- a/crates/parser/src/ion/decoder/utilities/common.rs +++ b/crates/parser/src/ion/decoder/utilities/common.rs @@ -1,4 +1,4 @@ -use cosmoz::{DecompressOptions, Decoder, compressed_size, decompress_into}; +use cosmoz::{DecompressOptions, Decoder, compressed_size}; use crate::ion::{ IonError, IonResult, @@ -92,8 +92,9 @@ pub(crate) fn decompress_zstd( budget.validate(comp.len(), expected)?; let mut out = vec![0u8; expected]; - let mut decoder = Decoder::new(); - let actual = decompress_into(comp, &mut out, &DecompressOptions::default(), &mut decoder) + let mut decoder = Decoder::new(&DecompressOptions::default()); + let actual = decoder + .decompress_into(comp, &mut out) .map_err(|err| IonError::from(format!("zstd decode failed: {err:?}")))?; if actual != expected { @@ -231,7 +232,7 @@ pub(crate) fn parse_accession_tail(accession: Option<&str>) -> AccessionTail { #[cfg(test)] mod tests { - use cosmoz::{CompressOptions, Encoder, compress_into, max_compressed_size}; + use cosmoz::{CompressOptions, Encoder}; use super::*; use crate::ion::decoder::utilities::decompression_limit::DecompressionLimit; @@ -239,8 +240,8 @@ mod tests { fn compress_to_frame(data: &[u8]) -> Vec { let options = CompressOptions::default(); let mut encoder = Encoder::new(&options).unwrap(); - let mut out = vec![0u8; max_compressed_size(data.len(), &options)]; - let written = compress_into(data, &mut out, &options, &mut encoder).unwrap(); + let mut out = vec![0u8; encoder.max_compressed_size(data.len())]; + let written = encoder.compress_into(data, &mut out).unwrap(); out.truncate(written); out } diff --git a/crates/parser/src/ion/decoder/utilities/spectrum_source.rs b/crates/parser/src/ion/decoder/utilities/spectrum_source.rs index 88c15a20..a5cb962d 100644 --- a/crates/parser/src/ion/decoder/utilities/spectrum_source.rs +++ b/crates/parser/src/ion/decoder/utilities/spectrum_source.rs @@ -9,6 +9,7 @@ fn acc(s: Option<&str>) -> u32 { parse_accession_tail(s).raw() } +#[non_exhaustive] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TimeUnit { Second, @@ -37,6 +38,7 @@ impl TimeUnit { } } +#[non_exhaustive] #[derive(Debug, Clone, Copy)] pub struct ScanSummary { pub rt: f64, @@ -52,6 +54,7 @@ pub struct ScanSummary { pub position_z: u32, } +#[cfg(test)] pub trait ScanSource { fn for_each_summary(&mut self, callback: &mut dyn FnMut(usize, ScanSummary)); fn load_scan(&mut self, index: usize, mz: &mut Vec, intensity: &mut Vec) -> bool; @@ -80,6 +83,7 @@ pub trait ScanSource { } } +#[cfg(test)] impl ScanSource for MzML { fn for_each_summary(&mut self, callback: &mut dyn FnMut(usize, ScanSummary)) { let Some(list) = self.run.spectrum_list.as_ref() else { @@ -225,6 +229,7 @@ pub(crate) fn summary_from_spectrum(spectrum: &Spectrum) -> ScanSummary { } } +#[cfg(test)] pub(crate) fn summary_from_spectra( spectra: &[Spectrum], callback: &mut dyn FnMut(usize, ScanSummary), @@ -273,6 +278,7 @@ fn parse_u32(s: Option<&str>) -> u32 { s.and_then(|v| v.parse().ok()).unwrap_or(0) } +#[cfg(test)] pub(crate) fn binary_pair(spectrum: &Spectrum) -> Option<(&NumericArray, &NumericArray)> { let list = spectrum.binary_data_array_list.as_ref()?; let mut mz = None; @@ -303,6 +309,7 @@ pub(crate) fn binary_pair(spectrum: &Spectrum) -> Option<(&NumericArray, &Numeri Some((mz?, intensity?)) } +#[cfg(test)] pub(crate) fn load_scan_from_spectra( spectra: &[Spectrum], index: usize, @@ -328,6 +335,7 @@ pub(crate) fn load_scan_from_spectra( true } +#[cfg(test)] fn extend_from_binary(data: &NumericArray, out: &mut Vec, max: usize) { match data { NumericArray::F64(v) => out.extend_from_slice(&v[..max]), diff --git a/crates/parser/src/ion/encoder/encode.rs b/crates/parser/src/ion/encoder/encode.rs index 23f9e8d2..462f0b58 100644 --- a/crates/parser/src/ion/encoder/encode.rs +++ b/crates/parser/src/ion/encoder/encode.rs @@ -17,7 +17,7 @@ use crate::{ ArrayPolicy, array_type_accession_from_binary_data_array, array_type_cv_code_from_binary_data_array, parse_accession_tail_raw, }, - output::SectionStorage, + output::{SectionStorage, WriteBytes}, }, filter_summary::{ChromatogramSummary, SpectrumSummary}, packing::{Dtype, Packing, PackingId, PackingInput, packing_for, raw::RAW as RAW_PACKING}, @@ -26,7 +26,7 @@ use crate::{ }, mzml::structs::{BinaryDataArray, Chromatogram, CvParam, NumericArray, NumericType, Spectrum}, }; -pub const TARGET_BLOCK_UNCOMPRESSED_BYTES: usize = 1024 * 1024; +pub(crate) const TARGET_BLOCK_UNCOMPRESSED_BYTES: usize = 1024 * 1024; pub const DEFAULT_MZ_WINDOW: f64 = 20.0; pub(crate) const SPEC_SUMMARY_SIZE: usize = 80; pub(crate) const CHROM_SUMMARY_SIZE: usize = 80; @@ -495,12 +495,13 @@ pub(crate) struct EncodedArrayAddress { } fn encode_variable_length_array( + output: &mut dyn WriteBytes, array_type: u32, data: ArrayData<'_>, dtype: u8, dtype_enum: Dtype, packing: &'static dyn Packing, - container: &mut BlockWriter<'_, DefaultCompressor>, + container: &mut BlockWriter, ) -> IonResult<(u32, u64, u32)> { let mut encoded = Vec::new(); match packing.id() { @@ -509,7 +510,7 @@ fn encode_variable_length_array( } let enc_len = u32::try_from(encoded.len()).map_err(|_| IonError::from("encoded array exceeds 4 GiB"))?; - let (bid, eoff) = container.add_item_to_box(array_type, encoded.len(), 1, |buf| { + let (bid, eoff) = container.add_item_to_box(output, array_type, encoded.len(), 1, |buf| { buf.extend_from_slice(&encoded); Ok(()) })?; @@ -532,16 +533,19 @@ fn write_fixed_array_payload( } } +#[allow(clippy::too_many_arguments)] fn encode_fixed_length_array( + output: &mut dyn WriteBytes, array_type: u32, data: ArrayData<'_>, dtype: u8, dtype_enum: Dtype, elem_bytes: usize, packing: &'static dyn Packing, - container: &mut BlockWriter<'_, DefaultCompressor>, + container: &mut BlockWriter, ) -> IonResult<(u32, u64, u32)> { let (bid, eoff) = container.add_item_to_box( + output, array_type, data.element_count() * elem_bytes, elem_bytes, @@ -599,10 +603,11 @@ fn resolve_array_encoding<'a>( } pub(crate) fn encode_single_array( + output: &mut dyn WriteBytes, bda: &BinaryDataArray, config: WriteOptions, policy: ArrayPolicy, - container: &mut BlockWriter<'_, DefaultCompressor>, + container: &mut BlockWriter, ) -> IonResult> { let Some(encoding) = resolve_array_encoding(bda, config, policy)? else { return Ok(None); @@ -617,10 +622,10 @@ pub(crate) fn encode_single_array( packing, } = encoding; let (block_id, element_offset, encoded_len) = if packing.is_variable_length() { - encode_variable_length_array(accession, data, dtype, dtype_enum, packing, container)? + encode_variable_length_array(output, accession, data, dtype, dtype_enum, packing, container)? } else { encode_fixed_length_array( - accession, data, dtype, dtype_enum, elem_bytes, packing, container, + output, accession, data, dtype, dtype_enum, elem_bytes, packing, container, )? }; Ok(Some(EncodedArrayAddress { @@ -741,11 +746,12 @@ pub(crate) fn window_ranges_for_item( } pub(crate) fn write_array_windows( + output: &mut dyn WriteBytes, bda: &BinaryDataArray, config: WriteOptions, policy: ArrayPolicy, windows: &[WindowRange], - container: &mut BlockWriter<'_, DefaultCompressor>, + container: &mut BlockWriter, ) -> IonResult> { let Some(encoding) = resolve_array_encoding(bda, config, policy)? else { return Ok(Vec::new()); @@ -769,6 +775,7 @@ pub(crate) fn write_array_windows( let segment_data = data.slice(window.start, window.end); let element_count = window.element_count(); let (block_id, element_offset) = container.add_item_to_window( + output, accession, window.window_index, element_count * elem_bytes, @@ -884,7 +891,7 @@ mod tests { let mut buf = Vec::new(); write_mzml_to_ion( &mzml, - WriteOptions { + &WriteOptions { compression_level: 0, force_f32: false, ..Default::default() @@ -903,7 +910,7 @@ mod tests { let mut buf = Vec::new(); write_mzml_to_ion( &mzml, - WriteOptions { + &WriteOptions { compression_level: 0, force_f32: false, ..Default::default() @@ -925,7 +932,7 @@ mod tests { let mut buf = Vec::new(); write_mzml_to_ion( &mzml, - WriteOptions { + &WriteOptions { compression_level: 0, force_f32: false, ..Default::default() @@ -947,7 +954,7 @@ mod tests { let mut buf = Vec::new(); write_mzml_to_ion( &mzml, - WriteOptions { + &WriteOptions { compression_level: 0, force_f32: false, ..Default::default() @@ -969,7 +976,7 @@ mod tests { let mut buf = Vec::new(); write_mzml_to_ion( &mzml, - WriteOptions { + &WriteOptions { compression_level: 0, force_f32: false, ..Default::default() @@ -987,7 +994,7 @@ mod tests { let mut buf = Vec::new(); write_mzml_to_ion( &mzml, - WriteOptions { + &WriteOptions { compression_level: 0, force_f32: false, ..Default::default() diff --git a/crates/parser/src/ion/encoder/ion_writer.rs b/crates/parser/src/ion/encoder/ion_writer.rs index ef009c5e..f8033473 100644 --- a/crates/parser/src/ion/encoder/ion_writer.rs +++ b/crates/parser/src/ion/encoder/ion_writer.rs @@ -1,4 +1,4 @@ -use std::borrow::Borrow; +use cosmoz::Encoder; pub(crate) struct SectionPlacement { pub(crate) offset: u64, @@ -10,7 +10,7 @@ pub(crate) struct SectionPlacement { use crate::{ accessions::{INTENSITY_ARRAY, MZ_ARRAY, TIME_ARRAY}, ion::{ - IonResult, + IonError, IonResult, encoder::{ encode::{ CHROM_SUMMARY_SIZE, EncodedArrayAddress, SPEC_SUMMARY_SIZE, WriteOptions, @@ -20,11 +20,11 @@ use crate::{ }, scan_stream::ScanStream, utilities::{ - BlockWriter, DefaultCompressor, SectionChunk, + BlockWriter, ContainerSummary, DefaultCompressor, SectionChunk, meta_collector::{ ArrayPolicy, GroupedSection, LOCAL_LIST_NODE_ID, MetaCollector, MetaGrouper, MzmlListItem, array_type_accession_from_binary_data_array, - compress_bytes_if_enabled, serialize_global_meta_with_counts, + compress_bytes_if_enabled, new_meta_encoder, serialize_global_meta_with_counts, }, output::{SectionStorage, WriteBytes}, tables::{ @@ -39,7 +39,10 @@ use crate::{ utilities::EmitAttributes, windowing::WindowRange, }, - mzml::structs::{BinaryDataArray, BinaryDataArrayList, Chromatogram, MzML, Spectrum}, + mzml::structs::{ + BinaryDataArray, BinaryDataArrayList, Chromatogram, ChromatogramList, MzML, Spectrum, + SpectrumList, + }, }; fn spec_summary_bytes(spec: &Spectrum) -> [u8; SPEC_SUMMARY_SIZE] { @@ -100,13 +103,14 @@ impl ArrayWriteState<'_> { } fn write_single_array( + output: &mut dyn WriteBytes, bda: &BinaryDataArray, config: WriteOptions, policy: ArrayPolicy, - container: &mut BlockWriter<'_, DefaultCompressor>, + container: &mut BlockWriter, state: &mut ArrayWriteState<'_>, ) -> IonResult<()> { - let Some(address) = encode_single_array(bda, config, policy, container)? else { + let Some(address) = encode_single_array(output, bda, config, policy, container)? else { return Ok(()); }; state.emit(&address)?; @@ -114,14 +118,15 @@ fn write_single_array( } fn write_windowed_array( + output: &mut dyn WriteBytes, bda: &BinaryDataArray, config: WriteOptions, policy: ArrayPolicy, windows: &[WindowRange], - container: &mut BlockWriter<'_, DefaultCompressor>, + container: &mut BlockWriter, state: &mut ArrayWriteState<'_>, ) -> IonResult<()> { - let addresses = write_array_windows(bda, config, policy, windows, container)?; + let addresses = write_array_windows(output, bda, config, policy, windows, container)?; for address in &addresses { state.emit(address)?; } @@ -130,12 +135,13 @@ fn write_windowed_array( #[allow(clippy::too_many_arguments)] fn encode_arrays_for( + output: &mut dyn WriteBytes, item: &T, spectrum_index: u32, config: WriteOptions, policy: ArrayPolicy, windowable: bool, - container: &mut BlockWriter<'_, DefaultCompressor>, + container: &mut BlockWriter, index: &mut IndexTable, state: &mut ArrayWriteState<'_>, window_directory: &mut WindowDirectory, @@ -160,9 +166,9 @@ where let address_start = *state.cursor; match windows.as_ref() { Some(windows) => { - write_windowed_array(bda, config, policy, windows, container, state)? + write_windowed_array(output, bda, config, policy, windows, container, state)? } - None => write_single_array(bda, config, policy, container, state)?, + None => write_single_array(output, bda, config, policy, container, state)?, } if *state.cursor > address_start { if accession == policy.x_array_accession { @@ -222,7 +228,7 @@ fn push_window_entries( } } -trait HasArrayList { +pub(crate) trait HasArrayList { fn array_list(&self) -> Option<&BinaryDataArrayList>; } @@ -248,10 +254,9 @@ struct ItemStream { count: usize, address_cursor: u64, seen: Vec, - container_offset: u64, - block_count: u64, - container_total: u64, - directory_crc32: u32, + container: BlockWriter, + container_offset: Option, + container_summary: Option, } struct StreamParts { @@ -283,6 +288,17 @@ impl ItemStream { } Ok(SectionChunk::memory(capacity)) }; + + let compressor = config.compression_mode()?; + let builder = BlockWriter::new(config.block_size, compressor, config.block_packing_id()); + #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] + let builder = builder.window_major()?; + let container = if config.parallel { + builder + } else { + builder.force_sequential() + }; + Ok(Self { summary: SummaryTable::new(table(summary_hint * summary_size)?), index: IndexTable::new(table(table_hint * 16)?), @@ -291,47 +307,78 @@ impl ItemStream { METADATA_GROUP_SIZE, config.compression_level, SectionChunk::memory(0), - ), + )?, window_directory: WindowDirectory::new(), windowable, count: 0, address_cursor: 0, seen: Vec::with_capacity(8), - container_offset: 0, - block_count: 0, - container_total: 0, - directory_crc32: 0, + container, + container_offset: None, + container_summary: None, }) } + #[cfg(test)] + fn pending_bytes(&self) -> usize { + self.container.pending_bytes() + } + + #[cfg(test)] + fn set_max_pending_bytes(&mut self, value: usize) { + self.container.set_max_pending_bytes(value); + } + + fn is_sealed(&self) -> bool { + self.container_summary.is_some() + } + + fn ensure_container_offset(&mut self, output: &mut dyn WriteBytes) -> IonResult<()> { + if self.container_offset.is_none() { + self.container_offset = Some(write_aligned(output, &[])?); + } + Ok(()) + } + + fn seal_container(&mut self, output: &mut dyn WriteBytes) -> IonResult<()> { + if self.container_summary.is_some() { + return Ok(()); + } + self.ensure_container_offset(output)?; + self.container_summary = Some(self.container.finish(output)?); + Ok(()) + } + #[allow(clippy::too_many_arguments)] fn add( &mut self, + output: &mut dyn WriteBytes, item: &T, config: WriteOptions, policy: ArrayPolicy, list_id: u32, list_schema: Option<&L>, collector: &mut MetaCollector, - container: &mut BlockWriter<'_, DefaultCompressor>, summary: &[u8], ) -> IonResult<()> where T: HasArrayList + MzmlListItem, L: EmitAttributes, { + self.ensure_container_offset(output)?; let mut state = ArrayWriteState { addresses: &mut self.addresses, cursor: &mut self.address_cursor, seen: &mut self.seen, }; encode_arrays_for( + output, item, self.count as u32, config, policy, self.windowable, - container, + &mut self.container, &mut self.index, &mut state, &mut self.window_directory, @@ -349,7 +396,11 @@ impl ItemStream { Ok(()) } - fn finish(self) -> IonResult { + fn finish(mut self, output: &mut dyn WriteBytes) -> IonResult { + self.seal_container(output)?; + let summary = self + .container_summary + .expect("seal_container always sets container_summary"); Ok(StreamParts { grouped: self.grouper.finish()?, summary: self.summary.finish(), @@ -358,10 +409,10 @@ impl ItemStream { window_directory: self.window_directory.finish()?, count: self.count, type_count: self.seen.len(), - container_offset: self.container_offset, - block_count: self.block_count, - container_total: self.container_total, - directory_crc32: self.directory_crc32, + container_offset: self.container_offset.unwrap_or(0), + block_count: summary.block_count as u64, + container_total: summary.total_bytes, + directory_crc32: summary.directory_crc32, }) } } @@ -377,6 +428,7 @@ fn write_chunk_maybe_compressed( output: &mut dyn WriteBytes, section: SectionChunk, compression_level: u8, + encoder: &mut Encoder, ) -> IonResult<(u64, u64, u32)> { #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] if section.is_spilled() { @@ -387,181 +439,164 @@ fn write_chunk_maybe_compressed( let offset = write_aligned(output, &raw)?; return Ok((offset, 0, crc32fast::hash(&raw))); } - let stored = compress_bytes_if_enabled(raw, compression_level); + let stored = compress_bytes_if_enabled(raw, compression_level, encoder); let crc32 = crc32fast::hash(&stored); let offset = write_aligned(output, &stored)?; Ok((offset, stored.len() as u64, crc32)) } -#[allow(clippy::too_many_arguments)] -fn write_list( - output: &mut dyn WriteBytes, - config: WriteOptions, - policy: ArrayPolicy, - list_id: u32, - list_schema: Option<&L>, - collector: &mut MetaCollector, - stream: &mut ItemStream, - items: I, - summary_of: fn(&T) -> [u8; SPEC_SUMMARY_SIZE], -) -> IonResult<()> -where - T: HasArrayList + MzmlListItem, - L: EmitAttributes, - B: Borrow, - I: Iterator>, -{ - stream.container_offset = write_aligned(output, &[])?; - let compressor = config.compression_mode()?; - let builder = BlockWriter::new( - output, - config.block_size, - compressor, - config.block_packing_id(), - ); - #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - let builder = builder.window_major()?; - let mut container = if config.parallel { - builder - } else { - builder.force_sequential() - }; - - let mut last_rt = f64::NEG_INFINITY; - let is_spectrum_stream = policy.x_array_accession == MZ_ARRAY; - for item in items { - let item = item?; - let item = item.borrow(); - if is_spectrum_stream && let Some(list) = item.array_list() { - check_spectrum_mz_order(&list.binary_data_arrays, stream.count)?; - } - let summary = summary_of(item); - if is_spectrum_stream { - check_spectrum_rt_order(&summary, stream.count, &mut last_rt)?; - } - stream.add( - item, - config, - policy, - list_id, - list_schema, - collector, - &mut container, - &summary, - )?; +fn schema_of_spectrum_list(list: &SpectrumList) -> SpectrumList { + SpectrumList { + count: list.count, + default_data_processing_ref: list.default_data_processing_ref.clone(), + spectra: Vec::new(), } +} - let summary = container.finish()?; - stream.block_count = summary.block_count as u64; - stream.container_total = summary.total_bytes; - stream.directory_crc32 = summary.directory_crc32; - Ok(()) +fn schema_of_chromatogram_list(list: &ChromatogramList) -> ChromatogramList { + ChromatogramList { + count: list.count, + default_data_processing_ref: list.default_data_processing_ref.clone(), + chromatograms: Vec::new(), + } } -pub struct IonWriter<'out> { - output: &'out mut dyn WriteBytes, +pub(crate) struct IonWriter { config: WriteOptions, + compressor: Encoder, collector: MetaCollector, spec_list_id: u32, chrom_list_id: u32, spec_stream: ItemStream, chrom_stream: ItemStream, + spec_schema: Option, + chrom_schema: Option, + last_spec_rt: f64, } -impl<'out> IonWriter<'out> { - pub fn create(output: &'out mut dyn WriteBytes, config: WriteOptions) -> IonResult { +impl IonWriter { + pub(crate) fn begin( + output: &mut dyn WriteBytes, + metadata: &MzML, + config: &WriteOptions, + ) -> IonResult { + let config = *config; allow_compression_level(config.compression_level)?; output.write(&[0u8; HEADER_SIZE])?; - let collector = MetaCollector::new(); - let spec_list_id = LOCAL_LIST_NODE_ID; - let chrom_list_id = LOCAL_LIST_NODE_ID; + let compressor = new_meta_encoder(config.compression_level)?; - Ok(Self { - output, + let mut writer = Self { config, - collector, - spec_list_id, - chrom_list_id, + compressor, + collector: MetaCollector::new(), + spec_list_id: LOCAL_LIST_NODE_ID, + chrom_list_id: LOCAL_LIST_NODE_ID, spec_stream: ItemStream::new(256, SPEC_SUMMARY_SIZE, 256, config, true)?, chrom_stream: ItemStream::new(32, CHROM_SUMMARY_SIZE, 32, config, false)?, - }) + spec_schema: None, + chrom_schema: None, + last_spec_rt: f64::NEG_INFINITY, + }; + writer.set_metadata(metadata); + Ok(writer) } - pub fn write_mzml(&mut self, mzml: &MzML) -> IonResult<()> { - let spectra = mzml - .run - .spectrum_list - .as_ref() - .map_or(&[][..], |list| &list.spectra); - let chroms = mzml + fn set_metadata(&mut self, metadata: &MzML) { + self.spec_schema = metadata.run.spectrum_list.as_ref().map(schema_of_spectrum_list); + self.chrom_schema = metadata .run .chromatogram_list .as_ref() - .map_or(&[][..], |list| &list.chromatograms); + .map(schema_of_chromatogram_list); + } - write_list( - self.output, + pub(crate) fn push_spectrum( + &mut self, + output: &mut dyn WriteBytes, + spectrum: &Spectrum, + ) -> IonResult<()> { + if self.spec_stream.is_sealed() { + return Err(IonError::from( + "cannot push a spectrum after a chromatogram has been written", + )); + } + if let Some(list) = spectrum.array_list() { + check_spectrum_mz_order(&list.binary_data_arrays, self.spec_stream.count)?; + } + let summary = spec_summary_bytes(spectrum); + check_spectrum_rt_order(&summary, self.spec_stream.count, &mut self.last_spec_rt)?; + self.spec_stream.add( + output, + spectrum, self.config, self.config.array_policy(MZ_ARRAY), self.spec_list_id, - mzml.run.spectrum_list.as_ref(), + self.spec_schema.as_ref(), &mut self.collector, - &mut self.spec_stream, - spectra.iter().map(Ok), - spec_summary_bytes, - )?; - write_list( - self.output, + &summary, + ) + } + + pub(crate) fn push_chromatogram( + &mut self, + output: &mut dyn WriteBytes, + chromatogram: &Chromatogram, + ) -> IonResult<()> { + self.spec_stream.seal_container(output)?; + let summary = chrom_summary_bytes(chromatogram); + self.chrom_stream.add( + output, + chromatogram, self.config, self.config.array_policy(TIME_ARRAY), self.chrom_list_id, - mzml.run.chromatogram_list.as_ref(), + self.chrom_schema.as_ref(), &mut self.collector, - &mut self.chrom_stream, - chroms.iter().map(Ok), - chrom_summary_bytes, - )?; + &summary, + ) + } - self.finish_inner(mzml) + pub(crate) fn write_mzml(&mut self, output: &mut dyn WriteBytes, mzml: &MzML) -> IonResult<()> { + if let Some(list) = &mzml.run.spectrum_list { + for spectrum in &list.spectra { + self.push_spectrum(output, spectrum)?; + } + } + if let Some(list) = &mzml.run.chromatogram_list { + for chromatogram in &list.chromatograms { + self.push_chromatogram(output, chromatogram)?; + } + } + Ok(()) } - fn write_reader(&mut self, scans: &mut dyn ScanStream) -> IonResult<()> { + pub(crate) fn write_stream( + &mut self, + output: &mut dyn WriteBytes, + scans: &mut dyn ScanStream, + ) -> IonResult<()> { let metadata = scans.metadata()?; - write_list( - self.output, - self.config, - self.config.array_policy(MZ_ARRAY), - self.spec_list_id, - metadata.run.spectrum_list.as_ref(), - &mut self.collector, - &mut self.spec_stream, - std::iter::from_fn(|| scans.next_spectrum().transpose()), - spec_summary_bytes, - )?; + self.set_metadata(&metadata); + while let Some(spectrum) = scans.next_spectrum()? { + self.push_spectrum(output, &spectrum)?; + } let metadata = scans.metadata()?; - write_list( - self.output, - self.config, - self.config.array_policy(TIME_ARRAY), - self.chrom_list_id, - metadata.run.chromatogram_list.as_ref(), - &mut self.collector, - &mut self.chrom_stream, - std::iter::from_fn(|| scans.next_chromatogram().transpose()), - chrom_summary_bytes, - )?; + self.set_metadata(&metadata); + while let Some(chromatogram) = scans.next_chromatogram()? { + self.push_chromatogram(output, &chromatogram)?; + } let metadata = scans.metadata()?; - self.finish_inner(&metadata) + self.finish(output, &metadata) } - pub fn write_stream(&mut self, scans: &mut dyn ScanStream) -> IonResult<()> { - self.write_reader(scans) - } - - fn write_window_directory(&mut self, bounds: SectionChunk) -> IonResult { + fn write_window_directory( + &mut self, + output: &mut dyn WriteBytes, + bounds: SectionChunk, + ) -> IonResult { let raw = bounds.into_vec()?; let plain_len = raw.len() as u64; if raw.is_empty() { @@ -572,9 +607,10 @@ impl<'out> IonWriter<'out> { crc32: crc32fast::hash(&[]), }); } - let stored = compress_bytes_if_enabled(raw, self.config.compression_level); + let stored = + compress_bytes_if_enabled(raw, self.config.compression_level, &mut self.compressor); let crc32 = crc32fast::hash(&stored); - let offset = write_aligned(self.output, &stored)?; + let offset = write_aligned(output, &stored)?; let length = stored.len() as u64; Ok(SectionPlacement { offset, @@ -584,22 +620,23 @@ impl<'out> IonWriter<'out> { }) } - fn finish_inner(&mut self, mzml: &MzML) -> IonResult<()> { + pub(crate) fn finish(&mut self, output: &mut dyn WriteBytes, mzml: &MzML) -> IonResult<()> { let (global_meta, global_counts) = self.collector.collect_global_meta(mzml); let raw_global = serialize_global_meta_with_counts(&global_counts, &global_meta)?; let global_uncompressed = raw_global.len() as u64; - let global_bytes = compress_bytes_if_enabled(raw_global, self.config.compression_level); + let global_bytes = + compress_bytes_if_enabled(raw_global, self.config.compression_level, &mut self.compressor); let spec = std::mem::replace( &mut self.spec_stream, - ItemStream::new(256, SPEC_SUMMARY_SIZE, 256, self.config, true)?, + ItemStream::new(1, SPEC_SUMMARY_SIZE, 1, self.config, true)?, ) - .finish()?; + .finish(output)?; let chrom = std::mem::replace( &mut self.chrom_stream, - ItemStream::new(32, CHROM_SUMMARY_SIZE, 32, self.config, false)?, + ItemStream::new(1, CHROM_SUMMARY_SIZE, 1, self.config, false)?, ) - .finish()?; + .finish(output)?; let spec_meta_crc32 = spec.grouped.crc32; let chrom_meta_crc32 = chrom.grouped.crc32; @@ -611,29 +648,59 @@ impl<'out> IonWriter<'out> { let compression_level = self.config.compression_level; let (off_spec_summary, len_spec_summary_stored, spec_summary_crc32) = - write_chunk_maybe_compressed(self.output, spec.summary, compression_level)?; + write_chunk_maybe_compressed( + output, + spec.summary, + compression_level, + &mut self.compressor, + )?; let (off_spec_entries, len_spec_entries_stored, spec_entries_crc32) = - write_chunk_maybe_compressed(self.output, spec.index, compression_level)?; + write_chunk_maybe_compressed( + output, + spec.index, + compression_level, + &mut self.compressor, + )?; let (off_spec_array_addresses, len_spec_array_addresses_stored, spec_array_addresses_crc32) = - write_chunk_maybe_compressed(self.output, spec.addresses, compression_level)?; + write_chunk_maybe_compressed( + output, + spec.addresses, + compression_level, + &mut self.compressor, + )?; let (off_chrom_summary, len_chrom_summary_stored, chrom_summary_crc32) = - write_chunk_maybe_compressed(self.output, chrom.summary, compression_level)?; + write_chunk_maybe_compressed( + output, + chrom.summary, + compression_level, + &mut self.compressor, + )?; let (off_chrom_entries, len_chrom_entries_stored, chrom_entries_crc32) = - write_chunk_maybe_compressed(self.output, chrom.index, compression_level)?; + write_chunk_maybe_compressed( + output, + chrom.index, + compression_level, + &mut self.compressor, + )?; let ( off_chrom_array_addresses, len_chrom_array_addresses_stored, chrom_array_addresses_crc32, - ) = write_chunk_maybe_compressed(self.output, chrom.addresses, compression_level)?; - let off_spec_meta = write_chunk(self.output, spec.grouped.section)?.0; - let off_chrom_meta = write_chunk(self.output, chrom.grouped.section)?.0; - let off_global_meta = write_aligned(self.output, &global_bytes)?; + ) = write_chunk_maybe_compressed( + output, + chrom.addresses, + compression_level, + &mut self.compressor, + )?; + let off_spec_meta = write_chunk(output, spec.grouped.section)?.0; + let off_chrom_meta = write_chunk(output, chrom.grouped.section)?.0; + let off_global_meta = write_aligned(output, &global_bytes)?; - let a1 = self.write_window_directory(spec.window_directory)?; - let b1 = self.write_window_directory(chrom.window_directory)?; + let a1 = self.write_window_directory(output, spec.window_directory)?; + let b1 = self.write_window_directory(output, chrom.window_directory)?; - self.output.write(&FILE_TRAILER)?; - let total_file_size = self.output.position()?; + output.write(&FILE_TRAILER)?; + let total_file_size = output.position()?; let header = Header { compression_codec: self.config.codec_id(), @@ -722,15 +789,114 @@ impl<'out> IonWriter<'out> { header.write(&mut header_bytes); let crc = crc32fast::hash(&header_bytes[0..1020]); header_bytes[1020..1024].copy_from_slice(&crc.to_le_bytes()); - self.output.patch(0, &header_bytes) + output.patch(0, &header_bytes) } } -pub fn write_mzml_to_ion( +pub(crate) fn write_mzml_to_ion( mzml: &MzML, - config: WriteOptions, + config: &WriteOptions, output: &mut dyn WriteBytes, ) -> IonResult<()> { - allow_compression_level(config.compression_level)?; - IonWriter::create(output, config)?.write_mzml(mzml) + let mut writer = IonWriter::begin(output, mzml, config)?; + writer.write_mzml(output, mzml)?; + writer.finish(output, mzml) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + ion::decoder::decode::IonReader, + mzml::structs::{BinaryDataArrayList, CvParam, NumericArray}, + }; + + fn make_bda(accession: &str, name: &str, data: Vec) -> BinaryDataArray { + BinaryDataArray { + cv_params: vec![CvParam { + cv_ref: Some("MS".to_string()), + accession: Some(accession.to_string()), + name: name.to_string(), + value: None, + unit_cv_ref: None, + unit_name: None, + unit_accession: None, + }], + binary: Some(NumericArray::F64(data)), + ..Default::default() + } + } + + fn make_spectrum(id: usize, rt: f64) -> Spectrum { + let mz = vec![100.0 + id as f64, 101.0 + id as f64, 102.0 + id as f64]; + let intensity = vec![1.0, 2.0, 3.0]; + Spectrum { + id: format!("scan={id}"), + index: Some(id as u32), + cv_params: vec![CvParam { + cv_ref: Some("MS".to_string()), + accession: Some("MS:1000016".to_string()), + name: "scan start time".to_string(), + value: Some(rt.to_string()), + unit_cv_ref: Some("UO".to_string()), + unit_accession: Some("UO:0000010".to_string()), + unit_name: Some("second".to_string()), + }], + binary_data_array_list: Some(BinaryDataArrayList { + count: Some(2), + binary_data_arrays: vec![ + make_bda("MS:1000514", "m/z array", mz), + make_bda("MS:1000515", "intensity array", intensity), + ], + }), + ..Default::default() + } + } + + #[test] + fn push_spectrum_streams_instead_of_buffering_the_whole_file() { + let block_size = 256; + let config = WriteOptions { + compression_level: 0, + block_size, + parallel: false, + ..Default::default() + }; + let mut output = Vec::new(); + let mut writer = IonWriter::begin(&mut output, &MzML::default(), &config).unwrap(); + // Flush every sealed block immediately, so the pending queue can never hold + // more than the block currently being assembled. + writer.spec_stream.set_max_pending_bytes(1); + + let spectrum_count = 500; + let mut max_pending_bytes_seen = 0usize; + for i in 0..spectrum_count { + writer + .push_spectrum(&mut output, &make_spectrum(i, i as f64)) + .unwrap(); + max_pending_bytes_seen = max_pending_bytes_seen.max(writer.spec_stream.pending_bytes()); + } + + // The bytes held by not-yet-flushed blocks must stay within one block's + // worth of data, regardless of how many spectra have been pushed so far: + // push_spectrum must stream one block at a time, not buffer the whole file. + assert!( + max_pending_bytes_seen <= block_size, + "pending bytes ({max_pending_bytes_seen}) exceeded one block ({block_size}) \ + after pushing {spectrum_count} spectra" + ); + + writer.finish(&mut output, &MzML::default()).unwrap(); + + let mut reader = + IonReader::from_bytes(&output, &crate::ion::decoder::decode::ReadOptions::default()) + .unwrap(); + assert_eq!(reader.spectrum_count(), spectrum_count as u64); + for i in [0usize, spectrum_count / 2, spectrum_count - 1] { + let spectrum = reader.spectrum(i).unwrap(); + let mz = reader.array(i, crate::ion::ArrayKind::Mz).unwrap(); + assert_eq!(spectrum.index, Some(i as u32)); + assert_eq!(mz, vec![100.0 + i as f64, 101.0 + i as f64, 102.0 + i as f64]); + } + } } diff --git a/crates/parser/src/ion/encoder/utilities/block_writer.rs b/crates/parser/src/ion/encoder/utilities/block_writer.rs index da4d1c9b..e04620dc 100644 --- a/crates/parser/src/ion/encoder/utilities/block_writer.rs +++ b/crates/parser/src/ion/encoder/utilities/block_writer.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use cosmoz::{CompressOptions, Encoder, compress_into, max_compressed_size}; +use cosmoz::{CompressOptions, Encoder}; #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] use rayon::prelude::*; @@ -53,7 +53,7 @@ pub(crate) trait BlockCompressor { pub(crate) struct DefaultCompressor { level: u8, - encoder: Box, + encoder: Encoder, } impl DefaultCompressor { @@ -72,14 +72,11 @@ impl DefaultCompressor { impl BlockCompressor for DefaultCompressor { fn compress(&mut self, input: &[u8], output: &mut Vec) -> IonResult { - let options = CompressOptions { - level: self.level, - checksum: false, - ..Default::default() - }; output.clear(); - output.resize(max_compressed_size(input.len(), &options), 0); - let written = compress_into(input, output, &options, &mut self.encoder) + output.resize(self.encoder.max_compressed_size(input.len()), 0); + let written = self + .encoder + .compress_into(input, output) .map_err(|err| IonError::from(format!("zstd encode failed: {err:?}")))?; output.truncate(written); Ok(written) @@ -367,8 +364,7 @@ pub(crate) struct ContainerSummary { pub(crate) directory_crc32: u32, } -pub(crate) struct BlockWriter<'output, C: BlockCompressor> { - output: &'output mut dyn WriteBytes, +pub(crate) struct BlockWriter { block_packing_id: PackingId, store: BlockStore, pending: Vec, @@ -381,15 +377,13 @@ pub(crate) struct BlockWriter<'output, C: BlockCompressor> { staging: Option, } -impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { +impl BlockWriter { pub(crate) fn new( - output: &'output mut dyn WriteBytes, max_block_uncompressed_size: usize, compressor: CompressionMode, block_packing_id: PackingId, ) -> Self { Self { - output, block_packing_id, store: BlockStore::new(max_block_uncompressed_size), pending: Vec::new(), @@ -403,6 +397,11 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { } } + #[cfg(test)] + pub(crate) fn pending_bytes(&self) -> usize { + self.pending_bytes + } + #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] pub(crate) fn window_major(mut self) -> IonResult { self.staging = Some(Staging { @@ -419,6 +418,7 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { pub(crate) fn add_item_to_box( &mut self, + output: &mut dyn WriteBytes, array_type: u32, item_byte_size: usize, element_size: usize, @@ -427,11 +427,12 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { where WriteAction: FnOnce(&mut Vec) -> IonResult<()>, { - self.add_item_to_window(array_type, 0, item_byte_size, element_size, write_action) + self.add_item_to_window(output, array_type, 0, item_byte_size, element_size, write_action) } pub(crate) fn add_item_to_window( &mut self, + output: &mut dyn WriteBytes, array_type: u32, window_index: u32, item_byte_size: usize, @@ -443,14 +444,15 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { { let group = BlockGroup::new(array_type, element_size, window_index); if item_byte_size > self.store.max_block_size { - self.add_isolated_block(group, item_byte_size, true, write_action) + self.add_isolated_block(output, group, item_byte_size, true, write_action) } else { - self.add_normal_item(group, item_byte_size, write_action) + self.add_normal_item(output, group, item_byte_size, write_action) } } fn add_isolated_block( &mut self, + output: &mut dyn WriteBytes, group: BlockGroup, item_byte_size: usize, compress_sequentially: bool, @@ -459,7 +461,7 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { where WriteAction: FnOnce(&mut Vec) -> IonResult<()>, { - self.seal_open_block_for_group(group)?; + self.seal_open_block_for_group(output, group)?; let block_id = self.store @@ -474,12 +476,13 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { .accumulated_data, )?; - self.seal_open_block_for_group(group)?; + self.seal_open_block_for_group(output, group)?; Ok((block_id, 0)) } fn add_normal_item( &mut self, + output: &mut dyn WriteBytes, group: BlockGroup, item_byte_size: usize, write_action: WriteAction, @@ -488,7 +491,7 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { WriteAction: FnOnce(&mut Vec) -> IonResult<()>, { if self.store.would_overflow(group, item_byte_size) { - self.seal_open_block_for_group(group)?; + self.seal_open_block_for_group(output, group)?; } self.store.ensure_open_block(group, item_byte_size)?; @@ -499,7 +502,11 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { Ok((block_id, element_offset)) } - fn seal_open_block_for_group(&mut self, group: BlockGroup) -> IonResult<()> { + fn seal_open_block_for_group( + &mut self, + output: &mut dyn WriteBytes, + group: BlockGroup, + ) -> IonResult<()> { let Some(active_block) = self.store.take_open_block(group) else { return Ok(()); }; @@ -516,7 +523,7 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { }); self.pending_bytes += data_len; if self.pending_bytes >= self.max_pending_bytes { - self.flush_pending()?; + self.flush_pending(output)?; } Ok(()) } @@ -611,11 +618,11 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { } #[cfg(test)] - fn set_max_pending_bytes(&mut self, value: usize) { + pub(crate) fn set_max_pending_bytes(&mut self, value: usize) { self.max_pending_bytes = value; } - fn flush_pending(&mut self) -> IonResult<()> { + fn flush_pending(&mut self, output: &mut dyn WriteBytes) -> IonResult<()> { if self.pending.is_empty() { return Ok(()); } @@ -659,7 +666,7 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { } for block in ready { - self.output.write(&block.bytes)?; + output.write(&block.bytes)?; self.store.seal( block.block_id, BlockDirEntry { @@ -676,7 +683,7 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { } #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - fn write_staged_blocks_in_window_order(&mut self) -> IonResult<()> { + fn write_staged_blocks_in_window_order(&mut self, output: &mut dyn WriteBytes) -> IonResult<()> { let Some(mut staging) = self.staging.take() else { return Ok(()); }; @@ -688,7 +695,7 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { for key in staging.keys { buffer.resize(key.staged_len as usize, 0); staging.spill.read_at(key.staged_offset, &mut buffer)?; - self.output.write(&buffer)?; + output.write(&buffer)?; self.store.seal( key.block_id, BlockDirEntry { @@ -703,20 +710,20 @@ impl<'output, C: BlockCompressor + Send + Sync> BlockWriter<'output, C> { Ok(()) } - pub(crate) fn finish(mut self) -> IonResult { + pub(crate) fn finish(&mut self, output: &mut dyn WriteBytes) -> IonResult { for group in self.store.open_blocks.open_groups_in_id_order() { - self.seal_open_block_for_group(group)?; + self.seal_open_block_for_group(output, group)?; } - self.flush_pending()?; + self.flush_pending(output)?; #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - self.write_staged_blocks_in_window_order()?; + self.write_staged_blocks_in_window_order(output)?; let block_count = self.store.block_count(); let mut directory_bytes = Vec::with_capacity(block_count as usize * BLOCK_DIRECTORY_ENTRY_SIZE); self.store.write_directory(&mut directory_bytes); let directory_crc32 = crc32fast::hash(&directory_bytes); - self.output.write(&directory_bytes)?; + output.write(&directory_bytes)?; let total_bytes = self.payload_bytes + directory_bytes.len() as u64; Ok(ContainerSummary { @@ -1025,7 +1032,6 @@ mod tests { let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, 8, CompressionMode::Compressed(CountingCompressor { forks: forks.clone(), @@ -1037,14 +1043,14 @@ mod tests { for i in 0..block_count { builder - .add_item_to_box(1, 8, 4, |buf| { + .add_item_to_box(&mut output, 1, 8, 4, |buf| { buf.extend_from_slice(&[(i % 256) as u8; 8]); Ok(()) }) .unwrap(); } - builder.finish().unwrap(); + builder.finish(&mut output).unwrap(); let fork_count = forks.load(Ordering::SeqCst); assert!( @@ -1059,14 +1065,13 @@ mod tests { fn block_writer_raw_single_item() { let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, 64 * 1024 * 1024, CompressionMode::::Raw, PackingId::Raw, ); let item_data = vec![1u8, 2, 3, 4, 5, 6, 7, 8]; let (block_id, element_offset) = builder - .add_item_to_box(1, item_data.len(), 8, |buf| { + .add_item_to_box(&mut output, 1, item_data.len(), 8, |buf| { buf.extend_from_slice(&item_data); Ok(()) }) @@ -1077,7 +1082,7 @@ mod tests { block_count, total_bytes, .. - } = builder.finish().unwrap(); + } = builder.finish(&mut output).unwrap(); assert_eq!(block_count, 1); assert!(total_bytes > 0); assert!(output.0.starts_with(&item_data)); @@ -1087,7 +1092,6 @@ mod tests { fn same_stride_different_array_types_get_different_blocks() { let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, 64 * 1024 * 1024, CompressionMode::::Raw, PackingId::Raw, @@ -1095,13 +1099,13 @@ mod tests { let mz_array_type = 1000514; let intensity_array_type = 1000515; let (mz_block, _) = builder - .add_item_to_box(mz_array_type, 8, 4, |buf| { + .add_item_to_box(&mut output, mz_array_type, 8, 4, |buf| { buf.extend_from_slice(&[0u8; 8]); Ok(()) }) .unwrap(); let (intensity_block, _) = builder - .add_item_to_box(intensity_array_type, 8, 4, |buf| { + .add_item_to_box(&mut output, intensity_array_type, 8, 4, |buf| { buf.extend_from_slice(&[0u8; 8]); Ok(()) }) @@ -1110,27 +1114,26 @@ mod tests { mz_block, intensity_block, "different array types must never share a block" ); - assert_eq!(builder.finish().unwrap().block_count, 2); + assert_eq!(builder.finish(&mut output).unwrap().block_count, 2); } #[test] fn same_type_different_windows_get_different_blocks() { let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, 64 * 1024 * 1024, CompressionMode::::Raw, PackingId::Raw, ); let array_type = 1000514; let (window_0_block, _) = builder - .add_item_to_window(array_type, 0, 8, 4, |buf| { + .add_item_to_window(&mut output, array_type, 0, 8, 4, |buf| { buf.extend_from_slice(&[0u8; 8]); Ok(()) }) .unwrap(); let (window_1_block, _) = builder - .add_item_to_window(array_type, 1, 8, 4, |buf| { + .add_item_to_window(&mut output, array_type, 1, 8, 4, |buf| { buf.extend_from_slice(&[0u8; 8]); Ok(()) }) @@ -1139,7 +1142,7 @@ mod tests { window_0_block, window_1_block, "different windows must never share a block" ); - let block_count = builder.finish().unwrap().block_count as usize; + let block_count = builder.finish(&mut output).unwrap().block_count as usize; assert_eq!(block_count, 2); let directory_start = output.0.len() - block_count * BLOCK_DIRECTORY_ENTRY_SIZE; @@ -1163,13 +1166,12 @@ mod tests { fn oversized_item_uses_sequential_path() { let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, 16, CompressionMode::::Raw, PackingId::Raw, ); builder - .add_item_to_box(1, 64, 8, |buf| { + .add_item_to_box(&mut output, 1, 64, 8, |buf| { buf.extend_from_slice(&[2u8; 64]); Ok(()) }) @@ -1185,25 +1187,24 @@ mod tests { fn block_writer_element_offsets_are_correct() { let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, 64 * 1024 * 1024, CompressionMode::::Raw, PackingId::Raw, ); let (_, first_offset) = builder - .add_item_to_box(1, 8, 8, |buf| { + .add_item_to_box(&mut output, 1, 8, 8, |buf| { buf.extend_from_slice(&[0u8; 8]); Ok(()) }) .unwrap(); let (_, second_offset) = builder - .add_item_to_box(1, 8, 8, |buf| { + .add_item_to_box(&mut output, 1, 8, 8, |buf| { buf.extend_from_slice(&[0u8; 8]); Ok(()) }) .unwrap(); let (_, third_offset) = builder - .add_item_to_box(1, 8, 8, |buf| { + .add_item_to_box(&mut output, 1, 8, 8, |buf| { buf.extend_from_slice(&[0u8; 8]); Ok(()) }) @@ -1217,19 +1218,18 @@ mod tests { fn block_writer_different_strides_get_different_blocks() { let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, 64 * 1024 * 1024, CompressionMode::::Raw, PackingId::Raw, ); let (four_byte_block_id, _) = builder - .add_item_to_box(1, 4, 4, |buf| { + .add_item_to_box(&mut output, 1, 4, 4, |buf| { buf.extend_from_slice(&[0u8; 4]); Ok(()) }) .unwrap(); let (eight_byte_block_id, _) = builder - .add_item_to_box(1, 8, 8, |buf| { + .add_item_to_box(&mut output, 1, 8, 8, |buf| { buf.extend_from_slice(&[0u8; 8]); Ok(()) }) @@ -1242,19 +1242,18 @@ mod tests { let max_block_size = 16usize; let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, max_block_size, CompressionMode::::Raw, PackingId::Raw, ); let (first_block_id, _) = builder - .add_item_to_box(1, 12, 4, |buf| { + .add_item_to_box(&mut output, 1, 12, 4, |buf| { buf.extend_from_slice(&[0u8; 12]); Ok(()) }) .unwrap(); let (second_block_id, _) = builder - .add_item_to_box(1, 12, 4, |buf| { + .add_item_to_box(&mut output, 1, 12, 4, |buf| { buf.extend_from_slice(&[0u8; 12]); Ok(()) }) @@ -1263,7 +1262,7 @@ mod tests { first_block_id, second_block_id, "overflow should have triggered a new block" ); - let total_block_count = builder.finish().unwrap().block_count; + let total_block_count = builder.finish(&mut output).unwrap().block_count; assert_eq!(total_block_count, 2); } @@ -1271,13 +1270,12 @@ mod tests { fn block_writer_finish_writes_directory() { let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, 64 * 1024 * 1024, CompressionMode::::Raw, PackingId::Raw, ); builder - .add_item_to_box(1, 8, 8, |buf| { + .add_item_to_box(&mut output, 1, 8, 8, |buf| { buf.extend_from_slice(&[0xAAu8; 8]); Ok(()) }) @@ -1286,7 +1284,7 @@ mod tests { block_count, total_bytes, .. - } = builder.finish().unwrap(); + } = builder.finish(&mut output).unwrap(); assert_eq!(block_count, 1); let expected_directory_size = BLOCK_DIRECTORY_ENTRY_SIZE as u64; assert_eq!(total_bytes, 8 + expected_directory_size); @@ -1295,8 +1293,7 @@ mod tests { #[test] fn block_writer_empty_produces_no_blocks() { let mut output = VecOutput(Vec::new()); - let builder = BlockWriter::new( - &mut output, + let mut builder = BlockWriter::new( 64 * 1024 * 1024, CompressionMode::::Raw, PackingId::Raw, @@ -1305,7 +1302,7 @@ mod tests { block_count, total_bytes, .. - } = builder.finish().unwrap(); + } = builder.finish(&mut output).unwrap(); assert_eq!(block_count, 0); assert_eq!(total_bytes, 0); assert!(output.0.is_empty()); @@ -1337,7 +1334,6 @@ mod tests { let window_count = 3u32; let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, 8, CompressionMode::::Raw, PackingId::Raw, @@ -1348,7 +1344,7 @@ mod tests { for round in 0..rounds { for window in 0..window_count { let (block_id, _) = builder - .add_item_to_window(1000514, window, 8, 8, |buf| { + .add_item_to_window(&mut output, 1000514, window, 8, 8, |buf| { buf.extend_from_slice(&[round as u8; 8]); Ok(()) }) @@ -1356,7 +1352,7 @@ mod tests { window_of.insert(block_id, window); } } - let block_count = builder.finish().unwrap().block_count as usize; + let block_count = builder.finish(&mut output).unwrap().block_count as usize; let payload_order = blocks_in_payload_order(&output.0, block_count, &window_of); let mut sorted = payload_order.clone(); @@ -1382,7 +1378,6 @@ mod tests { let blocks_per_batch = 6usize; let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, 8, CompressionMode::::Raw, PackingId::Raw, @@ -1393,7 +1388,7 @@ mod tests { for round in 0..rounds { for window in 0..window_count { let (block_id, _) = builder - .add_item_to_window(1000514, window, 8, 8, |buf| { + .add_item_to_window(&mut output, 1000514, window, 8, 8, |buf| { buf.extend_from_slice(&[round as u8; 8]); Ok(()) }) @@ -1401,7 +1396,7 @@ mod tests { window_of.insert(block_id, window); } } - let block_count = builder.finish().unwrap().block_count as usize; + let block_count = builder.finish(&mut output).unwrap().block_count as usize; let payload_order = blocks_in_payload_order(&output.0, block_count, &window_of); let batches: Vec<&[(u32, u32)]> = payload_order.chunks(blocks_per_batch).collect(); @@ -1437,7 +1432,6 @@ mod tests { let encode = |window_major: bool| -> (Vec, usize, std::collections::HashMap) { let mut output = VecOutput(Vec::new()); let builder = BlockWriter::new( - &mut output, 8, CompressionMode::::Raw, PackingId::Raw, @@ -1454,7 +1448,7 @@ mod tests { for window in 0..window_count { let content = [(round * window_count + window) as u8; 8]; let (block_id, _) = builder - .add_item_to_window(1000514, window, 8, 8, |buf| { + .add_item_to_window(&mut output, 1000514, window, 8, 8, |buf| { buf.extend_from_slice(&content); Ok(()) }) @@ -1462,7 +1456,7 @@ mod tests { window_of.insert(block_id, window); } } - let block_count = builder.finish().unwrap().block_count as usize; + let block_count = builder.finish(&mut output).unwrap().block_count as usize; (output.0, block_count, window_of) }; @@ -1509,7 +1503,6 @@ mod tests { let window_count = 3u32; let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, 8, CompressionMode::::Raw, PackingId::Raw, @@ -1524,7 +1517,7 @@ mod tests { for window in 0..window_count { let content = [(round * window_count + window) as u8; 8]; let (block_id, _) = builder - .add_item_to_window(1000514, window, 8, 8, |buf| { + .add_item_to_window(&mut output, 1000514, window, 8, 8, |buf| { buf.extend_from_slice(&content); Ok(()) }) @@ -1533,7 +1526,7 @@ mod tests { content_of.insert(block_id, content); } } - let block_count = builder.finish().unwrap().block_count as usize; + let block_count = builder.finish(&mut output).unwrap().block_count as usize; assert_eq!(block_count, (rounds * window_count) as usize); let raw = output.0; @@ -1572,19 +1565,18 @@ mod tests { let max_block_size = 16usize; let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, max_block_size, CompressionMode::::Raw, PackingId::Raw, ); let (block_id, element_offset) = builder - .add_item_to_box(1, 64, 8, |buf| { + .add_item_to_box(&mut output, 1, 64, 8, |buf| { buf.extend_from_slice(&[0xABu8; 64]); Ok(()) }) .unwrap(); assert_eq!(element_offset, 0); - let block_count = builder.finish().unwrap().block_count; + let block_count = builder.finish(&mut output).unwrap().block_count; assert_eq!(block_count, 1); assert_eq!(block_id, 0); assert!(output.0.starts_with(&[0xABu8; 64])); @@ -1595,25 +1587,24 @@ mod tests { let max_block_size = 16usize; let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, max_block_size, CompressionMode::::Raw, PackingId::Raw, ); let (bid_before, _) = builder - .add_item_to_box(1, 8, 8, |buf| { + .add_item_to_box(&mut output, 1, 8, 8, |buf| { buf.extend_from_slice(&[0x01u8; 8]); Ok(()) }) .unwrap(); let (bid_oversized, off_oversized) = builder - .add_item_to_box(1, 64, 8, |buf| { + .add_item_to_box(&mut output, 1, 64, 8, |buf| { buf.extend_from_slice(&[0x02u8; 64]); Ok(()) }) .unwrap(); let (bid_after, _) = builder - .add_item_to_box(1, 8, 8, |buf| { + .add_item_to_box(&mut output, 1, 8, 8, |buf| { buf.extend_from_slice(&[0x03u8; 8]); Ok(()) }) @@ -1621,7 +1612,7 @@ mod tests { assert_eq!(off_oversized, 0); assert_ne!(bid_before, bid_oversized); assert_ne!(bid_oversized, bid_after); - let block_count = builder.finish().unwrap().block_count; + let block_count = builder.finish(&mut output).unwrap().block_count; assert_eq!(block_count, 3); } @@ -1630,19 +1621,18 @@ mod tests { let max_block_size = 16usize; let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, max_block_size, CompressionMode::::Raw, PackingId::Raw, ); let (bid_a, off_a) = builder - .add_item_to_box(1, 64, 8, |buf| { + .add_item_to_box(&mut output, 1, 64, 8, |buf| { buf.extend_from_slice(&[0x11u8; 64]); Ok(()) }) .unwrap(); let (bid_b, off_b) = builder - .add_item_to_box(1, 64, 8, |buf| { + .add_item_to_box(&mut output, 1, 64, 8, |buf| { buf.extend_from_slice(&[0x22u8; 64]); Ok(()) }) @@ -1650,7 +1640,7 @@ mod tests { assert_eq!(off_a, 0); assert_eq!(off_b, 0); assert_ne!(bid_a, bid_b); - let block_count = builder.finish().unwrap().block_count; + let block_count = builder.finish(&mut output).unwrap().block_count; assert_eq!(block_count, 2); } @@ -1660,18 +1650,17 @@ mod tests { let item_size = 128usize; let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, max_block_size, CompressionMode::::Raw, PackingId::Raw, ); builder - .add_item_to_box(1, item_size, 8, |buf| { + .add_item_to_box(&mut output, 1, item_size, 8, |buf| { buf.extend_from_slice(&[0xFFu8; 128]); Ok(()) }) .unwrap(); - let block_count = builder.finish().unwrap().block_count; + let block_count = builder.finish(&mut output).unwrap().block_count; assert_eq!(block_count, 1); let directory_start = output.0.len() - BLOCK_DIRECTORY_ENTRY_SIZE; let uncomp_bytes = u64::from_le_bytes( @@ -1686,7 +1675,6 @@ mod tests { fn block_writer_seq_and_par_match() { let mut seq_out = VecOutput(Vec::new()); let mut seq = BlockWriter::new( - &mut seq_out, 8, CompressionMode::Compressed(PassthroughCompressor), PackingId::ByteShuffle, @@ -1695,42 +1683,41 @@ mod tests { let mut par_out = VecOutput(Vec::new()); let mut par = BlockWriter::new( - &mut par_out, 8, CompressionMode::Compressed(PassthroughCompressor), PackingId::ByteShuffle, ); par.set_par_min_blocks(0); - for builder in [&mut seq, &mut par] { + for (builder, out) in [(&mut seq, &mut seq_out), (&mut par, &mut par_out)] { builder - .add_item_to_box(1, 8, 4, |buf| { + .add_item_to_box(out, 1, 8, 4, |buf| { buf.extend_from_slice(&[1, 2, 3, 4, 5, 6, 7, 8]); Ok(()) }) .unwrap(); builder - .add_item_to_box(1, 8, 4, |buf| { + .add_item_to_box(out, 1, 8, 4, |buf| { buf.extend_from_slice(&[9, 10, 11, 12, 13, 14, 15, 16]); Ok(()) }) .unwrap(); builder - .add_item_to_box(1, 8, 4, |buf| { + .add_item_to_box(out, 1, 8, 4, |buf| { buf.extend_from_slice(&[17, 18, 19, 20, 21, 22, 23, 24]); Ok(()) }) .unwrap(); builder - .add_item_to_box(1, 8, 4, |buf| { + .add_item_to_box(out, 1, 8, 4, |buf| { buf.extend_from_slice(&[25, 26, 27, 28, 29, 30, 31, 32]); Ok(()) }) .unwrap(); } - let seq_res = seq.finish().unwrap(); - let par_res = par.finish().unwrap(); + let seq_res = seq.finish(&mut seq_out).unwrap(); + let par_res = par.finish(&mut par_out).unwrap(); assert_eq!(seq_res, par_res); assert_eq!(seq_out.0, par_out.0); @@ -1741,7 +1728,6 @@ mod tests { let item_count = 7usize; let mut output = VecOutput(Vec::new()); let mut builder = BlockWriter::new( - &mut output, 8, CompressionMode::::Raw, PackingId::Raw, @@ -1750,13 +1736,13 @@ mod tests { for i in 0..item_count { builder - .add_item_to_box(1, 8, 8, |buf| { + .add_item_to_box(&mut output, 1, 8, 8, |buf| { buf.extend_from_slice(&[i as u8; 8]); Ok(()) }) .unwrap(); } - let block_count = builder.finish().unwrap().block_count; + let block_count = builder.finish(&mut output).unwrap().block_count; assert_eq!(block_count as usize, item_count); let raw = output.0; diff --git a/crates/parser/src/ion/encoder/utilities/meta_collector/grouper.rs b/crates/parser/src/ion/encoder/utilities/meta_collector/grouper.rs index 4add4db9..bb546c0f 100644 --- a/crates/parser/src/ion/encoder/utilities/meta_collector/grouper.rs +++ b/crates/parser/src/ion/encoder/utilities/meta_collector/grouper.rs @@ -1,6 +1,8 @@ +use cosmoz::Encoder; + use super::{ super::output::SectionChunk, MetaParamBuffer, MetadataWriter, PackedMeta, PackedMetaBuilder, - compress_bytes_if_enabled, + compress_bytes_if_enabled, new_meta_encoder, }; use crate::ion::{ IonResult, @@ -22,6 +24,7 @@ pub(crate) struct GroupedSection { pub(crate) struct MetaGrouper { group_size: u32, level: u8, + encoder: Encoder, builder: PackedMetaBuilder, items_in_group: u32, payloads: SectionChunk, @@ -35,10 +38,11 @@ pub(crate) struct MetaGrouper { } impl MetaGrouper { - pub(crate) fn new(group_size: u32, level: u8, payloads: SectionChunk) -> Self { - Self { + pub(crate) fn new(group_size: u32, level: u8, payloads: SectionChunk) -> IonResult { + Ok(Self { group_size, level, + encoder: new_meta_encoder(level)?, builder: PackedMetaBuilder::new(), items_in_group: 0, payloads, @@ -49,7 +53,7 @@ impl MetaGrouper { row_count: 0, numeric_count: 0, string_count: 0, - } + }) } fn seal_group(&mut self) -> IonResult<()> { @@ -64,7 +68,7 @@ impl MetaGrouper { let raw_size = raw.len() as u64; self.uncompressed_size += raw_size; let payload_offset = self.payloads.len(); - let compressed = compress_bytes_if_enabled(raw, self.level); + let compressed = compress_bytes_if_enabled(raw, self.level, &mut self.encoder); self.directory.push(MetaGroupEntry { payload_offset, payload_size: compressed.len() as u64, diff --git a/crates/parser/src/ion/encoder/utilities/meta_collector/mod.rs b/crates/parser/src/ion/encoder/utilities/meta_collector/mod.rs index 2b60300b..6cce3156 100644 --- a/crates/parser/src/ion/encoder/utilities/meta_collector/mod.rs +++ b/crates/parser/src/ion/encoder/utilities/meta_collector/mod.rs @@ -7,12 +7,12 @@ mod tests; pub(crate) use grouper::{GroupedSection, MetaGrouper, serialize_global_meta_with_counts}; pub(crate) use schema::MzmlListItem; -use cosmoz::{CompressOptions, Encoder, compress_into, max_compressed_size}; +use cosmoz::{CompressOptions, Encoder}; use crate::{ accessions::{INTENSITY_ARRAY, MZ_ARRAY, TIME_ARRAY}, ion::{ - IonResult, + IonError, IonResult, attr_meta::{ AccessionTail, CV_CODE_UNKNOWN, CV_REF_ATTR, attr_cv_param, cv_ref_code_from_str, parse_accession_tail, @@ -615,19 +615,21 @@ impl MetadataWriter for PackedMetaBuilder { } } -pub(crate) fn compress_bytes_if_enabled(bytes: Vec, level: u8) -> Vec { +pub(crate) fn compress_bytes_if_enabled(bytes: Vec, level: u8, encoder: &mut Encoder) -> Vec { if level == 0 { return bytes; } + let mut out = vec![0u8; encoder.max_compressed_size(bytes.len())]; + let written = encoder.compress_into(&bytes, &mut out).expect("zstd compression failed"); + out.truncate(written); + out +} + +pub(crate) fn new_meta_encoder(level: u8) -> IonResult { let options = CompressOptions { - level: get_supported_compression_level(level), + level: get_supported_compression_level(level.max(1)), checksum: false, ..Default::default() }; - let mut encoder = Encoder::new(&options).expect("zstd compression failed"); - let mut out = vec![0u8; max_compressed_size(bytes.len(), &options)]; - let written = compress_into(&bytes, &mut out, &options, &mut encoder) - .expect("zstd compression failed"); - out.truncate(written); - out + Encoder::new(&options).map_err(|err| IonError::from(format!("zstd start error: {err:?}"))) } diff --git a/crates/parser/src/ion/encoder/utilities/meta_collector/tests.rs b/crates/parser/src/ion/encoder/utilities/meta_collector/tests.rs index 1102ea5c..c3bbb420 100644 --- a/crates/parser/src/ion/encoder/utilities/meta_collector/tests.rs +++ b/crates/parser/src/ion/encoder/utilities/meta_collector/tests.rs @@ -142,7 +142,8 @@ fn encode_user_param_with_value_uses_separator() { #[test] fn compress_bytes_if_enabled_level_zero_is_identity() { let input = vec![1u8, 2, 3, 4]; - assert_eq!(compress_bytes_if_enabled(input.clone(), 0), input); + let mut encoder = new_meta_encoder(0).unwrap(); + assert_eq!(compress_bytes_if_enabled(input.clone(), 0, &mut encoder), input); } #[test] @@ -243,7 +244,7 @@ fn three_item_grouped() -> grouper::GroupedSection { unit_name: None, unit_accession: None, }; - let mut grouper = grouper::MetaGrouper::new(1, 0, SectionChunk::memory(0)); + let mut grouper = grouper::MetaGrouper::new(1, 0, SectionChunk::memory(0)).unwrap(); for (index, value) in ["10.5", "20.5", "30.5"].iter().enumerate() { let mut buffer = MetaParamBuffer::new(); buffer.push(TagId::CvParam, (index + 1) as u32, 0, make_cv(value)); @@ -443,7 +444,7 @@ fn group_local_node_ids_across_group_boundaries() { let mut output = Vec::new(); write_mzml_to_ion( &mzml, - WriteOptions { + &WriteOptions { compression_level: 0, force_f32: false, block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, @@ -467,7 +468,7 @@ fn group_local_node_ids_across_group_boundaries() { } let mut decoder = - IonReader::open(&output, ReadOptions::default()).expect("failed to open decoder"); + IonReader::from_bytes(&output, &ReadOptions::default()).expect("failed to open decoder"); let first_rows = decoder .spectrum_metadata_at(0) @@ -569,7 +570,7 @@ fn product_own_cv_params_parent_to_product_list_not_to_the_product_itself() { let mut output = Vec::new(); write_mzml_to_ion( &mzml, - WriteOptions { + &WriteOptions { compression_level: 0, force_f32: false, block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, @@ -587,7 +588,7 @@ fn product_own_cv_params_parent_to_product_list_not_to_the_product_itself() { }; let mut decoder = - IonReader::open(&output, ReadOptions::default()).expect("failed to open decoder"); + IonReader::from_bytes(&output, &ReadOptions::default()).expect("failed to open decoder"); let rows = decoder .spectrum_metadata_at(0) .expect("failed to read spectrum metadata"); diff --git a/crates/parser/src/ion/encoder/utilities/mod.rs b/crates/parser/src/ion/encoder/utilities/mod.rs index 2f77ff4b..da2d39db 100644 --- a/crates/parser/src/ion/encoder/utilities/mod.rs +++ b/crates/parser/src/ion/encoder/utilities/mod.rs @@ -1,5 +1,5 @@ pub(crate) mod block_writer; -pub(crate) use block_writer::{BlockWriter, CompressionMode, DefaultCompressor}; +pub(crate) use block_writer::{BlockWriter, CompressionMode, ContainerSummary, DefaultCompressor}; pub(crate) mod output; #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] pub use output::FileWriter; diff --git a/crates/parser/src/ion/encoder/utilities/output.rs b/crates/parser/src/ion/encoder/utilities/output.rs index 010fa710..ed26894e 100644 --- a/crates/parser/src/ion/encoder/utilities/output.rs +++ b/crates/parser/src/ion/encoder/utilities/output.rs @@ -26,6 +26,7 @@ pub struct FileWriter { #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] impl FileWriter { + #[allow(dead_code)] pub fn open(path: &str) -> IonResult { Self::open_path(Path::new(path)) } @@ -199,7 +200,7 @@ impl SectionChunk { .create_new(true) .open(&path) .map_err(|err| IonError::from(format!("cannot create '{}': {err}", path.display())))?; - let packer = Compressor::to( + let packer = Compressor::write_to( BufWriter::with_capacity(1 << 20, file), &CompressOptions { level: get_supported_compression_level(level), @@ -239,6 +240,7 @@ impl SectionChunk { } } + #[allow(dead_code)] pub(crate) fn is_spilled(&self) -> bool { #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] return matches!(self, SectionChunk::Spilled(_)); @@ -317,7 +319,7 @@ fn pad_to_alignment(output: &mut dyn WriteBytes) -> IonResult { #[cfg(all(test, not(all(target_arch = "wasm32", not(target_os = "wasi")))))] mod tests { - use cosmoz::{DecompressOptions, Decoder, decompress_into}; + use cosmoz::{DecompressOptions, Decoder}; use super::*; @@ -392,10 +394,8 @@ mod tests { assert_eq!(crc32, crc32fast::hash(&output)); let mut restored = vec![0u8; written.len()]; - let mut decoder = Decoder::new(); - let restored_length = - decompress_into(&output, &mut restored, &DecompressOptions::default(), &mut decoder) - .unwrap(); + let mut decoder = Decoder::new(&DecompressOptions::default()); + let restored_length = decoder.decompress_into(&output, &mut restored).unwrap(); assert_eq!(restored_length, written.len()); assert_eq!(restored, written); } diff --git a/crates/parser/src/ion/error.rs b/crates/parser/src/ion/error.rs index 3f0490a8..54c552c5 100644 --- a/crates/parser/src/ion/error.rs +++ b/crates/parser/src/ion/error.rs @@ -3,6 +3,7 @@ use std::{ fmt::{Display, Formatter}, }; +#[non_exhaustive] #[derive(Debug, Clone, PartialEq, Eq)] pub enum IonError { Msg(String), @@ -16,6 +17,7 @@ pub enum IonError { MissingChromatogramBounds, BadChromatogramBoundsChecksum, MalformedChromatogramBounds(String), + OutOfRange { index: usize, count: u64 }, } pub type IonResult = Result; @@ -52,6 +54,9 @@ impl Display for IonError { Self::MalformedChromatogramBounds(reason) => { write!(f, "chromatogram bounds (A0) malformed: {reason}") } + Self::OutOfRange { index, count } => { + write!(f, "index {index} out of range (count is {count})") + } } } } diff --git a/crates/parser/src/ion/filter_summary.rs b/crates/parser/src/ion/filter_summary.rs index fdf89c12..14a42236 100644 --- a/crates/parser/src/ion/filter_summary.rs +++ b/crates/parser/src/ion/filter_summary.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; +#[non_exhaustive] #[derive(Debug, Clone, Serialize, Deserialize, Copy)] pub struct SpectrumSummary { pub rt: f64, @@ -15,6 +16,7 @@ pub struct SpectrumSummary { pub position_z: u32, } +#[non_exhaustive] #[derive(Debug, Clone, Serialize, Deserialize, Copy)] pub struct ChromatogramSummary { pub lowest_mz: f64, diff --git a/crates/parser/src/ion/header.rs b/crates/parser/src/ion/header.rs index 7156e0a8..b6840829 100644 --- a/crates/parser/src/ion/header.rs +++ b/crates/parser/src/ion/header.rs @@ -298,7 +298,7 @@ impl Header { return Err("header: expected little-endian endianness_flag=0".into()); } - let format_version = get_version_from_header(bytes).unwrap(); + let format_version = version_of(bytes).unwrap(); allow_version(format_version)?; let compression_codec = h[HEADER_CODEC_ID]; let compression_level = h[HEADER_COMPRESSION_LEVEL]; @@ -701,7 +701,7 @@ pub(crate) fn parse_header(bytes: &[u8]) -> IonResult

{ } #[inline] -pub fn get_version_from_header(bytes: &[u8]) -> Option { +pub fn version_of(bytes: &[u8]) -> Option { let end = HEADER_FORMAT_VERSION_OFFSET + 2; if bytes.len() < end { return None; @@ -711,7 +711,7 @@ pub fn get_version_from_header(bytes: &[u8]) -> Option { Some(u16::from_le_bytes(buf)) } -pub fn update_header_version(header: &mut [u8]) -> IonResult { +pub fn set_version(header: &mut [u8]) -> IonResult { if header.len() < HEADER_SIZE { return Err(format!( "header: need {HEADER_SIZE} bytes to update the format version, got {}", @@ -730,7 +730,7 @@ pub fn update_header_version(header: &mut [u8]) -> IonResult { ) .into()); } - let version = get_version_from_header(header).unwrap(); + let version = version_of(header).unwrap(); if version == CURRENT_VERSION { return Ok(false); } @@ -742,7 +742,8 @@ pub fn update_header_version(header: &mut [u8]) -> IonResult { } #[inline] -pub fn get_total_file_size_from_header(bytes: &[u8]) -> Option { +#[allow(dead_code)] +pub(crate) fn get_total_file_size_from_header(bytes: &[u8]) -> Option { let end = HEADER_TOTAL_FILE_SIZE + 8; if bytes.len() < end { return None; @@ -1062,7 +1063,7 @@ mod tests { #[test] fn get_version_returns_none_on_short_buffer() { let too_short = [0u8; HEADER_FORMAT_VERSION_OFFSET + 1]; - assert_eq!(get_version_from_header(&too_short), None); + assert_eq!(version_of(&too_short), None); } #[test] @@ -1070,7 +1071,7 @@ mod tests { let mut bytes = [0u8; HEADER_SIZE]; bytes[HEADER_FORMAT_VERSION_OFFSET..HEADER_FORMAT_VERSION_OFFSET + 2] .copy_from_slice(&CURRENT_VERSION.to_le_bytes()); - assert_eq!(get_version_from_header(&bytes), Some(CURRENT_VERSION)); + assert_eq!(version_of(&bytes), Some(CURRENT_VERSION)); } #[test] @@ -1078,13 +1079,13 @@ mod tests { let mut bytes = [0u8; HEADER_SIZE]; bytes[HEADER_FORMAT_VERSION_OFFSET..HEADER_FORMAT_VERSION_OFFSET + 2] .copy_from_slice(&u16::MAX.to_le_bytes()); - assert_eq!(get_version_from_header(&bytes), Some(u16::MAX)); + assert_eq!(version_of(&bytes), Some(u16::MAX)); } #[test] fn get_version_handles_exact_minimum_buffer_length() { let bytes = [0u8; HEADER_FORMAT_VERSION_OFFSET + 2]; - assert_eq!(get_version_from_header(&bytes), Some(0)); + assert_eq!(version_of(&bytes), Some(0)); } #[test] @@ -1493,8 +1494,8 @@ mod tests { #[test] fn update_rewrites_older_supported_version_to_current() { let mut buf = header_bytes_at_version(MIN_SUPPORTED_VERSION); - assert_eq!(update_header_version(&mut buf), Ok(true)); - assert_eq!(get_version_from_header(&buf), Some(CURRENT_VERSION)); + assert_eq!(set_version(&mut buf), Ok(true)); + assert_eq!(version_of(&buf), Some(CURRENT_VERSION)); assert_eq!( read_u32_at(&buf, HEADER_CRC32), crc32fast::hash(&buf[0..HEADER_CRC32]) @@ -1506,7 +1507,7 @@ mod tests { fn update_leaves_current_version_untouched() { let mut buf = header_bytes_at_version(CURRENT_VERSION); let before = buf; - assert_eq!(update_header_version(&mut buf), Ok(false)); + assert_eq!(set_version(&mut buf), Ok(false)); assert_eq!(buf, before); } @@ -1515,7 +1516,7 @@ mod tests { let mut buf = header_bytes_at_version(MAX_SUPPORTED_VERSION + 1); let before = buf; assert_eq!( - update_header_version(&mut buf), + set_version(&mut buf), Err(IonError::UnsupportedFormatVersion( MAX_SUPPORTED_VERSION + 1 )) @@ -1528,7 +1529,7 @@ mod tests { let mut buf = header_bytes_at_version(MIN_SUPPORTED_VERSION); buf[HEADER_CRC32] ^= 0xff; let before = buf; - let error = update_header_version(&mut buf).unwrap_err(); + let error = set_version(&mut buf).unwrap_err(); assert!(error.contains("header_crc32 mismatch"), "{error}"); assert_eq!(buf, before); } @@ -1537,9 +1538,9 @@ mod tests { fn update_rejects_bad_signature_and_short_buffer() { let mut buf = header_bytes_at_version(MIN_SUPPORTED_VERSION); buf[0] = b'X'; - assert!(update_header_version(&mut buf).is_err()); + assert!(set_version(&mut buf).is_err()); let mut short = [0u8; HEADER_SIZE - 1]; - assert!(update_header_version(&mut short).is_err()); + assert!(set_version(&mut short).is_err()); } } diff --git a/crates/parser/src/ion/mod.rs b/crates/parser/src/ion/mod.rs index 4750cb0e..b3f36257 100644 --- a/crates/parser/src/ion/mod.rs +++ b/crates/parser/src/ion/mod.rs @@ -1,48 +1,45 @@ +#![allow(unused_imports)] pub(crate) mod array_kind; pub(crate) mod byte_transpose; -pub mod filter_summary; +pub(crate) mod filter_summary; pub(crate) mod header; pub(crate) mod meta_groups; pub(crate) mod packing; -pub mod range; +pub(crate) mod range; pub(crate) mod windowing; pub use array_kind::ArrayKind; pub use filter_summary::{ChromatogramSummary, SpectrumSummary}; pub use range::{ByteRange, Range}; pub(crate) mod decoder; pub(crate) mod format; -pub mod scan; +pub(crate) mod scan; +pub use scan::{ScanSummary, TimeUnit}; +#[cfg(test)] +pub(crate) use scan::ScanSource; pub(crate) mod version_generated; pub(crate) use decoder::utilities; #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] -pub use decoder::utilities::byte_source::FileSource; -pub use decoder::{ - decode::{ - ArrayAddress, DataXY, IonReader, ItemKind, ItemSlice, Pixel, Query, ReadOptions, Select, - Window, coalesce_byte_ranges, open_ranges, - }, - utilities::{ - byte_source::{BytesSource, CallbackSource, ReadBytes, SourceBytes}, - decompression_limit::{DEFAULT_MAX_UNCOMPRESSED_SIZE, DecompressionLimit}, - }, -}; -pub use header::{ - HEADER_FORMAT_VERSION_OFFSET, get_total_file_size_from_header, get_version_from_header, - update_header_version, +pub(crate) use decoder::utilities::byte_source::FileSource; +pub use decoder::decode::{DataXY, IonReader, ReadOptions, ScanQuery, Select, Window, header_ranges, merge_ranges}; +pub use decoder::utilities::{ + byte_source::{CallbackSource, ReadBytes}, + decompression_limit::{DEFAULT_MAX_UNCOMPRESSED_SIZE, DecompressionLimit}, }; +pub(crate) use decoder::utilities::byte_source::{BytesSource, SourceBytes}; +pub use header::{HEADER_FORMAT_VERSION_OFFSET, set_version, version_of}; +pub(crate) use header::get_total_file_size_from_header; pub(crate) mod encoder; #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] -pub use encoder::utilities::FileWriter; -pub use encoder::{ - encode::{DEFAULT_MZ_WINDOW, TARGET_BLOCK_UNCOMPRESSED_BYTES, WriteOptions}, - ion_writer::{IonWriter, write_mzml_to_ion}, - scan_stream::{MemoryReader, ScanStream}, - utilities::{SectionStorage, WriteBytes}, -}; +pub(crate) use encoder::utilities::FileWriter; +pub use encoder::encode::{DEFAULT_MZ_WINDOW, WriteOptions}; +pub(crate) use encoder::ion_writer::IonWriter; +pub(crate) use encoder::scan_stream::MemoryReader; +pub use encoder::scan_stream::ScanStream; +pub use encoder::utilities::{SectionStorage, WriteBytes}; pub use format::{ CODEC_NONE, CODEC_ZSTD, CURRENT_VERSION, FILE_SIGNATURE, FILE_TRAILER, HEADER_SIZE, MAX_SUPPORTED_VERSION, MIN_SUPPORTED_VERSION, allow_version, is_supported, }; pub(crate) mod attr_meta; -pub mod error; +pub(crate) mod error; pub use error::{IonError, IonResult}; diff --git a/crates/parser/src/ion/range.rs b/crates/parser/src/ion/range.rs index db8a1081..bc3c19aa 100644 --- a/crates/parser/src/ion/range.rs +++ b/crates/parser/src/ion/range.rs @@ -1,10 +1,11 @@ +#[non_exhaustive] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct ByteRange { pub offset: u64, pub length: u64, } -#[derive(Debug, Clone, Copy, PartialEq)] +#[derive(Debug, Clone, Copy, PartialEq, Default)] pub struct Range { pub from: f64, pub to: f64, diff --git a/crates/parser/src/ion/scan.rs b/crates/parser/src/ion/scan.rs index 8a9fff22..3d78bc4b 100644 --- a/crates/parser/src/ion/scan.rs +++ b/crates/parser/src/ion/scan.rs @@ -1 +1,3 @@ -pub use crate::ion::decoder::utilities::spectrum_source::{ScanSource, ScanSummary, TimeUnit}; +#[cfg(test)] +pub use crate::ion::decoder::utilities::spectrum_source::ScanSource; +pub use crate::ion::decoder::utilities::spectrum_source::{ScanSummary, TimeUnit}; diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 15345b0b..8cae1015 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -1,27 +1,17 @@ +mod api; pub(crate) mod accessions; -pub mod convert; -pub mod ion; +pub(crate) mod ion; pub mod mzml; -pub mod utilities; +pub(crate) mod utilities; -pub use convert::{ConvertKind, ConvertOptions, convert}; +pub use api::{ConvertKind, ConvertOptions, IonReader, IonWriter, ReadOptions, ScanQuery, WriteOptions, convert}; +#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] +pub use api::{convert_file, read, write}; pub use ion::{ - ArrayAddress, ArrayKind, ByteRange, BytesSource, CallbackSource, ChromatogramSummary, DataXY, - IonError, IonReader, IonResult, ItemKind, ItemSlice, Pixel, Query, Range, ReadBytes, - ReadOptions, Select, SourceBytes, SpectrumSummary, Window, coalesce_byte_ranges, + ArrayKind, ChromatogramSummary, DataXY, DecompressionLimit, IonError, IonResult, Range, + ScanStream, ScanSummary, SectionStorage, Select, SpectrumSummary, TimeUnit, Window, decoder::decode::{Metadatum, MetadatumValue}, - encoder::{ - encode::{DEFAULT_MZ_WINDOW, WriteOptions}, - ion_writer::{IonWriter, write_mzml_to_ion}, - scan_stream::{MemoryReader, ScanStream}, - utilities::{SectionStorage, WriteBytes}, - }, - open_ranges, - scan::{ScanSource, ScanSummary, TimeUnit}, -}; -#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] -pub use ion::{FileSource, FileWriter}; -pub use mzml::{ - BinToMzmlError, ParseError, bin_to_mzml, parse_indexed_mzml, parse_mzml, - structs::{Chromatogram, MzML, NumericArray, Spectrum}, }; + +pub mod format; +pub mod source; diff --git a/crates/parser/src/mzml/schema.rs b/crates/parser/src/mzml/schema.rs index 79aea055..75de4928 100644 --- a/crates/parser/src/mzml/schema.rs +++ b/crates/parser/src/mzml/schema.rs @@ -17,9 +17,10 @@ pub(crate) fn schema() -> &'static SchemaTree { } #[repr(u8)] +#[non_exhaustive] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub(crate) enum TagId { +pub enum TagId { FileContent = 0, SourceFile = 1, Contact = 2, diff --git a/crates/parser/src/source.rs b/crates/parser/src/source.rs new file mode 100644 index 00000000..1348971e --- /dev/null +++ b/crates/parser/src/source.rs @@ -0,0 +1 @@ +pub use crate::ion::{ByteRange, CallbackSource, ReadBytes, WriteBytes, header_ranges, merge_ranges}; diff --git a/crates/parser/src/utilities/test.rs b/crates/parser/src/utilities/test.rs index a97beb14..6e46b25b 100644 --- a/crates/parser/src/utilities/test.rs +++ b/crates/parser/src/utilities/test.rs @@ -33,8 +33,8 @@ pub(crate) fn parse_ion_as_mzml(cache: &'static OnceLock, path: &str) -> & cache.get_or_init(|| { let bytes: &'static [u8] = Box::leak(load_mzml_bytes(path).into_boxed_slice()); - let mut decoder = IonReader::open(bytes, ReadOptions::default()) - .unwrap_or_else(|e| panic!("IonReader::open failed: {e}")); + let mut decoder = IonReader::from_bytes(bytes, &ReadOptions::default()) + .unwrap_or_else(|e| panic!("IonReader::from_bytes failed: {e}")); decoder .to_mzml() diff --git a/crates/parser/tests/api_contract.rs b/crates/parser/tests/api_contract.rs deleted file mode 100644 index 710c4400..00000000 --- a/crates/parser/tests/api_contract.rs +++ /dev/null @@ -1,661 +0,0 @@ -mod common; - -use std::sync::Arc; - -use common::{ - assertions::{assert_mzml_semantic_eq, assert_mzml_structural_eq}, - decode_ion, encode_to_ion, test_files, -}; -use ionic::{ - ByteRange, BytesSource, CallbackSource, Range, ScanSource, - ion::{ - IonError, IonReader, IonWriter, MemoryReader, ReadOptions, SectionStorage, - TARGET_BLOCK_UNCOMPRESSED_BYTES, WriteOptions, open_ranges, - }, -}; - -fn default_config() -> ReadOptions { - ReadOptions::default() -} - -fn segmented_write_config() -> WriteOptions { - WriteOptions { - compression_level: 0, - force_f32: false, - block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, - parallel: false, - section_storage: SectionStorage::Memory, - mz_window: 5.0, - } -} - -fn stream_write_config() -> WriteOptions { - WriteOptions { - compression_level: 0, - force_f32: false, - block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, - parallel: false, - section_storage: SectionStorage::Memory, - mz_window: 0.0, - } -} - -fn encode_with_segments(src: &ionic::mzml::structs::MzML) -> Vec { - let mut bytes = Vec::new(); - let config = segmented_write_config(); - let mut writer = IonWriter::create(&mut bytes, config).expect("writer begin must succeed"); - let mut reader = MemoryReader::new(src.clone()); - writer - .write_stream(&mut reader) - .expect("write_stream must succeed"); - bytes -} - -fn windowed_write_config() -> WriteOptions { - WriteOptions { - compression_level: 9, - force_f32: false, - block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, - parallel: false, - section_storage: SectionStorage::Memory, - mz_window: 50.0, - } -} - -fn encode_windowed(src: &ionic::mzml::structs::MzML) -> Vec { - let mut bytes = Vec::new(); - let config = windowed_write_config(); - let mut writer = IonWriter::create(&mut bytes, config).expect("writer begin must succeed"); - let mut reader = MemoryReader::new(src.clone()); - writer - .write_stream(&mut reader) - .expect("write_stream must succeed"); - bytes -} - -#[test] -fn windowed_roundtrip_gives_semantic_equal_mzml() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_windowed(src); - let decoded = decode_ion(&bytes).expect("decode must succeed"); - assert_mzml_semantic_eq(src, &decoded); -} - -#[test] -fn reencode_through_decode_preserves_data() { - let src = test_files::tiny_pwiz_11(); - let first = encode_to_ion(src, 9, false); - let decoded_first = decode_ion(&first).expect("decode first must succeed"); - let second = encode_windowed(&decoded_first); - let decoded_second = decode_ion(&second).expect("decode second must succeed"); - assert_mzml_semantic_eq(src, &decoded_second); -} - -#[test] -fn root_level_imports_cover_a_full_read_workflow() { - let src = test_files::tiny_pwiz_11(); - - let mut bytes = Vec::new(); - ionic::write_mzml_to_ion(src, ionic::WriteOptions::default(), &mut bytes) - .expect("encode must succeed"); - - let mut reader = - ionic::IonReader::open(&bytes, ionic::ReadOptions::default()).expect("open must succeed"); - assert!(reader.spectrum_count() > 0); - - let _spectrum: Option = reader.spectrum(0).expect("get_spectrum"); - - reader.require_bounds().expect("bounds must load"); - let peaks: ionic::DataXY = reader - .read_window(0, Range { from: 0.0, to: 1e9 }) - .expect("read_mz_range"); - assert_eq!(peaks.x.len(), peaks.y.len()); - - let _mzml: ionic::MzML = reader.to_mzml().expect("to_mzml"); -} - -#[test] -fn encode_is_deterministic() { - let src = test_files::tiny_pwiz_11(); - let first = encode_to_ion(src, 12, false); - let second = encode_to_ion(src, 12, false); - assert_eq!( - first, second, - "two identical encodes must produce byte-identical output" - ); -} - -#[test] -fn roundtrip_gives_semantic_equal_mzml() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 9, false); - let decoded = decode_ion(&bytes).expect("decode must succeed"); - assert_mzml_semantic_eq(src, &decoded); -} - -#[test] -fn roundtrip_f32_mode_preserves_structure() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 9, true); - let decoded = decode_ion(&bytes).expect("decode must succeed"); - assert_mzml_structural_eq(src, &decoded); -} - -#[test] -fn open_bytes_to_mzml_matches_reference() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 6, false); - let reference = decode_ion(&bytes).expect("reference decode must succeed"); - let arc: Arc<[u8]> = Arc::from(bytes.as_slice()); - let mut decoder = IonReader::open_source(Arc::new(BytesSource::new(arc)), default_config()) - .expect("open_bytes must succeed"); - let result = decoder.to_mzml().expect("to_mzml must succeed"); - assert_mzml_semantic_eq(&reference, &result); -} - -#[test] -fn open_via_callback_to_mzml_matches_reference() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 6, false); - let reference = decode_ion(&bytes).expect("reference decode must succeed"); - let shared: Arc<[u8]> = Arc::from(bytes.as_slice()); - let callback_bytes = shared.clone(); - let mut decoder = IonReader::open_source( - Arc::new(CallbackSource::new(move |range: ByteRange| { - let start = range.offset as usize; - let end = start + range.length as usize; - callback_bytes - .get(start..end) - .map(|slice| slice.to_vec()) - .ok_or_else(|| IonError::from("callback: read out of bounds")) - })), - default_config(), - ) - .expect("open_remote must succeed"); - let result = decoder.to_mzml().expect("to_mzml must succeed"); - assert_mzml_semantic_eq(&reference, &result); -} - -#[test] -fn arc_and_callback_open_paths_give_equal_results() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 6, false); - - let arc: Arc<[u8]> = Arc::from(bytes.as_slice()); - let mut arc_decoder = - IonReader::open_source(Arc::new(BytesSource::new(arc.clone())), default_config()) - .expect("open_bytes must succeed"); - let arc_mzml = arc_decoder.to_mzml().expect("arc to_mzml must succeed"); - - let callback_data = arc.clone(); - let mut cb_decoder = IonReader::open_source( - Arc::new(CallbackSource::new(move |range: ByteRange| { - let start = range.offset as usize; - let end = start + range.length as usize; - callback_data - .get(start..end) - .map(|slice| slice.to_vec()) - .ok_or_else(|| IonError::from("callback: out of bounds")) - })), - default_config(), - ) - .expect("open_remote must succeed"); - let cb_mzml = cb_decoder.to_mzml().expect("callback to_mzml must succeed"); - - assert_mzml_semantic_eq(&arc_mzml, &cb_mzml); -} - -#[test] -fn open_source_with_bytes_source_shares_the_arc_instead_of_copying() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 6, false); - let shared: Arc<[u8]> = Arc::from(bytes.as_slice()); - let pointer_before = Arc::as_ptr(&shared); - let strong_count_before = Arc::strong_count(&shared); - - let reader = - IonReader::open_source(Arc::new(BytesSource::new(shared.clone())), default_config()) - .expect("open_source must succeed"); - - assert_eq!( - Arc::as_ptr(&shared), - pointer_before, - "the shared buffer must stay at the same address" - ); - assert!( - Arc::strong_count(&shared) > strong_count_before, - "BytesSource must hold a clone of the Arc rather than a fresh copy of the bytes" - ); - - drop(reader); - assert_eq!( - Arc::strong_count(&shared), - strong_count_before, - "dropping the reader must release the shared clone" - ); -} - -#[test] -fn spectrum_count_and_chromatogram_count_are_stable() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 6, false); - let arc: Arc<[u8]> = Arc::from(bytes.as_slice()); - let decoder = IonReader::open_source(Arc::new(BytesSource::new(arc)), default_config()) - .expect("open must succeed"); - let spec_count = decoder.spectrum_count(); - let chrom_count = decoder.chromatogram_count(); - assert!(spec_count > 0, "spectrum count must be positive"); - let _ = chrom_count; -} - -#[test] -fn scan_source_summary_count_equals_spectrum_count() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 6, false); - let arc: Arc<[u8]> = Arc::from(bytes.as_slice()); - let mut decoder = IonReader::open_source(Arc::new(BytesSource::new(arc)), default_config()) - .expect("open must succeed"); - let total = decoder.spectrum_count(); - let mut counted = 0usize; - decoder.for_each_summary(&mut |_, _| counted += 1); - assert_eq!( - counted as u64, total, - "for_each_summary count must equal spectrum_count" - ); -} - -#[test] -fn load_scan_gives_arrays_for_first_spectrum() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 6, false); - let arc: Arc<[u8]> = Arc::from(bytes.as_slice()); - let mut decoder = IonReader::open_source(Arc::new(BytesSource::new(arc)), default_config()) - .expect("open must succeed"); - let mut mz = Vec::new(); - let mut intensity = Vec::new(); - let loaded = decoder.load_scan(0, &mut mz, &mut intensity); - assert!(loaded, "load_scan must return true for index 0"); - assert!(!mz.is_empty(), "mz must not be empty"); - assert_eq!( - mz.len(), - intensity.len(), - "mz and intensity must have equal length" - ); -} - -#[test] -fn plan_open_ranges_includes_header_range() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 6, false); - let ranges = open_ranges(&bytes).expect("open_ranges must succeed"); - assert!(!ranges.is_empty(), "ranges must not be empty"); - assert_eq!(ranges[0].offset, 0, "first range must start at byte 0"); - assert_eq!(ranges[0].length, 1024, "first range must span 1024 bytes"); -} - -#[test] -fn plan_open_ranges_all_ranges_fit_in_file() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 6, false); - let file_len = bytes.len() as u64; - let ranges = open_ranges(&bytes).expect("open_ranges must succeed"); - for range in &ranges { - let end = range.offset + range.length; - assert!( - end <= file_len, - "range (offset={}, length={}) extends past file end ({file_len})", - range.offset, - range.length, - ); - } -} - -#[test] -fn write_via_memory_reader_matches_direct_encode() { - let src = test_files::tiny_pwiz_11(); - let direct_bytes = encode_to_ion(src, 0, false); - let reference = decode_ion(&direct_bytes).expect("reference decode must succeed"); - - let mut stream_bytes = Vec::new(); - let mut reader = MemoryReader::new(src.clone()); - let mut writer = - IonWriter::create(&mut stream_bytes, stream_write_config()).expect("begin must succeed"); - writer - .write_stream(&mut reader) - .expect("write_stream must succeed"); - let stream_decoded = decode_ion(&stream_bytes).expect("stream decode must succeed"); - assert_mzml_semantic_eq(&reference, &stream_decoded); -} - -#[test] -fn require_bounds_succeeds_for_segmented_file() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_with_segments(src); - let arc: Arc<[u8]> = Arc::from(bytes.as_slice()); - let mut decoder = IonReader::open_source(Arc::new(BytesSource::new(arc)), default_config()) - .expect("open must succeed"); - decoder - .require_bounds() - .expect("require_bounds must succeed when file has window bounds"); -} - -#[test] -fn mz_range_read_returns_data_for_segmented_file() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_with_segments(src); - let arc: Arc<[u8]> = Arc::from(bytes.as_slice()); - let mut decoder = IonReader::open_source(Arc::new(BytesSource::new(arc)), default_config()) - .expect("open must succeed"); - decoder - .require_bounds() - .expect("require_bounds must succeed"); - let window = decoder - .read_window(0, Range { from: 0.0, to: 1e9 }) - .expect("read_mz_range must succeed"); - assert_eq!( - window.x.len(), - window.y.len(), - "window mz and intensity must have equal length" - ); - assert!( - !window.x.is_empty(), - "wide window must return data for a non-empty spectrum" - ); -} - -#[test] -fn mz_range_matches_get_spectrum_for_full_range() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_with_segments(src); - let arc: Arc<[u8]> = Arc::from(bytes.as_slice()); - let mut decoder = IonReader::open_source(Arc::new(BytesSource::new(arc)), default_config()) - .expect("open must succeed"); - decoder - .require_bounds() - .expect("require_bounds must succeed"); - - let spectrum = decoder - .spectrum(0) - .expect("get_spectrum must succeed") - .expect("first spectrum must exist"); - let full_len = spectrum - .binary_data_array_list - .as_ref() - .and_then(|list| list.binary_data_arrays.first()) - .and_then(|bda| bda.binary.as_ref()) - .map(|bin| bin.len()) - .unwrap_or(0); - - let window = decoder - .read_window(0, Range { from: 0.0, to: 1e9 }) - .expect("wide window must succeed"); - assert_eq!( - window.x.len(), - full_len, - "wide mz window must return as many points as the full spectrum" - ); -} - -#[test] -fn mz_range_block_ranges_are_non_empty_for_segmented_file() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_with_segments(src); - let arc: Arc<[u8]> = Arc::from(bytes.as_slice()); - let mut decoder = IonReader::open_source(Arc::new(BytesSource::new(arc)), default_config()) - .expect("open must succeed"); - decoder - .require_bounds() - .expect("require_bounds must succeed"); - let ranges = decoder - .byte_ranges(0, Range { from: 0.0, to: 1e9 }) - .expect("plan_mz_range must succeed"); - assert!( - !ranges.is_empty(), - "wide window must return at least one block range" - ); -} - -#[test] -fn spec_summary_matches_source_facts() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 6, false); - let arc: Arc<[u8]> = Arc::from(bytes.as_slice()); - let decoder = IonReader::open_source(Arc::new(BytesSource::new(arc)), default_config()) - .expect("open must succeed"); - - let source_spectra = common::spectra(src); - assert_eq!( - decoder.spectrum_count() as usize, - source_spectra.len(), - "summary count must match source spectrum count" - ); - - for (index, source) in source_spectra.iter().enumerate() { - let summary = decoder - .spectrum_summary(index) - .expect("spec_summary must return Some for a valid index"); - - let expected_level = source.ms_level.unwrap_or(0) as u8; - assert_eq!( - summary.ms_level, expected_level, - "spectrum {index}: stored ms_level must match source ms_level" - ); - - if let Some(expected_rt) = common::scan_start_time_raw(source) { - assert!( - (summary.rt - expected_rt).abs() <= 1e-6, - "spectrum {index}: stored rt {} must match source raw value {expected_rt}", - summary.rt, - ); - } - - assert_eq!( - summary.rt_unit, - common::scan_start_time_unit_code(source), - "spectrum {index}: stored rt_unit must match the source scan-start-time unit", - ); - } -} - -#[test] -fn format_version_is_supported() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 6, false); - let arc: Arc<[u8]> = Arc::from(bytes.as_slice()); - let decoder = IonReader::open_source(Arc::new(BytesSource::new(arc)), default_config()) - .expect("open must succeed"); - let version = decoder.format_version(); - assert!( - ionic::ion::is_supported(version), - "format_version must be within supported range" - ); -} - -struct GoldenEncode { - name: &'static str, - path: &'static str, - encoded_len: usize, - encoded_fnv: u64, -} - -const GOLDEN_ENCODES: &[GoldenEncode] = &[ - GoldenEncode { - name: "tiny_pwiz_10", - path: "crates/parser/data/mzml/tiny.pwiz.1.0.mzML", - encoded_len: 11464, - encoded_fnv: 0x8809_d202_308a_8999, - }, - GoldenEncode { - name: "tiny_pwiz_11", - path: "crates/parser/data/mzml/tiny.pwiz.1.1.mzML", - encoded_len: 12560, - encoded_fnv: 0x537f_7db5_ad76_3e4c, - }, - GoldenEncode { - name: "anpc_test", - path: "crates/parser/data/mzml/test.mzML", - encoded_len: 8832, - encoded_fnv: 0x7f6a_6713_6c50_2cb5, - }, -]; - -#[test] -fn encoded_bytes_stay_byte_for_byte_stable() { - for golden in GOLDEN_ENCODES { - let src = common::parse_test_file(golden.path); - let bytes = encode_to_ion(&src, 0, false); - assert_eq!( - bytes.len(), - golden.encoded_len, - "{}: encoded length changed (format drift)", - golden.name, - ); - assert_eq!( - common::fnv64_bytes(&bytes), - golden.encoded_fnv, - "{}: encoded bytes changed (format drift)", - golden.name, - ); - } -} - -#[test] -fn spec_array_addresses_crc_tamper_fails_closed_with_verify_on_and_opens_with_verify_off() { - let src = test_files::tiny_pwiz_11(); - let bytes = encode_to_ion(src, 0, false); - - let off_spec_array_addresses = u64::from_le_bytes(bytes[80..88].try_into().unwrap()) as usize; - let len_spec_array_addresses = u64::from_le_bytes(bytes[88..96].try_into().unwrap()) as usize; - assert!( - len_spec_array_addresses > 0, - "spec_array_addresses must be non-empty for this test to be meaningful" - ); - - let mut tampered = bytes.clone(); - tampered[off_spec_array_addresses] ^= 0xFF; - - let verify_on = ReadOptions { - verify_checksums: true, - ..ReadOptions::default() - }; - match IonReader::open(&tampered, verify_on) { - Ok(_) => { - panic!("tampered spec_array_addresses must be rejected when verify_checksums is true") - } - Err(err) => { - let msg = err.to_string(); - assert!( - msg.contains("spec_array_addresses") && msg.contains("crc"), - "error message must name spec_array_addresses and crc, got: {msg}" - ); - } - } - - let verify_off = ReadOptions { - verify_checksums: false, - ..ReadOptions::default() - }; - IonReader::open(&tampered, verify_off) - .expect("tampered spec_array_addresses must open when verify_checksums is false"); -} - -#[test] -fn public_api_surface_is_frozen() { - let mz = ionic::Range { from: 0.0, to: 1.0 }; - let _byte = ionic::ByteRange { - offset: 0, - length: 0, - }; - let pixel = ionic::Pixel { - x: mz, - y: mz, - z: mz, - }; - let _all = ionic::Select::All; - let _rt = ionic::Select::Rt(mz); - let _area = ionic::Select::Area(pixel); - - fn time_unit_is_frozen(unit: ionic::TimeUnit) -> u8 { - match unit { - ionic::TimeUnit::Second => 1, - ionic::TimeUnit::Minute => 2, - ionic::TimeUnit::Millisecond => 3, - ionic::TimeUnit::Other => 0, - } - } - let _ = time_unit_is_frozen(ionic::TimeUnit::Second); - - fn numeric_array_is_frozen(array: &ionic::NumericArray) -> usize { - match array { - ionic::NumericArray::F64(values) => values.len(), - ionic::NumericArray::F32(values) => values.len(), - ionic::NumericArray::F16(values) => values.len(), - ionic::NumericArray::I16(values) => values.len(), - ionic::NumericArray::I32(values) => values.len(), - ionic::NumericArray::I64(values) => values.len(), - } - } - - let data = ionic::DataXY { - x: ionic::NumericArray::F64(vec![1.0]), - y: ionic::NumericArray::F64(vec![2.0]), - }; - let _: Vec = data.x.to_f64(); - let _ = numeric_array_is_frozen(&data.y); - - let src = test_files::tiny_pwiz_11(); - let mut bytes = Vec::new(); - ionic::write_mzml_to_ion(src, ionic::WriteOptions::default(), &mut bytes) - .expect("write_mzml_to_ion must succeed"); - let mut reader = - ionic::IonReader::open(&bytes, ionic::ReadOptions::default()).expect("open must succeed"); - - let _spectrum_count: u64 = reader.spectrum_count(); - let _chromatogram_count: u64 = reader.chromatogram_count(); - - reader.for_each_summary(&mut |index: usize, summary: ionic::ScanSummary| { - let _: usize = index; - let _: f64 = summary.rt; - let _: ionic::TimeUnit = summary.rt_unit; - let _: u8 = summary.ms_level; - let _: u32 = summary.position_x; - }); - - let _read: ionic::DataXY = reader.read_window(0, mz).expect("read_window must succeed"); - - reader - .scans_in( - mz, - ionic::Select::All, - None, - &mut |window: &ionic::Window| { - let _: usize = window.index; - let _: &ionic::ScanSummary = window.summary; - let _: &[f64] = window.mz; - let _: &[f64] = window.intensity; - }, - ) - .expect("scans_in must succeed"); - - let _spectrum: Option = reader.spectrum(0).expect("spectrum must succeed"); - let _ranges: Vec = - reader.byte_ranges(0, mz).expect("byte_ranges must succeed"); - let _mzml: ionic::MzML = reader.to_mzml().expect("to_mzml must succeed"); - - let mut output = Vec::new(); - let mut writer = ionic::IonWriter::create(&mut output, ionic::WriteOptions::default()) - .expect("create must succeed"); - let mut stream = ionic::MemoryReader::new(reader.to_mzml().expect("to_mzml must succeed")); - writer - .write_stream(&mut stream) - .expect("write_stream must succeed"); - - let mut output2 = Vec::new(); - ionic::IonWriter::create(&mut output2, ionic::WriteOptions::default()) - .expect("create must succeed") - .write_mzml(src) - .expect("write_mzml must succeed"); - - let _open: Vec = - ionic::open_ranges(&bytes[..1024]).expect("open_ranges must succeed"); -} diff --git a/crates/parser/tests/api_surface.rs b/crates/parser/tests/api_surface.rs new file mode 100644 index 00000000..44dc2d10 --- /dev/null +++ b/crates/parser/tests/api_surface.rs @@ -0,0 +1,200 @@ +mod common; + +use std::path::PathBuf; + +use common::helpers::{build_mzml, make_chromatogram_f64, make_spectrum_f64}; +use ionic::{ + ArrayKind, ChromatogramSummary, ConvertKind, ConvertOptions, DataXY, DecompressionLimit, + IonError, IonReader, IonResult, IonWriter, Metadatum, MetadatumValue, Range, ReadOptions, + ScanQuery, ScanStream, ScanSummary, SectionStorage, Select, SpectrumSummary, TimeUnit, Window, + WriteOptions, + mzml::structs::MzML, +}; + +fn temp_path(name: &str) -> PathBuf { + std::env::temp_dir().join(format!("ionic-api-surface-{}-{name}", std::process::id())) +} + +fn sample_mzml() -> MzML { + let mz: Vec = (0..50).map(|i| 100.0 + i as f64 * 0.5).collect(); + let intensity: Vec = (0..50).map(|i| i as f64).collect(); + let time: Vec = (0..10).map(|i| i as f64 * 0.1).collect(); + let tic: Vec = (0..10).map(|i| i as f64 * 10.0).collect(); + build_mzml( + vec![make_spectrum_f64("scan=1", mz, intensity)], + vec![make_chromatogram_f64("tic", time, tic)], + ) +} + +#[test] +fn root_read_and_write_round_trip_a_file() { + let path = temp_path("read_write.ion"); + let mzml = sample_mzml(); + + ionic::write(&path, &mzml, &WriteOptions::default()).expect("write must succeed"); + let round_tripped = ionic::read(&path).expect("read must succeed"); + let _ = std::fs::remove_file(&path); + + assert_eq!(round_tripped.run.spectrum_list.unwrap().spectra.len(), 1); +} + +#[test] +fn create_write_spectrum_finish_round_trips_and_open_reads_it_back() { + let path = temp_path("create_push.ion"); + let mz: Vec = (0..20).map(|i| 200.0 + i as f64).collect(); + let intensity: Vec = (0..20).map(|i| i as f64 * 2.0).collect(); + let spectrum = make_spectrum_f64("scan=1", mz.clone(), intensity); + let metadata = build_mzml(vec![], vec![]); + + let mut writer = + IonWriter::create(&path, &metadata, &WriteOptions::default()).expect("create must succeed"); + writer + .write_spectrum(&spectrum) + .expect("write_spectrum must succeed"); + writer.finish().expect("finish must succeed"); + + let mut reader = IonReader::open(&path, &ReadOptions::default()).expect("open must succeed"); + let _ = std::fs::remove_file(&path); + + assert_eq!(reader.spectrum_count(), 1); + let got_mz = reader.array(0, ArrayKind::Mz).expect("array must succeed"); + assert_eq!(got_mz, mz); +} + +#[test] +fn array_into_reuses_the_callers_buffer() { + let mz: Vec = (0..30).map(|i| 300.0 + i as f64).collect(); + let intensity: Vec = (0..30).map(|i| i as f64).collect(); + let spectrum = make_spectrum_f64("scan=1", mz.clone(), intensity); + let mzml = build_mzml(vec![], vec![]); + + let mut bytes = Vec::new(); + let mut writer = IonWriter::to(&mut bytes, &mzml, &WriteOptions::default()).unwrap(); + writer.write_spectrum(&spectrum).unwrap(); + writer.finish().unwrap(); + + let mut reader = IonReader::from_bytes(&bytes, &ReadOptions::default()).unwrap(); + let mut buf = Vec::with_capacity(4096); + let addr_before = buf.as_ptr(); + reader.array_into(0, ArrayKind::Mz, &mut buf).unwrap(); + + assert_eq!(buf, mz); + assert_eq!( + buf.as_ptr(), + addr_before, + "array_into must reuse the caller's allocation" + ); +} + +#[test] +fn convert_and_convert_file_round_trip_bytes_and_paths() { + let mzml = sample_mzml(); + let mut ion_bytes = Vec::new(); + let mut writer = IonWriter::to(&mut ion_bytes, &mzml, &WriteOptions::default()).unwrap(); + for spectrum in &mzml.run.spectrum_list.as_ref().unwrap().spectra { + writer.write_spectrum(spectrum).unwrap(); + } + for chromatogram in &mzml.run.chromatogram_list.as_ref().unwrap().chromatograms { + writer.write_chromatogram(chromatogram).unwrap(); + } + writer.finish().unwrap(); + + let xml = ionic::convert(&ion_bytes, &ConvertOptions::default()).expect("convert must succeed"); + assert!(xml.starts_with(b" = (0..15).map(|i| 400.0 + i as f64).collect(); + let intensity: Vec = (0..15).map(|i| i as f64).collect(); + let spectrum = make_spectrum_f64("scan=1", mz, intensity); + let mzml = build_mzml(vec![], vec![]); + + let mut bytes = Vec::new(); + let mut writer = IonWriter::to(&mut bytes, &mzml, &WriteOptions::default()).unwrap(); + writer.write_spectrum(&spectrum).unwrap(); + writer.finish().unwrap(); + + let mut reader = IonReader::from_bytes(&bytes, &ReadOptions::default()).unwrap(); + let mut seen = 0usize; + reader + .scans_in( + &ScanQuery { + mz: Range { + from: 0.0, + to: f64::MAX, + }, + select: Select::All, + ms_level: None, + }, + |_window| seen += 1, + ) + .unwrap(); + assert_eq!(seen, 1); + + let metadata: MzML = reader.metadata().unwrap(); + assert_eq!(metadata.run.id, "test-run"); + + let summaries: Vec = reader.spectrum_summaries().unwrap(); + assert_eq!(summaries.len(), 1); + let chrom_summaries: Vec = reader.chromatogram_summaries().unwrap(); + assert!(chrom_summaries.is_empty()); +} + +#[test] +fn root_items_are_reachable_and_usable() { + let _kind: ConvertKind = ConvertKind::Auto; + let _options = ConvertOptions::default(); + let _read_options = ReadOptions::default(); + let _write_options = WriteOptions::default(); + let _array_kind = ArrayKind::Mz; + let _range = Range { from: 0.0, to: 1.0 }; + let _select = Select::All; + let _section_storage = SectionStorage::Memory; + let _decompression_limit = DecompressionLimit::default(); + let _query = ScanQuery::default(); + + let error: IonError = IonError::from("example error"); + let _result: IonResult<()> = Err(error); + + fn _accepts_scan_stream(_stream: &mut dyn ScanStream) {} + + fn _accepts_window(window: &Window) { + let _index: usize = window.index; + let _summary: &ScanSummary = window.summary; + let _mz: &[f64] = window.mz; + let _intensity: &[f64] = window.intensity; + } + + fn _accepts_data_xy(data: &DataXY) { + let _x = data.x.to_f64(); + let _y = data.y.to_f64(); + } + + fn _accepts_summaries(spectrum: &SpectrumSummary, chromatogram: &ChromatogramSummary) { + let _rt: f64 = spectrum.rt; + let _ms_level: u8 = spectrum.ms_level; + let _lowest_mz: f64 = chromatogram.lowest_mz; + } + + fn _accepts_metadatum(metadatum: &Metadatum) { + let _id: u32 = metadatum.id; + match &metadatum.value { + MetadatumValue::Number(_) | MetadatumValue::Text(_) | MetadatumValue::Empty => {} + _ => {} + } + } + + fn _accepts_time_unit(_unit: TimeUnit) {} +} diff --git a/crates/parser/tests/array_roundtrips.rs b/crates/parser/tests/array_roundtrips.rs deleted file mode 100644 index d2864cf8..00000000 --- a/crates/parser/tests/array_roundtrips.rs +++ /dev/null @@ -1,551 +0,0 @@ -mod common; - -use common::{ - binary_ext::BinaryDataExt, - decode_ion, encode_to_ion, first_spectrum_binary, - helpers::{build_mzml, make_spectrum_f64, mzml_with_single_array}, - roundtrip, -}; -use ionic::{ - ion::{IonReader, ReadOptions, WriteOptions, write_mzml_to_ion}, - mzml::structs::*, -}; -use proptest::prelude::*; - -#[test] -fn roundtrip_f64_array() { - let values = vec![1.0_f64, -2.5, 0.0, f64::MAX, f64::MIN, std::f64::consts::PI]; - let len = values.len(); - let mzml = mzml_with_single_array(NumericType::Float64, NumericArray::F64(values.clone()), len); - let out = roundtrip(&mzml); - let bin = first_spectrum_binary(&out).expect("should have binary data"); - let got = bin.to_f64_vec(); - assert_eq!(got, values); -} - -#[test] -fn roundtrip_f32_array() { - let values = vec![1.0_f32, -2.5, 0.0, f32::MAX, f32::MIN, std::f32::consts::PI]; - let len = values.len(); - let mzml = mzml_with_single_array(NumericType::Float32, NumericArray::F32(values.clone()), len); - let out = roundtrip(&mzml); - let bin = first_spectrum_binary(&out).expect("should have binary data"); - let got = bin.to_f64_vec(); - let expected: Vec = values.iter().map(|v| *v as f64).collect(); - assert_eq!(got, expected); -} - -#[test] -fn array_filter_label_matches_applied_transform() { - let raw_filter = 0u8; - let f64_dtype = 1u8; - let f32_dtype = 2u8; - - let f32_values = vec![10.0_f32, 11.0, 12.0, 13.0]; - let f32_len = f32_values.len(); - let f32_mzml = - mzml_with_single_array(NumericType::Float32, NumericArray::F32(f32_values), f32_len); - - let f64_values = vec![100.0_f64, 100.5, 101.0, 101.5]; - let f64_len = f64_values.len(); - let f64_mzml = - mzml_with_single_array(NumericType::Float64, NumericArray::F64(f64_values), f64_len); - - let f32_bytes = encode_to_ion(&f32_mzml, 9, false); - let f32_decoder = IonReader::open(&f32_bytes, ReadOptions::default()).expect("open f32 ion"); - let f32_refs = f32_decoder - .spectrum_array_addresses(0) - .expect("f32 array refs"); - - let f64_bytes = encode_to_ion(&f64_mzml, 9, false); - let f64_decoder = IonReader::open(&f64_bytes, ReadOptions::default()).expect("open f64 ion"); - let f64_refs = f64_decoder - .spectrum_array_addresses(0) - .expect("f64 array refs"); - - assert!( - f32_refs - .iter() - .any(|a| a.dtype() == f32_dtype && a.array_filter() == raw_filter), - "f32 intensity must be tagged raw" - ); - assert!( - f64_refs - .iter() - .any(|a| a.dtype() == f64_dtype && a.array_filter() == raw_filter), - "f64 intensity must be tagged raw (intensity is never delta-shuffled)" - ); -} - -#[test] -fn roundtrip_i64_array() { - let values = vec![0_i64, 1, -1, i64::MAX, i64::MIN, 42]; - let len = values.len(); - let mzml = mzml_with_single_array(NumericType::Int64, NumericArray::I64(values.clone()), len); - let out = roundtrip(&mzml); - let bin = first_spectrum_binary(&out).expect("should have binary data"); - match bin { - NumericArray::I64(got) => assert_eq!(got, &values), - other => { - let got = other.to_f64_vec(); - let expected: Vec = values.iter().map(|v| *v as f64).collect(); - assert_eq!( - got, expected, - "i64 roundtrip values differ (via f64 conversion)" - ); - } - } -} - -#[test] -fn roundtrip_i32_array() { - let values = vec![0_i32, 1, -1, i32::MAX, i32::MIN, 42]; - let len = values.len(); - let mzml = mzml_with_single_array(NumericType::Int32, NumericArray::I32(values.clone()), len); - let out = roundtrip(&mzml); - let bin = first_spectrum_binary(&out).expect("should have binary data"); - match bin { - NumericArray::I32(got) => assert_eq!(got, &values), - other => { - let got = other.to_f64_vec(); - let expected: Vec = values.iter().map(|v| *v as f64).collect(); - assert_eq!(got, expected, "i32 roundtrip values differ"); - } - } -} - -#[test] -fn roundtrip_i16_array() { - let values = vec![0_i16, 1, -1, i16::MAX, i16::MIN, 42]; - let len = values.len(); - let mzml = mzml_with_single_array(NumericType::Int16, NumericArray::I16(values.clone()), len); - let out = roundtrip(&mzml); - let bin = first_spectrum_binary(&out).expect("should have binary data"); - match bin { - NumericArray::I16(got) => assert_eq!(got, &values), - other => { - let got = other.to_f64_vec(); - let expected: Vec = values.iter().map(|v| *v as f64).collect(); - assert_eq!(got, expected, "i16 roundtrip values differ"); - } - } -} - -#[test] -fn roundtrip_single_element_per_type() { - let cases: Vec<(NumericType, NumericArray)> = vec![ - (NumericType::Float64, NumericArray::F64(vec![42.0])), - (NumericType::Float32, NumericArray::F32(vec![42.0])), - (NumericType::Int64, NumericArray::I64(vec![42])), - (NumericType::Int32, NumericArray::I32(vec![42])), - (NumericType::Int16, NumericArray::I16(vec![42])), - ]; - - for (nt, bin) in cases { - let mzml = mzml_with_single_array(nt, bin.clone(), 1); - let out = roundtrip(&mzml); - let got = first_spectrum_binary(&out).expect("should have binary data"); - let expected_f64 = bin.to_f64_vec(); - let got_f64 = got.to_f64_vec(); - assert_eq!( - got_f64, expected_f64, - "single-element roundtrip failed for {nt:?}" - ); - } -} -#[test] -fn roundtrip_at_compression_levels() { - let values = vec![100.0_f64, 200.0, 300.0, 400.0, 500.0]; - let len = values.len(); - let mzml = mzml_with_single_array(NumericType::Float64, NumericArray::F64(values.clone()), len); - - for level in [0, 3, 10, 22] { - let mut buf = Vec::new(); - write_mzml_to_ion( - &mzml, - WriteOptions { - compression_level: level, - force_f32: false, - ..Default::default() - }, - &mut buf, - ) - .unwrap_or_else(|e| panic!("encode at level {level} failed: {e}")); - let mut decoder = IonReader::open(&buf, ReadOptions::default()) - .unwrap_or_else(|e| panic!("decoder open at level {level} failed: {e}")); - let decoded = decoder - .to_mzml() - .unwrap_or_else(|e| panic!("to_mzml at level {level} failed: {e}")); - let got = first_spectrum_binary(&decoded).expect("should have binary data"); - assert_eq!( - got.to_f64_vec(), - values, - "values differ at compression level {level}" - ); - } -} -#[test] -fn roundtrip_force_f32_downcasts() { - let values = vec![100.0_f64, 200.0, 300.0]; - let len = values.len(); - let mzml = mzml_with_single_array(NumericType::Float64, NumericArray::F64(values.clone()), len); - - let mut buf = Vec::new(); - write_mzml_to_ion( - &mzml, - WriteOptions { - compression_level: 0, - force_f32: true, - ..Default::default() - }, - &mut buf, - ) - .expect("encode with force_f32"); - let mut decoder = IonReader::open(&buf, ReadOptions::default()).expect("decoder open"); - let decoded = decoder.to_mzml().expect("to_mzml"); - - let bin = first_spectrum_binary(&decoded).expect("should have binary data"); - let got = bin.to_f64_vec(); - for (i, (g, e)) in got.iter().zip(values.iter()).enumerate() { - let expected_f32 = *e as f32 as f64; - assert!( - (g - expected_f32).abs() < 1e-6, - "index {i}: force_f32 value mismatch: {g} vs {expected_f32}" - ); - } -} - -fn encode_decode_mz_compressed(mz: Vec) -> Vec { - let len = mz.len(); - let mzml = mzml_with_single_array(NumericType::Float64, NumericArray::F64(mz), len); - let buf = encode_to_ion(&mzml, 3, false); - let decoded = decode_ion(&buf).unwrap(); - first_spectrum_binary(&decoded).unwrap().to_f64_vec() -} - -#[test] -fn delta_mz_single_element_is_bit_exact() { - let input = vec![503.42f64]; - let got = encode_decode_mz_compressed(input.clone()); - assert_eq!(got[0].to_bits(), input[0].to_bits()); -} - -#[test] -fn delta_mz_two_elements_are_bit_exact() { - let input = vec![100.0f64, 200.5]; - let got = encode_decode_mz_compressed(input.clone()); - for (a, b) in got.iter().zip(input.iter()) { - assert_eq!(a.to_bits(), b.to_bits()); - } -} - -#[test] -fn delta_mz_monotonic_array_is_bit_exact() { - let input: Vec = (0..10_000).map(|i| 100.0 + i as f64 * 0.01).collect(); - let got = encode_decode_mz_compressed(input.clone()); - for (a, b) in got.iter().zip(input.iter()) { - assert_eq!(a.to_bits(), b.to_bits()); - } -} - -#[test] -fn delta_mz_special_values_are_bit_exact() { - let input = vec![f64::NAN, f64::INFINITY, f64::NEG_INFINITY, -0.0f64, 0.0f64]; - let got = encode_decode_mz_compressed(input.clone()); - for (a, b) in got.iter().zip(input.iter()) { - assert_eq!(a.to_bits(), b.to_bits()); - } -} - -#[test] -fn delta_mz_via_for_each_scan_is_bit_exact() { - use ionic::{IonReader, ScanSource}; - let input: Vec = (0..500).map(|i| 100.0 + i as f64 * 0.05).collect(); - let intensity: Vec = vec![1.0; input.len()]; - let mut spectrum = make_spectrum_f64("scan=1", input.clone(), intensity); - spectrum.scan_list = Some(ScanList { - count: Some(1), - scans: vec![Scan { - cv_params: vec![CvParam { - cv_ref: Some("MS".to_string()), - accession: Some("MS:1000016".to_string()), - name: "scan start time".to_string(), - value: Some("1.0".to_string()), - unit_accession: Some("UO:0000031".to_string()), - ..Default::default() - }], - ..Default::default() - }], - ..Default::default() - }); - let mzml = build_mzml(vec![spectrum], vec![]); - let buf = encode_to_ion(&mzml, 3, false); - let mut ion = IonReader::open(&buf, ReadOptions::default()).unwrap(); - let mut got_mz: Vec = Vec::new(); - ion.for_each_in_range(0.0, f64::MAX, 0, |_, mz: &[f64], _| { - got_mz = mz.to_vec(); - }); - assert_eq!(got_mz.len(), input.len()); - for (a, b) in got_mz.iter().zip(input.iter()) { - assert_eq!(a.to_bits(), b.to_bits()); - } -} - -#[test] -fn delta_on_mz_raw_on_intensity() { - use ionic::ion::IonReader as IonDecoder; - let mz: Vec = (0..100).map(|i| 100.0 + i as f64).collect(); - let intensity: Vec = (0..100).map(|i| (i * 10) as f64).collect(); - let mzml = build_mzml( - vec![make_spectrum_f64("scan=1", mz.clone(), intensity.clone())], - vec![], - ); - let buf = encode_to_ion(&mzml, 3, false); - let mut decoder = IonDecoder::open(&buf, ReadOptions::default()).unwrap(); - let refs = decoder.spectrum_array_addresses(0).unwrap(); - let mz_address = refs.iter().find(|r| r.array_type() == 1_000_514).unwrap(); - let int_ref = refs.iter().find(|r| r.array_type() == 1_000_515).unwrap(); - assert_eq!( - mz_address.array_filter(), - 2, - "m/z must use DeltaShuffle filter" - ); - assert_eq!( - int_ref.array_filter(), - 0, - "intensity must use raw filter (intensity is never delta-shuffled)" - ); - let mut got_mz = Vec::new(); - decoder - .read_spectrum_values(mz_address, &mut got_mz) - .unwrap(); - let mut got_int = Vec::new(); - decoder.read_spectrum_values(int_ref, &mut got_int).unwrap(); - for (a, b) in got_mz.iter().zip(mz.iter()) { - assert_eq!(a.to_bits(), b.to_bits()); - } - for (a, b) in got_int.iter().zip(intensity.iter()) { - assert_eq!(a.to_bits(), b.to_bits()); - } -} - -#[test] -fn format_version_always_matches_current() { - use ionic::ion::{CURRENT_VERSION, HEADER_FORMAT_VERSION_OFFSET}; - let values = vec![1.0_f64, 2.0, 3.0, 4.0, 5.0]; - let len = values.len(); - let mzml = mzml_with_single_array(NumericType::Float64, NumericArray::F64(values.clone()), len); - for level in [0u8, 3, 22] { - let buf = encode_to_ion(&mzml, level, false); - let format_version = u16::from_le_bytes( - buf[HEADER_FORMAT_VERSION_OFFSET..HEADER_FORMAT_VERSION_OFFSET + 2] - .try_into() - .unwrap(), - ); - assert_eq!( - format_version, CURRENT_VERSION, - "format_version must match CURRENT_VERSION at compression level {level}" - ); - } - let out = roundtrip(&mzml); - let bin = first_spectrum_binary(&out).expect("binary data"); - for (a, b) in bin.to_f64_vec().iter().zip(values.iter()) { - assert_eq!(a.to_bits(), b.to_bits()); - } -} - -#[test] -fn delta_not_applied_without_compression() { - use ionic::ion::IonReader as IonDecoder; - let mz: Vec = (0..10).map(|i| 100.0 + i as f64).collect(); - let intensity = vec![1.0_f64; mz.len()]; - let mzml = build_mzml( - vec![make_spectrum_f64("scan=1", mz.clone(), intensity)], - vec![], - ); - let buf = encode_to_ion(&mzml, 0, false); - let mut decoder = IonDecoder::open(&buf, ReadOptions::default()).unwrap(); - let refs = decoder.spectrum_array_addresses(0).unwrap(); - let mz_address = refs.iter().find(|r| r.array_type() == 1_000_514).unwrap(); - assert_eq!(mz_address.array_filter(), 0, "no delta without compression"); - let mut got = Vec::new(); - decoder.read_spectrum_values(mz_address, &mut got).unwrap(); - for (a, b) in got.iter().zip(mz.iter()) { - assert_eq!(a.to_bits(), b.to_bits()); - } -} - -#[test] -fn delta_shuffle_applied_to_time_array() { - use common::helpers::make_chromatogram_f64; - use ionic::ion::IonReader as IonDecoder; - let time: Vec = (0..100).map(|i| i as f64 * 0.1).collect(); - let intensity: Vec = vec![1.0; time.len()]; - let mzml = build_mzml( - vec![], - vec![make_chromatogram_f64("tic", time.clone(), intensity)], - ); - let buf = encode_to_ion(&mzml, 3, false); - let mut decoder = IonDecoder::open(&buf, ReadOptions::default()).unwrap(); - let refs = decoder.chromatogram_array_addresses(0).unwrap(); - let time_ref = refs.iter().find(|r| r.array_type() == 1_000_595).unwrap(); - assert_eq!( - time_ref.array_filter(), - 2, - "time must use DeltaShuffle filter" - ); - let mut got = Vec::new(); - decoder - .read_chromatogram_values(time_ref, &mut got) - .unwrap(); - for (a, b) in got.iter().zip(time.iter()) { - assert_eq!(a.to_bits(), b.to_bits()); - } -} - -#[test] -fn read_chromatogram_array_keeps_native_f32_width_33() { - use common::helpers::make_chromatogram_f64; - let time: Vec = (0..64).map(|i| i as f64 * 0.5).collect(); - let intensity: Vec = (0..64).map(|i| i as f64).collect(); - let mzml = build_mzml(vec![], vec![make_chromatogram_f64("tic", time, intensity)]); - let buf = encode_to_ion(&mzml, 3, true); - let mut decoder = IonReader::open(&buf, ReadOptions::default()).unwrap(); - let refs = decoder.chromatogram_array_addresses(0).unwrap(); - let intensity_ref = refs.iter().find(|r| r.array_type() == 1_000_515).unwrap(); - let native = decoder.read_chromatogram_array(intensity_ref).unwrap(); - assert!( - matches!(native, NumericArray::F32(_)), - "read_chromatogram_array must keep the stored native f32 width" - ); - let mut widened = Vec::new(); - decoder - .read_chromatogram_values(intensity_ref, &mut widened) - .unwrap(); - assert_eq!( - widened.len(), - 64, - "read_chromatogram_values must still widen to f64" - ); -} - -proptest! { - #![proptest_config(ProptestConfig::with_cases(32))] - - #[test] - fn proptest_arbitrary_f64_roundtrip( - values in prop::collection::vec(prop::num::f64::ANY, 1..64) - ) { - let len = values.len(); - let mzml = mzml_with_single_array( - NumericType::Float64, - NumericArray::F64(values.clone()), - len, - ); - let out = roundtrip(&mzml); - let bin = first_spectrum_binary(&out).expect("should have binary data"); - let got = bin.to_f64_vec(); - prop_assert_eq!(got.len(), values.len()); - for (i, (g, e)) in got.iter().zip(values.iter()).enumerate() { - if e.is_nan() { - prop_assert!(g.is_nan(), "index {}: expected NaN", i); - } else { - prop_assert_eq!(g.to_bits(), e.to_bits(), "index {}: bit mismatch", i); - } - } - } - - #[test] - fn proptest_delta2vbyte_mz_roundtrip( - mz in prop::collection::vec(0.0f64..100_000.0, 3..256) - ) { - let mut sorted_mz = mz.clone(); - sorted_mz.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - let intensity = vec![1.0f64; sorted_mz.len()]; - let mzml = build_mzml( - vec![make_spectrum_f64("scan=1", sorted_mz.clone(), intensity)], - vec![], - ); - let buf = encode_to_ion(&mzml, 3, false); - let got = decode_ion(&buf).unwrap(); - let bin = first_spectrum_binary(&got).expect("should have binary data"); - let decoded = bin.to_f64_vec(); - prop_assert_eq!(decoded.len(), sorted_mz.len()); - for (i, (g, e)) in decoded.iter().zip(sorted_mz.iter()).enumerate() { - if e.is_nan() { - prop_assert!(g.is_nan(), "index {}: expected NaN", i); - } else { - prop_assert_eq!(g.to_bits(), e.to_bits(), "index {}: bit mismatch", i); - } - } - } - - #[test] - fn proptest_chimp_time_roundtrip( - time in prop::collection::vec(prop::num::f64::ANY, 2..256) - ) { - use common::helpers::make_chromatogram_f64; - use common::first_chrom_array_values_by_accession; - let intensity = vec![1.0f64; time.len()]; - let mzml = build_mzml( - vec![], - vec![make_chromatogram_f64("tic", time.clone(), intensity)], - ); - let buf = encode_to_ion(&mzml, 3, false); - let got = decode_ion(&buf).unwrap(); - let chrom_list = got.run.chromatogram_list.unwrap(); - let decoded = first_chrom_array_values_by_accession(&chrom_list.chromatograms[0], "MS:1000595"); - prop_assert_eq!(decoded.len(), time.len()); - for (i, (g, e)) in decoded.iter().zip(time.iter()).enumerate() { - if e.is_nan() { - prop_assert!(g.is_nan(), "index {}: expected NaN", i); - } else { - prop_assert_eq!(g.to_bits(), e.to_bits(), "index {}: bit mismatch", i); - } - } - } - - #[test] - fn proptest_arbitrary_i32_roundtrip( - values in prop::collection::vec(prop::num::i32::ANY, 1..64) - ) { - let len = values.len(); - let mzml = mzml_with_single_array( - NumericType::Int32, - NumericArray::I32(values.clone()), - len, - ); - let out = roundtrip(&mzml); - let bin = first_spectrum_binary(&out).expect("should have binary data"); - match bin { - NumericArray::I32(got) => prop_assert_eq!(got, &values), - other => { - let got = other.to_f64_vec(); - let expected: Vec = values.iter().map(|v| *v as f64).collect(); - prop_assert_eq!(got, expected); - } - } - } - - #[test] - fn proptest_arbitrary_i16_roundtrip( - values in prop::collection::vec(prop::num::i16::ANY, 1..64) - ) { - let len = values.len(); - let mzml = mzml_with_single_array( - NumericType::Int16, - NumericArray::I16(values.clone()), - len, - ); - let out = roundtrip(&mzml); - let bin = first_spectrum_binary(&out).expect("should have binary data"); - match bin { - NumericArray::I16(got) => prop_assert_eq!(got, &values), - other => { - let got = other.to_f64_vec(); - let expected: Vec = values.iter().map(|v| *v as f64).collect(); - prop_assert_eq!(got, expected); - } - } - } -} diff --git a/crates/parser/tests/binary_encoder.rs b/crates/parser/tests/binary_encoder.rs index e2a3cf49..b6e0c5ce 100644 --- a/crates/parser/tests/binary_encoder.rs +++ b/crates/parser/tests/binary_encoder.rs @@ -3,7 +3,7 @@ mod common; use common::{ decode_ion, encode_to_ion, test_files, top_level_software_ids, top_level_source_file_ids, }; -use ionic::ion::{FILE_SIGNATURE, HEADER_SIZE}; +use ionic::format::{FILE_SIGNATURE, HEADER_SIZE}; #[test] fn ion_header_signature_is_correct() { diff --git a/crates/parser/tests/common/mod.rs b/crates/parser/tests/common/mod.rs index 4140e9c1..c57e32dd 100644 --- a/crates/parser/tests/common/mod.rs +++ b/crates/parser/tests/common/mod.rs @@ -13,7 +13,7 @@ use std::{collections::BTreeSet, fs, path::PathBuf, sync::OnceLock}; #[allow(unused_imports)] pub(crate) use binary_ext::BinaryDataExt; use ionic::{ - ion::{IonReader, IonResult, ReadOptions, WriteOptions, write_mzml_to_ion}, + IonReader, IonResult, ReadOptions, WriteOptions, mzml::{ parse_mzml::{parse_indexed_mzml, parse_mzml}, structs::*, @@ -50,6 +50,25 @@ pub(crate) fn parse_indexed(rel: &str) -> IndexedmzML { .unwrap_or_else(|e| panic!("parse_indexed_mzml failed for {rel}: {e}")) } +pub(crate) fn write_mzml_to_ion( + mzml: &MzML, + options: WriteOptions, + output: &mut Vec, +) -> IonResult<()> { + let mut writer = ionic::IonWriter::to(output, mzml, &options)?; + if let Some(list) = &mzml.run.spectrum_list { + for spectrum in &list.spectra { + writer.write_spectrum(spectrum)?; + } + } + if let Some(list) = &mzml.run.chromatogram_list { + for chromatogram in &list.chromatograms { + writer.write_chromatogram(chromatogram)?; + } + } + writer.finish() +} + pub(crate) fn encode_to_ion(mzml: &MzML, compression_level: u8, force_f32: bool) -> Vec { let mut out = Vec::new(); write_mzml_to_ion( @@ -66,7 +85,7 @@ pub(crate) fn encode_to_ion(mzml: &MzML, compression_level: u8, force_f32: bool) } pub(crate) fn decode_ion(bytes: &[u8]) -> IonResult { - let mut decoder = IonReader::open(bytes, ReadOptions::default())?; + let mut decoder = IonReader::from_bytes(bytes, &ReadOptions::default())?; decoder.to_mzml() } diff --git a/crates/parser/tests/concurrent_encode.rs b/crates/parser/tests/concurrent_encode.rs index 66ae8223..6f947199 100644 --- a/crates/parser/tests/concurrent_encode.rs +++ b/crates/parser/tests/concurrent_encode.rs @@ -2,11 +2,8 @@ mod common; use std::{sync::Arc, thread}; -use common::{parse_test_file, test_files::PWIZ_TEST_FILES}; -use ionic::{ - ion::{IonReader, ReadOptions, SectionStorage, WriteOptions, write_mzml_to_ion}, - mzml::structs::MzML, -}; +use common::{parse_test_file, test_files::PWIZ_TEST_FILES, write_mzml_to_ion}; +use ionic::{IonReader, ReadOptions, SectionStorage, WriteOptions, mzml::structs::MzML}; const THREADS: usize = 4; @@ -59,12 +56,12 @@ fn encoding_the_same_file_on_many_threads_gives_identical_bytes() { fn every_concurrently_encoded_file_keeps_its_spectrum_bounds() { let source = Arc::new(parse_test_file(PWIZ_TEST_FILES[4]).clone()); let expected_count = encode(&source); - let expected_count = IonReader::open(&expected_count, ReadOptions::default()) + let expected_count = IonReader::from_bytes(&expected_count, &ReadOptions::default()) .expect("reference open must succeed") .spectrum_count(); for (thread_index, bytes) in encode_on_threads(source, THREADS).into_iter().enumerate() { - let mut reader = IonReader::open(&bytes, ReadOptions::default()).unwrap_or_else(|error| { + let mut reader = IonReader::from_bytes(&bytes, &ReadOptions::default()).unwrap_or_else(|error| { panic!("thread {thread_index} wrote an unreadable file: {error}") }); assert_eq!( @@ -88,7 +85,7 @@ fn encoding_different_files_at_the_same_time_keeps_every_result_valid() { let expected: Vec = sources .iter() .map(|source| { - IonReader::open(&encode(source), ReadOptions::default()) + IonReader::from_bytes(&encode(source), &ReadOptions::default()) .expect("reference open must succeed") .spectrum_count() }) @@ -103,7 +100,7 @@ fn encoding_different_files_at_the_same_time_keeps_every_result_valid() { for handle in handles { let (index, bytes) = handle.join().expect("encoder thread must not panic"); let name = PWIZ_TEST_FILES[index]; - let mut reader = IonReader::open(&bytes, ReadOptions::default()) + let mut reader = IonReader::from_bytes(&bytes, &ReadOptions::default()) .unwrap_or_else(|error| panic!("{name} came back unreadable: {error}")); assert_eq!( reader.spectrum_count(), diff --git a/crates/parser/tests/convert.rs b/crates/parser/tests/convert.rs index 47d212f5..d5c043d8 100644 --- a/crates/parser/tests/convert.rs +++ b/crates/parser/tests/convert.rs @@ -1,16 +1,11 @@ mod common; -use std::{ - fs, - path::{Path, PathBuf}, -}; +use std::{fs, path::PathBuf}; use common::{ encode_to_ion, parse_test_file, read_test_file, repo_root, test_files::ALL_TEST_FILES, }; -use ionic::{ - ConvertKind, ConvertOptions, IonReader, ReadOptions, bin_to_mzml, ion::FILE_SIGNATURE, -}; +use ionic::{ConvertKind, ConvertOptions, IonReader, ReadOptions, format::FILE_SIGNATURE, mzml::bin_to_mzml}; const MZML: &str = "crates/parser/data/mzml/tiny.pwiz.1.1.mzML"; const ION: &str = "crates/parser/data/ion/tiny.msdata.mzML0.99.9.ion"; @@ -23,13 +18,6 @@ fn temp_path(name: &str) -> PathBuf { std::env::temp_dir().join(format!("ionic-convert-{}-{name}", std::process::id())) } -fn options_with_output(path: &Path) -> ConvertOptions { - ConvertOptions { - output: Some(path.to_path_buf()), - ..Default::default() - } -} - fn options_with_kind(kind: ConvertKind) -> ConvertOptions { ConvertOptions { kind, @@ -42,7 +30,7 @@ fn expected_ion_bytes() -> Vec { } fn expected_mzml_xml() -> Vec { - let mut reader = IonReader::open_file(&fixture_path(ION), ReadOptions::default()).unwrap(); + let mut reader = IonReader::open(fixture_path(ION), &ReadOptions::default()).unwrap(); bin_to_mzml(&reader.to_mzml().unwrap()).unwrap() } @@ -55,56 +43,38 @@ fn copy_fixture(relative_path: &str, name: &str) -> PathBuf { #[test] fn mzml_path_to_ion_path_writes_the_file() { let out = temp_path("a.ion"); - let result = ionic::convert(fixture_path(MZML), options_with_output(&out)).unwrap(); + ionic::convert_file(fixture_path(MZML), &out, &ConvertOptions::default()).unwrap(); let written = fs::read(&out).unwrap(); fs::remove_file(&out).unwrap(); - assert!(result.is_none()); assert_eq!(written, expected_ion_bytes()); } #[test] fn ion_path_to_mzml_path_writes_the_file() { let out = temp_path("b.mzML"); - let result = ionic::convert(fixture_path(ION), options_with_output(&out)).unwrap(); + ionic::convert_file(fixture_path(ION), &out, &ConvertOptions::default()).unwrap(); let written = fs::read(&out).unwrap(); fs::remove_file(&out).unwrap(); - assert!(result.is_none()); assert_eq!(written, expected_mzml_xml()); } -#[test] -fn mzml_path_to_memory_returns_the_ion_bytes() { - let result = ionic::convert(fixture_path(MZML), ConvertOptions::default()).unwrap(); - assert_eq!(result, Some(expected_ion_bytes())); -} - #[test] fn mzml_bytes_to_memory_returns_the_ion_bytes() { - let result = ionic::convert(&read_test_file(MZML), ConvertOptions::default()).unwrap(); - assert_eq!(result, Some(expected_ion_bytes())); -} - -#[test] -fn ion_path_to_memory_returns_the_xml() { - let result = ionic::convert(fixture_path(ION), ConvertOptions::default()).unwrap(); - assert_eq!(result, Some(expected_mzml_xml())); + let result = ionic::convert(&read_test_file(MZML), &ConvertOptions::default()).unwrap(); + assert_eq!(result, expected_ion_bytes()); } #[test] fn ion_bytes_to_memory_returns_the_xml() { - let result = ionic::convert(&read_test_file(ION), ConvertOptions::default()).unwrap(); - assert_eq!(result, Some(expected_mzml_xml())); + let result = ionic::convert(&read_test_file(ION), &ConvertOptions::default()).unwrap(); + assert_eq!(result, expected_mzml_xml()); } #[test] fn auto_reads_the_direction_from_the_file_signature_for_a_buffer() { - let ion = ionic::convert(&read_test_file(MZML), ConvertOptions::default()) - .unwrap() - .unwrap(); + let ion = ionic::convert(&read_test_file(MZML), &ConvertOptions::default()).unwrap(); assert!(ion.starts_with(&FILE_SIGNATURE)); - let xml = ionic::convert(&read_test_file(ION), ConvertOptions::default()) - .unwrap() - .unwrap(); + let xml = ionic::convert(&read_test_file(ION), &ConvertOptions::default()).unwrap(); assert!(xml.starts_with(b" = Vec::new(); - assert!(ionic::convert(&empty, options_with_kind(ConvertKind::IonToMzml)).is_err()); + assert!(ionic::convert(&empty, &options_with_kind(ConvertKind::IonToMzml)).is_err()); } diff --git a/crates/parser/tests/decode_rejection.rs b/crates/parser/tests/decode_rejection.rs index ad598eb1..5683bcf5 100644 --- a/crates/parser/tests/decode_rejection.rs +++ b/crates/parser/tests/decode_rejection.rs @@ -2,7 +2,8 @@ mod common; use common::{assertions::assert_mzml_semantic_eq, decode_ion, encode_to_ion, test_files}; use ionic::{ - ion::{HEADER_FORMAT_VERSION_OFFSET, IonReader, IonResult, MAX_SUPPORTED_VERSION, ReadOptions}, + IonReader, IonResult, ReadOptions, + format::{HEADER_FORMAT_VERSION_OFFSET, MAX_SUPPORTED_VERSION}, mzml::structs::MzML, }; @@ -43,7 +44,7 @@ fn decode_ion_without_checksum_verification(bytes: &[u8]) -> IonResult { verify_checksums: false, ..ReadOptions::default() }; - let mut decoder = IonReader::open(bytes, options)?; + let mut decoder = IonReader::from_bytes(bytes, &options)?; decoder.to_mzml() } @@ -70,7 +71,7 @@ fn rejects_unsupported_format_version() { #[test] fn accepts_all_supported_format_versions() { - use ionic::ion::{MIN_SUPPORTED_VERSION, allow_version}; + use ionic::format::{MIN_SUPPORTED_VERSION, allow_version}; for version in MIN_SUPPORTED_VERSION..=MAX_SUPPORTED_VERSION { assert!( allow_version(version).is_ok(), @@ -161,7 +162,7 @@ fn rejects_huge_spec_entry_count_with_checksums_off_4() { }; bytes[off + 8..off + 16].copy_from_slice(&u64::MAX.to_le_bytes()); - let err = IonReader::open(&bytes, config.clone()) + let err = IonReader::from_bytes(&bytes, &config.clone()) .map(|_| ()) .expect_err("decode must reject a huge spec entry count without checksums"); assert!(err.contains("overflow"), "unexpected decode error: {err}"); @@ -170,7 +171,7 @@ fn rejects_huge_spec_entry_count_with_checksums_off_4() { let wrapping_count = table_len / 32 + (1u64 << 59); bytes[off..off + 8].copy_from_slice(&0u64.to_le_bytes()); bytes[off + 8..off + 16].copy_from_slice(&wrapping_count.to_le_bytes()); - let err = IonReader::open(&bytes, config) + let err = IonReader::from_bytes(&bytes, &config) .map(|_| ()) .expect_err("decode must reject a count whose product wraps to the table size"); assert!(err.contains("overflow"), "unexpected decode error: {err}"); diff --git a/crates/parser/tests/empty_inputs.rs b/crates/parser/tests/empty_inputs.rs index 0d9ba329..6cdd9cb6 100644 --- a/crates/parser/tests/empty_inputs.rs +++ b/crates/parser/tests/empty_inputs.rs @@ -1,7 +1,8 @@ mod common; +use common::write_mzml_to_ion; use ionic::{ - ion::{IonReader, ReadOptions, WriteOptions, write_mzml_to_ion}, + IonReader, ReadOptions, WriteOptions, mzml::{ bin_to_mzml::bin_to_mzml, parse_mzml::{parse_indexed_mzml, parse_mzml}, @@ -137,7 +138,7 @@ fn encode_mzml_no_arrays() { #[test] fn decoder_open_empty_bytes() { - let result = IonReader::open(b"", ReadOptions::default()); + let result = IonReader::from_bytes(b"", &ReadOptions::default()); assert!( result.is_err(), "empty bytes should not be a valid Ion container" @@ -147,7 +148,7 @@ fn decoder_open_empty_bytes() { #[test] fn decoder_open_garbage() { let garbage = vec![0xDE, 0xAD, 0xBE, 0xEF]; - let result = IonReader::open(&garbage, ReadOptions::default()); + let result = IonReader::from_bytes(&garbage, &ReadOptions::default()); assert!( result.is_err(), "garbage should not be a valid Ion container" @@ -156,7 +157,7 @@ fn decoder_open_garbage() { #[test] fn decoder_open_too_small() { - let result = IonReader::open(&[0x01, 0x02, 0x03], ReadOptions::default()); + let result = IonReader::from_bytes(&[0x01, 0x02, 0x03], &ReadOptions::default()); assert!( result.is_err(), "3 bytes should not be a valid Ion container" @@ -185,7 +186,7 @@ fn roundtrip_empty_mzml() { ) .expect("encode should succeed"); - let mut decoder = IonReader::open(&buf, ReadOptions::default()).expect("decoder should open"); + let mut decoder = IonReader::from_bytes(&buf, &ReadOptions::default()).expect("decoder should open"); let decoded = decoder.to_mzml().expect("to_mzml should succeed"); assert_eq!(decoded.run.id, "roundtrip-empty"); } diff --git a/crates/parser/tests/file_encoder.rs b/crates/parser/tests/file_encoder.rs index 06d44b04..e0690531 100644 --- a/crates/parser/tests/file_encoder.rs +++ b/crates/parser/tests/file_encoder.rs @@ -2,11 +2,30 @@ mod common; use std::fs; -use common::helpers::{minimal_file_description, synthetic_binary_data_array}; -use ionic::{ - ion::{FileWriter, IonReader, ReadOptions, WriteOptions, write_mzml_to_ion}, - mzml::structs::*, +use common::{ + helpers::{minimal_file_description, synthetic_binary_data_array}, + write_mzml_to_ion, }; +use ionic::{IonReader, ReadOptions, WriteOptions, mzml::structs::*}; + +fn write_via_push( + mzml: &MzML, + path: &std::path::Path, + options: &WriteOptions, +) -> ionic::IonResult<()> { + let mut writer = ionic::IonWriter::create(path, mzml, options)?; + if let Some(list) = &mzml.run.spectrum_list { + for spectrum in &list.spectra { + writer.write_spectrum(spectrum)?; + } + } + if let Some(list) = &mzml.run.chromatogram_list { + for chromatogram in &list.chromatograms { + writer.write_chromatogram(chromatogram)?; + } + } + writer.finish() +} #[test] fn memory_mode_roundtrip_multi_spectrum() { @@ -25,7 +44,7 @@ fn memory_mode_roundtrip_multi_spectrum() { .expect("encode should succeed"); assert!(!buf.is_empty(), "output should not be empty"); - let mut decoder = IonReader::open(&buf, ReadOptions::default()).expect("decoder open"); + let mut decoder = IonReader::from_bytes(&buf, &ReadOptions::default()).expect("decoder open"); let decoded = decoder.to_mzml().expect("to_mzml"); let orig_spectra = common::spectra(&mzml); @@ -48,26 +67,21 @@ fn streaming_mode_roundtrip_via_tempfile() { let temp_dir = std::env::temp_dir(); let temp_path = temp_dir.join("ionic_test_streaming.ion"); - let mut file_output = FileWriter::open(temp_path.to_str().unwrap()).unwrap_or_else(|e| { - panic!("failed to create FileWriter: {e}"); - }); - - write_mzml_to_ion( + write_via_push( &mzml, - WriteOptions { + &temp_path, + &WriteOptions { compression_level: 3, force_f32: false, ..Default::default() }, - &mut file_output, ) .expect("streaming encode should succeed"); - drop(file_output); let bytes = fs::read(&temp_path).expect("should read temp file"); assert!(!bytes.is_empty(), "file should not be empty"); - let mut decoder = IonReader::open(&bytes, ReadOptions::default()).expect("decoder open"); + let mut decoder = IonReader::from_bytes(&bytes, &ReadOptions::default()).expect("decoder open"); let decoded = decoder.to_mzml().expect("to_mzml"); assert_eq!( @@ -96,26 +110,23 @@ fn memory_and_streaming_produce_equivalent_results() { .expect("memory encode"); let temp_path = std::env::temp_dir().join("ionic_test_equiv.ion"); - let mut file_output = - FileWriter::open(temp_path.to_str().unwrap()).expect("create file output"); - write_mzml_to_ion( + write_via_push( &mzml, - WriteOptions { + &temp_path, + &WriteOptions { compression_level: 0, force_f32: false, ..Default::default() }, - &mut file_output, ) .expect("streaming encode"); - drop(file_output); // ensure flush let stream_buf = fs::read(&temp_path).expect("read temp file"); - let mut mem_decoder = IonReader::open(&mem_buf, ReadOptions::default()).expect("mem decoder"); + let mut mem_decoder = IonReader::from_bytes(&mem_buf, &ReadOptions::default()).expect("mem decoder"); let mem_decoded = mem_decoder.to_mzml().expect("mem to_mzml"); let mut stream_decoder = - IonReader::open(&stream_buf, ReadOptions::default()).expect("stream decoder"); + IonReader::from_bytes(&stream_buf, &ReadOptions::default()).expect("stream decoder"); let stream_decoded = stream_decoder.to_mzml().expect("stream to_mzml"); let diffs = common::canonical_diff_paths(&mem_decoded, &stream_decoded); @@ -182,7 +193,7 @@ fn large_array_roundtrip_stress() { ) .expect("encode large array"); - let mut decoder = IonReader::open(&buf, ReadOptions::default()).expect("decoder open"); + let mut decoder = IonReader::from_bytes(&buf, &ReadOptions::default()).expect("decoder open"); let decoded = decoder.to_mzml().expect("to_mzml"); let dec_spectra = common::spectra(&decoded); diff --git a/crates/parser/tests/ion_stream_writer.rs b/crates/parser/tests/ion_stream_writer.rs index 7b9ee26b..5fc619b7 100644 --- a/crates/parser/tests/ion_stream_writer.rs +++ b/crates/parser/tests/ion_stream_writer.rs @@ -4,16 +4,15 @@ use std::fs; use common::{canonical_diff_paths, decode_ion}; use ionic::{ - IonWriter, - ion::{IonReader, ReadOptions, SectionStorage, TARGET_BLOCK_UNCOMPRESSED_BYTES, WriteOptions}, - mzml::{MzmlReader, parse_mzml::parse_mzml}, + IonReader, IonWriter, ReadOptions, SectionStorage, WriteOptions, + mzml::{MzmlReader, parse_mzml::parse_mzml, structs::MzML}, }; fn config() -> WriteOptions { WriteOptions { compression_level: 0, force_f32: false, - block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, + block_size: WriteOptions::default().block_size, parallel: false, section_storage: SectionStorage::Memory, mz_window: 0.0, @@ -60,7 +59,7 @@ fn stream_writer_roundtrips_like_mzml_writer() { let mzml = parse_mzml(coordinate_xml()).unwrap(); let mut reader = MzmlReader::from_mzml(mzml.clone()); let mut bytes = Vec::new(); - let mut writer = IonWriter::create(&mut bytes, config()).unwrap(); + let mut writer = IonWriter::to(&mut bytes, &MzML::default(), &config()).unwrap(); writer.write_stream(&mut reader).unwrap(); let decoded = decode_ion(&bytes).unwrap(); let diffs = canonical_diff_paths(&mzml, &decoded); @@ -72,9 +71,9 @@ fn spectrum_summary_keeps_coordinates() { let mzml = parse_mzml(coordinate_xml()).unwrap(); let mut reader = MzmlReader::from_mzml(mzml); let mut bytes = Vec::new(); - let mut writer = IonWriter::create(&mut bytes, config()).unwrap(); + let mut writer = IonWriter::to(&mut bytes, &MzML::default(), &config()).unwrap(); writer.write_stream(&mut reader).unwrap(); - let ion = IonReader::open(&bytes, ReadOptions::default()).unwrap(); + let ion = IonReader::from_bytes(&bytes, &ReadOptions::default()).unwrap(); let summary = ion.spectrum_summary(0).unwrap(); assert_eq!(summary.position_x, 11); assert_eq!(summary.position_y, 22); @@ -90,7 +89,7 @@ fn file_stream_reader_roundtrips_spectra_then_chromatograms() { let mzml = parse_mzml(mixed_xml()).unwrap(); let mut reader = MzmlReader::open(&path).unwrap(); let mut bytes = Vec::new(); - let mut writer = IonWriter::create(&mut bytes, config()).unwrap(); + let mut writer = IonWriter::to(&mut bytes, &MzML::default(), &config()).unwrap(); writer.write_stream(&mut reader).unwrap(); let decoded = decode_ion(&bytes).unwrap(); let diffs = canonical_diff_paths(&mzml, &decoded); diff --git a/crates/parser/tests/ion_writer_differ.rs b/crates/parser/tests/ion_writer_differ.rs index ea61f424..10363c85 100644 --- a/crates/parser/tests/ion_writer_differ.rs +++ b/crates/parser/tests/ion_writer_differ.rs @@ -2,17 +2,14 @@ mod common; use std::path::PathBuf; -use common::canonical_diff_paths; +use common::{canonical_diff_paths, write_mzml_to_ion}; use ionic::{ - ion::{ - IonReader, ReadOptions, SectionStorage, TARGET_BLOCK_UNCOMPRESSED_BYTES, WriteOptions, - write_mzml_to_ion, - }, + IonReader, ReadOptions, SectionStorage, WriteOptions, mzml::parse_mzml::parse_mzml, }; fn decode_ion_bytes(bytes: &[u8]) -> ionic::mzml::structs::MzML { - let mut ion = IonReader::open(bytes, ReadOptions::default()).unwrap(); + let mut ion = IonReader::from_bytes(bytes, &ReadOptions::default()).unwrap(); ion.to_mzml().unwrap() } @@ -40,7 +37,7 @@ fn config_for_level(compression_level: u8) -> WriteOptions { WriteOptions { compression_level, force_f32: false, - block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, + block_size: WriteOptions::default().block_size, parallel: false, section_storage: SectionStorage::Memory, mz_window: 0.0, diff --git a/crates/parser/tests/mz_order_rejection.rs b/crates/parser/tests/mz_order_rejection.rs index d3fb0a03..158e4f78 100644 --- a/crates/parser/tests/mz_order_rejection.rs +++ b/crates/parser/tests/mz_order_rejection.rs @@ -1,16 +1,10 @@ mod common; -use std::sync::Arc; - use ionic::{ - BytesSource, - ion::{ - IonReader, IonWriter, MemoryReader, Range, ReadOptions, SectionStorage, - TARGET_BLOCK_UNCOMPRESSED_BYTES, WriteOptions, - }, + IonReader, IonResult, IonWriter, Range, ReadOptions, ScanStream, SectionStorage, WriteOptions, mzml::structs::{ - BinaryDataArray, BinaryDataArrayList, CvParam, MzML, NumericArray, Run, Spectrum, - SpectrumList, + BinaryDataArray, BinaryDataArrayList, Chromatogram, CvParam, MzML, NumericArray, Run, + Spectrum, SpectrumList, }, }; @@ -18,13 +12,72 @@ fn write_config() -> WriteOptions { WriteOptions { compression_level: 0, force_f32: false, - block_size: TARGET_BLOCK_UNCOMPRESSED_BYTES, + block_size: WriteOptions::default().block_size, parallel: false, section_storage: SectionStorage::Memory, mz_window: 0.0, } } +struct OnceStream { + metadata: MzML, + spectra: std::vec::IntoIter, + chromatograms: std::vec::IntoIter, +} + +impl OnceStream { + fn new(mzml: MzML) -> Self { + let spectra = mzml + .run + .spectrum_list + .clone() + .map(|list| list.spectra) + .unwrap_or_default() + .into_iter(); + let chromatograms = mzml + .run + .chromatogram_list + .clone() + .map(|list| list.chromatograms) + .unwrap_or_default() + .into_iter(); + Self { + metadata: mzml, + spectra, + chromatograms, + } + } +} + +impl ScanStream for OnceStream { + fn metadata(&mut self) -> IonResult { + Ok(self.metadata.clone()) + } + + fn next_spectrum(&mut self) -> IonResult> { + Ok(self.spectra.next()) + } + + fn next_chromatogram(&mut self) -> IonResult> { + Ok(self.chromatograms.next()) + } +} + +fn write_via_push(mzml: &MzML, options: &WriteOptions, output: &mut Vec) -> IonResult<()> { + let mut writer = IonWriter::to(output, &MzML::default(), options)?; + if let Some(list) = &mzml.run.spectrum_list { + for spectrum in &list.spectra { + writer.write_spectrum(spectrum)?; + } + } + if let Some(list) = &mzml.run.chromatogram_list { + for chromatogram in &list.chromatograms { + writer.write_chromatogram(chromatogram)?; + } + } + writer.finish() +} + fn mz_cv_param() -> CvParam { CvParam { cv_ref: Some("MS".to_string()), @@ -101,9 +154,7 @@ fn write_mzml_rejects_unsorted_mz_spectrum() { vec![1.0, 2.0, 3.0], )]); let mut bytes = Vec::new(); - let result = IonWriter::create(&mut bytes, write_config()) - .expect("begin must succeed") - .write_mzml(&mzml); + let result = write_via_push(&mzml, &write_config(), &mut bytes); match result { Err(err) => { let msg = err.to_string(); @@ -127,9 +178,9 @@ fn write_stream_rejects_unsorted_mz_spectrum() { vec![100.0, 99.0, 101.0], vec![1.0, 2.0, 3.0], )]); - let mut reader = MemoryReader::new(mzml); + let mut reader = OnceStream::new(mzml); let mut bytes = Vec::new(); - let result = IonWriter::create(&mut bytes, write_config()) + let result = IonWriter::to(&mut bytes, &MzML::default(), &write_config()) .expect("begin must succeed") .write_stream(&mut reader); match result { @@ -155,9 +206,7 @@ fn write_mzml_rejects_second_spectrum_unsorted_names_correct_index() { spectrum_with_mz("scan=2", vec![100.0, 99.0, 101.0], vec![1.0, 2.0, 3.0]), ]); let mut bytes = Vec::new(); - let result = IonWriter::create(&mut bytes, write_config()) - .expect("begin must succeed") - .write_mzml(&mzml); + let result = write_via_push(&mzml, &write_config(), &mut bytes); match result { Err(err) => { let msg = err.to_string(); @@ -178,13 +227,9 @@ fn write_mzml_accepts_sorted_mz_spectrum_and_roundtrips() { vec![1.0, 2.0, 3.0], )]); let mut bytes = Vec::new(); - IonWriter::create(&mut bytes, write_config()) - .expect("begin must succeed") - .write_mzml(&mzml) - .expect("sorted m/z must be accepted"); + write_via_push(&mzml, &write_config(), &mut bytes).expect("sorted m/z must be accepted"); assert!(!bytes.is_empty(), "output must be non-empty"); - let arc: Arc<[u8]> = Arc::from(bytes.as_slice()); - IonReader::open_source(Arc::new(BytesSource::new(arc)), ReadOptions::default()) + IonReader::from_bytes(&bytes, &ReadOptions::default()) .expect("file must open after encoding sorted m/z"); } @@ -196,14 +241,9 @@ fn read_mz_range_works_on_sorted_spectrum() { vec![1.0, 2.0, 3.0], )]); let mut bytes = Vec::new(); - IonWriter::create(&mut bytes, write_config()) - .expect("begin must succeed") - .write_mzml(&mzml) - .expect("sorted m/z must be accepted"); - let arc: Arc<[u8]> = Arc::from(bytes.as_slice()); + write_via_push(&mzml, &write_config(), &mut bytes).expect("sorted m/z must be accepted"); let mut decoder = - IonReader::open_source(Arc::new(BytesSource::new(arc)), ReadOptions::default()) - .expect("open must succeed"); + IonReader::from_bytes(&bytes, &ReadOptions::default()).expect("open must succeed"); decoder.require_bounds().expect("window bounds must exist"); let window = decoder .read_window( @@ -231,9 +271,7 @@ fn write_mzml_accepts_equal_adjacent_mz_values() { vec![1.0, 2.0, 3.0], )]); let mut bytes = Vec::new(); - IonWriter::create(&mut bytes, write_config()) - .expect("begin must succeed") - .write_mzml(&mzml) + write_via_push(&mzml, &write_config(), &mut bytes) .expect("non-decreasing (equal adjacent values) must be accepted"); assert!(!bytes.is_empty()); } diff --git a/crates/parser/tests/roundtrip_ion.rs b/crates/parser/tests/roundtrip_ion.rs index d0f594b9..73d1f425 100644 --- a/crates/parser/tests/roundtrip_ion.rs +++ b/crates/parser/tests/roundtrip_ion.rs @@ -1,7 +1,7 @@ mod common; use common::{assertions::*, chromatograms, decode_ion, encode_to_ion, spectra, test_files}; -use ionic::ion::FILE_SIGNATURE; +use ionic::format::FILE_SIGNATURE; #[test] fn tiny_11_level12_with_header_check() { From 632a07bb1dfb928e609caa6230e6d9c67f5777ca Mon Sep 17 00:00:00 2001 From: josoriom Date: Wed, 23 Sep 2026 01:24:02 +0800 Subject: [PATCH 2/2] refactor!: upgrade cosmoz to 0.2.0 + read metadata on demand + simplify IonReader API --- Cargo.toml | 2 +- README.md | 117 ++-- config.json | 8 + crates/cli/Cargo.toml | 1 + crates/cli/src/main.rs | 288 ++++----- crates/cli/src/utilities/color.rs | 43 ++ crates/cli/src/utilities/header.rs | 164 ++--- crates/cli/src/utilities/mod.rs | 1 + crates/parser/examples/bench_window.rs | 10 +- crates/parser/src/api/convert.rs | 45 +- crates/parser/src/api/writer.rs | 54 +- .../parser/src/ion/decoder/decode/arrays.rs | 351 +++++------ crates/parser/src/ion/decoder/decode/mod.rs | 524 ++++++++-------- .../parser/src/ion/decoder/decode/spectra.rs | 296 ++------- crates/parser/src/ion/decoder/decode/tests.rs | 571 ++++++++++++++---- .../parser/src/ion/decoder/decode/to_mzml.rs | 67 +- .../parser/src/ion/decoder/decode/windows.rs | 357 +++++------ .../src/ion/decoder/utilities/common.rs | 31 +- .../decoder/utilities/meta_group_reader.rs | 217 +++---- .../parser/src/ion/decoder/utilities/mod.rs | 10 +- .../utilities/parse_chromatogram_list.rs | 53 +- .../decoder/utilities/parse_spectrum_list.rs | 51 +- .../ion/decoder/utilities/spectrum_source.rs | 157 +---- .../{parse_header.rs => header_parse.rs} | 6 +- .../src/ion/decoder/utilities/tests/meta.rs | 69 +-- .../src/ion/decoder/utilities/tests/mod.rs | 2 +- .../utilities/tests/parse_file_description.rs | 4 +- .../decoder/utilities/tests/parse_metadata.rs | 10 +- crates/parser/src/ion/encoder/encode.rs | 25 +- crates/parser/src/ion/encoder/ion_writer.rs | 33 +- .../src/ion/encoder/utilities/block_writer.rs | 2 - .../encoder/utilities/meta_collector/mod.rs | 1 - .../encoder/utilities/meta_collector/tests.rs | 107 ++-- .../src/ion/encoder/utilities/output.rs | 5 - crates/parser/src/ion/header.rs | 154 ++--- crates/parser/src/ion/meta_groups.rs | 24 +- crates/parser/src/ion/mod.rs | 17 +- crates/parser/src/ion/packing/byte_shuffle.rs | 12 +- .../parser/src/ion/packing/delta_shuffle.rs | 45 +- crates/parser/src/ion/packing/mod.rs | 58 +- crates/parser/src/ion/packing/raw.rs | 41 +- crates/parser/src/ion/scan.rs | 2 - crates/parser/src/ion/version_generated.rs | 1 - crates/parser/src/mzml/bin_to_mzml.rs | 74 +-- crates/parser/src/mzml/schema.rs | 8 +- crates/parser/src/mzml/structs.rs | 17 - crates/parser/src/utilities/test.rs | 3 +- crates/parser/tests/api_surface.rs | 12 +- crates/parser/tests/decode_rejection.rs | 18 + crates/parser/tests/ion_stream_writer.rs | 6 +- crates/parser/tests/mz_order_rejection.rs | 7 +- crates/xtask/Cargo.toml | 1 + crates/xtask/src/main.rs | 3 +- 53 files changed, 1955 insertions(+), 2230 deletions(-) create mode 100644 crates/cli/src/utilities/color.rs rename crates/parser/src/ion/decoder/utilities/tests/{parse_header.rs => header_parse.rs} (94%) diff --git a/Cargo.toml b/Cargo.toml index 477d1bf7..afd095d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ authors = [ "Jorge Osorio M " ] edition = "2024" -rust-version = "1.85" +rust-version = "1.88" repository = "https://github.com/phenological/ionic" homepage = "https://github.com/phenological/ionic" diff --git a/README.md b/README.md index a6e5b376..bb52e7ff 100644 --- a/README.md +++ b/README.md @@ -46,34 +46,23 @@ A command-line tool for converting mzML files to Ionic. See the [CLI](crates/cli ## Usage -| Item | Available | What it does | -|---|---|---| -| `ionic::read` / `ionic::write` | std | Whole `.ion` file to/from an `MzML`, in one call. | -| `ionic::convert` | everywhere | Convert an in-memory buffer between mzML and Ionic. | -| `ionic::convert_file` | std | Convert between two paths, streaming to disk. | -| `IonReader::open` | std | Open a `.ion` file by path (memory-mapped). | -| `IonReader::from_bytes` | everywhere | Open a `.ion` file already in memory. | -| `IonReader::new` | everywhere | Open from any `ionic::source::ReadBytes` (partial/remote reads). | -| `IonWriter::create` | std | Write a new `.ion` file by path. | -| `IonWriter::to` | everywhere | Write into any `ionic::source::WriteBytes` sink, such as a `Vec`. | -| `ionic::mzml` | everywhere | mzML types and parser/serializer: `MzML`, `Spectrum`, `Chromatogram`, `NumericArray`, `parse_mzml`, `bin_to_mzml`, ... | -| `ionic::source` | everywhere | Partial/remote-read building blocks: `ReadBytes`, `WriteBytes`, `ByteRange`, `CallbackSource`, `header_ranges`, `merge_ranges`. | -| `ionic::format` | everywhere | File-format constants used by tooling: `CURRENT_VERSION`, `HEADER_SIZE`, `FILE_SIGNATURE`, `is_supported`, ... | +| Item | What it does | +|---|---| +| `ionic::read` / `ionic::write` | Whole `.ion` file to/from an `MzML`, in one call. (std) | +| `ionic::convert` | Convert an in-memory buffer between mzML and Ionic. | +| `ionic::convert_file` | Convert between two paths, streaming to disk. (std) | +| `IonReader::open` | Open a `.ion` file by path (memory-mapped). (std) | +| `IonReader::from_bytes` | Open a `.ion` file already in memory. | +| `IonReader::new` | Open from any `ionic::source::ReadBytes` (partial/remote reads). | +| `IonWriter::create` | Write a new `.ion` file by path. (std) | +| `IonWriter::to` | Write into any `ionic::source::WriteBytes` sink, such as a `Vec`. | +| `ionic::mzml` | mzML types and parser/serializer: `MzML`, `Spectrum`, `Chromatogram`, `NumericArray`, `parse_mzml`, `bin_to_mzml`, ... | +| `ionic::source` | Partial/remote-read building blocks: `ReadBytes`, `WriteBytes`, `ByteRange`, `CallbackSource`, `header_ranges`, `merge_ranges`. | +| `ionic::format` | File-format constants used by tooling: `CURRENT_VERSION`, `HEADER_SIZE`, `FILE_SIGNATURE`, `is_supported`, ... | "std" means the item needs a real filesystem and is not available on `wasm32-unknown-unknown`; "everywhere" means it also works there. -### Read, one call - -```rust -fn main() -> Result<(), Box> { - let mzml = ionic::read("run.ion")?; - let count = mzml.run.spectrum_list.map_or(0, |list| list.spectra.len()); - println!("{count} spectra"); - Ok(()) -} -``` - -### Reader with options +### Read ```rust use ionic::{ArrayKind, IonReader, ReadOptions}; @@ -84,13 +73,30 @@ fn main() -> Result<(), Box> { ..ReadOptions::default() }; let mut reader = IonReader::open("run.ion", &options)?; - let mz = reader.array(0, ArrayKind::Mz)?; - let intensity = reader.array(0, ArrayKind::Intensity)?; + let mz = reader.spectrum_array(0, ArrayKind::Mz)?; + let intensity = reader.spectrum_array(0, ArrayKind::Intensity)?; println!("{} points", mz.len().min(intensity.len())); Ok(()) } ``` +### Streaming reader + +Read one spectrum at a time. Memory stays small, even for large files. + +```rust +use ionic::{IonReader, ReadOptions}; + +fn main() -> Result<(), Box> { + let mut reader = IonReader::open("run.ion", &ReadOptions::default())?; + for index in 0..reader.spectrum_count() as usize { + let spectrum = reader.spectrum_metadata_at(index)?; + println!("{}", spectrum.id); + } + Ok(()) +} +``` + `ReadOptions::default()`: | Field | Default | @@ -100,7 +106,26 @@ fn main() -> Result<(), Box> { | `parallel` | `true` | | `decompression_limit` | `DecompressionLimit::default()` (2 GiB uncompressed cap) | -### Write, one call +### Partial reads — ionic::source + +`byte_ranges` and `eic_byte_ranges` turn a query into the exact byte ranges a remote source +would need to fetch, without reading them; that is what lets a viewer such as +[ion-beam](https://github.com/phenological/ion-beam) show only the bytes it downloaded. + +```rust +use ionic::{IonReader, Range, ReadOptions}; + +fn main() -> Result<(), Box> { + let mut reader = IonReader::open("run.ion", &ReadOptions::default())?; + let ranges = reader.byte_ranges(0, Range { from: 200.0, to: 400.0 })?; + for range in ranges { + println!("fetch bytes {}..{}", range.offset, range.offset + range.length); + } + Ok(()) +} +``` + +### Write ```rust use ionic::WriteOptions; @@ -114,20 +139,19 @@ fn main() -> Result<(), Box> { ### Streaming writer -The metadata passed to `create`/`to` carries run-level information; spectra and chromatograms -come from `write_spectrum`/`write_chromatogram` (or `write_stream`), not from the metadata -argument. `finish` is required — `Drop` does not finish the file. +The metadata passed to `create`/`to` holds run-level data only. Add spectra and chromatograms +with `write_spectrum`/`write_chromatogram`, then call `finish`. `Drop` does not finish the file. +`write_stream` writes a whole stream and finishes the file in one call. ```rust -use ionic::{IonReader, IonWriter, ReadOptions, WriteOptions}; +use ionic::{IonWriter, WriteOptions, mzml::parse_mzml}; fn main() -> Result<(), Box> { - let mut reader = IonReader::open("run.ion", &ReadOptions::default())?; - let metadata = reader.metadata()?; + let mzml = parse_mzml(&std::fs::read("run.mzML")?)?; - let mut writer = IonWriter::create("out.ion", &metadata, &WriteOptions::default())?; - for index in 0..reader.spectrum_count() as usize { - writer.write_spectrum(&reader.spectrum(index)?)?; + let mut writer = IonWriter::create("out.ion", &mzml, &WriteOptions::default())?; + for spectrum in mzml.run.spectrum_list.iter().flat_map(|list| &list.spectra) { + writer.write_spectrum(spectrum)?; } writer.finish()?; Ok(()) @@ -168,25 +192,6 @@ fn main() -> Result<(), Box> { | `read` | `ReadOptions::default()` | | `write` | `WriteOptions::default()` | -### Partial reads — ionic::source - -`byte_ranges` and `eic_byte_ranges` turn a query into the exact byte ranges a remote source -would need to fetch, without reading them; that is what lets a viewer such as -[ion-beam](https://github.com/phenological/ion-beam) show only the bytes it downloaded. - -```rust -use ionic::{IonReader, Range, ReadOptions}; - -fn main() -> Result<(), Box> { - let mut reader = IonReader::open("run.ion", &ReadOptions::default())?; - let ranges = reader.byte_ranges(0, Range { from: 200.0, to: 400.0 })?; - for range in ranges { - println!("fetch bytes {}..{}", range.offset, range.offset + range.length); - } - Ok(()) -} -``` - ## How the format works A fixed 1024-byte header stores the byte offset of every section. A reader uses these offsets to go directly to the index, then decompresses only the blocks a query needs. [spec/README.MD](spec/README.MD) specifies the design; [spec/v1.md](spec/v1.md) gives the exact byte layout. diff --git a/config.json b/config.json index 52e4f920..557abd66 100644 --- a/config.json +++ b/config.json @@ -1,4 +1,12 @@ [ + { + "package": "0.2.0", + "format": { + "current": 1, + "min_supported": 0, + "max_supported": 1 + } + }, { "package": "0.1.3", "format": { diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 970f0fb5..62211cd5 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -3,6 +3,7 @@ name = "cli" version.workspace = true license.workspace = true edition.workspace = true +rust-version.workspace = true publish = false [dependencies] diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 06061d33..34994671 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,9 +1,9 @@ use std::{ fs::{self, File, OpenOptions}, - io::{IsTerminal, Read, Seek, SeekFrom, Write, stderr, stdout}, + io::{Read, Seek, SeekFrom, Write, stderr, stdout}, path::{Path, PathBuf}, sync::{ - Mutex, OnceLock, + Mutex, atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}, }, time::Instant, @@ -20,13 +20,13 @@ use ionic::{ mzml::{parse_mzml::parse_mzml, structs::*}, }; use mimalloc::MiMalloc; -use rayon::{ThreadPoolBuilder, prelude::*}; +use rayon::{ThreadPool, ThreadPoolBuilder, prelude::*}; use regex::Regex; use serde::Serialize; mod utilities; -use utilities::{TempOutput, check_ion_file, ion_file_is_valid, sweep_orphans}; +use utilities::{TempOutput, check_ion_file, color, ion_file_is_valid, sweep_orphans}; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; @@ -35,38 +35,6 @@ const VERSION: &str = env!("CARGO_PKG_VERSION"); const MB: f64 = 1024.0 * 1024.0; -static COLOR_ENABLED: OnceLock = OnceLock::new(); - -fn color_enabled() -> bool { - *COLOR_ENABLED.get_or_init(|| { - std::env::var_os("NO_COLOR").is_none() && stdout().is_terminal() && stderr().is_terminal() - }) -} - -fn ansi(code: &'static str) -> &'static str { - if color_enabled() { code } else { "" } -} - -fn ansi_reset() -> &'static str { - ansi("\x1b[0m") -} - -fn ansi_green() -> &'static str { - ansi("\x1b[1;32m") -} - -fn ansi_yellow() -> &'static str { - ansi("\x1b[1;33m") -} - -fn ansi_red() -> &'static str { - ansi("\x1b[1;31m") -} - -fn ansi_blue() -> &'static str { - ansi("\x1b[1;34m") -} - const AFTER_HELP: &str = " \x1b[1;33mQUICK REFERENCE\x1b[0m (full flags are in `ionic convert --help` / `ionic cat --help`) @@ -255,12 +223,6 @@ impl SectionStorageArg { } } -#[derive(Clone, Copy)] -enum Encoding { - WithinFileParallel, - FileLevelParallel, -} - #[derive(Args)] #[command( group( @@ -293,8 +255,6 @@ struct CatArgs { } fn main() -> std::process::ExitCode { - color_enabled(); - let mut cmd = Cli::command(); cmd = cmd .styles(cli_styles()) @@ -357,7 +317,7 @@ fn cat(cmd: CatArgs) -> Result<(), String> { if let Some(n) = cmd.chrom_full { return cat_chromatogram(&file_path, n, true); } - let mut mzml = read_mzml_or_ion(&file_path)?; + let mut mzml = read_mzml_or_ion(&file_path, cmd.full)?; if !cmd.full { trim_mzml_for_cat(&mut mzml); } @@ -366,21 +326,15 @@ fn cat(cmd: CatArgs) -> Result<(), String> { fn cat_spectrum(file_path: &Path, index_1based: u32, with_arrays: bool) -> Result<(), String> { let index = (index_1based - 1) as usize; - let mut spectrum = load_spectrum_at(file_path, index)? + let spectrum = load_spectrum_at(file_path, index, with_arrays)? .ok_or_else(|| format!("spectrum {index_1based} is out of range"))?; - if !with_arrays { - spectrum.binary_data_array_list = None; - } print_json_compact(&spectrum) } fn cat_chromatogram(file_path: &Path, index_1based: u32, with_arrays: bool) -> Result<(), String> { let index = (index_1based - 1) as usize; - let mut chromatogram = load_chromatogram_at(file_path, index)? + let chromatogram = load_chromatogram_at(file_path, index, with_arrays)? .ok_or_else(|| format!("chromatogram {index_1based} is out of range"))?; - if !with_arrays { - chromatogram.binary_data_array_list = None; - } print_json_compact(&chromatogram) } @@ -392,36 +346,79 @@ fn ion_result_to_option(result: ionic::IonResult) -> Result, Str } } -fn load_spectrum_at(file_path: &Path, index: usize) -> Result, String> { +fn load_spectrum_at( + file_path: &Path, + index: usize, + with_arrays: bool, +) -> Result, String> { match file_ext_lower(file_path).as_str() { "ion" => { let mut ion = IonReader::open(file_path, &ReadOptions::default()) .map_err(|e| format!("IonReader::open failed: {e}"))?; - ion_result_to_option(ion.spectrum(index)) + if with_arrays { + let mut mzml = ion.to_mzml().map_err(|e| format!("to_mzml failed: {e}"))?; + Ok(mzml.run.spectrum_list.as_mut().and_then(|l| { + (index < l.spectra.len()).then(|| std::mem::take(&mut l.spectra[index])) + })) + } else { + let spectrum = ion_result_to_option(ion.spectrum_metadata_at(index))?; + Ok(spectrum.map(|mut spectrum| { + spectrum.binary_data_array_list = None; + spectrum + })) + } } "mzml" => { let bytes = fs::read(file_path).map_err(|e| format!("read failed: {e}"))?; let mut mzml = parse_mzml(&bytes).map_err(|e| format!("parse_mzml failed: {e}"))?; Ok(mzml.run.spectrum_list.as_mut().and_then(|l| { - (index < l.spectra.len()).then(|| std::mem::take(&mut l.spectra[index])) + (index < l.spectra.len()).then(|| { + let mut spectrum = std::mem::take(&mut l.spectra[index]); + if !with_arrays { + spectrum.binary_data_array_list = None; + } + spectrum + }) })) } other => Err(format!("unsupported file extension: {other:?}")), } } -fn load_chromatogram_at(file_path: &Path, index: usize) -> Result, String> { +fn load_chromatogram_at( + file_path: &Path, + index: usize, + with_arrays: bool, +) -> Result, String> { match file_ext_lower(file_path).as_str() { "ion" => { let mut ion = IonReader::open(file_path, &ReadOptions::default()) .map_err(|e| format!("IonReader::open failed: {e}"))?; - ion_result_to_option(ion.chromatogram(index)) + if with_arrays { + let mut mzml = ion.to_mzml().map_err(|e| format!("to_mzml failed: {e}"))?; + Ok(mzml.run.chromatogram_list.as_mut().and_then(|l| { + (index < l.chromatograms.len()) + .then(|| std::mem::take(&mut l.chromatograms[index])) + })) + } else { + let chromatogram = ion_result_to_option(ion.chromatogram_metadata_at(index))?; + Ok(chromatogram.map(|mut chromatogram| { + chromatogram.binary_data_array_list = None; + chromatogram + })) + } } "mzml" => { let bytes = fs::read(file_path).map_err(|e| format!("read failed: {e}"))?; let mut mzml = parse_mzml(&bytes).map_err(|e| format!("parse_mzml failed: {e}"))?; Ok(mzml.run.chromatogram_list.as_mut().and_then(|l| { - (index < l.chromatograms.len()).then(|| std::mem::take(&mut l.chromatograms[index])) + (index < l.chromatograms.len()).then(|| { + let mut chromatogram = std::mem::take(&mut l.chromatograms[index]); + if !with_arrays { + chromatogram.binary_data_array_list = None; + } + chromatogram + }) })) } other => Err(format!("unsupported file extension: {other:?}")), @@ -435,12 +432,12 @@ fn file_ext_lower(path: &Path) -> String { .to_ascii_lowercase() } -fn out_name_for_mzml_file(path: &Path, out_ext: &str) -> Option { +fn out_name_for_mzml_file(path: &Path) -> Option { if file_ext_lower(path) != "mzml" { return None; } let stem = path.file_stem()?.to_string_lossy(); - Some(format!("{stem}.{out_ext}")) + Some(format!("{stem}.ion")) } fn out_name_for_bin_file_as_mzml(path: &Path) -> Option { @@ -470,13 +467,31 @@ fn in_place_output_root(input_root: &Path) -> PathBuf { } } -fn read_mzml_or_ion(file_path: &Path) -> Result { +fn read_mzml_or_ion(file_path: &Path, with_items: bool) -> Result { let ext = file_ext_lower(file_path); if ext == "ion" { - let ion = IonReader::open(file_path, &ReadOptions::default()) + let mut ion = IonReader::open(file_path, &ReadOptions::default()) .map_err(|e| format!("IonReader::open failed: {e}"))?; - return ion.metadata().map_err(|e| format!("metadata failed: {e}")); + let mut mzml = ion + .global_metadata() + .map_err(|e| format!("metadata failed: {e}"))?; + if !with_items { + return Ok(mzml); + } + if let Some(list) = mzml.run.spectrum_list.as_mut() { + list.spectra = (0..ion.spectrum_count() as usize) + .map(|index| ion.spectrum_metadata_at(index)) + .collect::>() + .map_err(|e| format!("spectrum metadata failed: {e}"))?; + } + if let Some(list) = mzml.run.chromatogram_list.as_mut() { + list.chromatograms = (0..ion.chromatogram_count() as usize) + .map(|index| ion.chromatogram_metadata_at(index)) + .collect::>() + .map_err(|e| format!("chromatogram metadata failed: {e}"))?; + } + return Ok(mzml); } if ext == "mzml" { let bytes = fs::read(file_path).map_err(|e| format!("read failed: {e}"))?; @@ -491,16 +506,16 @@ fn read_mzml_or_ion(file_path: &Path) -> Result { fn write_mzml_as_ion( input_path: &Path, output_path: &Path, - config: WriteOptions, + options: WriteOptions, ) -> Result<(), String> { sweep_orphans(output_path)?; let temp_output = TempOutput::new(output_path)?; - let options = ConvertOptions { + let convert_options = ConvertOptions { kind: ConvertKind::MzmlToIon, - write: config, + write: options, ..Default::default() }; - ionic::convert_file(input_path, temp_output.path(), &options).map_err(|error| error.to_string())?; + ionic::convert_file(input_path, temp_output.path(), &convert_options).map_err(|error| error.to_string())?; temp_output.move_to(output_path) } @@ -722,26 +737,6 @@ fn size_to_bytes(value: f64, unit_bytes: f64, flag: &str) -> Result Self { - Self { - reset: ansi_reset(), - green: ansi_green(), - yellow: ansi_yellow(), - red: ansi_red(), - blue: ansi_blue(), - } - } -} - struct ConvertCounters { print_lock: Mutex<()>, done: AtomicUsize, @@ -801,7 +796,6 @@ fn print_convert_totals(counters: &ConvertCounters, elapsed: std::time::Duration struct ConversionJob<'a> { counters: &'a ConvertCounters, - palette: &'a Palette, input_root: &'a Path, output_root: &'a Path, overwrite: bool, @@ -877,16 +871,16 @@ impl<'a> ConversionJob<'a> { let n = self.counters.next_step(); let elapsed_s = t0.elapsed().as_secs_f64(); - let (tag, color) = if fixing_bad_output { - ("[fixed]", self.palette.blue) + let (tag, tag_color) = if fixing_bad_output { + ("[fixed]", color::blue()) } else { - ("[ok]", self.palette.green) + ("[ok]", color::green()) }; let name = basename(&out_path); - let reset = self.palette.reset; + let reset = color::reset(); let total = self.total; let line = format!( - "{color}{tag}{reset} [{n}/{total}] output: {name} input={in_mb:.2} MB, output={out_mb:.2} MB, time={elapsed_s:.3}s" + "{tag_color}{tag}{reset} [{n}/{total}] output: {name} input={in_mb:.2} MB, output={out_mb:.2} MB, time={elapsed_s:.3}s" ); print_progress_line(&self.counters.print_lock, false, &line); } @@ -896,8 +890,8 @@ impl<'a> ConversionJob<'a> { self.counters.failed.fetch_add(1, Ordering::Relaxed); let n = self.counters.next_step(); let name = basename(path); - let red = self.palette.red; - let reset = self.palette.reset; + let red = color::red(); + let reset = color::reset(); let total = self.total; let line = format!("{red}[error]{reset} [{n}/{total}] {name}: {message}"); print_progress_line(&self.counters.print_lock, true, &line); @@ -907,8 +901,8 @@ impl<'a> ConversionJob<'a> { self.counters.skipped.fetch_add(1, Ordering::Relaxed); let n = self.counters.next_step(); let name = basename(in_path); - let yellow = self.palette.yellow; - let reset = self.palette.reset; + let yellow = color::yellow(); + let reset = color::reset(); let total = self.total; let line = format!("{yellow}[skipped]{reset} [{n}/{total}] {name}"); print_progress_line(&self.counters.print_lock, false, &line); @@ -920,8 +914,8 @@ impl<'a> ConversionJob<'a> { let in_mb = megabytes_of(in_path); let out_mb = out_len as f64 / MB; let name = basename(out_path); - let yellow = self.palette.yellow; - let reset = self.palette.reset; + let yellow = color::yellow(); + let reset = color::reset(); let total = self.total; let line = format!( "{yellow}[skipped]{reset} [{n}/{total}] {name} input={in_mb:.2} MB, output={out_mb:.2} MB" @@ -930,6 +924,30 @@ impl<'a> ConversionJob<'a> { } } +fn run_jobs( + job: &ConversionJob, + files: &[PathBuf], + parallel_inside_file: bool, + pool: &ThreadPool, + t_all: Instant, + work: impl Fn(&PathBuf) + Sync, +) -> Result<(), String> { + pool.install(|| { + if parallel_inside_file { + files.iter().for_each(&work); + } else { + files.par_iter().for_each(&work); + } + }); + + print_convert_totals(job.counters, t_all.elapsed()); + + if job.counters.had_any_failure() { + return Err("some files failed".to_string()); + } + Ok(()) +} + fn convert(cmd: ConvertArgs) -> Result<(), String> { let cwd = std::env::current_dir().map_err(|e| format!("get current dir failed: {e}"))?; @@ -966,11 +984,7 @@ fn convert(cmd: ConvertArgs) -> Result<(), String> { .build() .map_err(|e| format!("rayon thread pool init failed: {e}"))?; - let encoding = if cmd.many_files { - Encoding::FileLevelParallel - } else { - Encoding::WithinFileParallel - }; + let parallel_inside_file = !cmd.many_files; let t_all = Instant::now(); @@ -980,13 +994,9 @@ fn convert(cmd: ConvertArgs) -> Result<(), String> { let ion_to_mzml = cmd.which.ion_to_mzml; let update = cmd.which.update; - let palette = Palette::current(); let counters = ConvertCounters::new(); if mzml_to_ion { - let out_ext = "ion"; - let f32_compress = cmd.force_f32; - let files = collect_files_with_exts(&input_root, &["mzml"], filter.as_deref())?; if files.is_empty() { return Err(format!( @@ -997,7 +1007,6 @@ fn convert(cmd: ConvertArgs) -> Result<(), String> { let job = ConversionJob { counters: &counters, - palette: &palette, input_root: &input_root, output_root: &output_root, overwrite: cmd.overwrite, @@ -1006,17 +1015,17 @@ fn convert(cmd: ConvertArgs) -> Result<(), String> { let config = WriteOptions { compression_level: cmd.compression_level, - force_f32: f32_compress, + force_f32: cmd.force_f32, block_size, - parallel: matches!(encoding, Encoding::WithinFileParallel), + parallel: parallel_inside_file, section_storage: cmd.section_storage.storage(), mz_window: cmd.mz_window, }; - let convert_mzml_to_ion = |in_path: &PathBuf| { + let work = |in_path: &PathBuf| { job.run( in_path, - |p| out_name_for_mzml_file(p, out_ext), + out_name_for_mzml_file, |_in_path, out_path, _file_len| ion_file_is_valid(out_path), |in_path, out_path| { write_mzml_as_ion(in_path, out_path, config) @@ -1026,17 +1035,7 @@ fn convert(cmd: ConvertArgs) -> Result<(), String> { ); }; - pool.install(|| match encoding { - Encoding::FileLevelParallel => files.par_iter().for_each(convert_mzml_to_ion), - Encoding::WithinFileParallel => files.iter().for_each(convert_mzml_to_ion), - }); - - print_convert_totals(&counters, t_all.elapsed()); - - if counters.had_any_failure() { - return Err("some files failed".to_string()); - } - return Ok(()); + return run_jobs(&job, &files, parallel_inside_file, &pool, t_all, work); } if ion_to_mzml { @@ -1050,30 +1049,24 @@ fn convert(cmd: ConvertArgs) -> Result<(), String> { let job = ConversionJob { counters: &counters, - palette: &palette, input_root: &input_root, output_root: &output_root, overwrite: cmd.overwrite, total: files.len(), }; - let convert_ion_to_mzml = |in_path: &PathBuf| { + let work = |in_path: &PathBuf| { job.run( in_path, out_name_for_bin_file_as_mzml, |in_path, out_path, file_len| { - mzml_output_matches_source( - in_path, - out_path, - file_len, - matches!(encoding, Encoding::WithinFileParallel), - ) + mzml_output_matches_source(in_path, out_path, file_len, parallel_inside_file) }, |in_path, out_path| { let options = ConvertOptions { kind: ConvertKind::IonToMzml, read: ReadOptions { - parallel: matches!(encoding, Encoding::WithinFileParallel), + parallel: parallel_inside_file, ..ReadOptions::default() }, ..Default::default() @@ -1085,17 +1078,7 @@ fn convert(cmd: ConvertArgs) -> Result<(), String> { ); }; - pool.install(|| match encoding { - Encoding::FileLevelParallel => files.par_iter().for_each(convert_ion_to_mzml), - Encoding::WithinFileParallel => files.iter().for_each(convert_ion_to_mzml), - }); - - print_convert_totals(&counters, t_all.elapsed()); - - if counters.had_any_failure() { - return Err("some files failed".to_string()); - } - return Ok(()); + return run_jobs(&job, &files, parallel_inside_file, &pool, t_all, work); } if update { @@ -1109,14 +1092,13 @@ fn convert(cmd: ConvertArgs) -> Result<(), String> { let job = ConversionJob { counters: &counters, - palette: &palette, input_root: &input_root, output_root: &output_root, overwrite: cmd.overwrite || update_in_place, total: files.len(), }; - let update_ion_file = |in_path: &PathBuf| { + let work = |in_path: &PathBuf| { job.run( in_path, out_name_for_ion_file_as_updated_ion, @@ -1130,17 +1112,7 @@ fn convert(cmd: ConvertArgs) -> Result<(), String> { ); }; - pool.install(|| match encoding { - Encoding::FileLevelParallel => files.par_iter().for_each(update_ion_file), - Encoding::WithinFileParallel => files.iter().for_each(update_ion_file), - }); - - print_convert_totals(&counters, t_all.elapsed()); - - if counters.had_any_failure() { - return Err("some files failed".to_string()); - } - return Ok(()); + return run_jobs(&job, &files, parallel_inside_file, &pool, t_all, work); } Err("no convert mode selected".to_string()) diff --git a/crates/cli/src/utilities/color.rs b/crates/cli/src/utilities/color.rs new file mode 100644 index 00000000..6cd158dc --- /dev/null +++ b/crates/cli/src/utilities/color.rs @@ -0,0 +1,43 @@ +use std::{io::IsTerminal, sync::OnceLock}; + +static COLOR_ENABLED: OnceLock = OnceLock::new(); + +fn color_enabled() -> bool { + *COLOR_ENABLED.get_or_init(|| { + std::env::var_os("NO_COLOR").is_none() + && std::io::stdout().is_terminal() + && std::io::stderr().is_terminal() + }) +} + +fn ansi(code: &'static str) -> &'static str { + if color_enabled() { code } else { "" } +} + +pub(crate) fn reset() -> &'static str { + ansi("\x1b[0m") +} + +pub(crate) fn green() -> &'static str { + ansi("\x1b[1;32m") +} + +pub(crate) fn yellow() -> &'static str { + ansi("\x1b[1;33m") +} + +pub(crate) fn red() -> &'static str { + ansi("\x1b[1;31m") +} + +pub(crate) fn blue() -> &'static str { + ansi("\x1b[1;34m") +} + +pub(crate) fn bold() -> &'static str { + ansi("\x1b[1m") +} + +pub(crate) fn dim() -> &'static str { + ansi("\x1b[2m") +} diff --git a/crates/cli/src/utilities/header.rs b/crates/cli/src/utilities/header.rs index fda02a87..c4036ef2 100644 --- a/crates/cli/src/utilities/header.rs +++ b/crates/cli/src/utilities/header.rs @@ -1,4 +1,4 @@ -use std::{fs::File, io::IsTerminal, path::Path, sync::OnceLock}; +use std::{fs::File, path::Path}; use cosmoz::{DecompressOptions, Decoder}; @@ -6,39 +6,7 @@ use ionic::format::{ CODEC_NONE, CODEC_ZSTD, FILE_SIGNATURE, FILE_TRAILER, HEADER_SIZE, is_supported, version_of, }; -static COLOR_ENABLED: OnceLock = OnceLock::new(); - -fn color_enabled() -> bool { - *COLOR_ENABLED.get_or_init(|| { - std::env::var_os("NO_COLOR").is_none() - && std::io::stdout().is_terminal() - && std::io::stderr().is_terminal() - }) -} - -fn ansi(code: &'static str) -> &'static str { - if color_enabled() { code } else { "" } -} - -fn reset() -> &'static str { - ansi("\x1b[0m") -} - -fn green() -> &'static str { - ansi("\x1b[1;32m") -} - -fn red() -> &'static str { - ansi("\x1b[1;31m") -} - -fn bold() -> &'static str { - ansi("\x1b[1m") -} - -fn dim() -> &'static str { - ansi("\x1b[2m") -} +use super::color::{bold, dim, green, red, reset}; const SPEC_SUMMARY_ROW: usize = 80; const INDEX_ENTRY_ROW: usize = 16; @@ -55,7 +23,7 @@ struct FixedSections { fn max_address_index(entries: &[u8]) -> u64 { let mut total = 0u64; - for entry in entries.chunks_exact(INDEX_ENTRY_ROW) { + for entry in entries.as_chunks::().0 { let first = u64::from_le_bytes(entry[0..8].try_into().unwrap()); let count = u64::from_le_bytes(entry[8..16].try_into().unwrap()); total = total.max(first.saturating_add(count)); @@ -63,29 +31,33 @@ fn max_address_index(entries: &[u8]) -> u64 { total } -fn read_fixed_section( - bytes: &[u8], - header: &[u8], +struct FixedSectionSpec { off_at: usize, len_at: usize, plain_count: u64, record: usize, +} + +fn read_fixed_section( + bytes: &[u8], + header: &[u8], + spec: FixedSectionSpec, compressed: bool, decoder: &mut Decoder, ) -> Result, String> { - let off = u64_at(header, off_at) as usize; - let len = u64_at(header, len_at) as usize; + let off = u64_at(header, spec.off_at) as usize; + let len = u64_at(header, spec.len_at) as usize; let end = off.checked_add(len).ok_or("offset overflow")?; let stored = bytes.get(off..end).ok_or("section out of bounds")?; - if len > 0 && off % 8 != 0 { + if len > 0 && !off.is_multiple_of(8) { return Err("offset not 8-byte aligned".into()); } if !compressed { return Ok(stored.to_vec()); } - let plain_len = usize::try_from(plain_count) + let plain_len = usize::try_from(spec.plain_count) .ok() - .and_then(|count| count.checked_mul(record)) + .and_then(|count| count.checked_mul(spec.record)) .ok_or("section size overflow")?; if plain_len == 0 { return Ok(Vec::new()); @@ -120,10 +92,12 @@ fn read_address_section( read_fixed_section( bytes, header, - off_at, - len_at, - address_count, - ARRAY_ADDRESS_ROW, + FixedSectionSpec { + off_at, + len_at, + plain_count: address_count, + record: ARRAY_ADDRESS_ROW, + }, compressed, decoder, ) @@ -137,20 +111,24 @@ fn resolve_fixed_sections(bytes: &[u8], header: &[u8], decoder: &mut Decoder) -> let spec_summary = read_fixed_section( bytes, header, - 48, - 56, - spec_count, - SPEC_SUMMARY_ROW, + FixedSectionSpec { + off_at: 48, + len_at: 56, + plain_count: spec_count, + record: SPEC_SUMMARY_ROW, + }, compressed, decoder, ); let spec_entries = read_fixed_section( bytes, header, - 64, - 72, - spec_count, - INDEX_ENTRY_ROW, + FixedSectionSpec { + off_at: 64, + len_at: 72, + plain_count: spec_count, + record: INDEX_ENTRY_ROW, + }, compressed, decoder, ); @@ -159,20 +137,24 @@ fn resolve_fixed_sections(bytes: &[u8], header: &[u8], decoder: &mut Decoder) -> let chrom_summary = read_fixed_section( bytes, header, - 112, - 120, - chrom_count, - SPEC_SUMMARY_ROW, + FixedSectionSpec { + off_at: 112, + len_at: 120, + plain_count: chrom_count, + record: SPEC_SUMMARY_ROW, + }, compressed, decoder, ); let chrom_entries = read_fixed_section( bytes, header, - 128, - 136, - chrom_count, - INDEX_ENTRY_ROW, + FixedSectionSpec { + off_at: 128, + len_at: 136, + plain_count: chrom_count, + record: INDEX_ENTRY_ROW, + }, compressed, decoder, ); @@ -203,7 +185,7 @@ fn check_fixed( ) -> Result<(), String> { let plain = section.as_ref().map_err(|why| why.clone())?; let len = plain.len() as u64; - if len % record != 0 { + if !len.is_multiple_of(record) { return Err(format!("length {len} is not a multiple of {record}")); } if let Some(expected_count) = count { @@ -222,17 +204,11 @@ struct Section { name: &'static str, offset: u64, size: u64, - ok: bool, } impl Section { fn new(name: &'static str, offset: u64, size: u64) -> Self { - Self { - name, - offset, - size, - ok: true, - } + Self { name, offset, size } } } @@ -289,7 +265,7 @@ struct HeaderView<'a> { impl<'a> HeaderView<'a> { fn new(bytes: &'a [u8]) -> Self { let header = &bytes[..HEADER_SIZE]; - let sections = build_sections(header, u64_at(header, 400)); + let sections = build_sections(header); let mut decoder = Decoder::new(&DecompressOptions::default()); let spec_window_dir = open_window_directory(bytes, header, 32, 40, 384, &mut decoder); let chrom_window_dir = open_window_directory(bytes, header, 96, 104, 392, &mut decoder); @@ -408,7 +384,7 @@ impl<'a> HeaderView<'a> { let mut max_z: u32 = 0; if let Ok(buf) = &self.fixed.spec_summary { - for row in buf.chunks_exact(SPEC_SUMMARY_ROW) { + for row in buf.as_chunks::().0 { let rt = f64::from_le_bytes(row[0..8].try_into().unwrap()); let x = u32::from_le_bytes(row[42..46].try_into().unwrap()); let y = u32::from_le_bytes(row[46..50].try_into().unwrap()); @@ -500,8 +476,8 @@ fn open_window_directory( Ok(top_window) } -fn build_sections(header: &[u8], total_file_size: u64) -> Vec
{ - let mut sections = vec![ +fn build_sections(header: &[u8]) -> Vec
{ + let sections = vec![ Section::new( "A0 — spectrum m/z-window directory", u64_at(header, 32), @@ -549,28 +525,6 @@ fn build_sections(header: &[u8], total_file_size: u64) -> Vec
{ Section::new("chrom_container", u64_at(header, 224), u64_at(header, 232)), ]; - let trailer_start = total_file_size.saturating_sub(8); - for s in &mut sections { - s.ok = s - .offset - .checked_add(s.size) - .is_some_and(|end| end <= trailer_start) - && (s.size == 0 || s.offset % 8 == 0); - } - - let mut order: Vec = (0..sections.len()) - .filter(|&i| sections[i].size > 0) - .collect(); - order.sort_by_key(|&i| sections[i].offset); - for pair in order.windows(2) { - let (l, r) = (pair[0], pair[1]); - let end = sections[l].offset.saturating_add(sections[l].size); - if end > sections[r].offset { - sections[l].ok = false; - sections[r].ok = false; - } - } - sections } @@ -651,16 +605,24 @@ fn section_check_results(view: &HeaderView<'_>) -> [Result<(), String>; 8] { SPEC_SUMMARY_ROW as u64, Some(spec_count), ), - check_fixed(&view.fixed.spec_entries, 16, Some(spec_count)), - check_fixed(&view.fixed.spec_addresses, 32, None), + check_fixed( + &view.fixed.spec_entries, + INDEX_ENTRY_ROW as u64, + Some(spec_count), + ), + check_fixed(&view.fixed.spec_addresses, ARRAY_ADDRESS_ROW as u64, None), view.chrom_window_dir.clone().map(|_| ()), check_fixed( &view.fixed.chrom_summary, SPEC_SUMMARY_ROW as u64, Some(chrom_count), ), - check_fixed(&view.fixed.chrom_entries, 16, Some(chrom_count)), - check_fixed(&view.fixed.chrom_addresses, 32, None), + check_fixed( + &view.fixed.chrom_entries, + INDEX_ENTRY_ROW as u64, + Some(chrom_count), + ), + check_fixed(&view.fixed.chrom_addresses, ARRAY_ADDRESS_ROW as u64, None), ] } diff --git a/crates/cli/src/utilities/mod.rs b/crates/cli/src/utilities/mod.rs index 390513df..69679eaa 100644 --- a/crates/cli/src/utilities/mod.rs +++ b/crates/cli/src/utilities/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod color; mod header; mod temp_output; diff --git a/crates/parser/examples/bench_window.rs b/crates/parser/examples/bench_window.rs index 284b8d86..b9ec660e 100644 --- a/crates/parser/examples/bench_window.rs +++ b/crates/parser/examples/bench_window.rs @@ -1,6 +1,6 @@ use std::{path::Path, time::Instant}; -use ionic::{IonReader, Range, ReadOptions}; +use ionic::{ArrayKind, IonReader, Range, ReadOptions}; fn main() { let mut args = std::env::args().skip(1); @@ -74,8 +74,10 @@ struct Reading { } fn read_full(ion: &mut IonReader, index: usize) -> Result, String> { - ion.read_window( + ion.spectrum_window( index, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: 0.0, to: f64::MAX, @@ -97,8 +99,10 @@ fn measure(ion: &mut IonReader, sample: usize, low: f64, high: f64) -> Result ConvertKind { - if kind != ConvertKind::Auto { +fn resolve_kind(wanted: ConvertKind, from_extension: Option, is_ion: bool) -> ConvertKind { + if wanted != ConvertKind::Auto { + return wanted; + } + if let Some(kind) = from_extension { return kind; } - if bytes.starts_with(&FILE_SIGNATURE) { + if is_ion { ConvertKind::IonToMzml } else { ConvertKind::MzmlToIon @@ -17,12 +23,13 @@ fn resolve_kind(bytes: &[u8], kind: ConvertKind) -> ConvertKind { } pub fn convert(input: &[u8], options: &ConvertOptions) -> IonResult> { - if resolve_kind(input, options.kind) == ConvertKind::IonToMzml { - let mut reader = crate::ion::decoder::decode::IonReader::from_bytes(input, &options.read)?; + let is_ion = input.starts_with(&FILE_SIGNATURE); + if resolve_kind(options.kind, None, is_ion) == ConvertKind::IonToMzml { + let mut reader = IonReader::from_bytes(input, &options.read)?; let mzml = reader.to_mzml()?; return Ok(bin_to_mzml(&mzml)?); } - if input.starts_with(&FILE_SIGNATURE) { + if is_ion { return Err(IonError::from("input is already an ion file")); } let mzml = parse_mzml(input)?; @@ -39,10 +46,8 @@ pub fn convert_file( ) -> IonResult<()> { use std::io::Read; - use crate::{ - ion::{encoder::ion_writer::IonWriter, encoder::utilities::FileWriter}, - mzml::{MzmlReader, structs::MzML}, - }; + use crate::mzml::{MzmlReader, structs::MzML}; + use super::writer::IonWriter; let input = input.as_ref(); let output = output.as_ref(); @@ -56,18 +61,10 @@ pub fn convert_file( }; let is_ion = start.starts_with(&FILE_SIGNATURE); - let kind = if options.kind != ConvertKind::Auto { - options.kind - } else if let Some(kind) = kind_from_extension(input) { - kind - } else if is_ion { - ConvertKind::IonToMzml - } else { - ConvertKind::MzmlToIon - }; + let kind = resolve_kind(options.kind, kind_from_extension(input), is_ion); if kind == ConvertKind::IonToMzml { - let mut reader = crate::ion::decoder::decode::IonReader::open(input, &options.read)?; + let mut reader = IonReader::open(input, &options.read)?; let mzml = reader.to_mzml()?; let xml = bin_to_mzml(&mzml)?; std::fs::write(output, &xml)?; @@ -78,10 +75,8 @@ pub fn convert_file( return Err(IonError::from("input is already an ion file")); } - let mut file = FileWriter::open_path(output)?; - let mut writer = IonWriter::begin(&mut file, &MzML::default(), &options.write)?; - writer.write_stream(&mut file, &mut MzmlReader::open(input)?)?; - file.flush()?; + IonWriter::create(output, &MzML::default(), &options.write)? + .write_stream(&mut MzmlReader::open(input)?)?; Ok(()) } diff --git a/crates/parser/src/api/writer.rs b/crates/parser/src/api/writer.rs index 21442c40..3953b592 100644 --- a/crates/parser/src/api/writer.rs +++ b/crates/parser/src/api/writer.rs @@ -4,7 +4,7 @@ use std::path::Path; use crate::{ ion::{ IonResult, - encoder::{ion_writer::IonWriter as LowWriter, scan_stream::ScanStream, utilities::WriteBytes}, + encoder::{ion_writer::IonEncoder, scan_stream::ScanStream, utilities::WriteBytes}, }, mzml::structs::{Chromatogram, MzML, Spectrum}, }; @@ -14,34 +14,34 @@ use crate::ion::encoder::utilities::FileWriter; use super::options::WriteOptions; -enum Sink<'out> { +enum Output<'out> { Borrowed(&'out mut dyn WriteBytes), #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] Owned(FileWriter), } -impl WriteBytes for Sink<'_> { +impl WriteBytes for Output<'_> { fn write(&mut self, bytes: &[u8]) -> IonResult<()> { match self { - Sink::Borrowed(output) => output.write(bytes), + Output::Borrowed(output) => output.write(bytes), #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - Sink::Owned(file) => file.write(bytes), + Output::Owned(file) => file.write(bytes), } } fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()> { match self { - Sink::Borrowed(output) => output.patch(at, bytes), + Output::Borrowed(output) => output.patch(at, bytes), #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - Sink::Owned(file) => file.patch(at, bytes), + Output::Owned(file) => file.patch(at, bytes), } } fn position(&mut self) -> IonResult { match self { - Sink::Borrowed(output) => output.position(), + Output::Borrowed(output) => output.position(), #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - Sink::Owned(file) => file.position(), + Output::Owned(file) => file.position(), } } } @@ -54,8 +54,8 @@ fn strip_lists(metadata: &MzML) -> MzML { } pub struct IonWriter<'out> { - sink: Sink<'out>, - low: LowWriter, + output: Output<'out>, + encoder: IonEncoder, metadata: MzML, } @@ -67,11 +67,11 @@ impl IonWriter<'static> { options: &WriteOptions, ) -> IonResult { let file = FileWriter::open_path(path.as_ref())?; - let mut sink = Sink::Owned(file); - let low = LowWriter::begin(&mut sink, metadata, options)?; + let mut output = Output::Owned(file); + let encoder = IonEncoder::begin(&mut output, metadata, options)?; Ok(Self { - sink, - low, + output, + encoder, metadata: strip_lists(metadata), }) } @@ -83,32 +83,36 @@ impl<'out> IonWriter<'out> { metadata: &MzML, options: &WriteOptions, ) -> IonResult { - let mut sink = Sink::Borrowed(output); - let low = LowWriter::begin(&mut sink, metadata, options)?; + let mut output = Output::Borrowed(output); + let encoder = IonEncoder::begin(&mut output, metadata, options)?; Ok(Self { - sink, - low, + output, + encoder, metadata: strip_lists(metadata), }) } pub fn write_spectrum(&mut self, spectrum: &Spectrum) -> IonResult<()> { - self.low.push_spectrum(&mut self.sink, spectrum) + self.encoder.push_spectrum(&mut self.output, spectrum) } pub fn write_chromatogram(&mut self, chromatogram: &Chromatogram) -> IonResult<()> { - self.low.push_chromatogram(&mut self.sink, chromatogram) + self.encoder.push_chromatogram(&mut self.output, chromatogram) } - pub fn write_stream(&mut self, scans: &mut dyn ScanStream) -> IonResult<()> { - self.low.write_stream(&mut self.sink, scans) + pub fn write_stream(mut self, scans: &mut dyn ScanStream) -> IonResult<()> { + self.encoder.write_stream(&mut self.output, scans)?; + self.flush_file() } pub fn finish(mut self) -> IonResult<()> { - self.low.finish(&mut self.sink, &self.metadata)?; + self.encoder.finish(&mut self.output, &self.metadata)?; + self.flush_file() + } + fn flush_file(&mut self) -> IonResult<()> { #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - if let Sink::Owned(file) = &mut self.sink { + if let Output::Owned(file) = &mut self.output { file.flush()?; } diff --git a/crates/parser/src/ion/decoder/decode/arrays.rs b/crates/parser/src/ion/decoder/decode/arrays.rs index aa4b3f8d..1971834d 100644 --- a/crates/parser/src/ion/decoder/decode/arrays.rs +++ b/crates/parser/src/ion/decoder/decode/arrays.rs @@ -29,12 +29,12 @@ impl ArrayAddress { } #[derive(Debug, Clone)] -pub struct ArrayGroup { - pub array_type: u32, - pub array_cv_code: u8, - pub dtype: u8, - pub array_filter: u8, - pub refs: Vec, +pub(crate) struct ArrayGroup { + pub(crate) array_type: u32, + pub(crate) array_cv_code: u8, + pub(crate) dtype: u8, + pub(crate) array_filter: u8, + pub(crate) refs: Vec, } #[derive(Clone)] @@ -223,41 +223,6 @@ pub(crate) fn group_arrays(refs: &[ArrayAddress]) -> IonResult> Ok(groups) } -pub(crate) fn read_group_decoded_bytes( - group: &ArrayGroup, - container: &mut BlockReader, -) -> IonResult> { - if let [array_address] = group.refs.as_slice() { - let (element_offset, count, stride) = address_read_params(array_address); - let raw = container.get_array_bytes_from_block( - array_address.block_id, - element_offset, - count, - stride, - "read_group_decoded_bytes", - )?; - let unfiltered = unfilter_array_bytes(raw, group.dtype, group.array_filter)?; - return Ok(unfiltered.into_owned()); - } - - let mut decoded = Vec::new(); - for array_address in &group.refs { - let (element_offset, count, stride) = address_read_params(array_address); - let raw = container.get_array_bytes_from_block( - array_address.block_id, - element_offset, - count, - stride, - "read_group_decoded_bytes", - )?; - let unfiltered = unfilter_array_bytes(raw, group.dtype, group.array_filter)?; - decoded.reserve(unfiltered.len()); - decoded.extend_from_slice(&unfiltered); - } - - Ok(decoded) -} - #[inline] pub(crate) fn dtype_stride(dtype: u8) -> usize { match dtype { @@ -312,50 +277,56 @@ pub(crate) fn decode_into( match dtype { FILE_DTYPE_F64 => { buf.reserve(bytes.len() / 8); - buf.extend( - bytes - .chunks_exact(8) - .map(|c| f64::from_le_bytes(c.try_into().unwrap())), - ); + buf.extend(bytes.as_chunks::<8>().0.iter().map(|c| f64::from_le_bytes(*c))); } FILE_DTYPE_F32 => { buf.reserve(bytes.len() / 4); buf.extend( bytes - .chunks_exact(4) - .map(|c| f32::from_le_bytes(c.try_into().unwrap()) as f64), + .as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c) as f64), ); } FILE_DTYPE_F16 => { buf.reserve(bytes.len() / 2); buf.extend( bytes - .chunks_exact(2) - .map(|c| f16_bits_to_f64(u16::from_le_bytes(c.try_into().unwrap()))), + .as_chunks::<2>() + .0 + .iter() + .map(|c| f16_bits_to_f64(u16::from_le_bytes(*c))), ); } FILE_DTYPE_I16 => { buf.reserve(bytes.len() / 2); buf.extend( bytes - .chunks_exact(2) - .map(|c| i16::from_le_bytes(c.try_into().unwrap()) as f64), + .as_chunks::<2>() + .0 + .iter() + .map(|c| i16::from_le_bytes(*c) as f64), ); } FILE_DTYPE_I32 => { buf.reserve(bytes.len() / 4); buf.extend( bytes - .chunks_exact(4) - .map(|c| i32::from_le_bytes(c.try_into().unwrap()) as f64), + .as_chunks::<4>() + .0 + .iter() + .map(|c| i32::from_le_bytes(*c) as f64), ); } FILE_DTYPE_I64 => { buf.reserve(bytes.len() / 8); buf.extend( bytes - .chunks_exact(8) - .map(|c| i64::from_le_bytes(c.try_into().unwrap()) as f64), + .as_chunks::<8>() + .0 + .iter() + .map(|c| i64::from_le_bytes(*c) as f64), ); } _ => { @@ -368,82 +339,26 @@ pub(crate) fn decode_into( Ok(()) } -#[cfg(test)] -fn collect_entry_array_addresses( - entry_bytes: &[u8], - address_bytes: &[u8], -) -> Option> { - let ref_start = - usize::try_from(u64::from_le_bytes(entry_bytes[0..8].try_into().unwrap())).ok()?; - let address_count = - usize::try_from(u64::from_le_bytes(entry_bytes[8..16].try_into().unwrap())).ok()?; - let start = ref_start.checked_mul(ARRAY_ADDRESS_BYTES)?; - let span = address_count.checked_mul(ARRAY_ADDRESS_BYTES)?; - let end = start.checked_add(span)?; - let mut refs = Vec::with_capacity(address_count); - for bytes in address_bytes - .get(start..end)? - .chunks_exact(ARRAY_ADDRESS_BYTES) - { - refs.push(parse_array_address(bytes)); - } - Some(refs) -} - -#[cfg(test)] -pub(crate) fn read_scan_arrays( - container: &mut dyn ContainerAccess, - entry_bytes: &[u8], - address_bytes: &[u8], - mz: &mut Vec, - intensity: &mut Vec, -) -> bool { - mz.clear(); - intensity.clear(); - let Some(refs) = collect_entry_array_addresses(entry_bytes, address_bytes) else { - return false; - }; - let Ok(groups) = group_arrays(&refs) else { - return false; +fn read_group_values( + refs: &[ArrayAddress], + out: &mut Vec, + mut read_one: impl FnMut(&ArrayAddress, &mut Vec) -> IonResult<()>, +) -> IonResult<()> { + let Some((first, rest)) = refs.split_first() else { + out.clear(); + return Ok(()); }; - let mut segment = Vec::new(); - for group in &groups { - let target = match group.array_type { - ACC_MZ => &mut *mz, - ACC_INT => &mut *intensity, - _ => continue, - }; - for array_address in &group.refs { - let (element_offset, count, stride) = address_read_params(array_address); - let raw = match container.get_array_bytes_from_block( - array_address.block_id, - element_offset, - count, - stride, - "scan", - ) { - Ok(raw) => raw, - Err(_) => return false, - }; - if decode_into( - &mut segment, - raw, - array_address.dtype, - array_address.array_filter, - ) - .is_err() - { - return false; - } - target.extend_from_slice(&segment); - } + read_one(first, out)?; + let mut window = Vec::new(); + for array_address in rest { + read_one(array_address, &mut window)?; + out.extend_from_slice(&window); } - mz.len().min(intensity.len()) > 0 + Ok(()) } impl IonReader { - #[allow(private_interfaces)] - pub fn spectrum_array_addresses(&self, index: usize) -> Option> { + pub(crate) fn spectrum_array_addresses(&self, index: usize) -> Option> { if index >= self.header.spectrum_count as usize { return None; } @@ -451,21 +366,7 @@ impl IonReader { .map(ArrayAddressList::into_vec) } - #[allow(private_interfaces)] - pub fn chromatogram_array_addresses(&self, index: usize) -> Option> { - if index >= self.header.chrom_count as usize { - return None; - } - read_array_addresses_from_buffers( - &self.chrom_entries_buf, - &self.chrom_array_addresses, - index, - ) - .map(ArrayAddressList::into_vec) - } - - #[allow(private_interfaces)] - pub fn read_spectrum_values( + pub(crate) fn read_spectrum_values( &mut self, array_address: &ArrayAddress, out: &mut Vec, @@ -481,8 +382,7 @@ impl IonReader { decode_into(out, raw, array_address.dtype, array_address.array_filter) } - #[allow(private_interfaces)] - pub fn read_spectrum_array(&mut self, array_address: &ArrayAddress) -> IonResult { + pub(crate) fn read_spectrum_array(&mut self, array_address: &ArrayAddress) -> IonResult { let (element_offset, count, stride) = address_read_params(array_address); let raw = self.spec_container.get_array_bytes_from_block( array_address.block_id, @@ -495,32 +395,67 @@ impl IonReader { super::to_mzml::decoded_bytes_to_binary_data(&values, array_address.dtype) } - #[allow(private_interfaces)] - pub fn read_chromatogram_values( + pub(crate) fn read_spectrum_group_values( &mut self, - array_address: &ArrayAddress, + group: &ArrayGroup, out: &mut Vec, ) -> IonResult<()> { - let container = self - .chrom_container - .as_mut() - .ok_or_else(|| IonError::from("no chromatogram container"))?; - let (element_offset, count, stride) = address_read_params(array_address); - let raw = container.get_array_bytes_from_block( - array_address.block_id, - element_offset, - count, - stride, - "read_chromatogram_values", - )?; - decode_into(out, raw, array_address.dtype, array_address.array_filter) + read_group_values(&group.refs, out, |address, buf| { + self.read_spectrum_values(address, buf) + }) } - #[allow(private_interfaces)] - pub fn read_chromatogram_array( + pub fn spectrum_array( + &mut self, + spectrum_index: usize, + array_type: impl Into, + ) -> IonResult> { + let mut values = Vec::new(); + self.spectrum_array_into(spectrum_index, array_type, &mut values)?; + Ok(values) + } + + pub fn spectrum_array_into( + &mut self, + spectrum_index: usize, + array_type: impl Into, + out: &mut Vec, + ) -> IonResult<()> { + out.clear(); + let array_type = array_type.into(); + if spectrum_index >= self.header.spectrum_count as usize { + return Err(IonError::OutOfRange { + index: spectrum_index, + count: self.header.spectrum_count, + }); + } + + let Some(array_addresses) = read_array_addresses_from_buffers( + &self.spec_entries_buf, + &self.spec_array_addresses, + spectrum_index, + ) else { + return Ok(()); + }; + + let groups = group_arrays(array_addresses.as_slice())?; + + for group in groups { + if group.array_type != array_type { + continue; + } + self.read_spectrum_group_values(&group, out)?; + return Ok(()); + } + + Ok(()) + } + + pub(crate) fn read_chromatogram_values( &mut self, array_address: &ArrayAddress, - ) -> IonResult { + out: &mut Vec, + ) -> IonResult<()> { let container = self .chrom_container .as_mut() @@ -531,55 +466,50 @@ impl IonReader { element_offset, count, stride, - "read_chromatogram_array", + "read_chromatogram_values", )?; - let values = unfilter_array_bytes(raw, array_address.dtype, array_address.array_filter)?; - super::to_mzml::decoded_bytes_to_binary_data(&values, array_address.dtype) + decode_into(out, raw, array_address.dtype, array_address.array_filter) } - pub(crate) fn read_group_values( + pub(crate) fn read_chromatogram_group_values( &mut self, group: &ArrayGroup, out: &mut Vec, ) -> IonResult<()> { - let Some((first, rest)) = group.refs.split_first() else { - out.clear(); - return Ok(()); - }; - self.read_spectrum_values(first, out)?; - let mut window = Vec::new(); - for array_address in rest { - self.read_spectrum_values(array_address, &mut window)?; - out.extend_from_slice(&window); - } - Ok(()) + read_group_values(&group.refs, out, |address, buf| { + self.read_chromatogram_values(address, buf) + }) } - pub fn array(&mut self, spectrum_index: usize, array_type: impl Into) -> IonResult> { + pub fn chromatogram_array( + &mut self, + chromatogram_index: usize, + array_type: impl Into, + ) -> IonResult> { let mut values = Vec::new(); - self.array_into(spectrum_index, array_type, &mut values)?; + self.chromatogram_array_into(chromatogram_index, array_type, &mut values)?; Ok(values) } - pub fn array_into( + pub fn chromatogram_array_into( &mut self, - spectrum_index: usize, + chromatogram_index: usize, array_type: impl Into, out: &mut Vec, ) -> IonResult<()> { out.clear(); let array_type = array_type.into(); - if spectrum_index >= self.header.spectrum_count as usize { + if chromatogram_index >= self.header.chrom_count as usize { return Err(IonError::OutOfRange { - index: spectrum_index, - count: self.header.spectrum_count, + index: chromatogram_index, + count: self.header.chrom_count, }); } let Some(array_addresses) = read_array_addresses_from_buffers( - &self.spec_entries_buf, - &self.spec_array_addresses, - spectrum_index, + &self.chrom_entries_buf, + &self.chrom_array_addresses, + chromatogram_index, ) else { return Ok(()); }; @@ -590,7 +520,7 @@ impl IonReader { if group.array_type != array_type { continue; } - self.read_group_values(&group, out)?; + self.read_chromatogram_group_values(&group, out)?; return Ok(()); } @@ -598,6 +528,41 @@ impl IonReader { } } +#[cfg(test)] +impl IonReader { + pub(crate) fn chromatogram_array_addresses(&self, index: usize) -> Option> { + if index >= self.header.chrom_count as usize { + return None; + } + read_array_addresses_from_buffers( + &self.chrom_entries_buf, + &self.chrom_array_addresses, + index, + ) + .map(ArrayAddressList::into_vec) + } + + pub(crate) fn read_chromatogram_array( + &mut self, + array_address: &ArrayAddress, + ) -> IonResult { + let container = self + .chrom_container + .as_mut() + .ok_or_else(|| IonError::from("no chromatogram container"))?; + let (element_offset, count, stride) = address_read_params(array_address); + let raw = container.get_array_bytes_from_block( + array_address.block_id, + element_offset, + count, + stride, + "read_chromatogram_array", + )?; + let values = unfilter_array_bytes(raw, array_address.dtype, array_address.array_filter)?; + super::to_mzml::decoded_bytes_to_binary_data(&values, array_address.dtype) + } +} + #[cfg(test)] mod tests { use super::*; @@ -657,8 +622,10 @@ mod tests { ); let out: Vec = unfiltered - .chunks_exact(8) - .map(|c| f64::from_le_bytes(c.try_into().unwrap())) + .as_chunks::<8>() + .0 + .iter() + .map(|c| f64::from_le_bytes(*c)) .collect(); assert_eq!(out.len(), data.len()); for (a, b) in out.iter().zip(data.iter()) { @@ -689,7 +656,7 @@ mod tests { mod array_roundtrips { use crate::{ - ArrayKind, IonReader, Range, ReadOptions, ScanQuery, Select, WriteOptions, + IonReader, Range, ReadOptions, ScanQuery, Select, WriteOptions, format::{CURRENT_VERSION, HEADER_FORMAT_VERSION_OFFSET}, ion::encoder::ion_writer::write_mzml_to_ion, mzml::structs::{ diff --git a/crates/parser/src/ion/decoder/decode/mod.rs b/crates/parser/src/ion/decoder/decode/mod.rs index 836c0f25..e25e5c5c 100644 --- a/crates/parser/src/ion/decoder/decode/mod.rs +++ b/crates/parser/src/ion/decoder/decode/mod.rs @@ -2,8 +2,6 @@ use std::sync::Arc; #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] use crate::ion::decoder::utilities::byte_source::FileSource; -#[cfg(test)] -use crate::ion::decoder::utilities::spectrum_source::ScanSource; use crate::{ accessions::{ FLOAT_16BIT, FLOAT_32BIT, FLOAT_64BIT, INT_16BIT, INT_32BIT, INT_64BIT, format_accession, @@ -16,7 +14,7 @@ use crate::{ ACC_ATTR_START_TIME_STAMP, AccessionTail, parse_accession_tail, }, decoder::utilities::{ - byte_source::{BytesSource, ReadBytes}, + byte_source::{BytesSource, ReadBytes, SourceBytes}, common::decompress_zstd, }, encoder::encode::{ @@ -25,7 +23,7 @@ use crate::{ }, filter_summary::{ChromatogramSummary, SpectrumSummary}, format::{CODEC_NONE, CODEC_ZSTD, FILE_TRAILER}, - meta_groups::MetaTotals, + meta_groups::meta_directory_range, packing::PackingId, utilities::{ Header, MetaGroupReader, @@ -37,12 +35,12 @@ use crate::{ common::get_attr_text, decompression_limit::DecompressionLimit, meta_column_layout::MetaColumnLayout, - parse_chromatogram, parse_chromatogram_list, parse_cv_and_user_params, parse_cv_list, - parse_data_processing_list, parse_file_description, - parse_global_metadata::parse_global_metadata, - parse_header, parse_instrument_list, parse_referenceable_param_group_list, - parse_sample_list, parse_scan_settings_list, parse_software_list, parse_spectrum, - parse_spectrum_list, + parse_chromatogram, parse_chromatogram_list, parse_chromatogram_list_header, + parse_cv_and_user_params, parse_cv_list, parse_data_processing_list, + parse_file_description, parse_global_metadata::parse_global_metadata, + parse_instrument_list, parse_referenceable_param_group_list, parse_sample_list, + parse_scan_settings_list, parse_software_list, parse_spectrum, parse_spectrum_list, + parse_spectrum_list_header, spectrum_source::{ScanSummary, TimeUnit, f16_bits_to_f64}, window_directory::WindowDirectory, }, @@ -50,6 +48,7 @@ use crate::{ mzml::{schema::TagId, structs::*}, }; +#[cfg(test)] pub(crate) const ACC_MZ: u32 = 1_000_514; pub(crate) const ACC_INT: u32 = 1_000_515; pub(crate) const INDEX_ENTRY_BYTES: usize = 16; @@ -67,7 +66,7 @@ fn check_crc(bytes: &[u8], expected: u32, name: &str) -> IonResult<()> { fn max_address_index(entries_buf: &[u8]) -> u64 { let mut total = 0u64; - for entry in entries_buf.chunks_exact(INDEX_ENTRY_BYTES) { + for entry in entries_buf.as_chunks::().0 { let first = u64::from_le_bytes(entry[0..8].try_into().unwrap()); let count = u64::from_le_bytes(entry[8..16].try_into().unwrap()); total = total.max(first.saturating_add(count)); @@ -88,7 +87,7 @@ fn check_address_table(entries_buf: &[u8], table: &[u8], name: &str) -> IonResul } fn check_array_dtypes(address_bytes: &[u8], name: &'static str) -> IonResult<()> { - for record in address_bytes.chunks_exact(ARRAY_ADDRESS_BYTES) { + for record in address_bytes.as_chunks::().0 { crate::ion::packing::Dtype::from_byte(record[24]).map_err(|_| IonError::BadDtype { dtype: record[24], kind: name, @@ -108,6 +107,187 @@ fn decompress_fixed_section( Ok(Arc::from(plain)) } +type FixedSectionBuffers = (Arc<[u8]>, Arc<[u8]>, Arc<[u8]>, Arc<[u8]>, Arc<[u8]>, Arc<[u8]>); + +struct FixedSections { + spec_summary: Arc<[u8]>, + spec_entries: Arc<[u8]>, + spec_array_addresses: Arc<[u8]>, + chrom_summary: Arc<[u8]>, + chrom_entries: Arc<[u8]>, + chrom_array_addresses: Arc<[u8]>, +} + +fn check_integrity(header: &Header, source: &Arc) -> IonResult<()> { + let layout_failures = check_section_layout(header); + if !layout_failures.is_empty() { + return Err(IonError::from(format!( + "header: file integrity validation failed ({} check(s)):\n{}", + layout_failures.len(), + layout_failures.join("\n") + ))); + } + let trailer_length = FILE_TRAILER.len() as u64; + let trailer = source.read(ByteRange { + offset: header.total_file_size.saturating_sub(trailer_length), + length: trailer_length, + })?; + if trailer[..] != FILE_TRAILER { + return Err(IonError::from("header: missing or invalid file trailer")); + } + if let Some(actual_len) = source.total_len() + && actual_len != header.total_file_size + { + return Err(IonError::from(format!( + "header: total_file_size ({}) does not match source length ({actual_len})", + header.total_file_size + ))); + } + Ok(()) +} + +fn read_fixed_sections( + header: &Header, + source: &Arc, + options: &ReadOptions, +) -> IonResult { + let spec_summary_buf = source.read(ByteRange { + offset: header.off_spec_summary, + length: header.len_spec_summary, + })?; + let chrom_summary_buf = source.read(ByteRange { + offset: header.off_chrom_summary, + length: header.len_chrom_summary, + })?; + let spec_entries_buf = source.read(ByteRange { + offset: header.off_spec_entries, + length: header.len_spec_entries, + })?; + let spec_array_addresses = source.read(ByteRange { + offset: header.off_spec_array_addresses, + length: header.len_spec_array_addresses, + })?; + let chrom_entries_buf = source.read(ByteRange { + offset: header.off_chrom_entries, + length: header.len_chrom_entries, + })?; + let chrom_array_addresses = source.read(ByteRange { + offset: header.off_chrom_array_addresses, + length: header.len_chrom_array_addresses, + })?; + + if options.verify_checksums { + check_crc(&spec_summary_buf, header.spec_summary_crc32, "spec_summary")?; + check_crc(&spec_entries_buf, header.spec_entries_crc32, "spec_entries")?; + check_crc( + &spec_array_addresses, + header.spec_array_addresses_crc32, + "spec_array_addresses", + )?; + check_crc( + &chrom_summary_buf, + header.chrom_summary_crc32, + "chrom_summary", + )?; + check_crc( + &chrom_entries_buf, + header.chrom_entries_crc32, + "chrom_entries", + )?; + check_crc( + &chrom_array_addresses, + header.chrom_array_addresses_crc32, + "chrom_array_addresses", + )?; + } + + let ( + spec_summary_plain, + spec_entries_plain, + spec_array_addresses_plain, + chrom_summary_plain, + chrom_entries_plain, + chrom_array_addresses_plain, + ): FixedSectionBuffers = if header.compression_codec == CODEC_ZSTD { + let spec_summary_plain = decompress_fixed_section( + &spec_summary_buf, + header.spectrum_count as usize * SPEC_SUMMARY_SIZE, + options.decompression_limit, + "spec_summary", + )?; + let spec_entries_plain = decompress_fixed_section( + &spec_entries_buf, + header.spectrum_count as usize * INDEX_ENTRY_BYTES, + options.decompression_limit, + "spec_entries", + )?; + let spec_array_addresses_plain = decompress_fixed_section( + &spec_array_addresses, + max_address_index(&spec_entries_plain) as usize * ARRAY_ADDRESS_BYTES, + options.decompression_limit, + "spec_array_addresses", + )?; + let chrom_summary_plain = decompress_fixed_section( + &chrom_summary_buf, + header.chrom_count as usize * CHROM_SUMMARY_SIZE, + options.decompression_limit, + "chrom_summary", + )?; + let chrom_entries_plain = decompress_fixed_section( + &chrom_entries_buf, + header.chrom_count as usize * INDEX_ENTRY_BYTES, + options.decompression_limit, + "chrom_entries", + )?; + let chrom_array_addresses_plain = decompress_fixed_section( + &chrom_array_addresses, + max_address_index(&chrom_entries_plain) as usize * ARRAY_ADDRESS_BYTES, + options.decompression_limit, + "chrom_array_addresses", + )?; + ( + spec_summary_plain, + spec_entries_plain, + spec_array_addresses_plain, + chrom_summary_plain, + chrom_entries_plain, + chrom_array_addresses_plain, + ) + } else { + ( + spec_summary_buf.into_arc(), + spec_entries_buf.into_arc(), + spec_array_addresses.into_arc(), + chrom_summary_buf.into_arc(), + chrom_entries_buf.into_arc(), + chrom_array_addresses.into_arc(), + ) + }; + + check_address_table( + &spec_entries_plain, + &spec_array_addresses_plain, + "spec_array_addresses", + )?; + check_address_table( + &chrom_entries_plain, + &chrom_array_addresses_plain, + "chrom_array_addresses", + )?; + + check_array_dtypes(&spec_array_addresses_plain, "spec array dtype")?; + check_array_dtypes(&chrom_array_addresses_plain, "chrom array dtype")?; + + Ok(FixedSections { + spec_summary: spec_summary_plain, + spec_entries: spec_entries_plain, + spec_array_addresses: spec_array_addresses_plain, + chrom_summary: chrom_summary_plain, + chrom_entries: chrom_entries_plain, + chrom_array_addresses: chrom_array_addresses_plain, + }) +} + mod arrays; mod spectra; mod to_mzml; @@ -116,16 +296,12 @@ mod windows; pub(crate) use arrays::ArrayAddress; pub(crate) use arrays::{ ArrayGroup, address_read_params, dtype_stride, group_arrays, parse_array_address, - read_array_addresses_from_buffers, read_group_decoded_bytes, unfilter_array_bytes, + read_array_addresses_from_buffers, unfilter_array_bytes, }; -#[cfg(test)] -pub(crate) use arrays::read_scan_arrays; -pub(crate) use to_mzml::attach_logical_array; pub use to_mzml::{Metadatum, MetadatumValue}; pub use windows::{DataXY, ScanQuery, Select, Window}; #[cfg(test)] -pub(crate) use windows::{ItemKind, ItemSlice}; -pub(crate) use windows::Pixel; +pub(crate) use windows::ItemKind; #[derive(Debug, Clone)] pub struct ReadOptions { @@ -166,216 +342,59 @@ pub struct IonReader { pub(crate) spec_array_addresses: Arc<[u8]>, pub(crate) chrom_entries_buf: Arc<[u8]>, pub(crate) chrom_array_addresses: Arc<[u8]>, - pub(crate) global_meta_buf: Arc<[u8]>, + pub(crate) global_meta: std::cell::OnceCell, pub(crate) spec_container: BlockReader, pub(crate) chrom_container: Option>, - pub(crate) spec_meta_reader: MetaGroupReader, - pub(crate) chrom_meta_reader: MetaGroupReader, - #[allow(dead_code)] - pub(crate) mz_values: Vec, - #[allow(dead_code)] - pub(crate) int_values: Vec, - pub(crate) parallel: bool, - pub(crate) decompression_limit: DecompressionLimit, + pub(crate) spec_meta: MetaGroupReader, + pub(crate) chrom_meta: MetaGroupReader, + pub(crate) options: ReadOptions, pub(crate) spec_rt_finite_ascending: std::cell::Cell>, } impl IonReader { - pub fn from_bytes(bytes: &[u8], config: &ReadOptions) -> IonResult { + pub fn from_bytes(bytes: &[u8], options: &ReadOptions) -> IonResult { let source = Arc::new(BytesSource::new(Arc::from(bytes))) as Arc; - Self::new(source, config) + Self::new(source, options) } #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] - pub fn open(path: impl AsRef, config: &ReadOptions) -> IonResult { + pub fn open(path: impl AsRef, options: &ReadOptions) -> IonResult { let path = path.as_ref(); let file = std::fs::File::open(path).map_err(|e| IonError::from(e.to_string()))?; let map = unsafe { memmap2::Mmap::map(&file).map_err(|e| IonError::from(e.to_string()))? }; let source = Arc::new(FileSource::new(map)) as Arc; - Self::new(source, config) + Self::new(source, options) } - pub fn new(source: Arc, config: &ReadOptions) -> IonResult { + pub fn new(source: Arc, options: &ReadOptions) -> IonResult { let header_buf = source.read(ByteRange { offset: 0, length: 1024, })?; - let header = parse_header(&header_buf)?; + let header = Header::parse(&header_buf)?; let block_packing_id = PackingId::from_byte(header.default_array_filter)?; - let spec_summary_buf = source.read(ByteRange { - offset: header.off_spec_summary, - length: header.len_spec_summary, - })?; - let chrom_summary_buf = source.read(ByteRange { - offset: header.off_chrom_summary, - length: header.len_chrom_summary, - })?; - let spec_entries_buf = source.read(ByteRange { - offset: header.off_spec_entries, - length: header.len_spec_entries, - })?; - let spec_array_addresses = source.read(ByteRange { - offset: header.off_spec_array_addresses, - length: header.len_spec_array_addresses, - })?; - let chrom_entries_buf = source.read(ByteRange { - offset: header.off_chrom_entries, - length: header.len_chrom_entries, - })?; - let chrom_array_addresses = source.read(ByteRange { - offset: header.off_chrom_array_addresses, - length: header.len_chrom_array_addresses, - })?; - let global_meta_buf = source.read(ByteRange { - offset: header.off_global_meta, - length: header.len_global_meta, - })?; - let spec_meta_buf = source.read(ByteRange { - offset: header.off_spec_meta, - length: header.len_spec_meta, - })?; - let chrom_meta_buf = source.read(ByteRange { - offset: header.off_chrom_meta, - length: header.len_chrom_meta, - })?; - - if config.verify_checksums { - let layout_failures = check_section_layout(&header); - if !layout_failures.is_empty() { - return Err(IonError::from(format!( - "header: file integrity validation failed ({} check(s)):\n{}", - layout_failures.len(), - layout_failures.join("\n") - ))); - } - let trailer_length = FILE_TRAILER.len() as u64; - let trailer = source.read(ByteRange { - offset: header.total_file_size.saturating_sub(trailer_length), - length: trailer_length, - })?; - if trailer[..] != FILE_TRAILER { - return Err(IonError::from("header: missing or invalid file trailer")); - } - if let Some(actual_len) = source.total_len() { - if actual_len != header.total_file_size { - return Err(IonError::from(format!( - "header: total_file_size ({}) does not match source length ({actual_len})", - header.total_file_size - ))); - } - } - check_crc(&spec_summary_buf, header.spec_summary_crc32, "spec_summary")?; - check_crc(&spec_entries_buf, header.spec_entries_crc32, "spec_entries")?; - check_crc( - &spec_array_addresses, - header.spec_array_addresses_crc32, - "spec_array_addresses", - )?; - check_crc( - &chrom_summary_buf, - header.chrom_summary_crc32, - "chrom_summary", - )?; - check_crc( - &chrom_entries_buf, - header.chrom_entries_crc32, - "chrom_entries", - )?; - check_crc( - &chrom_array_addresses, - header.chrom_array_addresses_crc32, - "chrom_array_addresses", - )?; - check_crc(&spec_meta_buf, header.spec_meta_crc32, "spec_meta")?; - check_crc(&chrom_meta_buf, header.chrom_meta_crc32, "chrom_meta")?; - check_crc(&global_meta_buf, header.global_meta_crc32, "global_meta")?; + if options.verify_checksums { + check_integrity(&header, &source)?; } - let ( - spec_summary_plain, - spec_entries_plain, - spec_array_addresses_plain, - chrom_summary_plain, - chrom_entries_plain, - chrom_array_addresses_plain, - ): ( - Arc<[u8]>, - Arc<[u8]>, - Arc<[u8]>, - Arc<[u8]>, - Arc<[u8]>, - Arc<[u8]>, - ) = if header.compression_codec == CODEC_ZSTD { - let spec_summary_plain = decompress_fixed_section( - &spec_summary_buf, - header.spectrum_count as usize * SPEC_SUMMARY_SIZE, - config.decompression_limit, - "spec_summary", - )?; - let spec_entries_plain = decompress_fixed_section( - &spec_entries_buf, - header.spectrum_count as usize * INDEX_ENTRY_BYTES, - config.decompression_limit, - "spec_entries", - )?; - let spec_array_addresses_plain = decompress_fixed_section( - &spec_array_addresses, - max_address_index(&spec_entries_plain) as usize * ARRAY_ADDRESS_BYTES, - config.decompression_limit, - "spec_array_addresses", - )?; - let chrom_summary_plain = decompress_fixed_section( - &chrom_summary_buf, - header.chrom_count as usize * CHROM_SUMMARY_SIZE, - config.decompression_limit, - "chrom_summary", - )?; - let chrom_entries_plain = decompress_fixed_section( - &chrom_entries_buf, - header.chrom_count as usize * INDEX_ENTRY_BYTES, - config.decompression_limit, - "chrom_entries", - )?; - let chrom_array_addresses_plain = decompress_fixed_section( - &chrom_array_addresses, - max_address_index(&chrom_entries_plain) as usize * ARRAY_ADDRESS_BYTES, - config.decompression_limit, - "chrom_array_addresses", - )?; - ( - spec_summary_plain, - spec_entries_plain, - spec_array_addresses_plain, - chrom_summary_plain, - chrom_entries_plain, - chrom_array_addresses_plain, - ) - } else { - ( - spec_summary_buf.into_arc(), - spec_entries_buf.into_arc(), - spec_array_addresses.into_arc(), - chrom_summary_buf.into_arc(), - chrom_entries_buf.into_arc(), - chrom_array_addresses.into_arc(), - ) - }; - - check_address_table( - &spec_entries_plain, - &spec_array_addresses_plain, - "spec_array_addresses", + let fixed = read_fixed_sections(&header, &source, options)?; + let meta_column_layout = MetaColumnLayout::new(); + let spec_meta = MetaGroupReader::open( + source.clone(), + header.spec_meta_section(), + header.compression_codec, + options, + meta_column_layout, )?; - check_address_table( - &chrom_entries_plain, - &chrom_array_addresses_plain, - "chrom_array_addresses", + let chrom_meta = MetaGroupReader::open( + source.clone(), + header.chrom_meta_section(), + header.compression_codec, + options, + meta_column_layout, )?; - check_array_dtypes(&spec_array_addresses_plain, "spec array dtype")?; - check_array_dtypes(&chrom_array_addresses_plain, "chrom array dtype")?; - let spec_container = BlockReader::new( source.clone(), header.off_spec_container, @@ -384,11 +403,11 @@ impl IonReader { header.spec_directory_crc32, header.compression_codec, block_packing_id, - config.verify_checksums, + options.verify_checksums, "spec", DefaultBlockProcessor, - config.max_cached_bytes, - config.decompression_limit, + options.max_cached_bytes, + options.decompression_limit, )?; let chrom_container = if header.chrom_block_count > 0 && header.len_chrom_container > 0 { @@ -400,74 +419,33 @@ impl IonReader { header.chrom_directory_crc32, header.compression_codec, block_packing_id, - config.verify_checksums, + options.verify_checksums, "chrom", DefaultBlockProcessor, - config.max_cached_bytes, - config.decompression_limit, + options.max_cached_bytes, + options.decompression_limit, )?) } else { None }; - let meta_column_layout = MetaColumnLayout::new(); - - let spec_meta_reader = MetaGroupReader::new( - spec_meta_buf.into_arc(), - header.spec_meta_group_count, - header.meta_group_size, - header.spectrum_count, - MetaTotals { - rows: header.spec_meta_count, - numeric: header.spec_meta_numeric_count, - string: header.spec_meta_string_count, - uncompressed: header.spec_meta_uncompressed_bytes, - }, - header.compression_codec, - config.verify_checksums, - config.decompression_limit, - config.max_cached_bytes, - meta_column_layout, - )?; - - let chrom_meta_reader = MetaGroupReader::new( - chrom_meta_buf.into_arc(), - header.chrom_meta_group_count, - header.meta_group_size, - header.chrom_count, - MetaTotals { - rows: header.chrom_meta_count, - numeric: header.chrom_meta_numeric_count, - string: header.chrom_meta_string_count, - uncompressed: header.chrom_meta_uncompressed_bytes, - }, - header.compression_codec, - config.verify_checksums, - config.decompression_limit, - config.max_cached_bytes, - meta_column_layout, - )?; - Ok(Self { header, source, spec_window_directory: WindowDirectoryCache::Unloaded, chrom_window_directory: WindowDirectoryCache::Unloaded, - spec_summary_buf: spec_summary_plain, - chrom_summary_buf: chrom_summary_plain, - spec_entries_buf: spec_entries_plain, - spec_array_addresses: spec_array_addresses_plain, - chrom_entries_buf: chrom_entries_plain, - chrom_array_addresses: chrom_array_addresses_plain, - global_meta_buf: global_meta_buf.into_arc(), + spec_summary_buf: fixed.spec_summary, + chrom_summary_buf: fixed.chrom_summary, + spec_entries_buf: fixed.spec_entries, + spec_array_addresses: fixed.spec_array_addresses, + chrom_entries_buf: fixed.chrom_entries, + chrom_array_addresses: fixed.chrom_array_addresses, + global_meta: std::cell::OnceCell::new(), spec_container, chrom_container, - spec_meta_reader, - chrom_meta_reader, - mz_values: Vec::new(), - int_values: Vec::new(), - parallel: config.parallel, - decompression_limit: config.decompression_limit, + spec_meta, + chrom_meta, + options: options.clone(), spec_rt_finite_ascending: std::cell::Cell::new(None), }) } @@ -601,7 +579,7 @@ impl IonReader { } Ok(bytes.to_vec()) } - CODEC_ZSTD => decompress_zstd(bytes, plain_len, self.decompression_limit), + CODEC_ZSTD => decompress_zstd(bytes, plain_len, self.options.decompression_limit), _ => Err("window bounds: unsupported codec".into()), } } @@ -635,7 +613,7 @@ pub fn header_ranges(header_bytes: &[u8]) -> IonResult> { if header_bytes.len() < 1024 { return Err("header_ranges: needs at least 1024 header bytes".into()); } - let header = parse_header(&header_bytes[..1024])?; + let header = Header::parse(&header_bytes[..1024])?; open_byte_ranges(&header) } @@ -669,18 +647,20 @@ pub(crate) fn open_byte_ranges(header: &Header) -> IonResult> { offset: header.off_chrom_array_addresses, length: header.len_chrom_array_addresses, }, - ByteRange { - offset: header.off_global_meta, - length: header.len_global_meta, - }, - ByteRange { - offset: header.off_spec_meta, - length: header.len_spec_meta, - }, - ByteRange { - offset: header.off_chrom_meta, - length: header.len_chrom_meta, - }, + meta_directory_range( + ByteRange { + offset: header.off_spec_meta, + length: header.len_spec_meta, + }, + header.spec_meta_group_count, + )?, + meta_directory_range( + ByteRange { + offset: header.off_chrom_meta, + length: header.len_chrom_meta, + }, + header.chrom_meta_group_count, + )?, ]; if header.len_spec_window_directory > 0 { diff --git a/crates/parser/src/ion/decoder/decode/spectra.rs b/crates/parser/src/ion/decoder/decode/spectra.rs index ec5e8b1b..9ccf4ac9 100644 --- a/crates/parser/src/ion/decoder/decode/spectra.rs +++ b/crates/parser/src/ion/decoder/decode/spectra.rs @@ -90,91 +90,6 @@ pub(crate) fn build_one_chromatogram( )) } -#[cfg(test)] -pub(crate) struct ScanIterator<'a, 'd> { - pub(crate) summary_chunks: std::slice::ChunksExact<'a, u8>, - pub(crate) entry_chunks: std::slice::ChunksExact<'a, u8>, - pub(crate) address_bytes: &'a [u8], - pub(crate) container: &'d mut dyn ContainerAccess, - pub(crate) mz_values: &'d mut Vec, - pub(crate) int_values: &'d mut Vec, - pub(crate) rt_min: f64, - pub(crate) rt_max: f64, - pub(crate) ms_level: u8, -} - -#[cfg(test)] -impl<'a, 'd> ScanIterator<'a, 'd> { - #[allow(clippy::too_many_arguments)] - pub(crate) fn new( - summary_bytes: &'a [u8], - entry_bytes: &'a [u8], - address_bytes: &'a [u8], - container: &'d mut dyn ContainerAccess, - mz_values: &'d mut Vec, - int_values: &'d mut Vec, - rt_min: f64, - rt_max: f64, - ms_level: u8, - ) -> Self { - Self { - summary_chunks: summary_bytes.chunks_exact(SPEC_SUMMARY_SIZE), - entry_chunks: entry_bytes.chunks_exact(INDEX_ENTRY_BYTES), - address_bytes, - container, - mz_values, - int_values, - rt_min, - rt_max, - ms_level, - } - } - - pub(crate) fn run(&mut self, callback: &mut F) - where - F: FnMut(&ScanSummary, &[f64], &[f64]), - { - for (summary_bytes, entry_bytes) in - self.summary_chunks.by_ref().zip(self.entry_chunks.by_ref()) - { - let summary = parse_spec_summary(summary_bytes); - if !summary.rt.is_finite() || summary.rt < self.rt_min || summary.rt > self.rt_max { - continue; - } - if self.ms_level != 0 && summary.ms_level != self.ms_level { - continue; - } - if !read_scan_arrays( - self.container, - entry_bytes, - self.address_bytes, - self.mz_values, - self.int_values, - ) { - continue; - } - let len = self.mz_values.len().min(self.int_values.len()); - if len == 0 { - continue; - } - let summary = ScanSummary { - rt: summary.rt, - rt_unit: TimeUnit::from_code(summary.rt_unit), - ms_level: summary.ms_level, - polarity: summary.polarity, - base_peak_mz: summary.base_peak_mz, - selected_ion_mz: summary.selected_ion_mz, - base_peak_int: summary.base_peak_int, - total_ion_current: summary.total_ion_current, - position_x: summary.position_x, - position_y: summary.position_y, - position_z: summary.position_z, - }; - callback(&summary, &self.mz_values[..len], &self.int_values[..len]); - } - } -} - impl IonReader { pub fn spectrum_summary(&self, index: usize) -> Option { let b = slice_summary( @@ -226,8 +141,10 @@ impl IonReader { } Ok(self .spec_summary_buf - .chunks_exact(SPEC_SUMMARY_SIZE) - .map(parse_spec_summary) + .as_chunks::() + .0 + .iter() + .map(|c| parse_spec_summary(c)) .collect()) } @@ -243,208 +160,99 @@ impl IonReader { } Ok(self .chrom_summary_buf - .chunks_exact(CHROM_SUMMARY_SIZE) - .map(parse_chrom_summary) + .as_chunks::() + .0 + .iter() + .map(|c| parse_chrom_summary(c)) .collect()) } - pub(crate) fn global_metadata(&self) -> IonResult> { + fn global_meta_bytes(&self) -> IonResult<&[u8]> { + if let Some(bytes) = self.global_meta.get() { + return Ok(bytes); + } + let bytes = self.source.read(ByteRange { + offset: self.header.off_global_meta, + length: self.header.len_global_meta, + })?; + if self.options.verify_checksums { + check_crc(&bytes, self.header.global_meta_crc32, "global_meta")?; + } + Ok(self.global_meta.get_or_init(|| bytes)) + } + + pub(crate) fn global_metadata_rows(&self) -> IonResult> { parse_global_metadata( - &self.global_meta_buf, + self.global_meta_bytes()?, 0, self.header.global_meta_count, self.header.global_meta_numeric_count, self.header.global_meta_string_count, self.header.compression_codec, self.header.global_meta_uncompressed_bytes, - self.decompression_limit, + self.options.decompression_limit, MetaColumnLayout::new().without_ids_reset(), ) } - pub fn spectrum_metadata(&self) -> IonResult> { - self.spec_meta_reader.read_all() - } - - pub fn chromatogram_metadata(&self) -> IonResult> { - self.chrom_meta_reader.read_all() - } - pub(crate) fn spectrum_metadata_grouped(&self) -> IonResult>> { - self.spec_meta_reader.read_all_grouped() + self.spec_meta.read_all_grouped() } pub(crate) fn chromatogram_metadata_grouped(&self) -> IonResult>> { - self.chrom_meta_reader.read_all_grouped() + self.chrom_meta.read_all_grouped() } - pub fn spectrum_metadata_at(&mut self, index: usize) -> IonResult> { - self.spec_meta_reader.read_item(index as u64) + pub(crate) fn spectrum_metadata_rows_at(&mut self, index: usize) -> IonResult> { + self.spec_meta.read_item(index as u64) } - pub fn chromatogram_metadata_at(&mut self, index: usize) -> IonResult> { - self.chrom_meta_reader.read_item(index as u64) + pub(crate) fn chromatogram_metadata_rows_at( + &mut self, + index: usize, + ) -> IonResult> { + self.chrom_meta.read_item(index as u64) } - pub fn spectrum(&mut self, index: usize) -> IonResult { + pub fn spectrum_metadata_at(&mut self, index: usize) -> IonResult { if index >= self.header.spectrum_count as usize { return Err(IonError::OutOfRange { index, count: self.header.spectrum_count, }); } - let rows = self.spec_meta_reader.read_item(index as u64)?; - let Some(mut spectrum) = build_one_spectrum(&rows, index) else { - return Err(IonError::from(format!( + let rows = self.spectrum_metadata_rows_at(index)?; + build_one_spectrum(&rows, index).ok_or_else(|| { + IonError::from(format!( "spectrum {index}: metadata is missing the spectrum tag" - ))); - }; - - if let Some(array_addresses) = read_array_addresses_from_buffers( - &self.spec_entries_buf, - &self.spec_array_addresses, - index, - ) { - let groups = group_arrays(array_addresses.as_slice())?; - let bd_list = spectrum - .binary_data_array_list - .get_or_insert_with(BinaryDataArrayList::default); - for group in groups { - let decoded = read_group_decoded_bytes(&group, &mut self.spec_container)?; - attach_logical_array( - bd_list, - group.array_type, - group.array_cv_code, - group.dtype, - &decoded, - )?; - } - bd_list.count = Some(bd_list.binary_data_arrays.len()); - } - Ok(spectrum) + )) + }) } - pub fn chromatogram(&mut self, index: usize) -> IonResult { + pub fn chromatogram_metadata_at(&mut self, index: usize) -> IonResult { if index >= self.header.chrom_count as usize { return Err(IonError::OutOfRange { index, count: self.header.chrom_count, }); } - let rows = self.chrom_meta_reader.read_item(index as u64)?; - let Some(mut chromatogram) = build_one_chromatogram(&rows, index) else { - return Err(IonError::from(format!( + let rows = self.chromatogram_metadata_rows_at(index)?; + build_one_chromatogram(&rows, index).ok_or_else(|| { + IonError::from(format!( "chromatogram {index}: metadata is missing the chromatogram tag" - ))); - }; - - if let (Some(array_addresses), Some(container)) = ( - read_array_addresses_from_buffers( - &self.chrom_entries_buf, - &self.chrom_array_addresses, - index, - ), - self.chrom_container.as_mut(), - ) { - let groups = group_arrays(array_addresses.as_slice())?; - let bd_list = chromatogram - .binary_data_array_list - .get_or_insert_with(BinaryDataArrayList::default); - for group in groups { - let decoded = read_group_decoded_bytes(&group, container)?; - attach_logical_array( - bd_list, - group.array_type, - group.array_cv_code, - group.dtype, - &decoded, - )?; - } - bd_list.count = Some(bd_list.binary_data_arrays.len()); - } - Ok(chromatogram) + )) + }) } } #[cfg(test)] -impl ScanSource for IonReader { - fn for_each_summary(&mut self, callback: &mut dyn FnMut(usize, ScanSummary)) { - for (index, chunk) in self - .spec_summary_buf - .chunks_exact(SPEC_SUMMARY_SIZE) - .enumerate() - { - let summary = parse_spec_summary(chunk); - callback( - index, - ScanSummary { - rt: summary.rt, - rt_unit: TimeUnit::from_code(summary.rt_unit), - base_peak_mz: summary.base_peak_mz, - selected_ion_mz: summary.selected_ion_mz, - base_peak_int: summary.base_peak_int, - total_ion_current: summary.total_ion_current, - ms_level: summary.ms_level, - polarity: summary.polarity, - position_x: summary.position_x, - position_y: summary.position_y, - position_z: summary.position_z, - }, - ); - } - } - - fn load_scan(&mut self, index: usize, mz: &mut Vec, intensity: &mut Vec) -> bool { - let count = match usize::try_from(self.header.spectrum_count) { - Ok(c) => c, - Err(_) => return false, - }; - if index >= count { - return false; - } - let all_entries = self.spec_entries_buf.as_ref(); - let array_address_bytes = self.spec_array_addresses.as_ref(); - let entry_start = index * INDEX_ENTRY_BYTES; - let Some(entry) = all_entries.get(entry_start..entry_start + INDEX_ENTRY_BYTES) else { - return false; - }; - read_scan_arrays( - &mut self.spec_container, - entry, - array_address_bytes, - mz, - intensity, - ) +impl IonReader { + pub(crate) fn spectrum_metadata(&self) -> IonResult> { + self.spec_meta.read_all() } - fn for_each_in_range(&mut self, rt_min: f64, rt_max: f64, ms_level: u8, mut callback: F) - where - Self: Sized, - F: FnMut(&ScanSummary, &[f64], &[f64]), - { - let summary_bytes = self.spec_summary_buf.as_ref(); - let _count = match usize::try_from(self.header.spectrum_count) { - Ok(count) => count, - Err(_) => return, - }; - let entry_bytes = self.spec_entries_buf.as_ref(); - let array_address_bytes = self.spec_array_addresses.as_ref(); - let (container, mz_values, int_values) = ( - &mut self.spec_container as &mut dyn ContainerAccess, - &mut self.mz_values, - &mut self.int_values, - ); - ScanIterator::new( - summary_bytes, - entry_bytes, - array_address_bytes, - container, - mz_values, - int_values, - rt_min, - rt_max, - ms_level, - ) - .run(&mut callback); + pub(crate) fn chromatogram_metadata(&self) -> IonResult> { + self.chrom_meta.read_all() } } + diff --git a/crates/parser/src/ion/decoder/decode/tests.rs b/crates/parser/src/ion/decoder/decode/tests.rs index dc6766fe..d234dca7 100644 --- a/crates/parser/src/ion/decoder/decode/tests.rs +++ b/crates/parser/src/ion/decoder/decode/tests.rs @@ -1,10 +1,11 @@ use super::{arrays::decode_into, *}; use crate::{ ion::{ + ArrayKind, CallbackSource, encoder::utilities::SectionStorage, format::{CODEC_NONE, CODEC_ZSTD}, }, - mzml::structs::{BinaryDataArray, CvParam, NumericArray}, + mzml::structs::{BinaryDataArray, BinaryDataArrayList, CvParam, NumericArray}, }; const BYTES: &[u8] = include_bytes!("../../../../data/ion/test.ion"); @@ -98,27 +99,60 @@ fn to_mzml_keeps_data_type_cv_param_in_place() { } #[test] -fn get_spectrum_lazy_matches_full_conversion() { +fn spectrum_metadata_at_and_spectrum_array_match_full_conversion() { let mut reader = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let full = reader.to_mzml().unwrap(); let full_spectra = full.run.spectrum_list.expect("spectrum list").spectra; for (index, full_spectrum) in full_spectra.iter().enumerate() { - let lazy = reader.spectrum(index).expect("spectrum present"); + let lazy = reader.spectrum_metadata_at(index).expect("spectrum present"); + let mut expected_metadata = full_spectrum.clone(); + drop_binaries(&mut expected_metadata.binary_data_array_list); + assert!(lazy.binary_data_array_list.is_some()); assert_eq!( format!("{lazy:?}"), - format!("{full_spectrum:?}"), - "spectrum {index} differs between lazy and full paths" + format!("{expected_metadata:?}"), + "spectrum {index} metadata differs between lazy and full paths" ); + + let mz = reader + .spectrum_array(index, crate::ion::ArrayKind::Mz) + .unwrap(); + let expected_mz = find_binary_data_array( + full_spectrum.binary_data_array_list.as_ref(), + crate::accessions::MZ_ARRAY, + ) + .map(|array| array.binary.as_ref().unwrap().to_f64()) + .unwrap_or_default(); + assert_eq!(mz, expected_mz, "spectrum {index} mz differs"); + } +} + +fn drop_binaries(list: &mut Option) { + for array in list.iter_mut().flat_map(|list| &mut list.binary_data_arrays) { + array.binary = None; } } +fn find_binary_data_array( + list: Option<&BinaryDataArrayList>, + accession_tail: u32, +) -> Option<&BinaryDataArray> { + let accession = crate::accessions::format_accession(accession_tail); + list?.binary_data_arrays.iter().find(|array| { + array + .cv_params + .iter() + .any(|p| p.accession.as_deref() == Some(accession.as_str())) + }) +} + #[test] fn metadata_at_matches_filtered_full_read() { let mut reader = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let all_spectra = reader.spectrum_metadata().unwrap(); for index in 0..reader.spectrum_count() as usize { - let one = reader.spectrum_metadata_at(index).unwrap(); + let one = reader.spectrum_metadata_rows_at(index).unwrap(); let expected: Vec<_> = all_spectra .iter() .filter(|row| row.item_index as usize == index) @@ -129,7 +163,7 @@ fn metadata_at_matches_filtered_full_read() { let all_chroms = reader.chromatogram_metadata().unwrap(); for index in 0..reader.chromatogram_count() as usize { - let one = reader.chromatogram_metadata_at(index).unwrap(); + let one = reader.chromatogram_metadata_rows_at(index).unwrap(); let expected: Vec<_> = all_chroms .iter() .filter(|row| row.item_index as usize == index) @@ -184,12 +218,21 @@ fn read_array_produces_mz_values() { fn for_each_scan_yields_matching_scans() { let mut d = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let mut count = 0usize; - d.for_each_in_range(0.0, f64::MAX, 0, |summary, mz, int| { - assert!(summary.rt.is_finite()); - assert!(!mz.is_empty()); - assert_eq!(mz.len(), int.len()); + let query = ScanQuery { + mz: Range { + from: 0.0, + to: f64::MAX, + }, + select: Select::All, + ms_level: None, + }; + d.scans_in(&query, |window| { + assert!(window.summary.rt.is_finite()); + assert!(!window.mz.is_empty()); + assert_eq!(window.mz.len(), window.intensity.len()); count += 1; - }); + }) + .unwrap(); assert_eq!(count, d.spectrum_count() as usize); } @@ -197,9 +240,18 @@ fn for_each_scan_yields_matching_scans() { fn for_each_scan_filters_by_ms_level() { let mut d = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); let mut count = 0usize; - d.for_each_in_range(0.0, f64::MAX, 1, |_, _, _| { + let query = ScanQuery { + mz: Range { + from: 0.0, + to: f64::MAX, + }, + select: Select::All, + ms_level: Some(1), + }; + d.scans_in(&query, |_window| { count += 1; - }); + }) + .unwrap(); let expected = (0..d.spectrum_count() as usize) .filter(|&i| d.spectrum_summary(i).is_some_and(|r| r.ms_level == 1)) .count(); @@ -218,7 +270,7 @@ fn reencoded_fixture() -> Vec { } #[test] -fn scans_in_visits_every_scan_and_matches_read_window() { +fn scans_in_visits_every_scan_and_matches_spectrum_window() { let bytes = reencoded_fixture(); let mut reader = IonReader::from_bytes(&bytes, &ReadOptions::default()).unwrap(); let mz = Range { @@ -233,7 +285,9 @@ fn scans_in_visits_every_scan_and_matches_read_window() { .unwrap(); assert_eq!(seen.len(), reader.spectrum_count() as usize); for (index, mz_values, intensity_values) in &seen { - let direct = reader.read_window(*index, mz).unwrap(); + let direct = reader + .spectrum_window(*index, ArrayKind::Mz, ArrayKind::Intensity, mz) + .unwrap(); assert_eq!(*mz_values, direct.x.to_f64()); assert_eq!(*intensity_values, direct.y.to_f64()); } @@ -278,7 +332,7 @@ fn spec_array_addresses_store_cv_code_in_32_byte_records() { 0, "A3 length is whole records" ); - for record in table.chunks_exact(ARRAY_ADDRESS_BYTES) { + for record in table.as_chunks::().0 { assert_eq!( record[31], crate::ion::attr_meta::CV_CODE_MS, @@ -307,7 +361,7 @@ fn to_mzml_produces_valid_structure() { #[test] fn global_metadata_returns_entries() { let d = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); - assert!(!d.global_metadata().unwrap().is_empty()); + assert!(!d.global_metadata_rows().unwrap().is_empty()); } #[test] @@ -774,6 +828,16 @@ fn make_split_bda(accession: &str, name: &str, data: Vec) -> BinaryDataArra } fn encode_one_spectrum_windowed(mz: Vec, int: Vec, mz_window: f64) -> Vec { + encode_arrays_windowed( + vec![ + make_split_bda("MS:1000514", "m/z array", mz), + make_split_bda("MS:1000515", "intensity array", int), + ], + mz_window, + ) +} + +fn encode_arrays_windowed(arrays: Vec, mz_window: f64) -> Vec { use crate::{ ion::encoder::{ encode::{TARGET_BLOCK_UNCOMPRESSED_BYTES, WriteOptions}, @@ -785,11 +849,8 @@ fn encode_one_spectrum_windowed(mz: Vec, int: Vec, mz_window: f64) -> let spectrum = Spectrum { id: "split_ms1".to_string(), binary_data_array_list: Some(BinaryDataArrayList { - count: Some(2), - binary_data_arrays: vec![ - make_split_bda("MS:1000514", "m/z array", mz), - make_split_bda("MS:1000515", "intensity array", int), - ], + count: Some(arrays.len()), + binary_data_arrays: arrays, }), ..Default::default() }; @@ -1050,7 +1111,7 @@ fn split_mz_array_roundtrips_through_to_mzml() { } #[test] -fn split_mz_array_roundtrips_through_get_spectrum() { +fn split_mz_array_roundtrips_through_spectrum_array() { let n = 50_000; let mz: Vec = (0..n).map(|i| 100.0 + i as f64 * 0.001).collect(); let int: Vec = (0..n).map(|i| (i % 777) as f64).collect(); @@ -1058,40 +1119,13 @@ fn split_mz_array_roundtrips_through_get_spectrum() { let encoded = encode_one_spectrum_windowed(mz.clone(), int.clone(), 10.0); let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); - let spectrum = decoder.spectrum(0).unwrap(); - let arrays = &spectrum - .binary_data_array_list - .as_ref() - .unwrap() - .binary_data_arrays; - - let mz_arrays: Vec<_> = arrays - .iter() - .filter(|a| { - a.cv_params - .iter() - .any(|cv| cv.accession.as_deref() == Some("MS:1000514")) - }) - .collect(); - assert_eq!(mz_arrays.len(), 1); - let NumericArray::F64(mz_out) = mz_arrays[0].binary.as_ref().unwrap() else { - panic!("expected F64 mz"); - }; - assert_eq!(mz_out, &mz); + let mz_out = decoder.spectrum_array(0, crate::accessions::MZ_ARRAY).unwrap(); + assert_eq!(mz_out, mz); - let int_arrays: Vec<_> = arrays - .iter() - .filter(|a| { - a.cv_params - .iter() - .any(|cv| cv.accession.as_deref() == Some("MS:1000515")) - }) - .collect(); - assert_eq!(int_arrays.len(), 1); - let NumericArray::F64(int_out) = int_arrays[0].binary.as_ref().unwrap() else { - panic!("expected F64 intensity"); - }; - assert_eq!(int_out, &int); + let int_out = decoder + .spectrum_array(0, crate::accessions::INTENSITY_ARRAY) + .unwrap(); + assert_eq!(int_out, int); } #[test] @@ -1104,7 +1138,7 @@ fn read_spectrum_logical_array_joins_split_segments() { let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let mz_out = decoder - .array(0, crate::accessions::MZ_ARRAY) + .spectrum_array(0, crate::accessions::MZ_ARRAY) .unwrap(); assert_eq!(mz_out, mz); } @@ -1130,13 +1164,13 @@ fn split_mz_array_roundtrips_with_disk_staged_bounds() { let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let mz_out = decoder - .array(0, crate::accessions::MZ_ARRAY) + .spectrum_array(0, crate::accessions::MZ_ARRAY) .unwrap(); assert_eq!(mz_out, mz); } #[test] -fn read_window_handles_fractional_mz_window() { +fn spectrum_window_handles_fractional_mz_window() { let mz: Vec = (0..2000).map(|i| 100.0 + i as f64 * 0.01).collect(); let int: Vec = (0..2000).map(|i| (i * 3) as f64).collect(); let encoded = encode_one_spectrum_windowed(mz.clone(), int.clone(), 2.5); @@ -1144,8 +1178,10 @@ fn read_window_handles_fractional_mz_window() { for (low, high) in [(103.0, 107.0), (100.0, 120.0), (115.5, 116.5)] { let got = decoder - .read_window( + .spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: low, to: high, @@ -1158,6 +1194,44 @@ fn read_window_handles_fractional_mz_window() { } } +#[test] +fn spectrum_window_matches_filtered_spectrum_arrays() { + use crate::accessions::MEAN_ION_MOBILITY_ARRAY; + + let mz: Vec = (0..2000).map(|i| 100.0 + i as f64 * 0.01).collect(); + let int: Vec = (0..2000).map(|i| (i * 3) as f64).collect(); + let mobility: Vec = (0..2000).map(|i| ((i * 7) % 500) as f64 * 0.002).collect(); + let encoded = encode_arrays_windowed( + vec![ + make_split_bda("MS:1000514", "m/z array", mz), + make_split_bda("MS:1000515", "intensity array", int), + make_split_bda("MS:1002816", "mean ion mobility array", mobility), + ], + 2.5, + ); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); + + let mz_kind = ArrayKind::Mz.accession(); + let intensity_kind = ArrayKind::Intensity.accession(); + let kinds = [ + (mz_kind, MEAN_ION_MOBILITY_ARRAY), + (MEAN_ION_MOBILITY_ARRAY, mz_kind), + (intensity_kind, mz_kind), + ]; + for (x_kind, y_kind) in kinds { + let x = decoder.spectrum_array(0, x_kind).unwrap(); + let y = decoder.spectrum_array(0, y_kind).unwrap(); + for (low, high) in [(103.0, 107.0), (0.2, 0.6), (100.0, 5000.0)] { + let got = decoder + .spectrum_window(0, x_kind, y_kind, Range { from: low, to: high }) + .unwrap(); + let (expected_x, expected_y) = brute_force_window(&x, &y, low, high); + assert_eq!(got.x.to_f64(), expected_x, "x {x_kind} for [{low}, {high}]"); + assert_eq!(got.y.to_f64(), expected_y, "y {y_kind} for [{low}, {high}]"); + } + } +} + fn brute_force_window(mz: &[f64], int: &[f64], low: f64, high: f64) -> (Vec, Vec) { let mut mz_out = Vec::new(); let mut int_out = Vec::new(); @@ -1189,8 +1263,10 @@ fn window_fast_path_matches_brute_force() { ]; for (low, high) in windows { let got = decoder - .read_window( + .spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: low, to: high, @@ -1230,8 +1306,10 @@ fn window_errors_when_bounds_missing() { decoder.spec_window_directory = WindowDirectoryCache::Missing; assert_eq!( - decoder.read_window( + decoder.spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: 120.0, to: 130.0 @@ -1250,8 +1328,10 @@ fn window_on_unsplit_array_uses_fallback_and_is_correct() { let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let got = decoder - .read_window( + .spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: 102.0, to: 105.0, @@ -1272,8 +1352,10 @@ fn window_out_of_range_index_errors() { let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); assert!( decoder - .read_window( + .spectrum_window( 5, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: 100.0, to: 200.0 @@ -1297,8 +1379,10 @@ fn reader_reads_mz_range() { ) .unwrap(); let got = reader - .read_window( + .spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: 120.0, to: 130.0, @@ -1319,8 +1403,10 @@ fn a1_window_directory_pairs_windows_with_mz_and_intensity_refs() { let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); let _ = decoder - .read_window( + .spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: 120.0, to: 130.0, @@ -1361,22 +1447,18 @@ fn a1_window_directory_pairs_windows_with_mz_and_intensity_refs() { } #[test] -fn generic_window_matches_read_mz_range() { +fn spectrum_window_returns_values_in_range() { let n = 50_000; let mz: Vec = (0..n).map(|i| 100.0 + i as f64 * 0.001).collect(); let int: Vec = (0..n).map(|i| i as f64).collect(); - let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - - let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); - - let generic = decoder - .read_spectrum_window(0, crate::accessions::MZ_ARRAY, ACC_INT, 120.0, 130.0) - .unwrap(); + let encoded = encode_one_spectrum_windowed(mz.clone(), int.clone(), 10.0); let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); - let mz_range = decoder - .read_window( + let result = decoder + .spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: 120.0, to: 130.0, @@ -1384,24 +1466,20 @@ fn generic_window_matches_read_mz_range() { ) .unwrap(); - assert_eq!(generic.x, mz_range.x); - assert_eq!(generic.y, mz_range.y); -} - -#[test] -fn generic_window_missing_accession_returns_empty() { - let mz: Vec = (0..10).map(|i| 100.0 + i as f64).collect(); - let int: Vec = (0..10).map(|i| i as f64).collect(); - let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - - let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); - - let got = decoder - .read_spectrum_window(0, 99_999_999u32, ACC_INT, 1.0, 2.0) - .unwrap(); + let expected_mz: Vec = mz + .iter() + .copied() + .filter(|&m| (120.0..=130.0).contains(&m)) + .collect(); + let expected_int: Vec = mz + .iter() + .zip(int.iter()) + .filter(|&(&m, _)| (120.0..=130.0).contains(&m)) + .map(|(_, &i)| i) + .collect(); - assert!(got.x.is_empty()); - assert!(got.y.is_empty()); + assert_eq!(result.x.to_f64(), expected_mz); + assert_eq!(result.y.to_f64(), expected_int); } fn spec_directory_range(header: &Header) -> (usize, usize) { @@ -1420,7 +1498,7 @@ fn directory_crc_roundtrips() { let int: Vec = (0..n).map(|i| i as f64).collect(); let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); let (start, end) = spec_directory_range(&header); let computed = crc32fast::hash(&encoded[start..end]); @@ -1435,7 +1513,7 @@ fn flipped_directory_offset_is_caught_before_any_read() { let int: Vec = (0..n).map(|i| i as f64).collect(); let mut encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); let (start, _end) = spec_directory_range(&header); encoded[start] ^= 0xFF; @@ -1452,7 +1530,7 @@ fn verify_off_skips_directory_check() { let int: Vec = (0..n).map(|i| i as f64).collect(); let mut encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); let (start, _end) = spec_directory_range(&header); encoded[start] ^= 0xFF; @@ -1469,7 +1547,7 @@ fn empty_container_directory_crc_is_consistent() { let int: Vec = (0..10).map(|i| i as f64).collect(); let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); assert_eq!(header.chrom_block_count, 0); assert_eq!(header.chrom_directory_crc32, crc32fast::hash(&[])); assert!(IonReader::from_bytes(&encoded, &ReadOptions::default()).is_ok()); @@ -1481,7 +1559,7 @@ fn a1_window_directory_section_is_written() { let int: Vec = (0..1000).map(|i| i as f64).collect(); let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); assert!( header.len_spec_window_directory > 0, @@ -1556,7 +1634,7 @@ fn candidate_items_empty_on_a1_crc_failure() { let mut encoded = encode_one_spectrum_windowed(mz, int, 10.0); let a1_offset = { - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); header.off_spec_window_directory as usize }; @@ -1595,7 +1673,7 @@ fn ensure_chrom_window_directory_distinguishes_missing_from_bad_checksum() { let mut encoded = encode_one_chromatogram_windowed(time, int, 10.0); let b1_offset = { - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); header.off_chrom_window_directory as usize }; assert!(b1_offset > 0, "chromatogram window directory should exist"); @@ -1633,7 +1711,7 @@ fn b1_header_fields_are_populated() { let int: Vec = (0..n).map(|i| i as f64).collect(); let encoded = encode_one_spectrum_windowed(mz, int, 10.0); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); assert_eq!( header.off_chrom_window_directory, 0, @@ -1757,7 +1835,7 @@ fn require_bounds_errors_when_a1_missing() { #[test] fn require_bounds_errors_on_bad_checksum() { let (_, _, mut encoded) = split_file_with_a1(); - let a1_offset = parse_header(&encoded[..1024]) + let a1_offset = Header::parse(&encoded[..1024]) .unwrap() .off_spec_window_directory as usize; assert!(a1_offset > 0); @@ -1788,8 +1866,10 @@ fn read_mz_range_matches_brute_force() { for (low, high) in [(120.0, 130.0), (100.0, 149.999), (130.5, 130.5)] { let got = decoder - .read_window( + .spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: low, to: high, @@ -1816,8 +1896,10 @@ fn read_mz_range_errors_when_a1_missing() { let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); decoder.spec_window_directory = WindowDirectoryCache::Missing; assert_eq!( - decoder.read_window( + decoder.spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: 120.0, to: 130.0 @@ -1834,8 +1916,10 @@ fn read_mz_range_errors_when_window_directory_malformed() { decoder.spec_window_directory = WindowDirectoryCache::Malformed("bad window directory".to_string()); - let result = decoder.read_window( + let result = decoder.spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: 100.0, to: 200.0, @@ -1850,8 +1934,10 @@ fn read_mz_range_errors_on_low_above_high() { let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); assert!( decoder - .read_window( + .spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: 130.0, to: 120.0 @@ -1867,8 +1953,10 @@ fn read_mz_range_errors_on_non_finite_bounds() { let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); assert!( decoder - .read_window( + .spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: f64::NAN, to: 130.0 @@ -1878,8 +1966,10 @@ fn read_mz_range_errors_on_non_finite_bounds() { ); assert!( decoder - .read_window( + .spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: 120.0, to: f64::INFINITY @@ -1953,7 +2043,7 @@ fn plan_and_read_mz_range_use_same_segments() { #[test] fn plan_open_ranges_includes_spec_a1() { let (_, _, encoded) = split_file_with_a1(); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); assert!(header.len_spec_window_directory > 0); let ranges = header_ranges(&encoded[..1024]).unwrap(); @@ -1993,12 +2083,12 @@ fn candidate_items_carries_intensity_address() { let paired = windows.iter().any(|window| { window.mz_address.block_id == mz_address.block_id && window.mz_address.element_offset == mz_address.element_offset - && window.intensity_address.block_id == intensity_address.block_id - && window.intensity_address.element_offset == intensity_address.element_offset + && window.y_address.block_id == intensity_address.block_id + && window.y_address.element_offset == intensity_address.element_offset }); assert!( paired, - "slice {slice:?} does not match any read_window pair" + "slice {slice:?} does not match any spectrum window pair" ); } } @@ -2235,7 +2325,9 @@ fn window_major_roundtrip_decodes_correctly() { to: f64::MAX, }; for scan_index in 0..rts.len() { - let read = reader.read_window(scan_index, everything).unwrap(); + let read = reader + .spectrum_window(scan_index, ArrayKind::Mz, ArrayKind::Intensity, everything) + .unwrap(); let NumericArray::F64(mz) = read.x else { panic!("expected an f64 mz array"); }; @@ -2433,7 +2525,7 @@ fn coalesce_merges_adjacent_and_respects_gap() { #[test] fn coalesced_open_ranges_cover_every_open_range() { let (_, _, encoded) = split_file_with_a1(); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); let planned = header_ranges(&encoded[..1024]).unwrap(); let alignment_gap = 8; @@ -2441,6 +2533,9 @@ fn coalesced_open_ranges_cover_every_open_range() { merge_ranges(&mut coalesced, alignment_gap); for range in &planned { + if range.length == 0 { + continue; + } let range_end = range.offset + range.length; let covered = coalesced.iter().any(|merged| { merged.offset <= range.offset && range_end <= merged.offset + merged.length @@ -2448,22 +2543,73 @@ fn coalesced_open_ranges_cover_every_open_range() { assert!(covered, "range {range:?} is not covered by {coalesced:?}"); } - assert_eq!(coalesced.len(), 2); - assert_eq!(coalesced[0].offset, 0); - assert_eq!(coalesced[0].length, 1024); - - let (spec_directory_start, _) = spec_directory_range(&header); - assert_eq!(coalesced[1].offset, spec_directory_start as u64); assert_eq!( - coalesced[1].offset + coalesced[1].length, - header.total_file_size + coalesced[0], + ByteRange { + offset: 0, + length: 1024, + } ); + + let not_overlapped = |unplanned: ByteRange| { + if unplanned.length == 0 { + return; + } + let unplanned_end = unplanned.offset + unplanned.length; + let overlaps = coalesced.iter().any(|merged| { + merged.offset < unplanned_end && unplanned.offset < merged.offset + merged.length + }); + assert!( + !overlaps, + "unplanned range {unplanned:?} overlaps {coalesced:?}" + ); + }; + + let spec_meta_group_entries = header.spec_meta_group_count * 32; + not_overlapped(ByteRange { + offset: header.off_spec_meta, + length: header.len_spec_meta - spec_meta_group_entries, + }); + + let chrom_meta_group_entries = header.chrom_meta_group_count * 32; + not_overlapped(ByteRange { + offset: header.off_chrom_meta, + length: header.len_chrom_meta - chrom_meta_group_entries, + }); + + not_overlapped(ByteRange { + offset: header.off_global_meta, + length: header.len_global_meta, + }); +} + +#[test] +fn open_reads_only_planned_ranges() { + let (_, _, encoded) = split_file_with_a1(); + let seen = Arc::new(std::sync::Mutex::new(Vec::new())); + let log = seen.clone(); + let bytes: Arc<[u8]> = Arc::from(encoded.as_slice()); + let source = CallbackSource::new(move |range: ByteRange| { + log.lock().unwrap().push(range); + Ok(bytes[range.offset as usize..(range.offset + range.length) as usize].to_vec()) + }); + IonReader::new(Arc::new(source), &ReadOptions::default()).unwrap(); + + let planned = header_ranges(&encoded[..1024]).unwrap(); + for range in seen.lock().unwrap().iter() { + assert!( + planned + .iter() + .any(|p| p.offset <= range.offset && range.offset + range.length <= p.offset + p.length), + "{range:?} not planned" + ); + } } #[test] fn plan_open_ranges_includes_container_directories() { let (_, _, encoded) = split_file_with_a1(); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); let (start, end) = spec_directory_range(&header); let ranges = header_ranges(&encoded[..1024]).unwrap(); @@ -2494,8 +2640,10 @@ fn reader_plans_and_reads_mz_range() { assert!(!plan.is_empty()); let got = reader - .read_window( + .spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: 120.0, to: 130.0, @@ -2559,7 +2707,7 @@ fn roundtrip_compressed_sections_are_smaller_and_lossless() { ) .unwrap(); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); assert_eq!(header.compression_codec, CODEC_ZSTD); assert_eq!(header.spectrum_count, spectrum_count as u64); @@ -2679,7 +2827,7 @@ fn compression_disabled_keeps_sections_raw() { ) .unwrap(); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); assert_eq!(header.compression_codec, CODEC_NONE); assert_eq!( header.len_spec_summary, @@ -2723,7 +2871,7 @@ fn compressed_roundtrip_edge_cases() { let mut encoded = Vec::new(); write_mzml_to_ion(&mzml_in, &compressed_options, &mut encoded).unwrap(); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); assert_eq!(header.compression_codec, CODEC_ZSTD); assert_eq!(header.len_chrom_summary, 0); assert_eq!(header.len_chrom_entries, 0); @@ -2741,7 +2889,7 @@ fn compressed_roundtrip_edge_cases() { let mut encoded = Vec::new(); write_mzml_to_ion(&mzml_in, &compressed_options, &mut encoded).unwrap(); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); assert_eq!(header.compression_codec, CODEC_ZSTD); assert_eq!(header.len_spec_summary, 0); assert_eq!(header.len_spec_entries, 0); @@ -2778,7 +2926,7 @@ fn compressed_roundtrip_edge_cases() { let mut encoded = Vec::new(); write_mzml_to_ion(&mzml_in, &compressed_options, &mut encoded).unwrap(); - let header = parse_header(&encoded[..1024]).unwrap(); + let header = Header::parse(&encoded[..1024]).unwrap(); assert_eq!(header.compression_codec, CODEC_ZSTD); assert_eq!(header.len_spec_array_addresses, 0); @@ -2834,3 +2982,166 @@ fn ms_level_round_trips_without_cvparam_3() { assert_eq!(decoded_spectrum.ms_level, Some(2)); } + +#[test] +fn chromatogram_metadata_at_and_chromatogram_array_match_full_conversion() { + let mut reader = IonReader::from_bytes(BYTES, &ReadOptions::default()).unwrap(); + let full = reader.to_mzml().unwrap(); + let full_chromatograms = full + .run + .chromatogram_list + .expect("chromatogram list") + .chromatograms; + for (index, full_chromatogram) in full_chromatograms.iter().enumerate() { + let metadata_only = reader.chromatogram_metadata_at(index).unwrap(); + let mut expected_metadata = full_chromatogram.clone(); + drop_binaries(&mut expected_metadata.binary_data_array_list); + assert!(metadata_only.binary_data_array_list.is_some()); + assert_eq!( + format!("{metadata_only:?}"), + format!("{expected_metadata:?}"), + "chromatogram {index} metadata differs between lazy and full paths" + ); + + let time = reader + .chromatogram_array(index, crate::ion::ArrayKind::Time) + .unwrap(); + let expected_time = find_binary_data_array( + full_chromatogram.binary_data_array_list.as_ref(), + crate::accessions::TIME_ARRAY, + ) + .map(|array| array.binary.as_ref().unwrap().to_f64()) + .unwrap_or_default(); + assert_eq!(time, expected_time, "chromatogram {index} time differs"); + + let intensity = reader + .chromatogram_array(index, crate::ion::ArrayKind::Intensity) + .unwrap(); + let expected_intensity = find_binary_data_array( + full_chromatogram.binary_data_array_list.as_ref(), + crate::accessions::INTENSITY_ARRAY, + ) + .map(|array| array.binary.as_ref().unwrap().to_f64()) + .unwrap_or_default(); + assert_eq!( + intensity, expected_intensity, + "chromatogram {index} intensity differs" + ); + } +} + +#[test] +fn metadata_at_reads_no_data_blocks() { + let bytes: Arc<[u8]> = Arc::from(BYTES); + let seen = Arc::new(std::sync::Mutex::new(Vec::new())); + let log = seen.clone(); + let cb_bytes = bytes.clone(); + let source = CallbackSource::new(move |range: ByteRange| { + log.lock().unwrap().push(range); + Ok(cb_bytes[range.offset as usize..(range.offset + range.length) as usize].to_vec()) + }); + let mut reader = IonReader::new(Arc::new(source), &ReadOptions::default()).unwrap(); + let c_start = reader.header.off_spec_meta; + let c_end = c_start + reader.header.len_spec_meta; + let d_start = reader.header.off_chrom_meta; + let d_end = d_start + reader.header.len_chrom_meta; + + seen.lock().unwrap().clear(); + reader.spectrum_metadata_at(0).unwrap(); + + for range in seen.lock().unwrap().iter() { + assert!( + range.offset >= c_start && range.offset + range.length <= c_end, + "{range:?} outside C section [{c_start},{c_end})" + ); + } + + seen.lock().unwrap().clear(); + reader.chromatogram_metadata_at(0).unwrap(); + + for range in seen.lock().unwrap().iter() { + assert!( + range.offset >= d_start && range.offset + range.length <= d_end, + "{range:?} outside D section [{d_start},{d_end})" + ); + } +} + +#[test] +fn global_metadata_reads_list_headers_from_metadata_sections_only() { + let bytes: Arc<[u8]> = Arc::from(BYTES); + let seen = Arc::new(std::sync::Mutex::new(Vec::new())); + let log = seen.clone(); + let cb_bytes = bytes.clone(); + let source = CallbackSource::new(move |range: ByteRange| { + log.lock().unwrap().push(range); + Ok(cb_bytes[range.offset as usize..(range.offset + range.length) as usize].to_vec()) + }); + let mut reader = IonReader::new(Arc::new(source), &ReadOptions::default()).unwrap(); + let header = &reader.header; + let sections = [ + (header.off_spec_meta, header.len_spec_meta), + (header.off_chrom_meta, header.len_chrom_meta), + (header.off_global_meta, header.len_global_meta), + ]; + + seen.lock().unwrap().clear(); + let metadata = reader.global_metadata().unwrap(); + + for range in seen.lock().unwrap().iter() { + assert!( + sections.iter().any(|&(start, length)| { + range.offset >= start && range.offset + range.length <= start + length + }), + "{range:?} outside the metadata sections" + ); + } + + let full = reader.to_mzml().unwrap(); + let spectrum_list = metadata.run.spectrum_list.expect("spectrum list"); + let full_spectrum_list = full.run.spectrum_list.expect("full spectrum list"); + assert!(spectrum_list.spectra.is_empty()); + assert_eq!(spectrum_list.count, full_spectrum_list.count); + assert_eq!( + spectrum_list.default_data_processing_ref, + full_spectrum_list.default_data_processing_ref + ); + let chromatogram_list = metadata.run.chromatogram_list.expect("chromatogram list"); + let full_chromatogram_list = full.run.chromatogram_list.expect("full chromatogram list"); + assert!(chromatogram_list.chromatograms.is_empty()); + assert_eq!(chromatogram_list.count, full_chromatogram_list.count); + assert_eq!( + chromatogram_list.default_data_processing_ref, + full_chromatogram_list.default_data_processing_ref + ); +} + +#[test] +fn spectrum_window_keeps_file_dtype_on_every_path() { + let mz: Vec = (0..200).map(|i| 100.0 + i as f64).collect(); + let mut intensity = make_split_bda("MS:1000515", "intensity array", Vec::new()); + intensity.binary = Some(NumericArray::F32((0..200).map(|i| i as f32).collect())); + let encoded = encode_arrays_windowed( + vec![make_split_bda("MS:1000514", "m/z array", mz), intensity], + 10.0, + ); + let mut decoder = IonReader::from_bytes(&encoded, &ReadOptions::default()).unwrap(); + let everything = Range { + from: 0.0, + to: 1000.0, + }; + + let fast = decoder + .spectrum_window(0, ArrayKind::Mz, ArrayKind::Intensity, everything) + .unwrap(); + let filtered = decoder + .spectrum_window(0, ArrayKind::Intensity, ArrayKind::Mz, everything) + .unwrap(); + + assert!(matches!(fast.x, NumericArray::F64(_))); + assert!(matches!(fast.y, NumericArray::F32(_))); + assert!(matches!(filtered.x, NumericArray::F32(_))); + assert!(matches!(filtered.y, NumericArray::F64(_))); + assert_eq!(filtered.x, fast.y); + assert_eq!(filtered.y, fast.x); +} diff --git a/crates/parser/src/ion/decoder/decode/to_mzml.rs b/crates/parser/src/ion/decoder/decode/to_mzml.rs index e774c0e7..71cc96ac 100644 --- a/crates/parser/src/ion/decoder/decode/to_mzml.rs +++ b/crates/parser/src/ion/decoder/decode/to_mzml.rs @@ -36,8 +36,8 @@ impl<'d> MzmlConverter<'d> { Self { decoder } } - pub(crate) fn metadata_only(decoder: &IonReader) -> IonResult { - let global_meta = decoder.global_metadata()?; + fn run_level(decoder: &IonReader) -> IonResult { + let global_meta = decoder.global_metadata_rows()?; let global_lookup = ChildrenLookup::new(&global_meta); let meta_refs: Vec<&Metadatum> = global_meta.iter().collect(); let policy = DefaultMetadataPolicy; @@ -58,10 +58,6 @@ impl<'d> MzmlConverter<'d> { global_lookup.get_param_rows_into(&owner_rows, run_id, &policy, &mut param_buffer); let (cv_params, user_params) = parse_cv_and_user_params(¶m_buffer); - let spectrum_list = assemble_spectrum_list(&decoder.spectrum_metadata_grouped()?, &policy); - let chromatogram_list = - assemble_chromatogram_list(&decoder.chromatogram_metadata_grouped()?, &policy); - let source_file_ref_list = parse_run_source_file_refs(&owner_rows, &global_lookup, run_id); Ok(MzML { @@ -99,14 +95,19 @@ impl<'d> MzmlConverter<'d> { cv_params, user_params, source_file_ref_list, - spectrum_list, - chromatogram_list, + spectrum_list: None, + chromatogram_list: None, }, }) } pub(crate) fn full(&mut self) -> IonResult { - let mut mzml = Self::metadata_only(self.decoder)?; + let mut mzml = Self::run_level(self.decoder)?; + let policy = DefaultMetadataPolicy; + mzml.run.spectrum_list = + assemble_spectrum_list(&self.decoder.spectrum_metadata_grouped()?, &policy); + mzml.run.chromatogram_list = + assemble_chromatogram_list(&self.decoder.chromatogram_metadata_grouped()?, &policy); if let Some(spectrum_list) = mzml.run.spectrum_list.as_mut() { attach_binaries( @@ -115,7 +116,7 @@ impl<'d> MzmlConverter<'d> { &mut spectrum_list.spectra, &self.decoder.spec_container, "spec", - self.decoder.parallel, + self.decoder.options.parallel, )?; } @@ -129,7 +130,7 @@ impl<'d> MzmlConverter<'d> { &mut chrom_list.chromatograms, container, "chrom", - self.decoder.parallel, + self.decoder.options.parallel, )?; } @@ -156,10 +157,10 @@ fn assemble_spectrum_list( Some(existing) => existing.spectra.extend(list.spectra), } } - if let Some(list) = combined.as_mut() { - if list.count.is_none() { - list.count = Some(list.spectra.len()); - } + if let Some(list) = combined.as_mut() + && list.count.is_none() + { + list.count = Some(list.spectra.len()); } combined } @@ -183,10 +184,10 @@ fn assemble_chromatogram_list( Some(existing) => existing.chromatograms.extend(list.chromatograms), } } - if let Some(list) = combined.as_mut() { - if list.count.is_none() { - list.count = Some(list.chromatograms.len()); - } + if let Some(list) = combined.as_mut() + && list.count.is_none() + { + list.count = Some(list.chromatograms.len()); } combined } @@ -376,7 +377,7 @@ fn unfiltered_ref_bytes<'d>( unfilter_array_bytes(raw, group.dtype, group.array_filter) } -pub(crate) fn attach_logical_array( +fn attach_logical_array( binary_array_list: &mut BinaryDataArrayList, array_type: u32, array_cv_code: u8, @@ -525,6 +526,19 @@ fn binary_array_has_type(binary_array: &BinaryDataArray, array_type: u32) -> boo .any(|param| parse_accession_tail(param.accession.as_deref()).raw() == array_type) } +type ListHeader<'l, L> = Option<(L, &'l [u32])>; + +fn list_header( + rows: &[Metadatum], + parse: for<'l> fn(&OwnerRows, &'l ChildrenLookup) -> ListHeader<'l, L>, +) -> Option { + let mut owner_rows = OwnerRows::with_capacity(rows.len()); + for row in rows { + owner_rows.insert(row.id, row); + } + parse(&owner_rows, &ChildrenLookup::new(rows)).map(|(list, _)| list) +} + fn parse_run_source_file_refs( owner_rows: &OwnerRows, lookup: &ChildrenLookup, @@ -568,8 +582,17 @@ fn parse_run_source_file_refs( } impl IonReader { - pub fn metadata(&self) -> IonResult { - MzmlConverter::metadata_only(self) + pub fn global_metadata(&self) -> IonResult { + let mut mzml = MzmlConverter::run_level(self)?; + mzml.run.spectrum_list = list_header( + &self.spec_meta.read_first_group()?, + parse_spectrum_list_header, + ); + mzml.run.chromatogram_list = list_header( + &self.chrom_meta.read_first_group()?, + parse_chromatogram_list_header, + ); + Ok(mzml) } pub fn to_mzml(&mut self) -> IonResult { diff --git a/crates/parser/src/ion/decoder/decode/windows.rs b/crates/parser/src/ion/decoder/decode/windows.rs index f5b2f71c..47318e55 100644 --- a/crates/parser/src/ion/decoder/decode/windows.rs +++ b/crates/parser/src/ion/decoder/decode/windows.rs @@ -1,5 +1,8 @@ use super::*; -use crate::ion::windowing::window_index; +use crate::{ + accessions::MZ_ARRAY, + ion::{ArrayKind, decoder::utilities::window_directory::WindowEntry, windowing::window_index}, +}; fn window_span(width: f64, from: f64, to: f64, window_count: usize) -> Option<(usize, usize)> { if window_count == 0 { @@ -22,15 +25,6 @@ pub struct DataXY { pub y: NumericArray, } -impl DataXY { - pub(crate) fn empty() -> Self { - Self { - x: NumericArray::F64(Vec::new()), - y: NumericArray::F64(Vec::new()), - } - } -} - impl NumericArray { pub fn to_f64(&self) -> Vec { match self { @@ -55,21 +49,12 @@ impl NumericArray { } } -#[derive(Debug, Clone, Copy, PartialEq)] -pub(crate) struct Pixel { - pub(crate) x: Range, - pub(crate) y: Range, - pub(crate) z: Range, -} - #[non_exhaustive] -#[allow(private_interfaces)] #[derive(Debug, Clone, Copy, PartialEq, Default)] pub enum Select { #[default] All, Rt(Range), - Area(Pixel), } #[derive(Debug, Clone, Copy, PartialEq, Default)] @@ -87,20 +72,10 @@ pub struct Window<'a> { pub intensity: &'a [f64], } -fn position_in_range(value: u32, range: Range) -> bool { - let value = value as f64; - value >= range.from && value <= range.to -} - fn scan_is_selected(select: &Select, summary: &ScanSummary) -> bool { match select { Select::All => true, Select::Rt(range) => summary.rt >= range.from && summary.rt <= range.to, - Select::Area(pixel) => { - position_in_range(summary.position_x, pixel.x) - && position_in_range(summary.position_y, pixel.y) - && position_in_range(summary.position_z, pixel.z) - } } } @@ -138,7 +113,7 @@ pub(crate) struct ItemSlice { #[derive(Debug, Clone, Copy)] pub(crate) struct WindowRead { pub(crate) mz_address: ArrayAddress, - pub(crate) intensity_address: ArrayAddress, + pub(crate) y_address: ArrayAddress, } fn empty_array(dtype: u8) -> NumericArray { @@ -152,14 +127,6 @@ fn empty_array(dtype: u8) -> NumericArray { } } -fn group_dtype(group: &ArrayGroup) -> u8 { - group - .refs - .first() - .map(|array_address| array_address.dtype) - .unwrap_or(FILE_DTYPE_F64) -} - fn value_at(array: &NumericArray, index: usize) -> f64 { match array { NumericArray::F64(values) => values[index], @@ -220,8 +187,15 @@ fn range_in_sorted(x: &NumericArray, low: f64, high: f64, paired: usize) -> (usi ) } -fn is_sorted(x: &NumericArray, len: usize) -> bool { - (1..len).all(|index| value_at(x, index - 1) <= value_at(x, index)) +fn array_run(addresses: &[ArrayAddress], kind: u32) -> Option<(usize, &[ArrayAddress])> { + let start = addresses.iter().position(|address| { + address.array_type == kind && address.continues_previous_segment == 0 + })?; + let length = addresses[start + 1..] + .iter() + .take_while(|address| address.continues_previous_segment == 1) + .count(); + Some((start, &addresses[start..=start + length])) } pub(crate) fn keep_pairs_sorted( @@ -238,89 +212,71 @@ pub(crate) fn keep_pairs_sorted( append_range(y_out, y, start, end); } -pub(crate) fn keep_pairs( - x: &NumericArray, - y: &NumericArray, - low: f64, - high: f64, - x_out: &mut NumericArray, - y_out: &mut NumericArray, -) { - let paired = x.len().min(y.len()); - if is_sorted(x, paired) { - keep_pairs_sorted(x, y, low, high, x_out, y_out); - return; - } - for index in 0..paired { - let value = value_at(x, index); - if value >= low && value <= high { - append_range(x_out, x, index, index + 1); - append_range(y_out, y, index, index + 1); - } - } -} - impl IonReader { - pub fn read_spectrum_window( - &mut self, - index: usize, - x_array_accession: impl Into, - y_array_accession: impl Into, - low: f64, - high: f64, - ) -> IonResult { - let x_array_accession = x_array_accession.into(); - let y_array_accession = y_array_accession.into(); - self.read_spectrum_window_inner(index, x_array_accession, y_array_accession, low, high) - } - pub fn require_bounds(&mut self) -> IonResult<()> { self.ensure_spec_window_directory() } - pub(crate) fn get_spectrum_mz_windows( - &mut self, - scan_index: usize, - mz_from: f64, - mz_to: f64, - ) -> IonResult> { - if !mz_from.is_finite() || !mz_to.is_finite() { - return Err("mz window bounds must be finite".into()); + fn check_window(&self, index: usize, range: Range) -> IonResult<()> { + if !range.from.is_finite() || !range.to.is_finite() { + return Err("window bounds must be finite".into()); } - if mz_from > mz_to { - return Err("mz window: from is greater than to".into()); + if range.from > range.to { + return Err("window: from is greater than to".into()); } - if scan_index >= self.header.spectrum_count as usize { + if index >= self.header.spectrum_count as usize { return Err("spectrum index out of range".into()); } + Ok(()) + } + fn window_rows(&mut self, index: usize, mz: Range) -> IonResult> { self.require_bounds()?; let width = self.header.target_mz_window as f64; - let rows = { - let window_directory = match &self.spec_window_directory { - WindowDirectoryCache::Loaded(index) => index, - _ => return Err(IonError::MissingSpectrumBounds), - }; - let Some((window_low, window_high)) = - window_span(width, mz_from, mz_to, window_directory.window_count()) - else { - return Ok(Vec::new()); - }; - let mut rows = Vec::new(); - for window in window_low..=window_high { - if let Some(row) = window_directory.find_in_window(window, scan_index as u32) { - rows.push(row); - } + let window_directory = match &self.spec_window_directory { + WindowDirectoryCache::Loaded(index) => index, + _ => return Err(IonError::MissingSpectrumBounds), + }; + let Some((window_low, window_high)) = + window_span(width, mz.from, mz.to, window_directory.window_count()) + else { + return Ok(Vec::new()); + }; + let mut rows = Vec::new(); + for window in window_low..=window_high { + if let Some(row) = window_directory.find_in_window(window, index as u32) { + rows.push(row); } - rows + } + Ok(rows) + } + + pub(crate) fn get_spectrum_mz_windows( + &mut self, + scan_index: usize, + mz_from: f64, + mz_to: f64, + ) -> IonResult> { + let mz = Range { + from: mz_from, + to: mz_to, }; + self.check_window(scan_index, mz)?; + let rows = self.window_rows(scan_index, mz)?; + self.window_reads(&rows, |row| row.intensity_address) + } + fn window_reads( + &self, + rows: &[WindowEntry], + y_address: impl Fn(&WindowEntry) -> u32, + ) -> IonResult> { let mut windows = Vec::with_capacity(rows.len()); for row in rows { windows.push(WindowRead { mz_address: self.array_address_at(row.mz_address)?, - intensity_address: self.array_address_at(row.intensity_address)?, + y_address: self.array_address_at(y_address(row))?, }); } Ok(windows) @@ -408,7 +364,7 @@ impl IonReader { let mut block_ids = Vec::with_capacity(windows.len() * 2); for window in &windows { block_ids.push(window.mz_address.block_id); - block_ids.push(window.intensity_address.block_id); + block_ids.push(window.y_address.block_id); } block_ids.sort_unstable(); block_ids.dedup(); @@ -421,38 +377,125 @@ impl IonReader { Ok(ranges) } - pub fn read_window(&mut self, index: usize, mz: Range) -> IonResult { - let windows = self.get_spectrum_mz_windows(index, mz.from, mz.to)?; + pub fn spectrum_window( + &mut self, + index: usize, + x_kind: impl Into, + y_kind: impl Into, + range: Range, + ) -> IonResult { + let (x_kind, y_kind) = (x_kind.into(), y_kind.into()); + self.check_window(index, range)?; + if x_kind != MZ_ARRAY { + return self.filtered_window(index, x_kind, y_kind, range); + } + if y_kind == ACC_INT { + return self.mz_window(index, range, |row| row.intensity_address); + } + match self.y_shift(index, y_kind) { + Some(shift) => self.mz_window(index, range, |row| { + row.mz_address.wrapping_add_signed(shift) + }), + None => self.filtered_window(index, x_kind, y_kind, range), + } + } - let mz_dtype = windows - .first() - .map(|s| s.mz_address.dtype) - .unwrap_or(FILE_DTYPE_F64); - let intensity_dtype = windows + fn mz_window( + &mut self, + index: usize, + mz: Range, + y_address: impl Fn(&WindowEntry) -> u32, + ) -> IonResult { + let rows = self.window_rows(index, mz)?; + let windows = self.window_reads(&rows, y_address)?; + let (mz_dtype, y_dtype) = windows .first() - .map(|s| s.intensity_address.dtype) - .unwrap_or(FILE_DTYPE_F64); + .map(|window| (window.mz_address.dtype, window.y_address.dtype)) + .unwrap_or((FILE_DTYPE_F64, FILE_DTYPE_F64)); let mut mz_out = empty_array(mz_dtype); - let mut intensity_out = empty_array(intensity_dtype); + let mut y_out = empty_array(y_dtype); for window in windows { let mz_segment = self.read_spectrum_array(&window.mz_address)?; - let intensity_segment = self.read_spectrum_array(&window.intensity_address)?; + let y_segment = self.read_spectrum_array(&window.y_address)?; keep_pairs_sorted( &mz_segment, - &intensity_segment, + &y_segment, mz.from, mz.to, &mut mz_out, - &mut intensity_out, + &mut y_out, ); } Ok(DataXY { x: mz_out, - y: intensity_out, + y: y_out, }) } + fn y_shift(&self, index: usize, y_kind: u32) -> Option { + let addresses = read_array_addresses_from_buffers( + &self.spec_entries_buf, + &self.spec_array_addresses, + index, + )?; + let addresses = addresses.as_slice(); + let (mz_start, mz_run) = array_run(addresses, MZ_ARRAY)?; + array_run(addresses, ACC_INT)?; + let (y_start, y_run) = array_run(addresses, y_kind)?; + let same_split = mz_run + .iter() + .map(|address| address.element_count) + .eq(y_run.iter().map(|address| address.element_count)); + same_split.then(|| y_start as i32 - mz_start as i32) + } + + fn filtered_window( + &mut self, + index: usize, + x_kind: u32, + y_kind: u32, + range: Range, + ) -> IonResult { + let groups = match read_array_addresses_from_buffers( + &self.spec_entries_buf, + &self.spec_array_addresses, + index, + ) { + Some(addresses) => group_arrays(addresses.as_slice())?, + None => Vec::new(), + }; + let x_group = groups.iter().find(|group| group.array_type == x_kind); + let y_group = groups.iter().find(|group| group.array_type == y_kind); + let (Some(x_group), Some(y_group)) = (x_group, y_group) else { + return Ok(DataXY { + x: empty_array(FILE_DTYPE_F64), + y: empty_array(FILE_DTYPE_F64), + }); + }; + let x = self.read_spectrum_group(x_group)?; + let y = self.read_spectrum_group(y_group)?; + let mut x_out = empty_array(x_group.dtype); + let mut y_out = empty_array(y_group.dtype); + for position in 0..x.len().min(y.len()) { + let value = value_at(&x, position); + if value >= range.from && value <= range.to { + append_range(&mut x_out, &x, position, position + 1); + append_range(&mut y_out, &y, position, position + 1); + } + } + Ok(DataXY { x: x_out, y: y_out }) + } + + fn read_spectrum_group(&mut self, group: &ArrayGroup) -> IonResult { + let mut values = empty_array(group.dtype); + for address in &group.refs { + let segment = self.read_spectrum_array(address)?; + append_range(&mut values, &segment, 0, segment.len()); + } + Ok(values) + } + pub fn scans_in( &mut self, query: &ScanQuery, @@ -485,7 +528,8 @@ impl IonReader { if !has_mz || !has_intensity { continue; } - let data = self.read_window(index, query.mz)?; + let data = + self.spectrum_window(index, ArrayKind::Mz, ArrayKind::Intensity, query.mz)?; mz_out.clear(); data.x.extend_f64(&mut mz_out); intensity_out.clear(); @@ -501,81 +545,6 @@ impl IonReader { Ok(()) } - pub(crate) fn read_spectrum_window_inner( - &mut self, - index: usize, - x_array_accession: u32, - y_array_accession: u32, - low: f64, - high: f64, - ) -> IonResult { - if index >= self.header.spectrum_count as usize { - return Err("spectrum index out of range".into()); - } - if low > high { - return Ok(DataXY::empty()); - } - if x_array_accession == ACC_MZ && y_array_accession == ACC_INT { - return self.read_window( - index, - Range { - from: low, - to: high, - }, - ); - } - - let Some(array_addresses) = read_array_addresses_from_buffers( - &self.spec_entries_buf, - &self.spec_array_addresses, - index, - ) else { - return Ok(DataXY::empty()); - }; - let groups = group_arrays(array_addresses.as_slice())?; - - let mut x_group = None; - let mut y_group = None; - for group in &groups { - if group.array_type == x_array_accession && x_group.is_none() { - x_group = Some(group); - } else if group.array_type == y_array_accession && y_group.is_none() { - y_group = Some(group); - } - } - - let (Some(x_group), Some(y_group)) = (x_group, y_group) else { - return Ok(DataXY::empty()); - }; - - self.read_full_window(x_group, y_group, low, high) - } - - pub(crate) fn read_full_window( - &mut self, - x_group: &ArrayGroup, - y_group: &ArrayGroup, - low: f64, - high: f64, - ) -> IonResult { - let x = self.read_group_typed(x_group)?; - let y = self.read_group_typed(y_group)?; - - let mut x_out = empty_array(group_dtype(x_group)); - let mut y_out = empty_array(group_dtype(y_group)); - keep_pairs(&x, &y, low, high, &mut x_out, &mut y_out); - Ok(DataXY { x: x_out, y: y_out }) - } - - fn read_group_typed(&mut self, group: &ArrayGroup) -> IonResult { - let mut out = empty_array(group_dtype(group)); - for array_address in &group.refs { - let window = self.read_spectrum_array(array_address)?; - append_range(&mut out, &window, 0, window.len()); - } - Ok(out) - } - #[cfg(test)] pub(crate) fn candidate_items( &mut self, diff --git a/crates/parser/src/ion/decoder/utilities/common.rs b/crates/parser/src/ion/decoder/utilities/common.rs index afdaa4c2..701df879 100644 --- a/crates/parser/src/ion/decoder/utilities/common.rs +++ b/crates/parser/src/ion/decoder/utilities/common.rs @@ -2,7 +2,7 @@ use cosmoz::{DecompressOptions, Decoder, compressed_size}; use crate::ion::{ IonError, IonResult, - attr_meta::{AccessionTail, CV_CODE_UNKNOWN, cv_ref_code_from_str}, + attr_meta::{AccessionTail, CV_CODE_UNKNOWN, cv_ref_code_from_str, parse_accession_tail}, decoder::{ decode::{Metadatum, MetadatumValue}, utilities::decompression_limit::DecompressionLimit, @@ -60,8 +60,8 @@ pub(crate) fn read_u32_vec(bytes: &[u8], pos: &mut usize, n: usize) -> IonResult .ok_or_else(|| IonError::from("u32 vector length overflows usize"))?; let raw = take(bytes, pos, byte_len, "u32 vector")?; let mut out = Vec::with_capacity(n); - for chunk in raw.chunks_exact(4) { - out.push(u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])); + for chunk in raw.as_chunks::<4>().0 { + out.push(u32::from_le_bytes(*chunk)); } Ok(out) } @@ -73,8 +73,8 @@ pub(crate) fn read_f64_vec(bytes: &[u8], pos: &mut usize, n: usize) -> IonResult .ok_or_else(|| IonError::from("f64 vector length overflows usize"))?; let raw = take(bytes, pos, byte_len, "f64 vector")?; let mut out = Vec::with_capacity(n); - for chunk in raw.chunks_exact(8) { - out.push(f64::from_le_bytes(chunk.try_into().unwrap())); + for chunk in raw.as_chunks::<8>().0 { + out.push(f64::from_le_bytes(*chunk)); } Ok(out) } @@ -209,27 +209,6 @@ pub(crate) fn sum_string_lengths(string_lengths: &[u32]) -> IonResult { Ok(total) } -#[inline] -pub(crate) fn parse_accession_tail(accession: Option<&str>) -> AccessionTail { - let s = accession.unwrap_or(""); - let tail = s.rsplit_once(':').map(|(_, t)| t).unwrap_or(s); - let mut v: u32 = 0; - let mut saw = false; - for b in tail.bytes() { - if b.is_ascii_digit() { - saw = true; - v = match v - .checked_mul(10) - .and_then(|x| x.checked_add((b - b'0') as u32)) - { - Some(n) => n, - None => return AccessionTail::from_raw(0), - }; - } - } - AccessionTail::from_raw(if saw { v } else { 0 }) -} - #[cfg(test)] mod tests { use cosmoz::{CompressOptions, Encoder}; diff --git a/crates/parser/src/ion/decoder/utilities/meta_group_reader.rs b/crates/parser/src/ion/decoder/utilities/meta_group_reader.rs index 22818f6f..301ca121 100644 --- a/crates/parser/src/ion/decoder/utilities/meta_group_reader.rs +++ b/crates/parser/src/ion/decoder/utilities/meta_group_reader.rs @@ -5,10 +5,11 @@ use std::{ }; use crate::ion::{ - IonError, IonResult, + ByteRange, IonError, IonResult, decoder::{ - decode::{Metadatum, MetadatumValue}, + decode::{Metadatum, MetadatumValue, ReadOptions}, utilities::{ + byte_source::ReadBytes, common::decompress_zstd_allow_aligned_padding, decompression_limit::DecompressionLimit, meta_column_layout::MetaColumnLayout, @@ -16,18 +17,17 @@ use crate::ion::{ }, }, meta_groups::{ - META_GROUP_ENTRY_SIZE, META_GROUP_HEADER_SIZE, MetaGroupEntry, MetaTotals, group_count_for, - group_of_item, item_range_of_group, read_group_header, + META_GROUP_ENTRY_SIZE, META_GROUP_HEADER_SIZE, MetaGroupEntry, MetaSection, + group_count_for, group_of_item, item_range_of_group, meta_directory_range, + read_group_header, }, }; pub(crate) struct MetaGroupReader { - section: Arc<[u8]>, + source: Arc, + section: MetaSection, directory: Vec, - group_size: u32, - item_count: u64, - totals: MetaTotals, - payload_end: usize, + payload_end: u64, compression_codec: u8, verify_checksums: bool, budget: DecompressionLimit, @@ -63,90 +63,98 @@ struct GroupCache { } impl MetaGroupReader { - #[allow(clippy::too_many_arguments)] - pub(crate) fn new( - section: Arc<[u8]>, - group_count: u64, - group_size: u32, - item_count: u64, - totals: MetaTotals, + pub(crate) fn open( + source: Arc, + section: MetaSection, compression_codec: u8, - verify_checksums: bool, - budget: DecompressionLimit, - max_cached_bytes: usize, + options: &ReadOptions, layout: MetaColumnLayout, ) -> IonResult { - if item_count > 0 && group_size == 0 { + if section.item_count > 0 && section.group_size == 0 { return Err(IonError::from("metadata groups: group size is zero")); } - if group_count != group_count_for(item_count, group_size) { + if section.group_count != group_count_for(section.item_count, section.group_size) { return Err(IonError::from( "metadata groups: group count does not match item count", )); } - let directory = read_directory(§ion, group_count)?; - let payload_end = section.len() - directory.len() * META_GROUP_ENTRY_SIZE; + let directory_range = meta_directory_range(section.range, section.group_count)?; + let directory: Vec = source + .read(directory_range)? + .as_chunks::() + .0 + .iter() + .map(|entry| MetaGroupEntry::read_from(entry)) + .collect(); let mut uncompressed_sum: u64 = 0; for entry in &directory { uncompressed_sum = uncompressed_sum .checked_add(entry.uncompressed_size) .ok_or_else(|| IonError::from("metadata groups: uncompressed total overflows"))?; } - if uncompressed_sum != totals.uncompressed { + if uncompressed_sum != section.totals.uncompressed { return Err(IonError::from( "metadata groups: uncompressed total does not match header", )); } + let payload_end = directory_range.offset - section.range.offset; Ok(Self { + source, section, directory, - group_size, - item_count, - totals, payload_end, compression_codec, - verify_checksums, - budget, + verify_checksums: options.verify_checksums, + budget: options.decompression_limit, cache: GroupCache { groups: HashMap::new(), order: VecDeque::new(), used_bytes: 0, - max_bytes: max_cached_bytes, + max_bytes: options.max_cached_bytes, }, layout, }) } + #[cfg(test)] pub(crate) fn read_all(&self) -> IonResult> { - let mut rows = Vec::new(); - let mut seen = MetaCounts::default(); - for group_index in 0..self.directory.len() as u64 { - let bytes = self.decode_group(group_index)?; - let (meta_count, numeric_count, string_count) = read_group_header(&bytes)?; - seen.add(meta_count, numeric_count, string_count); - rows.extend(self.parse_group(&bytes, group_index)?); - } - self.check_totals(&seen)?; - Ok(rows) + Ok(self.read_all_grouped()?.into_iter().flatten().collect()) } pub(crate) fn read_all_grouped(&self) -> IonResult>> { + let payloads = self.source.read(ByteRange { + offset: self.section.range.offset, + length: self.payload_end, + })?; let mut groups = Vec::with_capacity(self.directory.len()); let mut seen = MetaCounts::default(); - for group_index in 0..self.directory.len() as u64 { - let bytes = self.decode_group(group_index)?; + for (group_index, entry) in self.directory.iter().enumerate() { + let range = self.payload_range(entry)?; + let start = (range.offset - self.section.range.offset) as usize; + let end = start + range.length as usize; + let payload = payloads + .get(start..end) + .ok_or_else(|| IonError::from("metadata groups: payload out of bounds"))?; + let bytes = self.decode_payload(entry, payload)?; let (meta_count, numeric_count, string_count) = read_group_header(&bytes)?; seen.add(meta_count, numeric_count, string_count); - groups.push(self.parse_group(&bytes, group_index)?); + groups.push(self.parse_group(&bytes, group_index as u64)?); } self.check_totals(&seen)?; Ok(groups) } + pub(crate) fn read_first_group(&self) -> IonResult> { + if self.directory.is_empty() { + return Ok(Vec::new()); + } + self.read_group(0) + } + fn check_totals(&self, seen: &MetaCounts) -> IonResult<()> { - if seen.rows != self.totals.rows - || seen.numeric != self.totals.numeric - || seen.string != self.totals.string + if seen.rows != self.section.totals.rows + || seen.numeric != self.section.totals.numeric + || seen.string != self.section.totals.string { return Err(IonError::from( "metadata groups: row totals do not match header", @@ -156,10 +164,10 @@ impl MetaGroupReader { } pub(crate) fn read_item(&mut self, item_index: u64) -> IonResult> { - if item_index >= self.item_count { + if item_index >= self.section.item_count { return Ok(Vec::new()); } - let group_index = group_of_item(item_index, self.group_size); + let group_index = group_of_item(item_index, self.section.group_size); let rows = self.get_cached_rows(group_index)?; let start = rows.partition_point(|row| (row.item_index as u64) < item_index); let end = rows.partition_point(|row| (row.item_index as u64) <= item_index); @@ -170,11 +178,7 @@ impl MetaGroupReader { if let Some(group) = self.cache.groups.get(&group_index) { return Ok(group.rows.clone()); } - let rows = { - let bytes = self.decode_group(group_index)?; - let result: Arc<[Metadatum]> = Arc::from(self.parse_group(&bytes, group_index)?); - result - }; + let rows: Arc<[Metadatum]> = Arc::from(self.read_group(group_index)?); let footprint = group_footprint(&rows); self.store_in_cache(group_index, rows.clone(), footprint); Ok(rows) @@ -196,12 +200,37 @@ impl MetaGroupReader { } } - fn decode_group(&self, group_index: u64) -> IonResult> { + fn read_group(&self, group_index: u64) -> IonResult> { let entry = self .directory .get(group_index as usize) .ok_or_else(|| IonError::from("metadata groups: group index out of range"))?; - let payload = self.get_payload(entry)?; + let payload = self.source.read(self.payload_range(entry)?)?; + let bytes = self.decode_payload(entry, &payload)?; + self.parse_group(&bytes, group_index) + } + + fn payload_range(&self, entry: &MetaGroupEntry) -> IonResult { + let end = entry + .payload_offset + .checked_add(entry.payload_size) + .ok_or_else(|| IonError::from("metadata groups: payload overflows"))?; + if end > self.payload_end { + return Err(IonError::from( + "metadata groups: payload overlaps the directory", + )); + } + Ok(ByteRange { + offset: self.section.range.offset + entry.payload_offset, + length: entry.payload_size, + }) + } + + fn decode_payload<'a>( + &self, + entry: &MetaGroupEntry, + payload: &'a [u8], + ) -> IonResult> { if self.verify_checksums && crc32fast::hash(payload) != entry.checksum { return Err(IonError::from("metadata groups: group checksum mismatch")); } @@ -223,7 +252,7 @@ impl MetaGroupReader { fn parse_group(&self, bytes: &[u8], group_index: u64) -> IonResult> { let (meta_count, numeric_count, string_count) = read_group_header(bytes)?; let (item_start, item_end) = - item_range_of_group(group_index, self.group_size, self.item_count); + item_range_of_group(group_index, self.section.group_size, self.section.item_count); let mut rows = parse_metadata( &bytes[META_GROUP_HEADER_SIZE..], item_end - item_start, @@ -241,24 +270,6 @@ impl MetaGroupReader { } Ok(rows) } - - fn get_payload(&self, entry: &MetaGroupEntry) -> IonResult<&[u8]> { - let start = usize::try_from(entry.payload_offset) - .map_err(|_| IonError::from("metadata groups: payload offset out of range"))?; - let size = usize::try_from(entry.payload_size) - .map_err(|_| IonError::from("metadata groups: payload size out of range"))?; - let end = start - .checked_add(size) - .ok_or_else(|| IonError::from("metadata groups: payload overflows"))?; - if end > self.payload_end { - return Err(IonError::from( - "metadata groups: payload overlaps the directory", - )); - } - self.section - .get(start..end) - .ok_or_else(|| IonError::from("metadata groups: payload out of bounds")) - } } fn group_footprint(rows: &[Metadatum]) -> usize { @@ -278,39 +289,21 @@ fn group_footprint(rows: &[Metadatum]) -> usize { total } -fn read_directory(section: &[u8], group_count: u64) -> IonResult> { - if group_count == 0 { - return Ok(Vec::new()); - } - let group_count = usize::try_from(group_count) - .map_err(|_| IonError::from("metadata groups: group count out of range"))?; - let directory_size = group_count - .checked_mul(META_GROUP_ENTRY_SIZE) - .ok_or_else(|| IonError::from("metadata groups: directory overflows"))?; - if section.len() < directory_size { - return Err(IonError::from( - "metadata groups: section smaller than directory", - )); - } - let directory_start = section.len() - directory_size; - let mut directory = Vec::with_capacity(group_count); - for index in 0..group_count { - let at = directory_start + index * META_GROUP_ENTRY_SIZE; - directory.push(MetaGroupEntry::read_from( - §ion[at..at + META_GROUP_ENTRY_SIZE], - )); - } - Ok(directory) -} - #[cfg(test)] mod tests { use std::sync::Arc; use super::MetaGroupReader; use crate::ion::{ - DecompressionLimit, IonResult, decoder::utilities::meta_column_layout::MetaColumnLayout, - format::CODEC_NONE, meta_groups::MetaTotals, + ByteRange, DecompressionLimit, IonResult, + decoder::{ + decode::ReadOptions, + utilities::{ + byte_source::BytesSource, meta_column_layout::MetaColumnLayout, + }, + }, + format::CODEC_NONE, + meta_groups::{MetaSection, MetaTotals}, }; fn no_totals() -> MetaTotals { @@ -323,16 +316,26 @@ mod tests { } fn new_reader(group_count: u64, group_size: u32, item_count: u64) -> IonResult<()> { - MetaGroupReader::new( - Arc::from(&[][..]), + let section = MetaSection { + range: ByteRange { + offset: 0, + length: 0, + }, group_count, group_size, item_count, - no_totals(), + totals: no_totals(), + }; + MetaGroupReader::open( + Arc::new(BytesSource::new(Arc::from(&[][..]))), + section, CODEC_NONE, - false, - DecompressionLimit::default(), - 1024, + &ReadOptions { + max_cached_bytes: 1024, + verify_checksums: false, + parallel: true, + decompression_limit: DecompressionLimit::default(), + }, MetaColumnLayout::new(), ) .map(|_| ()) diff --git a/crates/parser/src/ion/decoder/utilities/mod.rs b/crates/parser/src/ion/decoder/utilities/mod.rs index dd576e18..683bca91 100644 --- a/crates/parser/src/ion/decoder/utilities/mod.rs +++ b/crates/parser/src/ion/decoder/utilities/mod.rs @@ -1,5 +1,5 @@ pub mod byte_source; -pub(crate) use crate::ion::header::{Header, check_section_layout, parse_header}; +pub(crate) use crate::ion::header::{Header, check_section_layout}; pub(crate) mod common; pub(crate) mod meta_column_layout; pub(crate) mod meta_group_reader; @@ -17,9 +17,13 @@ pub(crate) use parse_precursor_list::parse_precursor_list; pub(crate) mod parse_product_list; pub(crate) use parse_product_list::parse_product_list; pub(crate) mod parse_spectrum_list; -pub(crate) use parse_spectrum_list::{parse_spectrum, parse_spectrum_list}; +pub(crate) use parse_spectrum_list::{ + parse_spectrum, parse_spectrum_list, parse_spectrum_list_header, +}; pub(crate) mod parse_chromatogram_list; -pub(crate) use parse_chromatogram_list::{parse_chromatogram, parse_chromatogram_list}; +pub(crate) use parse_chromatogram_list::{ + parse_chromatogram, parse_chromatogram_list, parse_chromatogram_list_header, +}; pub(crate) mod assign_attributes; #[cfg(test)] pub(crate) use assign_attributes::assign_attributes; diff --git a/crates/parser/src/ion/decoder/utilities/parse_chromatogram_list.rs b/crates/parser/src/ion/decoder/utilities/parse_chromatogram_list.rs index 73d53b10..a1f84030 100644 --- a/crates/parser/src/ion/decoder/utilities/parse_chromatogram_list.rs +++ b/crates/parser/src/ion/decoder/utilities/parse_chromatogram_list.rs @@ -35,24 +35,11 @@ pub(crate) fn parse_chromatogram_list( owner_rows.insert(entry.id, entry); } - let list_id = children_lookup - .all_ids(TagId::ChromatogramList) - .first() - .copied()?; - - let chromatogram_ids = children_lookup.ids_for(list_id, TagId::Chromatogram); - if chromatogram_ids.is_empty() { - return None; - } - - let list_rows = owner_rows.get(list_id); - let count = get_attr_u32(list_rows, ACC_ATTR_COUNT).map(|v| v as usize); - let default_data_processing_ref = - get_attr_text(list_rows, ACC_ATTR_DEFAULT_DATA_PROCESSING_REF); + let (mut list, chromatogram_ids) = parse_chromatogram_list_header(&owner_rows, children_lookup)?; let mut param_buffer: Vec<&Metadatum> = Vec::new(); - let chromatograms = chromatogram_ids + list.chromatograms = chromatogram_ids .iter() .enumerate() .map(|(index, &chromatogram_id)| { @@ -65,13 +52,37 @@ pub(crate) fn parse_chromatogram_list( &mut param_buffer, ) }) - .collect::>(); + .collect(); - Some(ChromatogramList { - count: count.or(Some(chromatograms.len())), - default_data_processing_ref, - chromatograms, - }) + Some(list) +} + +pub(crate) fn parse_chromatogram_list_header<'l>( + owner_rows: &OwnerRows, + children_lookup: &'l ChildrenLookup, +) -> Option<(ChromatogramList, &'l [u32])> { + let list_id = children_lookup + .all_ids(TagId::ChromatogramList) + .first() + .copied()?; + + let chromatogram_ids = children_lookup.ids_for(list_id, TagId::Chromatogram); + if chromatogram_ids.is_empty() { + return None; + } + + let list_rows = owner_rows.get(list_id); + let list = ChromatogramList { + count: get_attr_u32(list_rows, ACC_ATTR_COUNT) + .map(|v| v as usize) + .or(Some(chromatogram_ids.len())), + default_data_processing_ref: get_attr_text( + list_rows, + ACC_ATTR_DEFAULT_DATA_PROCESSING_REF, + ), + chromatograms: Vec::new(), + }; + Some((list, chromatogram_ids)) } #[inline] diff --git a/crates/parser/src/ion/decoder/utilities/parse_spectrum_list.rs b/crates/parser/src/ion/decoder/utilities/parse_spectrum_list.rs index ce39b1c6..83173f63 100644 --- a/crates/parser/src/ion/decoder/utilities/parse_spectrum_list.rs +++ b/crates/parser/src/ion/decoder/utilities/parse_spectrum_list.rs @@ -35,24 +35,11 @@ pub(crate) fn parse_spectrum_list( owner_rows.insert(entry.id, entry); } - let list_id = children_lookup - .all_ids(TagId::SpectrumList) - .first() - .copied()?; - - let spectrum_ids = children_lookup.ids_for(list_id, TagId::Spectrum); - if spectrum_ids.is_empty() { - return None; - } - - let list_rows = owner_rows.get(list_id); - let count = get_attr_u32(list_rows, ACC_ATTR_COUNT).map(|v| v as usize); - let default_data_processing_ref = - get_attr_text(list_rows, ACC_ATTR_DEFAULT_DATA_PROCESSING_REF); + let (mut list, spectrum_ids) = parse_spectrum_list_header(&owner_rows, children_lookup)?; let mut param_buffer: Vec<&Metadatum> = Vec::new(); - let spectra: Vec = spectrum_ids + list.spectra = spectrum_ids .iter() .enumerate() .map(|(index, &spectrum_id)| { @@ -67,11 +54,35 @@ pub(crate) fn parse_spectrum_list( }) .collect(); - Some(SpectrumList { - count: count.or(Some(spectra.len())), - default_data_processing_ref, - spectra, - }) + Some(list) +} + +pub(crate) fn parse_spectrum_list_header<'l>( + owner_rows: &OwnerRows, + children_lookup: &'l ChildrenLookup, +) -> Option<(SpectrumList, &'l [u32])> { + let list_id = children_lookup + .all_ids(TagId::SpectrumList) + .first() + .copied()?; + + let spectrum_ids = children_lookup.ids_for(list_id, TagId::Spectrum); + if spectrum_ids.is_empty() { + return None; + } + + let list_rows = owner_rows.get(list_id); + let list = SpectrumList { + count: get_attr_u32(list_rows, ACC_ATTR_COUNT) + .map(|v| v as usize) + .or(Some(spectrum_ids.len())), + default_data_processing_ref: get_attr_text( + list_rows, + ACC_ATTR_DEFAULT_DATA_PROCESSING_REF, + ), + spectra: Vec::new(), + }; + Some((list, spectrum_ids)) } #[inline] diff --git a/crates/parser/src/ion/decoder/utilities/spectrum_source.rs b/crates/parser/src/ion/decoder/utilities/spectrum_source.rs index a5cb962d..e54900e3 100644 --- a/crates/parser/src/ion/decoder/utilities/spectrum_source.rs +++ b/crates/parser/src/ion/decoder/utilities/spectrum_source.rs @@ -1,7 +1,7 @@ use crate::{ accessions as acc_const, ion::attr_meta::parse_accession_tail, - mzml::structs::{CvParam, MzML, NumericArray, Spectrum}, + mzml::structs::{CvParam, Spectrum}, }; #[inline] @@ -54,82 +54,6 @@ pub struct ScanSummary { pub position_z: u32, } -#[cfg(test)] -pub trait ScanSource { - fn for_each_summary(&mut self, callback: &mut dyn FnMut(usize, ScanSummary)); - fn load_scan(&mut self, index: usize, mz: &mut Vec, intensity: &mut Vec) -> bool; - - fn for_each_in_range(&mut self, rt_min: f64, rt_max: f64, ms_level: u8, mut callback: F) - where - Self: Sized, - F: FnMut(&ScanSummary, &[f64], &[f64]), - { - let mut matching: Vec<(usize, ScanSummary)> = Vec::new(); - self.for_each_summary(&mut |index, summary| { - if summary.rt >= rt_min - && summary.rt <= rt_max - && (ms_level == 0 || summary.ms_level == ms_level) - { - matching.push((index, summary)); - } - }); - let mut mz = Vec::new(); - let mut intensity = Vec::new(); - for (index, summary) in &matching { - if self.load_scan(*index, &mut mz, &mut intensity) { - callback(summary, &mz, &intensity); - } - } - } -} - -#[cfg(test)] -impl ScanSource for MzML { - fn for_each_summary(&mut self, callback: &mut dyn FnMut(usize, ScanSummary)) { - let Some(list) = self.run.spectrum_list.as_ref() else { - return; - }; - summary_from_spectra(&list.spectra, callback); - } - - fn load_scan(&mut self, index: usize, mz: &mut Vec, intensity: &mut Vec) -> bool { - let spectra = self - .run - .spectrum_list - .as_ref() - .map(|l| l.spectra.as_slice()) - .unwrap_or_default(); - load_scan_from_spectra(spectra, index, mz, intensity) - } - - fn for_each_in_range(&mut self, rt_min: f64, rt_max: f64, ms_level: u8, mut callback: F) - where - Self: Sized, - F: FnMut(&ScanSummary, &[f64], &[f64]), - { - let spectra = self - .run - .spectrum_list - .as_ref() - .map(|l| l.spectra.as_slice()) - .unwrap_or_default(); - let mut mz = Vec::new(); - let mut intensity = Vec::new(); - for (index, spectrum) in spectra.iter().enumerate() { - let summary = summary_from_spectrum(spectrum); - if summary.rt < rt_min - || summary.rt > rt_max - || (ms_level != 0 && summary.ms_level != ms_level) - { - continue; - } - if load_scan_from_spectra(spectra, index, &mut mz, &mut intensity) { - callback(&summary, &mz, &intensity); - } - } - } -} - pub(crate) fn summary_from_spectrum(spectrum: &Spectrum) -> ScanSummary { let mut rt = f64::NAN; let mut rt_unit = TimeUnit::Other; @@ -229,16 +153,6 @@ pub(crate) fn summary_from_spectrum(spectrum: &Spectrum) -> ScanSummary { } } -#[cfg(test)] -pub(crate) fn summary_from_spectra( - spectra: &[Spectrum], - callback: &mut dyn FnMut(usize, ScanSummary), -) { - for (index, spectrum) in spectra.iter().enumerate() { - callback(index, summary_from_spectrum(spectrum)); - } -} - #[inline] fn rt_from_params(params: &[CvParam]) -> Option<(f64, TimeUnit)> { for param in params { @@ -278,75 +192,6 @@ fn parse_u32(s: Option<&str>) -> u32 { s.and_then(|v| v.parse().ok()).unwrap_or(0) } -#[cfg(test)] -pub(crate) fn binary_pair(spectrum: &Spectrum) -> Option<(&NumericArray, &NumericArray)> { - let list = spectrum.binary_data_array_list.as_ref()?; - let mut mz = None; - let mut intensity = None; - for array in &list.binary_data_arrays { - if mz.is_some() && intensity.is_some() { - break; - } - let mut is_mz = false; - let mut is_intensity = false; - for param in &array.cv_params { - match acc(param.accession.as_deref()) { - acc_const::MZ_ARRAY => is_mz = true, - acc_const::INTENSITY_ARRAY => is_intensity = true, - _ => {} - } - if is_mz && is_intensity { - break; - } - } - if is_mz { - mz = array.binary.as_ref(); - } - if is_intensity { - intensity = array.binary.as_ref(); - } - } - Some((mz?, intensity?)) -} - -#[cfg(test)] -pub(crate) fn load_scan_from_spectra( - spectra: &[Spectrum], - index: usize, - mz: &mut Vec, - intensity: &mut Vec, -) -> bool { - let Some(spectrum) = spectra.get(index) else { - return false; - }; - let Some((mz_data, int_data)) = binary_pair(spectrum) else { - return false; - }; - let len = mz_data.len().min(int_data.len()); - if len == 0 { - return false; - } - mz.clear(); - mz.reserve(len); - extend_from_binary(mz_data, mz, len); - intensity.clear(); - intensity.reserve(len); - extend_from_binary(int_data, intensity, len); - true -} - -#[cfg(test)] -fn extend_from_binary(data: &NumericArray, out: &mut Vec, max: usize) { - match data { - NumericArray::F64(v) => out.extend_from_slice(&v[..max]), - NumericArray::F32(v) => out.extend(v[..max].iter().map(|&x| x as f64)), - NumericArray::I16(v) => out.extend(v[..max].iter().map(|&x| x as f64)), - NumericArray::I32(v) => out.extend(v[..max].iter().map(|&x| x as f64)), - NumericArray::I64(v) => out.extend(v[..max].iter().map(|&x| x as f64)), - NumericArray::F16(v) => out.extend(v[..max].iter().copied().map(f16_bits_to_f64)), - } -} - pub(crate) fn f16_bits_to_f64(bits: u16) -> f64 { let sign = ((bits & 0x8000) as u32) << 16; let exp = (bits & 0x7C00) >> 10; diff --git a/crates/parser/src/ion/decoder/utilities/tests/parse_header.rs b/crates/parser/src/ion/decoder/utilities/tests/header_parse.rs similarity index 94% rename from crates/parser/src/ion/decoder/utilities/tests/parse_header.rs rename to crates/parser/src/ion/decoder/utilities/tests/header_parse.rs index 7cf3476c..0673efc3 100644 --- a/crates/parser/src/ion/decoder/utilities/tests/parse_header.rs +++ b/crates/parser/src/ion/decoder/utilities/tests/header_parse.rs @@ -1,6 +1,6 @@ use std::{fs, path::PathBuf}; -use crate::ion::{format::FILE_SIGNATURE, utilities::parse_header}; +use crate::ion::utilities::Header; const PATH: &str = "data/ion/test.ion"; @@ -12,10 +12,8 @@ fn read_bytes(path: &str) -> Vec { #[test] fn check_header() { let bytes = read_bytes(PATH); - let header = parse_header(&bytes).expect("parse_header failed"); + let header = Header::parse(&bytes).expect("Header::parse failed"); - assert_eq!(header.file_signature, FILE_SIGNATURE); - assert_eq!(header.endianness_flag, 0); assert_eq!(header.spectrum_count, 2); assert_eq!(header.chrom_count, 2); assert_eq!(header.spec_meta_count, 71); diff --git a/crates/parser/src/ion/decoder/utilities/tests/meta.rs b/crates/parser/src/ion/decoder/utilities/tests/meta.rs index 7f14e618..30d8986c 100644 --- a/crates/parser/src/ion/decoder/utilities/tests/meta.rs +++ b/crates/parser/src/ion/decoder/utilities/tests/meta.rs @@ -1,67 +1,20 @@ -use std::{fs, path::PathBuf, sync::Arc}; - -use crate::ion::{ - DecompressionLimit, - decoder::decode::Metadatum, - meta_groups::MetaTotals, - utilities::{MetaGroupReader, meta_column_layout::MetaColumnLayout, parse_header}, -}; +use crate::ion::decoder::decode::{IonReader, Metadatum, ReadOptions}; fn read_file(path: &str) -> Vec { - let full = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(path); - fs::read(&full).unwrap_or_else(|error| panic!("cannot read {full:?}: {error}")) + let full = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(path); + std::fs::read(&full).unwrap_or_else(|error| panic!("cannot read {full:?}: {error}")) } pub(super) fn spectra_metadata(path: &str) -> Vec { - let bytes = read_file(path); - let header = parse_header(&bytes).expect("parse_header failed"); - let start = header.off_spec_meta as usize; - let end = start + header.len_spec_meta as usize; - MetaGroupReader::new( - Arc::from(&bytes[start..end]), - header.spec_meta_group_count, - header.meta_group_size, - header.spectrum_count, - MetaTotals { - rows: header.spec_meta_count, - numeric: header.spec_meta_numeric_count, - string: header.spec_meta_string_count, - uncompressed: header.spec_meta_uncompressed_bytes, - }, - header.compression_codec, - true, - DecompressionLimit::default(), - 64 * 1024 * 1024, - MetaColumnLayout::new(), - ) - .expect("build spectra metadata reader") - .read_all() - .expect("read spectra metadata") + IonReader::from_bytes(&read_file(path), &ReadOptions::default()) + .expect("open") + .spectrum_metadata() + .expect("read spectra metadata") } pub(super) fn chromatograms_metadata(path: &str) -> Vec { - let bytes = read_file(path); - let header = parse_header(&bytes).expect("parse_header failed"); - let start = header.off_chrom_meta as usize; - let end = start + header.len_chrom_meta as usize; - MetaGroupReader::new( - Arc::from(&bytes[start..end]), - header.chrom_meta_group_count, - header.meta_group_size, - header.chrom_count, - MetaTotals { - rows: header.chrom_meta_count, - numeric: header.chrom_meta_numeric_count, - string: header.chrom_meta_string_count, - uncompressed: header.chrom_meta_uncompressed_bytes, - }, - header.compression_codec, - true, - DecompressionLimit::default(), - 64 * 1024 * 1024, - MetaColumnLayout::new(), - ) - .expect("build chromatograms metadata reader") - .read_all() - .expect("read chromatograms metadata") + IonReader::from_bytes(&read_file(path), &ReadOptions::default()) + .expect("open") + .chromatogram_metadata() + .expect("read chromatograms metadata") } diff --git a/crates/parser/src/ion/decoder/utilities/tests/mod.rs b/crates/parser/src/ion/decoder/utilities/tests/mod.rs index 7f0440d3..e88f0ecf 100644 --- a/crates/parser/src/ion/decoder/utilities/tests/mod.rs +++ b/crates/parser/src/ion/decoder/utilities/tests/mod.rs @@ -1,6 +1,6 @@ mod meta; mod parse_binary_array_list; -mod parse_header; +mod header_parse; mod parse_scan_list; mod parse_precursor_list; diff --git a/crates/parser/src/ion/decoder/utilities/tests/parse_file_description.rs b/crates/parser/src/ion/decoder/utilities/tests/parse_file_description.rs index 8d9fd4c0..ef495f91 100644 --- a/crates/parser/src/ion/decoder/utilities/tests/parse_file_description.rs +++ b/crates/parser/src/ion/decoder/utilities/tests/parse_file_description.rs @@ -6,11 +6,11 @@ use crate::ion::{ DecompressionLimit, decoder::decode::Metadatum, utilities::{ + Header, children_lookup::{ChildrenLookup, DefaultMetadataPolicy}, meta_column_layout::MetaColumnLayout, parse_file_description::parse_file_description, parse_global_metadata::parse_global_metadata, - parse_header, }, }; @@ -23,7 +23,7 @@ fn read_bytes(path: &str) -> Vec { fn parse_global_metadata_from_test_file() -> Vec { let bytes = read_bytes(PATH); - let header = parse_header(&bytes).expect("parse_header failed"); + let header = Header::parse(&bytes).expect("Header::parse failed"); let start = header.off_global_meta as usize; let len = header.len_global_meta as usize; diff --git a/crates/parser/src/ion/decoder/utilities/tests/parse_metadata.rs b/crates/parser/src/ion/decoder/utilities/tests/parse_metadata.rs index da8263d4..f99bd9ab 100644 --- a/crates/parser/src/ion/decoder/utilities/tests/parse_metadata.rs +++ b/crates/parser/src/ion/decoder/utilities/tests/parse_metadata.rs @@ -4,7 +4,7 @@ use crate::ion::{ DecompressionLimit, attr_meta::*, decoder::decode::{Metadatum, MetadatumValue}, - utilities::{parse_header, parse_metadata}, + utilities::{Header, parse_metadata}, }; use crate::mzml::schema::TagId; @@ -73,7 +73,7 @@ fn parse_metadata_section_from_test_file( #[test] fn check_first_spectrum() { let header_bytes = read_bytes(PATH); - let header = parse_header(&header_bytes).expect("parse_header failed"); + let header = Header::parse(&header_bytes).expect("Header::parse failed"); let spec_meta = parse_metadata_section_from_test_file( header.off_spec_meta, header.off_chrom_meta, @@ -278,7 +278,7 @@ fn check_first_spectrum() { #[test] fn check_second_spectrum() { let header_bytes = read_bytes(PATH); - let header = parse_header(&header_bytes).expect("parse_header failed"); + let header = Header::parse(&header_bytes).expect("Header::parse failed"); let spec_meta = parse_metadata_section_from_test_file( header.off_spec_meta, header.off_chrom_meta, @@ -614,7 +614,7 @@ fn assert_chromatogram_binary_data_array_list(meta: &[Metadatum], item_index: u3 #[test] fn check_first_chromatogram() { let header_bytes = read_bytes(PATH); - let header = parse_header(&header_bytes).expect("parse_header failed"); + let header = Header::parse(&header_bytes).expect("Header::parse failed"); let chrom_meta = parse_metadata_section_from_test_file( header.off_chrom_meta, header.off_global_meta, @@ -676,7 +676,7 @@ fn check_first_chromatogram() { #[test] fn check_second_chromatogram() { let header_bytes = read_bytes(PATH); - let header = parse_header(&header_bytes).expect("parse_header failed"); + let header = Header::parse(&header_bytes).expect("Header::parse failed"); let chrom_meta = parse_metadata_section_from_test_file( header.off_chrom_meta, header.off_global_meta, diff --git a/crates/parser/src/ion/encoder/encode.rs b/crates/parser/src/ion/encoder/encode.rs index 462f0b58..9d5a4400 100644 --- a/crates/parser/src/ion/encoder/encode.rs +++ b/crates/parser/src/ion/encoder/encode.rs @@ -207,13 +207,7 @@ impl WriteOptions { } fn block_shuffle_is_enabled(self) -> bool { - if !self.compression_is_enabled() { - return false; - } - !matches!( - std::env::var("IONIC_BLOCK_SHUFFLE").as_deref(), - Ok("off") | Ok("raw") | Ok("0") | Ok("none") - ) + self.compression_is_enabled() } pub(crate) fn array_filter_id(self) -> u8 { @@ -249,20 +243,6 @@ impl WriteOptions { force_f32: self.force_f32, } } - - pub fn fast() -> Self { - Self { - compression_level: 3, - ..Default::default() - } - } - - pub fn max_ratio() -> Self { - Self { - compression_level: 22, - ..Default::default() - } - } } #[derive(Copy, Clone)] @@ -446,7 +426,6 @@ fn write_array_data(buf: &mut Vec, data: ArrayData<'_>, dtype: u8) { (FILE_DTYPE_I16, ArrayData::I16(e)) => write_i16_slice_le(buf, e), (FILE_DTYPE_I32, ArrayData::I32(e)) => write_i32_slice_le(buf, e), (FILE_DTYPE_I64, ArrayData::I64(e)) => write_i64_slice_le(buf, e), - // SAFETY: `validate_array_dtype` is always called before this function. _ => unreachable!("write_array_data called with unvalidated dtype/data combination"), } } @@ -582,7 +561,7 @@ fn resolve_array_encoding<'a>( let elem_bytes = element_byte_size_for_dtype(dtype); let dtype_enum = Dtype::from_byte(dtype).unwrap_or(Dtype::F64); let requested: &'static dyn Packing = if config.compression_is_enabled() { - packing_for(accession, dtype_enum, data.element_count()) + packing_for(accession, dtype_enum) } else { &RAW_PACKING }; diff --git a/crates/parser/src/ion/encoder/ion_writer.rs b/crates/parser/src/ion/encoder/ion_writer.rs index f8033473..ac14a9a7 100644 --- a/crates/parser/src/ion/encoder/ion_writer.rs +++ b/crates/parser/src/ion/encoder/ion_writer.rs @@ -461,7 +461,7 @@ fn schema_of_chromatogram_list(list: &ChromatogramList) -> ChromatogramList { } } -pub(crate) struct IonWriter { +pub(crate) struct IonEncoder { config: WriteOptions, compressor: Encoder, collector: MetaCollector, @@ -474,26 +474,26 @@ pub(crate) struct IonWriter { last_spec_rt: f64, } -impl IonWriter { +impl IonEncoder { pub(crate) fn begin( output: &mut dyn WriteBytes, metadata: &MzML, - config: &WriteOptions, + options: &WriteOptions, ) -> IonResult { - let config = *config; - allow_compression_level(config.compression_level)?; + let options = *options; + allow_compression_level(options.compression_level)?; output.write(&[0u8; HEADER_SIZE])?; - let compressor = new_meta_encoder(config.compression_level)?; + let compressor = new_meta_encoder(options.compression_level)?; let mut writer = Self { - config, + config: options, compressor, collector: MetaCollector::new(), spec_list_id: LOCAL_LIST_NODE_ID, chrom_list_id: LOCAL_LIST_NODE_ID, - spec_stream: ItemStream::new(256, SPEC_SUMMARY_SIZE, 256, config, true)?, - chrom_stream: ItemStream::new(32, CHROM_SUMMARY_SIZE, 32, config, false)?, + spec_stream: ItemStream::new(256, SPEC_SUMMARY_SIZE, 256, options, true)?, + chrom_stream: ItemStream::new(32, CHROM_SUMMARY_SIZE, 32, options, false)?, spec_schema: None, chrom_schema: None, last_spec_rt: f64::NEG_INFINITY, @@ -795,10 +795,10 @@ impl IonWriter { pub(crate) fn write_mzml_to_ion( mzml: &MzML, - config: &WriteOptions, + options: &WriteOptions, output: &mut dyn WriteBytes, ) -> IonResult<()> { - let mut writer = IonWriter::begin(output, mzml, config)?; + let mut writer = IonEncoder::begin(output, mzml, options)?; writer.write_mzml(output, mzml)?; writer.finish(output, mzml) } @@ -863,9 +863,7 @@ mod tests { ..Default::default() }; let mut output = Vec::new(); - let mut writer = IonWriter::begin(&mut output, &MzML::default(), &config).unwrap(); - // Flush every sealed block immediately, so the pending queue can never hold - // more than the block currently being assembled. + let mut writer = IonEncoder::begin(&mut output, &MzML::default(), &config).unwrap(); writer.spec_stream.set_max_pending_bytes(1); let spectrum_count = 500; @@ -877,9 +875,6 @@ mod tests { max_pending_bytes_seen = max_pending_bytes_seen.max(writer.spec_stream.pending_bytes()); } - // The bytes held by not-yet-flushed blocks must stay within one block's - // worth of data, regardless of how many spectra have been pushed so far: - // push_spectrum must stream one block at a time, not buffer the whole file. assert!( max_pending_bytes_seen <= block_size, "pending bytes ({max_pending_bytes_seen}) exceeded one block ({block_size}) \ @@ -893,8 +888,8 @@ mod tests { .unwrap(); assert_eq!(reader.spectrum_count(), spectrum_count as u64); for i in [0usize, spectrum_count / 2, spectrum_count - 1] { - let spectrum = reader.spectrum(i).unwrap(); - let mz = reader.array(i, crate::ion::ArrayKind::Mz).unwrap(); + let spectrum = reader.spectrum_metadata_at(i).unwrap(); + let mz = reader.spectrum_array(i, crate::ion::ArrayKind::Mz).unwrap(); assert_eq!(spectrum.index, Some(i as u32)); assert_eq!(mz, vec![100.0 + i as f64, 101.0 + i as f64, 102.0 + i as f64]); } diff --git a/crates/parser/src/ion/encoder/utilities/block_writer.rs b/crates/parser/src/ion/encoder/utilities/block_writer.rs index e04620dc..78eb0690 100644 --- a/crates/parser/src/ion/encoder/utilities/block_writer.rs +++ b/crates/parser/src/ion/encoder/utilities/block_writer.rs @@ -62,7 +62,6 @@ impl DefaultCompressor { let options = CompressOptions { level, checksum: false, - ..Default::default() }; let encoder = Encoder::new(&options) .map_err(|err| IonError::from(format!("zstd start error: {err:?}")))?; @@ -674,7 +673,6 @@ impl BlockWriter { payload_size: block.bytes.len() as u64, uncompressed_len_bytes: block.raw_len, checksum: block.checksum, - ..Default::default() }, )?; self.payload_bytes += block.bytes.len() as u64; diff --git a/crates/parser/src/ion/encoder/utilities/meta_collector/mod.rs b/crates/parser/src/ion/encoder/utilities/meta_collector/mod.rs index 6cce3156..3636402c 100644 --- a/crates/parser/src/ion/encoder/utilities/meta_collector/mod.rs +++ b/crates/parser/src/ion/encoder/utilities/meta_collector/mod.rs @@ -629,7 +629,6 @@ pub(crate) fn new_meta_encoder(level: u8) -> IonResult { let options = CompressOptions { level: get_supported_compression_level(level.max(1)), checksum: false, - ..Default::default() }; Encoder::new(&options).map_err(|err| IonError::from(format!("zstd start error: {err:?}"))) } diff --git a/crates/parser/src/ion/encoder/utilities/meta_collector/tests.rs b/crates/parser/src/ion/encoder/utilities/meta_collector/tests.rs index c3bbb420..4dc5c3ad 100644 --- a/crates/parser/src/ion/encoder/utilities/meta_collector/tests.rs +++ b/crates/parser/src/ion/encoder/utilities/meta_collector/tests.rs @@ -183,35 +183,54 @@ fn collector_global_meta_on_empty_mzml_produces_run_buffer() { assert!(!meta.ids.is_empty()); } -#[test] -fn grouped_metadata_keeps_values_local_to_each_group() { +fn open_grouped( + bytes: &[u8], + group_count: u64, + totals: crate::ion::meta_groups::MetaTotals, +) -> crate::ion::IonResult { use crate::ion::{ - DecompressionLimit, - decoder::decode::MetadatumValue, + ByteRange, + decoder::{decode::ReadOptions, utilities::byte_source::BytesSource}, format::CODEC_NONE, - meta_groups::MetaTotals, + meta_groups::MetaSection, utilities::{MetaGroupReader, meta_column_layout::MetaColumnLayout}, }; + let section = MetaSection { + range: ByteRange { + offset: 0, + length: bytes.len() as u64, + }, + group_count, + group_size: 1, + item_count: 3, + totals, + }; + MetaGroupReader::open( + Arc::new(BytesSource::new(Arc::from(bytes))), + section, + CODEC_NONE, + &ReadOptions::default(), + MetaColumnLayout::new(), + ) +} + +#[test] +fn grouped_metadata_keeps_values_local_to_each_group() { + use crate::ion::{decoder::decode::MetadatumValue, meta_groups::MetaTotals}; + let grouped = three_item_grouped(); assert_eq!(grouped.group_count, 3); - let mut reader = MetaGroupReader::new( - Arc::from(grouped_bytes(&grouped)), + let mut reader = open_grouped( + grouped_bytes(&grouped), grouped.group_count, - 1, - 3, MetaTotals { rows: 3, numeric: 3, string: 0, uncompressed: grouped.uncompressed_size, }, - CODEC_NONE, - true, - DecompressionLimit::default(), - 64 * 1024 * 1024, - MetaColumnLayout::new(), ) .unwrap(); @@ -260,60 +279,36 @@ fn grouped_bytes(grouped: &grouper::GroupedSection) -> &[u8] { #[test] fn metadata_reader_rejects_wrong_uncompressed_total() { - use crate::ion::{ - DecompressionLimit, - format::CODEC_NONE, - meta_groups::MetaTotals, - utilities::{MetaGroupReader, meta_column_layout::MetaColumnLayout}, - }; + use crate::ion::meta_groups::MetaTotals; let grouped = three_item_grouped(); - let result = MetaGroupReader::new( - Arc::from(grouped_bytes(&grouped)), + let result = open_grouped( + grouped_bytes(&grouped), grouped.group_count, - 1, - 3, MetaTotals { rows: 3, numeric: 3, string: 0, uncompressed: grouped.uncompressed_size + 1, }, - CODEC_NONE, - true, - DecompressionLimit::default(), - 64 * 1024 * 1024, - MetaColumnLayout::new(), ); assert!(result.is_err()); } #[test] fn metadata_reader_rejects_wrong_row_total() { - use crate::ion::{ - DecompressionLimit, - format::CODEC_NONE, - meta_groups::MetaTotals, - utilities::{MetaGroupReader, meta_column_layout::MetaColumnLayout}, - }; + use crate::ion::meta_groups::MetaTotals; let grouped = three_item_grouped(); - let reader = MetaGroupReader::new( - Arc::from(grouped_bytes(&grouped)), + let reader = open_grouped( + grouped_bytes(&grouped), grouped.group_count, - 1, - 3, MetaTotals { rows: 99, numeric: 3, string: 0, uncompressed: grouped.uncompressed_size, }, - CODEC_NONE, - true, - DecompressionLimit::default(), - 64 * 1024 * 1024, - MetaColumnLayout::new(), ) .unwrap(); assert!(reader.read_all().is_err()); @@ -321,12 +316,7 @@ fn metadata_reader_rejects_wrong_row_total() { #[test] fn metadata_reader_rejects_payload_into_directory() { - use crate::ion::{ - DecompressionLimit, - format::CODEC_NONE, - meta_groups::{META_GROUP_ENTRY_SIZE, MetaTotals}, - utilities::{MetaGroupReader, meta_column_layout::MetaColumnLayout}, - }; + use crate::ion::meta_groups::{META_GROUP_ENTRY_SIZE, MetaTotals}; let grouped = three_item_grouped(); let mut bytes = grouped_bytes(&grouped).to_vec(); @@ -334,22 +324,15 @@ fn metadata_reader_rejects_payload_into_directory() { bytes[directory_start..directory_start + 8] .copy_from_slice(&(directory_start as u64).to_le_bytes()); - let mut reader = MetaGroupReader::new( - Arc::from(bytes.as_slice()), + let mut reader = open_grouped( + &bytes, grouped.group_count, - 1, - 3, MetaTotals { rows: 3, numeric: 3, string: 0, uncompressed: grouped.uncompressed_size, }, - CODEC_NONE, - true, - DecompressionLimit::default(), - 64 * 1024 * 1024, - MetaColumnLayout::new(), ) .unwrap(); assert!(reader.read_item(0).is_err()); @@ -471,10 +454,10 @@ fn group_local_node_ids_across_group_boundaries() { IonReader::from_bytes(&output, &ReadOptions::default()).expect("failed to open decoder"); let first_rows = decoder - .spectrum_metadata_at(0) + .spectrum_metadata_rows_at(0) .expect("failed to read first spectrum metadata"); let second_rows = decoder - .spectrum_metadata_at(8192) + .spectrum_metadata_rows_at(8192) .expect("failed to read second spectrum metadata"); let first_list = find_row(&first_rows, TagId::SpectrumList, LOCAL_LIST_NODE_ID); @@ -590,7 +573,7 @@ fn product_own_cv_params_parent_to_product_list_not_to_the_product_itself() { let mut decoder = IonReader::from_bytes(&output, &ReadOptions::default()).expect("failed to open decoder"); let rows = decoder - .spectrum_metadata_at(0) + .spectrum_metadata_rows_at(0) .expect("failed to read spectrum metadata"); let product_row = rows diff --git a/crates/parser/src/ion/encoder/utilities/output.rs b/crates/parser/src/ion/encoder/utilities/output.rs index ed26894e..8bdf5166 100644 --- a/crates/parser/src/ion/encoder/utilities/output.rs +++ b/crates/parser/src/ion/encoder/utilities/output.rs @@ -26,11 +26,6 @@ pub struct FileWriter { #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] impl FileWriter { - #[allow(dead_code)] - pub fn open(path: &str) -> IonResult { - Self::open_path(Path::new(path)) - } - pub fn open_path(path: &Path) -> IonResult { let file = File::create(path).map_err(|err| { IonError::from(format!( diff --git a/crates/parser/src/ion/header.rs b/crates/parser/src/ion/header.rs index b6840829..20c64316 100644 --- a/crates/parser/src/ion/header.rs +++ b/crates/parser/src/ion/header.rs @@ -1,10 +1,11 @@ use crate::ion::{ - IonResult, + ByteRange, IonResult, decoder::decode::INDEX_ENTRY_BYTES, encoder::encode::{CHROM_SUMMARY_SIZE, SPEC_SUMMARY_SIZE}, format::{ CODEC_ZSTD, CURRENT_VERSION, FILE_SIGNATURE, HEADER_SIZE, allow_compression, allow_version, }, + meta_groups::{MetaSection, MetaTotals}, }; const BLOCK_DIRECTORY_ENTRY_SIZE_U64: u64 = 32; @@ -124,12 +125,8 @@ const _: () = assert!(HEADER_CHROM_META_CRC32 == 1012); const _: () = assert!(HEADER_GLOBAL_META_CRC32 == 1016); const _: () = assert!(HEADER_CRC32 == 1020); -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub(crate) struct Header { - #[allow(dead_code)] - pub(crate) file_signature: [u8; 8], - #[allow(dead_code)] - pub(crate) endianness_flag: u8, pub(crate) format_version: u16, pub(crate) compression_codec: u8, pub(crate) compression_level: u8, @@ -202,85 +199,6 @@ pub(crate) struct Header { pub(crate) header_crc32: u32, } -impl Default for Header { - fn default() -> Self { - Self { - file_signature: [0u8; 8], - endianness_flag: 0, - format_version: 0, - compression_codec: 0, - compression_level: 0, - default_array_filter: 0, - target_block_uncompressed_bytes: 0, - off_spec_summary: 0, - len_spec_summary: 0, - off_chrom_summary: 0, - len_chrom_summary: 0, - off_spec_entries: 0, - len_spec_entries: 0, - off_spec_array_addresses: 0, - len_spec_array_addresses: 0, - off_chrom_entries: 0, - len_chrom_entries: 0, - off_chrom_array_addresses: 0, - len_chrom_array_addresses: 0, - off_spec_meta: 0, - len_spec_meta: 0, - off_chrom_meta: 0, - len_chrom_meta: 0, - off_global_meta: 0, - len_global_meta: 0, - off_spec_container: 0, - len_spec_container: 0, - off_chrom_container: 0, - len_chrom_container: 0, - spec_block_count: 0, - chrom_block_count: 0, - spectrum_count: 0, - chrom_count: 0, - spec_meta_count: 0, - spec_meta_numeric_count: 0, - spec_meta_string_count: 0, - chrom_meta_count: 0, - chrom_meta_numeric_count: 0, - chrom_meta_string_count: 0, - global_meta_count: 0, - global_meta_numeric_count: 0, - global_meta_string_count: 0, - spec_array_type_count: 0, - chrom_array_type_count: 0, - spec_meta_uncompressed_bytes: 0, - chrom_meta_uncompressed_bytes: 0, - global_meta_uncompressed_bytes: 0, - total_file_size: 0, - meta_group_size: 0, - spec_meta_group_count: 0, - chrom_meta_group_count: 0, - spec_summary_crc32: 0, - spec_entries_crc32: 0, - spec_array_addresses_crc32: 0, - chrom_summary_crc32: 0, - chrom_entries_crc32: 0, - chrom_array_addresses_crc32: 0, - target_mz_window: 0, - spec_directory_crc32: 0, - chrom_directory_crc32: 0, - off_spec_window_directory: 0, - len_spec_window_directory: 0, - off_chrom_window_directory: 0, - len_chrom_window_directory: 0, - plain_len_spec_window_directory: 0, - plain_len_chrom_window_directory: 0, - spec_window_directory_crc32: 0, - chrom_window_directory_crc32: 0, - spec_meta_crc32: 0, - chrom_meta_crc32: 0, - global_meta_crc32: 0, - header_crc32: 0, - } - } -} - impl Header { pub(crate) fn parse(bytes: &[u8]) -> IonResult { if bytes.len() < HEADER_SIZE { @@ -289,12 +207,10 @@ impl Header { let h = &bytes[..HEADER_SIZE]; - let file_signature = <[u8; 8]>::try_from(&h[0..8]).unwrap(); - if file_signature != FILE_SIGNATURE { + if h[0..8] != FILE_SIGNATURE { return Err("header: invalid file signature".into()); } - let endianness_flag = h[8]; - if endianness_flag != 0 { + if h[8] != 0 { return Err("header: expected little-endian endianness_flag=0".into()); } @@ -401,8 +317,6 @@ impl Header { let header_crc32 = read_u32_at(h, HEADER_CRC32); let header = Header { - file_signature, - endianness_flag, format_version, compression_codec, compression_level, @@ -487,6 +401,42 @@ impl Header { Ok(header) } + pub(crate) fn spec_meta_section(&self) -> MetaSection { + MetaSection { + range: ByteRange { + offset: self.off_spec_meta, + length: self.len_spec_meta, + }, + group_count: self.spec_meta_group_count, + group_size: self.meta_group_size, + item_count: self.spectrum_count, + totals: MetaTotals { + rows: self.spec_meta_count, + numeric: self.spec_meta_numeric_count, + string: self.spec_meta_string_count, + uncompressed: self.spec_meta_uncompressed_bytes, + }, + } + } + + pub(crate) fn chrom_meta_section(&self) -> MetaSection { + MetaSection { + range: ByteRange { + offset: self.off_chrom_meta, + length: self.len_chrom_meta, + }, + group_count: self.chrom_meta_group_count, + group_size: self.meta_group_size, + item_count: self.chrom_count, + totals: MetaTotals { + rows: self.chrom_meta_count, + numeric: self.chrom_meta_numeric_count, + string: self.chrom_meta_string_count, + uncompressed: self.chrom_meta_uncompressed_bytes, + }, + } + } + pub(crate) fn write(&self, buf: &mut [u8]) { buf[0..FILE_SIGNATURE.len()].copy_from_slice(&FILE_SIGNATURE); buf[8] = 0u8; @@ -696,10 +646,6 @@ impl Header { } } -pub(crate) fn parse_header(bytes: &[u8]) -> IonResult
{ - Header::parse(bytes) -} - #[inline] pub fn version_of(bytes: &[u8]) -> Option { let end = HEADER_FORMAT_VERSION_OFFSET + 2; @@ -741,18 +687,6 @@ pub fn set_version(header: &mut [u8]) -> IonResult { Ok(true) } -#[inline] -#[allow(dead_code)] -pub(crate) fn get_total_file_size_from_header(bytes: &[u8]) -> Option { - let end = HEADER_TOTAL_FILE_SIZE + 8; - if bytes.len() < end { - return None; - } - let mut buf = [0u8; 8]; - buf.copy_from_slice(&bytes[HEADER_TOTAL_FILE_SIZE..end]); - Some(u64::from_le_bytes(buf)) -} - pub(crate) fn check_section_layout(h: &Header) -> Vec { let mut failures = Vec::new(); @@ -1027,8 +961,6 @@ mod tests { fn header_bytes_with(build: impl FnOnce(&mut Header)) -> [u8; HEADER_SIZE] { let mut header = Header { - file_signature: FILE_SIGNATURE, - endianness_flag: 0, format_version: CURRENT_VERSION, ..Header::default() }; @@ -1168,8 +1100,6 @@ mod tests { #[test] fn write_then_parse_round_trips_every_field() { let mut original = Header { - file_signature: FILE_SIGNATURE, - endianness_flag: 0, format_version: CURRENT_VERSION, compression_codec: 1, compression_level: 3, @@ -1250,8 +1180,6 @@ mod tests { let parsed = Header::parse(&buf).expect("round-trip parse failed"); - assert_eq!(parsed.file_signature, original.file_signature); - assert_eq!(parsed.endianness_flag, original.endianness_flag); assert_eq!(parsed.format_version, original.format_version); assert_eq!(parsed.compression_codec, original.compression_codec); assert_eq!(parsed.compression_level, original.compression_level); diff --git a/crates/parser/src/ion/meta_groups.rs b/crates/parser/src/ion/meta_groups.rs index c8b468e2..073ffd8e 100644 --- a/crates/parser/src/ion/meta_groups.rs +++ b/crates/parser/src/ion/meta_groups.rs @@ -1,4 +1,4 @@ -use crate::ion::{IonError, IonResult}; +use crate::ion::{ByteRange, IonError, IonResult}; pub(crate) const METADATA_GROUP_SIZE: u32 = 8192; pub(crate) const META_GROUP_ENTRY_SIZE: usize = 32; @@ -37,6 +37,28 @@ pub(crate) struct MetaTotals { pub(crate) uncompressed: u64, } +pub(crate) struct MetaSection { + pub(crate) range: ByteRange, + pub(crate) group_count: u64, + pub(crate) group_size: u32, + pub(crate) item_count: u64, + pub(crate) totals: MetaTotals, +} + +pub(crate) fn meta_directory_range(section: ByteRange, group_count: u64) -> IonResult { + let length = group_count + .checked_mul(META_GROUP_ENTRY_SIZE as u64) + .ok_or_else(|| IonError::from("metadata groups: directory size overflows"))?; + let offset = section + .length + .checked_sub(length) + .ok_or_else(|| IonError::from("metadata groups: section smaller than directory"))?; + Ok(ByteRange { + offset: section.offset + offset, + length, + }) +} + pub(crate) fn group_count_for(item_count: u64, group_size: u32) -> u64 { let group_size = group_size as u64; if group_size == 0 || item_count == 0 { diff --git a/crates/parser/src/ion/mod.rs b/crates/parser/src/ion/mod.rs index b3f36257..f5132333 100644 --- a/crates/parser/src/ion/mod.rs +++ b/crates/parser/src/ion/mod.rs @@ -1,4 +1,3 @@ -#![allow(unused_imports)] pub(crate) mod array_kind; pub(crate) mod byte_transpose; pub(crate) mod filter_summary; @@ -14,26 +13,16 @@ pub(crate) mod decoder; pub(crate) mod format; pub(crate) mod scan; pub use scan::{ScanSummary, TimeUnit}; -#[cfg(test)] -pub(crate) use scan::ScanSource; pub(crate) mod version_generated; pub(crate) use decoder::utilities; -#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] -pub(crate) use decoder::utilities::byte_source::FileSource; -pub use decoder::decode::{DataXY, IonReader, ReadOptions, ScanQuery, Select, Window, header_ranges, merge_ranges}; +pub use decoder::decode::{DataXY, Select, Window, header_ranges, merge_ranges}; pub use decoder::utilities::{ byte_source::{CallbackSource, ReadBytes}, - decompression_limit::{DEFAULT_MAX_UNCOMPRESSED_SIZE, DecompressionLimit}, + decompression_limit::DecompressionLimit, }; -pub(crate) use decoder::utilities::byte_source::{BytesSource, SourceBytes}; pub use header::{HEADER_FORMAT_VERSION_OFFSET, set_version, version_of}; -pub(crate) use header::get_total_file_size_from_header; pub(crate) mod encoder; -#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] -pub(crate) use encoder::utilities::FileWriter; -pub use encoder::encode::{DEFAULT_MZ_WINDOW, WriteOptions}; -pub(crate) use encoder::ion_writer::IonWriter; -pub(crate) use encoder::scan_stream::MemoryReader; +pub use encoder::encode::DEFAULT_MZ_WINDOW; pub use encoder::scan_stream::ScanStream; pub use encoder::utilities::{SectionStorage, WriteBytes}; pub use format::{ diff --git a/crates/parser/src/ion/packing/byte_shuffle.rs b/crates/parser/src/ion/packing/byte_shuffle.rs index a98a1d03..04770d87 100644 --- a/crates/parser/src/ion/packing/byte_shuffle.rs +++ b/crates/parser/src/ion/packing/byte_shuffle.rs @@ -22,7 +22,7 @@ impl Packing for ByteShuffle { fn decode(&self, input: &[u8], dtype: Dtype, out: &mut Vec) -> IonResult<()> { let stride = dtype.byte_stride(); - if input.len() % stride != 0 { + if !input.len().is_multiple_of(stride) { return Err(IonError::from( "byte shuffle: input length is not a multiple of the stride", )); @@ -37,10 +37,6 @@ fn typed_to_le_bytes_and_stride(input: PackingInput<'_>) -> (Vec, usize) { match input { PackingInput::F64(s) => (s.iter().flat_map(|v| v.to_le_bytes()).collect(), 8), PackingInput::F32(s) => (s.iter().flat_map(|v| v.to_le_bytes()).collect(), 4), - PackingInput::I64(s) => (s.iter().flat_map(|v| v.to_le_bytes()).collect(), 8), - PackingInput::I32(s) => (s.iter().flat_map(|v| v.to_le_bytes()).collect(), 4), - PackingInput::I16(s) => (s.iter().flat_map(|v| v.to_le_bytes()).collect(), 2), - PackingInput::Bytes(b) => (b.to_vec(), 1), } } @@ -55,8 +51,10 @@ mod tests { .unwrap(); let mut dec = Vec::new(); BYTE_SHUFFLE.decode(&enc, Dtype::F64, &mut dec).unwrap(); - dec.chunks_exact(8) - .map(|c| f64::from_le_bytes(c.try_into().unwrap())) + dec.as_chunks::<8>() + .0 + .iter() + .map(|c| f64::from_le_bytes(*c)) .collect() } diff --git a/crates/parser/src/ion/packing/delta_shuffle.rs b/crates/parser/src/ion/packing/delta_shuffle.rs index a04c7442..74960448 100644 --- a/crates/parser/src/ion/packing/delta_shuffle.rs +++ b/crates/parser/src/ion/packing/delta_shuffle.rs @@ -12,16 +12,24 @@ fn encode_delta(bits: impl Iterator, out: &mut Vec) } } -fn decode_delta(input: &[u8], mut emit: impl FnMut(W)) { - let mut prev = W::default(); - for chunk in input.chunks_exact(W::BYTES) { - prev = DeltaWord::wrapping_add(prev, W::from_le_chunk(chunk)); +fn decode_delta_u64(input: &[u8], mut emit: impl FnMut(u64)) { + let mut prev = 0u64; + for chunk in input.as_chunks::<8>().0 { + prev = prev.wrapping_add(u64::from_le_bytes(*chunk)); + emit(prev); + } +} + +fn decode_delta_u32(input: &[u8], mut emit: impl FnMut(u32)) { + let mut prev = 0u32; + for chunk in input.as_chunks::<4>().0 { + prev = prev.wrapping_add(u32::from_le_bytes(*chunk)); emit(prev); } } fn require_aligned_input(input: &[u8]) -> IonResult<()> { - if input.len() % W::BYTES == 0 { + if input.len().is_multiple_of(W::BYTES) { Ok(()) } else { Err(IonError::from( @@ -51,7 +59,6 @@ impl Packing for DeltaShuffle { encode_delta::(v.iter().map(|x| x.to_bits()), out); Ok(()) } - _ => Err(IonError::from("delta filter needs f32 or f64 input")), } } @@ -60,7 +67,7 @@ impl Packing for DeltaShuffle { Dtype::F64 => { require_aligned_input::(input)?; out.reserve(input.len()); - decode_delta::(input, |w| { + decode_delta_u64(input, |w| { out.extend_from_slice(&f64::from_bits(w).to_le_bytes()) }); Ok(()) @@ -68,7 +75,7 @@ impl Packing for DeltaShuffle { Dtype::F32 => { require_aligned_input::(input)?; out.reserve(input.len()); - decode_delta::(input, |w| { + decode_delta_u32(input, |w| { out.extend_from_slice(&f32::from_bits(w).to_le_bytes()) }); Ok(()) @@ -92,8 +99,10 @@ mod tests { .unwrap(); let mut dec = Vec::new(); DELTA_SHUFFLE.decode(&enc, Dtype::F64, &mut dec).unwrap(); - dec.chunks_exact(8) - .map(|c| f64::from_le_bytes(c.try_into().unwrap())) + dec.as_chunks::<8>() + .0 + .iter() + .map(|c| f64::from_le_bytes(*c)) .collect() } @@ -105,8 +114,10 @@ mod tests { assert_eq!(enc.len(), input.len() * 4); let mut dec = Vec::new(); DELTA_SHUFFLE.decode(&enc, Dtype::F32, &mut dec).unwrap(); - dec.chunks_exact(4) - .map(|c| f32::from_le_bytes(c.try_into().unwrap())) + dec.as_chunks::<4>() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)) .collect() } @@ -217,24 +228,24 @@ mod tests { use super::super::packing_for; use crate::accessions::{INTENSITY_ARRAY, MZ_ARRAY}; assert_eq!( - packing_for(MZ_ARRAY, Dtype::F64, 100).id(), + packing_for(MZ_ARRAY, Dtype::F64).id(), PackingId::DeltaShuffle ); assert_eq!( - packing_for(MZ_ARRAY, Dtype::F32, 100).id(), + packing_for(MZ_ARRAY, Dtype::F32).id(), PackingId::DeltaShuffle, "f32 m/z must also use delta-shuffle" ); assert_eq!( - packing_for(INTENSITY_ARRAY, Dtype::F32, 100).id(), + packing_for(INTENSITY_ARRAY, Dtype::F32).id(), PackingId::Raw, "intensity must stay raw" ); assert_eq!( - packing_for(INTENSITY_ARRAY, Dtype::F64, 100).id(), + packing_for(INTENSITY_ARRAY, Dtype::F64).id(), PackingId::Raw, "intensity is never delta-shuffled, even at f64" ); - assert_eq!(packing_for(0, Dtype::I32, 1).id(), PackingId::Raw); + assert_eq!(packing_for(0, Dtype::I32).id(), PackingId::Raw); } } diff --git a/crates/parser/src/ion/packing/mod.rs b/crates/parser/src/ion/packing/mod.rs index 0373e78a..4ad4dbae 100644 --- a/crates/parser/src/ion/packing/mod.rs +++ b/crates/parser/src/ion/packing/mod.rs @@ -1,12 +1,9 @@ use crate::ion::{IonError, IonResult}; -#[allow(dead_code)] pub(crate) trait DeltaWord: Copy + Default { const BYTES: usize; fn wrapping_sub(self, rhs: Self) -> Self; - fn wrapping_add(self, rhs: Self) -> Self; fn to_le_bytes_into(self, out: &mut Vec); - fn from_le_chunk(chunk: &[u8]) -> Self; } impl DeltaWord for u32 { @@ -14,15 +11,9 @@ impl DeltaWord for u32 { fn wrapping_sub(self, rhs: Self) -> Self { self.wrapping_sub(rhs) } - fn wrapping_add(self, rhs: Self) -> Self { - self.wrapping_add(rhs) - } fn to_le_bytes_into(self, out: &mut Vec) { out.extend_from_slice(&self.to_le_bytes()); } - fn from_le_chunk(chunk: &[u8]) -> Self { - u32::from_le_bytes(chunk.try_into().unwrap()) - } } impl DeltaWord for u64 { @@ -30,15 +21,9 @@ impl DeltaWord for u64 { fn wrapping_sub(self, rhs: Self) -> Self { self.wrapping_sub(rhs) } - fn wrapping_add(self, rhs: Self) -> Self { - self.wrapping_add(rhs) - } fn to_le_bytes_into(self, out: &mut Vec) { out.extend_from_slice(&self.to_le_bytes()); } - fn from_le_chunk(chunk: &[u8]) -> Self { - u64::from_le_bytes(chunk.try_into().unwrap()) - } } pub(crate) mod byte_shuffle; @@ -72,7 +57,6 @@ impl Dtype { } } - #[allow(dead_code)] pub(crate) fn byte_stride(self) -> usize { match self { Self::F64 | Self::I64 => 8, @@ -101,17 +85,11 @@ impl PackingId { } } -#[allow(dead_code)] pub(crate) enum PackingInput<'a> { F32(&'a [f32]), F64(&'a [f64]), - I16(&'a [i16]), - I32(&'a [i32]), - I64(&'a [i64]), - Bytes(&'a [u8]), } -#[allow(dead_code)] pub(crate) trait Packing: Send + Sync { fn id(&self) -> PackingId; @@ -126,35 +104,11 @@ pub(crate) trait Packing: Send + Sync { fn decode(&self, input: &[u8], dtype: Dtype, out: &mut Vec) -> IonResult<()>; } -pub(crate) fn packing_for( - array_type: u32, - dtype: Dtype, - _element_count: usize, -) -> &'static dyn Packing { +pub(crate) fn packing_for(array_type: u32, dtype: Dtype) -> &'static dyn Packing { use crate::accessions::{ - INTENSITY_ARRAY, ION_MOBILITY_ARRAY, MEAN_ION_MOBILITY_ARRAY, MZ_ARRAY, - RAW_ION_MOBILITY_ARRAY, RAW_ION_MOBILITY_DRIFT_TIME_ARRAY, TIME_ARRAY, + ION_MOBILITY_ARRAY, MEAN_ION_MOBILITY_ARRAY, MZ_ARRAY, RAW_ION_MOBILITY_ARRAY, + RAW_ION_MOBILITY_DRIFT_TIME_ARRAY, TIME_ARRAY, }; - let env_key = match array_type { - MZ_ARRAY => Some("IONIC_MZ_CODEC"), - INTENSITY_ARRAY => Some("IONIC_INTENSITY_CODEC"), - TIME_ARRAY => Some("IONIC_RT_CODEC"), - ION_MOBILITY_ARRAY - | MEAN_ION_MOBILITY_ARRAY - | RAW_ION_MOBILITY_ARRAY - | RAW_ION_MOBILITY_DRIFT_TIME_ARRAY => Some("IONIC_ION_MOBILITY_CODEC"), - _ => None, - }; - - if let Some(key) = env_key - && let Ok(v) = std::env::var(key) - { - return match v.as_str() { - "raw" => &raw::RAW, - "byte_shuffle" => &byte_shuffle::BYTE_SHUFFLE, - _ => &delta_shuffle::DELTA_SHUFFLE, - }; - } let is_delta_axis = matches!( array_type, MZ_ARRAY @@ -219,11 +173,11 @@ mod tests { #[test] fn packing_for_dtype_dispatch() { assert_eq!( - packing_for(MZ_ARRAY, Dtype::F64, 100).id(), + packing_for(MZ_ARRAY, Dtype::F64).id(), PackingId::DeltaShuffle ); - assert_eq!(packing_for(0, Dtype::F32, 1).id(), PackingId::Raw); - assert_eq!(packing_for(0, Dtype::I32, 1).id(), PackingId::Raw); + assert_eq!(packing_for(0, Dtype::F32).id(), PackingId::Raw); + assert_eq!(packing_for(0, Dtype::I32).id(), PackingId::Raw); } #[test] diff --git a/crates/parser/src/ion/packing/raw.rs b/crates/parser/src/ion/packing/raw.rs index 3ae63b30..ce03dce5 100644 --- a/crates/parser/src/ion/packing/raw.rs +++ b/crates/parser/src/ion/packing/raw.rs @@ -13,8 +13,13 @@ impl Packing for Raw { } fn encode(&self, input: PackingInput<'_>, out: &mut Vec) -> IonResult<()> { - if let PackingInput::Bytes(b) = input { - out.extend_from_slice(b); + match input { + PackingInput::F64(values) => { + out.extend(values.iter().flat_map(|v| v.to_le_bytes())) + } + PackingInput::F32(values) => { + out.extend(values.iter().flat_map(|v| v.to_le_bytes())) + } } Ok(()) } @@ -30,19 +35,25 @@ mod tests { use super::*; #[test] - fn encode_copies_bytes() { + fn encode_writes_f64_le_bytes() { let mut out = Vec::new(); - RAW.encode(PackingInput::Bytes(&[1, 2, 3, 4]), &mut out) + RAW.encode(PackingInput::F64(&[1.0, 2.0]), &mut out) .unwrap(); - assert_eq!(out, [1, 2, 3, 4]); + let mut expected = Vec::new(); + expected.extend_from_slice(&1.0f64.to_le_bytes()); + expected.extend_from_slice(&2.0f64.to_le_bytes()); + assert_eq!(out, expected); } #[test] - fn encode_non_bytes_input_is_noop() { + fn encode_writes_f32_le_bytes() { let mut out = Vec::new(); - RAW.encode(PackingInput::F64(&[1.0, 2.0]), &mut out) + RAW.encode(PackingInput::F32(&[1.0, 2.0]), &mut out) .unwrap(); - assert!(out.is_empty()); + let mut expected = Vec::new(); + expected.extend_from_slice(&1.0f32.to_le_bytes()); + expected.extend_from_slice(&2.0f32.to_le_bytes()); + assert_eq!(out, expected); } #[test] @@ -60,12 +71,18 @@ mod tests { } #[test] - fn bytes_roundtrip() { - let input = b"hello world 12345678"; + fn f64_roundtrip() { + let input = [1.5f64, -2.25, 3.0]; let mut enc = Vec::new(); - RAW.encode(PackingInput::Bytes(input), &mut enc).unwrap(); + RAW.encode(PackingInput::F64(&input), &mut enc).unwrap(); let mut dec = Vec::new(); RAW.decode(&enc, Dtype::F64, &mut dec).unwrap(); - assert_eq!(dec.as_slice(), input.as_ref()); + let got: Vec = dec + .as_chunks::<8>() + .0 + .iter() + .map(|c| f64::from_le_bytes(*c)) + .collect(); + assert_eq!(got, input); } } diff --git a/crates/parser/src/ion/scan.rs b/crates/parser/src/ion/scan.rs index 3d78bc4b..43e3d15b 100644 --- a/crates/parser/src/ion/scan.rs +++ b/crates/parser/src/ion/scan.rs @@ -1,3 +1 @@ -#[cfg(test)] -pub use crate::ion::decoder::utilities::spectrum_source::ScanSource; pub use crate::ion::decoder::utilities::spectrum_source::{ScanSummary, TimeUnit}; diff --git a/crates/parser/src/ion/version_generated.rs b/crates/parser/src/ion/version_generated.rs index 09c4f99e..6665dd78 100644 --- a/crates/parser/src/ion/version_generated.rs +++ b/crates/parser/src/ion/version_generated.rs @@ -1,4 +1,3 @@ -// generated by `make sync` from config.json — do not edit pub const CURRENT_VERSION: u16 = 1; pub const MIN_SUPPORTED_VERSION: u16 = 0; pub const MAX_SUPPORTED_VERSION: u16 = 1; diff --git a/crates/parser/src/mzml/bin_to_mzml.rs b/crates/parser/src/mzml/bin_to_mzml.rs index 71ce4336..4e6b8456 100644 --- a/crates/parser/src/mzml/bin_to_mzml.rs +++ b/crates/parser/src/mzml/bin_to_mzml.rs @@ -1476,43 +1476,6 @@ fn write_user_params( Ok(()) } -#[cfg(test)] -mod tests { - use super::*; - - fn render_cv_param(cv: &CvParam) -> String { - let mut writer = Writer::new(Vec::new()); - write_cv_param(&mut writer, cv).unwrap(); - String::from_utf8(writer.into_inner()).unwrap() - } - - #[test] - fn valueless_cvparam_omits_value_attribute_7() { - let valueless = CvParam { - cv_ref: Some("MS".to_string()), - accession: Some("MS:1000511".to_string()), - name: "ms level".to_string(), - value: None, - ..Default::default() - }; - let xml = render_cv_param(&valueless); - assert!( - !xml.contains("value="), - "a valueless cvParam must not emit a value attribute, got: {xml}" - ); - - let with_value = CvParam { - value: Some("x".to_string()), - ..valueless - }; - let xml = render_cv_param(&with_value); - assert!( - xml.contains(r#"value="x""#), - "a cvParam with a value must emit it verbatim, got: {xml}" - ); - } -} - fn write_cv_container( writer: &mut Writer>, tag_name: &str, @@ -1612,3 +1575,40 @@ fn write_file_checksum(writer: &mut Writer>) -> Result<(), BinToMzmlErro Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn render_cv_param(cv: &CvParam) -> String { + let mut writer = Writer::new(Vec::new()); + write_cv_param(&mut writer, cv).unwrap(); + String::from_utf8(writer.into_inner()).unwrap() + } + + #[test] + fn valueless_cvparam_omits_value_attribute_7() { + let valueless = CvParam { + cv_ref: Some("MS".to_string()), + accession: Some("MS:1000511".to_string()), + name: "ms level".to_string(), + value: None, + ..Default::default() + }; + let xml = render_cv_param(&valueless); + assert!( + !xml.contains("value="), + "a valueless cvParam must not emit a value attribute, got: {xml}" + ); + + let with_value = CvParam { + value: Some("x".to_string()), + ..valueless + }; + let xml = render_cv_param(&with_value); + assert!( + xml.contains(r#"value="x""#), + "a cvParam with a value must emit it verbatim, got: {xml}" + ); + } +} diff --git a/crates/parser/src/mzml/schema.rs b/crates/parser/src/mzml/schema.rs index 75de4928..88234aca 100644 --- a/crates/parser/src/mzml/schema.rs +++ b/crates/parser/src/mzml/schema.rs @@ -329,9 +329,7 @@ impl SchemaNode { if self.child_key_by_tag.len() != 256 { self.child_key_by_tag = default_child_key_by_tag(); } else { - for x in &mut self.child_key_by_tag { - *x = None; - } + self.child_key_by_tag.fill(None); } for (child_key, child) in &mut self.children { @@ -368,9 +366,7 @@ impl SchemaTree { if self.root_key_by_tag.len() != 256 { self.root_key_by_tag = default_key_by_tag(); } else { - for x in &mut self.root_key_by_tag { - *x = None; - } + self.root_key_by_tag.fill(None); } for (root_key, root_node) in self.roots.iter_mut() { diff --git a/crates/parser/src/mzml/structs.rs b/crates/parser/src/mzml/structs.rs index a9ab6a3b..0919362f 100644 --- a/crates/parser/src/mzml/structs.rs +++ b/crates/parser/src/mzml/structs.rs @@ -17,23 +17,6 @@ pub struct MzML { pub run: Run, } -impl MzML { - pub fn from_spectra(spectra: Vec) -> Self { - let count = spectra.len(); - Self { - run: Run { - spectrum_list: Some(SpectrumList { - count: Some(count), - spectra, - ..Default::default() - }), - ..Default::default() - }, - ..Default::default() - } - } -} - #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct CvList { pub count: Option, diff --git a/crates/parser/src/utilities/test.rs b/crates/parser/src/utilities/test.rs index 6e46b25b..21d85a50 100644 --- a/crates/parser/src/utilities/test.rs +++ b/crates/parser/src/utilities/test.rs @@ -2,7 +2,8 @@ use core::str::FromStr; use std::{fs, path::PathBuf, sync::OnceLock}; use crate::{ - ion::{ReadOptions, decoder::decode::IonReader}, + ReadOptions, + ion::decoder::decode::IonReader, mzml::{ parse_mzml::parse_mzml, structs::{ diff --git a/crates/parser/tests/api_surface.rs b/crates/parser/tests/api_surface.rs index 44dc2d10..09db4263 100644 --- a/crates/parser/tests/api_surface.rs +++ b/crates/parser/tests/api_surface.rs @@ -57,12 +57,14 @@ fn create_write_spectrum_finish_round_trips_and_open_reads_it_back() { let _ = std::fs::remove_file(&path); assert_eq!(reader.spectrum_count(), 1); - let got_mz = reader.array(0, ArrayKind::Mz).expect("array must succeed"); + let got_mz = reader + .spectrum_array(0, ArrayKind::Mz) + .expect("spectrum_array must succeed"); assert_eq!(got_mz, mz); } #[test] -fn array_into_reuses_the_callers_buffer() { +fn spectrum_array_into_reuses_the_callers_buffer() { let mz: Vec = (0..30).map(|i| 300.0 + i as f64).collect(); let intensity: Vec = (0..30).map(|i| i as f64).collect(); let spectrum = make_spectrum_f64("scan=1", mz.clone(), intensity); @@ -76,13 +78,13 @@ fn array_into_reuses_the_callers_buffer() { let mut reader = IonReader::from_bytes(&bytes, &ReadOptions::default()).unwrap(); let mut buf = Vec::with_capacity(4096); let addr_before = buf.as_ptr(); - reader.array_into(0, ArrayKind::Mz, &mut buf).unwrap(); + reader.spectrum_array_into(0, ArrayKind::Mz, &mut buf).unwrap(); assert_eq!(buf, mz); assert_eq!( buf.as_ptr(), addr_before, - "array_into must reuse the caller's allocation" + "spectrum_array_into must reuse the caller's allocation" ); } @@ -143,7 +145,7 @@ fn scans_in_and_metadata_are_reachable() { .unwrap(); assert_eq!(seen, 1); - let metadata: MzML = reader.metadata().unwrap(); + let metadata: MzML = reader.global_metadata().unwrap(); assert_eq!(metadata.run.id, "test-run"); let summaries: Vec = reader.spectrum_summaries().unwrap(); diff --git a/crates/parser/tests/decode_rejection.rs b/crates/parser/tests/decode_rejection.rs index 5683bcf5..1024ba6b 100644 --- a/crates/parser/tests/decode_rejection.rs +++ b/crates/parser/tests/decode_rejection.rs @@ -127,6 +127,24 @@ fn rejects_corrupted_global_meta() { ); } +#[test] +fn corrupt_spectrum_metadata_fails_on_read_not_on_open() { + let mut bytes = encode_to_ion(test_files::tiny_pwiz_11(), 0, false); + let off = read_header_u64(&bytes, 160) as usize; + let len = read_header_u64(&bytes, 168) as usize; + bytes[off + len / 4] ^= 0xFF; + let mut reader = IonReader::from_bytes(&bytes, &ReadOptions::default()) + .expect("open must not read spectrum metadata"); + let err = reader + .spectrum_metadata_at(0) + .map(|_| ()) + .expect_err("first group read must fail"); + assert!( + err.contains("group checksum mismatch"), + "unexpected decode error: {err}" + ); +} + #[test] fn rejects_corrupted_trailer() { let mut bytes = encode_to_ion(test_files::tiny_pwiz_11(), 9, false); diff --git a/crates/parser/tests/ion_stream_writer.rs b/crates/parser/tests/ion_stream_writer.rs index 5fc619b7..9d90e932 100644 --- a/crates/parser/tests/ion_stream_writer.rs +++ b/crates/parser/tests/ion_stream_writer.rs @@ -59,7 +59,7 @@ fn stream_writer_roundtrips_like_mzml_writer() { let mzml = parse_mzml(coordinate_xml()).unwrap(); let mut reader = MzmlReader::from_mzml(mzml.clone()); let mut bytes = Vec::new(); - let mut writer = IonWriter::to(&mut bytes, &MzML::default(), &config()).unwrap(); + let writer = IonWriter::to(&mut bytes, &MzML::default(), &config()).unwrap(); writer.write_stream(&mut reader).unwrap(); let decoded = decode_ion(&bytes).unwrap(); let diffs = canonical_diff_paths(&mzml, &decoded); @@ -71,7 +71,7 @@ fn spectrum_summary_keeps_coordinates() { let mzml = parse_mzml(coordinate_xml()).unwrap(); let mut reader = MzmlReader::from_mzml(mzml); let mut bytes = Vec::new(); - let mut writer = IonWriter::to(&mut bytes, &MzML::default(), &config()).unwrap(); + let writer = IonWriter::to(&mut bytes, &MzML::default(), &config()).unwrap(); writer.write_stream(&mut reader).unwrap(); let ion = IonReader::from_bytes(&bytes, &ReadOptions::default()).unwrap(); let summary = ion.spectrum_summary(0).unwrap(); @@ -89,7 +89,7 @@ fn file_stream_reader_roundtrips_spectra_then_chromatograms() { let mzml = parse_mzml(mixed_xml()).unwrap(); let mut reader = MzmlReader::open(&path).unwrap(); let mut bytes = Vec::new(); - let mut writer = IonWriter::to(&mut bytes, &MzML::default(), &config()).unwrap(); + let writer = IonWriter::to(&mut bytes, &MzML::default(), &config()).unwrap(); writer.write_stream(&mut reader).unwrap(); let decoded = decode_ion(&bytes).unwrap(); let diffs = canonical_diff_paths(&mzml, &decoded); diff --git a/crates/parser/tests/mz_order_rejection.rs b/crates/parser/tests/mz_order_rejection.rs index 158e4f78..b8ac74b6 100644 --- a/crates/parser/tests/mz_order_rejection.rs +++ b/crates/parser/tests/mz_order_rejection.rs @@ -1,7 +1,8 @@ mod common; use ionic::{ - IonReader, IonResult, IonWriter, Range, ReadOptions, ScanStream, SectionStorage, WriteOptions, + ArrayKind, IonReader, IonResult, IonWriter, Range, ReadOptions, ScanStream, SectionStorage, + WriteOptions, mzml::structs::{ BinaryDataArray, BinaryDataArrayList, Chromatogram, CvParam, MzML, NumericArray, Run, Spectrum, SpectrumList, @@ -246,8 +247,10 @@ fn read_mz_range_works_on_sorted_spectrum() { IonReader::from_bytes(&bytes, &ReadOptions::default()).expect("open must succeed"); decoder.require_bounds().expect("window bounds must exist"); let window = decoder - .read_window( + .spectrum_window( 0, + ArrayKind::Mz, + ArrayKind::Intensity, Range { from: 150.0, to: 250.0, diff --git a/crates/xtask/Cargo.toml b/crates/xtask/Cargo.toml index a7693619..cf31cd26 100644 --- a/crates/xtask/Cargo.toml +++ b/crates/xtask/Cargo.toml @@ -3,6 +3,7 @@ name = "xtask" version.workspace = true license.workspace = true edition.workspace = true +rust-version.workspace = true publish = false [dependencies] diff --git a/crates/xtask/src/main.rs b/crates/xtask/src/main.rs index 6838e503..4f82d918 100644 --- a/crates/xtask/src/main.rs +++ b/crates/xtask/src/main.rs @@ -165,8 +165,7 @@ fn cargo_path(root: &Path) -> PathBuf { fn render_generated(release: &Release) -> String { format!( - "// generated by `make sync` from config.json — do not edit\n\ - pub const CURRENT_VERSION: u16 = {};\n\ + "pub const CURRENT_VERSION: u16 = {};\n\ pub const MIN_SUPPORTED_VERSION: u16 = {};\n\ pub const MAX_SUPPORTED_VERSION: u16 = {};\n", release.format.current, release.format.min_supported, release.format.max_supported