Merge pull request #156 from ArcInstitute/integrate-cbq

Integrate cbq

Author: noamteyssier

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bqtools"
-version = "0.4.15"
+version = "0.5.0"
 edition = "2021"
 license = "MIT"
 authors = ["Noam Teyssier <noam.teyssier@arcinstitute.org>"]
@@ -12,20 +12,20 @@ keywords = ["bioinformatics", "binseq", "fastx", "file-format", "sequencing"]
 [dependencies]
 aho-corasick = "1.1.4"
 anyhow = "1.0.100"
-binseq = "0.8.3"
-clap = { version = "4.5.53", features = ["derive"] }
+binseq = "0.9.0"
+clap = { version = "4.5.54", features = ["derive"] }
 csv = "1.4.0"
 env_logger = "0.11.8"
 fixedbitset = "0.5.7"
 gzp = "2.0.2"
 is-terminal = "0.4.17"
-libc = "0.2.178"
+libc = "0.2.180"
 log = "0.4.29"
 memmap2 = "0.9.9"
 niffler = "3.0.0"
-nix = { version = "0.30.1", features = ["fs"] }
+nix = { version = "0.31.1", features = ["fs"] }
 num_cpus = "1.17.0"
-paraseq = "0.4.7"
+paraseq = "0.4.8"
 parking_lot = "0.12.5"
 rand = "0.9.2"
 regex = "1.12.2"
@@ -35,10 +35,10 @@ walkdir = "2.5.0"
 zstd = { version = "0.13.3", features = ["zstdmt"] }
 
 [dev-dependencies]
-bon = "3.8.1"
+bon = "3.8.2"
 itertools = "0.14.0"
 nucgen = "0.2.0"
-tempfile = "3.23.0"
+tempfile = "3.24.0"
 
 [features]
 default = ["htslib", "gcs"]
@@ -47,6 +47,7 @@ htslib = ["paraseq/htslib"]
 gcs = ["paraseq/gcs"]
 
 [profile.release]
+# debug = true
 lto = true
 codegen-units = 1
 

README.md

@@ -8,24 +8,40 @@ A command-line utility for working with BINSEQ files.
 ## Overview
 
 bqtools provides tools to encode, decode, manipulate, and analyze [BINSEQ](https://github.com/arcinstitute/binseq) files.
-It supports both (`*.bq`) and (`*.vbq`) files and makes use of the [`binseq`](https://crates.io/crates/binseq) library.
+It supports all BINSEQ variants (`*.bq`, `*.cbq`, `*.vbq`) and makes use of the [`binseq`](https://crates.io/crates/binseq) library.
 
 BINSEQ is a binary file format family designed for high-performance processing of DNA sequences.
 It currently has two variants: BQ and VBQ.
 
-- **BQ (\*.bq)**: Optimized for _fixed-length_ DNA sequences **without** quality scores.
-- **VBQ (\*.vbq)**: Optimized for _variable-length_ DNA sequences **with optional** quality scores.
+- **BQ (\*.bq)**: Optimized for _fixed-length_ DNA sequences **without** quality scores (2bit/4bit).
+- **VBQ (\*.vbq)**: Optimized for _variable-length_ DNA sequences **with optional** quality scores, headers with 2bit/4bit.
+- **CBQ (\*.cbq)**: Optimized for _variable-length_ DNA sequences **with optional** quality scores, headers with 2bit + N.
 
-Both support single and paired sequences and make use of two-bit or four-bit encoding for efficient nucleotide packing using [`bitnuc`](https://crates.io/crates/bitnuc) and efficient parallel FASTX processing using [`paraseq`](https://crates.io/crates/paraseq).
+All support single and paired sequences and make use of two-bit or four-bit encoding for efficient nucleotide packing using [`bitnuc`](https://crates.io/crates/bitnuc) and efficient parallel FASTX processing using [`paraseq`](https://crates.io/crates/paraseq).
 
 For more information about BINSEQ, see our [preprint](https://www.biorxiv.org/content/10.1101/2025.04.08.647863v1) where we describe the format family and its applications.
 
+### Description of variants
+
+> TL;DR: `*.cbq` is the recommended format for most applications.
+
+For most applications the BINSEQ variant of choice is `*.cbq`.
+This format is lossless by default and supports variable-length sequences.
+It achieves better compression than `*.vbq` and `*.bq` by using blocked-columnar compression of sequence attributes.
+It can optionally exclude quality scores and headers (but they are included by default).
+For an overview of the format check out the [BINSEQ docs](https://docs.rs/binseq/latest/binseq/cbq/index.html).
+
+If your application _only requires sequences_ and has _fixed-length_ reads then `*.bq` is the best choice.
+It is the _fastest_ variant but _is lossy_ by design.
+
+> Note: `*.vbq` was originally designed for variable-length sequences with quality scores and headers, but it is now deprecated in favor of `*.cbq` which is more compressable, lossless, and has faster decoding.
+
 ## Features
 
 - **Encode**: Convert FASTA or FASTQ files to a BINSEQ format
 - **Decode**: Convert a BINSEQ file back to FASTA, FASTQ, or TSV format
 - **Cat**: Concatenate multiple BINSEQ files
-- **Count**: Count records in a BINSEQ file
+- **Info**: Show information and statistics about a BINSEQ file.
 - **Grep**: Search for fixed-string, regex, or fuzzy matches in BINSEQ files.
 - **Pipe**: Create named-pipes for efficient data processing with legacy tools that don't support BINSEQ.
 
@@ -96,7 +112,9 @@ bqtools --help
 bqtools encode --help
 bqtools decode --help
 bqtools cat --help
-bqtools count --help
+bqtools info --help
+bqtools grep --help
+bqtools pipe --help
 ```
 
 ### Encoding
@@ -236,12 +254,18 @@ Combine multiple BINSEQ files:
 bqtools cat file1.bq file2.bq file3.bq -o combined.bq
 ```
 
-### Counting
+### Information and Statistics
 
-Count records in a BINSEQ file:
+Show information and statistics about a BINSEQ file.
 
 ```bash
-bqtools count input.bq
+bqtools info input.cbq
+
+# print out the VBQ index
+bqtools info input.vbq --show-index
+
+# print out the CBQ block headers
+bqtools info input.cbq --show-headers
 ```
 
 ### Grep
@@ -382,4 +406,4 @@ ls fifo_*.fq | xargs -P 4 -I {} sh -c 'legacy-tool {} > {.}.out'
 - Auto-scales to CPU count with `-p0` (default)
 - Pipes can be read sequentially _or_ in parallel without blocking.
 
-Note: This feature is not available on Windows.
+> Note: This feature is not available on Windows.

src/cli/cli.rs

@@ -8,7 +8,7 @@ use clap::{
 
 use crate::cli::PipeCommand;
 
-use super::{CatCommand, CountCommand, DecodeCommand, EncodeCommand, GrepCommand, SampleCommand};
+use super::{CatCommand, DecodeCommand, EncodeCommand, GrepCommand, InfoCommand, SampleCommand};
 
 // Configures Clap v3-style help menu colors
 const STYLES: Styles = Styles::styled()
@@ -33,7 +33,7 @@ pub enum Commands {
 
     Cat(CatCommand),
 
-    Count(CountCommand),
+    Info(InfoCommand),
 
     Grep(GrepCommand),
 

src/cli/encode.rs

@@ -19,7 +19,7 @@ impl EncodeCommand {
         if let Some(mode) = self.output.mode {
             Ok(mode)
         } else if self.input.recursive {
-            Ok(BinseqMode::VBinseq)
+            Ok(BinseqMode::default())
         } else {
             self.output.mode()
         }

src/cli/count.rs src/cli/info.rssrc/cli/count.rs renamed to src/cli/info.rs

@@ -3,23 +3,27 @@ use clap::Parser;
 use super::InputBinseq;
 
 #[derive(Parser, Debug)]
-/// Count the number of records in a BINSEQ file.
-pub struct CountCommand {
+/// Show information about a BINSEQ file.
+pub struct InfoCommand {
     #[clap(flatten)]
     pub input: InputBinseq,
 
     #[clap(flatten)]
-    pub opts: CountOpts,
+    pub opts: InfoOpts,
 }
 
 #[derive(Parser, Debug)]
-#[clap(next_help_heading = "COUNT OPTIONS")]
-pub struct CountOpts {
+#[clap(next_help_heading = "INFO OPTIONS")]
+pub struct InfoOpts {
     /// Only print the number of records in the file
     #[clap(short, long)]
     pub num: bool,
 
     /// Print the index of the file
     #[clap(long)]
     pub show_index: bool,
+
+    /// Print the block headers of the file
+    #[clap(long, conflicts_with = "show_index")]
+    pub show_headers: bool,
 }

src/cli/mod.rs

@@ -1,22 +1,22 @@
 mod cat;
 mod cli;
-mod count;
 mod decode;
 mod encode;
 mod formats;
 mod grep;
+mod info;
 mod input;
 mod output;
 mod pipe;
 mod sample;
 
 pub use cat::CatCommand;
 pub use cli::{Cli, Commands};
-pub use count::CountCommand;
 pub use decode::DecodeCommand;
 pub use encode::EncodeCommand;
 pub use formats::FileFormat;
 pub use grep::GrepCommand;
+pub use info::InfoCommand;
 pub use input::{InputBinseq, InputFile, MultiInputBinseq};
 pub use output::{BinseqMode, Mate, OutputBinseq, OutputFile};
 pub use pipe::PipeCommand;

src/cli/output.rs

@@ -150,16 +150,22 @@ pub struct OutputBinseq {
     pub mode: Option<BinseqMode>,
 
     /// Policy for handling Ns in sequences
+    ///
+    /// Used by bq+vbq
     #[clap(short = 'p', long, default_value = "r")]
     pub policy: PolicyWrapper,
 
     /// Encoding bitsize (2 or 4 bits per nucleotide)
+    ///
+    /// Used by bq+vbq
     #[clap(short = 'S', long, default_value = "2")]
     bitsize: u8,
 
-    /// Include sequence names (headers) in the vbq file
+    /// Exclude sequence names (headers) in the binseq file
+    ///
+    /// Used by vbq+cbq
     #[clap(short = 'H', long)]
-    pub headers: bool,
+    skip_headers: bool,
 
     /// Skip ZSTD compression of VBQ blocks (default: compressed)
     ///
@@ -169,13 +175,13 @@ pub struct OutputBinseq {
 
     /// Skip inclusion of quality scores (default: included)
     ///
-    /// Only used by vbq.
+    /// Used by vbq+cbq
     #[clap(short = 'Q', long)]
     pub skip_quality: bool,
 
-    /// VBQ virtual block size (in bytes)
+    /// Virtual block size (in bytes)
     ///
-    /// Only used by vbq
+    /// Used by vbq+cbq
     #[clap(short = 'B', long, value_parser = parse_memory_size, default_value = "128K")]
     block_size: usize,
 
@@ -189,6 +195,8 @@ pub struct OutputBinseq {
     /// The compression level is between 1 and 22, with 3 being the default.
     /// Higher levels provide better compression at the cost of speed.
     /// Level 0 disables compression.
+    ///
+    /// Used by vbq+cbq
     #[clap(short, long, default_value = "3")]
     pub level: i32,
 
@@ -225,6 +233,14 @@ impl OutputBinseq {
         }
     }
 
+    pub fn headers(&self) -> bool {
+        if self.archive {
+            true
+        } else {
+            !self.skip_headers
+        }
+    }
+
     pub fn block_size(&self) -> usize {
         if self.archive {
             200 * 1024 * 1024
@@ -319,18 +335,21 @@ impl From<PolicyWrapper> for Policy {
 #[derive(Debug, Clone, Copy, ValueEnum, Default, PartialEq)]
 pub enum BinseqMode {
     #[clap(name = "bq")]
-    Binseq,
+    Bq,
     #[clap(name = "vbq")]
+    Vbq,
+    #[clap(name = "cbq")]
     #[default]
-    VBinseq,
+    Cbq,
 }
 impl BinseqMode {
     pub fn determine(path: &str) -> Result<Self> {
         let pathbuf = Path::new(path);
         if let Some(ext) = pathbuf.extension() {
             match ext.to_str() {
-                Some("bq") => Ok(Self::Binseq),
-                Some("vbq") => Ok(Self::VBinseq),
+                Some("bq") => Ok(Self::Bq),
+                Some("vbq") => Ok(Self::Vbq),
+                Some("cbq") => Ok(Self::Cbq),
                 _ => bail!("Could not determine BINSEQ output mode from path: {path}"),
             }
         } else {
@@ -339,8 +358,18 @@ impl BinseqMode {
     }
     pub fn extension(&self) -> &str {
         match self {
-            Self::Binseq => ".bq",
-            Self::VBinseq => ".vbq",
+            Self::Bq => ".bq",
+            Self::Vbq => ".vbq",
+            Self::Cbq => ".cbq",
+        }
+    }
+}
+impl Into<binseq::write::Format> for BinseqMode {
+    fn into(self) -> binseq::write::Format {
+        match self {
+            Self::Bq => binseq::write::Format::Bq,
+            Self::Vbq => binseq::write::Format::Vbq,
+            Self::Cbq => binseq::write::Format::Cbq,
         }
     }
 }