Merge pull request #170 from ArcInstitute/bqtools-0.5.4

Bqtools 0.5.4

Author: noamteyssier

CLAUDE.md

@@ -44,7 +44,9 @@ Build without defaults: `cargo build --no-default-features -F fuzzy,gcs`
 
 **Parallel processing**: Commands use the `paraseq` crate's `ParallelProcessor` trait for embarrassingly parallel batch processing. Each command has a `processor.rs` implementing this trait with thread-local buffers and `Arc<Mutex<T>>` for shared global state.
 
-**Grep backends**: The grep command uses a `PatternMatcher` enum dispatching to three backends — `regex`, `aho-corasick` (fixed-string, multi-pattern), and `sassy` (fuzzy, feature-gated). The same pattern applies to `PatternCounter` for the `-P` pattern-count mode.
+**Grep backends**: The grep command uses a `PatternMatcher` enum dispatching to three backends — `regex`, `aho-corasick` (fixed-string, multi-pattern), and `sassy` (fuzzy, feature-gated). The same pattern applies to `PatternCounter` for the `-P` pattern-count mode. All backends accept `PatternCollection` which carries optional pattern names (from FASTA headers).
+
+**Pattern types**: `patterns.rs` defines `Pattern` (name + sequence) and `PatternCollection` (newtype over `Vec<Pattern>`) with methods `.bytes()`, `.regexes()`, `.names()`. Pattern files (`--file`, `--sfile`, `--xfile`) auto-detect FASTA vs plain text. FASTA headers become pattern names; plain text patterns have no name and fall back to the pattern string in output.
 
 **Encode modes**: Encoding dispatches across atomic (single/paired files), recursive (directory walk via `walkdir`), manifest (file list), and batch (multi-file thread distribution) modes.
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bqtools"
-version = "0.5.3"
+version = "0.5.4"
 edition = "2021"
 license = "MIT"
 authors = ["Noam Teyssier <noam.teyssier@arcinstitute.org>"]

README.md

@@ -320,6 +320,7 @@ bqtools grep input.bq "ACGTACGT" -zi
 ```
 
 `bqtools` can also handle a large collection of patterns which can be provided on the CLI as a file.
+Pattern files can be either **plain text** (one pattern per line) or **FASTA** format (sequences are used as patterns, auto-detected).
 You can provide files for either primary/extended, just primary, or just extended patterns with the relevant flags.
 Notably this will match _solely_ with OR logic.
 This can be used also with fuzzy matching as well as with pattern counting described below.
@@ -329,9 +330,12 @@ If your patterns are all fixed strings (and not regex), you can improve performa
 This will use the more efficient [Aho-Corasick algorithm](https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm) to match patterns.
 
 ```bash
-# Run grep with patterns from a file
+# Run grep with patterns from a plain text file (one pattern per line)
 bqtools grep input.bq --file patterns.txt
 
+# Run grep with patterns from a FASTA file (sequences used as patterns)
+bqtools grep input.bq --file patterns.fa
+
 # Run grep with patterns from a file (primary)
 bqtools grep input.bq --sfile patterns.txt
 
@@ -387,7 +391,13 @@ bqtools grep input.bq --file patterns.txt -P
 bqtools grep input.bq --file patterns.txt -Px
 ```
 
-The output of pattern count is a TSV with three columns: [Pattern, Count, Fraction of Total]
+The output of pattern count is a TSV with three columns: [Name, Count, Fraction of Total].
+When patterns are loaded from a FASTA file, the FASTA sequence headers are used as names; otherwise, the pattern string itself is used.
+
+```bash
+# Count patterns from a FASTA file (names column shows FASTA headers)
+bqtools grep input.bq --file patterns.fa -P
+```
 
 ### Pipe
 

src/cli/grep.rs

@@ -1,7 +1,16 @@
-use anyhow::{bail, Result};
+use std::{
+    fs,
+    io::{self, Read},
+};
+
+use anyhow::Result;
 use clap::Parser;
+use paraseq::{fasta, Record};
 
-use crate::{cli::FileFormat, commands::grep::SimpleRange};
+use crate::{
+    cli::FileFormat,
+    commands::grep::{Pattern, PatternCollection, SimpleRange},
+};
 
 use super::{InputBinseq, OutputFile};
 
@@ -121,51 +130,6 @@ impl GrepArgs {
         }
         Ok(())
     }
-    fn chain_regex(
-        &self,
-        cli_patterns: &[String],
-        filetype: PatternFileType,
-    ) -> Result<Vec<regex::bytes::Regex>> {
-        let mut all_patterns = cli_patterns
-            .iter()
-            .map(std::borrow::ToOwned::to_owned)
-            .collect::<Vec<String>>();
-        if !self.file_args.empty_file(filetype) {
-            all_patterns.extend(self.file_args.read_file_patterns(filetype)?);
-        }
-
-        // all patterns are kept separate for:
-        // 1. AND logic
-        // 2. Individual pattern counting
-        if self.and_logic() || self.pattern_count {
-            Ok(all_patterns
-                .iter()
-                .map(|s| {
-                    regex::bytes::Regex::new(s).expect("Could not build regex from pattern: {s}")
-                })
-                .collect())
-
-        // for OR logic they can be compiled into a single regex for performance
-        } else {
-            let global_pattern = all_patterns.join("|");
-            if global_pattern.is_empty() {
-                Ok(vec![])
-            } else {
-                Ok(vec![regex::bytes::Regex::new(&global_pattern).expect(
-                    "Could not build regex from pattern: {global_pattern}",
-                )])
-            }
-        }
-    }
-    pub fn bytes_reg1(&self) -> Result<Vec<regex::bytes::Regex>> {
-        self.chain_regex(&self.reg1, PatternFileType::SFile)
-    }
-    pub fn bytes_reg2(&self) -> Result<Vec<regex::bytes::Regex>> {
-        self.chain_regex(&self.reg2, PatternFileType::XFile)
-    }
-    pub fn bytes_reg(&self) -> Result<Vec<regex::bytes::Regex>> {
-        self.chain_regex(&self.reg, PatternFileType::File)
-    }
     pub fn and_logic(&self) -> bool {
         if self.file_args.empty() {
             !self.or_logic
@@ -177,27 +141,30 @@ impl GrepArgs {
 }
 
 impl GrepArgs {
-    fn chain_bytes(
+    fn chain_patterns(
         &self,
         cli_patterns: &[String],
         filetype: PatternFileType,
-    ) -> Result<Vec<Vec<u8>>> {
-        let bytes_iter = cli_patterns.iter().map(|s| s.as_bytes().to_vec());
+    ) -> Result<PatternCollection> {
+        let cli_iter = cli_patterns.iter().map(|s| Pattern {
+            name: None,
+            sequence: s.as_bytes().to_vec(),
+        });
         if self.file_args.empty_file(filetype) {
-            Ok(bytes_iter.collect())
+            Ok(PatternCollection(cli_iter.collect()))
         } else {
-            let patterns = self.file_args.patterns(filetype)?;
-            Ok(bytes_iter.chain(patterns).collect())
+            let file_patterns = self.file_args.patterns(filetype)?;
+            Ok(PatternCollection(cli_iter.chain(file_patterns).collect()))
         }
     }
-    pub fn bytes_pat1(&self) -> Result<Vec<Vec<u8>>> {
-        self.chain_bytes(&self.reg1, PatternFileType::SFile)
+    pub fn patterns_m1(&self) -> Result<PatternCollection> {
+        self.chain_patterns(&self.reg1, PatternFileType::SFile)
     }
-    pub fn bytes_pat2(&self) -> Result<Vec<Vec<u8>>> {
-        self.chain_bytes(&self.reg2, PatternFileType::XFile)
+    pub fn patterns_m2(&self) -> Result<PatternCollection> {
+        self.chain_patterns(&self.reg2, PatternFileType::XFile)
     }
-    pub fn bytes_pat(&self) -> Result<Vec<Vec<u8>>> {
-        self.chain_bytes(&self.reg, PatternFileType::File)
+    pub fn patterns(&self) -> Result<PatternCollection> {
+        self.chain_patterns(&self.reg, PatternFileType::File)
     }
 }
 
@@ -229,28 +196,32 @@ pub struct FuzzyArgs {
 pub struct PatternFileArgs {
     /// File of patterns to search for
     ///
-    /// This assumes one pattern per line.
+    /// Accepts a plain text file (one pattern per line) or a FASTA file
+    /// (sequences are used as patterns). FASTA files are auto-detected.
     /// Patterns may be regex or literal (fuzzy doesn't support regex).
     /// These will match against either primary or extended sequence.
     #[clap(long)]
     pub file: Option<String>,
 
     /// File of patterns to search for in primary sequence
     ///
-    /// This assumes one pattern per line.
+    /// Accepts a plain text file (one pattern per line) or a FASTA file
+    /// (sequences are used as patterns). FASTA files are auto-detected.
     /// Patterns may be regex or literal (fuzzy doesn't support regex).
     #[clap(long)]
     pub sfile: Option<String>,
 
     /// File of patterns to search for in extended sequence
     ///
-    /// This assumes one pattern per line.
+    /// Accepts a plain text file (one pattern per line) or a FASTA file
+    /// (sequences are used as patterns). FASTA files are auto-detected.
     /// Patterns may be regex or literal (fuzzy doesn't support regex).
     #[clap(long)]
     pub xfile: Option<String>,
 }
+
 impl PatternFileArgs {
-    fn empty(&self) -> bool {
+    pub(crate) fn empty(&self) -> bool {
         self.file.is_none() && self.sfile.is_none() && self.xfile.is_none()
     }
 
@@ -262,34 +233,62 @@ impl PatternFileArgs {
         }
     }
 
-    fn read_file(&self, filetype: PatternFileType) -> Result<String> {
+    fn file_path(&self, filetype: PatternFileType) -> Result<&str> {
         let file = match filetype {
             PatternFileType::File => &self.file,
             PatternFileType::SFile => &self.sfile,
             PatternFileType::XFile => &self.xfile,
         };
-        if let Some(file) = file {
-            Ok(std::fs::read_to_string(file)?)
-        } else {
-            bail!("Specified file type {filetype:?} not provided at CLI")
-        }
+        file.as_deref()
+            .ok_or_else(|| anyhow::anyhow!("Specified file type {filetype:?} not provided at CLI"))
     }
 
-    fn read_file_patterns(&self, filetype: PatternFileType) -> Result<Vec<String>> {
-        let contents = self.read_file(filetype)?;
-        Ok(contents
-            .lines()
-            .map(std::string::ToString::to_string)
-            .collect())
+    /// Returns true if the file starts with '>' (FASTA format).
+    fn is_fasta(path: &str) -> Result<bool> {
+        let file = fs::File::open(path)?;
+        // only take up to 10 bytes to determine fasta status
+        for byte in io::BufReader::new(file).bytes().take(10) {
+            let b = byte?;
+            if b != b'\n' && b != b'\r' {
+                return Ok(b == b'>');
+            }
+        }
+        Ok(false)
     }
 
-    fn patterns(&self, filetype: PatternFileType) -> Result<Vec<Vec<u8>>> {
-        let contents = self.read_file(filetype)?;
-        let mut patterns = Vec::new();
-        for line in contents.lines() {
-            patterns.push(line.as_bytes().to_vec());
+    /// Load patterns from a file, auto-detecting FASTA vs plain text.
+    fn load_patterns(path: &str) -> Result<Vec<Pattern>> {
+        if Self::is_fasta(path)? {
+            let mut reader = fasta::Reader::from_path(path)?;
+            let mut rset = fasta::RecordSet::default();
+            let mut patterns = Vec::new();
+
+            while rset.fill(&mut reader)? {
+                for record in rset.iter() {
+                    let record = record?;
+                    patterns.push(Pattern {
+                        name: Some(record.id_str().to_string()),
+                        sequence: record.seq().into_owned(),
+                    });
+                }
+            }
+
+            Ok(patterns)
+        } else {
+            let contents = std::fs::read_to_string(path)?;
+            Ok(contents
+                .lines()
+                .map(|line| Pattern {
+                    name: None,
+                    sequence: line.as_bytes().to_vec(),
+                })
+                .collect())
         }
-        Ok(patterns)
+    }
+
+    fn patterns(&self, filetype: PatternFileType) -> Result<Vec<Pattern>> {
+        let path = self.file_path(filetype)?;
+        Self::load_patterns(path)
     }
 }
 

src/cli/input.rs

@@ -272,17 +272,15 @@ impl Span {
         if let Some(start) = self.start {
             if start > max_records {
                 error!(
-                    "Provided start ({}) exceeds maximum number of records ({})",
-                    start, max_records
+                    "Provided start ({start}) exceeds maximum number of records ({max_records})"
                 );
                 bail!("Maximum number of records exceeded")
             }
         }
         if let Some(end) = self.end {
             if end > max_records {
                 warn!(
-                    "Clipping provided endpoint ({}) to maximum number of records ({})",
-                    end, max_records
+                    "Clipping provided endpoint ({end}) to maximum number of records ({max_records})"
                 );
             }
             self.end = Some(end.min(max_records));

src/cli/output.rs

@@ -364,12 +364,12 @@ impl BinseqMode {
         }
     }
 }
-impl Into<binseq::write::Format> for BinseqMode {
-    fn into(self) -> binseq::write::Format {
-        match self {
-            Self::Bq => binseq::write::Format::Bq,
-            Self::Vbq => binseq::write::Format::Vbq,
-            Self::Cbq => binseq::write::Format::Cbq,
+impl From<BinseqMode> for binseq::write::Format {
+    fn from(val: BinseqMode) -> Self {
+        match val {
+            BinseqMode::Bq => binseq::write::Format::Bq,
+            BinseqMode::Vbq => binseq::write::Format::Vbq,
+            BinseqMode::Cbq => binseq::write::Format::Cbq,
         }
     }
 }

src/commands/cat/mod.rs

@@ -80,9 +80,9 @@ fn record_vbq_header(paths: &[String]) -> Result<vbq::FileHeader> {
     for path in &paths[1..] {
         let reader = vbq::MmapReader::new(path)?;
         if reader.header() != header {
-            error!("Inconsistent header found for path: {}", path);
+            error!("Inconsistent header found for path: {path}");
             warn!("Note: The first VBQ used in `cat` will be considered as the reference header. All subsequent VBQs must have the same header.");
-            bail!("Inconsistent header found for path: {}", path);
+            bail!("Inconsistent header found for path: {path}");
         }
     }
     Ok(header)
@@ -97,9 +97,9 @@ fn record_cbq_header(paths: &[String]) -> Result<cbq::FileHeader> {
     for path in &paths[1..] {
         let reader = cbq::MmapReader::new(path)?;
         if reader.header() != header {
-            error!("Inconsistent header found for path: {}", path);
+            error!("Inconsistent header found for path: {path}");
             warn!("Note: The first CBQ used in `cat` will be considered as the reference header. All subsequent CBQs must have the same header.");
-            bail!("Inconsistent header found for path: {}", path);
+            bail!("Inconsistent header found for path: {path}");
         }
     }
     Ok(header)

src/commands/decode/mod.rs

@@ -65,7 +65,7 @@ pub fn run(args: &DecodeCommand) -> Result<()> {
             proc.clone(),
             args.output.threads(),
             span.get_range(num_records)?,
-        )?
+        )?;
     } else {
         reader.process_parallel(proc.clone(), args.output.threads())?;
     }

src/commands/encode/encode.rs

@@ -55,7 +55,7 @@ pub fn encode_collection(
         }
     } else {
         bail!("All input files must have the same format.");
-    };
+    }
     let ohandle = match_output(opath)?;
     let mut builder = BinseqWriterBuilder::new(mode.into())
         .block_size(config.block_size)
@@ -82,12 +82,12 @@ pub fn encode_collection(
                 let inner = collection.inner_mut();
                 let slen = get_sequence_len(&mut inner[0])?;
                 let xlen = get_sequence_len(&mut inner[1])?;
-                builder = builder.slen(slen as u32).xlen(xlen as u32)
+                builder = builder.slen(slen as u32).xlen(xlen as u32);
             }
             fastx::CollectionType::Interleaved => {
                 let inner = collection.inner_mut();
                 let (slen, xlen) = get_interleaved_sequence_len(&mut inner[0])?;
-                builder = builder.slen(slen as u32).xlen(xlen as u32)
+                builder = builder.slen(slen).xlen(xlen);
             }
             _ => {
                 bail!("Unsupported collection type found in `encode_collection_bq`");

src/commands/encode/processor.rs

@@ -29,8 +29,8 @@ impl<W: Write + Send> Clone for Encoder<W> {
     fn clone(&self) -> Self {
         Self {
             t_writer: self.t_writer.clone(),
-            t_count: self.t_count.clone(),
-            t_skip: self.t_skip.clone(),
+            t_count: self.t_count,
+            t_skip: self.t_skip,
             writer: self.writer.clone(),
             count: self.count.clone(),
             skip: self.skip.clone(),