Merge pull request #166 from ArcInstitute/autodetermine-fixed-regex

Autodetermine fixed regex

Author: noamteyssier

CLAUDE.md

@@ -52,18 +52,45 @@ Build without defaults: `cargo build --no-default-features -F fuzzy,gcs`
 
 ### Core Dependencies
 
-| Crate | Role |
-|-------|------|
-| `binseq` | BINSEQ format read/write |
-| `bitnuc` | 2-bit/4-bit nucleotide encoding |
+| Crate     | Role                             |
+| --------- | -------------------------------- |
+| `binseq`  | BINSEQ format read/write         |
+| `bitnuc`  | 2-bit/4-bit nucleotide encoding  |
 | `paraseq` | Parallel FASTX/BINSEQ processing |
-| `clap` | CLI argument parsing (derive) |
-| `anyhow` | Error handling throughout |
+| `clap`    | CLI argument parsing (derive)    |
+| `anyhow`  | Error handling throughout        |
 
 ### Testing
 
 Integration tests live in `tests/`. `tests/common.rs` provides a builder (`write_fastx()`) for generating random FASTQ/FASTA test data with configurable compression (none, gzip, zstd). Tests use cartesian products over format/compression/mode combinations. Dev dependencies: `bon` (builder macro), `nucgen` (random sequences), `tempfile`, `itertools`.
 
+### Generating Test Data
+
+Random FASTQ/FASTA test data can be created on the CLI with `nucgen` (`cargo install nucgen` if not already installed).
+
+```bash
+# generates 10,000 reads of length 150
+nucgen -n 10000 -l 150 some.fq
+# generates 30,000 paired-reads of length 50 and 200
+nucgen -n 30000 -l 50 -L 200 some_R1.fq some_R2.fq
+```
+
+These can then be ingested with `bqtools encode`:
+
+```bash
+bqtools encode some.fq -o some.cbq
+bqtools encode some_R1.fq some_R2.fq -o some.cbq
+```
+
+### Benchmarking Changes
+
+Make use of `hyperfine` (`cargo install hyperfine` if not already installed) to measure performance of binaries after changes.
+
+```bash
+# Measures decoding performance
+hyperfine --warmup 3 --runs 10 "bqtools decode some.cbq > /dev/null"
+```
+
 ## Contribution Guide
 
 When making changes, keep the following documentation in sync:

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bqtools"
-version = "0.5.2"
+version = "0.5.3"
 edition = "2021"
 license = "MIT"
 authors = ["Noam Teyssier <[email protected]>"]

src/cli/grep.rs

@@ -79,6 +79,7 @@ pub struct GrepArgs {
     /// Denotes patterns are fixed strings (non-regex)
     ///
     /// Allows usage of Aho-Corasick algorithm for efficient matching.
+    /// This is auto-detected when all patterns are literal strings.
     #[clap(short = 'x', long)]
     pub fixed: bool,
 

src/commands/grep/mod.rs

@@ -24,6 +24,22 @@ use crate::{
 use anyhow::Result;
 use binseq::prelude::*;
 
+/// Returns true if the pattern is a fixed DNA string (only ACGT).
+fn is_fixed(pattern: &[u8]) -> bool {
+    !pattern.is_empty()
+        && pattern
+            .iter()
+            .all(|b| matches!(b, b'A' | b'C' | b'G' | b'T'))
+}
+
+/// Returns true if all patterns across multiple sets are fixed DNA strings.
+fn all_patterns_fixed(pattern_sets: &[&[Vec<u8>]]) -> bool {
+    pattern_sets
+        .iter()
+        .flat_map(|s| s.iter())
+        .all(|p| is_fixed(p))
+}
+
 fn build_counter(args: &GrepCommand) -> Result<PatternCounter> {
     #[cfg(feature = "fuzzy")]
     if args.grep.fuzzy_args.fuzzy {
@@ -38,14 +54,17 @@ fn build_counter(args: &GrepCommand) -> Result<PatternCounter> {
         return Ok(PatternCounter::Fuzzy(counter));
     }
 
-    if args.grep.fixed {
-        let counter = AhoCorasickPatternCounter::new(
-            args.grep.bytes_pat1()?,
-            args.grep.bytes_pat2()?,
-            args.grep.bytes_pat()?,
-            args.grep.no_dfa,
-            args.grep.invert,
-        )?;
+    let pat1 = args.grep.bytes_pat1()?;
+    let pat2 = args.grep.bytes_pat2()?;
+    let pat = args.grep.bytes_pat()?;
+    let use_fixed = args.grep.fixed || all_patterns_fixed(&[&pat1, &pat2, &pat]);
+    if !args.grep.fixed && use_fixed {
+        log::debug!("All patterns are fixed strings — auto-selecting Aho-Corasick");
+    }
+
+    if use_fixed {
+        let counter =
+            AhoCorasickPatternCounter::new(pat1, pat2, pat, args.grep.no_dfa, args.grep.invert)?;
         Ok(PatternCounter::AhoCorasick(counter))
     } else {
         let counter = RegexPatternCounter::new(
@@ -89,17 +108,25 @@ fn build_matcher(args: &GrepCommand) -> Result<PatternMatcher> {
         return Ok(PatternMatcher::Fuzzy(matcher));
     }
 
-    if args.grep.fixed && !args.grep.and_logic() {
+    let pat1 = args.grep.bytes_pat1()?;
+    let pat2 = args.grep.bytes_pat2()?;
+    let pat = args.grep.bytes_pat()?;
+    let use_fixed = args.grep.fixed || all_patterns_fixed(&[&pat1, &pat2, &pat]);
+    if !args.grep.fixed && use_fixed {
+        log::debug!("All patterns are fixed strings — auto-selecting Aho-Corasick");
+    }
+
+    if use_fixed && !args.grep.and_logic() {
         let matcher = AhoCorasickMatcher::new(
-            args.grep.bytes_pat1()?,
-            args.grep.bytes_pat2()?,
-            args.grep.bytes_pat()?,
+            pat1,
+            pat2,
+            pat,
             args.grep.no_dfa,
             args.grep.range.map_or(0, |r| r.offset()),
         )?;
         Ok(PatternMatcher::AhoCorasick(matcher))
     } else {
-        if args.grep.fixed {
+        if use_fixed {
             warn!("`-x/--fixed provided but ignored when using AND logic");
         }
         let matcher = RegexMatcher::new(
@@ -168,3 +195,63 @@ pub fn run(args: &GrepCommand) -> Result<()> {
         run_grep(args, reader, writer, format, mate)
     }
 }
+
+#[cfg(test)]
+mod fixed_detection_tests {
+    use super::{all_patterns_fixed, is_fixed};
+
+    #[test]
+    fn test_fixed_dna_strings() {
+        assert!(is_fixed(b"ACGTACGT"));
+        assert!(is_fixed(b"AAAAAAAAAA"));
+        assert!(is_fixed(b"ACGT"));
+    }
+
+    #[test]
+    fn test_empty_string() {
+        assert!(!is_fixed(b""));
+    }
+
+    #[test]
+    fn test_iupac_ambiguity_codes() {
+        assert!(!is_fixed(b"ACGTNRYW"));
+        assert!(!is_fixed(b"ACGN"));
+    }
+
+    #[test]
+    fn test_lowercase_not_fixed() {
+        assert!(!is_fixed(b"acgt"));
+    }
+
+    #[test]
+    fn test_regex_patterns_not_fixed() {
+        assert!(!is_fixed(b"AC.GT"));
+        assert!(!is_fixed(b"AC[GT]"));
+        assert!(!is_fixed(b"A{3}"));
+        assert!(!is_fixed(b"^ACGT"));
+        assert!(!is_fixed(b"ACG|TGA"));
+        assert!(!is_fixed(b"(ACG)"));
+        assert!(!is_fixed(b"AC\\dGT"));
+    }
+
+    #[test]
+    fn test_all_patterns_fixed() {
+        let p1 = vec![b"ACGT".to_vec(), b"TTTT".to_vec()];
+        let p2 = vec![b"GGGG".to_vec()];
+        assert!(all_patterns_fixed(&[&p1, &p2]));
+    }
+
+    #[test]
+    fn test_all_patterns_fixed_with_regex() {
+        let p1 = vec![b"ACGT".to_vec(), b"AC.GT".to_vec()];
+        let p2 = vec![b"GGGG".to_vec()];
+        assert!(!all_patterns_fixed(&[&p1, &p2]));
+    }
+
+    #[test]
+    fn test_all_patterns_fixed_empty_sets() {
+        let p1: Vec<Vec<u8>> = vec![];
+        let p2: Vec<Vec<u8>> = vec![];
+        assert!(all_patterns_fixed(&[&p1, &p2]));
+    }
+}