Reorganize.

Author: pmfirestone

examples/constrained_generation.rs

@@ -4,11 +4,8 @@
 use candle_core::IntDType;
 use std::fs;
 use syncode_core::{
-    bytes::restore_bytes,
-    grammar::EBNFParser,
-    mask::grammar_mask,
-    mask_store,
-    types::{Lexer, Parser},
+    bytes::restore_bytes, grammar::EBNFParser, lexer::Lexer, mask::grammar_mask, mask_store,
+    parser::Parser,
 };
 use tokenizers::{Tokenizer, tokenizer};
 

examples/construct_mask_store.rs

@@ -1,11 +1,9 @@
 use rayon::prelude::*;
-use serde_json::{de, ser};
-use std::collections::HashSet;
 use std::fs;
 use syncode_core::bytes::restore_bytes;
 use syncode_core::grammar::EBNFParser;
 use syncode_core::mask::{dfa_mask_store, grammar_mask};
-use syncode_core::types::{Lexer, Parser};
+use syncode_core::{lexer::Lexer, parser::Parser};
 use tokenizers::Tokenizer;
 
 fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {

src/generate.rs

@@ -9,10 +9,10 @@ use tokenizers::Tokenizer;
 
 use crate::{
     bytes::restore_bytes,
+    grammar::Grammar,
     lexer::Lexer,
     mask::{DFAMaskStore, grammar_mask},
-    types::{Parser},
-    grammar::Grammar
+    parser::Parser,
 };
 
 /// Generate a sequence constrained by a given grammar.

src/lib.rs

@@ -9,9 +9,9 @@
 use bytes::restore_bytes;
 use grammar::EBNFParser;
 use mask::{DFAMaskStore, dfa_mask_store};
+use parser::Parser;
 use std::fs;
 use tokenizers::Tokenizer;
-use types::Parser;
 
 pub mod bytes;
 pub mod dfa;
@@ -24,7 +24,6 @@ pub mod production;
 pub mod table;
 pub mod terminal;
 pub mod token;
-pub mod types;
 
 pub fn mask_store(model_id: &str, grammar_file: &str) -> DFAMaskStore {
     let tokenizer = Tokenizer::from_pretrained(model_id, None).unwrap();

src/mask.rs

@@ -3,9 +3,9 @@
 
 use crate::Parser;
 use crate::dfa::all_dfa_states;
-use crate::token::Token;
-use crate::terminal::Terminal;
 use crate::grammar::Grammar;
+use crate::terminal::Terminal;
+use crate::token::Token;
 
 use core::iter::Iterator;
 use regex_automata::dfa::dense;

src/parser.rs

@@ -6,10 +6,10 @@ use std::cell::LazyCell;
 use std::collections::HashSet;
 use std::fmt;
 
-use crate::table::{LRTables, ActionTable, GotoTable, Action};
+use crate::grammar::Grammar;
+use crate::table::{Action, ActionTable, GotoTable, LRTables};
 use crate::terminal::Terminal;
 use crate::token::Token;
-use crate::grammar::Grammar;
 
 /// The Parser with its tables.
 ///
@@ -40,7 +40,6 @@ impl Grammar {
     }
 }
 
-
 /// A special empty token for convenience.
 const EMPTY_TOKEN: LazyCell<Token> = LazyCell::new(|| Token {
     value: [].into(),
@@ -353,10 +352,9 @@ mod tests {
     extern crate test;
     use std::collections::{HashMap, HashSet};
 
-    use std::fs;
-
     use super::*;
-    use crate::grammar::EBNFParser;
+    use crate::lexer::Lexer;
+    use crate::production::Production;
     // Terminal definitions to be used throughout tests. Commented out ones may
     // come in handy in future tests but are commented to avoid dead code warnings.
     fn word() -> Terminal {

src/production.rs

@@ -0,0 +1,26 @@
+use std::sync::Arc;
+
+/// A single production of the grammar.
+#[derive(Clone, Debug, Hash, Eq, PartialEq)]
+pub struct Production {
+    /// The left hand side of the production.
+    pub lhs: Arc<String>,
+    /// The right hand side of the production.
+    pub rhs: Arc<Vec<String>>,
+    // The priority of this production. Used to resolve conflicts when constructing the table.
+    // pub priority: i32,
+}
+
+impl Production {
+    /// Convenience constructor for tests.
+    pub fn new(lhs: &str, rhs: Vec<&str>) -> Production {
+        Production {
+            lhs: lhs.to_string().into(),
+            rhs: rhs
+                .iter()
+                .map(|s| s.to_string())
+                .collect::<Vec<String>>()
+                .into(),
+        }
+    }
+}

src/table.rs

@@ -10,9 +10,10 @@
 //! is work to be done here in cleaning the code and improving its efficiency;
 //! right now the goal is functioning code, nothing more, nothing less.
 
-use std::collections::{HashMap, HashSet};
-use crate::production::Production;
 use crate::grammar::Grammar;
+use crate::production::Production;
+use std::collections::{HashMap, HashSet};
+
 use rayon::prelude::*;
 
 /// An item of the item set for LR parsing.
@@ -45,8 +46,6 @@ pub type ActionTable = HashMap<(usize, String), Action>;
 /// A goto table is a map from a (state_id, nonterminal) pair to a state_id.
 pub type GotoTable = HashMap<(usize, String), usize>;
 
-
-
 pub const AUGMENTED_START_SYMBOL: &str = "supersecretnewstart";
 
 /// Hold state in order to save repeat computations.
@@ -436,9 +435,8 @@ impl LRTables {
 
 #[cfg(test)]
 mod tests {
-    use std::fs;
-
     use super::*;
+    use crate::terminal::Terminal;
     use Action::*;
 
     /// (4.55) from the Dragon Book 2e, section 4.7.2, p. 263.

src/token.rs

@@ -0,0 +1,25 @@
+use crate::terminal::Terminal;
+use std::sync::Arc;
+
+/// A lexical token, what the lexer breaks the input into.
+#[derive(Clone, Debug, PartialEq)]
+pub struct Token {
+    /// The content of the token.
+    pub value: Arc<[u8]>,
+    /// The type of terminal that this is in the grammar. None if this token
+    /// couldn't be lexed, which can happen in the case that this is the
+    /// unlexable remainder.
+    pub terminal: Option<Terminal>,
+    /// Where in the input the token begins.
+    pub start_pos: usize,
+    /// Where in the input the token ends.
+    pub end_pos: usize,
+    /// The line of the input the token begins on.
+    pub line: usize,
+    /// The line of the input the token ends on.
+    pub end_line: usize,
+    /// The column of the input the token begins on.
+    pub column: usize,
+    /// The column of the input the token ends on.
+    pub end_column: usize,
+}

tests/generation.rs

@@ -7,7 +7,11 @@ use syncode_core::bytes::restore_bytes;
 use syncode_core::grammar::EBNFParser;
 use syncode_core::mask::{dfa_mask_store, grammar_mask};
 use syncode_core::mask_store;
-use syncode_core::types::*;
+use syncode_core::parser::Parser;
+use syncode_core::lexer::Lexer;
+use syncode_core::grammar::Grammar;
+use syncode_core::production::Production;
+use syncode_core::terminal::Terminal;
 use tokenizers::Tokenizer;
 
 #[bench]