Pass a single integration tests.

Author: pmfirestone

examples/construct_mask_store.rs

@@ -26,7 +26,7 @@ fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         panic!()
     };
 
-    let Ok(lexer) = Lexer::new(&grammar.terminals) else {
+    let Ok(lexer) = Lexer::new(&grammar.terminals, &grammar.ignore_terminals) else {
         panic!()
     };
 

grammars/calc.lark

@@ -12,15 +12,6 @@ start: expr
        | "(" expr ")"
 
 
-#DIGIT: "0".."9"
-#HEXDIGIT: "a".."f"|"A".."F"|DIGIT
-#INT: DIGIT+
-#SIGNED_INT: ["+"|"-"] INT
-#DECIMAL: INT "." INT? | "." INT
-#_EXP: ("e"|"E") SIGNED_INT
-#FLOAT: INT _EXP | DECIMAL _EXP?
-#NUMBER: FLOAT | INT
-
 NUMBER: /(([0-9]+[eE][+-]?[0-9]+)|([0-9]\.[0-9]+|\.[0-9]+)([eE][+-]?[0-9]+)?)|[0-9]+/
 
 WS: /[ \t\f\r\n]+/

src/dfa.rs

@@ -15,7 +15,7 @@ use std::collections::VecDeque;
 /// method that returns an iterator over all states. The suggested alternative
 /// is to traverse the graph manually. See
 /// <https://github.com/rust-lang/regex/discussions/1223>.
-fn states(dfa: &dense::DFA<Vec<u32>>) -> Vec<StateID> {
+pub fn states(dfa: &dense::DFA<Vec<u32>>) -> Vec<StateID> {
     let mut queue: VecDeque<StateID> = VecDeque::new();
     let mut explored: Vec<StateID> = Vec::new();
 

src/grammar.rs

@@ -165,7 +165,7 @@ impl EBNFParser {
         self.grammar.symbol_set.dedup();
 
         // Make ignore terminals ignored, if any. This is a non-trivial task.
-        self.handle_ignore_terminals();
+        // self.handle_ignore_terminals();
 
         // Include the special EOF symbol.
         self.grammar.symbol_set.push("$".into());
@@ -264,6 +264,10 @@ impl EBNFParser {
             self.cur_priority,
         ));
 
+        self.grammar
+            .symbol_set
+            .push(self.name_stack.last().unwrap().to_string());
+
         // We are no longer parsing a token.
         self.name_stack.pop();
         self.cur_priority = 0;
@@ -300,7 +304,9 @@ impl EBNFParser {
                     self.report_parse_error("Expected token after %ignore statement.");
                     return;
                 };
-                self.ignore_terminals.push(token.as_str().to_string());
+                self.grammar
+                    .ignore_terminals
+                    .push(token.as_str().to_string());
                 self.consume(token.as_str().len());
             }
             "%de" => {
@@ -811,19 +817,23 @@ impl EBNFParser {
     /// but that is probably the most common use case.
     fn expand_extends(&mut self) {}
 
-    /// Insert the ignore terminals into each production.
-    ///
-    /// We do this instead of passing the ignore terminals on to the lexer so
-    /// that there aren't multiple kinds of terminals (lexer and parser) that
-    /// we have to juggle when building a mask: the ignore information is
-    /// already integrated directly into the grammar itself; the %ignore
-    /// notation is just syntactic sugar for putting the named terminal at the
-    /// beginning of each production and after each symbol in each production,
-    /// except for productions that are already empty (to prevent infinite
-    /// regression).
-    fn handle_ignore_terminals(&mut self) {
-        if !self.ignore_terminals.is_empty() {}
-    }
+    // /// Put the ignore terminals into the grammar as terminals.
+    // ///
+    // /// We do this at the end of parsing in case the ignore statement comes
+    // /// before the definition of the terminal it references.
+    // fn handle_ignore_terminals(&mut self) {
+    //     if !self.ignore_terminals.is_empty() {
+    //         for terminal_name in &self.ignore_terminals {
+    //             let Some(terminal) = self.grammar.terminal_from_name(&terminal_name) else {
+    //                 self.report_parse_error(&*format!(
+    //                     "The ignore terminal {terminal_name} is not defined"
+    //                 ));
+    //                 return;
+    //             };
+    //             self.grammar.ignore_terminals.push(terminal);
+    //         }
+    //     }
+    // }
 
     /// The symbols that "descend" from this one. A symbol descends
     /// from a symbol if it is on the right hand side of

src/mask.rs

@@ -144,30 +144,38 @@ pub fn dfa_mask(
 pub fn dfa_mask_store(
     lexical_terminals: &Vec<Terminal>,
     model_vocabulary: &Vec<Vec<u8>>,
-    parser: &Parser,
+    _parser: &Parser,
     _length_of_terminal_sequences: usize,
 ) -> DFAMaskStore {
     let all_states = all_dfa_states(lexical_terminals);
     let mut store: DFAMaskStore = HashMap::new();
     for (terminal, state_id) in &all_states {
+        let dfa = &terminal.dfa;
         // For now, hard-code the lookahead of two terminals.
         // let next_terminals = parser.next_terminals(&terminal.name);
+	// Lookahead of zero terminals.
+	let accept_sequence_names = vec![];
+	let accept_sequence_terminals = vec![];
+	store.insert(
+            (terminal.name.clone(), *state_id, accept_sequence_names),
+            dfa_mask(dfa, state_id, &accept_sequence_terminals, &model_vocabulary),
+        );
         for next_terminal in lexical_terminals {
+            // Lookahead of one terminal.
+            let accept_sequence_names = vec![next_terminal.name.clone()];
+            let accept_sequence_terminals = vec![next_terminal.clone()];
+            store.insert(
+                (terminal.name.clone(), *state_id, accept_sequence_names),
+                dfa_mask(dfa, state_id, &accept_sequence_terminals, &model_vocabulary),
+            );
             // let after_next_terminals = parser.next_terminals(next_terminal);
             for after_next_terminal in lexical_terminals {
+		// Lookahead of two terminals.
                 let accept_sequence_names =
                     vec![next_terminal.name.clone(), after_next_terminal.name.clone()];
-                let accept_sequence_terminals = vec![
-                    parser
-                        .grammar
-                        .terminal_from_name(&next_terminal.name)
-                        .unwrap(),
-                    parser
-                        .grammar
-                        .terminal_from_name(&after_next_terminal.name)
-                        .unwrap(),
-                ];
-                let dfa = &terminal.dfa;
+                let accept_sequence_terminals =
+                    vec![next_terminal.clone(), after_next_terminal.clone()];
+
                 store.insert(
                     (terminal.name.clone(), *state_id, accept_sequence_names),
                     dfa_mask(dfa, state_id, &accept_sequence_terminals, &model_vocabulary),
@@ -202,11 +210,18 @@ pub fn grammar_mask(
                 // Get the relevant mask out of the store.
                 mask_store
                     .get(&(
-                        first_terminal.name,
+                        first_terminal.name.clone(),
                         end_state,
                         accept_sequence[1..].to_vec(),
                     ))
-                    .unwrap(),
+                    .unwrap_or_else(|| {
+                        panic!(
+                            "The mask store does not contain the key ({}, {:?}, {:#?}).",
+                            first_terminal.name,
+                            end_state,
+                            accept_sequence[1..].to_vec()
+                        )
+                    }),
             )
             .enumerate()
             {

src/parser.rs

@@ -27,6 +27,10 @@ impl Parser {
 
     /// Return all the terminals that could come after this one, regardless of
     /// the state the parser is in.
+    ///
+    /// FIXME: This only works for shift actions and fails (stack underflow)
+    /// for reduces. Reimplement this as a cleverer algorithm at the level of
+    /// the grammar.
     pub fn next_terminals(&self, terminal: &String) -> Vec<String> {
         let states_that_accept_this_terminal: Vec<usize> = self
             .action_table
@@ -73,6 +77,10 @@ impl Parser {
         // consolidating the error-managing boiler plate.
         let mut state_stack = state_stack;
 
+        // if self.grammar.ignore_terminals.contains(terminal) {
+        //     // Consume the terminal without modifying the state stack.
+        // }
+
         loop {
             // Get the current state.
             let Some(state) = state_stack.last() else {
@@ -159,10 +167,20 @@ impl Parser {
         let mut a0: Vec<String> = Vec::new();
         let mut a1: Vec<String> = Vec::new();
 
-        let last_token = tokens[tokens.len() - 1].clone();
+        let last_token = tokens.last().unwrap().clone();
         let mut state_stack = vec![self.start_state];
 
-        for token in &tokens[..] {
+        // eprintln!("{:#?}", state_stack);
+        for token in &tokens {
+            // Skip ignore terminals.
+            if self
+                .grammar
+                .ignore_terminals
+                .contains(&token.clone().terminal.unwrap().name)
+            {
+                continue;
+            }
+
             let terminal = token.clone().terminal.unwrap().name.clone();
             // FIXME: There must be a less horrid way to do this.
             let Ok(new_state_stack) = self.next(&terminal, state_stack) else {
@@ -171,14 +189,15 @@ impl Parser {
             state_stack = new_state_stack;
             a0 = a1;
             a1 = self.follow(&state_stack);
+            // eprintln!("{:#?}", state_stack);
         }
 
         // There are two cases for accept sequences. See section 4.5 of the
         // paper and Algorithm 4, lines 15-21.
         let mut accept_sequences: HashSet<Vec<String>> = HashSet::new();
         if last_token == remainder {
             // Case 1: the remainder is the last lexical token.
-            let remainder_type = remainder.clone().terminal.unwrap().name;
+            let remainder_type = last_token.clone().terminal.unwrap().name;
             for terminal in a1 {
                 accept_sequences.insert(vec![remainder_type.clone(), terminal]);
             }
@@ -191,6 +210,13 @@ impl Parser {
                 accept_sequences.insert(vec![terminal]);
             }
         }
+
+        for ignore_terminal in &self.grammar.ignore_terminals {
+            // We deal with ignore terminals by adding a special 1-long accept
+            // sequence for each ignore terminal.
+            accept_sequences.insert(vec![ignore_terminal.to_string()]);
+        }
+
         Ok(accept_sequences)
     }
 }
@@ -438,8 +464,6 @@ mod tests {
             action_table,
             goto_table,
             start_state: 0,
-            // FIXME: Make this actually what it's supposed to be; for now,
-            // just make the compiler happy.
             grammar: Grammar {
                 symbol_set: vec![
                     "goal".into(),
@@ -451,11 +475,12 @@ mod tests {
                     "DEC_NUMBER".into(),
                     "WORD".into(),
                     "$".into(),
+                    "SPACE".into(),
                 ],
-                terminals: vec![plus(), star(), dec_number(), word(), eof()],
+                terminals: vec![plus(), star(), dec_number(), word(), eof(), space()],
                 start_symbol: "goal".to_string(),
                 productions: calc_rules(),
-                ignore_terminals: vec![],
+                ignore_terminals: vec!["SPACE".into()],
             },
         }
     }
@@ -502,7 +527,8 @@ mod tests {
     fn end_to_end_parse() {
         let parser = calc_parser();
         let terminals = vec![word(), star(), dec_number(), plus(), space()];
-        let Ok(lexer) = Lexer::new(&terminals) else {
+        let ignore_terminals = vec!["SPACE".to_string()];
+        let Ok(lexer) = Lexer::new(&terminals, &ignore_terminals) else {
             panic!()
         };
 
@@ -529,18 +555,11 @@ mod tests {
             },
             remainder
         );
-        // It's not clear to me exactly what the semantics are here
-        // w.r.t. whitespace and other ignored terminals. Also, how do we deal
-        // with the possibilty that the remainder could change lexical type,
-        // even to types that aren't permitted? As it is, the algorithm allows
-        // a DEC_NUMBER to change to a WORD, even though that isn't actually
-        // possible in the grammar: semantically, what this says is that
-        // instead of being a DEC_NUMBER, the last lexical token could have
-        // been a WORD, which is technically true. Nevertheless, we already
-        // have enough information to know that the last token couldn't
-        // possibly become a WORD and could only continue to be a DEC_NUMBER.
         assert_eq!(
             HashSet::from([
+                vec!["DEC_NUMBER".to_string()],
+                vec!["WORD".to_string()],
+                vec!["SPACE".to_string()],
                 vec!["DEC_NUMBER".to_string(), "STAR".to_string()],
                 vec!["DEC_NUMBER".to_string(), "PLUS".to_string()],
                 vec!["DEC_NUMBER".to_string(), "$".to_string()],

src/table.rs

@@ -342,12 +342,12 @@ fn checked_insert(state_id: usize, terminal: String, action: Action, table: &mut
         // As a special case, if the action we were going to insert is the same
         // as the one that's already there, we give a warning and continue.
         if table.get(&(state_id, terminal.clone())) == Some(&action) {
-            eprintln!(
-                "WARNING: While inserting an action into the action table, we found the same action we were going to insert.\nstate_id: {state_id}, terminal: {:#?}, existing action: {:#?}, new action: {:#?}",
-                terminal.clone(),
-                table.get(&(state_id, terminal.clone())),
-                action
-            );
+            // eprintln!(
+            //     "WARNING: While inserting an action into the action table, we found the same action we were going to insert.\nstate_id: {state_id}, terminal: {:#?}, existing action: {:#?}, new action: {:#?}",
+            //     terminal.clone(),
+            //     table.get(&(state_id, terminal.clone())),
+            //     action
+            // );
         } else {
             panic!(
                 "While inserting an action into the action table, discovered a conflicting action:\nstate_id: {state_id}, terminal: {:#?}, existing action: {:#?}, new action: {:#?}",

src/types.rs

@@ -90,7 +90,7 @@ pub struct Grammar {
     /// The termnals that the lexer should ignore. FIXME: It's an aberration
     /// that this is here, because this field is not part of the abstract idea
     /// of what a grammar is.
-    pub ignore_terminals: Vec<Terminal>,
+    pub ignore_terminals: Vec<String>,
 }
 
 /// An item of the item set for LR parsing.
@@ -145,6 +145,8 @@ pub struct Parser {
 pub struct Lexer {
     /// The terminals this lexer recognizes.
     pub terminals: Vec<Terminal>,
+    /// The terminals this lexer ignores.
+    pub ignore_terminals: Vec<String>,
     /// The terminals that contain newlines.
     pub newline_types: HashSet<Terminal>,
     /// The DFA for matching patterns.

tests/generation.rs

@@ -1,4 +1,6 @@
 use bstr::ByteSlice;
+use regex_automata::dfa::Automaton;
+use syncode_core::dfa::{all_dfa_states, states};
 // tests/generation.rs
 use syncode_core::grammar::EBNFParser;
 use syncode_core::mask::{dfa_mask_store, grammar_mask};
@@ -29,27 +31,34 @@ fn test_calc() {
     };
 
     // eprintln!("action_table: {:#?}", parser.action_table);
+    // eprintln!("goto_table: {:#?}", parser.goto_table);
 
-    let Ok(lexer) = Lexer::new(&grammar.terminals) else {
+    let Ok(lexer) = Lexer::new(&grammar.terminals, &grammar.ignore_terminals) else {
         panic!()
     };
 
     let mask_store = dfa_mask_store(&grammar.terminals, &model_vocabulary, &parser, 2);
 
-    let prefix = b"1+1";
+    let prefix = b"1 + ";
 
     let Ok((tokens, remainder)) = lexer.lex(prefix) else {
         panic!()
     };
 
-    // eprintln!("remainder: {}", remainder.value.as_bstr());
-    // eprintln!("tokens: {:#?}", tokens);
+    eprintln!("remainder: {:#?}", remainder);
+    eprintln!("tokens: {:#?}", tokens);
 
     let Ok(accept_sequences) = parser.parse(tokens, remainder.clone()) else {
         panic!()
     };
 
-    // eprintln!("{:#?}", accept_sequences);
+    eprintln!("accept_sequences: {:#?}", accept_sequences);
+
+    // let number = grammar.terminal_from_name(&"NUMBER".to_string()).unwrap();
+
+    // let state = number.advance(number.start_state(), b"22");
+
+    // eprintln!("{:?}", states(&number.dfa));
 
     let mask = grammar_mask(
         &accept_sequences,
@@ -104,7 +113,7 @@ fn test_dfa_mask_store() {
         panic!()
     };
     // println!("{:#?}", parser);
-    let Ok(lexer) = Lexer::new(&grammar.terminals) else {
+    let Ok(lexer) = Lexer::new(&grammar.terminals, &grammar.ignore_terminals) else {
         panic!()
     };
     let store = dfa_mask_store(&lexical_terminals, &model_vocabulary, &parser, 2);