More ergonomic morph alternative without clone

examples/array_language.rs

@@ -52,15 +52,15 @@ type Environment = HashMap<String, Vec<i128>>;
 
 /* ANCHOR: callbacks */
 /// Parse lexer slice as an i128
-fn number_callback<'s>(lex: &mut Lexer<'s, Token>) -> Result<i128, LexingError<'s>> {
+fn number_callback<'s>(lex: &mut Lexer<'s, '_, Token>) -> Result<i128, LexingError<'s>> {
     let source = lex.slice();
     let res = source.parse();
     res.map_err(|err| LexingError::InvalidInteger { err, source })
 }
 
 /// Look up the lexer slice in the variable environment,
 /// yielding a borrow of the variable's value.
-fn var_callback<'s, 'a>(lex: &mut Lexer<'s, Token<'a>>) -> Result<&'a [i128], LexingError<'s>> {
+fn var_callback<'s, 'a>(lex: &mut Lexer<'s, '_, Token<'a>>) -> Result<&'a [i128], LexingError<'s>> {
     match lex.extras.get(lex.slice()) {
         Some(arr) => Ok(arr.as_slice()),
         None => Err(LexingError::UnknownVariable(lex.slice())),

examples/custom_error.rs

@@ -31,7 +31,7 @@ impl From<ParseIntError> for LexingError {
 }
 
 impl LexingError {
-    fn from_lexer(lex: &mut logos::Lexer<'_, Token>) -> Self {
+    fn from_lexer(lex: &mut logos::Lexer<'_, '_, Token>) -> Self {
         LexingError::NonAsciiCharacter(lex.slice().chars().next().unwrap())
     }
 }

examples/json.rs

@@ -86,7 +86,7 @@ enum Value {
 
 /* ANCHOR: value */
 /// Parse a token stream into a JSON value.
-fn parse_value(lexer: &mut Lexer<'_, Token>) -> Result<Value> {
+fn parse_value(lexer: &mut Lexer<'_, '_, Token>) -> Result<Value> {
     if let Some(token) = lexer.next() {
         match token {
             Ok(Token::Bool(b)) => Ok(Value::Bool(b)),
@@ -111,7 +111,7 @@ fn parse_value(lexer: &mut Lexer<'_, Token>) -> Result<Value> {
 /// a valid terminator is found.
 ///
 /// > NOTE: we assume '[' was consumed.
-fn parse_array(lexer: &mut Lexer<'_, Token>) -> Result<Value> {
+fn parse_array(lexer: &mut Lexer<'_, '_, Token>) -> Result<Value> {
     let mut array = Vec::new();
     let span = lexer.span();
     let mut awaits_comma = false;
@@ -165,7 +165,7 @@ fn parse_array(lexer: &mut Lexer<'_, Token>) -> Result<Value> {
 /// a valid terminator is found.
 ///
 /// > NOTE: we assume '{' was consumed.
-fn parse_object(lexer: &mut Lexer<'_, Token>) -> Result<Value> {
+fn parse_object(lexer: &mut Lexer<'_, '_, Token>) -> Result<Value> {
     let mut map = HashMap::new();
     let span = lexer.span();
     let mut awaits_comma = false;

examples/json_borrowed.rs

@@ -82,7 +82,7 @@ enum Value<'source> {
 
 /* ANCHOR: value */
 /// Parse a token stream into a JSON value.
-fn parse_value<'source>(lexer: &mut Lexer<'source, Token<'source>>) -> Result<Value<'source>> {
+fn parse_value<'source>(lexer: &mut Lexer<'source, '_, Token<'source>>) -> Result<Value<'source>> {
     if let Some(token) = lexer.next() {
         match token {
             Ok(Token::Bool(b)) => Ok(Value::Bool(b)),
@@ -107,7 +107,7 @@ fn parse_value<'source>(lexer: &mut Lexer<'source, Token<'source>>) -> Result<Va
 /// a valid terminator is found.
 ///
 /// > NOTE: we assume '[' was consumed.
-fn parse_array<'source>(lexer: &mut Lexer<'source, Token<'source>>) -> Result<Value<'source>> {
+fn parse_array<'source>(lexer: &mut Lexer<'source, '_, Token<'source>>) -> Result<Value<'source>> {
     let mut array = Vec::new();
     let span = lexer.span();
     let mut awaits_comma = false;
@@ -161,7 +161,7 @@ fn parse_array<'source>(lexer: &mut Lexer<'source, Token<'source>>) -> Result<Va
 /// a valid terminator is found.
 ///
 /// > NOTE: we assume '{' was consumed.
-fn parse_object<'source>(lexer: &mut Lexer<'source, Token<'source>>) -> Result<Value<'source>> {
+fn parse_object<'source>(lexer: &mut Lexer<'source, '_, Token<'source>>) -> Result<Value<'source>> {
     let mut map = HashMap::new();
     let span = lexer.span();
     let mut awaits_comma = false;

logos-cli/tests/snapshots/tests__tailcall-fmt.snap

@@ -8,10 +8,10 @@ enum Token {
 }
 impl<'s> ::logos::Logos<'s> for Token {
     type Error = ();
-    type Extras = ();
+    type Extras<'extras> = ();
     type Source = ::core::primitive::str;
     fn lex(
-        lex: &mut ::logos::Lexer<'s, Self>,
+        lex: &mut ::logos::Lexer<'s, '_, Self>,
     ) -> ::core::option::Option<::core::result::Result<Self, <Self as ::logos::Logos<'s>>::Error>>
     {
         use core::option::Option as _Option;
@@ -93,12 +93,12 @@ impl<'s> ::logos::Logos<'s> for Token {
             }};
         }
         #[inline]
-        fn _make_error<'s>(lex: &mut _Lexer<'s, Token>) -> <Token as Logos<'s>>::Error {
+        fn _make_error<'s>(lex: &mut _Lexer<'s, '_, Token>) -> <Token as Logos<'s>>::Error {
             <<Token as Logos<'s>>::Error as ::core::default::Default>::default()
         }
         #[inline]
         fn _get_action<'s>(
-            lex: &mut _Lexer<'s, Token>,
+            lex: &mut _Lexer<'s, '_, Token>,
             offset: ::core::primitive::usize,
             context: _Option<LogosLeaf>,
         ) -> CallbackResult<'s, Token> {
@@ -115,7 +115,7 @@ impl<'s> ::logos::Logos<'s> for Token {
             Leaf0 = 0isize,
         }
         fn state0<'s>(
-            lex: &mut _Lexer<'s, Token>,
+            lex: &mut _Lexer<'s, '_, Token>,
             mut offset: ::core::primitive::usize,
             mut context: _Option<LogosLeaf>,
         ) -> _Option<_Result<Token, <Token as Logos<'s>>::Error>> {
@@ -133,7 +133,7 @@ impl<'s> ::logos::Logos<'s> for Token {
             _take_action!(lex, offset, context, state)
         }
         fn state1<'s>(
-            lex: &mut _Lexer<'s, Token>,
+            lex: &mut _Lexer<'s, '_, Token>,
             mut offset: ::core::primitive::usize,
             mut context: _Option<LogosLeaf>,
         ) -> _Option<_Result<Token, <Token as Logos<'s>>::Error>> {
@@ -148,7 +148,7 @@ impl<'s> ::logos::Logos<'s> for Token {
             _take_action!(lex, offset, context, state)
         }
         fn state2<'s>(
-            lex: &mut _Lexer<'s, Token>,
+            lex: &mut _Lexer<'s, '_, Token>,
             mut offset: ::core::primitive::usize,
             mut context: _Option<LogosLeaf>,
         ) -> _Option<_Result<Token, <Token as Logos<'s>>::Error>> {
@@ -161,7 +161,7 @@ impl<'s> ::logos::Logos<'s> for Token {
             _take_action!(lex, offset, context, state)
         }
         fn state3<'s>(
-            lex: &mut _Lexer<'s, Token>,
+            lex: &mut _Lexer<'s, '_, Token>,
             mut offset: ::core::primitive::usize,
             mut context: _Option<LogosLeaf>,
         ) -> _Option<_Result<Token, <Token as Logos<'s>>::Error>> {

logos-cli/tests/snapshots/tests__tailcall-nofmt.snap

@@ -2,4 +2,4 @@
 source: logos-cli/tests/tests.rs
 expression: output
 ---
-# [derive (Debug , Clone , Copy , PartialEq)] enum Token { Letter , }impl < 's > :: logos :: Logos < 's > for Token { type Error = () ; type Extras = () ; type Source = :: core :: primitive :: str ; fn lex (lex : & mut :: logos :: Lexer < 's , Self >) -> :: core :: option :: Option < :: core :: result :: Result < Self , < Self as :: logos :: Logos < 's >> :: Error >> { use :: logos :: internal :: { LexerInternal , CallbackRetVal , CallbackResult , SkipRetVal , SkipResult , } ; use :: core :: result :: Result as _Result ; use :: core :: option :: Option as _Option ; use :: logos :: Lexer as _Lexer ; use :: logos :: Logos ; macro_rules ! _fast_loop { ($ lex : ident , $ test : ident , $ offset : ident) => { 'fast_loop : { while let _Option :: Some (arr) = $ lex . read :: < & [:: core :: primitive :: u8 ; 8usize] > ($ offset) { if $ test (arr [0usize]) { $ offset += 0usize ; break 'fast_loop ; } if $ test (arr [1usize]) { $ offset += 1usize ; break 'fast_loop ; } if $ test (arr [2usize]) { $ offset += 2usize ; break 'fast_loop ; } if $ test (arr [3usize]) { $ offset += 3usize ; break 'fast_loop ; } if $ test (arr [4usize]) { $ offset += 4usize ; break 'fast_loop ; } if $ test (arr [5usize]) { $ offset += 5usize ; break 'fast_loop ; } if $ test (arr [6usize]) { $ offset += 6usize ; break 'fast_loop ; } if $ test (arr [7usize]) { $ offset += 7usize ; break 'fast_loop ; } $ offset += 8usize ; } while let _Option :: Some (byte) = $ lex . read :: < :: core :: primitive :: u8 > ($ offset) { if $ test (byte) { break 'fast_loop ; } $ offset += 1 ; } } } ; } macro_rules ! _take_action { ($ lex : ident , $ offset : ident , $ context : ident , $ state : ident) => { { let action = _get_action ($ lex , $ offset , $ context) ; match action { CallbackResult :: Emit (tok) => { return _Option :: Some (_Result :: Ok (tok)) ; } , CallbackResult :: Skip => { $ lex . trivia () ; $ offset = $ lex . offset () ; $ context = _Option :: None ; return state0 ($ lex , $ offset , $ context) ; } , CallbackResult :: Error (err) => { return _Option :: Some (_Result :: Err (err)) ; } , CallbackResult :: DefaultError => { return _Option :: Some (_Result :: Err (_make_error ($ lex))) ; } , } } } } # [inline] fn _make_error < 's > (lex : & mut _Lexer < 's , Token >) -> < Token as Logos < 's >> :: Error { << Token as Logos < 's >> :: Error as :: core :: default :: Default > :: default () } # [inline] fn _get_action < 's > (lex : & mut _Lexer < 's , Token > , offset : :: core :: primitive :: usize , context : _Option < LogosLeaf >) -> CallbackResult < 's , Token > { match context { _Option :: None => { lex . end_to_boundary (offset . max (lex . offset () + 1)) ; CallbackResult :: Error (_make_error (lex)) } , _Option :: Some (LogosLeaf :: Leaf0) => { CallbackResult :: Emit (Token :: Letter) } } } # [derive (:: core :: clone :: Clone , :: core :: marker :: Copy)] enum LogosLeaf { Leaf0 = 0isize } fn state0 < 's > (lex : & mut _Lexer < 's , Token > , mut offset : :: core :: primitive :: usize , mut context : _Option < LogosLeaf >) -> _Option < _Result < Token , < Token as Logos < 's >> :: Error >> { let other = lex . read :: < :: core :: primitive :: u8 > (offset) ; if let _Option :: Some (byte) = other { if (byte == b'a') { offset += 1 ; return state3 (lex , offset , context) ; } } else { if lex . offset () == offset { return _Option :: None } } _take_action ! (lex , offset , context , state) } fn state1 < 's > (lex : & mut _Lexer < 's , Token > , mut offset : :: core :: primitive :: usize , mut context : _Option < LogosLeaf >) -> _Option < _Result < Token , < Token as Logos < 's >> :: Error >> { let other = lex . read :: < :: core :: primitive :: u8 > (offset) ; if let _Option :: Some (byte) = other { if (byte == b'z') { offset += 1 ; return state2 (lex , offset , context) ; } } else { } _take_action ! (lex , offset , context , state) } fn state2 < 's > (lex : & mut _Lexer < 's , Token > , mut offset : :: core :: primitive :: usize , mut context : _Option < LogosLeaf >) -> _Option < _Result < Token , < Token as Logos < 's >> :: Error >> { lex . end (offset) ; context = _Option :: Some (LogosLeaf :: Leaf0) ; let other = lex . read :: < :: core :: primitive :: u8 > (offset) ; if let _Option :: Some (byte) = other { } else { } _take_action ! (lex , offset , context , state) } fn state3 < 's > (lex : & mut _Lexer < 's , Token > , mut offset : :: core :: primitive :: usize , mut context : _Option < LogosLeaf >) -> _Option < _Result < Token , < Token as Logos < 's >> :: Error >> { let other = lex . read :: < :: core :: primitive :: u8 > (offset) ; if let _Option :: Some (byte) = other { if (byte == b'-') { offset += 1 ; return state1 (lex , offset , context) ; } } else { } _take_action ! (lex , offset , context , state) } state0 (lex , lex . offset () , _Option :: None) } }
+# [derive (Debug , Clone , Copy , PartialEq)] enum Token { Letter , }impl < 's > :: logos :: Logos < 's > for Token { type Error = () ; type Extras < 'extras > = () ; type Source = :: core :: primitive :: str ; fn lex (lex : & mut :: logos :: Lexer < 's , '_ , Self >) -> :: core :: option :: Option < :: core :: result :: Result < Self , < Self as :: logos :: Logos < 's >> :: Error >> { use :: logos :: internal :: { LexerInternal , CallbackRetVal , CallbackResult , SkipRetVal , SkipResult , } ; use :: core :: result :: Result as _Result ; use :: core :: option :: Option as _Option ; use :: logos :: Lexer as _Lexer ; use :: logos :: Logos ; macro_rules ! _fast_loop { ($ lex : ident , $ test : ident , $ offset : ident) => { 'fast_loop : { while let _Option :: Some (arr) = $ lex . read :: < & [:: core :: primitive :: u8 ; 8usize] > ($ offset) { if $ test (arr [0usize]) { $ offset += 0usize ; break 'fast_loop ; } if $ test (arr [1usize]) { $ offset += 1usize ; break 'fast_loop ; } if $ test (arr [2usize]) { $ offset += 2usize ; break 'fast_loop ; } if $ test (arr [3usize]) { $ offset += 3usize ; break 'fast_loop ; } if $ test (arr [4usize]) { $ offset += 4usize ; break 'fast_loop ; } if $ test (arr [5usize]) { $ offset += 5usize ; break 'fast_loop ; } if $ test (arr [6usize]) { $ offset += 6usize ; break 'fast_loop ; } if $ test (arr [7usize]) { $ offset += 7usize ; break 'fast_loop ; } $ offset += 8usize ; } while let _Option :: Some (byte) = $ lex . read :: < :: core :: primitive :: u8 > ($ offset) { if $ test (byte) { break 'fast_loop ; } $ offset += 1 ; } } } ; } macro_rules ! _take_action { ($ lex : ident , $ offset : ident , $ context : ident , $ state : ident) => { { let action = _get_action ($ lex , $ offset , $ context) ; match action { CallbackResult :: Emit (tok) => { return _Option :: Some (_Result :: Ok (tok)) ; } , CallbackResult :: Skip => { $ lex . trivia () ; $ offset = $ lex . offset () ; $ context = _Option :: None ; return state0 ($ lex , $ offset , $ context) ; } , CallbackResult :: Error (err) => { return _Option :: Some (_Result :: Err (err)) ; } , CallbackResult :: DefaultError => { return _Option :: Some (_Result :: Err (_make_error ($ lex))) ; } , } } } } # [inline] fn _make_error < 's > (lex : & mut _Lexer < 's , '_ , Token >) -> < Token as Logos < 's >> :: Error { << Token as Logos < 's >> :: Error as :: core :: default :: Default > :: default () } # [inline] fn _get_action < 's > (lex : & mut _Lexer < 's , '_ , Token > , offset : :: core :: primitive :: usize , context : _Option < LogosLeaf >) -> CallbackResult < 's , Token > { match context { _Option :: None => { lex . end_to_boundary (offset . max (lex . offset () + 1)) ; CallbackResult :: Error (_make_error (lex)) } , _Option :: Some (LogosLeaf :: Leaf0) => { CallbackResult :: Emit (Token :: Letter) } } } # [derive (:: core :: clone :: Clone , :: core :: marker :: Copy)] enum LogosLeaf { Leaf0 = 0isize } fn state0 < 's > (lex : & mut _Lexer < 's , '_ , Token > , mut offset : :: core :: primitive :: usize , mut context : _Option < LogosLeaf >) -> _Option < _Result < Token , < Token as Logos < 's >> :: Error >> { let other = lex . read :: < :: core :: primitive :: u8 > (offset) ; if let _Option :: Some (byte) = other { if (byte == b'a') { offset += 1 ; return state3 (lex , offset , context) ; } } else { if lex . offset () == offset { return _Option :: None } } _take_action ! (lex , offset , context , state) } fn state1 < 's > (lex : & mut _Lexer < 's , '_ , Token > , mut offset : :: core :: primitive :: usize , mut context : _Option < LogosLeaf >) -> _Option < _Result < Token , < Token as Logos < 's >> :: Error >> { let other = lex . read :: < :: core :: primitive :: u8 > (offset) ; if let _Option :: Some (byte) = other { if (byte == b'z') { offset += 1 ; return state2 (lex , offset , context) ; } } else { } _take_action ! (lex , offset , context , state) } fn state2 < 's > (lex : & mut _Lexer < 's , '_ , Token > , mut offset : :: core :: primitive :: usize , mut context : _Option < LogosLeaf >) -> _Option < _Result < Token , < Token as Logos < 's >> :: Error >> { lex . end (offset) ; context = _Option :: Some (LogosLeaf :: Leaf0) ; let other = lex . read :: < :: core :: primitive :: u8 > (offset) ; if let _Option :: Some (byte) = other { } else { } _take_action ! (lex , offset , context , state) } fn state3 < 's > (lex : & mut _Lexer < 's , '_ , Token > , mut offset : :: core :: primitive :: usize , mut context : _Option < LogosLeaf >) -> _Option < _Result < Token , < Token as Logos < 's >> :: Error >> { let other = lex . read :: < :: core :: primitive :: u8 > (offset) ; if let _Option :: Some (byte) = other { if (byte == b'-') { offset += 1 ; return state1 (lex , offset , context) ; } } else { } _take_action ! (lex , offset , context , state) } state0 (lex , lex . offset () , _Option :: None) } }

logos-codegen/src/generator/mod.rs

@@ -4,7 +4,7 @@ use fast_loop::fast_loop_macro;
 use fnv::FnvHashMap as Map;
 use proc_macro2::TokenStream;
 use quote::quote;
-use syn::Ident;
+use syn::{Ident, Lifetime};
 
 use crate::graph::{ByteClass, Graph, State, StateType};
 use crate::leaf::{Callback, InlineCallback};
@@ -26,7 +26,7 @@ pub struct Generator<'a> {
     /// Name of the type with any generics it might need
     this: &'a TokenStream,
     /// Lifetime of the lexer source
-    source_lifetime: &'a TokenStream,
+    source_lifetime: &'a Lifetime,
     /// All lifetime bounds needed to create nested items
     lifetime_bounds: &'a TokenStream,
     /// Reference to the graph with all the nodes
@@ -47,7 +47,8 @@ impl<'a> Generator<'a> {
         config: Config,
         name: &'a Ident,
         this: &'a TokenStream,
-        source_lifetime: &'a TokenStream,
+        source_lifetime: &'a Lifetime,
+        _extras_lifetime: &'a Lifetime,
         lifetime_bounds: &'a TokenStream,
         graph: &'a Graph,
         error_callback: &'a Option<Callback>,
@@ -216,11 +217,11 @@ impl<'a> Generator<'a> {
 
         quote! {
             #[inline]
-            fn _make_error #lt_bounds (lex: &mut _Lexer<#src_lt, #this>) -> <#this as Logos<#src_lt>>::Error {
+            fn _make_error #lt_bounds (lex: &mut _Lexer<#src_lt, '_, #this>) -> <#this as Logos<#src_lt>>::Error {
                 #error_body
             }
             #[inline]
-            fn _get_action #lt_bounds (lex: &mut _Lexer<#src_lt, #this>, offset: ::core::primitive::usize, context: _Option<LogosLeaf>)
+            fn _get_action #lt_bounds (lex: &mut _Lexer<#src_lt, '_, #this>, offset: ::core::primitive::usize, context: _Option<LogosLeaf>)
                 -> CallbackResult<#src_lt, #this>
             {
                 match context {
@@ -314,7 +315,7 @@ impl<'a> Generator<'a> {
             let src_lt = self.source_lifetime;
             let lt_bounds = self.lifetime_bounds;
             quote! {
-                fn #this_ident #lt_bounds (lex: &mut _Lexer<#src_lt, #this>, mut offset: ::core::primitive::usize, mut context: _Option<LogosLeaf>)
+                fn #this_ident #lt_bounds (lex: &mut _Lexer<#src_lt, '_, #this>, mut offset: ::core::primitive::usize, mut context: _Option<LogosLeaf>)
                     -> _Option<_Result<#this, <#this as Logos<#src_lt>>::Error>> {
                     #fast_loop
                     #setup

logos-codegen/src/lib.rs

@@ -33,10 +33,10 @@ use parser::Parser;
 use pattern::Pattern;
 use quote::ToTokens;
 
-use proc_macro2::{TokenStream, TokenTree};
+use proc_macro2::{Span, TokenStream, TokenTree};
 use quote::quote;
 use syn::spanned::Spanned;
-use syn::{parse_quote, LitBool};
+use syn::{parse_quote, Lifetime, LitBool};
 use syn::{Fields, ItemEnum};
 
 use crate::graph::Config;
@@ -287,17 +287,18 @@ pub fn generate(input: TokenStream) -> TokenStream {
     let this = quote!(#name #generics);
     let lt_bounds = parser.lifetime_bounds();
     let src_lt = parser.source_lifetime();
+    let ext_lt = Lifetime::new("'extras", Span::mixed_site());
 
     let impl_logos = |body| {
         quote! {
             impl #lt_bounds #logos_path::Logos<#src_lt> for #this {
                 type Error = #error_type;
 
-                type Extras = #extras;
+                type Extras<#ext_lt> = #extras;
 
                 type Source = #source;
 
-                fn lex(lex: &mut #logos_path::Lexer<#src_lt, Self>)
+                fn lex(lex: &mut #logos_path::Lexer<#src_lt, '_, Self>)
                     -> ::core::option::Option<::core::result::Result<Self, <Self as #logos_path::Logos<#src_lt>>::Error>> {
                     #body
                 }
@@ -404,6 +405,7 @@ pub fn generate(input: TokenStream) -> TokenStream {
         name,
         &this,
         &src_lt,
+        &ext_lt,
         &lt_bounds,
         &graph,
         &error_callback,