parser revamp adding peek_* fns

Author: ModProg

.github/workflows/test.yaml

@@ -18,12 +18,6 @@ jobs:
         rust:
           - stable
           - nightly
-        features:
-          - ''
-          - --no-default-features
-          - --no-default-features --features proc-macro
-          - --no-default-features --features proc-macro2
-          - --no-default-features --features parser
         include:
           - rust: nightly
             cargo_flags: -Z minimal-versions
@@ -42,11 +36,14 @@ jobs:
       - uses: hecrj/setup-rust-action@v1
         with:
           rust-version: ${{ matrix.rust }}
+      - uses: bruxisma/setup-cargo-hack@v1
+        with:
+          cargo-hack-version: "0.5"
       - name: Build
-        run: cargo build ${{ matrix.features }} ${{ matrix.cargo_flags }}
+        run: cargo hack build --feature-powerset ${{ matrix.cargo_flags }}
       - name: Test
-        run: cargo test ${{ matrix.features }} --all-targets --no-fail-fast --workspace
+        run: cargo test --feature-powerset --all-targets --no-fail-fast --workspace
       - name: Doc Test
-        run: cargo test ${{ matrix.features }} --doc --no-fail-fast --workspace
+        run: cargo test --feature-powerset --doc --no-fail-fast --workspace
       - name: Build Docs
-        run: cargo doc ${{ matrix.features }} --workspace --no-deps
+        run: cargo doc --feature-powerset --workspace --no-deps

CHANGELOG.md

@@ -4,23 +4,32 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-<!-- ## [Unreleased] -->
+## [Unreleased]
+### Added
+- `quote::ToTokens` implementation for `TokenParser` (`quote` is a new default feature)
+- `peek_{token}` and `peek_n_{token}` to `TokenParser`
+
+### Changed
+- **Breaking Change** added const generic buffer size to `TokenParser`
+- **Breaking Change** `Peeker::peek` takes `&[TokenTree]` instead of `TokenParser`
+- `TokenParser` peeking supports `n` greater than stack buffer, allowing spilling to heap
+- increased default `TokenParser` peek buffer to `6`
 
 ## [0.6.0] - 2023-04-29
-- `Parser::next_keyword(v)`
+- `TokenParser::next_keyword(v)`
 
 ## [0.5.2] - 2023-04-06
 ### Fixed
-- `Parser::peek_n()` always returned `None`
-- `Parser::next_{token}` did not work correctly because of `peek_n`
+- `TokenParser::peek_n()` always returned `None`
+- `TokenParser::next_{token}` did not work correctly because of `peek_n`
 
 ## [0.5.1] - 2023-03-02
 ### Fixed
-- `Parser::next_string()` did not consume token
+- `TokenParser::next_string()` did not consume token
 
 ## [0.5.0] - 2023-03-01
 ### Added
-- Parser for parsing simple rust structures
+- `TokenParser` for parsing simple rust structures
 - `assert_tokens!` macro
 - `TokenTreeExt::into_<token>()` functions 
 - `Delimited` trait for accessing `TokenTree::Group`

Cargo.toml

@@ -14,13 +14,13 @@ include = ["src/**/*", "LICENSE-*", "README.md", "CHANGELOG.md"]
 
 [dependencies]
 proc-macro2 = { version = "1", optional = true }
-smallvec = { version = "1", optional = true }
+quote = { version = "1", optional = true }
+smallvec = { version = "1.4", optional = true, features = ["const_generics"] }
 
 [features]
-default = ["proc-macro2", "proc-macro", "parser"]
+default = ["proc-macro2", "proc-macro", "parser", "quote"]
 proc-macro = []
 parser = ["smallvec", "proc-macro2"]
-proc-macro2 = ["dep:proc-macro2"]
 
 [dev-dependencies]
 quote = "1"

src/assert.rs

@@ -22,7 +22,7 @@ macro_rules! assert_tokens {
 #[allow(clippy::module_name_repetitions)]
 macro_rules! assert_tokens {
     ($lhs:expr, {$($rhs:tt)*}) => {
-        let mut lhs = $crate::TokenParser::new($lhs);
+        let mut lhs = $crate::TokenParser::new_generic::<3, _, _>($lhs);
         assert_tokens!(@O lhs, "", $($rhs)*);
     };
     (@E $prefix:expr, $expected:tt, $found:tt) => {
@@ -33,7 +33,7 @@ macro_rules! assert_tokens {
     };
     (@G $lhs:ident, $fn:ident, $aggr:expr, $sym:literal, $group:tt, {$($inner:tt)*}, $($rhs:tt)*) => {
         if let Some(lhs) = $lhs.$fn() {
-            let mut lhs = $crate::TokenParser::from(lhs);
+            let mut lhs = $crate::TokenParser::<_, 3>::from(lhs);
             assert_tokens!(@O lhs, concat!($aggr, ' ', $sym), $($inner)*);
         } else if let Some(lhs) = $lhs.next() {
             assert_tokens!(@E $aggr, ($group), lhs);

src/lib.rs

@@ -2,8 +2,7 @@
 //!
 //! E.g. [pushing tokens onto `TokenStream`](TokenStreamExt::push) and [testing
 //! for specific punctuation on `TokenTree` and Punct](TokenTreePunct)
-#![warn(clippy::pedantic)]
-#![deny(missing_docs)]
+#![warn(clippy::pedantic, missing_docs)]
 #![cfg_attr(docsrs, feature(doc_auto_cfg))]
 
 #[cfg(feature = "proc-macro")]
@@ -36,18 +35,18 @@ macro_rules! attr {
 }
 
 macro_rules! trait_def {
-    ($item_attr:tt, $trait:ident, $($fn_attr:tt, $fn:ident, $args:tt, $($ret:ty)?),*) => {
+    ($item_attr:tt, $trait:ident, $($fn_attr:tt, $fn:ident, $({$($gen:tt)*})?, $args:tt, $($ret:ty)?),*) => {
         attr!($item_attr,
         pub trait $trait {
-            $(attr!($fn_attr, fn $fn $args $(-> $ret)?;);)*
+            $(attr!($fn_attr, fn $fn $($($gen)*)? $args $(-> $ret)?;);)*
         });
     };
 }
 
 macro_rules! trait_impl {
-    ($trait:ident, $type:ident, $($fn_attr:tt, $fn:ident, $args:tt, $($ret:ty)?, $stmts:tt),*) => {
+    ($trait:ident, $type:ident, $($fn_attr:tt, $fn:ident, $({$($gen:tt)*})?, $args:tt, $($ret:ty)?, $stmts:tt),*) => {
         impl $trait for $type {
-            $(attr!($fn_attr, fn $fn $args $(-> $ret)? $stmts);)*
+            $(attr!($fn_attr, fn $fn $($($gen)*)? $args $(-> $ret)? $stmts);)*
         }
     };
 }
@@ -60,16 +59,16 @@ macro_rules! impl_via_trait {
             fn $fn:ident $args:tt $(-> $ret:ty)? { $($stmts:tt)* })*
         }
     )+) => {
-        once!($((trait_def!(($($trait_attr)*), $trait, $(($($fn_attr)*), $fn, $args, $($ret)?),*);))+);
+        once!($((trait_def!(($($trait_attr)*), $trait, $(($($fn_attr)*), $fn,, $args, $($ret)?),*);))+);
         #[cfg(feature = "proc-macro")]
         const _: () = {
             use proc_macro::*;
-            $(trait_impl!($trait, $type, $(($($fn_attr)*), $fn, $args, $($ret)?, {$($stmts)*}),*);)+
+            $(trait_impl!($trait, $type, $(($($fn_attr)*), $fn,, $args, $($ret)?, {$($stmts)*}),*);)+
         };
         #[cfg(feature = "proc-macro2")]
         const _:() = {
             use proc_macro2::*;
-            $(trait_impl!($trait, $type, $(($($fn_attr)*), $fn, $args, $($ret)?, {$($stmts)*}),*);)+
+            $(trait_impl!($trait, $type, $(($($fn_attr)*), $fn,, $args, $($ret)?, {$($stmts)*}),*);)+
         };
     };
     (
@@ -78,7 +77,7 @@ macro_rules! impl_via_trait {
                 $(#$trait_attr:tt)*
                 impl $trait:ident$($doc:literal)?, $trait2:ident$($doc2:literal)?  for $type:ident {
                     $($(#$fn_attr:tt)*
-                    fn $fn:ident $args:tt $(-> $ret:ty)? { $($stmts:tt)* })*
+                    fn $fn:ident $({$($gen:tt)*})? ($($args:tt)*) $(-> $ret:ty)? { $($stmts:tt)* })*
                 }
             )+
         }
@@ -88,16 +87,16 @@ macro_rules! impl_via_trait {
         #[cfg(feature = "proc-macro")]
         mod $mod {
             use proc_macro::*;
-            once!($((trait_def!(($($trait_attr)* $([doc=$doc])?), $trait, $(($($fn_attr)*), $fn, $args, $($ret)?),*);))+);
-            $(trait_impl!($trait, $type, $(($($fn_attr)*), $fn, $args, $($ret)?, {$($stmts)*}),*);)+
+            once!($((trait_def!(($($trait_attr)* $([doc=$doc])?), $trait, $(($($fn_attr)*), $fn, $({$($gen)*})?, ($($args)*), $($ret)?),*);))+);
+            $(trait_impl!($trait, $type, $(($($fn_attr)*), $fn, $({$($gen)*})?, ($($args)*), $($ret)?, {$($stmts)*}),*);)+
         }
         #[cfg(feature = "proc-macro2")]
         once!(($(pub use $mod2::$trait2;)+));
         #[cfg(feature = "proc-macro2")]
         mod $mod2 {
             use proc_macro2::*;
-            once!($((trait_def!(($($trait_attr)*$([doc=$doc2])?), $trait2, $(($($fn_attr)*), $fn, $args, $($ret)?),*);))+);
-            $(trait_impl!($trait2, $type, $(($($fn_attr)*), $fn, $args, $($ret)?, {$($stmts)*}),*);)+
+            once!($((trait_def!(($($trait_attr)*$([doc=$doc2])?), $trait2, $(($($fn_attr)*), $fn, $({$($gen)*})?, ($($args)*), $($ret)?),*);))+);
+            $(trait_impl!($trait2, $type, $(($($fn_attr)*), $fn, $({$($gen)*})?, ($($args)*), $($ret)?, {$($stmts)*}),*);)+
         }
     };
 }
@@ -106,16 +105,25 @@ impl_via_trait! {
     mod token_stream_ext, token_stream2_ext {
         /// Generic extensions for
         impl TokenStreamExt "[`proc_macro::TokenStream`]", TokenStream2Ext "[`proc_macro2::TokenStream`]" for TokenStream {
-            /// Pushes a single [`TokenTree`] onto the token stream
+            /// Pushes a single [`TokenTree`] onto the token stream.
             fn push(&mut self, token: TokenTree) {
                 self.extend(std::iter::once(token))
             }
-            /// Creates a [`TokenParser`](crate::TokenParser) from this token stream
+            /// Creates a [`TokenParser`](crate::TokenParser) from this token stream.
             #[cfg(feature = "parser")]
             fn parser(self) -> crate::TokenParser<proc_macro2::token_stream::IntoIter> {
                 #[allow(clippy::useless_conversion)]
                 proc_macro2::TokenStream::from(self).into()
             }
+
+            /// Creates a [`TokenParser`](crate::TokenParser) from this token stream.
+            ///
+            /// Allows to specify the length of the [peeker buffer](crate::TokenParser#peeking).
+            #[cfg(feature = "parser")]
+            fn parser_generic{<const PEEKER_LEN: usize>}(self) -> crate::TokenParser<proc_macro2::token_stream::IntoIter, PEEKER_LEN> {
+                #[allow(clippy::useless_conversion)]
+                proc_macro2::TokenStream::from(self).into()
+            }
         }
     }
 }