Skip to content

Commit 7d0bcd6

Browse files
ModProgModProg
committed
TokenTreeLiteral
1 parent bddc0ed commit 7d0bcd6

File tree

3 files changed

+135
-90
lines changed

3 files changed

+135
-90
lines changed

‎CHANGELOG.md‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
66

77
## [Unreleased]
88
### Added
9-
- `*_tt` functions to `TokenParser`
10-
- `next_macro_rules_tt()` to `TokenParser`
9+
- `*_tt` functions to `TokenParser`.
10+
- `next_macro_rules_tt()` to `TokenParser`.
11+
- `TokenTreeLiteral` with `is_string()` and `string()`.
12+
13+
### Changed
14+
- **Breaking Change** Sealed extension traits.
1115

1216
### Fixed
13-
- `assert_expansion!` was failing on nightly
17+
- `assert_expansion!` was failing on nightly.
1418

1519
## [0.8.0] - 2023-05-14
1620
### Fixed

‎src/lib.rs‎

Lines changed: 124 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
#![cfg_attr(docsrs, feature(doc_auto_cfg))]
1010
#![deny(rustdoc::all)]
1111

12+
use std::num::ParseIntError;
13+
use std::str::FromStr;
14+
1215
#[cfg(doc)]
1316
use proc_macro2::{Punct, Spacing};
1417

@@ -29,6 +32,19 @@ mod assert;
2932
#[doc(hidden)]
3033
pub mod __private;
3134

35+
mod sealed {
36+
pub trait Sealed {}
37+
38+
macro_rules! sealed {
39+
[$($ty:ident),* $(,)?] => {$(
40+
impl Sealed for proc_macro::$ty {}
41+
impl Sealed for proc_macro2::$ty {}
42+
)*};
43+
}
44+
45+
sealed![TokenStream, TokenTree, Punct, Literal, Group];
46+
}
47+
3248
macro_rules! once {
3349
(($($tts:tt)*) $($tail:tt)*) => {
3450
$($tts)*
@@ -44,7 +60,7 @@ macro_rules! attr {
4460
macro_rules! trait_def {
4561
($item_attr:tt, $trait:ident, $($fn_attr:tt, $fn:ident, $({$($gen:tt)*})?, $args:tt, $($ret:ty)?),*) => {
4662
attr!($item_attr,
47-
pub trait $trait {
63+
pub trait $trait: crate::sealed::Sealed {
4864
$(attr!($fn_attr, fn $fn $($($gen)*)? $args $(-> $ret)?;);)*
4965
});
5066
};
@@ -63,19 +79,19 @@ macro_rules! impl_via_trait {
6379
$(#$trait_attr:tt)*
6480
impl $trait:ident for $type:ident {
6581
$($(#$fn_attr:tt)*
66-
fn $fn:ident $args:tt $(-> $ret:ty)? { $($stmts:tt)* })*
82+
fn $fn:ident $({$($gen:tt)*})? ($($args:tt)*) $(-> $ret:ty)? { $($stmts:tt)* })*
6783
}
6884
)+) => {
69-
once!($((trait_def!(($($trait_attr)*), $trait, $(($($fn_attr)*), $fn,, $args, $($ret)?),*);))+);
85+
once!($((trait_def!(($($trait_attr)*), $trait, $(($($fn_attr)*), $fn,$({$($gen)*})?, ($($args)*), $($ret)?),*);))+);
7086
#[cfg(feature = "proc-macro")]
7187
const _: () = {
7288
use proc_macro::*;
73-
$(trait_impl!($trait, $type, $(($($fn_attr)*), $fn,, $args, $($ret)?, {$($stmts)*}),*);)+
89+
$(trait_impl!($trait, $type, $(($($fn_attr)*), $fn, $({$($gen)*})?, ($($args)*), $($ret)?, {$($stmts)*}),*);)+
7490
};
7591
#[cfg(feature = "proc-macro2")]
7692
const _:() = {
7793
use proc_macro2::*;
78-
$(trait_impl!($trait, $type, $(($($fn_attr)*), $fn,, $args, $($ret)?, {$($stmts)*}),*);)+
94+
$(trait_impl!($trait, $type, $(($($fn_attr)*), $fn, $({$($gen)*})?, ($($args)*), $($ret)?, {$($stmts)*}),*);)+
7995
};
8096
};
8197
(
@@ -287,6 +303,109 @@ delimited![
287303
None as is_implicitly_delimited: " no delimiters (`Ø ... Ø`)"
288304
];
289305

306+
impl_via_trait! {
307+
/// Trait to parse literals
308+
impl TokenTreeLiteral for TokenTree {
309+
/// Tests if the token is a string literal.
310+
#[must_use]
311+
fn is_string(&self) -> bool {
312+
self.literal().is_some_and(TokenTreeLiteral::is_string)
313+
}
314+
315+
/// Returns the string contents if it is a string literal.
316+
#[must_use]
317+
fn string(&self) -> Option<String> {
318+
self.literal().and_then(TokenTreeLiteral::string)
319+
}
320+
}
321+
322+
impl TokenTreeLiteral for Literal {
323+
fn is_string(&self) -> bool {
324+
let s = self.to_string();
325+
s.starts_with('"') || s.starts_with("r\"") || s.starts_with("r#")
326+
}
327+
fn string(&self) -> Option<String> {
328+
let lit = self.to_string();
329+
if lit.starts_with('"') {
330+
Some(resolve_escapes(&lit[1..lit.len() - 1]))
331+
} else if lit.starts_with('r') {
332+
let pounds = lit.chars().skip(1).take_while(|&c| c == '#').count();
333+
Some(lit[2 + pounds..lit.len() - pounds - 1].to_owned())
334+
} else {
335+
None
336+
}
337+
}
338+
}
339+
}
340+
341+
// Implemented following https://doc.rust-lang.org/reference/tokens.html#string-literals
342+
// #[allow(clippy::needless_continue)]
343+
fn resolve_escapes(mut s: &str) -> String {
344+
let mut out = String::new();
345+
while !s.is_empty() {
346+
if s.starts_with('\\') {
347+
match s.as_bytes()[1] {
348+
b'x' => {
349+
out.push(
350+
char::from_u32(u32::from_str_radix(&s[2..=3], 16).expect("valid escape"))
351+
.expect("valid escape"),
352+
);
353+
s = &s[4..];
354+
}
355+
b'u' => {
356+
let len = s[3..].find('}').expect("valid escape");
357+
out.push(
358+
char::from_u32(u32::from_str_radix(&s[3..len], 16).expect("valid escape"))
359+
.expect("valid escape"),
360+
);
361+
s = &s[3 + len..];
362+
}
363+
b'n' => {
364+
out.push('\n');
365+
s = &s[2..];
366+
}
367+
b'r' => {
368+
out.push('\r');
369+
s = &s[2..];
370+
}
371+
b't' => {
372+
out.push('\t');
373+
s = &s[2..];
374+
}
375+
b'\\' => {
376+
out.push('\\');
377+
s = &s[2..];
378+
}
379+
b'0' => {
380+
out.push('\0');
381+
s = &s[2..];
382+
}
383+
b'\'' => {
384+
out.push('\'');
385+
s = &s[2..];
386+
}
387+
b'"' => {
388+
out.push('"');
389+
s = &s[2..];
390+
}
391+
b'\n' => {
392+
s = &s[..s[2..]
393+
.find(|c: char| !c.is_ascii_whitespace())
394+
.unwrap_or(s.len())];
395+
}
396+
c => unreachable!(
397+
"TokenStream string literals should only contain valid escapes, found `\\{c}`"
398+
),
399+
}
400+
} else {
401+
let len = s.find('\\').unwrap_or(s.len());
402+
out.push_str(&s[..len]);
403+
s = &s[len..];
404+
}
405+
}
406+
out
407+
}
408+
290409
#[cfg(all(test, feature = "proc-macro2"))]
291410
mod test {
292411
use proc_macro2::{Punct, Spacing, TokenTree};

‎src/parser.rs‎

Lines changed: 4 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use proc_macro2::Spacing;
77
use proc_macro2::{token_stream, Group, Ident, Literal, Punct, TokenStream, TokenTree};
88
use smallvec::{smallvec, SmallVec};
99

10-
use crate::{Delimited, TokenStream2Ext, TokenTree2Ext, TokenTreePunct};
10+
use crate::{Delimited, TokenStream2Ext, TokenTree2Ext, TokenTreeLiteral, TokenTreePunct};
1111

1212
// TODO move implementation in a trait implemented on both
1313
// Peekable<token_stream::IntoIter>s
@@ -949,20 +949,9 @@ where
949949
/// Returns the next string literal
950950
#[must_use]
951951
pub fn next_string(&mut self) -> Option<String> {
952-
if !self.peek()?.is_literal() {
953-
return None;
954-
}
955-
let lit = self.peek()?.to_string();
956-
if lit.starts_with('"') {
957-
self.next();
958-
Some(resolve_escapes(&lit[1..lit.len() - 1]))
959-
} else if lit.starts_with('r') {
960-
self.next();
961-
let pounds = lit.chars().skip(1).take_while(|&c| c == '#').count();
962-
Some(lit[2 + pounds..lit.len() - pounds - 1].to_owned())
963-
} else {
964-
None
965-
}
952+
let lit = self.peek().and_then(TokenTreeLiteral::string)?;
953+
self.next();
954+
Some(lit)
966955
}
967956

968957
/// Returns the next boolean literal
@@ -975,73 +964,6 @@ where
975964
.map(|t| matches!(t.ident(), Some(ident) if ident == "true"))
976965
}
977966
}
978-
// Implemented following https://doc.rust-lang.org/reference/tokens.html#string-literals
979-
// #[allow(clippy::needless_continue)]
980-
fn resolve_escapes(mut s: &str) -> String {
981-
let mut out = String::new();
982-
while !s.is_empty() {
983-
if s.starts_with('\\') {
984-
match s.as_bytes()[1] {
985-
b'x' => {
986-
out.push(
987-
char::from_u32(u32::from_str_radix(&s[2..=3], 16).expect("valid escape"))
988-
.expect("valid escape"),
989-
);
990-
s = &s[4..];
991-
}
992-
b'u' => {
993-
let len = s[3..].find('}').expect("valid escape");
994-
out.push(
995-
char::from_u32(u32::from_str_radix(&s[3..len], 16).expect("valid escape"))
996-
.expect("valid escape"),
997-
);
998-
s = &s[3 + len..];
999-
}
1000-
b'n' => {
1001-
out.push('\n');
1002-
s = &s[2..];
1003-
}
1004-
b'r' => {
1005-
out.push('\r');
1006-
s = &s[2..];
1007-
}
1008-
b't' => {
1009-
out.push('\t');
1010-
s = &s[2..];
1011-
}
1012-
b'\\' => {
1013-
out.push('\\');
1014-
s = &s[2..];
1015-
}
1016-
b'0' => {
1017-
out.push('\0');
1018-
s = &s[2..];
1019-
}
1020-
b'\'' => {
1021-
out.push('\'');
1022-
s = &s[2..];
1023-
}
1024-
b'"' => {
1025-
out.push('"');
1026-
s = &s[2..];
1027-
}
1028-
b'\n' => {
1029-
s = &s[..s[2..]
1030-
.find(|c: char| !c.is_ascii_whitespace())
1031-
.unwrap_or(s.len())];
1032-
}
1033-
c => unreachable!(
1034-
"TokenStream string literals should only contain valid escapes, found `\\{c}`"
1035-
),
1036-
}
1037-
} else {
1038-
let len = s.find('\\').unwrap_or(s.len());
1039-
out.push_str(&s[..len]);
1040-
s = &s[len..];
1041-
}
1042-
}
1043-
out
1044-
}
1045967

1046968
impl<I, const PEEKER_LEN: usize> TokenParser<I, PEEKER_LEN>
1047969
where

0 commit comments

Comments
 (0)