From 900f0e2ec3657f5d4555632b475fd49b585af466 Mon Sep 17 00:00:00 2001 From: Tomoaki Kobayashi Date: Tue, 21 Oct 2025 21:47:58 +0900 Subject: [PATCH 1/4] Fix parsing logic in `proc_macro::quote` --- library/proc_macro/src/quote.rs | 459 +++++++++++++++---- tests/ui/proc-macro/quote/auxiliary/basic.rs | 20 + 2 files changed, 402 insertions(+), 77 deletions(-) diff --git a/library/proc_macro/src/quote.rs b/library/proc_macro/src/quote.rs index dbb55cd9fb300..fb05021735624 100644 --- a/library/proc_macro/src/quote.rs +++ b/library/proc_macro/src/quote.rs @@ -4,6 +4,8 @@ //! This quasiquoter uses macros 2.0 hygiene to reliably access //! items from `proc_macro`, to build a `proc_macro::TokenStream`. +use core::iter::Peekable; + use crate::{ BitOr, Delimiter, Group, Ident, Literal, Punct, Spacing, Span, ToTokens, TokenStream, TokenTree, }; @@ -284,88 +286,16 @@ pub fn quote(stream: TokenStream) -> TokenStream { let mut after_dollar = false; let mut tokens = crate::TokenStream::new(); - let mut iter = stream.into_iter().peekable(); + let mut iter: Peekable = stream.into_iter().peekable(); while let Some(tree) = iter.next() { if after_dollar { after_dollar = false; match tree { - TokenTree::Group(tt) => { + TokenTree::Group(ref tt) => { // Handles repetition by expanding `$( CONTENTS ) SEP_OPT *` to `{ REP_EXPANDED }`. let contents = tt.stream(); - - // The `*` token is also consumed here. - let sep_opt: Option = match (iter.next(), iter.peek()) { - (Some(TokenTree::Punct(sep)), Some(TokenTree::Punct(star))) - if sep.spacing() == Spacing::Joint && star.as_char() == '*' => - { - iter.next(); - Some(sep) - } - (Some(TokenTree::Punct(star)), _) if star.as_char() == '*' => None, - _ => panic!("`$(...)` must be followed by `*` in `quote!`"), - }; - - let mut rep_expanded = TokenStream::new(); - - // Append setup code for a `while`, where recursively quoted `CONTENTS` - // and `SEP_OPT` are repeatedly processed, to `REP_EXPANDED`. - let meta_vars = collect_meta_vars(contents.clone()); - minimal_quote!( - use crate::ext::*; - (@ if sep_opt.is_some() { - minimal_quote!(let mut _i = 0usize;) - } else { - minimal_quote!(();) - }) - let has_iter = crate::ThereIsNoIteratorInRepetition; - ) - .to_tokens(&mut rep_expanded); - for meta_var in &meta_vars { - minimal_quote!( - #[allow(unused_mut)] - let (mut (@ meta_var), i) = (@ meta_var).quote_into_iter(); - let has_iter = has_iter | i; - ) - .to_tokens(&mut rep_expanded); - } - minimal_quote!(let _: crate::HasIterator = has_iter;) - .to_tokens(&mut rep_expanded); - - // Append the `while` to `REP_EXPANDED`. - let mut while_body = TokenStream::new(); - for meta_var in &meta_vars { - minimal_quote!( - let (@ meta_var) = match (@ meta_var).next() { - Some(_x) => crate::RepInterp(_x), - None => break, - }; - ) - .to_tokens(&mut while_body); - } - minimal_quote!( - (@ if let Some(sep) = sep_opt { - minimal_quote!( - if _i > 0 { - (@ minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new( - (@ TokenTree::from(Literal::character(sep.as_char()))), - (@ minimal_quote!(crate::Spacing::Alone)), - )), &mut ts);)) - } - _i += 1; - ) - } else { - minimal_quote!(();) - }) - (@ quote(contents.clone())).to_tokens(&mut ts); - ) - .to_tokens(&mut while_body); - rep_expanded.extend(vec![ - TokenTree::Ident(Ident::new("while", Span::call_site())), - TokenTree::Ident(Ident::new("true", Span::call_site())), - TokenTree::Group(Group::new(Delimiter::Brace, while_body)), - ]); - - minimal_quote!((@ TokenTree::Group(Group::new(Delimiter::Brace, rep_expanded)))).to_tokens(&mut tokens); + consume_dollar_group_sep_star(contents.clone(), &mut iter) + .to_tokens(&mut tokens); continue; } TokenTree::Ident(_) => { @@ -373,7 +303,7 @@ pub fn quote(stream: TokenStream) -> TokenStream { .to_tokens(&mut tokens); continue; } - TokenTree::Punct(ref tt) if tt.as_char() == '$' => {} + TokenTree::Punct(ref tt) if tt.as_char() == '$' => {} // Escape `$` via `$$`. _ => panic!( "`$` must be followed by an ident or `$` or a repetition group in `quote!`" ), @@ -477,6 +407,381 @@ fn collect_meta_vars(content_stream: TokenStream) -> Vec { vars } +/// Consume a `$( CONTENTS ) SEP_OPT *` accordingly. +fn consume_dollar_group_sep_star( + content_stream: TokenStream, + iter: &mut Peekable, +) -> TokenStream { + let mut tokens = crate::TokenStream::new(); + + let mut current_contents: TokenStream = content_stream; + let mut sep_cand = Vec::new(); + let mut is_sep_confirmed = false; + loop { + match (iter.next(), iter.peek().cloned()) { + // If a valid `*` is found, expand the current_contents and consume the `*`. + (Some(TokenTree::Punct(star)), opt) + if star.as_char() == '*' && !matches!(opt, Some(TokenTree::Punct(_))) => + { + let sep_opt: Option = + (!sep_cand.is_empty()).then(|| sep_cand.into_iter().collect::()); + + expand_dollar_group_sep_star(current_contents.clone(), sep_opt) + .to_tokens(&mut tokens); + break; + } + (Some(TokenTree::Punct(star)), Some(TokenTree::Punct(not_assign))) + if star.as_char() == '*' + && !(star.spacing() == Spacing::Joint && not_assign.as_char() == '=') => + { + let sep_opt: Option = + (!sep_cand.is_empty()).then(|| sep_cand.into_iter().collect::()); + + expand_dollar_group_sep_star(current_contents.clone(), sep_opt) + .to_tokens(&mut tokens); + break; + } + + // If the next `$( CONTENTS )` is found before the `*`, consume the current `$( CONTENTS )` literally. + // Then move to consume the next `$( CONTENTS )`. + (Some(TokenTree::Punct(dollar)), Some(TokenTree::Group(next_group))) + if dollar.as_char() == '$' => + { + minimal_quote!( + crate::ToTokens::to_tokens(&TokenTree::from(Punct::new('$', Spacing::Joint)), &mut ts); + ) + .to_tokens(&mut tokens); + + quote( + [TokenTree::Group(Group::new(Delimiter::Brace, current_contents.clone()))] + .into_iter() + .collect::(), + ) + .to_tokens(&mut tokens); + iter.next(); + + let sep_opt: Option = (!sep_cand.is_empty()) + .then(|| sep_cand.clone().into_iter().collect::()); + sep_opt.into_iter().for_each(|sep| { + quote(sep).to_tokens(&mut tokens); + }); + sep_cand = Vec::new(); + is_sep_confirmed = false; + + // Move to consume the next `$( CONTENTS )`. + current_contents = next_group.stream().clone(); + } + + // Add the current token to the separator candidate until the separator is confirmed. + (Some(x), _) if !is_sep_confirmed => { + sep_cand.push(x.clone()); + is_sep_confirmed = is_valid_sep(sep_cand.as_slice()); + + let mut current = Some(x); + while is_sep_confirmed { + if let Some(TokenTree::Punct(ref punct)) = current + && punct.spacing() == Spacing::Joint + { + match iter.peek() { + Some(next) => { + sep_cand.push(next.clone()); + is_sep_confirmed = is_valid_sep(sep_cand.as_slice()); + + if !is_sep_confirmed { + sep_cand.pop(); + if let Some(TokenTree::Punct(ref p)) = sep_cand.pop() { + let mut new_p = Punct::new(p.as_char(), Spacing::Alone); + new_p.set_span(p.span()); + sep_cand.push(TokenTree::Punct(new_p)); + } + + is_sep_confirmed = true; + break; + } + current = iter.next(); + } + None => break, + } + } else { + break; + } + } + } + + // Consume the current `$( CONTENTS )` literally without a `*`. + (x, _) => { + minimal_quote!( + crate::ToTokens::to_tokens(&TokenTree::from(Punct::new('$', Spacing::Joint)), &mut ts); + ) + .to_tokens(&mut tokens); + quote( + [TokenTree::Group(Group::new(Delimiter::Brace, current_contents.clone()))] + .into_iter() + .collect::(), + ) + .to_tokens(&mut tokens); + + let sep_opt: Option = (!sep_cand.is_empty()) + .then(|| sep_cand.clone().into_iter().collect::()); + sep_opt.into_iter().for_each(|sep| { + quote(sep).to_tokens(&mut tokens); + }); + + // Recover the unconsumed token `x`. + let mut new_stream = x.into_iter().collect::(); + new_stream.extend(iter.by_ref()); + *iter = new_stream.into_iter().peekable(); + break; + } + } + } + + return tokens; +} + +/// Determine if the given token sequence is a valid separator. +fn is_valid_sep(ts: &[TokenTree]) -> bool { + match ts { + [TokenTree::Punct(t1), TokenTree::Punct(t2), TokenTree::Punct(t3)] + if t1.as_char() == '.' + && t1.spacing() == Spacing::Joint + && t2.as_char() == '.' + && t2.spacing() == Spacing::Joint + && t3.as_char() == '.' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2), TokenTree::Punct(t3)] + if t1.as_char() == '.' + && t1.spacing() == Spacing::Joint + && t2.as_char() == '.' + && t2.spacing() == Spacing::Joint + && t3.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2), TokenTree::Punct(t3)] + if t1.as_char() == '<' + && t1.spacing() == Spacing::Joint + && t2.as_char() == '<' + && t2.spacing() == Spacing::Joint + && t3.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2), TokenTree::Punct(t3)] + if t1.as_char() == '>' + && t1.spacing() == Spacing::Joint + && t2.as_char() == '>' + && t2.spacing() == Spacing::Joint + && t3.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == ':' && t1.spacing() == Spacing::Joint && t2.as_char() == ':' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '+' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '&' && t1.spacing() == Spacing::Joint && t2.as_char() == '&' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '&' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '^' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '/' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '.' && t1.spacing() == Spacing::Joint && t2.as_char() == '.' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '=' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '>' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '<' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '*' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '!' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '|' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '|' && t1.spacing() == Spacing::Joint && t2.as_char() == '|' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '-' && t1.spacing() == Spacing::Joint && t2.as_char() == '>' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '<' && t1.spacing() == Spacing::Joint && t2.as_char() == '-' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '%' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '=' && t1.spacing() == Spacing::Joint && t2.as_char() == '>' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '<' && t1.spacing() == Spacing::Joint && t2.as_char() == '<' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '>' && t1.spacing() == Spacing::Joint && t2.as_char() == '>' => + { + true + } + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.as_char() == '-' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => + { + true + } + [TokenTree::Punct(single_quote), TokenTree::Ident(_)] // lifetime + if single_quote.as_char() == '\'' && single_quote.spacing() == Spacing::Joint => + { + true + } + [TokenTree::Punct(t1)] if t1.as_char() == '#' => true, + [TokenTree::Punct(t1)] if t1.as_char() == ',' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '.' => true, + [TokenTree::Punct(t1)] if t1.as_char() == ';' => true, + [TokenTree::Punct(t1)] if t1.as_char() == ':' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '+' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '@' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '!' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '^' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '&' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '/' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '=' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '>' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '<' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '|' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '?' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '%' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '*' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '-' => true, + [TokenTree::Punct(t1)] if t1.as_char() == '_' => true, + [TokenTree::Ident(_)] => true, + [TokenTree::Group(_)] => true, + [_x] => true, + _ => false, + } +} + +fn expand_dollar_group_sep_star( + contents: TokenStream, + sep_opt: Option, +) -> TokenStream { + let mut tokens = crate::TokenStream::new(); + + let mut rep_expanded = TokenStream::new(); + + // Append setup code for a `while`, where recursively quoted `CONTENTS` + // and `SEP_OPT` are repeatedly processed, to `REP_EXPANDED`. + let meta_vars = collect_meta_vars(contents.clone()); + minimal_quote!( + use crate::ext::*; + (@ if sep_opt.is_some() { + minimal_quote!(let mut _i = 0usize;) + } else { + minimal_quote!(();) + }) + let has_iter = crate::ThereIsNoIteratorInRepetition; + ) + .to_tokens(&mut rep_expanded); + for meta_var in &meta_vars { + minimal_quote!( + #[allow(unused_mut)] + let (mut (@ meta_var), i) = (@ meta_var).quote_into_iter(); + let has_iter = has_iter | i; + ) + .to_tokens(&mut rep_expanded); + } + minimal_quote!(let _: crate::HasIterator = has_iter;).to_tokens(&mut rep_expanded); + + // Append the `while` to `REP_EXPANDED`. + let mut while_body = TokenStream::new(); + for meta_var in &meta_vars { + minimal_quote!( + let (@ meta_var) = match (@ meta_var).next() { + Some(_x) => crate::RepInterp(_x), + None => break, + }; + ) + .to_tokens(&mut while_body); + } + minimal_quote!( + (@ if let Some(sep) = sep_opt { + minimal_quote!( + if _i > 0 { + (@ quote(sep)).to_tokens(&mut ts); + } + _i += 1; + ) + } else { + minimal_quote!(();) + }) + (@ quote(contents.clone())).to_tokens(&mut ts); + ) + .to_tokens(&mut while_body); + rep_expanded.extend(vec![ + TokenTree::Ident(Ident::new("while", Span::call_site())), + TokenTree::Ident(Ident::new("true", Span::call_site())), + TokenTree::Group(Group::new(Delimiter::Brace, while_body)), + ]); + + minimal_quote!((@ TokenTree::Group(Group::new(Delimiter::Brace, rep_expanded)))) + .to_tokens(&mut tokens); + return tokens; +} + /// Quote a `Span` into a `TokenStream`. /// This is needed to implement a custom quoter. #[unstable(feature = "proc_macro_quote", issue = "54722")] diff --git a/tests/ui/proc-macro/quote/auxiliary/basic.rs b/tests/ui/proc-macro/quote/auxiliary/basic.rs index c50bb964eab06..7dac65e68917f 100644 --- a/tests/ui/proc-macro/quote/auxiliary/basic.rs +++ b/tests/ui/proc-macro/quote/auxiliary/basic.rs @@ -11,6 +11,9 @@ use proc_macro::*; #[proc_macro] pub fn run_tests(_: TokenStream) -> TokenStream { + test_sep(); + test_dollar_dollar(); + test_quote_impl(); test_substitution(); test_iter(); @@ -50,6 +53,23 @@ pub fn run_tests(_: TokenStream) -> TokenStream { TokenStream::new() } +fn test_sep() { + let iter = ["a", "b"].into_iter(); + let tokens = quote!($($iter) << *); + + let expected = "\"a\" << \"b\""; + assert_eq!(expected, tokens.to_string()); +} + +fn test_dollar_dollar() { + let iter = ["a", "b"].into_iter(); + let x = "X"; + let tokens = quote!($$ x $($$ x $iter)*); + + let expected = "$x $x \"a\" $x \"b\""; + assert_eq!(expected, tokens.to_string()); +} + // Based on https://github.com/dtolnay/quote/blob/0245506323a3616daa2ee41c6ad0b871e4d78ae4/tests/test.rs // // FIXME(quote): From 1673552aff6bc9e6046f978f53d0769f6139a18c Mon Sep 17 00:00:00 2001 From: Tomoaki Kobayashi Date: Thu, 6 Nov 2025 21:39:01 +0900 Subject: [PATCH 2/4] [TO BE SQUASHED] Fix --- library/proc_macro/src/quote.rs | 332 ++++++------------ tests/ui/proc-macro/quote/auxiliary/basic.rs | 46 ++- .../not-quotable-repetition-group-brace.rs | 10 + ...not-quotable-repetition-group-brace.stderr | 23 ++ .../not-quotable-repetition-group-bracket.rs | 10 + ...t-quotable-repetition-group-bracket.stderr | 23 ++ 6 files changed, 214 insertions(+), 230 deletions(-) create mode 100644 tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.rs create mode 100644 tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.stderr create mode 100644 tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.rs create mode 100644 tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.stderr diff --git a/library/proc_macro/src/quote.rs b/library/proc_macro/src/quote.rs index fb05021735624..0343432de8c4b 100644 --- a/library/proc_macro/src/quote.rs +++ b/library/proc_macro/src/quote.rs @@ -285,19 +285,29 @@ pub fn quote(stream: TokenStream) -> TokenStream { let proc_macro_crate = minimal_quote!(crate); let mut after_dollar = false; - let mut tokens = crate::TokenStream::new(); - let mut iter: Peekable = stream.into_iter().peekable(); + let mut tokens = TokenStream::new(); + let mut iter: Peekable<::IntoIter> = stream.into_iter().peekable(); while let Some(tree) = iter.next() { if after_dollar { after_dollar = false; match tree { - TokenTree::Group(ref tt) => { - // Handles repetition by expanding `$( CONTENTS ) SEP_OPT *` to `{ REP_EXPANDED }`. - let contents = tt.stream(); - consume_dollar_group_sep_star(contents.clone(), &mut iter) + TokenTree::Group(ref tt) if tt.delimiter() == Delimiter::Parenthesis => { + consume_dollar_group_sep_star(tt.stream().clone(), &mut iter) .to_tokens(&mut tokens); continue; } + TokenTree::Group(_) => { + minimal_quote!( + crate::ToTokens::to_tokens(&TokenTree::from(Punct::new('$', Spacing::Joint)), &mut ts); + ) + .to_tokens(&mut tokens); + minimal_quote!((@ + quote(TokenStream::from(tree)) + ).to_tokens(&mut ts); + ) + .to_tokens(&mut tokens); + continue; + } TokenTree::Ident(_) => { minimal_quote!(crate::ToTokens::to_tokens(&(@ tree), &mut ts);) .to_tokens(&mut tokens); @@ -380,20 +390,34 @@ pub fn quote(stream: TokenStream) -> TokenStream { } } -/// Helper function to support macro repetitions like `$( CONTENTS ) SEP_OPT *` in `quote!`. +/// Helper function to support macro repetitions like `$( CONTENTS ) SEP *` in `quote!`. /// Recursively collects all `Ident`s (meta-variables) that follow a `$` /// from the given `CONTENTS` stream, preserving their order of appearance. fn collect_meta_vars(content_stream: TokenStream) -> Vec { fn helper(stream: TokenStream, out: &mut Vec) { + let mut after_dollar = false; + let mut iter = stream.into_iter().peekable(); - while let Some(tree) = iter.next() { - match &tree { - TokenTree::Punct(tt) if tt.as_char() == '$' => { - if let Some(TokenTree::Ident(id)) = iter.peek() { + while let Some(ref tree) = iter.next() { + if after_dollar { + after_dollar = false; + match tree { + TokenTree::Ident(id) => { out.push(id.clone()); iter.next(); + continue; } + // TokenTree::Punct(tt) if tt.as_char() == '$' => {} // Escape `$` via `$$`. + _ => {} } + } else if let TokenTree::Punct(tt) = tree + && tt.as_char() == '$' + { + after_dollar = true; + continue; + } + + match tree { TokenTree::Group(tt) => { helper(tt.stream(), out); } @@ -407,12 +431,12 @@ fn collect_meta_vars(content_stream: TokenStream) -> Vec { vars } -/// Consume a `$( CONTENTS ) SEP_OPT *` accordingly. +/// Consume a `$( CONTENTS ) SEP *` accordingly. It handles repetition by expanding `$( CONTENTS ) SEP *` to `{ REP_EXPANDED }`. fn consume_dollar_group_sep_star( content_stream: TokenStream, - iter: &mut Peekable, + iter: &mut Peekable<::IntoIter>, ) -> TokenStream { - let mut tokens = crate::TokenStream::new(); + let mut tokens = TokenStream::new(); let mut current_contents: TokenStream = content_stream; let mut sep_cand = Vec::new(); @@ -423,10 +447,7 @@ fn consume_dollar_group_sep_star( (Some(TokenTree::Punct(star)), opt) if star.as_char() == '*' && !matches!(opt, Some(TokenTree::Punct(_))) => { - let sep_opt: Option = - (!sep_cand.is_empty()).then(|| sep_cand.into_iter().collect::()); - - expand_dollar_group_sep_star(current_contents.clone(), sep_opt) + expand_dollar_group_sep_star(current_contents, TokenStream::from_iter(sep_cand)) .to_tokens(&mut tokens); break; } @@ -434,42 +455,39 @@ fn consume_dollar_group_sep_star( if star.as_char() == '*' && !(star.spacing() == Spacing::Joint && not_assign.as_char() == '=') => { - let sep_opt: Option = - (!sep_cand.is_empty()).then(|| sep_cand.into_iter().collect::()); - - expand_dollar_group_sep_star(current_contents.clone(), sep_opt) + expand_dollar_group_sep_star(current_contents, TokenStream::from_iter(sep_cand)) .to_tokens(&mut tokens); break; } - // If the next `$( CONTENTS )` is found before the `*`, consume the current `$( CONTENTS )` literally. - // Then move to consume the next `$( CONTENTS )`. + // If the next `$( CONTENTS )` is found before the `*`, consume the current `$( CONTENTS ) SEP` literally. + // Then move to consume the next `$( CONTENTS ) SEP *` . (Some(TokenTree::Punct(dollar)), Some(TokenTree::Group(next_group))) - if dollar.as_char() == '$' => + if dollar.as_char() == '$' && next_group.delimiter() == Delimiter::Parenthesis => { minimal_quote!( crate::ToTokens::to_tokens(&TokenTree::from(Punct::new('$', Spacing::Joint)), &mut ts); ) .to_tokens(&mut tokens); - - quote( - [TokenTree::Group(Group::new(Delimiter::Brace, current_contents.clone()))] - .into_iter() - .collect::(), + minimal_quote!((@ + quote( + TokenStream::from(TokenTree::Group(Group::new(Delimiter::Parenthesis, current_contents)))) + ).to_tokens(&mut ts); ) .to_tokens(&mut tokens); - iter.next(); + let sep = TokenStream::from_iter(sep_cand); + if !sep.is_empty() { + minimal_quote!((@ + quote(sep)).to_tokens(&mut ts); + ) + .to_tokens(&mut tokens); + } - let sep_opt: Option = (!sep_cand.is_empty()) - .then(|| sep_cand.clone().into_iter().collect::()); - sep_opt.into_iter().for_each(|sep| { - quote(sep).to_tokens(&mut tokens); - }); + // Move to consume the next `$( CONTENTS ) SEP *`. + current_contents = next_group.stream(); + iter.next(); sep_cand = Vec::new(); is_sep_confirmed = false; - - // Move to consume the next `$( CONTENTS )`. - current_contents = next_group.stream().clone(); } // Add the current token to the separator candidate until the separator is confirmed. @@ -508,24 +526,25 @@ fn consume_dollar_group_sep_star( } } - // Consume the current `$( CONTENTS )` literally without a `*`. + // Consume the current `$( CONTENTS ) SEP` literally without a `*`. (x, _) => { minimal_quote!( crate::ToTokens::to_tokens(&TokenTree::from(Punct::new('$', Spacing::Joint)), &mut ts); ) .to_tokens(&mut tokens); - quote( - [TokenTree::Group(Group::new(Delimiter::Brace, current_contents.clone()))] - .into_iter() - .collect::(), + minimal_quote!((@ + quote( + TokenStream::from(TokenTree::Group(Group::new(Delimiter::Parenthesis, current_contents)))) + ).to_tokens(&mut ts); ) .to_tokens(&mut tokens); - - let sep_opt: Option = (!sep_cand.is_empty()) - .then(|| sep_cand.clone().into_iter().collect::()); - sep_opt.into_iter().for_each(|sep| { - quote(sep).to_tokens(&mut tokens); - }); + let sep = TokenStream::from_iter(sep_cand); + if !sep.is_empty() { + minimal_quote!((@ + quote(sep)).to_tokens(&mut ts); + ) + .to_tokens(&mut tokens); + } // Recover the unconsumed token `x`. let mut new_stream = x.into_iter().collect::(); @@ -535,203 +554,64 @@ fn consume_dollar_group_sep_star( } } } - - return tokens; + tokens } /// Determine if the given token sequence is a valid separator. fn is_valid_sep(ts: &[TokenTree]) -> bool { match ts { [TokenTree::Punct(t1), TokenTree::Punct(t2), TokenTree::Punct(t3)] - if t1.as_char() == '.' - && t1.spacing() == Spacing::Joint - && t2.as_char() == '.' - && t2.spacing() == Spacing::Joint - && t3.as_char() == '.' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2), TokenTree::Punct(t3)] - if t1.as_char() == '.' - && t1.spacing() == Spacing::Joint - && t2.as_char() == '.' - && t2.spacing() == Spacing::Joint - && t3.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2), TokenTree::Punct(t3)] - if t1.as_char() == '<' - && t1.spacing() == Spacing::Joint - && t2.as_char() == '<' - && t2.spacing() == Spacing::Joint - && t3.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2), TokenTree::Punct(t3)] - if t1.as_char() == '>' - && t1.spacing() == Spacing::Joint - && t2.as_char() == '>' - && t2.spacing() == Spacing::Joint - && t3.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == ':' && t1.spacing() == Spacing::Joint && t2.as_char() == ':' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '+' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '&' && t1.spacing() == Spacing::Joint && t2.as_char() == '&' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '&' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '^' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '/' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '.' && t1.spacing() == Spacing::Joint && t2.as_char() == '.' => + if t1.spacing() == Spacing::Joint && t2.spacing() == Spacing::Joint => { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '=' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '>' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '<' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '*' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '!' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '|' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '|' && t1.spacing() == Spacing::Joint && t2.as_char() == '|' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '-' && t1.spacing() == Spacing::Joint && t2.as_char() == '>' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '<' && t1.spacing() == Spacing::Joint && t2.as_char() == '-' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '%' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '=' && t1.spacing() == Spacing::Joint && t2.as_char() == '>' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '<' && t1.spacing() == Spacing::Joint && t2.as_char() == '<' => - { - true - } - [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '>' && t1.spacing() == Spacing::Joint && t2.as_char() == '>' => - { - true + matches!( + [t1.as_char(), t2.as_char(), t3.as_char()], + ['.', '.', '.'] | ['.', '.', '='] | ['<', '<', '='] | ['>', '>', '='] + ) } [TokenTree::Punct(t1), TokenTree::Punct(t2)] - if t1.as_char() == '-' && t1.spacing() == Spacing::Joint && t2.as_char() == '=' => - { - true + if t1.spacing() == Spacing::Joint => + { + matches!( + [t1.as_char(), t2.as_char()], + [':', ':'] | ['+', '='] | ['&', '&'] | ['&', '='] | ['^', '='] | ['/', '='] + | ['.', '.'] | ['=', '='] | ['>', '='] | ['<', '='] | ['*', '='] | ['!', '='] + | ['|', '='] | ['|', '|'] | ['-', '>'] | ['<', '-'] | ['%', '='] | ['=', '>'] + | ['<', '<'] | ['>', '>'] | ['-', '='] + ) } [TokenTree::Punct(single_quote), TokenTree::Ident(_)] // lifetime if single_quote.as_char() == '\'' && single_quote.spacing() == Spacing::Joint => { true } - [TokenTree::Punct(t1)] if t1.as_char() == '#' => true, - [TokenTree::Punct(t1)] if t1.as_char() == ',' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '.' => true, - [TokenTree::Punct(t1)] if t1.as_char() == ';' => true, - [TokenTree::Punct(t1)] if t1.as_char() == ':' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '+' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '@' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '!' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '^' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '&' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '/' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '=' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '>' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '<' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '|' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '?' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '%' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '*' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '-' => true, - [TokenTree::Punct(t1)] if t1.as_char() == '_' => true, - [TokenTree::Ident(_)] => true, - [TokenTree::Group(_)] => true, - [_x] => true, + [TokenTree::Punct(t1)] => + { + matches!( + t1.as_char(), + // The LEGAL_CHARS except '\'' are available here. + '#' | ',' | '.' | ';' | ':' | '+' | '@' | '!' | '^' | '&' | '/' | + '=' | '>' | '<' | '|' | '?' | '%' | '*' | '-' | '_' | '~' + ) + } + [TokenTree::Ident(_)] | [TokenTree::Group(_)] | [TokenTree::Literal(_)] => true, _ => false, } } -fn expand_dollar_group_sep_star( - contents: TokenStream, - sep_opt: Option, -) -> TokenStream { - let mut tokens = crate::TokenStream::new(); +fn expand_dollar_group_sep_star(contents: TokenStream, sep: TokenStream) -> TokenStream { + let mut tokens = TokenStream::new(); let mut rep_expanded = TokenStream::new(); // Append setup code for a `while`, where recursively quoted `CONTENTS` - // and `SEP_OPT` are repeatedly processed, to `REP_EXPANDED`. + // and `SEP` are repeatedly processed, to `REP_EXPANDED`. let meta_vars = collect_meta_vars(contents.clone()); minimal_quote!( use crate::ext::*; - (@ if sep_opt.is_some() { - minimal_quote!(let mut _i = 0usize;) + (@ if !sep.is_empty() { + Some(minimal_quote!(let mut _i = 0usize;)) } else { - minimal_quote!(();) + None }) let has_iter = crate::ThereIsNoIteratorInRepetition; ) @@ -758,17 +638,17 @@ fn expand_dollar_group_sep_star( .to_tokens(&mut while_body); } minimal_quote!( - (@ if let Some(sep) = sep_opt { - minimal_quote!( + (@ if !sep.is_empty() { + Some(minimal_quote!( if _i > 0 { (@ quote(sep)).to_tokens(&mut ts); } _i += 1; - ) + )) } else { - minimal_quote!(();) + None }) - (@ quote(contents.clone())).to_tokens(&mut ts); + (@ quote(contents)).to_tokens(&mut ts); ) .to_tokens(&mut while_body); rep_expanded.extend(vec![ @@ -779,7 +659,7 @@ fn expand_dollar_group_sep_star( minimal_quote!((@ TokenTree::Group(Group::new(Delimiter::Brace, rep_expanded)))) .to_tokens(&mut tokens); - return tokens; + tokens } /// Quote a `Span` into a `TokenStream`. diff --git a/tests/ui/proc-macro/quote/auxiliary/basic.rs b/tests/ui/proc-macro/quote/auxiliary/basic.rs index 7dac65e68917f..668f403b7ffc4 100644 --- a/tests/ui/proc-macro/quote/auxiliary/basic.rs +++ b/tests/ui/proc-macro/quote/auxiliary/basic.rs @@ -11,8 +11,12 @@ use proc_macro::*; #[proc_macro] pub fn run_tests(_: TokenStream) -> TokenStream { - test_sep(); - test_dollar_dollar(); + test_sep1(); + test_sep2(); + test_sep_group1(); + test_sep_group2(); + test_dollar_dollar1(); + test_dollar_dollar2(); test_quote_impl(); test_substitution(); @@ -53,7 +57,7 @@ pub fn run_tests(_: TokenStream) -> TokenStream { TokenStream::new() } -fn test_sep() { +fn test_sep1() { let iter = ["a", "b"].into_iter(); let tokens = quote!($($iter) << *); @@ -61,15 +65,49 @@ fn test_sep() { assert_eq!(expected, tokens.to_string()); } -fn test_dollar_dollar() { +fn test_sep2() { + let iter1 = ["a", "b"].into_iter(); + let iter2 = [1, 2, 3]; + let tokens = quote!($($iter1) ($($iter1) $($iter2),* A) *); + + let expected = "\"a\" ($(\"b\") 1i32, 2i32, 3i32 A) \"b\""; + assert_eq!(expected, tokens.to_string()); +} + +fn test_sep_group1() { + let x = "X"; let iter = ["a", "b"].into_iter(); + let tokens = quote!($($x) >> $($iter) << *); + + let expected = "$(\"X\") >> \"a\" << \"b\""; + assert_eq!(expected, tokens.to_string()); +} + +fn test_sep_group2() { let x = "X"; + let iter = ["a", "b"].into_iter(); + let tokens = quote!($($x) >> > $($iter) << *); + + let expected = "$(\"X\") >> > \"a\" << \"b\""; + assert_eq!(expected, tokens.to_string()); +} + +fn test_dollar_dollar1() { + let iter = ["a", "b"].into_iter(); let tokens = quote!($$ x $($$ x $iter)*); let expected = "$x $x \"a\" $x \"b\""; assert_eq!(expected, tokens.to_string()); } +fn test_dollar_dollar2() { + let iter = ["a", "b", "c"].into_iter(); + let tokens = quote!($($iter) $$ *); + + let expected = "\"a\" $ \"b\" $ \"c\""; + assert_eq!(expected, tokens.to_string()); +} + // Based on https://github.com/dtolnay/quote/blob/0245506323a3616daa2ee41c6ad0b871e4d78ae4/tests/test.rs // // FIXME(quote): diff --git a/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.rs b/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.rs new file mode 100644 index 0000000000000..84dfeeb2e6519 --- /dev/null +++ b/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.rs @@ -0,0 +1,10 @@ +#![feature(proc_macro_quote)] + +extern crate proc_macro; + +use proc_macro::quote; + +fn main() { + let arr = [1, 2, 3]; + let _ = quote! { ${$arr}* }; //~ ERROR the trait bound `[{integer}; 3]: ToTokens` is not satisfied [E0277] +} diff --git a/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.stderr b/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.stderr new file mode 100644 index 0000000000000..c05f17bcddf10 --- /dev/null +++ b/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.stderr @@ -0,0 +1,23 @@ +error[E0277]: the trait bound `[{integer}; 3]: ToTokens` is not satisfied + --> $DIR/not-quotable-repetition-group-brace.rs:9:13 + | +LL | let _ = quote! { ${$arr}* }; + | ^^^^^^^^^^^^^^^^^^^ + | | + | the trait `ToTokens` is not implemented for `[{integer}; 3]` + | required by a bound introduced by this call + | + = help: the following other types implement trait `ToTokens`: + &T + &mut T + Box + CString + Cow<'_, T> + Option + Rc + bool + and 24 others + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0277`. diff --git a/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.rs b/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.rs new file mode 100644 index 0000000000000..84dfeeb2e6519 --- /dev/null +++ b/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.rs @@ -0,0 +1,10 @@ +#![feature(proc_macro_quote)] + +extern crate proc_macro; + +use proc_macro::quote; + +fn main() { + let arr = [1, 2, 3]; + let _ = quote! { ${$arr}* }; //~ ERROR the trait bound `[{integer}; 3]: ToTokens` is not satisfied [E0277] +} diff --git a/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.stderr b/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.stderr new file mode 100644 index 0000000000000..1aeeee1f3cad5 --- /dev/null +++ b/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.stderr @@ -0,0 +1,23 @@ +error[E0277]: the trait bound `[{integer}; 3]: ToTokens` is not satisfied + --> $DIR/not-quotable-repetition-group-bracket.rs:9:13 + | +LL | let _ = quote! { ${$arr}* }; + | ^^^^^^^^^^^^^^^^^^^ + | | + | the trait `ToTokens` is not implemented for `[{integer}; 3]` + | required by a bound introduced by this call + | + = help: the following other types implement trait `ToTokens`: + &T + &mut T + Box + CString + Cow<'_, T> + Option + Rc + bool + and 24 others + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0277`. From 93fd70ad7d978d4ebe6830af81057aba3706ee99 Mon Sep 17 00:00:00 2001 From: Tomoaki Kobayashi Date: Wed, 19 Nov 2025 01:48:46 +0900 Subject: [PATCH 3/4] [TO BE SQUASHED] Fix2 --- library/proc_macro/src/quote.rs | 327 +++++++++--------- tests/ui/proc-macro/quote/auxiliary/basic.rs | 38 +- .../not-quotable-repetition-group-brace.rs | 2 +- ...not-quotable-repetition-group-brace.stderr | 12 +- .../not-quotable-repetition-group-bracket.rs | 2 +- ...t-quotable-repetition-group-bracket.stderr | 12 +- .../not-quotable-repetition-group-none.rs | 16 + .../not-quotable-repetition-group-none.stderr | 27 ++ ...t-quotable-repetition-invalid-separator.rs | 10 + ...otable-repetition-invalid-separator.stderr | 23 ++ 10 files changed, 286 insertions(+), 183 deletions(-) create mode 100644 tests/ui/proc-macro/quote/not-quotable-repetition-group-none.rs create mode 100644 tests/ui/proc-macro/quote/not-quotable-repetition-group-none.stderr create mode 100644 tests/ui/proc-macro/quote/not-quotable-repetition-invalid-separator.rs create mode 100644 tests/ui/proc-macro/quote/not-quotable-repetition-invalid-separator.stderr diff --git a/library/proc_macro/src/quote.rs b/library/proc_macro/src/quote.rs index 0343432de8c4b..72419fa5a8b24 100644 --- a/library/proc_macro/src/quote.rs +++ b/library/proc_macro/src/quote.rs @@ -292,8 +292,7 @@ pub fn quote(stream: TokenStream) -> TokenStream { after_dollar = false; match tree { TokenTree::Group(ref tt) if tt.delimiter() == Delimiter::Parenthesis => { - consume_dollar_group_sep_star(tt.stream().clone(), &mut iter) - .to_tokens(&mut tokens); + consume_dollar_group_sep_star(tt.stream(), &mut iter).to_tokens(&mut tokens); continue; } TokenTree::Group(_) => { @@ -390,174 +389,140 @@ pub fn quote(stream: TokenStream) -> TokenStream { } } -/// Helper function to support macro repetitions like `$( CONTENTS ) SEP *` in `quote!`. -/// Recursively collects all `Ident`s (meta-variables) that follow a `$` -/// from the given `CONTENTS` stream, preserving their order of appearance. -fn collect_meta_vars(content_stream: TokenStream) -> Vec { - fn helper(stream: TokenStream, out: &mut Vec) { - let mut after_dollar = false; - - let mut iter = stream.into_iter().peekable(); - while let Some(ref tree) = iter.next() { - if after_dollar { - after_dollar = false; - match tree { - TokenTree::Ident(id) => { - out.push(id.clone()); - iter.next(); - continue; - } - // TokenTree::Punct(tt) if tt.as_char() == '$' => {} // Escape `$` via `$$`. - _ => {} - } - } else if let TokenTree::Punct(tt) = tree - && tt.as_char() == '$' - { - after_dollar = true; - continue; - } - - match tree { - TokenTree::Group(tt) => { - helper(tt.stream(), out); - } - _ => {} - } - } - } - - let mut vars = Vec::new(); - helper(content_stream, &mut vars); - vars -} - /// Consume a `$( CONTENTS ) SEP *` accordingly. It handles repetition by expanding `$( CONTENTS ) SEP *` to `{ REP_EXPANDED }`. fn consume_dollar_group_sep_star( content_stream: TokenStream, - iter: &mut Peekable<::IntoIter>, + sep_iter: &mut Peekable<::IntoIter>, ) -> TokenStream { let mut tokens = TokenStream::new(); let mut current_contents: TokenStream = content_stream; let mut sep_cand = Vec::new(); - let mut is_sep_confirmed = false; - loop { - match (iter.next(), iter.peek().cloned()) { - // If a valid `*` is found, expand the current_contents and consume the `*`. - (Some(TokenTree::Punct(star)), opt) - if star.as_char() == '*' && !matches!(opt, Some(TokenTree::Punct(_))) => - { - expand_dollar_group_sep_star(current_contents, TokenStream::from_iter(sep_cand)) - .to_tokens(&mut tokens); - break; - } - (Some(TokenTree::Punct(star)), Some(TokenTree::Punct(not_assign))) - if star.as_char() == '*' - && !(star.spacing() == Spacing::Joint && not_assign.as_char() == '=') => - { - expand_dollar_group_sep_star(current_contents, TokenStream::from_iter(sep_cand)) - .to_tokens(&mut tokens); - break; - } - // If the next `$( CONTENTS )` is found before the `*`, consume the current `$( CONTENTS ) SEP` literally. - // Then move to consume the next `$( CONTENTS ) SEP *` . - (Some(TokenTree::Punct(dollar)), Some(TokenTree::Group(next_group))) - if dollar.as_char() == '$' && next_group.delimiter() == Delimiter::Parenthesis => - { - minimal_quote!( - crate::ToTokens::to_tokens(&TokenTree::from(Punct::new('$', Spacing::Joint)), &mut ts); - ) - .to_tokens(&mut tokens); - minimal_quote!((@ - quote( - TokenStream::from(TokenTree::Group(Group::new(Delimiter::Parenthesis, current_contents)))) - ).to_tokens(&mut ts); - ) - .to_tokens(&mut tokens); - let sep = TokenStream::from_iter(sep_cand); - if !sep.is_empty() { - minimal_quote!((@ - quote(sep)).to_tokens(&mut ts); - ) - .to_tokens(&mut tokens); + loop { + match sep_iter.peek() { + // Check for repetition `*` + Some(TokenTree::Punct(star)) if star.as_char() == '*' => { + let star_clone = star.clone(); + + // Verify it's actually a repetition star, not part of `*=` + let mut peek_iter = sep_iter.clone(); + peek_iter.next(); // Skip the `*` + let is_repetition = !matches!( + (star_clone.spacing(), peek_iter.peek()), + (Spacing::Joint, Some(TokenTree::Punct(eq))) if eq.as_char() == '=' + ); + + if is_repetition && is_valid_sep(&sep_cand) { + sep_iter.next(); // Consume the `*` + expand_dollar_group_sep_star( + &mut tokens, + current_contents, + TokenStream::from_iter(sep_cand), + ); + return tokens; } - - // Move to consume the next `$( CONTENTS ) SEP *`. - current_contents = next_group.stream(); - iter.next(); - sep_cand = Vec::new(); - is_sep_confirmed = false; } - // Add the current token to the separator candidate until the separator is confirmed. - (Some(x), _) if !is_sep_confirmed => { - sep_cand.push(x.clone()); - is_sep_confirmed = is_valid_sep(sep_cand.as_slice()); - - let mut current = Some(x); - while is_sep_confirmed { - if let Some(TokenTree::Punct(ref punct)) = current - && punct.spacing() == Spacing::Joint - { - match iter.peek() { - Some(next) => { - sep_cand.push(next.clone()); - is_sep_confirmed = is_valid_sep(sep_cand.as_slice()); - - if !is_sep_confirmed { - sep_cand.pop(); - if let Some(TokenTree::Punct(ref p)) = sep_cand.pop() { - let mut new_p = Punct::new(p.as_char(), Spacing::Alone); - new_p.set_span(p.span()); - sep_cand.push(TokenTree::Punct(new_p)); - } - - is_sep_confirmed = true; - break; - } - current = iter.next(); - } - None => break, - } - } else { - break; - } + // Check for repeated `$( CONTENTS )` pattern + Some(TokenTree::Punct(dollar)) if dollar.as_char() == '$' => { + let mut peek_iter = sep_iter.clone(); + peek_iter.next(); // Skip `$` + if let Some(TokenTree::Group(next_contents)) = peek_iter.peek() + && next_contents.delimiter() == Delimiter::Parenthesis + // NOTE: only single separator of `$` does not match this `is_valid_sep`` to escape via `$$`. + && is_valid_sep(&sep_cand) + { + // Output current group without expansion + output_dollar_group_sep(&mut tokens, current_contents, sep_cand); + + // Move to the next `$( CONTENTS ) SEP *` + sep_iter.next(); // Consume the `$` + sep_iter.next(); // Consume the `next_contents` + current_contents = next_contents.stream(); + sep_cand = Vec::new(); + continue; } } - // Consume the current `$( CONTENTS ) SEP` literally without a `*`. - (x, _) => { - minimal_quote!( - crate::ToTokens::to_tokens(&TokenTree::from(Punct::new('$', Spacing::Joint)), &mut ts); - ) - .to_tokens(&mut tokens); - minimal_quote!((@ - quote( - TokenStream::from(TokenTree::Group(Group::new(Delimiter::Parenthesis, current_contents)))) - ).to_tokens(&mut ts); - ) - .to_tokens(&mut tokens); - let sep = TokenStream::from_iter(sep_cand); - if !sep.is_empty() { - minimal_quote!((@ - quote(sep)).to_tokens(&mut ts); - ) - .to_tokens(&mut tokens); - } + _ => {} + } - // Recover the unconsumed token `x`. - let mut new_stream = x.into_iter().collect::(); - new_stream.extend(iter.by_ref()); - *iter = new_stream.into_iter().peekable(); + // Try to extend separator with any other token + if let Some(token) = sep_iter.peek() { + sep_cand.push(token.clone()); + if is_valid_sep_prefix(&sep_cand) { + sep_iter.next(); + } else { + // Can't extend separator further, backtrack and exit + sep_cand.pop(); break; } } } + + // No repetition found, output `$( CONTENTS ) SEP *` without expansion + output_dollar_group_sep(&mut tokens, current_contents, sep_cand); tokens } -/// Determine if the given token sequence is a valid separator. +/// Output `$( CONTENTS ) SEP *` without expansion +fn output_dollar_group_sep( + tokens: &mut TokenStream, + contents: TokenStream, + sep_cand: Vec, +) { + minimal_quote!( + crate::ToTokens::to_tokens(&TokenTree::from(Punct::new('$', Spacing::Joint)), &mut ts); + ) + .to_tokens(tokens); + + minimal_quote!( + (@ quote(TokenStream::from(TokenTree::Group(Group::new(Delimiter::Parenthesis, contents))))).to_tokens(&mut ts); + ) + .to_tokens(tokens); + + let sep = TokenStream::from_iter(sep_cand); + if !sep.is_empty() { + minimal_quote!( + (@ quote(sep)).to_tokens(&mut ts); + ) + .to_tokens(tokens); + } +} + +/// Check if a token sequence could be a valid separator prefix (for greedy matching). +fn is_valid_sep_prefix(ts: &[TokenTree]) -> bool { + // First check if it's a complete valid separator + if is_valid_sep(ts) { + return true; + } + + // Then check if it could be extended to a longer separator + match ts { + // Single punctuation that could be extended + [TokenTree::Punct(t1)] if t1.spacing() == Spacing::Joint => { + matches!( + t1.as_char(), + '.' | '<' | '>' | ':' | '+' | '&' | '^' | + '/' | '=' | '*' | '!' | '|' | '-' | '%' | '\'' | + // Escape `$` via `$$`. + '$' + ) + } + + // Two punctuation that could be extended to three + [TokenTree::Punct(t1), TokenTree::Punct(t2)] + if t1.spacing() == Spacing::Joint && t2.spacing() == Spacing::Joint => + { + matches!([t1.as_char(), t2.as_char()], ['.', '.'] | ['<', '<'] | ['>', '>']) + } + + _ => false, + } +} + +/// Determine if the given token sequence is a complete valid separator. fn is_valid_sep(ts: &[TokenTree]) -> bool { match ts { [TokenTree::Punct(t1), TokenTree::Punct(t2), TokenTree::Punct(t3)] @@ -576,7 +541,9 @@ fn is_valid_sep(ts: &[TokenTree]) -> bool { [':', ':'] | ['+', '='] | ['&', '&'] | ['&', '='] | ['^', '='] | ['/', '='] | ['.', '.'] | ['=', '='] | ['>', '='] | ['<', '='] | ['*', '='] | ['!', '='] | ['|', '='] | ['|', '|'] | ['-', '>'] | ['<', '-'] | ['%', '='] | ['=', '>'] - | ['<', '<'] | ['>', '>'] | ['-', '='] + | ['<', '<'] | ['>', '>'] | ['-', '='] | + // Escape `$` via `$$`. + ['$', '$'] ) } [TokenTree::Punct(single_quote), TokenTree::Ident(_)] // lifetime @@ -594,18 +561,16 @@ fn is_valid_sep(ts: &[TokenTree]) -> bool { ) } [TokenTree::Ident(_)] | [TokenTree::Group(_)] | [TokenTree::Literal(_)] => true, + [] => true, // Empty separator is valid _ => false, } } -fn expand_dollar_group_sep_star(contents: TokenStream, sep: TokenStream) -> TokenStream { - let mut tokens = TokenStream::new(); - +fn expand_dollar_group_sep_star(tokens: &mut TokenStream, contents: TokenStream, sep: TokenStream) { let mut rep_expanded = TokenStream::new(); - - // Append setup code for a `while`, where recursively quoted `CONTENTS` - // and `SEP` are repeatedly processed, to `REP_EXPANDED`. let meta_vars = collect_meta_vars(contents.clone()); + + // Construct setup code for the `while_loop` where recursively quoted `CONTENTS` and `SEP` are repeatedly processed minimal_quote!( use crate::ext::*; (@ if !sep.is_empty() { @@ -626,7 +591,7 @@ fn expand_dollar_group_sep_star(contents: TokenStream, sep: TokenStream) -> Toke } minimal_quote!(let _: crate::HasIterator = has_iter;).to_tokens(&mut rep_expanded); - // Append the `while` to `REP_EXPANDED`. + // Construct the `while_loop` let mut while_body = TokenStream::new(); for meta_var in &meta_vars { minimal_quote!( @@ -641,7 +606,9 @@ fn expand_dollar_group_sep_star(contents: TokenStream, sep: TokenStream) -> Toke (@ if !sep.is_empty() { Some(minimal_quote!( if _i > 0 { - (@ quote(sep)).to_tokens(&mut ts); + (@ + quote(TokenTree::Group(Group::new(Delimiter::None, sep)).into()) // Adjust spacing by Delimiter::None + ).to_tokens(&mut ts); } _i += 1; )) @@ -651,15 +618,55 @@ fn expand_dollar_group_sep_star(contents: TokenStream, sep: TokenStream) -> Toke (@ quote(contents)).to_tokens(&mut ts); ) .to_tokens(&mut while_body); - rep_expanded.extend(vec![ + let while_loop = vec![ TokenTree::Ident(Ident::new("while", Span::call_site())), TokenTree::Ident(Ident::new("true", Span::call_site())), TokenTree::Group(Group::new(Delimiter::Brace, while_body)), - ]); + ]; + rep_expanded.extend(while_loop); minimal_quote!((@ TokenTree::Group(Group::new(Delimiter::Brace, rep_expanded)))) - .to_tokens(&mut tokens); - tokens + .to_tokens(tokens); +} + +/// Helper function to support macro repetitions like `$( CONTENTS ) SEP *` in `quote!`. +/// Recursively collects all `Ident`s (meta-variables) that follow a `$` +/// from the given `CONTENTS` stream, preserving their order of appearance. +fn collect_meta_vars(content_stream: TokenStream) -> Vec { + fn helper(stream: TokenStream, out: &mut Vec) { + let mut after_dollar = false; + + let mut iter = stream.into_iter().peekable(); + while let Some(ref tree) = iter.next() { + if after_dollar { + after_dollar = false; + match tree { + TokenTree::Ident(id) => { + out.push(id.clone()); + continue; + } + // TokenTree::Punct(tt) if tt.as_char() == '$' => {} // Escape `$` via `$$`. + _ => {} + } + } else if let TokenTree::Punct(tt) = tree + && tt.as_char() == '$' + { + after_dollar = true; + continue; + } + + match tree { + TokenTree::Group(tt) => { + helper(tt.stream(), out); + } + _ => {} + } + } + } + + let mut vars = Vec::new(); + helper(content_stream, &mut vars); + vars } /// Quote a `Span` into a `TokenStream`. diff --git a/tests/ui/proc-macro/quote/auxiliary/basic.rs b/tests/ui/proc-macro/quote/auxiliary/basic.rs index 668f403b7ffc4..3b200e6db3984 100644 --- a/tests/ui/proc-macro/quote/auxiliary/basic.rs +++ b/tests/ui/proc-macro/quote/auxiliary/basic.rs @@ -17,6 +17,8 @@ pub fn run_tests(_: TokenStream) -> TokenStream { test_sep_group2(); test_dollar_dollar1(); test_dollar_dollar2(); + test_dollar_dollar3(); + test_dollar_dollar4(); test_quote_impl(); test_substitution(); @@ -70,7 +72,7 @@ fn test_sep2() { let iter2 = [1, 2, 3]; let tokens = quote!($($iter1) ($($iter1) $($iter2),* A) *); - let expected = "\"a\" ($(\"b\") 1i32, 2i32, 3i32 A) \"b\""; + let expected = "\"a\" ($(\"b\") 1i32 , 2i32 , 3i32 A) \"b\""; assert_eq!(expected, tokens.to_string()); } @@ -108,6 +110,24 @@ fn test_dollar_dollar2() { assert_eq!(expected, tokens.to_string()); } +fn test_dollar_dollar3() { + let x = "X"; + let iter = ["a", "b", "c"].into_iter(); + let tokens = quote!($($x)$$($x),*); + + let expected = "$(\"X\") $ (\"X\"),*"; + assert_eq!(expected, tokens.to_string()); +} + +fn test_dollar_dollar4() { + let x = "X"; + let iter = ["a", "b", "c"].into_iter(); + let tokens = quote!($($x)$$y$($iter)*); + + let expected = "$(\"X\") $y \"a\" \"b\" \"c\""; + assert_eq!(expected, tokens.to_string()); +} + // Based on https://github.com/dtolnay/quote/blob/0245506323a3616daa2ee41c6ad0b871e4d78ae4/tests/test.rs // // FIXME(quote): @@ -164,7 +184,7 @@ fn test_iter() { assert_eq!("X, X, X, X,", quote!($($primes,)*).to_string()); - assert_eq!("X, X, X, X", quote!($($primes),*).to_string()); + assert_eq!("X , X , X , X", quote!($($primes),*).to_string()); } fn test_array() { @@ -378,7 +398,7 @@ fn test_fancy_repetition() { $($foo: $bar),* }; - let expected = r#""a" : true, "b" : false"#; + let expected = r#""a" : true , "b" : false"#; assert_eq!(expected, tokens.to_string()); } @@ -391,7 +411,7 @@ fn test_nested_fancy_repetition() { ),* }; - let expected = "'a' 'b' 'c', 'x' 'y' 'z'"; + let expected = "'a' 'b' 'c' , 'x' 'y' 'z'"; assert_eq!(expected, tokens.to_string()); } @@ -403,7 +423,7 @@ fn test_duplicate_name_repetition() { $($foo: $foo),* }; - let expected = r#""a" : "a", "b" : "b" "a" : "a", "b" : "b""#; + let expected = r#""a" : "a" , "b" : "b" "a" : "a" , "b" : "b""#; assert_eq!(expected, tokens.to_string()); } @@ -414,7 +434,7 @@ fn test_duplicate_name_repetition_no_copy() { $($foo: $foo),* }; - let expected = r#""a" : "a", "b" : "b""#; + let expected = r#""a" : "a" , "b" : "b""#; assert_eq!(expected, tokens.to_string()); } @@ -427,7 +447,7 @@ fn test_btreeset_repetition() { $($set: $set),* }; - let expected = r#""a" : "a", "b" : "b""#; + let expected = r#""a" : "a" , "b" : "b""#; assert_eq!(expected, tokens.to_string()); } @@ -436,7 +456,7 @@ fn test_variable_name_conflict() { // fine, if a little confusing when debugging. let _i = vec!['a', 'b']; let tokens = quote! { $($_i),* }; - let expected = "'a', 'b'"; + let expected = "'a' , 'b'"; assert_eq!(expected, tokens.to_string()); } @@ -448,7 +468,7 @@ fn test_nonrep_in_repetition() { $($rep $rep : $nonrep $nonrep),* }; - let expected = r#""a" "a" : "c" "c", "b" "b" : "c" "c""#; + let expected = r#""a" "a" : "c" "c" , "b" "b" : "c" "c""#; assert_eq!(expected, tokens.to_string()); } diff --git a/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.rs b/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.rs index 84dfeeb2e6519..0d988d6cc954b 100644 --- a/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.rs +++ b/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.rs @@ -6,5 +6,5 @@ use proc_macro::quote; fn main() { let arr = [1, 2, 3]; - let _ = quote! { ${$arr}* }; //~ ERROR the trait bound `[{integer}; 3]: ToTokens` is not satisfied [E0277] + quote! { ${$arr}* }; //~ ERROR the trait bound `[{integer}; 3]: ToTokens` is not satisfied [E0277] } diff --git a/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.stderr b/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.stderr index c05f17bcddf10..a175ca8401dcc 100644 --- a/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.stderr +++ b/tests/ui/proc-macro/quote/not-quotable-repetition-group-brace.stderr @@ -1,11 +1,11 @@ error[E0277]: the trait bound `[{integer}; 3]: ToTokens` is not satisfied - --> $DIR/not-quotable-repetition-group-brace.rs:9:13 + --> $DIR/not-quotable-repetition-group-brace.rs:9:5 | -LL | let _ = quote! { ${$arr}* }; - | ^^^^^^^^^^^^^^^^^^^ - | | - | the trait `ToTokens` is not implemented for `[{integer}; 3]` - | required by a bound introduced by this call +LL | quote! { ${$arr}* }; + | ^^^^^^^^^^^^^^^^^^^ + | | + | the trait `ToTokens` is not implemented for `[{integer}; 3]` + | required by a bound introduced by this call | = help: the following other types implement trait `ToTokens`: &T diff --git a/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.rs b/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.rs index 84dfeeb2e6519..ccfb91132ed04 100644 --- a/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.rs +++ b/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.rs @@ -6,5 +6,5 @@ use proc_macro::quote; fn main() { let arr = [1, 2, 3]; - let _ = quote! { ${$arr}* }; //~ ERROR the trait bound `[{integer}; 3]: ToTokens` is not satisfied [E0277] + quote! { $[$arr]* }; //~ ERROR the trait bound `[{integer}; 3]: ToTokens` is not satisfied [E0277] } diff --git a/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.stderr b/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.stderr index 1aeeee1f3cad5..2ea626a70978d 100644 --- a/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.stderr +++ b/tests/ui/proc-macro/quote/not-quotable-repetition-group-bracket.stderr @@ -1,11 +1,11 @@ error[E0277]: the trait bound `[{integer}; 3]: ToTokens` is not satisfied - --> $DIR/not-quotable-repetition-group-bracket.rs:9:13 + --> $DIR/not-quotable-repetition-group-bracket.rs:9:5 | -LL | let _ = quote! { ${$arr}* }; - | ^^^^^^^^^^^^^^^^^^^ - | | - | the trait `ToTokens` is not implemented for `[{integer}; 3]` - | required by a bound introduced by this call +LL | quote! { $[$arr]* }; + | ^^^^^^^^^^^^^^^^^^^ + | | + | the trait `ToTokens` is not implemented for `[{integer}; 3]` + | required by a bound introduced by this call | = help: the following other types implement trait `ToTokens`: &T diff --git a/tests/ui/proc-macro/quote/not-quotable-repetition-group-none.rs b/tests/ui/proc-macro/quote/not-quotable-repetition-group-none.rs new file mode 100644 index 0000000000000..3440247da899f --- /dev/null +++ b/tests/ui/proc-macro/quote/not-quotable-repetition-group-none.rs @@ -0,0 +1,16 @@ +#![feature(proc_macro_quote)] + +extern crate proc_macro; + +use proc_macro::quote; + +macro_rules! do_quote { + ($dollar:tt $content:expr) => { + proc_macro::quote!($dollar $content *) //~ ERROR the trait bound `[{integer}; 3]: ToTokens` is not satisfied [E0277] + }; +} + +fn main() { + let arr = [1, 2, 3]; + do_quote!($ f!($arr)); +} diff --git a/tests/ui/proc-macro/quote/not-quotable-repetition-group-none.stderr b/tests/ui/proc-macro/quote/not-quotable-repetition-group-none.stderr new file mode 100644 index 0000000000000..f6d5cf9597e81 --- /dev/null +++ b/tests/ui/proc-macro/quote/not-quotable-repetition-group-none.stderr @@ -0,0 +1,27 @@ +error[E0277]: the trait bound `[{integer}; 3]: ToTokens` is not satisfied + --> $DIR/not-quotable-repetition-group-none.rs:9:9 + | +LL | proc_macro::quote!($dollar $content *) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `ToTokens` is not implemented for `[{integer}; 3]` +... +LL | do_quote!($ f!($arr)); + | --------------------- + | | + | required by a bound introduced by this call + | in this macro invocation + | + = help: the following other types implement trait `ToTokens`: + &T + &mut T + Box + CString + Cow<'_, T> + Option + Rc + bool + and 24 others + = note: this error originates in the macro `proc_macro::quote` which comes from the expansion of the macro `do_quote` (in Nightly builds, run with -Z macro-backtrace for more info) + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0277`. diff --git a/tests/ui/proc-macro/quote/not-quotable-repetition-invalid-separator.rs b/tests/ui/proc-macro/quote/not-quotable-repetition-invalid-separator.rs new file mode 100644 index 0000000000000..6796f7a5ca704 --- /dev/null +++ b/tests/ui/proc-macro/quote/not-quotable-repetition-invalid-separator.rs @@ -0,0 +1,10 @@ +#![feature(proc_macro_quote)] + +extern crate proc_macro; + +use proc_macro::quote; + +fn main() { + let arr = [1, 2, 3]; + quote! { $($arr) $$ x .. false [] * }; //~ ERROR the trait bound `[{integer}; 3]: ToTokens` is not satisfied [E0277] +} diff --git a/tests/ui/proc-macro/quote/not-quotable-repetition-invalid-separator.stderr b/tests/ui/proc-macro/quote/not-quotable-repetition-invalid-separator.stderr new file mode 100644 index 0000000000000..717012c0b1281 --- /dev/null +++ b/tests/ui/proc-macro/quote/not-quotable-repetition-invalid-separator.stderr @@ -0,0 +1,23 @@ +error[E0277]: the trait bound `[{integer}; 3]: ToTokens` is not satisfied + --> $DIR/not-quotable-repetition-invalid-separator.rs:9:5 + | +LL | quote! { $($arr) $$ x .. false [] * }; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | | + | the trait `ToTokens` is not implemented for `[{integer}; 3]` + | required by a bound introduced by this call + | + = help: the following other types implement trait `ToTokens`: + &T + &mut T + Box + CString + Cow<'_, T> + Option + Rc + bool + and 24 others + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0277`. From 3613f223bf52da90deb4de0f5495ca6e6ea57910 Mon Sep 17 00:00:00 2001 From: Tomoaki Kobayashi Date: Wed, 19 Nov 2025 09:29:34 +0900 Subject: [PATCH 4/4] [TO BE SQUASHED] Modify comments --- library/proc_macro/src/quote.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/library/proc_macro/src/quote.rs b/library/proc_macro/src/quote.rs index 72419fa5a8b24..8ff691adb9c1f 100644 --- a/library/proc_macro/src/quote.rs +++ b/library/proc_macro/src/quote.rs @@ -389,7 +389,7 @@ pub fn quote(stream: TokenStream) -> TokenStream { } } -/// Consume a `$( CONTENTS ) SEP *` accordingly. It handles repetition by expanding `$( CONTENTS ) SEP *` to `{ REP_EXPANDED }`. +/// Consume a `$( CONTENTS ) SEP *` accordingly. fn consume_dollar_group_sep_star( content_stream: TokenStream, sep_iter: &mut Peekable<::IntoIter>, @@ -427,13 +427,13 @@ fn consume_dollar_group_sep_star( // Check for repeated `$( CONTENTS )` pattern Some(TokenTree::Punct(dollar)) if dollar.as_char() == '$' => { let mut peek_iter = sep_iter.clone(); - peek_iter.next(); // Skip `$` + peek_iter.next(); // Skip the `$` if let Some(TokenTree::Group(next_contents)) = peek_iter.peek() && next_contents.delimiter() == Delimiter::Parenthesis - // NOTE: only single separator of `$` does not match this `is_valid_sep`` to escape via `$$`. + // NOTE: Only a separator of a single `$` does not match this `is_valid_sep` because it is escaped via `$$`. && is_valid_sep(&sep_cand) { - // Output current group without expansion + // Output the current `$( CONTENTS ) SEP` without expansion output_dollar_group_sep(&mut tokens, current_contents, sep_cand); // Move to the next `$( CONTENTS ) SEP *` @@ -448,7 +448,7 @@ fn consume_dollar_group_sep_star( _ => {} } - // Try to extend separator with any other token + // Try to extend separator if let Some(token) = sep_iter.peek() { sep_cand.push(token.clone()); if is_valid_sep_prefix(&sep_cand) { @@ -461,12 +461,12 @@ fn consume_dollar_group_sep_star( } } - // No repetition found, output `$( CONTENTS ) SEP *` without expansion + // No repetition found, output `$( CONTENTS ) SEP` without expansion output_dollar_group_sep(&mut tokens, current_contents, sep_cand); tokens } -/// Output `$( CONTENTS ) SEP *` without expansion +/// Output `$( CONTENTS ) SEP` without expansion. fn output_dollar_group_sep( tokens: &mut TokenStream, contents: TokenStream, @@ -566,6 +566,7 @@ fn is_valid_sep(ts: &[TokenTree]) -> bool { } } +/// Handle repetition by expanding `$( CONTENTS ) SEP *` to `{ REP_EXPANDED }`. fn expand_dollar_group_sep_star(tokens: &mut TokenStream, contents: TokenStream, sep: TokenStream) { let mut rep_expanded = TokenStream::new(); let meta_vars = collect_meta_vars(contents.clone());