Merge pull request #20163 from ChayimFriedman2/parser-per-token-edition

fix: Use per-token, not global, edition in the parser

Author: Veykril

src/tools/rust-analyzer/crates/hir-def/src/macro_expansion_tests/mod.rs

@@ -372,7 +372,6 @@ impl ProcMacroExpander for IdentityWhenValidProcMacroExpander {
             subtree,
             syntax_bridge::TopEntryPoint::MacroItems,
             &mut |_| span::Edition::CURRENT,
-            span::Edition::CURRENT,
         );
         if parse.errors().is_empty() {
             Ok(subtree.clone())
@@ -413,10 +412,7 @@ fn regression_20171() {
         #dollar_crate::panic::panic_2021!();
     }}
         };
-    token_tree_to_syntax_node(
-        &tt,
-        syntax_bridge::TopEntryPoint::MacroStmts,
-        &mut |_| Edition::CURRENT,
-        Edition::CURRENT,
-    );
+    token_tree_to_syntax_node(&tt, syntax_bridge::TopEntryPoint::MacroStmts, &mut |_| {
+        Edition::CURRENT
+    });
 }

src/tools/rust-analyzer/crates/hir-expand/src/builtin/derive_macro.rs

@@ -392,12 +392,7 @@ fn to_adt_syntax(
     tt: &tt::TopSubtree,
     call_site: Span,
 ) -> Result<(ast::Adt, span::SpanMap<SyntaxContext>), ExpandError> {
-    let (parsed, tm) = crate::db::token_tree_to_syntax_node(
-        db,
-        tt,
-        crate::ExpandTo::Items,
-        parser::Edition::CURRENT_FIXME,
-    );
+    let (parsed, tm) = crate::db::token_tree_to_syntax_node(db, tt, crate::ExpandTo::Items);
     let macro_items = ast::MacroItems::cast(parsed.syntax_node())
         .ok_or_else(|| ExpandError::other(call_site, "invalid item definition"))?;
     let item =

src/tools/rust-analyzer/crates/hir-expand/src/db.rs

@@ -315,8 +315,7 @@ pub fn expand_speculative(
     let expand_to = loc.expand_to();
 
     fixup::reverse_fixups(&mut speculative_expansion.value, &undo_info);
-    let (node, rev_tmap) =
-        token_tree_to_syntax_node(db, &speculative_expansion.value, expand_to, loc.def.edition);
+    let (node, rev_tmap) = token_tree_to_syntax_node(db, &speculative_expansion.value, expand_to);
 
     let syntax_node = node.syntax_node();
     let token = rev_tmap
@@ -358,7 +357,6 @@ fn parse_macro_expansion(
 ) -> ExpandResult<(Parse<SyntaxNode>, Arc<ExpansionSpanMap>)> {
     let _p = tracing::info_span!("parse_macro_expansion").entered();
     let loc = db.lookup_intern_macro_call(macro_file);
-    let def_edition = loc.def.edition;
     let expand_to = loc.expand_to();
     let mbe::ValueResult { value: (tt, matched_arm), err } = macro_expand(db, macro_file, loc);
 
@@ -369,7 +367,6 @@ fn parse_macro_expansion(
             CowArc::Owned(it) => it,
         },
         expand_to,
-        def_edition,
     );
     rev_token_map.matched_arm = matched_arm;
 
@@ -731,7 +728,6 @@ pub(crate) fn token_tree_to_syntax_node(
     db: &dyn ExpandDatabase,
     tt: &tt::TopSubtree,
     expand_to: ExpandTo,
-    edition: parser::Edition,
 ) -> (Parse<SyntaxNode>, ExpansionSpanMap) {
     let entry_point = match expand_to {
         ExpandTo::Statements => syntax_bridge::TopEntryPoint::MacroStmts,
@@ -740,7 +736,7 @@ pub(crate) fn token_tree_to_syntax_node(
         ExpandTo::Type => syntax_bridge::TopEntryPoint::Type,
         ExpandTo::Expr => syntax_bridge::TopEntryPoint::Expr,
     };
-    syntax_bridge::token_tree_to_syntax_node(tt, entry_point, &mut |ctx| ctx.edition(db), edition)
+    syntax_bridge::token_tree_to_syntax_node(tt, entry_point, &mut |ctx| ctx.edition(db))
 }
 
 fn check_tt_count(tt: &tt::TopSubtree) -> Result<(), ExpandResult<()>> {

src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs

@@ -535,7 +535,6 @@ mod tests {
             &tt,
             syntax_bridge::TopEntryPoint::MacroItems,
             &mut |_| parser::Edition::CURRENT,
-            parser::Edition::CURRENT,
         );
         assert!(
             parse.errors().is_empty(),

src/tools/rust-analyzer/crates/mbe/src/lib.rs

@@ -399,7 +399,7 @@ pub fn expect_fragment<'t>(
     let buffer = tt_iter.remaining();
     // FIXME: Pass the correct edition per token. Due to the split between mbe and hir-expand it's complicated.
     let parser_input = to_parser_input(buffer, &mut |_ctx| edition);
-    let tree_traversal = entry_point.parse(&parser_input, edition);
+    let tree_traversal = entry_point.parse(&parser_input);
     let mut cursor = buffer.cursor();
     let mut error = false;
     for step in tree_traversal.iter() {

src/tools/rust-analyzer/crates/mbe/src/tests.rs

@@ -66,12 +66,8 @@ fn check_(
     if render_debug {
         format_to!(expect_res, "{:#?}\n\n", res.value.0);
     }
-    let (node, _) = syntax_bridge::token_tree_to_syntax_node(
-        &res.value.0,
-        parse,
-        &mut |_| def_edition,
-        def_edition,
-    );
+    let (node, _) =
+        syntax_bridge::token_tree_to_syntax_node(&res.value.0, parse, &mut |_| def_edition);
     format_to!(
         expect_res,
         "{}",

src/tools/rust-analyzer/crates/parser/src/grammar/generic_args.rs

@@ -59,7 +59,7 @@ pub(crate) fn generic_arg(p: &mut Parser<'_>) -> bool {
 
         // test edition_2015_dyn_prefix_inside_generic_arg 2015
         // type A = Foo<dyn T>;
-        T![ident] if !p.edition().at_least_2018() && types::is_dyn_weak(p) => type_arg(p),
+        T![ident] if !p.current_edition().at_least_2018() && types::is_dyn_weak(p) => type_arg(p),
         // test macro_inside_generic_arg
         // type A = Foo<syn::Token![_]>;
         k if PATH_NAME_REF_KINDS.contains(k) => {

src/tools/rust-analyzer/crates/parser/src/grammar/types.rs

@@ -54,7 +54,9 @@ fn type_with_bounds_cond(p: &mut Parser<'_>, allow_bounds: bool) {
         T![dyn] => dyn_trait_type(p),
         // Some path types are not allowed to have bounds (no plus)
         T![<] => path_type_bounds(p, allow_bounds),
-        T![ident] if !p.edition().at_least_2018() && is_dyn_weak(p) => dyn_trait_type_weak(p),
+        T![ident] if !p.current_edition().at_least_2018() && is_dyn_weak(p) => {
+            dyn_trait_type_weak(p)
+        }
         _ if paths::is_path_start(p) => path_or_macro_type(p, allow_bounds),
         LIFETIME_IDENT if p.nth_at(1, T![+]) => bare_dyn_trait_type(p),
         _ => {

src/tools/rust-analyzer/crates/parser/src/input.rs

@@ -1,5 +1,7 @@
 //! See [`Input`].
 
+use edition::Edition;
+
 use crate::SyntaxKind;
 
 #[allow(non_camel_case_types)]
@@ -16,6 +18,7 @@ pub struct Input {
     kind: Vec<SyntaxKind>,
     joint: Vec<bits>,
     contextual_kind: Vec<SyntaxKind>,
+    edition: Vec<Edition>,
 }
 
 /// `pub` impl used by callers to create `Tokens`.
@@ -26,15 +29,16 @@ impl Input {
             kind: Vec::with_capacity(capacity),
             joint: Vec::with_capacity(capacity / size_of::<bits>()),
             contextual_kind: Vec::with_capacity(capacity),
+            edition: Vec::with_capacity(capacity),
         }
     }
     #[inline]
-    pub fn push(&mut self, kind: SyntaxKind) {
-        self.push_impl(kind, SyntaxKind::EOF)
+    pub fn push(&mut self, kind: SyntaxKind, edition: Edition) {
+        self.push_impl(kind, SyntaxKind::EOF, edition)
     }
     #[inline]
-    pub fn push_ident(&mut self, contextual_kind: SyntaxKind) {
-        self.push_impl(SyntaxKind::IDENT, contextual_kind)
+    pub fn push_ident(&mut self, contextual_kind: SyntaxKind, edition: Edition) {
+        self.push_impl(SyntaxKind::IDENT, contextual_kind, edition)
     }
     /// Sets jointness for the last token we've pushed.
     ///
@@ -59,13 +63,14 @@ impl Input {
         self.joint[idx] |= 1 << b_idx;
     }
     #[inline]
-    fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind) {
+    fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind, edition: Edition) {
         let idx = self.len();
         if idx.is_multiple_of(bits::BITS as usize) {
             self.joint.push(0);
         }
         self.kind.push(kind);
         self.contextual_kind.push(contextual_kind);
+        self.edition.push(edition);
     }
 }
 
@@ -77,6 +82,9 @@ impl Input {
     pub(crate) fn contextual_kind(&self, idx: usize) -> SyntaxKind {
         self.contextual_kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
     }
+    pub(crate) fn edition(&self, idx: usize) -> Edition {
+        self.edition[idx]
+    }
     pub(crate) fn is_joint(&self, n: usize) -> bool {
         let (idx, b_idx) = self.bit_index(n);
         self.joint[idx] & (1 << b_idx) != 0

src/tools/rust-analyzer/crates/parser/src/lib.rs

@@ -88,7 +88,7 @@ pub enum TopEntryPoint {
 }
 
 impl TopEntryPoint {
-    pub fn parse(&self, input: &Input, edition: Edition) -> Output {
+    pub fn parse(&self, input: &Input) -> Output {
         let _p = tracing::info_span!("TopEntryPoint::parse", ?self).entered();
         let entry_point: fn(&'_ mut parser::Parser<'_>) = match self {
             TopEntryPoint::SourceFile => grammar::entry::top::source_file,
@@ -99,7 +99,7 @@ impl TopEntryPoint {
             TopEntryPoint::Expr => grammar::entry::top::expr,
             TopEntryPoint::MetaItem => grammar::entry::top::meta_item,
         };
-        let mut p = parser::Parser::new(input, edition);
+        let mut p = parser::Parser::new(input);
         entry_point(&mut p);
         let events = p.finish();
         let res = event::process(events);
@@ -151,7 +151,7 @@ pub enum PrefixEntryPoint {
 }
 
 impl PrefixEntryPoint {
-    pub fn parse(&self, input: &Input, edition: Edition) -> Output {
+    pub fn parse(&self, input: &Input) -> Output {
         let entry_point: fn(&'_ mut parser::Parser<'_>) = match self {
             PrefixEntryPoint::Vis => grammar::entry::prefix::vis,
             PrefixEntryPoint::Block => grammar::entry::prefix::block,
@@ -164,7 +164,7 @@ impl PrefixEntryPoint {
             PrefixEntryPoint::Item => grammar::entry::prefix::item,
             PrefixEntryPoint::MetaItem => grammar::entry::prefix::meta_item,
         };
-        let mut p = parser::Parser::new(input, edition);
+        let mut p = parser::Parser::new(input);
         entry_point(&mut p);
         let events = p.finish();
         event::process(events)
@@ -188,9 +188,9 @@ impl Reparser {
     ///
     /// Tokens must start with `{`, end with `}` and form a valid brace
     /// sequence.
-    pub fn parse(self, tokens: &Input, edition: Edition) -> Output {
+    pub fn parse(self, tokens: &Input) -> Output {
         let Reparser(r) = self;
-        let mut p = parser::Parser::new(tokens, edition);
+        let mut p = parser::Parser::new(tokens);
         r(&mut p);
         let events = p.finish();
         event::process(events)