diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index de44a2031ab8..99196210e004 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -915,16 +915,6 @@ impl Stmt { } } - pub fn tokens_mut(&mut self) -> Option<&mut LazyTokenStream> { - match self.kind { - StmtKind::Local(ref mut local) => local.tokens.as_mut(), - StmtKind::Item(ref mut item) => item.tokens.as_mut(), - StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => expr.tokens.as_mut(), - StmtKind::Empty => None, - StmtKind::MacCall(ref mut mac) => mac.tokens.as_mut(), - } - } - pub fn has_trailing_semicolon(&self) -> bool { match &self.kind { StmtKind::Semi(_) => true, diff --git a/compiler/rustc_ast/src/ast_like.rs b/compiler/rustc_ast/src/ast_like.rs index 6649cda69a0b..a71f2ac98150 100644 --- a/compiler/rustc_ast/src/ast_like.rs +++ b/compiler/rustc_ast/src/ast_like.rs @@ -11,13 +11,7 @@ use super::{AttrVec, Attribute, Stmt, StmtKind}; pub trait AstLike: Sized { fn attrs(&self) -> &[Attribute]; fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec)); - /// Called by `Parser::collect_tokens` to store the collected - /// tokens inside an AST node - fn finalize_tokens(&mut self, _tokens: LazyTokenStream) { - // This default impl makes this trait easier to implement - // in tools like `rust-analyzer` - panic!("`finalize_tokens` is not supported!") - } + fn tokens_mut(&mut self) -> Option<&mut Option>; } impl AstLike for P { @@ -27,8 +21,8 @@ impl AstLike for P { fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec)) { (**self).visit_attrs(f); } - fn finalize_tokens(&mut self, tokens: LazyTokenStream) { - (**self).finalize_tokens(tokens) + fn tokens_mut(&mut self) -> Option<&mut Option> { + (**self).tokens_mut() } } @@ -42,12 +36,12 @@ fn visit_attrvec(attrs: &mut AttrVec, f: impl FnOnce(&mut Vec)) { impl AstLike for StmtKind { fn attrs(&self) -> &[Attribute] { - match *self { - StmtKind::Local(ref local) => local.attrs(), - StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => expr.attrs(), - StmtKind::Item(ref item) => item.attrs(), + match self { + StmtKind::Local(local) => local.attrs(), + StmtKind::Expr(expr) | StmtKind::Semi(expr) => expr.attrs(), + StmtKind::Item(item) => item.attrs(), StmtKind::Empty => &[], - StmtKind::MacCall(ref mac) => &*mac.attrs, + StmtKind::MacCall(mac) => &mac.attrs, } } @@ -60,17 +54,14 @@ impl AstLike for StmtKind { StmtKind::MacCall(mac) => visit_attrvec(&mut mac.attrs, f), } } - fn finalize_tokens(&mut self, tokens: LazyTokenStream) { - let stmt_tokens = match self { - StmtKind::Local(ref mut local) => &mut local.tokens, - StmtKind::Item(ref mut item) => &mut item.tokens, - StmtKind::Expr(ref mut expr) | StmtKind::Semi(ref mut expr) => &mut expr.tokens, - StmtKind::Empty => return, - StmtKind::MacCall(ref mut mac) => &mut mac.tokens, - }; - if stmt_tokens.is_none() { - *stmt_tokens = Some(tokens); - } + fn tokens_mut(&mut self) -> Option<&mut Option> { + Some(match self { + StmtKind::Local(local) => &mut local.tokens, + StmtKind::Item(item) => &mut item.tokens, + StmtKind::Expr(expr) | StmtKind::Semi(expr) => &mut expr.tokens, + StmtKind::Empty => return None, + StmtKind::MacCall(mac) => &mut mac.tokens, + }) } } @@ -82,8 +73,8 @@ impl AstLike for Stmt { fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec)) { self.kind.visit_attrs(f); } - fn finalize_tokens(&mut self, tokens: LazyTokenStream) { - self.kind.finalize_tokens(tokens) + fn tokens_mut(&mut self) -> Option<&mut Option> { + self.kind.tokens_mut() } } @@ -92,17 +83,13 @@ impl AstLike for Attribute { &[] } fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec)) {} - fn finalize_tokens(&mut self, tokens: LazyTokenStream) { - match &mut self.kind { - AttrKind::Normal(_, attr_tokens) => { - if attr_tokens.is_none() { - *attr_tokens = Some(tokens); - } + fn tokens_mut(&mut self) -> Option<&mut Option> { + Some(match &mut self.kind { + AttrKind::Normal(_, tokens) => tokens, + kind @ AttrKind::DocComment(..) => { + panic!("Called tokens_mut on doc comment attr {:?}", kind) } - AttrKind::DocComment(..) => { - panic!("Called finalize_tokens on doc comment attr {:?}", self) - } - } + }) } } @@ -115,10 +102,8 @@ impl AstLike for Option { inner.visit_attrs(f); } } - fn finalize_tokens(&mut self, tokens: LazyTokenStream) { - if let Some(inner) = self { - inner.finalize_tokens(tokens); - } + fn tokens_mut(&mut self) -> Option<&mut Option> { + self.as_mut().and_then(|inner| inner.tokens_mut()) } } @@ -152,11 +137,8 @@ macro_rules! derive_has_tokens_and_attrs { VecOrAttrVec::visit(&mut self.attrs, f) } - fn finalize_tokens(&mut self, tokens: LazyTokenStream) { - if self.tokens.is_none() { - self.tokens = Some(tokens); - } - + fn tokens_mut(&mut self) -> Option<&mut Option> { + Some(&mut self.tokens) } } )* } @@ -173,7 +155,9 @@ macro_rules! derive_has_attrs_no_tokens { VecOrAttrVec::visit(&mut self.attrs, f) } - fn finalize_tokens(&mut self, _tokens: LazyTokenStream) {} + fn tokens_mut(&mut self) -> Option<&mut Option> { + None + } } )* } } @@ -185,14 +169,10 @@ macro_rules! derive_has_tokens_no_attrs { &[] } - fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec)) { - } - - fn finalize_tokens(&mut self, tokens: LazyTokenStream) { - if self.tokens.is_none() { - self.tokens = Some(tokens); - } + fn visit_attrs(&mut self, _f: impl FnOnce(&mut Vec)) {} + fn tokens_mut(&mut self) -> Option<&mut Option> { + Some(&mut self.tokens) } } )* } diff --git a/compiler/rustc_builtin_macros/src/derive.rs b/compiler/rustc_builtin_macros/src/derive.rs index fad64858ce3f..c307d8b2f530 100644 --- a/compiler/rustc_builtin_macros/src/derive.rs +++ b/compiler/rustc_builtin_macros/src/derive.rs @@ -1,4 +1,4 @@ -use rustc_ast::{self as ast, token, ItemKind, MetaItemKind, NestedMetaItem, StmtKind}; +use rustc_ast::{self as ast, token, AstLike, ItemKind, MetaItemKind, NestedMetaItem, StmtKind}; use rustc_errors::{struct_span_err, Applicability}; use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier}; use rustc_expand::config::StripUnconfigured; @@ -59,15 +59,9 @@ impl MultiItemModifier for Expander { // Erase the tokens if cfg-stripping modified the item // This will cause us to synthesize fake tokens // when `nt_to_tokenstream` is called on this item. - match &mut item { - Annotatable::Item(item) => item, - Annotatable::Stmt(stmt) => match &mut stmt.kind { - StmtKind::Item(item) => item, - _ => unreachable!(), - }, - _ => unreachable!(), + if let Some(tokens) = item.tokens_mut() { + *tokens = None; } - .tokens = None; } ExpandResult::Ready(vec![item]) } diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs index 929cc56294d3..ffdb30819b19 100644 --- a/compiler/rustc_expand/src/base.rs +++ b/compiler/rustc_expand/src/base.rs @@ -81,8 +81,22 @@ impl AstLike for Annotatable { } } - fn finalize_tokens(&mut self, tokens: LazyTokenStream) { - panic!("Called finalize_tokens on an Annotatable: {:?}", tokens); + fn tokens_mut(&mut self) -> Option<&mut Option> { + match self { + Annotatable::Item(item) => item.tokens_mut(), + Annotatable::TraitItem(trait_item) => trait_item.tokens_mut(), + Annotatable::ImplItem(impl_item) => impl_item.tokens_mut(), + Annotatable::ForeignItem(foreign_item) => foreign_item.tokens_mut(), + Annotatable::Stmt(stmt) => stmt.tokens_mut(), + Annotatable::Expr(expr) => expr.tokens_mut(), + Annotatable::Arm(arm) => arm.tokens_mut(), + Annotatable::Field(field) => field.tokens_mut(), + Annotatable::FieldPat(fp) => fp.tokens_mut(), + Annotatable::GenericParam(gp) => gp.tokens_mut(), + Annotatable::Param(p) => p.tokens_mut(), + Annotatable::StructField(sf) => sf.tokens_mut(), + Annotatable::Variant(v) => v.tokens_mut(), + } } } diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index f45d8d6c7a00..7512f46988c9 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -72,6 +72,10 @@ impl<'a> Parser<'a> { let cursor_snapshot = self.token_cursor.clone(); let (mut ret, trailing_token) = f(self, attrs.attrs)?; + let tokens = match ret.tokens_mut() { + Some(tokens) if tokens.is_none() => tokens, + _ => return Ok(ret), + }; // Produces a `TokenStream` on-demand. Using `cursor_snapshot` // and `num_calls`, we can reconstruct the `TokenStream` seen @@ -128,14 +132,14 @@ impl<'a> Parser<'a> { } } - let lazy_impl = LazyTokenStreamImpl { + *tokens = Some(LazyTokenStream::new(LazyTokenStreamImpl { start_token, num_calls, cursor_snapshot, desugar_doc_comments: self.desugar_doc_comments, append_unglued_token: self.token_cursor.append_unglued_token.clone(), - }; - ret.finalize_tokens(LazyTokenStream::new(lazy_impl)); + })); + Ok(ret) } }