From 49b6f677a50ac57051806eef1727b6a40bb2a196 Mon Sep 17 00:00:00 2001 From: Chayim Refael Friedman Date: Mon, 29 Dec 2025 04:14:12 +0200 Subject: [PATCH 1/2] Separate the public `tt` API from how they're stored And don't provide any insight into the actual storage (no references, no "cheat" methods like `flat_tokens()`), as we're going to rewrite it to be more memory efficient. --- .../rust-analyzer/crates/cfg/src/cfg_expr.rs | 11 +- .../hir-def/src/nameres/attr_resolution.rs | 2 +- .../crates/hir-def/src/nameres/collector.rs | 4 +- .../crates/hir-def/src/nameres/proc_macro.rs | 51 ++--- .../crates/hir-expand/src/attrs.rs | 19 +- .../crates/hir-expand/src/builtin/fn_macro.rs | 65 +++--- .../crates/hir-expand/src/builtin/quote.rs | 9 +- .../rust-analyzer/crates/hir-expand/src/db.rs | 14 +- .../crates/hir-expand/src/eager.rs | 2 +- .../crates/hir-expand/src/fixup.rs | 95 ++------- .../crates/hir-expand/src/mod_path.rs | 10 +- .../rust-analyzer/crates/mbe/src/benchmark.rs | 2 +- .../rust-analyzer/crates/mbe/src/expander.rs | 2 +- .../crates/mbe/src/expander/matcher.rs | 10 +- .../crates/mbe/src/expander/transcriber.rs | 22 ++- .../rust-analyzer/crates/mbe/src/parser.rs | 8 +- .../src/legacy_protocol/msg/flat.rs | 20 +- .../crates/proc-macro-api/src/lib.rs | 18 +- .../crates/syntax-bridge/src/lib.rs | 14 +- .../crates/syntax-bridge/src/tests.rs | 2 +- .../crates/test-fixture/src/lib.rs | 42 ++-- .../rust-analyzer/crates/tt/src/buffer.rs | 4 +- src/tools/rust-analyzer/crates/tt/src/iter.rs | 32 +-- src/tools/rust-analyzer/crates/tt/src/lib.rs | 185 +++++++++++++++--- 24 files changed, 350 insertions(+), 293 deletions(-) diff --git a/src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs b/src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs index 2f1fc8ae49e9..c5da8443a68f 100644 --- a/src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs +++ b/src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs @@ -197,17 +197,18 @@ fn next_cfg_expr(it: &mut tt::iter::TtIter<'_>) -> Option { Some(_) => return Some(CfgExpr::Invalid), }; - let ret = match it.peek() { + let mut it_clone = it.clone(); + let ret = match it_clone.next() { Some(TtElement::Leaf(tt::Leaf::Punct(punct))) // Don't consume on e.g. `=>`. if punct.char == '=' && (punct.spacing == tt::Spacing::Alone - || it.remaining().flat_tokens().get(1).is_none_or(|peek2| { - !matches!(peek2, tt::TokenTree::Leaf(tt::Leaf::Punct(_))) + || it_clone.peek().is_none_or(|peek2| { + !matches!(peek2, tt::TtElement::Leaf(tt::Leaf::Punct(_))) })) => { - match it.remaining().flat_tokens().get(1) { - Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => { + match it_clone.next() { + Some(tt::TtElement::Leaf(tt::Leaf::Literal(literal))) => { it.next(); it.next(); CfgAtom::KeyValue { key: name, value: literal.symbol.clone() }.into() diff --git a/src/tools/rust-analyzer/crates/hir-def/src/nameres/attr_resolution.rs b/src/tools/rust-analyzer/crates/hir-def/src/nameres/attr_resolution.rs index 1cbd2c10b5a1..062b55fcefdf 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/nameres/attr_resolution.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/nameres/attr_resolution.rs @@ -114,7 +114,7 @@ pub(super) fn attr_macro_as_call_id( let arg = match macro_attr.input.as_deref() { Some(AttrInput::TokenTree(tt)) => { let mut tt = tt.clone(); - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Invisible; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); Some(tt) } diff --git a/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs b/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs index a7f687a3168b..822da678d797 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs @@ -2466,8 +2466,8 @@ impl ModCollector<'_, '_> { None => { let explicit_name = attrs.by_key(sym::rustc_builtin_macro).tt_values().next().and_then(|tt| { - match tt.token_trees().flat_tokens().first() { - Some(tt::TokenTree::Leaf(tt::Leaf::Ident(name))) => Some(name), + match tt.token_trees().iter().next() { + Some(tt::TtElement::Leaf(tt::Leaf::Ident(name))) => Some(name), _ => None, } }); diff --git a/src/tools/rust-analyzer/crates/hir-def/src/nameres/proc_macro.rs b/src/tools/rust-analyzer/crates/hir-def/src/nameres/proc_macro.rs index cd45afe57d7c..91d664938bf2 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/nameres/proc_macro.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/nameres/proc_macro.rs @@ -2,10 +2,11 @@ use hir_expand::name::{AsName, Name}; use intern::sym; +use itertools::Itertools; use crate::{ item_tree::Attrs, - tt::{Leaf, TokenTree, TopSubtree, TtElement}, + tt::{Leaf, TopSubtree, TtElement}, }; #[derive(Debug, PartialEq, Eq)] @@ -61,35 +62,35 @@ impl Attrs<'_> { // This fn is intended for `#[proc_macro_derive(..)]` and `#[rustc_builtin_macro(..)]`, which have // the same structure. -#[rustfmt::skip] pub(crate) fn parse_macro_name_and_helper_attrs(tt: &TopSubtree) -> Option<(Name, Box<[Name]>)> { - match tt.token_trees().flat_tokens() { + if let Some([TtElement::Leaf(Leaf::Ident(trait_name))]) = + tt.token_trees().iter().collect_array() + { // `#[proc_macro_derive(Trait)]` // `#[rustc_builtin_macro(Trait)]` - [TokenTree::Leaf(Leaf::Ident(trait_name))] => Some((trait_name.as_name(), Box::new([]))), - + Some((trait_name.as_name(), Box::new([]))) + } else if let Some( + [ + TtElement::Leaf(Leaf::Ident(trait_name)), + TtElement::Leaf(Leaf::Punct(comma)), + TtElement::Leaf(Leaf::Ident(attributes)), + TtElement::Subtree(_, helpers), + ], + ) = tt.token_trees().iter().collect_array() + && comma.char == ',' + && attributes.sym == sym::attributes + { // `#[proc_macro_derive(Trait, attributes(helper1, helper2, ...))]` // `#[rustc_builtin_macro(Trait, attributes(helper1, helper2, ...))]` - [ - TokenTree::Leaf(Leaf::Ident(trait_name)), - TokenTree::Leaf(Leaf::Punct(comma)), - TokenTree::Leaf(Leaf::Ident(attributes)), - TokenTree::Subtree(_), - .. - ] if comma.char == ',' && attributes.sym == sym::attributes => - { - let helpers = tt::TokenTreesView::new(&tt.token_trees().flat_tokens()[3..]).try_into_subtree()?; - let helpers = helpers - .iter() - .filter_map(|tt| match tt { - TtElement::Leaf(Leaf::Ident(helper)) => Some(helper.as_name()), - _ => None, - }) - .collect::>(); + let helpers = helpers + .filter_map(|tt| match tt { + TtElement::Leaf(Leaf::Ident(helper)) => Some(helper.as_name()), + _ => None, + }) + .collect::>(); - Some((trait_name.as_name(), helpers)) - } - - _ => None, + Some((trait_name.as_name(), helpers)) + } else { + None } } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs b/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs index e1807cd2e1e9..5a440187203c 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs @@ -36,6 +36,7 @@ use base_db::Crate; use cfg::{CfgExpr, CfgOptions}; use either::Either; use intern::{Interned, Symbol}; +use itertools::Itertools; use mbe::{DelimiterKind, Punct}; use parser::T; use smallvec::SmallVec; @@ -453,10 +454,10 @@ impl Attr { } /// #[path(ident)] - pub fn single_ident_value(&self) -> Option<&tt::Ident> { + pub fn single_ident_value(&self) -> Option { match self.input.as_deref()? { - AttrInput::TokenTree(tt) => match tt.token_trees().flat_tokens() { - [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] => Some(ident), + AttrInput::TokenTree(tt) => match tt.token_trees().iter().collect_array() { + Some([tt::TtElement::Leaf(tt::Leaf::Ident(ident))]) => Some(ident), _ => None, }, _ => None, @@ -492,7 +493,7 @@ fn parse_path_comma_token_tree<'a>( args.token_trees() .split(|tt| matches!(tt, tt::TtElement::Leaf(tt::Leaf::Punct(Punct { char: ',', .. })))) .filter_map(move |tts| { - let span = tts.flat_tokens().first()?.first_span(); + let span = tts.first_span()?; Some((ModPath::from_tt(db, tts)?, span, tts)) }) } @@ -611,16 +612,12 @@ impl AttrId { else { return derive_attr_range; }; - let (Some(first_tt), Some(last_tt)) = - (derive_tts.flat_tokens().first(), derive_tts.flat_tokens().last()) + let (Some(first_span), Some(last_span)) = (derive_tts.first_span(), derive_tts.last_span()) else { return derive_attr_range; }; - let start = first_tt.first_span().range.start(); - let end = match last_tt { - tt::TokenTree::Leaf(it) => it.span().range.end(), - tt::TokenTree::Subtree(it) => it.delimiter.close.range.end(), - }; + let start = first_span.range.start(); + let end = last_span.range.end(); TextRange::new(start, end) } } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/builtin/fn_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/builtin/fn_macro.rs index 3d630cfc1c8e..011a9530c4d3 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/builtin/fn_macro.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/builtin/fn_macro.rs @@ -210,7 +210,7 @@ fn stringify_expand( tt: &tt::TopSubtree, span: Span, ) -> ExpandResult { - let pretty = ::tt::pretty(tt.token_trees().flat_tokens()); + let pretty = ::tt::pretty(tt.token_trees()); let expanded = quote! {span => #pretty @@ -283,7 +283,7 @@ fn format_args_expand( ) -> ExpandResult { let pound = mk_pound(span); let mut tt = tt.clone(); - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis); ExpandResult::ok(quote! {span => builtin #pound format_args #tt }) @@ -297,14 +297,14 @@ fn format_args_nl_expand( ) -> ExpandResult { let pound = mk_pound(span); let mut tt = tt.clone(); - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis; - if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - symbol: text, - kind: tt::LitKind::Str, - .. - }))) = tt.0.get_mut(1) + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis); + let lit = tt.as_token_trees().iter_flat_tokens().nth(1); + if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal( + mut lit @ tt::Literal { kind: tt::LitKind::Str, .. }, + ))) = lit { - *text = Symbol::intern(&format_smolstr!("{}\\n", text.as_str())); + lit.symbol = Symbol::intern(&format_smolstr!("{}\\n", lit.symbol.as_str())); + tt.set_token(1, lit.into()); } ExpandResult::ok(quote! {span => builtin #pound format_args #tt @@ -318,7 +318,7 @@ fn asm_expand( span: Span, ) -> ExpandResult { let mut tt = tt.clone(); - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis); let pound = mk_pound(span); let expanded = quote! {span => builtin #pound asm #tt @@ -333,7 +333,7 @@ fn global_asm_expand( span: Span, ) -> ExpandResult { let mut tt = tt.clone(); - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis); let pound = mk_pound(span); let expanded = quote! {span => builtin #pound global_asm #tt @@ -348,7 +348,7 @@ fn naked_asm_expand( span: Span, ) -> ExpandResult { let mut tt = tt.clone(); - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis); let pound = mk_pound(span); let expanded = quote! {span => builtin #pound naked_asm #tt @@ -478,11 +478,11 @@ fn unreachable_expand( // Pass the original arguments let mut subtree = tt.clone(); - *subtree.top_subtree_delimiter_mut() = tt::Delimiter { + subtree.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis); + subtree.set_top_subtree_delimiter_span(tt::DelimSpan { open: call_site_span, close: call_site_span, - kind: tt::DelimiterKind::Parenthesis, - }; + }); // Expand to a macro call `$crate::panic::panic_{edition}` let call = quote!(call_site_span =>#dollar_crate::panic::#mac! #subtree); @@ -518,16 +518,17 @@ fn compile_error_expand( tt: &tt::TopSubtree, span: Span, ) -> ExpandResult { - let err = match &*tt.0 { - [ - _, - tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - symbol: text, - span: _, - kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), - suffix: _, - })), - ] => ExpandError::other(span, Box::from(unescape_symbol(text).as_str())), + let err = match tt.iter().collect_array() { + Some( + [ + tt::TtElement::Leaf(tt::Leaf::Literal(tt::Literal { + symbol: text, + span: _, + kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), + suffix: _, + })), + ], + ) => ExpandError::other(span, Box::from(unescape_symbol(&text).as_str())), _ => ExpandError::other(span, "`compile_error!` argument must be a string"), }; @@ -556,7 +557,7 @@ fn concat_expand( // to ensure the right parsing order, so skip the parentheses here. Ideally we'd // implement rustc's model. cc https://github.com/rust-lang/rust-analyzer/pull/10623 if let TtElement::Subtree(subtree, subtree_iter) = &t - && let [tt::TokenTree::Leaf(tt)] = subtree_iter.remaining().flat_tokens() + && let Some([tt::TtElement::Leaf(tt)]) = subtree_iter.clone().collect_array() && subtree.delimiter.kind == tt::DelimiterKind::Parenthesis { t = TtElement::Leaf(tt); @@ -663,7 +664,7 @@ fn concat_bytes_expand( kind, suffix: _, })) => { - record_span(*span); + record_span(span); match kind { tt::LitKind::Byte => { if let Ok(b) = unescape_byte(text.as_str()) { @@ -679,7 +680,7 @@ fn concat_bytes_expand( bytes.extend(text.as_str().escape_debug()); } _ => { - err.get_or_insert(ExpandError::other(*span, "unexpected token")); + err.get_or_insert(ExpandError::other(span, "unexpected token")); break; } } @@ -733,7 +734,7 @@ fn concat_bytes_expand_subtree( if let Ok(b) = unescape_byte(text.as_str()) { bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32))); } - record_span(*span); + record_span(span); } TtElement::Leaf(tt::Leaf::Literal(tt::Literal { symbol: text, @@ -741,7 +742,7 @@ fn concat_bytes_expand_subtree( kind: tt::LitKind::Integer, suffix: _, })) => { - record_span(*span); + record_span(span); if let Ok(b) = text.as_str().parse::() { bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32))); } @@ -796,13 +797,13 @@ fn parse_string(tt: &tt::TopSubtree) -> Result<(Symbol, Span), ExpandError> { span, kind: tt::LitKind::Str, suffix: _, - })) => Ok((unescape_symbol(text), *span)), + })) => Ok((unescape_symbol(&text), span)), TtElement::Leaf(tt::Leaf::Literal(tt::Literal { symbol: text, span, kind: tt::LitKind::StrRaw(_), suffix: _, - })) => Ok((text.clone(), *span)), + })) => Ok((text.clone(), span)), TtElement::Leaf(l) => Err(*l.span()), TtElement::Subtree(tt, _) => Err(tt.delimiter.open.cover(tt.delimiter.close)), } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/builtin/quote.rs b/src/tools/rust-analyzer/crates/hir-expand/src/builtin/quote.rs index b1d46f5c4e76..4039b6c334ec 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/builtin/quote.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/builtin/quote.rs @@ -163,7 +163,7 @@ impl ToTokenTree for crate::tt::SubtreeView<'_> { impl ToTokenTree for crate::tt::TopSubtree { fn to_tokens(self, _: Span, builder: &mut TopSubtreeBuilder) { - builder.extend_tt_dangerous(self.0); + builder.extend_with_tt(self.as_token_trees()); } } @@ -172,10 +172,9 @@ impl ToTokenTree for crate::tt::TtElement<'_> { match self { crate::tt::TtElement::Leaf(leaf) => builder.push(leaf.clone()), crate::tt::TtElement::Subtree(subtree, subtree_iter) => { - builder.extend_tt_dangerous( - std::iter::once(crate::tt::TokenTree::Subtree(subtree.clone())) - .chain(subtree_iter.remaining().flat_tokens().iter().cloned()), - ); + builder.open(subtree.delimiter.kind, subtree.delimiter.open); + builder.extend_with_tt(subtree_iter.remaining()); + builder.close(subtree.delimiter.close); } } } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/db.rs b/src/tools/rust-analyzer/crates/hir-expand/src/db.rs index 5c517e671be4..40d44cd1dba2 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/db.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/db.rs @@ -237,7 +237,8 @@ pub fn expand_speculative( span, DocCommentDesugarMode::ProcMacro, ); - *tree.top_subtree_delimiter_mut() = tt::Delimiter::invisible_spanned(span); + tree.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); + tree.set_top_subtree_delimiter_span(tt::DelimSpan::from_single(span)); tree }, ) @@ -255,7 +256,7 @@ pub fn expand_speculative( span, DocCommentDesugarMode::ProcMacro, ); - attr_arg.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Invisible; + attr_arg.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); Some(attr_arg) } _ => None, @@ -270,7 +271,8 @@ pub fn expand_speculative( let mut speculative_expansion = match loc.def.kind { MacroDefKind::ProcMacro(ast, expander, _) => { let span = db.proc_macro_span(ast); - *tt.top_subtree_delimiter_mut() = tt::Delimiter::invisible_spanned(span); + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); + tt.set_top_subtree_delimiter_span(tt::DelimSpan::from_single(span)); expander.expand( db, loc.def.krate, @@ -430,7 +432,7 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult { ( Arc::new(tt::TopSubtree::from_token_trees( tt::Delimiter { open: span, close: span, kind }, - tt::TokenTreesView::new(&[]), + tt::TokenTreesView::empty(), )), SyntaxFixupUndoInfo::default(), span, @@ -478,7 +480,7 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult { ); if loc.def.is_proc_macro() { // proc macros expect their inputs without parentheses, MBEs expect it with them included - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Invisible; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); } return (Arc::new(tt), SyntaxFixupUndoInfo::NONE, span); } @@ -512,7 +514,7 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult { if loc.def.is_proc_macro() { // proc macros expect their inputs without parentheses, MBEs expect it with them included - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Invisible; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); } (Arc::new(tt), undo_info, span) diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/eager.rs b/src/tools/rust-analyzer/crates/hir-expand/src/eager.rs index 9b65bdac65c0..0b6124ebf359 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/eager.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/eager.rs @@ -96,7 +96,7 @@ pub fn expand_eager_macro_input( DocCommentDesugarMode::Mbe, ); - subtree.top_subtree_delimiter_mut().kind = crate::tt::DelimiterKind::Invisible; + subtree.set_top_subtree_delimiter_kind(crate::tt::DelimiterKind::Invisible); let loc = MacroCallLoc { def, diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs b/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs index 876d87093663..6afc6e5d5e7f 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs @@ -15,7 +15,7 @@ use syntax::{ }; use syntax_bridge::DocCommentDesugarMode; use triomphe::Arc; -use tt::Spacing; +use tt::{Spacing, TransformTtAction, transform_tt}; use crate::{ span_map::SpanMapRef, @@ -343,93 +343,29 @@ fn has_error_to_handle(node: &SyntaxNode) -> bool { pub(crate) fn reverse_fixups(tt: &mut TopSubtree, undo_info: &SyntaxFixupUndoInfo) { let Some(undo_info) = undo_info.original.as_deref() else { return }; let undo_info = &**undo_info; - let delimiter = tt.top_subtree_delimiter_mut(); + let top_subtree = tt.top_subtree(); + let open_span = top_subtree.delimiter.open; + let close_span = top_subtree.delimiter.close; #[allow(deprecated)] if never!( - delimiter.close.anchor.ast_id == FIXUP_DUMMY_AST_ID - || delimiter.open.anchor.ast_id == FIXUP_DUMMY_AST_ID + close_span.anchor.ast_id == FIXUP_DUMMY_AST_ID + || open_span.anchor.ast_id == FIXUP_DUMMY_AST_ID ) { let span = |file_id| Span { range: TextRange::empty(TextSize::new(0)), anchor: SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, ctx: SyntaxContext::root(span::Edition::Edition2015), }; - delimiter.open = span(delimiter.open.anchor.file_id); - delimiter.close = span(delimiter.close.anchor.file_id); + tt.set_top_subtree_delimiter_span(tt::DelimSpan { + open: span(open_span.anchor.file_id), + close: span(close_span.anchor.file_id), + }); } reverse_fixups_(tt, undo_info); } -#[derive(Debug)] -enum TransformTtAction<'a> { - Keep, - ReplaceWith(tt::TokenTreesView<'a>), -} - -impl TransformTtAction<'_> { - fn remove() -> Self { - Self::ReplaceWith(tt::TokenTreesView::new(&[])) - } -} - -/// This function takes a token tree, and calls `callback` with each token tree in it. -/// Then it does what the callback says: keeps the tt or replaces it with a (possibly empty) -/// tts view. -fn transform_tt<'a, 'b>( - tt: &'a mut Vec, - mut callback: impl FnMut(&mut tt::TokenTree) -> TransformTtAction<'b>, -) { - // We need to keep a stack of the currently open subtrees, because we need to update - // them if we change the number of items in them. - let mut subtrees_stack = Vec::new(); - let mut i = 0; - while i < tt.len() { - 'pop_finished_subtrees: while let Some(&subtree_idx) = subtrees_stack.last() { - let tt::TokenTree::Subtree(subtree) = &tt[subtree_idx] else { - unreachable!("non-subtree on subtrees stack"); - }; - if i >= subtree_idx + 1 + subtree.usize_len() { - subtrees_stack.pop(); - } else { - break 'pop_finished_subtrees; - } - } - - let action = callback(&mut tt[i]); - match action { - TransformTtAction::Keep => { - // This cannot be shared with the replaced case, because then we may push the same subtree - // twice, and will update it twice which will lead to errors. - if let tt::TokenTree::Subtree(_) = &tt[i] { - subtrees_stack.push(i); - } - - i += 1; - } - TransformTtAction::ReplaceWith(replacement) => { - let old_len = 1 + match &tt[i] { - tt::TokenTree::Leaf(_) => 0, - tt::TokenTree::Subtree(subtree) => subtree.usize_len(), - }; - let len_diff = replacement.len() as i64 - old_len as i64; - tt.splice(i..i + old_len, replacement.flat_tokens().iter().cloned()); - // Skip the newly inserted replacement, we don't want to visit it. - i += replacement.len(); - - for &subtree_idx in &subtrees_stack { - let tt::TokenTree::Subtree(subtree) = &mut tt[subtree_idx] else { - unreachable!("non-subtree on subtrees stack"); - }; - subtree.len = (i64::from(subtree.len) + len_diff).try_into().unwrap(); - } - } - } - } -} - fn reverse_fixups_(tt: &mut TopSubtree, undo_info: &[TopSubtree]) { - let mut tts = std::mem::take(&mut tt.0).into_vec(); - transform_tt(&mut tts, |tt| match tt { + transform_tt(tt, |tt| match tt { tt::TokenTree::Leaf(leaf) => { let span = leaf.span(); let is_real_leaf = span.anchor.ast_id != FIXUP_DUMMY_AST_ID; @@ -459,7 +395,6 @@ fn reverse_fixups_(tt: &mut TopSubtree, undo_info: &[TopSubtree]) { TransformTtAction::Keep } }); - tt.0 = tts.into_boxed_slice(); } #[cfg(test)] @@ -488,9 +423,9 @@ mod tests { } fn check_subtree_eq(a: &tt::TopSubtree, b: &tt::TopSubtree) -> bool { - let a = a.view().as_token_trees().flat_tokens(); - let b = b.view().as_token_trees().flat_tokens(); - a.len() == b.len() && std::iter::zip(a, b).all(|(a, b)| check_tt_eq(a, b)) + let a = a.view().as_token_trees().iter_flat_tokens(); + let b = b.view().as_token_trees().iter_flat_tokens(); + a.len() == b.len() && std::iter::zip(a, b).all(|(a, b)| check_tt_eq(&a, &b)) } fn check_tt_eq(a: &tt::TokenTree, b: &tt::TokenTree) -> bool { @@ -545,7 +480,7 @@ mod tests { // the fixed-up tree should not contain braces as punct // FIXME: should probably instead check that it's a valid punctuation character - for x in tt.token_trees().flat_tokens() { + for x in tt.token_trees().iter_flat_tokens() { match x { ::tt::TokenTree::Leaf(::tt::Leaf::Punct(punct)) => { assert!(!matches!(punct.char, '{' | '}' | '(' | ')' | '[' | ']')) diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/mod_path.rs b/src/tools/rust-analyzer/crates/hir-expand/src/mod_path.rs index e9805e3f86b8..51e449fe5085 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/mod_path.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/mod_path.rs @@ -355,16 +355,16 @@ fn convert_path_tt(db: &dyn ExpandDatabase, tt: tt::TokenTreesView<'_>) -> Optio tt::Leaf::Punct(tt::Punct { char: ':', .. }) => PathKind::Abs, _ => return None, }, - tt::Leaf::Ident(tt::Ident { sym: text, span, .. }) if *text == sym::dollar_crate => { + tt::Leaf::Ident(tt::Ident { sym: text, span, .. }) if text == sym::dollar_crate => { resolve_crate_root(db, span.ctx).map(PathKind::DollarCrate).unwrap_or(PathKind::Crate) } - tt::Leaf::Ident(tt::Ident { sym: text, .. }) if *text == sym::self_ => PathKind::SELF, - tt::Leaf::Ident(tt::Ident { sym: text, .. }) if *text == sym::super_ => { + tt::Leaf::Ident(tt::Ident { sym: text, .. }) if text == sym::self_ => PathKind::SELF, + tt::Leaf::Ident(tt::Ident { sym: text, .. }) if text == sym::super_ => { let mut deg = 1; while let Some(tt::Leaf::Ident(tt::Ident { sym: text, span, is_raw: _ })) = leaves.next() { - if *text != sym::super_ { + if text != sym::super_ { segments.push(Name::new_symbol(text.clone(), span.ctx)); break; } @@ -372,7 +372,7 @@ fn convert_path_tt(db: &dyn ExpandDatabase, tt: tt::TokenTreesView<'_>) -> Optio } PathKind::Super(deg) } - tt::Leaf::Ident(tt::Ident { sym: text, .. }) if *text == sym::crate_ => PathKind::Crate, + tt::Leaf::Ident(tt::Ident { sym: text, .. }) if text == sym::crate_ => PathKind::Crate, tt::Leaf::Ident(ident) => { segments.push(Name::new_symbol(ident.sym.clone(), ident.span.ctx)); PathKind::Plain diff --git a/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs b/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs index e0f1e616d851..6b018510be14 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs @@ -54,7 +54,7 @@ fn benchmark_expand_macro_rules() { .map(|(id, tt)| { let res = rules[&id].expand(&db, &tt, |_| (), MacroCallStyle::FnLike, DUMMY); assert!(res.err.is_none()); - res.value.0.0.len() + res.value.0.as_token_trees().len() }) .sum() }; diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander.rs b/src/tools/rust-analyzer/crates/mbe/src/expander.rs index 36c7871519aa..25e05df7b710 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/expander.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/expander.rs @@ -162,7 +162,7 @@ impl Fragment<'_> { Fragment::Tokens { tree, .. } => tree.len() == 0, Fragment::Expr(it) => it.len() == 0, Fragment::Path(it) => it.len() == 0, - Fragment::TokensOwned(it) => it.0.is_empty(), + Fragment::TokensOwned(_) => false, // A `TopSubtree` is never empty } } } diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs b/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs index c4c1e1f5654f..e845b1ab8d1f 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs @@ -517,7 +517,7 @@ fn match_loop_inner<'t>( } OpDelimited::Op(Op::Literal(lhs)) => { if let Ok(rhs) = src.clone().expect_leaf() { - if matches!(rhs, tt::Leaf::Literal(it) if it.symbol == lhs.symbol) { + if matches!(&rhs, tt::Leaf::Literal(it) if it.symbol == lhs.symbol) { item.dot.next(); } else { res.add_err(ExpandError::new( @@ -537,7 +537,7 @@ fn match_loop_inner<'t>( } OpDelimited::Op(Op::Ident(lhs)) => { if let Ok(rhs) = src.clone().expect_leaf() { - if matches!(rhs, tt::Leaf::Ident(it) if it.sym == lhs.sym) { + if matches!(&rhs, tt::Leaf::Ident(it) if it.sym == lhs.sym) { item.dot.next(); } else { res.add_err(ExpandError::new( @@ -701,7 +701,7 @@ fn match_loop<'t>( || !(bb_items.is_empty() || next_items.is_empty()) || bb_items.len() > 1; if has_leftover_tokens { - res.unmatched_tts += src.remaining().flat_tokens().len(); + res.unmatched_tts += src.remaining().len(); res.add_err(ExpandError::new(span.open, ExpandErrorKind::LeftoverTokens)); if let Some(error_recover_item) = error_recover_item { @@ -991,7 +991,7 @@ fn expect_tt(iter: &mut TtIter<'_>) -> Result<(), ()> { Ok(()) } -fn expect_lifetime<'a>(iter: &mut TtIter<'a>) -> Result<&'a tt::Ident, ()> { +fn expect_lifetime<'a>(iter: &mut TtIter<'a>) -> Result { let punct = iter.expect_single_punct()?; if punct.char != '\'' { return Err(()); @@ -1000,7 +1000,7 @@ fn expect_lifetime<'a>(iter: &mut TtIter<'a>) -> Result<&'a tt::Ident, ()> { } fn eat_char(iter: &mut TtIter<'_>, c: char) { - if matches!(iter.peek(), Some(TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char, .. }))) if *char == c) + if matches!(iter.peek(), Some(TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char, .. }))) if char == c) { iter.next().expect("already peeked"); } diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs b/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs index a0fd4550ba8c..38098e2c84da 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs @@ -3,6 +3,7 @@ use intern::{Symbol, sym}; use span::{Edition, Span}; +use stdx::itertools::Itertools; use tt::{Delimiter, TopSubtreeBuilder, iter::TtElement}; use super::TokensOrigin; @@ -324,7 +325,7 @@ fn expand_subtree( } _ => (None, None), }; - let value = match values { + let value = match &values { (Some(TtElement::Leaf(tt::Leaf::Ident(ident))), None) => { ident.sym.as_str() } @@ -412,15 +413,15 @@ fn expand_var( // Check if this is a simple negative literal (MINUS + LITERAL) // that should not be wrapped in parentheses let is_negative_literal = matches!( - sub.flat_tokens(), - [ - tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '-', .. })), - tt::TokenTree::Leaf(tt::Leaf::Literal(_)) - ] + sub.iter().collect_array(), + Some([ + tt::TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char: '-', .. })), + tt::TtElement::Leaf(tt::Leaf::Literal(_)) + ]) ); let wrap_in_parens = !is_negative_literal - && !matches!(sub.flat_tokens(), [tt::TokenTree::Leaf(_)]) + && !matches!(sub.iter().collect_array(), Some([tt::TtElement::Leaf(_)])) && sub.try_into_subtree().is_none_or(|it| { it.top_subtree().delimiter.kind == tt::DelimiterKind::Invisible }); @@ -560,8 +561,8 @@ fn fix_up_and_push_path_tt( // argument list and thus needs `::` between it and `FnOnce`. However in // today's Rust this type of path *semantically* cannot appear as a // top-level expression-context path, so we can safely ignore it. - if let [tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '<', .. }))] = - tt.flat_tokens() + if let Some([tt::TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char: '<', .. }))]) = + tt.iter().collect_array() { builder.extend([ tt::Leaf::Punct(tt::Punct { @@ -577,7 +578,8 @@ fn fix_up_and_push_path_tt( ]); } } - prev_was_ident = matches!(tt.flat_tokens(), [tt::TokenTree::Leaf(tt::Leaf::Ident(_))]); + prev_was_ident = + matches!(tt.iter().collect_array(), Some([tt::TtElement::Leaf(tt::Leaf::Ident(_))])); builder.extend_with_tt(tt); } } diff --git a/src/tools/rust-analyzer/crates/mbe/src/parser.rs b/src/tools/rust-analyzer/crates/mbe/src/parser.rs index ddf9afbf98ab..cecdd43a42a1 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/parser.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/parser.rs @@ -224,7 +224,7 @@ fn next_op( None => { return Ok(Op::Punct({ let mut res = ArrayVec::new(); - res.push(*p); + res.push(p); Box::new(res) })); } @@ -268,7 +268,7 @@ fn next_op( let id = ident.span; Op::Var { name, kind, id } } - tt::Leaf::Literal(lit) if is_boolean_literal(lit) => { + tt::Leaf::Literal(lit) if is_boolean_literal(&lit) => { let kind = eat_fragment_kind(edition, src, mode)?; let name = lit.symbol.clone(); let id = lit.span; @@ -282,7 +282,7 @@ fn next_op( } Mode::Template => Op::Punct({ let mut res = ArrayVec::new(); - res.push(*punct); + res.push(punct); Box::new(res) }), }, @@ -400,7 +400,7 @@ fn parse_repeat(src: &mut TtIter<'_>) -> Result<(Option, RepeatKind), '?' => RepeatKind::ZeroOrOne, _ => match &mut separator { Separator::Puncts(puncts) if puncts.len() < 3 => { - puncts.push(*punct); + puncts.push(punct); continue; } _ => return Err(ParseError::InvalidRepeat), diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs index 8c96b06700a9..ed4b7aff76fc 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs @@ -489,7 +489,7 @@ struct Writer<'a, 'span, S: SpanTransformer, W> { impl<'a, T: SpanTransformer> Writer<'a, '_, T, tt::iter::TtIter<'a>> { fn write_subtree(&mut self, root: tt::SubtreeView<'a>) { let subtree = root.top_subtree(); - self.enqueue(subtree, root.iter()); + self.enqueue(&subtree, root.iter()); while let Some((idx, len, subtree)) = self.work.pop_front() { self.subtree(idx, len, subtree); } @@ -504,7 +504,7 @@ impl<'a, T: SpanTransformer> Writer<'a, '_, T, tt::iter::TtIt for child in subtree { let idx_tag = match child { tt::iter::TtElement::Subtree(subtree, subtree_iter) => { - let idx = self.enqueue(subtree, subtree_iter); + let idx = self.enqueue(&subtree, subtree_iter); idx << 2 } tt::iter::TtElement::Leaf(leaf) => match leaf { @@ -513,8 +513,11 @@ impl<'a, T: SpanTransformer> Writer<'a, '_, T, tt::iter::TtIt let id = self.token_id_of(lit.span); let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { ( - self.intern(lit.symbol.as_str()), - lit.suffix.as_ref().map(|s| self.intern(s.as_str())).unwrap_or(!0), + self.intern_owned(lit.symbol.as_str().to_owned()), + lit.suffix + .as_ref() + .map(|s| self.intern_owned(s.as_str().to_owned())) + .unwrap_or(!0), ) } else { (self.intern_owned(format!("{lit}")), !0) @@ -549,11 +552,11 @@ impl<'a, T: SpanTransformer> Writer<'a, '_, T, tt::iter::TtIt let idx = self.ident.len() as u32; let id = self.token_id_of(ident.span); let text = if self.version >= EXTENDED_LEAF_DATA { - self.intern(ident.sym.as_str()) + self.intern_owned(ident.sym.as_str().to_owned()) } else if ident.is_raw.yes() { self.intern_owned(format!("r#{}", ident.sym.as_str(),)) } else { - self.intern(ident.sym.as_str()) + self.intern_owned(ident.sym.as_str().to_owned()) }; self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw.yes() }); (idx << 2) | 0b11 @@ -565,7 +568,7 @@ impl<'a, T: SpanTransformer> Writer<'a, '_, T, tt::iter::TtIt } } - fn enqueue(&mut self, subtree: &'a tt::Subtree, contents: tt::iter::TtIter<'a>) -> u32 { + fn enqueue(&mut self, subtree: &tt::Subtree, contents: tt::iter::TtIter<'a>) -> u32 { let idx = self.subtree.len(); let open = self.token_id_of(subtree.delimiter.open); let close = self.token_id_of(subtree.delimiter.close); @@ -582,6 +585,7 @@ impl<'a, T: SpanTransformer, U> Writer<'a, '_, T, U> { T::token_id_of(self.span_data_table, span) } + #[cfg(feature = "sysroot-abi")] pub(crate) fn intern(&mut self, text: &'a str) -> u32 { let table = &mut self.text; *self.string_table.entry(text.into()).or_insert_with(|| { @@ -840,7 +844,7 @@ impl> Reader<'_, T> { let (delimiter, mut res) = res[0].take().unwrap(); res.insert(0, tt::TokenTree::Subtree(tt::Subtree { delimiter, len: res.len() as u32 })); - tt::TopSubtree(res.into_boxed_slice()) + tt::TopSubtree::from_serialized(res) } } diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs index e0d20c82b9e2..f5fcc99f14a3 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs @@ -194,28 +194,14 @@ impl ProcMacro { /// On some server versions, the fixup ast id is different than ours. So change it to match. fn change_fixup_to_match_old_server(&self, tt: &mut tt::TopSubtree) { const OLD_FIXUP_AST_ID: ErasedFileAstId = ErasedFileAstId::from_raw(!0 - 1); - let change_ast_id = |ast_id: &mut ErasedFileAstId| { + tt.change_every_ast_id(|ast_id| { if *ast_id == FIXUP_ERASED_FILE_AST_ID_MARKER { *ast_id = OLD_FIXUP_AST_ID; } else if *ast_id == OLD_FIXUP_AST_ID { // Swap between them, that means no collision plus the change can be reversed by doing itself. *ast_id = FIXUP_ERASED_FILE_AST_ID_MARKER; } - }; - - for tt in &mut tt.0 { - match tt { - tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { span, .. })) - | tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { span, .. })) - | tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { span, .. })) => { - change_ast_id(&mut span.anchor.ast_id); - } - tt::TokenTree::Subtree(subtree) => { - change_ast_id(&mut subtree.delimiter.open.anchor.ast_id); - change_ast_id(&mut subtree.delimiter.close.anchor.ast_id); - } - } - } + }); } /// Expands the procedural macro by sending an expansion request to the server. diff --git a/src/tools/rust-analyzer/crates/syntax-bridge/src/lib.rs b/src/tools/rust-analyzer/crates/syntax-bridge/src/lib.rs index 82cf51f1627a..79b51a816ebe 100644 --- a/src/tools/rust-analyzer/crates/syntax-bridge/src/lib.rs +++ b/src/tools/rust-analyzer/crates/syntax-bridge/src/lib.rs @@ -223,7 +223,7 @@ where spacing: _, })) => { let found_expected_delimiter = - builder.expected_delimiters().enumerate().find(|(_, delim)| match delim.kind { + builder.expected_delimiters().enumerate().find(|(_, delim)| match delim { tt::DelimiterKind::Parenthesis => char == ')', tt::DelimiterKind::Brace => char == '}', tt::DelimiterKind::Bracket => char == ']', @@ -257,13 +257,11 @@ where } kind if kind.is_punct() && kind != UNDERSCORE => { let found_expected_delimiter = - builder.expected_delimiters().enumerate().find(|(_, delim)| { - match delim.kind { - tt::DelimiterKind::Parenthesis => kind == T![')'], - tt::DelimiterKind::Brace => kind == T!['}'], - tt::DelimiterKind::Bracket => kind == T![']'], - tt::DelimiterKind::Invisible => false, - } + builder.expected_delimiters().enumerate().find(|(_, delim)| match delim { + tt::DelimiterKind::Parenthesis => kind == T![')'], + tt::DelimiterKind::Brace => kind == T!['}'], + tt::DelimiterKind::Bracket => kind == T![']'], + tt::DelimiterKind::Invisible => false, }); // Current token is a closing delimiter that we expect, fix up the closing span diff --git a/src/tools/rust-analyzer/crates/syntax-bridge/src/tests.rs b/src/tools/rust-analyzer/crates/syntax-bridge/src/tests.rs index c8dc3131b59c..8c28e1c5aaac 100644 --- a/src/tools/rust-analyzer/crates/syntax-bridge/src/tests.rs +++ b/src/tools/rust-analyzer/crates/syntax-bridge/src/tests.rs @@ -30,7 +30,7 @@ fn check_punct_spacing(fixture: &str) { }) .collect(); - let mut cursor = Cursor::new(&subtree.0); + let mut cursor = Cursor::new(subtree.as_token_trees()); while !cursor.eof() { while let Some(token_tree) = cursor.token_tree() { if let tt::TokenTree::Leaf(Leaf::Punct(Punct { diff --git a/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs b/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs index 67f69d0fa98e..487171c3e3e8 100644 --- a/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs +++ b/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs @@ -770,7 +770,7 @@ impl ProcMacroExpander for Issue18089ProcMacroExpander { _: Span, _: String, ) -> Result { - let tt::TokenTree::Leaf(macro_name) = &subtree.0[2] else { + let Some(tt::TtElement::Leaf(macro_name)) = subtree.iter().nth(1) else { return Err(ProcMacroExpansionError::Panic("incorrect input".to_owned())); }; Ok(quote! { call_site => @@ -837,13 +837,14 @@ impl ProcMacroExpander for Issue18840ProcMacroExpander { // ``` // The span that was created by the fixup infra. - let fixed_up_span = fn_.token_trees().flat_tokens()[5].first_span(); + let mut iter = fn_.iter(); + iter.nth(2); + let (_, mut fn_body) = iter.expect_subtree().unwrap(); + let fixed_up_span = fn_body.nth(1).unwrap().first_span(); let mut result = quote! {fixed_up_span => ::core::compile_error! { "my cool compile_error!" } }; // Make it so we won't remove the top subtree when reversing fixups. - let top_subtree_delimiter_mut = result.top_subtree_delimiter_mut(); - top_subtree_delimiter_mut.open = def_site; - top_subtree_delimiter_mut.close = def_site; + result.set_top_subtree_delimiter_span(tt::DelimSpan::from_single(def_site)); Ok(result) } @@ -905,13 +906,14 @@ impl ProcMacroExpander for ShortenProcMacroExpander { _: Span, _: String, ) -> Result { - let mut result = input.0.clone(); - for it in &mut result { - if let TokenTree::Leaf(leaf) = it { - modify_leaf(leaf) + let mut result = input.clone(); + for (idx, it) in input.as_token_trees().iter_flat_tokens().enumerate() { + if let TokenTree::Leaf(mut leaf) = it { + modify_leaf(&mut leaf); + result.set_token(idx, leaf); } } - return Ok(tt::TopSubtree(result)); + return Ok(result); fn modify_leaf(leaf: &mut Leaf) { match leaf { @@ -948,7 +950,8 @@ impl ProcMacroExpander for Issue17479ProcMacroExpander { _: Span, _: String, ) -> Result { - let TokenTree::Leaf(Leaf::Literal(lit)) = &subtree.0[1] else { + let mut iter = subtree.iter(); + let Some(TtElement::Leaf(tt::Leaf::Literal(lit))) = iter.next() else { return Err(ProcMacroExpansionError::Panic("incorrect Input".into())); }; let symbol = &lit.symbol; @@ -980,10 +983,8 @@ impl ProcMacroExpander for Issue18898ProcMacroExpander { ) -> Result { let span = subtree .token_trees() - .flat_tokens() - .last() - .ok_or_else(|| ProcMacroExpansionError::Panic("malformed input".to_owned()))? - .first_span(); + .last_span() + .ok_or_else(|| ProcMacroExpansionError::Panic("malformed input".to_owned()))?; let overly_long_subtree = quote! {span => { let a = 5; @@ -1034,7 +1035,7 @@ impl ProcMacroExpander for DisallowCfgProcMacroExpander { _: Span, _: String, ) -> Result { - for tt in subtree.token_trees().flat_tokens() { + for tt in subtree.token_trees().iter_flat_tokens() { if let tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) = tt && (ident.sym == sym::cfg || ident.sym == sym::cfg_attr) { @@ -1066,20 +1067,23 @@ impl ProcMacroExpander for GenerateSuffixedTypeProcMacroExpander { _mixed_site: Span, _current_dir: String, ) -> Result { - let TokenTree::Leaf(Leaf::Ident(ident)) = &subtree.0[1] else { + let mut iter = subtree.iter(); + let Some(TtElement::Leaf(tt::Leaf::Ident(ident))) = iter.next() else { return Err(ProcMacroExpansionError::Panic("incorrect Input".into())); }; let ident = match ident.sym.as_str() { "struct" => { - let TokenTree::Leaf(Leaf::Ident(ident)) = &subtree.0[2] else { + let Some(TtElement::Leaf(tt::Leaf::Ident(ident))) = iter.next() else { return Err(ProcMacroExpansionError::Panic("incorrect Input".into())); }; ident } "enum" => { - let TokenTree::Leaf(Leaf::Ident(ident)) = &subtree.0[4] else { + iter.next(); + let (_, mut iter) = iter.expect_subtree().unwrap(); + let Some(TtElement::Leaf(tt::Leaf::Ident(ident))) = iter.next() else { return Err(ProcMacroExpansionError::Panic("incorrect Input".into())); }; ident diff --git a/src/tools/rust-analyzer/crates/tt/src/buffer.rs b/src/tools/rust-analyzer/crates/tt/src/buffer.rs index c464e5ece194..de6379b5cd14 100644 --- a/src/tools/rust-analyzer/crates/tt/src/buffer.rs +++ b/src/tools/rust-analyzer/crates/tt/src/buffer.rs @@ -10,8 +10,8 @@ pub struct Cursor<'a> { } impl<'a> Cursor<'a> { - pub fn new(buffer: &'a [TokenTree]) -> Self { - Self { buffer, index: 0, subtrees_stack: Vec::new() } + pub fn new(buffer: TokenTreesView<'a>) -> Self { + Self { buffer: buffer.0, index: 0, subtrees_stack: Vec::new() } } /// Check whether it is eof diff --git a/src/tools/rust-analyzer/crates/tt/src/iter.rs b/src/tools/rust-analyzer/crates/tt/src/iter.rs index 88c3c7f52e93..5ab9f94b63fd 100644 --- a/src/tools/rust-analyzer/crates/tt/src/iter.rs +++ b/src/tools/rust-analyzer/crates/tt/src/iter.rs @@ -36,28 +36,28 @@ impl<'a> TtIter<'a> { pub fn expect_char(&mut self, char: char) -> Result<(), ()> { match self.next() { - Some(TtElement::Leaf(&Leaf::Punct(Punct { char: c, .. }))) if c == char => Ok(()), + Some(TtElement::Leaf(Leaf::Punct(Punct { char: c, .. }))) if c == char => Ok(()), _ => Err(()), } } pub fn expect_any_char(&mut self, chars: &[char]) -> Result<(), ()> { match self.next() { - Some(TtElement::Leaf(Leaf::Punct(Punct { char: c, .. }))) if chars.contains(c) => { + Some(TtElement::Leaf(Leaf::Punct(Punct { char: c, .. }))) if chars.contains(&c) => { Ok(()) } _ => Err(()), } } - pub fn expect_subtree(&mut self) -> Result<(&'a Subtree, TtIter<'a>), ()> { + pub fn expect_subtree(&mut self) -> Result<(Subtree, TtIter<'a>), ()> { match self.next() { Some(TtElement::Subtree(subtree, iter)) => Ok((subtree, iter)), _ => Err(()), } } - pub fn expect_leaf(&mut self) -> Result<&'a Leaf, ()> { + pub fn expect_leaf(&mut self) -> Result { match self.next() { Some(TtElement::Leaf(it)) => Ok(it), _ => Err(()), @@ -78,30 +78,30 @@ impl<'a> TtIter<'a> { } } - pub fn expect_ident(&mut self) -> Result<&'a Ident, ()> { + pub fn expect_ident(&mut self) -> Result { match self.expect_leaf()? { Leaf::Ident(it) if it.sym != sym::underscore => Ok(it), _ => Err(()), } } - pub fn expect_ident_or_underscore(&mut self) -> Result<&'a Ident, ()> { + pub fn expect_ident_or_underscore(&mut self) -> Result { match self.expect_leaf()? { Leaf::Ident(it) => Ok(it), _ => Err(()), } } - pub fn expect_literal(&mut self) -> Result<&'a Leaf, ()> { + pub fn expect_literal(&mut self) -> Result { let it = self.expect_leaf()?; - match it { + match &it { Leaf::Literal(_) => Ok(it), Leaf::Ident(ident) if ident.sym == sym::true_ || ident.sym == sym::false_ => Ok(it), _ => Err(()), } } - pub fn expect_single_punct(&mut self) -> Result<&'a Punct, ()> { + pub fn expect_single_punct(&mut self) -> Result { match self.expect_leaf()? { Leaf::Punct(it) => Ok(it), _ => Err(()), @@ -113,7 +113,7 @@ impl<'a> TtIter<'a> { /// This method currently may return a single quotation, which is part of lifetime ident and /// conceptually not a punct in the context of mbe. Callers should handle this. pub fn expect_glued_punct(&mut self) -> Result, ()> { - let TtElement::Leaf(&Leaf::Punct(first)) = self.next().ok_or(())? else { + let TtElement::Leaf(Leaf::Punct(first)) = self.next().ok_or(())? else { return Err(()); }; @@ -168,11 +168,11 @@ impl<'a> TtIter<'a> { pub fn peek(&self) -> Option> { match self.inner.as_slice().first()? { - TokenTree::Leaf(leaf) => Some(TtElement::Leaf(leaf)), + TokenTree::Leaf(leaf) => Some(TtElement::Leaf(leaf.clone())), TokenTree::Subtree(subtree) => { let nested_iter = TtIter { inner: self.inner.as_slice()[1..][..subtree.usize_len()].iter() }; - Some(TtElement::Subtree(subtree, nested_iter)) + Some(TtElement::Subtree(*subtree, nested_iter)) } } } @@ -214,8 +214,8 @@ impl<'a> TtIter<'a> { #[derive(Clone)] pub enum TtElement<'a> { - Leaf(&'a Leaf), - Subtree(&'a Subtree, TtIter<'a>), + Leaf(Leaf), + Subtree(Subtree, TtIter<'a>), } impl fmt::Debug for TtElement<'_> { @@ -243,12 +243,12 @@ impl<'a> Iterator for TtIter<'a> { type Item = TtElement<'a>; fn next(&mut self) -> Option { match self.inner.next()? { - TokenTree::Leaf(leaf) => Some(TtElement::Leaf(leaf)), + TokenTree::Leaf(leaf) => Some(TtElement::Leaf(leaf.clone())), TokenTree::Subtree(subtree) => { let nested_iter = TtIter { inner: self.inner.as_slice()[..subtree.usize_len()].iter() }; self.inner = self.inner.as_slice()[subtree.usize_len()..].iter(); - Some(TtElement::Subtree(subtree, nested_iter)) + Some(TtElement::Subtree(*subtree, nested_iter)) } } } diff --git a/src/tools/rust-analyzer/crates/tt/src/lib.rs b/src/tools/rust-analyzer/crates/tt/src/lib.rs index 636f567f1aa7..97e0b34ad7b7 100644 --- a/src/tools/rust-analyzer/crates/tt/src/lib.rs +++ b/src/tools/rust-analyzer/crates/tt/src/lib.rs @@ -111,7 +111,7 @@ impl Leaf { } impl_from!(Literal, Punct, Ident for Leaf); -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Subtree { pub delimiter: Delimiter, /// Number of following token trees that belong to this subtree, excluding this subtree. @@ -125,7 +125,7 @@ impl Subtree { } #[derive(Clone, PartialEq, Eq, Hash)] -pub struct TopSubtree(pub Box<[TokenTree]>); +pub struct TopSubtree(Box<[TokenTree]>); impl TopSubtree { pub fn empty(span: DelimSpan) -> Self { @@ -147,6 +147,10 @@ impl TopSubtree { builder.build() } + pub fn from_serialized(tt: Vec) -> Self { + Self(tt.into_boxed_slice()) + } + pub fn from_subtree(subtree: SubtreeView<'_>) -> Self { Self(subtree.0.into()) } @@ -159,20 +163,55 @@ impl TopSubtree { self.view().iter() } - pub fn top_subtree(&self) -> &Subtree { + pub fn top_subtree(&self) -> Subtree { self.view().top_subtree() } - pub fn top_subtree_delimiter_mut(&mut self) -> &mut Delimiter { + pub fn set_top_subtree_delimiter_kind(&mut self, kind: DelimiterKind) { + self.top_subtree_mut().delimiter.kind = kind; + } + + pub fn set_top_subtree_delimiter_span(&mut self, span: DelimSpan) { + let top_subtree = self.top_subtree_mut(); + top_subtree.delimiter.open = span.open; + top_subtree.delimiter.close = span.close; + } + + fn top_subtree_mut(&mut self) -> &mut Subtree { let TokenTree::Subtree(subtree) = &mut self.0[0] else { unreachable!("the first token tree is always the top subtree"); }; - &mut subtree.delimiter + subtree + } + + pub fn set_token(&mut self, idx: usize, leaf: Leaf) { + assert!(matches!(self.0[idx], TokenTree::Leaf(_)), "cannot replace a subtree by a leaf"); + self.0[idx] = leaf.into(); } pub fn token_trees(&self) -> TokenTreesView<'_> { self.view().token_trees() } + + pub fn as_token_trees(&self) -> TokenTreesView<'_> { + self.view().as_token_trees() + } + + pub fn change_every_ast_id(&mut self, mut callback: impl FnMut(&mut span::ErasedFileAstId)) { + for tt in &mut self.0 { + match tt { + TokenTree::Leaf(Leaf::Ident(Ident { span, .. })) + | TokenTree::Leaf(Leaf::Literal(Literal { span, .. })) + | TokenTree::Leaf(Leaf::Punct(Punct { span, .. })) => { + callback(&mut span.anchor.ast_id); + } + TokenTree::Subtree(subtree) => { + callback(&mut subtree.delimiter.open.anchor.ast_id); + callback(&mut subtree.delimiter.close.anchor.ast_id); + } + } + } + } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -241,11 +280,6 @@ impl TopSubtreeBuilder { self.token_trees.extend(leaves.into_iter().map(TokenTree::Leaf)); } - /// This does not check the token trees are valid, beware! - pub fn extend_tt_dangerous(&mut self, tt: impl IntoIterator) { - self.token_trees.extend(tt); - } - pub fn extend_with_tt(&mut self, tt: TokenTreesView<'_>) { self.token_trees.extend(tt.0.iter().cloned()); } @@ -267,12 +301,12 @@ impl TopSubtreeBuilder { } } - pub fn expected_delimiters(&self) -> impl Iterator { + pub fn expected_delimiters(&self) -> impl Iterator { self.unclosed_subtree_indices.iter().rev().map(|&subtree_idx| { let TokenTree::Subtree(subtree) = &self.token_trees[subtree_idx] else { unreachable!("unclosed token tree is always a subtree") }; - &subtree.delimiter + subtree.delimiter.kind }) } @@ -330,7 +364,7 @@ pub struct SubtreeBuilderRestorePoint { pub struct TokenTreesView<'a>(&'a [TokenTree]); impl<'a> TokenTreesView<'a> { - pub fn new(tts: &'a [TokenTree]) -> Self { + fn new(tts: &'a [TokenTree]) -> Self { if cfg!(debug_assertions) { tts.iter().enumerate().for_each(|(idx, tt)| { if let TokenTree::Subtree(tt) = &tt { @@ -345,12 +379,16 @@ impl<'a> TokenTreesView<'a> { Self(tts) } + pub fn empty() -> Self { + Self(&[]) + } + pub fn iter(&self) -> TtIter<'a> { TtIter::new(self.0) } pub fn cursor(&self) -> Cursor<'a> { - Cursor::new(self.0) + Cursor::new(*self) } pub fn len(&self) -> usize { @@ -374,13 +412,6 @@ impl<'a> TokenTreesView<'a> { self.try_into_subtree().map(|subtree| subtree.strip_invisible()).unwrap_or(self) } - /// This returns a **flat** structure of tokens (subtrees will be represented by a single node - /// preceding their children), so it isn't suited for most use cases, only for matching leaves - /// at the beginning/end with no subtrees before them. If you need a structured pass, use [`TtIter`]. - pub fn flat_tokens(&self) -> &'a [TokenTree] { - self.0 - } - pub fn split( self, mut split_fn: impl FnMut(TtElement<'a>) -> bool, @@ -406,6 +437,21 @@ impl<'a> TokenTreesView<'a> { Some(result) }) } + + pub fn first_span(&self) -> Option { + Some(self.0.first()?.first_span()) + } + + pub fn last_span(&self) -> Option { + Some(match self.0.last()? { + TokenTree::Leaf(it) => *it.span(), + TokenTree::Subtree(it) => it.delimiter.close, + }) + } + + pub fn iter_flat_tokens(&self) -> impl ExactSizeIterator + use<'a> { + self.0.iter().cloned() + } } impl fmt::Debug for TokenTreesView<'_> { @@ -453,11 +499,11 @@ impl fmt::Display for TokenTreesView<'_> { match child { TtElement::Leaf(Leaf::Punct(p)) => { needs_space = p.spacing == Spacing::Alone; - fmt::Display::fmt(p, f)?; + fmt::Display::fmt(&p, f)?; } - TtElement::Leaf(leaf) => fmt::Display::fmt(leaf, f)?, + TtElement::Leaf(leaf) => fmt::Display::fmt(&leaf, f)?, TtElement::Subtree(subtree, subtree_iter) => { - subtree_display(subtree, f, subtree_iter)? + subtree_display(&subtree, f, subtree_iter)? } } } @@ -493,11 +539,11 @@ impl<'a> SubtreeView<'a> { TtIter::new(&self.0[1..]) } - pub fn top_subtree(&self) -> &'a Subtree { + pub fn top_subtree(&self) -> Subtree { let TokenTree::Subtree(subtree) = &self.0[0] else { unreachable!("the first token tree is always the top subtree"); }; - subtree + *subtree } pub fn strip_invisible(&self) -> TokenTreesView<'a> { @@ -791,7 +837,7 @@ fn print_debug_token(f: &mut fmt::Formatter<'_>, level: usize, tt: TtElement<'_> } }, TtElement::Subtree(subtree, subtree_iter) => { - print_debug_subtree(f, subtree, level, subtree_iter)?; + print_debug_subtree(f, &subtree, level, subtree_iter)?; } } @@ -956,7 +1002,7 @@ impl TopSubtree { } } -pub fn pretty(mut tkns: &[TokenTree]) -> String { +pub fn pretty(tkns: TokenTreesView<'_>) -> String { fn tokentree_to_text(tkn: &TokenTree, tkns: &mut &[TokenTree]) -> String { match tkn { TokenTree::Leaf(Leaf::Ident(ident)) => { @@ -966,7 +1012,7 @@ pub fn pretty(mut tkns: &[TokenTree]) -> String { TokenTree::Leaf(Leaf::Punct(punct)) => format!("{}", punct.char), TokenTree::Subtree(subtree) => { let (subtree_content, rest) = tkns.split_at(subtree.usize_len()); - let content = pretty(subtree_content); + let content = pretty(TokenTreesView(subtree_content)); *tkns = rest; let (open, close) = match subtree.delimiter.kind { DelimiterKind::Brace => ("{", "}"), @@ -979,6 +1025,7 @@ pub fn pretty(mut tkns: &[TokenTree]) -> String { } } + let mut tkns = tkns.0; let mut last = String::new(); let mut last_to_joint = true; @@ -994,3 +1041,83 @@ pub fn pretty(mut tkns: &[TokenTree]) -> String { } last } + +#[derive(Debug)] +pub enum TransformTtAction<'a> { + Keep, + ReplaceWith(TokenTreesView<'a>), +} + +impl TransformTtAction<'_> { + #[inline] + pub fn remove() -> Self { + Self::ReplaceWith(TokenTreesView::empty()) + } +} + +/// This function takes a token tree, and calls `callback` with each token tree in it. +/// Then it does what the callback says: keeps the tt or replaces it with a (possibly empty) +/// tts view. +pub fn transform_tt<'b>( + tt: &mut TopSubtree, + mut callback: impl FnMut(TokenTree) -> TransformTtAction<'b>, +) { + let mut tt_vec = std::mem::take(&mut tt.0).into_vec(); + + // We need to keep a stack of the currently open subtrees, because we need to update + // them if we change the number of items in them. + let mut subtrees_stack = Vec::new(); + let mut i = 0; + while i < tt_vec.len() { + 'pop_finished_subtrees: while let Some(&subtree_idx) = subtrees_stack.last() { + let TokenTree::Subtree(subtree) = &tt_vec[subtree_idx] else { + unreachable!("non-subtree on subtrees stack"); + }; + if i >= subtree_idx + 1 + subtree.usize_len() { + subtrees_stack.pop(); + } else { + break 'pop_finished_subtrees; + } + } + + let current = match &tt_vec[i] { + TokenTree::Leaf(leaf) => TokenTree::Leaf(match leaf { + Leaf::Literal(leaf) => Leaf::Literal(leaf.clone()), + Leaf::Punct(leaf) => Leaf::Punct(*leaf), + Leaf::Ident(leaf) => Leaf::Ident(leaf.clone()), + }), + TokenTree::Subtree(subtree) => TokenTree::Subtree(*subtree), + }; + let action = callback(current); + match action { + TransformTtAction::Keep => { + // This cannot be shared with the replaced case, because then we may push the same subtree + // twice, and will update it twice which will lead to errors. + if let TokenTree::Subtree(_) = &tt_vec[i] { + subtrees_stack.push(i); + } + + i += 1; + } + TransformTtAction::ReplaceWith(replacement) => { + let old_len = 1 + match &tt_vec[i] { + TokenTree::Leaf(_) => 0, + TokenTree::Subtree(subtree) => subtree.usize_len(), + }; + let len_diff = replacement.len() as i64 - old_len as i64; + tt_vec.splice(i..i + old_len, replacement.0.iter().cloned()); + // Skip the newly inserted replacement, we don't want to visit it. + i += replacement.len(); + + for &subtree_idx in &subtrees_stack { + let TokenTree::Subtree(subtree) = &mut tt_vec[subtree_idx] else { + unreachable!("non-subtree on subtrees stack"); + }; + subtree.len = (i64::from(subtree.len) + len_diff).try_into().unwrap(); + } + } + } + } + + tt.0 = tt_vec.into_boxed_slice(); +} From 175e297e58198883cc181ad01805c6f3280f2e61 Mon Sep 17 00:00:00 2001 From: Chayim Refael Friedman Date: Mon, 29 Dec 2025 05:05:47 +0200 Subject: [PATCH 2/2] Make `tt::Literal` use one `Symbol` for the text and the suffix That shrinks it, which is useless now (it's not the dominant factor), but will become important when we'll implement span compression. --- .../rust-analyzer/crates/cfg/src/cfg_expr.rs | 5 +- .../crates/hir-def/src/item_tree/attrs.rs | 2 +- .../crates/hir-def/src/nameres/collector.rs | 8 +- .../crates/hir-expand/src/attrs.rs | 33 ++--- .../crates/hir-expand/src/builtin/fn_macro.rs | 107 ++++++--------- .../crates/hir-expand/src/builtin/quote.rs | 10 +- .../crates/hir-expand/src/fixup.rs | 2 +- .../rust-analyzer/crates/mbe/src/benchmark.rs | 7 +- .../crates/mbe/src/expander/matcher.rs | 7 +- .../crates/mbe/src/expander/transcriber.rs | 18 ++- .../rust-analyzer/crates/mbe/src/parser.rs | 13 +- .../proc-macro-api/src/legacy_protocol/msg.rs | 25 ++-- .../src/legacy_protocol/msg/flat.rs | 28 ++-- .../crates/syntax-bridge/src/lib.rs | 11 +- .../syntax-bridge/src/to_parser_input.rs | 2 +- .../crates/test-fixture/src/lib.rs | 5 +- src/tools/rust-analyzer/crates/tt/src/lib.rs | 125 +++++++++--------- 17 files changed, 190 insertions(+), 218 deletions(-) diff --git a/src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs b/src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs index c5da8443a68f..d253f6f492c7 100644 --- a/src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs +++ b/src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs @@ -157,7 +157,8 @@ fn next_cfg_expr_from_ast( }, ctx: span::SyntaxContext::root(span::Edition::Edition2015), }; - let literal = tt::token_to_literal(literal.text(), dummy_span).symbol; + let literal = + Symbol::intern(tt::token_to_literal(literal.text(), dummy_span).text()); it.next(); CfgAtom::KeyValue { key: name, value: literal.clone() }.into() } else { @@ -211,7 +212,7 @@ fn next_cfg_expr(it: &mut tt::iter::TtIter<'_>) -> Option { Some(tt::TtElement::Leaf(tt::Leaf::Literal(literal))) => { it.next(); it.next(); - CfgAtom::KeyValue { key: name, value: literal.symbol.clone() }.into() + CfgAtom::KeyValue { key: name, value: Symbol::intern(literal.text()) }.into() } _ => return Some(CfgExpr::Invalid), } diff --git a/src/tools/rust-analyzer/crates/hir-def/src/item_tree/attrs.rs b/src/tools/rust-analyzer/crates/hir-def/src/item_tree/attrs.rs index bfebce001375..79076112847b 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/item_tree/attrs.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/item_tree/attrs.rs @@ -226,7 +226,7 @@ impl<'attr> AttrQuery<'attr> { } #[inline] - pub(crate) fn string_value_with_span(self) -> Option<(&'attr Symbol, span::Span)> { + pub(crate) fn string_value_with_span(self) -> Option<(&'attr str, span::Span)> { self.attrs().find_map(|attr| attr.string_value_with_span()) } diff --git a/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs b/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs index 822da678d797..4740e3b99e3d 100644 --- a/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs +++ b/src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs @@ -17,7 +17,7 @@ use hir_expand::{ name::{AsName, Name}, proc_macro::CustomProcMacroExpander, }; -use intern::{Interned, sym}; +use intern::{Interned, Symbol, sym}; use itertools::izip; use la_arena::Idx; use rustc_hash::{FxHashMap, FxHashSet}; @@ -292,13 +292,13 @@ impl<'db> DefCollector<'db> { match () { () if *attr_name == sym::recursion_limit => { if let Some(limit) = attr.string_value() - && let Ok(limit) = limit.as_str().parse() + && let Ok(limit) = limit.parse() { crate_data.recursion_limit = Some(limit); } } () if *attr_name == sym::crate_type => { - if attr.string_value() == Some(&sym::proc_dash_macro) { + if attr.string_value() == Some("proc-macro") { self.is_proc_macro = true; } } @@ -2460,7 +2460,7 @@ impl ModCollector<'_, '_> { let name; let name = match attrs.by_key(sym::rustc_builtin_macro).string_value_with_span() { Some((it, span)) => { - name = Name::new_symbol(it.clone(), span.ctx); + name = Name::new_symbol(Symbol::intern(it), span.ctx); &name } None => { diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs b/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs index 5a440187203c..e3f10b212904 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs @@ -35,7 +35,7 @@ use arrayvec::ArrayVec; use base_db::Crate; use cfg::{CfgExpr, CfgOptions}; use either::Either; -use intern::{Interned, Symbol}; +use intern::Interned; use itertools::Itertools; use mbe::{DelimiterKind, Punct}; use parser::T; @@ -417,37 +417,32 @@ impl fmt::Display for AttrInput { impl Attr { /// #[path = "string"] - pub fn string_value(&self) -> Option<&Symbol> { + pub fn string_value(&self) -> Option<&str> { match self.input.as_deref()? { - AttrInput::Literal(tt::Literal { - symbol: text, - kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), - .. - }) => Some(text), + AttrInput::Literal( + lit @ tt::Literal { kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), .. }, + ) => Some(lit.text()), _ => None, } } /// #[path = "string"] - pub fn string_value_with_span(&self) -> Option<(&Symbol, span::Span)> { + pub fn string_value_with_span(&self) -> Option<(&str, span::Span)> { match self.input.as_deref()? { - AttrInput::Literal(tt::Literal { - symbol: text, - kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), - span, - suffix: _, - }) => Some((text, *span)), + AttrInput::Literal( + lit @ tt::Literal { kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), span, .. }, + ) => Some((lit.text(), *span)), _ => None, } } pub fn string_value_unescape(&self) -> Option> { match self.input.as_deref()? { - AttrInput::Literal(tt::Literal { - symbol: text, kind: tt::LitKind::StrRaw(_), .. - }) => Some(Cow::Borrowed(text.as_str())), - AttrInput::Literal(tt::Literal { symbol: text, kind: tt::LitKind::Str, .. }) => { - unescape(text.as_str()) + AttrInput::Literal(lit @ tt::Literal { kind: tt::LitKind::StrRaw(_), .. }) => { + Some(Cow::Borrowed(lit.text())) + } + AttrInput::Literal(lit @ tt::Literal { kind: tt::LitKind::Str, .. }) => { + unescape(lit.text()) } _ => None, } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/builtin/fn_macro.rs b/src/tools/rust-analyzer/crates/hir-expand/src/builtin/fn_macro.rs index 011a9530c4d3..6e4b96b05088 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/builtin/fn_macro.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/builtin/fn_macro.rs @@ -1,5 +1,7 @@ //! Builtin macro +use std::borrow::Cow; + use base_db::AnchoredPath; use cfg::CfgExpr; use either::Either; @@ -13,7 +15,7 @@ use span::{Edition, FileId, Span}; use stdx::format_to; use syntax::{ format_smolstr, - unescape::{unescape_byte, unescape_char, unescape_str}, + unescape::{unescape_byte, unescape_char}, }; use syntax_bridge::syntax_node_to_token_tree; @@ -177,12 +179,7 @@ fn line_expand( // not incremental ExpandResult::ok(tt::TopSubtree::invisible_from_leaves( span, - [tt::Leaf::Literal(tt::Literal { - symbol: sym::INTEGER_0, - span, - kind: tt::LitKind::Integer, - suffix: Some(sym::u32), - })], + [tt::Leaf::Literal(tt::Literal::new("0", span, tt::LitKind::Integer, "u32"))], )) } @@ -303,7 +300,8 @@ fn format_args_nl_expand( mut lit @ tt::Literal { kind: tt::LitKind::Str, .. }, ))) = lit { - lit.symbol = Symbol::intern(&format_smolstr!("{}\\n", lit.symbol.as_str())); + let (text, suffix) = lit.text_and_suffix(); + lit.text_and_suffix = Symbol::intern(&format_smolstr!("{text}\\n{suffix}")); tt.set_token(1, lit.into()); } ExpandResult::ok(quote! {span => @@ -521,14 +519,11 @@ fn compile_error_expand( let err = match tt.iter().collect_array() { Some( [ - tt::TtElement::Leaf(tt::Leaf::Literal(tt::Literal { - symbol: text, - span: _, - kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), - suffix: _, - })), + tt::TtElement::Leaf(tt::Leaf::Literal( + lit @ tt::Literal { kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), .. }, + )), ], - ) => ExpandError::other(span, Box::from(unescape_symbol(&text).as_str())), + ) => ExpandError::other(span, Box::from(unescape_str(lit.text()))), _ => ExpandError::other(span, "`compile_error!` argument must be a string"), }; @@ -569,20 +564,20 @@ fn concat_expand( // as-is. match it.kind { tt::LitKind::Char => { - if let Ok(c) = unescape_char(it.symbol.as_str()) { + if let Ok(c) = unescape_char(it.text()) { text.push(c); } record_span(it.span); } tt::LitKind::Integer | tt::LitKind::Float => { - format_to!(text, "{}", it.symbol.as_str()) + format_to!(text, "{}", it.text()) } tt::LitKind::Str => { - text.push_str(unescape_symbol(&it.symbol).as_str()); + text.push_str(&unescape_str(it.text())); record_span(it.span); } tt::LitKind::StrRaw(_) => { - format_to!(text, "{}", it.symbol.as_str()); + format_to!(text, "{}", it.text()); record_span(it.span); } tt::LitKind::Byte @@ -620,7 +615,7 @@ fn concat_expand( TtElement::Leaf(tt::Leaf::Literal(it)) if matches!(it.kind, tt::LitKind::Integer | tt::LitKind::Float) => { - format_to!(text, "-{}", it.symbol.as_str()); + format_to!(text, "-{}", it.text()); record_span(punct.span.cover(it.span)); } _ => { @@ -658,26 +653,22 @@ fn concat_bytes_expand( }; for (i, t) in tt.iter().enumerate() { match t { - TtElement::Leaf(tt::Leaf::Literal(tt::Literal { - symbol: text, - span, - kind, - suffix: _, - })) => { + TtElement::Leaf(tt::Leaf::Literal(lit @ tt::Literal { span, kind, .. })) => { + let text = lit.text(); record_span(span); match kind { tt::LitKind::Byte => { - if let Ok(b) = unescape_byte(text.as_str()) { + if let Ok(b) = unescape_byte(text) { bytes.extend( b.escape_ascii().filter_map(|it| char::from_u32(it as u32)), ); } } tt::LitKind::ByteStr => { - bytes.push_str(text.as_str()); + bytes.push_str(text); } tt::LitKind::ByteStrRaw(_) => { - bytes.extend(text.as_str().escape_debug()); + bytes.extend(text.escape_debug()); } _ => { err.get_or_insert(ExpandError::other(span, "unexpected token")); @@ -706,12 +697,7 @@ fn concat_bytes_expand( ExpandResult { value: tt::TopSubtree::invisible_from_leaves( span, - [tt::Leaf::Literal(tt::Literal { - symbol: Symbol::intern(&bytes), - span, - kind: tt::LitKind::ByteStr, - suffix: None, - })], + [tt::Leaf::Literal(tt::Literal::new_no_suffix(&bytes, span, tt::LitKind::ByteStr))], ), err, } @@ -725,25 +711,19 @@ fn concat_bytes_expand_subtree( ) -> Result<(), ExpandError> { for (ti, tt) in tree_iter.enumerate() { match tt { - TtElement::Leaf(tt::Leaf::Literal(tt::Literal { - symbol: text, - span, - kind: tt::LitKind::Byte, - suffix: _, - })) => { - if let Ok(b) = unescape_byte(text.as_str()) { + TtElement::Leaf(tt::Leaf::Literal( + lit @ tt::Literal { span, kind: tt::LitKind::Byte, .. }, + )) => { + if let Ok(b) = unescape_byte(lit.text()) { bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32))); } record_span(span); } - TtElement::Leaf(tt::Leaf::Literal(tt::Literal { - symbol: text, - span, - kind: tt::LitKind::Integer, - suffix: _, - })) => { + TtElement::Leaf(tt::Leaf::Literal( + lit @ tt::Literal { span, kind: tt::LitKind::Integer, .. }, + )) => { record_span(span); - if let Ok(b) = text.as_str().parse::() { + if let Ok(b) = lit.text().parse::() { bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32))); } } @@ -792,18 +772,16 @@ fn parse_string(tt: &tt::TopSubtree) -> Result<(Symbol, Span), ExpandError> { } match tt { - TtElement::Leaf(tt::Leaf::Literal(tt::Literal { - symbol: text, + TtElement::Leaf(tt::Leaf::Literal(lit @ tt::Literal { span, kind: tt::LitKind::Str, - suffix: _, - })) => Ok((unescape_symbol(&text), span)), - TtElement::Leaf(tt::Leaf::Literal(tt::Literal { - symbol: text, + .. + })) => Ok((Symbol::intern(&unescape_str(lit.text())), span)), + TtElement::Leaf(tt::Leaf::Literal(lit @ tt::Literal { span, kind: tt::LitKind::StrRaw(_), - suffix: _, - })) => Ok((text.clone(), span)), + .. + })) => Ok((Symbol::intern(lit.text()), span)), TtElement::Leaf(l) => Err(*l.span()), TtElement::Subtree(tt, _) => Err(tt.delimiter.open.cover(tt.delimiter.close)), } @@ -855,10 +833,10 @@ fn include_bytes_expand( let res = tt::TopSubtree::invisible_from_leaves( span, [tt::Leaf::Literal(tt::Literal { - symbol: Symbol::empty(), + text_and_suffix: Symbol::empty(), span, kind: tt::LitKind::ByteStrRaw(1), - suffix: None, + suffix_len: 0, })], ); ExpandResult::ok(res) @@ -979,17 +957,16 @@ fn quote_expand( ) } -fn unescape_symbol(s: &Symbol) -> Symbol { - if s.as_str().contains('\\') { - let s = s.as_str(); +fn unescape_str(s: &str) -> Cow<'_, str> { + if s.contains('\\') { let mut buf = String::with_capacity(s.len()); - unescape_str(s, |_, c| { + syntax::unescape::unescape_str(s, |_, c| { if let Ok(c) = c { buf.push(c) } }); - Symbol::intern(&buf) + Cow::Owned(buf) } else { - s.clone() + Cow::Borrowed(s) } } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/builtin/quote.rs b/src/tools/rust-analyzer/crates/hir-expand/src/builtin/quote.rs index 4039b6c334ec..51c4e225168f 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/builtin/quote.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/builtin/quote.rs @@ -199,16 +199,16 @@ impl ToTokenTree for &T { } impl_to_to_tokentrees! { - span: u32 => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } }; - span: usize => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } }; - span: i32 => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } }; + span: u32 => self { crate::tt::Literal{text_and_suffix: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix_len: 0 } }; + span: usize => self { crate::tt::Literal{text_and_suffix: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix_len: 0 } }; + span: i32 => self { crate::tt::Literal{text_and_suffix: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix_len: 0 } }; span: bool => self { crate::tt::Ident{sym: if self { sym::true_ } else { sym::false_ }, span, is_raw: tt::IdentIsRaw::No } }; _span: crate::tt::Leaf => self { self }; _span: crate::tt::Literal => self { self }; _span: crate::tt::Ident => self { self }; _span: crate::tt::Punct => self { self }; - span: &str => self { crate::tt::Literal{symbol: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix: None }}; - span: String => self { crate::tt::Literal{symbol: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix: None }}; + span: &str => self { crate::tt::Literal{text_and_suffix: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix_len: 0 }}; + span: String => self { crate::tt::Literal{text_and_suffix: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix_len: 0 }}; span: Name => self { let (is_raw, s) = IdentIsRaw::split_from_symbol(self.as_str()); crate::tt::Ident{sym: Symbol::intern(s), span, is_raw } diff --git a/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs b/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs index 6afc6e5d5e7f..92ddd7fa8b07 100644 --- a/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs +++ b/src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs @@ -415,7 +415,7 @@ mod tests { // `TokenTree`s, see the last assertion in `check()`. fn check_leaf_eq(a: &tt::Leaf, b: &tt::Leaf) -> bool { match (a, b) { - (tt::Leaf::Literal(a), tt::Leaf::Literal(b)) => a.symbol == b.symbol, + (tt::Leaf::Literal(a), tt::Leaf::Literal(b)) => a.text_and_suffix == b.text_and_suffix, (tt::Leaf::Punct(a), tt::Leaf::Punct(b)) => a.char == b.char, (tt::Leaf::Ident(a), tt::Leaf::Ident(b)) => a.sym == b.sym, _ => false, diff --git a/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs b/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs index 6b018510be14..603fee73064e 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs @@ -236,12 +236,7 @@ fn invocation_fixtures( tt::Leaf::Punct(tt::Punct { span: DUMMY, char, spacing: tt::Spacing::Alone }) } fn make_literal(lit: &str) -> tt::Leaf { - tt::Leaf::Literal(tt::Literal { - span: DUMMY, - symbol: Symbol::intern(lit), - kind: tt::LitKind::Str, - suffix: None, - }) + tt::Leaf::Literal(tt::Literal::new_no_suffix(lit, DUMMY, tt::LitKind::Str)) } fn make_subtree(kind: tt::DelimiterKind, builder: &mut tt::TopSubtreeBuilder) { builder.open(kind, DUMMY); diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs b/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs index e845b1ab8d1f..8f6627a60fe6 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs @@ -517,7 +517,8 @@ fn match_loop_inner<'t>( } OpDelimited::Op(Op::Literal(lhs)) => { if let Ok(rhs) = src.clone().expect_leaf() { - if matches!(&rhs, tt::Leaf::Literal(it) if it.symbol == lhs.symbol) { + if matches!(&rhs, tt::Leaf::Literal(it) if it.text_and_suffix == lhs.text_and_suffix) + { item.dot.next(); } else { res.add_err(ExpandError::new( @@ -953,8 +954,8 @@ fn expect_separator(iter: &mut TtIter<'_>, separator: &Separator) -> bool { }, Separator::Literal(lhs) => match fork.expect_literal() { Ok(rhs) => match rhs { - tt::Leaf::Literal(rhs) => rhs.symbol == lhs.symbol, - tt::Leaf::Ident(rhs) => rhs.sym == lhs.symbol, + tt::Leaf::Literal(rhs) => rhs.text_and_suffix == lhs.text_and_suffix, + tt::Leaf::Ident(rhs) => rhs.sym == lhs.text_and_suffix, tt::Leaf::Punct(_) => false, }, Err(_) => false, diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs b/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs index 38098e2c84da..e8e7928c263f 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs @@ -222,10 +222,10 @@ fn expand_subtree( let index = ctx.nesting.get(ctx.nesting.len() - 1 - depth).map_or(0, |nest| nest.idx); builder.push(tt::Leaf::Literal(tt::Literal { - symbol: Symbol::integer(index), + text_and_suffix: Symbol::integer(index), span: ctx.call_site, kind: tt::LitKind::Integer, - suffix: None, + suffix_len: 0, })); } Op::Len { depth } => { @@ -234,10 +234,10 @@ fn expand_subtree( 0 }); builder.push(tt::Leaf::Literal(tt::Literal { - symbol: Symbol::integer(length), + text_and_suffix: Symbol::integer(length), span: ctx.call_site, kind: tt::LitKind::Integer, - suffix: None, + suffix_len: 0, })); } Op::Count { name, depth } => { @@ -278,9 +278,9 @@ fn expand_subtree( let res = count(binding, 0, depth.unwrap_or(0)); builder.push(tt::Leaf::Literal(tt::Literal { - symbol: Symbol::integer(res), + text_and_suffix: Symbol::integer(res), span: ctx.call_site, - suffix: None, + suffix_len: 0, kind: tt::LitKind::Integer, })); } @@ -294,7 +294,7 @@ fn expand_subtree( ConcatMetaVarExprElem::Literal(lit) => { // FIXME: This isn't really correct wrt. escaping, but that's what rustc does and anyway // escaping is used most of the times for characters that are invalid in identifiers. - concatenated.push_str(lit.symbol.as_str()) + concatenated.push_str(lit.text()) } ConcatMetaVarExprElem::Var(var) => { // Handling of repetitions in `${concat}` isn't fleshed out in rustc, so we currently @@ -329,9 +329,7 @@ fn expand_subtree( (Some(TtElement::Leaf(tt::Leaf::Ident(ident))), None) => { ident.sym.as_str() } - (Some(TtElement::Leaf(tt::Leaf::Literal(lit))), None) => { - lit.symbol.as_str() - } + (Some(TtElement::Leaf(tt::Leaf::Literal(lit))), None) => lit.text(), _ => { if err.is_none() { err = Some(ExpandError::binding_error( diff --git a/src/tools/rust-analyzer/crates/mbe/src/parser.rs b/src/tools/rust-analyzer/crates/mbe/src/parser.rs index cecdd43a42a1..796ee62d48e3 100644 --- a/src/tools/rust-analyzer/crates/mbe/src/parser.rs +++ b/src/tools/rust-analyzer/crates/mbe/src/parser.rs @@ -192,7 +192,7 @@ impl PartialEq for Separator { match (self, other) { (Ident(a), Ident(b)) => a.sym == b.sym, - (Literal(a), Literal(b)) => a.symbol == b.symbol, + (Literal(a), Literal(b)) => a.text_and_suffix == b.text_and_suffix, (Puncts(a), Puncts(b)) if a.len() == b.len() => { let a_iter = a.iter().map(|a| a.char); let b_iter = b.iter().map(|b| b.char); @@ -270,7 +270,7 @@ fn next_op( } tt::Leaf::Literal(lit) if is_boolean_literal(&lit) => { let kind = eat_fragment_kind(edition, src, mode)?; - let name = lit.symbol.clone(); + let name = lit.text_and_suffix.clone(); let id = lit.span; Op::Var { name, kind, id } } @@ -364,7 +364,7 @@ fn eat_fragment_kind( } fn is_boolean_literal(lit: &tt::Literal) -> bool { - matches!(lit.symbol.as_str(), "true" | "false") + lit.text_and_suffix == sym::true_ || lit.text_and_suffix == sym::false_ } fn parse_repeat(src: &mut TtIter<'_>) -> Result<(Option, RepeatKind), ParseError> { @@ -478,11 +478,12 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result { fn parse_depth(src: &mut TtIter<'_>) -> Result { if src.is_empty() { Ok(0) - } else if let tt::Leaf::Literal(tt::Literal { symbol: text, suffix: None, .. }) = - src.expect_literal()? + } else if let tt::Leaf::Literal(lit) = src.expect_literal()? + && let (text, suffix) = lit.text_and_suffix() + && suffix.is_empty() { // Suffixes are not allowed. - text.as_str().parse().map_err(|_| ()) + text.parse().map_err(|_| ()) } else { Err(()) } diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg.rs index ac627b9232a0..4146b619ec0c 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg.rs @@ -172,7 +172,7 @@ impl Message for Response {} #[cfg(test)] mod tests { - use intern::{Symbol, sym}; + use intern::Symbol; use span::{ Edition, ROOT_ERASED_FILE_AST_ID, Span, SpanAnchor, SyntaxContext, TextRange, TextSize, }; @@ -232,16 +232,15 @@ mod tests { } .into(), ); - builder.push(Leaf::Literal(Literal { - symbol: Symbol::intern("Foo"), - span: Span { + builder.push(Leaf::Literal(Literal::new_no_suffix( + "Foo", + Span { range: TextRange::at(TextSize::new(10), TextSize::of("\"Foo\"")), anchor, ctx: SyntaxContext::root(Edition::CURRENT), }, - kind: tt::LitKind::Str, - suffix: None, - })); + tt::LitKind::Str, + ))); builder.push(Leaf::Punct(Punct { char: '@', span: Span { @@ -267,16 +266,16 @@ mod tests { ctx: SyntaxContext::root(Edition::CURRENT), }, ); - builder.push(Leaf::Literal(Literal { - symbol: sym::INTEGER_0, - span: Span { + builder.push(Leaf::Literal(Literal::new( + "0", + Span { range: TextRange::at(TextSize::new(16), TextSize::of("0u32")), anchor, ctx: SyntaxContext::root(Edition::CURRENT), }, - kind: tt::LitKind::Integer, - suffix: Some(sym::u32), - })); + tt::LitKind::Integer, + "u32", + ))); builder.close(Span { range: TextRange::at(TextSize::new(20), TextSize::of(']')), anchor, diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs index ed4b7aff76fc..cd8944aa6170 100644 --- a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs +++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs @@ -512,12 +512,14 @@ impl<'a, T: SpanTransformer> Writer<'a, '_, T, tt::iter::TtIt let idx = self.literal.len() as u32; let id = self.token_id_of(lit.span); let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { + let (text, suffix) = lit.text_and_suffix(); ( - self.intern_owned(lit.symbol.as_str().to_owned()), - lit.suffix - .as_ref() - .map(|s| self.intern_owned(s.as_str().to_owned())) - .unwrap_or(!0), + self.intern_owned(text.to_owned()), + if suffix.is_empty() { + !0 + } else { + self.intern_owned(suffix.to_owned()) + }, ) } else { (self.intern_owned(format!("{lit}")), !0) @@ -774,10 +776,10 @@ impl> Reader<'_, T> { let span = read_span(repr.id); s.push( tt::Leaf::Literal(if self.version >= EXTENDED_LEAF_DATA { - tt::Literal { - symbol: Symbol::intern(text), + tt::Literal::new( + text, span, - kind: match u16::to_le_bytes(repr.kind) { + match u16::to_le_bytes(repr.kind) { [0, _] => Err(()), [1, _] => Byte, [2, _] => Char, @@ -791,14 +793,12 @@ impl> Reader<'_, T> { [10, r] => CStrRaw(r), _ => unreachable!(), }, - suffix: if repr.suffix != !0 { - Some(Symbol::intern( - self.text[repr.suffix as usize].as_str(), - )) + if repr.suffix != !0 { + self.text[repr.suffix as usize].as_str() } else { - None + "" }, - } + ) } else { tt::token_to_literal(text, span) }) diff --git a/src/tools/rust-analyzer/crates/syntax-bridge/src/lib.rs b/src/tools/rust-analyzer/crates/syntax-bridge/src/lib.rs index 79b51a816ebe..ce238eb932ae 100644 --- a/src/tools/rust-analyzer/crates/syntax-bridge/src/lib.rs +++ b/src/tools/rust-analyzer/crates/syntax-bridge/src/lib.rs @@ -442,7 +442,7 @@ fn convert_doc_comment( text = &text[0..text.len() - 2]; } let (text, kind) = desugar_doc_comment_text(text, mode); - let lit = tt::Literal { symbol: text, span, kind, suffix: None }; + let lit = tt::Literal { text_and_suffix: text, span, kind, suffix_len: 0 }; tt::Leaf::from(lit) }; @@ -867,12 +867,9 @@ impl TtTreeSink<'_> { /// This occurs when a float literal is used as a field access. fn float_split(&mut self, has_pseudo_dot: bool) { let (text, span) = match self.cursor.token_tree() { - Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - symbol: text, - span, - kind: tt::LitKind::Float, - suffix: _, - }))) => (text.as_str(), *span), + Some(tt::TokenTree::Leaf(tt::Leaf::Literal( + lit @ tt::Literal { span, kind: tt::LitKind::Float, .. }, + ))) => (lit.text(), *span), tt => unreachable!("{tt:?}"), }; // FIXME: Span splitting diff --git a/src/tools/rust-analyzer/crates/syntax-bridge/src/to_parser_input.rs b/src/tools/rust-analyzer/crates/syntax-bridge/src/to_parser_input.rs index 6883f71307c8..851a4af86439 100644 --- a/src/tools/rust-analyzer/crates/syntax-bridge/src/to_parser_input.rs +++ b/src/tools/rust-analyzer/crates/syntax-bridge/src/to_parser_input.rs @@ -52,7 +52,7 @@ pub fn to_parser_input( }; res.push(kind, ctx_edition(lit.span.ctx)); - if kind == FLOAT_NUMBER && !lit.symbol.as_str().ends_with('.') { + if kind == FLOAT_NUMBER && !lit.text().ends_with('.') { // Tag the token as joint if it is float with a fractional part // we use this jointness to inform the parser about what token split // event to emit when we encounter a float literal in a field access diff --git a/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs b/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs index 487171c3e3e8..b9c389c7694e 100644 --- a/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs +++ b/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs @@ -920,7 +920,8 @@ impl ProcMacroExpander for ShortenProcMacroExpander { Leaf::Literal(it) => { // XXX Currently replaces any literals with an empty string, but supporting // "shortening" other literals would be nice. - it.symbol = Symbol::empty(); + it.text_and_suffix = Symbol::empty(); + it.suffix_len = 0; } Leaf::Punct(_) => {} Leaf::Ident(it) => { @@ -954,7 +955,7 @@ impl ProcMacroExpander for Issue17479ProcMacroExpander { let Some(TtElement::Leaf(tt::Leaf::Literal(lit))) = iter.next() else { return Err(ProcMacroExpansionError::Panic("incorrect Input".into())); }; - let symbol = &lit.symbol; + let symbol = Symbol::intern(lit.text()); let span = lit.span; Ok(quote! { span => #symbol() diff --git a/src/tools/rust-analyzer/crates/tt/src/lib.rs b/src/tools/rust-analyzer/crates/tt/src/lib.rs index 97e0b34ad7b7..91fcec9327c3 100644 --- a/src/tools/rust-analyzer/crates/tt/src/lib.rs +++ b/src/tools/rust-analyzer/crates/tt/src/lib.rs @@ -18,6 +18,7 @@ pub mod iter; use std::fmt; +use arrayvec::ArrayString; use buffer::Cursor; use intern::Symbol; use stdx::{impl_from, itertools::Itertools as _}; @@ -617,17 +618,56 @@ pub enum DelimiterKind { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Literal { - // escaped - pub symbol: Symbol, + /// Escaped, text then suffix concatenated. + pub text_and_suffix: Symbol, pub span: Span, pub kind: LitKind, - pub suffix: Option, + pub suffix_len: u8, } -pub fn token_to_literal(text: &str, span: Span) -> Literal -where - Span: Copy, -{ +impl Literal { + #[inline] + pub fn text_and_suffix(&self) -> (&str, &str) { + let text_and_suffix = self.text_and_suffix.as_str(); + text_and_suffix.split_at(text_and_suffix.len() - usize::from(self.suffix_len)) + } + + #[inline] + pub fn text(&self) -> &str { + self.text_and_suffix().0 + } + + #[inline] + pub fn suffix(&self) -> &str { + self.text_and_suffix().1 + } + + pub fn new(text: &str, span: Span, kind: LitKind, suffix: &str) -> Self { + const MAX_INLINE_CAPACITY: usize = 30; + let text_and_suffix = if suffix.is_empty() { + Symbol::intern(text) + } else if (text.len() + suffix.len()) < MAX_INLINE_CAPACITY { + let mut text_and_suffix = ArrayString::::new(); + text_and_suffix.push_str(text); + text_and_suffix.push_str(suffix); + Symbol::intern(&text_and_suffix) + } else { + let mut text_and_suffix = String::with_capacity(text.len() + suffix.len()); + text_and_suffix.push_str(text); + text_and_suffix.push_str(suffix); + Symbol::intern(&text_and_suffix) + }; + + Self { text_and_suffix, span, kind, suffix_len: suffix.len().try_into().unwrap() } + } + + #[inline] + pub fn new_no_suffix(text: &str, span: Span, kind: LitKind) -> Self { + Self { text_and_suffix: Symbol::intern(text), span, kind, suffix_len: 0 } + } +} + +pub fn token_to_literal(text: &str, span: Span) -> Literal { use rustc_lexer::LiteralKind; let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next_tuple(); @@ -636,12 +676,7 @@ where .. },)) = token else { - return Literal { - span, - symbol: Symbol::intern(text), - kind: LitKind::Err(()), - suffix: None, - }; + return Literal::new_no_suffix(text, span, LitKind::Err(())); }; let (kind, start_offset, end_offset) = match kind { @@ -672,20 +707,15 @@ where let (lit, suffix) = text.split_at(suffix_start as usize); let lit = &lit[start_offset..lit.len() - end_offset]; let suffix = match suffix { - "" | "_" => None, + "" | "_" => "", // ill-suffixed literals _ if !matches!(kind, LitKind::Integer | LitKind::Float | LitKind::Err(_)) => { - return Literal { - span, - symbol: Symbol::intern(text), - kind: LitKind::Err(()), - suffix: None, - }; + return Literal::new_no_suffix(text, span, LitKind::Err(())); } - suffix => Some(Symbol::intern(suffix)), + suffix => suffix, }; - Literal { span, symbol: Symbol::intern(lit), kind, suffix } + Literal::new(lit, span, kind, suffix) } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -805,15 +835,8 @@ fn print_debug_token(f: &mut fmt::Formatter<'_>, level: usize, tt: TtElement<'_> match tt { TtElement::Leaf(leaf) => match leaf { Leaf::Literal(lit) => { - write!( - f, - "{}LITERAL {:?} {}{} {:#?}", - align, - lit.kind, - lit.symbol, - lit.suffix.as_ref().map(|it| it.as_str()).unwrap_or(""), - lit.span - )?; + let (text, suffix) = lit.text_and_suffix(); + write!(f, "{}LITERAL {:?} {}{} {:#?}", align, lit.kind, text, suffix, lit.span)?; } Leaf::Punct(punct) => { write!( @@ -875,44 +898,28 @@ impl fmt::Display for Ident { impl fmt::Display for Literal { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (text, suffix) = self.text_and_suffix(); match self.kind { - LitKind::Byte => write!(f, "b'{}'", self.symbol), - LitKind::Char => write!(f, "'{}'", self.symbol), - LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", self.symbol), - LitKind::Str => write!(f, "\"{}\"", self.symbol), - LitKind::ByteStr => write!(f, "b\"{}\"", self.symbol), - LitKind::CStr => write!(f, "c\"{}\"", self.symbol), + LitKind::Byte => write!(f, "b'{}'", text), + LitKind::Char => write!(f, "'{}'", text), + LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", text), + LitKind::Str => write!(f, "\"{}\"", text), + LitKind::ByteStr => write!(f, "b\"{}\"", text), + LitKind::CStr => write!(f, "c\"{}\"", text), LitKind::StrRaw(num_of_hashes) => { let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"r{0:# { let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"br{0:# { let num_of_hashes = num_of_hashes as usize; - write!( - f, - r#"cr{0:# { let s = match it { - Leaf::Literal(it) => it.symbol.to_string(), + Leaf::Literal(it) => it.text().to_owned(), Leaf::Punct(it) => it.char.to_string(), Leaf::Ident(it) => format!("{}{}", it.is_raw.as_str(), it.sym), };