user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #21362 from ChayimFriedman2/compress-spans-v2

Browse source 
internal: Preparations for span compression
This commit is contained in:
Lukas Wirth 2025-12-29 08:30:19 +00:00 • committed by GitHub
commit 00f80b4355
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 524 additions and 495 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/tools/rust-analyzer/crates/cfg/src/cfg_expr.rs
View file

@ -157,7 +157,8 @@ fn next_cfg_expr_from_ast(
},
ctx: span::SyntaxContext::root(span::Edition::Edition2015),
};
let literal = tt::token_to_literal(literal.text(), dummy_span).symbol;
let literal =
Symbol::intern(tt::token_to_literal(literal.text(), dummy_span).text());
it.next();
CfgAtom::KeyValue { key: name, value: literal.clone() }.into()
} else {
@ -197,20 +198,21 @@ fn next_cfg_expr(it: &mut tt::iter::TtIter<'_>) -> Option<CfgExpr> {
Some(_) => return Some(CfgExpr::Invalid),
};
let ret = match it.peek() {
let mut it_clone = it.clone();
let ret = match it_clone.next() {
Some(TtElement::Leaf(tt::Leaf::Punct(punct)))
// Don't consume on e.g. `=>`.
if punct.char == '='
&& (punct.spacing == tt::Spacing::Alone
|| it.remaining().flat_tokens().get(1).is_none_or(|peek2| {
!matches!(peek2, tt::TokenTree::Leaf(tt::Leaf::Punct(_)))
|| it_clone.peek().is_none_or(|peek2| {
!matches!(peek2, tt::TtElement::Leaf(tt::Leaf::Punct(_)))
})) =>
{
match it.remaining().flat_tokens().get(1) {
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => {
match it_clone.next() {
Some(tt::TtElement::Leaf(tt::Leaf::Literal(literal))) => {
it.next();
it.next();
CfgAtom::KeyValue { key: name, value: literal.symbol.clone() }.into()
CfgAtom::KeyValue { key: name, value: Symbol::intern(literal.text()) }.into()
}
_ => return Some(CfgExpr::Invalid),
}

2
src/tools/rust-analyzer/crates/hir-def/src/item_tree/attrs.rs
View file

@ -226,7 +226,7 @@ impl<'attr> AttrQuery<'attr> {
}
#[inline]
pub(crate) fn string_value_with_span(self) -> Option<(&'attr Symbol, span::Span)> {
pub(crate) fn string_value_with_span(self) -> Option<(&'attr str, span::Span)> {
self.attrs().find_map(|attr| attr.string_value_with_span())
}

2
src/tools/rust-analyzer/crates/hir-def/src/nameres/attr_resolution.rs
View file

@ -114,7 +114,7 @@ pub(super) fn attr_macro_as_call_id(
let arg = match macro_attr.input.as_deref() {
Some(AttrInput::TokenTree(tt)) => {
let mut tt = tt.clone();
tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Invisible;
tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible);
Some(tt)
}

12
src/tools/rust-analyzer/crates/hir-def/src/nameres/collector.rs
View file

@ -17,7 +17,7 @@ use hir_expand::{
name::{AsName, Name},
proc_macro::CustomProcMacroExpander,
};
use intern::{Interned, sym};
use intern::{Interned, Symbol, sym};
use itertools::izip;
use la_arena::Idx;
use rustc_hash::{FxHashMap, FxHashSet};
@ -292,13 +292,13 @@ impl<'db> DefCollector<'db> {
match () {
() if *attr_name == sym::recursion_limit => {
if let Some(limit) = attr.string_value()
&& let Ok(limit) = limit.as_str().parse()
&& let Ok(limit) = limit.parse()
{
crate_data.recursion_limit = Some(limit);
}
}
() if *attr_name == sym::crate_type => {
if attr.string_value() == Some(&sym::proc_dash_macro) {
if attr.string_value() == Some("proc-macro") {
self.is_proc_macro = true;
}
}
@ -2460,14 +2460,14 @@ impl ModCollector<'_, '_> {
let name;
let name = match attrs.by_key(sym::rustc_builtin_macro).string_value_with_span() {
Some((it, span)) => {
name = Name::new_symbol(it.clone(), span.ctx);
name = Name::new_symbol(Symbol::intern(it), span.ctx);
&name
}
None => {
let explicit_name =
attrs.by_key(sym::rustc_builtin_macro).tt_values().next().and_then(|tt| {
match tt.token_trees().flat_tokens().first() {
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(name))) => Some(name),
match tt.token_trees().iter().next() {
Some(tt::TtElement::Leaf(tt::Leaf::Ident(name))) => Some(name),
_ => None,
}
});

51
src/tools/rust-analyzer/crates/hir-def/src/nameres/proc_macro.rs
View file

@ -2,10 +2,11 @@
use hir_expand::name::{AsName, Name};
use intern::sym;
use itertools::Itertools;
use crate::{
item_tree::Attrs,
tt::{Leaf, TokenTree, TopSubtree, TtElement},
tt::{Leaf, TopSubtree, TtElement},
};
#[derive(Debug, PartialEq, Eq)]
@ -61,35 +62,35 @@ impl Attrs<'_> {
// This fn is intended for `#[proc_macro_derive(..)]` and `#[rustc_builtin_macro(..)]`, which have
// the same structure.
#[rustfmt::skip]
pub(crate) fn parse_macro_name_and_helper_attrs(tt: &TopSubtree) -> Option<(Name, Box<[Name]>)> {
match tt.token_trees().flat_tokens() {
if let Some([TtElement::Leaf(Leaf::Ident(trait_name))]) =
tt.token_trees().iter().collect_array()
{
// `#[proc_macro_derive(Trait)]`
// `#[rustc_builtin_macro(Trait)]`
[TokenTree::Leaf(Leaf::Ident(trait_name))] => Some((trait_name.as_name(), Box::new([]))),
Some((trait_name.as_name(), Box::new([])))
} else if let Some(
[
TtElement::Leaf(Leaf::Ident(trait_name)),
TtElement::Leaf(Leaf::Punct(comma)),
TtElement::Leaf(Leaf::Ident(attributes)),
TtElement::Subtree(_, helpers),
],
) = tt.token_trees().iter().collect_array()
&& comma.char == ','
&& attributes.sym == sym::attributes
{
// `#[proc_macro_derive(Trait, attributes(helper1, helper2, ...))]`
// `#[rustc_builtin_macro(Trait, attributes(helper1, helper2, ...))]`
[
TokenTree::Leaf(Leaf::Ident(trait_name)),
TokenTree::Leaf(Leaf::Punct(comma)),
TokenTree::Leaf(Leaf::Ident(attributes)),
TokenTree::Subtree(_),
..
] if comma.char == ',' && attributes.sym == sym::attributes =>
{
let helpers = tt::TokenTreesView::new(&tt.token_trees().flat_tokens()[3..]).try_into_subtree()?;
let helpers = helpers
.iter()
.filter_map(|tt| match tt {
TtElement::Leaf(Leaf::Ident(helper)) => Some(helper.as_name()),
_ => None,
})
.collect::<Box<[_]>>();
let helpers = helpers
.filter_map(|tt| match tt {
TtElement::Leaf(Leaf::Ident(helper)) => Some(helper.as_name()),
_ => None,
})
.collect::<Box<[_]>>();
Some((trait_name.as_name(), helpers))
}
_ => None,
Some((trait_name.as_name(), helpers))
} else {
None
}
}

52
src/tools/rust-analyzer/crates/hir-expand/src/attrs.rs
View file

@ -35,7 +35,8 @@ use arrayvec::ArrayVec;
use base_db::Crate;
use cfg::{CfgExpr, CfgOptions};
use either::Either;
use intern::{Interned, Symbol};
use intern::Interned;
use itertools::Itertools;
use mbe::{DelimiterKind, Punct};
use parser::T;
use smallvec::SmallVec;
@ -416,47 +417,42 @@ impl fmt::Display for AttrInput {
impl Attr {
/// #[path = "string"]
pub fn string_value(&self) -> Option<&Symbol> {
pub fn string_value(&self) -> Option<&str> {
match self.input.as_deref()? {
AttrInput::Literal(tt::Literal {
symbol: text,
kind: tt::LitKind::Str | tt::LitKind::StrRaw(_),
..
}) => Some(text),
AttrInput::Literal(
lit @ tt::Literal { kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), .. },
) => Some(lit.text()),
_ => None,
}
}
/// #[path = "string"]
pub fn string_value_with_span(&self) -> Option<(&Symbol, span::Span)> {
pub fn string_value_with_span(&self) -> Option<(&str, span::Span)> {
match self.input.as_deref()? {
AttrInput::Literal(tt::Literal {
symbol: text,
kind: tt::LitKind::Str | tt::LitKind::StrRaw(_),
span,
suffix: _,
}) => Some((text, *span)),
AttrInput::Literal(
lit @ tt::Literal { kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), span, .. },
) => Some((lit.text(), *span)),
_ => None,
}
}
pub fn string_value_unescape(&self) -> Option<Cow<'_, str>> {
match self.input.as_deref()? {
AttrInput::Literal(tt::Literal {
symbol: text, kind: tt::LitKind::StrRaw(_), ..
}) => Some(Cow::Borrowed(text.as_str())),
AttrInput::Literal(tt::Literal { symbol: text, kind: tt::LitKind::Str, .. }) => {
unescape(text.as_str())
AttrInput::Literal(lit @ tt::Literal { kind: tt::LitKind::StrRaw(_), .. }) => {
Some(Cow::Borrowed(lit.text()))
}
AttrInput::Literal(lit @ tt::Literal { kind: tt::LitKind::Str, .. }) => {
unescape(lit.text())
}
_ => None,
}
}
/// #[path(ident)]
pub fn single_ident_value(&self) -> Option<&tt::Ident> {
pub fn single_ident_value(&self) -> Option<tt::Ident> {
match self.input.as_deref()? {
AttrInput::TokenTree(tt) => match tt.token_trees().flat_tokens() {
[tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] => Some(ident),
AttrInput::TokenTree(tt) => match tt.token_trees().iter().collect_array() {
Some([tt::TtElement::Leaf(tt::Leaf::Ident(ident))]) => Some(ident),
_ => None,
},
_ => None,
@ -492,7 +488,7 @@ fn parse_path_comma_token_tree<'a>(
args.token_trees()
.split(|tt| matches!(tt, tt::TtElement::Leaf(tt::Leaf::Punct(Punct { char: ',', .. }))))
.filter_map(move |tts| {
let span = tts.flat_tokens().first()?.first_span();
let span = tts.first_span()?;
Some((ModPath::from_tt(db, tts)?, span, tts))
})
}
@ -611,16 +607,12 @@ impl AttrId {
else {
return derive_attr_range;
};
let (Some(first_tt), Some(last_tt)) =
(derive_tts.flat_tokens().first(), derive_tts.flat_tokens().last())
let (Some(first_span), Some(last_span)) = (derive_tts.first_span(), derive_tts.last_span())
else {
return derive_attr_range;
};
let start = first_tt.first_span().range.start();
let end = match last_tt {
tt::TokenTree::Leaf(it) => it.span().range.end(),
tt::TokenTree::Subtree(it) => it.delimiter.close.range.end(),
};
let start = first_span.range.start();
let end = last_span.range.end();
TextRange::new(start, end)
}
}

152
src/tools/rust-analyzer/crates/hir-expand/src/builtin/fn_macro.rs
View file

@ -1,5 +1,7 @@
//! Builtin macro
use std::borrow::Cow;
use base_db::AnchoredPath;
use cfg::CfgExpr;
use either::Either;
@ -13,7 +15,7 @@ use span::{Edition, FileId, Span};
use stdx::format_to;
use syntax::{
format_smolstr,
unescape::{unescape_byte, unescape_char, unescape_str},
unescape::{unescape_byte, unescape_char},
};
use syntax_bridge::syntax_node_to_token_tree;
@ -177,12 +179,7 @@ fn line_expand(
// not incremental
ExpandResult::ok(tt::TopSubtree::invisible_from_leaves(
span,
[tt::Leaf::Literal(tt::Literal {
symbol: sym::INTEGER_0,
span,
kind: tt::LitKind::Integer,
suffix: Some(sym::u32),
})],
[tt::Leaf::Literal(tt::Literal::new("0", span, tt::LitKind::Integer, "u32"))],
))
}
@ -210,7 +207,7 @@ fn stringify_expand(
tt: &tt::TopSubtree,
span: Span,
) -> ExpandResult<tt::TopSubtree> {
let pretty = ::tt::pretty(tt.token_trees().flat_tokens());
let pretty = ::tt::pretty(tt.token_trees());
let expanded = quote! {span =>
#pretty
@ -283,7 +280,7 @@ fn format_args_expand(
) -> ExpandResult<tt::TopSubtree> {
let pound = mk_pound(span);
let mut tt = tt.clone();
tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis;
tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis);
ExpandResult::ok(quote! {span =>
builtin #pound format_args #tt
})
@ -297,14 +294,15 @@ fn format_args_nl_expand(
) -> ExpandResult<tt::TopSubtree> {
let pound = mk_pound(span);
let mut tt = tt.clone();
tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis;
if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
symbol: text,
kind: tt::LitKind::Str,
..
}))) = tt.0.get_mut(1)
tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis);
let lit = tt.as_token_trees().iter_flat_tokens().nth(1);
if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(
mut lit @ tt::Literal { kind: tt::LitKind::Str, .. },
))) = lit
{
*text = Symbol::intern(&format_smolstr!("{}\\n", text.as_str()));
let (text, suffix) = lit.text_and_suffix();
lit.text_and_suffix = Symbol::intern(&format_smolstr!("{text}\\n{suffix}"));
tt.set_token(1, lit.into());
}
ExpandResult::ok(quote! {span =>
builtin #pound format_args #tt
@ -318,7 +316,7 @@ fn asm_expand(
span: Span,
) -> ExpandResult<tt::TopSubtree> {
let mut tt = tt.clone();
tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis;
tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis);
let pound = mk_pound(span);
let expanded = quote! {span =>
builtin #pound asm #tt
@ -333,7 +331,7 @@ fn global_asm_expand(
span: Span,
) -> ExpandResult<tt::TopSubtree> {
let mut tt = tt.clone();
tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis;
tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis);
let pound = mk_pound(span);
let expanded = quote! {span =>
builtin #pound global_asm #tt
@ -348,7 +346,7 @@ fn naked_asm_expand(
span: Span,
) -> ExpandResult<tt::TopSubtree> {
let mut tt = tt.clone();
tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis;
tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis);
let pound = mk_pound(span);
let expanded = quote! {span =>
builtin #pound naked_asm #tt
@ -478,11 +476,11 @@ fn unreachable_expand(
// Pass the original arguments
let mut subtree = tt.clone();
*subtree.top_subtree_delimiter_mut() = tt::Delimiter {
subtree.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis);
subtree.set_top_subtree_delimiter_span(tt::DelimSpan {
open: call_site_span,
close: call_site_span,
kind: tt::DelimiterKind::Parenthesis,
};
});
// Expand to a macro call `$crate::panic::panic_{edition}`
let call = quote!(call_site_span =>#dollar_crate::panic::#mac! #subtree);
@ -518,16 +516,14 @@ fn compile_error_expand(
tt: &tt::TopSubtree,
span: Span,
) -> ExpandResult<tt::TopSubtree> {
let err = match &*tt.0 {
[
_,
tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
symbol: text,
span: _,
kind: tt::LitKind::Str | tt::LitKind::StrRaw(_),
suffix: _,
})),
] => ExpandError::other(span, Box::from(unescape_symbol(text).as_str())),
let err = match tt.iter().collect_array() {
Some(
[
tt::TtElement::Leaf(tt::Leaf::Literal(
lit @ tt::Literal { kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), .. },
)),
],
) => ExpandError::other(span, Box::from(unescape_str(lit.text()))),
_ => ExpandError::other(span, "`compile_error!` argument must be a string"),
};
@ -556,7 +552,7 @@ fn concat_expand(
// to ensure the right parsing order, so skip the parentheses here. Ideally we'd
// implement rustc's model. cc https://github.com/rust-lang/rust-analyzer/pull/10623
if let TtElement::Subtree(subtree, subtree_iter) = &t
&& let [tt::TokenTree::Leaf(tt)] = subtree_iter.remaining().flat_tokens()
&& let Some([tt::TtElement::Leaf(tt)]) = subtree_iter.clone().collect_array()
&& subtree.delimiter.kind == tt::DelimiterKind::Parenthesis
{
t = TtElement::Leaf(tt);
@ -568,20 +564,20 @@ fn concat_expand(
// as-is.
match it.kind {
tt::LitKind::Char => {
if let Ok(c) = unescape_char(it.symbol.as_str()) {
if let Ok(c) = unescape_char(it.text()) {
text.push(c);
}
record_span(it.span);
}
tt::LitKind::Integer | tt::LitKind::Float => {
format_to!(text, "{}", it.symbol.as_str())
format_to!(text, "{}", it.text())
}
tt::LitKind::Str => {
text.push_str(unescape_symbol(&it.symbol).as_str());
text.push_str(&unescape_str(it.text()));
record_span(it.span);
}
tt::LitKind::StrRaw(_) => {
format_to!(text, "{}", it.symbol.as_str());
format_to!(text, "{}", it.text());
record_span(it.span);
}
tt::LitKind::Byte
@ -619,7 +615,7 @@ fn concat_expand(
TtElement::Leaf(tt::Leaf::Literal(it))
if matches!(it.kind, tt::LitKind::Integer | tt::LitKind::Float) =>
{
format_to!(text, "-{}", it.symbol.as_str());
format_to!(text, "-{}", it.text());
record_span(punct.span.cover(it.span));
}
_ => {
@ -657,29 +653,25 @@ fn concat_bytes_expand(
};
for (i, t) in tt.iter().enumerate() {
match t {
TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
symbol: text,
span,
kind,
suffix: _,
})) => {
record_span(*span);
TtElement::Leaf(tt::Leaf::Literal(lit @ tt::Literal { span, kind, .. })) => {
let text = lit.text();
record_span(span);
match kind {
tt::LitKind::Byte => {
if let Ok(b) = unescape_byte(text.as_str()) {
if let Ok(b) = unescape_byte(text) {
bytes.extend(
b.escape_ascii().filter_map(|it| char::from_u32(it as u32)),
);
}
}
tt::LitKind::ByteStr => {
bytes.push_str(text.as_str());
bytes.push_str(text);
}
tt::LitKind::ByteStrRaw(_) => {
bytes.extend(text.as_str().escape_debug());
bytes.extend(text.escape_debug());
}
_ => {
err.get_or_insert(ExpandError::other(*span, "unexpected token"));
err.get_or_insert(ExpandError::other(span, "unexpected token"));
break;
}
}
@ -705,12 +697,7 @@ fn concat_bytes_expand(
ExpandResult {
value: tt::TopSubtree::invisible_from_leaves(
span,
[tt::Leaf::Literal(tt::Literal {
symbol: Symbol::intern(&bytes),
span,
kind: tt::LitKind::ByteStr,
suffix: None,
})],
[tt::Leaf::Literal(tt::Literal::new_no_suffix(&bytes, span, tt::LitKind::ByteStr))],
),
err,
}
@ -724,25 +711,19 @@ fn concat_bytes_expand_subtree(
) -> Result<(), ExpandError> {
for (ti, tt) in tree_iter.enumerate() {
match tt {
TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
symbol: text,
span,
kind: tt::LitKind::Byte,
suffix: _,
})) => {
if let Ok(b) = unescape_byte(text.as_str()) {
TtElement::Leaf(tt::Leaf::Literal(
lit @ tt::Literal { span, kind: tt::LitKind::Byte, .. },
)) => {
if let Ok(b) = unescape_byte(lit.text()) {
bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32)));
}
record_span(*span);
record_span(span);
}
TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
symbol: text,
span,
kind: tt::LitKind::Integer,
suffix: _,
})) => {
record_span(*span);
if let Ok(b) = text.as_str().parse::<u8>() {
TtElement::Leaf(tt::Leaf::Literal(
lit @ tt::Literal { span, kind: tt::LitKind::Integer, .. },
)) => {
record_span(span);
if let Ok(b) = lit.text().parse::<u8>() {
bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32)));
}
}
@ -791,18 +772,16 @@ fn parse_string(tt: &tt::TopSubtree) -> Result<(Symbol, Span), ExpandError> {
}
match tt {
TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
symbol: text,
TtElement::Leaf(tt::Leaf::Literal(lit @ tt::Literal {
span,
kind: tt::LitKind::Str,
suffix: _,
})) => Ok((unescape_symbol(text), *span)),
TtElement::Leaf(tt::Leaf::Literal(tt::Literal {
symbol: text,
..
})) => Ok((Symbol::intern(&unescape_str(lit.text())), span)),
TtElement::Leaf(tt::Leaf::Literal(lit @ tt::Literal {
span,
kind: tt::LitKind::StrRaw(_),
suffix: _,
})) => Ok((text.clone(), *span)),
..
})) => Ok((Symbol::intern(lit.text()), span)),
TtElement::Leaf(l) => Err(*l.span()),
TtElement::Subtree(tt, _) => Err(tt.delimiter.open.cover(tt.delimiter.close)),
}
@ -854,10 +833,10 @@ fn include_bytes_expand(
let res = tt::TopSubtree::invisible_from_leaves(
span,
[tt::Leaf::Literal(tt::Literal {
symbol: Symbol::empty(),
text_and_suffix: Symbol::empty(),
span,
kind: tt::LitKind::ByteStrRaw(1),
suffix: None,
suffix_len: 0,
})],
);
ExpandResult::ok(res)
@ -978,17 +957,16 @@ fn quote_expand(
)
}
fn unescape_symbol(s: &Symbol) -> Symbol {
if s.as_str().contains('\\') {
let s = s.as_str();
fn unescape_str(s: &str) -> Cow<'_, str> {
if s.contains('\\') {
let mut buf = String::with_capacity(s.len());
unescape_str(s, |_, c| {
syntax::unescape::unescape_str(s, |_, c| {
if let Ok(c) = c {
buf.push(c)
}
});
Symbol::intern(&buf)
Cow::Owned(buf)
} else {
s.clone()
Cow::Borrowed(s)
}
}

19
src/tools/rust-analyzer/crates/hir-expand/src/builtin/quote.rs
View file

@ -163,7 +163,7 @@ impl ToTokenTree for crate::tt::SubtreeView<'_> {
impl ToTokenTree for crate::tt::TopSubtree {
fn to_tokens(self, _: Span, builder: &mut TopSubtreeBuilder) {
builder.extend_tt_dangerous(self.0);
builder.extend_with_tt(self.as_token_trees());
}
}
@ -172,10 +172,9 @@ impl ToTokenTree for crate::tt::TtElement<'_> {
match self {
crate::tt::TtElement::Leaf(leaf) => builder.push(leaf.clone()),
crate::tt::TtElement::Subtree(subtree, subtree_iter) => {
builder.extend_tt_dangerous(
std::iter::once(crate::tt::TokenTree::Subtree(subtree.clone()))
.chain(subtree_iter.remaining().flat_tokens().iter().cloned()),
);
builder.open(subtree.delimiter.kind, subtree.delimiter.open);
builder.extend_with_tt(subtree_iter.remaining());
builder.close(subtree.delimiter.close);
}
}
}
@ -200,16 +199,16 @@ impl<T: ToTokenTree + Clone> ToTokenTree for &T {
}
impl_to_to_tokentrees! {
span: u32 => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } };
span: usize => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } };
span: i32 => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } };
span: u32 => self { crate::tt::Literal{text_and_suffix: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix_len: 0 } };
span: usize => self { crate::tt::Literal{text_and_suffix: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix_len: 0 } };
span: i32 => self { crate::tt::Literal{text_and_suffix: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix_len: 0 } };
span: bool => self { crate::tt::Ident{sym: if self { sym::true_ } else { sym::false_ }, span, is_raw: tt::IdentIsRaw::No } };
_span: crate::tt::Leaf => self { self };
_span: crate::tt::Literal => self { self };
_span: crate::tt::Ident => self { self };
_span: crate::tt::Punct => self { self };
span: &str => self { crate::tt::Literal{symbol: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix: None }};
span: String => self { crate::tt::Literal{symbol: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix: None }};
span: &str => self { crate::tt::Literal{text_and_suffix: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix_len: 0 }};
span: String => self { crate::tt::Literal{text_and_suffix: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix_len: 0 }};
span: Name => self {
let (is_raw, s) = IdentIsRaw::split_from_symbol(self.as_str());
crate::tt::Ident{sym: Symbol::intern(s), span, is_raw }

14
src/tools/rust-analyzer/crates/hir-expand/src/db.rs
View file

@ -237,7 +237,8 @@ pub fn expand_speculative(
span,
DocCommentDesugarMode::ProcMacro,
);
*tree.top_subtree_delimiter_mut() = tt::Delimiter::invisible_spanned(span);
tree.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible);
tree.set_top_subtree_delimiter_span(tt::DelimSpan::from_single(span));
tree
},
)
@ -255,7 +256,7 @@ pub fn expand_speculative(
span,
DocCommentDesugarMode::ProcMacro,
);
attr_arg.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Invisible;
attr_arg.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible);
Some(attr_arg)
}
_ => None,
@ -270,7 +271,8 @@ pub fn expand_speculative(
let mut speculative_expansion = match loc.def.kind {
MacroDefKind::ProcMacro(ast, expander, _) => {
let span = db.proc_macro_span(ast);
*tt.top_subtree_delimiter_mut() = tt::Delimiter::invisible_spanned(span);
tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible);
tt.set_top_subtree_delimiter_span(tt::DelimSpan::from_single(span));
expander.expand(
db,
loc.def.krate,
@ -430,7 +432,7 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult {
(
Arc::new(tt::TopSubtree::from_token_trees(
tt::Delimiter { open: span, close: span, kind },
tt::TokenTreesView::new(&[]),
tt::TokenTreesView::empty(),
)),
SyntaxFixupUndoInfo::default(),
span,
@ -478,7 +480,7 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult {
);
if loc.def.is_proc_macro() {
// proc macros expect their inputs without parentheses, MBEs expect it with them included
tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Invisible;
tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible);
}
return (Arc::new(tt), SyntaxFixupUndoInfo::NONE, span);
}
@ -512,7 +514,7 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult {
if loc.def.is_proc_macro() {
// proc macros expect their inputs without parentheses, MBEs expect it with them included
tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Invisible;
tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible);
}
(Arc::new(tt), undo_info, span)

2
src/tools/rust-analyzer/crates/hir-expand/src/eager.rs
View file

@ -96,7 +96,7 @@ pub fn expand_eager_macro_input(
DocCommentDesugarMode::Mbe,
);
subtree.top_subtree_delimiter_mut().kind = crate::tt::DelimiterKind::Invisible;
subtree.set_top_subtree_delimiter_kind(crate::tt::DelimiterKind::Invisible);
let loc = MacroCallLoc {
def,

97
src/tools/rust-analyzer/crates/hir-expand/src/fixup.rs
View file

@ -15,7 +15,7 @@ use syntax::{
};
use syntax_bridge::DocCommentDesugarMode;
use triomphe::Arc;
use tt::Spacing;
use tt::{Spacing, TransformTtAction, transform_tt};
use crate::{
span_map::SpanMapRef,
@ -343,93 +343,29 @@ fn has_error_to_handle(node: &SyntaxNode) -> bool {
pub(crate) fn reverse_fixups(tt: &mut TopSubtree, undo_info: &SyntaxFixupUndoInfo) {
let Some(undo_info) = undo_info.original.as_deref() else { return };
let undo_info = &**undo_info;
let delimiter = tt.top_subtree_delimiter_mut();
let top_subtree = tt.top_subtree();
let open_span = top_subtree.delimiter.open;
let close_span = top_subtree.delimiter.close;
#[allow(deprecated)]
if never!(
delimiter.close.anchor.ast_id == FIXUP_DUMMY_AST_ID
|| delimiter.open.anchor.ast_id == FIXUP_DUMMY_AST_ID
close_span.anchor.ast_id == FIXUP_DUMMY_AST_ID
|| open_span.anchor.ast_id == FIXUP_DUMMY_AST_ID
) {
let span = |file_id| Span {
range: TextRange::empty(TextSize::new(0)),
anchor: SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID },
ctx: SyntaxContext::root(span::Edition::Edition2015),
};
delimiter.open = span(delimiter.open.anchor.file_id);
delimiter.close = span(delimiter.close.anchor.file_id);
tt.set_top_subtree_delimiter_span(tt::DelimSpan {
open: span(open_span.anchor.file_id),
close: span(close_span.anchor.file_id),
});
}
reverse_fixups_(tt, undo_info);
}
#[derive(Debug)]
enum TransformTtAction<'a> {
Keep,
ReplaceWith(tt::TokenTreesView<'a>),
}
impl TransformTtAction<'_> {
fn remove() -> Self {
Self::ReplaceWith(tt::TokenTreesView::new(&[]))
}
}
/// This function takes a token tree, and calls `callback` with each token tree in it.
/// Then it does what the callback says: keeps the tt or replaces it with a (possibly empty)
/// tts view.
fn transform_tt<'a, 'b>(
tt: &'a mut Vec<tt::TokenTree>,
mut callback: impl FnMut(&mut tt::TokenTree) -> TransformTtAction<'b>,
) {
// We need to keep a stack of the currently open subtrees, because we need to update
// them if we change the number of items in them.
let mut subtrees_stack = Vec::new();
let mut i = 0;
while i < tt.len() {
'pop_finished_subtrees: while let Some(&subtree_idx) = subtrees_stack.last() {
let tt::TokenTree::Subtree(subtree) = &tt[subtree_idx] else {
unreachable!("non-subtree on subtrees stack");
};
if i >= subtree_idx + 1 + subtree.usize_len() {
subtrees_stack.pop();
} else {
break 'pop_finished_subtrees;
}
}
let action = callback(&mut tt[i]);
match action {
TransformTtAction::Keep => {
// This cannot be shared with the replaced case, because then we may push the same subtree
// twice, and will update it twice which will lead to errors.
if let tt::TokenTree::Subtree(_) = &tt[i] {
subtrees_stack.push(i);
}
i += 1;
}
TransformTtAction::ReplaceWith(replacement) => {
let old_len = 1 + match &tt[i] {
tt::TokenTree::Leaf(_) => 0,
tt::TokenTree::Subtree(subtree) => subtree.usize_len(),
};
let len_diff = replacement.len() as i64 - old_len as i64;
tt.splice(i..i + old_len, replacement.flat_tokens().iter().cloned());
// Skip the newly inserted replacement, we don't want to visit it.
i += replacement.len();
for &subtree_idx in &subtrees_stack {
let tt::TokenTree::Subtree(subtree) = &mut tt[subtree_idx] else {
unreachable!("non-subtree on subtrees stack");
};
subtree.len = (i64::from(subtree.len) + len_diff).try_into().unwrap();
}
}
}
}
}
fn reverse_fixups_(tt: &mut TopSubtree, undo_info: &[TopSubtree]) {
let mut tts = std::mem::take(&mut tt.0).into_vec();
transform_tt(&mut tts, |tt| match tt {
transform_tt(tt, |tt| match tt {
tt::TokenTree::Leaf(leaf) => {
let span = leaf.span();
let is_real_leaf = span.anchor.ast_id != FIXUP_DUMMY_AST_ID;
@ -459,7 +395,6 @@ fn reverse_fixups_(tt: &mut TopSubtree, undo_info: &[TopSubtree]) {
TransformTtAction::Keep
}
});
tt.0 = tts.into_boxed_slice();
}
#[cfg(test)]
@ -480,7 +415,7 @@ mod tests {
// `TokenTree`s, see the last assertion in `check()`.
fn check_leaf_eq(a: &tt::Leaf, b: &tt::Leaf) -> bool {
match (a, b) {
(tt::Leaf::Literal(a), tt::Leaf::Literal(b)) => a.symbol == b.symbol,
(tt::Leaf::Literal(a), tt::Leaf::Literal(b)) => a.text_and_suffix == b.text_and_suffix,
(tt::Leaf::Punct(a), tt::Leaf::Punct(b)) => a.char == b.char,
(tt::Leaf::Ident(a), tt::Leaf::Ident(b)) => a.sym == b.sym,
_ => false,
@ -488,9 +423,9 @@ mod tests {
}
fn check_subtree_eq(a: &tt::TopSubtree, b: &tt::TopSubtree) -> bool {
let a = a.view().as_token_trees().flat_tokens();
let b = b.view().as_token_trees().flat_tokens();
a.len() == b.len() && std::iter::zip(a, b).all(|(a, b)| check_tt_eq(a, b))
let a = a.view().as_token_trees().iter_flat_tokens();
let b = b.view().as_token_trees().iter_flat_tokens();
a.len() == b.len() && std::iter::zip(a, b).all(|(a, b)| check_tt_eq(&a, &b))
}
fn check_tt_eq(a: &tt::TokenTree, b: &tt::TokenTree) -> bool {
@ -545,7 +480,7 @@ mod tests {
// the fixed-up tree should not contain braces as punct
// FIXME: should probably instead check that it's a valid punctuation character
for x in tt.token_trees().flat_tokens() {
for x in tt.token_trees().iter_flat_tokens() {
match x {
::tt::TokenTree::Leaf(::tt::Leaf::Punct(punct)) => {
assert!(!matches!(punct.char, '{' | '}' | '(' | ')' | '[' | ']'))

10
src/tools/rust-analyzer/crates/hir-expand/src/mod_path.rs
View file

@ -355,16 +355,16 @@ fn convert_path_tt(db: &dyn ExpandDatabase, tt: tt::TokenTreesView<'_>) -> Optio
tt::Leaf::Punct(tt::Punct { char: ':', .. }) => PathKind::Abs,
_ => return None,
},
tt::Leaf::Ident(tt::Ident { sym: text, span, .. }) if *text == sym::dollar_crate => {
tt::Leaf::Ident(tt::Ident { sym: text, span, .. }) if text == sym::dollar_crate => {
resolve_crate_root(db, span.ctx).map(PathKind::DollarCrate).unwrap_or(PathKind::Crate)
}
tt::Leaf::Ident(tt::Ident { sym: text, .. }) if *text == sym::self_ => PathKind::SELF,
tt::Leaf::Ident(tt::Ident { sym: text, .. }) if *text == sym::super_ => {
tt::Leaf::Ident(tt::Ident { sym: text, .. }) if text == sym::self_ => PathKind::SELF,
tt::Leaf::Ident(tt::Ident { sym: text, .. }) if text == sym::super_ => {
let mut deg = 1;
while let Some(tt::Leaf::Ident(tt::Ident { sym: text, span, is_raw: _ })) =
leaves.next()
{
if *text != sym::super_ {
if text != sym::super_ {
segments.push(Name::new_symbol(text.clone(), span.ctx));
break;
}
@ -372,7 +372,7 @@ fn convert_path_tt(db: &dyn ExpandDatabase, tt: tt::TokenTreesView<'_>) -> Optio
}
PathKind::Super(deg)
}
tt::Leaf::Ident(tt::Ident { sym: text, .. }) if *text == sym::crate_ => PathKind::Crate,
tt::Leaf::Ident(tt::Ident { sym: text, .. }) if text == sym::crate_ => PathKind::Crate,
tt::Leaf::Ident(ident) => {
segments.push(Name::new_symbol(ident.sym.clone(), ident.span.ctx));
PathKind::Plain

9
src/tools/rust-analyzer/crates/mbe/src/benchmark.rs
View file

@ -54,7 +54,7 @@ fn benchmark_expand_macro_rules() {
.map(|(id, tt)| {
let res = rules[&id].expand(&db, &tt, |_| (), MacroCallStyle::FnLike, DUMMY);
assert!(res.err.is_none());
res.value.0.0.len()
res.value.0.as_token_trees().len()
})
.sum()
};
@ -236,12 +236,7 @@ fn invocation_fixtures(
tt::Leaf::Punct(tt::Punct { span: DUMMY, char, spacing: tt::Spacing::Alone })
}
fn make_literal(lit: &str) -> tt::Leaf {
tt::Leaf::Literal(tt::Literal {
span: DUMMY,
symbol: Symbol::intern(lit),
kind: tt::LitKind::Str,
suffix: None,
})
tt::Leaf::Literal(tt::Literal::new_no_suffix(lit, DUMMY, tt::LitKind::Str))
}
fn make_subtree(kind: tt::DelimiterKind, builder: &mut tt::TopSubtreeBuilder) {
builder.open(kind, DUMMY);

2
src/tools/rust-analyzer/crates/mbe/src/expander.rs
View file

@ -162,7 +162,7 @@ impl Fragment<'_> {
Fragment::Tokens { tree, .. } => tree.len() == 0,
Fragment::Expr(it) => it.len() == 0,
Fragment::Path(it) => it.len() == 0,
Fragment::TokensOwned(it) => it.0.is_empty(),
Fragment::TokensOwned(_) => false, // A `TopSubtree` is never empty
}
}
}

15
src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs
View file

@ -517,7 +517,8 @@ fn match_loop_inner<'t>(
}
OpDelimited::Op(Op::Literal(lhs)) => {
if let Ok(rhs) = src.clone().expect_leaf() {
if matches!(rhs, tt::Leaf::Literal(it) if it.symbol == lhs.symbol) {
if matches!(&rhs, tt::Leaf::Literal(it) if it.text_and_suffix == lhs.text_and_suffix)
{
item.dot.next();
} else {
res.add_err(ExpandError::new(
@ -537,7 +538,7 @@ fn match_loop_inner<'t>(
}
OpDelimited::Op(Op::Ident(lhs)) => {
if let Ok(rhs) = src.clone().expect_leaf() {
if matches!(rhs, tt::Leaf::Ident(it) if it.sym == lhs.sym) {
if matches!(&rhs, tt::Leaf::Ident(it) if it.sym == lhs.sym) {
item.dot.next();
} else {
res.add_err(ExpandError::new(
@ -701,7 +702,7 @@ fn match_loop<'t>(
|| !(bb_items.is_empty() || next_items.is_empty())
|| bb_items.len() > 1;
if has_leftover_tokens {
res.unmatched_tts += src.remaining().flat_tokens().len();
res.unmatched_tts += src.remaining().len();
res.add_err(ExpandError::new(span.open, ExpandErrorKind::LeftoverTokens));
if let Some(error_recover_item) = error_recover_item {
@ -953,8 +954,8 @@ fn expect_separator(iter: &mut TtIter<'_>, separator: &Separator) -> bool {
},
Separator::Literal(lhs) => match fork.expect_literal() {
Ok(rhs) => match rhs {
tt::Leaf::Literal(rhs) => rhs.symbol == lhs.symbol,
tt::Leaf::Ident(rhs) => rhs.sym == lhs.symbol,
tt::Leaf::Literal(rhs) => rhs.text_and_suffix == lhs.text_and_suffix,
tt::Leaf::Ident(rhs) => rhs.sym == lhs.text_and_suffix,
tt::Leaf::Punct(_) => false,
},
Err(_) => false,
@ -991,7 +992,7 @@ fn expect_tt(iter: &mut TtIter<'_>) -> Result<(), ()> {
Ok(())
}
fn expect_lifetime<'a>(iter: &mut TtIter<'a>) -> Result<&'a tt::Ident, ()> {
fn expect_lifetime<'a>(iter: &mut TtIter<'a>) -> Result<tt::Ident, ()> {
let punct = iter.expect_single_punct()?;
if punct.char != '\'' {
return Err(());
@ -1000,7 +1001,7 @@ fn expect_lifetime<'a>(iter: &mut TtIter<'a>) -> Result<&'a tt::Ident, ()> {
}
fn eat_char(iter: &mut TtIter<'_>, c: char) {
if matches!(iter.peek(), Some(TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char, .. }))) if *char == c)
if matches!(iter.peek(), Some(TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char, .. }))) if char == c)
{
iter.next().expect("already peeked");
}

40
src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs
View file

@ -3,6 +3,7 @@
use intern::{Symbol, sym};
use span::{Edition, Span};
use stdx::itertools::Itertools;
use tt::{Delimiter, TopSubtreeBuilder, iter::TtElement};
use super::TokensOrigin;
@ -221,10 +222,10 @@ fn expand_subtree(
let index =
ctx.nesting.get(ctx.nesting.len() - 1 - depth).map_or(0, |nest| nest.idx);
builder.push(tt::Leaf::Literal(tt::Literal {
symbol: Symbol::integer(index),
text_and_suffix: Symbol::integer(index),
span: ctx.call_site,
kind: tt::LitKind::Integer,
suffix: None,
suffix_len: 0,
}));
}
Op::Len { depth } => {
@ -233,10 +234,10 @@ fn expand_subtree(
0
});
builder.push(tt::Leaf::Literal(tt::Literal {
symbol: Symbol::integer(length),
text_and_suffix: Symbol::integer(length),
span: ctx.call_site,
kind: tt::LitKind::Integer,
suffix: None,
suffix_len: 0,
}));
}
Op::Count { name, depth } => {
@ -277,9 +278,9 @@ fn expand_subtree(
let res = count(binding, 0, depth.unwrap_or(0));
builder.push(tt::Leaf::Literal(tt::Literal {
symbol: Symbol::integer(res),
text_and_suffix: Symbol::integer(res),
span: ctx.call_site,
suffix: None,
suffix_len: 0,
kind: tt::LitKind::Integer,
}));
}
@ -293,7 +294,7 @@ fn expand_subtree(
ConcatMetaVarExprElem::Literal(lit) => {
// FIXME: This isn't really correct wrt. escaping, but that's what rustc does and anyway
// escaping is used most of the times for characters that are invalid in identifiers.
concatenated.push_str(lit.symbol.as_str())
concatenated.push_str(lit.text())
}
ConcatMetaVarExprElem::Var(var) => {
// Handling of repetitions in `${concat}` isn't fleshed out in rustc, so we currently
@ -324,13 +325,11 @@ fn expand_subtree(
}
_ => (None, None),
};
let value = match values {
let value = match &values {
(Some(TtElement::Leaf(tt::Leaf::Ident(ident))), None) => {
ident.sym.as_str()
}
(Some(TtElement::Leaf(tt::Leaf::Literal(lit))), None) => {
lit.symbol.as_str()
}
(Some(TtElement::Leaf(tt::Leaf::Literal(lit))), None) => lit.text(),
_ => {
if err.is_none() {
err = Some(ExpandError::binding_error(
@ -412,15 +411,15 @@ fn expand_var(
// Check if this is a simple negative literal (MINUS + LITERAL)
// that should not be wrapped in parentheses
let is_negative_literal = matches!(
sub.flat_tokens(),
[
tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '-', .. })),
tt::TokenTree::Leaf(tt::Leaf::Literal(_))
]
sub.iter().collect_array(),
Some([
tt::TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char: '-', .. })),
tt::TtElement::Leaf(tt::Leaf::Literal(_))
])
);
let wrap_in_parens = !is_negative_literal
&& !matches!(sub.flat_tokens(), [tt::TokenTree::Leaf(_)])
&& !matches!(sub.iter().collect_array(), Some([tt::TtElement::Leaf(_)]))
&& sub.try_into_subtree().is_none_or(|it| {
it.top_subtree().delimiter.kind == tt::DelimiterKind::Invisible
});
@ -560,8 +559,8 @@ fn fix_up_and_push_path_tt(
// argument list and thus needs `::` between it and `FnOnce`. However in
// today's Rust this type of path *semantically* cannot appear as a
// top-level expression-context path, so we can safely ignore it.
if let [tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '<', .. }))] =
tt.flat_tokens()
if let Some([tt::TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char: '<', .. }))]) =
tt.iter().collect_array()
{
builder.extend([
tt::Leaf::Punct(tt::Punct {
@ -577,7 +576,8 @@ fn fix_up_and_push_path_tt(
]);
}
}
prev_was_ident = matches!(tt.flat_tokens(), [tt::TokenTree::Leaf(tt::Leaf::Ident(_))]);
prev_was_ident =
matches!(tt.iter().collect_array(), Some([tt::TtElement::Leaf(tt::Leaf::Ident(_))]));
builder.extend_with_tt(tt);
}
}

21
src/tools/rust-analyzer/crates/mbe/src/parser.rs
View file

@ -192,7 +192,7 @@ impl PartialEq for Separator {
match (self, other) {
(Ident(a), Ident(b)) => a.sym == b.sym,
(Literal(a), Literal(b)) => a.symbol == b.symbol,
(Literal(a), Literal(b)) => a.text_and_suffix == b.text_and_suffix,
(Puncts(a), Puncts(b)) if a.len() == b.len() => {
let a_iter = a.iter().map(|a| a.char);
let b_iter = b.iter().map(|b| b.char);
@ -224,7 +224,7 @@ fn next_op(
None => {
return Ok(Op::Punct({
let mut res = ArrayVec::new();
res.push(*p);
res.push(p);
Box::new(res)
}));
}
@ -268,9 +268,9 @@ fn next_op(
let id = ident.span;
Op::Var { name, kind, id }
}
tt::Leaf::Literal(lit) if is_boolean_literal(lit) => {
tt::Leaf::Literal(lit) if is_boolean_literal(&lit) => {
let kind = eat_fragment_kind(edition, src, mode)?;
let name = lit.symbol.clone();
let name = lit.text_and_suffix.clone();
let id = lit.span;
Op::Var { name, kind, id }
}
@ -282,7 +282,7 @@ fn next_op(
}
Mode::Template => Op::Punct({
let mut res = ArrayVec::new();
res.push(*punct);
res.push(punct);
Box::new(res)
}),
},
@ -364,7 +364,7 @@ fn eat_fragment_kind(
}
fn is_boolean_literal(lit: &tt::Literal) -> bool {
matches!(lit.symbol.as_str(), "true" | "false")
lit.text_and_suffix == sym::true_ || lit.text_and_suffix == sym::false_
}
fn parse_repeat(src: &mut TtIter<'_>) -> Result<(Option<Separator>, RepeatKind), ParseError> {
@ -400,7 +400,7 @@ fn parse_repeat(src: &mut TtIter<'_>) -> Result<(Option<Separator>, RepeatKind),
'?' => RepeatKind::ZeroOrOne,
_ => match &mut separator {
Separator::Puncts(puncts) if puncts.len() < 3 => {
puncts.push(*punct);
puncts.push(punct);
continue;
}
_ => return Err(ParseError::InvalidRepeat),
@ -478,11 +478,12 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> {
fn parse_depth(src: &mut TtIter<'_>) -> Result<usize, ()> {
if src.is_empty() {
Ok(0)
} else if let tt::Leaf::Literal(tt::Literal { symbol: text, suffix: None, .. }) =
src.expect_literal()?
} else if let tt::Leaf::Literal(lit) = src.expect_literal()?
&& let (text, suffix) = lit.text_and_suffix()
&& suffix.is_empty()
{
// Suffixes are not allowed.
text.as_str().parse().map_err(|_| ())
text.parse().map_err(|_| ())
} else {
Err(())
}

25
src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg.rs
View file

@ -172,7 +172,7 @@ impl Message for Response {}
#[cfg(test)]
mod tests {
use intern::{Symbol, sym};
use intern::Symbol;
use span::{
Edition, ROOT_ERASED_FILE_AST_ID, Span, SpanAnchor, SyntaxContext, TextRange, TextSize,
};
@ -232,16 +232,15 @@ mod tests {
}
.into(),
);
builder.push(Leaf::Literal(Literal {
symbol: Symbol::intern("Foo"),
span: Span {
builder.push(Leaf::Literal(Literal::new_no_suffix(
"Foo",
Span {
range: TextRange::at(TextSize::new(10), TextSize::of("\"Foo\"")),
anchor,
ctx: SyntaxContext::root(Edition::CURRENT),
},
kind: tt::LitKind::Str,
suffix: None,
}));
tt::LitKind::Str,
)));
builder.push(Leaf::Punct(Punct {
char: '@',
span: Span {
@ -267,16 +266,16 @@ mod tests {
ctx: SyntaxContext::root(Edition::CURRENT),
},
);
builder.push(Leaf::Literal(Literal {
symbol: sym::INTEGER_0,
span: Span {
builder.push(Leaf::Literal(Literal::new(
"0",
Span {
range: TextRange::at(TextSize::new(16), TextSize::of("0u32")),
anchor,
ctx: SyntaxContext::root(Edition::CURRENT),
},
kind: tt::LitKind::Integer,
suffix: Some(sym::u32),
}));
tt::LitKind::Integer,
"u32",
)));
builder.close(Span {
range: TextRange::at(TextSize::new(20), TextSize::of(']')),
anchor,

38
src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs
View file

@ -489,7 +489,7 @@ struct Writer<'a, 'span, S: SpanTransformer, W> {
impl<'a, T: SpanTransformer<Span = span::Span>> Writer<'a, '_, T, tt::iter::TtIter<'a>> {
fn write_subtree(&mut self, root: tt::SubtreeView<'a>) {
let subtree = root.top_subtree();
self.enqueue(subtree, root.iter());
self.enqueue(&subtree, root.iter());
while let Some((idx, len, subtree)) = self.work.pop_front() {
self.subtree(idx, len, subtree);
}
@ -504,7 +504,7 @@ impl<'a, T: SpanTransformer<Span = span::Span>> Writer<'a, '_, T, tt::iter::TtIt
for child in subtree {
let idx_tag = match child {
tt::iter::TtElement::Subtree(subtree, subtree_iter) => {
let idx = self.enqueue(subtree, subtree_iter);
let idx = self.enqueue(&subtree, subtree_iter);
idx << 2
}
tt::iter::TtElement::Leaf(leaf) => match leaf {
@ -512,9 +512,14 @@ impl<'a, T: SpanTransformer<Span = span::Span>> Writer<'a, '_, T, tt::iter::TtIt
let idx = self.literal.len() as u32;
let id = self.token_id_of(lit.span);
let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA {
let (text, suffix) = lit.text_and_suffix();
(
self.intern(lit.symbol.as_str()),
lit.suffix.as_ref().map(|s| self.intern(s.as_str())).unwrap_or(!0),
self.intern_owned(text.to_owned()),
if suffix.is_empty() {
!0
} else {
self.intern_owned(suffix.to_owned())
},
)
} else {
(self.intern_owned(format!("{lit}")), !0)
@ -549,11 +554,11 @@ impl<'a, T: SpanTransformer<Span = span::Span>> Writer<'a, '_, T, tt::iter::TtIt
let idx = self.ident.len() as u32;
let id = self.token_id_of(ident.span);
let text = if self.version >= EXTENDED_LEAF_DATA {
self.intern(ident.sym.as_str())
self.intern_owned(ident.sym.as_str().to_owned())
} else if ident.is_raw.yes() {
self.intern_owned(format!("r#{}", ident.sym.as_str(),))
} else {
self.intern(ident.sym.as_str())
self.intern_owned(ident.sym.as_str().to_owned())
};
self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw.yes() });
(idx << 2) | 0b11
@ -565,7 +570,7 @@ impl<'a, T: SpanTransformer<Span = span::Span>> Writer<'a, '_, T, tt::iter::TtIt
}
}
fn enqueue(&mut self, subtree: &'a tt::Subtree, contents: tt::iter::TtIter<'a>) -> u32 {
fn enqueue(&mut self, subtree: &tt::Subtree, contents: tt::iter::TtIter<'a>) -> u32 {
let idx = self.subtree.len();
let open = self.token_id_of(subtree.delimiter.open);
let close = self.token_id_of(subtree.delimiter.close);
@ -582,6 +587,7 @@ impl<'a, T: SpanTransformer, U> Writer<'a, '_, T, U> {
T::token_id_of(self.span_data_table, span)
}
#[cfg(feature = "sysroot-abi")]
pub(crate) fn intern(&mut self, text: &'a str) -> u32 {
let table = &mut self.text;
*self.string_table.entry(text.into()).or_insert_with(|| {
@ -770,10 +776,10 @@ impl<T: SpanTransformer<Span = span::Span>> Reader<'_, T> {
let span = read_span(repr.id);
s.push(
tt::Leaf::Literal(if self.version >= EXTENDED_LEAF_DATA {
tt::Literal {
symbol: Symbol::intern(text),
tt::Literal::new(
text,
span,
kind: match u16::to_le_bytes(repr.kind) {
match u16::to_le_bytes(repr.kind) {
[0, _] => Err(()),
[1, _] => Byte,
[2, _] => Char,
@ -787,14 +793,12 @@ impl<T: SpanTransformer<Span = span::Span>> Reader<'_, T> {
[10, r] => CStrRaw(r),
_ => unreachable!(),
},
suffix: if repr.suffix != !0 {
Some(Symbol::intern(
self.text[repr.suffix as usize].as_str(),
))
if repr.suffix != !0 {
self.text[repr.suffix as usize].as_str()
} else {
None
""
},
}
)
} else {
tt::token_to_literal(text, span)
})
@ -840,7 +844,7 @@ impl<T: SpanTransformer<Span = span::Span>> Reader<'_, T> {
let (delimiter, mut res) = res[0].take().unwrap();
res.insert(0, tt::TokenTree::Subtree(tt::Subtree { delimiter, len: res.len() as u32 }));
tt::TopSubtree(res.into_boxed_slice())
tt::TopSubtree::from_serialized(res)
}
}

18
src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
View file

@ -194,28 +194,14 @@ impl ProcMacro {
/// On some server versions, the fixup ast id is different than ours. So change it to match.
fn change_fixup_to_match_old_server(&self, tt: &mut tt::TopSubtree) {
const OLD_FIXUP_AST_ID: ErasedFileAstId = ErasedFileAstId::from_raw(!0 - 1);
let change_ast_id = |ast_id: &mut ErasedFileAstId| {
tt.change_every_ast_id(|ast_id| {
if *ast_id == FIXUP_ERASED_FILE_AST_ID_MARKER {
*ast_id = OLD_FIXUP_AST_ID;
} else if *ast_id == OLD_FIXUP_AST_ID {
// Swap between them, that means no collision plus the change can be reversed by doing itself.
*ast_id = FIXUP_ERASED_FILE_AST_ID_MARKER;
}
};
for tt in &mut tt.0 {
match tt {
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { span, .. }))
| tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { span, .. }))
| tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { span, .. })) => {
change_ast_id(&mut span.anchor.ast_id);
}
tt::TokenTree::Subtree(subtree) => {
change_ast_id(&mut subtree.delimiter.open.anchor.ast_id);
change_ast_id(&mut subtree.delimiter.close.anchor.ast_id);
}
}
}
});
}
/// Expands the procedural macro by sending an expansion request to the server.

25
src/tools/rust-analyzer/crates/syntax-bridge/src/lib.rs
View file

@ -223,7 +223,7 @@ where
spacing: _,
})) => {
let found_expected_delimiter =
builder.expected_delimiters().enumerate().find(|(_, delim)| match delim.kind {
builder.expected_delimiters().enumerate().find(|(_, delim)| match delim {
tt::DelimiterKind::Parenthesis => char == ')',
tt::DelimiterKind::Brace => char == '}',
tt::DelimiterKind::Bracket => char == ']',
@ -257,13 +257,11 @@ where
}
kind if kind.is_punct() && kind != UNDERSCORE => {
let found_expected_delimiter =
builder.expected_delimiters().enumerate().find(|(_, delim)| {
match delim.kind {
tt::DelimiterKind::Parenthesis => kind == T![')'],
tt::DelimiterKind::Brace => kind == T!['}'],
tt::DelimiterKind::Bracket => kind == T![']'],
tt::DelimiterKind::Invisible => false,
}
builder.expected_delimiters().enumerate().find(|(_, delim)| match delim {
tt::DelimiterKind::Parenthesis => kind == T![')'],
tt::DelimiterKind::Brace => kind == T!['}'],
tt::DelimiterKind::Bracket => kind == T![']'],
tt::DelimiterKind::Invisible => false,
});
// Current token is a closing delimiter that we expect, fix up the closing span
@ -444,7 +442,7 @@ fn convert_doc_comment(
text = &text[0..text.len() - 2];
}
let (text, kind) = desugar_doc_comment_text(text, mode);
let lit = tt::Literal { symbol: text, span, kind, suffix: None };
let lit = tt::Literal { text_and_suffix: text, span, kind, suffix_len: 0 };
tt::Leaf::from(lit)
};
@ -869,12 +867,9 @@ impl TtTreeSink<'_> {
/// This occurs when a float literal is used as a field access.
fn float_split(&mut self, has_pseudo_dot: bool) {
let (text, span) = match self.cursor.token_tree() {
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
symbol: text,
span,
kind: tt::LitKind::Float,
suffix: _,
}))) => (text.as_str(), *span),
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(
lit @ tt::Literal { span, kind: tt::LitKind::Float, .. },
))) => (lit.text(), *span),
tt => unreachable!("{tt:?}"),
};
// FIXME: Span splitting

2
src/tools/rust-analyzer/crates/syntax-bridge/src/tests.rs
View file

@ -30,7 +30,7 @@ fn check_punct_spacing(fixture: &str) {
})
.collect();
let mut cursor = Cursor::new(&subtree.0);
let mut cursor = Cursor::new(subtree.as_token_trees());
while !cursor.eof() {
while let Some(token_tree) = cursor.token_tree() {
if let tt::TokenTree::Leaf(Leaf::Punct(Punct {

2
src/tools/rust-analyzer/crates/syntax-bridge/src/to_parser_input.rs
View file

@ -52,7 +52,7 @@ pub fn to_parser_input(
};
res.push(kind, ctx_edition(lit.span.ctx));
if kind == FLOAT_NUMBER && !lit.symbol.as_str().ends_with('.') {
if kind == FLOAT_NUMBER && !lit.text().ends_with('.') {
// Tag the token as joint if it is float with a fractional part
// we use this jointness to inform the parser about what token split
// event to emit when we encounter a float literal in a field access

47
src/tools/rust-analyzer/crates/test-fixture/src/lib.rs
View file

@ -770,7 +770,7 @@ impl ProcMacroExpander for Issue18089ProcMacroExpander {
_: Span,
_: String,
) -> Result<TopSubtree, ProcMacroExpansionError> {
let tt::TokenTree::Leaf(macro_name) = &subtree.0[2] else {
let Some(tt::TtElement::Leaf(macro_name)) = subtree.iter().nth(1) else {
return Err(ProcMacroExpansionError::Panic("incorrect input".to_owned()));
};
Ok(quote! { call_site =>
@ -837,13 +837,14 @@ impl ProcMacroExpander for Issue18840ProcMacroExpander {
// ```
// The span that was created by the fixup infra.
let fixed_up_span = fn_.token_trees().flat_tokens()[5].first_span();
let mut iter = fn_.iter();
iter.nth(2);
let (_, mut fn_body) = iter.expect_subtree().unwrap();
let fixed_up_span = fn_body.nth(1).unwrap().first_span();
let mut result =
quote! {fixed_up_span => ::core::compile_error! { "my cool compile_error!" } };
// Make it so we won't remove the top subtree when reversing fixups.
let top_subtree_delimiter_mut = result.top_subtree_delimiter_mut();
top_subtree_delimiter_mut.open = def_site;
top_subtree_delimiter_mut.close = def_site;
result.set_top_subtree_delimiter_span(tt::DelimSpan::from_single(def_site));
Ok(result)
}
@ -905,20 +906,22 @@ impl ProcMacroExpander for ShortenProcMacroExpander {
_: Span,
_: String,
) -> Result<TopSubtree, ProcMacroExpansionError> {
let mut result = input.0.clone();
for it in &mut result {
if let TokenTree::Leaf(leaf) = it {
modify_leaf(leaf)
let mut result = input.clone();
for (idx, it) in input.as_token_trees().iter_flat_tokens().enumerate() {
if let TokenTree::Leaf(mut leaf) = it {
modify_leaf(&mut leaf);
result.set_token(idx, leaf);
}
}
return Ok(tt::TopSubtree(result));
return Ok(result);
fn modify_leaf(leaf: &mut Leaf) {
match leaf {
Leaf::Literal(it) => {
// XXX Currently replaces any literals with an empty string, but supporting
// "shortening" other literals would be nice.
it.symbol = Symbol::empty();
it.text_and_suffix = Symbol::empty();
it.suffix_len = 0;
}
Leaf::Punct(_) => {}
Leaf::Ident(it) => {
@ -948,10 +951,11 @@ impl ProcMacroExpander for Issue17479ProcMacroExpander {
_: Span,
_: String,
) -> Result<TopSubtree, ProcMacroExpansionError> {
let TokenTree::Leaf(Leaf::Literal(lit)) = &subtree.0[1] else {
let mut iter = subtree.iter();
let Some(TtElement::Leaf(tt::Leaf::Literal(lit))) = iter.next() else {
return Err(ProcMacroExpansionError::Panic("incorrect Input".into()));
};
let symbol = &lit.symbol;
let symbol = Symbol::intern(lit.text());
let span = lit.span;
Ok(quote! { span =>
#symbol()
@ -980,10 +984,8 @@ impl ProcMacroExpander for Issue18898ProcMacroExpander {
) -> Result<TopSubtree, ProcMacroExpansionError> {
let span = subtree
.token_trees()
.flat_tokens()
.last()
.ok_or_else(|| ProcMacroExpansionError::Panic("malformed input".to_owned()))?
.first_span();
.last_span()
.ok_or_else(|| ProcMacroExpansionError::Panic("malformed input".to_owned()))?;
let overly_long_subtree = quote! {span =>
{
let a = 5;
@ -1034,7 +1036,7 @@ impl ProcMacroExpander for DisallowCfgProcMacroExpander {
_: Span,
_: String,
) -> Result<TopSubtree, ProcMacroExpansionError> {
for tt in subtree.token_trees().flat_tokens() {
for tt in subtree.token_trees().iter_flat_tokens() {
if let tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) = tt
&& (ident.sym == sym::cfg || ident.sym == sym::cfg_attr)
{
@ -1066,20 +1068,23 @@ impl ProcMacroExpander for GenerateSuffixedTypeProcMacroExpander {
_mixed_site: Span,
_current_dir: String,
) -> Result<TopSubtree, ProcMacroExpansionError> {
let TokenTree::Leaf(Leaf::Ident(ident)) = &subtree.0[1] else {
let mut iter = subtree.iter();
let Some(TtElement::Leaf(tt::Leaf::Ident(ident))) = iter.next() else {
return Err(ProcMacroExpansionError::Panic("incorrect Input".into()));
};
let ident = match ident.sym.as_str() {
"struct" => {
let TokenTree::Leaf(Leaf::Ident(ident)) = &subtree.0[2] else {
let Some(TtElement::Leaf(tt::Leaf::Ident(ident))) = iter.next() else {
return Err(ProcMacroExpansionError::Panic("incorrect Input".into()));
};
ident
}
"enum" => {
let TokenTree::Leaf(Leaf::Ident(ident)) = &subtree.0[4] else {
iter.next();
let (_, mut iter) = iter.expect_subtree().unwrap();
let Some(TtElement::Leaf(tt::Leaf::Ident(ident))) = iter.next() else {
return Err(ProcMacroExpansionError::Panic("incorrect Input".into()));
};
ident

4
src/tools/rust-analyzer/crates/tt/src/buffer.rs
View file

@ -10,8 +10,8 @@ pub struct Cursor<'a> {
}
impl<'a> Cursor<'a> {
pub fn new(buffer: &'a [TokenTree]) -> Self {
Self { buffer, index: 0, subtrees_stack: Vec::new() }
pub fn new(buffer: TokenTreesView<'a>) -> Self {
Self { buffer: buffer.0, index: 0, subtrees_stack: Vec::new() }
}
/// Check whether it is eof

32
src/tools/rust-analyzer/crates/tt/src/iter.rs
View file

@ -36,28 +36,28 @@ impl<'a> TtIter<'a> {
pub fn expect_char(&mut self, char: char) -> Result<(), ()> {
match self.next() {
Some(TtElement::Leaf(&Leaf::Punct(Punct { char: c, .. }))) if c == char => Ok(()),
Some(TtElement::Leaf(Leaf::Punct(Punct { char: c, .. }))) if c == char => Ok(()),
_ => Err(()),
}
}
pub fn expect_any_char(&mut self, chars: &[char]) -> Result<(), ()> {
match self.next() {
Some(TtElement::Leaf(Leaf::Punct(Punct { char: c, .. }))) if chars.contains(c) => {
Some(TtElement::Leaf(Leaf::Punct(Punct { char: c, .. }))) if chars.contains(&c) => {
Ok(())
}
_ => Err(()),
}
}
pub fn expect_subtree(&mut self) -> Result<(&'a Subtree, TtIter<'a>), ()> {
pub fn expect_subtree(&mut self) -> Result<(Subtree, TtIter<'a>), ()> {
match self.next() {
Some(TtElement::Subtree(subtree, iter)) => Ok((subtree, iter)),
_ => Err(()),
}
}
pub fn expect_leaf(&mut self) -> Result<&'a Leaf, ()> {
pub fn expect_leaf(&mut self) -> Result<Leaf, ()> {
match self.next() {
Some(TtElement::Leaf(it)) => Ok(it),
_ => Err(()),
@ -78,30 +78,30 @@ impl<'a> TtIter<'a> {
}
}
pub fn expect_ident(&mut self) -> Result<&'a Ident, ()> {
pub fn expect_ident(&mut self) -> Result<Ident, ()> {
match self.expect_leaf()? {
Leaf::Ident(it) if it.sym != sym::underscore => Ok(it),
_ => Err(()),
}
}
pub fn expect_ident_or_underscore(&mut self) -> Result<&'a Ident, ()> {
pub fn expect_ident_or_underscore(&mut self) -> Result<Ident, ()> {
match self.expect_leaf()? {
Leaf::Ident(it) => Ok(it),
_ => Err(()),
}
}
pub fn expect_literal(&mut self) -> Result<&'a Leaf, ()> {
pub fn expect_literal(&mut self) -> Result<Leaf, ()> {
let it = self.expect_leaf()?;
match it {
match &it {
Leaf::Literal(_) => Ok(it),
Leaf::Ident(ident) if ident.sym == sym::true_ || ident.sym == sym::false_ => Ok(it),
_ => Err(()),
}
}
pub fn expect_single_punct(&mut self) -> Result<&'a Punct, ()> {
pub fn expect_single_punct(&mut self) -> Result<Punct, ()> {
match self.expect_leaf()? {
Leaf::Punct(it) => Ok(it),
_ => Err(()),
@ -113,7 +113,7 @@ impl<'a> TtIter<'a> {
/// This method currently may return a single quotation, which is part of lifetime ident and
/// conceptually not a punct in the context of mbe. Callers should handle this.
pub fn expect_glued_punct(&mut self) -> Result<ArrayVec<Punct, MAX_GLUED_PUNCT_LEN>, ()> {
let TtElement::Leaf(&Leaf::Punct(first)) = self.next().ok_or(())? else {
let TtElement::Leaf(Leaf::Punct(first)) = self.next().ok_or(())? else {
return Err(());
};
@ -168,11 +168,11 @@ impl<'a> TtIter<'a> {
pub fn peek(&self) -> Option<TtElement<'a>> {
match self.inner.as_slice().first()? {
TokenTree::Leaf(leaf) => Some(TtElement::Leaf(leaf)),
TokenTree::Leaf(leaf) => Some(TtElement::Leaf(leaf.clone())),
TokenTree::Subtree(subtree) => {
let nested_iter =
TtIter { inner: self.inner.as_slice()[1..][..subtree.usize_len()].iter() };
Some(TtElement::Subtree(subtree, nested_iter))
Some(TtElement::Subtree(*subtree, nested_iter))
}
}
}
@ -214,8 +214,8 @@ impl<'a> TtIter<'a> {
#[derive(Clone)]
pub enum TtElement<'a> {
Leaf(&'a Leaf),
Subtree(&'a Subtree, TtIter<'a>),
Leaf(Leaf),
Subtree(Subtree, TtIter<'a>),
}
impl fmt::Debug for TtElement<'_> {
@ -243,12 +243,12 @@ impl<'a> Iterator for TtIter<'a> {
type Item = TtElement<'a>;
fn next(&mut self) -> Option<Self::Item> {
match self.inner.next()? {
TokenTree::Leaf(leaf) => Some(TtElement::Leaf(leaf)),
TokenTree::Leaf(leaf) => Some(TtElement::Leaf(leaf.clone())),
TokenTree::Subtree(subtree) => {
let nested_iter =
TtIter { inner: self.inner.as_slice()[..subtree.usize_len()].iter() };
self.inner = self.inner.as_slice()[subtree.usize_len()..].iter();
Some(TtElement::Subtree(subtree, nested_iter))
Some(TtElement::Subtree(*subtree, nested_iter))
}
}
}

310
src/tools/rust-analyzer/crates/tt/src/lib.rs
View file

@ -18,6 +18,7 @@ pub mod iter;
use std::fmt;
use arrayvec::ArrayString;
use buffer::Cursor;
use intern::Symbol;
use stdx::{impl_from, itertools::Itertools as _};
@ -111,7 +112,7 @@ impl Leaf {
}
impl_from!(Literal, Punct, Ident for Leaf);
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Subtree {
pub delimiter: Delimiter,
/// Number of following token trees that belong to this subtree, excluding this subtree.
@ -125,7 +126,7 @@ impl Subtree {
}
#[derive(Clone, PartialEq, Eq, Hash)]
pub struct TopSubtree(pub Box<[TokenTree]>);
pub struct TopSubtree(Box<[TokenTree]>);
impl TopSubtree {
pub fn empty(span: DelimSpan) -> Self {
@ -147,6 +148,10 @@ impl TopSubtree {
builder.build()
}
pub fn from_serialized(tt: Vec<TokenTree>) -> Self {
Self(tt.into_boxed_slice())
}
pub fn from_subtree(subtree: SubtreeView<'_>) -> Self {
Self(subtree.0.into())
}
@ -159,20 +164,55 @@ impl TopSubtree {
self.view().iter()
}
pub fn top_subtree(&self) -> &Subtree {
pub fn top_subtree(&self) -> Subtree {
self.view().top_subtree()
}
pub fn top_subtree_delimiter_mut(&mut self) -> &mut Delimiter {
pub fn set_top_subtree_delimiter_kind(&mut self, kind: DelimiterKind) {
self.top_subtree_mut().delimiter.kind = kind;
}
pub fn set_top_subtree_delimiter_span(&mut self, span: DelimSpan) {
let top_subtree = self.top_subtree_mut();
top_subtree.delimiter.open = span.open;
top_subtree.delimiter.close = span.close;
}
fn top_subtree_mut(&mut self) -> &mut Subtree {
let TokenTree::Subtree(subtree) = &mut self.0[0] else {
unreachable!("the first token tree is always the top subtree");
};
&mut subtree.delimiter
subtree
}
pub fn set_token(&mut self, idx: usize, leaf: Leaf) {
assert!(matches!(self.0[idx], TokenTree::Leaf(_)), "cannot replace a subtree by a leaf");
self.0[idx] = leaf.into();
}
pub fn token_trees(&self) -> TokenTreesView<'_> {
self.view().token_trees()
}
pub fn as_token_trees(&self) -> TokenTreesView<'_> {
self.view().as_token_trees()
}
pub fn change_every_ast_id(&mut self, mut callback: impl FnMut(&mut span::ErasedFileAstId)) {
for tt in &mut self.0 {
match tt {
TokenTree::Leaf(Leaf::Ident(Ident { span, .. }))
| TokenTree::Leaf(Leaf::Literal(Literal { span, .. }))
| TokenTree::Leaf(Leaf::Punct(Punct { span, .. })) => {
callback(&mut span.anchor.ast_id);
}
TokenTree::Subtree(subtree) => {
callback(&mut subtree.delimiter.open.anchor.ast_id);
callback(&mut subtree.delimiter.close.anchor.ast_id);
}
}
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
@ -241,11 +281,6 @@ impl TopSubtreeBuilder {
self.token_trees.extend(leaves.into_iter().map(TokenTree::Leaf));
}
/// This does not check the token trees are valid, beware!
pub fn extend_tt_dangerous(&mut self, tt: impl IntoIterator<Item = TokenTree>) {
self.token_trees.extend(tt);
}
pub fn extend_with_tt(&mut self, tt: TokenTreesView<'_>) {
self.token_trees.extend(tt.0.iter().cloned());
}
@ -267,12 +302,12 @@ impl TopSubtreeBuilder {
}
}
pub fn expected_delimiters(&self) -> impl Iterator<Item = &Delimiter> {
pub fn expected_delimiters(&self) -> impl Iterator<Item = DelimiterKind> {
self.unclosed_subtree_indices.iter().rev().map(|&subtree_idx| {
let TokenTree::Subtree(subtree) = &self.token_trees[subtree_idx] else {
unreachable!("unclosed token tree is always a subtree")
};
&subtree.delimiter
subtree.delimiter.kind
})
}
@ -330,7 +365,7 @@ pub struct SubtreeBuilderRestorePoint {
pub struct TokenTreesView<'a>(&'a [TokenTree]);
impl<'a> TokenTreesView<'a> {
pub fn new(tts: &'a [TokenTree]) -> Self {
fn new(tts: &'a [TokenTree]) -> Self {
if cfg!(debug_assertions) {
tts.iter().enumerate().for_each(|(idx, tt)| {
if let TokenTree::Subtree(tt) = &tt {
@ -345,12 +380,16 @@ impl<'a> TokenTreesView<'a> {
Self(tts)
}
pub fn empty() -> Self {
Self(&[])
}
pub fn iter(&self) -> TtIter<'a> {
TtIter::new(self.0)
}
pub fn cursor(&self) -> Cursor<'a> {
Cursor::new(self.0)
Cursor::new(*self)
}
pub fn len(&self) -> usize {
@ -374,13 +413,6 @@ impl<'a> TokenTreesView<'a> {
self.try_into_subtree().map(|subtree| subtree.strip_invisible()).unwrap_or(self)
}
/// This returns a **flat** structure of tokens (subtrees will be represented by a single node
/// preceding their children), so it isn't suited for most use cases, only for matching leaves
/// at the beginning/end with no subtrees before them. If you need a structured pass, use [`TtIter`].
pub fn flat_tokens(&self) -> &'a [TokenTree] {
self.0
}
pub fn split(
self,
mut split_fn: impl FnMut(TtElement<'a>) -> bool,
@ -406,6 +438,21 @@ impl<'a> TokenTreesView<'a> {
Some(result)
})
}
pub fn first_span(&self) -> Option<Span> {
Some(self.0.first()?.first_span())
}
pub fn last_span(&self) -> Option<Span> {
Some(match self.0.last()? {
TokenTree::Leaf(it) => *it.span(),
TokenTree::Subtree(it) => it.delimiter.close,
})
}
pub fn iter_flat_tokens(&self) -> impl ExactSizeIterator<Item = TokenTree> + use<'a> {
self.0.iter().cloned()
}
}
impl fmt::Debug for TokenTreesView<'_> {
@ -453,11 +500,11 @@ impl fmt::Display for TokenTreesView<'_> {
match child {
TtElement::Leaf(Leaf::Punct(p)) => {
needs_space = p.spacing == Spacing::Alone;
fmt::Display::fmt(p, f)?;
fmt::Display::fmt(&p, f)?;
}
TtElement::Leaf(leaf) => fmt::Display::fmt(leaf, f)?,
TtElement::Leaf(leaf) => fmt::Display::fmt(&leaf, f)?,
TtElement::Subtree(subtree, subtree_iter) => {
subtree_display(subtree, f, subtree_iter)?
subtree_display(&subtree, f, subtree_iter)?
}
}
}
@ -493,11 +540,11 @@ impl<'a> SubtreeView<'a> {
TtIter::new(&self.0[1..])
}
pub fn top_subtree(&self) -> &'a Subtree {
pub fn top_subtree(&self) -> Subtree {
let TokenTree::Subtree(subtree) = &self.0[0] else {
unreachable!("the first token tree is always the top subtree");
};
subtree
*subtree
}
pub fn strip_invisible(&self) -> TokenTreesView<'a> {
@ -571,17 +618,56 @@ pub enum DelimiterKind {
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Literal {
// escaped
pub symbol: Symbol,
/// Escaped, text then suffix concatenated.
pub text_and_suffix: Symbol,
pub span: Span,
pub kind: LitKind,
pub suffix: Option<Symbol>,
pub suffix_len: u8,
}
pub fn token_to_literal(text: &str, span: Span) -> Literal
where
Span: Copy,
{
impl Literal {
#[inline]
pub fn text_and_suffix(&self) -> (&str, &str) {
let text_and_suffix = self.text_and_suffix.as_str();
text_and_suffix.split_at(text_and_suffix.len() - usize::from(self.suffix_len))
}
#[inline]
pub fn text(&self) -> &str {
self.text_and_suffix().0
}
#[inline]
pub fn suffix(&self) -> &str {
self.text_and_suffix().1
}
pub fn new(text: &str, span: Span, kind: LitKind, suffix: &str) -> Self {
const MAX_INLINE_CAPACITY: usize = 30;
let text_and_suffix = if suffix.is_empty() {
Symbol::intern(text)
} else if (text.len() + suffix.len()) < MAX_INLINE_CAPACITY {
let mut text_and_suffix = ArrayString::<MAX_INLINE_CAPACITY>::new();
text_and_suffix.push_str(text);
text_and_suffix.push_str(suffix);
Symbol::intern(&text_and_suffix)
} else {
let mut text_and_suffix = String::with_capacity(text.len() + suffix.len());
text_and_suffix.push_str(text);
text_and_suffix.push_str(suffix);
Symbol::intern(&text_and_suffix)
};
Self { text_and_suffix, span, kind, suffix_len: suffix.len().try_into().unwrap() }
}
#[inline]
pub fn new_no_suffix(text: &str, span: Span, kind: LitKind) -> Self {
Self { text_and_suffix: Symbol::intern(text), span, kind, suffix_len: 0 }
}
}
pub fn token_to_literal(text: &str, span: Span) -> Literal {
use rustc_lexer::LiteralKind;
let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next_tuple();
@ -590,12 +676,7 @@ where
..
},)) = token
else {
return Literal {
span,
symbol: Symbol::intern(text),
kind: LitKind::Err(()),
suffix: None,
};
return Literal::new_no_suffix(text, span, LitKind::Err(()));
};
let (kind, start_offset, end_offset) = match kind {
@ -626,20 +707,15 @@ where
let (lit, suffix) = text.split_at(suffix_start as usize);
let lit = &lit[start_offset..lit.len() - end_offset];
let suffix = match suffix {
"" | "_" => None,
"" | "_" => "",
// ill-suffixed literals
_ if !matches!(kind, LitKind::Integer | LitKind::Float | LitKind::Err(_)) => {
return Literal {
span,
symbol: Symbol::intern(text),
kind: LitKind::Err(()),
suffix: None,
};
return Literal::new_no_suffix(text, span, LitKind::Err(()));
}
suffix => Some(Symbol::intern(suffix)),
suffix => suffix,
};
Literal { span, symbol: Symbol::intern(lit), kind, suffix }
Literal::new(lit, span, kind, suffix)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@ -759,15 +835,8 @@ fn print_debug_token(f: &mut fmt::Formatter<'_>, level: usize, tt: TtElement<'_>
match tt {
TtElement::Leaf(leaf) => match leaf {
Leaf::Literal(lit) => {
write!(
f,
"{}LITERAL {:?} {}{} {:#?}",
align,
lit.kind,
lit.symbol,
lit.suffix.as_ref().map(|it| it.as_str()).unwrap_or(""),
lit.span
)?;
let (text, suffix) = lit.text_and_suffix();
write!(f, "{}LITERAL {:?} {}{} {:#?}", align, lit.kind, text, suffix, lit.span)?;
}
Leaf::Punct(punct) => {
write!(
@ -791,7 +860,7 @@ fn print_debug_token(f: &mut fmt::Formatter<'_>, level: usize, tt: TtElement<'_>
}
},
TtElement::Subtree(subtree, subtree_iter) => {
print_debug_subtree(f, subtree, level, subtree_iter)?;
print_debug_subtree(f, &subtree, level, subtree_iter)?;
}
}
@ -829,44 +898,28 @@ impl fmt::Display for Ident {
impl fmt::Display for Literal {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let (text, suffix) = self.text_and_suffix();
match self.kind {
LitKind::Byte => write!(f, "b'{}'", self.symbol),
LitKind::Char => write!(f, "'{}'", self.symbol),
LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", self.symbol),
LitKind::Str => write!(f, "\"{}\"", self.symbol),
LitKind::ByteStr => write!(f, "b\"{}\"", self.symbol),
LitKind::CStr => write!(f, "c\"{}\"", self.symbol),
LitKind::Byte => write!(f, "b'{}'", text),
LitKind::Char => write!(f, "'{}'", text),
LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", text),
LitKind::Str => write!(f, "\"{}\"", text),
LitKind::ByteStr => write!(f, "b\"{}\"", text),
LitKind::CStr => write!(f, "c\"{}\"", text),
LitKind::StrRaw(num_of_hashes) => {
let num_of_hashes = num_of_hashes as usize;
write!(
f,
r#"r{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,
"",
text = self.symbol
)
write!(f, r#"r{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#, "", text = text)
}
LitKind::ByteStrRaw(num_of_hashes) => {
let num_of_hashes = num_of_hashes as usize;
write!(
f,
r#"br{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,
"",
text = self.symbol
)
write!(f, r#"br{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#, "", text = text)
}
LitKind::CStrRaw(num_of_hashes) => {
let num_of_hashes = num_of_hashes as usize;
write!(
f,
r#"cr{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,
"",
text = self.symbol
)
write!(f, r#"cr{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#, "", text = text)
}
}?;
if let Some(suffix) = &self.suffix {
write!(f, "{suffix}")?;
}
write!(f, "{suffix}")?;
Ok(())
}
}
@ -921,7 +974,7 @@ impl TopSubtree {
match tt {
TokenTree::Leaf(it) => {
let s = match it {
Leaf::Literal(it) => it.symbol.to_string(),
Leaf::Literal(it) => it.text().to_owned(),
Leaf::Punct(it) => it.char.to_string(),
Leaf::Ident(it) => format!("{}{}", it.is_raw.as_str(), it.sym),
};
@ -956,7 +1009,7 @@ impl TopSubtree {
}
}
pub fn pretty(mut tkns: &[TokenTree]) -> String {
pub fn pretty(tkns: TokenTreesView<'_>) -> String {
fn tokentree_to_text(tkn: &TokenTree, tkns: &mut &[TokenTree]) -> String {
match tkn {
TokenTree::Leaf(Leaf::Ident(ident)) => {
@ -966,7 +1019,7 @@ pub fn pretty(mut tkns: &[TokenTree]) -> String {
TokenTree::Leaf(Leaf::Punct(punct)) => format!("{}", punct.char),
TokenTree::Subtree(subtree) => {
let (subtree_content, rest) = tkns.split_at(subtree.usize_len());
let content = pretty(subtree_content);
let content = pretty(TokenTreesView(subtree_content));
*tkns = rest;
let (open, close) = match subtree.delimiter.kind {
DelimiterKind::Brace => ("{", "}"),
@ -979,6 +1032,7 @@ pub fn pretty(mut tkns: &[TokenTree]) -> String {
}
}
let mut tkns = tkns.0;
let mut last = String::new();
let mut last_to_joint = true;
@ -994,3 +1048,83 @@ pub fn pretty(mut tkns: &[TokenTree]) -> String {
}
last
}
#[derive(Debug)]
pub enum TransformTtAction<'a> {
Keep,
ReplaceWith(TokenTreesView<'a>),
}
impl TransformTtAction<'_> {
#[inline]
pub fn remove() -> Self {
Self::ReplaceWith(TokenTreesView::empty())
}
}
/// This function takes a token tree, and calls `callback` with each token tree in it.
/// Then it does what the callback says: keeps the tt or replaces it with a (possibly empty)
/// tts view.
pub fn transform_tt<'b>(
tt: &mut TopSubtree,
mut callback: impl FnMut(TokenTree) -> TransformTtAction<'b>,
) {
let mut tt_vec = std::mem::take(&mut tt.0).into_vec();
// We need to keep a stack of the currently open subtrees, because we need to update
// them if we change the number of items in them.
let mut subtrees_stack = Vec::new();
let mut i = 0;
while i < tt_vec.len() {
'pop_finished_subtrees: while let Some(&subtree_idx) = subtrees_stack.last() {
let TokenTree::Subtree(subtree) = &tt_vec[subtree_idx] else {
unreachable!("non-subtree on subtrees stack");
};
if i >= subtree_idx + 1 + subtree.usize_len() {
subtrees_stack.pop();
} else {
break 'pop_finished_subtrees;
}
}
let current = match &tt_vec[i] {
TokenTree::Leaf(leaf) => TokenTree::Leaf(match leaf {
Leaf::Literal(leaf) => Leaf::Literal(leaf.clone()),
Leaf::Punct(leaf) => Leaf::Punct(*leaf),
Leaf::Ident(leaf) => Leaf::Ident(leaf.clone()),
}),
TokenTree::Subtree(subtree) => TokenTree::Subtree(*subtree),
};
let action = callback(current);
match action {
TransformTtAction::Keep => {
// This cannot be shared with the replaced case, because then we may push the same subtree
// twice, and will update it twice which will lead to errors.
if let TokenTree::Subtree(_) = &tt_vec[i] {
subtrees_stack.push(i);
}
i += 1;
}
TransformTtAction::ReplaceWith(replacement) => {
let old_len = 1 + match &tt_vec[i] {
TokenTree::Leaf(_) => 0,
TokenTree::Subtree(subtree) => subtree.usize_len(),
};
let len_diff = replacement.len() as i64 - old_len as i64;
tt_vec.splice(i..i + old_len, replacement.0.iter().cloned());
// Skip the newly inserted replacement, we don't want to visit it.
i += replacement.len();
for &subtree_idx in &subtrees_stack {
let TokenTree::Subtree(subtree) = &mut tt_vec[subtree_idx] else {
unreachable!("non-subtree on subtrees stack");
};
subtree.len = (i64::from(subtree.len) + len_diff).try_into().unwrap();
}
}
}
}
tt.0 = tt_vec.into_boxed_slice();
}