user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Put TokenStream stuff in a sensible order.

Browse source 
I.e. the type definition, then a single inherent `impl` block, then the
trait `impl` blocks.

The lack of sensible ordering here has bugged me for some time.
This commit is contained in:
Nicholas Nethercote 2025-08-29 14:25:40 +10:00
parent 16b5ac111c
commit 364a3be579
1 changed files with 65 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

132
compiler/rustc_ast/src/tokenstream.rs
View file

@ -90,17 +90,6 @@ impl TokenTree {
}
}
impl<CTX> HashStable<CTX> for TokenStream
where
CTX: crate::HashStableContext,
{
fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
for sub_tt in self.iter() {
sub_tt.hash_stable(hcx, hasher);
}
}
}
/// A lazy version of [`AttrTokenStream`], which defers creation of an actual
/// `AttrTokenStream` until it is needed.
#[derive(Clone)]
@ -547,10 +536,6 @@ pub struct AttrsTarget {
pub tokens: LazyAttrTokenStream,
}
/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s.
#[derive(Clone, Debug, Default, Encodable, Decodable)]
pub struct TokenStream(pub(crate) Arc<Vec<TokenTree>>);
/// Indicates whether a token can join with the following token to form a
/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to
/// guide pretty-printing, which is where the `JointHidden` value (which isn't
@ -611,58 +596,9 @@ pub enum Spacing {
JointHidden,
}
impl TokenStream {
/// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream`
/// separating the two arguments with a comma for diagnostic suggestions.
pub fn add_comma(&self) -> Option<(TokenStream, Span)> {
// Used to suggest if a user writes `foo!(a b);`
let mut suggestion = None;
let mut iter = self.0.iter().enumerate().peekable();
while let Some((pos, ts)) = iter.next() {
if let Some((_, next)) = iter.peek() {
let sp = match (&ts, &next) {
(_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue,
(
TokenTree::Token(token_left, Spacing::Alone),
TokenTree::Token(token_right, _),
) if (token_left.is_non_reserved_ident() || token_left.is_lit())
&& (token_right.is_non_reserved_ident() || token_right.is_lit()) =>
{
token_left.span
}
(TokenTree::Delimited(sp, ..), _) => sp.entire(),
_ => continue,
};
let sp = sp.shrink_to_hi();
let comma = TokenTree::token_alone(token::Comma, sp);
suggestion = Some((pos, comma, sp));
}
}
if let Some((pos, comma, sp)) = suggestion {
let mut new_stream = Vec::with_capacity(self.0.len() + 1);
let parts = self.0.split_at(pos + 1);
new_stream.extend_from_slice(parts.0);
new_stream.push(comma);
new_stream.extend_from_slice(parts.1);
return Some((TokenStream::new(new_stream), sp));
}
None
}
}
impl FromIterator<TokenTree> for TokenStream {
fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>())
}
}
impl Eq for TokenStream {}
impl PartialEq<TokenStream> for TokenStream {
fn eq(&self, other: &TokenStream) -> bool {
self.iter().eq(other.iter())
}
}
/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s.
#[derive(Clone, Debug, Default, Encodable, Decodable)]
pub struct TokenStream(pub(crate) Arc<Vec<TokenTree>>);
impl TokenStream {
pub fn new(tts: Vec<TokenTree>) -> TokenStream {
@ -838,6 +774,68 @@ impl TokenStream {
}
}
}
/// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream`
/// separating the two arguments with a comma for diagnostic suggestions.
pub fn add_comma(&self) -> Option<(TokenStream, Span)> {
// Used to suggest if a user writes `foo!(a b);`
let mut suggestion = None;
let mut iter = self.0.iter().enumerate().peekable();
while let Some((pos, ts)) = iter.next() {
if let Some((_, next)) = iter.peek() {
let sp = match (&ts, &next) {
(_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue,
(
TokenTree::Token(token_left, Spacing::Alone),
TokenTree::Token(token_right, _),
) if (token_left.is_non_reserved_ident() || token_left.is_lit())
&& (token_right.is_non_reserved_ident() || token_right.is_lit()) =>
{
token_left.span
}
(TokenTree::Delimited(sp, ..), _) => sp.entire(),
_ => continue,
};
let sp = sp.shrink_to_hi();
let comma = TokenTree::token_alone(token::Comma, sp);
suggestion = Some((pos, comma, sp));
}
}
if let Some((pos, comma, sp)) = suggestion {
let mut new_stream = Vec::with_capacity(self.0.len() + 1);
let parts = self.0.split_at(pos + 1);
new_stream.extend_from_slice(parts.0);
new_stream.push(comma);
new_stream.extend_from_slice(parts.1);
return Some((TokenStream::new(new_stream), sp));
}
None
}
}
impl PartialEq<TokenStream> for TokenStream {
fn eq(&self, other: &TokenStream) -> bool {
self.iter().eq(other.iter())
}
}
impl Eq for TokenStream {}
impl FromIterator<TokenTree> for TokenStream {
fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>())
}
}
impl<CTX> HashStable<CTX> for TokenStream
where
CTX: crate::HashStableContext,
{
fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
for sub_tt in self.iter() {
sub_tt.hash_stable(hcx, hasher);
}
}
}
#[derive(Clone)]