From a822d08bd1145f14838b5483582b574c8c12df52 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Fri, 30 Sep 2022 16:51:35 +1000
Subject: [PATCH] Remove `TokenStreamBuilder`.

It's now only used in one function. Also, the "should we glue the
tokens?" check is only necessary when pushing a `TokenTree::Token`, not
when pushing a `TokenTree::Delimited`.

As part of this, we now do the "should we glue the tokens?" check
immediately, which avoids having look back at the previous token. It
also puts all the logic dealing with token gluing in a single place.
---
 compiler/rustc_parse/src/lexer/tokentrees.rs | 57 +++++++-------------
 1 file changed, 20 insertions(+), 37 deletions(-)

diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs
index e3ccfc654623..0af52043d375 100644
--- a/compiler/rustc_parse/src/lexer/tokentrees.rs
+++ b/compiler/rustc_parse/src/lexer/tokentrees.rs
@@ -47,13 +47,13 @@ impl<'a> TokenTreesReader<'a> {
     // Parse a stream of tokens into a list of `TokenTree`s.
     fn parse_token_trees(&mut self, is_top_level: bool) -> PResult<'a, TokenStream> {
         self.token = self.string_reader.next_token().0;
-        let mut buf = TokenStreamBuilder::default();
+        let mut buf = Vec::new();
         loop {
             match self.token.kind {
                 token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)),
                 token::CloseDelim(delim) => {
                     return if !is_top_level {
-                        Ok(buf.into_token_stream())
+                        Ok(TokenStream::new(buf))
                     } else {
                         Err(self.close_delim_err(delim))
                     };
@@ -62,21 +62,28 @@ impl<'a> TokenTreesReader<'a> {
                     if !is_top_level {
                         self.eof_err().emit();
                     }
-                    return Ok(buf.into_token_stream());
+                    return Ok(TokenStream::new(buf));
                 }
                 _ => {
-                    // `this_spacing` for the returned token refers to whether the token is
-                    // immediately followed by another op token. It is determined by the
-                    // next token: its kind and its `preceded_by_whitespace` status.
-                    let (next_tok, is_next_tok_preceded_by_whitespace) =
-                        self.string_reader.next_token();
-                    let this_spacing = if is_next_tok_preceded_by_whitespace || !next_tok.is_op() {
-                        Spacing::Alone
-                    } else {
-                        Spacing::Joint
+                    // Get the next normal token. This might require getting multiple adjacent
+                    // single-char tokens and joining them together.
+                    let (this_spacing, next_tok) = loop {
+                        let (next_tok, is_next_tok_preceded_by_whitespace) =
+                            self.string_reader.next_token();
+                        if !is_next_tok_preceded_by_whitespace {
+                            if let Some(glued) = self.token.glue(&next_tok) {
+                                self.token = glued;
+                            } else {
+                                let this_spacing =
+                                    if next_tok.is_op() { Spacing::Joint } else { Spacing::Alone };
+                                break (this_spacing, next_tok);
+                            }
+                        } else {
+                            break (Spacing::Alone, next_tok);
+                        }
                     };
                     let this_tok = std::mem::replace(&mut self.token, next_tok);
-                    buf.push(TokenTree::Token(this_tok, this_spacing))
+                    buf.push(TokenTree::Token(this_tok, this_spacing));
                 }
             }
         }
@@ -249,27 +256,3 @@ impl<'a> TokenTreesReader<'a> {
         err
     }
 }
-
-#[derive(Default)]
-struct TokenStreamBuilder {
-    buf: Vec<TokenTree>,
-}
-
-impl TokenStreamBuilder {
-    #[inline(always)]
-    fn push(&mut self, tree: TokenTree) {
-        if let Some(TokenTree::Token(prev_token, Spacing::Joint)) = self.buf.last()
-            && let TokenTree::Token(token, joint) = &tree
-            && let Some(glued) = prev_token.glue(token)
-        {
-            self.buf.pop();
-            self.buf.push(TokenTree::Token(glued, *joint));
-        } else {
-            self.buf.push(tree)
-        }
-    }
-
-    fn into_token_stream(self) -> TokenStream {
-        TokenStream::new(self.buf)
-    }
-}