From 9b2b8a5afa833795b66267684615497c9547878d Mon Sep 17 00:00:00 2001
From: Aaron Hill <aa1ronham@gmail.com>
Date: Sun, 17 May 2020 15:51:01 -0400
Subject: [PATCH] Break tokens before checking if they are 'probably equal'

Fixes #68489

When checking two `TokenStreams` to see if they are 'probably equal',
we ignore the `IsJoint` information associated with each `TokenTree`.
However, the `IsJoint` information determines whether adjacent tokens
will be 'glued' (if possible) when construction the `TokenStream` - e.g.
`[Gt Gt]` can be 'glued' to `BinOp(Shr)`.

Since we are ignoring the `IsJoint` information, 'glued' and 'unglued'
tokens are equivalent for determining if two `TokenStreams` are
'probably equal'. Therefore, we need to 'unglue' all tokens in the
stream to avoid false negatives (which cause us to throw out the cached
tokens, losing span information).
---
 src/librustc_ast/tokenstream.rs               | 34 +++++++++++++++++--
 src/test/ui/proc-macro/turbo-proc-macro.rs    |  9 +++++
 .../ui/proc-macro/turbo-proc-macro.stderr     |  9 +++++
 3 files changed, 50 insertions(+), 2 deletions(-)
 create mode 100644 src/test/ui/proc-macro/turbo-proc-macro.rs
 create mode 100644 src/test/ui/proc-macro/turbo-proc-macro.stderr

diff --git a/src/librustc_ast/tokenstream.rs b/src/librustc_ast/tokenstream.rs
index 916a5ff6f46f..38483360c066 100644
--- a/src/librustc_ast/tokenstream.rs
+++ b/src/librustc_ast/tokenstream.rs
@@ -338,8 +338,38 @@ impl TokenStream {
             true
         }
 
-        let mut t1 = self.trees().filter(semantic_tree);
-        let mut t2 = other.trees().filter(semantic_tree);
+        // When comparing two `TokenStream`s, we ignore the `IsJoint` information.
+        //
+        // However, `rustc_parse::lexer::tokentrees::TokenStreamBuilder` will
+        // use `Token.glue` on adjacent tokens with the proper `IsJoint`.
+        // Since we are ignoreing `IsJoint`, a 'glued' token (e.g. `BinOp(Shr)`)
+        // and its 'split'/'unglued' compoenents (e.g. `Gt, Gt`) are equivalent
+        // when determining if two `TokenStream`s are 'probably equal'.
+        //
+        // Therefore, we use `break_two_token_op` to convert all tokens
+        // to the 'unglued' form (if it exists). This ensures that two
+        // `TokenStream`s which differ only in how their tokens are glued
+        // will be considered 'probably equal', which allows us to keep spans.
+        //
+        // This is important when the original `TokenStream` contained
+        // extra spaces (e.g. `f :: < Vec < _ > > ( ) ;'). These extra spaces
+        // will be omitted when we pretty-print, which can cause the original
+        // and reparsed `TokenStream`s to differ in the assignment of `IsJoint`,
+        // leading to some tokens being 'glued' together in one stream but not
+        // the other. See #68489 for more details.
+        fn break_tokens(tree: TokenTree) -> impl Iterator<Item = TokenTree> {
+            if let TokenTree::Token(token) = &tree {
+                if let Some((first, second)) = token.kind.break_two_token_op() {
+                    return SmallVec::from_buf([TokenTree::Token(Token::new(first, DUMMY_SP)), TokenTree::Token(Token::new(second, DUMMY_SP))]).into_iter()
+                }
+            }
+            let mut vec = SmallVec::<[_; 2]>::new();
+            vec.push(tree);
+            vec.into_iter()
+        }
+
+        let mut t1 = self.trees().filter(semantic_tree).flat_map(break_tokens);
+        let mut t2 = other.trees().filter(semantic_tree).flat_map(break_tokens);
         for (t1, t2) in t1.by_ref().zip(t2.by_ref()) {
             if !t1.probably_equal_for_proc_macro(&t2) {
                 return false;
diff --git a/src/test/ui/proc-macro/turbo-proc-macro.rs b/src/test/ui/proc-macro/turbo-proc-macro.rs
new file mode 100644
index 000000000000..a255955f38da
--- /dev/null
+++ b/src/test/ui/proc-macro/turbo-proc-macro.rs
@@ -0,0 +1,9 @@
+// aux-build:test-macros.rs
+
+extern crate test_macros;
+
+#[test_macros::recollect_attr]
+fn repro() {
+    f :: < Vec < _ > > ( ) ; //~ ERROR cannot find
+}
+fn main() {}
diff --git a/src/test/ui/proc-macro/turbo-proc-macro.stderr b/src/test/ui/proc-macro/turbo-proc-macro.stderr
new file mode 100644
index 000000000000..85c93b9345c3
--- /dev/null
+++ b/src/test/ui/proc-macro/turbo-proc-macro.stderr
@@ -0,0 +1,9 @@
+error[E0425]: cannot find function `f` in this scope
+  --> $DIR/turbo-proc-macro.rs:7:5
+   |
+LL |     f :: < Vec < _ > > ( ) ;
+   |     ^ not found in this scope
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0425`.