Auto merge of #50838 - alexcrichton:token-impls, r=eddyb

rustc: Fix joint-ness of stringified token-streams

This commit fixes `StringReader`'s parsing of tokens which have been stringified
through procedural macros. Whether or not a token tree is joint is defined by
span information, but when working with procedural macros these spans are often
dummy and/or overridden which means that they end up considering all operators
joint if they can!

The fix here is to track the raw source span as opposed to the overridden span.
With this information we can more accurately classify `Punct` structs as either
joint or not.

Closes #50700

src/test/run-pass-fulldeps/proc-macro/auxiliary/not-joint.rs

@@ -0,0 +1,40 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+#![feature(proc_macro)]
+
+extern crate proc_macro;
+
+use proc_macro::*;
+
+#[proc_macro]
+pub fn tokens(input: TokenStream) -> TokenStream {
+    assert_nothing_joint(input);
+    TokenStream::empty()
+}
+
+#[proc_macro_attribute]
+pub fn nothing(_: TokenStream, input: TokenStream) -> TokenStream {
+    assert_nothing_joint(input);
+    TokenStream::empty()
+}
+
+fn assert_nothing_joint(s: TokenStream) {
+    for tt in s {
+        match tt {
+            TokenTree::Group(g) => assert_nothing_joint(g.stream()),
+            TokenTree::Punct(p) => assert_eq!(p.spacing(), Spacing::Alone),
+            _ => {}
+        }
+    }
+}

src/test/run-pass-fulldeps/proc-macro/not-joint.rs

@@ -0,0 +1,35 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:not-joint.rs
+
+#![feature(proc_macro)]
+
+extern crate not_joint as bar;
+use bar::{tokens, nothing};
+
+tokens![< -];
+
+#[nothing]
+a![< -];
+
+#[nothing]
+b!{< -}
+
+#[nothing]
+c!(< -);
+
+#[nothing]
+fn foo() {
+    //! dox
+    let x = 2 < - 3;
+}
+
+fn main() {}