Rollup merge of #73569 - Aaron1011:fix/macro-rules-group, r=petrochenkov

Handle `macro_rules!` tokens consistently across crates

When we serialize a `macro_rules!` macro, we used a 'lowered' `TokenStream` for its body, which has all `Nonterminal`s expanded in-place via `nt_to_tokenstream`. This matters when an 'outer' `macro_rules!` macro expands to an 'inner' `macro_rules!` macro - the inner macro may use tokens captured from the 'outer' macro in its definition.

This means that invoking a foreign `macro_rules!` macro may use a different body `TokenStream` than when the same `macro_rules!` macro is invoked in the same crate. This difference is observable by proc-macros invoked by a `macro_rules!` macro - a `None`-delimited group will be seen in the same-crate case (inserted when convering `Nonterminal`s to the `proc_macro` crate's structs), but no `None`-delimited group in the cross-crate case.

To fix this inconsistency, we now insert `None`-delimited groups when 'lowering' a `Nonterminal` `macro_rules!` body, just as we do in `proc_macro_server`. Additionally, we no longer print extra spaces for `None`-delimited groups - as far as pretty-printing is concerned, they don't exist (only their contents do). This ensures that `Display` output of a `TokenStream` does not depend on which crate a `macro_rules!` macro was invoked from.

This PR is necessary in order to patch the `solana-genesis-programs` for the upcoming hygiene serialization breakage (https://github.com/rust-lang/rust/pull/72121#issuecomment-646924847). The `solana-genesis-programs` crate will need to use a proc macro to re-span certain tokens in a nested `macro_rules!`, which requires us to consistently use a `None`-delimited group.

See `src/test/ui/proc-macro/nested-macro-rules.rs` for an example of the kind of nested `macro_rules!` affected by this crate.

src/test/ui/macros/doc-comment.rs

@@ -0,0 +1,25 @@
+// check-pass
+// Tests that we properly handle a nested macro expansion
+// involving a `#[doc]` attribute
+#![deny(missing_docs)]
+//! Crate docs
+
+macro_rules! doc_comment {
+    ($x:expr, $($tt:tt)*) => {
+        #[doc = $x]
+        $($tt)*
+    }
+}
+
+macro_rules! make_comment {
+    () => {
+        doc_comment!("Function docs",
+            pub fn bar() {}
+        );
+    }
+}
+
+
+make_comment!();
+
+fn main() {}

src/test/ui/proc-macro/auxiliary/meta-delim.rs

@@ -0,0 +1,12 @@
+macro_rules! produce_it {
+    ($dollar_one:tt $foo:ident $my_name:ident) => {
+        #[macro_export]
+        macro_rules! meta_delim {
+            ($dollar_one ($dollar_one $my_name:ident)*) => {
+                stringify!($dollar_one ($dollar_one $my_name)*)
+            }
+        }
+    }
+}
+
+produce_it!($my_name name);

src/test/ui/proc-macro/auxiliary/nested-macro-rules.rs

@@ -0,0 +1,15 @@
+pub struct FirstStruct;
+
+#[macro_export]
+macro_rules! outer_macro {
+    ($name:ident) => {
+        #[macro_export]
+        macro_rules! inner_macro {
+            ($wrapper:ident) => {
+                $wrapper!($name)
+            }
+        }
+    }
+}
+
+outer_macro!(FirstStruct);

src/test/ui/proc-macro/auxiliary/test-macros.rs

@@ -101,6 +101,12 @@ pub fn print_bang(input: TokenStream) -> TokenStream {
     print_helper(input, "BANG")
 }
 
+#[proc_macro]
+pub fn print_bang_consume(input: TokenStream) -> TokenStream {
+    print_helper(input, "BANG");
+    TokenStream::new()
+}
+
 #[proc_macro_attribute]
 pub fn print_attr(_: TokenStream, input: TokenStream) -> TokenStream {
     print_helper(input, "ATTR")

src/test/ui/proc-macro/input-interpolated.stdout

@@ -1,5 +1,4 @@
 PRINT-BANG INPUT (DISPLAY): A
-PRINT-BANG RE-COLLECTED (DISPLAY):  A 
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,

src/test/ui/proc-macro/meta-delim.rs

@@ -0,0 +1,12 @@
+// aux-build:meta-delim.rs
+// edition:2018
+// run-pass
+
+// Tests that we can properly deserialize a macro with strange delimiters
+// See https://github.com/rust-lang/rust/pull/73569#issuecomment-650860457
+
+extern crate meta_delim;
+
+fn main() {
+    assert_eq!("a bunch of idents", meta_delim::meta_delim!(a bunch of idents));
+}

src/test/ui/proc-macro/nested-macro-rules.rs

@@ -0,0 +1,20 @@
+// run-pass
+// aux-build:nested-macro-rules.rs
+// aux-build:test-macros.rs
+// compile-flags: -Z span-debug
+// edition:2018
+
+extern crate nested_macro_rules;
+extern crate test_macros;
+
+use test_macros::print_bang;
+
+use nested_macro_rules::FirstStruct;
+struct SecondStruct;
+
+fn main() {
+    nested_macro_rules::inner_macro!(print_bang);
+
+    nested_macro_rules::outer_macro!(SecondStruct);
+    inner_macro!(print_bang);
+}

src/test/ui/proc-macro/nested-macro-rules.stdout

@@ -0,0 +1,26 @@
+PRINT-BANG INPUT (DISPLAY): FirstStruct
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Ident {
+                ident: "FirstStruct",
+                span: $DIR/auxiliary/nested-macro-rules.rs:15:14: 15:25 (#3),
+            },
+        ],
+        span: $DIR/auxiliary/nested-macro-rules.rs:9:27: 9:32 (#3),
+    },
+]
+PRINT-BANG INPUT (DISPLAY): SecondStruct
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Ident {
+                ident: "SecondStruct",
+                span: $DIR/nested-macro-rules.rs:18:38: 18:50 (#9),
+            },
+        ],
+        span: $DIR/auxiliary/nested-macro-rules.rs:9:27: 9:32 (#8),
+    },
+]

src/test/ui/proc-macro/nodelim-groups.rs

@@ -0,0 +1,19 @@
+// run-pass
+// aux-build:test-macros.rs
+// compile-flags: -Z span-debug
+// edition:2018
+//
+// Tests the pretty-printing behavior of inserting `NoDelim` groups
+
+extern crate test_macros;
+use test_macros::print_bang_consume;
+
+macro_rules! expand_it {
+    (($val1:expr) ($val2:expr)) => { expand_it!($val1 + $val2) };
+    ($val:expr) => { print_bang_consume!("hi" $val (1 + 1)) };
+}
+
+fn main() {
+    expand_it!(1 + (25) + 1);
+    expand_it!(("hello".len()) ("world".len()));
+}

src/test/ui/proc-macro/nodelim-groups.stdout

@@ -0,0 +1,156 @@
+PRINT-BANG INPUT (DISPLAY): "hi" 1 + (25) + 1 (1 + 1)
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Literal {
+        kind: Str,
+        symbol: "hi",
+        suffix: None,
+        span: $DIR/nodelim-groups.rs:13:42: 13:46 (#3),
+    },
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Literal {
+                kind: Integer,
+                symbol: "1",
+                suffix: None,
+                span: $DIR/nodelim-groups.rs:17:16: 17:17 (#0),
+            },
+            Punct {
+                ch: '+',
+                spacing: Alone,
+                span: $DIR/nodelim-groups.rs:17:18: 17:19 (#0),
+            },
+            Group {
+                delimiter: Parenthesis,
+                stream: TokenStream [
+                    Literal {
+                        kind: Integer,
+                        symbol: "25",
+                        suffix: None,
+                        span: $DIR/nodelim-groups.rs:17:21: 17:23 (#0),
+                    },
+                ],
+                span: $DIR/nodelim-groups.rs:17:20: 17:24 (#0),
+            },
+            Punct {
+                ch: '+',
+                spacing: Alone,
+                span: $DIR/nodelim-groups.rs:17:25: 17:26 (#0),
+            },
+            Literal {
+                kind: Integer,
+                symbol: "1",
+                suffix: None,
+                span: $DIR/nodelim-groups.rs:17:27: 17:28 (#0),
+            },
+        ],
+        span: $DIR/nodelim-groups.rs:13:47: 13:51 (#3),
+    },
+    Group {
+        delimiter: Parenthesis,
+        stream: TokenStream [
+            Literal {
+                kind: Integer,
+                symbol: "1",
+                suffix: None,
+                span: $DIR/nodelim-groups.rs:13:53: 13:54 (#3),
+            },
+            Punct {
+                ch: '+',
+                spacing: Alone,
+                span: $DIR/nodelim-groups.rs:13:55: 13:56 (#3),
+            },
+            Literal {
+                kind: Integer,
+                symbol: "1",
+                suffix: None,
+                span: $DIR/nodelim-groups.rs:13:57: 13:58 (#3),
+            },
+        ],
+        span: $DIR/nodelim-groups.rs:13:52: 13:59 (#3),
+    },
+]
+PRINT-BANG INPUT (DISPLAY): "hi" "hello".len() + "world".len() (1 + 1)
+PRINT-BANG RE-COLLECTED (DISPLAY): "hi" "hello" . len() + "world" . len() (1 + 1)
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Literal {
+        kind: Str,
+        symbol: "hi",
+        suffix: None,
+        span: $DIR/nodelim-groups.rs:13:42: 13:46 (#8),
+    },
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Literal {
+                kind: Str,
+                symbol: "hello",
+                suffix: None,
+                span: $DIR/nodelim-groups.rs:13:47: 13:51 (#8),
+            },
+            Punct {
+                ch: '.',
+                spacing: Alone,
+                span: $DIR/nodelim-groups.rs:13:47: 13:51 (#8),
+            },
+            Ident {
+                ident: "len",
+                span: $DIR/nodelim-groups.rs:13:47: 13:51 (#8),
+            },
+            Group {
+                delimiter: Parenthesis,
+                stream: TokenStream [],
+                span: $DIR/nodelim-groups.rs:13:47: 13:51 (#8),
+            },
+            Punct {
+                ch: '+',
+                spacing: Alone,
+                span: $DIR/nodelim-groups.rs:13:47: 13:51 (#8),
+            },
+            Literal {
+                kind: Str,
+                symbol: "world",
+                suffix: None,
+                span: $DIR/nodelim-groups.rs:13:47: 13:51 (#8),
+            },
+            Punct {
+                ch: '.',
+                spacing: Alone,
+                span: $DIR/nodelim-groups.rs:13:47: 13:51 (#8),
+            },
+            Ident {
+                ident: "len",
+                span: $DIR/nodelim-groups.rs:13:47: 13:51 (#8),
+            },
+            Group {
+                delimiter: Parenthesis,
+                stream: TokenStream [],
+                span: $DIR/nodelim-groups.rs:13:47: 13:51 (#8),
+            },
+        ],
+        span: $DIR/nodelim-groups.rs:13:47: 13:51 (#8),
+    },
+    Group {
+        delimiter: Parenthesis,
+        stream: TokenStream [
+            Literal {
+                kind: Integer,
+                symbol: "1",
+                suffix: None,
+                span: $DIR/nodelim-groups.rs:13:53: 13:54 (#8),
+            },
+            Punct {
+                ch: '+',
+                spacing: Alone,
+                span: $DIR/nodelim-groups.rs:13:55: 13:56 (#8),
+            },
+            Literal {
+                kind: Integer,
+                symbol: "1",
+                suffix: None,
+                span: $DIR/nodelim-groups.rs:13:57: 13:58 (#8),
+            },
+        ],
+        span: $DIR/nodelim-groups.rs:13:52: 13:59 (#8),
+    },
+]

src/test/ui/unsafe/ranged_ints_macro.rs

@@ -0,0 +1,16 @@
+// build-pass
+#![feature(rustc_attrs)]
+
+macro_rules! apply {
+    ($val:expr) => {
+        #[rustc_layout_scalar_valid_range_start($val)]
+        #[repr(transparent)]
+        pub(crate) struct NonZero<T>(pub(crate) T);
+    }
+}
+
+apply!(1);
+
+fn main() {
+    let _x = unsafe { NonZero(1) };
+}