proc_macro: Add an expand_expr method to TokenStream

This feature is aimed at giving proc macros access to powers similar to
those used by builtin macros such as `format_args!` or `concat!`. These
macros are able to accept macros in place of string literal parameters,
such as the format string, as they perform recursive macro expansion
while being expanded.

This can be especially useful in many cases thanks to helper macros like
`concat!`, `stringify!` and `include_str!` which are often used to
construct string literals at compile-time in user code.

For now, this method only allows expanding macros which produce
literals, although more expresisons will be supported before the method
is stabilized.

src/test/ui/proc-macro/auxiliary/expand-expr.rs

@@ -0,0 +1,80 @@
+// force-host
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+#![deny(warnings)]
+#![feature(proc_macro_expand, proc_macro_span)]
+
+extern crate proc_macro;
+
+use proc_macro::*;
+use std::str::FromStr;
+
+#[proc_macro]
+pub fn expand_expr_is(input: TokenStream) -> TokenStream {
+    let mut iter = input.into_iter();
+    let mut expected_tts = Vec::new();
+    loop {
+        match iter.next() {
+            Some(TokenTree::Punct(ref p)) if p.as_char() == ',' => break,
+            Some(tt) => expected_tts.push(tt),
+            None => panic!("expected comma"),
+        }
+    }
+
+    let expected = expected_tts.into_iter().collect::<TokenStream>();
+    let expanded = iter.collect::<TokenStream>().expand_expr().expect("expand_expr failed");
+    assert!(
+        expected.to_string() == expanded.to_string(),
+        "assert failed\nexpected: `{}`\nexpanded: `{}`",
+        expected.to_string(),
+        expanded.to_string()
+    );
+
+    TokenStream::new()
+}
+
+#[proc_macro]
+pub fn expand_expr_fail(input: TokenStream) -> TokenStream {
+    match input.expand_expr() {
+        Ok(ts) => panic!("expand_expr unexpectedly succeeded: `{}`", ts),
+        Err(_) => TokenStream::new(),
+    }
+}
+
+#[proc_macro]
+pub fn check_expand_expr_file(ts: TokenStream) -> TokenStream {
+    // Check that the passed in `file!()` invocation and a parsed `file!`
+    // invocation expand to the same literal.
+    let input_t = ts.expand_expr().expect("expand_expr failed on macro input").to_string();
+    let parse_t = TokenStream::from_str("file!{}")
+    .unwrap()
+        .expand_expr()
+        .expect("expand_expr failed on internal macro")
+        .to_string();
+    assert_eq!(input_t, parse_t);
+
+    // Check that the literal matches `Span::call_site().source_file().path()`
+    let expect_t =
+        Literal::string(&Span::call_site().source_file().path().to_string_lossy()).to_string();
+    assert_eq!(input_t, expect_t);
+
+    TokenStream::new()
+}
+
+#[proc_macro]
+pub fn recursive_expand(_: TokenStream) -> TokenStream {
+    // Recursively call until we hit the recursion limit and get an error.
+    //
+    // NOTE: This doesn't panic if expansion fails because that'll cause a very
+    // large number of errors to fill the output.
+    TokenStream::from_str("recursive_expand!{}")
+        .unwrap()
+        .expand_expr()
+        .unwrap_or(std::iter::once(TokenTree::Literal(Literal::u32_suffixed(0))).collect())
+}
+
+#[proc_macro]
+pub fn echo_pm(input: TokenStream) -> TokenStream {
+    input
+}

src/test/ui/proc-macro/auxiliary/included-file.txt

@@ -0,0 +1 @@
+Included file contents

src/test/ui/proc-macro/expand-expr.rs

@@ -0,0 +1,121 @@
+// aux-build:expand-expr.rs
+
+extern crate expand_expr;
+
+use expand_expr::{
+    check_expand_expr_file, echo_pm, expand_expr_fail, expand_expr_is, recursive_expand,
+};
+
+// Check builtin macros can be expanded.
+
+expand_expr_is!(11u32, line!());
+expand_expr_is!(24u32, column!());
+
+expand_expr_is!("Hello, World!", concat!("Hello, ", "World", "!"));
+expand_expr_is!("int10floats5.3booltrue", concat!("int", 10, "floats", 5.3, "bool", true));
+expand_expr_is!("Hello", concat!(r##"Hello"##));
+
+expand_expr_is!("Included file contents\n", include_str!("auxiliary/included-file.txt"));
+expand_expr_is!(b"Included file contents\n", include_bytes!("auxiliary/included-file.txt"));
+
+expand_expr_is!(
+    "contents: Included file contents\n",
+    concat!("contents: ", include_str!("auxiliary/included-file.txt"))
+);
+
+// Correct value is checked for multiple sources.
+check_expand_expr_file!(file!());
+
+expand_expr_is!("hello", stringify!(hello));
+expand_expr_is!("10 + 20", stringify!(10 + 20));
+
+macro_rules! echo_tts {
+    ($($t:tt)*) => { $($t)* };  //~ ERROR: expected expression, found `$`
+}
+
+macro_rules! echo_lit {
+    ($l:literal) => {
+        $l
+    };
+}
+
+macro_rules! echo_expr {
+    ($e:expr) => {
+        $e
+    };
+}
+
+macro_rules! simple_lit {
+    ($l:literal) => {
+        expand_expr_is!($l, $l);
+        expand_expr_is!($l, echo_lit!($l));
+        expand_expr_is!($l, echo_expr!($l));
+        expand_expr_is!($l, echo_tts!($l));
+        expand_expr_is!($l, echo_pm!($l));
+        const _: () = {
+            macro_rules! mac {
+                () => {
+                    $l
+                };
+            }
+            expand_expr_is!($l, mac!());
+            expand_expr_is!($l, echo_expr!(mac!()));
+            expand_expr_is!($l, echo_tts!(mac!()));
+            expand_expr_is!($l, echo_pm!(mac!()));
+        };
+    };
+}
+
+simple_lit!("Hello, World");
+simple_lit!('c');
+simple_lit!(b'c');
+simple_lit!(10);
+simple_lit!(10.0);
+simple_lit!(10.0f64);
+simple_lit!(-3.14159);
+simple_lit!(-3.5e10);
+simple_lit!(0xFEED);
+simple_lit!(-0xFEED);
+simple_lit!(0b0100);
+simple_lit!(-0b0100);
+simple_lit!("string");
+simple_lit!(r##"raw string"##);
+simple_lit!(b"byte string");
+simple_lit!(br##"raw byte string"##);
+simple_lit!(true);
+simple_lit!(false);
+
+// Ensure char escapes aren't normalized by expansion
+simple_lit!("\u{0}");
+simple_lit!("\0");
+simple_lit!("\x00");
+simple_lit!('\u{0}');
+simple_lit!('\0');
+simple_lit!('\x00');
+simple_lit!(b"\x00");
+simple_lit!(b"\0");
+simple_lit!(b'\x00');
+simple_lit!(b'\0');
+
+// Extra tokens after the string literal aren't ignored
+expand_expr_fail!("string"; hello); //~ ERROR: expected one of `.`, `?`, or an operator, found `;`
+
+// Invalid expressions produce errors in addition to returning `Err(())`.
+expand_expr_fail!($); //~ ERROR: expected expression, found `$`
+expand_expr_fail!(echo_tts!($));
+expand_expr_fail!(echo_pm!($)); //~ ERROR: expected expression, found `$`
+
+// We get errors reported and recover during macro expansion if the macro
+// doesn't produce a valid expression.
+expand_expr_is!("string", echo_tts!("string"; hello)); //~ ERROR: macro expansion ignores token `hello` and any following
+expand_expr_is!("string", echo_pm!("string"; hello)); //~ ERROR: macro expansion ignores token `;` and any following
+
+// For now, fail if a non-literal expression is expanded.
+expand_expr_fail!(arbitrary_expression() + "etc");
+expand_expr_fail!(echo_tts!(arbitrary_expression() + "etc"));
+expand_expr_fail!(echo_expr!(arbitrary_expression() + "etc"));
+expand_expr_fail!(echo_pm!(arbitrary_expression() + "etc"));
+
+const _: u32 = recursive_expand!(); //~ ERROR: recursion limit reached while expanding `recursive_expand!`
+
+fn main() {}

src/test/ui/proc-macro/expand-expr.stderr

@@ -0,0 +1,55 @@
+error: expected one of `.`, `?`, or an operator, found `;`
+  --> $DIR/expand-expr.rs:101:27
+   |
+LL | expand_expr_fail!("string"; hello);
+   |                           ^ expected one of `.`, `?`, or an operator
+
+error: expected expression, found `$`
+  --> $DIR/expand-expr.rs:104:19
+   |
+LL | expand_expr_fail!($);
+   |                   ^ expected expression
+
+error: expected expression, found `$`
+  --> $DIR/expand-expr.rs:33:23
+   |
+LL |     ($($t:tt)*) => { $($t)* };
+   |                       ^^^^ expected expression
+
+error: expected expression, found `$`
+  --> $DIR/expand-expr.rs:106:28
+   |
+LL | expand_expr_fail!(echo_pm!($));
+   |                            ^ expected expression
+
+error: macro expansion ignores token `hello` and any following
+  --> $DIR/expand-expr.rs:110:47
+   |
+LL | expand_expr_is!("string", echo_tts!("string"; hello));
+   |                           --------------------^^^^^-- help: you might be missing a semicolon here: `;`
+   |                           |
+   |                           caused by the macro expansion here
+   |
+   = note: the usage of `echo_tts!` is likely invalid in expression context
+
+error: macro expansion ignores token `;` and any following
+  --> $DIR/expand-expr.rs:111:44
+   |
+LL | expand_expr_is!("string", echo_pm!("string"; hello));
+   |                           -----------------^-------- help: you might be missing a semicolon here: `;`
+   |                           |
+   |                           caused by the macro expansion here
+   |
+   = note: the usage of `echo_pm!` is likely invalid in expression context
+
+error: recursion limit reached while expanding `recursive_expand!`
+  --> $DIR/expand-expr.rs:119:16
+   |
+LL | const _: u32 = recursive_expand!();
+   |                ^^^^^^^^^^^^^^^^^^^
+   |
+   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` attribute to your crate (`expand_expr`)
+   = note: this error originates in the macro `recursive_expand` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: aborting due to 7 previous errors
+