user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Auto merge of #52552 - eddyb:proc-macro-prep, r=alexcrichton

Browse source 
Prepare proc_macro for decoupling it from the rest of the compiler.

This is #49219 up to the point where the bridge is introduced. Aside from moving some code around, the largest change is the rewrite of `proc_macro::quote` to be simpler and do less introspection.

I'd like to also extend `quote!` with `${stmt;...;expr}` instead of just `$variable` (and maybe even `$(... $iter ...)*`), which seems pretty straight-forward now, but I don't know if/when I should.

r? @alexcrichton or @dtolnay cc @jseyfried @petrochenkov
This commit is contained in:
bors 2018-07-21 08:31:32 +00:00
commit bd455ef165
8 changed files with 465 additions and 570 deletions
Download patch file Download diff file Expand all files Collapse all files

48
src/libproc_macro/diagnostic.rs
View file

@ -10,7 +10,8 @@
use Span;
use rustc_errors as rustc;
use rustc_errors as errors;
use syntax_pos::MultiSpan;
/// An enum representing a diagnostic level.
#[unstable(feature = "proc_macro_diagnostic", issue = "38356")]
@ -97,38 +98,21 @@ impl Diagnostic {
/// Emit the diagnostic.
#[unstable(feature = "proc_macro_diagnostic", issue = "38356")]
pub fn emit(self) {
let level = self.level.to_internal();
let mut diag = errors::Diagnostic::new(level, &*self.message);
if let Some(span) = self.span {
diag.set_span(span.0);
}
for child in self.children {
let span = child.span.map_or(MultiSpan::new(), |s| s.0.into());
let level = child.level.to_internal();
diag.sub(level, &*child.message, span, None);
}
::__internal::with_sess(move |sess, _| {
let handler = &sess.span_diagnostic;
let level = __internal::level_to_internal_level(self.level);
let mut diag = rustc::DiagnosticBuilder::new(handler, level, &*self.message);
if let Some(span) = self.span {
diag.set_span(span.0);
}
for child in self.children {
let span = child.span.map(|s| s.0);
let level = __internal::level_to_internal_level(child.level);
diag.sub(level, &*child.message, span);
}
diag.emit();
errors::DiagnosticBuilder::new_diagnostic(&sess.span_diagnostic, diag).emit();
});
}
}
#[unstable(feature = "proc_macro_internals", issue = "27812")]
#[doc(hidden)]
pub mod __internal {
use super::{Level, rustc};
pub fn level_to_internal_level(level: Level) -> rustc::Level {
match level {
Level::Error => rustc::Level::Error,
Level::Warning => rustc::Level::Warning,
Level::Note => rustc::Level::Note,
Level::Help => rustc::Level::Help,
Level::__Nonexhaustive => unreachable!("Level::__Nonexhaustive")
}
}
}

348
src/libproc_macro/lib.rs
View file

@ -44,21 +44,24 @@ extern crate syntax_pos;
extern crate rustc_errors;
extern crate rustc_data_structures;
#[unstable(feature = "proc_macro_internals", issue = "27812")]
#[doc(hidden)]
pub mod rustc;
mod diagnostic;
#[unstable(feature = "proc_macro_diagnostic", issue = "38356")]
pub use diagnostic::{Diagnostic, Level};
use std::{ascii, fmt, iter};
use std::path::PathBuf;
use rustc_data_structures::sync::Lrc;
use std::str::FromStr;
use syntax::ast;
use syntax::errors::DiagnosticBuilder;
use syntax::parse::{self, token};
use syntax::symbol::{keywords, Symbol};
use syntax::symbol::Symbol;
use syntax::tokenstream;
use syntax::parse::lexer::{self, comments};
use syntax_pos::{FileMap, Pos, FileName};
/// The main type provided by this crate, representing an abstract stream of
@ -145,6 +148,9 @@ impl fmt::Debug for TokenStream {
}
}
#[unstable(feature = "proc_macro_quote", issue = "38356")]
pub use quote::{quote, quote_span};
/// Creates a token stream containing a single token tree.
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
impl From<TokenTree> for TokenStream {
@ -237,7 +243,7 @@ pub mod token_stream {
/// Unquoting is done with `$`, and works by taking the single next ident as the unquoted term.
/// To quote `$` itself, use `$$`.
///
/// This is a dummy macro, the actual implementation is in quote::Quoter
/// This is a dummy macro, the actual implementation is in `quote::quote`.`
#[unstable(feature = "proc_macro_quote", issue = "38356")]
#[macro_export]
macro_rules! quote { () => {} }
@ -246,13 +252,6 @@ macro_rules! quote { () => {} }
#[doc(hidden)]
mod quote;
/// Quote a `Span` into a `TokenStream`.
/// This is needed to implement a custom quoter.
#[unstable(feature = "proc_macro_quote", issue = "38356")]
pub fn quote_span(span: Span) -> TokenStream {
quote::Quote::quote(span)
}
/// A region of source code, along with macro expansion information.
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
#[derive(Copy, Clone)]
@ -425,8 +424,11 @@ impl SourceFile {
///
/// [`is_real`]: #method.is_real
#[unstable(feature = "proc_macro_span", issue = "38356")]
pub fn path(&self) -> &FileName {
&self.filemap.name
pub fn path(&self) -> PathBuf {
match self.filemap.name {
FileName::Real(ref path) => path.clone(),
_ => PathBuf::from(self.filemap.name.to_string())
}
}
/// Returns `true` if this source file is a real source file, and not generated by an external
@ -440,18 +442,12 @@ impl SourceFile {
}
}
#[unstable(feature = "proc_macro_span", issue = "38356")]
impl AsRef<FileName> for SourceFile {
fn as_ref(&self) -> &FileName {
self.path()
}
}
#[unstable(feature = "proc_macro_span", issue = "38356")]
impl fmt::Debug for SourceFile {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("SourceFile")
.field("path", self.path())
.field("path", &self.path())
.field("is_real", &self.is_real())
.finish()
}
@ -467,13 +463,6 @@ impl PartialEq for SourceFile {
#[unstable(feature = "proc_macro_span", issue = "38356")]
impl Eq for SourceFile {}
#[unstable(feature = "proc_macro_span", issue = "38356")]
impl PartialEq<FileName> for SourceFile {
fn eq(&self, other: &FileName) -> bool {
self.as_ref() == other
}
}
/// A single token or a delimited sequence of token trees (e.g. `[1, (), ..]`).
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
#[derive(Clone)]
@ -599,7 +588,7 @@ impl fmt::Display for TokenTree {
/// A delimited token stream.
///
/// A `Group` internally contains a `TokenStream` which is surrounded by `Delimiter`s.
#[derive(Clone, Debug)]
#[derive(Clone)]
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
pub struct Group {
delimiter: Delimiter,
@ -693,12 +682,23 @@ impl fmt::Display for Group {
}
}
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
impl fmt::Debug for Group {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("Group")
.field("delimiter", &self.delimiter())
.field("stream", &self.stream())
.field("span", &self.span())
.finish()
}
}
/// An `Punct` is an single punctuation character like `+`, `-` or `#`.
///
/// Multicharacter operators like `+=` are represented as two instances of `Punct` with different
/// forms of `Spacing` returned.
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
#[derive(Clone, Debug)]
#[derive(Clone)]
pub struct Punct {
ch: char,
spacing: Spacing,
@ -782,8 +782,19 @@ impl fmt::Display for Punct {
}
}
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
impl fmt::Debug for Punct {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("Punct")
.field("ch", &self.as_char())
.field("spacing", &self.spacing())
.field("span", &self.span())
.finish()
}
}
/// An identifier (`ident`).
#[derive(Clone, Debug)]
#[derive(Clone)]
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
pub struct Ident {
sym: Symbol,
@ -797,6 +808,16 @@ impl !Send for Ident {}
impl !Sync for Ident {}
impl Ident {
fn is_valid(string: &str) -> bool {
let mut chars = string.chars();
if let Some(start) = chars.next() {
(start == '_' || start.is_xid_start())
&& chars.all(|cont| cont == '_' || cont.is_xid_continue())
} else {
false
}
}
/// Creates a new `Ident` with the given `string` as well as the specified
/// `span`.
/// The `string` argument must be a valid identifier permitted by the
@ -818,26 +839,19 @@ impl Ident {
/// tokens, requires a `Span` to be specified at construction.
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
pub fn new(string: &str, span: Span) -> Ident {
if !lexer::is_valid_ident(string) {
if !Ident::is_valid(string) {
panic!("`{:?}` is not a valid identifier", string)
}
Ident {
sym: Symbol::intern(string),
span,
is_raw: false,
}
Ident::new_maybe_raw(string, span, false)
}
/// Same as `Ident::new`, but creates a raw identifier (`r#ident`).
#[unstable(feature = "proc_macro_raw_ident", issue = "38356")]
pub fn new_raw(string: &str, span: Span) -> Ident {
let mut ident = Ident::new(string, span);
if ident.sym == keywords::Underscore.name() ||
ast::Ident::with_empty_ctxt(ident.sym).is_path_segment_keyword() {
panic!("`{:?}` is not a valid raw identifier", string)
if !Ident::is_valid(string) {
panic!("`{:?}` is not a valid identifier", string)
}
ident.is_raw = true;
ident
Ident::new_maybe_raw(string, span, true)
}
/// Returns the span of this `Ident`, encompassing the entire string returned
@ -859,10 +873,17 @@ impl Ident {
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
impl fmt::Display for Ident {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
if self.is_raw {
f.write_str("r#")?;
}
self.sym.as_str().fmt(f)
TokenStream::from(TokenTree::from(self.clone())).fmt(f)
}
}
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
impl fmt::Debug for Ident {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("Ident")
.field("ident", &self.to_string())
.field("span", &self.span())
.finish()
}
}
@ -870,11 +891,12 @@ impl fmt::Display for Ident {
/// character (`'a'`), byte character (`b'a'`), an integer or floating point number
/// with or without a suffix (`1`, `1u8`, `2.3`, `2.3f32`).
/// Boolean literals like `true` and `false` do not belong here, they are `Ident`s.
// FIXME(eddyb) `Literal` should not expose internal `Debug` impls.
#[derive(Clone, Debug)]
#[stable(feature = "proc_macro_lib2", since = "1.29.0")]
pub struct Literal {
lit: token::Lit,
suffix: Option<ast::Name>,
suffix: Option<Symbol>,
span: Span,
}
@ -1122,236 +1144,6 @@ impl fmt::Display for Literal {
}
}
impl Delimiter {
fn from_internal(delim: token::DelimToken) -> Delimiter {
match delim {
token::Paren => Delimiter::Parenthesis,
token::Brace => Delimiter::Brace,
token::Bracket => Delimiter::Bracket,
token::NoDelim => Delimiter::None,
}
}
fn to_internal(self) -> token::DelimToken {
match self {
Delimiter::Parenthesis => token::Paren,
Delimiter::Brace => token::Brace,
Delimiter::Bracket => token::Bracket,
Delimiter::None => token::NoDelim,
}
}
}
impl TokenTree {
fn from_internal(stream: tokenstream::TokenStream, stack: &mut Vec<TokenTree>)
-> TokenTree {
use syntax::parse::token::*;
let (tree, is_joint) = stream.as_tree();
let (span, token) = match tree {
tokenstream::TokenTree::Token(span, token) => (span, token),
tokenstream::TokenTree::Delimited(span, delimed) => {
let delimiter = Delimiter::from_internal(delimed.delim);
let mut g = Group::new(delimiter, TokenStream(delimed.tts.into()));
g.set_span(Span(span));
return g.into()
}
};
let op_kind = if is_joint { Spacing::Joint } else { Spacing::Alone };
macro_rules! tt {
($e:expr) => ({
let mut x = TokenTree::from($e);
x.set_span(Span(span));
x
})
}
macro_rules! op {
($a:expr) => (tt!(Punct::new($a, op_kind)));
($a:expr, $b:expr) => ({
stack.push(tt!(Punct::new($b, op_kind)));
tt!(Punct::new($a, Spacing::Joint))
});
($a:expr, $b:expr, $c:expr) => ({
stack.push(tt!(Punct::new($c, op_kind)));
stack.push(tt!(Punct::new($b, Spacing::Joint)));
tt!(Punct::new($a, Spacing::Joint))
})
}
match token {
Eq => op!('='),
Lt => op!('<'),
Le => op!('<', '='),
EqEq => op!('=', '='),
Ne => op!('!', '='),
Ge => op!('>', '='),
Gt => op!('>'),
AndAnd => op!('&', '&'),
OrOr => op!('|', '|'),
Not => op!('!'),
Tilde => op!('~'),
BinOp(Plus) => op!('+'),
BinOp(Minus) => op!('-'),
BinOp(Star) => op!('*'),
BinOp(Slash) => op!('/'),
BinOp(Percent) => op!('%'),
BinOp(Caret) => op!('^'),
BinOp(And) => op!('&'),
BinOp(Or) => op!('|'),
BinOp(Shl) => op!('<', '<'),
BinOp(Shr) => op!('>', '>'),
BinOpEq(Plus) => op!('+', '='),
BinOpEq(Minus) => op!('-', '='),
BinOpEq(Star) => op!('*', '='),
BinOpEq(Slash) => op!('/', '='),
BinOpEq(Percent) => op!('%', '='),
BinOpEq(Caret) => op!('^', '='),
BinOpEq(And) => op!('&', '='),
BinOpEq(Or) => op!('|', '='),
BinOpEq(Shl) => op!('<', '<', '='),
BinOpEq(Shr) => op!('>', '>', '='),
At => op!('@'),
Dot => op!('.'),
DotDot => op!('.', '.'),
DotDotDot => op!('.', '.', '.'),
DotDotEq => op!('.', '.', '='),
Comma => op!(','),
Semi => op!(';'),
Colon => op!(':'),
ModSep => op!(':', ':'),
RArrow => op!('-', '>'),
LArrow => op!('<', '-'),
FatArrow => op!('=', '>'),
Pound => op!('#'),
Dollar => op!('$'),
Question => op!('?'),
SingleQuote => op!('\''),
Ident(ident, false) => {
tt!(self::Ident::new(&ident.as_str(), Span(span)))
}
Ident(ident, true) => {
tt!(self::Ident::new_raw(&ident.as_str(), Span(span)))
}
Lifetime(ident) => {
let ident = ident.without_first_quote();
stack.push(tt!(self::Ident::new(&ident.as_str(), Span(span))));
tt!(Punct::new('\'', Spacing::Joint))
}
Literal(lit, suffix) => tt!(self::Literal { lit, suffix, span: Span(span) }),
DocComment(c) => {
let style = comments::doc_comment_style(&c.as_str());
let stripped = comments::strip_doc_comment_decoration(&c.as_str());
let stream = vec![
tt!(self::Ident::new("doc", Span(span))),
tt!(Punct::new('=', Spacing::Alone)),
tt!(self::Literal::string(&stripped)),
].into_iter().collect();
stack.push(tt!(Group::new(Delimiter::Bracket, stream)));
if style == ast::AttrStyle::Inner {
stack.push(tt!(Punct::new('!', Spacing::Alone)));
}
tt!(Punct::new('#', Spacing::Alone))
}
Interpolated(_) => {
__internal::with_sess(|sess, _| {
let tts = token.interpolated_to_tokenstream(sess, span);
tt!(Group::new(Delimiter::None, TokenStream(tts)))
})
}
DotEq => op!('.', '='),
OpenDelim(..) | CloseDelim(..) => unreachable!(),
Whitespace | Comment | Shebang(..) | Eof => unreachable!(),
}
}
fn to_internal(self) -> tokenstream::TokenStream {
use syntax::parse::token::*;
use syntax::tokenstream::{TokenTree, Delimited};
let (ch, kind, span) = match self {
self::TokenTree::Punct(tt) => (tt.as_char(), tt.spacing(), tt.span()),
self::TokenTree::Group(tt) => {
return TokenTree::Delimited(tt.span.0, Delimited {
delim: tt.delimiter.to_internal(),
tts: tt.stream.0.into(),
}).into();
},
self::TokenTree::Ident(tt) => {
let token = Ident(ast::Ident::new(tt.sym, tt.span.0), tt.is_raw);
return TokenTree::Token(tt.span.0, token).into();
}
self::TokenTree::Literal(self::Literal {
lit: Lit::Integer(ref a),
suffix,
span,
})
if a.as_str().starts_with("-") =>
{
let minus = BinOp(BinOpToken::Minus);
let integer = Symbol::intern(&a.as_str()[1..]);
let integer = Literal(Lit::Integer(integer), suffix);
let a = TokenTree::Token(span.0, minus);
let b = TokenTree::Token(span.0, integer);
return vec![a, b].into_iter().collect()
}
self::TokenTree::Literal(self::Literal {
lit: Lit::Float(ref a),
suffix,
span,
})
if a.as_str().starts_with("-") =>
{
let minus = BinOp(BinOpToken::Minus);
let float = Symbol::intern(&a.as_str()[1..]);
let float = Literal(Lit::Float(float), suffix);
let a = TokenTree::Token(span.0, minus);
let b = TokenTree::Token(span.0, float);
return vec![a, b].into_iter().collect()
}
self::TokenTree::Literal(tt) => {
let token = Literal(tt.lit, tt.suffix);
return TokenTree::Token(tt.span.0, token).into()
}
};
let token = match ch {
'=' => Eq,
'<' => Lt,
'>' => Gt,
'!' => Not,
'~' => Tilde,
'+' => BinOp(Plus),
'-' => BinOp(Minus),
'*' => BinOp(Star),
'/' => BinOp(Slash),
'%' => BinOp(Percent),
'^' => BinOp(Caret),
'&' => BinOp(And),
'|' => BinOp(Or),
'@' => At,
'.' => Dot,
',' => Comma,
';' => Semi,
':' => Colon,
'#' => Pound,
'$' => Dollar,
'?' => Question,
'\'' => SingleQuote,
_ => unreachable!(),
};
let tree = TokenTree::Token(span.0, token);
match kind {
Spacing::Alone => tree.into(),
Spacing::Joint => tree.joint(),
}
}
}
/// Permanently unstable internal implementation details of this crate. This
/// should not be used.
///
@ -1364,8 +1156,6 @@ impl TokenTree {
#[unstable(feature = "proc_macro_internals", issue = "27812")]
#[doc(hidden)]
pub mod __internal {
pub use quote::{LiteralKind, SpannedSymbol, Quoter, unquote};
use std::cell::Cell;
use std::ptr;

340
src/libproc_macro/quote.rs
View file

@ -14,35 +14,26 @@
//! This quasiquoter uses macros 2.0 hygiene to reliably access
//! items from `proc_macro`, to build a `proc_macro::TokenStream`.
use {Delimiter, Literal, Spacing, Span, Ident, Punct, Group, TokenStream, TokenTree};
use {Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
use syntax::ext::base::{ExtCtxt, ProcMacro};
use syntax::parse::token;
use syntax::symbol::Symbol;
use syntax::tokenstream;
/// This is the actual quote!() proc macro
///
/// It is manually loaded in CStore::load_macro_untracked
pub struct Quoter;
pub fn unquote<T: Into<TokenStream> + Clone>(tokens: &T) -> TokenStream {
tokens.clone().into()
macro_rules! quote_tt {
(($($t:tt)*)) => { Group::new(Delimiter::Parenthesis, quote!($($t)*)) };
([$($t:tt)*]) => { Group::new(Delimiter::Bracket, quote!($($t)*)) };
({$($t:tt)*}) => { Group::new(Delimiter::Brace, quote!($($t)*)) };
(,) => { Punct::new(',', Spacing::Alone) };
(.) => { Punct::new('.', Spacing::Alone) };
(:) => { Punct::new(':', Spacing::Alone) };
(;) => { Punct::new(';', Spacing::Alone) };
(!) => { Punct::new('!', Spacing::Alone) };
(<) => { Punct::new('<', Spacing::Alone) };
(>) => { Punct::new('>', Spacing::Alone) };
(&) => { Punct::new('&', Spacing::Alone) };
(=) => { Punct::new('=', Spacing::Alone) };
($i:ident) => { Ident::new(stringify!($i), Span::def_site()) };
}
pub trait Quote {
fn quote(self) -> TokenStream;
}
macro_rules! tt2ts {
($e:expr) => (TokenStream::from(TokenTree::from($e)))
}
macro_rules! quote_tok {
(,) => { tt2ts!(Punct::new(',', Spacing::Alone)) };
(.) => { tt2ts!(Punct::new('.', Spacing::Alone)) };
(:) => { tt2ts!(Punct::new(':', Spacing::Alone)) };
(|) => { tt2ts!(Punct::new('|', Spacing::Alone)) };
macro_rules! quote_ts {
((@ $($t:tt)*)) => { $($t)* };
(::) => {
[
TokenTree::from(Punct::new(':', Spacing::Joint)),
@ -55,65 +46,45 @@ macro_rules! quote_tok {
})
.collect::<TokenStream>()
};
(!) => { tt2ts!(Punct::new('!', Spacing::Alone)) };
(<) => { tt2ts!(Punct::new('<', Spacing::Alone)) };
(>) => { tt2ts!(Punct::new('>', Spacing::Alone)) };
(_) => { tt2ts!(Punct::new('_', Spacing::Alone)) };
(0) => { tt2ts!(Literal::i8_unsuffixed(0)) };
(&) => { tt2ts!(Punct::new('&', Spacing::Alone)) };
($i:ident) => { tt2ts!(Ident::new(stringify!($i), Span::def_site())) };
}
macro_rules! quote_tree {
((unquote $($t:tt)*)) => { $($t)* };
((quote $($t:tt)*)) => { ($($t)*).quote() };
(($($t:tt)*)) => { tt2ts!(Group::new(Delimiter::Parenthesis, quote!($($t)*))) };
([$($t:tt)*]) => { tt2ts!(Group::new(Delimiter::Bracket, quote!($($t)*))) };
({$($t:tt)*}) => { tt2ts!(Group::new(Delimiter::Brace, quote!($($t)*))) };
($t:tt) => { quote_tok!($t) };
($t:tt) => { TokenTree::from(quote_tt!($t)) };
}
/// Simpler version of the real `quote!` macro, implemented solely
/// through `macro_rules`, for bootstrapping the real implementation
/// (see the `quote` function), which does not have access to the
/// real `quote!` macro due to the `proc_macro` crate not being
/// able to depend on itself.
///
/// Note: supported tokens are a subset of the real `quote!`, but
/// unquoting is different: instead of `$x`, this uses `(@ expr)`.
macro_rules! quote {
() => { TokenStream::new() };
($($t:tt)*) => {
[$(quote_tree!($t),)*].iter()
.cloned()
.flat_map(|x| x.into_iter())
.collect::<TokenStream>()
[
$(TokenStream::from(quote_ts!($t)),)*
].iter().cloned().collect::<TokenStream>()
};
}
impl ProcMacro for Quoter {
fn expand<'cx>(&self, cx: &'cx mut ExtCtxt,
_: ::syntax_pos::Span,
stream: tokenstream::TokenStream)
-> tokenstream::TokenStream {
::__internal::set_sess(cx, || TokenStream(stream).quote().0)
/// Quote a `TokenStream` into a `TokenStream`.
/// This is the actual `quote!()` proc macro.
///
/// It is manually loaded in `CStore::load_macro_untracked`.
#[unstable(feature = "proc_macro_quote", issue = "38356")]
pub fn quote(stream: TokenStream) -> TokenStream {
if stream.is_empty() {
return quote!(::TokenStream::new());
}
}
impl<T: Quote> Quote for Option<T> {
fn quote(self) -> TokenStream {
match self {
Some(t) => quote!(Some((quote t))),
None => quote!(None),
}
}
}
impl Quote for TokenStream {
fn quote(self) -> TokenStream {
if self.is_empty() {
return quote!(::TokenStream::new());
}
let mut after_dollar = false;
let tokens = self.into_iter().filter_map(|tree| {
let mut after_dollar = false;
let tokens = stream
.into_iter()
.filter_map(|tree| {
if after_dollar {
after_dollar = false;
match tree {
TokenTree::Ident(_) => {
let tree = TokenStream::from(tree);
return Some(quote!(::__internal::unquote(&(unquote tree)),));
return Some(quote!(Into::<::TokenStream>::into(
Clone::clone(&(@ tree))),));
}
TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
_ => panic!("`$` must be followed by an ident or `$` in `quote!`"),
@ -125,186 +96,55 @@ impl Quote for TokenStream {
}
}
Some(quote!(::TokenStream::from((quote tree)),))
}).flat_map(|t| t.into_iter()).collect::<TokenStream>();
Some(quote!(::TokenStream::from((@ match tree {
TokenTree::Punct(tt) => quote!(::TokenTree::Punct(::Punct::new(
(@ TokenTree::from(Literal::character(tt.as_char()))),
(@ match tt.spacing() {
Spacing::Alone => quote!(::Spacing::Alone),
Spacing::Joint => quote!(::Spacing::Joint),
}),
))),
TokenTree::Group(tt) => quote!(::TokenTree::Group(::Group::new(
(@ match tt.delimiter() {
Delimiter::Parenthesis => quote!(::Delimiter::Parenthesis),
Delimiter::Brace => quote!(::Delimiter::Brace),
Delimiter::Bracket => quote!(::Delimiter::Bracket),
Delimiter::None => quote!(::Delimiter::None),
}),
(@ quote(tt.stream())),
))),
TokenTree::Ident(tt) => quote!(::TokenTree::Ident(::Ident::new(
(@ TokenTree::from(Literal::string(&tt.to_string()))),
(@ quote_span(tt.span())),
))),
TokenTree::Literal(tt) => quote!(::TokenTree::Literal({
let mut iter = (@ TokenTree::from(Literal::string(&tt.to_string())))
.parse::<::TokenStream>()
.unwrap()
.into_iter();
if let (Some(::TokenTree::Literal(mut lit)), None) =
(iter.next(), iter.next())
{
lit.set_span((@ quote_span(tt.span())));
lit
} else {
unreachable!()
}
}))
})),))
})
.collect::<TokenStream>();
if after_dollar {
panic!("unexpected trailing `$` in `quote!`");
}
quote!(
[(unquote tokens)].iter()
.cloned()
.flat_map(|x| x.into_iter())
.collect::<::TokenStream>()
)
if after_dollar {
panic!("unexpected trailing `$` in `quote!`");
}
quote!([(@ tokens)].iter().cloned().collect::<::TokenStream>())
}
impl Quote for TokenTree {
fn quote(self) -> TokenStream {
match self {
TokenTree::Punct(tt) => quote!(::TokenTree::Punct( (quote tt) )),
TokenTree::Group(tt) => quote!(::TokenTree::Group( (quote tt) )),
TokenTree::Ident(tt) => quote!(::TokenTree::Ident( (quote tt) )),
TokenTree::Literal(tt) => quote!(::TokenTree::Literal( (quote tt) )),
}
}
}
impl Quote for char {
fn quote(self) -> TokenStream {
TokenTree::from(Literal::character(self)).into()
}
}
impl<'a> Quote for &'a str {
fn quote(self) -> TokenStream {
TokenTree::from(Literal::string(self)).into()
}
}
impl Quote for u16 {
fn quote(self) -> TokenStream {
TokenTree::from(Literal::u16_unsuffixed(self)).into()
}
}
impl Quote for Group {
fn quote(self) -> TokenStream {
quote!(::Group::new((quote self.delimiter()), (quote self.stream())))
}
}
impl Quote for Punct {
fn quote(self) -> TokenStream {
quote!(::Punct::new((quote self.as_char()), (quote self.spacing())))
}
}
impl Quote for Ident {
fn quote(self) -> TokenStream {
quote!(::Ident::new((quote self.sym.as_str()), (quote self.span())))
}
}
impl Quote for Span {
fn quote(self) -> TokenStream {
quote!(::Span::def_site())
}
}
macro_rules! literals {
($($i:ident),*; $($raw:ident),*) => {
pub struct SpannedSymbol {
sym: Symbol,
span: Span,
}
impl SpannedSymbol {
pub fn new(string: &str, span: Span) -> SpannedSymbol {
SpannedSymbol { sym: Symbol::intern(string), span }
}
}
impl Quote for SpannedSymbol {
fn quote(self) -> TokenStream {
quote!(::__internal::SpannedSymbol::new((quote self.sym.as_str()),
(quote self.span)))
}
}
pub enum LiteralKind {
$($i,)*
$($raw(u16),)*
}
impl LiteralKind {
pub fn with_contents_and_suffix(self, contents: SpannedSymbol,
suffix: Option<SpannedSymbol>) -> Literal {
let sym = contents.sym;
let suffix = suffix.map(|t| t.sym);
match self {
$(LiteralKind::$i => {
Literal {
lit: token::Lit::$i(sym),
suffix,
span: contents.span,
}
})*
$(LiteralKind::$raw(n) => {
Literal {
lit: token::Lit::$raw(sym, n),
suffix,
span: contents.span,
}
})*
}
}
}
impl Literal {
fn kind_contents_and_suffix(self) -> (LiteralKind, SpannedSymbol, Option<SpannedSymbol>)
{
let (kind, contents) = match self.lit {
$(token::Lit::$i(contents) => (LiteralKind::$i, contents),)*
$(token::Lit::$raw(contents, n) => (LiteralKind::$raw(n), contents),)*
};
let suffix = self.suffix.map(|sym| SpannedSymbol::new(&sym.as_str(), self.span()));
(kind, SpannedSymbol::new(&contents.as_str(), self.span()), suffix)
}
}
impl Quote for LiteralKind {
fn quote(self) -> TokenStream {
match self {
$(LiteralKind::$i => quote! {
::__internal::LiteralKind::$i
},)*
$(LiteralKind::$raw(n) => quote! {
::__internal::LiteralKind::$raw((quote n))
},)*
}
}
}
impl Quote for Literal {
fn quote(self) -> TokenStream {
let (kind, contents, suffix) = self.kind_contents_and_suffix();
quote! {
(quote kind).with_contents_and_suffix((quote contents), (quote suffix))
}
}
}
}
}
literals!(Byte, Char, Float, Str_, Integer, ByteStr; StrRaw, ByteStrRaw);
impl Quote for Delimiter {
fn quote(self) -> TokenStream {
macro_rules! gen_match {
($($i:ident),*) => {
match self {
$(Delimiter::$i => { quote!(::Delimiter::$i) })*
}
}
}
gen_match!(Parenthesis, Brace, Bracket, None)
}
}
impl Quote for Spacing {
fn quote(self) -> TokenStream {
macro_rules! gen_match {
($($i:ident),*) => {
match self {
$(Spacing::$i => { quote!(::Spacing::$i) })*
}
}
}
gen_match!(Alone, Joint)
}
/// Quote a `Span` into a `TokenStream`.
/// This is needed to implement a custom quoter.
#[unstable(feature = "proc_macro_quote", issue = "38356")]
pub fn quote_span(_: Span) -> TokenStream {
quote!(::Span::def_site())
}

284
src/libproc_macro/rustc.rs Normal file
View file

@ -0,0 +1,284 @@
// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use {Delimiter, Level, Spacing, Span, __internal};
use {Group, Ident, Literal, Punct, TokenTree};
use rustc_errors as errors;
use syntax::ast;
use syntax::parse::lexer::comments;
use syntax::parse::token;
use syntax::tokenstream;
use syntax_pos::symbol::{keywords, Symbol};
impl Ident {
pub(crate) fn new_maybe_raw(string: &str, span: Span, is_raw: bool) -> Ident {
let sym = Symbol::intern(string);
if is_raw
&& (sym == keywords::Underscore.name()
|| ast::Ident::with_empty_ctxt(sym).is_path_segment_keyword())
{
panic!("`{:?}` is not a valid raw identifier", string)
}
Ident { sym, span, is_raw }
}
}
impl Delimiter {
pub(crate) fn from_internal(delim: token::DelimToken) -> Delimiter {
match delim {
token::Paren => Delimiter::Parenthesis,
token::Brace => Delimiter::Brace,
token::Bracket => Delimiter::Bracket,
token::NoDelim => Delimiter::None,
}
}
pub(crate) fn to_internal(self) -> token::DelimToken {
match self {
Delimiter::Parenthesis => token::Paren,
Delimiter::Brace => token::Brace,
Delimiter::Bracket => token::Bracket,
Delimiter::None => token::NoDelim,
}
}
}
impl TokenTree {
pub(crate) fn from_internal(
stream: tokenstream::TokenStream,
stack: &mut Vec<TokenTree>,
) -> TokenTree {
use syntax::parse::token::*;
let (tree, is_joint) = stream.as_tree();
let (span, token) = match tree {
tokenstream::TokenTree::Token(span, token) => (span, token),
tokenstream::TokenTree::Delimited(span, delimed) => {
let delimiter = Delimiter::from_internal(delimed.delim);
let mut g = Group::new(delimiter, ::TokenStream(delimed.tts.into()));
g.set_span(Span(span));
return g.into();
}
};
let op_kind = if is_joint {
Spacing::Joint
} else {
Spacing::Alone
};
macro_rules! tt {
($e:expr) => {{
let mut x = TokenTree::from($e);
x.set_span(Span(span));
x
}};
}
macro_rules! op {
($a:expr) => {
tt!(Punct::new($a, op_kind))
};
($a:expr, $b:expr) => {{
stack.push(tt!(Punct::new($b, op_kind)));
tt!(Punct::new($a, Spacing::Joint))
}};
($a:expr, $b:expr, $c:expr) => {{
stack.push(tt!(Punct::new($c, op_kind)));
stack.push(tt!(Punct::new($b, Spacing::Joint)));
tt!(Punct::new($a, Spacing::Joint))
}};
}
match token {
Eq => op!('='),
Lt => op!('<'),
Le => op!('<', '='),
EqEq => op!('=', '='),
Ne => op!('!', '='),
Ge => op!('>', '='),
Gt => op!('>'),
AndAnd => op!('&', '&'),
OrOr => op!('|', '|'),
Not => op!('!'),
Tilde => op!('~'),
BinOp(Plus) => op!('+'),
BinOp(Minus) => op!('-'),
BinOp(Star) => op!('*'),
BinOp(Slash) => op!('/'),
BinOp(Percent) => op!('%'),
BinOp(Caret) => op!('^'),
BinOp(And) => op!('&'),
BinOp(Or) => op!('|'),
BinOp(Shl) => op!('<', '<'),
BinOp(Shr) => op!('>', '>'),
BinOpEq(Plus) => op!('+', '='),
BinOpEq(Minus) => op!('-', '='),
BinOpEq(Star) => op!('*', '='),
BinOpEq(Slash) => op!('/', '='),
BinOpEq(Percent) => op!('%', '='),
BinOpEq(Caret) => op!('^', '='),
BinOpEq(And) => op!('&', '='),
BinOpEq(Or) => op!('|', '='),
BinOpEq(Shl) => op!('<', '<', '='),
BinOpEq(Shr) => op!('>', '>', '='),
At => op!('@'),
Dot => op!('.'),
DotDot => op!('.', '.'),
DotDotDot => op!('.', '.', '.'),
DotDotEq => op!('.', '.', '='),
Comma => op!(','),
Semi => op!(';'),
Colon => op!(':'),
ModSep => op!(':', ':'),
RArrow => op!('-', '>'),
LArrow => op!('<', '-'),
FatArrow => op!('=', '>'),
Pound => op!('#'),
Dollar => op!('$'),
Question => op!('?'),
SingleQuote => op!('\''),
Ident(ident, false) => tt!(self::Ident::new(&ident.as_str(), Span(span))),
Ident(ident, true) => tt!(self::Ident::new_raw(&ident.as_str(), Span(span))),
Lifetime(ident) => {
let ident = ident.without_first_quote();
stack.push(tt!(self::Ident::new(&ident.as_str(), Span(span))));
tt!(Punct::new('\'', Spacing::Joint))
}
Literal(lit, suffix) => tt!(self::Literal {
lit,
suffix,
span: Span(span)
}),
DocComment(c) => {
let style = comments::doc_comment_style(&c.as_str());
let stripped = comments::strip_doc_comment_decoration(&c.as_str());
let stream = vec![
tt!(self::Ident::new("doc", Span(span))),
tt!(Punct::new('=', Spacing::Alone)),
tt!(self::Literal::string(&stripped)),
].into_iter()
.collect();
stack.push(tt!(Group::new(Delimiter::Bracket, stream)));
if style == ast::AttrStyle::Inner {
stack.push(tt!(Punct::new('!', Spacing::Alone)));
}
tt!(Punct::new('#', Spacing::Alone))
}
Interpolated(_) => __internal::with_sess(|sess, _| {
let tts = token.interpolated_to_tokenstream(sess, span);
tt!(Group::new(Delimiter::None, ::TokenStream(tts)))
}),
DotEq => op!('.', '='),
OpenDelim(..) | CloseDelim(..) => unreachable!(),
Whitespace | Comment | Shebang(..) | Eof => unreachable!(),
}
}
pub(crate) fn to_internal(self) -> tokenstream::TokenStream {
use syntax::parse::token::*;
use syntax::tokenstream::{Delimited, TokenTree};
let (ch, kind, span) = match self {
self::TokenTree::Punct(tt) => (tt.as_char(), tt.spacing(), tt.span()),
self::TokenTree::Group(tt) => {
return TokenTree::Delimited(
tt.span.0,
Delimited {
delim: tt.delimiter.to_internal(),
tts: tt.stream.0.into(),
},
).into();
}
self::TokenTree::Ident(tt) => {
let token = Ident(ast::Ident::new(tt.sym, tt.span.0), tt.is_raw);
return TokenTree::Token(tt.span.0, token).into();
}
self::TokenTree::Literal(self::Literal {
lit: Lit::Integer(ref a),
suffix,
span,
})
if a.as_str().starts_with("-") =>
{
let minus = BinOp(BinOpToken::Minus);
let integer = Symbol::intern(&a.as_str()[1..]);
let integer = Literal(Lit::Integer(integer), suffix);
let a = TokenTree::Token(span.0, minus);
let b = TokenTree::Token(span.0, integer);
return vec![a, b].into_iter().collect();
}
self::TokenTree::Literal(self::Literal {
lit: Lit::Float(ref a),
suffix,
span,
})
if a.as_str().starts_with("-") =>
{
let minus = BinOp(BinOpToken::Minus);
let float = Symbol::intern(&a.as_str()[1..]);
let float = Literal(Lit::Float(float), suffix);
let a = TokenTree::Token(span.0, minus);
let b = TokenTree::Token(span.0, float);
return vec![a, b].into_iter().collect();
}
self::TokenTree::Literal(tt) => {
let token = Literal(tt.lit, tt.suffix);
return TokenTree::Token(tt.span.0, token).into();
}
};
let token = match ch {
'=' => Eq,
'<' => Lt,
'>' => Gt,
'!' => Not,
'~' => Tilde,
'+' => BinOp(Plus),
'-' => BinOp(Minus),
'*' => BinOp(Star),
'/' => BinOp(Slash),
'%' => BinOp(Percent),
'^' => BinOp(Caret),
'&' => BinOp(And),
'|' => BinOp(Or),
'@' => At,
'.' => Dot,
',' => Comma,
';' => Semi,
':' => Colon,
'#' => Pound,
'$' => Dollar,
'?' => Question,
'\'' => SingleQuote,
_ => unreachable!(),
};
let tree = TokenTree::Token(span.0, token);
match kind {
Spacing::Alone => tree.into(),
Spacing::Joint => tree.joint(),
}
}
}
impl Level {
pub(crate) fn to_internal(self) -> errors::Level {
match self {
Level::Error => errors::Level::Error,
Level::Warning => errors::Level::Warning,
Level::Note => errors::Level::Note,
Level::Help => errors::Level::Help,
Level::__Nonexhaustive => unreachable!("Level::__Nonexhaustive"),
}
}
}

2
src/librustc_errors/diagnostic.rs
View file

@ -379,7 +379,7 @@ impl Diagnostic {
/// Convenience function for internal use, clients should use one of the
/// public methods above.
pub(crate) fn sub(&mut self,
pub fn sub(&mut self,
level: Level,
message: &str,
span: MultiSpan,

6
src/librustc_metadata/cstore_impl.rs
View file

@ -39,7 +39,6 @@ use syntax::ast;
use syntax::attr;
use syntax::codemap;
use syntax::edition::Edition;
use syntax::ext::base::SyntaxExtension;
use syntax::parse::filemap_to_stream;
use syntax::symbol::Symbol;
use syntax_pos::{Span, NO_EXPANSION, FileName};
@ -517,8 +516,11 @@ impl CrateStore for cstore::CStore {
return LoadedMacro::ProcMacro(proc_macros[id.index.to_proc_macro_index()].1.clone());
} else if data.name == "proc_macro" &&
self.get_crate_data(id.krate).item_name(id.index) == "quote" {
use syntax::ext::base::SyntaxExtension;
use syntax_ext::proc_macro_impl::BangProcMacro;
let ext = SyntaxExtension::ProcMacro {
expander: Box::new(::proc_macro::__internal::Quoter),
expander: Box::new(BangProcMacro { inner: ::proc_macro::quote }),
allow_internal_unstable: true,
edition: data.root.edition,
};

1
src/librustc_metadata/lib.rs
View file

@ -19,6 +19,7 @@
#![feature(libc)]
#![feature(macro_at_most_once_rep)]
#![feature(proc_macro_internals)]
#![feature(proc_macro_quote)]
#![feature(quote)]
#![feature(rustc_diagnostic_macros)]
#![feature(slice_sort_by_cached_key)]

6
src/libsyntax/parse/lexer/mod.rs
View file

@ -1775,12 +1775,6 @@ fn ident_continue(c: Option<char>) -> bool {
(c > '\x7f' && c.is_xid_continue())
}
// The string is a valid identifier or a lifetime identifier.
pub fn is_valid_ident(s: &str) -> bool {
let mut chars = s.chars();
ident_start(chars.next()) && chars.all(|ch| ident_continue(Some(ch)))
}
#[cfg(test)]
mod tests {
use super::*;