Introduce NonterminalKind

It encapsulate the (part of) the interface between the parser and
macro by example (macro_rules) parser.

The second bit is somewhat more general `parse_ast_fragment`, which is
the reason why we keep some `parse_xxx` functions as public.

src/librustc_parse/parser/expr.rs

@@ -1450,7 +1450,7 @@ impl<'a> Parser<'a> {
 
     /// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`).
     /// Keep this in sync with `Token::can_begin_literal_maybe_minus`.
-    pub fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> {
+    pub(super) fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> {
         maybe_whole_expr!(self);
 
         let lo = self.token.span;

src/librustc_parse/parser/mod.rs

@@ -1,6 +1,7 @@
 pub mod attr;
 mod expr;
 mod item;
+mod nonterminal;
 mod pat;
 mod path;
 mod ty;
@@ -10,6 +11,7 @@ mod generics;
 mod stmt;
 use diagnostics::Error;
 
+
 use crate::lexer::UnmatchedBrace;
 
 use log::debug;
@@ -958,7 +960,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a single token tree from the input.
-    pub fn parse_token_tree(&mut self) -> TokenTree {
+    pub(crate) fn parse_token_tree(&mut self) -> TokenTree {
         match self.token.kind {
             token::OpenDelim(..) => {
                 let frame = mem::replace(
@@ -1017,7 +1019,7 @@ impl<'a> Parser<'a> {
     /// If the following element can't be a tuple (i.e., it's a function definition), then
     /// it's not a tuple struct field), and the contents within the parentheses isn't valid,
     /// so emit a proper diagnostic.
-    pub fn parse_visibility(&mut self, fbt: FollowedByType) -> PResult<'a, Visibility> {
+    pub(crate) fn parse_visibility(&mut self, fbt: FollowedByType) -> PResult<'a, Visibility> {
         maybe_whole!(self, NtVis, |x| x);
 
         self.expected_tokens.push(TokenType::Keyword(kw::Crate));

src/librustc_parse/parser/nonterminal.rs

@@ -0,0 +1,163 @@
+use rustc_ast::ptr::P;
+use rustc_ast::token::{self, Nonterminal, NonterminalKind, Token};
+use rustc_ast_pretty::pprust;
+use rustc_errors::PResult;
+use rustc_span::symbol::{kw, Ident};
+
+use crate::parser::{FollowedByType, Parser, PathStyle};
+
+impl<'a> Parser<'a> {
+    /// Checks whether a non-terminal may begin with a particular token.
+    ///
+    /// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that
+    /// token. Be conservative (return true) if not sure.
+    pub fn nonterminal_may_begin_with(kind: NonterminalKind, token: &Token) -> bool {
+        /// Checks whether the non-terminal may contain a single (non-keyword) identifier.
+        fn may_be_ident(nt: &token::Nonterminal) -> bool {
+            match *nt {
+                token::NtItem(_) | token::NtBlock(_) | token::NtVis(_) | token::NtLifetime(_) => {
+                    false
+                }
+                _ => true,
+            }
+        }
+
+        match kind {
+            NonterminalKind::Expr => {
+                token.can_begin_expr()
+                // This exception is here for backwards compatibility.
+                && !token.is_keyword(kw::Let)
+            }
+            NonterminalKind::Ty => token.can_begin_type(),
+            NonterminalKind::Ident => get_macro_ident(token).is_some(),
+            NonterminalKind::Literal => token.can_begin_literal_maybe_minus(),
+            NonterminalKind::Vis => match token.kind {
+                // The follow-set of :vis + "priv" keyword + interpolated
+                token::Comma | token::Ident(..) | token::Interpolated(..) => true,
+                _ => token.can_begin_type(),
+            },
+            NonterminalKind::Block => match token.kind {
+                token::OpenDelim(token::Brace) => true,
+                token::Interpolated(ref nt) => match **nt {
+                    token::NtItem(_)
+                    | token::NtPat(_)
+                    | token::NtTy(_)
+                    | token::NtIdent(..)
+                    | token::NtMeta(_)
+                    | token::NtPath(_)
+                    | token::NtVis(_) => false, // none of these may start with '{'.
+                    _ => true,
+                },
+                _ => false,
+            },
+            NonterminalKind::Path | NonterminalKind::Meta => match token.kind {
+                token::ModSep | token::Ident(..) => true,
+                token::Interpolated(ref nt) => match **nt {
+                    token::NtPath(_) | token::NtMeta(_) => true,
+                    _ => may_be_ident(&nt),
+                },
+                _ => false,
+            },
+            NonterminalKind::Pat => match token.kind {
+                token::Ident(..) |                  // box, ref, mut, and other identifiers (can stricten)
+                token::OpenDelim(token::Paren) |    // tuple pattern
+                token::OpenDelim(token::Bracket) |  // slice pattern
+                token::BinOp(token::And) |          // reference
+                token::BinOp(token::Minus) |        // negative literal
+                token::AndAnd |                     // double reference
+                token::Literal(..) |                // literal
+                token::DotDot |                     // range pattern (future compat)
+                token::DotDotDot |                  // range pattern (future compat)
+                token::ModSep |                     // path
+                token::Lt |                         // path (UFCS constant)
+                token::BinOp(token::Shl) => true,   // path (double UFCS)
+                token::Interpolated(ref nt) => may_be_ident(nt),
+                _ => false,
+            },
+            NonterminalKind::Lifetime => match token.kind {
+                token::Lifetime(_) => true,
+                token::Interpolated(ref nt) => match **nt {
+                    token::NtLifetime(_) | token::NtTT(_) => true,
+                    _ => false,
+                },
+                _ => false,
+            },
+            NonterminalKind::TT | NonterminalKind::Item | NonterminalKind::Stmt => match token.kind
+            {
+                token::CloseDelim(_) => false,
+                _ => true,
+            },
+        }
+    }
+
+    pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, Nonterminal> {
+        // Any `Nonterminal` which stores its tokens (currently `NtItem` and `NtExpr`)
+        // needs to have them force-captured here.
+        // A `macro_rules!` invocation may pass a captured item/expr to a proc-macro,
+        // which requires having captured tokens available. Since we cannot determine
+        // in advance whether or not a proc-macro will be (transitively) invoked,
+        // we always capture tokens for any `Nonterminal` which needs them.
+        Ok(match kind {
+            NonterminalKind::Item => match self.collect_tokens(|this| this.parse_item())? {
+                (Some(mut item), tokens) => {
+                    // If we captured tokens during parsing (due to outer attributes),
+                    // use those.
+                    if item.tokens.is_none() {
+                        item.tokens = Some(tokens);
+                    }
+                    token::NtItem(item)
+                }
+                (None, _) => {
+                    return Err(self.struct_span_err(self.token.span, "expected an item keyword"));
+                }
+            },
+            NonterminalKind::Block => token::NtBlock(self.parse_block()?),
+            NonterminalKind::Stmt => match self.parse_stmt()? {
+                Some(s) => token::NtStmt(s),
+                None => return Err(self.struct_span_err(self.token.span, "expected a statement")),
+            },
+            NonterminalKind::Pat => token::NtPat(self.parse_pat(None)?),
+            NonterminalKind::Expr => {
+                let (mut expr, tokens) = self.collect_tokens(|this| this.parse_expr())?;
+                // If we captured tokens during parsing (due to outer attributes),
+                // use those.
+                if expr.tokens.is_none() {
+                    expr.tokens = Some(tokens);
+                }
+                token::NtExpr(expr)
+            }
+            NonterminalKind::Literal => token::NtLiteral(self.parse_literal_maybe_minus()?),
+            NonterminalKind::Ty => token::NtTy(self.parse_ty()?),
+            // this could be handled like a token, since it is one
+            NonterminalKind::Ident => {
+                if let Some((ident, is_raw)) = get_macro_ident(&self.token) {
+                    self.bump();
+                    token::NtIdent(ident, is_raw)
+                } else {
+                    let token_str = pprust::token_to_string(&self.token);
+                    let msg = &format!("expected ident, found {}", &token_str);
+                    return Err(self.struct_span_err(self.token.span, msg));
+                }
+            }
+            NonterminalKind::Path => token::NtPath(self.parse_path(PathStyle::Type)?),
+            NonterminalKind::Meta => token::NtMeta(P(self.parse_attr_item()?)),
+            NonterminalKind::TT => token::NtTT(self.parse_token_tree()),
+            NonterminalKind::Vis => token::NtVis(self.parse_visibility(FollowedByType::Yes)?),
+            NonterminalKind::Lifetime => {
+                if self.check_lifetime() {
+                    token::NtLifetime(self.expect_lifetime().ident)
+                } else {
+                    let token_str = pprust::token_to_string(&self.token);
+                    let msg = &format!("expected a lifetime, found `{}`", &token_str);
+                    return Err(self.struct_span_err(self.token.span, msg));
+                }
+            }
+        })
+    }
+}
+
+/// The token is an identifier, but not `_`.
+/// We prohibit passing `_` to macros expecting `ident` for now.
+fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
+    token.ident().filter(|(ident, _)| ident.name != kw::Underscore)
+}

src/librustc_parse/parser/path.rs

@@ -125,7 +125,7 @@ impl<'a> Parser<'a> {
     /// `a::b::C::<D>` (with disambiguator)
     /// `Fn(Args)` (without disambiguator)
     /// `Fn::(Args)` (with disambiguator)
-    pub fn parse_path(&mut self, style: PathStyle) -> PResult<'a, Path> {
+    pub(super) fn parse_path(&mut self, style: PathStyle) -> PResult<'a, Path> {
         maybe_whole!(self, NtPath, |path| {
             if style == PathStyle::Mod && path.segments.iter().any(|segment| segment.args.is_some())
             {

src/librustc_parse/parser/stmt.rs

@@ -21,7 +21,7 @@ use std::mem;
 impl<'a> Parser<'a> {
     /// Parses a statement. This stops just before trailing semicolons on everything but items.
     /// e.g., a `StmtKind::Semi` parses to a `StmtKind::Expr`, leaving the trailing `;` unconsumed.
-    pub fn parse_stmt(&mut self) -> PResult<'a, Option<Stmt>> {
+    pub(super) fn parse_stmt(&mut self) -> PResult<'a, Option<Stmt>> {
         Ok(self.parse_stmt_without_recovery().unwrap_or_else(|mut e| {
             e.emit();
             self.recover_stmt_(SemiColonMode::Break, BlockMode::Ignore);
@@ -247,7 +247,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a block. No inner attributes are allowed.
-    pub fn parse_block(&mut self) -> PResult<'a, P<Block>> {
+    pub(super) fn parse_block(&mut self) -> PResult<'a, P<Block>> {
         let (attrs, block) = self.parse_inner_attrs_and_block()?;
         if let [.., last] = &*attrs {
             self.error_on_forbidden_inner_attr(last.span, DEFAULT_INNER_ATTR_FORBIDDEN);

src/librustc_parse/parser/ty.rs

@@ -610,13 +610,13 @@ impl<'a> Parser<'a> {
         }
     }
 
-    pub fn check_lifetime(&mut self) -> bool {
+    pub(super) fn check_lifetime(&mut self) -> bool {
         self.expected_tokens.push(TokenType::Lifetime);
         self.token.is_lifetime()
     }
 
     /// Parses a single lifetime `'a` or panics.
-    pub fn expect_lifetime(&mut self) -> Lifetime {
+    pub(super) fn expect_lifetime(&mut self) -> Lifetime {
         if let Some(ident) = self.token.lifetime() {
             self.bump();
             Lifetime { ident, id: ast::DUMMY_NODE_ID }