From 760879bc88b2884275b59fc38e0c5b1a8632e4cd Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Thu, 18 Jan 2018 18:41:09 -0600 Subject: [PATCH] Allow `?` as a KleeneOp in the macro parser --- src/libsyntax/ext/tt/quoted.rs | 126 +++++++++++++++++++++------------ 1 file changed, 82 insertions(+), 44 deletions(-) diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index c55dfaba8f6b..f8ae8726bc1e 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -16,6 +16,7 @@ use symbol::keywords; use syntax_pos::{BytePos, Span, DUMMY_SP}; use tokenstream; +use std::iter::Peekable; use std::rc::Rc; /// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note @@ -78,6 +79,7 @@ pub enum KleeneOp { ZeroOrMore, /// Kleene plus (`+`) for one or more repetitions OneOrMore, + ZeroOrOne, } /// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)` @@ -183,7 +185,7 @@ pub fn parse( // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming // additional trees if need be. - let mut trees = input.trees(); + let mut trees = input.trees().peekable(); while let Some(tree) = trees.next() { let tree = parse_tree(tree, &mut trees, expect_matchers, sess); @@ -321,6 +323,34 @@ where } } +/// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return +/// `None`. +fn kleene_op(token: &token::Token) -> Option { + match *token { + token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), + token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), + token::Question => Some(KleeneOp::ZeroOrOne), + _ => None, + } +} + +/// Parse the next token tree of the input looking for a KleeneOp. Returns +/// +/// - Ok(Ok(op)) if the next token tree is a KleeneOp +/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp +/// - Err(span) if the next token tree is not a token +fn parse_kleene_op(input: &mut I, span: Span) -> Result, Span> + where I: Iterator, +{ + match input.next() { + Some(tokenstream::TokenTree::Token(span, tok)) => match kleene_op(&tok) { + Some(op) => Ok(Ok(op)), + None => Ok(Err((tok, span))), + } + tree => Err(tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span)), + } +} + /// Attempt to parse a single Kleene star, possibly with a separator. /// /// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the @@ -333,56 +363,64 @@ where /// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene /// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an /// error with the appropriate span is emitted to `sess` and a dummy value is returned. -fn parse_sep_and_kleene_op( - input: &mut I, - span: Span, - sess: &ParseSess, -) -> (Option, KleeneOp) -where - I: Iterator, +fn parse_sep_and_kleene_op(input: &mut Peekable, span: Span, sess: &ParseSess) + -> (Option, KleeneOp) + where I: Iterator, { - fn kleene_op(token: &token::Token) -> Option { - match *token { - token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), - token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), - _ => None, + // We basically look at two token trees here, denoted as #1 and #2 below + let span = match parse_kleene_op(input, span) { + // #1 is a `+` or `*` KleeneOp + // + // `?` is ambiguous: it could be a separator or a Kleene::ZeroOrOne, so we need to look + // ahead one more token to be sure. + Ok(Ok(op)) if op != KleeneOp::ZeroOrOne => return (None, op), + + // #1 is `?` token, but it could be a Kleene::ZeroOrOne without a separator or it could + // be a `?` separator followed by any Kleene operator. We need to look ahead 1 token to + // find out which. + Ok(Ok(op)) => { + // Lookahead at #2. If it is a KleenOp, then #1 is a separator. + let is_1_sep = if let Some(&tokenstream::TokenTree::Token(_, ref tok2)) = input.peek() { + kleene_op(tok2).is_some() + } else { + false + }; + + if is_1_sep { + // #1 is a separator and #2 should be a KleepeOp::* + // (N.B. We need to advance the input iterator.) + match parse_kleene_op(input, span) { + // #2 is a KleeneOp (this is the only valid option) :) + Ok(Ok(op)) => return (Some(token::Question), op), + + // #2 is a random token (this is an error) :( + Ok(Err((_, span))) => span, + + // #2 is not even a token at all :( + Err(span) => span, + } + } else { + // #2 is a random tree and #1 is KleeneOp::ZeroOrOne + return (None, op); + } } - } - // We attempt to look at the next two token trees in `input`. I will call the first #1 and the - // second #2. If #1 and #2 don't match a valid KleeneOp with/without separator, that is an - // error, and we should emit an error on the most specific span possible. - let span = match input.next() { - // #1 is a token - Some(tokenstream::TokenTree::Token(span, tok)) => match kleene_op(&tok) { - // #1 is a KleeneOp with no separator - Some(op) => return (None, op), + // #1 is a separator followed by #2, a KleeneOp + Ok(Err((tok, span))) => match parse_kleene_op(input, span) { + // #2 is a KleeneOp :D + Ok(Ok(op)) => return (Some(tok), op), - // #1 is not a KleeneOp, but may be a separator... need to look at #2 - None => match input.next() { - // #2 is a token - Some(tokenstream::TokenTree::Token(span, tok2)) => match kleene_op(&tok2) { - // #2 is a KleeneOp, so #1 must be a separator - Some(op) => return (Some(tok), op), + // #2 is a random token :( + Ok(Err((_, span))) => span, - // #2 is not a KleeneOp... error - None => span, - }, + // #2 is not a token at all :( + Err(span) => span, + } - // #2 is not a token at all... error - tree => tree.as_ref() - .map(tokenstream::TokenTree::span) - .unwrap_or(span), - }, - }, - - // #1 is not a token at all... error - tree => tree.as_ref() - .map(tokenstream::TokenTree::span) - .unwrap_or(span), + // #1 is not a token + Err(span) => span, }; - // Error... - sess.span_diagnostic.span_err(span, "expected `*` or `+`"); + sess.span_diagnostic.span_err(span, "expected one of: `*`, `+`, or `?`"); (None, KleeneOp::ZeroOrMore) }