Add expr, pat, ty and macro_stmts

crates/ra_mbe/src/lib.rs

@@ -44,7 +44,11 @@ pub use crate::syntax_bridge::{
     ast_to_token_tree,
     token_tree_to_ast_item_list,
     syntax_node_to_token_tree,
+    token_tree_to_expr,
+    token_tree_to_pat,
+    token_tree_to_ty,
     token_tree_to_macro_items,
+    token_tree_to_macro_stmts,
 };
 
 /// This struct contains AST for a single `macro_rules` definition. What might
@@ -450,6 +454,59 @@ MACRO_ITEMS@[0; 40)
         assert_expansion(&rules, "foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}");
     }
 
+    #[test]
+    fn test_tt_to_stmts() {
+        let rules = create_rules(
+            r#"
+        macro_rules! foo {
+            () => {
+                 let a = 0;
+                 a = 10 + 1;
+                 a
+            }
+        }
+"#,
+        );
+
+        let expanded = expand(&rules, "foo!{}");
+        let stmts = token_tree_to_macro_stmts(&expanded);
+
+        assert_eq!(
+            stmts.syntax().debug_dump().trim(),
+            r#"MACRO_STMTS@[0; 15)
+  LET_STMT@[0; 7)
+    LET_KW@[0; 3) "let"
+    BIND_PAT@[3; 4)
+      NAME@[3; 4)
+        IDENT@[3; 4) "a"
+    EQ@[4; 5) "="
+    LITERAL@[5; 6)
+      INT_NUMBER@[5; 6) "0"
+    SEMI@[6; 7) ";"
+  EXPR_STMT@[7; 14)
+    BIN_EXPR@[7; 13)
+      PATH_EXPR@[7; 8)
+        PATH@[7; 8)
+          PATH_SEGMENT@[7; 8)
+            NAME_REF@[7; 8)
+              IDENT@[7; 8) "a"
+      EQ@[8; 9) "="
+      BIN_EXPR@[9; 13)
+        LITERAL@[9; 11)
+          INT_NUMBER@[9; 11) "10"
+        PLUS@[11; 12) "+"
+        LITERAL@[12; 13)
+          INT_NUMBER@[12; 13) "1"
+    SEMI@[13; 14) ";"
+  EXPR_STMT@[14; 15)
+    PATH_EXPR@[14; 15)
+      PATH@[14; 15)
+        PATH_SEGMENT@[14; 15)
+          NAME_REF@[14; 15)
+            IDENT@[14; 15) "a""#,
+        );
+    }
+
     // The following tests are port from intellij-rust directly
     // https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt
 

crates/ra_mbe/src/syntax_bridge.rs

@@ -32,11 +32,11 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke
 
 // The following items are what `rustc` macro can be parsed into :
 // link: https://github.com/rust-lang/rust/blob/9ebf47851a357faa4cd97f4b1dc7835f6376e639/src/libsyntax/ext/expand.rs#L141
-// * Expr(P<ast::Expr>)
-// * Pat(P<ast::Pat>)
-// * Ty(P<ast::Ty>)
-// * Stmts(SmallVec<[ast::Stmt; 1]>)
-// * Items(SmallVec<[P<ast::Item>; 1]>)
+// * Expr(P<ast::Expr>)                     -> token_tree_to_expr
+// * Pat(P<ast::Pat>)                       -> token_tree_to_pat
+// * Ty(P<ast::Ty>)                         -> token_tree_to_ty
+// * Stmts(SmallVec<[ast::Stmt; 1]>)        -> token_tree_to_stmts
+// * Items(SmallVec<[P<ast::Item>; 1]>)     -> token_tree_to_items
 //
 // * TraitItems(SmallVec<[ast::TraitItem; 1]>)
 // * ImplItems(SmallVec<[ast::ImplItem; 1]>)
@@ -44,6 +44,42 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke
 //
 //
 
+/// Parses the token tree (result of macro expansion) to an expression
+pub fn token_tree_to_expr(tt: &tt::Subtree) -> TreeArc<ast::Expr> {
+    let token_source = SubtreeTokenSource::new(tt);
+    let mut tree_sink = TtTreeSink::new(token_source.querier());
+    ra_parser::parse_expr(&token_source, &mut tree_sink);
+    let syntax = tree_sink.inner.finish();
+    ast::Expr::cast(&syntax).unwrap().to_owned()
+}
+
+/// Parses the token tree (result of macro expansion) to a Pattern
+pub fn token_tree_to_pat(tt: &tt::Subtree) -> TreeArc<ast::Pat> {
+    let token_source = SubtreeTokenSource::new(tt);
+    let mut tree_sink = TtTreeSink::new(token_source.querier());
+    ra_parser::parse_pat(&token_source, &mut tree_sink);
+    let syntax = tree_sink.inner.finish();
+    ast::Pat::cast(&syntax).unwrap().to_owned()
+}
+
+/// Parses the token tree (result of macro expansion) to a Type
+pub fn token_tree_to_ty(tt: &tt::Subtree) -> TreeArc<ast::TypeRef> {
+    let token_source = SubtreeTokenSource::new(tt);
+    let mut tree_sink = TtTreeSink::new(token_source.querier());
+    ra_parser::parse_ty(&token_source, &mut tree_sink);
+    let syntax = tree_sink.inner.finish();
+    ast::TypeRef::cast(&syntax).unwrap().to_owned()
+}
+
+/// Parses the token tree (result of macro expansion) as a sequence of stmts
+pub fn token_tree_to_macro_stmts(tt: &tt::Subtree) -> TreeArc<ast::MacroStmts> {
+    let token_source = SubtreeTokenSource::new(tt);
+    let mut tree_sink = TtTreeSink::new(token_source.querier());
+    ra_parser::parse_macro_stmts(&token_source, &mut tree_sink);
+    let syntax = tree_sink.inner.finish();
+    ast::MacroStmts::cast(&syntax).unwrap().to_owned()
+}
+
 /// Parses the token tree (result of macro expansion) as a sequence of items
 pub fn token_tree_to_macro_items(tt: &tt::Subtree) -> TreeArc<ast::MacroItems> {
     let token_source = SubtreeTokenSource::new(tt);

crates/ra_parser/src/grammar.rs

@@ -55,6 +55,21 @@ pub(crate) fn macro_items(p: &mut Parser) {
     m.complete(p, MACRO_ITEMS);
 }
 
+pub(crate) fn macro_stmts(p: &mut Parser) {
+    let m = p.start();
+
+    while !p.at(EOF) {
+        if p.current() == SEMI {
+            p.bump();
+            continue;
+        }
+
+        expressions::stmt(p, expressions::StmtWithSemi::Optional);
+    }
+
+    m.complete(p, MACRO_STMTS);
+}
+
 pub(crate) fn path(p: &mut Parser) {
     paths::type_path(p);
 }
@@ -72,6 +87,11 @@ pub(crate) fn pattern(p: &mut Parser) {
 }
 
 pub(crate) fn stmt(p: &mut Parser, with_semi: bool) {
+    let with_semi = match with_semi {
+        true => expressions::StmtWithSemi::Yes,
+        false => expressions::StmtWithSemi::No,
+    };
+
     expressions::stmt(p, with_semi)
 }
 

crates/ra_parser/src/grammar/expressions.rs

@@ -4,6 +4,12 @@ pub(crate) use self::atom::match_arm_list;
 pub(super) use self::atom::{literal, LITERAL_FIRST};
 use super::*;
 
+pub(super) enum StmtWithSemi {
+    Yes,
+    No,
+    Optional,
+}
+
 const EXPR_FIRST: TokenSet = LHS_FIRST;
 
 pub(super) fn expr(p: &mut Parser) -> BlockLike {
@@ -48,7 +54,7 @@ fn is_expr_stmt_attr_allowed(kind: SyntaxKind) -> bool {
     }
 }
 
-pub(super) fn stmt(p: &mut Parser, with_semi: bool) {
+pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi) {
     // test block_items
     // fn a() { fn b() {} }
     let m = p.start();
@@ -111,13 +117,23 @@ pub(super) fn stmt(p: &mut Parser, with_semi: bool) {
         //     }
         //     test!{}
         // }
-        if with_semi {
-            if blocklike.is_block() {
-                p.eat(SEMI);
-            } else {
-                p.expect(SEMI);
+
+        match with_semi {
+            StmtWithSemi::Yes => {
+                if blocklike.is_block() {
+                    p.eat(SEMI);
+                } else {
+                    p.expect(SEMI);
+                }
+            }
+            StmtWithSemi::No => {}
+            StmtWithSemi::Optional => {
+                if p.at(SEMI) {
+                    p.eat(SEMI);
+                }
             }
         }
+
         m.complete(p, EXPR_STMT);
     }
 
@@ -128,7 +144,7 @@ pub(super) fn stmt(p: &mut Parser, with_semi: bool) {
     //     let c = 92;
     //     let d: i32 = 92;
     // }
-    fn let_stmt(p: &mut Parser, m: Marker, with_semi: bool) {
+    fn let_stmt(p: &mut Parser, m: Marker, with_semi: StmtWithSemi) {
         assert!(p.at(LET_KW));
         p.bump();
         patterns::pattern(p);
@@ -139,8 +155,16 @@ pub(super) fn stmt(p: &mut Parser, with_semi: bool) {
             expressions::expr(p);
         }
 
-        if with_semi {
-            p.expect(SEMI);
+        match with_semi {
+            StmtWithSemi::Yes => {
+                p.expect(SEMI);
+            }
+            StmtWithSemi::No => {}
+            StmtWithSemi::Optional => {
+                if p.at(SEMI) {
+                    p.eat(SEMI);
+                }
+            }
         }
         m.complete(p, LET_STMT);
     }
@@ -160,7 +184,7 @@ pub(crate) fn expr_block_contents(p: &mut Parser) {
             continue;
         }
 
-        stmt(p, true)
+        stmt(p, StmtWithSemi::Yes)
     }
 }
 

crates/ra_parser/src/lib.rs

@@ -102,6 +102,10 @@ pub fn parse_macro_items(token_source: &dyn TokenSource, tree_sink: &mut dyn Tre
     parse_from_tokens(token_source, tree_sink, grammar::macro_items);
 }
 
+pub fn parse_macro_stmts(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+    parse_from_tokens(token_source, tree_sink, grammar::macro_stmts);
+}
+
 /// A parsing function for a specific braced-block.
 pub struct Reparser(fn(&mut parser::Parser));