WIP: adding mark-cancelling for macro_rules

src/libsyntax/ast_util.rs

@@ -1010,6 +1010,16 @@ pub fn marksof(ctxt: SyntaxContext, stopname: Name, table: &SCTable) -> ~[Mrk] {
     }
 }
 
+/// Return the outer mark for a context with a mark at the outside.
+/// FAILS when outside is not a mark.
+pub fn mtwt_outer_mark(ctxt: SyntaxContext) -> Mrk {
+    let sctable = get_sctable();
+    match sctable.table[ctxt] {
+        ast::Mark(mrk,_) => mrk,
+        _ => fail!("can't retrieve outer mark when outside is not a mark")
+    }
+}
+
 /// Push a name... unless it matches the one on top, in which
 /// case pop and discard (so two of the same marks cancel)
 pub fn xorPush(marks: &mut ~[uint], mark: uint) {

src/libsyntax/ext/base.rs

@@ -152,8 +152,7 @@ pub fn syntax_expander_table() -> SyntaxEnv {
                                 pending_renames : @mut ~[]
                             }));
     syntax_expanders.insert(intern(&"macro_rules"),
-                            builtin_item_tt_no_ctxt(
-                                ext::tt::macro_rules::add_new_extension));
+                            @SE(IdentTT(ext::tt::macro_rules::add_new_extension, None)));
     syntax_expanders.insert(intern(&"fmt"),
                             builtin_normal_tt_no_ctxt(ext::fmt::expand_syntax_ext));
     syntax_expanders.insert(intern(&"format"),

src/libsyntax/ext/expand.rs

@@ -13,7 +13,8 @@ use ast::{Local, Ident, mac_invoc_tt};
 use ast::{item_mac, Mrk, Stmt_, StmtDecl, StmtMac, StmtExpr, StmtSemi};
 use ast::{token_tree};
 use ast;
-use ast_util::{new_rename, new_mark};
+use ast_util::{mtwt_outer_mark, new_rename, new_mark};
+use ast_util;
 use attr;
 use attr::AttrMetaMethods;
 use codemap;
@@ -1507,7 +1508,10 @@ pub fn renames_to_fold(renames : @mut ~[(ast::Ident,ast::Name)]) -> @AstFoldFns
 }
 
 // just a convenience:
-pub fn new_mark_folder(m : Mrk) -> @AstFoldFns { fun_to_ctxt_folder(@Marker{mark:m}) }
+pub fn new_mark_folder(m : Mrk) -> @AstFoldFns {
+    fun_to_ctxt_folder(@Marker{mark:m})
+}
+
 pub fn new_rename_folder(from : ast::Ident, to : ast::Name) -> @AstFoldFns {
     fun_to_ctxt_folder(@Renamer{from:from,to:to})
 }
@@ -1538,6 +1542,16 @@ pub fn replace_ctxts(expr : @ast::Expr, ctxt : SyntaxContext) -> @ast::Expr {
     fun_to_ctxt_folder(@Repainter{ctxt:ctxt}).fold_expr(expr)
 }
 
+// take the mark from the given ctxt (that has a mark at the outside),
+// and apply it to everything in the token trees, thereby cancelling
+// that mark.
+pub fn mtwt_cancel_outer_mark(tts: &[ast::token_tree], ctxt: ast::SyntaxContext)
+    -> ~[ast::token_tree] {
+    let outer_mark = mtwt_outer_mark(ctxt);
+    mark_tts(tts,outer_mark)
+}
+
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -1546,13 +1560,15 @@ mod test {
     use ast_util::{get_sctable, mtwt_marksof, mtwt_resolve, new_rename};
     use codemap;
     use codemap::Spanned;
+    use fold;
     use parse;
-    use parse::token::{gensym, intern, get_ident_interner, ident_to_str};
+    use parse::token::{fresh_mark, gensym, intern, get_ident_interner, ident_to_str};
+    use parse::token;
     use print::pprust;
     use std;
     use std::vec;
     use util::parser_testing::{string_to_crate, string_to_crate_and_sess, string_to_item};
-    use util::parser_testing::{string_to_pat, strs_to_idents};
+    use util::parser_testing::{string_to_pat, string_to_tts, strs_to_idents};
     use visit;
 
     // make sure that fail! is present
@@ -1651,6 +1667,28 @@ mod test {
         }
     }
 
+    #[test] fn cancel_outer_mark_test(){
+        let invalid_name = token::special_idents::invalid.name;
+        let ident_str = @"x";
+        let tts = string_to_tts(ident_str);
+        let fm = fresh_mark();
+        let marked_once = fold::fold_tts(tts,new_mark_folder(fm) as @fold::ast_fold);
+        assert_eq!(marked_once.len(),1);
+        let marked_once_ctxt =
+            match marked_once[0] {
+                ast::tt_tok(_,token::IDENT(id,_)) => id.ctxt,
+                _ => fail!(fmt!("unexpected shape for marked tts: %?",marked_once[0]))
+            };
+        assert_eq!(mtwt_marksof(marked_once_ctxt,invalid_name),~[fm]);
+        let remarked = mtwt_cancel_outer_mark(marked_once,marked_once_ctxt);
+        assert_eq!(remarked.len(),1);
+        match remarked[0] {
+            ast::tt_tok(_,token::IDENT(id,_)) =>
+            assert_eq!(mtwt_marksof(id.ctxt,invalid_name),~[]),
+            _ => fail!(fmt!("unexpected shape for marked tts: %?",remarked[0]))
+        }
+    }
+
     #[test]
     fn renaming () {
         let item_ast = string_to_crate(@"fn f() -> int { a }");

src/libsyntax/ext/tt/macro_rules.rs

@@ -14,6 +14,7 @@ use ast;
 use codemap::{Span, Spanned, dummy_sp};
 use ext::base::{ExtCtxt, MacResult, MRAny, MRDef, MacroDef, NormalTT};
 use ext::base;
+use ext::expand;
 use ext::tt::macro_parser::{error};
 use ext::tt::macro_parser::{named_match, matched_seq, matched_nonterminal};
 use ext::tt::macro_parser::{parse, parse_or_else, success, failure};
@@ -29,8 +30,10 @@ use print;
 pub fn add_new_extension(cx: @ExtCtxt,
                          sp: Span,
                          name: Ident,
-                         arg: ~[ast::token_tree])
+                         arg: ~[ast::token_tree],
+                         stx_ctxt: ast::SyntaxContext)
                       -> base::MacResult {
+    let arg = expand::mtwt_cancel_outer_mark(arg,stx_ctxt);
     // Wrap a matcher_ in a spanned to produce a matcher.
     // these spans won't matter, anyways
     fn ms(m: matcher_) -> matcher {