user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

libsyntax: Unconfigure tests during normal build

Browse source
This commit is contained in:
Vadim Petrochenkov 2019-08-02 00:26:40 +03:00
parent a332e224a3
commit 310b9fc760
22 changed files with 1277 additions and 1303 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/libsyntax/ast.rs
View file

@ -27,6 +27,9 @@ use std::fmt;
pub use rustc_target::abi::FloatTy;
#[cfg(test)]
mod tests;
#[derive(Clone, RustcEncodable, RustcDecodable, Copy)]
pub struct Label {
pub ident: Ident,
@ -2432,15 +2435,3 @@ impl ForeignItemKind {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
// Are ASTs encodable?
#[test]
fn check_asts_encodable() {
fn assert_encodable<T: rustc_serialize::Encodable>() {}
assert_encodable::<Crate>();
}
}

8
src/libsyntax/ast/tests.rs Normal file
View file

@ -0,0 +1,8 @@
use super::*;
// Are ASTs encodable?
#[test]
fn check_asts_encodable() {
fn assert_encodable<T: rustc_serialize::Encodable>() {}
assert_encodable::<Crate>();
}

8
src/libsyntax/lib.rs
View file

@ -33,6 +33,9 @@ pub use rustc_data_structures::thin_vec::ThinVec;
use ast::AttrId;
use syntax_pos::edition::Edition;
#[cfg(test)]
mod tests;
const MACRO_ARGUMENTS: Option<&'static str> = Some("macro arguments");
// A variant of 'try!' that panics on an Err. This is used as a crutch on the
@ -132,8 +135,6 @@ pub mod util {
pub mod lev_distance;
pub mod node_count;
pub mod parser;
#[cfg(test)]
pub mod parser_testing;
pub mod map_in_place;
}
@ -183,7 +184,4 @@ pub mod ext {
pub mod early_buffered_lints;
#[cfg(test)]
mod test_snippet;
__build_diagnostic_array! { libsyntax, DIAGNOSTICS }

77
src/libsyntax/mut_visit.rs
View file

@ -22,6 +22,9 @@ use rustc_data_structures::sync::Lrc;
use std::ops::DerefMut;
use std::{panic, process, ptr};
#[cfg(test)]
mod tests;
pub trait ExpectOne<A: Array> {
fn expect_one(self, err: &'static str) -> A::Item;
}
@ -1255,77 +1258,3 @@ pub fn noop_visit_vis<T: MutVisitor>(Spanned { node, span }: &mut Visibility, vi
}
vis.visit_span(span);
}
#[cfg(test)]
mod tests {
use crate::ast::{self, Ident};
use crate::util::parser_testing::{string_to_crate, matches_codepattern};
use crate::print::pprust;
use crate::mut_visit;
use crate::with_default_globals;
use super::*;
// this version doesn't care about getting comments or docstrings in.
fn fake_print_crate(s: &mut pprust::State<'_>,
krate: &ast::Crate) {
s.print_mod(&krate.module, &krate.attrs)
}
// change every identifier to "zz"
struct ToZzIdentMutVisitor;
impl MutVisitor for ToZzIdentMutVisitor {
fn visit_ident(&mut self, ident: &mut ast::Ident) {
*ident = Ident::from_str("zz");
}
fn visit_mac(&mut self, mac: &mut ast::Mac) {
mut_visit::noop_visit_mac(mac, self)
}
}
// maybe add to expand.rs...
macro_rules! assert_pred {
($pred:expr, $predname:expr, $a:expr , $b:expr) => (
{
let pred_val = $pred;
let a_val = $a;
let b_val = $b;
if !(pred_val(&a_val, &b_val)) {
panic!("expected args satisfying {}, got {} and {}",
$predname, a_val, b_val);
}
}
)
}
// make sure idents get transformed everywhere
#[test] fn ident_transformation () {
with_default_globals(|| {
let mut zz_visitor = ToZzIdentMutVisitor;
let mut krate = string_to_crate(
"#[a] mod b {fn c (d : e, f : g) {h!(i,j,k);l;m}}".to_string());
zz_visitor.visit_crate(&mut krate);
assert_pred!(
matches_codepattern,
"matches_codepattern",
pprust::to_string(|s| fake_print_crate(s, &krate)),
"#[zz]mod zz{fn zz(zz:zz,zz:zz){zz!(zz,zz,zz);zz;zz}}".to_string());
})
}
// even inside macro defs....
#[test] fn ident_transformation_in_defs () {
with_default_globals(|| {
let mut zz_visitor = ToZzIdentMutVisitor;
let mut krate = string_to_crate(
"macro_rules! a {(b $c:expr $(d $e:token)f+ => \
(g $(d $d $e)+))} ".to_string());
zz_visitor.visit_crate(&mut krate);
assert_pred!(
matches_codepattern,
"matches_codepattern",
pprust::to_string(|s| fake_print_crate(s, &krate)),
"macro_rules! zz{(zz$zz:zz$(zz $zz:zz)zz+=>(zz$(zz$zz$zz)+))}".to_string());
})
}
}

71
src/libsyntax/mut_visit/tests.rs Normal file
View file

@ -0,0 +1,71 @@
use super::*;
use crate::ast::{self, Ident};
use crate::tests::{string_to_crate, matches_codepattern};
use crate::print::pprust;
use crate::mut_visit;
use crate::with_default_globals;
// this version doesn't care about getting comments or docstrings in.
fn fake_print_crate(s: &mut pprust::State<'_>,
krate: &ast::Crate) {
s.print_mod(&krate.module, &krate.attrs)
}
// change every identifier to "zz"
struct ToZzIdentMutVisitor;
impl MutVisitor for ToZzIdentMutVisitor {
fn visit_ident(&mut self, ident: &mut ast::Ident) {
*ident = Ident::from_str("zz");
}
fn visit_mac(&mut self, mac: &mut ast::Mac) {
mut_visit::noop_visit_mac(mac, self)
}
}
// maybe add to expand.rs...
macro_rules! assert_pred {
($pred:expr, $predname:expr, $a:expr , $b:expr) => (
{
let pred_val = $pred;
let a_val = $a;
let b_val = $b;
if !(pred_val(&a_val, &b_val)) {
panic!("expected args satisfying {}, got {} and {}",
$predname, a_val, b_val);
}
}
)
}
// make sure idents get transformed everywhere
#[test] fn ident_transformation () {
with_default_globals(|| {
let mut zz_visitor = ToZzIdentMutVisitor;
let mut krate = string_to_crate(
"#[a] mod b {fn c (d : e, f : g) {h!(i,j,k);l;m}}".to_string());
zz_visitor.visit_crate(&mut krate);
assert_pred!(
matches_codepattern,
"matches_codepattern",
pprust::to_string(|s| fake_print_crate(s, &krate)),
"#[zz]mod zz{fn zz(zz:zz,zz:zz){zz!(zz,zz,zz);zz;zz}}".to_string());
})
}
// even inside macro defs....
#[test] fn ident_transformation_in_defs () {
with_default_globals(|| {
let mut zz_visitor = ToZzIdentMutVisitor;
let mut krate = string_to_crate(
"macro_rules! a {(b $c:expr $(d $e:token)f+ => \
(g $(d $d $e)+))} ".to_string());
zz_visitor.visit_crate(&mut krate);
assert_pred!(
matches_codepattern,
"matches_codepattern",
pprust::to_string(|s| fake_print_crate(s, &krate)),
"macro_rules! zz{(zz$zz:zz$(zz $zz:zz)zz+=>(zz$(zz$zz$zz)+))}".to_string());
})
}

54
src/libsyntax/parse/lexer/comments.rs
View file

@ -9,6 +9,9 @@ use syntax_pos::{BytePos, CharPos, Pos, FileName};
use std::usize;
#[cfg(test)]
mod tests;
#[derive(Clone, Copy, PartialEq, Debug)]
pub enum CommentStyle {
/// No code on either side of each line of the comment
@ -249,54 +252,3 @@ pub fn gather_comments(sess: &ParseSess, path: FileName, src: String) -> Vec<Com
comments
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_block_doc_comment_1() {
let comment = "/**\n * Test \n ** Test\n * Test\n*/";
let stripped = strip_doc_comment_decoration(comment);
assert_eq!(stripped, " Test \n* Test\n Test");
}
#[test]
fn test_block_doc_comment_2() {
let comment = "/**\n * Test\n * Test\n*/";
let stripped = strip_doc_comment_decoration(comment);
assert_eq!(stripped, " Test\n Test");
}
#[test]
fn test_block_doc_comment_3() {
let comment = "/**\n let a: *i32;\n *a = 5;\n*/";
let stripped = strip_doc_comment_decoration(comment);
assert_eq!(stripped, " let a: *i32;\n *a = 5;");
}
#[test]
fn test_block_doc_comment_4() {
let comment = "/*******************\n test\n *********************/";
let stripped = strip_doc_comment_decoration(comment);
assert_eq!(stripped, " test");
}
#[test]
fn test_line_doc_comment() {
let stripped = strip_doc_comment_decoration("/// test");
assert_eq!(stripped, " test");
let stripped = strip_doc_comment_decoration("///! test");
assert_eq!(stripped, " test");
let stripped = strip_doc_comment_decoration("// test");
assert_eq!(stripped, " test");
let stripped = strip_doc_comment_decoration("// test");
assert_eq!(stripped, " test");
let stripped = strip_doc_comment_decoration("///test");
assert_eq!(stripped, "test");
let stripped = strip_doc_comment_decoration("///!test");
assert_eq!(stripped, "test");
let stripped = strip_doc_comment_decoration("//test");
assert_eq!(stripped, "test");
}
}

47
src/libsyntax/parse/lexer/comments/tests.rs Normal file
View file

@ -0,0 +1,47 @@
use super::*;
#[test]
fn test_block_doc_comment_1() {
let comment = "/**\n * Test \n ** Test\n * Test\n*/";
let stripped = strip_doc_comment_decoration(comment);
assert_eq!(stripped, " Test \n* Test\n Test");
}
#[test]
fn test_block_doc_comment_2() {
let comment = "/**\n * Test\n * Test\n*/";
let stripped = strip_doc_comment_decoration(comment);
assert_eq!(stripped, " Test\n Test");
}
#[test]
fn test_block_doc_comment_3() {
let comment = "/**\n let a: *i32;\n *a = 5;\n*/";
let stripped = strip_doc_comment_decoration(comment);
assert_eq!(stripped, " let a: *i32;\n *a = 5;");
}
#[test]
fn test_block_doc_comment_4() {
let comment = "/*******************\n test\n *********************/";
let stripped = strip_doc_comment_decoration(comment);
assert_eq!(stripped, " test");
}
#[test]
fn test_line_doc_comment() {
let stripped = strip_doc_comment_decoration("/// test");
assert_eq!(stripped, " test");
let stripped = strip_doc_comment_decoration("///! test");
assert_eq!(stripped, " test");
let stripped = strip_doc_comment_decoration("// test");
assert_eq!(stripped, " test");
let stripped = strip_doc_comment_decoration("// test");
assert_eq!(stripped, " test");
let stripped = strip_doc_comment_decoration("///test");
assert_eq!(stripped, "test");
let stripped = strip_doc_comment_decoration("///!test");
assert_eq!(stripped, "test");
let stripped = strip_doc_comment_decoration("//test");
assert_eq!(stripped, "test");
}

262
src/libsyntax/parse/lexer/mod.rs
View file

@ -15,6 +15,9 @@ use std::convert::TryInto;
use rustc_data_structures::sync::Lrc;
use log::debug;
#[cfg(test)]
mod tests;
pub mod comments;
mod tokentrees;
mod unicode_chars;
@ -777,262 +780,3 @@ fn is_block_doc_comment(s: &str) -> bool {
debug!("is {:?} a doc comment? {}", s, res);
res
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ast::CrateConfig;
use crate::symbol::Symbol;
use crate::source_map::{SourceMap, FilePathMapping};
use crate::feature_gate::UnstableFeatures;
use crate::parse::token;
use crate::diagnostics::plugin::ErrorMap;
use crate::with_default_globals;
use std::io;
use std::path::PathBuf;
use syntax_pos::{BytePos, Span, NO_EXPANSION, edition::Edition};
use rustc_data_structures::fx::{FxHashSet, FxHashMap};
use rustc_data_structures::sync::{Lock, Once};
fn mk_sess(sm: Lrc<SourceMap>) -> ParseSess {
let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()),
Some(sm.clone()),
false,
false,
false);
ParseSess {
span_diagnostic: errors::Handler::with_emitter(true, None, Box::new(emitter)),
unstable_features: UnstableFeatures::from_environment(),
config: CrateConfig::default(),
included_mod_stack: Lock::new(Vec::new()),
source_map: sm,
missing_fragment_specifiers: Lock::new(FxHashSet::default()),
raw_identifier_spans: Lock::new(Vec::new()),
registered_diagnostics: Lock::new(ErrorMap::new()),
buffered_lints: Lock::new(vec![]),
edition: Edition::from_session(),
ambiguous_block_expr_parse: Lock::new(FxHashMap::default()),
param_attr_spans: Lock::new(Vec::new()),
let_chains_spans: Lock::new(Vec::new()),
async_closure_spans: Lock::new(Vec::new()),
injected_crate_name: Once::new(),
}
}
// open a string reader for the given string
fn setup<'a>(sm: &SourceMap,
sess: &'a ParseSess,
teststr: String)
-> StringReader<'a> {
let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr);
StringReader::new(sess, sf, None)
}
#[test]
fn t1() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
let mut string_reader = setup(&sm,
&sh,
"/* my source file */ fn main() { println!(\"zebra\"); }\n"
.to_string());
assert_eq!(string_reader.next_token(), token::Comment);
assert_eq!(string_reader.next_token(), token::Whitespace);
let tok1 = string_reader.next_token();
let tok2 = Token::new(
mk_ident("fn"),
Span::new(BytePos(21), BytePos(23), NO_EXPANSION),
);
assert_eq!(tok1.kind, tok2.kind);
assert_eq!(tok1.span, tok2.span);
assert_eq!(string_reader.next_token(), token::Whitespace);
// read another token:
let tok3 = string_reader.next_token();
assert_eq!(string_reader.pos.clone(), BytePos(28));
let tok4 = Token::new(
mk_ident("main"),
Span::new(BytePos(24), BytePos(28), NO_EXPANSION),
);
assert_eq!(tok3.kind, tok4.kind);
assert_eq!(tok3.span, tok4.span);
assert_eq!(string_reader.next_token(), token::OpenDelim(token::Paren));
assert_eq!(string_reader.pos.clone(), BytePos(29))
})
}
// check that the given reader produces the desired stream
// of tokens (stop checking after exhausting the expected vec)
fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<TokenKind>) {
for expected_tok in &expected {
assert_eq!(&string_reader.next_token(), expected_tok);
}
}
// make the identifier by looking up the string in the interner
fn mk_ident(id: &str) -> TokenKind {
token::Ident(Symbol::intern(id), false)
}
fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind {
TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern))
}
#[test]
fn doublecolonparsing() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
check_tokenization(setup(&sm, &sh, "a b".to_string()),
vec![mk_ident("a"), token::Whitespace, mk_ident("b")]);
})
}
#[test]
fn dcparsing_2() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
check_tokenization(setup(&sm, &sh, "a::b".to_string()),
vec![mk_ident("a"), token::ModSep, mk_ident("b")]);
})
}
#[test]
fn dcparsing_3() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
check_tokenization(setup(&sm, &sh, "a ::b".to_string()),
vec![mk_ident("a"), token::Whitespace, token::ModSep, mk_ident("b")]);
})
}
#[test]
fn dcparsing_4() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
check_tokenization(setup(&sm, &sh, "a:: b".to_string()),
vec![mk_ident("a"), token::ModSep, token::Whitespace, mk_ident("b")]);
})
}
#[test]
fn character_a() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "'a'".to_string()).next_token(),
mk_lit(token::Char, "a", None));
})
}
#[test]
fn character_space() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "' '".to_string()).next_token(),
mk_lit(token::Char, " ", None));
})
}
#[test]
fn character_escaped() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "'\\n'".to_string()).next_token(),
mk_lit(token::Char, "\\n", None));
})
}
#[test]
fn lifetime_name() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "'abc".to_string()).next_token(),
token::Lifetime(Symbol::intern("'abc")));
})
}
#[test]
fn raw_string() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token(),
mk_lit(token::StrRaw(3), "\"#a\\b\x00c\"", None));
})
}
#[test]
fn literal_suffixes() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
macro_rules! test {
($input: expr, $tok_type: ident, $tok_contents: expr) => {{
assert_eq!(setup(&sm, &sh, format!("{}suffix", $input)).next_token(),
mk_lit(token::$tok_type, $tok_contents, Some("suffix")));
// with a whitespace separator:
assert_eq!(setup(&sm, &sh, format!("{} suffix", $input)).next_token(),
mk_lit(token::$tok_type, $tok_contents, None));
}}
}
test!("'a'", Char, "a");
test!("b'a'", Byte, "a");
test!("\"a\"", Str, "a");
test!("b\"a\"", ByteStr, "a");
test!("1234", Integer, "1234");
test!("0b101", Integer, "0b101");
test!("0xABC", Integer, "0xABC");
test!("1.0", Float, "1.0");
test!("1.0e10", Float, "1.0e10");
assert_eq!(setup(&sm, &sh, "2us".to_string()).next_token(),
mk_lit(token::Integer, "2", Some("us")));
assert_eq!(setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token(),
mk_lit(token::StrRaw(3), "raw", Some("suffix")));
assert_eq!(setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token(),
mk_lit(token::ByteStrRaw(3), "raw", Some("suffix")));
})
}
#[test]
fn line_doc_comments() {
assert!(is_doc_comment("///"));
assert!(is_doc_comment("/// blah"));
assert!(!is_doc_comment("////"));
}
#[test]
fn nested_block_comments() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
let mut lexer = setup(&sm, &sh, "/* /* */ */'a'".to_string());
assert_eq!(lexer.next_token(), token::Comment);
assert_eq!(lexer.next_token(), mk_lit(token::Char, "a", None));
})
}
#[test]
fn crlf_comments() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
let mut lexer = setup(&sm, &sh, "// test\r\n/// test\r\n".to_string());
let comment = lexer.next_token();
assert_eq!(comment.kind, token::Comment);
assert_eq!((comment.span.lo(), comment.span.hi()), (BytePos(0), BytePos(7)));
assert_eq!(lexer.next_token(), token::Whitespace);
assert_eq!(lexer.next_token(), token::DocComment(Symbol::intern("/// test")));
})
}
}

255
src/libsyntax/parse/lexer/tests.rs Normal file
View file

@ -0,0 +1,255 @@
use super::*;
use crate::ast::CrateConfig;
use crate::symbol::Symbol;
use crate::source_map::{SourceMap, FilePathMapping};
use crate::feature_gate::UnstableFeatures;
use crate::parse::token;
use crate::diagnostics::plugin::ErrorMap;
use crate::with_default_globals;
use std::io;
use std::path::PathBuf;
use syntax_pos::{BytePos, Span, NO_EXPANSION, edition::Edition};
use rustc_data_structures::fx::{FxHashSet, FxHashMap};
use rustc_data_structures::sync::{Lock, Once};
fn mk_sess(sm: Lrc<SourceMap>) -> ParseSess {
let emitter = errors::emitter::EmitterWriter::new(Box::new(io::sink()),
Some(sm.clone()),
false,
false,
false);
ParseSess {
span_diagnostic: errors::Handler::with_emitter(true, None, Box::new(emitter)),
unstable_features: UnstableFeatures::from_environment(),
config: CrateConfig::default(),
included_mod_stack: Lock::new(Vec::new()),
source_map: sm,
missing_fragment_specifiers: Lock::new(FxHashSet::default()),
raw_identifier_spans: Lock::new(Vec::new()),
registered_diagnostics: Lock::new(ErrorMap::new()),
buffered_lints: Lock::new(vec![]),
edition: Edition::from_session(),
ambiguous_block_expr_parse: Lock::new(FxHashMap::default()),
param_attr_spans: Lock::new(Vec::new()),
let_chains_spans: Lock::new(Vec::new()),
async_closure_spans: Lock::new(Vec::new()),
injected_crate_name: Once::new(),
}
}
// open a string reader for the given string
fn setup<'a>(sm: &SourceMap,
sess: &'a ParseSess,
teststr: String)
-> StringReader<'a> {
let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr);
StringReader::new(sess, sf, None)
}
#[test]
fn t1() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
let mut string_reader = setup(&sm,
&sh,
"/* my source file */ fn main() { println!(\"zebra\"); }\n"
.to_string());
assert_eq!(string_reader.next_token(), token::Comment);
assert_eq!(string_reader.next_token(), token::Whitespace);
let tok1 = string_reader.next_token();
let tok2 = Token::new(
mk_ident("fn"),
Span::new(BytePos(21), BytePos(23), NO_EXPANSION),
);
assert_eq!(tok1.kind, tok2.kind);
assert_eq!(tok1.span, tok2.span);
assert_eq!(string_reader.next_token(), token::Whitespace);
// read another token:
let tok3 = string_reader.next_token();
assert_eq!(string_reader.pos.clone(), BytePos(28));
let tok4 = Token::new(
mk_ident("main"),
Span::new(BytePos(24), BytePos(28), NO_EXPANSION),
);
assert_eq!(tok3.kind, tok4.kind);
assert_eq!(tok3.span, tok4.span);
assert_eq!(string_reader.next_token(), token::OpenDelim(token::Paren));
assert_eq!(string_reader.pos.clone(), BytePos(29))
})
}
// check that the given reader produces the desired stream
// of tokens (stop checking after exhausting the expected vec)
fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<TokenKind>) {
for expected_tok in &expected {
assert_eq!(&string_reader.next_token(), expected_tok);
}
}
// make the identifier by looking up the string in the interner
fn mk_ident(id: &str) -> TokenKind {
token::Ident(Symbol::intern(id), false)
}
fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind {
TokenKind::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern))
}
#[test]
fn doublecolonparsing() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
check_tokenization(setup(&sm, &sh, "a b".to_string()),
vec![mk_ident("a"), token::Whitespace, mk_ident("b")]);
})
}
#[test]
fn dcparsing_2() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
check_tokenization(setup(&sm, &sh, "a::b".to_string()),
vec![mk_ident("a"), token::ModSep, mk_ident("b")]);
})
}
#[test]
fn dcparsing_3() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
check_tokenization(setup(&sm, &sh, "a ::b".to_string()),
vec![mk_ident("a"), token::Whitespace, token::ModSep, mk_ident("b")]);
})
}
#[test]
fn dcparsing_4() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
check_tokenization(setup(&sm, &sh, "a:: b".to_string()),
vec![mk_ident("a"), token::ModSep, token::Whitespace, mk_ident("b")]);
})
}
#[test]
fn character_a() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "'a'".to_string()).next_token(),
mk_lit(token::Char, "a", None));
})
}
#[test]
fn character_space() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "' '".to_string()).next_token(),
mk_lit(token::Char, " ", None));
})
}
#[test]
fn character_escaped() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "'\\n'".to_string()).next_token(),
mk_lit(token::Char, "\\n", None));
})
}
#[test]
fn lifetime_name() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "'abc".to_string()).next_token(),
token::Lifetime(Symbol::intern("'abc")));
})
}
#[test]
fn raw_string() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
assert_eq!(setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token(),
mk_lit(token::StrRaw(3), "\"#a\\b\x00c\"", None));
})
}
#[test]
fn literal_suffixes() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
macro_rules! test {
($input: expr, $tok_type: ident, $tok_contents: expr) => {{
assert_eq!(setup(&sm, &sh, format!("{}suffix", $input)).next_token(),
mk_lit(token::$tok_type, $tok_contents, Some("suffix")));
// with a whitespace separator:
assert_eq!(setup(&sm, &sh, format!("{} suffix", $input)).next_token(),
mk_lit(token::$tok_type, $tok_contents, None));
}}
}
test!("'a'", Char, "a");
test!("b'a'", Byte, "a");
test!("\"a\"", Str, "a");
test!("b\"a\"", ByteStr, "a");
test!("1234", Integer, "1234");
test!("0b101", Integer, "0b101");
test!("0xABC", Integer, "0xABC");
test!("1.0", Float, "1.0");
test!("1.0e10", Float, "1.0e10");
assert_eq!(setup(&sm, &sh, "2us".to_string()).next_token(),
mk_lit(token::Integer, "2", Some("us")));
assert_eq!(setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token(),
mk_lit(token::StrRaw(3), "raw", Some("suffix")));
assert_eq!(setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token(),
mk_lit(token::ByteStrRaw(3), "raw", Some("suffix")));
})
}
#[test]
fn line_doc_comments() {
assert!(is_doc_comment("///"));
assert!(is_doc_comment("/// blah"));
assert!(!is_doc_comment("////"));
}
#[test]
fn nested_block_comments() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
let mut lexer = setup(&sm, &sh, "/* /* */ */'a'".to_string());
assert_eq!(lexer.next_token(), token::Comment);
assert_eq!(lexer.next_token(), mk_lit(token::Char, "a", None));
})
}
#[test]
fn crlf_comments() {
with_default_globals(|| {
let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
let sh = mk_sess(sm.clone());
let mut lexer = setup(&sm, &sh, "// test\r\n/// test\r\n".to_string());
let comment = lexer.next_token();
assert_eq!(comment.kind, token::Comment);
assert_eq!((comment.span.lo(), comment.span.hi()), (BytePos(0), BytePos(7)));
assert_eq!(lexer.next_token(), token::Whitespace);
assert_eq!(lexer.next_token(), token::DocComment(Symbol::intern("/// test")));
})
}

296
src/libsyntax/parse/mod.rs
View file

@ -22,7 +22,8 @@ use std::borrow::Cow;
use std::path::{Path, PathBuf};
use std::str;
pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>;
#[cfg(test)]
mod tests;
#[macro_use]
pub mod parser;
@ -35,6 +36,8 @@ crate mod diagnostics;
crate mod literal;
crate mod unescape_error_reporting;
pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>;
/// Info about a parsing session.
pub struct ParseSess {
pub span_diagnostic: Handler,
@ -389,294 +392,3 @@ impl SeqSep {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ast::{self, Name, PatKind};
use crate::attr::first_attr_value_str_by_name;
use crate::ptr::P;
use crate::parse::token::Token;
use crate::print::pprust::item_to_string;
use crate::symbol::{kw, sym};
use crate::tokenstream::{DelimSpan, TokenTree};
use crate::util::parser_testing::string_to_stream;
use crate::util::parser_testing::{string_to_expr, string_to_item};
use crate::with_default_globals;
use syntax_pos::{Span, BytePos, Pos, NO_EXPANSION};
/// Parses an item.
///
/// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and `Err`
/// when a syntax error occurred.
fn parse_item_from_source_str(name: FileName, source: String, sess: &ParseSess)
-> PResult<'_, Option<P<ast::Item>>> {
new_parser_from_source_str(sess, name, source).parse_item()
}
// produce a syntax_pos::span
fn sp(a: u32, b: u32) -> Span {
Span::new(BytePos(a), BytePos(b), NO_EXPANSION)
}
#[should_panic]
#[test] fn bad_path_expr_1() {
with_default_globals(|| {
string_to_expr("::abc::def::return".to_string());
})
}
// check the token-tree-ization of macros
#[test]
fn string_to_tts_macro () {
with_default_globals(|| {
let tts: Vec<_> =
string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect();
let tts: &[TokenTree] = &tts[..];
match tts {
[
TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }),
TokenTree::Token(Token { kind: token::Not, .. }),
TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }),
TokenTree::Delimited(_, macro_delim, macro_tts)
]
if name_macro_rules == &sym::macro_rules && name_zip.as_str() == "zip" => {
let tts = &macro_tts.trees().collect::<Vec<_>>();
match &tts[..] {
[
TokenTree::Delimited(_, first_delim, first_tts),
TokenTree::Token(Token { kind: token::FatArrow, .. }),
TokenTree::Delimited(_, second_delim, second_tts),
]
if macro_delim == &token::Paren => {
let tts = &first_tts.trees().collect::<Vec<_>>();
match &tts[..] {
[
TokenTree::Token(Token { kind: token::Dollar, .. }),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }),
]
if first_delim == &token::Paren && name.as_str() == "a" => {},
_ => panic!("value 3: {:?} {:?}", first_delim, first_tts),
}
let tts = &second_tts.trees().collect::<Vec<_>>();
match &tts[..] {
[
TokenTree::Token(Token { kind: token::Dollar, .. }),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }),
]
if second_delim == &token::Paren && name.as_str() == "a" => {},
_ => panic!("value 4: {:?} {:?}", second_delim, second_tts),
}
},
_ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts),
}
},
_ => panic!("value: {:?}",tts),
}
})
}
#[test]
fn string_to_tts_1() {
with_default_globals(|| {
let tts = string_to_stream("fn a (b : i32) { b; }".to_string());
let expected = TokenStream::new(vec![
TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(),
TokenTree::token(token::Ident(Name::intern("a"), false), sp(3, 4)).into(),
TokenTree::Delimited(
DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
token::DelimToken::Paren,
TokenStream::new(vec![
TokenTree::token(token::Ident(Name::intern("b"), false), sp(6, 7)).into(),
TokenTree::token(token::Colon, sp(8, 9)).into(),
TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(),
]).into(),
).into(),
TokenTree::Delimited(
DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
token::DelimToken::Brace,
TokenStream::new(vec![
TokenTree::token(token::Ident(Name::intern("b"), false), sp(17, 18)).into(),
TokenTree::token(token::Semi, sp(18, 19)).into(),
]).into(),
).into()
]);
assert_eq!(tts, expected);
})
}
#[test] fn parse_use() {
with_default_globals(|| {
let use_s = "use foo::bar::baz;";
let vitem = string_to_item(use_s.to_string()).unwrap();
let vitem_s = item_to_string(&vitem);
assert_eq!(&vitem_s[..], use_s);
let use_s = "use foo::bar as baz;";
let vitem = string_to_item(use_s.to_string()).unwrap();
let vitem_s = item_to_string(&vitem);
assert_eq!(&vitem_s[..], use_s);
})
}
#[test] fn parse_extern_crate() {
with_default_globals(|| {
let ex_s = "extern crate foo;";
let vitem = string_to_item(ex_s.to_string()).unwrap();
let vitem_s = item_to_string(&vitem);
assert_eq!(&vitem_s[..], ex_s);
let ex_s = "extern crate foo as bar;";
let vitem = string_to_item(ex_s.to_string()).unwrap();
let vitem_s = item_to_string(&vitem);
assert_eq!(&vitem_s[..], ex_s);
})
}
fn get_spans_of_pat_idents(src: &str) -> Vec<Span> {
let item = string_to_item(src.to_string()).unwrap();
struct PatIdentVisitor {
spans: Vec<Span>
}
impl<'a> crate::visit::Visitor<'a> for PatIdentVisitor {
fn visit_pat(&mut self, p: &'a ast::Pat) {
match p.node {
PatKind::Ident(_ , ref spannedident, _) => {
self.spans.push(spannedident.span.clone());
}
_ => {
crate::visit::walk_pat(self, p);
}
}
}
}
let mut v = PatIdentVisitor { spans: Vec::new() };
crate::visit::walk_item(&mut v, &item);
return v.spans;
}
#[test] fn span_of_self_arg_pat_idents_are_correct() {
with_default_globals(|| {
let srcs = ["impl z { fn a (&self, &myarg: i32) {} }",
"impl z { fn a (&mut self, &myarg: i32) {} }",
"impl z { fn a (&'a self, &myarg: i32) {} }",
"impl z { fn a (self, &myarg: i32) {} }",
"impl z { fn a (self: Foo, &myarg: i32) {} }",
];
for &src in &srcs {
let spans = get_spans_of_pat_idents(src);
let (lo, hi) = (spans[0].lo(), spans[0].hi());
assert!("self" == &src[lo.to_usize()..hi.to_usize()],
"\"{}\" != \"self\". src=\"{}\"",
&src[lo.to_usize()..hi.to_usize()], src)
}
})
}
#[test] fn parse_exprs () {
with_default_globals(|| {
// just make sure that they parse....
string_to_expr("3 + 4".to_string());
string_to_expr("a::z.froob(b,&(987+3))".to_string());
})
}
#[test] fn attrs_fix_bug () {
with_default_globals(|| {
string_to_item("pub fn mk_file_writer(path: &Path, flags: &[FileFlag])
-> Result<Box<Writer>, String> {
#[cfg(windows)]
fn wb() -> c_int {
(O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int
}
#[cfg(unix)]
fn wb() -> c_int { O_WRONLY as c_int }
let mut fflags: c_int = wb();
}".to_string());
})
}
#[test] fn crlf_doc_comments() {
with_default_globals(|| {
let sess = ParseSess::new(FilePathMapping::empty());
let name_1 = FileName::Custom("crlf_source_1".to_string());
let source = "/// doc comment\r\nfn foo() {}".to_string();
let item = parse_item_from_source_str(name_1, source, &sess)
.unwrap().unwrap();
let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap();
assert_eq!(doc.as_str(), "/// doc comment");
let name_2 = FileName::Custom("crlf_source_2".to_string());
let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string();
let item = parse_item_from_source_str(name_2, source, &sess)
.unwrap().unwrap();
let docs = item.attrs.iter().filter(|a| a.path == sym::doc)
.map(|a| a.value_str().unwrap().to_string()).collect::<Vec<_>>();
let b: &[_] = &["/// doc comment".to_string(), "/// line 2".to_string()];
assert_eq!(&docs[..], b);
let name_3 = FileName::Custom("clrf_source_3".to_string());
let source = "/** doc comment\r\n * with CRLF */\r\nfn foo() {}".to_string();
let item = parse_item_from_source_str(name_3, source, &sess).unwrap().unwrap();
let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap();
assert_eq!(doc.as_str(), "/** doc comment\n * with CRLF */");
});
}
#[test]
fn ttdelim_span() {
fn parse_expr_from_source_str(
name: FileName, source: String, sess: &ParseSess
) -> PResult<'_, P<ast::Expr>> {
new_parser_from_source_str(sess, name, source).parse_expr()
}
with_default_globals(|| {
let sess = ParseSess::new(FilePathMapping::empty());
let expr = parse_expr_from_source_str(PathBuf::from("foo").into(),
"foo!( fn main() { body } )".to_string(), &sess).unwrap();
let tts: Vec<_> = match expr.node {
ast::ExprKind::Mac(ref mac) => mac.node.stream().trees().collect(),
_ => panic!("not a macro"),
};
let span = tts.iter().rev().next().unwrap().span();
match sess.source_map().span_to_snippet(span) {
Ok(s) => assert_eq!(&s[..], "{ body }"),
Err(_) => panic!("could not get snippet"),
}
});
}
// This tests that when parsing a string (rather than a file) we don't try
// and read in a file for a module declaration and just parse a stub.
// See `recurse_into_file_modules` in the parser.
#[test]
fn out_of_line_mod() {
with_default_globals(|| {
let sess = ParseSess::new(FilePathMapping::empty());
let item = parse_item_from_source_str(
PathBuf::from("foo").into(),
"mod foo { struct S; mod this_does_not_exist; }".to_owned(),
&sess,
).unwrap().unwrap();
if let ast::ItemKind::Mod(ref m) = item.node {
assert!(m.items.len() == 2);
} else {
panic!();
}
});
}
}

339
src/libsyntax/parse/tests.rs Normal file
View file

@ -0,0 +1,339 @@
use super::*;
use crate::ast::{self, Name, PatKind};
use crate::attr::first_attr_value_str_by_name;
use crate::parse::{ParseSess, PResult};
use crate::parse::new_parser_from_source_str;
use crate::parse::token::Token;
use crate::print::pprust::item_to_string;
use crate::ptr::P;
use crate::source_map::FilePathMapping;
use crate::symbol::{kw, sym};
use crate::tests::{matches_codepattern, string_to_stream, with_error_checking_parse};
use crate::tokenstream::{DelimSpan, TokenTree, TokenStream};
use crate::with_default_globals;
use syntax_pos::{Span, BytePos, Pos, NO_EXPANSION};
use std::path::PathBuf;
/// Parses an item.
///
/// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and `Err`
/// when a syntax error occurred.
fn parse_item_from_source_str(name: FileName, source: String, sess: &ParseSess)
-> PResult<'_, Option<P<ast::Item>>> {
new_parser_from_source_str(sess, name, source).parse_item()
}
// produce a syntax_pos::span
fn sp(a: u32, b: u32) -> Span {
Span::new(BytePos(a), BytePos(b), NO_EXPANSION)
}
/// Parse a string, return an expr
fn string_to_expr(source_str : String) -> P<ast::Expr> {
let ps = ParseSess::new(FilePathMapping::empty());
with_error_checking_parse(source_str, &ps, |p| {
p.parse_expr()
})
}
/// Parse a string, return an item
fn string_to_item(source_str : String) -> Option<P<ast::Item>> {
let ps = ParseSess::new(FilePathMapping::empty());
with_error_checking_parse(source_str, &ps, |p| {
p.parse_item()
})
}
#[should_panic]
#[test] fn bad_path_expr_1() {
with_default_globals(|| {
string_to_expr("::abc::def::return".to_string());
})
}
// check the token-tree-ization of macros
#[test]
fn string_to_tts_macro () {
with_default_globals(|| {
let tts: Vec<_> =
string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect();
let tts: &[TokenTree] = &tts[..];
match tts {
[
TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }),
TokenTree::Token(Token { kind: token::Not, .. }),
TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }),
TokenTree::Delimited(_, macro_delim, macro_tts)
]
if name_macro_rules == &sym::macro_rules && name_zip.as_str() == "zip" => {
let tts = &macro_tts.trees().collect::<Vec<_>>();
match &tts[..] {
[
TokenTree::Delimited(_, first_delim, first_tts),
TokenTree::Token(Token { kind: token::FatArrow, .. }),
TokenTree::Delimited(_, second_delim, second_tts),
]
if macro_delim == &token::Paren => {
let tts = &first_tts.trees().collect::<Vec<_>>();
match &tts[..] {
[
TokenTree::Token(Token { kind: token::Dollar, .. }),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }),
]
if first_delim == &token::Paren && name.as_str() == "a" => {},
_ => panic!("value 3: {:?} {:?}", first_delim, first_tts),
}
let tts = &second_tts.trees().collect::<Vec<_>>();
match &tts[..] {
[
TokenTree::Token(Token { kind: token::Dollar, .. }),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }),
]
if second_delim == &token::Paren && name.as_str() == "a" => {},
_ => panic!("value 4: {:?} {:?}", second_delim, second_tts),
}
},
_ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts),
}
},
_ => panic!("value: {:?}",tts),
}
})
}
#[test]
fn string_to_tts_1() {
with_default_globals(|| {
let tts = string_to_stream("fn a (b : i32) { b; }".to_string());
let expected = TokenStream::new(vec![
TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(),
TokenTree::token(token::Ident(Name::intern("a"), false), sp(3, 4)).into(),
TokenTree::Delimited(
DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
token::DelimToken::Paren,
TokenStream::new(vec![
TokenTree::token(token::Ident(Name::intern("b"), false), sp(6, 7)).into(),
TokenTree::token(token::Colon, sp(8, 9)).into(),
TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(),
]).into(),
).into(),
TokenTree::Delimited(
DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
token::DelimToken::Brace,
TokenStream::new(vec![
TokenTree::token(token::Ident(Name::intern("b"), false), sp(17, 18)).into(),
TokenTree::token(token::Semi, sp(18, 19)).into(),
]).into(),
).into()
]);
assert_eq!(tts, expected);
})
}
#[test] fn parse_use() {
with_default_globals(|| {
let use_s = "use foo::bar::baz;";
let vitem = string_to_item(use_s.to_string()).unwrap();
let vitem_s = item_to_string(&vitem);
assert_eq!(&vitem_s[..], use_s);
let use_s = "use foo::bar as baz;";
let vitem = string_to_item(use_s.to_string()).unwrap();
let vitem_s = item_to_string(&vitem);
assert_eq!(&vitem_s[..], use_s);
})
}
#[test] fn parse_extern_crate() {
with_default_globals(|| {
let ex_s = "extern crate foo;";
let vitem = string_to_item(ex_s.to_string()).unwrap();
let vitem_s = item_to_string(&vitem);
assert_eq!(&vitem_s[..], ex_s);
let ex_s = "extern crate foo as bar;";
let vitem = string_to_item(ex_s.to_string()).unwrap();
let vitem_s = item_to_string(&vitem);
assert_eq!(&vitem_s[..], ex_s);
})
}
fn get_spans_of_pat_idents(src: &str) -> Vec<Span> {
let item = string_to_item(src.to_string()).unwrap();
struct PatIdentVisitor {
spans: Vec<Span>
}
impl<'a> crate::visit::Visitor<'a> for PatIdentVisitor {
fn visit_pat(&mut self, p: &'a ast::Pat) {
match p.node {
PatKind::Ident(_ , ref spannedident, _) => {
self.spans.push(spannedident.span.clone());
}
_ => {
crate::visit::walk_pat(self, p);
}
}
}
}
let mut v = PatIdentVisitor { spans: Vec::new() };
crate::visit::walk_item(&mut v, &item);
return v.spans;
}
#[test] fn span_of_self_arg_pat_idents_are_correct() {
with_default_globals(|| {
let srcs = ["impl z { fn a (&self, &myarg: i32) {} }",
"impl z { fn a (&mut self, &myarg: i32) {} }",
"impl z { fn a (&'a self, &myarg: i32) {} }",
"impl z { fn a (self, &myarg: i32) {} }",
"impl z { fn a (self: Foo, &myarg: i32) {} }",
];
for &src in &srcs {
let spans = get_spans_of_pat_idents(src);
let (lo, hi) = (spans[0].lo(), spans[0].hi());
assert!("self" == &src[lo.to_usize()..hi.to_usize()],
"\"{}\" != \"self\". src=\"{}\"",
&src[lo.to_usize()..hi.to_usize()], src)
}
})
}
#[test] fn parse_exprs () {
with_default_globals(|| {
// just make sure that they parse....
string_to_expr("3 + 4".to_string());
string_to_expr("a::z.froob(b,&(987+3))".to_string());
})
}
#[test] fn attrs_fix_bug () {
with_default_globals(|| {
string_to_item("pub fn mk_file_writer(path: &Path, flags: &[FileFlag])
-> Result<Box<Writer>, String> {
#[cfg(windows)]
fn wb() -> c_int {
(O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int
}
#[cfg(unix)]
fn wb() -> c_int { O_WRONLY as c_int }
let mut fflags: c_int = wb();
}".to_string());
})
}
#[test] fn crlf_doc_comments() {
with_default_globals(|| {
let sess = ParseSess::new(FilePathMapping::empty());
let name_1 = FileName::Custom("crlf_source_1".to_string());
let source = "/// doc comment\r\nfn foo() {}".to_string();
let item = parse_item_from_source_str(name_1, source, &sess)
.unwrap().unwrap();
let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap();
assert_eq!(doc.as_str(), "/// doc comment");
let name_2 = FileName::Custom("crlf_source_2".to_string());
let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string();
let item = parse_item_from_source_str(name_2, source, &sess)
.unwrap().unwrap();
let docs = item.attrs.iter().filter(|a| a.path == sym::doc)
.map(|a| a.value_str().unwrap().to_string()).collect::<Vec<_>>();
let b: &[_] = &["/// doc comment".to_string(), "/// line 2".to_string()];
assert_eq!(&docs[..], b);
let name_3 = FileName::Custom("clrf_source_3".to_string());
let source = "/** doc comment\r\n * with CRLF */\r\nfn foo() {}".to_string();
let item = parse_item_from_source_str(name_3, source, &sess).unwrap().unwrap();
let doc = first_attr_value_str_by_name(&item.attrs, sym::doc).unwrap();
assert_eq!(doc.as_str(), "/** doc comment\n * with CRLF */");
});
}
#[test]
fn ttdelim_span() {
fn parse_expr_from_source_str(
name: FileName, source: String, sess: &ParseSess
) -> PResult<'_, P<ast::Expr>> {
new_parser_from_source_str(sess, name, source).parse_expr()
}
with_default_globals(|| {
let sess = ParseSess::new(FilePathMapping::empty());
let expr = parse_expr_from_source_str(PathBuf::from("foo").into(),
"foo!( fn main() { body } )".to_string(), &sess).unwrap();
let tts: Vec<_> = match expr.node {
ast::ExprKind::Mac(ref mac) => mac.node.stream().trees().collect(),
_ => panic!("not a macro"),
};
let span = tts.iter().rev().next().unwrap().span();
match sess.source_map().span_to_snippet(span) {
Ok(s) => assert_eq!(&s[..], "{ body }"),
Err(_) => panic!("could not get snippet"),
}
});
}
// This tests that when parsing a string (rather than a file) we don't try
// and read in a file for a module declaration and just parse a stub.
// See `recurse_into_file_modules` in the parser.
#[test]
fn out_of_line_mod() {
with_default_globals(|| {
let sess = ParseSess::new(FilePathMapping::empty());
let item = parse_item_from_source_str(
PathBuf::from("foo").into(),
"mod foo { struct S; mod this_does_not_exist; }".to_owned(),
&sess,
).unwrap().unwrap();
if let ast::ItemKind::Mod(ref m) = item.node {
assert!(m.items.len() == 2);
} else {
panic!();
}
});
}
#[test]
fn eqmodws() {
assert_eq!(matches_codepattern("",""),true);
assert_eq!(matches_codepattern("","a"),false);
assert_eq!(matches_codepattern("a",""),false);
assert_eq!(matches_codepattern("a","a"),true);
assert_eq!(matches_codepattern("a b","a \n\t\r b"),true);
assert_eq!(matches_codepattern("a b ","a \n\t\r b"),true);
assert_eq!(matches_codepattern("a b","a \n\t\r b "),false);
assert_eq!(matches_codepattern("a b","a b"),true);
assert_eq!(matches_codepattern("ab","a b"),false);
assert_eq!(matches_codepattern("a b","ab"),true);
assert_eq!(matches_codepattern(" a b","ab"),true);
}
#[test]
fn pattern_whitespace() {
assert_eq!(matches_codepattern("","\x0C"), false);
assert_eq!(matches_codepattern("a b ","a \u{0085}\n\t\r b"),true);
assert_eq!(matches_codepattern("a b","a \u{0085}\n\t\r b "),false);
}
#[test]
fn non_pattern_whitespace() {
// These have the property 'White_Space' but not 'Pattern_White_Space'
assert_eq!(matches_codepattern("a b","a\u{2002}b"), false);
assert_eq!(matches_codepattern("a b","a\u{2002}b"), false);
assert_eq!(matches_codepattern("\u{205F}a b","ab"), false);
assert_eq!(matches_codepattern("a \u{3000}b","ab"), false);
}

60
src/libsyntax/print/pprust.rs
View file

@ -19,6 +19,9 @@ use syntax_pos::{DUMMY_SP, FileName, Span};
use std::borrow::Cow;
#[cfg(test)]
mod tests;
pub enum MacHeader<'a> {
Path(&'a ast::Path),
Keyword(&'static str),
@ -2888,60 +2891,3 @@ impl<'a> State<'a> {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ast;
use crate::source_map;
use crate::with_default_globals;
use syntax_pos;
#[test]
fn test_fun_to_string() {
with_default_globals(|| {
let abba_ident = ast::Ident::from_str("abba");
let decl = ast::FnDecl {
inputs: Vec::new(),
output: ast::FunctionRetTy::Default(syntax_pos::DUMMY_SP),
c_variadic: false
};
let generics = ast::Generics::default();
assert_eq!(
fun_to_string(
&decl,
ast::FnHeader {
unsafety: ast::Unsafety::Normal,
constness: source_map::dummy_spanned(ast::Constness::NotConst),
asyncness: source_map::dummy_spanned(ast::IsAsync::NotAsync),
abi: Abi::Rust,
},
abba_ident,
&generics
),
"fn abba()"
);
})
}
#[test]
fn test_variant_to_string() {
with_default_globals(|| {
let ident = ast::Ident::from_str("principal_skinner");
let var = source_map::respan(syntax_pos::DUMMY_SP, ast::Variant_ {
ident,
attrs: Vec::new(),
id: ast::DUMMY_NODE_ID,
// making this up as I go.... ?
data: ast::VariantData::Unit(ast::DUMMY_NODE_ID),
disr_expr: None,
});
let varstr = variant_to_string(&var);
assert_eq!(varstr, "principal_skinner");
})
}
}

53
src/libsyntax/print/pprust/tests.rs Normal file
View file

@ -0,0 +1,53 @@
use super::*;
use crate::ast;
use crate::source_map;
use crate::with_default_globals;
use syntax_pos;
#[test]
fn test_fun_to_string() {
with_default_globals(|| {
let abba_ident = ast::Ident::from_str("abba");
let decl = ast::FnDecl {
inputs: Vec::new(),
output: ast::FunctionRetTy::Default(syntax_pos::DUMMY_SP),
c_variadic: false
};
let generics = ast::Generics::default();
assert_eq!(
fun_to_string(
&decl,
ast::FnHeader {
unsafety: ast::Unsafety::Normal,
constness: source_map::dummy_spanned(ast::Constness::NotConst),
asyncness: source_map::dummy_spanned(ast::IsAsync::NotAsync),
abi: Abi::Rust,
},
abba_ident,
&generics
),
"fn abba()"
);
})
}
#[test]
fn test_variant_to_string() {
with_default_globals(|| {
let ident = ast::Ident::from_str("principal_skinner");
let var = source_map::respan(syntax_pos::DUMMY_SP, ast::Variant_ {
ident,
attrs: Vec::new(),
id: ast::DUMMY_NODE_ID,
// making this up as I go.... ?
data: ast::VariantData::Unit(ast::DUMMY_NODE_ID),
disr_expr: None,
});
let varstr = variant_to_string(&var);
assert_eq!(varstr, "principal_skinner");
})
}

223
src/libsyntax/source_map.rs
View file

@ -24,6 +24,9 @@ use log::debug;
use errors::SourceMapper;
#[cfg(test)]
mod tests;
/// Returns the span itself if it doesn't come from a macro expansion,
/// otherwise return the call site span up to the `enclosing_sp` by
/// following the `expn_info` chain.
@ -1020,223 +1023,3 @@ impl FilePathMapping {
(path, false)
}
}
// _____________________________________________________________________________
// Tests
//
#[cfg(test)]
mod tests {
use super::*;
use rustc_data_structures::sync::Lrc;
fn init_source_map() -> SourceMap {
let sm = SourceMap::new(FilePathMapping::empty());
sm.new_source_file(PathBuf::from("blork.rs").into(),
"first line.\nsecond line".to_string());
sm.new_source_file(PathBuf::from("empty.rs").into(),
String::new());
sm.new_source_file(PathBuf::from("blork2.rs").into(),
"first line.\nsecond line".to_string());
sm
}
#[test]
fn t3() {
// Test lookup_byte_offset
let sm = init_source_map();
let srcfbp1 = sm.lookup_byte_offset(BytePos(23));
assert_eq!(srcfbp1.sf.name, PathBuf::from("blork.rs").into());
assert_eq!(srcfbp1.pos, BytePos(23));
let srcfbp1 = sm.lookup_byte_offset(BytePos(24));
assert_eq!(srcfbp1.sf.name, PathBuf::from("empty.rs").into());
assert_eq!(srcfbp1.pos, BytePos(0));
let srcfbp2 = sm.lookup_byte_offset(BytePos(25));
assert_eq!(srcfbp2.sf.name, PathBuf::from("blork2.rs").into());
assert_eq!(srcfbp2.pos, BytePos(0));
}
#[test]
fn t4() {
// Test bytepos_to_file_charpos
let sm = init_source_map();
let cp1 = sm.bytepos_to_file_charpos(BytePos(22));
assert_eq!(cp1, CharPos(22));
let cp2 = sm.bytepos_to_file_charpos(BytePos(25));
assert_eq!(cp2, CharPos(0));
}
#[test]
fn t5() {
// Test zero-length source_files.
let sm = init_source_map();
let loc1 = sm.lookup_char_pos(BytePos(22));
assert_eq!(loc1.file.name, PathBuf::from("blork.rs").into());
assert_eq!(loc1.line, 2);
assert_eq!(loc1.col, CharPos(10));
let loc2 = sm.lookup_char_pos(BytePos(25));
assert_eq!(loc2.file.name, PathBuf::from("blork2.rs").into());
assert_eq!(loc2.line, 1);
assert_eq!(loc2.col, CharPos(0));
}
fn init_source_map_mbc() -> SourceMap {
let sm = SourceMap::new(FilePathMapping::empty());
// € is a three byte utf8 char.
sm.new_source_file(PathBuf::from("blork.rs").into(),
"fir€st €€€€ line.\nsecond line".to_string());
sm.new_source_file(PathBuf::from("blork2.rs").into(),
"first line€€.\n€ second line".to_string());
sm
}
#[test]
fn t6() {
// Test bytepos_to_file_charpos in the presence of multi-byte chars
let sm = init_source_map_mbc();
let cp1 = sm.bytepos_to_file_charpos(BytePos(3));
assert_eq!(cp1, CharPos(3));
let cp2 = sm.bytepos_to_file_charpos(BytePos(6));
assert_eq!(cp2, CharPos(4));
let cp3 = sm.bytepos_to_file_charpos(BytePos(56));
assert_eq!(cp3, CharPos(12));
let cp4 = sm.bytepos_to_file_charpos(BytePos(61));
assert_eq!(cp4, CharPos(15));
}
#[test]
fn t7() {
// Test span_to_lines for a span ending at the end of source_file
let sm = init_source_map();
let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION);
let file_lines = sm.span_to_lines(span).unwrap();
assert_eq!(file_lines.file.name, PathBuf::from("blork.rs").into());
assert_eq!(file_lines.lines.len(), 1);
assert_eq!(file_lines.lines[0].line_index, 1);
}
/// Given a string like " ~~~~~~~~~~~~ ", produces a span
/// converting that range. The idea is that the string has the same
/// length as the input, and we uncover the byte positions. Note
/// that this can span lines and so on.
fn span_from_selection(input: &str, selection: &str) -> Span {
assert_eq!(input.len(), selection.len());
let left_index = selection.find('~').unwrap() as u32;
let right_index = selection.rfind('~').map(|x|x as u32).unwrap_or(left_index);
Span::new(BytePos(left_index), BytePos(right_index + 1), NO_EXPANSION)
}
/// Tests span_to_snippet and span_to_lines for a span converting 3
/// lines in the middle of a file.
#[test]
fn span_to_snippet_and_lines_spanning_multiple_lines() {
let sm = SourceMap::new(FilePathMapping::empty());
let inputtext = "aaaaa\nbbbbBB\nCCC\nDDDDDddddd\neee\n";
let selection = " \n ~~\n~~~\n~~~~~ \n \n";
sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_string());
let span = span_from_selection(inputtext, selection);
// check that we are extracting the text we thought we were extracting
assert_eq!(&sm.span_to_snippet(span).unwrap(), "BB\nCCC\nDDDDD");
// check that span_to_lines gives us the complete result with the lines/cols we expected
let lines = sm.span_to_lines(span).unwrap();
let expected = vec![
LineInfo { line_index: 1, start_col: CharPos(4), end_col: CharPos(6) },
LineInfo { line_index: 2, start_col: CharPos(0), end_col: CharPos(3) },
LineInfo { line_index: 3, start_col: CharPos(0), end_col: CharPos(5) }
];
assert_eq!(lines.lines, expected);
}
#[test]
fn t8() {
// Test span_to_snippet for a span ending at the end of source_file
let sm = init_source_map();
let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION);
let snippet = sm.span_to_snippet(span);
assert_eq!(snippet, Ok("second line".to_string()));
}
#[test]
fn t9() {
// Test span_to_str for a span ending at the end of source_file
let sm = init_source_map();
let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION);
let sstr = sm.span_to_string(span);
assert_eq!(sstr, "blork.rs:2:1: 2:12");
}
/// Tests failing to merge two spans on different lines
#[test]
fn span_merging_fail() {
let sm = SourceMap::new(FilePathMapping::empty());
let inputtext = "bbbb BB\ncc CCC\n";
let selection1 = " ~~\n \n";
let selection2 = " \n ~~~\n";
sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_owned());
let span1 = span_from_selection(inputtext, selection1);
let span2 = span_from_selection(inputtext, selection2);
assert!(sm.merge_spans(span1, span2).is_none());
}
/// Returns the span corresponding to the `n`th occurrence of
/// `substring` in `source_text`.
trait SourceMapExtension {
fn span_substr(&self,
file: &Lrc<SourceFile>,
source_text: &str,
substring: &str,
n: usize)
-> Span;
}
impl SourceMapExtension for SourceMap {
fn span_substr(&self,
file: &Lrc<SourceFile>,
source_text: &str,
substring: &str,
n: usize)
-> Span
{
println!("span_substr(file={:?}/{:?}, substring={:?}, n={})",
file.name, file.start_pos, substring, n);
let mut i = 0;
let mut hi = 0;
loop {
let offset = source_text[hi..].find(substring).unwrap_or_else(|| {
panic!("source_text `{}` does not have {} occurrences of `{}`, only {}",
source_text, n, substring, i);
});
let lo = hi + offset;
hi = lo + substring.len();
if i == n {
let span = Span::new(
BytePos(lo as u32 + file.start_pos.0),
BytePos(hi as u32 + file.start_pos.0),
NO_EXPANSION,
);
assert_eq!(&self.span_to_snippet(span).unwrap()[..],
substring);
return span;
}
i += 1;
}
}
}
}

213
src/libsyntax/source_map/tests.rs Normal file
View file

@ -0,0 +1,213 @@
use super::*;
use rustc_data_structures::sync::Lrc;
fn init_source_map() -> SourceMap {
let sm = SourceMap::new(FilePathMapping::empty());
sm.new_source_file(PathBuf::from("blork.rs").into(),
"first line.\nsecond line".to_string());
sm.new_source_file(PathBuf::from("empty.rs").into(),
String::new());
sm.new_source_file(PathBuf::from("blork2.rs").into(),
"first line.\nsecond line".to_string());
sm
}
#[test]
fn t3() {
// Test lookup_byte_offset
let sm = init_source_map();
let srcfbp1 = sm.lookup_byte_offset(BytePos(23));
assert_eq!(srcfbp1.sf.name, PathBuf::from("blork.rs").into());
assert_eq!(srcfbp1.pos, BytePos(23));
let srcfbp1 = sm.lookup_byte_offset(BytePos(24));
assert_eq!(srcfbp1.sf.name, PathBuf::from("empty.rs").into());
assert_eq!(srcfbp1.pos, BytePos(0));
let srcfbp2 = sm.lookup_byte_offset(BytePos(25));
assert_eq!(srcfbp2.sf.name, PathBuf::from("blork2.rs").into());
assert_eq!(srcfbp2.pos, BytePos(0));
}
#[test]
fn t4() {
// Test bytepos_to_file_charpos
let sm = init_source_map();
let cp1 = sm.bytepos_to_file_charpos(BytePos(22));
assert_eq!(cp1, CharPos(22));
let cp2 = sm.bytepos_to_file_charpos(BytePos(25));
assert_eq!(cp2, CharPos(0));
}
#[test]
fn t5() {
// Test zero-length source_files.
let sm = init_source_map();
let loc1 = sm.lookup_char_pos(BytePos(22));
assert_eq!(loc1.file.name, PathBuf::from("blork.rs").into());
assert_eq!(loc1.line, 2);
assert_eq!(loc1.col, CharPos(10));
let loc2 = sm.lookup_char_pos(BytePos(25));
assert_eq!(loc2.file.name, PathBuf::from("blork2.rs").into());
assert_eq!(loc2.line, 1);
assert_eq!(loc2.col, CharPos(0));
}
fn init_source_map_mbc() -> SourceMap {
let sm = SourceMap::new(FilePathMapping::empty());
// € is a three byte utf8 char.
sm.new_source_file(PathBuf::from("blork.rs").into(),
"fir€st €€€€ line.\nsecond line".to_string());
sm.new_source_file(PathBuf::from("blork2.rs").into(),
"first line€€.\n€ second line".to_string());
sm
}
#[test]
fn t6() {
// Test bytepos_to_file_charpos in the presence of multi-byte chars
let sm = init_source_map_mbc();
let cp1 = sm.bytepos_to_file_charpos(BytePos(3));
assert_eq!(cp1, CharPos(3));
let cp2 = sm.bytepos_to_file_charpos(BytePos(6));
assert_eq!(cp2, CharPos(4));
let cp3 = sm.bytepos_to_file_charpos(BytePos(56));
assert_eq!(cp3, CharPos(12));
let cp4 = sm.bytepos_to_file_charpos(BytePos(61));
assert_eq!(cp4, CharPos(15));
}
#[test]
fn t7() {
// Test span_to_lines for a span ending at the end of source_file
let sm = init_source_map();
let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION);
let file_lines = sm.span_to_lines(span).unwrap();
assert_eq!(file_lines.file.name, PathBuf::from("blork.rs").into());
assert_eq!(file_lines.lines.len(), 1);
assert_eq!(file_lines.lines[0].line_index, 1);
}
/// Given a string like " ~~~~~~~~~~~~ ", produces a span
/// converting that range. The idea is that the string has the same
/// length as the input, and we uncover the byte positions. Note
/// that this can span lines and so on.
fn span_from_selection(input: &str, selection: &str) -> Span {
assert_eq!(input.len(), selection.len());
let left_index = selection.find('~').unwrap() as u32;
let right_index = selection.rfind('~').map(|x|x as u32).unwrap_or(left_index);
Span::new(BytePos(left_index), BytePos(right_index + 1), NO_EXPANSION)
}
/// Tests span_to_snippet and span_to_lines for a span converting 3
/// lines in the middle of a file.
#[test]
fn span_to_snippet_and_lines_spanning_multiple_lines() {
let sm = SourceMap::new(FilePathMapping::empty());
let inputtext = "aaaaa\nbbbbBB\nCCC\nDDDDDddddd\neee\n";
let selection = " \n ~~\n~~~\n~~~~~ \n \n";
sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_string());
let span = span_from_selection(inputtext, selection);
// check that we are extracting the text we thought we were extracting
assert_eq!(&sm.span_to_snippet(span).unwrap(), "BB\nCCC\nDDDDD");
// check that span_to_lines gives us the complete result with the lines/cols we expected
let lines = sm.span_to_lines(span).unwrap();
let expected = vec![
LineInfo { line_index: 1, start_col: CharPos(4), end_col: CharPos(6) },
LineInfo { line_index: 2, start_col: CharPos(0), end_col: CharPos(3) },
LineInfo { line_index: 3, start_col: CharPos(0), end_col: CharPos(5) }
];
assert_eq!(lines.lines, expected);
}
#[test]
fn t8() {
// Test span_to_snippet for a span ending at the end of source_file
let sm = init_source_map();
let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION);
let snippet = sm.span_to_snippet(span);
assert_eq!(snippet, Ok("second line".to_string()));
}
#[test]
fn t9() {
// Test span_to_str for a span ending at the end of source_file
let sm = init_source_map();
let span = Span::new(BytePos(12), BytePos(23), NO_EXPANSION);
let sstr = sm.span_to_string(span);
assert_eq!(sstr, "blork.rs:2:1: 2:12");
}
/// Tests failing to merge two spans on different lines
#[test]
fn span_merging_fail() {
let sm = SourceMap::new(FilePathMapping::empty());
let inputtext = "bbbb BB\ncc CCC\n";
let selection1 = " ~~\n \n";
let selection2 = " \n ~~~\n";
sm.new_source_file(Path::new("blork.rs").to_owned().into(), inputtext.to_owned());
let span1 = span_from_selection(inputtext, selection1);
let span2 = span_from_selection(inputtext, selection2);
assert!(sm.merge_spans(span1, span2).is_none());
}
/// Returns the span corresponding to the `n`th occurrence of
/// `substring` in `source_text`.
trait SourceMapExtension {
fn span_substr(&self,
file: &Lrc<SourceFile>,
source_text: &str,
substring: &str,
n: usize)
-> Span;
}
impl SourceMapExtension for SourceMap {
fn span_substr(&self,
file: &Lrc<SourceFile>,
source_text: &str,
substring: &str,
n: usize)
-> Span
{
println!("span_substr(file={:?}/{:?}, substring={:?}, n={})",
file.name, file.start_pos, substring, n);
let mut i = 0;
let mut hi = 0;
loop {
let offset = source_text[hi..].find(substring).unwrap_or_else(|| {
panic!("source_text `{}` does not have {} occurrences of `{}`, only {}",
source_text, n, substring, i);
});
let lo = hi + offset;
hi = lo + substring.len();
if i == n {
let span = Span::new(
BytePos(lo as u32 + file.start_pos.0),
BytePos(hi as u32 + file.start_pos.0),
NO_EXPANSION,
);
assert_eq!(&self.span_to_snippet(span).unwrap()[..],
substring);
return span;
}
i += 1;
}
}
}

98
src/libsyntax/test_snippet.rs → src/libsyntax/tests.rs
View file

@ -1,16 +1,106 @@
use crate::{ast, panictry};
use crate::parse::{ParseSess, PResult, source_file_to_stream};
use crate::parse::new_parser_from_source_str;
use crate::parse::parser::Parser;
use crate::source_map::{SourceMap, FilePathMapping};
use crate::tokenstream::TokenStream;
use crate::with_default_globals;
use errors::Handler;
use errors::emitter::EmitterWriter;
use errors::Handler;
use rustc_data_structures::sync::Lrc;
use syntax_pos::{BytePos, NO_EXPANSION, Span, MultiSpan};
use std::io;
use std::io::prelude::*;
use rustc_data_structures::sync::Lrc;
use std::iter::Peekable;
use std::path::{Path, PathBuf};
use std::str;
use std::sync::{Arc, Mutex};
use std::path::Path;
use syntax_pos::{BytePos, NO_EXPANSION, Span, MultiSpan};
/// Map string to parser (via tts)
fn string_to_parser(ps: &ParseSess, source_str: String) -> Parser<'_> {
new_parser_from_source_str(ps, PathBuf::from("bogofile").into(), source_str)
}
crate fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T where
F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>,
{
let mut p = string_to_parser(&ps, s);
let x = panictry!(f(&mut p));
p.sess.span_diagnostic.abort_if_errors();
x
}
/// Map a string to tts, using a made-up filename:
crate fn string_to_stream(source_str: String) -> TokenStream {
let ps = ParseSess::new(FilePathMapping::empty());
source_file_to_stream(
&ps,
ps.source_map().new_source_file(PathBuf::from("bogofile").into(),
source_str,
), None).0
}
/// Parse a string, return a crate.
crate fn string_to_crate(source_str : String) -> ast::Crate {
let ps = ParseSess::new(FilePathMapping::empty());
with_error_checking_parse(source_str, &ps, |p| {
p.parse_crate_mod()
})
}
/// Does the given string match the pattern? whitespace in the first string
/// may be deleted or replaced with other whitespace to match the pattern.
/// This function is relatively Unicode-ignorant; fortunately, the careful design
/// of UTF-8 mitigates this ignorance. It doesn't do NKF-normalization(?).
crate fn matches_codepattern(a : &str, b : &str) -> bool {
let mut a_iter = a.chars().peekable();
let mut b_iter = b.chars().peekable();
loop {
let (a, b) = match (a_iter.peek(), b_iter.peek()) {
(None, None) => return true,
(None, _) => return false,
(Some(&a), None) => {
if is_pattern_whitespace(a) {
break // trailing whitespace check is out of loop for borrowck
} else {
return false
}
}
(Some(&a), Some(&b)) => (a, b)
};
if is_pattern_whitespace(a) && is_pattern_whitespace(b) {
// skip whitespace for a and b
scan_for_non_ws_or_end(&mut a_iter);
scan_for_non_ws_or_end(&mut b_iter);
} else if is_pattern_whitespace(a) {
// skip whitespace for a
scan_for_non_ws_or_end(&mut a_iter);
} else if a == b {
a_iter.next();
b_iter.next();
} else {
return false
}
}
// check if a has *only* trailing whitespace
a_iter.all(is_pattern_whitespace)
}
/// Advances the given peekable `Iterator` until it reaches a non-whitespace character
fn scan_for_non_ws_or_end<I: Iterator<Item = char>>(iter: &mut Peekable<I>) {
while iter.peek().copied().map(|c| is_pattern_whitespace(c)) == Some(true) {
iter.next();
}
}
fn is_pattern_whitespace(c: char) -> bool {
rustc_lexer::character_properties::is_whitespace(c)
}
/// Identify a position in the text by the Nth occurrence of a string.
struct Position {

114
src/libsyntax/tokenstream.rs
View file

@ -29,6 +29,9 @@ use smallvec::{SmallVec, smallvec};
use std::borrow::Cow;
use std::{fmt, iter, mem};
#[cfg(test)]
mod tests;
/// When the main rust parser encounters a syntax-extension invocation, it
/// parses the arguments to the invocation as a token-tree. This is a very
/// loose structure, such that all sorts of different AST-fragments can
@ -552,114 +555,3 @@ impl DelimSpan {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::ast::Name;
use crate::with_default_globals;
use crate::util::parser_testing::string_to_stream;
use syntax_pos::{Span, BytePos, NO_EXPANSION};
fn string_to_ts(string: &str) -> TokenStream {
string_to_stream(string.to_owned())
}
fn sp(a: u32, b: u32) -> Span {
Span::new(BytePos(a), BytePos(b), NO_EXPANSION)
}
#[test]
fn test_concat() {
with_default_globals(|| {
let test_res = string_to_ts("foo::bar::baz");
let test_fst = string_to_ts("foo::bar");
let test_snd = string_to_ts("::baz");
let eq_res = TokenStream::from_streams(smallvec![test_fst, test_snd]);
assert_eq!(test_res.trees().count(), 5);
assert_eq!(eq_res.trees().count(), 5);
assert_eq!(test_res.eq_unspanned(&eq_res), true);
})
}
#[test]
fn test_to_from_bijection() {
with_default_globals(|| {
let test_start = string_to_ts("foo::bar(baz)");
let test_end = test_start.trees().collect();
assert_eq!(test_start, test_end)
})
}
#[test]
fn test_eq_0() {
with_default_globals(|| {
let test_res = string_to_ts("foo");
let test_eqs = string_to_ts("foo");
assert_eq!(test_res, test_eqs)
})
}
#[test]
fn test_eq_1() {
with_default_globals(|| {
let test_res = string_to_ts("::bar::baz");
let test_eqs = string_to_ts("::bar::baz");
assert_eq!(test_res, test_eqs)
})
}
#[test]
fn test_eq_3() {
with_default_globals(|| {
let test_res = string_to_ts("");
let test_eqs = string_to_ts("");
assert_eq!(test_res, test_eqs)
})
}
#[test]
fn test_diseq_0() {
with_default_globals(|| {
let test_res = string_to_ts("::bar::baz");
let test_eqs = string_to_ts("bar::baz");
assert_eq!(test_res == test_eqs, false)
})
}
#[test]
fn test_diseq_1() {
with_default_globals(|| {
let test_res = string_to_ts("(bar,baz)");
let test_eqs = string_to_ts("bar,baz");
assert_eq!(test_res == test_eqs, false)
})
}
#[test]
fn test_is_empty() {
with_default_globals(|| {
let test0: TokenStream = Vec::<TokenTree>::new().into_iter().collect();
let test1: TokenStream =
TokenTree::token(token::Ident(Name::intern("a"), false), sp(0, 1)).into();
let test2 = string_to_ts("foo(bar::baz)");
assert_eq!(test0.is_empty(), true);
assert_eq!(test1.is_empty(), false);
assert_eq!(test2.is_empty(), false);
})
}
#[test]
fn test_dotdotdot() {
with_default_globals(|| {
let mut builder = TokenStreamBuilder::new();
builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint());
builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint());
builder.push(TokenTree::token(token::Dot, sp(2, 3)));
let stream = builder.build();
assert!(stream.eq_unspanned(&string_to_ts("...")));
assert_eq!(stream.trees().count(), 1);
})
}
}

108
src/libsyntax/tokenstream/tests.rs Normal file
View file

@ -0,0 +1,108 @@
use super::*;
use crate::ast::Name;
use crate::with_default_globals;
use crate::tests::string_to_stream;
use syntax_pos::{Span, BytePos, NO_EXPANSION};
fn string_to_ts(string: &str) -> TokenStream {
string_to_stream(string.to_owned())
}
fn sp(a: u32, b: u32) -> Span {
Span::new(BytePos(a), BytePos(b), NO_EXPANSION)
}
#[test]
fn test_concat() {
with_default_globals(|| {
let test_res = string_to_ts("foo::bar::baz");
let test_fst = string_to_ts("foo::bar");
let test_snd = string_to_ts("::baz");
let eq_res = TokenStream::from_streams(smallvec![test_fst, test_snd]);
assert_eq!(test_res.trees().count(), 5);
assert_eq!(eq_res.trees().count(), 5);
assert_eq!(test_res.eq_unspanned(&eq_res), true);
})
}
#[test]
fn test_to_from_bijection() {
with_default_globals(|| {
let test_start = string_to_ts("foo::bar(baz)");
let test_end = test_start.trees().collect();
assert_eq!(test_start, test_end)
})
}
#[test]
fn test_eq_0() {
with_default_globals(|| {
let test_res = string_to_ts("foo");
let test_eqs = string_to_ts("foo");
assert_eq!(test_res, test_eqs)
})
}
#[test]
fn test_eq_1() {
with_default_globals(|| {
let test_res = string_to_ts("::bar::baz");
let test_eqs = string_to_ts("::bar::baz");
assert_eq!(test_res, test_eqs)
})
}
#[test]
fn test_eq_3() {
with_default_globals(|| {
let test_res = string_to_ts("");
let test_eqs = string_to_ts("");
assert_eq!(test_res, test_eqs)
})
}
#[test]
fn test_diseq_0() {
with_default_globals(|| {
let test_res = string_to_ts("::bar::baz");
let test_eqs = string_to_ts("bar::baz");
assert_eq!(test_res == test_eqs, false)
})
}
#[test]
fn test_diseq_1() {
with_default_globals(|| {
let test_res = string_to_ts("(bar,baz)");
let test_eqs = string_to_ts("bar,baz");
assert_eq!(test_res == test_eqs, false)
})
}
#[test]
fn test_is_empty() {
with_default_globals(|| {
let test0: TokenStream = Vec::<TokenTree>::new().into_iter().collect();
let test1: TokenStream =
TokenTree::token(token::Ident(Name::intern("a"), false), sp(0, 1)).into();
let test2 = string_to_ts("foo(bar::baz)");
assert_eq!(test0.is_empty(), true);
assert_eq!(test1.is_empty(), false);
assert_eq!(test2.is_empty(), false);
})
}
#[test]
fn test_dotdotdot() {
with_default_globals(|| {
let mut builder = TokenStreamBuilder::new();
builder.push(TokenTree::token(token::Dot, sp(0, 1)).joint());
builder.push(TokenTree::token(token::Dot, sp(1, 2)).joint());
builder.push(TokenTree::token(token::Dot, sp(2, 3)));
let stream = builder.build();
assert!(stream.eq_unspanned(&string_to_ts("...")));
assert_eq!(stream.trees().count(), 1);
})
}

60
src/libsyntax/util/lev_distance.rs
View file

@ -1,6 +1,9 @@
use std::cmp;
use crate::symbol::Symbol;
#[cfg(test)]
mod tests;
/// Finds the Levenshtein distance between two strings
pub fn lev_distance(a: &str, b: &str) -> usize {
// cases which don't require further computation
@ -77,60 +80,3 @@ pub fn find_best_match_for_name<'a, T>(iter_names: T,
if let Some((candidate, _)) = levenstein_match { Some(candidate) } else { None }
}
}
#[test]
fn test_lev_distance() {
use std::char::{from_u32, MAX};
// Test bytelength agnosticity
for c in (0..MAX as u32)
.filter_map(|i| from_u32(i))
.map(|i| i.to_string()) {
assert_eq!(lev_distance(&c[..], &c[..]), 0);
}
let a = "\nMäry häd ä little lämb\n\nLittle lämb\n";
let b = "\nMary häd ä little lämb\n\nLittle lämb\n";
let c = "Mary häd ä little lämb\n\nLittle lämb\n";
assert_eq!(lev_distance(a, b), 1);
assert_eq!(lev_distance(b, a), 1);
assert_eq!(lev_distance(a, c), 2);
assert_eq!(lev_distance(c, a), 2);
assert_eq!(lev_distance(b, c), 1);
assert_eq!(lev_distance(c, b), 1);
}
#[test]
fn test_find_best_match_for_name() {
use crate::with_default_globals;
with_default_globals(|| {
let input = vec![Symbol::intern("aaab"), Symbol::intern("aaabc")];
assert_eq!(
find_best_match_for_name(input.iter(), "aaaa", None),
Some(Symbol::intern("aaab"))
);
assert_eq!(
find_best_match_for_name(input.iter(), "1111111111", None),
None
);
let input = vec![Symbol::intern("aAAA")];
assert_eq!(
find_best_match_for_name(input.iter(), "AAAA", None),
Some(Symbol::intern("aAAA"))
);
let input = vec![Symbol::intern("AAAA")];
// Returns None because `lev_distance > max_dist / 3`
assert_eq!(
find_best_match_for_name(input.iter(), "aaaa", None),
None
);
let input = vec![Symbol::intern("AAAA")];
assert_eq!(
find_best_match_for_name(input.iter(), "aaaa", Some(4)),
Some(Symbol::intern("AAAA"))
);
})
}

58
src/libsyntax/util/lev_distance/tests.rs Normal file
View file

@ -0,0 +1,58 @@
use super::*;
#[test]
fn test_lev_distance() {
use std::char::{from_u32, MAX};
// Test bytelength agnosticity
for c in (0..MAX as u32)
.filter_map(|i| from_u32(i))
.map(|i| i.to_string()) {
assert_eq!(lev_distance(&c[..], &c[..]), 0);
}
let a = "\nMäry häd ä little lämb\n\nLittle lämb\n";
let b = "\nMary häd ä little lämb\n\nLittle lämb\n";
let c = "Mary häd ä little lämb\n\nLittle lämb\n";
assert_eq!(lev_distance(a, b), 1);
assert_eq!(lev_distance(b, a), 1);
assert_eq!(lev_distance(a, c), 2);
assert_eq!(lev_distance(c, a), 2);
assert_eq!(lev_distance(b, c), 1);
assert_eq!(lev_distance(c, b), 1);
}
#[test]
fn test_find_best_match_for_name() {
use crate::with_default_globals;
with_default_globals(|| {
let input = vec![Symbol::intern("aaab"), Symbol::intern("aaabc")];
assert_eq!(
find_best_match_for_name(input.iter(), "aaaa", None),
Some(Symbol::intern("aaab"))
);
assert_eq!(
find_best_match_for_name(input.iter(), "1111111111", None),
None
);
let input = vec![Symbol::intern("aAAA")];
assert_eq!(
find_best_match_for_name(input.iter(), "AAAA", None),
Some(Symbol::intern("aAAA"))
);
let input = vec![Symbol::intern("AAAA")];
// Returns None because `lev_distance > max_dist / 3`
assert_eq!(
find_best_match_for_name(input.iter(), "aaaa", None),
None
);
let input = vec![Symbol::intern("AAAA")];
assert_eq!(
find_best_match_for_name(input.iter(), "aaaa", Some(4)),
Some(Symbol::intern("AAAA"))
);
})
}

160
src/libsyntax/util/parser_testing.rs
View file

@ -1,160 +0,0 @@
use crate::ast::{self, Ident};
use crate::source_map::FilePathMapping;
use crate::parse::{ParseSess, PResult, source_file_to_stream};
use crate::parse::new_parser_from_source_str;
use crate::parse::parser::Parser;
use crate::ptr::P;
use crate::tokenstream::TokenStream;
use std::iter::Peekable;
use std::path::PathBuf;
/// Map a string to tts, using a made-up filename:
pub fn string_to_stream(source_str: String) -> TokenStream {
let ps = ParseSess::new(FilePathMapping::empty());
source_file_to_stream(
&ps,
ps.source_map().new_source_file(PathBuf::from("bogofile").into(),
source_str,
), None).0
}
/// Map string to parser (via tts)
pub fn string_to_parser(ps: &ParseSess, source_str: String) -> Parser<'_> {
new_parser_from_source_str(ps, PathBuf::from("bogofile").into(), source_str)
}
fn with_error_checking_parse<'a, T, F>(s: String, ps: &'a ParseSess, f: F) -> T where
F: FnOnce(&mut Parser<'a>) -> PResult<'a, T>,
{
let mut p = string_to_parser(&ps, s);
let x = panictry!(f(&mut p));
p.sess.span_diagnostic.abort_if_errors();
x
}
/// Parse a string, return a crate.
pub fn string_to_crate(source_str : String) -> ast::Crate {
let ps = ParseSess::new(FilePathMapping::empty());
with_error_checking_parse(source_str, &ps, |p| {
p.parse_crate_mod()
})
}
/// Parse a string, return an expr
pub fn string_to_expr(source_str : String) -> P<ast::Expr> {
let ps = ParseSess::new(FilePathMapping::empty());
with_error_checking_parse(source_str, &ps, |p| {
p.parse_expr()
})
}
/// Parse a string, return an item
pub fn string_to_item(source_str : String) -> Option<P<ast::Item>> {
let ps = ParseSess::new(FilePathMapping::empty());
with_error_checking_parse(source_str, &ps, |p| {
p.parse_item()
})
}
/// Parse a string, return a pat. Uses "irrefutable"... which doesn't
/// (currently) affect parsing.
pub fn string_to_pat(source_str: String) -> P<ast::Pat> {
let ps = ParseSess::new(FilePathMapping::empty());
with_error_checking_parse(source_str, &ps, |p| {
p.parse_pat(None)
})
}
/// Converts a vector of strings to a vector of Ident's
pub fn strs_to_idents(ids: Vec<&str> ) -> Vec<Ident> {
ids.iter().map(|u| Ident::from_str(*u)).collect()
}
/// Does the given string match the pattern? whitespace in the first string
/// may be deleted or replaced with other whitespace to match the pattern.
/// This function is relatively Unicode-ignorant; fortunately, the careful design
/// of UTF-8 mitigates this ignorance. It doesn't do NKF-normalization(?).
pub fn matches_codepattern(a : &str, b : &str) -> bool {
let mut a_iter = a.chars().peekable();
let mut b_iter = b.chars().peekable();
loop {
let (a, b) = match (a_iter.peek(), b_iter.peek()) {
(None, None) => return true,
(None, _) => return false,
(Some(&a), None) => {
if is_pattern_whitespace(a) {
break // trailing whitespace check is out of loop for borrowck
} else {
return false
}
}
(Some(&a), Some(&b)) => (a, b)
};
if is_pattern_whitespace(a) && is_pattern_whitespace(b) {
// skip whitespace for a and b
scan_for_non_ws_or_end(&mut a_iter);
scan_for_non_ws_or_end(&mut b_iter);
} else if is_pattern_whitespace(a) {
// skip whitespace for a
scan_for_non_ws_or_end(&mut a_iter);
} else if a == b {
a_iter.next();
b_iter.next();
} else {
return false
}
}
// check if a has *only* trailing whitespace
a_iter.all(is_pattern_whitespace)
}
/// Advances the given peekable `Iterator` until it reaches a non-whitespace character
fn scan_for_non_ws_or_end<I: Iterator<Item = char>>(iter: &mut Peekable<I>) {
while iter.peek().copied().map(|c| is_pattern_whitespace(c)) == Some(true) {
iter.next();
}
}
pub fn is_pattern_whitespace(c: char) -> bool {
rustc_lexer::character_properties::is_whitespace(c)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn eqmodws() {
assert_eq!(matches_codepattern("",""),true);
assert_eq!(matches_codepattern("","a"),false);
assert_eq!(matches_codepattern("a",""),false);
assert_eq!(matches_codepattern("a","a"),true);
assert_eq!(matches_codepattern("a b","a \n\t\r b"),true);
assert_eq!(matches_codepattern("a b ","a \n\t\r b"),true);
assert_eq!(matches_codepattern("a b","a \n\t\r b "),false);
assert_eq!(matches_codepattern("a b","a b"),true);
assert_eq!(matches_codepattern("ab","a b"),false);
assert_eq!(matches_codepattern("a b","ab"),true);
assert_eq!(matches_codepattern(" a b","ab"),true);
}
#[test]
fn pattern_whitespace() {
assert_eq!(matches_codepattern("","\x0C"), false);
assert_eq!(matches_codepattern("a b ","a \u{0085}\n\t\r b"),true);
assert_eq!(matches_codepattern("a b","a \u{0085}\n\t\r b "),false);
}
#[test]
fn non_pattern_whitespace() {
// These have the property 'White_Space' but not 'Pattern_White_Space'
assert_eq!(matches_codepattern("a b","a\u{2002}b"), false);
assert_eq!(matches_codepattern("a b","a\u{2002}b"), false);
assert_eq!(matches_codepattern("\u{205F}a b","ab"), false);
assert_eq!(matches_codepattern("a \u{3000}b","ab"), false);
}
}

1
src/tools/tidy/src/unit_tests.rs
View file

@ -28,7 +28,6 @@ pub fn check(root_path: &Path, bad: &mut bool) {
let fixme = [
"liballoc",
"libstd",
"libsyntax",
];
let mut skip = |path: &Path| {