user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add a Rayon thread pool

Browse source
This commit is contained in:
John Kåre Alsaker 2018-04-26 00:49:52 +02:00
parent 3df199680a
commit 022dff47e3
13 changed files with 430 additions and 344 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/librustdoc/clean/mod.rs
View file

@ -697,7 +697,7 @@ impl<'a> FromIterator<&'a DocFragment> for String {
pub struct Attributes {
pub doc_strings: Vec<DocFragment>,
pub other_attrs: Vec<ast::Attribute>,
pub cfg: Option<Rc<Cfg>>,
pub cfg: Option<Arc<Cfg>>,
pub span: Option<syntax_pos::Span>,
/// map from Rust paths to resolved defs and potential URL fragments
pub links: Vec<(String, Option<DefId>, Option<String>)>,
@ -848,7 +848,7 @@ impl Attributes {
Attributes {
doc_strings,
other_attrs,
cfg: if cfg == Cfg::True { None } else { Some(Rc::new(cfg)) },
cfg: if cfg == Cfg::True { None } else { Some(Arc::new(cfg)) },
span: sp,
links: vec![],
}

303
src/librustdoc/core.rs
View file

@ -161,161 +161,162 @@ pub fn run_core(search_paths: SearchPaths,
edition,
..config::basic_options().clone()
};
let codemap = Lrc::new(codemap::CodeMap::new(sessopts.file_path_mapping()));
let emitter: Box<dyn Emitter + sync::Send> = match error_format {
ErrorOutputType::HumanReadable(color_config) => Box::new(
EmitterWriter::stderr(
color_config,
Some(codemap.clone()),
false,
sessopts.debugging_opts.teach,
).ui_testing(sessopts.debugging_opts.ui_testing)
),
ErrorOutputType::Json(pretty) => Box::new(
JsonEmitter::stderr(
None,
codemap.clone(),
pretty,
sessopts.debugging_opts.suggestion_applicability,
).ui_testing(sessopts.debugging_opts.ui_testing)
),
ErrorOutputType::Short(color_config) => Box::new(
EmitterWriter::stderr(color_config, Some(codemap.clone()), true, false)
),
};
let diagnostic_handler = errors::Handler::with_emitter_and_flags(
emitter,
errors::HandlerFlags {
can_emit_warnings: true,
treat_err_as_bug: false,
external_macro_backtrace: false,
..Default::default()
},
);
let mut sess = session::build_session_(
sessopts, cpath, diagnostic_handler, codemap,
);
let trans = rustc_driver::get_trans(&sess);
let cstore = Rc::new(CStore::new(trans.metadata_loader()));
rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
let mut cfg = config::build_configuration(&sess, config::parse_cfgspecs(cfgs));
target_features::add_configuration(&mut cfg, &sess, &*trans);
sess.parse_sess.config = cfg;
let control = &driver::CompileController::basic();
let krate = panictry!(driver::phase_1_parse_input(control, &sess, &input));
let name = ::rustc_trans_utils::link::find_crate_name(Some(&sess), &krate.attrs, &input);
let mut crate_loader = CrateLoader::new(&sess, &cstore, &name);
let resolver_arenas = resolve::Resolver::arenas();
let result = driver::phase_2_configure_and_expand_inner(&sess,
&cstore,
krate,
None,
&name,
None,
resolve::MakeGlobMap::No,
&resolver_arenas,
&mut crate_loader,
|_| Ok(()));
let driver::InnerExpansionResult {
mut hir_forest,
resolver,
..
} = abort_on_err(result, &sess);
// We need to hold on to the complete resolver, so we clone everything
// for the analysis passes to use. Suboptimal, but necessary in the
// current architecture.
let defs = resolver.definitions.clone();
let resolutions = ty::Resolutions {
freevars: resolver.freevars.clone(),
export_map: resolver.export_map.clone(),
trait_map: resolver.trait_map.clone(),
maybe_unused_trait_imports: resolver.maybe_unused_trait_imports.clone(),
maybe_unused_extern_crates: resolver.maybe_unused_extern_crates.clone(),
};
let analysis = ty::CrateAnalysis {
access_levels: Lrc::new(AccessLevels::default()),
name: name.to_string(),
glob_map: if resolver.make_glob_map { Some(resolver.glob_map.clone()) } else { None },
};
let arenas = AllArenas::new();
let hir_map = hir_map::map_crate(&sess, &*cstore, &mut hir_forest, &defs);
let output_filenames = driver::build_output_filenames(&input,
&None,
&None,
&[],
&sess);
let resolver = RefCell::new(resolver);
abort_on_err(driver::phase_3_run_analysis_passes(&*trans,
control,
&sess,
&*cstore,
hir_map,
analysis,
resolutions,
&arenas,
&name,
&output_filenames,
|tcx, analysis, _, result| {
if let Err(_) = result {
sess.fatal("Compilation failed, aborting rustdoc");
}
let ty::CrateAnalysis { access_levels, .. } = analysis;
// Convert from a NodeId set to a DefId set since we don't always have easy access
// to the map from defid -> nodeid
let access_levels = AccessLevels {
map: access_levels.map.iter()
.map(|(&k, &v)| (tcx.hir.local_def_id(k), v))
.collect()
driver::spawn_thread_pool(sessopts, move |sessopts| {
let codemap = Lrc::new(codemap::CodeMap::new(sessopts.file_path_mapping()));
let emitter: Box<dyn Emitter + sync::Send> = match error_format {
ErrorOutputType::HumanReadable(color_config) => Box::new(
EmitterWriter::stderr(
color_config,
Some(codemap.clone()),
false,
sessopts.debugging_opts.teach,
).ui_testing(sessopts.debugging_opts.ui_testing)
),
ErrorOutputType::Json(pretty) => Box::new(
JsonEmitter::stderr(
None,
codemap.clone(),
pretty,
sessopts.debugging_opts.suggestion_applicability,
).ui_testing(sessopts.debugging_opts.ui_testing)
),
ErrorOutputType::Short(color_config) => Box::new(
EmitterWriter::stderr(color_config, Some(codemap.clone()), true, false)
),
};
let send_trait = if crate_name == Some("core".to_string()) {
clean::get_trait_def_id(&tcx, &["marker", "Send"], true)
} else {
clean::get_trait_def_id(&tcx, &["core", "marker", "Send"], false)
let diagnostic_handler = errors::Handler::with_emitter_and_flags(
emitter,
errors::HandlerFlags {
can_emit_warnings: true,
treat_err_as_bug: false,
external_macro_backtrace: false,
..Default::default()
},
);
let mut sess = session::build_session_(
sessopts, cpath, diagnostic_handler, codemap,
);
let trans = rustc_driver::get_trans(&sess);
let cstore = Rc::new(CStore::new(trans.metadata_loader()));
rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
let mut cfg = config::build_configuration(&sess, config::parse_cfgspecs(cfgs));
target_features::add_configuration(&mut cfg, &sess, &*trans);
sess.parse_sess.config = cfg;
let control = &driver::CompileController::basic();
let krate = panictry!(driver::phase_1_parse_input(control, &sess, &input));
let name = ::rustc_trans_utils::link::find_crate_name(Some(&sess), &krate.attrs, &input);
let mut crate_loader = CrateLoader::new(&sess, &cstore, &name);
let resolver_arenas = resolve::Resolver::arenas();
let result = driver::phase_2_configure_and_expand_inner(&sess,
&cstore,
krate,
None,
&name,
None,
resolve::MakeGlobMap::No,
&resolver_arenas,
&mut crate_loader,
|_| Ok(()));
let driver::InnerExpansionResult {
mut hir_forest,
resolver,
..
} = abort_on_err(result, &sess);
// We need to hold on to the complete resolver, so we clone everything
// for the analysis passes to use. Suboptimal, but necessary in the
// current architecture.
let defs = resolver.definitions.clone();
let resolutions = ty::Resolutions {
freevars: resolver.freevars.clone(),
export_map: resolver.export_map.clone(),
trait_map: resolver.trait_map.clone(),
maybe_unused_trait_imports: resolver.maybe_unused_trait_imports.clone(),
maybe_unused_extern_crates: resolver.maybe_unused_extern_crates.clone(),
};
let analysis = ty::CrateAnalysis {
access_levels: Lrc::new(AccessLevels::default()),
name: name.to_string(),
glob_map: if resolver.make_glob_map { Some(resolver.glob_map.clone()) } else { None },
};
let ctxt = DocContext {
tcx,
resolver: &resolver,
crate_name,
cstore: cstore.clone(),
populated_all_crate_impls: Cell::new(false),
access_levels: RefCell::new(access_levels),
external_traits: Default::default(),
active_extern_traits: Default::default(),
renderinfo: Default::default(),
ty_substs: Default::default(),
lt_substs: Default::default(),
impl_trait_bounds: Default::default(),
mod_ids: Default::default(),
send_trait: send_trait,
fake_def_ids: RefCell::new(FxHashMap()),
all_fake_def_ids: RefCell::new(FxHashSet()),
generated_synthetics: RefCell::new(FxHashSet()),
};
debug!("crate: {:?}", tcx.hir.krate());
let arenas = AllArenas::new();
let hir_map = hir_map::map_crate(&sess, &*cstore, &mut hir_forest, &defs);
let output_filenames = driver::build_output_filenames(&input,
&None,
&None,
&[],
&sess);
let krate = {
let mut v = RustdocVisitor::new(&*cstore, &ctxt);
v.visit(tcx.hir.krate());
v.clean(&ctxt)
};
let resolver = RefCell::new(resolver);
(krate, ctxt.renderinfo.into_inner())
}), &sess)
abort_on_err(driver::phase_3_run_analysis_passes(&*trans,
control,
&sess,
&*cstore,
hir_map,
analysis,
resolutions,
&arenas,
&name,
&output_filenames,
|tcx, analysis, _, result| {
if let Err(_) = result {
sess.fatal("Compilation failed, aborting rustdoc");
}
let ty::CrateAnalysis { access_levels, .. } = analysis;
// Convert from a NodeId set to a DefId set since we don't always have easy access
// to the map from defid -> nodeid
let access_levels = AccessLevels {
map: access_levels.map.iter()
.map(|(&k, &v)| (tcx.hir.local_def_id(k), v))
.collect()
};
let send_trait = if crate_name == Some("core".to_string()) {
clean::get_trait_def_id(&tcx, &["marker", "Send"], true)
} else {
clean::get_trait_def_id(&tcx, &["core", "marker", "Send"], false)
};
let ctxt = DocContext {
tcx,
resolver: &resolver,
crate_name,
cstore: cstore.clone(),
populated_all_crate_impls: Cell::new(false),
access_levels: RefCell::new(access_levels),
external_traits: Default::default(),
active_extern_traits: Default::default(),
renderinfo: Default::default(),
ty_substs: Default::default(),
lt_substs: Default::default(),
impl_trait_bounds: Default::default(),
mod_ids: Default::default(),
send_trait: send_trait,
fake_def_ids: RefCell::new(FxHashMap()),
all_fake_def_ids: RefCell::new(FxHashSet()),
generated_synthetics: RefCell::new(FxHashSet()),
};
debug!("crate: {:?}", tcx.hir.krate());
let krate = {
let mut v = RustdocVisitor::new(&*cstore, &ctxt);
v.visit(tcx.hir.krate());
v.clean(&ctxt)
};
(krate, ctxt.renderinfo.into_inner())
}), &sess)
})
}

2
src/librustdoc/lib.rs
View file

@ -26,6 +26,8 @@
#![feature(vec_remove_item)]
#![feature(entry_and_modify)]
#![recursion_limit="256"]
extern crate arena;
extern crate getopts;
extern crate env_logger;

8
src/librustdoc/passes/propagate_doc_cfg.rs
View file

@ -8,7 +8,7 @@
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use std::rc::Rc;
use std::sync::Arc;
use clean::{Crate, Item};
use clean::cfg::Cfg;
@ -20,7 +20,7 @@ pub fn propagate_doc_cfg(cr: Crate) -> PluginResult {
}
struct CfgPropagator {
parent_cfg: Option<Rc<Cfg>>,
parent_cfg: Option<Arc<Cfg>>,
}
impl DocFolder for CfgPropagator {
@ -31,8 +31,8 @@ impl DocFolder for CfgPropagator {
(None, None) => None,
(Some(rc), None) | (None, Some(rc)) => Some(rc),
(Some(mut a), Some(b)) => {
let b = Rc::try_unwrap(b).unwrap_or_else(|rc| Cfg::clone(&rc));
*Rc::make_mut(&mut a) &= b;
let b = Arc::try_unwrap(b).unwrap_or_else(|rc| Cfg::clone(&rc));
*Arc::make_mut(&mut a) &= b;
Some(a)
}
};

319
src/librustdoc/test.rs
View file

@ -85,77 +85,80 @@ pub fn run(input_path: &Path,
edition,
..config::basic_options().clone()
};
driver::spawn_thread_pool(sessopts, |sessopts| {
let codemap = Lrc::new(CodeMap::new(sessopts.file_path_mapping()));
let handler =
errors::Handler::with_tty_emitter(ColorConfig::Auto,
true, false,
Some(codemap.clone()));
let codemap = Lrc::new(CodeMap::new(sessopts.file_path_mapping()));
let handler =
errors::Handler::with_tty_emitter(ColorConfig::Auto,
true, false,
Some(codemap.clone()));
let mut sess = session::build_session_(
sessopts, Some(input_path.to_owned()), handler, codemap.clone(),
);
let trans = rustc_driver::get_trans(&sess);
let cstore = CStore::new(trans.metadata_loader());
rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
let mut sess = session::build_session_(
sessopts, Some(input_path.to_owned()), handler, codemap.clone(),
);
let trans = rustc_driver::get_trans(&sess);
let cstore = CStore::new(trans.metadata_loader());
rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
let mut cfg = config::build_configuration(&sess, config::parse_cfgspecs(cfgs.clone()));
target_features::add_configuration(&mut cfg, &sess, &*trans);
sess.parse_sess.config = cfg;
let mut cfg = config::build_configuration(&sess, config::parse_cfgspecs(cfgs.clone()));
target_features::add_configuration(&mut cfg, &sess, &*trans);
sess.parse_sess.config = cfg;
let krate = panictry!(driver::phase_1_parse_input(&driver::CompileController::basic(),
&sess,
&input));
let driver::ExpansionResult { defs, mut hir_forest, .. } = {
phase_2_configure_and_expand(
&sess,
&cstore,
krate,
None,
"rustdoc-test",
None,
MakeGlobMap::No,
|_| Ok(()),
).expect("phase_2_configure_and_expand aborted in rustdoc!")
};
let crate_name = crate_name.unwrap_or_else(|| {
::rustc_trans_utils::link::find_crate_name(None, &hir_forest.krate().attrs, &input)
});
let mut opts = scrape_test_config(hir_forest.krate());
opts.display_warnings |= display_warnings;
let mut collector = Collector::new(crate_name,
cfgs,
libs,
cg,
externs,
false,
opts,
maybe_sysroot,
Some(codemap),
None,
linker,
edition);
{
let map = hir::map::map_crate(&sess, &cstore, &mut hir_forest, &defs);
let krate = map.krate();
let mut hir_collector = HirCollector {
sess: &sess,
collector: &mut collector,
map: &map
let krate = panictry!(driver::phase_1_parse_input(&driver::CompileController::basic(),
&sess,
&input));
let driver::ExpansionResult { defs, mut hir_forest, .. } = {
phase_2_configure_and_expand(
&sess,
&cstore,
krate,
None,
"rustdoc-test",
None,
MakeGlobMap::No,
|_| Ok(()),
).expect("phase_2_configure_and_expand aborted in rustdoc!")
};
hir_collector.visit_testable("".to_string(), &krate.attrs, |this| {
intravisit::walk_crate(this, krate);
let crate_name = crate_name.unwrap_or_else(|| {
::rustc_trans_utils::link::find_crate_name(None, &hir_forest.krate().attrs, &input)
});
}
let mut opts = scrape_test_config(hir_forest.krate());
opts.display_warnings |= display_warnings;
let mut collector = Collector::new(
crate_name,
cfgs,
libs,
cg,
externs,
false,
opts,
maybe_sysroot,
Some(codemap),
None,
linker,
edition
);
test_args.insert(0, "rustdoctest".to_string());
{
let map = hir::map::map_crate(&sess, &cstore, &mut hir_forest, &defs);
let krate = map.krate();
let mut hir_collector = HirCollector {
sess: &sess,
collector: &mut collector,
map: &map
};
hir_collector.visit_testable("".to_string(), &krate.attrs, |this| {
intravisit::walk_crate(this, krate);
});
}
testing::test_main(&test_args,
collector.tests.into_iter().collect(),
testing::Options::new().display_output(display_warnings));
0
test_args.insert(0, "rustdoctest".to_string());
testing::test_main(&test_args,
collector.tests.into_iter().collect(),
testing::Options::new().display_output(display_warnings));
0
})
}
// Look for #![doc(test(no_crate_inject))], used by crates in the std facade
@ -229,102 +232,106 @@ fn run_test(test: &str, cratename: &str, filename: &FileName, line: usize,
..config::basic_options().clone()
};
// Shuffle around a few input and output handles here. We're going to pass
// an explicit handle into rustc to collect output messages, but we also
// want to catch the error message that rustc prints when it fails.
//
// We take our thread-local stderr (likely set by the test runner) and replace
// it with a sink that is also passed to rustc itself. When this function
// returns the output of the sink is copied onto the output of our own thread.
//
// The basic idea is to not use a default Handler for rustc, and then also
// not print things by default to the actual stderr.
struct Sink(Arc<Mutex<Vec<u8>>>);
impl Write for Sink {
fn write(&mut self, data: &[u8]) -> io::Result<usize> {
Write::write(&mut *self.0.lock().unwrap(), data)
let (libdir, outdir) = driver::spawn_thread_pool(sessopts, |sessopts| {
// Shuffle around a few input and output handles here. We're going to pass
// an explicit handle into rustc to collect output messages, but we also
// want to catch the error message that rustc prints when it fails.
//
// We take our thread-local stderr (likely set by the test runner) and replace
// it with a sink that is also passed to rustc itself. When this function
// returns the output of the sink is copied onto the output of our own thread.
//
// The basic idea is to not use a default Handler for rustc, and then also
// not print things by default to the actual stderr.
struct Sink(Arc<Mutex<Vec<u8>>>);
impl Write for Sink {
fn write(&mut self, data: &[u8]) -> io::Result<usize> {
Write::write(&mut *self.0.lock().unwrap(), data)
}
fn flush(&mut self) -> io::Result<()> { Ok(()) }
}
fn flush(&mut self) -> io::Result<()> { Ok(()) }
}
struct Bomb(Arc<Mutex<Vec<u8>>>, Box<Write+Send>);
impl Drop for Bomb {
fn drop(&mut self) {
let _ = self.1.write_all(&self.0.lock().unwrap());
}
}
let data = Arc::new(Mutex::new(Vec::new()));
let codemap = Lrc::new(CodeMap::new_doctest(
sessopts.file_path_mapping(), filename.clone(), line as isize - line_offset as isize
));
let emitter = errors::emitter::EmitterWriter::new(box Sink(data.clone()),
Some(codemap.clone()),
false,
false);
let old = io::set_panic(Some(box Sink(data.clone())));
let _bomb = Bomb(data.clone(), old.unwrap_or(box io::stdout()));
// Compile the code
let diagnostic_handler = errors::Handler::with_emitter(true, false, box emitter);
let mut sess = session::build_session_(
sessopts, None, diagnostic_handler, codemap,
);
let trans = rustc_driver::get_trans(&sess);
let cstore = CStore::new(trans.metadata_loader());
rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
let outdir = Mutex::new(TempDir::new("rustdoctest").ok().expect("rustdoc needs a tempdir"));
let libdir = sess.target_filesearch(PathKind::All).get_lib_path();
let mut control = driver::CompileController::basic();
let mut cfg = config::build_configuration(&sess, config::parse_cfgspecs(cfgs.clone()));
target_features::add_configuration(&mut cfg, &sess, &*trans);
sess.parse_sess.config = cfg;
let out = Some(outdir.lock().unwrap().path().to_path_buf());
if no_run {
control.after_analysis.stop = Compilation::Stop;
}
let res = panic::catch_unwind(AssertUnwindSafe(|| {
driver::compile_input(
trans,
&sess,
&cstore,
&None,
&input,
&out,
&None,
None,
&control
)
}));
let compile_result = match res {
Ok(Ok(())) | Ok(Err(CompileIncomplete::Stopped)) => Ok(()),
Err(_) | Ok(Err(CompileIncomplete::Errored(_))) => Err(())
};
match (compile_result, compile_fail) {
(Ok(()), true) => {
panic!("test compiled while it wasn't supposed to")
}
(Ok(()), false) => {}
(Err(()), true) => {
if error_codes.len() > 0 {
let out = String::from_utf8(data.lock().unwrap().to_vec()).unwrap();
error_codes.retain(|err| !out.contains(err));
struct Bomb(Arc<Mutex<Vec<u8>>>, Box<Write+Send>);
impl Drop for Bomb {
fn drop(&mut self) {
let _ = self.1.write_all(&self.0.lock().unwrap());
}
}
(Err(()), false) => {
panic!("couldn't compile the test")
}
}
let data = Arc::new(Mutex::new(Vec::new()));
let codemap = Lrc::new(CodeMap::new_doctest(
sessopts.file_path_mapping(), filename.clone(), line as isize - line_offset as isize
));
let emitter = errors::emitter::EmitterWriter::new(box Sink(data.clone()),
Some(codemap.clone()),
false,
false);
let old = io::set_panic(Some(box Sink(data.clone())));
let _bomb = Bomb(data.clone(), old.unwrap_or(box io::stdout()));
if error_codes.len() > 0 {
panic!("Some expected error codes were not found: {:?}", error_codes);
}
// Compile the code
let diagnostic_handler = errors::Handler::with_emitter(true, false, box emitter);
let mut sess = session::build_session_(
sessopts, None, diagnostic_handler, codemap,
);
let trans = rustc_driver::get_trans(&sess);
let cstore = CStore::new(trans.metadata_loader());
rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
let outdir = Mutex::new(TempDir::new("rustdoctest").ok().expect("rustdoc needs a tempdir"));
let libdir = sess.target_filesearch(PathKind::All).get_lib_path();
let mut control = driver::CompileController::basic();
let mut cfg = config::build_configuration(&sess, config::parse_cfgspecs(cfgs.clone()));
target_features::add_configuration(&mut cfg, &sess, &*trans);
sess.parse_sess.config = cfg;
let out = Some(outdir.lock().unwrap().path().to_path_buf());
if no_run {
control.after_analysis.stop = Compilation::Stop;
}
let res = panic::catch_unwind(AssertUnwindSafe(|| {
driver::compile_input(
trans,
&sess,
&cstore,
&None,
&input,
&out,
&None,
None,
&control
)
}));
let compile_result = match res {
Ok(Ok(())) | Ok(Err(CompileIncomplete::Stopped)) => Ok(()),
Err(_) | Ok(Err(CompileIncomplete::Errored(_))) => Err(())
};
match (compile_result, compile_fail) {
(Ok(()), true) => {
panic!("test compiled while it wasn't supposed to")
}
(Ok(()), false) => {}
(Err(()), true) => {
if error_codes.len() > 0 {
let out = String::from_utf8(data.lock().unwrap().to_vec()).unwrap();
error_codes.retain(|err| !out.contains(err));
}
}
(Err(()), false) => {
panic!("couldn't compile the test")
}
}
if error_codes.len() > 0 {
panic!("Some expected error codes were not found: {:?}", error_codes);
}
(libdir, outdir)
});
if no_run { return }