From 99d02fe583f4747f67debc1973a3eb3ca62e2005 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 10 Sep 2018 20:14:31 +0300 Subject: [PATCH 01/18] start query-based modules --- crates/libanalysis/Cargo.toml | 1 + crates/libanalysis/src/db.rs | 121 +++++++++++++++ crates/libanalysis/src/lib.rs | 3 + crates/libanalysis/src/module_map.rs | 59 ++++---- crates/libanalysis/src/module_map_db.rs | 189 ++++++++++++++++++++++++ crates/libanalysis/tests/tests.rs | 35 +++-- crates/server/Cargo.toml | 2 +- 7 files changed, 365 insertions(+), 45 deletions(-) create mode 100644 crates/libanalysis/src/db.rs create mode 100644 crates/libanalysis/src/module_map_db.rs diff --git a/crates/libanalysis/Cargo.toml b/crates/libanalysis/Cargo.toml index 4d565e95fccd..4c92951b148f 100644 --- a/crates/libanalysis/Cargo.toml +++ b/crates/libanalysis/Cargo.toml @@ -11,6 +11,7 @@ parking_lot = "0.6.3" once_cell = "0.1.5" rayon = "1.0.2" fst = "0.3.1" +im = "12.0.0" libsyntax2 = { path = "../libsyntax2" } libeditor = { path = "../libeditor" } diff --git a/crates/libanalysis/src/db.rs b/crates/libanalysis/src/db.rs new file mode 100644 index 000000000000..335c79e76da5 --- /dev/null +++ b/crates/libanalysis/src/db.rs @@ -0,0 +1,121 @@ +use std::{ + hash::Hash, + sync::Arc, +}; +use libsyntax2::{File}; +use im; +use { + FileId, + imp::{FileResolverImp}, +}; + +#[derive(Clone)] +pub(crate) struct Db { + file_resolver: FileResolverImp, + files: im::HashMap>, +} + +impl Db { + pub(crate) fn new() -> Db { + Db { + file_resolver: FileResolverImp::default(), + files: im::HashMap::new(), + } + } + pub(crate) fn change_file(&mut self, file_id: FileId, text: Option) { + match text { + None => { + self.files.remove(&file_id); + } + Some(text) => { + self.files.insert(file_id, Arc::new(text)); + } + } + } + pub(crate) fn set_file_resolver(&mut self, file_resolver: FileResolverImp) { + self.file_resolver = file_resolver + } + pub(crate) fn query_ctx(&self) -> QueryCtx { + QueryCtx { db: self.clone() } + } +} + +pub(crate) struct QueryCtx { + db: Db +} + +impl QueryCtx { + pub(crate) fn get(&self, params: &Q::Params) -> Q::Output { + Q::get(self, params) + } +} + +pub(crate) trait Query { + const ID: u32; + type Params: Hash; + type Output; +} + +pub(crate) trait Get: Query { + fn get(ctx: &QueryCtx, params: &Self::Params) -> Self::Output; +} + +impl Get for T { + fn get(ctx: &QueryCtx, params: &Self::Params) -> Self::Output { + Self::eval(ctx, params) + } +} + +pub(crate) trait Eval: Query { + fn eval(ctx: &QueryCtx, params: &Self::Params) -> Self::Output; +} + +pub(crate) struct DbFiles { + db: Db, +} + +impl DbFiles { + pub(crate) fn iter<'a>(&'a self) -> impl Iterator + 'a { + self.db.files.keys().cloned() + } + pub(crate) fn file_resolver(&self) -> FileResolverImp { + self.db.file_resolver.clone() + } +} + +pub(crate) enum Files {} +impl Query for Files { + const ID: u32 = 1; + type Params = (); + type Output = DbFiles; +} +impl Get for Files { + fn get(ctx: &QueryCtx, _params: &()) -> DbFiles { + DbFiles { db: ctx.db.clone() } + } +} + +enum FileText {} +impl Query for FileText { + const ID: u32 = 10; + type Params = FileId; + type Output = Arc; +} +impl Get for FileText { + fn get(ctx: &QueryCtx, file_id: &FileId) -> Arc { + ctx.db.files[file_id].clone() + } +} + +pub(crate) enum FileSyntax {} +impl Query for FileSyntax { + const ID: u32 = 20; + type Params = FileId; + type Output = File; +} +impl Eval for FileSyntax { + fn eval(ctx: &QueryCtx, file_id: &FileId) -> File { + let text = ctx.get::(file_id); + File::parse(&text) + } +} diff --git a/crates/libanalysis/src/lib.rs b/crates/libanalysis/src/lib.rs index 80cde079f9c1..68cf31e08f57 100644 --- a/crates/libanalysis/src/lib.rs +++ b/crates/libanalysis/src/lib.rs @@ -9,12 +9,15 @@ extern crate rayon; extern crate relative_path; #[macro_use] extern crate crossbeam_channel; +extern crate im; mod symbol_index; mod module_map; +mod module_map_db; mod imp; mod job; mod roots; +mod db; use std::{ sync::Arc, diff --git a/crates/libanalysis/src/module_map.rs b/crates/libanalysis/src/module_map.rs index 9acebd6e2647..79b88cac2ff2 100644 --- a/crates/libanalysis/src/module_map.rs +++ b/crates/libanalysis/src/module_map.rs @@ -244,31 +244,38 @@ impl Link { self.points_to = Vec::new(); return; } - - let mod_name = file_resolver.file_stem(self.owner.0); - let is_dir_owner = - mod_name == "mod" || mod_name == "lib" || mod_name == "main"; - - let file_mod = RelativePathBuf::from(format!("../{}.rs", self.name())); - let dir_mod = RelativePathBuf::from(format!("../{}/mod.rs", self.name())); - if is_dir_owner { - self.points_to = [&file_mod, &dir_mod].iter() - .filter_map(|path| file_resolver.resolve(self.owner.0, path)) - .map(ModuleId) - .collect(); - self.problem = if self.points_to.is_empty() { - Some(Problem::UnresolvedModule { - candidate: file_mod, - }) - } else { - None - } - } else { - self.points_to = Vec::new(); - self.problem = Some(Problem::NotDirOwner { - move_to: RelativePathBuf::from(format!("../{}/mod.rs", mod_name)), - candidate: file_mod, - }); - } + let (points_to, problem) = resolve_submodule(self.owner.0, &self.name(), file_resolver); + self.problem = problem; + self.points_to = points_to.into_iter().map(ModuleId).collect(); } } + +pub(crate) fn resolve_submodule(file_id: FileId, name: &SmolStr, file_resolver: &FileResolverImp) -> (Vec, Option) { + let mod_name = file_resolver.file_stem(file_id); + let is_dir_owner = + mod_name == "mod" || mod_name == "lib" || mod_name == "main"; + + let file_mod = RelativePathBuf::from(format!("../{}.rs", name)); + let dir_mod = RelativePathBuf::from(format!("../{}/mod.rs", name)); + let points_to: Vec; + let problem: Option; + if is_dir_owner { + points_to = [&file_mod, &dir_mod].iter() + .filter_map(|path| file_resolver.resolve(file_id, path)) + .collect(); + problem = if points_to.is_empty() { + Some(Problem::UnresolvedModule { + candidate: file_mod, + }) + } else { + None + } + } else { + points_to = Vec::new(); + problem = Some(Problem::NotDirOwner { + move_to: RelativePathBuf::from(format!("../{}/mod.rs", mod_name)), + candidate: file_mod, + }); + } + (points_to, problem) +} diff --git a/crates/libanalysis/src/module_map_db.rs b/crates/libanalysis/src/module_map_db.rs new file mode 100644 index 000000000000..1ef87ab3f2f8 --- /dev/null +++ b/crates/libanalysis/src/module_map_db.rs @@ -0,0 +1,189 @@ +use std::sync::Arc; +use { + FileId, + db::{Query, Eval, QueryCtx, FileSyntax, Files}, + module_map::resolve_submodule, +}; + +enum ModuleDescr {} +impl Query for ModuleDescr { + const ID: u32 = 30; + type Params = FileId; + type Output = Arc; +} + +enum ResolveSubmodule {} +impl Query for ResolveSubmodule { + const ID: u32 = 31; + type Params = (FileId, descr::Submodule); + type Output = Arc>; +} + +enum ParentModule {} +impl Query for ParentModule { + const ID: u32 = 40; + type Params = FileId; + type Output = Arc>; +} + +impl Eval for ModuleDescr { + fn eval(ctx: &QueryCtx, file_id: &FileId) -> Arc { + let file = ctx.get::(file_id); + Arc::new(descr::ModuleDescr::new(file.ast())) + } +} + +impl Eval for ResolveSubmodule { + fn eval(ctx: &QueryCtx, &(file_id, ref submodule): &(FileId, descr::Submodule)) -> Arc> { + let files = ctx.get::(&()); + let res = resolve_submodule(file_id, &submodule.name, &files.file_resolver()).0; + Arc::new(res) + } +} + +impl Eval for ParentModule { + fn eval(ctx: &QueryCtx, file_id: &FileId) -> Arc> { + let files = ctx.get::(&()); + let res = files.iter() + .map(|parent_id| (parent_id, ctx.get::(&parent_id))) + .filter(|(parent_id, descr)| { + descr.submodules.iter() + .any(|subm| { + ctx.get::(&(*parent_id, subm.clone())) + .iter() + .any(|it| it == file_id) + }) + }) + .map(|(id, _)| id) + .collect(); + Arc::new(res) + } +} + +mod descr { + use libsyntax2::{ + SmolStr, + ast::{self, NameOwner}, + }; + + pub struct ModuleDescr { + pub submodules: Vec + } + + impl ModuleDescr { + pub fn new(root: ast::Root) -> ModuleDescr { + let submodules = root + .modules() + .filter_map(|module| { + let name = module.name()?.text(); + if !module.has_semi() { + return None; + } + Some(Submodule { name }) + }).collect(); + + ModuleDescr { submodules } } + } + + #[derive(Clone, Hash)] + pub struct Submodule { + pub name: SmolStr, + } + +} + +#[cfg(test)] +mod tests { + use super::*; + use im; + use relative_path::{RelativePath, RelativePathBuf}; + use { + db::Db, + imp::FileResolverImp, + FileId, FileResolver, + }; + + #[derive(Debug)] + struct FileMap(im::HashMap); + + impl FileResolver for FileMap { + fn file_stem(&self, file_id: FileId) -> String { + self.0[&file_id].file_stem().unwrap().to_string() + } + fn resolve(&self, file_id: FileId, rel: &RelativePath) -> Option { + let path = self.0[&file_id].join(rel).normalize(); + self.0.iter() + .filter_map(|&(id, ref p)| Some(id).filter(|_| p == &path)) + .next() + } + } + + struct Fixture { + next_file_id: u32, + fm: im::HashMap, + db: Db, + } + + impl Fixture { + fn new() -> Fixture { + Fixture { + next_file_id: 1, + fm: im::HashMap::new(), + db: Db::new(), + } + } + fn add_file(&mut self, path: &str, text: &str) -> FileId { + assert!(path.starts_with("/")); + let file_id = FileId(self.next_file_id); + self.next_file_id += 1; + self.fm.insert(file_id, RelativePathBuf::from(&path[1..])); + self.db.change_file(file_id, Some(text.to_string())); + self.db.set_file_resolver(FileResolverImp::new( + Arc::new(FileMap(self.fm.clone())) + )); + + file_id + } + fn remove_file(&mut self, file_id: FileId) { + self.fm.remove(&file_id); + self.db.change_file(file_id, None); + self.db.set_file_resolver(FileResolverImp::new( + Arc::new(FileMap(self.fm.clone())) + )) + } + fn change_file(&mut self, file_id: FileId, new_text: &str) { + self.db.change_file(file_id, Some(new_text.to_string())); + } + fn check_parent_modules(&self, file_id: FileId, expected: &[FileId]) { + let ctx = self.db.query_ctx(); + let actual = ctx.get::(&file_id); + assert_eq!(actual.as_slice(), expected); + } + } + + #[test] + fn test_parent_module() { + let mut f = Fixture::new(); + let foo = f.add_file("/foo.rs", ""); + f.check_parent_modules(foo, &[]); + + let lib = f.add_file("/lib.rs", "mod foo;"); + f.check_parent_modules(foo, &[lib]); + + f.change_file(lib, ""); + f.check_parent_modules(foo, &[]); + + f.change_file(lib, "mod foo;"); + f.check_parent_modules(foo, &[lib]); + + f.change_file(lib, "mod bar;"); + f.check_parent_modules(foo, &[]); + + f.change_file(lib, "mod foo;"); + f.check_parent_modules(foo, &[lib]); + + f.remove_file(lib); + f.check_parent_modules(foo, &[]); + } + +} diff --git a/crates/libanalysis/tests/tests.rs b/crates/libanalysis/tests/tests.rs index 00efe059ccc5..547f85958ea7 100644 --- a/crates/libanalysis/tests/tests.rs +++ b/crates/libanalysis/tests/tests.rs @@ -14,24 +14,6 @@ use test_utils::assert_eq_dbg; #[derive(Debug)] struct FileMap(Vec<(FileId, RelativePathBuf)>); -fn analysis_host(files: &'static [(&'static str, &'static str)]) -> AnalysisHost { - let mut host = AnalysisHost::new(); - let mut file_map = Vec::new(); - for (id, &(path, contents)) in files.iter().enumerate() { - let file_id = FileId((id + 1) as u32); - assert!(path.starts_with('/')); - let path = RelativePathBuf::from_path(&path[1..]).unwrap(); - host.change_file(file_id, Some(contents.to_string())); - file_map.push((file_id, path)); - } - host.set_file_resolver(Arc::new(FileMap(file_map))); - host -} - -fn analysis(files: &'static [(&'static str, &'static str)]) -> Analysis { - analysis_host(files).analysis() -} - impl FileMap { fn iter<'a>(&'a self) -> impl Iterator + 'a { self.0.iter().map(|(id, path)| (*id, path.as_relative_path())) @@ -56,6 +38,23 @@ impl FileResolver for FileMap { } } +fn analysis_host(files: &'static [(&'static str, &'static str)]) -> AnalysisHost { + let mut host = AnalysisHost::new(); + let mut file_map = Vec::new(); + for (id, &(path, contents)) in files.iter().enumerate() { + let file_id = FileId((id + 1) as u32); + assert!(path.starts_with('/')); + let path = RelativePathBuf::from_path(&path[1..]).unwrap(); + host.change_file(file_id, Some(contents.to_string())); + file_map.push((file_id, path)); + } + host.set_file_resolver(Arc::new(FileMap(file_map))); + host +} + +fn analysis(files: &'static [(&'static str, &'static str)]) -> Analysis { + analysis_host(files).analysis() +} #[test] fn test_resolve_module() { diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 9aeea9a9b1d9..fc20730b8880 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -17,7 +17,7 @@ log = "0.4.3" url_serde = "0.2.0" languageserver-types = "0.49.0" walkdir = "2.2.0" -im = { version = "11.0.1", features = ["arc"] } +im = "12.0.0" cargo_metadata = "0.6.0" text_unit = { version = "0.1.2", features = ["serde"] } smol_str = { version = "0.1.5", features = ["serde"] } From 3ae3b3eb0682a4550578b4c35dc6e099d8a04e66 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 10 Sep 2018 20:48:32 +0300 Subject: [PATCH 02/18] initial query tracing --- crates/libanalysis/src/db.rs | 28 +++++++++++++++-- crates/libanalysis/src/module_map_db.rs | 40 ++++++++++++++++++------- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/crates/libanalysis/src/db.rs b/crates/libanalysis/src/db.rs index 335c79e76da5..5e3c8fb7ac2b 100644 --- a/crates/libanalysis/src/db.rs +++ b/crates/libanalysis/src/db.rs @@ -1,6 +1,7 @@ use std::{ hash::Hash, sync::Arc, + cell::RefCell, }; use libsyntax2::{File}; use im; @@ -36,17 +37,38 @@ impl Db { self.file_resolver = file_resolver } pub(crate) fn query_ctx(&self) -> QueryCtx { - QueryCtx { db: self.clone() } + QueryCtx { + db: self.clone(), + trace: RefCell::new(Vec::new()), + } } } pub(crate) struct QueryCtx { - db: Db + db: Db, + pub(crate) trace: RefCell>, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct TraceEvent { + pub(crate) query_id: u32, + pub(crate) kind: TraceEventKind +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TraceEventKind { + Start, Finish } impl QueryCtx { pub(crate) fn get(&self, params: &Q::Params) -> Q::Output { - Q::get(self, params) + self.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Start }); + let res = Q::get(self, params); + self.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Finish }); + res + } + fn trace(&self, event: TraceEvent) { + self.trace.borrow_mut().push(event) } } diff --git a/crates/libanalysis/src/module_map_db.rs b/crates/libanalysis/src/module_map_db.rs index 1ef87ab3f2f8..25dbe8dd4274 100644 --- a/crates/libanalysis/src/module_map_db.rs +++ b/crates/libanalysis/src/module_map_db.rs @@ -94,14 +94,15 @@ mod descr { #[cfg(test)] mod tests { - use super::*; + use std::collections::HashMap; use im; use relative_path::{RelativePath, RelativePathBuf}; use { - db::Db, + db::{Query, Db, TraceEventKind}, imp::FileResolverImp, FileId, FileResolver, }; + use super::*; #[derive(Debug)] struct FileMap(im::HashMap); @@ -154,10 +155,29 @@ mod tests { fn change_file(&mut self, file_id: FileId, new_text: &str) { self.db.change_file(file_id, Some(new_text.to_string())); } - fn check_parent_modules(&self, file_id: FileId, expected: &[FileId]) { + fn check_parent_modules( + &self, + file_id: FileId, + expected: &[FileId], + queries: &[(u32, u64)] + ) { let ctx = self.db.query_ctx(); let actual = ctx.get::(&file_id); assert_eq!(actual.as_slice(), expected); + let mut counts = HashMap::new(); + ctx.trace.borrow().iter() + .filter(|event| event.kind == TraceEventKind::Start) + .for_each(|event| *counts.entry(event.query_id).or_insert(0) += 1); + for &(query_id, expected_count) in queries.iter() { + let actual_count = *counts.get(&query_id).unwrap_or(&0); + assert_eq!( + actual_count, + expected_count, + "counts for {} differ", + query_id, + ) + } + } } @@ -165,25 +185,25 @@ mod tests { fn test_parent_module() { let mut f = Fixture::new(); let foo = f.add_file("/foo.rs", ""); - f.check_parent_modules(foo, &[]); + f.check_parent_modules(foo, &[], &[(FileSyntax::ID, 1)]); let lib = f.add_file("/lib.rs", "mod foo;"); - f.check_parent_modules(foo, &[lib]); + f.check_parent_modules(foo, &[lib], &[(FileSyntax::ID, 2)]); f.change_file(lib, ""); - f.check_parent_modules(foo, &[]); + f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 2)]); f.change_file(lib, "mod foo;"); - f.check_parent_modules(foo, &[lib]); + f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 2)]); f.change_file(lib, "mod bar;"); - f.check_parent_modules(foo, &[]); + f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 2)]); f.change_file(lib, "mod foo;"); - f.check_parent_modules(foo, &[lib]); + f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 2)]); f.remove_file(lib); - f.check_parent_modules(foo, &[]); + f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 1)]); } } From db14b4270c0b328f89a3a2262f2814b2f80b083c Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 10 Sep 2018 21:53:33 +0300 Subject: [PATCH 03/18] Add simplisitc global modification caching --- crates/libanalysis/src/db.rs | 116 +++++++++++++++++++----- crates/libanalysis/src/lib.rs | 2 +- crates/libanalysis/src/module_map_db.rs | 24 +++-- 3 files changed, 111 insertions(+), 31 deletions(-) diff --git a/crates/libanalysis/src/db.rs b/crates/libanalysis/src/db.rs index 5e3c8fb7ac2b..31c73c40287b 100644 --- a/crates/libanalysis/src/db.rs +++ b/crates/libanalysis/src/db.rs @@ -2,50 +2,95 @@ use std::{ hash::Hash, sync::Arc, cell::RefCell, + fmt::Debug, }; +use parking_lot::Mutex; use libsyntax2::{File}; use im; use { FileId, imp::{FileResolverImp}, + module_map_db::ModuleDescr, }; -#[derive(Clone)] -pub(crate) struct Db { - file_resolver: FileResolverImp, - files: im::HashMap>, +#[derive(Debug)] +pub(crate) struct DbHost { + db: Arc, } -impl Db { - pub(crate) fn new() -> Db { - Db { +impl DbHost { + pub(crate) fn new() -> DbHost { + let db = Db { file_resolver: FileResolverImp::default(), files: im::HashMap::new(), - } + cache: Mutex::new(Cache::new()) + }; + DbHost { db: Arc::new(db) } } pub(crate) fn change_file(&mut self, file_id: FileId, text: Option) { + let db = self.db_mut(); match text { None => { - self.files.remove(&file_id); + db.files.remove(&file_id); } Some(text) => { - self.files.insert(file_id, Arc::new(text)); + db.files.insert(file_id, Arc::new(text)); } } } pub(crate) fn set_file_resolver(&mut self, file_resolver: FileResolverImp) { - self.file_resolver = file_resolver + let db = self.db_mut(); + db.file_resolver = file_resolver } pub(crate) fn query_ctx(&self) -> QueryCtx { QueryCtx { - db: self.clone(), + db: Arc::clone(&self.db), trace: RefCell::new(Vec::new()), } } + fn db_mut(&mut self) -> &mut Db { + // NB: this "forks" the database & clears the cache + let db = Arc::make_mut(&mut self.db); + *db.cache.get_mut() = Default::default(); + db + } +} + +#[derive(Debug)] +pub(crate) struct Db { + file_resolver: FileResolverImp, + files: im::HashMap>, + cache: Mutex, +} + +impl Clone for Db { + fn clone(&self) -> Db { + Db { + file_resolver: self.file_resolver.clone(), + files: self.files.clone(), + cache: Mutex::new(Cache::new()), + } + } +} + +#[derive(Clone, Default, Debug)] +pub(crate) struct Cache { + pub(crate) module_descr: QueryCache +} +#[allow(type_alias_bounds)] +pub(crate) type QueryCache = im::HashMap< + ::Params, + ::Output +>; + +impl Cache { + fn new() -> Cache { + Default::default() + } } pub(crate) struct QueryCtx { - db: Db, + db: Arc, pub(crate) trace: RefCell>, } @@ -62,9 +107,7 @@ pub(crate) enum TraceEventKind { impl QueryCtx { pub(crate) fn get(&self, params: &Q::Params) -> Q::Output { - self.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Start }); let res = Q::get(self, params); - self.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Finish }); res } fn trace(&self, event: TraceEvent) { @@ -74,26 +117,55 @@ impl QueryCtx { pub(crate) trait Query { const ID: u32; - type Params: Hash; - type Output; + type Params: Hash + Eq + Debug; + type Output: Debug; } pub(crate) trait Get: Query { fn get(ctx: &QueryCtx, params: &Self::Params) -> Self::Output; } -impl Get for T { +impl Get for T +where + T::Params: Clone, + T::Output: Clone, +{ fn get(ctx: &QueryCtx, params: &Self::Params) -> Self::Output { - Self::eval(ctx, params) + { + let mut cache = ctx.db.cache.lock(); + if let Some(cache) = Self::cache(&mut cache) { + if let Some(res) = cache.get(params) { + return res.clone(); + } + } + } + ctx.trace(TraceEvent { query_id: Self::ID, kind: TraceEventKind::Start }); + let res = Self::eval(ctx, params); + ctx.trace(TraceEvent { query_id: Self::ID, kind: TraceEventKind::Finish }); + + let mut cache = ctx.db.cache.lock(); + if let Some(cache) = Self::cache(&mut cache) { + cache.insert(params.clone(), res.clone()); + } + + res } } -pub(crate) trait Eval: Query { +pub(crate) trait Eval: Query +where + Self::Params: Clone, + Self::Output: Clone, + { + fn cache(_cache: &mut Cache) -> Option<&mut QueryCache> { + None + } fn eval(ctx: &QueryCtx, params: &Self::Params) -> Self::Output; } +#[derive(Debug)] pub(crate) struct DbFiles { - db: Db, + db: Arc, } impl DbFiles { @@ -113,7 +185,7 @@ impl Query for Files { } impl Get for Files { fn get(ctx: &QueryCtx, _params: &()) -> DbFiles { - DbFiles { db: ctx.db.clone() } + DbFiles { db: Arc::clone(&ctx.db) } } } diff --git a/crates/libanalysis/src/lib.rs b/crates/libanalysis/src/lib.rs index 68cf31e08f57..3e77006c5ace 100644 --- a/crates/libanalysis/src/lib.rs +++ b/crates/libanalysis/src/lib.rs @@ -13,7 +13,7 @@ extern crate im; mod symbol_index; mod module_map; -mod module_map_db; +pub(crate) mod module_map_db; mod imp; mod job; mod roots; diff --git a/crates/libanalysis/src/module_map_db.rs b/crates/libanalysis/src/module_map_db.rs index 25dbe8dd4274..14b156b43631 100644 --- a/crates/libanalysis/src/module_map_db.rs +++ b/crates/libanalysis/src/module_map_db.rs @@ -1,11 +1,14 @@ use std::sync::Arc; use { FileId, - db::{Query, Eval, QueryCtx, FileSyntax, Files}, + db::{ + Query, Eval, QueryCtx, FileSyntax, Files, + Cache, QueryCache, + }, module_map::resolve_submodule, }; -enum ModuleDescr {} +pub(crate) enum ModuleDescr {} impl Query for ModuleDescr { const ID: u32 = 30; type Params = FileId; @@ -27,6 +30,9 @@ impl Query for ParentModule { } impl Eval for ModuleDescr { + fn cache(cache: &mut Cache) -> Option<&mut QueryCache> { + Some(&mut cache.module_descr) + } fn eval(ctx: &QueryCtx, file_id: &FileId) -> Arc { let file = ctx.get::(file_id); Arc::new(descr::ModuleDescr::new(file.ast())) @@ -66,6 +72,7 @@ mod descr { ast::{self, NameOwner}, }; + #[derive(Debug)] pub struct ModuleDescr { pub submodules: Vec } @@ -85,7 +92,7 @@ mod descr { ModuleDescr { submodules } } } - #[derive(Clone, Hash)] + #[derive(Clone, Hash, PartialEq, Eq, Debug)] pub struct Submodule { pub name: SmolStr, } @@ -98,7 +105,7 @@ mod tests { use im; use relative_path::{RelativePath, RelativePathBuf}; use { - db::{Query, Db, TraceEventKind}, + db::{Query, DbHost, TraceEventKind}, imp::FileResolverImp, FileId, FileResolver, }; @@ -122,7 +129,7 @@ mod tests { struct Fixture { next_file_id: u32, fm: im::HashMap, - db: Db, + db: DbHost, } impl Fixture { @@ -130,7 +137,7 @@ mod tests { Fixture { next_file_id: 1, fm: im::HashMap::new(), - db: Db::new(), + db: DbHost::new(), } } fn add_file(&mut self, path: &str, text: &str) -> FileId { @@ -185,10 +192,11 @@ mod tests { fn test_parent_module() { let mut f = Fixture::new(); let foo = f.add_file("/foo.rs", ""); - f.check_parent_modules(foo, &[], &[(FileSyntax::ID, 1)]); + f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 1)]); let lib = f.add_file("/lib.rs", "mod foo;"); - f.check_parent_modules(foo, &[lib], &[(FileSyntax::ID, 2)]); + f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 2)]); + f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 0)]); f.change_file(lib, ""); f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 2)]); From c81d0d51bf05791b6ed39376d67d6e2876dd2a1e Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 11 Sep 2018 10:31:04 +0300 Subject: [PATCH 04/18] add deps tracking --- crates/libanalysis/src/db.rs | 148 +++++++++++++++++++----- crates/libanalysis/src/module_map_db.rs | 45 ++++--- crates/libsyntax2/src/lib.rs | 2 +- 3 files changed, 148 insertions(+), 47 deletions(-) diff --git a/crates/libanalysis/src/db.rs b/crates/libanalysis/src/db.rs index 31c73c40287b..bfff5357fe02 100644 --- a/crates/libanalysis/src/db.rs +++ b/crates/libanalysis/src/db.rs @@ -1,5 +1,5 @@ use std::{ - hash::Hash, + hash::{Hash, Hasher}, sync::Arc, cell::RefCell, fmt::Debug, @@ -45,17 +45,35 @@ impl DbHost { pub(crate) fn query_ctx(&self) -> QueryCtx { QueryCtx { db: Arc::clone(&self.db), + stack: RefCell::new(Vec::new()), trace: RefCell::new(Vec::new()), } } fn db_mut(&mut self) -> &mut Db { - // NB: this "forks" the database & clears the cache + // NB: this "forks" the database let db = Arc::make_mut(&mut self.db); - *db.cache.get_mut() = Default::default(); + db.cache.get_mut().gen += 1; db } } +type QueryInvocationId = (u32, u64); +type Gen = u64; +type OutputHash = u64; + +fn id(params: &Q::Params) -> QueryInvocationId { + use std::collections::hash_map::DefaultHasher; + let mut hasher = DefaultHasher::new(); + params.hash(&mut hasher); + (Q::ID, hasher.finish()) +} +fn output_hash(output: &Q::Output) -> OutputHash { + use std::collections::hash_map::DefaultHasher; + let mut hasher = DefaultHasher::new(); + output.hash(&mut hasher); + hasher.finish() +} + #[derive(Debug)] pub(crate) struct Db { file_resolver: FileResolverImp, @@ -73,9 +91,13 @@ impl Clone for Db { } } -#[derive(Clone, Default, Debug)] + +#[derive(Default, Debug)] pub(crate) struct Cache { - pub(crate) module_descr: QueryCache + pub(crate) module_descr: QueryCache, + gen: Gen, + green: im::HashMap, + deps: im::HashMap>, } #[allow(type_alias_bounds)] pub(crate) type QueryCache = im::HashMap< @@ -91,6 +113,7 @@ impl Cache { pub(crate) struct QueryCtx { db: Arc, + stack: RefCell>, pub(crate) trace: RefCell>, } @@ -102,12 +125,28 @@ pub(crate) struct TraceEvent { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum TraceEventKind { - Start, Finish + Start, Evaluating, Finish } impl QueryCtx { pub(crate) fn get(&self, params: &Q::Params) -> Q::Output { + let me = id::(params); + eprintln!("eval: {:?}", me); + let parent = self.stack.borrow().last().map(|&id| id); + self.stack.borrow_mut().push(me); + self.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Start }); let res = Q::get(self, params); + self.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Finish }); + if let Some(parent) = parent { + let h = output_hash::(&res); + let mut cache = self.db.cache.lock(); + cache.deps + .entry(parent) + .or_insert(Vec::new()) + .push((me, h)) + } + let also_me = self.stack.borrow_mut().pop(); + assert_eq!(also_me, Some(me)); res } fn trace(&self, event: TraceEvent) { @@ -118,47 +157,80 @@ impl QueryCtx { pub(crate) trait Query { const ID: u32; type Params: Hash + Eq + Debug; - type Output: Debug; + type Output: Hash + Debug; } pub(crate) trait Get: Query { fn get(ctx: &QueryCtx, params: &Self::Params) -> Self::Output; } -impl Get for T +impl Get for Q where - T::Params: Clone, - T::Output: Clone, + Q::Params: Clone, + Q::Output: Clone, { fn get(ctx: &QueryCtx, params: &Self::Params) -> Self::Output { - { - let mut cache = ctx.db.cache.lock(); - if let Some(cache) = Self::cache(&mut cache) { - if let Some(res) = cache.get(params) { - return res.clone(); - } - } + if !Self::cacheable() { + ctx.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Evaluating }); + return Self::eval(ctx, params); } - ctx.trace(TraceEvent { query_id: Self::ID, kind: TraceEventKind::Start }); + + if let Some(res) = try_reuse::(ctx, params) { + return res; + } + + ctx.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Evaluating }); let res = Self::eval(ctx, params); - ctx.trace(TraceEvent { query_id: Self::ID, kind: TraceEventKind::Finish }); let mut cache = ctx.db.cache.lock(); - if let Some(cache) = Self::cache(&mut cache) { - cache.insert(params.clone(), res.clone()); - } - + let gen = cache.gen; + let output_hash = output_hash::(&res); + let id = id::(params); + cache.green.insert(id, (gen, output_hash)); + let cache = Self::cache(&mut cache); + cache.insert(params.clone(), res.clone()); res } } +fn try_reuse(ctx: &QueryCtx, params: &Q::Params) -> Option +where + Q::Params: Clone, + Q::Output: Clone, +{ + let id = id::(params); + let mut cache = ctx.db.cache.lock(); + let curr_gen = cache.gen; + let old_hash = match *cache.green.get(&id)? { + (gen, _) if gen == curr_gen => { + return Some(Q::cache(&mut cache)[params].clone()); + } + (_, hash) => hash, + }; + let deps_are_fresh = cache.deps[&id] + .iter() + .all(|&(dep_id, dep_hash)| { + match cache.green.get(&dep_id) { + //TODO: store the value of parameters, and re-execute the query + Some((gen, hash)) if gen == &curr_gen && hash == &dep_hash => true, + _ => false, + } + }); + if !deps_are_fresh { + return None; + } + cache.green.insert(id, (curr_gen, old_hash)); + Some(Q::cache(&mut cache)[params].clone()) +} + pub(crate) trait Eval: Query where Self::Params: Clone, Self::Output: Clone, - { - fn cache(_cache: &mut Cache) -> Option<&mut QueryCache> { - None +{ + fn cacheable() -> bool { false } + fn cache(_cache: &mut Cache) -> &mut QueryCache { + unimplemented!() } fn eval(ctx: &QueryCtx, params: &Self::Params) -> Self::Output; } @@ -168,6 +240,12 @@ pub(crate) struct DbFiles { db: Arc, } +impl Hash for DbFiles { + fn hash(&self, hasher: &mut H) { + self.db.cache.lock().gen.hash(hasher) + } +} + impl DbFiles { pub(crate) fn iter<'a>(&'a self) -> impl Iterator + 'a { self.db.files.keys().cloned() @@ -184,8 +262,14 @@ impl Query for Files { type Output = DbFiles; } impl Get for Files { - fn get(ctx: &QueryCtx, _params: &()) -> DbFiles { - DbFiles { db: Arc::clone(&ctx.db) } + fn get(ctx: &QueryCtx, params: &()) -> DbFiles { + let res = DbFiles { db: Arc::clone(&ctx.db) }; + let id = id::(params); + let hash = output_hash::(&res); + let mut cache = ctx.db.cache.lock(); + let gen = cache.gen; + cache.green.insert(id, (gen, hash)); + res } } @@ -197,7 +281,13 @@ impl Query for FileText { } impl Get for FileText { fn get(ctx: &QueryCtx, file_id: &FileId) -> Arc { - ctx.db.files[file_id].clone() + let res = ctx.db.files[file_id].clone(); + let id = id::(file_id); + let hash = output_hash::(&res); + let mut cache = ctx.db.cache.lock(); + let gen = cache.gen; + cache.green.insert(id, (gen, hash)); + res } } diff --git a/crates/libanalysis/src/module_map_db.rs b/crates/libanalysis/src/module_map_db.rs index 14b156b43631..27f19f96e78f 100644 --- a/crates/libanalysis/src/module_map_db.rs +++ b/crates/libanalysis/src/module_map_db.rs @@ -30,8 +30,9 @@ impl Query for ParentModule { } impl Eval for ModuleDescr { - fn cache(cache: &mut Cache) -> Option<&mut QueryCache> { - Some(&mut cache.module_descr) + fn cacheable() -> bool { true } + fn cache(cache: &mut Cache) -> &mut QueryCache { + &mut cache.module_descr } fn eval(ctx: &QueryCtx, file_id: &FileId) -> Arc { let file = ctx.get::(file_id); @@ -72,7 +73,7 @@ mod descr { ast::{self, NameOwner}, }; - #[derive(Debug)] + #[derive(Debug, Hash)] pub struct ModuleDescr { pub submodules: Vec } @@ -168,12 +169,13 @@ mod tests { expected: &[FileId], queries: &[(u32, u64)] ) { + eprintln!(); let ctx = self.db.query_ctx(); let actual = ctx.get::(&file_id); assert_eq!(actual.as_slice(), expected); let mut counts = HashMap::new(); ctx.trace.borrow().iter() - .filter(|event| event.kind == TraceEventKind::Start) + .filter(|event| event.kind == TraceEventKind::Evaluating) .for_each(|event| *counts.entry(event.query_id).or_insert(0) += 1); for &(query_id, expected_count) in queries.iter() { let actual_count = *counts.get(&query_id).unwrap_or(&0); @@ -192,26 +194,35 @@ mod tests { fn test_parent_module() { let mut f = Fixture::new(); let foo = f.add_file("/foo.rs", ""); - f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 1)]); + f.check_parent_modules(foo, &[], &[ + (ModuleDescr::ID, 1), + (FileSyntax::ID, 1), + ]); let lib = f.add_file("/lib.rs", "mod foo;"); - f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 2)]); - f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 0)]); + f.check_parent_modules(foo, &[lib], &[ + (ModuleDescr::ID, 1), + (FileSyntax::ID, 2), + ]); + // f.check_parent_modules(foo, &[lib], &[ + // (ModuleDescr::ID, 0), + // (FileSyntax::ID, 2), + // ]); - f.change_file(lib, ""); - f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 2)]); + // f.change_file(lib, ""); + // f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 2)]); - f.change_file(lib, "mod foo;"); - f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 2)]); + // f.change_file(lib, "mod foo;"); + // f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 2)]); - f.change_file(lib, "mod bar;"); - f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 2)]); + // f.change_file(lib, "mod bar;"); + // f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 2)]); - f.change_file(lib, "mod foo;"); - f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 2)]); + // f.change_file(lib, "mod foo;"); + // f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 2)]); - f.remove_file(lib); - f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 1)]); + // f.remove_file(lib); + // f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 1)]); } } diff --git a/crates/libsyntax2/src/lib.rs b/crates/libsyntax2/src/lib.rs index fd58cb4fa6e0..e761fa35831d 100644 --- a/crates/libsyntax2/src/lib.rs +++ b/crates/libsyntax2/src/lib.rs @@ -57,7 +57,7 @@ use { parser_api::Parser, }; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Hash)] pub struct File { root: SyntaxNode } From dbdf72e2e2fb3ebc1a5cdeac4e70108371bb91fb Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 11 Sep 2018 11:31:07 +0300 Subject: [PATCH 05/18] fix dep tracking --- crates/libanalysis/src/db.rs | 26 ++++++++++++------------- crates/libanalysis/src/module_map_db.rs | 21 +++++--------------- 2 files changed, 17 insertions(+), 30 deletions(-) diff --git a/crates/libanalysis/src/db.rs b/crates/libanalysis/src/db.rs index bfff5357fe02..2f344f788789 100644 --- a/crates/libanalysis/src/db.rs +++ b/crates/libanalysis/src/db.rs @@ -113,7 +113,7 @@ impl Cache { pub(crate) struct QueryCtx { db: Arc, - stack: RefCell>, + stack: RefCell)>>, pub(crate) trace: RefCell>, } @@ -131,22 +131,16 @@ pub(crate) enum TraceEventKind { impl QueryCtx { pub(crate) fn get(&self, params: &Q::Params) -> Q::Output { let me = id::(params); - eprintln!("eval: {:?}", me); - let parent = self.stack.borrow().last().map(|&id| id); - self.stack.borrow_mut().push(me); self.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Start }); let res = Q::get(self, params); self.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Finish }); - if let Some(parent) = parent { - let h = output_hash::(&res); - let mut cache = self.db.cache.lock(); - cache.deps - .entry(parent) - .or_insert(Vec::new()) - .push((me, h)) + { + let mut stack = self.stack.borrow_mut(); + if let Some((_, ref mut deps)) = stack.last_mut() { + deps.push((me, output_hash::(&res))); + } } - let also_me = self.stack.borrow_mut().pop(); - assert_eq!(also_me, Some(me)); + res } fn trace(&self, event: TraceEvent) { @@ -179,10 +173,14 @@ where return res; } + let me = id::(params); ctx.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Evaluating }); + ctx.stack.borrow_mut().push((me, Vec::new())); let res = Self::eval(ctx, params); - + let (also_me, deps) = ctx.stack.borrow_mut().pop().unwrap(); + assert_eq!(also_me, me); let mut cache = ctx.db.cache.lock(); + cache.deps.insert(me, deps); let gen = cache.gen; let output_hash = output_hash::(&res); let id = id::(params); diff --git a/crates/libanalysis/src/module_map_db.rs b/crates/libanalysis/src/module_map_db.rs index 27f19f96e78f..72173c7bcf0a 100644 --- a/crates/libanalysis/src/module_map_db.rs +++ b/crates/libanalysis/src/module_map_db.rs @@ -169,7 +169,6 @@ mod tests { expected: &[FileId], queries: &[(u32, u64)] ) { - eprintln!(); let ctx = self.db.query_ctx(); let actual = ctx.get::(&file_id); assert_eq!(actual.as_slice(), expected); @@ -194,23 +193,14 @@ mod tests { fn test_parent_module() { let mut f = Fixture::new(); let foo = f.add_file("/foo.rs", ""); - f.check_parent_modules(foo, &[], &[ - (ModuleDescr::ID, 1), - (FileSyntax::ID, 1), - ]); + // f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 1)]); let lib = f.add_file("/lib.rs", "mod foo;"); - f.check_parent_modules(foo, &[lib], &[ - (ModuleDescr::ID, 1), - (FileSyntax::ID, 2), - ]); - // f.check_parent_modules(foo, &[lib], &[ - // (ModuleDescr::ID, 0), - // (FileSyntax::ID, 2), - // ]); + f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 2)]); + f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 0)]); - // f.change_file(lib, ""); - // f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 2)]); + f.change_file(lib, ""); + f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 2)]); // f.change_file(lib, "mod foo;"); // f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 2)]); @@ -224,5 +214,4 @@ mod tests { // f.remove_file(lib); // f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 1)]); } - } From 907d44a75113d318102ff05a66b4dcdafa1b5e7f Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 12 Sep 2018 17:13:11 +0300 Subject: [PATCH 06/18] any-cache --- crates/libanalysis/src/db.rs | 35 +++++++++++++------------ crates/libanalysis/src/module_map_db.rs | 4 --- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/crates/libanalysis/src/db.rs b/crates/libanalysis/src/db.rs index 2f344f788789..d67cc189cde6 100644 --- a/crates/libanalysis/src/db.rs +++ b/crates/libanalysis/src/db.rs @@ -3,6 +3,7 @@ use std::{ sync::Arc, cell::RefCell, fmt::Debug, + any::Any, }; use parking_lot::Mutex; use libsyntax2::{File}; @@ -10,7 +11,6 @@ use im; use { FileId, imp::{FileResolverImp}, - module_map_db::ModuleDescr, }; #[derive(Debug)] @@ -94,11 +94,13 @@ impl Clone for Db { #[derive(Default, Debug)] pub(crate) struct Cache { - pub(crate) module_descr: QueryCache, gen: Gen, green: im::HashMap, deps: im::HashMap>, + results: im::HashMap>, } + + #[allow(type_alias_bounds)] pub(crate) type QueryCache = im::HashMap< ::Params, @@ -109,6 +111,15 @@ impl Cache { fn new() -> Cache { Default::default() } + + fn get_result(&self, id: QueryInvocationId) -> Q::Output + where + Q::Output: Clone + { + let res = &self.results[&id]; + let res = res.downcast_ref::().unwrap(); + res.clone() + } } pub(crate) struct QueryCtx { @@ -150,8 +161,8 @@ impl QueryCtx { pub(crate) trait Query { const ID: u32; - type Params: Hash + Eq + Debug; - type Output: Hash + Debug; + type Params: Hash + Eq + Debug + Any + 'static; + type Output: Hash + Debug + Any + 'static; } pub(crate) trait Get: Query { @@ -164,11 +175,6 @@ where Q::Output: Clone, { fn get(ctx: &QueryCtx, params: &Self::Params) -> Self::Output { - if !Self::cacheable() { - ctx.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Evaluating }); - return Self::eval(ctx, params); - } - if let Some(res) = try_reuse::(ctx, params) { return res; } @@ -185,8 +191,7 @@ where let output_hash = output_hash::(&res); let id = id::(params); cache.green.insert(id, (gen, output_hash)); - let cache = Self::cache(&mut cache); - cache.insert(params.clone(), res.clone()); + cache.results.insert(me, Arc::new(res.clone())); res } } @@ -201,7 +206,7 @@ where let curr_gen = cache.gen; let old_hash = match *cache.green.get(&id)? { (gen, _) if gen == curr_gen => { - return Some(Q::cache(&mut cache)[params].clone()); + return Some(cache.get_result::(id)); } (_, hash) => hash, }; @@ -218,7 +223,7 @@ where return None; } cache.green.insert(id, (curr_gen, old_hash)); - Some(Q::cache(&mut cache)[params].clone()) + Some(cache.get_result::(id)) } pub(crate) trait Eval: Query @@ -226,10 +231,6 @@ where Self::Params: Clone, Self::Output: Clone, { - fn cacheable() -> bool { false } - fn cache(_cache: &mut Cache) -> &mut QueryCache { - unimplemented!() - } fn eval(ctx: &QueryCtx, params: &Self::Params) -> Self::Output; } diff --git a/crates/libanalysis/src/module_map_db.rs b/crates/libanalysis/src/module_map_db.rs index 72173c7bcf0a..4d4bd91046f6 100644 --- a/crates/libanalysis/src/module_map_db.rs +++ b/crates/libanalysis/src/module_map_db.rs @@ -30,10 +30,6 @@ impl Query for ParentModule { } impl Eval for ModuleDescr { - fn cacheable() -> bool { true } - fn cache(cache: &mut Cache) -> &mut QueryCache { - &mut cache.module_descr - } fn eval(ctx: &QueryCtx, file_id: &FileId) -> Arc { let file = ctx.get::(file_id); Arc::new(descr::ModuleDescr::new(file.ast())) From 0e493160c0cdbaa71f61af64fd7c439410e8c8b1 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 12 Sep 2018 17:16:58 +0300 Subject: [PATCH 07/18] store params in the graph --- crates/libanalysis/src/db.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/libanalysis/src/db.rs b/crates/libanalysis/src/db.rs index d67cc189cde6..d30e75fe2b6b 100644 --- a/crates/libanalysis/src/db.rs +++ b/crates/libanalysis/src/db.rs @@ -91,12 +91,13 @@ impl Clone for Db { } } +type QueryDeps = Vec<(QueryInvocationId, Arc, OutputHash)>; #[derive(Default, Debug)] pub(crate) struct Cache { gen: Gen, green: im::HashMap, - deps: im::HashMap>, + deps: im::HashMap, results: im::HashMap>, } @@ -124,7 +125,7 @@ impl Cache { pub(crate) struct QueryCtx { db: Arc, - stack: RefCell)>>, + stack: RefCell>, pub(crate) trace: RefCell>, } @@ -148,7 +149,8 @@ impl QueryCtx { { let mut stack = self.stack.borrow_mut(); if let Some((_, ref mut deps)) = stack.last_mut() { - deps.push((me, output_hash::(&res))); + let params = Arc::new(params.clone()); + deps.push((me, params, output_hash::(&res))); } } @@ -161,7 +163,7 @@ impl QueryCtx { pub(crate) trait Query { const ID: u32; - type Params: Hash + Eq + Debug + Any + 'static; + type Params: Hash + Eq + Debug + Clone + Any + 'static; type Output: Hash + Debug + Any + 'static; } @@ -212,7 +214,7 @@ where }; let deps_are_fresh = cache.deps[&id] .iter() - .all(|&(dep_id, dep_hash)| { + .all(|&(dep_id, _, dep_hash)| { match cache.green.get(&dep_id) { //TODO: store the value of parameters, and re-execute the query Some((gen, hash)) if gen == &curr_gen && hash == &dep_hash => true, From 8cf9c2719652d298006d51bc82a32908ab4e5335 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 12 Sep 2018 21:50:15 +0300 Subject: [PATCH 08/18] generic salsa algo --- crates/salsa/Cargo.toml | 8 + crates/salsa/src/lib.rs | 238 ++++++++++++++++++++++++++++++ crates/salsa/tests/integration.rs | 153 +++++++++++++++++++ 3 files changed, 399 insertions(+) create mode 100644 crates/salsa/Cargo.toml create mode 100644 crates/salsa/src/lib.rs create mode 100644 crates/salsa/tests/integration.rs diff --git a/crates/salsa/Cargo.toml b/crates/salsa/Cargo.toml new file mode 100644 index 000000000000..9eb83234f0cb --- /dev/null +++ b/crates/salsa/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "salsa" +version = "0.1.0" +authors = ["Aleksey Kladov "] + +[dependencies] +parking_lot = "0.6.3" +im = "12.0.0" diff --git a/crates/salsa/src/lib.rs b/crates/salsa/src/lib.rs new file mode 100644 index 000000000000..69c7b35fa1b0 --- /dev/null +++ b/crates/salsa/src/lib.rs @@ -0,0 +1,238 @@ +extern crate im; +extern crate parking_lot; + +use std::{ + sync::Arc, + any::Any, + collections::HashMap, + cell::RefCell, +}; +use parking_lot::Mutex; + +type GroundQueryFn = fn(&T, &(Any + Send + Sync + 'static)) -> (Box, OutputFingerprint); +type QueryFn = fn(&QueryCtx, &(Any + Send + Sync + 'static)) -> (Box, OutputFingerprint); + +#[derive(Debug)] +pub struct Db { + db: Arc>, + query_config: Arc>, +} + +pub struct QueryConfig { + ground_fn: HashMap>, + query_fn: HashMap>, +} + +impl ::std::fmt::Debug for QueryConfig { + fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + ::std::fmt::Display::fmt("QueryConfig { ... }", f) + } +} + +#[derive(Debug)] +struct DbState { + ground_data: T, + gen: Gen, + graph: Mutex)>>, +} + +#[derive(Debug)] +struct QueryRecord { + params: Arc, + output: Arc, + output_fingerprint: OutputFingerprint, + deps: Vec<(QueryId, OutputFingerprint)>, +} + +impl DbState { + fn record( + &self, + query_id: QueryId, + params: Arc, + output: Arc, + output_fingerprint: OutputFingerprint, + deps: Vec<(QueryId, OutputFingerprint)>, + ) { + let gen = self.gen; + let record = QueryRecord { + params, + output, + output_fingerprint, + deps, + }; + self.graph.lock().insert(query_id, (gen, Arc::new(record))); + } +} + +impl QueryConfig { + pub fn new() -> Self { + QueryConfig { + ground_fn: HashMap::new(), + query_fn: HashMap::new(), + } + } + pub fn with_ground_query( + mut self, + query_type: QueryTypeId, + query_fn: GroundQueryFn + ) -> Self { + let prev = self.ground_fn.insert(query_type, query_fn); + assert!(prev.is_none()); + self + } + pub fn with_query( + mut self, + query_type: QueryTypeId, + query_fn: QueryFn, + ) -> Self { + let prev = self.query_fn.insert(query_type, query_fn); + assert!(prev.is_none()); + self + } +} + +pub struct QueryCtx { + db: Arc>, + query_config: Arc>, + stack: RefCell>>, + executed: RefCell>, +} + +impl QueryCtx { + fn new(db: &Db) -> QueryCtx { + QueryCtx { + db: Arc::clone(&db.db), + query_config: Arc::clone(&db.query_config), + stack: RefCell::new(vec![Vec::new()]), + executed: RefCell::new(Vec::new()), + } + } + pub fn get( + &self, + query_id: QueryId, + params: Arc, + ) -> Arc { + let (res, output_fingerprint) = self.get_inner(query_id, params); + self.record_dep(query_id, output_fingerprint); + res + } + + pub fn get_inner( + &self, + query_id: QueryId, + params: Arc, + ) -> (Arc, OutputFingerprint) { + let (gen, record) = { + let guard = self.db.graph.lock(); + match guard.get(&query_id).map(|it| it.clone()){ + None => { + drop(guard); + return self.force(query_id, params); + }, + Some(it) => it, + } + }; + if gen == self.db.gen { + return (record.output.clone(), record.output_fingerprint) + } + if self.query_config.ground_fn.contains_key(&query_id.0) { + return self.force(query_id, params); + } + for (dep_query_id, prev_fingerprint) in record.deps.iter().cloned() { + let dep_params: Arc = { + let guard = self.db.graph.lock(); + guard[&dep_query_id] + .1 + .params + .clone() + }; + if prev_fingerprint != self.get_inner(dep_query_id, dep_params).1 { + return self.force(query_id, params) + } + } + let gen = self.db.gen; + { + let mut guard = self.db.graph.lock(); + guard[&query_id].0 = gen; + } + (record.output.clone(), record.output_fingerprint) + } + fn force( + &self, + query_id: QueryId, + params: Arc, + ) -> (Arc, OutputFingerprint) { + self.executed.borrow_mut().push(query_id.0); + self.stack.borrow_mut().push(Vec::new()); + + let (res, output_fingerprint) = if let Some(f) = self.ground_query_fn_by_type(query_id.0) { + f(&self.db.ground_data, &*params) + } else if let Some(f) = self.query_fn_by_type(query_id.0) { + f(self, &*params) + } else { + panic!("unknown query type: {:?}", query_id.0); + }; + + let res: Arc = res.into(); + + let deps = self.stack.borrow_mut().pop().unwrap(); + self.db.record(query_id, params, res.clone(), output_fingerprint, deps); + (res, output_fingerprint) + } + fn ground_query_fn_by_type(&self, query_type: QueryTypeId) -> Option> { + self.query_config.ground_fn.get(&query_type).map(|&it| it) + } + fn query_fn_by_type(&self, query_type: QueryTypeId) -> Option> { + self.query_config.query_fn.get(&query_type).map(|&it| it) + } + fn record_dep( + &self, + query_id: QueryId, + output_fingerprint: OutputFingerprint, + ) -> () { + let mut stack = self.stack.borrow_mut(); + let deps = stack.last_mut().unwrap(); + deps.push((query_id, output_fingerprint)) + } +} + +impl Db { + pub fn new(query_config: QueryConfig, ground_data: T) -> Db { + Db { + db: Arc::new(DbState { ground_data, gen: Gen(0), graph: Default::default() }), + query_config: Arc::new(query_config), + } + } + + pub fn with_ground_data(&self, ground_data: T) -> Db { + let gen = Gen(self.db.gen.0 + 1); + let graph = self.db.graph.lock().clone(); + let graph = Mutex::new(graph); + Db { + db: Arc::new(DbState { ground_data, gen, graph }), + query_config: Arc::clone(&self.query_config) + } + } + pub fn get( + &self, + query_id: QueryId, + params: Box, + ) -> (Arc, Vec) { + let ctx = QueryCtx::new(self); + let res = ctx.get(query_id, params.into()); + let executed = ::std::mem::replace(&mut *ctx.executed.borrow_mut(), Vec::new()); + (res, executed) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +struct Gen(u64); +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct InputFingerprint(pub u64); +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct OutputFingerprint(pub u64); +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct QueryTypeId(pub u16); +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct QueryId(pub QueryTypeId, pub InputFingerprint); + diff --git a/crates/salsa/tests/integration.rs b/crates/salsa/tests/integration.rs new file mode 100644 index 000000000000..7241eca3850f --- /dev/null +++ b/crates/salsa/tests/integration.rs @@ -0,0 +1,153 @@ +extern crate salsa; +use std::{ + sync::Arc, + collections::hash_map::{HashMap, DefaultHasher}, + any::Any, + hash::{Hash, Hasher}, +}; + +type State = HashMap; +const GET_TEXT: salsa::QueryTypeId = salsa::QueryTypeId(1); +const GET_FILES: salsa::QueryTypeId = salsa::QueryTypeId(2); +const FILE_NEWLINES: salsa::QueryTypeId = salsa::QueryTypeId(3); +const TOTAL_NEWLINES: salsa::QueryTypeId = salsa::QueryTypeId(4); + +fn mk_ground_query( + state: &State, + params: &(Any + Send + Sync + 'static), + f: fn(&State, &T) -> R, +) -> (Box, salsa::OutputFingerprint) +where + T: 'static, + R: Hash + Send + Sync + 'static, +{ + let params = params.downcast_ref().unwrap(); + let result = f(state, params); + let fingerprint = o_print(&result); + (Box::new(result), fingerprint) +} + +fn get(db: &salsa::Db, query_type: salsa::QueryTypeId, param: T) -> (Arc, Vec) +where + T: Hash + Send + Sync + 'static, + R: Send + Sync + 'static, +{ + let i_print = i_print(¶m); + let param = Box::new(param); + let (res, trace) = db.get(salsa::QueryId(query_type, i_print), param); + (res.downcast().unwrap(), trace) +} + +struct QueryCtx<'a>(&'a salsa::QueryCtx); + +impl<'a> QueryCtx<'a> { + fn get_text(&self, id: u32) -> Arc { + let i_print = i_print(&id); + let text = self.0.get(salsa::QueryId(GET_TEXT, i_print), Arc::new(id)); + text.downcast().unwrap() + } + fn get_files(&self) -> Arc> { + let i_print = i_print(&()); + let files = self.0.get(salsa::QueryId(GET_FILES, i_print), Arc::new(())); + let res = files.downcast().unwrap(); + res + } + fn get_n_lines(&self, id: u32) -> usize { + let i_print = i_print(&id); + let n_lines = self.0.get(salsa::QueryId(FILE_NEWLINES, i_print), Arc::new(id)); + *n_lines.downcast().unwrap() + } +} + +fn mk_query( + query_ctx: &salsa::QueryCtx, + params: &(Any + Send + Sync + 'static), + f: fn(QueryCtx, &T) -> R, +) -> (Box, salsa::OutputFingerprint) +where + T: 'static, + R: Hash + Send + Sync + 'static, +{ + let params: &T = params.downcast_ref().unwrap(); + let query_ctx = QueryCtx(query_ctx); + let result = f(query_ctx, params); + let fingerprint = o_print(&result); + (Box::new(result), fingerprint) +} + +fn mk_queries() -> salsa::QueryConfig { + salsa::QueryConfig::::new() + .with_ground_query(GET_TEXT, |state, id| { + mk_ground_query::(state, id, |state, id| state[id].clone()) + }) + .with_ground_query(GET_FILES, |state, id| { + mk_ground_query::<(), Vec>(state, id, |state, &()| state.keys().cloned().collect()) + }) + .with_query(FILE_NEWLINES, |query_ctx, id| { + mk_query(query_ctx, id, |query_ctx, &id| { + let text = query_ctx.get_text(id); + text.lines().count() + }) + }) + .with_query(TOTAL_NEWLINES, |query_ctx, id| { + mk_query(query_ctx, id, |query_ctx, &()| { + let mut total = 0; + for &id in query_ctx.get_files().iter() { + total += query_ctx.get_n_lines(id) + } + total + }) + }) +} + +#[test] +fn test_number_of_lines() { + let mut state = State::new(); + let db = salsa::Db::new(mk_queries(), state.clone()); + let (newlines, trace) = get::<(), usize>(&db, TOTAL_NEWLINES, ()); + assert_eq!(*newlines, 0); + assert_eq!(trace.len(), 2); + let (newlines, trace) = get::<(), usize>(&db, TOTAL_NEWLINES, ()); + assert_eq!(*newlines, 0); + assert_eq!(trace.len(), 0); + + state.insert(1, "hello\nworld".to_string()); + let db = db.with_ground_data(state.clone()); + let (newlines, trace) = get::<(), usize>(&db, TOTAL_NEWLINES, ()); + assert_eq!(*newlines, 2); + assert_eq!(trace.len(), 4); + + state.insert(2, "spam\neggs".to_string()); + let db = db.with_ground_data(state.clone()); + let (newlines, trace) = get::<(), usize>(&db, TOTAL_NEWLINES, ()); + assert_eq!(*newlines, 4); + assert_eq!(trace.len(), 5); + + for i in 0..10 { + state.insert(i + 10, "spam".to_string()); + } + let db = db.with_ground_data(state.clone()); + let (newlines, trace) = get::<(), usize>(&db, TOTAL_NEWLINES, ()); + assert_eq!(*newlines, 14); + assert_eq!(trace.len(), 24); + + state.insert(15, String::new()); + let db = db.with_ground_data(state.clone()); + let (newlines, trace) = get::<(), usize>(&db, TOTAL_NEWLINES, ()); + assert_eq!(*newlines, 13); + assert_eq!(trace.len(), 15); +} + +fn o_print(x: &T) -> salsa::OutputFingerprint { + let mut hasher = DefaultHasher::new(); + x.hash(&mut hasher); + let hash = hasher.finish(); + salsa::OutputFingerprint(hash) +} + +fn i_print(x: &T) -> salsa::InputFingerprint { + let mut hasher = DefaultHasher::new(); + x.hash(&mut hasher); + let hash = hasher.finish(); + salsa::InputFingerprint(hash) +} From cecc7ad5b20e693cb8d962187bd83b9ac234de97 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 12 Sep 2018 22:11:26 +0300 Subject: [PATCH 09/18] be generic over data --- crates/salsa/src/lib.rs | 91 ++++++++++++++++--------------- crates/salsa/tests/integration.rs | 25 +++++---- 2 files changed, 61 insertions(+), 55 deletions(-) diff --git a/crates/salsa/src/lib.rs b/crates/salsa/src/lib.rs index 69c7b35fa1b0..a54f2a06fca1 100644 --- a/crates/salsa/src/lib.rs +++ b/crates/salsa/src/lib.rs @@ -3,53 +3,52 @@ extern crate parking_lot; use std::{ sync::Arc, - any::Any, collections::HashMap, cell::RefCell, }; use parking_lot::Mutex; -type GroundQueryFn = fn(&T, &(Any + Send + Sync + 'static)) -> (Box, OutputFingerprint); -type QueryFn = fn(&QueryCtx, &(Any + Send + Sync + 'static)) -> (Box, OutputFingerprint); +type GroundQueryFn = fn(&T, &D) -> (D, OutputFingerprint); +type QueryFn = fn(&QueryCtx, &D) -> (D, OutputFingerprint); #[derive(Debug)] -pub struct Db { - db: Arc>, - query_config: Arc>, +pub struct Db { + db: Arc>, + query_config: Arc>, } -pub struct QueryConfig { - ground_fn: HashMap>, - query_fn: HashMap>, +pub struct QueryConfig { + ground_fn: HashMap>, + query_fn: HashMap>, } -impl ::std::fmt::Debug for QueryConfig { +impl ::std::fmt::Debug for QueryConfig { fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { ::std::fmt::Display::fmt("QueryConfig { ... }", f) } } #[derive(Debug)] -struct DbState { +struct DbState { ground_data: T, gen: Gen, - graph: Mutex)>>, + graph: Mutex>)>>, } #[derive(Debug)] -struct QueryRecord { - params: Arc, - output: Arc, +struct QueryRecord { + params: D, + output: D, output_fingerprint: OutputFingerprint, deps: Vec<(QueryId, OutputFingerprint)>, } -impl DbState { +impl DbState { fn record( &self, query_id: QueryId, - params: Arc, - output: Arc, + params: D, + output: D, output_fingerprint: OutputFingerprint, deps: Vec<(QueryId, OutputFingerprint)>, ) { @@ -64,7 +63,7 @@ impl DbState { } } -impl QueryConfig { +impl QueryConfig { pub fn new() -> Self { QueryConfig { ground_fn: HashMap::new(), @@ -74,7 +73,7 @@ impl QueryConfig { pub fn with_ground_query( mut self, query_type: QueryTypeId, - query_fn: GroundQueryFn + query_fn: GroundQueryFn ) -> Self { let prev = self.ground_fn.insert(query_type, query_fn); assert!(prev.is_none()); @@ -83,7 +82,7 @@ impl QueryConfig { pub fn with_query( mut self, query_type: QueryTypeId, - query_fn: QueryFn, + query_fn: QueryFn, ) -> Self { let prev = self.query_fn.insert(query_type, query_fn); assert!(prev.is_none()); @@ -91,15 +90,18 @@ impl QueryConfig { } } -pub struct QueryCtx { - db: Arc>, - query_config: Arc>, +pub struct QueryCtx { + db: Arc>, + query_config: Arc>, stack: RefCell>>, executed: RefCell>, } -impl QueryCtx { - fn new(db: &Db) -> QueryCtx { +impl QueryCtx +where + D: Clone +{ + fn new(db: &Db) -> QueryCtx { QueryCtx { db: Arc::clone(&db.db), query_config: Arc::clone(&db.query_config), @@ -110,8 +112,8 @@ impl QueryCtx { pub fn get( &self, query_id: QueryId, - params: Arc, - ) -> Arc { + params: D, + ) -> D { let (res, output_fingerprint) = self.get_inner(query_id, params); self.record_dep(query_id, output_fingerprint); res @@ -120,8 +122,8 @@ impl QueryCtx { pub fn get_inner( &self, query_id: QueryId, - params: Arc, - ) -> (Arc, OutputFingerprint) { + params: D, + ) -> (D, OutputFingerprint) { let (gen, record) = { let guard = self.db.graph.lock(); match guard.get(&query_id).map(|it| it.clone()){ @@ -139,7 +141,7 @@ impl QueryCtx { return self.force(query_id, params); } for (dep_query_id, prev_fingerprint) in record.deps.iter().cloned() { - let dep_params: Arc = { + let dep_params: D = { let guard = self.db.graph.lock(); guard[&dep_query_id] .1 @@ -160,29 +162,29 @@ impl QueryCtx { fn force( &self, query_id: QueryId, - params: Arc, - ) -> (Arc, OutputFingerprint) { + params: D, + ) -> (D, OutputFingerprint) { self.executed.borrow_mut().push(query_id.0); self.stack.borrow_mut().push(Vec::new()); let (res, output_fingerprint) = if let Some(f) = self.ground_query_fn_by_type(query_id.0) { - f(&self.db.ground_data, &*params) + f(&self.db.ground_data, ¶ms) } else if let Some(f) = self.query_fn_by_type(query_id.0) { - f(self, &*params) + f(self, ¶ms) } else { panic!("unknown query type: {:?}", query_id.0); }; - let res: Arc = res.into(); + let res: D = res.into(); let deps = self.stack.borrow_mut().pop().unwrap(); self.db.record(query_id, params, res.clone(), output_fingerprint, deps); (res, output_fingerprint) } - fn ground_query_fn_by_type(&self, query_type: QueryTypeId) -> Option> { + fn ground_query_fn_by_type(&self, query_type: QueryTypeId) -> Option> { self.query_config.ground_fn.get(&query_type).map(|&it| it) } - fn query_fn_by_type(&self, query_type: QueryTypeId) -> Option> { + fn query_fn_by_type(&self, query_type: QueryTypeId) -> Option> { self.query_config.query_fn.get(&query_type).map(|&it| it) } fn record_dep( @@ -196,15 +198,18 @@ impl QueryCtx { } } -impl Db { - pub fn new(query_config: QueryConfig, ground_data: T) -> Db { +impl Db +where + D: Clone +{ + pub fn new(query_config: QueryConfig, ground_data: T) -> Db { Db { db: Arc::new(DbState { ground_data, gen: Gen(0), graph: Default::default() }), query_config: Arc::new(query_config), } } - pub fn with_ground_data(&self, ground_data: T) -> Db { + pub fn with_ground_data(&self, ground_data: T) -> Db { let gen = Gen(self.db.gen.0 + 1); let graph = self.db.graph.lock().clone(); let graph = Mutex::new(graph); @@ -216,8 +221,8 @@ impl Db { pub fn get( &self, query_id: QueryId, - params: Box, - ) -> (Arc, Vec) { + params: D, + ) -> (D, Vec) { let ctx = QueryCtx::new(self); let res = ctx.get(query_id, params.into()); let executed = ::std::mem::replace(&mut *ctx.executed.borrow_mut(), Vec::new()); diff --git a/crates/salsa/tests/integration.rs b/crates/salsa/tests/integration.rs index 7241eca3850f..2872d39133f5 100644 --- a/crates/salsa/tests/integration.rs +++ b/crates/salsa/tests/integration.rs @@ -7,6 +7,7 @@ use std::{ }; type State = HashMap; +type Data = Arc; const GET_TEXT: salsa::QueryTypeId = salsa::QueryTypeId(1); const GET_FILES: salsa::QueryTypeId = salsa::QueryTypeId(2); const FILE_NEWLINES: salsa::QueryTypeId = salsa::QueryTypeId(3); @@ -14,9 +15,9 @@ const TOTAL_NEWLINES: salsa::QueryTypeId = salsa::QueryTypeId(4); fn mk_ground_query( state: &State, - params: &(Any + Send + Sync + 'static), + params: &Data, f: fn(&State, &T) -> R, -) -> (Box, salsa::OutputFingerprint) +) -> (Data, salsa::OutputFingerprint) where T: 'static, R: Hash + Send + Sync + 'static, @@ -24,21 +25,21 @@ where let params = params.downcast_ref().unwrap(); let result = f(state, params); let fingerprint = o_print(&result); - (Box::new(result), fingerprint) + (Arc::new(result), fingerprint) } -fn get(db: &salsa::Db, query_type: salsa::QueryTypeId, param: T) -> (Arc, Vec) +fn get(db: &salsa::Db, query_type: salsa::QueryTypeId, param: T) -> (Arc, Vec) where T: Hash + Send + Sync + 'static, R: Send + Sync + 'static, { let i_print = i_print(¶m); - let param = Box::new(param); + let param = Arc::new(param); let (res, trace) = db.get(salsa::QueryId(query_type, i_print), param); (res.downcast().unwrap(), trace) } -struct QueryCtx<'a>(&'a salsa::QueryCtx); +struct QueryCtx<'a>(&'a salsa::QueryCtx); impl<'a> QueryCtx<'a> { fn get_text(&self, id: u32) -> Arc { @@ -60,10 +61,10 @@ impl<'a> QueryCtx<'a> { } fn mk_query( - query_ctx: &salsa::QueryCtx, - params: &(Any + Send + Sync + 'static), + query_ctx: &salsa::QueryCtx, + params: &Data, f: fn(QueryCtx, &T) -> R, -) -> (Box, salsa::OutputFingerprint) +) -> (Data, salsa::OutputFingerprint) where T: 'static, R: Hash + Send + Sync + 'static, @@ -72,11 +73,11 @@ where let query_ctx = QueryCtx(query_ctx); let result = f(query_ctx, params); let fingerprint = o_print(&result); - (Box::new(result), fingerprint) + (Arc::new(result), fingerprint) } -fn mk_queries() -> salsa::QueryConfig { - salsa::QueryConfig::::new() +fn mk_queries() -> salsa::QueryConfig { + salsa::QueryConfig::::new() .with_ground_query(GET_TEXT, |state, id| { mk_ground_query::(state, id, |state, id| state[id].clone()) }) From 60fdfec32759d5e006eae9fe09a87b1a28b19983 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 12 Sep 2018 22:30:48 +0300 Subject: [PATCH 10/18] eager invalidation --- crates/salsa/src/lib.rs | 56 ++++++++++++++++++++++++++++--- crates/salsa/tests/integration.rs | 32 +++++++++++++----- 2 files changed, 76 insertions(+), 12 deletions(-) diff --git a/crates/salsa/src/lib.rs b/crates/salsa/src/lib.rs index a54f2a06fca1..5de3c7774772 100644 --- a/crates/salsa/src/lib.rs +++ b/crates/salsa/src/lib.rs @@ -3,7 +3,7 @@ extern crate parking_lot; use std::{ sync::Arc, - collections::HashMap, + collections::{HashSet, HashMap}, cell::RefCell, }; use parking_lot::Mutex; @@ -138,7 +138,16 @@ where return (record.output.clone(), record.output_fingerprint) } if self.query_config.ground_fn.contains_key(&query_id.0) { - return self.force(query_id, params); + let (invalidated, record) = { + let guard = self.db.graph.lock(); + let (gen, ref record) = guard[&query_id]; + (gen == INVALIDATED, record.clone()) + }; + if invalidated { + return self.force(query_id, params); + } else { + return (record.output.clone(), record.output_fingerprint); + } } for (dep_query_id, prev_fingerprint) in record.deps.iter().cloned() { let dep_params: D = { @@ -198,6 +207,28 @@ where } } +pub struct Invalidations { + types: HashSet, + ids: Vec, +} + +impl Invalidations { + pub fn new() -> Invalidations { + Invalidations { + types: HashSet::new(), + ids: Vec::new(), + } + } + pub fn invalidate( + &mut self, + query_type: QueryTypeId, + params: impl Iterator, + ) { + self.types.insert(query_type); + self.ids.extend(params.map(|it| QueryId(query_type, it))) + } +} + impl Db where D: Clone @@ -209,9 +240,25 @@ where } } - pub fn with_ground_data(&self, ground_data: T) -> Db { + pub fn with_ground_data( + &self, + ground_data: T, + invalidations: Invalidations, + ) -> Db { + for id in self.query_config.ground_fn.keys() { + assert!( + invalidations.types.contains(id), + "all ground queries must be invalidated" + ); + } + let gen = Gen(self.db.gen.0 + 1); - let graph = self.db.graph.lock().clone(); + let mut graph = self.db.graph.lock().clone(); + for id in invalidations.ids { + if let Some((gen, _)) = graph.get_mut(&id) { + *gen = INVALIDATED; + } + } let graph = Mutex::new(graph); Db { db: Arc::new(DbState { ground_data, gen, graph }), @@ -232,6 +279,7 @@ where #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] struct Gen(u64); +const INVALIDATED: Gen = Gen(!0); #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct InputFingerprint(pub u64); #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] diff --git a/crates/salsa/tests/integration.rs b/crates/salsa/tests/integration.rs index 2872d39133f5..3cec330e64b3 100644 --- a/crates/salsa/tests/integration.rs +++ b/crates/salsa/tests/integration.rs @@ -1,5 +1,6 @@ extern crate salsa; use std::{ + iter::once, sync::Arc, collections::hash_map::{HashMap, DefaultHasher}, any::Any, @@ -113,30 +114,45 @@ fn test_number_of_lines() { assert_eq!(trace.len(), 0); state.insert(1, "hello\nworld".to_string()); - let db = db.with_ground_data(state.clone()); + let mut inv = salsa::Invalidations::new(); + inv.invalidate(GET_TEXT, once(i_print(&1u32))); + inv.invalidate(GET_FILES, once(i_print(&()))); + let db = db.with_ground_data(state.clone(), inv); let (newlines, trace) = get::<(), usize>(&db, TOTAL_NEWLINES, ()); assert_eq!(*newlines, 2); assert_eq!(trace.len(), 4); state.insert(2, "spam\neggs".to_string()); - let db = db.with_ground_data(state.clone()); + let mut inv = salsa::Invalidations::new(); + inv.invalidate(GET_TEXT, once(i_print(&2u32))); + inv.invalidate(GET_FILES, once(i_print(&()))); + let db = db.with_ground_data(state.clone(), inv); let (newlines, trace) = get::<(), usize>(&db, TOTAL_NEWLINES, ()); assert_eq!(*newlines, 4); - assert_eq!(trace.len(), 5); + assert_eq!(trace.len(), 4); + let mut invs = vec![]; for i in 0..10 { - state.insert(i + 10, "spam".to_string()); + let id = i + 10; + invs.push(i_print(&id)); + state.insert(id, "spam".to_string()); } - let db = db.with_ground_data(state.clone()); + let mut inv = salsa::Invalidations::new(); + inv.invalidate(GET_TEXT, invs.into_iter()); + inv.invalidate(GET_FILES, once(i_print(&()))); + let db = db.with_ground_data(state.clone(), inv); let (newlines, trace) = get::<(), usize>(&db, TOTAL_NEWLINES, ()); assert_eq!(*newlines, 14); - assert_eq!(trace.len(), 24); + assert_eq!(trace.len(), 22); state.insert(15, String::new()); - let db = db.with_ground_data(state.clone()); + let mut inv = salsa::Invalidations::new(); + inv.invalidate(GET_TEXT, once(i_print(&15u32))); + inv.invalidate(GET_FILES, once(i_print(&()))); + let db = db.with_ground_data(state.clone(), inv); let (newlines, trace) = get::<(), usize>(&db, TOTAL_NEWLINES, ()); assert_eq!(*newlines, 13); - assert_eq!(trace.len(), 15); + assert_eq!(trace.len(), 4); } fn o_print(x: &T) -> salsa::OutputFingerprint { From 8c737255ff876fc61f8dc8a7d33252476a4b4c8d Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 13 Sep 2018 22:58:36 +0300 Subject: [PATCH 11/18] use salsa for new module map --- crates/libanalysis/Cargo.toml | 1 + crates/libanalysis/src/db.rs | 306 ------------------------ crates/libanalysis/src/db/mod.rs | 196 +++++++++++++++ crates/libanalysis/src/db/queries.rs | 43 ++++ crates/libanalysis/src/lib.rs | 1 + crates/libanalysis/src/module_map_db.rs | 132 +++++----- crates/salsa/src/lib.rs | 20 +- crates/salsa/tests/integration.rs | 16 +- 8 files changed, 319 insertions(+), 396 deletions(-) delete mode 100644 crates/libanalysis/src/db.rs create mode 100644 crates/libanalysis/src/db/mod.rs create mode 100644 crates/libanalysis/src/db/queries.rs diff --git a/crates/libanalysis/Cargo.toml b/crates/libanalysis/Cargo.toml index 4c92951b148f..88f29d7c869a 100644 --- a/crates/libanalysis/Cargo.toml +++ b/crates/libanalysis/Cargo.toml @@ -14,6 +14,7 @@ fst = "0.3.1" im = "12.0.0" libsyntax2 = { path = "../libsyntax2" } libeditor = { path = "../libeditor" } +salsa = { path = "../salsa" } [dev-dependencies] test_utils = { path = "../test_utils" } diff --git a/crates/libanalysis/src/db.rs b/crates/libanalysis/src/db.rs deleted file mode 100644 index d30e75fe2b6b..000000000000 --- a/crates/libanalysis/src/db.rs +++ /dev/null @@ -1,306 +0,0 @@ -use std::{ - hash::{Hash, Hasher}, - sync::Arc, - cell::RefCell, - fmt::Debug, - any::Any, -}; -use parking_lot::Mutex; -use libsyntax2::{File}; -use im; -use { - FileId, - imp::{FileResolverImp}, -}; - -#[derive(Debug)] -pub(crate) struct DbHost { - db: Arc, -} - -impl DbHost { - pub(crate) fn new() -> DbHost { - let db = Db { - file_resolver: FileResolverImp::default(), - files: im::HashMap::new(), - cache: Mutex::new(Cache::new()) - }; - DbHost { db: Arc::new(db) } - } - pub(crate) fn change_file(&mut self, file_id: FileId, text: Option) { - let db = self.db_mut(); - match text { - None => { - db.files.remove(&file_id); - } - Some(text) => { - db.files.insert(file_id, Arc::new(text)); - } - } - } - pub(crate) fn set_file_resolver(&mut self, file_resolver: FileResolverImp) { - let db = self.db_mut(); - db.file_resolver = file_resolver - } - pub(crate) fn query_ctx(&self) -> QueryCtx { - QueryCtx { - db: Arc::clone(&self.db), - stack: RefCell::new(Vec::new()), - trace: RefCell::new(Vec::new()), - } - } - fn db_mut(&mut self) -> &mut Db { - // NB: this "forks" the database - let db = Arc::make_mut(&mut self.db); - db.cache.get_mut().gen += 1; - db - } -} - -type QueryInvocationId = (u32, u64); -type Gen = u64; -type OutputHash = u64; - -fn id(params: &Q::Params) -> QueryInvocationId { - use std::collections::hash_map::DefaultHasher; - let mut hasher = DefaultHasher::new(); - params.hash(&mut hasher); - (Q::ID, hasher.finish()) -} -fn output_hash(output: &Q::Output) -> OutputHash { - use std::collections::hash_map::DefaultHasher; - let mut hasher = DefaultHasher::new(); - output.hash(&mut hasher); - hasher.finish() -} - -#[derive(Debug)] -pub(crate) struct Db { - file_resolver: FileResolverImp, - files: im::HashMap>, - cache: Mutex, -} - -impl Clone for Db { - fn clone(&self) -> Db { - Db { - file_resolver: self.file_resolver.clone(), - files: self.files.clone(), - cache: Mutex::new(Cache::new()), - } - } -} - -type QueryDeps = Vec<(QueryInvocationId, Arc, OutputHash)>; - -#[derive(Default, Debug)] -pub(crate) struct Cache { - gen: Gen, - green: im::HashMap, - deps: im::HashMap, - results: im::HashMap>, -} - - -#[allow(type_alias_bounds)] -pub(crate) type QueryCache = im::HashMap< - ::Params, - ::Output ->; - -impl Cache { - fn new() -> Cache { - Default::default() - } - - fn get_result(&self, id: QueryInvocationId) -> Q::Output - where - Q::Output: Clone - { - let res = &self.results[&id]; - let res = res.downcast_ref::().unwrap(); - res.clone() - } -} - -pub(crate) struct QueryCtx { - db: Arc, - stack: RefCell>, - pub(crate) trace: RefCell>, -} - -#[derive(Clone, Copy, Debug)] -pub(crate) struct TraceEvent { - pub(crate) query_id: u32, - pub(crate) kind: TraceEventKind -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum TraceEventKind { - Start, Evaluating, Finish -} - -impl QueryCtx { - pub(crate) fn get(&self, params: &Q::Params) -> Q::Output { - let me = id::(params); - self.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Start }); - let res = Q::get(self, params); - self.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Finish }); - { - let mut stack = self.stack.borrow_mut(); - if let Some((_, ref mut deps)) = stack.last_mut() { - let params = Arc::new(params.clone()); - deps.push((me, params, output_hash::(&res))); - } - } - - res - } - fn trace(&self, event: TraceEvent) { - self.trace.borrow_mut().push(event) - } -} - -pub(crate) trait Query { - const ID: u32; - type Params: Hash + Eq + Debug + Clone + Any + 'static; - type Output: Hash + Debug + Any + 'static; -} - -pub(crate) trait Get: Query { - fn get(ctx: &QueryCtx, params: &Self::Params) -> Self::Output; -} - -impl Get for Q -where - Q::Params: Clone, - Q::Output: Clone, -{ - fn get(ctx: &QueryCtx, params: &Self::Params) -> Self::Output { - if let Some(res) = try_reuse::(ctx, params) { - return res; - } - - let me = id::(params); - ctx.trace(TraceEvent { query_id: Q::ID, kind: TraceEventKind::Evaluating }); - ctx.stack.borrow_mut().push((me, Vec::new())); - let res = Self::eval(ctx, params); - let (also_me, deps) = ctx.stack.borrow_mut().pop().unwrap(); - assert_eq!(also_me, me); - let mut cache = ctx.db.cache.lock(); - cache.deps.insert(me, deps); - let gen = cache.gen; - let output_hash = output_hash::(&res); - let id = id::(params); - cache.green.insert(id, (gen, output_hash)); - cache.results.insert(me, Arc::new(res.clone())); - res - } -} - -fn try_reuse(ctx: &QueryCtx, params: &Q::Params) -> Option -where - Q::Params: Clone, - Q::Output: Clone, -{ - let id = id::(params); - let mut cache = ctx.db.cache.lock(); - let curr_gen = cache.gen; - let old_hash = match *cache.green.get(&id)? { - (gen, _) if gen == curr_gen => { - return Some(cache.get_result::(id)); - } - (_, hash) => hash, - }; - let deps_are_fresh = cache.deps[&id] - .iter() - .all(|&(dep_id, _, dep_hash)| { - match cache.green.get(&dep_id) { - //TODO: store the value of parameters, and re-execute the query - Some((gen, hash)) if gen == &curr_gen && hash == &dep_hash => true, - _ => false, - } - }); - if !deps_are_fresh { - return None; - } - cache.green.insert(id, (curr_gen, old_hash)); - Some(cache.get_result::(id)) -} - -pub(crate) trait Eval: Query -where - Self::Params: Clone, - Self::Output: Clone, -{ - fn eval(ctx: &QueryCtx, params: &Self::Params) -> Self::Output; -} - -#[derive(Debug)] -pub(crate) struct DbFiles { - db: Arc, -} - -impl Hash for DbFiles { - fn hash(&self, hasher: &mut H) { - self.db.cache.lock().gen.hash(hasher) - } -} - -impl DbFiles { - pub(crate) fn iter<'a>(&'a self) -> impl Iterator + 'a { - self.db.files.keys().cloned() - } - pub(crate) fn file_resolver(&self) -> FileResolverImp { - self.db.file_resolver.clone() - } -} - -pub(crate) enum Files {} -impl Query for Files { - const ID: u32 = 1; - type Params = (); - type Output = DbFiles; -} -impl Get for Files { - fn get(ctx: &QueryCtx, params: &()) -> DbFiles { - let res = DbFiles { db: Arc::clone(&ctx.db) }; - let id = id::(params); - let hash = output_hash::(&res); - let mut cache = ctx.db.cache.lock(); - let gen = cache.gen; - cache.green.insert(id, (gen, hash)); - res - } -} - -enum FileText {} -impl Query for FileText { - const ID: u32 = 10; - type Params = FileId; - type Output = Arc; -} -impl Get for FileText { - fn get(ctx: &QueryCtx, file_id: &FileId) -> Arc { - let res = ctx.db.files[file_id].clone(); - let id = id::(file_id); - let hash = output_hash::(&res); - let mut cache = ctx.db.cache.lock(); - let gen = cache.gen; - cache.green.insert(id, (gen, hash)); - res - } -} - -pub(crate) enum FileSyntax {} -impl Query for FileSyntax { - const ID: u32 = 20; - type Params = FileId; - type Output = File; -} -impl Eval for FileSyntax { - fn eval(ctx: &QueryCtx, file_id: &FileId) -> File { - let text = ctx.get::(file_id); - File::parse(&text) - } -} diff --git a/crates/libanalysis/src/db/mod.rs b/crates/libanalysis/src/db/mod.rs new file mode 100644 index 000000000000..f68aab61c0f1 --- /dev/null +++ b/crates/libanalysis/src/db/mod.rs @@ -0,0 +1,196 @@ +mod queries; + +use std::{ + hash::{Hash}, + sync::Arc, + fmt::Debug, + any::Any, + iter, +}; +use im; +use salsa; +use { + FileId, + imp::{FileResolverImp}, +}; + + +#[derive(Clone, Default)] +pub(crate) struct State { + pub(crate) resolver: FileResolverImp, + pub(crate) file_map: im::HashMap>, +} + +type Data = Arc; + +pub(crate) struct QueryCtx<'a> { + inner: &'a salsa::QueryCtx +} + +pub(crate) struct Db { + inner: salsa::Db +} + +struct GroundQuery { + id: u16, + f: fn(&State, &T) -> R, + h: fn(&R) -> u64, +} + +pub(crate) struct Query { + pub(crate) id: u16, + pub(crate) f: fn(QueryCtx, &T) -> R, +} + +impl Db { + pub(crate) fn new(state: State) -> Db { + Db { inner: salsa::Db::new(query_config(), state) } + } + pub(crate) fn state(&self) -> &State { + self.inner.ground_data() + } + pub(crate) fn with_state( + &self, + new_state: State, + updated_files: &[FileId], + file_set_changed: bool, + ) -> Db { + let mut inv = salsa::Invalidations::new(); + if file_set_changed { + inv.invalidate( + salsa::QueryTypeId(queries::FILE_SET.id), + iter::once(salsa::InputFingerprint(hash(&()))), + ); + } else { + inv.invalidate( + salsa::QueryTypeId(queries::FILE_SET.id), + iter::empty(), + ); + } + inv.invalidate( + salsa::QueryTypeId(queries::FILE_TEXT.id), + updated_files.iter().map(hash).map(salsa::InputFingerprint), + ); + Db { inner: self.inner.with_ground_data(new_state, inv) } + } + pub(crate) fn get(&self, q: Query, params: T) -> (Arc, Vec) + where + T: Hash + Send + Sync + 'static, + R: Send + Sync + 'static, + { + let query_id = salsa::QueryId( + salsa::QueryTypeId(q.id), + salsa::InputFingerprint(hash(¶ms)), + ); + let params = Arc::new(params); + let (res, events) = self.inner.get(query_id, params); + let res = res.downcast().unwrap(); + let events = events.into_iter().map(|it| it.0).collect(); + (res, events) + } + +} + +impl<'a> QueryCtx<'a> { + fn get_g(&self, q: GroundQuery, params: T) -> Arc + where + T: Hash + Send + Sync + 'static, + R: Send + Sync + 'static, + { + let query_id = salsa::QueryId( + salsa::QueryTypeId(q.id), + salsa::InputFingerprint(hash(¶ms)), + ); + let res = self.inner.get(query_id, Arc::new(params)); + res.downcast().unwrap() + } + pub(crate) fn get(&self, q: Query, params: T) -> Arc + where + T: Hash + Send + Sync + 'static, + R: Send + Sync + 'static, + { + let query_id = salsa::QueryId( + salsa::QueryTypeId(q.id), + salsa::InputFingerprint(hash(¶ms)), + ); + let res = self.inner.get(query_id, Arc::new(params)); + res.downcast().unwrap() + } +} + +fn query_config() -> salsa::QueryConfig { + let mut res = salsa::QueryConfig::new(); + let queries: Vec = vec![ + queries::FILE_TEXT.into(), + queries::FILE_SET.into(), + ]; + for q in queries { + res = res.with_ground_query(q.query_type, q.f) + } + let queries: Vec = vec![ + queries::FILE_SYNTAX.into(), + ::module_map_db::MODULE_DESCR.into(), + ::module_map_db::RESOLVE_SUBMODULE.into(), + ::module_map_db::PARENT_MODULE.into(), + ]; + for q in queries { + res = res.with_query(q.query_type, q.f); + } + res +} + +struct SalsaGroundQuery { + query_type: salsa::QueryTypeId, + f: Box (Data, salsa::OutputFingerprint) + Send + Sync + 'static>, +} + +impl From> for SalsaGroundQuery +where + T: Send + Sync + 'static, + R: Send + Sync + 'static, +{ + fn from(q: GroundQuery) -> SalsaGroundQuery + { + SalsaGroundQuery { + query_type: salsa::QueryTypeId(q.id), + f: Box::new(move |state, data| { + let data: &T = data.downcast_ref().unwrap(); + let res = (q.f)(state, data); + let h = (q.h)(&res); + (Arc::new(res), salsa::OutputFingerprint(h)) + }) + } + } +} + +struct SalsaQuery { + query_type: salsa::QueryTypeId, + f: Box, &Data) -> (Data, salsa::OutputFingerprint) + Send + Sync + 'static>, +} + +impl From> for SalsaQuery +where + T: Hash + Send + Sync + 'static, + R: Hash + Send + Sync + 'static, +{ + fn from(q: Query) -> SalsaQuery + { + SalsaQuery { + query_type: salsa::QueryTypeId(q.id), + f: Box::new(move |ctx, data| { + let ctx = QueryCtx { inner: ctx }; + let data: &T = data.downcast_ref().unwrap(); + let res = (q.f)(ctx, data); + let h = hash(&res); + (Arc::new(res), salsa::OutputFingerprint(h)) + }) + } + } +} + +fn hash(x: &T) -> u64 { + use std::hash::Hasher; + let mut hasher = ::std::collections::hash_map::DefaultHasher::new(); + ::std::hash::Hash::hash(x, &mut hasher); + hasher.finish() +} diff --git a/crates/libanalysis/src/db/queries.rs b/crates/libanalysis/src/db/queries.rs new file mode 100644 index 000000000000..2d4aac6e93e2 --- /dev/null +++ b/crates/libanalysis/src/db/queries.rs @@ -0,0 +1,43 @@ +use std::sync::Arc; +use libsyntax2::{File}; +use { + FileId, FileResolverImp, + db::{Query, GroundQuery, QueryCtx, hash}, +}; + + +impl<'a> QueryCtx<'a> { + pub(crate) fn file_set(&self) -> Arc<(Vec, FileResolverImp)> { + self.get_g(FILE_SET, ()) + } + pub(crate) fn file_text(&self, file_id: FileId) -> Arc { + Arc::clone(&*self.get_g(FILE_TEXT, file_id)) + } + pub(crate) fn file_syntax(&self, file_id: FileId) -> File { + (&*self.get(FILE_SYNTAX, file_id)).clone() + } +} + +pub(super) const FILE_TEXT: GroundQuery> = GroundQuery { + id: 10, + f: |state, id| state.file_map[&id].clone(), + h: hash, +}; + +pub(super) const FILE_SET: GroundQuery<(), (Vec, FileResolverImp)> = GroundQuery { + id: 11, + f: |state, &()| { + let files = state.file_map.keys().cloned().collect(); + let resolver = state.resolver.clone(); + (files, resolver) + }, + h: |(files, _)| hash(files), +}; + +pub(super) const FILE_SYNTAX: Query = Query { + id: 20, + f: |ctx, file_id: &FileId| { + let text = ctx.file_text(*file_id); + File::parse(&*text) + } +}; diff --git a/crates/libanalysis/src/lib.rs b/crates/libanalysis/src/lib.rs index 3e77006c5ace..bca5d4c1d678 100644 --- a/crates/libanalysis/src/lib.rs +++ b/crates/libanalysis/src/lib.rs @@ -10,6 +10,7 @@ extern crate relative_path; #[macro_use] extern crate crossbeam_channel; extern crate im; +extern crate salsa; mod symbol_index; mod module_map; diff --git a/crates/libanalysis/src/module_map_db.rs b/crates/libanalysis/src/module_map_db.rs index 4d4bd91046f6..ff69cc0d2085 100644 --- a/crates/libanalysis/src/module_map_db.rs +++ b/crates/libanalysis/src/module_map_db.rs @@ -2,66 +2,55 @@ use std::sync::Arc; use { FileId, db::{ - Query, Eval, QueryCtx, FileSyntax, Files, - Cache, QueryCache, + Query, QueryCtx }, module_map::resolve_submodule, }; -pub(crate) enum ModuleDescr {} -impl Query for ModuleDescr { - const ID: u32 = 30; - type Params = FileId; - type Output = Arc; -} - -enum ResolveSubmodule {} -impl Query for ResolveSubmodule { - const ID: u32 = 31; - type Params = (FileId, descr::Submodule); - type Output = Arc>; -} - -enum ParentModule {} -impl Query for ParentModule { - const ID: u32 = 40; - type Params = FileId; - type Output = Arc>; -} - -impl Eval for ModuleDescr { - fn eval(ctx: &QueryCtx, file_id: &FileId) -> Arc { - let file = ctx.get::(file_id); - Arc::new(descr::ModuleDescr::new(file.ast())) +impl<'a> QueryCtx<'a> { + fn module_descr(&self, file_id: FileId) -> Arc { + self.get(MODULE_DESCR, file_id) + } + fn resolve_submodule(&self, file_id: FileId, submod: descr::Submodule) -> Arc> { + self.get(RESOLVE_SUBMODULE, (file_id, submod)) } } -impl Eval for ResolveSubmodule { - fn eval(ctx: &QueryCtx, &(file_id, ref submodule): &(FileId, descr::Submodule)) -> Arc> { - let files = ctx.get::(&()); - let res = resolve_submodule(file_id, &submodule.name, &files.file_resolver()).0; - Arc::new(res) +pub(crate) const MODULE_DESCR: Query = Query { + id: 30, + f: |ctx, &file_id| { + let file = ctx.file_syntax(file_id); + descr::ModuleDescr::new(file.ast()) } -} +}; -impl Eval for ParentModule { - fn eval(ctx: &QueryCtx, file_id: &FileId) -> Arc> { - let files = ctx.get::(&()); - let res = files.iter() - .map(|parent_id| (parent_id, ctx.get::(&parent_id))) +pub(crate) const RESOLVE_SUBMODULE: Query<(FileId, descr::Submodule), Vec> = Query { + id: 31, + f: |ctx, params| { + let files = ctx.file_set(); + resolve_submodule(params.0, ¶ms.1.name, &files.1).0 + } +}; + +pub(crate) const PARENT_MODULE: Query> = Query { + id: 40, + f: |ctx, file_id| { + let files = ctx.file_set(); + let res = files.0.iter() + .map(|&parent_id| (parent_id, ctx.module_descr(parent_id))) .filter(|(parent_id, descr)| { descr.submodules.iter() .any(|subm| { - ctx.get::(&(*parent_id, subm.clone())) + ctx.resolve_submodule(*parent_id, subm.clone()) .iter() .any(|it| it == file_id) }) }) .map(|(id, _)| id) .collect(); - Arc::new(res) + res } -} +}; mod descr { use libsyntax2::{ @@ -102,7 +91,7 @@ mod tests { use im; use relative_path::{RelativePath, RelativePathBuf}; use { - db::{Query, DbHost, TraceEventKind}, + db::{Query, Db, State}, imp::FileResolverImp, FileId, FileResolver, }; @@ -126,7 +115,7 @@ mod tests { struct Fixture { next_file_id: u32, fm: im::HashMap, - db: DbHost, + db: Db, } impl Fixture { @@ -134,7 +123,7 @@ mod tests { Fixture { next_file_id: 1, fm: im::HashMap::new(), - db: DbHost::new(), + db: Db::new(State::default()), } } fn add_file(&mut self, path: &str, text: &str) -> FileId { @@ -142,36 +131,39 @@ mod tests { let file_id = FileId(self.next_file_id); self.next_file_id += 1; self.fm.insert(file_id, RelativePathBuf::from(&path[1..])); - self.db.change_file(file_id, Some(text.to_string())); - self.db.set_file_resolver(FileResolverImp::new( + let mut new_state = self.db.state().clone(); + new_state.file_map.insert(file_id, text.to_string().into_boxed_str().into()); + new_state.resolver = FileResolverImp::new( Arc::new(FileMap(self.fm.clone())) - )); - + ); + self.db = self.db.with_state(new_state, &[file_id], true); file_id } fn remove_file(&mut self, file_id: FileId) { self.fm.remove(&file_id); - self.db.change_file(file_id, None); - self.db.set_file_resolver(FileResolverImp::new( + let mut new_state = self.db.state().clone(); + new_state.file_map.remove(&file_id); + new_state.resolver = FileResolverImp::new( Arc::new(FileMap(self.fm.clone())) - )) + ); + self.db = self.db.with_state(new_state, &[file_id], true); } fn change_file(&mut self, file_id: FileId, new_text: &str) { - self.db.change_file(file_id, Some(new_text.to_string())); + let mut new_state = self.db.state().clone(); + new_state.file_map.insert(file_id, new_text.to_string().into_boxed_str().into()); + self.db = self.db.with_state(new_state, &[file_id], false); } fn check_parent_modules( &self, file_id: FileId, expected: &[FileId], - queries: &[(u32, u64)] + queries: &[(u16, u64)] ) { - let ctx = self.db.query_ctx(); - let actual = ctx.get::(&file_id); + let (actual, events) = self.db.get(PARENT_MODULE, file_id); assert_eq!(actual.as_slice(), expected); let mut counts = HashMap::new(); - ctx.trace.borrow().iter() - .filter(|event| event.kind == TraceEventKind::Evaluating) - .for_each(|event| *counts.entry(event.query_id).or_insert(0) += 1); + events.into_iter() + .for_each(|event| *counts.entry(event).or_insert(0) += 1); for &(query_id, expected_count) in queries.iter() { let actual_count = *counts.get(&query_id).unwrap_or(&0); assert_eq!( @@ -189,25 +181,25 @@ mod tests { fn test_parent_module() { let mut f = Fixture::new(); let foo = f.add_file("/foo.rs", ""); - // f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 1)]); + f.check_parent_modules(foo, &[], &[(MODULE_DESCR.id, 1)]); let lib = f.add_file("/lib.rs", "mod foo;"); - f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 2)]); - f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 0)]); + f.check_parent_modules(foo, &[lib], &[(MODULE_DESCR.id, 1)]); + f.check_parent_modules(foo, &[lib], &[(MODULE_DESCR.id, 0)]); f.change_file(lib, ""); - f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 2)]); + f.check_parent_modules(foo, &[], &[(MODULE_DESCR.id, 1)]); - // f.change_file(lib, "mod foo;"); - // f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 2)]); + f.change_file(lib, "mod foo;"); + f.check_parent_modules(foo, &[lib], &[(MODULE_DESCR.id, 1)]); - // f.change_file(lib, "mod bar;"); - // f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 2)]); + f.change_file(lib, "mod bar;"); + f.check_parent_modules(foo, &[], &[(MODULE_DESCR.id, 1)]); - // f.change_file(lib, "mod foo;"); - // f.check_parent_modules(foo, &[lib], &[(ModuleDescr::ID, 2)]); + f.change_file(lib, "mod foo;"); + f.check_parent_modules(foo, &[lib], &[(MODULE_DESCR.id, 1)]); - // f.remove_file(lib); - // f.check_parent_modules(foo, &[], &[(ModuleDescr::ID, 1)]); + f.remove_file(lib); + f.check_parent_modules(foo, &[], &[(MODULE_DESCR.id, 0)]); } } diff --git a/crates/salsa/src/lib.rs b/crates/salsa/src/lib.rs index 5de3c7774772..75815e8bd2d4 100644 --- a/crates/salsa/src/lib.rs +++ b/crates/salsa/src/lib.rs @@ -8,8 +8,8 @@ use std::{ }; use parking_lot::Mutex; -type GroundQueryFn = fn(&T, &D) -> (D, OutputFingerprint); -type QueryFn = fn(&QueryCtx, &D) -> (D, OutputFingerprint); +type GroundQueryFn = Box (D, OutputFingerprint) + Send + Sync + 'static>; +type QueryFn = Box, &D) -> (D, OutputFingerprint) + Send + Sync + 'static>; #[derive(Debug)] pub struct Db { @@ -119,7 +119,7 @@ where res } - pub fn get_inner( + fn get_inner( &self, query_id: QueryId, params: D, @@ -176,9 +176,9 @@ where self.executed.borrow_mut().push(query_id.0); self.stack.borrow_mut().push(Vec::new()); - let (res, output_fingerprint) = if let Some(f) = self.ground_query_fn_by_type(query_id.0) { + let (res, output_fingerprint) = if let Some(f) = self.query_config.ground_fn.get(&query_id.0) { f(&self.db.ground_data, ¶ms) - } else if let Some(f) = self.query_fn_by_type(query_id.0) { + } else if let Some(f) = self.query_config.query_fn.get(&query_id.0) { f(self, ¶ms) } else { panic!("unknown query type: {:?}", query_id.0); @@ -190,12 +190,6 @@ where self.db.record(query_id, params, res.clone(), output_fingerprint, deps); (res, output_fingerprint) } - fn ground_query_fn_by_type(&self, query_type: QueryTypeId) -> Option> { - self.query_config.ground_fn.get(&query_type).map(|&it| it) - } - fn query_fn_by_type(&self, query_type: QueryTypeId) -> Option> { - self.query_config.query_fn.get(&query_type).map(|&it| it) - } fn record_dep( &self, query_id: QueryId, @@ -239,7 +233,9 @@ where query_config: Arc::new(query_config), } } - + pub fn ground_data(&self) -> &T { + &self.db.ground_data + } pub fn with_ground_data( &self, ground_data: T, diff --git a/crates/salsa/tests/integration.rs b/crates/salsa/tests/integration.rs index 3cec330e64b3..aed9219bee1f 100644 --- a/crates/salsa/tests/integration.rs +++ b/crates/salsa/tests/integration.rs @@ -79,19 +79,19 @@ where fn mk_queries() -> salsa::QueryConfig { salsa::QueryConfig::::new() - .with_ground_query(GET_TEXT, |state, id| { + .with_ground_query(GET_TEXT, Box::new(|state, id| { mk_ground_query::(state, id, |state, id| state[id].clone()) - }) - .with_ground_query(GET_FILES, |state, id| { + })) + .with_ground_query(GET_FILES, Box::new(|state, id| { mk_ground_query::<(), Vec>(state, id, |state, &()| state.keys().cloned().collect()) - }) - .with_query(FILE_NEWLINES, |query_ctx, id| { + })) + .with_query(FILE_NEWLINES, Box::new(|query_ctx, id| { mk_query(query_ctx, id, |query_ctx, &id| { let text = query_ctx.get_text(id); text.lines().count() }) - }) - .with_query(TOTAL_NEWLINES, |query_ctx, id| { + })) + .with_query(TOTAL_NEWLINES, Box::new(|query_ctx, id| { mk_query(query_ctx, id, |query_ctx, &()| { let mut total = 0; for &id in query_ctx.get_files().iter() { @@ -99,7 +99,7 @@ fn mk_queries() -> salsa::QueryConfig { } total }) - }) + })) } #[test] From 47be3a3a24de1eb28e1575db1571d934765f6d53 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 15 Sep 2018 13:38:31 +0300 Subject: [PATCH 12/18] renames --- crates/libanalysis/src/db/mod.rs | 24 +++++------ crates/libanalysis/src/module_map_db/descr.rs | 29 +++++++++++++ .../mod.rs} | 43 ++++--------------- 3 files changed, 49 insertions(+), 47 deletions(-) create mode 100644 crates/libanalysis/src/module_map_db/descr.rs rename crates/libanalysis/src/{module_map_db.rs => module_map_db/mod.rs} (87%) diff --git a/crates/libanalysis/src/db/mod.rs b/crates/libanalysis/src/db/mod.rs index f68aab61c0f1..3198272b7d64 100644 --- a/crates/libanalysis/src/db/mod.rs +++ b/crates/libanalysis/src/db/mod.rs @@ -120,38 +120,36 @@ impl<'a> QueryCtx<'a> { fn query_config() -> salsa::QueryConfig { let mut res = salsa::QueryConfig::new(); - let queries: Vec = vec![ + let queries: Vec = vec![ queries::FILE_TEXT.into(), queries::FILE_SET.into(), ]; for q in queries { res = res.with_ground_query(q.query_type, q.f) } - let queries: Vec = vec![ + let mut queries: Vec = vec![ queries::FILE_SYNTAX.into(), - ::module_map_db::MODULE_DESCR.into(), - ::module_map_db::RESOLVE_SUBMODULE.into(), - ::module_map_db::PARENT_MODULE.into(), ]; + ::module_map_db::queries(&mut queries); for q in queries { res = res.with_query(q.query_type, q.f); } res } -struct SalsaGroundQuery { +struct BoxedGroundQuery { query_type: salsa::QueryTypeId, f: Box (Data, salsa::OutputFingerprint) + Send + Sync + 'static>, } -impl From> for SalsaGroundQuery +impl From> for BoxedGroundQuery where T: Send + Sync + 'static, R: Send + Sync + 'static, { - fn from(q: GroundQuery) -> SalsaGroundQuery + fn from(q: GroundQuery) -> BoxedGroundQuery { - SalsaGroundQuery { + BoxedGroundQuery { query_type: salsa::QueryTypeId(q.id), f: Box::new(move |state, data| { let data: &T = data.downcast_ref().unwrap(); @@ -163,19 +161,19 @@ where } } -struct SalsaQuery { +pub(crate) struct BoxedQuery { query_type: salsa::QueryTypeId, f: Box, &Data) -> (Data, salsa::OutputFingerprint) + Send + Sync + 'static>, } -impl From> for SalsaQuery +impl From> for BoxedQuery where T: Hash + Send + Sync + 'static, R: Hash + Send + Sync + 'static, { - fn from(q: Query) -> SalsaQuery + fn from(q: Query) -> BoxedQuery { - SalsaQuery { + BoxedQuery { query_type: salsa::QueryTypeId(q.id), f: Box::new(move |ctx, data| { let ctx = QueryCtx { inner: ctx }; diff --git a/crates/libanalysis/src/module_map_db/descr.rs b/crates/libanalysis/src/module_map_db/descr.rs new file mode 100644 index 000000000000..fb298a315d8e --- /dev/null +++ b/crates/libanalysis/src/module_map_db/descr.rs @@ -0,0 +1,29 @@ +use libsyntax2::{ + SmolStr, + ast::{self, NameOwner}, +}; + +#[derive(Debug, Hash)] +pub struct ModuleDescr { + pub submodules: Vec +} + +impl ModuleDescr { + pub fn new(root: ast::Root) -> ModuleDescr { + let submodules = root + .modules() + .filter_map(|module| { + let name = module.name()?.text(); + if !module.has_semi() { + return None; + } + Some(Submodule { name }) + }).collect(); + + ModuleDescr { submodules } } +} + +#[derive(Clone, Hash, PartialEq, Eq, Debug)] +pub struct Submodule { + pub name: SmolStr, +} diff --git a/crates/libanalysis/src/module_map_db.rs b/crates/libanalysis/src/module_map_db/mod.rs similarity index 87% rename from crates/libanalysis/src/module_map_db.rs rename to crates/libanalysis/src/module_map_db/mod.rs index ff69cc0d2085..777f7a38ad4a 100644 --- a/crates/libanalysis/src/module_map_db.rs +++ b/crates/libanalysis/src/module_map_db/mod.rs @@ -1,12 +1,20 @@ +mod descr; + use std::sync::Arc; use { FileId, db::{ - Query, QueryCtx + BoxedQuery, Query, QueryCtx }, module_map::resolve_submodule, }; +pub(crate) fn queries(acc: &mut Vec) { + acc.push(MODULE_DESCR.into()); + acc.push(RESOLVE_SUBMODULE.into()); + acc.push(PARENT_MODULE.into()); +} + impl<'a> QueryCtx<'a> { fn module_descr(&self, file_id: FileId) -> Arc { self.get(MODULE_DESCR, file_id) @@ -52,39 +60,6 @@ pub(crate) const PARENT_MODULE: Query> = Query { } }; -mod descr { - use libsyntax2::{ - SmolStr, - ast::{self, NameOwner}, - }; - - #[derive(Debug, Hash)] - pub struct ModuleDescr { - pub submodules: Vec - } - - impl ModuleDescr { - pub fn new(root: ast::Root) -> ModuleDescr { - let submodules = root - .modules() - .filter_map(|module| { - let name = module.name()?.text(); - if !module.has_semi() { - return None; - } - Some(Submodule { name }) - }).collect(); - - ModuleDescr { submodules } } - } - - #[derive(Clone, Hash, PartialEq, Eq, Debug)] - pub struct Submodule { - pub name: SmolStr, - } - -} - #[cfg(test)] mod tests { use std::collections::HashMap; From 0d7b1e442d1449a48e0b73b3db6ea270520ea039 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 15 Sep 2018 13:55:34 +0300 Subject: [PATCH 13/18] minor --- crates/libanalysis/src/db/mod.rs | 3 ++- crates/libanalysis/src/module_map_db/mod.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/libanalysis/src/db/mod.rs b/crates/libanalysis/src/db/mod.rs index 3198272b7d64..38ba40273668 100644 --- a/crates/libanalysis/src/db/mod.rs +++ b/crates/libanalysis/src/db/mod.rs @@ -43,7 +43,8 @@ pub(crate) struct Query { } impl Db { - pub(crate) fn new(state: State) -> Db { + pub(crate) fn new() -> Db { + let state = Default::default(); Db { inner: salsa::Db::new(query_config(), state) } } pub(crate) fn state(&self) -> &State { diff --git a/crates/libanalysis/src/module_map_db/mod.rs b/crates/libanalysis/src/module_map_db/mod.rs index 777f7a38ad4a..02ac06c5fcd3 100644 --- a/crates/libanalysis/src/module_map_db/mod.rs +++ b/crates/libanalysis/src/module_map_db/mod.rs @@ -98,7 +98,7 @@ mod tests { Fixture { next_file_id: 1, fm: im::HashMap::new(), - db: Db::new(State::default()), + db: Db::new(), } } fn add_file(&mut self, path: &str, text: &str) -> FileId { From d59413c895e7b49ed2ad01be35871e417a57a43c Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 15 Sep 2018 17:21:47 +0300 Subject: [PATCH 14/18] yet another db api --- crates/libanalysis/src/db/imp.rs | 155 ++++++++++++++ crates/libanalysis/src/db/mod.rs | 222 ++++++-------------- crates/libanalysis/src/db/queries.rs | 43 ---- crates/libanalysis/src/module_map_db/mod.rs | 104 +++++---- crates/salsa/src/lib.rs | 12 +- 5 files changed, 276 insertions(+), 260 deletions(-) create mode 100644 crates/libanalysis/src/db/imp.rs delete mode 100644 crates/libanalysis/src/db/queries.rs diff --git a/crates/libanalysis/src/db/imp.rs b/crates/libanalysis/src/db/imp.rs new file mode 100644 index 000000000000..1b4ee5cf31d2 --- /dev/null +++ b/crates/libanalysis/src/db/imp.rs @@ -0,0 +1,155 @@ +use std::{ + sync::Arc, + any::Any, + hash::{Hash, Hasher}, + collections::hash_map::{DefaultHasher, HashMap}, + iter, +}; +use salsa; +use {FileId, imp::FileResolverImp}; +use super::{State, Query, QueryCtx}; + +pub(super) type Data = Arc; + +#[derive(Debug)] +pub(super) struct Db { + names: Arc>, + pub(super) imp: salsa::Db, +} + +impl Db { + pub(super) fn new(mut reg: QueryRegistry) -> Db { + let config = reg.config.take().unwrap(); + Db { + names: Arc::new(reg.names), + imp: salsa::Db::new(config, State::default()) + } + } + pub(crate) fn with_changes(&self, new_state: State, changed_files: &[FileId], resolver_changed: bool) -> Db { + let names = self.names.clone(); + let mut invalidations = salsa::Invalidations::new(); + invalidations.invalidate(FILE_TEXT, changed_files.iter().map(hash).map(salsa::InputFingerprint)); + if resolver_changed { + invalidations.invalidate(FILE_SET, iter::once(salsa::InputFingerprint(hash(&())))); + } else { + invalidations.invalidate(FILE_SET, iter::empty()); + } + let imp = self.imp.with_ground_data( + new_state, + invalidations, + ); + Db { names, imp } + } + pub(super) fn extract_trace(&self, ctx: &salsa::QueryCtx) -> Vec<&'static str> { + ctx.trace().into_iter().map(|it| self.names[&it]).collect() + } +} + +pub(crate) trait EvalQuery { + type Params; + type Output; + fn query_type(&self) -> salsa::QueryTypeId; + fn f(&self) -> salsa::QueryFn; + fn get(&self, &QueryCtx, Self::Params) -> Arc; +} + +impl EvalQuery for Query +where + T: Hash + Send + Sync + 'static, + R: Hash + Send + Sync + 'static, +{ + type Params = T; + type Output = R; + fn query_type(&self) -> salsa::QueryTypeId { + salsa::QueryTypeId(self.0) + } + fn f(&self) -> salsa::QueryFn { + let f = self.1; + Box::new(move |ctx, data| { + let ctx = QueryCtx { imp: ctx }; + let data: &T = data.downcast_ref().unwrap(); + let res = f(ctx, data); + let h = hash(&res); + (Arc::new(res), salsa::OutputFingerprint(h)) + }) + } + fn get(&self, ctx: &QueryCtx, params: Self::Params) -> Arc { + let query_id = salsa::QueryId( + self.query_type(), + salsa::InputFingerprint(hash(¶ms)), + ); + let res = ctx.imp.get(query_id, Arc::new(params)); + res.downcast().unwrap() + } +} + +pub(super) struct QueryRegistry { + config: Option>, + names: HashMap, +} + +impl QueryRegistry { + pub(super) fn new() -> QueryRegistry { + let mut config = salsa::QueryConfig::::new(); + config = config.with_ground_query( + FILE_TEXT, Box::new(|state, params| { + let file_id: &FileId = params.downcast_ref().unwrap(); + let res = state.file_map[file_id].clone(); + let fingerprint = salsa::OutputFingerprint(hash(&res)); + (res, fingerprint) + }) + ); + config = config.with_ground_query( + FILE_SET, Box::new(|state, _params| { + let file_ids: Vec = state.file_map.keys().cloned().collect(); + let hash = hash(&file_ids); + let file_resolver = state.file_resolver.clone(); + let res = (file_ids, file_resolver); + let fingerprint = salsa::OutputFingerprint(hash); + (Arc::new(res), fingerprint) + }) + ); + let mut names = HashMap::new(); + names.insert(FILE_TEXT, "FILE_TEXT"); + names.insert(FILE_SET, "FILE_SET"); + QueryRegistry { config: Some(config), names } + } + pub(super) fn add(&mut self, q: Q, name: &'static str) { + let id = q.query_type(); + let prev = self.names.insert(id, name); + assert!(prev.is_none(), "duplicate query: {:?}", id); + let config = self.config.take().unwrap(); + let config = config.with_query(id, q.f()); + self.config= Some(config); + } + pub(super) fn finish(mut self) -> salsa::QueryConfig { + self.config.take().unwrap() + } +} + +fn hash(x: &T) -> u64 { + let mut hasher = DefaultHasher::new(); + x.hash(&mut hasher); + hasher.finish() +} + +const FILE_TEXT: salsa::QueryTypeId = salsa::QueryTypeId(0); +pub(super) fn file_text(ctx: QueryCtx, file_id: FileId) -> Arc { + let query_id = salsa::QueryId( + FILE_TEXT, + salsa::InputFingerprint(hash(&file_id)), + ); + let res = ctx.imp.get(query_id, Arc::new(file_id)); + res.downcast().unwrap() +} + +const FILE_SET: salsa::QueryTypeId = salsa::QueryTypeId(1); +pub(super) fn file_set(ctx: QueryCtx) -> Arc<(Vec, FileResolverImp)> { + let query_id = salsa::QueryId( + FILE_SET, + salsa::InputFingerprint(hash(&())), + ); + let res = ctx.imp.get(query_id, Arc::new(())); + res.downcast().unwrap() +} + diff --git a/crates/libanalysis/src/db/mod.rs b/crates/libanalysis/src/db/mod.rs index 38ba40273668..a775b5f75a3d 100644 --- a/crates/libanalysis/src/db/mod.rs +++ b/crates/libanalysis/src/db/mod.rs @@ -1,195 +1,99 @@ -mod queries; +mod imp; use std::{ - hash::{Hash}, sync::Arc, - fmt::Debug, - any::Any, - iter, }; use im; use salsa; -use { - FileId, - imp::{FileResolverImp}, -}; +use {FileId, imp::FileResolverImp}; - -#[derive(Clone, Default)] +#[derive(Debug, Default, Clone)] pub(crate) struct State { - pub(crate) resolver: FileResolverImp, - pub(crate) file_map: im::HashMap>, -} - -type Data = Arc; - -pub(crate) struct QueryCtx<'a> { - inner: &'a salsa::QueryCtx + pub(crate) file_map: im::HashMap>, + pub(crate) file_resolver: FileResolverImp } +#[derive(Debug)] pub(crate) struct Db { - inner: salsa::Db + imp: imp::Db, } -struct GroundQuery { - id: u16, - f: fn(&State, &T) -> R, - h: fn(&R) -> u64, +#[derive(Clone, Copy)] +pub(crate) struct QueryCtx<'a> { + imp: &'a salsa::QueryCtx, } -pub(crate) struct Query { - pub(crate) id: u16, - pub(crate) f: fn(QueryCtx, &T) -> R, +pub(crate) struct Query(pub(crate) u16, pub(crate) fn(QueryCtx, &T) -> R); + +pub(crate) struct QueryRegistry { + imp: imp::QueryRegistry, } impl Db { pub(crate) fn new() -> Db { - let state = Default::default(); - Db { inner: salsa::Db::new(query_config(), state) } + let reg = QueryRegistry::new(); + Db { imp: imp::Db::new(reg.imp) } } pub(crate) fn state(&self) -> &State { - self.inner.ground_data() + self.imp.imp.ground_data() } - pub(crate) fn with_state( - &self, - new_state: State, - updated_files: &[FileId], - file_set_changed: bool, - ) -> Db { - let mut inv = salsa::Invalidations::new(); - if file_set_changed { - inv.invalidate( - salsa::QueryTypeId(queries::FILE_SET.id), - iter::once(salsa::InputFingerprint(hash(&()))), - ); - } else { - inv.invalidate( - salsa::QueryTypeId(queries::FILE_SET.id), - iter::empty(), - ); - } - inv.invalidate( - salsa::QueryTypeId(queries::FILE_TEXT.id), - updated_files.iter().map(hash).map(salsa::InputFingerprint), - ); - Db { inner: self.inner.with_ground_data(new_state, inv) } + pub(crate) fn with_changes(&self, new_state: State, changed_files: &[FileId], resolver_changed: bool) -> Db { + Db { imp: self.imp.with_changes(new_state, changed_files, resolver_changed) } } - pub(crate) fn get(&self, q: Query, params: T) -> (Arc, Vec) - where - T: Hash + Send + Sync + 'static, - R: Send + Sync + 'static, - { - let query_id = salsa::QueryId( - salsa::QueryTypeId(q.id), - salsa::InputFingerprint(hash(¶ms)), - ); - let params = Arc::new(params); - let (res, events) = self.inner.get(query_id, params); - let res = res.downcast().unwrap(); - let events = events.into_iter().map(|it| it.0).collect(); - (res, events) + pub(crate) fn make_query R, R>(&self, f: F) -> R { + let ctx = QueryCtx { imp: &self.imp.imp.query_ctx() }; + f(ctx) + } + pub(crate) fn trace_query R, R>(&self, f: F) -> (R, Vec<&'static str>) { + let ctx = QueryCtx { imp: &self.imp.imp.query_ctx() }; + let res = f(ctx); + let trace = self.imp.extract_trace(ctx.imp); + (res, trace) } - } impl<'a> QueryCtx<'a> { - fn get_g(&self, q: GroundQuery, params: T) -> Arc - where - T: Hash + Send + Sync + 'static, - R: Send + Sync + 'static, - { - let query_id = salsa::QueryId( - salsa::QueryTypeId(q.id), - salsa::InputFingerprint(hash(¶ms)), - ); - let res = self.inner.get(query_id, Arc::new(params)); - res.downcast().unwrap() - } - pub(crate) fn get(&self, q: Query, params: T) -> Arc - where - T: Hash + Send + Sync + 'static, - R: Send + Sync + 'static, - { - let query_id = salsa::QueryId( - salsa::QueryTypeId(q.id), - salsa::InputFingerprint(hash(¶ms)), - ); - let res = self.inner.get(query_id, Arc::new(params)); - res.downcast().unwrap() + pub(crate) fn get(&self, q: Q, params: Q::Params) -> Arc { + q.get(self, params) } } -fn query_config() -> salsa::QueryConfig { - let mut res = salsa::QueryConfig::new(); - let queries: Vec = vec![ - queries::FILE_TEXT.into(), - queries::FILE_SET.into(), - ]; - for q in queries { - res = res.with_ground_query(q.query_type, q.f) - } - let mut queries: Vec = vec![ - queries::FILE_SYNTAX.into(), - ]; - ::module_map_db::queries(&mut queries); - for q in queries { - res = res.with_query(q.query_type, q.f); - } - res +pub(crate) fn file_text(ctx: QueryCtx, file_id: FileId) -> Arc { + imp::file_text(ctx, file_id) } -struct BoxedGroundQuery { - query_type: salsa::QueryTypeId, - f: Box (Data, salsa::OutputFingerprint) + Send + Sync + 'static>, +pub(crate) fn file_set(ctx: QueryCtx) -> Arc<(Vec, FileResolverImp)> { + imp::file_set(ctx) +} +pub(crate) use self::queries::file_syntax; + +mod queries { + use libsyntax2::File; + use {FileId}; + use super::{Query, QueryCtx, QueryRegistry, file_text}; + + pub(crate) fn register_queries(reg: &mut QueryRegistry) { + reg.add(FILE_SYNTAX, "FILE_SYNTAX") + } + + pub(crate) fn file_syntax(ctx: QueryCtx, file_id: FileId) -> File { + (&*ctx.get(FILE_SYNTAX, file_id)).clone() + } + + pub(super) const FILE_SYNTAX: Query = Query(16, |ctx, file_id: &FileId| { + let text = file_text(ctx, *file_id); + File::parse(&*text) + }); } -impl From> for BoxedGroundQuery -where - T: Send + Sync + 'static, - R: Send + Sync + 'static, -{ - fn from(q: GroundQuery) -> BoxedGroundQuery - { - BoxedGroundQuery { - query_type: salsa::QueryTypeId(q.id), - f: Box::new(move |state, data| { - let data: &T = data.downcast_ref().unwrap(); - let res = (q.f)(state, data); - let h = (q.h)(&res); - (Arc::new(res), salsa::OutputFingerprint(h)) - }) - } +impl QueryRegistry { + fn new() -> QueryRegistry { + let mut reg = QueryRegistry { imp: imp::QueryRegistry::new() }; + queries::register_queries(&mut reg); + ::module_map_db::register_queries(&mut reg); + reg + } + pub(crate) fn add(&mut self, q: Q, name: &'static str) { + self.imp.add(q, name) } } - -pub(crate) struct BoxedQuery { - query_type: salsa::QueryTypeId, - f: Box, &Data) -> (Data, salsa::OutputFingerprint) + Send + Sync + 'static>, -} - -impl From> for BoxedQuery -where - T: Hash + Send + Sync + 'static, - R: Hash + Send + Sync + 'static, -{ - fn from(q: Query) -> BoxedQuery - { - BoxedQuery { - query_type: salsa::QueryTypeId(q.id), - f: Box::new(move |ctx, data| { - let ctx = QueryCtx { inner: ctx }; - let data: &T = data.downcast_ref().unwrap(); - let res = (q.f)(ctx, data); - let h = hash(&res); - (Arc::new(res), salsa::OutputFingerprint(h)) - }) - } - } -} - -fn hash(x: &T) -> u64 { - use std::hash::Hasher; - let mut hasher = ::std::collections::hash_map::DefaultHasher::new(); - ::std::hash::Hash::hash(x, &mut hasher); - hasher.finish() -} diff --git a/crates/libanalysis/src/db/queries.rs b/crates/libanalysis/src/db/queries.rs deleted file mode 100644 index 2d4aac6e93e2..000000000000 --- a/crates/libanalysis/src/db/queries.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::sync::Arc; -use libsyntax2::{File}; -use { - FileId, FileResolverImp, - db::{Query, GroundQuery, QueryCtx, hash}, -}; - - -impl<'a> QueryCtx<'a> { - pub(crate) fn file_set(&self) -> Arc<(Vec, FileResolverImp)> { - self.get_g(FILE_SET, ()) - } - pub(crate) fn file_text(&self, file_id: FileId) -> Arc { - Arc::clone(&*self.get_g(FILE_TEXT, file_id)) - } - pub(crate) fn file_syntax(&self, file_id: FileId) -> File { - (&*self.get(FILE_SYNTAX, file_id)).clone() - } -} - -pub(super) const FILE_TEXT: GroundQuery> = GroundQuery { - id: 10, - f: |state, id| state.file_map[&id].clone(), - h: hash, -}; - -pub(super) const FILE_SET: GroundQuery<(), (Vec, FileResolverImp)> = GroundQuery { - id: 11, - f: |state, &()| { - let files = state.file_map.keys().cloned().collect(); - let resolver = state.resolver.clone(); - (files, resolver) - }, - h: |(files, _)| hash(files), -}; - -pub(super) const FILE_SYNTAX: Query = Query { - id: 20, - f: |ctx, file_id: &FileId| { - let text = ctx.file_text(*file_id); - File::parse(&*text) - } -}; diff --git a/crates/libanalysis/src/module_map_db/mod.rs b/crates/libanalysis/src/module_map_db/mod.rs index 02ac06c5fcd3..5560e4a34a65 100644 --- a/crates/libanalysis/src/module_map_db/mod.rs +++ b/crates/libanalysis/src/module_map_db/mod.rs @@ -4,15 +4,16 @@ use std::sync::Arc; use { FileId, db::{ - BoxedQuery, Query, QueryCtx + Query, QueryRegistry, QueryCtx, + file_syntax, file_set }, module_map::resolve_submodule, }; -pub(crate) fn queries(acc: &mut Vec) { - acc.push(MODULE_DESCR.into()); - acc.push(RESOLVE_SUBMODULE.into()); - acc.push(PARENT_MODULE.into()); +pub(crate) fn register_queries(reg: &mut QueryRegistry) { + reg.add(MODULE_DESCR, "MODULE_DESCR"); + reg.add(RESOLVE_SUBMODULE, "RESOLVE_SUBMODULE"); + reg.add(PARENT_MODULE, "PARENT_MODULE"); } impl<'a> QueryCtx<'a> { @@ -24,41 +25,32 @@ impl<'a> QueryCtx<'a> { } } -pub(crate) const MODULE_DESCR: Query = Query { - id: 30, - f: |ctx, &file_id| { - let file = ctx.file_syntax(file_id); - descr::ModuleDescr::new(file.ast()) - } -}; +const MODULE_DESCR: Query = Query(30, |ctx, &file_id| { + let file = file_syntax(ctx, file_id); + descr::ModuleDescr::new(file.ast()) +}); -pub(crate) const RESOLVE_SUBMODULE: Query<(FileId, descr::Submodule), Vec> = Query { - id: 31, - f: |ctx, params| { - let files = ctx.file_set(); - resolve_submodule(params.0, ¶ms.1.name, &files.1).0 - } -}; +const RESOLVE_SUBMODULE: Query<(FileId, descr::Submodule), Vec> = Query(31, |ctx, params| { + let files = file_set(ctx); + resolve_submodule(params.0, ¶ms.1.name, &files.1).0 +}); -pub(crate) const PARENT_MODULE: Query> = Query { - id: 40, - f: |ctx, file_id| { - let files = ctx.file_set(); - let res = files.0.iter() - .map(|&parent_id| (parent_id, ctx.module_descr(parent_id))) - .filter(|(parent_id, descr)| { - descr.submodules.iter() - .any(|subm| { - ctx.resolve_submodule(*parent_id, subm.clone()) - .iter() - .any(|it| it == file_id) - }) - }) - .map(|(id, _)| id) - .collect(); - res - } -}; +const PARENT_MODULE: Query> = Query(40, |ctx, file_id| { + let files = file_set(ctx); + let res = files.0.iter() + .map(|&parent_id| (parent_id, ctx.module_descr(parent_id))) + .filter(|(parent_id, descr)| { + descr.submodules.iter() + .any(|subm| { + ctx.resolve_submodule(*parent_id, subm.clone()) + .iter() + .any(|it| it == file_id) + }) + }) + .map(|(id, _)| id) + .collect(); + res +}); #[cfg(test)] mod tests { @@ -107,34 +99,36 @@ mod tests { self.next_file_id += 1; self.fm.insert(file_id, RelativePathBuf::from(&path[1..])); let mut new_state = self.db.state().clone(); - new_state.file_map.insert(file_id, text.to_string().into_boxed_str().into()); - new_state.resolver = FileResolverImp::new( + new_state.file_map.insert(file_id, Arc::new(text.to_string())); + new_state.file_resolver = FileResolverImp::new( Arc::new(FileMap(self.fm.clone())) ); - self.db = self.db.with_state(new_state, &[file_id], true); + self.db = self.db.with_changes(new_state, &[file_id], true); file_id } fn remove_file(&mut self, file_id: FileId) { self.fm.remove(&file_id); let mut new_state = self.db.state().clone(); new_state.file_map.remove(&file_id); - new_state.resolver = FileResolverImp::new( + new_state.file_resolver = FileResolverImp::new( Arc::new(FileMap(self.fm.clone())) ); - self.db = self.db.with_state(new_state, &[file_id], true); + self.db = self.db.with_changes(new_state, &[file_id], true); } fn change_file(&mut self, file_id: FileId, new_text: &str) { let mut new_state = self.db.state().clone(); - new_state.file_map.insert(file_id, new_text.to_string().into_boxed_str().into()); - self.db = self.db.with_state(new_state, &[file_id], false); + new_state.file_map.insert(file_id, Arc::new(new_text.to_string())); + self.db = self.db.with_changes(new_state, &[file_id], false); } fn check_parent_modules( &self, file_id: FileId, expected: &[FileId], - queries: &[(u16, u64)] + queries: &[(&'static str, u64)] ) { - let (actual, events) = self.db.get(PARENT_MODULE, file_id); + let (actual, events) = self.db.trace_query(|ctx| { + ctx.get(PARENT_MODULE, file_id) + }); assert_eq!(actual.as_slice(), expected); let mut counts = HashMap::new(); events.into_iter() @@ -156,25 +150,25 @@ mod tests { fn test_parent_module() { let mut f = Fixture::new(); let foo = f.add_file("/foo.rs", ""); - f.check_parent_modules(foo, &[], &[(MODULE_DESCR.id, 1)]); + f.check_parent_modules(foo, &[], &[("MODULE_DESCR", 1)]); let lib = f.add_file("/lib.rs", "mod foo;"); - f.check_parent_modules(foo, &[lib], &[(MODULE_DESCR.id, 1)]); - f.check_parent_modules(foo, &[lib], &[(MODULE_DESCR.id, 0)]); + f.check_parent_modules(foo, &[lib], &[("MODULE_DESCR", 1)]); + f.check_parent_modules(foo, &[lib], &[("MODULE_DESCR", 0)]); f.change_file(lib, ""); - f.check_parent_modules(foo, &[], &[(MODULE_DESCR.id, 1)]); + f.check_parent_modules(foo, &[], &[("MODULE_DESCR", 1)]); f.change_file(lib, "mod foo;"); - f.check_parent_modules(foo, &[lib], &[(MODULE_DESCR.id, 1)]); + f.check_parent_modules(foo, &[lib], &[("MODULE_DESCR", 1)]); f.change_file(lib, "mod bar;"); - f.check_parent_modules(foo, &[], &[(MODULE_DESCR.id, 1)]); + f.check_parent_modules(foo, &[], &[("MODULE_DESCR", 1)]); f.change_file(lib, "mod foo;"); - f.check_parent_modules(foo, &[lib], &[(MODULE_DESCR.id, 1)]); + f.check_parent_modules(foo, &[lib], &[("MODULE_DESCR", 1)]); f.remove_file(lib); - f.check_parent_modules(foo, &[], &[(MODULE_DESCR.id, 0)]); + f.check_parent_modules(foo, &[], &[("MODULE_DESCR", 0)]); } } diff --git a/crates/salsa/src/lib.rs b/crates/salsa/src/lib.rs index 75815e8bd2d4..35deed37417c 100644 --- a/crates/salsa/src/lib.rs +++ b/crates/salsa/src/lib.rs @@ -8,8 +8,8 @@ use std::{ }; use parking_lot::Mutex; -type GroundQueryFn = Box (D, OutputFingerprint) + Send + Sync + 'static>; -type QueryFn = Box, &D) -> (D, OutputFingerprint) + Send + Sync + 'static>; +pub type GroundQueryFn = Box (D, OutputFingerprint) + Send + Sync + 'static>; +pub type QueryFn = Box, &D) -> (D, OutputFingerprint) + Send + Sync + 'static>; #[derive(Debug)] pub struct Db { @@ -118,6 +118,9 @@ where self.record_dep(query_id, output_fingerprint); res } + pub fn trace(&self) -> Vec { + ::std::mem::replace(&mut *self.executed.borrow_mut(), Vec::new()) + } fn get_inner( &self, @@ -261,12 +264,15 @@ where query_config: Arc::clone(&self.query_config) } } + pub fn query_ctx(&self) -> QueryCtx { + QueryCtx::new(self) + } pub fn get( &self, query_id: QueryId, params: D, ) -> (D, Vec) { - let ctx = QueryCtx::new(self); + let ctx = self.query_ctx(); let res = ctx.get(query_id, params.into()); let executed = ::std::mem::replace(&mut *ctx.executed.borrow_mut(), Vec::new()); (res, executed) From 58674dc3c415142dbdd93b990d9f5b4fe10bef69 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 15 Sep 2018 20:29:22 +0300 Subject: [PATCH 15/18] ModuleTreeDescriptor --- crates/libanalysis/src/db/mod.rs | 12 +- crates/libanalysis/src/descriptors.rs | 217 ++++++++++++++++++ crates/libanalysis/src/imp.rs | 124 +++++----- crates/libanalysis/src/lib.rs | 1 + crates/libanalysis/src/module_map_db/descr.rs | 29 --- crates/libanalysis/src/module_map_db/mod.rs | 47 ++-- crates/libanalysis/src/roots.rs | 12 +- crates/libeditor/src/line_index.rs | 2 +- 8 files changed, 306 insertions(+), 138 deletions(-) create mode 100644 crates/libanalysis/src/descriptors.rs delete mode 100644 crates/libanalysis/src/module_map_db/descr.rs diff --git a/crates/libanalysis/src/db/mod.rs b/crates/libanalysis/src/db/mod.rs index a775b5f75a3d..1111a4f87469 100644 --- a/crates/libanalysis/src/db/mod.rs +++ b/crates/libanalysis/src/db/mod.rs @@ -68,22 +68,32 @@ pub(crate) fn file_set(ctx: QueryCtx) -> Arc<(Vec, FileResolverImp)> { pub(crate) use self::queries::file_syntax; mod queries { + use std::sync::Arc; use libsyntax2::File; + use libeditor::LineIndex; use {FileId}; use super::{Query, QueryCtx, QueryRegistry, file_text}; pub(crate) fn register_queries(reg: &mut QueryRegistry) { - reg.add(FILE_SYNTAX, "FILE_SYNTAX") + reg.add(FILE_SYNTAX, "FILE_SYNTAX"); + reg.add(FILE_LINES, "FILE_LINES"); } pub(crate) fn file_syntax(ctx: QueryCtx, file_id: FileId) -> File { (&*ctx.get(FILE_SYNTAX, file_id)).clone() } + pub(crate) fn file_lines(ctx: QueryCtx, file_id: FileId) -> Arc { + ctx.get(FILE_LINES, file_id) + } pub(super) const FILE_SYNTAX: Query = Query(16, |ctx, file_id: &FileId| { let text = file_text(ctx, *file_id); File::parse(&*text) }); + pub(super) const FILE_LINES: Query = Query(17, |ctx, file_id: &FileId| { + let text = file_text(ctx, *file_id); + LineIndex::new(&*text) + }); } impl QueryRegistry { diff --git a/crates/libanalysis/src/descriptors.rs b/crates/libanalysis/src/descriptors.rs new file mode 100644 index 000000000000..e21ee728ff2c --- /dev/null +++ b/crates/libanalysis/src/descriptors.rs @@ -0,0 +1,217 @@ +use std::{ + collections::BTreeMap, +}; +use relative_path::RelativePathBuf; +use libsyntax2::{ + SmolStr, + ast::{self, NameOwner}, +}; +use { + FileId, + imp::FileResolverImp, +}; + +#[derive(Debug, Hash)] +pub struct ModuleDescriptor { + pub submodules: Vec +} + +impl ModuleDescriptor { + pub fn new(root: ast::Root) -> ModuleDescriptor { + let submodules = modules(root) + .map(|(name, _)| Submodule { name }) + .collect(); + + ModuleDescriptor { submodules } } +} + +fn modules<'a>(root: ast::Root<'a>) -> impl Iterator)> { + root + .modules() + .filter_map(|module| { + let name = module.name()?.text(); + if !module.has_semi() { + return None; + } + Some((name, module)) + }) +} + +#[derive(Clone, Hash, PartialEq, Eq, Debug)] +pub struct Submodule { + pub name: SmolStr, +} + +#[derive(Hash)] +pub(crate) struct ModuleTreeDescriptor { + nodes: Vec, + links: Vec, + file_id2node: BTreeMap, +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +struct Node(usize); +#[derive(Hash)] +struct NodeData { + file_id: FileId, + links: Vec, + parents: Vec +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct Link(usize); +#[derive(Hash)] +struct LinkData { + owner: Node, + name: SmolStr, + points_to: Vec, + problem: Option, +} + + +#[derive(Clone, Debug, Hash)] +pub enum Problem { + UnresolvedModule { + candidate: RelativePathBuf, + }, + NotDirOwner { + move_to: RelativePathBuf, + candidate: RelativePathBuf, + } +} + +impl ModuleTreeDescriptor { + pub(crate) fn new<'a>( + files: impl Iterator + Clone, + file_resolver: &FileResolverImp, + ) -> ModuleTreeDescriptor { + let mut file_id2node = BTreeMap::new(); + let mut nodes: Vec = files.clone().enumerate() + .map(|(idx, (file_id, _))| { + file_id2node.insert(file_id, Node(idx)); + NodeData { + file_id, + links: Vec::new(), + parents: Vec::new(), + } + }) + .collect(); + let mut links = Vec::new(); + + for (idx, (file_id, descr)) in files.enumerate() { + let owner = Node(idx); + for sub in descr.submodules.iter() { + let link = Link(links.len()); + nodes[owner.0].links.push(link); + let (points_to, problem) = resolve_submodule(file_id, &sub.name, file_resolver); + let points_to = points_to + .into_iter() + .map(|file_id| { + let node = file_id2node[&file_id]; + nodes[node.0].parents.push(link); + node + }) + .collect(); + + links.push(LinkData { + owner, + name: sub.name.clone(), + points_to, + problem, + }) + + } + } + + ModuleTreeDescriptor { + nodes, links, file_id2node + } + } + + pub(crate) fn parent_modules(&self, file_id: FileId) -> Vec { + let node = self.file_id2node[&file_id]; + self.node(node) + .parents + .clone() + } + pub(crate) fn child_module_by_name(&self, file_id: FileId, name: &str) -> Vec { + let node = self.file_id2node[&file_id]; + self.node(node) + .links + .iter() + .filter(|it| it.name(self) == name) + .map(|link| link.owner(self)) + .collect() + } + pub(crate) fn problems<'a, 'b>(&'b self, file_id: FileId, root: ast::Root<'a>) -> Vec<(ast::Name<'a>, &'b Problem)> { + let node = self.file_id2node[&file_id]; + self.node(node) + .links + .iter() + .filter_map(|&link| { + let problem = self.link(link).problem.as_ref()?; + let name = link.bind_source(self, root).name()?; + Some((name, problem)) + }) + .collect() + } + + fn node(&self, node: Node) -> &NodeData { + &self.nodes[node.0] + } + fn link(&self, link: Link) -> &LinkData { + &self.links[link.0] + } +} + +impl Link { + pub(crate) fn name(self, tree: &ModuleTreeDescriptor) -> SmolStr { + tree.link(self).name.clone() + } + pub(crate) fn owner(self, tree: &ModuleTreeDescriptor) -> FileId { + let owner = tree.link(self).owner; + tree.node(owner).file_id + } + pub(crate) fn bind_source<'a>(self, tree: &ModuleTreeDescriptor, root: ast::Root<'a>) -> ast::Module<'a> { + modules(root) + .filter(|(name, _)| name == &tree.link(self).name) + .next() + .unwrap() + .1 + } +} + + +fn resolve_submodule( + file_id: FileId, + name: &SmolStr, + file_resolver: &FileResolverImp +) -> (Vec, Option) { + let mod_name = file_resolver.file_stem(file_id); + let is_dir_owner = + mod_name == "mod" || mod_name == "lib" || mod_name == "main"; + + let file_mod = RelativePathBuf::from(format!("../{}.rs", name)); + let dir_mod = RelativePathBuf::from(format!("../{}/mod.rs", name)); + let points_to: Vec; + let problem: Option; + if is_dir_owner { + points_to = [&file_mod, &dir_mod].iter() + .filter_map(|path| file_resolver.resolve(file_id, path)) + .collect(); + problem = if points_to.is_empty() { + Some(Problem::UnresolvedModule { + candidate: file_mod, + }) + } else { + None + } + } else { + points_to = Vec::new(); + problem = Some(Problem::NotDirOwner { + move_to: RelativePathBuf::from(format!("../{}/mod.rs", mod_name)), + candidate: file_mod, + }); + } + (points_to, problem) +} diff --git a/crates/libanalysis/src/imp.rs b/crates/libanalysis/src/imp.rs index 3e65ee14ab74..8734813f49ce 100644 --- a/crates/libanalysis/src/imp.rs +++ b/crates/libanalysis/src/imp.rs @@ -18,8 +18,8 @@ use libsyntax2::{ use { FileId, FileResolver, Query, Diagnostic, SourceChange, SourceFileEdit, Position, FileSystemEdit, JobToken, CrateGraph, CrateId, - module_map::{ModuleMap, Problem}, roots::{SourceRoot, ReadonlySourceRoot, WritableSourceRoot}, + descriptors::{ModuleTreeDescriptor, Problem}, }; @@ -148,25 +148,24 @@ impl AnalysisImpl { } pub fn parent_module(&self, file_id: FileId) -> Vec<(FileId, FileSymbol)> { let root = self.root(file_id); - let module_map = root.module_map(); - let id = module_map.file2module(file_id); - module_map - .parent_modules(id, &|file_id| root.syntax(file_id)) - .into_iter() - .map(|(id, name, node)| { - let id = module_map.module2file(id); + let module_tree = root.module_tree(); + module_tree.parent_modules(file_id) + .iter() + .map(|link| { + let file_id = link.owner(&module_tree); + let syntax = root.syntax(file_id); + let decl = link.bind_source(&module_tree, syntax.ast()); let sym = FileSymbol { - name, - node_range: node.range(), + name: link.name(&module_tree), + node_range: decl.syntax().range(), kind: MODULE, }; - (id, sym) + (file_id, sym) }) .collect() } - pub fn crate_for(&self, file_id: FileId) -> Vec { - let module_map = self.root(file_id).module_map(); + let module_tree = self.root(file_id).module_tree(); let crate_graph = &self.data.crate_graph; let mut res = Vec::new(); let mut work = VecDeque::new(); @@ -177,11 +176,10 @@ impl AnalysisImpl { res.push(crate_id); continue; } - let mid = module_map.file2module(id); - let parents = module_map - .parent_module_ids(mid, &|file_id| self.file_syntax(file_id)) + let parents = module_tree + .parent_modules(id) .into_iter() - .map(|id| module_map.module2file(id)) + .map(|link| link.owner(&module_tree)) .filter(|&id| visited.insert(id)); work.extend(parents); } @@ -197,7 +195,7 @@ impl AnalysisImpl { token: &JobToken, ) -> Vec<(FileId, FileSymbol)> { let root = self.root(file_id); - let module_map = root.module_map(); + let module_tree = root.module_tree(); let file = root.syntax(file_id); let syntax = file.syntax(); if let Some(name_ref) = find_node_at_offset::(syntax, offset) { @@ -206,7 +204,7 @@ impl AnalysisImpl { if let Some(name) = find_node_at_offset::(syntax, offset) { if let Some(module) = name.syntax().parent().and_then(ast::Module::cast) { if module.has_semi() { - let file_ids = self.resolve_module(module_map, file_id, module); + let file_ids = self.resolve_module(&*module_tree, file_id, module); let res = file_ids.into_iter().map(|id| { let name = module.name() @@ -229,7 +227,7 @@ impl AnalysisImpl { pub fn diagnostics(&self, file_id: FileId) -> Vec { let root = self.root(file_id); - let module_map = root.module_map(); + let module_tree = root.module_tree(); let syntax = root.syntax(file_id); let mut res = libeditor::diagnostics(&syntax) @@ -237,47 +235,43 @@ impl AnalysisImpl { .map(|d| Diagnostic { range: d.range, message: d.msg, fix: None }) .collect::>(); - module_map.problems( - file_id, - &|file_id| self.file_syntax(file_id), - |name_node, problem| { - let diag = match problem { - Problem::UnresolvedModule { candidate } => { - let create_file = FileSystemEdit::CreateFile { - anchor: file_id, - path: candidate.clone(), - }; - let fix = SourceChange { - label: "create module".to_string(), - source_file_edits: Vec::new(), - file_system_edits: vec![create_file], - cursor_position: None, - }; - Diagnostic { - range: name_node.syntax().range(), - message: "unresolved module".to_string(), - fix: Some(fix), - } + for (name_node, problem) in module_tree.problems(file_id, syntax.ast()) { + let diag = match problem { + Problem::UnresolvedModule { candidate } => { + let create_file = FileSystemEdit::CreateFile { + anchor: file_id, + path: candidate.clone(), + }; + let fix = SourceChange { + label: "create module".to_string(), + source_file_edits: Vec::new(), + file_system_edits: vec![create_file], + cursor_position: None, + }; + Diagnostic { + range: name_node.syntax().range(), + message: "unresolved module".to_string(), + fix: Some(fix), } - Problem::NotDirOwner { move_to, candidate } => { - let move_file = FileSystemEdit::MoveFile { file: file_id, path: move_to.clone() }; - let create_file = FileSystemEdit::CreateFile { anchor: file_id, path: move_to.join(candidate) }; - let fix = SourceChange { - label: "move file and create module".to_string(), - source_file_edits: Vec::new(), - file_system_edits: vec![move_file, create_file], - cursor_position: None, - }; - Diagnostic { - range: name_node.syntax().range(), - message: "can't declare module at this location".to_string(), - fix: Some(fix), - } + } + Problem::NotDirOwner { move_to, candidate } => { + let move_file = FileSystemEdit::MoveFile { file: file_id, path: move_to.clone() }; + let create_file = FileSystemEdit::CreateFile { anchor: file_id, path: move_to.join(candidate) }; + let fix = SourceChange { + label: "move file and create module".to_string(), + source_file_edits: Vec::new(), + file_system_edits: vec![move_file, create_file], + cursor_position: None, + }; + Diagnostic { + range: name_node.syntax().range(), + message: "can't declare module at this location".to_string(), + fix: Some(fix), } - }; - res.push(diag) - } - ); + } + }; + res.push(diag) + } res } @@ -307,20 +301,12 @@ impl AnalysisImpl { self.world_symbols(query, token) } - fn resolve_module(&self, module_map: &ModuleMap, file_id: FileId, module: ast::Module) -> Vec { + fn resolve_module(&self, module_tree: &ModuleTreeDescriptor, file_id: FileId, module: ast::Module) -> Vec { let name = match module.name() { Some(name) => name.text(), None => return Vec::new(), }; - let id = module_map.file2module(file_id); - module_map - .child_module_by_name( - id, name.as_str(), - &|file_id| self.file_syntax(file_id), - ) - .into_iter() - .map(|id| module_map.module2file(id)) - .collect() + module_tree.child_module_by_name(file_id, name.as_str()) } fn reindex(&self) { diff --git a/crates/libanalysis/src/lib.rs b/crates/libanalysis/src/lib.rs index bca5d4c1d678..ea3a28702237 100644 --- a/crates/libanalysis/src/lib.rs +++ b/crates/libanalysis/src/lib.rs @@ -19,6 +19,7 @@ mod imp; mod job; mod roots; mod db; +mod descriptors; use std::{ sync::Arc, diff --git a/crates/libanalysis/src/module_map_db/descr.rs b/crates/libanalysis/src/module_map_db/descr.rs deleted file mode 100644 index fb298a315d8e..000000000000 --- a/crates/libanalysis/src/module_map_db/descr.rs +++ /dev/null @@ -1,29 +0,0 @@ -use libsyntax2::{ - SmolStr, - ast::{self, NameOwner}, -}; - -#[derive(Debug, Hash)] -pub struct ModuleDescr { - pub submodules: Vec -} - -impl ModuleDescr { - pub fn new(root: ast::Root) -> ModuleDescr { - let submodules = root - .modules() - .filter_map(|module| { - let name = module.name()?.text(); - if !module.has_semi() { - return None; - } - Some(Submodule { name }) - }).collect(); - - ModuleDescr { submodules } } -} - -#[derive(Clone, Hash, PartialEq, Eq, Debug)] -pub struct Submodule { - pub name: SmolStr, -} diff --git a/crates/libanalysis/src/module_map_db/mod.rs b/crates/libanalysis/src/module_map_db/mod.rs index 5560e4a34a65..adad943da50a 100644 --- a/crates/libanalysis/src/module_map_db/mod.rs +++ b/crates/libanalysis/src/module_map_db/mod.rs @@ -1,5 +1,3 @@ -mod descr; - use std::sync::Arc; use { FileId, @@ -7,49 +5,36 @@ use { Query, QueryRegistry, QueryCtx, file_syntax, file_set }, - module_map::resolve_submodule, + descriptors::{ModuleDescriptor, ModuleTreeDescriptor} }; pub(crate) fn register_queries(reg: &mut QueryRegistry) { reg.add(MODULE_DESCR, "MODULE_DESCR"); - reg.add(RESOLVE_SUBMODULE, "RESOLVE_SUBMODULE"); - reg.add(PARENT_MODULE, "PARENT_MODULE"); +} + +pub(crate) fn module_tree(ctx: QueryCtx) -> Arc { + ctx.get(MODULE_TREE, ()) } impl<'a> QueryCtx<'a> { - fn module_descr(&self, file_id: FileId) -> Arc { + fn module_descr(&self, file_id: FileId) -> Arc { self.get(MODULE_DESCR, file_id) } - fn resolve_submodule(&self, file_id: FileId, submod: descr::Submodule) -> Arc> { - self.get(RESOLVE_SUBMODULE, (file_id, submod)) - } } -const MODULE_DESCR: Query = Query(30, |ctx, &file_id| { +const MODULE_DESCR: Query = Query(30, |ctx, &file_id| { let file = file_syntax(ctx, file_id); - descr::ModuleDescr::new(file.ast()) + ModuleDescriptor::new(file.ast()) }); -const RESOLVE_SUBMODULE: Query<(FileId, descr::Submodule), Vec> = Query(31, |ctx, params| { - let files = file_set(ctx); - resolve_submodule(params.0, ¶ms.1.name, &files.1).0 -}); - -const PARENT_MODULE: Query> = Query(40, |ctx, file_id| { - let files = file_set(ctx); - let res = files.0.iter() - .map(|&parent_id| (parent_id, ctx.module_descr(parent_id))) - .filter(|(parent_id, descr)| { - descr.submodules.iter() - .any(|subm| { - ctx.resolve_submodule(*parent_id, subm.clone()) - .iter() - .any(|it| it == file_id) - }) - }) - .map(|(id, _)| id) - .collect(); - res +const MODULE_TREE: Query<(), ModuleTreeDescriptor> = Query(31, |ctx, _| { + let file_set = file_set(ctx); + let mut files = Vec::new(); + for &file_id in file_set.0.iter() { + let module_descr = ctx.get(MODULE_DESCR, file_id); + files.push((file_id, module_descr)); + } + ModuleTreeDescriptor::new(files.iter().map(|(file_id, descr)| (*file_id, &**descr)), &file_set.1) }); #[cfg(test)] diff --git a/crates/libanalysis/src/roots.rs b/crates/libanalysis/src/roots.rs index 629a697c5adb..b925fd24d53a 100644 --- a/crates/libanalysis/src/roots.rs +++ b/crates/libanalysis/src/roots.rs @@ -15,11 +15,15 @@ use { imp::FileResolverImp, module_map::{ModuleMap, ChangeKind}, symbol_index::SymbolIndex, + descriptors::ModuleTreeDescriptor, }; pub(crate) trait SourceRoot { fn contains(&self, file_id: FileId) -> bool; - fn module_map(&self) -> &ModuleMap; + fn module_tree(&self) -> Arc { + unimplemented!() + } + // fn module_map(&self) -> &ModuleMap; fn lines(&self, file_id: FileId) -> &LineIndex; fn syntax(&self, file_id: FileId) -> &File; fn symbols<'a>(&'a self, acc: &mut Vec<&'a SymbolIndex>); @@ -74,9 +78,6 @@ impl SourceRoot for WritableSourceRoot { fn contains(&self, file_id: FileId) -> bool { self.file_map.contains_key(&file_id) } - fn module_map(&self) -> &ModuleMap { - &self.module_map - } fn lines(&self, file_id: FileId) -> &LineIndex { self.data(file_id).lines() } @@ -175,9 +176,6 @@ impl SourceRoot for ReadonlySourceRoot { fn contains(&self, file_id: FileId) -> bool { self.file_map.contains_key(&file_id) } - fn module_map(&self) -> &ModuleMap { - &self.module_map - } fn lines(&self, file_id: FileId) -> &LineIndex { self.data(file_id).lines() } diff --git a/crates/libeditor/src/line_index.rs b/crates/libeditor/src/line_index.rs index 801726aa59b9..9cd8da3a893f 100644 --- a/crates/libeditor/src/line_index.rs +++ b/crates/libeditor/src/line_index.rs @@ -1,7 +1,7 @@ use superslice::Ext; use ::TextUnit; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Hash)] pub struct LineIndex { newlines: Vec, } From 3ebeb0db8de65434db79c86aeed5473bfb21334f Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 15 Sep 2018 20:56:19 +0300 Subject: [PATCH 16/18] move readonly source to module tree descr --- crates/libanalysis/src/descriptors.rs | 10 ++++----- crates/libanalysis/src/roots.rs | 32 +++++++++++++++++---------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/crates/libanalysis/src/descriptors.rs b/crates/libanalysis/src/descriptors.rs index e21ee728ff2c..200f21576b75 100644 --- a/crates/libanalysis/src/descriptors.rs +++ b/crates/libanalysis/src/descriptors.rs @@ -42,25 +42,25 @@ pub struct Submodule { pub name: SmolStr, } -#[derive(Hash)] +#[derive(Hash, Debug)] pub(crate) struct ModuleTreeDescriptor { nodes: Vec, links: Vec, file_id2node: BTreeMap, } -#[derive(Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] struct Node(usize); -#[derive(Hash)] +#[derive(Hash, Debug)] struct NodeData { file_id: FileId, links: Vec, parents: Vec } -#[derive(Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub(crate) struct Link(usize); -#[derive(Hash)] +#[derive(Hash, Debug)] struct LinkData { owner: Node, name: SmolStr, diff --git a/crates/libanalysis/src/roots.rs b/crates/libanalysis/src/roots.rs index b925fd24d53a..e9527eeff443 100644 --- a/crates/libanalysis/src/roots.rs +++ b/crates/libanalysis/src/roots.rs @@ -15,7 +15,7 @@ use { imp::FileResolverImp, module_map::{ModuleMap, ChangeKind}, symbol_index::SymbolIndex, - descriptors::ModuleTreeDescriptor, + descriptors::{ModuleDescriptor, ModuleTreeDescriptor}, }; pub(crate) trait SourceRoot { @@ -137,30 +137,35 @@ impl FileData { pub(crate) struct ReadonlySourceRoot { symbol_index: SymbolIndex, file_map: HashMap, - module_map: ModuleMap, + module_tree: Arc, } impl ReadonlySourceRoot { pub(crate) fn new(files: Vec<(FileId, String)>, file_resolver: FileResolverImp) -> ReadonlySourceRoot { - let mut module_map = ModuleMap::new(); - module_map.set_file_resolver(file_resolver); - let symbol_index = SymbolIndex::for_files( - files.par_iter().map(|(file_id, text)| { - (*file_id, File::parse(text)) + let modules = files.par_iter() + .map(|(file_id, text)| { + let syntax = File::parse(text); + let mod_descr = ModuleDescriptor::new(syntax.ast()); + (*file_id, syntax, mod_descr) }) + .collect::>(); + let module_tree = ModuleTreeDescriptor::new( + modules.iter().map(|it| (it.0, &it.2)), + &file_resolver, + ); + + let symbol_index = SymbolIndex::for_files( + modules.par_iter().map(|it| (it.0, it.1.clone())) ); let file_map: HashMap = files .into_iter() - .map(|(id, text)| { - module_map.update_file(id, ChangeKind::Insert); - (id, FileData::new(text)) - }) + .map(|(id, text)| (id, FileData::new(text))) .collect(); ReadonlySourceRoot { symbol_index, file_map, - module_map, + module_tree: Arc::new(module_tree), } } @@ -173,6 +178,9 @@ impl ReadonlySourceRoot { } impl SourceRoot for ReadonlySourceRoot { + fn module_tree(&self) -> Arc { + Arc::clone(&self.module_tree) + } fn contains(&self, file_id: FileId) -> bool { self.file_map.contains_key(&file_id) } From e69ff21207d83864e13f6c8631733f4f0c32ba0d Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 15 Sep 2018 23:19:41 +0300 Subject: [PATCH 17/18] kill old module_map --- crates/libanalysis/src/db/imp.rs | 3 - crates/libanalysis/src/db/mod.rs | 42 +- crates/libanalysis/src/descriptors.rs | 5 +- crates/libanalysis/src/imp.rs | 22 +- crates/libanalysis/src/lib.rs | 20 +- crates/libanalysis/src/module_map.rs | 408 +++++++------------- crates/libanalysis/src/module_map_db/mod.rs | 159 -------- crates/libanalysis/src/queries.rs | 28 ++ crates/libanalysis/src/roots.rs | 116 +++--- 9 files changed, 251 insertions(+), 552 deletions(-) delete mode 100644 crates/libanalysis/src/module_map_db/mod.rs create mode 100644 crates/libanalysis/src/queries.rs diff --git a/crates/libanalysis/src/db/imp.rs b/crates/libanalysis/src/db/imp.rs index 1b4ee5cf31d2..f26be104638a 100644 --- a/crates/libanalysis/src/db/imp.rs +++ b/crates/libanalysis/src/db/imp.rs @@ -122,9 +122,6 @@ impl QueryRegistry { let config = config.with_query(id, q.f()); self.config= Some(config); } - pub(super) fn finish(mut self) -> salsa::QueryConfig { - self.config.take().unwrap() - } } fn hash(x: &T) -> u64 { diff --git a/crates/libanalysis/src/db/mod.rs b/crates/libanalysis/src/db/mod.rs index 1111a4f87469..22769d1120e5 100644 --- a/crates/libanalysis/src/db/mod.rs +++ b/crates/libanalysis/src/db/mod.rs @@ -29,6 +29,12 @@ pub(crate) struct QueryRegistry { imp: imp::QueryRegistry, } +impl Default for Db { + fn default() -> Db { + Db::new() + } +} + impl Db { pub(crate) fn new() -> Db { let reg = QueryRegistry::new(); @@ -44,6 +50,7 @@ impl Db { let ctx = QueryCtx { imp: &self.imp.imp.query_ctx() }; f(ctx) } + #[allow(unused)] pub(crate) fn trace_query R, R>(&self, f: F) -> (R, Vec<&'static str>) { let ctx = QueryCtx { imp: &self.imp.imp.query_ctx() }; let res = f(ctx); @@ -65,42 +72,11 @@ pub(crate) fn file_text(ctx: QueryCtx, file_id: FileId) -> Arc { pub(crate) fn file_set(ctx: QueryCtx) -> Arc<(Vec, FileResolverImp)> { imp::file_set(ctx) } -pub(crate) use self::queries::file_syntax; - -mod queries { - use std::sync::Arc; - use libsyntax2::File; - use libeditor::LineIndex; - use {FileId}; - use super::{Query, QueryCtx, QueryRegistry, file_text}; - - pub(crate) fn register_queries(reg: &mut QueryRegistry) { - reg.add(FILE_SYNTAX, "FILE_SYNTAX"); - reg.add(FILE_LINES, "FILE_LINES"); - } - - pub(crate) fn file_syntax(ctx: QueryCtx, file_id: FileId) -> File { - (&*ctx.get(FILE_SYNTAX, file_id)).clone() - } - pub(crate) fn file_lines(ctx: QueryCtx, file_id: FileId) -> Arc { - ctx.get(FILE_LINES, file_id) - } - - pub(super) const FILE_SYNTAX: Query = Query(16, |ctx, file_id: &FileId| { - let text = file_text(ctx, *file_id); - File::parse(&*text) - }); - pub(super) const FILE_LINES: Query = Query(17, |ctx, file_id: &FileId| { - let text = file_text(ctx, *file_id); - LineIndex::new(&*text) - }); -} - impl QueryRegistry { fn new() -> QueryRegistry { let mut reg = QueryRegistry { imp: imp::QueryRegistry::new() }; - queries::register_queries(&mut reg); - ::module_map_db::register_queries(&mut reg); + ::queries::register_queries(&mut reg); + ::module_map::register_queries(&mut reg); reg } pub(crate) fn add(&mut self, q: Q, name: &'static str) { diff --git a/crates/libanalysis/src/descriptors.rs b/crates/libanalysis/src/descriptors.rs index 200f21576b75..93a4158e4412 100644 --- a/crates/libanalysis/src/descriptors.rs +++ b/crates/libanalysis/src/descriptors.rs @@ -140,7 +140,7 @@ impl ModuleTreeDescriptor { .links .iter() .filter(|it| it.name(self) == name) - .map(|link| link.owner(self)) + .flat_map(|link| link.points_to(self).iter().map(|&node| self.node(node).file_id)) .collect() } pub(crate) fn problems<'a, 'b>(&'b self, file_id: FileId, root: ast::Root<'a>) -> Vec<(ast::Name<'a>, &'b Problem)> { @@ -172,6 +172,9 @@ impl Link { let owner = tree.link(self).owner; tree.node(owner).file_id } + fn points_to(self, tree: &ModuleTreeDescriptor) -> &[Node] { + &tree.link(self).points_to + } pub(crate) fn bind_source<'a>(self, tree: &ModuleTreeDescriptor, root: ast::Root<'a>) -> ast::Module<'a> { modules(root) .filter(|(name, _)| name == &tree.link(self).name) diff --git a/crates/libanalysis/src/imp.rs b/crates/libanalysis/src/imp.rs index 8734813f49ce..6f3191fe7311 100644 --- a/crates/libanalysis/src/imp.rs +++ b/crates/libanalysis/src/imp.rs @@ -5,6 +5,7 @@ use std::{ }, fmt, collections::{HashSet, VecDeque}, + iter, }; use relative_path::RelativePath; @@ -75,14 +76,12 @@ impl AnalysisHostImpl { } pub fn change_files(&mut self, changes: &mut dyn Iterator)>) { let data = self.data_mut(); - for (file_id, text) in changes { - data.root.update(file_id, text); - } + data.root = Arc::new(data.root.apply_changes(changes, None)); } pub fn set_file_resolver(&mut self, resolver: FileResolverImp) { let data = self.data_mut(); data.file_resolver = resolver.clone(); - data.root.set_file_resolver(resolver); + data.root = Arc::new(data.root.apply_changes(&mut iter::empty(), Some(resolver))); } pub fn set_crate_graph(&mut self, graph: CrateGraph) { let mut visited = HashSet::new(); @@ -124,18 +123,17 @@ impl Clone for AnalysisImpl { impl AnalysisImpl { fn root(&self, file_id: FileId) -> &SourceRoot { if self.data.root.contains(file_id) { - return &self.data.root; + return &*self.data.root; } &**self.data.libs.iter().find(|it| it.contains(file_id)).unwrap() } - pub fn file_syntax(&self, file_id: FileId) -> &File { + pub fn file_syntax(&self, file_id: FileId) -> File { self.root(file_id).syntax(file_id) } - pub fn file_line_index(&self, file_id: FileId) -> &LineIndex { + pub fn file_line_index(&self, file_id: FileId) -> Arc { self.root(file_id).lines(file_id) } pub fn world_symbols(&self, query: Query, token: &JobToken) -> Vec<(FileId, FileSymbol)> { - self.reindex(); let mut buf = Vec::new(); if query.libs { self.data.libs.iter() @@ -308,19 +306,13 @@ impl AnalysisImpl { }; module_tree.child_module_by_name(file_id, name.as_str()) } - - fn reindex(&self) { - if self.needs_reindex.compare_and_swap(true, false, SeqCst) { - self.data.root.reindex(); - } - } } #[derive(Default, Clone, Debug)] struct WorldData { file_resolver: FileResolverImp, crate_graph: CrateGraph, - root: WritableSourceRoot, + root: Arc, libs: Vec>, } diff --git a/crates/libanalysis/src/lib.rs b/crates/libanalysis/src/lib.rs index ea3a28702237..b4b7a6a304e0 100644 --- a/crates/libanalysis/src/lib.rs +++ b/crates/libanalysis/src/lib.rs @@ -14,11 +14,11 @@ extern crate salsa; mod symbol_index; mod module_map; -pub(crate) mod module_map_db; mod imp; mod job; mod roots; mod db; +mod queries; mod descriptors; use std::{ @@ -166,8 +166,8 @@ impl Analysis { pub fn file_syntax(&self, file_id: FileId) -> File { self.imp.file_syntax(file_id).clone() } - pub fn file_line_index(&self, file_id: FileId) -> LineIndex { - self.imp.file_line_index(file_id).clone() + pub fn file_line_index(&self, file_id: FileId) -> Arc { + self.imp.file_line_index(file_id) } pub fn extend_selection(&self, file: &File, range: TextRange) -> TextRange { libeditor::extend_selection(file, range).unwrap_or(range) @@ -177,19 +177,19 @@ impl Analysis { } pub fn syntax_tree(&self, file_id: FileId) -> String { let file = self.imp.file_syntax(file_id); - libeditor::syntax_tree(file) + libeditor::syntax_tree(&file) } pub fn join_lines(&self, file_id: FileId, range: TextRange) -> SourceChange { let file = self.imp.file_syntax(file_id); - SourceChange::from_local_edit(file_id, "join lines", libeditor::join_lines(file, range)) + SourceChange::from_local_edit(file_id, "join lines", libeditor::join_lines(&file, range)) } pub fn on_eq_typed(&self, file_id: FileId, offset: TextUnit) -> Option { let file = self.imp.file_syntax(file_id); - Some(SourceChange::from_local_edit(file_id, "add semicolon", libeditor::on_eq_typed(file, offset)?)) + Some(SourceChange::from_local_edit(file_id, "add semicolon", libeditor::on_eq_typed(&file, offset)?)) } pub fn file_structure(&self, file_id: FileId) -> Vec { let file = self.imp.file_syntax(file_id); - libeditor::file_structure(file) + libeditor::file_structure(&file) } pub fn symbol_search(&self, query: Query, token: &JobToken) -> Vec<(FileId, FileSymbol)> { self.imp.world_symbols(query, token) @@ -208,15 +208,15 @@ impl Analysis { } pub fn runnables(&self, file_id: FileId) -> Vec { let file = self.imp.file_syntax(file_id); - libeditor::runnables(file) + libeditor::runnables(&file) } pub fn highlight(&self, file_id: FileId) -> Vec { let file = self.imp.file_syntax(file_id); - libeditor::highlight(file) + libeditor::highlight(&file) } pub fn completions(&self, file_id: FileId, offset: TextUnit) -> Option> { let file = self.imp.file_syntax(file_id); - libeditor::scope_completion(file, offset) + libeditor::scope_completion(&file, offset) } pub fn assists(&self, file_id: FileId, range: TextRange) -> Vec { self.imp.assists(file_id, range) diff --git a/crates/libanalysis/src/module_map.rs b/crates/libanalysis/src/module_map.rs index 79b88cac2ff2..a21f55fff1e6 100644 --- a/crates/libanalysis/src/module_map.rs +++ b/crates/libanalysis/src/module_map.rs @@ -1,281 +1,157 @@ -use relative_path::RelativePathBuf; -use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; -use libsyntax2::{ - File, - ast::{self, AstNode, NameOwner}, - SyntaxNode, SmolStr, -}; -use {FileId, imp::FileResolverImp}; - -type SyntaxProvider<'a> = dyn Fn(FileId) -> &'a File + 'a; - -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] -pub struct ModuleId(FileId); - -#[derive(Debug, Default)] -pub struct ModuleMap { - state: RwLock, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ChangeKind { - Delete, Insert, Update -} - -impl Clone for ModuleMap { - fn clone(&self) -> ModuleMap { - let state = self.state.read().clone(); - ModuleMap { state: RwLock::new(state) } - } -} - -#[derive(Clone, Debug, Default)] -struct State { - file_resolver: FileResolverImp, - changes: Vec<(FileId, ChangeKind)>, - links: Vec, -} - -#[derive(Clone, Debug)] -struct Link { - owner: ModuleId, - syntax: SyntaxNode, - points_to: Vec, - problem: Option, -} - -#[derive(Clone, Debug)] -pub enum Problem { - UnresolvedModule { - candidate: RelativePathBuf, +use std::sync::Arc; +use { + FileId, + db::{ + Query, QueryRegistry, QueryCtx, + file_set }, - NotDirOwner { - move_to: RelativePathBuf, - candidate: RelativePathBuf, - } + queries::file_syntax, + descriptors::{ModuleDescriptor, ModuleTreeDescriptor}, +}; + +pub(crate) fn register_queries(reg: &mut QueryRegistry) { + reg.add(MODULE_DESCR, "MODULE_DESCR"); + reg.add(MODULE_TREE, "MODULE_TREE"); } -impl ModuleMap { - pub fn new() -> ModuleMap { - Default::default() +pub(crate) fn module_tree(ctx: QueryCtx) -> Arc { + ctx.get(MODULE_TREE, ()) +} + +const MODULE_DESCR: Query = Query(30, |ctx, &file_id| { + let file = file_syntax(ctx, file_id); + ModuleDescriptor::new(file.ast()) +}); + +const MODULE_TREE: Query<(), ModuleTreeDescriptor> = Query(31, |ctx, _| { + let file_set = file_set(ctx); + let mut files = Vec::new(); + for &file_id in file_set.0.iter() { + let module_descr = ctx.get(MODULE_DESCR, file_id); + files.push((file_id, module_descr)); } - pub fn update_file(&mut self, file_id: FileId, change_kind: ChangeKind) { - self.state.get_mut().changes.push((file_id, change_kind)); - } - pub(crate) fn set_file_resolver(&mut self, file_resolver: FileResolverImp) { - self.state.get_mut().file_resolver = file_resolver; - } - pub fn module2file(&self, m: ModuleId) -> FileId { - m.0 - } - pub fn file2module(&self, file_id: FileId) -> ModuleId { - ModuleId(file_id) - } - pub fn child_module_by_name<'a>( - &self, - parent_mod: ModuleId, - child_mod: &str, - syntax_provider: &SyntaxProvider, - ) -> Vec { - self.links(syntax_provider) - .links - .iter() - .filter(|link| link.owner == parent_mod) - .filter(|link| link.name() == child_mod) - .filter_map(|it| it.points_to.first()) - .map(|&it| it) - .collect() + ModuleTreeDescriptor::new(files.iter().map(|(file_id, descr)| (*file_id, &**descr)), &file_set.1) +}); + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use im; + use relative_path::{RelativePath, RelativePathBuf}; + use { + db::{Db}, + imp::FileResolverImp, + FileId, FileResolver, + }; + use super::*; + + #[derive(Debug)] + struct FileMap(im::HashMap); + + impl FileResolver for FileMap { + fn file_stem(&self, file_id: FileId) -> String { + self.0[&file_id].file_stem().unwrap().to_string() + } + fn resolve(&self, file_id: FileId, rel: &RelativePath) -> Option { + let path = self.0[&file_id].join(rel).normalize(); + self.0.iter() + .filter_map(|&(id, ref p)| Some(id).filter(|_| p == &path)) + .next() + } } - pub fn parent_modules( - &self, - m: ModuleId, - syntax_provider: &SyntaxProvider, - ) -> Vec<(ModuleId, SmolStr, SyntaxNode)> { - let mut res = Vec::new(); - self.for_each_parent_link(m, syntax_provider, |link| { - res.push( - (link.owner, link.name().clone(), link.syntax.clone()) - ) - }); - res + struct Fixture { + next_file_id: u32, + fm: im::HashMap, + db: Db, } - pub fn parent_module_ids( - &self, - m: ModuleId, - syntax_provider: &SyntaxProvider, - ) -> Vec { - let mut res = Vec::new(); - self.for_each_parent_link(m, syntax_provider, |link| res.push(link.owner)); - res - } - - fn for_each_parent_link( - &self, - m: ModuleId, - syntax_provider: &SyntaxProvider, - f: impl FnMut(&Link) - ) { - self.links(syntax_provider) - .links - .iter() - .filter(move |link| link.points_to.iter().any(|&it| it == m)) - .for_each(f) - } - - pub fn problems( - &self, - file: FileId, - syntax_provider: &SyntaxProvider, - mut cb: impl FnMut(ast::Name, &Problem), - ) { - let module = self.file2module(file); - let links = self.links(syntax_provider); - links - .links - .iter() - .filter(|link| link.owner == module) - .filter_map(|link| { - let problem = link.problem.as_ref()?; - Some((link, problem)) - }) - .for_each(|(link, problem)| cb(link.name_node(), problem)) - } - - fn links( - &self, - syntax_provider: &SyntaxProvider, - ) -> RwLockReadGuard { - { - let guard = self.state.read(); - if guard.changes.is_empty() { - return guard; + impl Fixture { + fn new() -> Fixture { + Fixture { + next_file_id: 1, + fm: im::HashMap::new(), + db: Db::new(), } } - let mut guard = self.state.write(); - if !guard.changes.is_empty() { - guard.apply_changes(syntax_provider); + fn add_file(&mut self, path: &str, text: &str) -> FileId { + assert!(path.starts_with("/")); + let file_id = FileId(self.next_file_id); + self.next_file_id += 1; + self.fm.insert(file_id, RelativePathBuf::from(&path[1..])); + let mut new_state = self.db.state().clone(); + new_state.file_map.insert(file_id, Arc::new(text.to_string())); + new_state.file_resolver = FileResolverImp::new( + Arc::new(FileMap(self.fm.clone())) + ); + self.db = self.db.with_changes(new_state, &[file_id], true); + file_id } - assert!(guard.changes.is_empty()); - RwLockWriteGuard::downgrade(guard) - } -} - -impl State { - pub fn apply_changes( - &mut self, - syntax_provider: &SyntaxProvider, - ) { - let mut reresolve = false; - for (file_id, kind) in self.changes.drain(..) { - let mod_id = ModuleId(file_id); - self.links.retain(|link| link.owner != mod_id); - match kind { - ChangeKind::Delete => { - for link in self.links.iter_mut() { - link.points_to.retain(|&x| x != mod_id); - } - } - ChangeKind::Insert => { - let file = syntax_provider(file_id); - self.links.extend( - file - .ast() - .modules() - .filter_map(|it| Link::new(mod_id, it)) - ); - reresolve = true; - } - ChangeKind::Update => { - let file = syntax_provider(file_id); - let resolver = &self.file_resolver; - self.links.extend( - file - .ast() - .modules() - .filter_map(|it| Link::new(mod_id, it)) - .map(|mut link| { - link.resolve(resolver); - link - }) - ); - } - } - } - if reresolve { - for link in self.links.iter_mut() { - link.resolve(&self.file_resolver) + fn remove_file(&mut self, file_id: FileId) { + self.fm.remove(&file_id); + let mut new_state = self.db.state().clone(); + new_state.file_map.remove(&file_id); + new_state.file_resolver = FileResolverImp::new( + Arc::new(FileMap(self.fm.clone())) + ); + self.db = self.db.with_changes(new_state, &[file_id], true); + } + fn change_file(&mut self, file_id: FileId, new_text: &str) { + let mut new_state = self.db.state().clone(); + new_state.file_map.insert(file_id, Arc::new(new_text.to_string())); + self.db = self.db.with_changes(new_state, &[file_id], false); + } + fn check_parent_modules( + &self, + file_id: FileId, + expected: &[FileId], + queries: &[(&'static str, u64)] + ) { + let (tree, events) = self.db.trace_query(|ctx| module_tree(ctx)); + let actual = tree.parent_modules(file_id) + .into_iter() + .map(|link| link.owner(&tree)) + .collect::>(); + assert_eq!(actual.as_slice(), expected); + let mut counts = HashMap::new(); + events.into_iter() + .for_each(|event| *counts.entry(event).or_insert(0) += 1); + for &(query_id, expected_count) in queries.iter() { + let actual_count = *counts.get(&query_id).unwrap_or(&0); + assert_eq!( + actual_count, + expected_count, + "counts for {} differ", + query_id, + ) } + } } + + #[test] + fn test_parent_module() { + let mut f = Fixture::new(); + let foo = f.add_file("/foo.rs", ""); + f.check_parent_modules(foo, &[], &[("MODULE_DESCR", 1)]); + + let lib = f.add_file("/lib.rs", "mod foo;"); + f.check_parent_modules(foo, &[lib], &[("MODULE_DESCR", 1)]); + f.check_parent_modules(foo, &[lib], &[("MODULE_DESCR", 0)]); + + f.change_file(lib, ""); + f.check_parent_modules(foo, &[], &[("MODULE_DESCR", 1)]); + + f.change_file(lib, "mod foo;"); + f.check_parent_modules(foo, &[lib], &[("MODULE_DESCR", 1)]); + + f.change_file(lib, "mod bar;"); + f.check_parent_modules(foo, &[], &[("MODULE_DESCR", 1)]); + + f.change_file(lib, "mod foo;"); + f.check_parent_modules(foo, &[lib], &[("MODULE_DESCR", 1)]); + + f.remove_file(lib); + f.check_parent_modules(foo, &[], &[("MODULE_DESCR", 0)]); + } } - -impl Link { - fn new(owner: ModuleId, module: ast::Module) -> Option { - if module.name().is_none() { - return None; - } - let link = Link { - owner, - syntax: module.syntax().owned(), - points_to: Vec::new(), - problem: None, - }; - Some(link) - } - - fn name(&self) -> SmolStr { - self.name_node().text() - } - - fn name_node(&self) -> ast::Name { - self.ast().name().unwrap() - } - - fn ast(&self) -> ast::Module { - ast::Module::cast(self.syntax.borrowed()) - .unwrap() - } - - fn resolve(&mut self, file_resolver: &FileResolverImp) { - if !self.ast().has_semi() { - self.problem = None; - self.points_to = Vec::new(); - return; - } - let (points_to, problem) = resolve_submodule(self.owner.0, &self.name(), file_resolver); - self.problem = problem; - self.points_to = points_to.into_iter().map(ModuleId).collect(); - } -} - -pub(crate) fn resolve_submodule(file_id: FileId, name: &SmolStr, file_resolver: &FileResolverImp) -> (Vec, Option) { - let mod_name = file_resolver.file_stem(file_id); - let is_dir_owner = - mod_name == "mod" || mod_name == "lib" || mod_name == "main"; - - let file_mod = RelativePathBuf::from(format!("../{}.rs", name)); - let dir_mod = RelativePathBuf::from(format!("../{}/mod.rs", name)); - let points_to: Vec; - let problem: Option; - if is_dir_owner { - points_to = [&file_mod, &dir_mod].iter() - .filter_map(|path| file_resolver.resolve(file_id, path)) - .collect(); - problem = if points_to.is_empty() { - Some(Problem::UnresolvedModule { - candidate: file_mod, - }) - } else { - None - } - } else { - points_to = Vec::new(); - problem = Some(Problem::NotDirOwner { - move_to: RelativePathBuf::from(format!("../{}/mod.rs", mod_name)), - candidate: file_mod, - }); - } - (points_to, problem) -} diff --git a/crates/libanalysis/src/module_map_db/mod.rs b/crates/libanalysis/src/module_map_db/mod.rs deleted file mode 100644 index adad943da50a..000000000000 --- a/crates/libanalysis/src/module_map_db/mod.rs +++ /dev/null @@ -1,159 +0,0 @@ -use std::sync::Arc; -use { - FileId, - db::{ - Query, QueryRegistry, QueryCtx, - file_syntax, file_set - }, - descriptors::{ModuleDescriptor, ModuleTreeDescriptor} -}; - -pub(crate) fn register_queries(reg: &mut QueryRegistry) { - reg.add(MODULE_DESCR, "MODULE_DESCR"); -} - -pub(crate) fn module_tree(ctx: QueryCtx) -> Arc { - ctx.get(MODULE_TREE, ()) -} - -impl<'a> QueryCtx<'a> { - fn module_descr(&self, file_id: FileId) -> Arc { - self.get(MODULE_DESCR, file_id) - } -} - -const MODULE_DESCR: Query = Query(30, |ctx, &file_id| { - let file = file_syntax(ctx, file_id); - ModuleDescriptor::new(file.ast()) -}); - -const MODULE_TREE: Query<(), ModuleTreeDescriptor> = Query(31, |ctx, _| { - let file_set = file_set(ctx); - let mut files = Vec::new(); - for &file_id in file_set.0.iter() { - let module_descr = ctx.get(MODULE_DESCR, file_id); - files.push((file_id, module_descr)); - } - ModuleTreeDescriptor::new(files.iter().map(|(file_id, descr)| (*file_id, &**descr)), &file_set.1) -}); - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use im; - use relative_path::{RelativePath, RelativePathBuf}; - use { - db::{Query, Db, State}, - imp::FileResolverImp, - FileId, FileResolver, - }; - use super::*; - - #[derive(Debug)] - struct FileMap(im::HashMap); - - impl FileResolver for FileMap { - fn file_stem(&self, file_id: FileId) -> String { - self.0[&file_id].file_stem().unwrap().to_string() - } - fn resolve(&self, file_id: FileId, rel: &RelativePath) -> Option { - let path = self.0[&file_id].join(rel).normalize(); - self.0.iter() - .filter_map(|&(id, ref p)| Some(id).filter(|_| p == &path)) - .next() - } - } - - struct Fixture { - next_file_id: u32, - fm: im::HashMap, - db: Db, - } - - impl Fixture { - fn new() -> Fixture { - Fixture { - next_file_id: 1, - fm: im::HashMap::new(), - db: Db::new(), - } - } - fn add_file(&mut self, path: &str, text: &str) -> FileId { - assert!(path.starts_with("/")); - let file_id = FileId(self.next_file_id); - self.next_file_id += 1; - self.fm.insert(file_id, RelativePathBuf::from(&path[1..])); - let mut new_state = self.db.state().clone(); - new_state.file_map.insert(file_id, Arc::new(text.to_string())); - new_state.file_resolver = FileResolverImp::new( - Arc::new(FileMap(self.fm.clone())) - ); - self.db = self.db.with_changes(new_state, &[file_id], true); - file_id - } - fn remove_file(&mut self, file_id: FileId) { - self.fm.remove(&file_id); - let mut new_state = self.db.state().clone(); - new_state.file_map.remove(&file_id); - new_state.file_resolver = FileResolverImp::new( - Arc::new(FileMap(self.fm.clone())) - ); - self.db = self.db.with_changes(new_state, &[file_id], true); - } - fn change_file(&mut self, file_id: FileId, new_text: &str) { - let mut new_state = self.db.state().clone(); - new_state.file_map.insert(file_id, Arc::new(new_text.to_string())); - self.db = self.db.with_changes(new_state, &[file_id], false); - } - fn check_parent_modules( - &self, - file_id: FileId, - expected: &[FileId], - queries: &[(&'static str, u64)] - ) { - let (actual, events) = self.db.trace_query(|ctx| { - ctx.get(PARENT_MODULE, file_id) - }); - assert_eq!(actual.as_slice(), expected); - let mut counts = HashMap::new(); - events.into_iter() - .for_each(|event| *counts.entry(event).or_insert(0) += 1); - for &(query_id, expected_count) in queries.iter() { - let actual_count = *counts.get(&query_id).unwrap_or(&0); - assert_eq!( - actual_count, - expected_count, - "counts for {} differ", - query_id, - ) - } - - } - } - - #[test] - fn test_parent_module() { - let mut f = Fixture::new(); - let foo = f.add_file("/foo.rs", ""); - f.check_parent_modules(foo, &[], &[("MODULE_DESCR", 1)]); - - let lib = f.add_file("/lib.rs", "mod foo;"); - f.check_parent_modules(foo, &[lib], &[("MODULE_DESCR", 1)]); - f.check_parent_modules(foo, &[lib], &[("MODULE_DESCR", 0)]); - - f.change_file(lib, ""); - f.check_parent_modules(foo, &[], &[("MODULE_DESCR", 1)]); - - f.change_file(lib, "mod foo;"); - f.check_parent_modules(foo, &[lib], &[("MODULE_DESCR", 1)]); - - f.change_file(lib, "mod bar;"); - f.check_parent_modules(foo, &[], &[("MODULE_DESCR", 1)]); - - f.change_file(lib, "mod foo;"); - f.check_parent_modules(foo, &[lib], &[("MODULE_DESCR", 1)]); - - f.remove_file(lib); - f.check_parent_modules(foo, &[], &[("MODULE_DESCR", 0)]); - } -} diff --git a/crates/libanalysis/src/queries.rs b/crates/libanalysis/src/queries.rs new file mode 100644 index 000000000000..9d6754fd47c1 --- /dev/null +++ b/crates/libanalysis/src/queries.rs @@ -0,0 +1,28 @@ +use std::sync::Arc; +use libsyntax2::File; +use libeditor::LineIndex; +use { + FileId, + db::{Query, QueryCtx, QueryRegistry, file_text}, +}; + +pub(crate) fn register_queries(reg: &mut QueryRegistry) { + reg.add(FILE_SYNTAX, "FILE_SYNTAX"); + reg.add(FILE_LINES, "FILE_LINES"); +} + +pub(crate) fn file_syntax(ctx: QueryCtx, file_id: FileId) -> File { + (&*ctx.get(FILE_SYNTAX, file_id)).clone() +} +pub(crate) fn file_lines(ctx: QueryCtx, file_id: FileId) -> Arc { + ctx.get(FILE_LINES, file_id) +} + +const FILE_SYNTAX: Query = Query(16, |ctx, file_id: &FileId| { + let text = file_text(ctx, *file_id); + File::parse(&*text) +}); +const FILE_LINES: Query = Query(17, |ctx, file_id: &FileId| { + let text = file_text(ctx, *file_id); + LineIndex::new(&*text) +}); diff --git a/crates/libanalysis/src/roots.rs b/crates/libanalysis/src/roots.rs index e9527eeff443..0e7253ba2fb7 100644 --- a/crates/libanalysis/src/roots.rs +++ b/crates/libanalysis/src/roots.rs @@ -1,6 +1,5 @@ use std::{ collections::HashMap, - time::Instant, sync::Arc, panic, }; @@ -13,83 +12,74 @@ use libsyntax2::File; use { FileId, imp::FileResolverImp, - module_map::{ModuleMap, ChangeKind}, symbol_index::SymbolIndex, descriptors::{ModuleDescriptor, ModuleTreeDescriptor}, + db::Db, }; pub(crate) trait SourceRoot { fn contains(&self, file_id: FileId) -> bool; - fn module_tree(&self) -> Arc { - unimplemented!() - } - // fn module_map(&self) -> &ModuleMap; - fn lines(&self, file_id: FileId) -> &LineIndex; - fn syntax(&self, file_id: FileId) -> &File; + fn module_tree(&self) -> Arc; + fn lines(&self, file_id: FileId) -> Arc; + fn syntax(&self, file_id: FileId) -> File; fn symbols<'a>(&'a self, acc: &mut Vec<&'a SymbolIndex>); } -#[derive(Clone, Default, Debug)] +#[derive(Default, Debug)] pub(crate) struct WritableSourceRoot { - file_map: HashMap)>>, - module_map: ModuleMap, + db: Db, } impl WritableSourceRoot { - pub fn update(&mut self, file_id: FileId, text: Option) { - let change_kind = if self.file_map.remove(&file_id).is_some() { - if text.is_some() { - ChangeKind::Update - } else { - ChangeKind::Delete - } - } else { - ChangeKind::Insert - }; - self.module_map.update_file(file_id, change_kind); - self.file_map.remove(&file_id); - if let Some(text) = text { - let file_data = FileData::new(text); - self.file_map.insert(file_id, Arc::new((file_data, Default::default()))); - } - } - pub fn set_file_resolver(&mut self, file_resolver: FileResolverImp) { - self.module_map.set_file_resolver(file_resolver) - } - pub fn reindex(&self) { - let now = Instant::now(); - self.file_map - .par_iter() - .for_each(|(&file_id, data)| { - symbols(file_id, data); - }); - info!("parallel indexing took {:?}", now.elapsed()); + pub fn apply_changes( + &self, + changes: &mut dyn Iterator)>, + file_resolver: Option, + ) -> WritableSourceRoot { + let resolver_changed = file_resolver.is_some(); + let mut changed_files = Vec::new(); + let mut new_state = self.db.state().clone(); - } - fn data(&self, file_id: FileId) -> &FileData { - match self.file_map.get(&file_id) { - Some(data) => &data.0, - None => panic!("unknown file: {:?}", file_id), + for (file_id, text) in changes { + changed_files.push(file_id); + match text { + Some(text) => { + new_state.file_map.insert(file_id, Arc::new(text)); + }, + None => { + new_state.file_map.remove(&file_id); + } + } + } + if let Some(file_resolver) = file_resolver { + new_state.file_resolver = file_resolver + } + WritableSourceRoot { + db: self.db.with_changes(new_state, &changed_files, resolver_changed) } } } impl SourceRoot for WritableSourceRoot { + fn module_tree(&self) -> Arc { + self.db.make_query(::module_map::module_tree) + } + fn contains(&self, file_id: FileId) -> bool { - self.file_map.contains_key(&file_id) + self.db.state().file_map.contains_key(&file_id) } - fn lines(&self, file_id: FileId) -> &LineIndex { - self.data(file_id).lines() + fn lines(&self, file_id: FileId) -> Arc { + self.db.make_query(|ctx| ::queries::file_lines(ctx, file_id)) } - fn syntax(&self, file_id: FileId) -> &File { - self.data(file_id).syntax() + fn syntax(&self, file_id: FileId) -> File { + self.db.make_query(|ctx| ::queries::file_syntax(ctx, file_id)) } fn symbols<'a>(&'a self, acc: &mut Vec<&'a SymbolIndex>) { - acc.extend( - self.file_map - .iter() - .map(|(&file_id, data)| symbols(file_id, data)) - ) + // acc.extend( + // self.file_map + // .iter() + // .map(|(&file_id, data)| symbols(file_id, data)) + // ) } } @@ -101,7 +91,7 @@ fn symbols(file_id: FileId, (data, symbols): &(FileData, OnceCell)) #[derive(Debug)] struct FileData { text: String, - lines: OnceCell, + lines: OnceCell>, syntax: OnceCell, } @@ -113,8 +103,8 @@ impl FileData { lines: OnceCell::new(), } } - fn lines(&self) -> &LineIndex { - self.lines.get_or_init(|| LineIndex::new(&self.text)) + fn lines(&self) -> &Arc { + self.lines.get_or_init(|| Arc::new(LineIndex::new(&self.text))) } fn syntax(&self) -> &File { let text = &self.text; @@ -127,10 +117,6 @@ impl FileData { } } } - fn syntax_transient(&self) -> File { - self.syntax.get().map(|s| s.clone()) - .unwrap_or_else(|| File::parse(&self.text)) - } } #[derive(Debug)] @@ -184,11 +170,11 @@ impl SourceRoot for ReadonlySourceRoot { fn contains(&self, file_id: FileId) -> bool { self.file_map.contains_key(&file_id) } - fn lines(&self, file_id: FileId) -> &LineIndex { - self.data(file_id).lines() + fn lines(&self, file_id: FileId) -> Arc { + Arc::clone(self.data(file_id).lines()) } - fn syntax(&self, file_id: FileId) -> &File { - self.data(file_id).syntax() + fn syntax(&self, file_id: FileId) -> File { + self.data(file_id).syntax().clone() } fn symbols<'a>(&'a self, acc: &mut Vec<&'a SymbolIndex>) { acc.push(&self.symbol_index) From fcdf3a52b4b61a39474950486ea0edf5ebf33bea Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 15 Sep 2018 23:42:01 +0300 Subject: [PATCH 18/18] everysalsa --- crates/libanalysis/src/queries.rs | 21 ++++++++++++++----- crates/libanalysis/src/roots.rs | 28 +++++++++++--------------- crates/libanalysis/src/symbol_index.rs | 12 ++++++++++- crates/libeditor/src/symbols.rs | 2 +- 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/crates/libanalysis/src/queries.rs b/crates/libanalysis/src/queries.rs index 9d6754fd47c1..0b60316e6044 100644 --- a/crates/libanalysis/src/queries.rs +++ b/crates/libanalysis/src/queries.rs @@ -3,13 +3,11 @@ use libsyntax2::File; use libeditor::LineIndex; use { FileId, - db::{Query, QueryCtx, QueryRegistry, file_text}, + db::{Query, QueryCtx, QueryRegistry}, + symbol_index::SymbolIndex, }; -pub(crate) fn register_queries(reg: &mut QueryRegistry) { - reg.add(FILE_SYNTAX, "FILE_SYNTAX"); - reg.add(FILE_LINES, "FILE_LINES"); -} +pub(crate) use db::{file_text, file_set}; pub(crate) fn file_syntax(ctx: QueryCtx, file_id: FileId) -> File { (&*ctx.get(FILE_SYNTAX, file_id)).clone() @@ -17,6 +15,9 @@ pub(crate) fn file_syntax(ctx: QueryCtx, file_id: FileId) -> File { pub(crate) fn file_lines(ctx: QueryCtx, file_id: FileId) -> Arc { ctx.get(FILE_LINES, file_id) } +pub(crate) fn file_symbols(ctx: QueryCtx, file_id: FileId) -> Arc { + ctx.get(FILE_SYMBOLS, file_id) +} const FILE_SYNTAX: Query = Query(16, |ctx, file_id: &FileId| { let text = file_text(ctx, *file_id); @@ -26,3 +27,13 @@ const FILE_LINES: Query = Query(17, |ctx, file_id: &FileId| { let text = file_text(ctx, *file_id); LineIndex::new(&*text) }); +const FILE_SYMBOLS: Query = Query(18, |ctx, file_id: &FileId| { + let syntax = file_syntax(ctx, *file_id); + SymbolIndex::for_file(*file_id, syntax) +}); + +pub(crate) fn register_queries(reg: &mut QueryRegistry) { + reg.add(FILE_SYNTAX, "FILE_SYNTAX"); + reg.add(FILE_LINES, "FILE_LINES"); + reg.add(FILE_SYMBOLS, "FILE_SYMBOLS"); +} diff --git a/crates/libanalysis/src/roots.rs b/crates/libanalysis/src/roots.rs index 0e7253ba2fb7..191d0d821a34 100644 --- a/crates/libanalysis/src/roots.rs +++ b/crates/libanalysis/src/roots.rs @@ -22,7 +22,7 @@ pub(crate) trait SourceRoot { fn module_tree(&self) -> Arc; fn lines(&self, file_id: FileId) -> Arc; fn syntax(&self, file_id: FileId) -> File; - fn symbols<'a>(&'a self, acc: &mut Vec<&'a SymbolIndex>); + fn symbols(&self, acc: &mut Vec>); } #[derive(Default, Debug)] @@ -74,20 +74,16 @@ impl SourceRoot for WritableSourceRoot { fn syntax(&self, file_id: FileId) -> File { self.db.make_query(|ctx| ::queries::file_syntax(ctx, file_id)) } - fn symbols<'a>(&'a self, acc: &mut Vec<&'a SymbolIndex>) { - // acc.extend( - // self.file_map - // .iter() - // .map(|(&file_id, data)| symbols(file_id, data)) - // ) + fn symbols<'a>(&'a self, acc: &mut Vec>) { + self.db.make_query(|ctx| { + let file_set = ::queries::file_set(ctx); + let syms = file_set.0.iter() + .map(|file_id| ::queries::file_symbols(ctx, *file_id)); + acc.extend(syms); + }); } } -fn symbols(file_id: FileId, (data, symbols): &(FileData, OnceCell)) -> &SymbolIndex { - let syntax = data.syntax_transient(); - symbols.get_or_init(|| SymbolIndex::for_file(file_id, syntax)) -} - #[derive(Debug)] struct FileData { text: String, @@ -121,7 +117,7 @@ impl FileData { #[derive(Debug)] pub(crate) struct ReadonlySourceRoot { - symbol_index: SymbolIndex, + symbol_index: Arc, file_map: HashMap, module_tree: Arc, } @@ -149,7 +145,7 @@ impl ReadonlySourceRoot { .collect(); ReadonlySourceRoot { - symbol_index, + symbol_index: Arc::new(symbol_index), file_map, module_tree: Arc::new(module_tree), } @@ -176,7 +172,7 @@ impl SourceRoot for ReadonlySourceRoot { fn syntax(&self, file_id: FileId) -> File { self.data(file_id).syntax().clone() } - fn symbols<'a>(&'a self, acc: &mut Vec<&'a SymbolIndex>) { - acc.push(&self.symbol_index) + fn symbols(&self, acc: &mut Vec>) { + acc.push(Arc::clone(&self.symbol_index)) } } diff --git a/crates/libanalysis/src/symbol_index.rs b/crates/libanalysis/src/symbol_index.rs index 4c93761aad2c..d22187ac0b22 100644 --- a/crates/libanalysis/src/symbol_index.rs +++ b/crates/libanalysis/src/symbol_index.rs @@ -1,3 +1,7 @@ +use std::{ + sync::Arc, + hash::{Hash, Hasher}, +}; use libeditor::{FileSymbol, file_symbols}; use libsyntax2::{ File, @@ -13,6 +17,12 @@ pub(crate) struct SymbolIndex { map: fst::Map, } +impl Hash for SymbolIndex { + fn hash(&self, hasher: &mut H) { + self.symbols.hash(hasher) + } +} + impl SymbolIndex { pub(crate) fn for_files(files: impl ParallelIterator) -> SymbolIndex { let mut symbols = files @@ -43,7 +53,7 @@ impl SymbolIndex { impl Query { pub(crate) fn search( self, - indices: &[&SymbolIndex], + indices: &[Arc], token: &JobToken, ) -> Vec<(FileId, FileSymbol)> { diff --git a/crates/libeditor/src/symbols.rs b/crates/libeditor/src/symbols.rs index 28b86c004522..2f9cc9233657 100644 --- a/crates/libeditor/src/symbols.rs +++ b/crates/libeditor/src/symbols.rs @@ -17,7 +17,7 @@ pub struct StructureNode { pub kind: SyntaxKind, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Hash)] pub struct FileSymbol { pub name: SmolStr, pub node_range: TextRange,