user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Auto merge of #146038 - notriddle:polarity, r=GuillaumeGomez

Browse source 
rustdoc-search: split function inverted index by input/output

Fixes rust-lang/rust#146015

With a patch applied to count the number of unifications, and running the query `Option<T>, (T -> U) -> Option<U>`

before: performed unifyFunctionType on 17484 functions
after: performed unifyFunctionType on 3055 functions

preview:
https://notriddle.com/rustdoc-html-demo-12/polarity/doc/std/index.html
https://notriddle.com/rustdoc-html-demo-12/polarity/compiler-doc/rustc_hir/index.html
This commit is contained in:
bors 2025-08-31 20:28:35 +00:00
commit 07d246fc6d
3 changed files with 267 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

255
src/librustdoc/html/render/search_index.rs
View file

@ -289,8 +289,26 @@ impl SerializedSearchIndex {
(Some(self_type_data), None) => Some(self_type_data),
(None, Some(other_type_data)) => Some(TypeData {
search_unbox: other_type_data.search_unbox,
inverted_function_signature_index: other_type_data
.inverted_function_signature_index
inverted_function_inputs_index: other_type_data
.inverted_function_inputs_index
.iter()
.cloned()
.map(|mut list: Vec<u32>| {
for fnid in &mut list {
assert!(
other.function_data
[usize::try_from(*fnid).unwrap()]
.is_some(),
);
// this is valid because we call `self.push()` once, exactly, for every entry,
// even if we're just pushing a tombstone
*fnid += u32::try_from(other_entryid_offset).unwrap();
}
list
})
.collect(),
inverted_function_output_index: other_type_data
.inverted_function_output_index
.iter()
.cloned()
.map(|mut list: Vec<u32>| {
@ -310,18 +328,42 @@ impl SerializedSearchIndex {
}),
(Some(mut self_type_data), Some(other_type_data)) => {
for (size, other_list) in other_type_data
.inverted_function_signature_index
.inverted_function_inputs_index
.iter()
.enumerate()
{
while self_type_data.inverted_function_signature_index.len()
while self_type_data.inverted_function_inputs_index.len()
<= size
{
self_type_data
.inverted_function_signature_index
.inverted_function_inputs_index
.push(Vec::new());
}
self_type_data.inverted_function_signature_index[size].extend(
self_type_data.inverted_function_inputs_index[size].extend(
other_list.iter().copied().map(|fnid| {
assert!(
other.function_data[usize::try_from(fnid).unwrap()]
.is_some(),
);
// this is valid because we call `self.push()` once, exactly, for every entry,
// even if we're just pushing a tombstone
fnid + u32::try_from(other_entryid_offset).unwrap()
}),
)
}
for (size, other_list) in other_type_data
.inverted_function_output_index
.iter()
.enumerate()
{
while self_type_data.inverted_function_output_index.len()
<= size
{
self_type_data
.inverted_function_output_index
.push(Vec::new());
}
self_type_data.inverted_function_output_index[size].extend(
other_list.iter().copied().map(|fnid| {
assert!(
other.function_data[usize::try_from(fnid).unwrap()]
@ -443,8 +485,25 @@ impl SerializedSearchIndex {
param_names: function_data.param_names.clone(),
}),
other.type_data[other_entryid].as_ref().map(|type_data| TypeData {
inverted_function_signature_index: type_data
.inverted_function_signature_index
inverted_function_inputs_index: type_data
.inverted_function_inputs_index
.iter()
.cloned()
.map(|mut list| {
for fnid in &mut list {
assert!(
other.function_data[usize::try_from(*fnid).unwrap()]
.is_some(),
);
// this is valid because we call `self.push()` once, exactly, for every entry,
// even if we're just pushing a tombstone
*fnid += u32::try_from(other_entryid_offset).unwrap();
}
list
})
.collect(),
inverted_function_output_index: type_data
.inverted_function_output_index
.iter()
.cloned()
.map(|mut list| {
@ -599,9 +658,13 @@ impl SerializedSearchIndex {
},
),
self.type_data[id].as_ref().map(
|TypeData { search_unbox, inverted_function_signature_index }| {
let inverted_function_signature_index: Vec<Vec<u32>> =
inverted_function_signature_index
|TypeData {
search_unbox,
inverted_function_inputs_index,
inverted_function_output_index,
}| {
let inverted_function_inputs_index: Vec<Vec<u32>> =
inverted_function_inputs_index
.iter()
.cloned()
.map(|mut list| {
@ -615,7 +678,26 @@ impl SerializedSearchIndex {
list
})
.collect();
TypeData { search_unbox: *search_unbox, inverted_function_signature_index }
let inverted_function_output_index: Vec<Vec<u32>> =
inverted_function_output_index
.iter()
.cloned()
.map(|mut list| {
for id in &mut list {
*id = u32::try_from(
*map.get(&usize::try_from(*id).unwrap()).unwrap(),
)
.unwrap();
}
list.sort();
list
})
.collect();
TypeData {
search_unbox: *search_unbox,
inverted_function_inputs_index,
inverted_function_output_index,
}
},
),
self.alias_pointers[id].and_then(|alias| map.get(&alias).copied()),
@ -934,18 +1016,20 @@ struct TypeData {
/// | `Unboxable<Inner>` | yes | no | no |
/// | `Inner<Unboxable>` | no | no | yes |
search_unbox: bool,
/// List of functions that mention this type in their type signature.
/// List of functions that mention this type in their type signature,
/// on the left side of the `->` arrow.
///
/// - The outermost list has one entry per alpha-normalized generic.
///
/// - The second layer is sorted by number of types that appear in the
/// - The outer layer is sorted by number of types that appear in the
/// type signature. The search engine iterates over these in order from
/// smallest to largest. Functions with less stuff in their type
/// signature are more likely to be what the user wants, because we never
/// show functions that are *missing* parts of the query, so removing..
///
/// - The final layer is the list of functions.
inverted_function_signature_index: Vec<Vec<u32>>,
/// - The inner layer is the list of functions.
inverted_function_inputs_index: Vec<Vec<u32>>,
/// List of functions that mention this type in their type signature,
/// on the right side of the `->` arrow.
inverted_function_output_index: Vec<Vec<u32>>,
}
impl Serialize for TypeData {
@ -953,15 +1037,21 @@ impl Serialize for TypeData {
where
S: Serializer,
{
if self.search_unbox || !self.inverted_function_signature_index.is_empty() {
if self.search_unbox
|| !self.inverted_function_inputs_index.is_empty()
|| !self.inverted_function_output_index.is_empty()
{
let mut seq = serializer.serialize_seq(None)?;
if !self.inverted_function_signature_index.is_empty() {
let mut buf = Vec::new();
encode::write_postings_to_string(&self.inverted_function_signature_index, &mut buf);
let mut serialized_result = Vec::new();
stringdex_internals::encode::write_base64_to_bytes(&buf, &mut serialized_result);
seq.serialize_element(&String::from_utf8(serialized_result).unwrap())?;
}
let mut buf = Vec::new();
encode::write_postings_to_string(&self.inverted_function_inputs_index, &mut buf);
let mut serialized_result = Vec::new();
stringdex_internals::encode::write_base64_to_bytes(&buf, &mut serialized_result);
seq.serialize_element(&str::from_utf8(&serialized_result).unwrap())?;
buf.clear();
serialized_result.clear();
encode::write_postings_to_string(&self.inverted_function_output_index, &mut buf);
stringdex_internals::encode::write_base64_to_bytes(&buf, &mut serialized_result);
seq.serialize_element(&str::from_utf8(&serialized_result).unwrap())?;
if self.search_unbox {
seq.serialize_element(&1)?;
}
@ -984,21 +1074,39 @@ impl<'de> Deserialize<'de> for TypeData {
write!(formatter, "type data")
}
fn visit_none<E>(self) -> Result<TypeData, E> {
Ok(TypeData { inverted_function_signature_index: vec![], search_unbox: false })
Ok(TypeData {
inverted_function_inputs_index: vec![],
inverted_function_output_index: vec![],
search_unbox: false,
})
}
fn visit_seq<A: de::SeqAccess<'de>>(self, mut v: A) -> Result<TypeData, A::Error> {
let inverted_function_signature_index: String =
let inverted_function_inputs_index: String =
v.next_element()?.unwrap_or(String::new());
let inverted_function_output_index: String =
v.next_element()?.unwrap_or(String::new());
let search_unbox: u32 = v.next_element()?.unwrap_or(0);
let mut idx: Vec<u8> = Vec::new();
stringdex_internals::decode::read_base64_from_bytes(
inverted_function_signature_index.as_bytes(),
inverted_function_inputs_index.as_bytes(),
&mut idx,
)
.unwrap();
let mut inverted_function_signature_index = Vec::new();
encode::read_postings_from_string(&mut inverted_function_signature_index, &idx);
Ok(TypeData { inverted_function_signature_index, search_unbox: search_unbox == 1 })
let mut inverted_function_inputs_index = Vec::new();
encode::read_postings_from_string(&mut inverted_function_inputs_index, &idx);
idx.clear();
stringdex_internals::decode::read_base64_from_bytes(
inverted_function_output_index.as_bytes(),
&mut idx,
)
.unwrap();
let mut inverted_function_output_index = Vec::new();
encode::read_postings_from_string(&mut inverted_function_output_index, &idx);
Ok(TypeData {
inverted_function_inputs_index,
inverted_function_output_index,
search_unbox: search_unbox == 1,
})
}
}
deserializer.deserialize_any(TypeDataVisitor)
@ -1222,8 +1330,16 @@ pub(crate) fn build_index(
let index = *index.get();
serialized_index.descs[index] = crate_doc;
for type_data in serialized_index.type_data.iter_mut() {
if let Some(TypeData { inverted_function_signature_index, .. }) = type_data {
for list in &mut inverted_function_signature_index[..] {
if let Some(TypeData {
inverted_function_inputs_index,
inverted_function_output_index,
..
}) = type_data
{
for list in inverted_function_inputs_index
.iter_mut()
.chain(inverted_function_output_index.iter_mut())
{
list.retain(|fnid| {
serialized_index.entry_data[usize::try_from(*fnid).unwrap()]
.as_ref()
@ -1449,7 +1565,8 @@ pub(crate) fn build_index(
if serialized_index.type_data[id].as_mut().is_none() {
serialized_index.type_data[id] = Some(TypeData {
search_unbox,
inverted_function_signature_index: Vec::new(),
inverted_function_inputs_index: Vec::new(),
inverted_function_output_index: Vec::new(),
});
} else if search_unbox {
serialized_index.type_data[id].as_mut().unwrap().search_unbox = true;
@ -1473,7 +1590,11 @@ pub(crate) fn build_index(
None
},
},
TypeData { search_unbox, inverted_function_signature_index: Vec::new() },
TypeData {
inverted_function_inputs_index: Vec::new(),
inverted_function_output_index: Vec::new(),
search_unbox,
},
);
pathid
}
@ -1695,13 +1816,14 @@ pub(crate) fn build_index(
}
}
if let Some(search_type) = &mut item.search_type {
let mut used_in_function_signature = BTreeSet::new();
let mut used_in_function_inputs = BTreeSet::new();
let mut used_in_function_output = BTreeSet::new();
for item in &mut search_type.inputs {
convert_render_type(
item,
cache,
&mut serialized_index,
&mut used_in_function_signature,
&mut used_in_function_inputs,
tcx,
);
}
@ -1710,20 +1832,44 @@ pub(crate) fn build_index(
item,
cache,
&mut serialized_index,
&mut used_in_function_signature,
&mut used_in_function_output,
tcx,
);
}
let mut used_in_constraints = Vec::new();
for constraint in &mut search_type.where_clause {
let mut used_in_constraint = BTreeSet::new();
for trait_ in &mut constraint[..] {
convert_render_type(
trait_,
cache,
&mut serialized_index,
&mut used_in_function_signature,
&mut used_in_constraint,
tcx,
);
}
used_in_constraints.push(used_in_constraint);
}
loop {
let mut inserted_any = false;
for (i, used_in_constraint) in used_in_constraints.iter().enumerate() {
let id = !(i as isize);
if used_in_function_inputs.contains(&id)
&& !used_in_function_inputs.is_superset(&used_in_constraint)
{
used_in_function_inputs.extend(used_in_constraint.iter().copied());
inserted_any = true;
}
if used_in_function_output.contains(&id)
&& !used_in_function_output.is_superset(&used_in_constraint)
{
used_in_function_output.extend(used_in_constraint.iter().copied());
inserted_any = true;
}
}
if !inserted_any {
break;
}
}
let search_type_size = search_type.size() +
// Artificially give struct fields a size of 8 instead of their real
@ -1746,13 +1892,13 @@ pub(crate) fn build_index(
.map(|sym| sym.map(|sym| sym.to_string()).unwrap_or(String::new()))
.collect::<Vec<String>>(),
});
for index in used_in_function_signature {
for index in used_in_function_inputs {
let postings = if index >= 0 {
assert!(serialized_index.path_data[index as usize].is_some());
&mut serialized_index.type_data[index as usize]
.as_mut()
.unwrap()
.inverted_function_signature_index
.inverted_function_inputs_index
} else {
let generic_id = usize::try_from(-index).unwrap() - 1;
for _ in serialized_index.generic_inverted_index.len()..=generic_id {
@ -1763,7 +1909,30 @@ pub(crate) fn build_index(
while postings.len() <= search_type_size {
postings.push(Vec::new());
}
postings[search_type_size].push(new_entry_id as u32);
if postings[search_type_size].last() != Some(&(new_entry_id as u32)) {
postings[search_type_size].push(new_entry_id as u32);
}
}
for index in used_in_function_output {
let postings = if index >= 0 {
assert!(serialized_index.path_data[index as usize].is_some());
&mut serialized_index.type_data[index as usize]
.as_mut()
.unwrap()
.inverted_function_output_index
} else {
let generic_id = usize::try_from(-index).unwrap() - 1;
for _ in serialized_index.generic_inverted_index.len()..=generic_id {
serialized_index.generic_inverted_index.push(Vec::new());
}
&mut serialized_index.generic_inverted_index[generic_id]
};
while postings.len() <= search_type_size {
postings.push(Vec::new());
}
if postings[search_type_size].last() != Some(&(new_entry_id as u32)) {
postings[search_type_size].push(new_entry_id as u32);
}
}
}
}

5
src/librustdoc/html/static/js/rustdoc.d.ts vendored
View file

@ -270,9 +270,12 @@ declare namespace rustdoc {
*/
interface TypeData {
searchUnbox: boolean,
invertedFunctionSignatureIndex: RoaringBitmap[],
invertedFunctionInputsIndex: RoaringBitmap[],
invertedFunctionOutputIndex: RoaringBitmap[],
}
type TypeInvertedIndexPolarity = "invertedFunctionInputsIndex" | "invertedFunctionOutputIndex";
/**
* A search entry of some sort.
*/

79
src/librustdoc/html/static/js/search.js
View file

@ -1696,7 +1696,7 @@ class DocSearch {
}
/**
* function_signature, param_names
* @type {[string, number] | [number] | [string] | [] | null}
* @type {[string, string, number] | [string, string] | [] | null}
*/
const raw = JSON.parse(encoded);
@ -1705,32 +1705,46 @@ class DocSearch {
}
let searchUnbox = false;
const invertedFunctionSignatureIndex = [];
const invertedFunctionInputsIndex = [];
const invertedFunctionOutputIndex = [];
if (typeof raw[0] === "string") {
if (raw[1]) {
if (raw[2]) {
searchUnbox = true;
}
// the inverted function signature index is a list of bitmaps,
// by number of types that appear in the function
let i = 0;
const pb = makeUint8ArrayFromBase64(raw[0]);
const l = pb.length;
let pb = makeUint8ArrayFromBase64(raw[0]);
let l = pb.length;
while (i < l) {
if (pb[i] === 0) {
invertedFunctionSignatureIndex.push(RoaringBitmap.empty());
invertedFunctionInputsIndex.push(RoaringBitmap.empty());
i += 1;
} else {
const bitmap = new RoaringBitmap(pb, i);
i += bitmap.consumed_len_bytes;
invertedFunctionSignatureIndex.push(bitmap);
invertedFunctionInputsIndex.push(bitmap);
}
}
i = 0;
pb = makeUint8ArrayFromBase64(raw[1]);
l = pb.length;
while (i < l) {
if (pb[i] === 0) {
invertedFunctionOutputIndex.push(RoaringBitmap.empty());
i += 1;
} else {
const bitmap = new RoaringBitmap(pb, i);
i += bitmap.consumed_len_bytes;
invertedFunctionOutputIndex.push(bitmap);
}
}
} else if (raw[0]) {
searchUnbox = true;
}
return { searchUnbox, invertedFunctionSignatureIndex };
return { searchUnbox, invertedFunctionInputsIndex, invertedFunctionOutputIndex };
}
/**
@ -4009,14 +4023,19 @@ class DocSearch {
* or anything else. This function returns all possible permutations.
*
* @param {rustdoc.ParserQueryElement|null} elem
* @param {rustdoc.TypeInvertedIndexPolarity} polarity
* @returns {Promise<PostingsList<rustdoc.QueryElement>[]>}
*/
const unpackPostingsList = async elem => {
const unpackPostingsList = async(elem, polarity) => {
if (!elem) {
return empty_postings_list;
}
const typeFilter = itemTypeFromName(elem.typeFilter);
const searchResults = await index.search(elem.normalizedPathLast);
const [searchResults, upla, uplb] = await Promise.all([
index.search(elem.normalizedPathLast),
unpackPostingsListAll(elem.generics, polarity),
unpackPostingsListBindings(elem.bindings, polarity),
]);
/**
* @type {Promise<[
* number,
@ -4039,7 +4058,7 @@ class DocSearch {
const types = (await Promise.all(typePromises))
.filter(([_id, name, ty, path]) =>
name !== null && name.toLowerCase() === elem.pathLast &&
ty && !ty.invertedFunctionSignatureIndex.every(bitmap => {
ty && !ty[polarity].every(bitmap => {
return bitmap.isEmpty();
}) &&
path && path.ty !== TY_ASSOCTYPE &&
@ -4078,7 +4097,7 @@ class DocSearch {
this.getPathData(id),
]);
if (name !== null && ty !== null && path !== null &&
!ty.invertedFunctionSignatureIndex.every(bitmap => {
!ty[polarity].every(bitmap => {
return bitmap.isEmpty();
}) &&
path.ty !== TY_ASSOCTYPE
@ -4176,18 +4195,16 @@ class DocSearch {
/** @type {PostingsList<rustdoc.QueryElement>[]} */
const results = [];
for (const [id, _name, typeData] of types) {
if (!typeData || typeData.invertedFunctionSignatureIndex.every(bitmap => {
if (!typeData || typeData[polarity].every(bitmap => {
return bitmap.isEmpty();
})) {
continue;
}
const upla = await unpackPostingsListAll(elem.generics);
const uplb = await unpackPostingsListBindings(elem.bindings);
for (const {invertedIndex: genericsIdx, queryElem: generics} of upla) {
for (const {invertedIndex: bindingsIdx, queryElem: bindings} of uplb) {
results.push({
invertedIndex: intersectInvertedIndexes(
typeData.invertedFunctionSignatureIndex,
typeData[polarity],
genericsIdx,
bindingsIdx,
),
@ -4219,15 +4236,16 @@ class DocSearch {
* take the intersection of this bitmap.
*
* @param {(rustdoc.ParserQueryElement|null)[]|null} elems
* @param {rustdoc.TypeInvertedIndexPolarity} polarity
* @returns {Promise<PostingsList<rustdoc.QueryElement[]>[]>}
*/
const unpackPostingsListAll = async elems => {
const unpackPostingsListAll = async(elems, polarity) => {
if (!elems || elems.length === 0) {
return nested_everything_postings_list;
}
const [firstPostingsList, remainingAll] = await Promise.all([
unpackPostingsList(elems[0]),
unpackPostingsListAll(elems.slice(1)),
unpackPostingsList(elems[0], polarity),
unpackPostingsListAll(elems.slice(1), polarity),
]);
/** @type {PostingsList<rustdoc.QueryElement[]>[]} */
const results = [];
@ -4261,11 +4279,12 @@ class DocSearch {
* Before passing an actual parser item to it, make sure to clone the map.
*
* @param {Map<string, rustdoc.ParserQueryElement[]>} elems
* @param {rustdoc.TypeInvertedIndexPolarity} polarity
* @returns {Promise<PostingsList<
* Map<number, rustdoc.QueryElement[]>,
* >[]>}
*/
const unpackPostingsListBindings = async elems => {
const unpackPostingsListBindings = async(elems, polarity) => {
if (!elems) {
return [{
invertedIndex: everything_inverted_index,
@ -4286,19 +4305,23 @@ class DocSearch {
queryElem: new Map(),
}];
}
const firstKeyIds = await index.search(firstKey);
// HEADS UP!
// We must put this map back the way we found it before returning,
// otherwise things break.
elems.delete(firstKey);
const [firstKeyIds, firstPostingsList, remainingAll] = await Promise.all([
index.search(firstKey),
unpackPostingsListAll(firstList, polarity),
unpackPostingsListBindings(elems, polarity),
]);
if (!firstKeyIds) {
elems.set(firstKey, firstList);
// User specified a non-existent key.
return [{
invertedIndex: empty_inverted_index,
queryElem: new Map(),
}];
}
elems.delete(firstKey);
const [firstPostingsList, remainingAll] = await Promise.all([
unpackPostingsListAll(firstList),
unpackPostingsListBindings(elems),
]);
/** @type {PostingsList<Map<number, rustdoc.QueryElement[]>>[]} */
const results = [];
for (const keyId of firstKeyIds.matches().entries()) {
@ -4335,8 +4358,8 @@ class DocSearch {
// finally, we can do the actual unification loop
const [allInputs, allOutput] = await Promise.all([
unpackPostingsListAll(inputs),
unpackPostingsListAll(output),
unpackPostingsListAll(inputs, "invertedFunctionInputsIndex"),
unpackPostingsListAll(output, "invertedFunctionOutputIndex"),
]);
let checkCounter = 0;
/**