initial implementation of mergable rustdoc cci

src/librustdoc/Cargo.toml

@@ -16,7 +16,7 @@ minifier = "0.3.0"
 pulldown-cmark-old = { version = "0.9.6", package = "pulldown-cmark", default-features = false }
 regex = "1"
 rustdoc-json-types = { path = "../rustdoc-json-types" }
-serde_json = "1.0"
+serde_json = { version = "1.0", features = ["preserve_order"] }
 serde = { version = "1.0", features = ["derive"] }
 smallvec = "1.8.1"
 tempfile = "3"

src/librustdoc/clean/types.rs

@@ -128,7 +128,7 @@ pub(crate) struct ExternalCrate {
 }
 
 impl ExternalCrate {
-    const LOCAL: Self = Self { crate_num: LOCAL_CRATE };
+    pub(crate) const LOCAL: Self = Self { crate_num: LOCAL_CRATE };
 
     #[inline]
     pub(crate) fn def_id(&self) -> DefId {

src/librustdoc/config.rs

@@ -730,7 +730,6 @@ impl Options {
         let extern_html_root_takes_precedence =
             matches.opt_present("extern-html-root-takes-precedence");
         let html_no_source = matches.opt_present("html-no-source");
-
         if generate_link_to_definition && (show_coverage || output_format != OutputFormat::Html) {
             dcx.fatal(
                 "--generate-link-to-definition option can only be used with HTML output format",

src/librustdoc/html/render/context.rs

@@ -14,7 +14,6 @@ use rustc_span::edition::Edition;
 use rustc_span::{sym, FileName, Symbol};
 
 use super::print_item::{full_path, item_path, print_item};
-use super::search_index::build_index;
 use super::sidebar::{print_sidebar, sidebar_module_like, Sidebar};
 use super::write_shared::write_shared;
 use super::{collect_spans_and_sources, scrape_examples_help, AllTypes, LinkFromSrc, StylePath};
@@ -573,13 +572,7 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
         }
 
         if !no_emit_shared {
-            // Build our search index
-            let index = build_index(&krate, &mut Rc::get_mut(&mut cx.shared).unwrap().cache, tcx);
-
-            // Write shared runs within a flock; disable thread dispatching of IO temporarily.
-            Rc::get_mut(&mut cx.shared).unwrap().fs.set_sync_only(true);
-            write_shared(&mut cx, &krate, index, &md_opts)?;
-            Rc::get_mut(&mut cx.shared).unwrap().fs.set_sync_only(false);
+            write_shared(&mut cx, &krate, &md_opts, tcx)?;
         }
 
         Ok((cx, krate))
@@ -729,6 +722,7 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
         );
         shared.fs.write(help_file, v)?;
 
+        // if to avoid writing files to doc root unless we're on the final invocation
         if shared.layout.scrape_examples_extension {
             page.title = "About scraped examples";
             page.description = "How the scraped examples feature works in Rustdoc";

src/librustdoc/html/render/mod.rs

@@ -31,6 +31,8 @@ mod tests;
 mod context;
 mod print_item;
 pub(crate) mod sidebar;
+mod sorted_json;
+mod sorted_template;
 mod span_map;
 mod type_layout;
 mod write_shared;

src/librustdoc/html/render/search_index.rs

@@ -18,6 +18,7 @@ use crate::formats::cache::{Cache, OrphanImplItem};
 use crate::formats::item_type::ItemType;
 use crate::html::format::join_with_double_colon;
 use crate::html::markdown::short_markdown_summary;
+use crate::html::render::sorted_json::SortedJson;
 use crate::html::render::{self, IndexItem, IndexItemFunctionType, RenderType, RenderTypeId};
 
 /// The serialized search description sharded version
@@ -46,7 +47,7 @@ use crate::html::render::{self, IndexItem, IndexItemFunctionType, RenderType, Re
 /// [2]: https://en.wikipedia.org/wiki/Sliding_window_protocol#Basic_concept
 /// [3]: https://learn.microsoft.com/en-us/troubleshoot/windows-server/networking/description-tcp-features
 pub(crate) struct SerializedSearchIndex {
-    pub(crate) index: String,
+    pub(crate) index: SortedJson,
     pub(crate) desc: Vec<(usize, String)>,
 }
 
@@ -683,24 +684,19 @@ pub(crate) fn build_index<'tcx>(
     // The index, which is actually used to search, is JSON
     // It uses `JSON.parse(..)` to actually load, since JSON
     // parses faster than the full JavaScript syntax.
-    let index = format!(
-        r#"["{}",{}]"#,
-        krate.name(tcx),
-        serde_json::to_string(&CrateData {
-            items: crate_items,
-            paths: crate_paths,
-            aliases: &aliases,
-            associated_item_disambiguators: &associated_item_disambiguators,
-            desc_index,
-            empty_desc,
-        })
-        .expect("failed serde conversion")
-        // All these `replace` calls are because we have to go through JS string for JSON content.
-        .replace('\\', r"\\")
-        .replace('\'', r"\'")
-        // We need to escape double quotes for the JSON.
-        .replace("\\\"", "\\\\\"")
-    );
+    let crate_name = krate.name(tcx);
+    let data = CrateData {
+        items: crate_items,
+        paths: crate_paths,
+        aliases: &aliases,
+        associated_item_disambiguators: &associated_item_disambiguators,
+        desc_index,
+        empty_desc,
+    };
+    let index = SortedJson::array_unsorted([
+        SortedJson::serialize(crate_name.as_str()),
+        SortedJson::serialize(data),
+    ]);
     SerializedSearchIndex { index, desc }
 }
 

src/librustdoc/html/render/sorted_json.rs

@@ -0,0 +1,82 @@
+use itertools::Itertools as _;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::borrow::Borrow;
+use std::fmt;
+
+/// Prerenedered json.
+///
+/// Arrays are sorted by their stringified entries, and objects are sorted by their stringified
+/// keys.
+///
+/// Must use serde_json with the preserve_order feature.
+///
+/// Both the Display and serde_json::to_string implementations write the serialized json
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+#[serde(from = "Value")]
+#[serde(into = "Value")]
+pub(crate) struct SortedJson(String);
+
+impl SortedJson {
+    /// If you pass in an array, it will not be sorted.
+    pub(crate) fn serialize<T: Serialize>(item: T) -> Self {
+        SortedJson(serde_json::to_string(&item).unwrap())
+    }
+
+    /// Serializes and sorts
+    pub(crate) fn array<T: Borrow<SortedJson>, I: IntoIterator<Item = T>>(items: I) -> Self {
+        let items = items
+            .into_iter()
+            .sorted_unstable_by(|a, b| a.borrow().cmp(&b.borrow()))
+            .format_with(",", |item, f| f(item.borrow()));
+        SortedJson(format!("[{}]", items))
+    }
+
+    pub(crate) fn array_unsorted<T: Borrow<SortedJson>, I: IntoIterator<Item = T>>(
+        items: I,
+    ) -> Self {
+        let items = items.into_iter().format_with(",", |item, f| f(item.borrow()));
+        SortedJson(format!("[{items}]"))
+    }
+}
+
+impl fmt::Display for SortedJson {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+impl From<Value> for SortedJson {
+    fn from(value: Value) -> Self {
+        SortedJson(serde_json::to_string(&value).unwrap())
+    }
+}
+
+impl From<SortedJson> for Value {
+    fn from(json: SortedJson) -> Self {
+        serde_json::from_str(&json.0).unwrap()
+    }
+}
+
+/// For use in JSON.parse('{...}').
+///
+/// JSON.parse supposedly loads faster than raw JS source,
+/// so this is used for large objects.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub(crate) struct EscapedJson(SortedJson);
+
+impl From<SortedJson> for EscapedJson {
+    fn from(json: SortedJson) -> Self {
+        EscapedJson(json)
+    }
+}
+
+impl fmt::Display for EscapedJson {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // All these `replace` calls are because we have to go through JS string
+        // for JSON content.
+        // We need to escape double quotes for the JSON
+        let json = self.0.0.replace('\\', r"\\").replace('\'', r"\'").replace("\\\"", "\\\\\"");
+        write!(f, "{}", json)
+    }
+}

src/librustdoc/html/render/sorted_template.rs

@@ -0,0 +1,136 @@
+use std::collections::BTreeSet;
+use std::fmt;
+use std::marker::PhantomData;
+use std::str::FromStr;
+
+use serde::{Deserialize, Serialize};
+
+/// Append-only templates for sorted, deduplicated lists of items.
+///
+/// Last line of the rendered output is a comment encoding the next insertion point.
+#[derive(Debug, Clone)]
+pub(crate) struct SortedTemplate<F> {
+    format: PhantomData<F>,
+    before: String,
+    after: String,
+    contents: BTreeSet<String>,
+}
+
+/// Written to last line of file to specify the location of each fragment
+#[derive(Serialize, Deserialize, Debug, Clone)]
+struct Offset {
+    /// Index of the first byte in the template
+    start: usize,
+    /// The length of each fragment in the encoded template, including the separator
+    delta: Vec<usize>,
+}
+
+impl<F> SortedTemplate<F> {
+    /// Generate this template from arbitary text.
+    /// Will insert wherever the substring `magic` can be found.
+    /// Errors if it does not appear exactly once.
+    pub(crate) fn magic(template: &str, magic: &str) -> Result<Self, Error> {
+        let mut split = template.split(magic);
+        let before = split.next().ok_or(Error)?;
+        let after = split.next().ok_or(Error)?;
+        if split.next().is_some() {
+            return Err(Error);
+        }
+        Ok(Self::before_after(before, after))
+    }
+
+    /// Template will insert contents between `before` and `after`
+    pub(crate) fn before_after<S: ToString, T: ToString>(before: S, after: T) -> Self {
+        let before = before.to_string();
+        let after = after.to_string();
+        SortedTemplate { format: PhantomData, before, after, contents: Default::default() }
+    }
+}
+
+impl<F: FileFormat> SortedTemplate<F> {
+    /// Adds this text to the template
+    pub(crate) fn append(&mut self, insert: String) {
+        self.contents.insert(insert);
+    }
+}
+
+impl<F: FileFormat> fmt::Display for SortedTemplate<F> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut delta = Vec::default();
+        write!(f, "{}", self.before)?;
+        let contents: Vec<_> = self.contents.iter().collect();
+        let mut sep = "";
+        for content in contents {
+            delta.push(sep.len() + content.len());
+            write!(f, "{}{}", sep, content)?;
+            sep = F::SEPARATOR;
+        }
+        let offset = Offset { start: self.before.len(), delta };
+        let offset = serde_json::to_string(&offset).unwrap();
+        write!(f, "{}\n{}{}{}", self.after, F::COMMENT_START, offset, F::COMMENT_END)?;
+        Ok(())
+    }
+}
+
+fn checked_split_at(s: &str, index: usize) -> Option<(&str, &str)> {
+    s.is_char_boundary(index).then(|| s.split_at(index))
+}
+
+impl<F: FileFormat> FromStr for SortedTemplate<F> {
+    type Err = Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let (s, offset) = s.rsplit_once("\n").ok_or(Error)?;
+        let offset = offset.strip_prefix(F::COMMENT_START).ok_or(Error)?;
+        let offset = offset.strip_suffix(F::COMMENT_END).ok_or(Error)?;
+        let offset: Offset = serde_json::from_str(&offset).map_err(|_| Error)?;
+        let (before, mut s) = checked_split_at(s, offset.start).ok_or(Error)?;
+        let mut contents = BTreeSet::default();
+        let mut sep = "";
+        for &index in offset.delta.iter() {
+            let (content, rest) = checked_split_at(s, index).ok_or(Error)?;
+            s = rest;
+            let content = content.strip_prefix(sep).ok_or(Error)?;
+            contents.insert(content.to_string());
+            sep = F::SEPARATOR;
+        }
+        Ok(SortedTemplate {
+            format: PhantomData,
+            before: before.to_string(),
+            after: s.to_string(),
+            contents,
+        })
+    }
+}
+
+pub(crate) trait FileFormat {
+    const COMMENT_START: &'static str;
+    const COMMENT_END: &'static str;
+    const SEPARATOR: &'static str;
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct Html;
+
+impl FileFormat for Html {
+    const COMMENT_START: &'static str = "<!--";
+    const COMMENT_END: &'static str = "-->";
+    const SEPARATOR: &'static str = "";
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct Js;
+
+impl FileFormat for Js {
+    const COMMENT_START: &'static str = "//";
+    const COMMENT_END: &'static str = "";
+    const SEPARATOR: &'static str = ",";
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct Error;
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "invalid template")
+    }
+}

src/librustdoc/html/render/tests.rs

@@ -52,3 +52,274 @@ fn test_all_types_prints_header_once() {
 
     assert_eq!(1, buffer.into_inner().matches("List of all items").count());
 }
+
+mod sorted_json {
+    use super::super::sorted_json::*;
+
+    fn check(json: SortedJson, serialized: &str) {
+        assert_eq!(json.to_string(), serialized);
+        assert_eq!(serde_json::to_string(&json).unwrap(), serialized);
+
+        let json = json.to_string();
+        let json: SortedJson = serde_json::from_str(&json).unwrap();
+
+        assert_eq!(json.to_string(), serialized);
+        assert_eq!(serde_json::to_string(&json).unwrap(), serialized);
+
+        let json = serde_json::to_string(&json).unwrap();
+        let json: SortedJson = serde_json::from_str(&json).unwrap();
+
+        assert_eq!(json.to_string(), serialized);
+        assert_eq!(serde_json::to_string(&json).unwrap(), serialized);
+    }
+
+    #[test]
+    fn escape_json_number() {
+        let json = SortedJson::serialize(3);
+        let json = EscapedJson::from(json);
+        assert_eq!(format!("{json}"), "3");
+    }
+
+    #[test]
+    fn escape_json_single_quote() {
+        let json = SortedJson::serialize("he's");
+        let json = EscapedJson::from(json);
+        assert_eq!(format!("{json}"), r#""he\'s""#);
+    }
+
+    #[test]
+    fn escape_json_array() {
+        let json = SortedJson::serialize([1, 2, 3]);
+        let json = EscapedJson::from(json);
+        assert_eq!(format!("{json}"), r#"[1,2,3]"#);
+    }
+
+    #[test]
+    fn escape_json_string() {
+        let json = SortedJson::serialize(r#"he"llo"#);
+        let json = EscapedJson::from(json);
+        assert_eq!(format!("{json}"), r#""he\\\"llo""#);
+    }
+
+    #[test]
+    fn escape_json_string_escaped() {
+        let json = SortedJson::serialize(r#"he\"llo"#);
+        let json = EscapedJson::from(json);
+        assert_eq!(format!("{json}"), r#""he\\\\\\\"llo""#);
+    }
+
+    #[test]
+    fn escape_json_string_escaped_escaped() {
+        let json = SortedJson::serialize(r#"he\\"llo"#);
+        let json = EscapedJson::from(json);
+        assert_eq!(format!("{json}"), r#""he\\\\\\\\\\\"llo""#);
+    }
+
+    #[test]
+    fn number() {
+        let json = SortedJson::serialize(3);
+        let serialized = "3";
+        check(json, serialized);
+    }
+
+    #[test]
+    fn boolean() {
+        let json = SortedJson::serialize(true);
+        let serialized = "true";
+        check(json, serialized);
+    }
+
+    #[test]
+    fn string() {
+        let json = SortedJson::serialize("he\"llo");
+        let serialized = r#""he\"llo""#;
+        check(json, serialized);
+    }
+
+    #[test]
+    fn serialize_array() {
+        let json = SortedJson::serialize([3, 1, 2]);
+        let serialized = "[3,1,2]";
+        check(json, serialized);
+    }
+
+    #[test]
+    fn sorted_array() {
+        let items = ["c", "a", "b"];
+        let serialized = r#"["a","b","c"]"#;
+        let items: Vec<SortedJson> = items.into_iter().map(SortedJson::serialize).collect();
+        let json = SortedJson::array(items);
+        check(json, serialized);
+    }
+
+    #[test]
+    fn nested_array() {
+        let a = SortedJson::serialize(3);
+        let b = SortedJson::serialize(2);
+        let c = SortedJson::serialize(1);
+        let d = SortedJson::serialize([1, 3, 2]);
+        let json = SortedJson::array([a, b, c, d]);
+        let serialized = r#"[1,2,3,[1,3,2]]"#;
+        check(json, serialized);
+    }
+
+    #[test]
+    fn array_unsorted() {
+        let items = ["c", "a", "b"];
+        let serialized = r#"["c","a","b"]"#;
+        let items: Vec<SortedJson> = items.into_iter().map(SortedJson::serialize).collect();
+        let json = SortedJson::array_unsorted(items);
+        check(json, serialized);
+    }
+}
+
+mod sorted_template {
+    use super::super::sorted_template::*;
+    use std::str::FromStr;
+
+    fn is_comment_js(s: &str) -> bool {
+        s.starts_with("//")
+    }
+
+    fn is_comment_html(s: &str) -> bool {
+        // not correct but good enough for these tests
+        s.starts_with("<!--") && s.ends_with("-->")
+    }
+
+    #[test]
+    fn html_from_empty() {
+        let inserts = ["<p>hello</p>", "<p>kind</p>", "<p>hello</p>", "<p>world</p>"];
+        let mut template = SortedTemplate::<Html>::before_after("", "");
+        for insert in inserts {
+            template.append(insert.to_string());
+        }
+        let template = format!("{template}");
+        let (template, end) = template.rsplit_once("\n").unwrap();
+        assert_eq!(template, "<p>hello</p><p>kind</p><p>world</p>");
+        assert!(is_comment_html(end));
+        assert!(!end.contains("\n"));
+    }
+
+    #[test]
+    fn html_page() {
+        let inserts = ["<p>hello</p>", "<p>kind</p>", "<p>world</p>"];
+        let before = "<html><head></head><body>";
+        let after = "</body>";
+        let mut template = SortedTemplate::<Html>::before_after(before, after);
+        for insert in inserts {
+            template.append(insert.to_string());
+        }
+        let template = format!("{template}");
+        let (template, end) = template.rsplit_once("\n").unwrap();
+        assert_eq!(template, format!("{before}{}{after}", inserts.join("")));
+        assert!(is_comment_html(end));
+        assert!(!end.contains("\n"));
+    }
+
+    #[test]
+    fn js_from_empty() {
+        let inserts = ["1", "2", "2", "2", "3", "1"];
+        let mut template = SortedTemplate::<Js>::before_after("", "");
+        for insert in inserts {
+            template.append(insert.to_string());
+        }
+        let template = format!("{template}");
+        let (template, end) = template.rsplit_once("\n").unwrap();
+        assert_eq!(template, "1,2,3");
+        assert!(is_comment_js(end));
+        assert!(!end.contains("\n"));
+    }
+
+    #[test]
+    fn js_empty_array() {
+        let template = SortedTemplate::<Js>::before_after("[", "]");
+        let template = format!("{template}");
+        let (template, end) = template.rsplit_once("\n").unwrap();
+        assert_eq!(template, format!("[]"));
+        assert!(is_comment_js(end));
+        assert!(!end.contains("\n"));
+    }
+
+    #[test]
+    fn js_number_array() {
+        let inserts = ["1", "2", "3"];
+        let mut template = SortedTemplate::<Js>::before_after("[", "]");
+        for insert in inserts {
+            template.append(insert.to_string());
+        }
+        let template = format!("{template}");
+        let (template, end) = template.rsplit_once("\n").unwrap();
+        assert_eq!(template, format!("[1,2,3]"));
+        assert!(is_comment_js(end));
+        assert!(!end.contains("\n"));
+    }
+
+    #[test]
+    fn magic_js_number_array() {
+        let inserts = ["1", "1"];
+        let mut template = SortedTemplate::<Js>::magic("[#]", "#").unwrap();
+        for insert in inserts {
+            template.append(insert.to_string());
+        }
+        let template = format!("{template}");
+        let (template, end) = template.rsplit_once("\n").unwrap();
+        assert_eq!(template, format!("[1]"));
+        assert!(is_comment_js(end));
+        assert!(!end.contains("\n"));
+    }
+
+    #[test]
+    fn round_trip_js() {
+        let inserts = ["1", "2", "3"];
+        let mut template = SortedTemplate::<Js>::before_after("[", "]");
+        for insert in inserts {
+            template.append(insert.to_string());
+        }
+        let template1 = format!("{template}");
+        let mut template = SortedTemplate::<Js>::from_str(&template1).unwrap();
+        assert_eq!(template1, format!("{template}"));
+        template.append("4".to_string());
+        let template = format!("{template}");
+        let (template, end) = template.rsplit_once("\n").unwrap();
+        assert_eq!(template, "[1,2,3,4]");
+        assert!(is_comment_js(end));
+    }
+
+    #[test]
+    fn round_trip_html() {
+        let inserts = ["<p>hello</p>", "<p>kind</p>", "<p>world</p>", "<p>kind</p>"];
+        let before = "<html><head></head><body>";
+        let after = "</body>";
+        let mut template = SortedTemplate::<Html>::before_after(before, after);
+        template.append(inserts[0].to_string());
+        template.append(inserts[1].to_string());
+        let template = format!("{template}");
+        let mut template = SortedTemplate::<Html>::from_str(&template).unwrap();
+        template.append(inserts[2].to_string());
+        let template = format!("{template}");
+        let (template, end) = template.rsplit_once("\n").unwrap();
+        assert_eq!(template, format!("{before}<p>hello</p><p>kind</p><p>world</p>{after}"));
+        assert!(is_comment_html(end));
+    }
+
+    #[test]
+    fn blank_js() {
+        let inserts = ["1", "2", "3"];
+        let template = SortedTemplate::<Js>::before_after("", "");
+        let template = format!("{template}");
+        let (t, _) = template.rsplit_once("\n").unwrap();
+        assert_eq!(t, "");
+        let mut template = SortedTemplate::<Js>::from_str(&template).unwrap();
+        for insert in inserts {
+            template.append(insert.to_string());
+        }
+        let template1 = format!("{template}");
+        let mut template = SortedTemplate::<Js>::from_str(&template1).unwrap();
+        assert_eq!(template1, format!("{template}"));
+        template.append("4".to_string());
+        let template = format!("{template}");
+        let (template, end) = template.rsplit_once("\n").unwrap();
+        assert_eq!(template, "1,2,3,4");
+        assert!(is_comment_js(end));
+    }
+}