Rollup merge of #148234 - notriddle:doc-cci, r=GuillaumeGomez

rustdoc: make mergeable crate info more usable

Part of https://github.com/rust-lang/rust/issues/130676

Adds documentation and a feature change aimed at making this usable without breaking backwards compat.

cc ``@EtomicBomb``

src/doc/rustdoc/src/unstable-features.md

@@ -197,6 +197,37 @@ themselves marked as unstable. To use any of these options, pass `-Z unstable-op
 the flag in question to Rustdoc on the command-line. To do this from Cargo, you can either use the
 `RUSTDOCFLAGS` environment variable or the `cargo rustdoc` command.
 
+### `--merge`, `--parts-out-dir`, and `--include-parts-dir`
+
+These options control how rustdoc handles files that combine data from multiple crates.
+
+By default, they act like `--merge=shared` is set, and `--parts-out-dir` and `--include-parts-dir`
+are turned off. The `--merge=shared` mode causes rustdoc to load the existing data in the out-dir,
+combine the new crate data into it, and write the result. This is very easy to use in scripts that
+manually invoke rustdoc, but it's also slow, because it performs O(crates) work on
+every crate, meaning it performs O(crates<sup>2</sup>) work.
+
+```console
+$ rustdoc crate1.rs --out-dir=doc
+$ cat doc/search.index/crateNames/*
+rd_("fcrate1")
+$ rustdoc crate2.rs --out-dir=doc
+$ cat doc/search.index/crateNames/*
+rd_("fcrate1fcrate2")
+```
+
+To delay shared-data merging until the end of a build, so that you only have to perform O(crates)
+work, use `--merge=none` on every crate except the last one, which will use `--merge=finalize`.
+
+```console
+$ rustdoc +nightly crate1.rs --merge=none --parts-out-dir=crate1.d -Zunstable-options
+$ cat doc/search.index/crateNames/*
+cat: 'doc/search.index/crateNames/*': No such file or directory
+$ rustdoc +nightly crate2.rs --merge=finalize --include-parts-dir=crate1.d -Zunstable-options
+$ cat doc/search.index/crateNames/*
+rd_("fcrate1fcrate2")
+```
+
 ### `--document-hidden-items`: Show items that are `#[doc(hidden)]`
 <span id="document-hidden-items"></span>
 

src/librustdoc/config.rs

@@ -978,15 +978,16 @@ fn parse_extern_html_roots(
     Ok(externs)
 }
 
-/// Path directly to crate-info file.
+/// Path directly to crate-info directory.
 ///
-/// For example, `/home/user/project/target/doc.parts/<crate>/crate-info`.
+/// For example, `/home/user/project/target/doc.parts`.
+/// Each crate has its info stored in a file called `CRATENAME.json`.
 #[derive(Clone, Debug)]
 pub(crate) struct PathToParts(pub(crate) PathBuf);
 
 impl PathToParts {
     fn from_flag(path: String) -> Result<PathToParts, String> {
-        let mut path = PathBuf::from(path);
+        let path = PathBuf::from(path);
         // check here is for diagnostics
         if path.exists() && !path.is_dir() {
             Err(format!(
@@ -995,20 +996,22 @@ impl PathToParts {
             ))
         } else {
             // if it doesn't exist, we'll create it. worry about that in write_shared
-            path.push("crate-info");
             Ok(PathToParts(path))
         }
     }
 }
 
-/// Reports error if --include-parts-dir / crate-info is not a file
+/// Reports error if --include-parts-dir is not a directory
 fn parse_include_parts_dir(m: &getopts::Matches) -> Result<Vec<PathToParts>, String> {
     let mut ret = Vec::new();
     for p in m.opt_strs("include-parts-dir") {
         let p = PathToParts::from_flag(p)?;
         // this is just for diagnostic
-        if !p.0.is_file() {
-            return Err(format!("--include-parts-dir expected {} to be a file", p.0.display()));
+        if !p.0.is_dir() {
+            return Err(format!(
+                "--include-parts-dir expected {} to be a directory",
+                p.0.display()
+            ));
         }
         ret.push(p);
     }

src/librustdoc/html/render/write_shared.rs

@@ -14,7 +14,7 @@
 //!    or contains "invocation-specific".
 
 use std::cell::RefCell;
-use std::ffi::OsString;
+use std::ffi::{OsStr, OsString};
 use std::fs::File;
 use std::io::{self, Write as _};
 use std::iter::once;
@@ -84,9 +84,11 @@ pub(crate) fn write_shared(
     };
 
     if let Some(parts_out_dir) = &opt.parts_out_dir {
-        create_parents(&parts_out_dir.0)?;
+        let mut parts_out_file = parts_out_dir.0.clone();
+        parts_out_file.push(&format!("{crate_name}.json"));
+        create_parents(&parts_out_file)?;
         try_err!(
-            fs::write(&parts_out_dir.0, serde_json::to_string(&info).unwrap()),
+            fs::write(&parts_out_file, serde_json::to_string(&info).unwrap()),
             &parts_out_dir.0
         );
     }
@@ -238,13 +240,25 @@ impl CrateInfo {
     pub(crate) fn read_many(parts_paths: &[PathToParts]) -> Result<Vec<Self>, Error> {
         parts_paths
             .iter()
-            .map(|parts_path| {
-                let path = &parts_path.0;
-                let parts = try_err!(fs::read(path), &path);
-                let parts: CrateInfo = try_err!(serde_json::from_slice(&parts), &path);
-                Ok::<_, Error>(parts)
+            .fold(Ok(Vec::new()), |acc, parts_path| {
+                let mut acc = acc?;
+                let dir = &parts_path.0;
+                acc.append(&mut try_err!(std::fs::read_dir(dir), dir.as_path())
+                    .filter_map(|file| {
+                        let to_crate_info = |file: Result<std::fs::DirEntry, std::io::Error>| -> Result<Option<CrateInfo>, Error> {
+                            let file = try_err!(file, dir.as_path());
+                            if file.path().extension() != Some(OsStr::new("json")) {
+                                return Ok(None);
+                            }
+                            let parts = try_err!(fs::read(file.path()), file.path());
+                            let parts: CrateInfo = try_err!(serde_json::from_slice(&parts), file.path());
+                            Ok(Some(parts))
+                        };
+                        to_crate_info(file).transpose()
+                    })
+                    .collect::<Result<Vec<CrateInfo>, Error>>()?);
+                Ok(acc)
             })
-            .collect::<Result<Vec<CrateInfo>, Error>>()
     }
 }
 

tests/run-make/rustdoc-merge-directory/dep1.rs

@@ -0,0 +1,4 @@
+//@ hasraw crates.js 'dep1'
+//@ hasraw search.index/name/*.js 'Dep1'
+//@ has dep1/index.html
+pub struct Dep1;

tests/run-make/rustdoc-merge-directory/dep2.rs

@@ -0,0 +1,4 @@
+//@ hasraw crates.js 'dep1'
+//@ hasraw search.index/name/*.js 'Dep1'
+//@ has dep2/index.html
+pub struct Dep2;

tests/run-make/rustdoc-merge-directory/dep_missing.rs

@@ -0,0 +1,4 @@
+//@ !hasraw crates.js 'dep_missing'
+//@ !hasraw search.index/name/*.js 'DepMissing'
+//@ has dep_missing/index.html
+pub struct DepMissing;

tests/run-make/rustdoc-merge-directory/rmake.rs

@@ -0,0 +1,46 @@
+// Running --merge=finalize without an input crate root should not trigger ICE.
+// Issue: https://github.com/rust-lang/rust/issues/146646
+
+//@ needs-target-std
+
+use run_make_support::{htmldocck, path, rustdoc};
+
+fn main() {
+    let out_dir = path("out");
+    let merged_dir = path("merged");
+    let parts_out_dir = path("parts");
+
+    rustdoc()
+        .input("dep1.rs")
+        .out_dir(&out_dir)
+        .arg("-Zunstable-options")
+        .arg(format!("--parts-out-dir={}", parts_out_dir.display()))
+        .arg("--merge=none")
+        .run();
+    assert!(parts_out_dir.join("dep1.json").exists());
+
+    rustdoc()
+        .input("dep2.rs")
+        .out_dir(&out_dir)
+        .arg("-Zunstable-options")
+        .arg(format!("--parts-out-dir={}", parts_out_dir.display()))
+        .arg("--merge=none")
+        .run();
+    assert!(parts_out_dir.join("dep2.json").exists());
+
+    // dep_missing is different, because --parts-out-dir is not supplied
+    rustdoc().input("dep_missing.rs").out_dir(&out_dir).run();
+    assert!(parts_out_dir.join("dep2.json").exists());
+
+    let output = rustdoc()
+        .arg("-Zunstable-options")
+        .out_dir(&out_dir)
+        .arg(format!("--include-parts-dir={}", parts_out_dir.display()))
+        .arg("--merge=finalize")
+        .run();
+    output.assert_stderr_not_contains("error: the compiler unexpectedly panicked. this is a bug.");
+
+    htmldocck().arg(&out_dir).arg("dep1.rs").run();
+    htmldocck().arg(&out_dir).arg("dep2.rs").run();
+    htmldocck().arg(out_dir).arg("dep_missing.rs").run();
+}

tests/run-make/rustdoc-merge-no-input-finalize/rmake.rs

@@ -16,7 +16,7 @@ fn main() {
         .arg(format!("--parts-out-dir={}", parts_out_dir.display()))
         .arg("--merge=none")
         .run();
-    assert!(parts_out_dir.join("crate-info").exists());
+    assert!(parts_out_dir.join("sierra.json").exists());
 
     let output = rustdoc()
         .arg("-Zunstable-options")