Rollup merge of #151137 - osiewicz:151090-checksum-freshness-binary-files, r=jdonszelmann

checksum-freshness: Fix invalid checksum calculation for binary files

Admittedly this is not the cleanest way to achieve this, but SourceMap is quite intertwined with source files being represented as Strings.
Tracking issue: https://github.com/rust-lang/cargo/issues/14136
Closes: rust-lang/rust#151090

compiler/rustc_builtin_macros/src/source_util.rs

@@ -275,7 +275,15 @@ fn load_binary_file(
         }
     };
     match cx.source_map().load_binary_file(&resolved_path) {
-        Ok(data) => Ok(data),
+        Ok(data) => {
+            cx.sess
+                .psess
+                .file_depinfo
+                .borrow_mut()
+                .insert(Symbol::intern(&resolved_path.to_string_lossy()));
+
+            Ok(data)
+        }
         Err(io_err) => {
             let mut err = cx.dcx().struct_span_err(
                 macro_span,

compiler/rustc_interface/src/passes.rs

@@ -9,6 +9,7 @@ use rustc_ast::{self as ast, CRATE_NODE_ID};
 use rustc_attr_parsing::{AttributeParser, Early, ShouldEmit};
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::{CodegenResults, CrateInfo};
+use rustc_data_structures::indexmap::IndexMap;
 use rustc_data_structures::jobserver::Proxy;
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{AppendOnlyIndexVec, FreezeLock, WorkerLocal};
@@ -584,7 +585,7 @@ fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[P
     let result: io::Result<()> = try {
         // Build a list of files used to compile the output and
         // write Makefile-compatible dependency rules
-        let mut files: Vec<(String, u64, Option<SourceFileHash>)> = sess
+        let mut files: IndexMap<String, (u64, Option<SourceFileHash>)> = sess
             .source_map()
             .files()
             .iter()
@@ -593,10 +594,12 @@ fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[P
             .map(|fmap| {
                 (
                     escape_dep_filename(&fmap.name.prefer_local_unconditionally().to_string()),
-                    // This needs to be unnormalized,
-                    // as external tools wouldn't know how rustc normalizes them
-                    fmap.unnormalized_source_len as u64,
-                    fmap.checksum_hash,
+                    (
+                        // This needs to be unnormalized,
+                        // as external tools wouldn't know how rustc normalizes them
+                        fmap.unnormalized_source_len as u64,
+                        fmap.checksum_hash,
+                    ),
                 )
             })
             .collect();
@@ -614,7 +617,7 @@ fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[P
         fn hash_iter_files<P: AsRef<Path>>(
             it: impl Iterator<Item = P>,
             checksum_hash_algo: Option<SourceFileHashAlgorithm>,
-        ) -> impl Iterator<Item = (P, u64, Option<SourceFileHash>)> {
+        ) -> impl Iterator<Item = (P, (u64, Option<SourceFileHash>))> {
             it.map(move |path| {
                 match checksum_hash_algo.and_then(|algo| {
                     fs::File::open(path.as_ref())
@@ -630,8 +633,8 @@ fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[P
                         })
                         .ok()
                 }) {
-                    Some((file_len, checksum)) => (path, file_len, Some(checksum)),
-                    None => (path, 0, None),
+                    Some((file_len, checksum)) => (path, (file_len, Some(checksum))),
+                    None => (path, (0, None)),
                 }
             })
         }
@@ -705,18 +708,14 @@ fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[P
                     file,
                     "{}: {}\n",
                     path.display(),
-                    files
-                        .iter()
-                        .map(|(path, _file_len, _checksum_hash_algo)| path.as_str())
-                        .intersperse(" ")
-                        .collect::<String>()
+                    files.keys().map(String::as_str).intersperse(" ").collect::<String>()
                 )?;
             }
 
             // Emit a fake target for each input file to the compilation. This
             // prevents `make` from spitting out an error if a file is later
             // deleted. For more info see #28735
-            for (path, _file_len, _checksum_hash_algo) in &files {
+            for path in files.keys() {
                 writeln!(file, "{path}:")?;
             }
 
@@ -745,7 +744,7 @@ fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[P
             if sess.opts.unstable_opts.checksum_hash_algorithm().is_some() {
                 files
                     .iter()
-                    .filter_map(|(path, file_len, hash_algo)| {
+                    .filter_map(|(path, (file_len, hash_algo))| {
                         hash_algo.map(|hash_algo| (path, file_len, hash_algo))
                     })
                     .try_for_each(|(path, file_len, checksum_hash)| {

tests/run-make/checksum-freshness/binary_file

@@ -0,0 +1 @@
+binary<EFBFBD>

tests/run-make/checksum-freshness/expected.d

@@ -1,6 +1,8 @@
-lib.d: lib.rs foo.rs
+lib.d: lib.rs foo.rs binary_file
 
 lib.rs:
 foo.rs:
-# checksum:blake3=94af75ee4ed805434484c3de51c9025278e5c3ada2315e2592052e102168a503 file_len:120 lib.rs
+binary_file:
+# checksum:blake3=4ac56f3f877798fb762d714c7bcb72e70133f4cc585f80dbd99c07755ae2c7f6 file_len:222 lib.rs
 # checksum:blake3=2720e17bfda4f3b2a5c96bb61b7e76ed8ebe3359b34128c0e5d8032c090a4f1a file_len:119 foo.rs
+# checksum:blake3=119a5db8711914922c5b1c1908be4958175c5afa95c08888de594725329b5439 file_len:7 binary_file

tests/run-make/checksum-freshness/lib.rs

@@ -1,7 +1,8 @@
 // A basic library to be used in tests with no real purpose.
 
 mod foo;
-
+// Binary file with invalid UTF-8 sequence.
+static BINARY_FILE: &[u8] = include_bytes!("binary_file");
 pub fn sum(a: i32, b: i32) -> i32 {
     a + b
 }