Add rlib digest to identify Rust object files

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -58,7 +58,7 @@ use super::command::Command;
 use super::linker::{self, Linker};
 use super::metadata::{MetadataPosition, create_wrapper_file};
 use super::rpath::{self, RPathConfig};
-use super::{apple, versioned_llvm_target};
+use super::{apple, rlib_digest, versioned_llvm_target};
 use crate::base::needs_allocator_shim_for_linking;
 use crate::{
     CodegenLintLevels, CompiledModule, CompiledModules, CrateInfo, NativeLib, errors,
@@ -329,8 +329,13 @@ fn link_rlib<'a>(
         RlibFlavor::StaticlibBase => None,
     };
 
+    let mut rust_object_files: Vec<String> = Vec::new();
+
     for m in &compiled_modules.modules {
         if let Some(obj) = m.object.as_ref() {
+            if let Some(name) = obj.file_name().and_then(|n| n.to_str()) {
+                rust_object_files.push(name.to_string());
+            }
             ab.add_file(obj);
         }
 
@@ -442,6 +447,17 @@ fn link_rlib<'a>(
         ab.add_file(&lib)
     }
 
+    // Add the rlib digest as the very last member. This records which archive
+    // members are Rust object files, replacing filename-based heuristics.
+    if matches!(flavor, RlibFlavor::Normal) {
+        let digest = rlib_digest::RlibDigest { rust_object_files };
+        let digest_data = digest.encode();
+        let (wrapper, _) =
+            create_wrapper_file(sess, rlib_digest::SECTION.to_string(), &digest_data);
+        let digest_file = emit_wrapper_file(sess, &wrapper, tmpdir.as_ref(), rlib_digest::FILENAME);
+        ab.add_file(&digest_file);
+    }
+
     ab
 }
 
@@ -486,6 +502,7 @@ fn link_staticlib(
         let relevant_libs: FxIndexSet<_> = relevant.filter_map(|lib| lib.filename).collect();
 
         let bundled_libs: FxIndexSet<_> = native_libs.filter_map(|lib| lib.filename).collect();
+        let digest = rlib_digest::read(path);
         ab.add_archive(
             path,
             Box::new(move |fname: &str| {
@@ -494,11 +511,22 @@ fn link_staticlib(
                     return true;
                 }
 
-                // Don't include Rust objects if LTO is enabled
-                if lto && looks_like_rust_object_file(fname) {
+                // Ignore the rlib digest file.
+                if fname == rlib_digest::FILENAME {
                     return true;
                 }
 
+                // Don't include Rust objects if LTO is enabled.
+                if lto {
+                    let is_rust_object = match &digest {
+                        Some(d) => d.rust_object_files.iter().any(|f| f == fname),
+                        None => looks_like_rust_object_file(fname),
+                    };
+                    if is_rust_object {
+                        return true;
+                    }
+                }
+
                 // Skip objects for bundled libs.
                 if bundled_libs.contains(&Symbol::intern(fname)) {
                     return true;
@@ -3157,6 +3185,10 @@ fn add_static_crate(
                     return true;
                 }
 
+                if f == rlib_digest::FILENAME {
+                    return true;
+                }
+
                 let canonical = f.replace('-', "_");
 
                 let is_rust_object =

compiler/rustc_codegen_ssa/src/back/mod.rs

@@ -9,6 +9,7 @@ pub mod link;
 pub(crate) mod linker;
 pub mod lto;
 pub mod metadata;
+pub(crate) mod rlib_digest;
 pub(crate) mod rpath;
 pub mod symbol_export;
 pub mod write;

compiler/rustc_codegen_ssa/src/back/rlib_digest.rs

@@ -0,0 +1,60 @@
+//! Late-metadata archive member that lists which rlib entries are Rust object files,
+//! replacing the `looks_like_rust_object_file` filename heuristic.
+//! See <https://github.com/rust-lang/rust/issues/138243>.
+
+use std::fs::File;
+use std::path::Path;
+
+use object::read::archive::ArchiveFile;
+use rustc_data_structures::memmap::Mmap;
+use rustc_serialize::opaque::mem_encoder::MemEncoder;
+use rustc_serialize::opaque::{MAGIC_END_BYTES, MemDecoder};
+use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
+
+use super::metadata::search_for_section;
+
+pub(crate) const FILENAME: &str = "lib.rlib-digest";
+pub(crate) const SECTION: &str = ".rlib-digest";
+const VERSION: u8 = 1;
+
+pub(crate) struct RlibDigest {
+    pub rust_object_files: Vec<String>,
+}
+
+impl RlibDigest {
+    pub(crate) fn encode(&self) -> Vec<u8> {
+        let mut encoder = MemEncoder::new();
+        encoder.emit_u8(VERSION);
+        self.rust_object_files.encode(&mut encoder);
+        let mut data = encoder.finish();
+        data.extend_from_slice(MAGIC_END_BYTES);
+        data
+    }
+
+    pub(crate) fn decode(data: &[u8]) -> Option<RlibDigest> {
+        let mut decoder = MemDecoder::new(data, 0).ok()?;
+        let version = decoder.read_u8();
+        if version != VERSION {
+            return None;
+        }
+        let rust_object_files = Vec::<String>::decode(&mut decoder);
+        Some(RlibDigest { rust_object_files })
+    }
+}
+
+/// Returns `None` for rlibs without a digest (for older compilers).
+pub(crate) fn read(rlib_path: &Path) -> Option<RlibDigest> {
+    let file = File::open(rlib_path).ok()?;
+    let mmap = unsafe { Mmap::map(file).ok()? };
+    let archive = ArchiveFile::parse(&*mmap).ok()?;
+
+    for entry in archive.members() {
+        let entry = entry.ok()?;
+        if entry.name() == FILENAME.as_bytes() {
+            let data = entry.data(&*mmap).ok()?;
+            let section_data = search_for_section(rlib_path, data, SECTION).ok()?;
+            return RlibDigest::decode(section_data);
+        }
+    }
+    None
+}