Auto merge of #145093 - nikic:dead-on-return, r=nnethercote

Set dead_on_return attribute for indirect arguments

Set the dead_on_return attribute (added in LLVM 21) for arguments that are passed indirectly, but not byval.

This indicates that the value of the argument on return does not matter, enabling additional dead store elimination.

From LangRef:

> This attribute indicates that the memory pointed to by the argument is dead upon function return, both upon normal return and if the calls unwinds, meaning that the caller will not depend on its contents. Stores that would be observable either on the return path or on the unwind path may be elided.
>
> Specifically, the behavior is as-if any memory written through the pointer during the execution of the function is overwritten with a poison value upon function return. The caller may access the memory, but any load not preceded by a store will return poison.
>
> This attribute does not imply aliasing properties. For pointer arguments that do not alias other memory locations, noalias attribute may be used in conjunction. Conversely, this attribute always implies dead_on_unwind.
>
> This attribute cannot be applied to return values.

This fixes parts of https://github.com/rust-lang/rust/issues/96497.

compiler/rustc_codegen_llvm/src/abi.rs

@@ -24,6 +24,7 @@ use crate::attributes::{self, llfn_attrs_from_instance};
 use crate::builder::Builder;
 use crate::context::CodegenCx;
 use crate::llvm::{self, Attribute, AttributePlace};
+use crate::llvm_util;
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
@@ -500,7 +501,16 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     }
                 }
                 PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
-                    apply(attrs);
+                    let i = apply(attrs);
+                    if cx.sess().opts.optimize != config::OptLevel::No
+                        && llvm_util::get_version() >= (21, 0, 0)
+                    {
+                        attributes::apply_to_llfn(
+                            llfn,
+                            llvm::AttributePlace::Argument(i),
+                            &[llvm::AttributeKind::DeadOnReturn.create_attr(cx.llcx)],
+                        );
+                    }
                 }
                 PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
                     assert!(!on_stack);

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -249,6 +249,7 @@ pub(crate) enum AttributeKind {
     FnRetThunkExtern = 41,
     Writable = 42,
     DeadOnUnwind = 43,
+    DeadOnReturn = 44,
 }
 
 /// LLVMIntPredicate

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -277,6 +277,7 @@ enum class LLVMRustAttributeKind {
   FnRetThunkExtern = 41,
   Writable = 42,
   DeadOnUnwind = 43,
+  DeadOnReturn = 44,
 };
 
 static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) {
@@ -369,6 +370,12 @@ static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) {
     return Attribute::Writable;
   case LLVMRustAttributeKind::DeadOnUnwind:
     return Attribute::DeadOnUnwind;
+  case LLVMRustAttributeKind::DeadOnReturn:
+#if LLVM_VERSION_GE(21, 0)
+    return Attribute::DeadOnReturn;
+#else
+    report_fatal_error("DeadOnReturn attribute requires LLVM 21 or later");
+#endif
   }
   report_fatal_error("bad LLVMRustAttributeKind");
 }

tests/codegen-llvm/addr-of-mutate.rs

@@ -5,7 +5,7 @@
 // Test for the absence of `readonly` on the argument when it is mutated via `&raw const`.
 // See <https://github.com/rust-lang/rust/issues/111502>.
 
-// CHECK: i8 @foo(ptr noalias{{( nocapture)?}} noundef align 1{{( captures\(none\))?}} dereferenceable(128) %x)
+// CHECK: i8 @foo(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef align 1{{( captures\(none\))?}} dereferenceable(128) %x)
 #[no_mangle]
 pub fn foo(x: [u8; 128]) -> u8 {
     let ptr = core::ptr::addr_of!(x).cast_mut();
@@ -15,7 +15,7 @@ pub fn foo(x: [u8; 128]) -> u8 {
     x[0]
 }
 
-// CHECK: i1 @second(ptr noalias{{( nocapture)?}} noundef align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b)
+// CHECK: i1 @second(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b)
 #[no_mangle]
 pub unsafe fn second(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool {
     let b_bool_ptr = core::ptr::addr_of!(a_ptr_and_b.1.1).cast_mut();
@@ -24,7 +24,7 @@ pub unsafe fn second(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool {
 }
 
 // If going through a deref (and there are no other mutating accesses), then `readonly` is fine.
-// CHECK: i1 @third(ptr noalias{{( nocapture)?}} noundef readonly align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b)
+// CHECK: i1 @third(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef readonly align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b)
 #[no_mangle]
 pub unsafe fn third(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool {
     let b_bool_ptr = core::ptr::addr_of!((*a_ptr_and_b.0).1).cast_mut();

tests/codegen-llvm/dead_on_return.rs

@@ -0,0 +1,31 @@
+//@ compile-flags: -C opt-level=3
+//@ min-llvm-version: 21
+
+#![crate_type = "lib"]
+#![allow(unused_assignments, unused_variables)]
+
+// Check that the old string is deallocated, but a new one is not initialized.
+#[unsafe(no_mangle)]
+pub fn test_str_new(mut s: String) {
+    // CHECK-LABEL: @test_str_new
+    // CHECK: __rust_dealloc
+    // CHECK-NOT: store
+    s = String::new();
+}
+
+#[unsafe(no_mangle)]
+pub fn test_str_take(mut x: String) -> String {
+    // CHECK-LABEL: @test_str_take
+    // CHECK-NEXT: {{.*}}:
+    // CHECK-NEXT: call void @llvm.memcpy
+    // CHECK-NEXT: ret
+    core::mem::take(&mut x)
+}
+
+#[unsafe(no_mangle)]
+pub fn test_array_store(mut x: [u32; 100]) {
+    // CHECK-LABEL: @test_array_store
+    // CHECK-NEXT: {{.*}}:
+    // CHECK-NEXT: ret
+    x[0] = 1;
+}

tests/codegen-llvm/function-arguments.rs

@@ -134,7 +134,7 @@ pub fn mutable_notunpin_borrow(_: &mut NotUnpin) {}
 #[no_mangle]
 pub fn notunpin_borrow(_: &NotUnpin) {}
 
-// CHECK: @indirect_struct(ptr noalias{{( nocapture)?}} noundef readonly align 4{{( captures\(none\))?}} dereferenceable(32) %_1)
+// CHECK: @indirect_struct(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef readonly align 4{{( captures\(none\))?}} dereferenceable(32) %_1)
 #[no_mangle]
 pub fn indirect_struct(_: S) {}
 

tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs

@@ -256,7 +256,7 @@ pub struct IntDoubleInt {
     c: i32,
 }
 
-// CHECK: define void @f_int_double_int_s_arg(ptr noalias{{( nocapture)?}} noundef align 8{{( captures\(none\))?}} dereferenceable(24) %a)
+// CHECK: define void @f_int_double_int_s_arg(ptr{{( dead_on_return)?}} noalias{{( nocapture)?}} noundef align 8{{( captures\(none\))?}} dereferenceable(24) %a)
 #[no_mangle]
 pub extern "C" fn f_int_double_int_s_arg(a: IntDoubleInt) {}