error on non-rustic ABIs using unsized parameters

compiler/rustc_abi/src/canon_abi.rs

@@ -51,6 +51,20 @@ pub enum CanonAbi {
     X86(X86Call),
 }
 
+impl CanonAbi {
+    pub fn is_rustic_abi(self) -> bool {
+        match self {
+            CanonAbi::Rust | CanonAbi::RustCold => true,
+            CanonAbi::C
+            | CanonAbi::Custom
+            | CanonAbi::Arm(_)
+            | CanonAbi::GpuKernel
+            | CanonAbi::Interrupt(_)
+            | CanonAbi::X86(_) => false,
+        }
+    }
+}
+
 impl fmt::Display for CanonAbi {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         // convert to the ExternAbi that *shares a string* with this CanonAbi.

compiler/rustc_monomorphize/messages.ftl

@@ -12,6 +12,17 @@ monomorphize_abi_error_disabled_vector_type =
   } here
   .help = consider enabling it globally (`-C target-feature=+{$required_feature}`) or locally (`#[target_feature(enable="{$required_feature}")]`)
 
+monomorphize_abi_error_unsupported_unsized_parameter =
+  this function {$is_call ->
+    [true] call
+    *[false] definition
+  } uses unsized type `{$ty}` which is not supported with the chosen ABI
+  .label = function {$is_call ->
+    [true] called
+    *[false] defined
+  } here
+  .help = only rustic ABIs support unsized parameters
+
 monomorphize_abi_error_unsupported_vector_type =
   this function {$is_call ->
     [true] call

compiler/rustc_monomorphize/src/errors.rs

@@ -80,6 +80,18 @@ pub(crate) struct AbiErrorDisabledVectorType<'a> {
     pub is_call: bool,
 }
 
+#[derive(Diagnostic)]
+#[diag(monomorphize_abi_error_unsupported_unsized_parameter)]
+#[help]
+pub(crate) struct AbiErrorUnsupportedUnsizedParameter<'a> {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub ty: Ty<'a>,
+    /// Whether this is a problem at a call site or at a declaration.
+    pub is_call: bool,
+}
+
 #[derive(Diagnostic)]
 #[diag(monomorphize_abi_error_unsupported_vector_type)]
 pub(crate) struct AbiErrorUnsupportedVectorType<'a> {

compiler/rustc_monomorphize/src/mono_checks/abi_check.rs

@@ -78,8 +78,37 @@ fn do_check_simd_vector_abi<'tcx>(
     }
 }
 
-/// Checks that the ABI of a given instance of a function does not contain vector-passed arguments
-/// or return values for which the corresponding target feature is not enabled.
+/// Emit an error when a non-rustic ABI has unsized parameters.
+/// Unsized types do not have a stable layout, so should not be used with stable ABIs.
+/// `is_call` indicates whether this is a call-site check or a definition-site check;
+/// this is only relevant for the wording in the emitted error.
+fn do_check_unsized_params<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+    is_call: bool,
+    loc: impl Fn() -> (Span, HirId),
+) {
+    // Unsized parameters are allowed with the (unstable) "Rust" (and similar) ABIs.
+    if fn_abi.conv.is_rustic_abi() {
+        return;
+    }
+
+    for arg_abi in fn_abi.args.iter() {
+        if !arg_abi.layout.layout.is_sized() {
+            let (span, _hir_id) = loc();
+            tcx.dcx().emit_err(errors::AbiErrorUnsupportedUnsizedParameter {
+                span,
+                ty: arg_abi.layout.ty,
+                is_call,
+            });
+        }
+    }
+}
+
+/// Checks the ABI of an Instance, emitting an error when:
+///
+/// - a non-rustic ABI uses unsized parameters
+/// - the signature requires target features that are not enabled
 fn check_instance_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
     let typing_env = ty::TypingEnv::fully_monomorphized();
     let Ok(abi) = tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty())))
@@ -102,11 +131,14 @@ fn check_instance_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
             def_id.as_local().map(|did| tcx.local_def_id_to_hir_id(did)).unwrap_or(CRATE_HIR_ID),
         )
     };
+    do_check_unsized_params(tcx, abi, /*is_call*/ false, loc);
     do_check_simd_vector_abi(tcx, abi, instance.def_id(), /*is_call*/ false, loc);
 }
 
-/// Checks that a call expression does not try to pass a vector-passed argument which requires a
-/// target feature that the caller does not have, as doing so causes UB because of ABI mismatch.
+/// Check the ABI at a call site, emitting an error when:
+///
+/// - a non-rustic ABI uses unsized parameters
+/// - the signature requires target features that are not enabled
 fn check_call_site_abi<'tcx>(
     tcx: TyCtxt<'tcx>,
     callee: Ty<'tcx>,
@@ -140,6 +172,7 @@ fn check_call_site_abi<'tcx>(
         // ABI failed to compute; this will not get through codegen.
         return;
     };
+    do_check_unsized_params(tcx, callee_abi, /*is_call*/ true, loc);
     do_check_simd_vector_abi(tcx, callee_abi, caller.def_id(), /*is_call*/ true, loc);
 }
 

tests/ui/abi/non-rustic-unsized.rs

@@ -0,0 +1,66 @@
+//@ add-minicore
+//@ build-fail
+#![no_core]
+#![crate_type = "lib"]
+#![feature(no_core, unsized_fn_params)]
+#![allow(improper_ctypes_definitions, improper_ctypes)]
+
+extern crate minicore;
+use minicore::*;
+
+fn rust(_: [u8]) {}
+extern "C" fn c(_: [u8]) {}
+//~^ ERROR this function definition uses unsized type `[u8]` which is not supported with the chosen ABI
+extern "system" fn system(_: [u8]) {}
+//~^ ERROR this function definition uses unsized type `[u8]` which is not supported with the chosen ABI
+
+#[repr(C)]
+struct CustomUnsized {
+    a: i64,
+    b: [u8],
+}
+
+extern "C" fn c_custom_unsized(x: CustomUnsized) {}
+//~^ ERROR this function definition uses unsized type `CustomUnsized` which is not supported with the chosen ABI
+
+#[unsafe(no_mangle)]
+fn entry(x: [u8], y: [u8], z: [u8], w: CustomUnsized) {
+    rust(x);
+    c(y);
+    //~^ ERROR this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+    system(z);
+    //~^ ERROR this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+    c_custom_unsized(w);
+    //~^ ERROR this function call uses unsized type `CustomUnsized` which is not supported with the chosen ABI
+}
+
+#[unsafe(no_mangle)]
+fn test_fn_ptr(rust: extern "Rust" fn(_: [u8]), c: extern "C" fn(_: [u8]), x: [u8], y: [u8]) {
+    rust(x);
+    c(y);
+    //~^ ERROR this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+}
+
+#[unsafe(no_mangle)]
+fn test_extern(x: [u8], y: [u8]) {
+    unsafe extern "Rust" {
+        safe fn rust(_: [u8]);
+    }
+
+    unsafe extern "system" {
+        safe fn system(_: [u8]);
+    }
+
+    rust(x);
+    system(y);
+    //~^ ERROR this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+}
+
+extern "C" fn c_polymorphic<T: ?Sized>(_: T) {}
+//~^ ERROR this function definition uses unsized type `[u8]` which is not supported with the chosen ABI
+
+#[unsafe(no_mangle)]
+fn test_polymorphic(x: [u8]) {
+    c_polymorphic(x);
+    //~^ ERROR this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+}

tests/ui/abi/non-rustic-unsized.stderr

@@ -0,0 +1,88 @@
+error: this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+  --> $DIR/non-rustic-unsized.rs:29:5
+   |
+LL |     c(y);
+   |     ^^^^ function called here
+   |
+   = help: only rustic ABIs support unsized parameters
+
+error: this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+  --> $DIR/non-rustic-unsized.rs:31:5
+   |
+LL |     system(z);
+   |     ^^^^^^^^^ function called here
+   |
+   = help: only rustic ABIs support unsized parameters
+
+error: this function call uses unsized type `CustomUnsized` which is not supported with the chosen ABI
+  --> $DIR/non-rustic-unsized.rs:33:5
+   |
+LL |     c_custom_unsized(w);
+   |     ^^^^^^^^^^^^^^^^^^^ function called here
+   |
+   = help: only rustic ABIs support unsized parameters
+
+error: this function definition uses unsized type `[u8]` which is not supported with the chosen ABI
+  --> $DIR/non-rustic-unsized.rs:12:1
+   |
+LL | extern "C" fn c(_: [u8]) {}
+   | ^^^^^^^^^^^^^^^^^^^^^^^^ function defined here
+   |
+   = help: only rustic ABIs support unsized parameters
+
+error: this function definition uses unsized type `[u8]` which is not supported with the chosen ABI
+  --> $DIR/non-rustic-unsized.rs:14:1
+   |
+LL | extern "system" fn system(_: [u8]) {}
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function defined here
+   |
+   = help: only rustic ABIs support unsized parameters
+
+error: this function definition uses unsized type `CustomUnsized` which is not supported with the chosen ABI
+  --> $DIR/non-rustic-unsized.rs:23:1
+   |
+LL | extern "C" fn c_custom_unsized(x: CustomUnsized) {}
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function defined here
+   |
+   = help: only rustic ABIs support unsized parameters
+
+error: this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+  --> $DIR/non-rustic-unsized.rs:40:5
+   |
+LL |     c(y);
+   |     ^^^^ function called here
+   |
+   = help: only rustic ABIs support unsized parameters
+
+error: this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+  --> $DIR/non-rustic-unsized.rs:55:5
+   |
+LL |     system(y);
+   |     ^^^^^^^^^ function called here
+   |
+   = help: only rustic ABIs support unsized parameters
+
+error: this function call uses unsized type `[u8]` which is not supported with the chosen ABI
+  --> $DIR/non-rustic-unsized.rs:64:5
+   |
+LL |     c_polymorphic(x);
+   |     ^^^^^^^^^^^^^^^^ function called here
+   |
+   = help: only rustic ABIs support unsized parameters
+
+error: this function definition uses unsized type `[u8]` which is not supported with the chosen ABI
+  --> $DIR/non-rustic-unsized.rs:59:1
+   |
+LL | extern "C" fn c_polymorphic<T: ?Sized>(_: T) {}
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ function defined here
+   |
+   = help: only rustic ABIs support unsized parameters
+
+note: the above error was encountered while instantiating `fn c_polymorphic::<[u8]>`
+  --> $DIR/non-rustic-unsized.rs:64:5
+   |
+LL |     c_polymorphic(x);
+   |     ^^^^^^^^^^^^^^^^
+
+error: aborting due to 10 previous errors
+