Auto merge of rust-lang#120944 - compiler-errors:inliner-abi, r=<try>

Check that the ABI of the instance we are inlining is correct

When computing the `CallSite` in the mir inliner, double check that the instance of the function that we are inlining is compatible with the signature from the trait definition that we acquire from the MIR.

Fixes rust-lang#120940

r? `@oli-obk` or `@cjgillot`

Author: bors

compiler/rustc_mir_transform/src/inline.rs

@@ -2,6 +2,7 @@
 use crate::deref_separator::deref_finder;
 use rustc_attr::InlineAttr;
 use rustc_const_eval::transform::validate::validate_types;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_index::bit_set::BitSet;
 use rustc_index::Idx;
@@ -384,6 +385,17 @@ impl<'tcx> Inliner<'tcx> {
                 }
 
                 let fn_sig = self.tcx.fn_sig(def_id).instantiate(self.tcx, args);
+
+                // Additionally, check that the body that we're inlining actually agrees
+                // with the ABI of the trait that the item comes from.
+                if let InstanceDef::Item(instance_def_id) = callee.def
+                    && self.tcx.def_kind(instance_def_id) == DefKind::AssocFn
+                    && let instance_fn_sig = self.tcx.fn_sig(instance_def_id).skip_binder()
+                    && instance_fn_sig.abi() != fn_sig.abi()
+                {
+                    return None;
+                }
+
                 let source_info = SourceInfo { span: fn_span, ..terminator.source_info };
 
                 return Some(CallSite { callee, fn_sig, block: bb, source_info });

tests/ui/mir/inline-wrong-abi.rs

@@ -0,0 +1,32 @@
+// compile-flags: -Zpolymorphize=on -Zinline-mir=yes -Zmir-opt-level=0
+
+#![feature(fn_traits, unboxed_closures)]
+struct Foo<T>(T);
+
+impl<T: Copy> Fn<()> for Foo<T> {
+    extern "C" fn call(&self, _: ()) -> T {
+        //~^ ERROR method `call` has an incompatible type for trait
+        match *self {
+            Foo(t) => t,
+        }
+    }
+}
+
+impl<T: Copy> FnMut<()> for Foo<T> {
+    extern "rust-call" fn call_mut(&mut self, _: ()) -> T {
+        self.call(())
+    }
+}
+
+impl<T: Copy> FnOnce<()> for Foo<T> {
+    type Output = T;
+
+    extern "rust-call" fn call_once(self, _: ()) -> T {
+        self.call(())
+    }
+}
+
+fn main() {
+    let t: u8 = 1;
+    println!("{}", Foo(t)());
+}

tests/ui/mir/inline-wrong-abi.stderr

@@ -0,0 +1,12 @@
+error[E0053]: method `call` has an incompatible type for trait
+  --> $DIR/inline-wrong-abi.rs:7:5
+   |
+LL |     extern "C" fn call(&self, _: ()) -> T {
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected "rust-call" fn, found "C" fn
+   |
+   = note: expected signature `extern "rust-call" fn(&Foo<_>, ()) -> _`
+              found signature `extern "C" fn(&Foo<_>, ()) -> _`
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0053`.