[mini] Dynamically allocate a buffer for large runtime invoke results (#58215)

* [mini] Dynamically allocate a buffer for large runtime invoke results

   If the return type is a struct that's bigger than our buffer, malloc a buffer for it instead of using a fixed-size stack buffer

   Also take the ref-return logic from #52501 and add it to the non-LLVM runtime-invoke

   This makes the `System.Runtime` testsuite (particularly `InvokeRefReturnNetcoreTests`) pass on M1 MacCatalyst FullAOT.

  Related to #58190

* Throw nullbyrefreturn exception for non-LLVM dyn invoke

   Fixes various tests in InvokeRefReturnNetcoreTests

Author: lambdageek

src/mono/mono/mini/mini-runtime.c

@@ -2998,6 +2998,8 @@ typedef struct {
 	gpointer *wrapper_arg;
 } RuntimeInvokeInfo;
 
+#define MONO_SIZEOF_DYN_CALL_RET_BUF 256
+
 static RuntimeInvokeInfo*
 create_runtime_invoke_info (MonoMethod *method, gpointer compiled_method, gboolean callee_gsharedvt, gboolean use_interp, MonoError *error)
 {
@@ -3156,8 +3158,9 @@ mono_llvmonly_runtime_invoke (MonoMethod *method, RuntimeInvokeInfo *info, void
 {
 	MonoMethodSignature *sig = info->sig;
 	MonoObject *(*runtime_invoke) (MonoObject *this_obj, void **params, MonoObject **exc, void* compiled_method);
+	gboolean retval_malloc = FALSE;
 	gpointer retval_ptr;
-	guint8 retval [256];
+	guint8 retval [MONO_SIZEOF_DYN_CALL_RET_BUF];
 	int i, pindex;
 
 	error_init (error);
@@ -3184,7 +3187,21 @@ mono_llvmonly_runtime_invoke (MonoMethod *method, RuntimeInvokeInfo *info, void
 	if (sig->hasthis)
 		args [pindex ++] = &obj;
 	if (sig->ret->type != MONO_TYPE_VOID) {
-		retval_ptr = &retval;
+		if (info->ret_box_class && !sig->ret->byref &&
+		    (sig->ret->type == MONO_TYPE_VALUETYPE ||
+		     (sig->ret->type == MONO_TYPE_GENERICINST && !MONO_TYPE_IS_REFERENCE (sig->ret)))) {
+			// if the return type is a struct and its too big for the stack buffer, malloc instead
+			MonoClass *ret_klass = mono_class_from_mono_type_internal (sig->ret);
+			g_assert (!mono_class_has_failure (ret_klass));
+			int32_t inst_size = mono_class_instance_size (ret_klass);
+			if (inst_size > MONO_SIZEOF_DYN_CALL_RET_BUF) {
+				retval_malloc = TRUE;
+				retval_ptr = g_new0 (guint8, inst_size);
+				g_assert (retval_ptr);
+			}
+		}
+		if (!retval_malloc)
+			retval_ptr = &retval;
 		args [pindex ++] = &retval_ptr;
 	}
 	for (i = 0; i < sig->param_count; ++i) {
@@ -3234,7 +3251,10 @@ mono_llvmonly_runtime_invoke (MonoMethod *method, RuntimeInvokeInfo *info, void
 			if (sig->ret->byref) {
 				return mono_value_box_checked (info->ret_box_class, *(gpointer*)retval, error);
 			} else {
-				return mono_value_box_checked (info->ret_box_class, retval, error);
+				MonoObject *ret = mono_value_box_checked (info->ret_box_class, retval_ptr, error);
+				if (retval_malloc)
+					g_free (retval_ptr);
+				return ret;
 			}
 		} else {
 			if (sig->ret->byref)
@@ -3397,7 +3417,25 @@ mono_jit_runtime_invoke (MonoMethod *method, void *obj, void **params, MonoObjec
 		gpointer *args;
 		int i, pindex, buf_size;
 		guint8 *buf;
-		guint8 retval [256];
+		guint8 retbuf [MONO_SIZEOF_DYN_CALL_RET_BUF];
+		guint8 *retval = &retbuf[0];
+		gboolean retval_malloc = FALSE;
+
+		/* if the return value is too big, put it in a dynamically allocated temporary */
+		if (info->ret_box_class && !sig->ret->byref &&
+		    (sig->ret->type == MONO_TYPE_VALUETYPE ||
+		     (sig->ret->type == MONO_TYPE_GENERICINST && !MONO_TYPE_IS_REFERENCE (sig->ret)))) {
+			// if the return type is a struct and its too big for the stack buffer, malloc instead
+			MonoClass *ret_klass = mono_class_from_mono_type_internal (sig->ret);
+			g_assert (!mono_class_has_failure (ret_klass));
+			int32_t inst_size = mono_class_instance_size (ret_klass);
+			if (inst_size > MONO_SIZEOF_DYN_CALL_RET_BUF) {
+				retval_malloc = TRUE;
+				retval = g_new0 (guint8, inst_size);
+				g_assert (retval);
+			}
+		}
+
 
 		/* Convert the arguments to the format expected by start_dyn_call () */
 		args = (void **)g_alloca ((sig->param_count + sig->hasthis) * sizeof (gpointer));
@@ -3433,10 +3471,31 @@ mono_jit_runtime_invoke (MonoMethod *method, void *obj, void **params, MonoObjec
 			return NULL;
 		}
 
-		if (info->ret_box_class)
-			return mono_value_box_checked (info->ret_box_class, retval, error);
-		else
-			return *(MonoObject**)retval;
+		if (sig->ret->byref) {
+			if (*(gpointer*)retval == NULL) {
+				MonoClass *klass = mono_class_get_nullbyrefreturn_ex_class ();
+				MonoObject *ex = mono_object_new_checked (klass, error);
+				mono_error_assert_ok (error);
+				mono_error_set_exception_instance (error, (MonoException*)ex);
+				return NULL;
+			}
+		}
+
+		if (info->ret_box_class) {
+			if (sig->ret->byref) {
+				return mono_value_box_checked (info->ret_box_class, *(gpointer*)retval, error);
+			} else {
+				MonoObject *boxed_ret = mono_value_box_checked (info->ret_box_class, retval, error);
+				if (retval_malloc)
+					g_free (retval);
+				return boxed_ret;
+			}
+		} else {
+			if (sig->ret->byref)
+				return **(MonoObject***)retval;
+			else
+				return *(MonoObject**)retval;
+		}
 	}
 #endif