[NativeAOT] Objective-C Marshal: object tracker support (#78280)

* NativeAOT Objective-C Marshal: object tracker support
* Remove support for registering multiple begin/end callbacks, since the API was not designed with multiple managed worlds in mind.
* Use more asserts where it should not be possible for there to be no callback.
* Add missing GC-trigger disable when calling managed callouts.
* Simplify initialization logic
* Add comments to CoreCLR and NativeAOT about constants that should have the same size.
* Remove superfluous use of C++ templates.
* Use the C# unmanaged function pointer types in more places, improving type safety
* Add SuppressGCTransition and fix compile error by adding cast

Co-authored-by: Michal Strehovský <[email protected]>
Co-authored-by: Aaron Robinson <[email protected]>

Author: 3 people

src/coreclr/nativeaot/Bootstrap/main.cpp

@@ -112,6 +112,11 @@ extern "C" void OnFirstChanceException();
 extern "C" void OnUnhandledException();
 extern "C" void IDynamicCastableIsInterfaceImplemented();
 extern "C" void IDynamicCastableGetInterfaceImplementation();
+#ifdef FEATURE_OBJCMARSHAL
+extern "C" void ObjectiveCMarshalTryGetTaggedMemory();
+extern "C" void ObjectiveCMarshalGetIsTrackedReferenceCallback();
+extern "C" void ObjectiveCMarshalGetOnEnteredFinalizerQueueCallback();
+#endif
 
 typedef void(*pfn)();
 
@@ -126,6 +131,15 @@ static const pfn c_classlibFunctions[] = {
     &OnUnhandledException,
     &IDynamicCastableIsInterfaceImplemented,
     &IDynamicCastableGetInterfaceImplementation,
+#ifdef FEATURE_OBJCMARSHAL
+    &ObjectiveCMarshalTryGetTaggedMemory,
+    &ObjectiveCMarshalGetIsTrackedReferenceCallback,
+    &ObjectiveCMarshalGetOnEnteredFinalizerQueueCallback,
+#else
+    nullptr,
+    nullptr,
+    nullptr,
+#endif
 };
 
 #ifndef _countof

src/coreclr/nativeaot/CMakeLists.txt

@@ -34,6 +34,7 @@ if(CLR_CMAKE_HOST_UNIX)
 
   if(CLR_CMAKE_TARGET_OSX)
     add_definitions(-D_XOPEN_SOURCE)
+    add_definitions(-DFEATURE_OBJCMARSHAL)
   endif(CLR_CMAKE_TARGET_OSX)
 
   if(CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_I386)

src/coreclr/nativeaot/Common/src/Internal/Runtime/MethodTable.cs

@@ -791,6 +791,14 @@ internal bool IsHFA
             }
         }
 
+        internal bool IsTrackedReferenceWithFinalizer
+        {
+            get
+            {
+                return (ExtendedFlags & (ushort)EETypeFlagsEx.IsTrackedReferenceWithFinalizerFlag) != 0;
+            }
+        }
+
         internal uint ValueTypeFieldPadding
         {
             get

src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs

@@ -44,6 +44,9 @@ internal enum ClassLibFunctionId
         OnUnhandledException = 7,
         IDynamicCastableIsInterfaceImplemented = 8,
         IDynamicCastableGetInterfaceImplementation = 9,
+        ObjectiveCMarshalTryGetTaggedMemory = 10,
+        ObjectiveCMarshalGetIsTrackedReferenceCallback = 11,
+        ObjectiveCMarshalGetOnEnteredFinalizerQueueCallback = 12,
     }
 
     internal static class InternalCalls

src/coreclr/nativeaot/Runtime/CMakeLists.txt

@@ -170,6 +170,12 @@ else()
   set(ASM_SUFFIX S)
 endif()
 
+if (CLR_CMAKE_TARGET_OSX)
+  list(APPEND COMMON_RUNTIME_SOURCES
+    interoplibinterface_objc.cpp
+  )
+endif (CLR_CMAKE_TARGET_OSX)
+
 if (CLR_CMAKE_TARGET_ARCH_AMD64 AND CLR_CMAKE_TARGET_WIN32)
   list(APPEND COMMON_RUNTIME_SOURCES
     ${GC_DIR}/vxsort/isa_detection.cpp

src/coreclr/nativeaot/Runtime/Crst.h

@@ -18,6 +18,7 @@ enum CrstType
     CrstInterfaceDispatchGlobalLists,
     CrstStressLog,
     CrstRestrictedCallouts,
+    CrstObjectiveCMarshalCallouts,
     CrstGcStressControl,
     CrstSuspendEE,
     CrstCastCache,

src/coreclr/nativeaot/Runtime/GCHelpers.cpp

@@ -18,6 +18,7 @@
 #include "varint.h"
 #include "regdisplay.h"
 #include "StackFrameIterator.h"
+#include "interoplibinterface.h"
 
 #include "thread.h"
 #include "RWLock.h"
@@ -132,6 +133,13 @@ COOP_PINVOKE_HELPER(void, RhUnregisterGcCallout, (GcRestrictedCalloutKind eKind,
     RestrictedCallouts::UnregisterGcCallout(eKind, pCallout);
 }
 
+#ifdef FEATURE_OBJCMARSHAL
+COOP_PINVOKE_HELPER(FC_BOOL_RET, RhRegisterObjectiveCMarshalBeginEndCallback, (void * pCallback))
+{
+    FC_RETURN_BOOL(ObjCMarshalNative::RegisterBeginEndCallback(pCallback));
+}
+#endif
+
 COOP_PINVOKE_HELPER(int32_t, RhGetLohCompactionMode, ())
 {
     return GCHeapUtilities::GetGCHeap()->GetLOHCompactionMode();

src/coreclr/nativeaot/Runtime/ICodeManager.h

@@ -154,6 +154,9 @@ enum class ClasslibFunctionId
     OnUnhandledException = 7,
     IDynamicCastableIsInterfaceImplemented = 8,
     IDynamicCastableGetInterfaceImplementation = 9,
+    ObjectiveCMarshalTryGetTaggedMemory = 10,
+    ObjectiveCMarshalGetIsTrackedReferenceCallback = 11,
+    ObjectiveCMarshalGetOnEnteredFinalizerQueueCallback = 12,
 };
 
 enum class AssociatedDataFlags : unsigned char

src/coreclr/nativeaot/Runtime/gcrhenv.cpp

@@ -45,6 +45,7 @@
 #include "daccess.h"
 
 #include "GCMemoryHelpers.h"
+#include "interoplibinterface.h"
 
 #include "holder.h"
 #include "volatile.h"
@@ -682,13 +683,25 @@ void GCToEEInterface::GcStartWork(int condemned, int /*max_gen*/)
 
 void GCToEEInterface::BeforeGcScanRoots(int condemned, bool is_bgc, bool is_concurrent)
 {
+#ifdef FEATURE_OBJCMARSHAL
+    if (!is_concurrent)
+    {
+        ObjCMarshalNative::BeforeRefCountedHandleCallbacks();
+    }
+#endif
 }
 
 // EE can perform post stack scanning action, while the user threads are still suspended
-void GCToEEInterface::AfterGcScanRoots(int condemned, int /*max_gen*/, ScanContext* /*sc*/)
+void GCToEEInterface::AfterGcScanRoots(int condemned, int /*max_gen*/, ScanContext* sc)
 {
     // Invoke any registered callouts for the end of the mark phase.
     RestrictedCallouts::InvokeGcCallouts(GCRC_AfterMarkPhase, condemned);
+#ifdef FEATURE_OBJCMARSHAL
+    if (!sc->concurrent)
+    {
+        ObjCMarshalNative::AfterRefCountedHandleCallbacks();
+    }
+#endif
 }
 
 void GCToEEInterface::GcDone(int condemned)
@@ -699,6 +712,11 @@ void GCToEEInterface::GcDone(int condemned)
 
 bool GCToEEInterface::RefCountedHandleCallbacks(Object * pObject)
 {
+#ifdef FEATURE_OBJCMARSHAL
+    bool isReferenced = false;
+    if (ObjCMarshalNative::IsTrackedReference(pObject, &isReferenced))
+        return isReferenced;
+#endif // FEATURE_OBJCMARSHAL
     return RestrictedCallouts::InvokeRefCountedHandleCallbacks(pObject);
 }
 
@@ -1153,6 +1171,14 @@ void GCToEEInterface::HandleFatalError(unsigned int exitCode)
 
 bool GCToEEInterface::EagerFinalized(Object* obj)
 {
+#ifdef FEATURE_OBJCMARSHAL
+    if (obj->GetGCSafeMethodTable()->IsTrackedReferenceWithFinalizer())
+    {
+        ObjCMarshalNative::OnEnteredFinalizerQueue(obj);
+        return false;
+    }
+#endif
+
     if (!obj->GetGCSafeMethodTable()->HasEagerFinalizer())
         return false;
 

src/coreclr/nativeaot/Runtime/inc/MethodTable.h

@@ -194,6 +194,7 @@ class MethodTable
     {
         HasEagerFinalizerFlag = 0x0001,
         HasCriticalFinalizerFlag = 0x0002,
+        IsTrackedReferenceWithFinalizerFlag = 0x0004,
     };
 
 public:
@@ -269,6 +270,11 @@ class MethodTable
         return (m_uFlags & HasCriticalFinalizerFlag) && !HasComponentSize();
     }
 
+    bool IsTrackedReferenceWithFinalizer()
+    {
+        return (m_uFlags & IsTrackedReferenceWithFinalizerFlag) && !HasComponentSize();
+    }
+
     bool  HasComponentSize()
     {
         static_assert(HasComponentSizeFlag == (MethodTable::Flags)(1 << 31), "we assume that HasComponentSizeFlag matches the sign bit");