Support DllImportSearchPath.AssemblyDirectory for NativeAOT applications

src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/NativeLibrary.NativeAot.Windows.cs

@@ -13,26 +13,35 @@ private static IntPtr LoadLibraryHelper(string libraryName, int flags, ref LoadL
         {
             IntPtr hmod;
 
+            // Disable the OS dialogs when failing to load. This matches CoreCLR.
+            uint prev;
+            bool set = Interop.Kernel32.SetThreadErrorMode(Interop.Kernel32.SEM_FAILCRITICALERRORS | Interop.Kernel32.SEM_NOOPENFILEERRORBOX, out prev);
             if (((uint)flags & 0xFFFFFF00) != 0)
             {
                 hmod = Interop.Kernel32.LoadLibraryEx(libraryName, IntPtr.Zero, (int)((uint)flags & 0xFFFFFF00));
                 if (hmod != IntPtr.Zero)
                 {
-                    return hmod;
+                    goto exit;
                 }
 
-                int lastError = Marshal.GetLastWin32Error();
+                int lastError = Marshal.GetLastPInvokeError();
                 if (lastError != Interop.Errors.ERROR_INVALID_PARAMETER)
                 {
                     errorTracker.TrackErrorCode(lastError);
-                    return hmod;
+                    goto exit;
                 }
             }
 
             hmod = Interop.Kernel32.LoadLibraryEx(libraryName, IntPtr.Zero, flags & 0xFF);
             if (hmod == IntPtr.Zero)
             {
-                errorTracker.TrackErrorCode(Marshal.GetLastWin32Error());
+                errorTracker.TrackErrorCode(Marshal.GetLastPInvokeError());
+            }
+
+        exit:
+            if (set)
+            {
+                Interop.Kernel32.SetThreadErrorMode(prev, out _);
             }
 
             return hmod;

src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/NativeLibrary.NativeAot.cs

@@ -97,7 +97,12 @@ internal static IntPtr LoadBySearch(Assembly callingAssembly, bool searchAssembl
                 else if ((callingAssembly != null) && searchAssemblyDirectory)
                 {
                     // Try to load the module alongside the assembly where the PInvoke was declared.
-                    // This only makes sense in dynamic scenarios (JIT/interpreter), so leaving this out for now.
+                    // For PInvokes where the DllImportSearchPath.AssemblyDirectory is specified, look next to the application.
+                    ret = LoadLibraryHelper(Path.Combine(AppContext.BaseDirectory, currLibNameVariation), loadWithAlteredPathFlags | dllImportSearchPathFlags, ref errorTracker);
+                    if (ret != IntPtr.Zero)
+                    {
+                        return ret;
+                    }
                 }
 
                 ret = LoadLibraryHelper(currLibNameVariation, dllImportSearchPathFlags, ref errorTracker);

src/libraries/Common/src/Interop/Windows/Kernel32/Interop.SetThreadErrorMode.cs

@@ -14,6 +14,7 @@ internal static partial bool SetThreadErrorMode(
             uint dwNewMode,
             out uint lpOldMode);
 
-        internal const uint SEM_FAILCRITICALERRORS = 1;
+        internal const int SEM_FAILCRITICALERRORS = 0x00000001;
+        internal const int SEM_NOOPENFILEERRORBOX = 0x00008000;
     }
 }

src/tests/Interop/DllImportSearchPaths/DllImportSearchPathsTest.cs

@@ -39,6 +39,13 @@ public static void AssemblyDirectory_Found()
         Assert.Equal(3, sum);
     }
 
+    [ConditionalFact(typeof(TestLibrary.Utilities), nameof(TestLibrary.Utilities.IsNativeAot))]
+    public static void AssemblyDirectoryAot_Found()
+    {
+        int sum = NativeLibraryPInvokeAot.Sum(1, 2);
+        Assert.Equal(3, sum);
+    }
+
     [Fact]
     [PlatformSpecific(TestPlatforms.Windows)]
     public static void AssemblyDirectory_Fallback_Found()
@@ -70,3 +77,18 @@ public static int Sum(int a, int b)
     [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
     static extern int NativeSum(int arg1, int arg2);
 }
+
+public class NativeLibraryPInvokeAot
+{
+    public static int Sum(int a, int b)
+    {
+        return NativeSum(a, b);
+    }
+
+    // For NativeAOT, validate the case where the native library is next to the AOT application.
+    // The passing of DllImportSearchPath.System32 is done to ensure on Windows the runtime won't fallback
+    // and try to search the application directory by default.
+    [DllImport(NativeLibraryToLoad.Name + "-in-native")]
+    [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory | DllImportSearchPath.System32)]
+    static extern int NativeSum(int arg1, int arg2);
+}

src/tests/Interop/DllImportSearchPaths/DllImportSearchPathsTest.csproj

@@ -14,16 +14,24 @@
 
   <Target Name="SetUpSubdirectoryNative" AfterTargets="CopyNativeProjectBinaries">
     <ItemGroup>
-      <_FilesToMove Include="$(OutDir)/libNativeLibrary.*" />
-      <_FilesToMove Include="$(OutDir)/NativeLibrary.*" />
+      <NativeLibrariesToMove Include="$(OutDir)/libNativeLibrary.*" />
+      <NativeLibrariesToMove Include="$(OutDir)/NativeLibrary.*" />
     </ItemGroup>
-    <Move SourceFiles="@(_FilesToMove)" DestinationFiles="@(_FilesToMove -> '$(LibrarySubdirectory)/%(Filename)%(Extension)')"/>
+    <Move SourceFiles="@(NativeLibrariesToMove)" DestinationFiles="@(NativeLibrariesToMove -> '$(LibrarySubdirectory)/%(Filename)%(Extension)')"/>
   </Target>
 
   <Target Name="SetUpSubdirectoryManaged" AfterTargets="Build">
     <ItemGroup>
-      <_FilesToCopy Include="$(OutDir)/$(TargetName).dll" />
+      <AssembliesToCopy Include="$(OutDir)/$(TargetName).dll" />
     </ItemGroup>
-    <Copy SourceFiles="@(_FilesToCopy)" DestinationFiles="@(_FilesToCopy -> '$(LibrarySubdirectory)/%(Filename)%(Extension)')"/>
+    <Copy SourceFiles="@(AssembliesToCopy)" DestinationFiles="@(AssembliesToCopy -> '$(LibrarySubdirectory)/%(Filename)%(Extension)')"/>
+  </Target>
+
+  <Target Name="SetUpAOTDirectory" Condition="'$(TestBuildMode)' == 'nativeaot'" AfterTargets="Build">
+    <ItemGroup>
+      <NativeLibrariesToCopy Include="$(LibrarySubdirectory)/libNativeLibrary.*" />
+      <NativeLibrariesToCopy Include="$(LibrarySubdirectory)/NativeLibrary.*" />
+    </ItemGroup>
+    <Copy SourceFiles="@(NativeLibrariesToCopy)" DestinationFiles="@(NativeLibrariesToCopy -> '$(NativeOutputPath)/%(Filename)-in-native%(Extension)')" />
   </Target>
 </Project>

src/tests/Interop/NativeLibrary/API/NativeLibraryTests.cs

@@ -189,6 +189,16 @@ public void LoadLibrary_AssemblyDirectory()
             EXPECT(TryLoadLibrary_WithAssembly(libName, assemblyInSubdirectory, DllImportSearchPath.AssemblyDirectory));
         }
 
+        if (TestLibrary.Utilities.IsNativeAot)
+        {
+            // For NativeAOT, validate the case where the native library is next to the AOT application.
+            // The passing of DllImportSearchPath.System32 is done to ensure on Windows the runtime won't fallback
+            // and try to search the application directory by default.
+            string libNameAot = $"{NativeLibraryToLoad.Name}-in-native";
+            EXPECT(LoadLibrary_WithAssembly(libNameAot, assembly, DllImportSearchPath.AssemblyDirectory | DllImportSearchPath.System32));
+            EXPECT(TryLoadLibrary_WithAssembly(libNameAot, assembly, DllImportSearchPath.AssemblyDirectory | DllImportSearchPath.System32));
+        }
+
         if (OperatingSystem.IsWindows())
         {
             string currentDirectory = Environment.CurrentDirectory;

src/tests/Interop/NativeLibrary/API/NativeLibraryTests.csproj

@@ -24,10 +24,10 @@
 
   <Target Name="SetUpSubdirectoryNative" AfterTargets="CopyNativeProjectBinaries">
     <ItemGroup>
-      <AssembliesToCopy Include="$(OutDir)/libNativeLibrary.*" />
-      <AssembliesToCopy Include="$(OutDir)/NativeLibrary.*" />
+      <NativeLibrariesToCopy Include="$(OutDir)/libNativeLibrary.*" />
+      <NativeLibrariesToCopy Include="$(OutDir)/NativeLibrary.*" />
     </ItemGroup>
-    <Copy SourceFiles="@(AssembliesToCopy)" DestinationFiles="@(AssembliesToCopy -> '$(LibrarySubdirectory)/%(Filename)$(FileNameSuffix)%(Extension)')" />
+    <Copy SourceFiles="@(NativeLibrariesToCopy)" DestinationFiles="@(NativeLibrariesToCopy -> '$(LibrarySubdirectory)/%(Filename)$(FileNameSuffix)%(Extension)')" />
   </Target>
 
   <Target Name="SetUpSubdirectoryManaged" AfterTargets="Build">
@@ -36,4 +36,12 @@
     </ItemGroup>
     <Copy SourceFiles="@(AssembliesToCopy)" DestinationFiles="@(AssembliesToCopy -> '$(LibrarySubdirectory)/%(Filename)$(FileNameSuffix)%(Extension)')" />
   </Target>
+
+  <Target Name="SetUpAOTDirectory" Condition="'$(TestBuildMode)' == 'nativeaot'" AfterTargets="Build">
+    <ItemGroup>
+      <NativeLibrariesToCopyAOT Include="$(OutDir)/libNativeLibrary.*" />
+      <NativeLibrariesToCopyAOT Include="$(OutDir)/NativeLibrary.*" />
+    </ItemGroup>
+    <Copy SourceFiles="@(NativeLibrariesToCopyAOT)" DestinationFiles="@(NativeLibrariesToCopyAOT -> '$(NativeOutputPath)/%(Filename)-in-native%(Extension)')" />
+  </Target>
 </Project>

src/tests/issues.targets

@@ -778,9 +778,6 @@
         <ExcludeList Include="$(XunitTestBinBase)/Interop/MarshalAPI/FunctionPointer/FunctionPtrTest/*">
             <Issue>https://github.com/dotnet/runtimelab/issues/164</Issue>
         </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)/Interop/NativeLibrary/API/NativeLibraryTests/*">
-            <Issue>https://github.com/dotnet/runtimelab/issues/165</Issue>
-        </ExcludeList>
         <ExcludeList Include="$(XunitTestBinBase)/Interop/NativeLibrary/AssemblyLoadContext/ResolveUnmanagedDllTests/*">
             <Issue>https://github.com/dotnet/runtimelab/issues/165</Issue>
         </ExcludeList>