New code to perform managed <-> java lookups (typemap)

Xamarin.Assembly needs to "translate" managed types to Java types and
vice versa in order to provide a bridge between the two world. So far it
has been done using a straightforward (and fast) method of performing
the lookups - all the type pairs were stored in two tables of the same
size, with all type names padded to the width of the longest name so
that the `bsearch` C function can be used to quickly perform a binary
search over the data set. This approach works very well at the expense
of data size (shorter strings are 0-padded to the maximum width) and a
slightly degraded performace because of the requirement to perform
string comparisons. Furthermore, the lookup required that reflection is
used to obtain full managed type name (when translating from managed to
Java) or to get a `Type` instance from type name (when translating from
Java to managed).

For Release builds all the above data is placed in the
`libxamarin-app.so` library, for Debug builds it is also placed in two
files - one for each direction of lookup, described above.

This commit is a slight improvement over the above scheme. It eliminates
reflection from the process by using managed type tokens (which are
integers) and using UUID/Guid of the module in which the type is found.
This allows us to perform the binary search over the set of 20 bytes (16
bytes for the UUID and 4 bytes for the token ID) for managed to Java
lookups and a single string comparison + binary search over a set of
integers for the Java to managed lookup.

Java type names must still be used because Java doesn't provide any
equivalent to the .NET's type token and module UUID. Those names are
still 0-padded to the width of the longest name but there are no longer
duplicated. Managed type names are eliminated completely.

If Xamarin.Android Instant Run is not used (which is the case for OSS
code) for Debug builds, the operation is performed in the same way for
both Release and Debug builds. If, however, Instant Run is in effect,
the type maps are stored in several files with the .typemap extension -
one per **module**. The files contain both the Java to managed maps as
well as managed to Java maps (which use indexes into the Java to managed
maps). All of those files are loaded during Debug app startup and used
to construct a dataset which is the searched during all the lookups.

TODO:

  - Performance changes
  - Size changes:
  - monodroid PR

Author: grendello

.external

@@ -1,2 +1,2 @@
-xamarin/monodroid:master@0f6725edfa09559afad58233d43f385122e31e8d
+grendello/monodroid:new-typemap@efe3956ce2e557b5f8286f984731e43c6ee402f7
 mono/mono:2019-10@18920a83f423fb864a2263948737681968f5b2c8

src/Mono.Android/Android.Runtime/AndroidRuntime.cs

@@ -234,7 +234,7 @@ protected override IEnumerable<string> GetSimpleReferences (Type type)
 			foreach (var simpleRef in base.GetSimpleReferences (type)) {
 				yield return simpleRef;
 			}
-			var j = JNIEnv.monodroid_typemap_managed_to_java (type.FullName + ", " + type.Assembly.GetName ().Name);
+			var j = JNIEnv.monodroid_typemap_managed_to_java (type.Module.ModuleVersionId.ToByteArray (), type.MetadataToken);
 			if (j != IntPtr.Zero) {
 				yield return Marshal.PtrToStringAnsi (j);
 			}

src/Mono.Android/Android.Runtime/JNIEnv.cs

@@ -632,13 +632,13 @@ public static string GetClassNameFromInstance (IntPtr jobject)
 		}
 
 		[DllImport ("__Internal", CallingConvention = CallingConvention.Cdecl)]
-		internal static extern IntPtr monodroid_typemap_managed_to_java (string managed);
+		internal static extern IntPtr monodroid_typemap_managed_to_java (byte[] mvid, int token);
 
 		public static string GetJniName (Type type)
 		{
 			if (type == null)
 				throw new ArgumentNullException ("type");
-			var java  = monodroid_typemap_managed_to_java (type.FullName + ", " + type.Assembly.GetName ().Name);
+			var java  = monodroid_typemap_managed_to_java (type.Module.ModuleVersionId.ToByteArray (), type.MetadataToken);
 			return java == IntPtr.Zero
 				? JavaNativeTypeManager.ToJniName (type)
 				: Marshal.PtrToStringAnsi (java);

src/Mono.Android/Java.Interop/TypeManager.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Reflection;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using Java.Interop.Tools.TypeNameMappings;
 
@@ -203,22 +204,22 @@ static Exception CreateJavaLocationException ()
 				return new JavaLocationException (loc.ToString ());
 		}
 
-		[DllImport ("__Internal", CallingConvention = CallingConvention.Cdecl)]
-		internal static extern IntPtr monodroid_typemap_java_to_managed (string java);
+		[MethodImplAttribute(MethodImplOptions.InternalCall)]
+		static extern Type monodroid_typemap_java_to_managed (string java_type_name);
 
 		internal static Type GetJavaToManagedType (string class_name)
 		{
-			var t = monodroid_typemap_java_to_managed (class_name);
-			if (t != IntPtr.Zero)
-				return Type.GetType (Marshal.PtrToStringAnsi (t));
+			Type type = monodroid_typemap_java_to_managed (class_name);
+			if (type != null)
+				return type;
 
 			if (!JNIEnv.IsRunningOnDesktop) {
 				return null;
 			}
 
 			__TypeRegistrations.RegisterPackages ();
 
-			var type    = (Type) null;
+			type = null;
 			int ls      = class_name.LastIndexOf ('/');
 			var package = ls >= 0 ? class_name.Substring (0, ls) : "";
 			List<Converter<string, Type>> mappers;

src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs

@@ -42,6 +42,9 @@ public class GenerateJavaStubs : AndroidTask
 		[Required]
 		public string [] SupportedAbis { get; set; }
 
+		[Required]
+		public bool GenerateNativeAssembly { get; set; }
+
 		public string ManifestTemplate { get; set; }
 		public string[] MergedManifestDocuments { get; set; }
 
@@ -55,6 +58,13 @@ public class GenerateJavaStubs : AndroidTask
 
 		public string AndroidSdkPlatform { get; set; }
 		public string OutputDirectory { get; set; }
+
+		[Required]
+		public string NativeOutputDirectory { get; set; }
+
+		[Output]
+		public string[] GeneratedBinaryTypeMaps { get; set; }
+
 		public string MergedAndroidManifestOutput { get; set; }
 
 		public bool EmbedAssemblies { get; set; }
@@ -75,11 +85,12 @@ public class GenerateJavaStubs : AndroidTask
 
 		public override bool RunTask ()
 		{
+			var typemapGen = new GenerateTypeMaps ();
 			try {
 				// We're going to do 3 steps here instead of separate tasks so
 				// we can share the list of JLO TypeDefinitions between them
 				using (var res = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: true)) {
-					Run (res);
+					Run (res, typemapGen);
 				}
 			} catch (XamarinAndroidException e) {
 				Log.LogCodedError (string.Format ("XA{0:0000}", e.Code), e.MessageWithoutCode);
@@ -92,14 +103,13 @@ public override bool RunTask ()
 				// by ensuring that the target outputs have been deleted.
 				Files.DeleteFile (MergedAndroidManifestOutput, Log);
 				Files.DeleteFile (AcwMapFile, Log);
-				Files.DeleteFile (Path.Combine (OutputDirectory, "typemap.jm"), Log);
-				Files.DeleteFile (Path.Combine (OutputDirectory, "typemap.mj"), Log);
+				typemapGen.CleanupOnError (NativeOutputDirectory);
 			}
 
 			return !Log.HasLoggedErrors;
 		}
 
-		void Run (DirectoryAssemblyResolver res)
+		void Run (DirectoryAssemblyResolver res, GenerateTypeMaps typemapGen)
 		{
 			PackageNamingPolicy pnp;
 			JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64;
@@ -141,14 +151,16 @@ void Run (DirectoryAssemblyResolver res)
 				assemblies.Add (asm.ItemSpec);
 			}
 
+			// Step 0 - Generate type maps (this should eventually live in a separate task)
+			typemapGen.Run (Log, res, ResolvedAssemblies, NativeOutputDirectory, SupportedAbis, GenerateNativeAssembly);
+			GeneratedBinaryTypeMaps = typemapGen.GeneratedBinaryTypeMaps;
+
 			// Step 1 - Find all the JLO types
 			var scanner = new JavaTypeScanner (this.CreateTaskLogger ()) {
 				ErrorOnCustomJavaObject     = ErrorOnCustomJavaObject,
 			};
 			var all_java_types = scanner.GetJavaTypes (assemblies, res);
 
-			WriteTypeMappings (all_java_types);
-
 			var java_types = all_java_types
 				.Where (t => !JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (t))
 				.ToArray ();
@@ -300,76 +312,5 @@ void SaveResource (string resource, string filename, string destDir, Func<string
 			template = applyTemplate (template);
 			MonoAndroidHelper.CopyIfStringChanged (template, Path.Combine (destDir, filename));
 		}
-
-		void WriteTypeMappings (List<TypeDefinition> types)
-		{
-			void logger (TraceLevel level, string value) => Log.LogDebugMessage (value);
-			TypeNameMapGenerator createTypeMapGenerator () => UseSharedRuntime ?
-				new TypeNameMapGenerator (types, logger) :
-				new TypeNameMapGenerator (ResolvedAssemblies.Select (p => p.ItemSpec), logger);
-			using (var gen = createTypeMapGenerator ()) {
-				using (var ms = new MemoryStream ()) {
-					UpdateWhenChanged (Path.Combine (OutputDirectory, "typemap.jm"), "jm", ms, gen.WriteJavaToManaged);
-					UpdateWhenChanged (Path.Combine (OutputDirectory, "typemap.mj"), "mj", ms, gen.WriteManagedToJava);
-				}
-			}
-		}
-
-		void UpdateWhenChanged (string path, string type, MemoryStream ms, Action<Stream> generator)
-		{
-			if (!EmbedAssemblies) {
-				ms.SetLength (0);
-				generator (ms);
-				MonoAndroidHelper.CopyIfStreamChanged (ms, path);
-			}
-
-			string dataFilePath = $"{path}.inc";
-			using (var stream = new NativeAssemblyDataStream ()) {
-				if (EmbedAssemblies) {
-					generator (stream);
-					stream.EndOfFile ();
-					MonoAndroidHelper.CopyIfStreamChanged (stream, dataFilePath);
-				} else {
-					stream.EmptyFile ();
-				}
-
-				var generatedFiles = new List <ITaskItem> ();
-				string mappingFieldName = $"{type}_typemap";
-				string dataFileName = Path.GetFileName (dataFilePath);
-				NativeAssemblerTargetProvider asmTargetProvider;
-				var utf8Encoding = new UTF8Encoding (false);
-				foreach (string abi in SupportedAbis) {
-					ms.SetLength (0);
-					switch (abi.Trim ()) {
-						case "armeabi-v7a":
-							asmTargetProvider = new ARMNativeAssemblerTargetProvider (is64Bit: false);
-							break;
-
-						case "arm64-v8a":
-							asmTargetProvider = new ARMNativeAssemblerTargetProvider (is64Bit: true);
-							break;
-
-						case "x86":
-							asmTargetProvider = new X86NativeAssemblerTargetProvider (is64Bit: false);
-							break;
-
-						case "x86_64":
-							asmTargetProvider = new X86NativeAssemblerTargetProvider (is64Bit: true);
-							break;
-
-						default:
-							throw new InvalidOperationException ($"Unknown ABI {abi}");
-					}
-
-					var asmgen = new TypeMappingNativeAssemblyGenerator (asmTargetProvider, stream, dataFileName, stream.MapByteCount, mappingFieldName);
-					asmgen.EmbedAssemblies = EmbedAssemblies;
-					string asmFileName = $"{path}.{abi.Trim ()}.s";
-					using (var sw = new StreamWriter (ms, utf8Encoding, bufferSize: 8192, leaveOpen: true)) {
-						asmgen.Write (sw, dataFileName);
-						MonoAndroidHelper.CopyIfStreamChanged (ms, asmFileName);
-					}
-				}
-			}
-		}
 	}
 }

src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs

@@ -58,6 +58,9 @@ public class GeneratePackageManagerJava : AndroidTask
 		[Required]
 		public bool EnablePreloadAssembliesDefault { get; set; }
 
+		[Required]
+		public bool InstantRunEnabled { get; set; }
+
 		public string BoundExceptionType { get; set; }
 
 		public string PackageNamingPolicy { get; set; }
@@ -252,7 +255,8 @@ void AddEnvironment ()
 				foreach (string abi in SupportedAbis) {
 					ms.SetLength (0);
 					NativeAssemblerTargetProvider asmTargetProvider;
-					string asmFileName = Path.Combine (EnvironmentOutputDirectory, $"environment.{abi.ToLowerInvariant ()}.s");
+					string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{abi.ToLowerInvariant ()}");
+					string asmFilePath = $"{baseAsmFilePath}.s";
 					switch (abi.Trim ()) {
 						case "armeabi-v7a":
 							asmTargetProvider = new ARMNativeAssemblerTargetProvider (false);
@@ -274,7 +278,7 @@ void AddEnvironment ()
 							throw new InvalidOperationException ($"Unknown ABI {abi}");
 					}
 
-					var asmgen = new ApplicationConfigNativeAssemblyGenerator (asmTargetProvider, environmentVariables, systemProperties) {
+					var asmgen = new ApplicationConfigNativeAssemblyGenerator (asmTargetProvider, baseAsmFilePath, environmentVariables, systemProperties) {
 						IsBundledApp = IsBundledApplication,
 						UsesMonoAOT = usesMonoAOT,
 						UsesMonoLLVM = EnableLLVM,
@@ -284,11 +288,12 @@ void AddEnvironment ()
 						BrokenExceptionTransitions = brokenExceptionTransitions,
 						PackageNamingPolicy = pnp,
 						BoundExceptionType = boundExceptionType,
+						InstantRunEnabled = InstantRunEnabled,
 					};
 
 					using (var sw = new StreamWriter (ms, utf8Encoding, bufferSize: 8192, leaveOpen: true)) {
-						asmgen.Write (sw, asmFileName);
-						MonoAndroidHelper.CopyIfStreamChanged (ms, asmFileName);
+						asmgen.Write (sw);
+						MonoAndroidHelper.CopyIfStreamChanged (ms, asmFilePath);
 					}
 
 				}

src/Xamarin.Android.Build.Tasks/Tasks/GenerateTypeMaps.cs

@@ -0,0 +1,96 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+using Java.Interop.Tools.Cecil;
+using Java.Interop.Tools.Diagnostics;
+using Xamarin.Android.Tools;
+
+namespace Xamarin.Android.Tasks
+{
+    public class GenerateTypeMaps : AndroidTask
+	{
+		public override string TaskPrefix => "GTM";
+
+		[Required]
+		public ITaskItem[] ResolvedAssemblies { get; set; }
+
+		[Required]
+		public ITaskItem[] ResolvedUserAssemblies { get; set; }
+
+		[Required]
+		public ITaskItem [] FrameworkDirectories { get; set; }
+
+		[Required]
+		public string[] SupportedAbis { get; set; }
+
+		[Required]
+		public string OutputDirectory { get; set; }
+
+		[Required]
+		public bool GenerateNativeAssembly { get; set; }
+
+		[Output]
+		public string[] GeneratedBinaryTypeMaps { get; set; }
+
+		public bool ErrorOnCustomJavaObject { get; set; }
+
+		public override bool RunTask ()
+		{
+			try {
+				var res = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: true);
+				foreach (var dir in FrameworkDirectories) {
+					if (Directory.Exists (dir.ItemSpec))
+						res.SearchDirectories.Add (dir.ItemSpec);
+				}
+
+				Run (Log, res, ResolvedAssemblies, OutputDirectory, SupportedAbis, GenerateNativeAssembly);
+			} catch (XamarinAndroidException e) {
+				Log.LogCodedError (string.Format ("XA{0:0000}", e.Code), e.MessageWithoutCode);
+				if (MonoAndroidHelper.LogInternalExceptions)
+					Log.LogMessage (e.ToString ());
+			}
+
+			if (Log.HasLoggedErrors) {
+				CleanupOnError (OutputDirectory);
+				return false;
+			}
+
+			return true;
+		}
+
+		public void CleanupOnError (string outputDirectory)
+		{
+			// Ensure that on a rebuild, we don't *skip* the `_GenerateJavaStubs` target,
+			// by ensuring that the target outputs have been deleted.
+			Files.DeleteFile (Path.Combine (outputDirectory, "typemap.index"), Log);
+			foreach (string file in Directory.EnumerateFiles (outputDirectory, "*.typemap")) {
+				Files.DeleteFile (file, Log);
+			}
+		}
+
+		public void Run (TaskLoggingHelper log, DirectoryAssemblyResolver res, ITaskItem[] resolvedAssemblies, string outputDirectory, string[] supportedAbis, bool generateNativeAssembly)
+		{
+			var interestingAssemblies = new HashSet<string> (StringComparer.OrdinalIgnoreCase);
+
+			foreach (ITaskItem assembly in resolvedAssemblies) {
+				res.Load (assembly.ItemSpec);
+				if (String.Compare ("MonoAndroid", assembly.GetMetadata ("TargetFrameworkIdentifier"), StringComparison.Ordinal) != 0)
+					continue;
+				if (interestingAssemblies.Contains (assembly.ItemSpec))
+					continue;
+				interestingAssemblies.Add (assembly.ItemSpec);
+			}
+
+			var tmg = new TypeMapGenerator ((string message) => log.LogDebugMessage (message), supportedAbis);
+			if (!tmg.Generate (res, interestingAssemblies, outputDirectory, generateNativeAssembly))
+				throw new XamarinAndroidException (99999, "Failed to generate type maps");
+			GeneratedBinaryTypeMaps = tmg.GeneratedBinaryTypeMaps.ToArray ();
+		}
+	}
+}