New code to perform managed <-> java lookups (typemap)

Xamarin.Assembly needs to "translate" managed types to Java types and
vice versa in order to provide a bridge between the two world. So far it
has been done using a straightforward (and fast) method of performing
the lookups - all the type pairs were stored in two tables of the same
size, with all type names padded to the width of the longest name so
that the `bsearch` C function can be used to quickly perform a binary
search over the data set. This approach works very well at the expense
of data size (shorter strings are 0-padded to the maximum width) and a
slightly degraded performace because of the requirement to perform
string comparisons. Furthermore, the lookup required that reflection is
used to obtain full managed type name (when translating from managed to
Java) or to get a `Type` instance from type name (when translating from
Java to managed).

For Release builds all the above data is placed in the
`libxamarin-app.so` library, for Debug builds it is also placed in two
files - one for each direction of lookup, described above.

This commit is a slight improvement over the above scheme. It eliminates
reflection from the process by using managed type tokens (which are
integers) and using UUID/Guid of the module in which the type is found.
This allows us to perform the binary search over the set of 20 bytes (16
bytes for the UUID and 4 bytes for the token ID) for managed to Java
lookups and a single string comparison + binary search over a set of
integers for the Java to managed lookup.

Java type names must still be used because Java doesn't provide any
equivalent to the .NET's type token and module UUID. Those names are
still 0-padded to the width of the longest name but there are no longer
duplicated. Managed type names are eliminated completely.

If Xamarin.Android Instant Run is not used (which is the case for OSS
code) for Debug builds, the operation is performed in the same way for
both Release and Debug builds. If, however, Instant Run is in effect,
the type maps are stored in several files with the .typemap extension -
one per **module**. The files contain both the Java to managed maps as
well as managed to Java maps (which use indexes into the Java to managed
maps). All of those files are loaded during Debug app startup and used
to construct a dataset which is the searched during all the lookups.

TODO:

  - Performance changes
  - Size changes:
  - monodroid PR

Author: grendello

.external

@@ -1,2 +1,2 @@
-xamarin/monodroid:master@0f6725edfa09559afad58233d43f385122e31e8d
+grendello/monodroid:new-typemap@efe3956ce2e557b5f8286f984731e43c6ee402f7
 mono/mono:2019-10@18920a83f423fb864a2263948737681968f5b2c8

src/Mono.Android/Android.Runtime/AndroidRuntime.cs

@@ -234,7 +234,7 @@ protected override IEnumerable<string> GetSimpleReferences (Type type)
 			foreach (var simpleRef in base.GetSimpleReferences (type)) {
 				yield return simpleRef;
 			}
-			var j = JNIEnv.monodroid_typemap_managed_to_java (type.FullName + ", " + type.Assembly.GetName ().Name);
+			var j = JNIEnv.monodroid_typemap_managed_to_java (type.Module.ModuleVersionId.ToByteArray (), type.MetadataToken);
 			if (j != IntPtr.Zero) {
 				yield return Marshal.PtrToStringAnsi (j);
 			}

src/Mono.Android/Android.Runtime/JNIEnv.cs

@@ -632,13 +632,13 @@ public static string GetClassNameFromInstance (IntPtr jobject)
 		}
 
 		[DllImport ("__Internal", CallingConvention = CallingConvention.Cdecl)]
-		internal static extern IntPtr monodroid_typemap_managed_to_java (string managed);
+		internal static extern IntPtr monodroid_typemap_managed_to_java (byte[] mvid, int token);
 
 		public static string GetJniName (Type type)
 		{
 			if (type == null)
 				throw new ArgumentNullException ("type");
-			var java  = monodroid_typemap_managed_to_java (type.FullName + ", " + type.Assembly.GetName ().Name);
+			var java  = monodroid_typemap_managed_to_java (type.Module.ModuleVersionId.ToByteArray (), type.MetadataToken);
 			return java == IntPtr.Zero
 				? JavaNativeTypeManager.ToJniName (type)
 				: Marshal.PtrToStringAnsi (java);

src/Mono.Android/Java.Interop/TypeManager.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Reflection;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using Java.Interop.Tools.TypeNameMappings;
 
@@ -203,22 +204,22 @@ static Exception CreateJavaLocationException ()
 				return new JavaLocationException (loc.ToString ());
 		}
 
-		[DllImport ("__Internal", CallingConvention = CallingConvention.Cdecl)]
-		internal static extern IntPtr monodroid_typemap_java_to_managed (string java);
+		[MethodImplAttribute(MethodImplOptions.InternalCall)]
+		static extern Type monodroid_typemap_java_to_managed (string java_type_name);
 
 		internal static Type GetJavaToManagedType (string class_name)
 		{
-			var t = monodroid_typemap_java_to_managed (class_name);
-			if (t != IntPtr.Zero)
-				return Type.GetType (Marshal.PtrToStringAnsi (t));
+			Type type = monodroid_typemap_java_to_managed (class_name);
+			if (type != null)
+				return type;
 
 			if (!JNIEnv.IsRunningOnDesktop) {
 				return null;
 			}
 
 			__TypeRegistrations.RegisterPackages ();
 
-			var type    = (Type) null;
+			type = null;
 			int ls      = class_name.LastIndexOf ('/');
 			var package = ls >= 0 ? class_name.Substring (0, ls) : "";
 			List<Converter<string, Type>> mappers;

src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs

@@ -147,8 +147,6 @@ void Run (DirectoryAssemblyResolver res)
 			};
 			var all_java_types = scanner.GetJavaTypes (assemblies, res);
 
-			WriteTypeMappings (all_java_types);
-
 			var java_types = all_java_types
 				.Where (t => !JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (t))
 				.ToArray ();
@@ -300,76 +298,5 @@ void SaveResource (string resource, string filename, string destDir, Func<string
 			template = applyTemplate (template);
 			MonoAndroidHelper.CopyIfStringChanged (template, Path.Combine (destDir, filename));
 		}
-
-		void WriteTypeMappings (List<TypeDefinition> types)
-		{
-			void logger (TraceLevel level, string value) => Log.LogDebugMessage (value);
-			TypeNameMapGenerator createTypeMapGenerator () => UseSharedRuntime ?
-				new TypeNameMapGenerator (types, logger) :
-				new TypeNameMapGenerator (ResolvedAssemblies.Select (p => p.ItemSpec), logger);
-			using (var gen = createTypeMapGenerator ()) {
-				using (var ms = new MemoryStream ()) {
-					UpdateWhenChanged (Path.Combine (OutputDirectory, "typemap.jm"), "jm", ms, gen.WriteJavaToManaged);
-					UpdateWhenChanged (Path.Combine (OutputDirectory, "typemap.mj"), "mj", ms, gen.WriteManagedToJava);
-				}
-			}
-		}
-
-		void UpdateWhenChanged (string path, string type, MemoryStream ms, Action<Stream> generator)
-		{
-			if (!EmbedAssemblies) {
-				ms.SetLength (0);
-				generator (ms);
-				MonoAndroidHelper.CopyIfStreamChanged (ms, path);
-			}
-
-			string dataFilePath = $"{path}.inc";
-			using (var stream = new NativeAssemblyDataStream ()) {
-				if (EmbedAssemblies) {
-					generator (stream);
-					stream.EndOfFile ();
-					MonoAndroidHelper.CopyIfStreamChanged (stream, dataFilePath);
-				} else {
-					stream.EmptyFile ();
-				}
-
-				var generatedFiles = new List <ITaskItem> ();
-				string mappingFieldName = $"{type}_typemap";
-				string dataFileName = Path.GetFileName (dataFilePath);
-				NativeAssemblerTargetProvider asmTargetProvider;
-				var utf8Encoding = new UTF8Encoding (false);
-				foreach (string abi in SupportedAbis) {
-					ms.SetLength (0);
-					switch (abi.Trim ()) {
-						case "armeabi-v7a":
-							asmTargetProvider = new ARMNativeAssemblerTargetProvider (is64Bit: false);
-							break;
-
-						case "arm64-v8a":
-							asmTargetProvider = new ARMNativeAssemblerTargetProvider (is64Bit: true);
-							break;
-
-						case "x86":
-							asmTargetProvider = new X86NativeAssemblerTargetProvider (is64Bit: false);
-							break;
-
-						case "x86_64":
-							asmTargetProvider = new X86NativeAssemblerTargetProvider (is64Bit: true);
-							break;
-
-						default:
-							throw new InvalidOperationException ($"Unknown ABI {abi}");
-					}
-
-					var asmgen = new TypeMappingNativeAssemblyGenerator (asmTargetProvider, stream, dataFileName, stream.MapByteCount, mappingFieldName);
-					asmgen.EmbedAssemblies = EmbedAssemblies;
-					string asmFileName = $"{path}.{abi.Trim ()}.s";
-					using (var sw = new StreamWriter (ms, utf8Encoding, bufferSize: 8192, leaveOpen: true)) {
-						asmgen.Write (sw, dataFileName);
-						MonoAndroidHelper.CopyIfStreamChanged (ms, asmFileName);
-					}
-				}
-			}
-		}
 	}
 }

src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs

@@ -58,6 +58,9 @@ public class GeneratePackageManagerJava : AndroidTask
 		[Required]
 		public bool EnablePreloadAssembliesDefault { get; set; }
 
+		[Required]
+		public bool InstantRunEnabled { get; set; }
+
 		public string BoundExceptionType { get; set; }
 
 		public string PackageNamingPolicy { get; set; }
@@ -252,7 +255,8 @@ void AddEnvironment ()
 				foreach (string abi in SupportedAbis) {
 					ms.SetLength (0);
 					NativeAssemblerTargetProvider asmTargetProvider;
-					string asmFileName = Path.Combine (EnvironmentOutputDirectory, $"environment.{abi.ToLowerInvariant ()}.s");
+					string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{abi.ToLowerInvariant ()}");
+					string asmFilePath = $"{baseAsmFilePath}.s";
 					switch (abi.Trim ()) {
 						case "armeabi-v7a":
 							asmTargetProvider = new ARMNativeAssemblerTargetProvider (false);
@@ -274,7 +278,7 @@ void AddEnvironment ()
 							throw new InvalidOperationException ($"Unknown ABI {abi}");
 					}
 
-					var asmgen = new ApplicationConfigNativeAssemblyGenerator (asmTargetProvider, environmentVariables, systemProperties) {
+					var asmgen = new ApplicationConfigNativeAssemblyGenerator (asmTargetProvider, baseAsmFilePath, environmentVariables, systemProperties) {
 						IsBundledApp = IsBundledApplication,
 						UsesMonoAOT = usesMonoAOT,
 						UsesMonoLLVM = EnableLLVM,
@@ -284,11 +288,12 @@ void AddEnvironment ()
 						BrokenExceptionTransitions = brokenExceptionTransitions,
 						PackageNamingPolicy = pnp,
 						BoundExceptionType = boundExceptionType,
+						InstantRunEnabled = InstantRunEnabled,
 					};
 
 					using (var sw = new StreamWriter (ms, utf8Encoding, bufferSize: 8192, leaveOpen: true)) {
-						asmgen.Write (sw, asmFileName);
-						MonoAndroidHelper.CopyIfStreamChanged (ms, asmFileName);
+						asmgen.Write (sw);
+						MonoAndroidHelper.CopyIfStreamChanged (ms, asmFilePath);
 					}
 
 				}

src/Xamarin.Android.Build.Tasks/Tasks/GenerateTypeMaps.cs

@@ -0,0 +1,89 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+
+using Microsoft.Build.Framework;
+
+using Java.Interop.Tools.Cecil;
+using Java.Interop.Tools.Diagnostics;
+using Xamarin.Android.Tools;
+
+namespace Xamarin.Android.Tasks
+{
+    public class GenerateTypeMaps : AndroidTask
+	{
+		public override string TaskPrefix => "GTM";
+
+		[Required]
+		public ITaskItem[] ResolvedAssemblies { get; set; }
+
+		[Required]
+		public ITaskItem[] ResolvedUserAssemblies { get; set; }
+
+		[Required]
+		public ITaskItem [] FrameworkDirectories { get; set; }
+
+		[Required]
+		public string[] SupportedAbis { get; set; }
+
+		[Required]
+		public string OutputDirectory { get; set; }
+
+		[Required]
+		public bool GenerateNativeAssembly { get; set; }
+
+		[Output]
+		public string[] GeneratedBinaryTypeMaps { get; set; }
+
+		public bool ErrorOnCustomJavaObject { get; set; }
+
+		public override bool RunTask ()
+		{
+			try {
+				Run ();
+			} catch (XamarinAndroidException e) {
+				Log.LogCodedError (string.Format ("XA{0:0000}", e.Code), e.MessageWithoutCode);
+				if (MonoAndroidHelper.LogInternalExceptions)
+					Log.LogMessage (e.ToString ());
+			}
+
+			if (Log.HasLoggedErrors) {
+				// Ensure that on a rebuild, we don't *skip* the `_GenerateJavaStubs` target,
+				// by ensuring that the target outputs have been deleted.
+				Files.DeleteFile (Path.Combine (OutputDirectory, "typemap.index"), Log);
+				foreach (string file in Directory.EnumerateFiles (OutputDirectory, "*.typemap")) {
+					Files.DeleteFile (file, Log);
+				}
+			}
+
+			return !Log.HasLoggedErrors;
+		}
+
+		void Run ()
+		{
+			var interestingAssemblies = new HashSet<string> (StringComparer.OrdinalIgnoreCase);
+
+			var res = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: true);
+			foreach (var dir in FrameworkDirectories) {
+				if (Directory.Exists (dir.ItemSpec))
+					res.SearchDirectories.Add (dir.ItemSpec);
+			}
+
+			foreach (ITaskItem assembly in ResolvedAssemblies) {
+				res.Load (assembly.ItemSpec);
+				if (String.Compare ("MonoAndroid", assembly.GetMetadata ("TargetFrameworkIdentifier"), StringComparison.Ordinal) != 0)
+					continue;
+				if (interestingAssemblies.Contains (assembly.ItemSpec))
+					continue;
+				interestingAssemblies.Add (assembly.ItemSpec);
+			}
+
+			var tmg = new TypeMapGenerator ((string message) => Log.LogDebugMessage (message), SupportedAbis);
+			if (!tmg.Generate (res, interestingAssemblies, OutputDirectory, GenerateNativeAssembly))
+				throw new XamarinAndroidException (99999, "Failed to generate type maps");
+			GeneratedBinaryTypeMaps = tmg.GeneratedBinaryTypeMaps.ToArray ();
+		}
+	}
+}

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs

@@ -22,13 +22,14 @@ public sealed class ApplicationConfig
 			public bool   uses_assembly_preload;
 			public bool   is_a_bundled_app;
 			public bool   broken_exception_transitions;
+			public bool   instant_run_enabled;
 			public byte   bound_stream_io_exception_type;
 			public uint   package_naming_policy;
 			public uint   environment_variable_count;
 			public uint   system_property_count;
 			public string android_package_name;
 		};
-		const uint ApplicationConfigFieldCount = 10;
+		const uint ApplicationConfigFieldCount = 11;
 
 		static readonly object ndkInitLock = new object ();
 		static readonly char[] readElfFieldSeparator = new [] { ' ', '\t' };
@@ -49,10 +50,11 @@ public sealed class ApplicationConfig
 			"app_environment_variables",
 			"app_system_properties",
 			"application_config",
-			"jm_typemap",
-			"jm_typemap_header",
-			"mj_typemap",
-			"mj_typemap_header",
+			"map_modules",
+			"map_module_count",
+			"java_type_count",
+			"java_name_width",
+			"map_java",
 			"mono_aot_mode_name",
 		};
 
@@ -136,27 +138,32 @@ static ApplicationConfig ReadApplicationConfig (string envFile)
 						ret.broken_exception_transitions = ConvertFieldToBool ("broken_exception_transitions", envFile, i, field [1]);
 						break;
 
-					case 5: // bound_stream_io_exception_type: byte / .byte
+					case 5: // instant_run_enabled: bool / .byte
+						AssertFieldType (envFile, ".byte", field [0], i);
+						ret.instant_run_enabled = ConvertFieldToBool ("instant_run_enabled", envFile, i, field [1]);
+						break;
+
+					case 6: // bound_stream_io_exception_type: byte / .byte
 						AssertFieldType (envFile, ".byte", field [0], i);
 						ret.bound_stream_io_exception_type = ConvertFieldToByte ("bound_stream_io_exception_type", envFile, i, field [1]);
 						break;
 
-					case 6: // package_naming_policy: uint32_t / .word | .long
+					case 7: // package_naming_policy: uint32_t / .word | .long
 						Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}");
 						ret.package_naming_policy = ConvertFieldToUInt32 ("package_naming_policy", envFile, i, field [1]);
 						break;
 
-					case 7: // environment_variable_count: uint32_t / .word | .long
+					case 8: // environment_variable_count: uint32_t / .word | .long
 						Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}");
 						ret.environment_variable_count = ConvertFieldToUInt32 ("environment_variable_count", envFile, i, field [1]);
 						break;
 
-					case 8: // system_property_count: uint32_t / .word | .long
+					case 9: // system_property_count: uint32_t / .word | .long
 						Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}");
 						ret.system_property_count = ConvertFieldToUInt32 ("system_property_count", envFile, i, field [1]);
 						break;
 
-					case 9: // android_package_name: string / [pointer type]
+					case 10: // android_package_name: string / [pointer type]
 						Assert.IsTrue (expectedPointerTypes.Contains (field [0]), $"Unexpected pointer field type in '{envFile}:{i}': {field [0]}");
 						pointers.Add (field [1].Trim ());
 						break;

src/Xamarin.Android.Build.Tasks/Utilities/ARMNativeAssemblerTargetProvider.cs

@@ -5,9 +5,15 @@ namespace Xamarin.Android.Tasks
 {
 	class ARMNativeAssemblerTargetProvider : NativeAssemblerTargetProvider
 	{
+		const string ARMV7a = "armeabi-v7a";
+		const string ARMV8a = "arm64-v8a";
+
 		public override bool Is64Bit { get; }
 		public override string PointerFieldType { get; }
 		public override string TypePrefix { get; }
+		public override string AbiName => Is64Bit ? ARMV8a : ARMV7a;
+		public override uint MapModulesAlignBits => Is64Bit ? 3u : 2u;
+		public override uint MapJavaAlignBits { get; } = 2;
 
 		public ARMNativeAssemblerTargetProvider (bool is64Bit)
 		{