diff --git a/Microsoft.Toolkit.HighPerformance/Extensions/ArrayExtensions.2D.cs b/Microsoft.Toolkit.HighPerformance/Extensions/ArrayExtensions.2D.cs index 157f36d61c1..a7a4baa075d 100644 --- a/Microsoft.Toolkit.HighPerformance/Extensions/ArrayExtensions.2D.cs +++ b/Microsoft.Toolkit.HighPerformance/Extensions/ArrayExtensions.2D.cs @@ -66,13 +66,13 @@ public static ref T DangerousGetReference(this T[,] array) /// [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref T DangerousGetReferenceAt(this T[,] array, int i, int j) + public static unsafe ref T DangerousGetReferenceAt(this T[,] array, int i, int j) { #if NETCORE_RUNTIME var arrayData = Unsafe.As(array); int offset = (i * arrayData.Width) + j; ref T r0 = ref Unsafe.As(ref arrayData.Data); - ref T ri = ref Unsafe.Add(ref r0, offset); + ref T ri = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)offset); return ref ri; #else @@ -82,10 +82,7 @@ public static ref T DangerousGetReferenceAt(this T[,] array, int i, int j) return ref array[i, j]; } - unsafe - { - return ref Unsafe.AsRef(null); - } + return ref Unsafe.AsRef(null); #endif } @@ -274,11 +271,11 @@ public static Span AsSpan(this T[,] array) /// The number of occurrences of in . [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Count(this T[,] array, T value) + public static unsafe int Count(this T[,] array, T value) where T : IEquatable { ref T r0 = ref array.DangerousGetReference(); - IntPtr length = (IntPtr)array.Length; + IntPtr length = (IntPtr)(void*)(uint)array.Length; return SpanHelper.Count(ref r0, length, value); } @@ -293,11 +290,11 @@ public static int Count(this T[,] array, T value) /// The Djb2 hash is fully deterministic and with no random components. [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetDjb2HashCode(this T[,] array) + public static unsafe int GetDjb2HashCode(this T[,] array) where T : notnull { ref T r0 = ref array.DangerousGetReference(); - IntPtr length = (IntPtr)array.Length; + IntPtr length = (IntPtr)(void*)(uint)array.Length; return SpanHelper.GetDjb2HashCode(ref r0, length); } diff --git a/Microsoft.Toolkit.HighPerformance/Extensions/ArrayExtensions.cs b/Microsoft.Toolkit.HighPerformance/Extensions/ArrayExtensions.cs index d29b2290dca..73e3b8ed73f 100644 --- a/Microsoft.Toolkit.HighPerformance/Extensions/ArrayExtensions.cs +++ b/Microsoft.Toolkit.HighPerformance/Extensions/ArrayExtensions.cs @@ -62,12 +62,12 @@ public static ref T DangerousGetReference(this T[] array) /// This method doesn't do any bounds checks, therefore it is responsibility of the caller to ensure the parameter is valid. [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref T DangerousGetReferenceAt(this T[] array, int i) + public static unsafe ref T DangerousGetReferenceAt(this T[] array, int i) { #if NETCORE_RUNTIME var arrayData = Unsafe.As(array); ref T r0 = ref Unsafe.As(ref arrayData.Data); - ref T ri = ref Unsafe.Add(ref r0, i); + ref T ri = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)i); return ref ri; #else @@ -76,10 +76,7 @@ public static ref T DangerousGetReferenceAt(this T[] array, int i) return ref array[i]; } - unsafe - { - return ref Unsafe.AsRef(null); - } + return ref Unsafe.AsRef(null); #endif } @@ -114,11 +111,11 @@ private sealed class RawArrayData /// The number of occurrences of in . [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Count(this T[] array, T value) + public static unsafe int Count(this T[] array, T value) where T : IEquatable { ref T r0 = ref array.DangerousGetReference(); - IntPtr length = (IntPtr)array.Length; + IntPtr length = (IntPtr)(void*)(uint)array.Length; return SpanHelper.Count(ref r0, length, value); } @@ -185,11 +182,11 @@ public static SpanTokenizer Tokenize(this T[] array, T separator) /// The Djb2 hash is fully deterministic and with no random components. [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetDjb2HashCode(this T[] array) + public static unsafe int GetDjb2HashCode(this T[] array) where T : notnull { ref T r0 = ref array.DangerousGetReference(); - IntPtr length = (IntPtr)array.Length; + IntPtr length = (IntPtr)(void*)(uint)array.Length; return SpanHelper.GetDjb2HashCode(ref r0, length); } diff --git a/Microsoft.Toolkit.HighPerformance/Extensions/ReadOnlySpanExtensions.cs b/Microsoft.Toolkit.HighPerformance/Extensions/ReadOnlySpanExtensions.cs index b2a07767c01..47428db40a2 100644 --- a/Microsoft.Toolkit.HighPerformance/Extensions/ReadOnlySpanExtensions.cs +++ b/Microsoft.Toolkit.HighPerformance/Extensions/ReadOnlySpanExtensions.cs @@ -40,10 +40,40 @@ public static ref T DangerousGetReference(this ReadOnlySpan span) /// This method doesn't do any bounds checks, therefore it is responsibility of the caller to ensure the parameter is valid. [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref T DangerousGetReferenceAt(this ReadOnlySpan span, int i) + public static unsafe ref T DangerousGetReferenceAt(this ReadOnlySpan span, int i) { + // Here we assume the input index will never be negative, so we do an unsafe cast to + // force the JIT to skip the sign extension when going from int to native int. + // On .NET Core 3.1, if we only use Unsafe.Add(ref r0, i), we get the following: + // ============================= + // L0000: mov rax, [rcx] + // L0003: movsxd rdx, edx + // L0006: lea rax, [rax+rdx*4] + // L000a: ret + // ============================= + // Note the movsxd (move with sign extension) to expand the index passed in edx to + // the whole rdx register. This is unnecessary and more expensive than just a mov, + // which when done to a large register size automatically zeroes the upper bits. + // With the (IntPtr)(void*)(uint) cast, we get the following codegen instead: + // ============================= + // L0000: mov rax, [rcx] + // L0003: mov edx, edx + // L0005: lea rax, [rax+rdx*4] + // L0009: ret + // ============================= + // Here we can see how the index is extended to a native integer with just a mov, + // which effectively only zeroes the upper bits of the same register used as source. + // These three casts are a bit verbose, but they do the trick on both 32 bit and 64 + // bit architectures, producing optimal code in both cases (they are either completely + // elided on 32 bit systems, or result in the correct register expansion when on 64 bit). + // We first do an unchecked conversion to uint (which is just a reinterpret-cast). We + // then cast to void*, which lets the following IntPtr cast avoid the range check on 32 bit + // (since uint could be out of range there if the original index was negative). The final + // result is a clean mov as shown above. This will eventually be natively supported by the + // JIT compiler (see https://github.com/dotnet/runtime/issues/38794), but doing this here + // still ensures the optimal codegen even on existing runtimes (eg. .NET Core 2.1 and 3.1). ref T r0 = ref MemoryMarshal.GetReference(span); - ref T ri = ref Unsafe.Add(ref r0, i); + ref T ri = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)i); return ref ri; } @@ -87,7 +117,7 @@ public static ref T DangerousGetReferenceAt(this ReadOnlySpan span, int i) /// [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref readonly T DangerousGetLookupReferenceAt(this ReadOnlySpan span, int i) + public static unsafe ref readonly T DangerousGetLookupReferenceAt(this ReadOnlySpan span, int i) { // Check whether the input is in range by first casting both // operands to uint and then comparing them, as this allows @@ -106,12 +136,12 @@ public static ref readonly T DangerousGetLookupReferenceAt(this ReadOnlySpan< // lookup table can just be assumed to always be false. bool isInRange = (uint)i < (uint)span.Length; byte rangeFlag = Unsafe.As(ref isInRange); - int - negativeFlag = rangeFlag - 1, + uint + negativeFlag = unchecked(rangeFlag - 1u), mask = ~negativeFlag, - offset = i & mask; + offset = (uint)i & mask; ref T r0 = ref MemoryMarshal.GetReference(span); - ref T r1 = ref Unsafe.Add(ref r0, offset); + ref T r1 = ref Unsafe.Add(ref r0, (IntPtr)(void*)offset); return ref r1; } @@ -165,11 +195,11 @@ public static unsafe int IndexOf(this ReadOnlySpan span, in T value) /// The number of occurrences of in . [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Count(this ReadOnlySpan span, T value) + public static unsafe int Count(this ReadOnlySpan span, T value) where T : IEquatable { ref T r0 = ref MemoryMarshal.GetReference(span); - IntPtr length = (IntPtr)span.Length; + IntPtr length = (IntPtr)(void*)(uint)span.Length; return SpanHelper.Count(ref r0, length, value); } @@ -291,11 +321,11 @@ public static ReadOnlySpanTokenizer Tokenize(this ReadOnlySpan span, T /// The Djb2 hash is fully deterministic and with no random components. [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetDjb2HashCode(this ReadOnlySpan span) + public static unsafe int GetDjb2HashCode(this ReadOnlySpan span) where T : notnull { ref T r0 = ref MemoryMarshal.GetReference(span); - IntPtr length = (IntPtr)span.Length; + IntPtr length = (IntPtr)(void*)(uint)span.Length; return SpanHelper.GetDjb2HashCode(ref r0, length); } diff --git a/Microsoft.Toolkit.HighPerformance/Extensions/SpanExtensions.cs b/Microsoft.Toolkit.HighPerformance/Extensions/SpanExtensions.cs index 8ce1cca75a0..52dac209876 100644 --- a/Microsoft.Toolkit.HighPerformance/Extensions/SpanExtensions.cs +++ b/Microsoft.Toolkit.HighPerformance/Extensions/SpanExtensions.cs @@ -40,10 +40,10 @@ public static ref T DangerousGetReference(this Span span) /// This method doesn't do any bounds checks, therefore it is responsibility of the caller to ensure the parameter is valid. [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref T DangerousGetReferenceAt(this Span span, int i) + public static unsafe ref T DangerousGetReferenceAt(this Span span, int i) { ref T r0 = ref MemoryMarshal.GetReference(span); - ref T ri = ref Unsafe.Add(ref r0, i); + ref T ri = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)i); return ref ri; } @@ -140,11 +140,11 @@ public static unsafe int IndexOf(this Span span, ref T value) /// The number of occurrences of in . [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Count(this Span span, T value) + public static unsafe int Count(this Span span, T value) where T : IEquatable { ref T r0 = ref MemoryMarshal.GetReference(span); - IntPtr length = (IntPtr)span.Length; + IntPtr length = (IntPtr)(void*)(uint)span.Length; return SpanHelper.Count(ref r0, length, value); } @@ -211,11 +211,11 @@ public static SpanTokenizer Tokenize(this Span span, T separator) /// The Djb2 hash is fully deterministic and with no random components. [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetDjb2HashCode(this Span span) + public static unsafe int GetDjb2HashCode(this Span span) where T : notnull { ref T r0 = ref MemoryMarshal.GetReference(span); - IntPtr length = (IntPtr)span.Length; + IntPtr length = (IntPtr)(void*)(uint)span.Length; return SpanHelper.GetDjb2HashCode(ref r0, length); } diff --git a/Microsoft.Toolkit.HighPerformance/Extensions/StringExtensions.cs b/Microsoft.Toolkit.HighPerformance/Extensions/StringExtensions.cs index 74ba2edd2fd..c9cdc114c36 100644 --- a/Microsoft.Toolkit.HighPerformance/Extensions/StringExtensions.cs +++ b/Microsoft.Toolkit.HighPerformance/Extensions/StringExtensions.cs @@ -48,7 +48,7 @@ public static ref char DangerousGetReference(this string text) /// This method doesn't do any bounds checks, therefore it is responsibility of the caller to ensure the parameter is valid. [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref char DangerousGetReferenceAt(this string text, int i) + public static unsafe ref char DangerousGetReferenceAt(this string text, int i) { #if NETCOREAPP3_1 ref char r0 = ref Unsafe.AsRef(text.GetPinnableReference()); @@ -57,7 +57,7 @@ public static ref char DangerousGetReferenceAt(this string text, int i) #else ref char r0 = ref MemoryMarshal.GetReference(text.AsSpan()); #endif - ref char ri = ref Unsafe.Add(ref r0, i); + ref char ri = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)i); return ref ri; } @@ -91,10 +91,10 @@ private sealed class RawStringData /// The number of occurrences of in . [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Count(this string text, char c) + public static unsafe int Count(this string text, char c) { ref char r0 = ref text.DangerousGetReference(); - IntPtr length = (IntPtr)text.Length; + IntPtr length = (IntPtr)(void*)(uint)text.Length; return SpanHelper.Count(ref r0, length, c); } @@ -157,10 +157,10 @@ public static ReadOnlySpanTokenizer Tokenize(this string text, char separa /// The Djb2 hash is fully deterministic and with no random components. [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetDjb2HashCode(this string text) + public static unsafe int GetDjb2HashCode(this string text) { ref char r0 = ref text.DangerousGetReference(); - IntPtr length = (IntPtr)text.Length; + IntPtr length = (IntPtr)(void*)(uint)text.Length; return SpanHelper.GetDjb2HashCode(ref r0, length); } diff --git a/Microsoft.Toolkit.HighPerformance/Helpers/HashCode{T}.cs b/Microsoft.Toolkit.HighPerformance/Helpers/HashCode{T}.cs index c2f96561722..3c2a43c32a4 100644 --- a/Microsoft.Toolkit.HighPerformance/Helpers/HashCode{T}.cs +++ b/Microsoft.Toolkit.HighPerformance/Helpers/HashCode{T}.cs @@ -57,7 +57,7 @@ public static int Combine(ReadOnlySpan span) /// The returned hash code is not processed through APIs. [Pure] [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int CombineValues(ReadOnlySpan span) + internal static unsafe int CombineValues(ReadOnlySpan span) { ref T r0 = ref MemoryMarshal.GetReference(span); @@ -67,13 +67,19 @@ internal static int CombineValues(ReadOnlySpan span) // compiler, so this branch will never actually be executed by the code. if (RuntimeHelpers.IsReferenceOrContainsReferences()) { - return SpanHelper.GetDjb2HashCode(ref r0, (IntPtr)span.Length); + return SpanHelper.GetDjb2HashCode(ref r0, (IntPtr)(void*)(uint)span.Length); } #endif - // Get the info for the target memory area to process + // Get the info for the target memory area to process. + // The line below is computing the total byte size for the span, + // and we cast both input factors to uint first to avoid sign extensions + // (they're both guaranteed to always be positive values), and to let the + // JIT avoid the 64 bit computation entirely when running in a 32 bit + // process. In that case it will just compute the byte size as a 32 bit + // multiplication with overflow, which is guaranteed never to happen anyway. ref byte rb = ref Unsafe.As(ref r0); - IntPtr length = (IntPtr)((long)span.Length * Unsafe.SizeOf()); + IntPtr length = (IntPtr)(void*)((uint)span.Length * (uint)Unsafe.SizeOf()); return SpanHelper.GetDjb2LikeByteHash(ref rb, length); } diff --git a/Microsoft.Toolkit.HighPerformance/Helpers/ParallelHelper.ForEach.IInAction.cs b/Microsoft.Toolkit.HighPerformance/Helpers/ParallelHelper.ForEach.IInAction.cs index d2a9be06eb6..79c72ca122b 100644 --- a/Microsoft.Toolkit.HighPerformance/Helpers/ParallelHelper.ForEach.IInAction.cs +++ b/Microsoft.Toolkit.HighPerformance/Helpers/ParallelHelper.ForEach.IInAction.cs @@ -136,7 +136,7 @@ public InActionInvoker( /// /// The index of the batch to process [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Invoke(int i) + public unsafe void Invoke(int i) { int low = i * this.batchSize, @@ -147,7 +147,7 @@ public void Invoke(int i) for (int j = low; j < end; j++) { - ref TItem rj = ref Unsafe.Add(ref r0, j); + ref TItem rj = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)j); Unsafe.AsRef(this.action).Invoke(rj); } diff --git a/Microsoft.Toolkit.HighPerformance/Helpers/ParallelHelper.ForEach.IRefAction.cs b/Microsoft.Toolkit.HighPerformance/Helpers/ParallelHelper.ForEach.IRefAction.cs index e6dae6624d8..18802dd0100 100644 --- a/Microsoft.Toolkit.HighPerformance/Helpers/ParallelHelper.ForEach.IRefAction.cs +++ b/Microsoft.Toolkit.HighPerformance/Helpers/ParallelHelper.ForEach.IRefAction.cs @@ -136,7 +136,7 @@ public RefActionInvoker( /// /// The index of the batch to process [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Invoke(int i) + public unsafe void Invoke(int i) { int low = i * this.batchSize, @@ -147,7 +147,7 @@ public void Invoke(int i) for (int j = low; j < end; j++) { - ref TItem rj = ref Unsafe.Add(ref r0, j); + ref TItem rj = ref Unsafe.Add(ref r0, (IntPtr)(void*)(uint)j); Unsafe.AsRef(this.action).Invoke(ref rj); } diff --git a/Microsoft.Toolkit.HighPerformance/Microsoft.Toolkit.HighPerformance.csproj b/Microsoft.Toolkit.HighPerformance/Microsoft.Toolkit.HighPerformance.csproj index a0d6bcb991c..7f2d3956c53 100644 --- a/Microsoft.Toolkit.HighPerformance/Microsoft.Toolkit.HighPerformance.csproj +++ b/Microsoft.Toolkit.HighPerformance/Microsoft.Toolkit.HighPerformance.csproj @@ -12,6 +12,7 @@ - MemoryBufferWriter<T>: an IBufferWriter<T>: implementation that can wrap external Memory<T>: instances. - MemoryOwner<T>: an IMemoryOwner<T> implementation with an embedded length and a fast Span<T> accessor. - SpanOwner<T>: a stack-only type with the ability to rent a buffer of a specified length and getting a Span<T> from it. + - StringPool: a configurable pool for string instances that be used to minimize allocations when creating multiple strings from char buffers. - String, array, Span<T>, Memory<T> extensions and more, all focused on high performance. - HashCode<T>: a SIMD-enabled extension of HashCode to quickly process sequences of values. - BitHelper: a class with helper methods to perform bit operations on numeric types.