Improve perf of Enumerable.Sum/Average/Max/Min for arrays and lists (#64624)

* Improve perf of Enumerable.Sum/Average/Max/Min for arrays and lists

It's very common to use these terminal functions for quick stats on arrays and lists of values.  Just the overhead of enumerating as an enumerable (involving multiple interface dispatch) per iteration is significant, and it's much faster to directly enumerate the contents of the array or the list.  In some cases, we can further use vectorization to speed up the processing.

This change:
- Adds a helper that does a fast check to see if it can extract a span from an enumerable that's actually an array or a list.  It could be augmented to detect other interesting types, but `T[]` and `List<T>` are the most relevant from the data I've seen, and we can fairly quickly do type checks to get the most benefit for a small amount of cost.
- Uses that helper in the int/long/float/double/decimal overloads of Sum/Average/Min/Max to add a span-based path.
- Vectorizes Sum for float and double
- Vectorizes Average for int, float, and double (the latter two via use of Sum)

* Address PR feedback

Author: stephentoub

src/libraries/System.Linq/src/System.Linq.csproj

@@ -100,6 +100,8 @@
     <Reference Include="System.Memory" />
     <Reference Include="System.Numerics.Vectors" />
     <Reference Include="System.Runtime" />
+    <Reference Include="System.Runtime.CompilerServices.Unsafe" />
     <Reference Include="System.Runtime.Extensions" />
+    <Reference Include="System.Runtime.InteropServices" />
   </ItemGroup>
 </Project>

src/libraries/System.Linq/src/System/Linq/Average.cs

@@ -2,16 +2,17 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Collections.Generic;
+using System.Numerics;
 
 namespace System.Linq
 {
     public static partial class Enumerable
     {
         public static double Average(this IEnumerable<int> source)
         {
-            if (source == null)
+            if (source.TryGetSpan(out ReadOnlySpan<int> span))
             {
-                ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source);
+                return Average(span);
             }
 
             using (IEnumerator<int> e = source.GetEnumerator())
@@ -36,6 +37,38 @@ public static double Average(this IEnumerable<int> source)
             }
         }
 
+        private static double Average(ReadOnlySpan<int> span)
+        {
+            if (span.IsEmpty)
+            {
+                ThrowHelper.ThrowNoElementsException();
+            }
+
+            long sum = 0;
+            int i = 0;
+
+            if (Vector.IsHardwareAccelerated && span.Length >= Vector<int>.Count)
+            {
+                Vector<long> sums = default;
+                do
+                {
+                    Vector.Widen(new Vector<int>(span.Slice(i)), out Vector<long> low, out Vector<long> high);
+                    sums += low;
+                    sums += high;
+                    i += Vector<int>.Count;
+                }
+                while (i <= span.Length - Vector<int>.Count);
+                sum += Vector.Sum(sums);
+            }
+
+            for (; (uint)i < (uint)span.Length; i++)
+            {
+                sum += span[i];
+            }
+
+            return (double)sum / span.Length;
+        }
+
         public static double? Average(this IEnumerable<int?> source)
         {
             if (source == null)
@@ -75,9 +108,9 @@ public static double Average(this IEnumerable<int> source)
 
         public static double Average(this IEnumerable<long> source)
         {
-            if (source == null)
+            if (source.TryGetSpan(out ReadOnlySpan<long> span))
             {
-                ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source);
+                return Average(span);
             }
 
             using (IEnumerator<long> e = source.GetEnumerator())
@@ -102,6 +135,22 @@ public static double Average(this IEnumerable<long> source)
             }
         }
 
+        private static double Average(ReadOnlySpan<long> span)
+        {
+            if (span.IsEmpty)
+            {
+                ThrowHelper.ThrowNoElementsException();
+            }
+
+            long sum = span[0];
+            for (int i = 1; i < span.Length; i++)
+            {
+                checked { sum += span[i]; }
+            }
+
+            return (double)sum / span.Length;
+        }
+
         public static double? Average(this IEnumerable<long?> source)
         {
             if (source == null)
@@ -141,9 +190,14 @@ public static double Average(this IEnumerable<long> source)
 
         public static float Average(this IEnumerable<float> source)
         {
-            if (source == null)
+            if (source.TryGetSpan(out ReadOnlySpan<float> span))
             {
-                ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source);
+                if (span.IsEmpty)
+                {
+                    ThrowHelper.ThrowNoElementsException();
+                }
+
+                return (float)(Sum(span) / span.Length);
             }
 
             using (IEnumerator<float> e = source.GetEnumerator())
@@ -204,9 +258,14 @@ public static float Average(this IEnumerable<float> source)
 
         public static double Average(this IEnumerable<double> source)
         {
-            if (source == null)
+            if (source.TryGetSpan(out ReadOnlySpan<double> span))
             {
-                ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source);
+                if (span.IsEmpty)
+                {
+                    ThrowHelper.ThrowNoElementsException();
+                }
+
+                return Sum(span) / span.Length;
             }
 
             using (IEnumerator<double> e = source.GetEnumerator())
@@ -270,9 +329,14 @@ public static double Average(this IEnumerable<double> source)
 
         public static decimal Average(this IEnumerable<decimal> source)
         {
-            if (source == null)
+            if (source.TryGetSpan(out ReadOnlySpan<decimal> span))
             {
-                ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source);
+                if (span.IsEmpty)
+                {
+                    ThrowHelper.ThrowNoElementsException();
+                }
+
+                return Sum(span) / span.Length;
             }
 
             using (IEnumerator<decimal> e = source.GetEnumerator())

src/libraries/System.Linq/src/System/Linq/Enumerable.cs

@@ -2,11 +2,52 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 
 namespace System.Linq
 {
     public static partial class Enumerable
     {
         public static IEnumerable<TSource> AsEnumerable<TSource>(this IEnumerable<TSource> source) => source;
+
+        /// <summary>Validates that source is not null and then tries to extract a span from the source.</summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)] // fast type checks that don't add a lot of overhead
+        private static bool TryGetSpan<TSource>(this IEnumerable<TSource> source, out ReadOnlySpan<TSource> span)
+            // This constraint isn't required, but the overheads involved here can be more substantial when TSource
+            // is a reference type and generic implementations are shared.  So for now we're protecting ourselves
+            // and forcing a conscious choice to remove this in the future, at which point it should be paired with
+            // sufficient performance testing.
+            where TSource : struct
+        {
+            if (source is null)
+            {
+                ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source);
+            }
+
+            // Use `GetType() == typeof(...)` rather than `is` to avoid cast helpers.  This is measurably cheaper
+            // but does mean we could end up missing some rare cases where we could get a span but don't (e.g. a uint[]
+            // masquerading as an int[]).  That's an acceptable tradeoff.  The Unsafe usage is only after we've
+            // validated the exact type; this could be changed to a cast in the future if the JIT starts to recognize it.
+            // We only pay the comparison/branching costs here for super common types we expect to be used frequently
+            // with LINQ methods.
+
+            bool result = true;
+            if (source.GetType() == typeof(TSource[]))
+            {
+                span = Unsafe.As<TSource[]>(source);
+            }
+            else if (source.GetType() == typeof(List<TSource>))
+            {
+                span = CollectionsMarshal.AsSpan(Unsafe.As<List<TSource>>(source));
+            }
+            else
+            {
+                span = default;
+                result = false;
+            }
+
+            return result;
+        }
     }
 }