Skip to content

Create vector constants inline and not via ROS<byte> #2419

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Mar 30, 2023
Merged

Create vector constants inline and not via ROS<byte> #2419

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
77ffeea
Create vector constants inline and not via ROS<byte>
gfoidl Mar 26, 2023
cdf6eed
Fixed build
gfoidl Mar 26, 2023
97ac149
Merge branch 'main' into vector-constants
gfoidl Mar 27, 2023
d617907
Merge branch 'main' into vector-constants
JimBobSquarePants Mar 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 54 additions & 57 deletions src/ImageSharp/Common/Helpers/SimdUtils.HwIntrinsics.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,33 +14,38 @@ internal static partial class SimdUtils
{
public static class HwIntrinsics
{
public static ReadOnlySpan<byte> PermuteMaskDeinterleave8x32 => new byte[] { 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0 };
[MethodImpl(MethodImplOptions.AggressiveInlining)] // too much IL for JIT to inline, so give a hint
public static Vector256<int> PermuteMaskDeinterleave8x32() => Vector256.Create(0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0).AsInt32();

public static ReadOnlySpan<byte> PermuteMaskEvenOdd8x32 => new byte[] { 0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0 };
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<uint> PermuteMaskEvenOdd8x32() => Vector256.Create(0, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0).AsUInt32();

public static ReadOnlySpan<byte> PermuteMaskSwitchInnerDWords8x32 => new byte[] { 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0 };
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<uint> PermuteMaskSwitchInnerDWords8x32() => Vector256.Create(0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0).AsUInt32();

private static ReadOnlySpan<byte> MoveFirst24BytesToSeparateLanes => new byte[] { 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0 };
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector256<uint> MoveFirst24BytesToSeparateLanes() => Vector256.Create(0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 7, 0, 0, 0).AsUInt32();

internal static ReadOnlySpan<byte> ExtractRgb => new byte[] { 0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, 0xFF, 0xFF, 0xFF, 0xFF };
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static Vector256<byte> ExtractRgb() => Vector256.Create(0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11, 0xFF, 0xFF, 0xFF, 0xFF);

private static ReadOnlySpan<byte> ShuffleMaskPad4Nx16 => new byte[] { 0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80 };
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> ShuffleMaskPad4Nx16() => Vector128.Create(0, 1, 2, 0x80, 3, 4, 5, 0x80, 6, 7, 8, 0x80, 9, 10, 11, 0x80);

private static ReadOnlySpan<byte> ShuffleMaskSlice4Nx16 => new byte[] { 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80 };
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> ShuffleMaskSlice4Nx16() => Vector128.Create(0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 0x80, 0x80, 0x80, 0x80);

private static ReadOnlySpan<byte> ShuffleMaskShiftAlpha =>
new byte[]
{
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15,
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15
};
#pragma warning disable SA1003, SA1116, SA1117 // Parameters should be on same line or separate lines
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector256<byte> ShuffleMaskShiftAlpha() => Vector256.Create((byte)
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15,
0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 3, 7, 11, 15);

public static ReadOnlySpan<byte> PermuteMaskShiftAlpha8x32 =>
new byte[]
{
0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0,
5, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0
};
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector256<uint> PermuteMaskShiftAlpha8x32() => Vector256.Create(
0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0,
5, 0, 0, 0, 6, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0).AsUInt32();
#pragma warning restore SA1003, SA1116, SA1117 // Parameters should be on same line or separate lines

/// <summary>
/// Shuffle single-precision (32-bit) floating-point elements in <paramref name="source"/>
Expand Down Expand Up @@ -190,7 +195,7 @@ public static void Shuffle4Slice3Reduce(
{
if (Ssse3.IsSupported)
{
int remainder = source.Length % (Vector128<byte>.Count * 4);
int remainder = source.Length & ((Vector128<byte>.Count * 4) - 1); // bit-hack for modulo

int sourceCount = source.Length - remainder;
int destCount = (int)((uint)sourceCount * 3 / 4);
Expand Down Expand Up @@ -254,7 +259,7 @@ private static void Shuffle4(
ref Vector128<float> destBase =
ref Unsafe.As<float, Vector128<float>>(ref MemoryMarshal.GetReference(dest));

nint n = (nint)(uint)dest.Length / Vector128<float>.Count;
nint n = (nint)((uint)dest.Length / (uint)Vector128<float>.Count);
nint m = Numerics.Modulo4(n);
nint u = n - m;

Expand Down Expand Up @@ -307,7 +312,7 @@ private static void Shuffle4(
ref Vector256<byte> destBase =
ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(dest));

nint n = (nint)(uint)dest.Length / Vector256<byte>.Count;
nint n = (nint)((uint)dest.Length / (uint)Vector256<byte>.Count);
nint m = Numerics.Modulo4(n);
nint u = n - m;

Expand Down Expand Up @@ -343,7 +348,7 @@ private static void Shuffle4(
ref Vector128<byte> destBase =
ref Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(dest));

nint n = (nint)(uint)dest.Length / Vector128<byte>.Count;
nint n = (nint)((uint)dest.Length / (uint)Vector128<byte>.Count);
nint m = Numerics.Modulo4(n);
nint u = n - m;

Expand Down Expand Up @@ -376,10 +381,8 @@ private static void Shuffle3(
{
if (Ssse3.IsSupported)
{
ref byte vmaskBase = ref MemoryMarshal.GetReference(ShuffleMaskPad4Nx16);
Vector128<byte> vmask = Unsafe.As<byte, Vector128<byte>>(ref vmaskBase);
ref byte vmaskoBase = ref MemoryMarshal.GetReference(ShuffleMaskSlice4Nx16);
Vector128<byte> vmasko = Unsafe.As<byte, Vector128<byte>>(ref vmaskoBase);
Vector128<byte> vmask = ShuffleMaskPad4Nx16();
Vector128<byte> vmasko = ShuffleMaskSlice4Nx16();
Vector128<byte> vmaske = Ssse3.AlignRight(vmasko, vmasko, 12);

Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
Expand Down Expand Up @@ -441,8 +444,7 @@ private static void Pad3Shuffle4(
{
if (Ssse3.IsSupported)
{
ref byte vmaskBase = ref MemoryMarshal.GetReference(ShuffleMaskPad4Nx16);
Vector128<byte> vmask = Unsafe.As<byte, Vector128<byte>>(ref vmaskBase);
Vector128<byte> vmask = ShuffleMaskPad4Nx16();
Vector128<byte> vfill = Vector128.Create(0xff000000ff000000ul).AsByte();

Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
Expand Down Expand Up @@ -485,8 +487,7 @@ private static void Shuffle4Slice3(
{
if (Ssse3.IsSupported)
{
ref byte vmaskoBase = ref MemoryMarshal.GetReference(ShuffleMaskSlice4Nx16);
Vector128<byte> vmasko = Unsafe.As<byte, Vector128<byte>>(ref vmaskoBase);
Vector128<byte> vmasko = ShuffleMaskSlice4Nx16();
Vector128<byte> vmaske = Ssse3.AlignRight(vmasko, vmasko, 12);

Span<byte> bytes = stackalloc byte[Vector128<byte>.Count];
Expand Down Expand Up @@ -543,9 +544,9 @@ private static void Shuffle4Slice3(
/// <returns>The <see cref="Vector256{T}"/>.</returns>
[MethodImpl(InliningOptions.AlwaysInline)]
public static Vector256<float> MultiplyAdd(
in Vector256<float> va,
in Vector256<float> vm0,
in Vector256<float> vm1)
Vector256<float> va,
Vector256<float> vm0,
Vector256<float> vm1)
{
if (Fma.IsSupported)
{
Expand Down Expand Up @@ -594,9 +595,9 @@ public static Vector128<float> MultiplyAdd(
/// <returns>The <see cref="Vector256{T}"/>.</returns>
[MethodImpl(InliningOptions.ShortMethod)]
public static Vector256<float> MultiplySubtract(
in Vector256<float> vs,
in Vector256<float> vm0,
in Vector256<float> vm1)
Vector256<float> vs,
Vector256<float> vm0,
Vector256<float> vm1)
{
if (Fma.IsSupported)
{
Expand All @@ -616,9 +617,9 @@ public static Vector256<float> MultiplySubtract(
/// <returns>The <see cref="Vector256{T}"/>.</returns>
[MethodImpl(InliningOptions.ShortMethod)]
public static Vector256<float> MultiplyAddNegated(
in Vector256<float> a,
in Vector256<float> b,
in Vector256<float> c)
Vector256<float> a,
Vector256<float> b,
Vector256<float> c)
{
if (Fma.IsSupported)
{
Expand Down Expand Up @@ -684,7 +685,7 @@ internal static unsafe void ByteToNormalizedFloat(
ref Vector256<float> destBase =
ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(dest));

var scale = Vector256.Create(1 / (float)byte.MaxValue);
Vector256<float> scale = Vector256.Create(1 / (float)byte.MaxValue);

for (nuint i = 0; i < n; i++)
{
Expand Down Expand Up @@ -717,7 +718,7 @@ internal static unsafe void ByteToNormalizedFloat(
ref Vector128<float> destBase =
ref Unsafe.As<float, Vector128<float>>(ref MemoryMarshal.GetReference(dest));

var scale = Vector128.Create(1 / (float)byte.MaxValue);
Vector128<float> scale = Vector128.Create(1 / (float)byte.MaxValue);
Vector128<byte> zero = Vector128<byte>.Zero;

for (nuint i = 0; i < n; i++)
Expand Down Expand Up @@ -819,9 +820,8 @@ internal static void NormalizedFloatToByteSaturate(
ref Vector256<byte> destBase =
ref Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(dest));

var scale = Vector256.Create((float)byte.MaxValue);
ref byte maskBase = ref MemoryMarshal.GetReference(PermuteMaskDeinterleave8x32);
Vector256<int> mask = Unsafe.As<byte, Vector256<int>>(ref maskBase);
Vector256<float> scale = Vector256.Create((float)byte.MaxValue);
Vector256<int> mask = PermuteMaskDeinterleave8x32();

for (nuint i = 0; i < n; i++)
{
Expand Down Expand Up @@ -858,7 +858,7 @@ internal static void NormalizedFloatToByteSaturate(
ref Vector128<byte> destBase =
ref Unsafe.As<byte, Vector128<byte>>(ref MemoryMarshal.GetReference(dest));

var scale = Vector128.Create((float)byte.MaxValue);
Vector128<float> scale = Vector128.Create((float)byte.MaxValue);

for (nuint i = 0; i < n; i++)
{
Expand Down Expand Up @@ -895,14 +895,12 @@ internal static void PackFromRgbPlanesAvx2Reduce(

nuint count = redChannel.Vector256Count<byte>();

ref byte control1Bytes = ref MemoryMarshal.GetReference(PermuteMaskEvenOdd8x32);
Vector256<uint> control1 = Unsafe.As<byte, Vector256<uint>>(ref control1Bytes);
Vector256<uint> control1 = PermuteMaskEvenOdd8x32();

ref byte control2Bytes = ref MemoryMarshal.GetReference(PermuteMaskShiftAlpha8x32);
Vector256<uint> control2 = Unsafe.As<byte, Vector256<uint>>(ref control2Bytes);
var a = Vector256.Create((byte)255);
Vector256<uint> control2 = PermuteMaskShiftAlpha8x32();
Vector256<byte> a = Vector256.Create((byte)255);

Vector256<byte> shuffleAlpha = Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(ShuffleMaskShiftAlpha));
Vector256<byte> shuffleAlpha = ShuffleMaskShiftAlpha();

for (nuint i = 0; i < count; i++)
{
Expand Down Expand Up @@ -966,9 +964,8 @@ internal static void PackFromRgbPlanesAvx2Reduce(
ref Vector256<byte> dBase = ref Unsafe.As<Rgba32, Vector256<byte>>(ref MemoryMarshal.GetReference(destination));

nuint count = redChannel.Vector256Count<byte>();
ref byte control1Bytes = ref MemoryMarshal.GetReference(PermuteMaskEvenOdd8x32);
Vector256<uint> control1 = Unsafe.As<byte, Vector256<uint>>(ref control1Bytes);
var a = Vector256.Create((byte)255);
Vector256<uint> control1 = PermuteMaskEvenOdd8x32();
Vector256<byte> a = Vector256.Create((byte)255);

for (nuint i = 0; i < count; i++)
{
Expand Down Expand Up @@ -1017,8 +1014,8 @@ internal static void UnpackToRgbPlanesAvx2Reduce(
ref Vector256<float> destGRef = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(greenChannel));
ref Vector256<float> destBRef = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(blueChannel));

Vector256<uint> extractToLanesMask = Unsafe.As<byte, Vector256<uint>>(ref MemoryMarshal.GetReference(MoveFirst24BytesToSeparateLanes));
Vector256<byte> extractRgbMask = Unsafe.As<byte, Vector256<byte>>(ref MemoryMarshal.GetReference(ExtractRgb));
Vector256<uint> extractToLanesMask = MoveFirst24BytesToSeparateLanes();
Vector256<byte> extractRgbMask = ExtractRgb();
Vector256<byte> rgb, rg, bx;
Vector256<float> r, g, b;

Expand Down
3 changes: 1 addition & 2 deletions tests/ImageSharp.Benchmarks/General/PixelConversion/PixelConversion_PackFromRgbPlanes.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,8 +207,7 @@ public void Rgba32_Avx2_Float()

nuint count = (uint)this.Count / (uint)Vector256<float>.Count;

ref byte control = ref MemoryMarshal.GetReference(SimdUtils.HwIntrinsics.PermuteMaskEvenOdd8x32);
Vector256<int> vcontrol = Unsafe.As<byte, Vector256<int>>(ref control);
Vector256<int> vcontrol = SimdUtils.HwIntrinsics.PermuteMaskEvenOdd8x32().AsInt32();

var va = Vector256.Create(1F);

Expand Down