// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System.Diagnostics; using System.Runtime.CompilerServices; using System.Numerics; #if !netstandard using Internal.Runtime.CompilerServices; #endif #if BIT64 using nuint = System.UInt64; #else using nuint = System.UInt32; #endif namespace System { internal static partial class SpanHelpers // .Char { public static int IndexOf(ref char searchSpace, int searchSpaceLength, ref char value, int valueLength) { Debug.Assert(searchSpaceLength >= 0); Debug.Assert(valueLength >= 0); if (valueLength == 0) return 0; // A zero-length sequence is always treated as "found" at the start of the search space. char valueHead = value; ref char valueTail = ref Unsafe.Add(ref value, 1); int valueTailLength = valueLength - 1; int remainingSearchSpaceLength = searchSpaceLength - valueTailLength; int index = 0; while (remainingSearchSpaceLength > 0) { // Do a quick search for the first element of "value". int relativeIndex = IndexOf(ref Unsafe.Add(ref searchSpace, index), valueHead, remainingSearchSpaceLength); if (relativeIndex == -1) break; remainingSearchSpaceLength -= relativeIndex; index += relativeIndex; if (remainingSearchSpaceLength <= 0) break; // The unsearched portion is now shorter than the sequence we're looking for. So it can't be there. // Found the first element of "value". See if the tail matches. if (SequenceEqual( ref Unsafe.As(ref Unsafe.Add(ref searchSpace, index + 1)), ref Unsafe.As(ref valueTail), (nuint)valueTailLength * 2)) { return index; // The tail matched. Return a successful find. } remainingSearchSpaceLength--; index++; } return -1; } public static unsafe int SequenceCompareTo(ref char first, int firstLength, ref char second, int secondLength) { Debug.Assert(firstLength >= 0); Debug.Assert(secondLength >= 0); int lengthDelta = firstLength - secondLength; if (Unsafe.AreSame(ref first, ref second)) goto Equal; IntPtr minLength = (IntPtr)((firstLength < secondLength) ? firstLength : secondLength); IntPtr i = (IntPtr)0; // Use IntPtr for arithmetic to avoid unnecessary 64->32->64 truncations if ((byte*)minLength >= (byte*)(sizeof(UIntPtr) / sizeof(char))) { if (Vector.IsHardwareAccelerated && (byte*)minLength >= (byte*)Vector.Count) { IntPtr nLength = minLength - Vector.Count; do { if (Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref first, i))) != Unsafe.ReadUnaligned>(ref Unsafe.As(ref Unsafe.Add(ref second, i)))) { break; } i += Vector.Count; } while ((byte*)nLength >= (byte*)i); } while ((byte*)minLength >= (byte*)(i + sizeof(UIntPtr) / sizeof(char))) { if (Unsafe.ReadUnaligned(ref Unsafe.As(ref Unsafe.Add(ref first, i))) != Unsafe.ReadUnaligned(ref Unsafe.As(ref Unsafe.Add(ref second, i)))) { break; } i += sizeof(UIntPtr) / sizeof(char); } } if (sizeof(UIntPtr) > sizeof(int) && (byte*)minLength >= (byte*)(i + sizeof(int) / sizeof(char))) { if (Unsafe.ReadUnaligned(ref Unsafe.As(ref Unsafe.Add(ref first, i))) == Unsafe.ReadUnaligned(ref Unsafe.As(ref Unsafe.Add(ref second, i)))) { i += sizeof(int) / sizeof(char); } } while ((byte*)i < (byte*)minLength) { int result = Unsafe.Add(ref first, i).CompareTo(Unsafe.Add(ref second, i)); if (result != 0) return result; i += 1; } Equal: return lengthDelta; } // Adapted from IndexOf(...) public static unsafe bool Contains(ref char searchSpace, char value, int length) { Debug.Assert(length >= 0); fixed (char* pChars = &searchSpace) { char* pCh = pChars; char* pEndCh = pCh + length; if (Vector.IsHardwareAccelerated && length >= Vector.Count * 2) { // Figure out how many characters to read sequentially until we are vector aligned // This is equivalent to: // unaligned = ((int)pCh % Unsafe.SizeOf>()) / elementsPerByte // length = (Vector.Count - unaligned) % Vector.Count const int elementsPerByte = sizeof(ushort) / sizeof(byte); int unaligned = ((int)pCh & (Unsafe.SizeOf>() - 1)) / elementsPerByte; length = (Vector.Count - unaligned) & (Vector.Count - 1); } SequentialScan: while (length >= 4) { length -= 4; if (value == *pCh || value == *(pCh + 1) || value == *(pCh + 2) || value == *(pCh + 3)) { goto Found; } pCh += 4; } while (length > 0) { length -= 1; if (value == *pCh) goto Found; pCh += 1; } // We get past SequentialScan only if IsHardwareAccelerated is true. However, we still have the redundant check to allow // the JIT to see that the code is unreachable and eliminate it when the platform does not have hardware accelerated. if (Vector.IsHardwareAccelerated && pCh < pEndCh) { // Get the highest multiple of Vector.Count that is within the search space. // That will be how many times we iterate in the loop below. // This is equivalent to: length = Vector.Count * ((int)(pEndCh - pCh) / Vector.Count) length = (int)((pEndCh - pCh) & ~(Vector.Count - 1)); // Get comparison Vector Vector vComparison = new Vector(value); while (length > 0) { // Using Unsafe.Read instead of ReadUnaligned since the search space is pinned and pCh is always vector aligned Debug.Assert(((int)pCh & (Unsafe.SizeOf>() - 1)) == 0); Vector vMatches = Vector.Equals(vComparison, Unsafe.Read>(pCh)); if (Vector.Zero.Equals(vMatches)) { pCh += Vector.Count; length -= Vector.Count; continue; } goto Found; } if (pCh < pEndCh) { length = (int)(pEndCh - pCh); goto SequentialScan; } } return false; Found: return true; } } public static unsafe int IndexOf(ref char searchSpace, char value, int length) { Debug.Assert(length >= 0); fixed (char* pChars = &searchSpace) { char* pCh = pChars; char* pEndCh = pCh + length; if (Vector.IsHardwareAccelerated && length >= Vector.Count * 2) { // Figure out how many characters to read sequentially until we are vector aligned // This is equivalent to: // unaligned = ((int)pCh % Unsafe.SizeOf>()) / elementsPerByte // length = (Vector.Count - unaligned) % Vector.Count const int elementsPerByte = sizeof(ushort) / sizeof(byte); int unaligned = ((int)pCh & (Unsafe.SizeOf>() - 1)) / elementsPerByte; length = (Vector.Count - unaligned) & (Vector.Count - 1); } SequentialScan: while (length >= 4) { length -= 4; if (pCh[0] == value) goto Found; if (pCh[1] == value) goto Found1; if (pCh[2] == value) goto Found2; if (pCh[3] == value) goto Found3; pCh += 4; } while (length > 0) { length--; if (pCh[0] == value) goto Found; pCh++; } // We get past SequentialScan only if IsHardwareAccelerated is true. However, we still have the redundant check to allow // the JIT to see that the code is unreachable and eliminate it when the platform does not have hardware accelerated. if (Vector.IsHardwareAccelerated && pCh < pEndCh) { // Get the highest multiple of Vector.Count that is within the search space. // That will be how many times we iterate in the loop below. // This is equivalent to: length = Vector.Count * ((int)(pEndCh - pCh) / Vector.Count) length = (int)((pEndCh - pCh) & ~(Vector.Count - 1)); // Get comparison Vector Vector vComparison = new Vector(value); while (length > 0) { // Using Unsafe.Read instead of ReadUnaligned since the search space is pinned and pCh is always vector aligned Debug.Assert(((int)pCh & (Unsafe.SizeOf>() - 1)) == 0); Vector vMatches = Vector.Equals(vComparison, Unsafe.Read>(pCh)); if (Vector.Zero.Equals(vMatches)) { pCh += Vector.Count; length -= Vector.Count; continue; } // Find offset of first match return (int)(pCh - pChars) + LocateFirstFoundChar(vMatches); } if (pCh < pEndCh) { length = (int)(pEndCh - pCh); goto SequentialScan; } } return -1; Found3: pCh++; Found2: pCh++; Found1: pCh++; Found: return (int)(pCh - pChars); } } public static unsafe int IndexOfAny(ref char searchSpace, char value0, char value1, int length) { Debug.Assert(length >= 0); fixed (char* pChars = &searchSpace) { char* pCh = pChars; char* pEndCh = pCh + length; if (Vector.IsHardwareAccelerated && length >= Vector.Count * 2) { // Figure out how many characters to read sequentially until we are vector aligned // This is equivalent to: // unaligned = ((int)pCh % Unsafe.SizeOf>()) / elementsPerByte // length = (Vector.Count - unaligned) % Vector.Count const int elementsPerByte = sizeof(ushort) / sizeof(byte); int unaligned = ((int)pCh & (Unsafe.SizeOf>() - 1)) / elementsPerByte; length = (Vector.Count - unaligned) & (Vector.Count - 1); } SequentialScan: while (length >= 4) { length -= 4; if (pCh[0] == value0 || pCh[0] == value1) goto Found; if (pCh[1] == value0 || pCh[1] == value1) goto Found1; if (pCh[2] == value0 || pCh[2] == value1) goto Found2; if (pCh[3] == value0 || pCh[3] == value1) goto Found3; pCh += 4; } while (length > 0) { length--; if (pCh[0] == value0 || pCh[0] == value1) goto Found; pCh++; } // We get past SequentialScan only if IsHardwareAccelerated is true. However, we still have the redundant check to allow // the JIT to see that the code is unreachable and eliminate it when the platform does not have hardware accelerated. if (Vector.IsHardwareAccelerated && pCh < pEndCh) { // Get the highest multiple of Vector.Count that is within the search space. // That will be how many times we iterate in the loop below. // This is equivalent to: length = Vector.Count * ((int)(pEndCh - pCh) / Vector.Count) length = (int)((pEndCh - pCh) & ~(Vector.Count - 1)); // Get comparison Vector Vector values0 = new Vector(value0); Vector values1 = new Vector(value1); while (length > 0) { // Using Unsafe.Read instead of ReadUnaligned since the search space is pinned and pCh is always vector aligned Debug.Assert(((int)pCh & (Unsafe.SizeOf>() - 1)) == 0); Vector vData = Unsafe.Read>(pCh); var vMatches = Vector.BitwiseOr( Vector.Equals(vData, values0), Vector.Equals(vData, values1)); if (Vector.Zero.Equals(vMatches)) { pCh += Vector.Count; length -= Vector.Count; continue; } // Find offset of first match return (int)(pCh - pChars) + LocateFirstFoundChar(vMatches); } if (pCh < pEndCh) { length = (int)(pEndCh - pCh); goto SequentialScan; } } return -1; Found3: pCh++; Found2: pCh++; Found1: pCh++; Found: return (int)(pCh - pChars); } } public static unsafe int IndexOfAny(ref char searchSpace, char value0, char value1, char value2, int length) { Debug.Assert(length >= 0); fixed (char* pChars = &searchSpace) { char* pCh = pChars; char* pEndCh = pCh + length; if (Vector.IsHardwareAccelerated && length >= Vector.Count * 2) { // Figure out how many characters to read sequentially until we are vector aligned // This is equivalent to: // unaligned = ((int)pCh % Unsafe.SizeOf>()) / elementsPerByte // length = (Vector.Count - unaligned) % Vector.Count const int elementsPerByte = sizeof(ushort) / sizeof(byte); int unaligned = ((int)pCh & (Unsafe.SizeOf>() - 1)) / elementsPerByte; length = (Vector.Count - unaligned) & (Vector.Count - 1); } SequentialScan: while (length >= 4) { length -= 4; if (pCh[0] == value0 || pCh[0] == value1 || pCh[0] == value2) goto Found; if (pCh[1] == value0 || pCh[1] == value1 || pCh[1] == value2) goto Found1; if (pCh[2] == value0 || pCh[2] == value1 || pCh[2] == value2) goto Found2; if (pCh[3] == value0 || pCh[3] == value1 || pCh[3] == value2) goto Found3; pCh += 4; } while (length > 0) { length--; if (pCh[0] == value0 || pCh[0] == value1 || pCh[0] == value2) goto Found; pCh++; } // We get past SequentialScan only if IsHardwareAccelerated is true. However, we still have the redundant check to allow // the JIT to see that the code is unreachable and eliminate it when the platform does not have hardware accelerated. if (Vector.IsHardwareAccelerated && pCh < pEndCh) { // Get the highest multiple of Vector.Count that is within the search space. // That will be how many times we iterate in the loop below. // This is equivalent to: length = Vector.Count * ((int)(pEndCh - pCh) / Vector.Count) length = (int)((pEndCh - pCh) & ~(Vector.Count - 1)); // Get comparison Vector Vector values0 = new Vector(value0); Vector values1 = new Vector(value1); Vector values2 = new Vector(value2); while (length > 0) { // Using Unsafe.Read instead of ReadUnaligned since the search space is pinned and pCh is always vector aligned Debug.Assert(((int)pCh & (Unsafe.SizeOf>() - 1)) == 0); Vector vData = Unsafe.Read>(pCh); var vMatches = Vector.BitwiseOr( Vector.BitwiseOr( Vector.Equals(vData, values0), Vector.Equals(vData, values1)), Vector.Equals(vData, values2)); if (Vector.Zero.Equals(vMatches)) { pCh += Vector.Count; length -= Vector.Count; continue; } // Find offset of first match return (int)(pCh - pChars) + LocateFirstFoundChar(vMatches); } if (pCh < pEndCh) { length = (int)(pEndCh - pCh); goto SequentialScan; } } return -1; Found3: pCh++; Found2: pCh++; Found1: pCh++; Found: return (int)(pCh - pChars); } } public static unsafe int IndexOfAny(ref char searchSpace, char value0, char value1, char value2, char value3, int length) { Debug.Assert(length >= 0); fixed (char* pChars = &searchSpace) { char* pCh = pChars; char* pEndCh = pCh + length; if (Vector.IsHardwareAccelerated && length >= Vector.Count * 2) { // Figure out how many characters to read sequentially until we are vector aligned // This is equivalent to: // unaligned = ((int)pCh % Unsafe.SizeOf>()) / elementsPerByte // length = (Vector.Count - unaligned) % Vector.Count const int elementsPerByte = sizeof(ushort) / sizeof(byte); int unaligned = ((int)pCh & (Unsafe.SizeOf>() - 1)) / elementsPerByte; length = (Vector.Count - unaligned) & (Vector.Count - 1); } SequentialScan: while (length >= 4) { length -= 4; if (pCh[0] == value0 || pCh[0] == value1 || pCh[0] == value2 || pCh[0] == value3) goto Found; if (pCh[1] == value0 || pCh[1] == value1 || pCh[1] == value2 || pCh[1] == value3) goto Found1; if (pCh[2] == value0 || pCh[2] == value1 || pCh[2] == value2 || pCh[2] == value3) goto Found2; if (pCh[3] == value0 || pCh[3] == value1 || pCh[3] == value2 || pCh[3] == value3) goto Found3; pCh += 4; } while (length > 0) { length--; if (pCh[0] == value0 || pCh[0] == value1 || pCh[0] == value2 || pCh[0] == value3) goto Found; pCh++; } // We get past SequentialScan only if IsHardwareAccelerated is true. However, we still have the redundant check to allow // the JIT to see that the code is unreachable and eliminate it when the platform does not have hardware accelerated. if (Vector.IsHardwareAccelerated && pCh < pEndCh) { // Get the highest multiple of Vector.Count that is within the search space. // That will be how many times we iterate in the loop below. // This is equivalent to: length = Vector.Count * ((int)(pEndCh - pCh) / Vector.Count) length = (int)((pEndCh - pCh) & ~(Vector.Count - 1)); // Get comparison Vector Vector values0 = new Vector(value0); Vector values1 = new Vector(value1); Vector values2 = new Vector(value2); Vector values3 = new Vector(value3); while (length > 0) { // Using Unsafe.Read instead of ReadUnaligned since the search space is pinned and pCh is always vector aligned Debug.Assert(((int)pCh & (Unsafe.SizeOf>() - 1)) == 0); Vector vData = Unsafe.Read>(pCh); var vMatches = Vector.BitwiseOr( Vector.BitwiseOr( Vector.BitwiseOr(Vector.Equals(vData, values0), Vector.Equals(vData, values1)), Vector.Equals(vData, values2)), Vector.Equals(vData, values3)); if (Vector.Zero.Equals(vMatches)) { pCh += Vector.Count; length -= Vector.Count; continue; } // Find offset of first match return (int)(pCh - pChars) + LocateFirstFoundChar(vMatches); } if (pCh < pEndCh) { length = (int)(pEndCh - pCh); goto SequentialScan; } } return -1; Found3: pCh++; Found2: pCh++; Found1: pCh++; Found: return (int)(pCh - pChars); } } public static unsafe int IndexOfAny(ref char searchSpace, char value0, char value1, char value2, char value3, char value4, int length) { Debug.Assert(length >= 0); fixed (char* pChars = &searchSpace) { char* pCh = pChars; char* pEndCh = pCh + length; if (Vector.IsHardwareAccelerated && length >= Vector.Count * 2) { // Figure out how many characters to read sequentially until we are vector aligned // This is equivalent to: // unaligned = ((int)pCh % Unsafe.SizeOf>()) / elementsPerByte // length = (Vector.Count - unaligned) % Vector.Count const int elementsPerByte = sizeof(ushort) / sizeof(byte); int unaligned = ((int)pCh & (Unsafe.SizeOf>() - 1)) / elementsPerByte; length = (Vector.Count - unaligned) & (Vector.Count - 1); } SequentialScan: while (length >= 4) { length -= 4; if (pCh[0] == value0 || pCh[0] == value1 || pCh[0] == value2 || pCh[0] == value3 || pCh[0] == value4) goto Found; if (pCh[1] == value0 || pCh[1] == value1 || pCh[1] == value2 || pCh[1] == value3 || pCh[1] == value4) goto Found1; if (pCh[2] == value0 || pCh[2] == value1 || pCh[2] == value2 || pCh[2] == value3 || pCh[2] == value4) goto Found2; if (pCh[3] == value0 || pCh[3] == value1 || pCh[3] == value2 || pCh[3] == value3 || pCh[3] == value4) goto Found3; pCh += 4; } while (length > 0) { length--; if (pCh[0] == value0 || pCh[0] == value1 || pCh[0] == value2 || pCh[0] == value3 || pCh[0] == value4) goto Found; pCh++; } // We get past SequentialScan only if IsHardwareAccelerated is true. However, we still have the redundant check to allow // the JIT to see that the code is unreachable and eliminate it when the platform does not have hardware accelerated. if (Vector.IsHardwareAccelerated && pCh < pEndCh) { // Get the highest multiple of Vector.Count that is within the search space. // That will be how many times we iterate in the loop below. // This is equivalent to: length = Vector.Count * ((int)(pEndCh - pCh) / Vector.Count) length = (int)((pEndCh - pCh) & ~(Vector.Count - 1)); // Get comparison Vector Vector values0 = new Vector(value0); Vector values1 = new Vector(value1); Vector values2 = new Vector(value2); Vector values3 = new Vector(value3); Vector values4 = new Vector(value4); while (length > 0) { // Using Unsafe.Read instead of ReadUnaligned since the search space is pinned and pCh is always vector aligned Debug.Assert(((int)pCh & (Unsafe.SizeOf>() - 1)) == 0); Vector vData = Unsafe.Read>(pCh); var vMatches = Vector.BitwiseOr( Vector.BitwiseOr( Vector.BitwiseOr( Vector.BitwiseOr(Vector.Equals(vData, values0), Vector.Equals(vData, values1)), Vector.Equals(vData, values2)), Vector.Equals(vData, values3)), Vector.Equals(vData, values4)); if (Vector.Zero.Equals(vMatches)) { pCh += Vector.Count; length -= Vector.Count; continue; } // Find offset of first match return (int)(pCh - pChars) + LocateFirstFoundChar(vMatches); } if (pCh < pEndCh) { length = (int)(pEndCh - pCh); goto SequentialScan; } } return -1; Found3: pCh++; Found2: pCh++; Found1: pCh++; Found: return (int)(pCh - pChars); } } public static unsafe int LastIndexOf(ref char searchSpace, char value, int length) { Debug.Assert(length >= 0); fixed (char* pChars = &searchSpace) { char* pCh = pChars + length; char* pEndCh = pChars; if (Vector.IsHardwareAccelerated && length >= Vector.Count * 2) { // Figure out how many characters to read sequentially from the end until we are vector aligned // This is equivalent to: length = ((int)pCh % Unsafe.SizeOf>()) / elementsPerByte const int elementsPerByte = sizeof(ushort) / sizeof(byte); length = ((int)pCh & (Unsafe.SizeOf>() - 1)) / elementsPerByte; } SequentialScan: while (length >= 4) { length -= 4; pCh -= 4; if (*(pCh + 3) == value) goto Found3; if (*(pCh + 2) == value) goto Found2; if (*(pCh + 1) == value) goto Found1; if (*pCh == value) goto Found; } while (length > 0) { length -= 1; pCh -= 1; if (*pCh == value) goto Found; } // We get past SequentialScan only if IsHardwareAccelerated is true. However, we still have the redundant check to allow // the JIT to see that the code is unreachable and eliminate it when the platform does not have hardware accelerated. if (Vector.IsHardwareAccelerated && pCh > pEndCh) { // Get the highest multiple of Vector.Count that is within the search space. // That will be how many times we iterate in the loop below. // This is equivalent to: length = Vector.Count * ((int)(pCh - pEndCh) / Vector.Count) length = (int)((pCh - pEndCh) & ~(Vector.Count - 1)); // Get comparison Vector Vector vComparison = new Vector(value); while (length > 0) { char* pStart = pCh - Vector.Count; // Using Unsafe.Read instead of ReadUnaligned since the search space is pinned and pCh (and hence pSart) is always vector aligned Debug.Assert(((int)pStart & (Unsafe.SizeOf>() - 1)) == 0); Vector vMatches = Vector.Equals(vComparison, Unsafe.Read>(pStart)); if (Vector.Zero.Equals(vMatches)) { pCh -= Vector.Count; length -= Vector.Count; continue; } // Find offset of last match return (int)(pStart - pEndCh) + LocateLastFoundChar(vMatches); } if (pCh > pEndCh) { length = (int)(pCh - pEndCh); goto SequentialScan; } } return -1; Found: return (int)(pCh - pEndCh); Found1: return (int)(pCh - pEndCh) + 1; Found2: return (int)(pCh - pEndCh) + 2; Found3: return (int)(pCh - pEndCh) + 3; } } // Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138 [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int LocateFirstFoundChar(Vector match) { var vector64 = Vector.AsVectorUInt64(match); ulong candidate = 0; int i = 0; // Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001 for (; i < Vector.Count; i++) { candidate = vector64[i]; if (candidate != 0) { break; } } // Single LEA instruction with jitted const (using function result) return i * 4 + LocateFirstFoundChar(candidate); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int LocateFirstFoundChar(ulong match) { unchecked { // Flag least significant power of two bit var powerOfTwoFlag = match ^ (match - 1); // Shift all powers of two into the high byte and extract return (int)((powerOfTwoFlag * XorPowerOfTwoToHighChar) >> 49); } } private const ulong XorPowerOfTwoToHighChar = (0x03ul | 0x02ul << 16 | 0x01ul << 32) + 1; // Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138 [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int LocateLastFoundChar(Vector match) { var vector64 = Vector.AsVectorUInt64(match); ulong candidate = 0; int i = Vector.Count - 1; // Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001 for (; i >= 0; i--) { candidate = vector64[i]; if (candidate != 0) { break; } } // Single LEA instruction with jitted const (using function result) return i * 4 + LocateLastFoundChar(candidate); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int LocateLastFoundChar(ulong match) { // Find the most significant char that has its highest bit set int index = 3; while ((long)match > 0) { match = match << 16; index--; } return index; } } }