// // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. // // // SIMD Support // // IMPORTANT NOTES AND CAVEATS: // // This implementation is preliminary, and may change dramatically. // // New JIT types, TYP_SIMDxx, are introduced, and the SIMD intrinsics are created as GT_SIMD nodes. // In this initial implementation, the SIMD types are kept as TYP_STRUCT, and only transformed to // SIMD types in Lowering. Eventually, we will want to make SIMD types "first class" types. // This will require ensuring that the JIT front-end can handle, and effectively optimize, the // SIMD type nodes and lclVars. // // Note that currently the "reference implementation" is the same as the runtime dll. As such, it is currently // providing implementations for those methods not currently supported by the JIT as intrinsics. // // These are currently recognized using string compares, in order to provide an implementation in the JIT // without taking a dependency on the VM. // Furthermore, in the CTP, in order to limit the impact of doing these string compares // against assembly names, we only look for the SIMDVector assembly if we are compiling a class constructor. This // makes it somewhat more "pay for play" but is a significant usability compromise. // This has been addressed for RTM by doing the assembly recognition in the VM. // -------------------------------------------------------------------------------------- #include "jitpch.h" #include "simd.h" #ifdef _MSC_VER #pragma hdrstop #endif #ifndef LEGACY_BACKEND // This file is ONLY used for the RyuJIT backend that uses the linear scan register allocator. #ifdef FEATURE_SIMD // Intrinsic Id to intrinsic info map const SIMDIntrinsicInfo simdIntrinsicInfoArray[] = { #define SIMD_INTRINSIC(mname, inst, id, name, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10) \ {SIMDIntrinsic##id, mname, inst, retType, argCount, arg1, arg2, arg3, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10}, #include "simdintrinsiclist.h" }; //------------------------------------------------------------------------ // getSIMDVectorLength: Get the length (number of elements of base type) of // SIMD Vector given its size and base (element) type. // // Arguments: // simdSize - size of the SIMD vector // baseType - type of the elements of the SIMD vector // // static int Compiler::getSIMDVectorLength(unsigned simdSize, var_types baseType) { return simdSize / genTypeSize(baseType); } //------------------------------------------------------------------------ // Get the length (number of elements of base type) of SIMD Vector given by typeHnd. // // Arguments: // typeHnd - type handle of the SIMD vector // int Compiler::getSIMDVectorLength(CORINFO_CLASS_HANDLE typeHnd) { unsigned sizeBytes = 0; var_types baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, &sizeBytes); return getSIMDVectorLength(sizeBytes, baseType); } //------------------------------------------------------------------------ // Get the preferred alignment of SIMD vector type for better performance. // // Arguments: // typeHnd - type handle of the SIMD vector // int Compiler::getSIMDTypeAlignment(CORINFO_CLASS_HANDLE typeHnd) { #ifdef _TARGET_AMD64_ // Fixed length vectors have the following alignment preference // Vector2/3 = 8 byte alignment // Vector4 = 16-byte alignment unsigned size = 0; var_types baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, &size); noway_assert(baseType != TYP_UNKNOWN); // preferred alignment for SSE2 128-bit vectors is 16-bytes if (size == 8) return 8; // As per Intel manual, AVX vectors preferred alignment is 32-bytes but on Amd64 // RSP/EBP is aligned at 16-bytes, therefore to align SIMD types at 32-bytes we need even // RSP/EBP to be 32-byte aligned. It is not clear whether additional stack space used in // aligning stack is worth the benefit and for now will use 16-byte alignment for AVX // 256-bit vectors with unaligned load/stores to/from memory. return 16; #else assert(!"getSIMDTypeAlignment() unimplemented on target arch"); unreached(); #endif } #ifdef RYUJIT_CTPBUILD // SIMDVector assembly // static volatile CORINFO_ASSEMBLY_HANDLE Compiler::SIMDAssemblyHandle = nullptr; //------------------------------------------------------------------------ // isSIMDModule: If the SIMDVector assembly has previously been found, and its handle // saved, check the moduleHnd to see if it is in the SIMDVector assembly. // // Arguments: // moduleHnd - The handle of the module we're interested in. // // Return Value: // Returns true if this module is in the SIMDVector assembly. // // Notes: // This will only return true if this is a class constructor, and it is referencing // a SIMD type, OR if we have already found the SIMDVector assembly. bool Compiler::isSIMDModule(CORINFO_MODULE_HANDLE moduleHnd) { assert(featureSIMD); const char* assemblyName; if (SIMDAssemblyHandle == nullptr) { // For the CTP, we only check for the SIMD assembly while compiling a class constructor. // This is because in the CTP we don't have VM support for identifying the SIMD assembly, // and we don't want to check on each method. if ((info.compFlags & CORINFO_FLG_CONSTRUCTOR) == 0 || (info.compFlags & CORINFO_FLG_STATIC) == 0) { return false; } CORINFO_ASSEMBLY_HANDLE assemblyHnd = info.compCompHnd->getModuleAssembly(moduleHnd); assemblyName = info.compCompHnd->getAssemblyName(assemblyHnd); if ((assemblyName != nullptr && strcmp(assemblyName, "System.Numerics.Vectors") == 0)) { SIMDAssemblyHandle = assemblyHnd; JITDUMP("Found SIMDVector (System.Numerics.Vectors) assembly.\n"); return true; } else { return false; } } else { CORINFO_ASSEMBLY_HANDLE assemblyHnd = info.compCompHnd->getModuleAssembly(moduleHnd); if (assemblyHnd == SIMDAssemblyHandle) { JITDUMP("\nKnown SIMDVector assembly.\n"); return true; } } return false; } #endif // RYUJIT_CTPBUILD //---------------------------------------------------------------------------------- // Return the base type and size of SIMD vector type given its type handle. // // Arguments: // typeHnd - The handle of the type we're interested in. // sizeBytes - out param // // Return Value: // base type of SIMD vector. // sizeBytes if non-null is set to size in bytes. // // TODO-Throughput: current implementation parses class name to find base type. Change // this when we implement SIMD intrinsic identification for the final // product. // var_types Compiler::getBaseTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd, unsigned *sizeBytes /*= nullptr */) { assert(featureSIMD); if (typeHnd == nullptr) { return TYP_UNKNOWN; } // fast path search using cached type handles of important types var_types simdBaseType = TYP_UNKNOWN; unsigned size = 0; // Early return if it is not a SIMD module. if (!isSIMDClass(typeHnd)) { return TYP_UNKNOWN; } // The most likely to be used type handles are looked up first followed by // less likely to be used type handles if (typeHnd == SIMDFloatHandle) { simdBaseType = TYP_FLOAT; JITDUMP(" Known type SIMD Vector\n"); } else if (typeHnd == SIMDIntHandle) { simdBaseType = TYP_INT; JITDUMP(" Known type SIMD Vector\n"); } else if (typeHnd == SIMDVector2Handle) { simdBaseType = TYP_FLOAT; size = 2*genTypeSize(TYP_FLOAT); assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE)); JITDUMP(" Known type Vector2\n"); } else if (typeHnd == SIMDVector3Handle) { simdBaseType = TYP_FLOAT; size = 3*genTypeSize(TYP_FLOAT); assert(size == info.compCompHnd->getClassSize(typeHnd)); JITDUMP(" Known type Vector3\n"); } else if (typeHnd == SIMDVector4Handle) { simdBaseType = TYP_FLOAT; size = 4*genTypeSize(TYP_FLOAT); assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE)); JITDUMP(" Known type Vector4\n"); } else if (typeHnd == SIMDVectorHandle) { JITDUMP(" Known type Vector\n"); } else if (typeHnd == SIMDUShortHandle) { simdBaseType = TYP_CHAR; JITDUMP(" Known type SIMD Vector\n"); } else if (typeHnd == SIMDUByteHandle) { simdBaseType = TYP_UBYTE; JITDUMP(" Known type SIMD Vector\n"); } else if (typeHnd == SIMDDoubleHandle) { simdBaseType = TYP_DOUBLE; JITDUMP(" Known type SIMD Vector\n"); } else if (typeHnd == SIMDLongHandle) { simdBaseType = TYP_LONG; JITDUMP(" Known type SIMD Vector\n"); } else if (typeHnd == SIMDShortHandle) { simdBaseType = TYP_SHORT; JITDUMP(" Known type SIMD Vector\n"); } else if (typeHnd == SIMDByteHandle) { simdBaseType = TYP_BYTE; JITDUMP(" Known type SIMD Vector\n"); } else if (typeHnd == SIMDUIntHandle) { simdBaseType = TYP_UINT; JITDUMP(" Known type SIMD Vector\n"); } else if (typeHnd == SIMDULongHandle) { simdBaseType = TYP_ULONG; JITDUMP(" Known type SIMD Vector\n"); } // slow path search if (simdBaseType == TYP_UNKNOWN) { // Doesn't match with any of the cached type handles. // Obtain base type by parsing fully qualified class name. // // TODO-Throughput: implement product shipping solution to query base type. WCHAR className[256] = {'\\0'}; WCHAR *pbuf = &className[0]; int len = sizeof(className)/sizeof(className[0]); info.compCompHnd->appendClassName(&pbuf, &len, typeHnd, TRUE, FALSE, FALSE); noway_assert(pbuf < &className[256]); JITDUMP("SIMD Candidate Type %S\n", className); if (wcsncmp(className, W("System.Numerics."), 16) == 0) { if (wcsncmp(&(className[16]), W("Vector`1["), 9) == 0) { if (wcsncmp(&(className[25]), W("System.Single"), 13) == 0) { SIMDFloatHandle = typeHnd; simdBaseType = TYP_FLOAT; JITDUMP(" Found type SIMD Vector\n"); } else if (wcsncmp(&(className[25]), W("System.Int32"), 12) == 0) { SIMDIntHandle = typeHnd; simdBaseType = TYP_INT; JITDUMP(" Found type SIMD Vector\n"); } else if (wcsncmp(&(className[25]), W("System.UInt16"), 13) == 0) { SIMDUShortHandle = typeHnd; simdBaseType = TYP_CHAR; JITDUMP(" Found type SIMD Vector\n"); } else if (wcsncmp(&(className[25]), W("System.Byte"), 11) == 0) { SIMDUByteHandle = typeHnd; simdBaseType = TYP_UBYTE; JITDUMP(" Found type SIMD Vector\n"); } else if (wcsncmp(&(className[25]), W("System.Double"), 13) == 0) { SIMDDoubleHandle = typeHnd; simdBaseType = TYP_DOUBLE; JITDUMP(" Found type SIMD Vector\n"); } else if (wcsncmp(&(className[25]), W("System.Int64"), 12) == 0) { SIMDLongHandle = typeHnd; simdBaseType = TYP_LONG; JITDUMP(" Found type SIMD Vector\n"); } else if (wcsncmp(&(className[25]), W("System.Int16"), 12) == 0) { SIMDShortHandle = typeHnd; simdBaseType = TYP_SHORT; JITDUMP(" Found type SIMD Vector\n"); } else if (wcsncmp(&(className[25]), W("System.SByte"), 12) == 0) { SIMDByteHandle = typeHnd; simdBaseType = TYP_BYTE; JITDUMP(" Found type SIMD Vector\n"); } else if (wcsncmp(&(className[25]), W("System.UInt32"), 13) == 0) { SIMDUIntHandle = typeHnd; simdBaseType = TYP_UINT; JITDUMP(" Found type SIMD Vector\n"); } else if (wcsncmp(&(className[25]), W("System.UInt64"), 13) == 0) { SIMDULongHandle = typeHnd; simdBaseType = TYP_ULONG; JITDUMP(" Found type SIMD Vector\n"); } else { JITDUMP(" Unknown SIMD Vector\n"); } } else if (wcsncmp(&(className[16]), W("Vector2"), 8) == 0) { SIMDVector2Handle = typeHnd; simdBaseType = TYP_FLOAT; size = 2*genTypeSize(TYP_FLOAT); assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE)); JITDUMP(" Found Vector2\n"); } else if (wcsncmp(&(className[16]), W("Vector3"), 8) == 0) { SIMDVector3Handle = typeHnd; simdBaseType = TYP_FLOAT; size = 3*genTypeSize(TYP_FLOAT); assert(size == info.compCompHnd->getClassSize(typeHnd)); JITDUMP(" Found Vector3\n"); } else if (wcsncmp(&(className[16]), W("Vector4"), 8) == 0) { SIMDVector4Handle = typeHnd; simdBaseType = TYP_FLOAT; size = 4*genTypeSize(TYP_FLOAT); assert(size == roundUp(info.compCompHnd->getClassSize(typeHnd), TARGET_POINTER_SIZE)); JITDUMP(" Found Vector4\n"); } else if (wcsncmp(&(className[16]), W("Vector"), 6) == 0) { SIMDVectorHandle = typeHnd; JITDUMP(" Found type Vector\n"); } else { JITDUMP(" Unknown SIMD Type\n"); } } } if (simdBaseType != TYP_UNKNOWN && sizeBytes != nullptr) { // If not a fixed size vector then its size is same as SIMD vector // register length in bytes if (size == 0) { size = getSIMDVectorRegisterByteLength(); } *sizeBytes = size; } return simdBaseType; } //-------------------------------------------------------------------------------------- // getSIMDIntrinsicInfo: get SIMD intrinsic info given the method handle. // // Arguments: // inOutTypeHnd - The handle of the type on which the method is invoked. This is an in-out param. // methodHnd - The handle of the method we're interested in. // sig - method signature info // isNewObj - whether this call represents a newboj constructor call // argCount - argument count - out pram // baseType - base type of the intrinsic - out param // sizeBytes - size of SIMD vector type on which the method is invoked - out param // // Return Value: // SIMDIntrinsicInfo struct initialized corresponding to methodHnd. // Sets SIMDIntrinsicInfo.id to SIMDIntrinsicInvalid if methodHnd doesn't correspond // to any SIMD intrinsic. Also, sets the out params inOutTypeHnd, argCount, baseType and // sizeBytes. // // Note that VectorMath class doesn't have a base type and first argument of the method // determines the SIMD vector type on which intrinsic is invoked. In such a case inOutTypeHnd // is modified by this routine. // // TODO-Throughput: The current implementation is based on method name string parsing. // Although we now have type identification from the VM, the parsing of intrinsic names // could be made more efficient. // const SIMDIntrinsicInfo* Compiler::getSIMDIntrinsicInfo(CORINFO_CLASS_HANDLE* inOutTypeHnd, CORINFO_METHOD_HANDLE methodHnd, CORINFO_SIG_INFO * sig, bool isNewObj, unsigned* argCount, var_types* baseType, unsigned* sizeBytes) { assert(featureSIMD); assert(baseType != nullptr); assert(sizeBytes != nullptr); // get baseType and size of the type CORINFO_CLASS_HANDLE typeHnd = *inOutTypeHnd; *baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes); bool isHWAcceleratedIntrinsic = false; if (typeHnd == SIMDVectorHandle) { // All of the supported intrinsics on this static class take a first argument that's a vector, // which determines the baseType. // The exception is the IsHardwareAccelerated property, which is handled as a special case. assert(*baseType == TYP_UNKNOWN); if(sig->numArgs == 0) { const SIMDIntrinsicInfo* hwAccelIntrinsicInfo = &(simdIntrinsicInfoArray[SIMDIntrinsicHWAccel]); if ((strcmp(eeGetMethodName(methodHnd, nullptr), hwAccelIntrinsicInfo->methodName) == 0) && JITtype2varType(sig->retType) == hwAccelIntrinsicInfo->retType) { // Sanity check assert(hwAccelIntrinsicInfo->argCount == 0 && hwAccelIntrinsicInfo->isInstMethod == false); return hwAccelIntrinsicInfo; } return nullptr; } else { typeHnd = info.compCompHnd->getArgClass(sig, sig->args); *inOutTypeHnd = typeHnd; *baseType = getBaseTypeAndSizeOfSIMDType(typeHnd, sizeBytes); } } if (*baseType == TYP_UNKNOWN) { JITDUMP("NOT a SIMD Intrinsic: unsupported baseType\n"); return nullptr; } // account for implicit "this" arg *argCount = sig->numArgs; if (sig->hasThis()) { *argCount += 1; } // Get the Intrinsic Id by parsing method name. // // TODO-Throughput: replace sequential search by binary search by arranging entries // sorted by method name. SIMDIntrinsicID intrinsicId = SIMDIntrinsicInvalid; const char* methodName = eeGetMethodName(methodHnd, nullptr); for (int i = SIMDIntrinsicNone+1; i < SIMDIntrinsicInvalid; ++i) { if (strcmp(methodName, simdIntrinsicInfoArray[i].methodName) == 0) { // Found an entry for the method; further check whether it is one of // the supported base types. bool found = false; for (int j=0; j < SIMD_INTRINSIC_MAX_BASETYPE_COUNT; ++j) { // Convention: if there are fewer base types supported than MAX_BASETYPE_COUNT, // the end of the list is marked by TYP_UNDEF. if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == TYP_UNDEF) break; if (simdIntrinsicInfoArray[i].supportedBaseTypes[j] == *baseType) { found = true; break; } } if (!found) { continue; } // Now, check the arguments. unsigned int fixedArgCnt = simdIntrinsicInfoArray[i].argCount; unsigned int expectedArgCnt = fixedArgCnt; // First handle SIMDIntrinsicInitN, where the arg count depends on the type. // The listed arg types include the vector and the first two init values, which is the expected number // for Vector2. For other cases, we'll check their types here. if (*argCount > expectedArgCnt) { if (i == SIMDIntrinsicInitN) { if (*argCount == 3 && typeHnd == SIMDVector2Handle) { expectedArgCnt = 3; } else if (*argCount == 4 && typeHnd == SIMDVector3Handle) { expectedArgCnt = 4; } else if (*argCount == 5 && typeHnd == SIMDVector4Handle) { expectedArgCnt = 5; } } else if (i == SIMDIntrinsicInitFixed) { if (*argCount == 4 && typeHnd == SIMDVector4Handle) { expectedArgCnt = 4; } } } if (*argCount != expectedArgCnt) { continue; } // Validate the types of individual args passed are what is expected of. // If any of the types don't match with what is expected, don't consider // as an intrinsic. This will make an older JIT with SIMD capabilities // resilient to breaking changes to SIMD managed API. // // Note that from IL type stack, args get popped in right to left order // whereas args get listed in method signatures in left to right order. int stackIndex = (expectedArgCnt - 1); // Track the arguments from the signature - we currently only use this to distinguish // integral and pointer types, both of which will by TYP_I_IMPL on the importer stack. CORINFO_ARG_LIST_HANDLE argLst = sig->args; CORINFO_CLASS_HANDLE argClass; for (unsigned int argIndex = 0; found == true && argIndex < expectedArgCnt; argIndex++) { bool isThisPtr = ((argIndex == 0) && sig->hasThis()); // In case of "newobj SIMDVector(T val)", thisPtr won't be present on type stack. // We don't check anything in that case. if (!isThisPtr || !isNewObj) { GenTreePtr arg = impStackTop(stackIndex).val; var_types expectedArgType; if (argIndex < fixedArgCnt) { // Convention: // - intrinsicInfo.argType[i] == TYP_UNDEF - intrinsic doesn't have a valid arg at position i // - intrinsicInfo.argType[i] == TYP_UNKNOWN - arg type should be same as basetype // Note that we pop the args off in reverse order. expectedArgType = simdIntrinsicInfoArray[i].argType[argIndex]; assert(expectedArgType != TYP_UNDEF); if (expectedArgType == TYP_UNKNOWN) { // JIT maps uint/ulong type vars to TYP_INT/TYP_LONG. expectedArgType = (*baseType == TYP_UINT || *baseType == TYP_ULONG) ? genActualType(*baseType) : *baseType; } } else { expectedArgType = *baseType; } var_types argType = arg->TypeGet(); if (!isThisPtr && argType == TYP_I_IMPL) { // The reference implementation has a constructor that takes a pointer. // We don't want to recognize that one. This requires us to look at the CorInfoType // in order to distinguish a signature with a pointer argument from one with an // integer argument of pointer size, both of which will be TYP_I_IMPL on the stack. // TODO-Review: This seems quite fragile. We should consider beefing up the checking // here. CorInfoType corType = strip(info.compCompHnd->getArgType(sig, argLst, &argClass)); if (corType == CORINFO_TYPE_PTR) { found = false; } } if (argType != expectedArgType) { found = false; } } if (argIndex != 0 || !sig->hasThis()) { argLst = info.compCompHnd->getArgNext(argLst); } stackIndex--; } // Cross check return type and static vs. instance is what we are expecting. // If not, don't consider it as an intrinsic. // Note that ret type of TYP_UNKNOWN means that it is not known apriori and must be same as baseType if (found) { var_types expectedRetType = simdIntrinsicInfoArray[i].retType; if (expectedRetType == TYP_UNKNOWN) { // JIT maps uint/ulong type vars to TYP_INT/TYP_LONG. expectedRetType = (*baseType == TYP_UINT || *baseType == TYP_ULONG) ? genActualType(*baseType) : *baseType; } if (JITtype2varType(sig->retType) != expectedRetType || sig->hasThis() != simdIntrinsicInfoArray[i].isInstMethod) { found = false; } } if (found) { intrinsicId = (SIMDIntrinsicID) i; break; } } } if (intrinsicId != SIMDIntrinsicInvalid) { JITDUMP("Method %s maps to SIMD intrinsic %s\n", methodName, simdIntrinsicNames[intrinsicId]); return &simdIntrinsicInfoArray[intrinsicId]; } else { JITDUMP("Method %s is NOT a SIMD intrinsic\n", methodName); } return nullptr; } //------------------------------------------------------------------------ // checkForSIMDType: Check lclVar to see if it is a SIMD type. // This calls getBaseTypeOfSIMDType to do most of the work. If it is a known // SIMD type, then lvSIMDType will be set to true on 'varDsc', lvBaseType // will be set to the base type (element type) and lvExactSize will be set to // size of the SIMD struct supported on the target machine. // // Arguments: // varDsc - The lclVar we are interested in. // typeHnd - The handle of the type of the lclVar. // // Return Value: // None. // void Compiler::checkForSIMDType(LclVarDsc* varDsc, CORINFO_CLASS_HANDLE typeHnd) { assert(featureSIMD); // We should be calling this once per struct. assert (!varDsc->lvIsSIMDType()); if (varDsc->lvType != TYP_BYREF && (varDsc->lvExactSize > getSIMDVectorRegisterByteLength() || varDsc->lvExactSize < 8 || varDsc->lvStructGcCount != 0)) { return; } unsigned sizeBytes = 0; var_types simdBaseType = getBaseTypeAndSizeOfSIMDType(typeHnd, &sizeBytes); if (simdBaseType != TYP_UNKNOWN) { assert(simdBaseType != TYP_UNKNOWN); varDsc->lvSIMDType = true; varDsc->lvBaseType = simdBaseType; // Set the size of the simd struct. // Note that the size of SIMD vector will dynamically depend on whether // the underlying machine supports SSE2 (16 bytes) or AVX (32 bytes). varDsc->lvExactSize = sizeBytes; #ifdef _TARGET_AMD64_ // Amd64: also indicate that we use floating point registers compFloatingPointUsed = true; #endif } } // Pops and returns GenTree node from importer's type stack. // Normalizes TYP_STRUCT value in case of GT_CALL, GT_RET_EXPR and arg nodes. // // Arguments: // expectAddr - if true indicates we are expecting type stack entry to be a TYP_BYREF. // GenTreePtr Compiler::impSIMDPopStack(bool expectAddr) { StackEntry se = impPopStack(); typeInfo ti = se.seTypeInfo; GenTreePtr tree = se.val; // If expectAddr is true implies what we have on stack is address and we need // SIMD type struct that it points to. if (expectAddr) { assert(tree->TypeGet() == TYP_BYREF); if (tree->OperGet() == GT_ADDR) { tree = tree->gtGetOp1(); } else { tree = gtNewOperNode(GT_IND, TYP_STRUCT, tree); } } bool isParam = false; // If we have a ldobj of a SIMD local we need to transform it. if (tree->OperGet() == GT_LDOBJ) { GenTree* addr = tree->gtOp.gtOp1; if ((addr->OperGet() == GT_ADDR) && isSIMDTypeLocal(addr->gtOp.gtOp1)) { tree = addr->gtOp.gtOp1; } } if (tree->OperGet() == GT_LCL_VAR) { unsigned lclNum = tree->AsLclVarCommon()->GetLclNum(); LclVarDsc* lclVarDsc = &lvaTable[lclNum]; isParam = lclVarDsc->lvIsParam; } // normalize TYP_STRUCT value if ((tree->TypeGet() == TYP_STRUCT) && (tree->OperGet() == GT_RET_EXPR || tree->OperGet() == GT_CALL || isParam)) { assert(ti.IsType(TI_STRUCT)); CORINFO_CLASS_HANDLE structType = ti.GetClassHandleForValueClass(); tree = impNormStructVal(tree, structType, (unsigned)CHECK_SPILL_ALL); } return tree; } // impSIMDGetFixed: Create a GT_SIMD tree for a Get property of SIMD vector with a fixed index. // // Arguments: // baseType - The base (element) type of the SIMD vector. // simdSize - The total size in bytes of the SIMD vector. // index - The index of the field to get. // // Return Value: // Returns a GT_SIMD node with the SIMDIntrinsicGetItem intrinsic id. // GenTreeSIMD* Compiler::impSIMDGetFixed(var_types baseType, unsigned simdSize, int index) { assert(simdSize >= ((index + 1) * genTypeSize(baseType))); // op1 is a SIMD source. GenTree* op1 = impSIMDPopStack(true); assert(op1->TypeGet() == TYP_STRUCT); GenTree* op2 = gtNewIconNode(index); GenTreeSIMD* simdTree = gtNewSIMDNode(baseType, op1, op2, SIMDIntrinsicGetItem, baseType, simdSize); return simdTree; } #ifdef _TARGET_AMD64_ // impSIMDLongRelOpEqual: transforms operands and returns the SIMD intrinsic to be applied on // transformed operands to obtain == comparison result. // // Argumens: // typeHnd - type handle of SIMD vector // size - SIMD vector size // op1 - in-out parameter; first operand // op2 - in-out parameter; second operand // // Return Value: // Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands // SIMDIntrinsicID Compiler::impSIMDLongRelOpEqual(CORINFO_CLASS_HANDLE typeHnd, unsigned size, GenTree** op1, GenTree** op2) { // There is no direct SSE2 support for comparing TYP_LONG vectors. // These have to be implemented interms of TYP_INT vector comparison operations. // // Equality(v1, v2): // tmp = (v1 == v2) i.e. compare for equality as if v1 and v2 are vector // result = BitwiseAnd(t, shuffle(t, (2, 3, 1 0))) // Shuffle is meant to swap the comparison results of low-32-bits and high 32-bits of respective long elements. // Compare vector as if they were vector and assign the result to a temp GenTree* compResult = gtNewSIMDNode(TYP_STRUCT, *op1, *op2, SIMDIntrinsicEqual, TYP_INT, size); unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Long ==")); lvaSetStruct(lclNum, typeHnd, false); GenTree* tmp = gtNewLclvNode(lclNum, TYP_STRUCT); GenTree* asg = gtNewTempAssign(lclNum, compResult); // op1 = GT_COMMA(tmp=compResult, tmp) // op2 = Shuffle(tmp, 0xB1) // IntrinsicId = BitwiseAnd *op1 = gtNewOperNode(GT_COMMA, TYP_STRUCT, asg, tmp); *op2 = gtNewSIMDNode(TYP_STRUCT, gtNewLclvNode(lclNum, TYP_STRUCT), gtNewIconNode(SHUFFLE_ZWYX, TYP_INT), SIMDIntrinsicShuffleSSE2, TYP_INT, size); return SIMDIntrinsicBitwiseAnd; } // impSIMDLongRelOpGreaterThan: transforms operands and returns the SIMD intrinsic to be applied on // transformed operands to obtain > comparison result. // // Argumens: // typeHnd - type handle of SIMD vector // size - SIMD vector size // op1 - in-out parameter; first operand // op2 - in-out parameter; second operand // // Return Value: // Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands // SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThan(CORINFO_CLASS_HANDLE typeHnd, unsigned size, GenTree** op1, GenTree** op2) { // GreaterThan(v1, v2) where v1 and v2 are vector long. // Let us consider the case of single long element comparison. // say L1 = (x1, y1) and L2 = (x2, y2) where x1, y1, x2, and y2 are 32-bit integers that comprise the longs L1 and L2. // // GreaterThan(L1, L2) can be expressed in terms of > relationship between 32-bit integers that comprise L1 and L2 as // = (x1, y1) > (x2, y2) // = (x1 > x2) || [(x1 == x2) && (y1 > y2)] - eq (1) // // t = (v1 > v2) 32-bit signed comparison // u = (v1 == v2) 32-bit sized element equality // v = (v1 > v2) 32-bit unsigned comparison // // z = shuffle(t, (3, 3, 1, 1)) - This corresponds to (x1 > x2) in eq(1) above // t1 = Shuffle(v, (2, 2, 0, 0)) - This corresponds to (y1 > y2) in eq(1) above // u1 = Shuffle(u, (3, 3, 1, 1)) - This corresponds to (x1 == x2) in eq(1) above // w = And(t1, u1) - This corresponds to [(x1 == x2) && (y1 > y2)] in eq(1) above // Result = BitwiseOr(z, w) // Since op1 and op2 gets used multiple times, make sure side effects are computed. GenTree* dupOp1 = nullptr; GenTree* dupOp2 = nullptr; GenTree* dupDupOp1 = nullptr; GenTree* dupDupOp2 = nullptr; if (((*op1)->gtFlags & GTF_SIDE_EFFECT) != 0) { dupOp1 = fgInsertCommaFormTemp(op1, typeHnd); dupDupOp1 = gtNewLclvNode(dupOp1->AsLclVarCommon()->GetLclNum(), TYP_STRUCT); } else { dupOp1 = gtCloneExpr(*op1); dupDupOp1 = gtCloneExpr(*op1); } if (((*op2)->gtFlags & GTF_SIDE_EFFECT) != 0) { dupOp2 = fgInsertCommaFormTemp(op2, typeHnd); dupDupOp2 = gtNewLclvNode(dupOp2->AsLclVarCommon()->GetLclNum(), TYP_STRUCT); } else { dupOp2 = gtCloneExpr(*op2); dupDupOp2 = gtCloneExpr(*op2); } assert(dupDupOp1 != nullptr && dupDupOp2 != nullptr); assert(dupOp1 != nullptr && dupOp2 != nullptr); assert(*op1 != nullptr && *op2 != nullptr); // v1GreaterThanv2Signed - signed 32-bit comparison GenTree* v1GreaterThanv2Signed = gtNewSIMDNode(TYP_STRUCT, *op1, *op2, SIMDIntrinsicGreaterThan, TYP_INT, size); // v1Equalsv2 - 32-bit equality GenTree* v1Equalsv2 = gtNewSIMDNode(TYP_STRUCT, dupOp1, dupOp2, SIMDIntrinsicEqual, TYP_INT, size); // v1GreaterThanv2Unsigned - unsigned 32-bit comparison var_types tempBaseType = TYP_UINT; SIMDIntrinsicID sid = impSIMDRelOp(SIMDIntrinsicGreaterThan, typeHnd, size, &tempBaseType, &dupDupOp1, &dupDupOp2); GenTree* v1GreaterThanv2Unsigned = gtNewSIMDNode(TYP_STRUCT, dupDupOp1, dupDupOp2, sid, tempBaseType, size); GenTree* z = gtNewSIMDNode(TYP_STRUCT, v1GreaterThanv2Signed, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size); GenTree* t1 = gtNewSIMDNode(TYP_STRUCT, v1GreaterThanv2Unsigned, gtNewIconNode(SHUFFLE_ZZXX, TYP_INT), SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size); GenTree* u1 = gtNewSIMDNode(TYP_STRUCT, v1Equalsv2, gtNewIconNode(SHUFFLE_WWYY, TYP_INT), SIMDIntrinsicShuffleSSE2, TYP_FLOAT, size); GenTree* w = gtNewSIMDNode(TYP_STRUCT, u1, t1, SIMDIntrinsicBitwiseAnd, TYP_INT, size); *op1 = z; *op2 = w; return SIMDIntrinsicBitwiseOr; } // impSIMDLongRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on // transformed operands to obtain >= comparison result. // // Argumens: // typeHnd - type handle of SIMD vector // size - SIMD vector size // op1 - in-out parameter; first operand // op2 - in-out parameter; second operand // // Return Value: // Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands // SIMDIntrinsicID Compiler::impSIMDLongRelOpGreaterThanOrEqual(CORINFO_CLASS_HANDLE typeHnd, unsigned size, GenTree** op1, GenTree** op2) { // expand this to (a == b) | (a > b) GenTree* dupOp1 = nullptr; GenTree* dupOp2 = nullptr; if (((*op1)->gtFlags & GTF_SIDE_EFFECT) != 0) { dupOp1 = fgInsertCommaFormTemp(op1, typeHnd); } else { dupOp1 = gtCloneExpr(*op1); } if (((*op2)->gtFlags & GTF_SIDE_EFFECT) != 0) { dupOp2 = fgInsertCommaFormTemp(op2, typeHnd); } else { dupOp2 = gtCloneExpr(*op2); } assert(dupOp1 != nullptr && dupOp2 != nullptr); assert(*op1 != nullptr && *op2 != nullptr); // (a==b) SIMDIntrinsicID id = impSIMDLongRelOpEqual(typeHnd, size, op1, op2); *op1 = gtNewSIMDNode(TYP_STRUCT, *op1, *op2, id, TYP_LONG, size); // (a > b) id = impSIMDLongRelOpGreaterThan(typeHnd, size, &dupOp1, &dupOp2); *op2 = gtNewSIMDNode(TYP_STRUCT, dupOp1, dupOp2, id, TYP_LONG, size); return SIMDIntrinsicBitwiseOr; } // impSIMDInt32OrSmallIntRelOpGreaterThanOrEqual: transforms operands and returns the SIMD intrinsic to be applied on // transformed operands to obtain >= comparison result in case of integer base type vectors // // Argumens: // typeHnd - type handle of SIMD vector // size - SIMD vector size // baseType - base type of SIMD vector // op1 - in-out parameter; first operand // op2 - in-out parameter; second operand // // Return Value: // Modifies in-out params op1, op2 and returns intrinsic ID to be applied to modified operands // SIMDIntrinsicID Compiler::impSIMDIntegralRelOpGreaterThanOrEqual(CORINFO_CLASS_HANDLE typeHnd, unsigned size, var_types baseType, GenTree** op1, GenTree** op2) { // This routine should be used only for integer base type vectors assert(varTypeIsIntegral(baseType)); if ((getSIMDInstructionSet() == InstructionSet_SSE2) && ((baseType == TYP_LONG) || baseType == TYP_UBYTE)) { return impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, op1, op2); } // expand this to (a == b) | (a > b) GenTree* dupOp1 = nullptr; GenTree* dupOp2 = nullptr; if (((*op1)->gtFlags & GTF_SIDE_EFFECT) != 0) { dupOp1 = fgInsertCommaFormTemp(op1, typeHnd); } else { dupOp1 = gtCloneExpr(*op1); } if (((*op2)->gtFlags & GTF_SIDE_EFFECT) != 0) { dupOp2 = fgInsertCommaFormTemp(op2, typeHnd); } else { dupOp2 = gtCloneExpr(*op2); } assert(dupOp1 != nullptr && dupOp2 != nullptr); assert(*op1 != nullptr && *op2 != nullptr); // (a==b) *op1 = gtNewSIMDNode(TYP_STRUCT, *op1, *op2, SIMDIntrinsicEqual, baseType, size); // (a > b) *op2 = gtNewSIMDNode(TYP_STRUCT, dupOp1, dupOp2, SIMDIntrinsicGreaterThan, baseType, size); return SIMDIntrinsicBitwiseOr; } #endif //_TARGET_AMD64_ // Transforms operands and returns the SIMD intrinsic to be applied on // transformed operands to obtain given relop result. // // Argumens: // relOpIntrinsicId - Relational operator SIMD intrinsic // typeHnd - type handle of SIMD vector // size - SIMD vector size // inOutBaseType - base type of SIMD vector // op1 - in-out parameter; first operand // op2 - in-out parameter; second operand // // Return Value: // Modifies in-out params op1, op2, inOutBaseType and returns intrinsic ID to be applied to modified operands // SIMDIntrinsicID Compiler::impSIMDRelOp(SIMDIntrinsicID relOpIntrinsicId, CORINFO_CLASS_HANDLE typeHnd, unsigned size, var_types* inOutBaseType, GenTree** op1, GenTree** op2) { assert(isRelOpSIMDIntrinsic(relOpIntrinsicId)); #ifdef _TARGET_AMD64_ SIMDIntrinsicID intrinsicID = relOpIntrinsicId; var_types baseType = *inOutBaseType; if (varTypeIsFloating(baseType)) { // SSE2/AVX doesn't support > and >= on vector float/double. // Therefore, we need to use < and <= with swapped operands if (relOpIntrinsicId == SIMDIntrinsicGreaterThan || relOpIntrinsicId == SIMDIntrinsicGreaterThanOrEqual) { GenTree* tmp = *op1; *op1 = *op2; *op2 = tmp; intrinsicID = (relOpIntrinsicId == SIMDIntrinsicGreaterThan) ? SIMDIntrinsicLessThan : SIMDIntrinsicLessThanOrEqual; } } else if (varTypeIsIntegral(baseType)) { // SSE/AVX doesn't support < and <= on integer base type vectors. // Therefore, we need to use > and >= with swapped operands. if (intrinsicID == SIMDIntrinsicLessThan || intrinsicID == SIMDIntrinsicLessThanOrEqual) { GenTree* tmp = *op1; *op1 = *op2; *op2 = tmp; intrinsicID = (relOpIntrinsicId == SIMDIntrinsicLessThan) ? SIMDIntrinsicGreaterThan : SIMDIntrinsicGreaterThanOrEqual; } if ((getSIMDInstructionSet() == InstructionSet_SSE2) && baseType == TYP_LONG) { // There is no direct SSE2 support for comparing TYP_LONG vectors. // These have to be implemented interms of TYP_INT vector comparison operations. if (intrinsicID == SIMDIntrinsicEqual) { intrinsicID = impSIMDLongRelOpEqual(typeHnd, size, op1, op2); } else if (intrinsicID == SIMDIntrinsicGreaterThan) { intrinsicID = impSIMDLongRelOpGreaterThan(typeHnd, size, op1, op2); } else if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual) { intrinsicID = impSIMDLongRelOpGreaterThanOrEqual(typeHnd, size, op1, op2); } else { unreached(); } } // SSE2 and AVX direct support for signed comparison of int32, int16 and int8 types else if (!varTypeIsUnsigned(baseType)) { if (intrinsicID == SIMDIntrinsicGreaterThanOrEqual) { intrinsicID = impSIMDIntegralRelOpGreaterThanOrEqual(typeHnd, size, baseType, op1, op2); } } else // unsigned { // Vector, Vector, Vector and Vector: // SSE2 supports > for signed comparison. Therefore, to use it for // comparing unsigned numbers, we subtract a constant from both the // operands such that the result fits within the corresponding signed // type. The resulting signed numbers are compared using SSE2 signed // comparison. // // Vector: constant to be subtracted is 2^7 // Vector constant to be subtracted is 2^15 // Vector constant to be subtracted is 2^31 // Vector constant to be subtracted is 2^63 // // We need to treat op1 and op2 as signed for comparison purpose after // the transformation. ssize_t constVal = 0; switch(baseType) { case TYP_UBYTE: constVal = 0x80808080; *inOutBaseType = TYP_BYTE; break; case TYP_CHAR: constVal = 0x80008000; *inOutBaseType = TYP_SHORT; break; case TYP_UINT: constVal = 0x80000000; *inOutBaseType = TYP_INT; break; case TYP_ULONG: constVal = 0x8000000000000000LL; *inOutBaseType = TYP_LONG; break; } assert(constVal != 0); // This transformation is not required for equality. if (intrinsicID != SIMDIntrinsicEqual) { // For constructing const vector use either long or int base type. var_types tempBaseType = (baseType == TYP_ULONG) ? TYP_LONG : TYP_INT; GenTree* initVal = gtNewIconNode(constVal); initVal->gtType = tempBaseType; GenTree* constVector = gtNewSIMDNode(TYP_STRUCT, initVal, nullptr, SIMDIntrinsicInit, tempBaseType, size); // Assign constVector to a temp, since we intend to use it more than once // TODO-CQ: We have quite a few such constant vectors constructed during // the importation of SIMD intrinsics. Make sure that we have a single // temp per distinct constant per method. GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd); // op1 = op1 - constVector // op2 = op2 - constVector *op1 = gtNewSIMDNode(TYP_STRUCT, *op1, constVector, SIMDIntrinsicSub, baseType, size); *op2 = gtNewSIMDNode(TYP_STRUCT, *op2, tmp, SIMDIntrinsicSub, baseType, size); } return impSIMDRelOp(intrinsicID, typeHnd, size, inOutBaseType, op1, op2); } } return intrinsicID; #else assert(!"impSIMDRelOp() unimplemented on target arch"); unreached(); #endif //_TARGET_AMD64_ } // Creates a GT_SIMD tree for Select operation // // Argumens: // typeHnd - type handle of SIMD vector // baseType - base type of SIMD vector // size - SIMD vector size // op1 - first operand = Condition vector vc // op2 - second operand = va // op3 - third operand = vb // // Return Value: // Returns GT_SIMD tree that computes Select(vc, va, vb) // GenTreePtr Compiler::impSIMDSelect(CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1, GenTree* op2, GenTree* op3) { assert(op1->TypeGet() == TYP_STRUCT); assert(op2->TypeGet() == TYP_STRUCT); assert(op3->TypeGet() == TYP_STRUCT); // Select(BitVector vc, va, vb) = (va & vc) | (vb & !vc) // Select(op1, op2, op3) = (op2 & op1) | (op3 & !op1) // = SIMDIntrinsicBitwiseOr(SIMDIntrinsicBitwiseAnd(op2, op1), SIMDIntrinsicBitwiseAndNot(op3, op1)) // // If Op1 has side effect, create an assignment to a temp GenTree* tmp = op1; GenTree* asg = nullptr; if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0) { unsigned lclNum = lvaGrabTemp(true DEBUGARG("SIMD Select")); tmp = gtNewLclvNode(lclNum, op1->TypeGet()); lvaSetStruct(lclNum, typeHnd, false); asg = gtNewTempAssign(lclNum, op1); } GenTree* andExpr = gtNewSIMDNode(TYP_STRUCT, op2, tmp, SIMDIntrinsicBitwiseAnd, baseType, size); GenTree* dupOp1 = gtCloneExpr(tmp); assert(dupOp1 != nullptr); GenTree* andNotExpr = gtNewSIMDNode(TYP_STRUCT, dupOp1, op3, SIMDIntrinsicBitwiseAndNot, baseType, size); GenTree* simdTree = gtNewSIMDNode(TYP_STRUCT, andExpr, andNotExpr, SIMDIntrinsicBitwiseOr, baseType, size); // If asg not null, create a GT_COMMA tree. if (asg != nullptr) { simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), asg, simdTree); } return simdTree; } // Creates a GT_SIMD tree for Min/Max operation // // Argumens: // IntrinsicId - SIMD intrinsic Id, either Min or Max // typeHnd - type handle of SIMD vector // baseType - base type of SIMD vector // size - SIMD vector size // op1 - first operand = va // op2 - second operand = vb // // Return Value: // Returns GT_SIMD tree that computes Max(va, vb) // GenTreePtr Compiler::impSIMDMinMax(SIMDIntrinsicID intrinsicId, CORINFO_CLASS_HANDLE typeHnd, var_types baseType, unsigned size, GenTree* op1, GenTree* op2) { assert(intrinsicId == SIMDIntrinsicMin || intrinsicId == SIMDIntrinsicMax); assert(op1->TypeGet() == TYP_STRUCT); assert(op2->TypeGet() == TYP_STRUCT); #ifdef _TARGET_AMD64_ // SSE2 has direct support for float/double/signed word/unsigned byte. // For other integer types we compute min/max as follows // // int32/uint32/int64/uint64: // compResult = (op1 < op2) in case of Min // (op1 > op2) in case of Max // Min/Max(op1, op2) = Select(compResult, op1, op2) // // unsigned word: // op1 = op1 - 2^15 ; to make it fit within a signed word // op2 = op2 - 2^15 ; to make it fit within a signed word // result = SSE2 signed word Min/Max(op1, op2) // result = result + 2^15 ; readjust it back // // signed byte: // op1 = op1 + 2^7 ; to make it unsigned // op1 = op1 + 2^7 ; to make it unsigned // result = SSE2 unsigned byte Min/Max(op1, op2) // result = result - 2^15 ; readjust it back GenTree* simdTree = nullptr; if (varTypeIsFloating(baseType) || baseType == TYP_SHORT || baseType == TYP_UBYTE) { // SSE2 has direct support simdTree = gtNewSIMDNode(TYP_STRUCT, op1, op2, intrinsicId, baseType, size); } else if (baseType == TYP_CHAR || baseType == TYP_BYTE) { int constVal; SIMDIntrinsicID operIntrinsic; SIMDIntrinsicID adjustIntrinsic; var_types minMaxOperBaseType; if (baseType == TYP_CHAR) { constVal = 0x80008000; operIntrinsic = SIMDIntrinsicSub; adjustIntrinsic = SIMDIntrinsicAdd; minMaxOperBaseType = TYP_SHORT; } else { assert(baseType == TYP_BYTE); constVal = 0x80808080; operIntrinsic = SIMDIntrinsicAdd; adjustIntrinsic = SIMDIntrinsicSub; minMaxOperBaseType = TYP_UBYTE; } GenTree* initVal = gtNewIconNode(constVal); GenTree* constVector = gtNewSIMDNode(TYP_STRUCT, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size); // Assign constVector to a temp, since we intend to use it more than once // TODO-CQ: We have quite a few such constant vectors constructed during // the importation of SIMD intrinsics. Make sure that we have a single // temp per distinct constant per method. GenTree* tmp = fgInsertCommaFormTemp(&constVector, typeHnd); // op1 = op1 - constVector // op2 = op2 - constVector op1 = gtNewSIMDNode(TYP_STRUCT, op1, constVector, operIntrinsic, baseType, size); op2 = gtNewSIMDNode(TYP_STRUCT, op2, tmp, operIntrinsic, baseType, size); // compute min/max of op1 and op2 considering them as if minMaxOperBaseType simdTree = gtNewSIMDNode(TYP_STRUCT, op1, op2, intrinsicId, minMaxOperBaseType, size); // re-adjust the value by adding or subtracting constVector tmp = gtNewLclvNode(tmp->AsLclVarCommon()->GetLclNum(), tmp->TypeGet()); simdTree = gtNewSIMDNode(TYP_STRUCT, simdTree, tmp, adjustIntrinsic, baseType, size); } else { GenTree* dupOp1 = nullptr; GenTree* dupOp2 = nullptr; GenTree* op1Assign = nullptr; GenTree* op2Assign = nullptr; unsigned op1LclNum; unsigned op2LclNum; if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0) { op1LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max")); dupOp1 = gtNewLclvNode(op1LclNum, op1->TypeGet()); lvaSetStruct(op1LclNum, typeHnd, false); op1Assign = gtNewTempAssign(op1LclNum, op1); op1 = gtNewLclvNode(op1LclNum, op1->TypeGet()); } else { dupOp1 = gtCloneExpr(op1); } if ((op2->gtFlags & GTF_SIDE_EFFECT) != 0) { op2LclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max")); dupOp2 = gtNewLclvNode(op2LclNum, op2->TypeGet()); lvaSetStruct(op2LclNum, typeHnd, false); op2Assign = gtNewTempAssign(op2LclNum, op2); op2 = gtNewLclvNode(op2LclNum, op2->TypeGet()); } else { dupOp2 = gtCloneExpr(op2); } SIMDIntrinsicID relOpIntrinsic = (intrinsicId == SIMDIntrinsicMin) ? SIMDIntrinsicLessThan : SIMDIntrinsicGreaterThan; var_types relOpBaseType = baseType; // compResult = op1 relOp op2 // simdTree = Select(compResult, op1, op2); assert(dupOp1 != nullptr); assert(dupOp2 != nullptr); relOpIntrinsic = impSIMDRelOp(relOpIntrinsic, typeHnd, size, &relOpBaseType, &dupOp1, &dupOp2); GenTree* compResult = gtNewSIMDNode(TYP_STRUCT, dupOp1, dupOp2, relOpIntrinsic, relOpBaseType, size); unsigned compResultLclNum = lvaGrabTemp(true DEBUGARG("SIMD Min/Max")); lvaSetStruct(compResultLclNum, typeHnd, false); GenTree* compResultAssign = gtNewTempAssign(compResultLclNum, compResult); compResult = gtNewLclvNode(compResultLclNum, compResult->TypeGet()); simdTree = impSIMDSelect(typeHnd, baseType, size, compResult, op1, op2); simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), compResultAssign, simdTree); // Now create comma trees if we have created assignments of op1/op2 to temps if (op2Assign != nullptr) { simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op2Assign, simdTree); } if (op1Assign != nullptr) { simdTree = gtNewOperNode(GT_COMMA, simdTree->TypeGet(), op1Assign, simdTree); } } assert(simdTree != nullptr); return simdTree; #else assert(!"impSIMDMinMax() unimplemented on target arch"); unreached(); #endif //_TARGET_AMD64_ } //------------------------------------------------------------------------ // getOp1ForConstructor: Get the op1 for a constructor call. // // Arguments: // opcode - the opcode being handled (needed to identify the CEE_NEWOBJ case) // newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object. // clsHnd - The handle of the class of the method. // // Return Value: // The tree node representing the object to be initialized with the constructor. // // Notes: // This method handles the differences between the CEE_NEWOBJ and constructor cases. // GenTreePtr Compiler::getOp1ForConstructor(OPCODE opcode, GenTreePtr newobjThis, CORINFO_CLASS_HANDLE clsHnd) { GenTree* op1; if (opcode == CEE_NEWOBJ) { op1 = newobjThis; assert(newobjThis->gtOper == GT_ADDR && newobjThis->gtOp.gtOp1->gtOper == GT_LCL_VAR); // push newobj result on type stack unsigned tmp = op1->gtOp.gtOp1->gtLclVarCommon.gtLclNum; impPushOnStack(gtNewLclvNode(tmp, lvaGetRealType(tmp)), verMakeTypeInfo(clsHnd).NormaliseForStack()); } else { op1 = impSIMDPopStack(); } assert(op1->TypeGet() == TYP_BYREF); return op1; } //------------------------------------------------------------------- // Set the flag that indicates that the lclVar referenced by this tree // is used in a SIMD intrinsic. // Arguments: // tree - GenTreePtr void Compiler::setLclRelatedToSIMDIntrinsic(GenTreePtr tree) { assert(tree->OperIsLocal()); unsigned lclNum = tree->AsLclVarCommon()->GetLclNum(); LclVarDsc* lclVarDsc = &lvaTable[lclNum]; lclVarDsc->lvUsedInSIMDIntrinsic = true; } //------------------------------------------------------------- // Check if two field nodes reference at the same memory location. // Notice that this check is just based on pattern matching. // Arguments: // op1 - GenTreePtr. // op2 - GenTreePtr. // Return Value: // If op1's parents node and op2's parents node are at the same location, return true. Otherwise, return false bool areFieldsParentsLocatedSame(GenTreePtr op1, GenTreePtr op2) { assert(op1->OperGet() == GT_FIELD); assert(op2->OperGet() == GT_FIELD); GenTreePtr op1ObjRef = op1->gtField.gtFldObj; GenTreePtr op2ObjRef = op2->gtField.gtFldObj; while (op1ObjRef != nullptr && op2ObjRef != nullptr) { if(op1ObjRef->OperGet() != op2ObjRef->OperGet()) { break; } else if (op1ObjRef->OperGet() == GT_ADDR) { op1ObjRef = op1ObjRef->gtOp.gtOp1; op2ObjRef = op2ObjRef->gtOp.gtOp1; } if (op1ObjRef->OperIsLocal() && op2ObjRef->OperIsLocal() && op1ObjRef->AsLclVarCommon()->GetLclNum() == op2ObjRef->AsLclVarCommon()->GetLclNum()) { return true; } else if (op1ObjRef->OperGet() == GT_FIELD && op2ObjRef->OperGet() == GT_FIELD && op1ObjRef->gtField.gtFldHnd == op2ObjRef->gtField.gtFldHnd) { op1ObjRef = op1ObjRef->gtField.gtFldObj; op2ObjRef = op2ObjRef->gtField.gtFldObj; continue; } else { break; } } return false; } //---------------------------------------------------------------------- // Check whether two field are contiguous // Arguments: // first - GenTreePtr. The Type of the node should be TYP_FLOAT // second - GenTreePtr. The Type of the node should be TYP_FLOAT // Return Value: // if the first field is located before second field, and they are located contiguously, // then return true. Otherwise, return false. bool Compiler::areFieldsContiguous(GenTreePtr first, GenTreePtr second) { assert(first->OperGet() == GT_FIELD); assert(second->OperGet() == GT_FIELD); assert(first->gtType == TYP_FLOAT); assert(second->gtType == TYP_FLOAT); var_types firstFieldType = first->gtType; var_types secondFieldType = second->gtType; unsigned firstFieldEndOffset = first->gtField.gtFldOffset + genTypeSize(firstFieldType); unsigned secondFieldOffset = second->gtField.gtFldOffset; if (firstFieldEndOffset == secondFieldOffset && firstFieldType == secondFieldType && areFieldsParentsLocatedSame(first, second)) { return true; } return false; } //------------------------------------------------------------------------------- // Check whether two array element nodes are located contiguously or not. // Arguments: // op1 - GenTreePtr. // op2 - GenTreePtr. // Return Value: // if the array element op1 is located before array element op2, and they are located contiguously, // then return true. Otherwise, return false. // TODO-CQ: // Right this can only check array element with const number as index. In future, // we should consider to allow this function to check the index using expression. bool Compiler::areArrayElementsLocatedContiguously(GenTreePtr op1, GenTreePtr op2) { noway_assert(op1->gtOper == GT_INDEX); noway_assert(op2->gtOper == GT_INDEX); GenTreeIndex* op1Index = op1->AsIndex(); GenTreeIndex* op2Index = op2->AsIndex(); GenTreePtr op1ArrayRef = op1Index->Arr(); GenTreePtr op2ArrayRef = op2Index->Arr(); assert(op1ArrayRef->TypeGet() == TYP_REF); assert(op2ArrayRef->TypeGet() == TYP_REF); GenTreePtr op1IndexNode = op1Index->Index(); GenTreePtr op2IndexNode = op2Index->Index(); if ((op1IndexNode->OperGet() == GT_CNS_INT && op2IndexNode->OperGet() == GT_CNS_INT) && op1IndexNode->gtIntCon.gtIconVal + 1 == op2IndexNode->gtIntCon.gtIconVal) { if (op1ArrayRef->OperGet() == GT_FIELD && op2ArrayRef->OperGet() == GT_FIELD && areFieldsParentsLocatedSame(op1ArrayRef, op2ArrayRef)) { return true; } else if (op1ArrayRef->OperIsLocal() && op2ArrayRef->OperIsLocal() && op1ArrayRef->AsLclVarCommon()->GetLclNum() == op2ArrayRef->AsLclVarCommon()->GetLclNum()) { return true; } } return false; } //------------------------------------------------------------------------------- // Check whether two argument nodes are located contiguously or not. // Arguments: // op1 - GenTreePtr. // op2 - GenTreePtr. // Return Value: // if the argument node op1 is located before argument node op2, and they are located contiguously, // then return true. Otherwise, return false. // TODO-CQ: // Right now this can only check field and array. In future we should add more cases. // bool Compiler::areArgumentsLocatedContiguously(GenTreePtr op1, GenTreePtr op2) { if(op1->OperGet() == GT_INDEX && op2->OperGet() == GT_INDEX) { return areArrayElementsLocatedContiguously(op1, op2); } else if(op1->OperGet() == GT_FIELD && op2->OperGet() == GT_FIELD) { return areFieldsContiguous(op1, op2); } return false; } //-------------------------------------------------------------------------------------------------------- // createAddressNodeForSIMDInit: Generate the address node(GT_LEA) if we want to intialize vector2, vector3 or vector4 from first argument's address. // // Arguments: // tree - GenTreePtr. This the tree node which is used to get the address for indir. // simdsize - unsigned. This the simd vector size. // arrayElementsCount - unsigned. This is used for generating the boundary check for array. // // Return value: // return the address node. // // TODO-CQ: // 1. Currently just support for GT_FIELD and GT_INDEX, because we can only verify the GT_INDEX node or GT_Field are located contiguously or not. // In future we should support more cases. // 2.Though it happens to just work fine front-end phases are not aware of GT_LEA node. Therefore, convert these to use GT_ADDR . GenTreePtr Compiler::createAddressNodeForSIMDInit(GenTreePtr tree, unsigned simdSize) { assert(tree->OperGet() == GT_FIELD || tree->OperGet() == GT_INDEX); GenTreePtr byrefNode = nullptr; GenTreePtr startIndex = nullptr; unsigned offset = 0; var_types baseType = tree->gtType; if (tree->OperGet() == GT_FIELD) { GenTreePtr objRef = tree->gtField.gtFldObj; if(objRef != nullptr && objRef->gtOper == GT_ADDR) { GenTreePtr obj = objRef->gtOp.gtOp1; // If the field is directly from a struct, then in this case, // we should set this struct's lvUsedInSIMDIntrinsic as true, // so that this sturct won't be promoted. // e.g. s.x x is a field, and s is a struct, then we should set the s's lvUsedInSIMDIntrinsic as true. // so that s won't be promoted. // Notice that if we have a case like s1.s2.x. s1 s2 are struct, and x is a field, then it is possible that s1 can be promoted, so that s2 can be promoted. // The reason for that is if we don't allow s1 to be promoted, then this will affect the other optimizations which are depend on s1's struct promotion. // TODO-CQ: // In future, we should optimize this case so that if there is a nested field like s1.s2.x and s1.s2.x's address is used for // initializing the vector, then s1 can be promoted but s2 can't. if(obj->gtType == TYP_STRUCT && obj->OperIsLocal()) { setLclRelatedToSIMDIntrinsic(obj); } } byrefNode = gtCloneExpr(tree->gtField.gtFldObj); assert(byrefNode != nullptr); offset = tree->gtField.gtFldOffset; } else if(tree->OperGet() == GT_INDEX) { GenTreePtr index = tree->AsIndex()->Index(); assert(index->OperGet() == GT_CNS_INT); GenTreePtr checkIndexExpr = nullptr; unsigned indexVal = (unsigned)(index->gtIntCon.gtIconVal); offset = indexVal * genTypeSize(tree->TypeGet()); GenTreePtr arrayRef = tree->AsIndex()->Arr(); // Generate the boundary check exception. // The length for boundary check should be the maximum index number which should be // (first argument's index number) + (how many array arguments we have) - 1 // = indexVal + arrayElementsCount - 1 unsigned arrayElementsCount = simdSize / genTypeSize(baseType); checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, indexVal + arrayElementsCount - 1); GenTreeArrLen* arrLen = new (this, GT_ARR_LENGTH) GenTreeArrLen(TYP_INT, arrayRef, (int)offsetof(CORINFO_Array, length)); GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK) GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, arrLen, checkIndexExpr); offset += offsetof(CORINFO_Array, u1Elems); byrefNode = gtNewOperNode(GT_COMMA, arrayRef->TypeGet(), arrBndsChk, gtCloneExpr(arrayRef)); } else { unreached(); } GenTreePtr address = new (this, GT_LEA) GenTreeAddrMode(TYP_BYREF, byrefNode, startIndex, genTypeSize(tree->TypeGet()), offset); return address; } //------------------------------------------------------------------------------- // impMarkContiguousSIMDFieldAssignments: Try to identify if there are contiguous // assignments from SIMD field to memory. If there are, then mark the related // lclvar so that it won't be promoted. // // Arguments: // stmt - GenTreePtr. Input statement node. void Compiler::impMarkContiguousSIMDFieldAssignments(GenTreePtr stmt) { GenTreePtr expr = stmt->gtStmt.gtStmtExpr; if (expr->OperGet() == GT_ASG && expr->TypeGet() == TYP_FLOAT) { GenTreePtr curDst = expr->gtOp.gtOp1; GenTreePtr curSrc = expr->gtOp.gtOp2; unsigned index = 0; var_types baseType = TYP_UNKNOWN; unsigned simdSize = 0; GenTreePtr srcSimdStructNode = getSIMDStructFromField(curSrc, &baseType, &index, &simdSize, true); if (srcSimdStructNode == nullptr || baseType != TYP_FLOAT) { fgPreviousCandidateSIMDFieldAsgStmt = nullptr; } else if (index == 0 && isSIMDTypeLocal(srcSimdStructNode)) { fgPreviousCandidateSIMDFieldAsgStmt = stmt; } else if (fgPreviousCandidateSIMDFieldAsgStmt != nullptr) { assert(index > 0); GenTreePtr prevAsgExpr = fgPreviousCandidateSIMDFieldAsgStmt->gtStmt.gtStmtExpr; GenTreePtr prevDst = prevAsgExpr->gtOp.gtOp1; GenTreePtr prevSrc = prevAsgExpr->gtOp.gtOp2; if (!areArgumentsLocatedContiguously(prevDst, curDst) || !areArgumentsLocatedContiguously(prevSrc, curSrc)) { fgPreviousCandidateSIMDFieldAsgStmt = nullptr; } else { if (index == (simdSize / genTypeSize(baseType) - 1)) { // Successfully found the pattern, mark the lclvar as UsedInSIMDIntrinsic if (srcSimdStructNode->OperIsLocal()) { setLclRelatedToSIMDIntrinsic(srcSimdStructNode); } if (curDst->OperGet() == GT_FIELD) { GenTreePtr objRef = curDst->gtField.gtFldObj; if (objRef != nullptr && objRef->gtOper == GT_ADDR) { GenTreePtr obj = objRef->gtOp.gtOp1; if (obj->gtType == TYP_STRUCT && obj->OperIsLocal()) { setLclRelatedToSIMDIntrinsic(obj); } } } } else { fgPreviousCandidateSIMDFieldAsgStmt = stmt; } } } } else { fgPreviousCandidateSIMDFieldAsgStmt = nullptr; } } //------------------------------------------------------------------------ // impSIMDIntrinsic: Check method to see if it is a SIMD method // // Arguments: // opcode - the opcode being handled (needed to identify the CEE_NEWOBJ case) // newobjThis - For CEE_NEWOBJ, this is the temp grabbed for the allocated uninitalized object. // clsHnd - The handle of the class of the method. // method - The handle of the method. // sig - The call signature for the method. // memberRef - The memberRef token for the method reference. // // Return Value: // If clsHnd is a known SIMD type, and 'method' is one of the methods that are // implemented as an intrinsic in the JIT, then return the tree that implements // it. // GenTreePtr Compiler::impSIMDIntrinsic(OPCODE opcode, GenTreePtr newobjThis, CORINFO_CLASS_HANDLE clsHnd, CORINFO_METHOD_HANDLE methodHnd, CORINFO_SIG_INFO * sig, int memberRef) { assert(featureSIMD); if (!isSIMDClass(clsHnd)) { return nullptr; } // For now SIMD Intrinsics are disabled in debuggable code. // TODO-Cleanup: add support for SIMD intrinsics regardless of debuggable code. // Note that we can't disable SIMD intrinsics if we are set to use AVX, because the // reference implementation won't match the vector size. if (!canUseAVX() && opts.compDbgCode) { return nullptr; } // Get base type and intrinsic Id var_types baseType = TYP_UNKNOWN; unsigned size = 0; unsigned argCount = 0; const SIMDIntrinsicInfo* intrinsicInfo = getSIMDIntrinsicInfo(&clsHnd, methodHnd, sig, (opcode == CEE_NEWOBJ), &argCount, &baseType, &size); if (intrinsicInfo == nullptr || intrinsicInfo->id == SIMDIntrinsicInvalid) { return nullptr; } SIMDIntrinsicID simdIntrinsicID = intrinsicInfo->id; bool instMethod = intrinsicInfo->isInstMethod; var_types callType = JITtype2varType(sig->retType); GenTree* simdTree = nullptr; GenTree* op1 = nullptr; GenTree* op2 = nullptr; GenTree* op3 = nullptr; GenTree* retVal = nullptr; GenTree* copyBlkDst = nullptr; bool doCopyBlk = false; switch(simdIntrinsicID) { case SIMDIntrinsicGetCount: { int length = getSIMDVectorLength(clsHnd); GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, length); retVal = intConstTree; } break; case SIMDIntrinsicGetZero: { baseType = genActualType(baseType); GenTree *initVal = gtNewZeroConNode(baseType); initVal->gtType = baseType; simdTree = gtNewSIMDNode(TYP_STRUCT, initVal, nullptr, SIMDIntrinsicInit, baseType, size); retVal = simdTree; } break; case SIMDIntrinsicGetOne: { GenTree *initVal; if (varTypeIsSmallInt(baseType)) { unsigned baseSize = genTypeSize(baseType); int val; if (baseSize == 1) { val = 0x01010101; } else { val = 0x00010001; } initVal = gtNewIconNode(val); } else { initVal = gtNewOneConNode(baseType); } baseType = genActualType(baseType); initVal->gtType = baseType; simdTree = gtNewSIMDNode(TYP_STRUCT, initVal, nullptr, SIMDIntrinsicInit, baseType, size); retVal = simdTree; } break; case SIMDIntrinsicGetAllOnes: { // Equivalent to (Vector) new Vector(0xffffffff); GenTree *initVal = gtNewIconNode(0xffffffff, TYP_INT); simdTree = gtNewSIMDNode(TYP_STRUCT, initVal, nullptr, SIMDIntrinsicInit, TYP_INT, size); if (baseType != TYP_INT) { // cast it to required baseType if different from TYP_INT simdTree = gtNewSIMDNode(genActualType(callType), simdTree, nullptr, SIMDIntrinsicCast, baseType, size); } retVal = simdTree; } break; case SIMDIntrinsicInit: case SIMDIntrinsicInitN: { // SIMDIntrinsicInit: // op2 - the initializer value // op1 - byref of vector // // SIMDIntrinsicInitN // op2 - list of initializer values stitched into a list // op1 - byref of vector bool initFromFirstArgIndir = false; if (simdIntrinsicID == SIMDIntrinsicInit) { op2 = impSIMDPopStack(); } else { assert(simdIntrinsicID == SIMDIntrinsicInitN); assert(baseType == TYP_FLOAT); unsigned initCount = argCount - 1; unsigned elementCount = getSIMDVectorLength(size, baseType); noway_assert(initCount == elementCount); GenTree* nextArg = op2; // Build a GT_LIST with the N values. // For efficient codegen that requires minimal internal regs, we build the list // with the args in reverse order. ArrayStack listArgs(this); for (unsigned i = 0; i < initCount; i++) { GenTree* nextArg = impSIMDPopStack(); assert(nextArg->TypeGet() == baseType); listArgs.Push(nextArg); } GenTree* list = nullptr; GenTreePtr firstArg = nullptr; GenTreePtr preArg = nullptr; int offset = 0; bool areArgsLocatedContiguously = true; for (unsigned i = 0; i < initCount; i++) { GenTree* nextArg = listArgs.Pop(); if (areArgsLocatedContiguously) { GenTreePtr curArg = nextArg; if(firstArg == nullptr) { firstArg = curArg; } if(preArg != nullptr) { areArgsLocatedContiguously = areArgumentsLocatedContiguously(preArg, curArg); } preArg = curArg; } list = new (this, GT_LIST) GenTreeOp(GT_LIST, baseType, nextArg, list); } if (areArgsLocatedContiguously && baseType == TYP_FLOAT) { // Since Vector2, Vector3 and Vector4's arguments type are only float, // we intialize the vector from first argument address, only when // the baseType is TYP_FLOAT and the arguments are located contiguously in memory initFromFirstArgIndir = true; GenTreePtr op2Address = createAddressNodeForSIMDInit(firstArg, size); var_types simdType = getSIMDTypeForSize(size); op2 = gtNewOperNode(GT_IND, simdType, op2Address); } else { op2 = list; } } op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd); assert(op1->TypeGet() == TYP_BYREF); assert(genActualType(op2->TypeGet()) == genActualType(baseType)||initFromFirstArgIndir); #if AVX_WITHOUT_AVX2 // NOTE: This #define, AVX_WITHOUT_AVX2, is never defined. This code is kept here // in case we decide to implement AVX support (32 byte vectors) with AVX only. // On AVX (as opposed to AVX2), broadcast is supported only for float and double, // and requires taking a mem address of the value. // If not a constant, take the addr of op2. if (simdIntrinsicID == SIMDIntrinsicInit && canUseAVX()) { if (!op2->OperIsConst()) { // It is better to assign op2 to a temp and take the addr of temp // rather than taking address of op2 since the latter would make op2 // address-taken and ineligible for register allocation. // // op2 = GT_COMMA(tmp=op2, GT_ADDR(tmp)) unsigned tmpNum = lvaGrabTemp(true DEBUGARG("Val addr for vector Init")); GenTreePtr asg = gtNewTempAssign(tmpNum, op2); GenTreePtr tmp = gtNewLclvNode(tmpNum, op2->TypeGet()); tmp = gtNewOperNode(GT_ADDR, TYP_BYREF,tmp); op2 = gtNewOperNode(GT_COMMA, TYP_BYREF, asg, tmp); } } #endif // For integral base types of size less than TYP_INT, expand the initializer // to fill size of TYP_INT bytes. if (varTypeIsSmallInt(baseType)) { // This case should occur only for Init intrinsic. assert(simdIntrinsicID == SIMDIntrinsicInit); unsigned baseSize = genTypeSize(baseType); int multiplier; if (baseSize == 1) { multiplier = 0x01010101; } else { assert(baseSize == 2); multiplier = 0x00010001; } GenTree* t1 = nullptr; if (baseType == TYP_BYTE) { // What we have is a signed byte initializer, // which when loaded to a reg will get sign extended to TYP_INT. // But what we need is the initializer without sign extended or // rather zero extended to 32-bits. t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xff, TYP_INT)); } else if (baseType == TYP_SHORT) { // What we have is a signed short initializer, // which when loaded to a reg will get sign extended to TYP_INT. // But what we need is the initializer without sign extended or // rather zero extended to 32-bits. t1 = gtNewOperNode(GT_AND, TYP_INT, op2, gtNewIconNode(0xffff, TYP_INT)); } else { assert(baseType == TYP_UBYTE || baseType == TYP_CHAR); t1 = gtNewCastNode(TYP_INT, op2, TYP_INT); } assert(t1 != nullptr); GenTree* t2 = gtNewIconNode(multiplier, TYP_INT); op2 = gtNewOperNode(GT_MUL, TYP_INT, t1, t2); // Construct a vector of TYP_INT with the new initializer and cast it back to vector of baseType simdTree = gtNewSIMDNode(TYP_STRUCT, op2, nullptr, simdIntrinsicID, TYP_INT, size); simdTree = gtNewSIMDNode(TYP_STRUCT, simdTree, nullptr, SIMDIntrinsicCast, baseType, size); } else { if (initFromFirstArgIndir) { simdTree = op2; if (op1->gtOp.gtOp1->OperIsLocal()) { // label the dst struct's lclvar is used for SIMD intrinsic, // so that this dst struct won't be promoted. setLclRelatedToSIMDIntrinsic(op1->gtOp.gtOp1); } } else { simdTree = gtNewSIMDNode(TYP_STRUCT, op2, nullptr, simdIntrinsicID, baseType, size); } } copyBlkDst = op1; doCopyBlk = true; } break; case SIMDIntrinsicInitArray: case SIMDIntrinsicInitArrayX: case SIMDIntrinsicCopyToArray: case SIMDIntrinsicCopyToArrayX: { // op3 - index into array in case of SIMDIntrinsicCopyToArrayX and SIMDIntrinsicInitArrayX // op2 - array itself // op1 - byref to vector struct unsigned int vectorLength = getSIMDVectorLength(size, baseType); // We will need to generate an array bounds check for the last array element will access. // (This constructor takes only the zero-based arrays.) // We'll either check against (vectorLength - 1) or (index + vectorLength - 1). GenTree* checkIndexExpr = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength - 1); // Get the index into the array. If it has been provided, it will be on the // top of the stack. Otherwise, it is null. if (argCount == 3) { GenTree* index = impSIMDPopStack(); assert(simdIntrinsicID == SIMDIntrinsicInitArrayX || simdIntrinsicID == SIMDIntrinsicCopyToArrayX); if ((index->gtFlags & GTF_SIDE_EFFECT) != 0) { op3 = fgInsertCommaFormTemp(&index); } else { op3 = gtCloneExpr(index); } checkIndexExpr = gtNewOperNode(GT_ADD, TYP_INT, index, checkIndexExpr); } else { // TODO-CQ: Here, or elsewhere, check for the pattern where op2 is a newly constructed array, and // change this to the InitN form. // op3 = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 0); op3 = nullptr; } // Clone the array for use in the bounds check. op2 = impSIMDPopStack(); assert(op2->TypeGet() == TYP_REF); GenTree* arrayRef = op2; GenTree* asg = nullptr; if ((arrayRef->gtFlags & GTF_SIDE_EFFECT) != 0) { op2 = fgInsertCommaFormTemp(&arrayRef); } else { op2 = gtCloneExpr(arrayRef); } assert(op2 != nullptr); // Insert a bounds check for index + offset - 1. // This must be a "normal" array. GenTreeArrLen* arrLen = new (this, GT_ARR_LENGTH) GenTreeArrLen(TYP_INT, arrayRef, (int)offsetof(CORINFO_Array, length)); GenTreeBoundsChk* arrBndsChk = new (this, GT_ARR_BOUNDS_CHECK) GenTreeBoundsChk(GT_ARR_BOUNDS_CHECK, TYP_VOID, arrLen, checkIndexExpr); // Create a GT_COMMA tree for the bounds check. op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), arrBndsChk, op2); if (simdIntrinsicID == SIMDIntrinsicInitArray || simdIntrinsicID == SIMDIntrinsicInitArrayX) { op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd); simdTree = gtNewSIMDNode(TYP_STRUCT, op2, op3, SIMDIntrinsicInitArray, baseType, size); copyBlkDst = op1; doCopyBlk = true; } else { assert(simdIntrinsicID == SIMDIntrinsicCopyToArray || simdIntrinsicID == SIMDIntrinsicCopyToArrayX); op1 = impSIMDPopStack(instMethod); assert(op1->TypeGet() == TYP_STRUCT); // copy vector (op1) to array (op2) starting at index (op3) simdTree = op1; // TODO-Cleanup: Though it happens to just work fine front-end phases are not aware of GT_LEA node. Therefore, convert these to use GT_ADDR . copyBlkDst = new (this, GT_LEA) GenTreeAddrMode(TYP_BYREF, op2, op3, genTypeSize(baseType), offsetof(CORINFO_Array, u1Elems)); doCopyBlk = true; } } break; case SIMDIntrinsicInitFixed: { // We are initializing a fixed-length vector VLarge with a smaller fixed-length vector VSmall, plus 1 or 2 additional floats. // op4 (optional) - float value for VLarge.W, if VLarge is Vector4, and VSmall is Vector2 // op3 - float value for VLarge.Z or VLarge.W // op2 - VSmall // op1 - byref of VLarge assert(baseType == TYP_FLOAT); unsigned elementByteCount = 4; GenTree* op4 = nullptr; if (argCount == 4) { op4 = impSIMDPopStack(); assert(op4->TypeGet() == TYP_FLOAT); } op3 = impSIMDPopStack(); assert(op3->TypeGet() == TYP_FLOAT); op2 = impSIMDPopStack(); assert(op2->TypeGet() == TYP_STRUCT); op1 = getOp1ForConstructor(opcode, newobjThis, clsHnd); // We are going to redefine the operands so that: // - op3 is the value that's going into the Z position, or null if it's a Vector4 constructor with a single operand, and // - op4 is the W position value, or null if this is a Vector3 constructor. if (size == 16 && argCount == 3) { op4 = op3; op3 = nullptr; } simdTree = op2; if (op3 != nullptr) { simdTree = gtNewSIMDNode(TYP_STRUCT, simdTree, op3, SIMDIntrinsicSetZ, baseType, size); } if (op4 != nullptr) { simdTree = gtNewSIMDNode(TYP_STRUCT, simdTree, op4, SIMDIntrinsicSetW, baseType, size); } copyBlkDst = op1; doCopyBlk = true; } break; case SIMDIntrinsicOpEquality: case SIMDIntrinsicInstEquals: { op2 = impSIMDPopStack(); op1 = impSIMDPopStack(instMethod); assert(op1->TypeGet() == TYP_STRUCT); assert(op2->TypeGet() == TYP_STRUCT); simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpEquality, baseType, size); retVal = simdTree; } break; case SIMDIntrinsicOpInEquality: { // op1 is the first operand // op2 is the second operand op2 = impSIMDPopStack(); op1 = impSIMDPopStack(instMethod); assert(op1->TypeGet() == TYP_STRUCT); assert(op2->TypeGet() == TYP_STRUCT); simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, SIMDIntrinsicOpInEquality, baseType, size); retVal = simdTree; } break; case SIMDIntrinsicEqual: case SIMDIntrinsicLessThan: case SIMDIntrinsicLessThanOrEqual: case SIMDIntrinsicGreaterThan: case SIMDIntrinsicGreaterThanOrEqual: { op2 = impSIMDPopStack(); op1 = impSIMDPopStack(instMethod); assert(op1->TypeGet() == TYP_STRUCT); assert(op2->TypeGet() == TYP_STRUCT); SIMDIntrinsicID intrinsicID = impSIMDRelOp(simdIntrinsicID, clsHnd, size, &baseType, &op1, &op2); simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, intrinsicID, baseType, size); retVal = simdTree; } break; case SIMDIntrinsicAdd: case SIMDIntrinsicSub: case SIMDIntrinsicMul: case SIMDIntrinsicDiv: case SIMDIntrinsicBitwiseAnd: case SIMDIntrinsicBitwiseAndNot: case SIMDIntrinsicBitwiseOr: case SIMDIntrinsicBitwiseXor: { #if defined(_TARGET_AMD64_) && defined(DEBUG) // check for the cases where we don't support intrinsics. // This check should be done before we make modifications to type stack. // Note that this is more of a double safety check for robustness since // we expect getSIMDIntrinsicInfo() to have filtered out intrinsics on // unsupported base types. If getSIMdIntrinsicInfo() doesn't filter due // to some bug, assert in chk/dbg will fire. if (!varTypeIsFloating(baseType)) { if (simdIntrinsicID == SIMDIntrinsicMul) { if ((baseType != TYP_INT) && (baseType != TYP_SHORT)) { // TODO-CQ: implement mul on these integer vectors. // Note that SSE2 has no direct support for these vectors. assert(!"Mul not supported on long/ulong/uint/small int vectors\n"); return nullptr; } } // common to all integer type vectors if (simdIntrinsicID == SIMDIntrinsicDiv) { // SSE2 doesn't support div on non-floating point vectors. assert(!"Div not supported on integer type vectors\n"); return nullptr; } } #endif //_TARGET_AMD64_ && DEBUG // op1 is the first operand; if instance method, op1 is "this" arg // op2 is the second operand op2 = impSIMDPopStack(); op1 = impSIMDPopStack(instMethod); assert(op1->TypeGet() == TYP_STRUCT); assert(op2->TypeGet() == TYP_STRUCT); simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, simdIntrinsicID, baseType, size); retVal = simdTree; } break; case SIMDIntrinsicSelect: { // op3 is a SIMD variable that is the second source // op2 is a SIMD variable that is the first source // op1 is a SIMD variable which is the bit mask. op3 = impSIMDPopStack(); op2 = impSIMDPopStack(); op1 = impSIMDPopStack(); retVal = impSIMDSelect(clsHnd, baseType, size, op1, op2, op3); } break; case SIMDIntrinsicMin: case SIMDIntrinsicMax: { // op1 is the first operand; if instance method, op1 is "this" arg // op2 is the second operand op2 = impSIMDPopStack(); op1 = impSIMDPopStack(instMethod); retVal = impSIMDMinMax(simdIntrinsicID, clsHnd, baseType, size, op1, op2); } break; case SIMDIntrinsicGetItem: { // op1 is a SIMD variable that is "this" arg // op2 is an index of TYP_INT op2 = impSIMDPopStack(); op1 = impSIMDPopStack(instMethod); unsigned int vectorLength = getSIMDVectorLength(size, baseType); if (!op2->IsCnsIntOrI() || op2->AsIntCon()->gtIconVal >= vectorLength) { // We need to bounds-check the length of the vector. // For that purpose, we need to clone the index expression. GenTree* index = op2; if ((index->gtFlags & GTF_SIDE_EFFECT) != 0) { op2 = fgInsertCommaFormTemp(&index); } else { op2 = gtCloneExpr(index); } GenTree* lengthNode = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, vectorLength); GenTreeBoundsChk* simdChk = new (this, GT_SIMD_CHK) GenTreeBoundsChk(GT_SIMD_CHK, TYP_VOID, lengthNode, index); // Create a GT_COMMA tree for the bounds check. op2 = gtNewOperNode(GT_COMMA, op2->TypeGet(), simdChk, op2); } assert(op1->TypeGet() == TYP_STRUCT); assert(op2->TypeGet() == TYP_INT); simdTree = gtNewSIMDNode(genActualType(callType), op1, op2, simdIntrinsicID, baseType, size); retVal = simdTree; } break; case SIMDIntrinsicDotProduct: { #if defined(_TARGET_AMD64_) && defined(DEBUG) // Right now dot product is supported only for float vectors. // See SIMDIntrinsicList.h for supported base types for this intrinsic. if (!varTypeIsFloating(baseType)) { assert(!"Dot product on integer type vectors not supported"); return nullptr; } #endif //_TARGET_AMD64_ && DEBUG // op1 is a SIMD variable that is the first source and also "this" arg. // op2 is a SIMD variable which is the second source. op2 = impSIMDPopStack(); op1 = impSIMDPopStack(instMethod); assert(op1->TypeGet() == TYP_STRUCT); assert(op2->TypeGet() == TYP_STRUCT); simdTree = gtNewSIMDNode(baseType, op1, op2, simdIntrinsicID, baseType, size); retVal = simdTree; } break; case SIMDIntrinsicSqrt: { #if defined(_TARGET_AMD64_) && defined(DEBUG) // SSE/AVX doesn't support sqrt on integer type vectors and hence // should never be seen as an intrinsic here. See SIMDIntrinsicList.h // for supported base types for this intrinsic. if (!varTypeIsFloating(baseType)) { assert(!"Sqrt not supported on integer vectors\n"); return nullptr; } #endif // _TARGET_AMD64_ && DEBUG op1 = impSIMDPopStack(); assert(op1->TypeGet() == TYP_STRUCT); retVal = gtNewSIMDNode(genActualType(callType), op1, nullptr, simdIntrinsicID, baseType, size); } break; case SIMDIntrinsicAbs: { op1 = impSIMDPopStack(); assert(op1->TypeGet() == TYP_STRUCT); #ifdef _TARGET_AMD64_ if (varTypeIsFloating(baseType)) { // Abs(vf) = vf & new SIMDVector(0x7fffffff); // Abs(vd) = vf & new SIMDVector(0x7fffffffffffffff); GenTree* bitMask = nullptr; if (baseType == TYP_FLOAT) { float f; static_assert_no_msg(sizeof(float) == sizeof(int)); *((int *)&f) = 0x7fffffff; bitMask = gtNewDconNode(f); } else if (baseType == TYP_DOUBLE) { double d; static_assert_no_msg(sizeof(double) == sizeof(__int64)); *((__int64*)&d) = 0x7fffffffffffffffLL; bitMask = gtNewDconNode(d); } assert(bitMask != nullptr); bitMask->gtType = baseType; GenTree* bitMaskVector = gtNewSIMDNode(TYP_STRUCT, bitMask, SIMDIntrinsicInit, baseType, size); retVal = gtNewSIMDNode(TYP_STRUCT, op1, bitMaskVector, SIMDIntrinsicBitwiseAnd, baseType, size); } else if (baseType == TYP_CHAR || baseType == TYP_UBYTE || baseType == TYP_UINT || baseType == TYP_ULONG) { // Abs is a no-op on unsigned integer type vectors retVal = op1; } else { // SSE/AVX doesn't support abs on signed integer vectors and hence // should never be seen as an intrinsic here. See SIMDIntrinsicList.h // for supported base types for this intrinsic. unreached(); } #else //!_TARGET_AMD64_ assert(!"Abs intrinsic on non-Amd64 target not implemented"); unreached(); #endif //!_TARGET_AMD64_ } break; case SIMDIntrinsicGetW: retVal = impSIMDGetFixed(baseType, size, 3); break; case SIMDIntrinsicGetZ: retVal = impSIMDGetFixed(baseType, size, 2); break; case SIMDIntrinsicGetY: retVal = impSIMDGetFixed(baseType, size, 1); break; case SIMDIntrinsicGetX: retVal = impSIMDGetFixed(baseType, size, 0); break; case SIMDIntrinsicSetW: case SIMDIntrinsicSetZ: case SIMDIntrinsicSetY: case SIMDIntrinsicSetX: { // op2 is the value to be set at indexTemp position // op1 is SIMD vector that is going to be modified, which is a byref // If op1 has a side-effect, then don't make it an intrinsic. // It would be in-efficient to read the entire vector into xmm reg, // modify it and write back entire xmm reg. // // TODO-CQ: revisit this later. op1 = impStackTop(1).val; if ((op1->gtFlags & GTF_SIDE_EFFECT) != 0) { return nullptr; } op2 = impSIMDPopStack(); op1 = impSIMDPopStack(instMethod); assert(op1->TypeGet() == TYP_STRUCT); assert(op2->TypeGet() == baseType); GenTree* src = gtCloneExpr(op1); assert(src != nullptr); simdTree = gtNewSIMDNode(TYP_STRUCT, src, op2, simdIntrinsicID, baseType, size); copyBlkDst = gtNewOperNode(GT_ADDR, TYP_BYREF, op1); doCopyBlk = true; } break; // Unary operators that take and return a Vector. case SIMDIntrinsicCast: { op1 = impSIMDPopStack(instMethod); assert(op1->TypeGet() == TYP_STRUCT); simdTree = gtNewSIMDNode(genActualType(callType), op1, nullptr, simdIntrinsicID, baseType, size); retVal = simdTree; } break; case SIMDIntrinsicHWAccel: { GenTreeIntCon* intConstTree = new (this, GT_CNS_INT) GenTreeIntCon(TYP_INT, 1); retVal = intConstTree; } break; default: assert(!"Unimplemented SIMD Intrinsic"); return nullptr; } #ifdef _TARGET_AMD64_ // Amd64: also indicate that we use floating point registers. // The need for setting this here is that a method may not have SIMD // type lclvars, but might be exercising SIMD intrinsics on fields of // SIMD type. // // e.g. public Vector ComplexVecFloat::sqabs() { return this.r * this.r + this.i * this.i; } compFloatingPointUsed = true; #endif // At this point, we have a tree that we are going to store into a destination. // TODO-Cleanup: Once we've "plumbed" SIMD types all the way through the front-end, this should // be a simple store or assignment. if (doCopyBlk) { retVal = gtNewBlkOpNode(GT_COPYBLK, copyBlkDst, gtNewOperNode(GT_ADDR, TYP_BYREF, simdTree), gtNewIconNode(getSIMDTypeSizeInBytes(clsHnd)), false); retVal->gtFlags |= ((simdTree->gtFlags | copyBlkDst->gtFlags) & GTF_ALL_EFFECT); } return retVal; } #endif // FEATURE_SIMD #endif // !LEGACY_BACKEND