Improve Vector128.ExtractMostSignificantBits for arm64

@tannergooding

Per discussion with @tannergooding on Discord

Vector128.ExtractMostSignificantBits is quite an important API that is typically used together with comparisons and TrailingZeroCount/LeadingZeroCount to detect positions of an element in a vector - typically used in various IndexOf-like algorithms, etc. Example:

static void PrintPostion()
{
    Vector128<byte> src = Vector128.Create((byte)0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
    Vector128<byte> val = Vector128.Create((byte)42);

    // prints 3 as the index of 42 is "3" in src vector
    Console.WriteLine(FirstMatch(src, val)); 
}

static int FirstMatch(Vector128<byte> src, Vector128<byte> val)
{
    Vector128<byte> eq = Vector128.Equals(src, val);
    return BitOperations.TrailingZeroCount(eq.ExtractMostSignificantBits());
}

Codegen for FirstMatch on x64:

       vpcmpeqb  xmm0, xmm0, xmm1
       vpmovmskb eax, xmm0
       tzcnt     eax, eax

Codegen for FirstMatch on arm64:

            cmeq    v16.16b, v0.16b, v1.16b
            ldr     q17, [@RWD00]
            and     v16.16b, v16.16b, v17.16b
            ldr     q17, [@RWD16]
            ushl    v16.16b, v16.16b, v17.16b
            movi    v17.4s, #0
            ext     v17.16b, v16.16b, v17.16b, #8
            addv    b17, v17.8b
            umov    w0, v17.b[0]
            lsl     w0, w0, #8
            addv    b16, v16.8b
            umov    w1, v16.b[0]
            orr     w0, w0, w1
            rbit    w0, w0
            clz     w0, w0

Because arm64 doesn't have a direct equivalent of movmsk. However, this particular case can be optimized because we know that input of ExtractMostSignificantBits is a comparison's result with all elements being either zero or all-bits-set, in the best case we can perform this smart trick from arm blog by @danlark1

            cmeq    v16.16b, v0.16b, v1.16b
            shrn    v16.8b, v16.8h, #4
            umov    x0, v16.d[0]
            rbit    x0, x0
            clz     x0, x0
            asr     w0, w0, #2

its C# equivalent:

static int FirstMatch(Vector128<byte> src, Vector128<byte> val)
{
    Vector128<byte> eq = Vector128.Equals(vector, val);
    ulong matches = AdvSimd.ShiftRightLogicalNarrowingLower(src.AsUInt16(), 4).AsUInt64().ToScalar();
    return BitOperations.TrailingZeroCount(matches) >> 2;
}

Performance impact

We expect a nice improvement for small inputs like what http parsing typically sees where we have to find positions of symbols like :, \n etc in relatively small inputs. For large inputs in most of our algorithms we use fast compare == Vector128<>.Zero checks to ignore chunks without matches.

category:cq
theme:vector-codegen
skill-level:intermediate
cost:small
impact:small

Benchmark:

    private static readonly byte[] httpHeader = Encoding.UTF8.GetBytes( 
""" 
Host: 127.0.0.1:5001 
Connection: keep-alive 
Cache-Control: max-age=0 
sec-ch-ua-mobile: ?0 
sec-ch-ua-platform: "Windows" 
Upgrade-Insecure-Requests: 1 
Sec-Fetch-Site: none 
Sec-Fetch-Mode: navigate 
Sec-Fetch-User: ?1 
Sec-Fetch-Dest: document 
Accept-Encoding: gzip, deflate, br 
Accept-Language: en-US,en;q=0.9 
"""); 
 
    [Benchmark] 
    public int CountHeaders() 
    { 
        ReadOnlySpan<byte> span = httpHeader.AsSpan(); 
        int newline = 0; 
        int count = 0; 
        while (newline != -1 && span.Length > newline) 
        { 
            span = span.Slice(newline + 1); 
            newline = span.IndexOfAny((byte)'\n', (byte)':'); // or just IndexOf((byte)'\n')
            count++; 
        } 
        return count; 
    }

Method	Job	Toolchain	Mean	Error	StdDev	Ratio
CountHeaders	Job-AWWEJK	/Core_Root/corerun	193.7 ns	2.58 ns	2.41 ns	0.80
CountHeaders	Job-DZXKTO	/Core_Root_base/corerun	242.2 ns	1.11 ns	1.04 ns	1.00

@JulieLeeMSFT

So the task here is to optimize TrailingZeroCount(comparison.ExtractMostSignificantBits()) in:

static int FirstMatch_old(Vector128<byte> src, Vector128<byte> val)
{
    Vector128<byte> eq = Vector128.Equals(src, val);
    return BitOperations.TrailingZeroCount(eq.ExtractMostSignificantBits());
}

to emit the same codegen as this function does:

static int FirstMatch_new(Vector128<byte> src, Vector128<byte> val)
{
    Vector128<byte> eq = Vector128.Equals(src, val);
    ulong matches = AdvSimd.ShiftRightLogicalNarrowingLower(src.AsUInt16(), 4).AsUInt64().ToScalar();
    return BitOperations.TrailingZeroCount(matches) >> 2;
}

Very first task here is to move ExtractMostSignificantBits expansion from importer to lower. Currently, ExtractMostSignificantBits on ARM64 is imported as the following IR:

STMT00000 ( 0x000[E-] ... ??? )
               [000006] -A---------                         *  ASG       simd16 (copy)
               [000005] D------N---                         +--*  LCL_VAR   simd16<System.Runtime.Intrinsics.Vector128`1> V03 tmp1         
               [000004] -----------                         \--*  HWINTRINSIC simd16 ubyte ShiftLogical
               [000003] -----------                            +--*  HWINTRINSIC simd16 ubyte And
               [000000] -----------                            |  +--*  LCL_VAR   simd16<System.Runtime.Intrinsics.Vector128`1> V00 arg0         
               [000001] -----------                            |  \--*  CNS_VEC   simd16<0x80808080, 0x80808080, 0x80808080, 0x80808080>
               [000002] -----------                            \--*  CNS_VEC   simd16<0xfcfbfaf9, 0x00fffefd, 0xfcfbfaf9, 0x00fffefd>

    [ 1]   6 (0x006) ret

STMT00001 ( ??? ... ??? )
               [000023] -----------                         *  RETURN    int   
               [000022] -----------                         \--*  OR        int   
               [000012] ---------U-                            +--*  CAST      int <- uint
               [000011] -----------                            |  \--*  HWINTRINSIC ubyte  ubyte ToScalar
               [000010] -----------                            |     \--*  HWINTRINSIC simd8  ubyte AddAcross
               [000009] -----------                            |        \--*  HWINTRINSIC simd8  ubyte GetLower
               [000008] -----------                            |           \--*  LCL_VAR   simd16<System.Runtime.Intrinsics.Vector128`1> V03 tmp1         
               [000021] -----------                            \--*  LSH       int   
               [000019] ---------U-                               +--*  CAST      int <- uint
               [000018] -----------                               |  \--*  HWINTRINSIC ubyte  ubyte ToScalar
               [000017] -----------                               |     \--*  HWINTRINSIC simd8  ubyte AddAcross
               [000016] -----------                               |        \--*  HWINTRINSIC simd8  ubyte GetLower
               [000015] -----------                               |           \--*  HWINTRINSIC simd16 ubyte ExtractVector128
               [000007] -----------                               |              +--*  LCL_VAR   simd16<System.Runtime.Intrinsics.Vector128`1> V03 tmp1         
               [000013] -----------                               |              +--*  CNS_VEC   simd16<0x00000000, 0x00000000, 0x00000000, 0x00000000>
               [000014] -----------                               |              \--*  CNS_INT   int    8
               [000020] -----------                               \--*  CNS_INT   int    8

Instead, it should be just * HWINTRINSIC simd16 int ExtractMostSignificantBits as is and is expanded to ^ in Lower. It will allow us to perform optimizations in VN on top of a single node instead of this huge tree.
Here is where it's expanded currently:

runtime/src/coreclr/jit/hwintrinsicarm64.cpp

Lines 943 to 1122 in 91df184

    
           case NI_Vector128_ExtractMostSignificantBits: 
        
           { 
        
               assert(sig->numArgs == 1); 
        
               // ARM64 doesn't have a single instruction that performs the behavior so we'll emulate it instead. 
        
               // To do this, we effectively perform the following steps: 
        
               // 1. tmp = input & 0x80         ; and the input to clear all but the most significant bit 
        
               // 2. tmp = tmp >> index         ; right shift each element by its index 
        
               // 3. tmp = sum(tmp)             ; sum the elements together 
        
               // For byte/sbyte, we also need to handle the fact that we can only shift by up to 8 
        
               // but for Vector128, we have 16 elements to handle. In that scenario, we will simply 
        
               // extract both scalars, and combine them via: (upper << 8) | lower 
        
               var_types simdType = getSIMDTypeForSize(simdSize); 
        
               op1 = impSIMDPopStack(simdType); 
        
               GenTreeVecCon* vecCon2 = gtNewVconNode(simdType); 
        
               GenTreeVecCon* vecCon3 = gtNewVconNode(simdType); 
        
               switch (simdBaseType) 
        
               { 
        
                   case TYP_BYTE: 
        
                   case TYP_UBYTE: 
        
                   { 
        
                       simdBaseType    = TYP_UBYTE; 
        
                       simdBaseJitType = CORINFO_TYPE_UBYTE; 
        
                       vecCon2->gtSimdVal.u64[0] = 0x8080808080808080; 
        
                       vecCon3->gtSimdVal.u64[0] = 0x00FFFEFDFCFBFAF9; 
        
                       if (simdSize == 16) 
        
                       { 
        
                           vecCon2->gtSimdVal.u64[1] = 0x8080808080808080; 
        
                           vecCon3->gtSimdVal.u64[1] = 0x00FFFEFDFCFBFAF9; 
        
                       } 
        
                       break; 
        
                   } 
        
                   case TYP_SHORT: 
        
                   case TYP_USHORT: 
        
                   { 
        
                       simdBaseType    = TYP_USHORT; 
        
                       simdBaseJitType = CORINFO_TYPE_USHORT; 
        
                       vecCon2->gtSimdVal.u64[0] = 0x8000800080008000; 
        
                       vecCon3->gtSimdVal.u64[0] = 0xFFF4FFF3FFF2FFF1; 
        
                       if (simdSize == 16) 
        
                       { 
        
                           vecCon2->gtSimdVal.u64[1] = 0x8000800080008000; 
        
                           vecCon3->gtSimdVal.u64[1] = 0xFFF8FFF7FFF6FFF5; 
        
                       } 
        
                       break; 
        
                   } 
        
                   case TYP_INT: 
        
                   case TYP_UINT: 
        
                   case TYP_FLOAT: 
        
                   { 
        
                       simdBaseType    = TYP_INT; 
        
                       simdBaseJitType = CORINFO_TYPE_INT; 
        
                       vecCon2->gtSimdVal.u64[0] = 0x8000000080000000; 
        
                       vecCon3->gtSimdVal.u64[0] = 0xFFFFFFE2FFFFFFE1; 
        
                       if (simdSize == 16) 
        
                       { 
        
                           vecCon2->gtSimdVal.u64[1] = 0x8000000080000000; 
        
                           vecCon3->gtSimdVal.u64[1] = 0xFFFFFFE4FFFFFFE3; 
        
                       } 
        
                       break; 
        
                   } 
        
                   case TYP_LONG: 
        
                   case TYP_ULONG: 
        
                   case TYP_DOUBLE: 
        
                   { 
        
                       simdBaseType    = TYP_LONG; 
        
                       simdBaseJitType = CORINFO_TYPE_LONG; 
        
                       vecCon2->gtSimdVal.u64[0] = 0x8000000000000000; 
        
                       vecCon3->gtSimdVal.u64[0] = 0xFFFFFFFFFFFFFFC1; 
        
                       if (simdSize == 16) 
        
                       { 
        
                           vecCon2->gtSimdVal.u64[1] = 0x8000000000000000; 
        
                           vecCon3->gtSimdVal.u64[1] = 0xFFFFFFFFFFFFFFC2; 
        
                       } 
        
                       break; 
        
                   } 
        
                   default: 
        
                   { 
        
                       unreached(); 
        
                   } 
        
               } 
        
               op3 = vecCon3; 
        
               op2 = vecCon2; 
        
               op1 = gtNewSimdHWIntrinsicNode(simdType, op1, op2, NI_AdvSimd_And, simdBaseJitType, simdSize, 
        
                                              /* isSimdAsHWIntrinsic */ false); 
        
               NamedIntrinsic shiftIntrinsic = NI_AdvSimd_ShiftLogical; 
        
               if ((simdSize == 8) && varTypeIsLong(simdBaseType)) 
        
               { 
        
                   shiftIntrinsic = NI_AdvSimd_ShiftLogicalScalar; 
        
               } 
        
               op1 = gtNewSimdHWIntrinsicNode(simdType, op1, op3, shiftIntrinsic, simdBaseJitType, simdSize, 
        
                                              /* isSimdAsHWIntrinsic */ false); 
        
               if (varTypeIsByte(simdBaseType) && (simdSize == 16)) 
        
               { 
        
                   CORINFO_CLASS_HANDLE simdClsHnd = gtGetStructHandleForSimdOrHW(TYP_SIMD16, simdBaseJitType); 
        
                   op1 = impCloneExpr(op1, &op2, simdClsHnd, CHECK_SPILL_ALL, 
        
                                      nullptr DEBUGARG("Clone op1 for vector extractmostsignificantbits")); 
        
                   op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_Vector128_GetLower, simdBaseJitType, simdSize, 
        
                                                  /* isSimdAsHWIntrinsic */ false); 
        
                   op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_AddAcross, simdBaseJitType, 8, 
        
                                                  /* isSimdAsHWIntrinsic */ false); 
        
                   op1 = gtNewSimdHWIntrinsicNode(simdBaseType, op1, NI_Vector64_ToScalar, simdBaseJitType, 8, 
        
                                                  /* isSimdAsHWIntrinsic */ false); 
        
                   op1 = gtNewCastNode(TYP_INT, op1, /* isUnsigned */ true, TYP_INT); 
        
                   GenTree* zero  = gtNewZeroConNode(TYP_SIMD16); 
        
                   ssize_t  index = 8 / genTypeSize(simdBaseType); 
        
                   op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, zero, gtNewIconNode(index), NI_AdvSimd_ExtractVector128, 
        
                                                  simdBaseJitType, simdSize, /* isSimdAsHWIntrinsic */ false); 
        
                   op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op2, NI_Vector128_GetLower, simdBaseJitType, simdSize, 
        
                                                  /* isSimdAsHWIntrinsic */ false); 
        
                   op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op2, NI_AdvSimd_Arm64_AddAcross, simdBaseJitType, 8, 
        
                                                  /* isSimdAsHWIntrinsic */ false); 
        
                   op2 = gtNewSimdHWIntrinsicNode(simdBaseType, op2, NI_Vector64_ToScalar, simdBaseJitType, 8, 
        
                                                  /* isSimdAsHWIntrinsic */ false); 
        
                   op2 = gtNewCastNode(TYP_INT, op2, /* isUnsigned */ true, TYP_INT); 
        
                   op2     = gtNewOperNode(GT_LSH, TYP_INT, op2, gtNewIconNode(8)); 
        
                   retNode = gtNewOperNode(GT_OR, TYP_INT, op1, op2); 
        
               } 
        
               else 
        
               { 
        
                   if (!varTypeIsLong(simdBaseType)) 
        
                   { 
        
                       if ((simdSize == 8) && ((simdBaseType == TYP_INT) || (simdBaseType == TYP_UINT))) 
        
                       { 
        
                           CORINFO_CLASS_HANDLE simdClsHnd = gtGetStructHandleForSimdOrHW(simdType, simdBaseJitType); 
        
                           op1 = impCloneExpr(op1, &op2, simdClsHnd, CHECK_SPILL_ALL, 
        
                                              nullptr DEBUGARG("Clone op1 for vector extractmostsignificantbits")); 
        
                           op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, op2, NI_AdvSimd_AddPairwise, simdBaseJitType, 
        
                                                          simdSize, /* isSimdAsHWIntrinsic */ false); 
        
                       } 
        
                       else 
        
                       { 
        
                           op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_AddAcross, simdBaseJitType, 
        
                                                          simdSize, /* isSimdAsHWIntrinsic */ false); 
        
                       } 
        
                   } 
        
                   else if (simdSize == 16) 
        
                   { 
        
                       op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_AddPairwiseScalar, simdBaseJitType, 
        
                                                      simdSize, /* isSimdAsHWIntrinsic */ false); 
        
                   } 
        
                   retNode = gtNewSimdHWIntrinsicNode(simdBaseType, op1, NI_Vector64_ToScalar, simdBaseJitType, 8, 
        
                                                      /* isSimdAsHWIntrinsic */ false); 
        
                   if ((simdBaseType != TYP_INT) && (simdBaseType != TYP_UINT)) 
        
                   { 
        
                       retNode = gtNewCastNode(TYP_INT, retNode, /* isUnsigned */ true, TYP_INT); 
        
                   } 
        
               } 
        
               break; 
        
           }

- it should be moved to Lower. Then the next step is simple to recognize a the final shape we want to opimize in VN.

cc @JulieLeeMSFT

Seems unlikely that this will make it into .NET 10, so am moving to .NET 11.

👍.

Notably we did make a few improvements already, but the remaining work (for .NET 11) is having morph/lowering take advantage of knowing we have ExtractMSB now (since we're no longer expanding it on import and instead carry it through).

Looking at this for .NET 11. One thing worth flagging before anyone picks it up: the specific rewrite proposed here (TrailingZeroCount(mask.ExtractMostSignificantBits()) → the shrn trick) is not semantics-preserving for the no-match case, so it can't be applied as a transparent JIT peephole.

The two forms agree only when there is a match. For an all-zero comparison result they differ:

// no-match input (no byte equals the search value)
old: TrailingZeroCount(eq.ExtractMostSignificantBits())                   // = TrailingZeroCount(0u) = 32
new: TrailingZeroCount(shrn(eq.AsUInt16(),4).AsUInt64().ToScalar()) >> 2  // = 64 >> 2 = 16

Verified empirically on current main (arm64):

no-match: old=32 new=16
match@3:  old=3  new=3

BitOperations.TrailingZeroCount(uint) is contractually 32 for input 0, so silently rewriting changes observable behavior. The hand-written FirstMatch_new works in BCL IndexOf-style code only because the caller guards/handles the no-match sentinel itself; the JIT can't assume that for arbitrary user code.

So a correct version needs one of:

A semantics-preserving sequence (keep the shrn lowering but add a small fixup so no-match still yields the original sentinel — e.g. for byte/Vector128, r + (r & 16) maps 16→32 while leaving match indices unchanged). This still drops the memory constant load the current movemask expansion needs.
A flow-sensitive rewrite that only fires where the mask is provably non-zero (e.g. inside an if (mask != 0) guard, as the BCL's own IndexOf is written).

I'm going to prototype option 1 (semantics-preserving, with benchmarking) since it keeps the win for the common matched case and removes the ldr of the per-lane shift constant.

Note

This comment was generated with the assistance of GitHub Copilot.

@AndyAyersMS

@AndyAyersMS see #129688 (comment)

@jonathandavies-arm

Update: I see this is already being handled by @jonathandavies-arm in #129688 (which I'd missed when I wrote the above — apologies for the noise). That PR covers the byte/short/int cases for both PopCount and TrailingZeroCount consumers and, notably, preserves the no-match semantics I was worried about: for TrailingZeroCount it builds a per-element "index+1 or sentinel 33" vector, does a horizontal min, then subtracts one, so the zero mask still yields 32. So my option-1 concern is already addressed there — no need for a separate change here.

Note

This comment was generated with the assistance of GitHub Copilot.

So is #129688 the direction we want to go here? My version was a bit different, but I can drop it if we're happy with that PR.

So is #129688 the direction we want to go here? My version was a bit different, but I can drop it if we're happy with that PR.

my point we already have a well optimized Vector*.*IndexOf* API that everyone should use. There are still valid scenarious we might optimize EMSB for. #129688 is doing that, but it needs a better way to detect that the input is all zeroes/ones (e.g. some code runs analysis with VN and leaves a flag on the intrinsic).

#129688 will fix this issue.

Navigation Menu

Improve Vector128.ExtractMostSignificantBits for arm64 #76047

Performance impact

1 remaining item

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

	case NI_Vector128_ExtractMostSignificantBits:
	{
	assert(sig->numArgs == 1);

	// ARM64 doesn't have a single instruction that performs the behavior so we'll emulate it instead.
	// To do this, we effectively perform the following steps:
	// 1. tmp = input & 0x80 ; and the input to clear all but the most significant bit
	// 2. tmp = tmp >> index ; right shift each element by its index
	// 3. tmp = sum(tmp) ; sum the elements together

	// For byte/sbyte, we also need to handle the fact that we can only shift by up to 8
	// but for Vector128, we have 16 elements to handle. In that scenario, we will simply
	// extract both scalars, and combine them via: (upper << 8) \| lower

	var_types simdType = getSIMDTypeForSize(simdSize);

	op1 = impSIMDPopStack(simdType);

	GenTreeVecCon* vecCon2 = gtNewVconNode(simdType);
	GenTreeVecCon* vecCon3 = gtNewVconNode(simdType);

	switch (simdBaseType)
	{
	case TYP_BYTE:
	case TYP_UBYTE:
	{
	simdBaseType = TYP_UBYTE;
	simdBaseJitType = CORINFO_TYPE_UBYTE;

	vecCon2->gtSimdVal.u64[0] = 0x8080808080808080;
	vecCon3->gtSimdVal.u64[0] = 0x00FFFEFDFCFBFAF9;

	if (simdSize == 16)
	{
	vecCon2->gtSimdVal.u64[1] = 0x8080808080808080;
	vecCon3->gtSimdVal.u64[1] = 0x00FFFEFDFCFBFAF9;
	}
	break;
	}

	case TYP_SHORT:
	case TYP_USHORT:
	{
	simdBaseType = TYP_USHORT;
	simdBaseJitType = CORINFO_TYPE_USHORT;

	vecCon2->gtSimdVal.u64[0] = 0x8000800080008000;
	vecCon3->gtSimdVal.u64[0] = 0xFFF4FFF3FFF2FFF1;

	if (simdSize == 16)
	{
	vecCon2->gtSimdVal.u64[1] = 0x8000800080008000;
	vecCon3->gtSimdVal.u64[1] = 0xFFF8FFF7FFF6FFF5;
	}
	break;
	}

	case TYP_INT:
	case TYP_UINT:
	case TYP_FLOAT:
	{
	simdBaseType = TYP_INT;
	simdBaseJitType = CORINFO_TYPE_INT;

	vecCon2->gtSimdVal.u64[0] = 0x8000000080000000;
	vecCon3->gtSimdVal.u64[0] = 0xFFFFFFE2FFFFFFE1;

	if (simdSize == 16)
	{
	vecCon2->gtSimdVal.u64[1] = 0x8000000080000000;
	vecCon3->gtSimdVal.u64[1] = 0xFFFFFFE4FFFFFFE3;
	}
	break;
	}

	case TYP_LONG:
	case TYP_ULONG:
	case TYP_DOUBLE:
	{
	simdBaseType = TYP_LONG;
	simdBaseJitType = CORINFO_TYPE_LONG;

	vecCon2->gtSimdVal.u64[0] = 0x8000000000000000;
	vecCon3->gtSimdVal.u64[0] = 0xFFFFFFFFFFFFFFC1;

	if (simdSize == 16)
	{
	vecCon2->gtSimdVal.u64[1] = 0x8000000000000000;
	vecCon3->gtSimdVal.u64[1] = 0xFFFFFFFFFFFFFFC2;
	}
	break;
	}

	default:
	{
	unreached();
	}
	}

	op3 = vecCon3;
	op2 = vecCon2;
	op1 = gtNewSimdHWIntrinsicNode(simdType, op1, op2, NI_AdvSimd_And, simdBaseJitType, simdSize,
	/* isSimdAsHWIntrinsic */ false);

	NamedIntrinsic shiftIntrinsic = NI_AdvSimd_ShiftLogical;

	if ((simdSize == 8) && varTypeIsLong(simdBaseType))
	{
	shiftIntrinsic = NI_AdvSimd_ShiftLogicalScalar;
	}

	op1 = gtNewSimdHWIntrinsicNode(simdType, op1, op3, shiftIntrinsic, simdBaseJitType, simdSize,
	/* isSimdAsHWIntrinsic */ false);

	if (varTypeIsByte(simdBaseType) && (simdSize == 16))
	{
	CORINFO_CLASS_HANDLE simdClsHnd = gtGetStructHandleForSimdOrHW(TYP_SIMD16, simdBaseJitType);

	op1 = impCloneExpr(op1, &op2, simdClsHnd, CHECK_SPILL_ALL,
	nullptr DEBUGARG("Clone op1 for vector extractmostsignificantbits"));

	op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_Vector128_GetLower, simdBaseJitType, simdSize,
	/* isSimdAsHWIntrinsic */ false);
	op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_AddAcross, simdBaseJitType, 8,
	/* isSimdAsHWIntrinsic */ false);
	op1 = gtNewSimdHWIntrinsicNode(simdBaseType, op1, NI_Vector64_ToScalar, simdBaseJitType, 8,
	/* isSimdAsHWIntrinsic */ false);
	op1 = gtNewCastNode(TYP_INT, op1, /* isUnsigned */ true, TYP_INT);

	GenTree* zero = gtNewZeroConNode(TYP_SIMD16);
	ssize_t index = 8 / genTypeSize(simdBaseType);

	op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, zero, gtNewIconNode(index), NI_AdvSimd_ExtractVector128,
	simdBaseJitType, simdSize, /* isSimdAsHWIntrinsic */ false);
	op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op2, NI_Vector128_GetLower, simdBaseJitType, simdSize,
	/* isSimdAsHWIntrinsic */ false);
	op2 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op2, NI_AdvSimd_Arm64_AddAcross, simdBaseJitType, 8,
	/* isSimdAsHWIntrinsic */ false);
	op2 = gtNewSimdHWIntrinsicNode(simdBaseType, op2, NI_Vector64_ToScalar, simdBaseJitType, 8,
	/* isSimdAsHWIntrinsic */ false);
	op2 = gtNewCastNode(TYP_INT, op2, /* isUnsigned */ true, TYP_INT);

	op2 = gtNewOperNode(GT_LSH, TYP_INT, op2, gtNewIconNode(8));
	retNode = gtNewOperNode(GT_OR, TYP_INT, op1, op2);
	}
	else
	{
	if (!varTypeIsLong(simdBaseType))
	{
	if ((simdSize == 8) && ((simdBaseType == TYP_INT) \|\| (simdBaseType == TYP_UINT)))
	{
	CORINFO_CLASS_HANDLE simdClsHnd = gtGetStructHandleForSimdOrHW(simdType, simdBaseJitType);

	op1 = impCloneExpr(op1, &op2, simdClsHnd, CHECK_SPILL_ALL,
	nullptr DEBUGARG("Clone op1 for vector extractmostsignificantbits"));
	op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, op2, NI_AdvSimd_AddPairwise, simdBaseJitType,
	simdSize, /* isSimdAsHWIntrinsic */ false);
	}
	else
	{
	op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_AddAcross, simdBaseJitType,
	simdSize, /* isSimdAsHWIntrinsic */ false);
	}
	}
	else if (simdSize == 16)
	{
	op1 = gtNewSimdHWIntrinsicNode(TYP_SIMD8, op1, NI_AdvSimd_Arm64_AddPairwiseScalar, simdBaseJitType,
	simdSize, /* isSimdAsHWIntrinsic */ false);
	}

	retNode = gtNewSimdHWIntrinsicNode(simdBaseType, op1, NI_Vector64_ToScalar, simdBaseJitType, 8,
	/* isSimdAsHWIntrinsic */ false);

	if ((simdBaseType != TYP_INT) && (simdBaseType != TYP_UINT))
	{
	retNode = gtNewCastNode(TYP_INT, retNode, /* isUnsigned */ true, TYP_INT);
	}
	}
	break;
	}

Improve Vector128.ExtractMostSignificantBits for arm64 #76047

Description

Performance impact

Activity

EgorBo commented on Sep 23, 2022

EgorBo commented on Mar 29, 2023

1 remaining item

AndyAyersMS commented on Aug 4, 2025

tannergooding commented on Aug 4, 2025

AndyAyersMS commented on Jun 24, 2026

EgorBo commented on Jun 24, 2026

AndyAyersMS commented on Jun 24, 2026

AndyAyersMS commented on Jun 24, 2026

EgorBo commented on Jun 24, 2026

JulieLeeMSFT commented on Jul 30, 2026

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions