Skip to content

Commit 8ccec0a

Browse files
snickolls-armsnickolls-arm
authored
Implement SVE2 BitwiseSelect, BitwiseSelectLeftInverted, BitwiseSelectRightInverted (#115775)
* Implement SVE2 BitwiseSelect, BitwiseSelectLeftInverted, BitwiseSelectRightInverted * Add missing PlatformNotSupported lines * Fix the list order
1 parent f0230db commit 8ccec0a

File tree

8 files changed

+826
-436
lines changed

8 files changed

+826
-436
lines changed

‎src/coreclr/jit/hwintrinsiccodegenarm64.cpp‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2661,13 +2661,27 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
26612661
case NI_Sve2_BitwiseClearXor:
26622662
if (targetReg != op1Reg)
26632663
{
2664-
assert(targetReg != op2Reg);
2664+
assert(targetReg != op2Reg && targetReg != op3Reg);
26652665
GetEmitter()->emitInsSve_R_R(INS_sve_movprfx, EA_SCALABLE, targetReg, op1Reg);
26662666
}
26672667
// Always use the lane size D. It's a bitwise operation so this is fine for all integer vector types.
26682668
GetEmitter()->emitInsSve_R_R_R(ins, emitSize, targetReg, op2Reg, op3Reg, INS_OPTS_SCALABLE_D);
26692669
break;
26702670

2671+
case NI_Sve2_BitwiseSelect:
2672+
case NI_Sve2_BitwiseSelectLeftInverted:
2673+
case NI_Sve2_BitwiseSelectRightInverted:
2674+
// op1: select, op2: left, op3: right
2675+
// Operation is destructive on the 'left' operand.
2676+
if (targetReg != op2Reg)
2677+
{
2678+
assert(targetReg != op3Reg && targetReg != op1Reg);
2679+
GetEmitter()->emitInsSve_R_R(INS_sve_movprfx, EA_SCALABLE, targetReg, op2Reg);
2680+
}
2681+
// Always use the lane size D. It's a bitwise operation so this is fine for all integer vector types.
2682+
GetEmitter()->emitInsSve_R_R_R(ins, emitSize, targetReg, op3Reg, op1Reg, INS_OPTS_SCALABLE_D);
2683+
break;
2684+
26712685
default:
26722686
unreached();
26732687
}

‎src/coreclr/jit/hwintrinsiclistarm64sve.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,9 @@ HARDWARE_INTRINSIC(Sve, ZipLow,
310310
// SVE2 Intrinsics
311311
#define FIRST_NI_Sve2 NI_Sve2_BitwiseClearXor
312312
HARDWARE_INTRINSIC(Sve2, BitwiseClearXor, -1, 3, {INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
313+
HARDWARE_INTRINSIC(Sve2, BitwiseSelect, -1, 3, {INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
314+
HARDWARE_INTRINSIC(Sve2, BitwiseSelectLeftInverted, -1, 3, {INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
315+
HARDWARE_INTRINSIC(Sve2, BitwiseSelectRightInverted, -1, 3, {INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_sve_bsl2n, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
313316
HARDWARE_INTRINSIC(Sve2, ShiftLeftAndInsert, -1, 3, {INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_sve_sli, INS_invalid, INS_invalid}, HW_Category_ShiftLeftByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
314317
#define LAST_NI_Sve2 NI_Sve2_ShiftLeftAndInsert
315318

‎src/coreclr/jit/lsraarm64.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2290,6 +2290,9 @@ GenTree* LinearScan::getDelayFreeOperand(GenTreeHWIntrinsic* intrinsicTree, bool
22902290
break;
22912291

22922292
case NI_Sve_CreateBreakPropagateMask:
2293+
case NI_Sve2_BitwiseSelect:
2294+
case NI_Sve2_BitwiseSelectLeftInverted:
2295+
case NI_Sve2_BitwiseSelectRightInverted:
22932296
// RMW operates on the second op.
22942297
assert(isRMW);
22952298
delayFreeOp = intrinsicTree->Op(2);

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.PlatformNotSupported.cs‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,159 @@ internal Arm64() { }
7878
/// </summary>
7979
public static unsafe Vector<ulong> BitwiseClearXor(Vector<ulong> xor, Vector<ulong> value, Vector<ulong> mask) { throw new PlatformNotSupportedException(); }
8080

81+
82+
// Bitwise select
83+
84+
/// <summary>
85+
/// svuint8_t svbsl[_u8](svuint8_t op1, svuint8_t op2, svuint8_t op3)
86+
/// BSL Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
87+
/// </summary>
88+
public static unsafe Vector<byte> BitwiseSelect(Vector<byte> select, Vector<byte> left, Vector<byte> right) { throw new PlatformNotSupportedException(); }
89+
90+
/// <summary>
91+
/// svint16_t svbsl[_s16](svint16_t op1, svint16_t op2, svint16_t op3)
92+
/// BSL Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
93+
/// </summary>
94+
public static unsafe Vector<short> BitwiseSelect(Vector<short> select, Vector<short> left, Vector<short> right) { throw new PlatformNotSupportedException(); }
95+
96+
/// <summary>
97+
/// svint32_t svbsl[_s32](svint32_t op1, svint32_t op2, svint32_t op3)
98+
/// BSL Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
99+
/// </summary>
100+
public static unsafe Vector<int> BitwiseSelect(Vector<int> select, Vector<int> left, Vector<int> right) { throw new PlatformNotSupportedException(); }
101+
102+
/// <summary>
103+
/// svint64_t svbsl[_s64](svint64_t op1, svint64_t op2, svint64_t op3)
104+
/// BSL Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
105+
/// </summary>
106+
public static unsafe Vector<long> BitwiseSelect(Vector<long> select, Vector<long> left, Vector<long> right) { throw new PlatformNotSupportedException(); }
107+
108+
/// <summary>
109+
/// svint8_t svbsl[_s8](svint8_t op1, svint8_t op2, svint8_t op3)
110+
/// BSL Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
111+
/// </summary>
112+
public static unsafe Vector<sbyte> BitwiseSelect(Vector<sbyte> select, Vector<sbyte> left, Vector<sbyte> right) { throw new PlatformNotSupportedException(); }
113+
114+
/// <summary>
115+
/// svuint16_t svbsl[_u16](svuint16_t op1, svuint16_t op2, svuint16_t op3)
116+
/// BSL Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
117+
/// </summary>
118+
public static unsafe Vector<ushort> BitwiseSelect(Vector<ushort> select, Vector<ushort> left, Vector<ushort> right) { throw new PlatformNotSupportedException(); }
119+
120+
/// <summary>
121+
/// svuint32_t svbsl[_u32](svuint32_t op1, svuint32_t op2, svuint32_t op3)
122+
/// BSL Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
123+
/// </summary>
124+
public static unsafe Vector<uint> BitwiseSelect(Vector<uint> select, Vector<uint> left, Vector<uint> right) { throw new PlatformNotSupportedException(); }
125+
126+
/// <summary>
127+
/// svuint64_t svbsl[_u64](svuint64_t op1, svuint64_t op2, svuint64_t op3)
128+
/// BSL Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
129+
/// </summary>
130+
public static unsafe Vector<ulong> BitwiseSelect(Vector<ulong> select, Vector<ulong> left, Vector<ulong> right) { throw new PlatformNotSupportedException(); }
131+
132+
133+
// Bitwise select with first input inverted
134+
135+
/// <summary>
136+
/// svuint8_t svbsl1n[_u8](svuint8_t op1, svuint8_t op2, svuint8_t op3)
137+
/// BSL1N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
138+
/// </summary>
139+
public static unsafe Vector<byte> BitwiseSelectLeftInverted(Vector<byte> select, Vector<byte> left, Vector<byte> right) { throw new PlatformNotSupportedException(); }
140+
141+
/// <summary>
142+
/// svint16_t svbsl1n[_s16](svint16_t op1, svint16_t op2, svint16_t op3)
143+
/// BSL1N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
144+
/// </summary>
145+
public static unsafe Vector<short> BitwiseSelectLeftInverted(Vector<short> select, Vector<short> left, Vector<short> right) { throw new PlatformNotSupportedException(); }
146+
147+
/// <summary>
148+
/// svint32_t svbsl1n[_s32](svint32_t op1, svint32_t op2, svint32_t op3)
149+
/// BSL1N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
150+
/// </summary>
151+
public static unsafe Vector<int> BitwiseSelectLeftInverted(Vector<int> select, Vector<int> left, Vector<int> right) { throw new PlatformNotSupportedException(); }
152+
153+
/// <summary>
154+
/// svint64_t svbsl1n[_s64](svint64_t op1, svint64_t op2, svint64_t op3)
155+
/// BSL1N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
156+
/// </summary>
157+
public static unsafe Vector<long> BitwiseSelectLeftInverted(Vector<long> select, Vector<long> left, Vector<long> right) { throw new PlatformNotSupportedException(); }
158+
159+
/// <summary>
160+
/// svint8_t svbsl1n[_s8](svint8_t op1, svint8_t op2, svint8_t op3)
161+
/// BSL1N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
162+
/// </summary>
163+
public static unsafe Vector<sbyte> BitwiseSelectLeftInverted(Vector<sbyte> select, Vector<sbyte> left, Vector<sbyte> right) { throw new PlatformNotSupportedException(); }
164+
165+
/// <summary>
166+
/// svuint16_t svbsl1n[_u16](svuint16_t op1, svuint16_t op2, svuint16_t op3)
167+
/// BSL1N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
168+
/// </summary>
169+
public static unsafe Vector<ushort> BitwiseSelectLeftInverted(Vector<ushort> select, Vector<ushort> left, Vector<ushort> right) { throw new PlatformNotSupportedException(); }
170+
171+
/// <summary>
172+
/// svuint32_t svbsl1n[_u32](svuint32_t op1, svuint32_t op2, svuint32_t op3)
173+
/// BSL1N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
174+
/// </summary>
175+
public static unsafe Vector<uint> BitwiseSelectLeftInverted(Vector<uint> select, Vector<uint> left, Vector<uint> right) { throw new PlatformNotSupportedException(); }
176+
177+
/// <summary>
178+
/// svuint64_t svbsl1n[_u64](svuint64_t op1, svuint64_t op2, svuint64_t op3)
179+
/// BSL1N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
180+
/// </summary>
181+
public static unsafe Vector<ulong> BitwiseSelectLeftInverted(Vector<ulong> select, Vector<ulong> left, Vector<ulong> right) { throw new PlatformNotSupportedException(); }
182+
183+
184+
// Bitwise select with second input inverted
185+
186+
/// <summary>
187+
/// svuint8_t svbsl2n[_u8](svuint8_t op1, svuint8_t op2, svuint8_t op3)
188+
/// BSL2N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
189+
/// </summary>
190+
public static unsafe Vector<byte> BitwiseSelectRightInverted(Vector<byte> select, Vector<byte> left, Vector<byte> right) { throw new PlatformNotSupportedException(); }
191+
192+
/// <summary>
193+
/// svint16_t svbsl2n[_s16](svint16_t op1, svint16_t op2, svint16_t op3)
194+
/// BSL2N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
195+
/// </summary>
196+
public static unsafe Vector<short> BitwiseSelectRightInverted(Vector<short> select, Vector<short> left, Vector<short> right) { throw new PlatformNotSupportedException(); }
197+
198+
/// <summary>
199+
/// svint32_t svbsl2n[_s32](svint32_t op1, svint32_t op2, svint32_t op3)
200+
/// BSL2N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
201+
/// </summary>
202+
public static unsafe Vector<int> BitwiseSelectRightInverted(Vector<int> select, Vector<int> left, Vector<int> right) { throw new PlatformNotSupportedException(); }
203+
204+
/// <summary>
205+
/// svint64_t svbsl2n[_s64](svint64_t op1, svint64_t op2, svint64_t op3)
206+
/// BSL2N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
207+
/// </summary>
208+
public static unsafe Vector<long> BitwiseSelectRightInverted(Vector<long> select, Vector<long> left, Vector<long> right) { throw new PlatformNotSupportedException(); }
209+
210+
/// <summary>
211+
/// svint8_t svbsl2n[_s8](svint8_t op1, svint8_t op2, svint8_t op3)
212+
/// BSL2N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
213+
/// </summary>
214+
public static unsafe Vector<sbyte> BitwiseSelectRightInverted(Vector<sbyte> select, Vector<sbyte> left, Vector<sbyte> right) { throw new PlatformNotSupportedException(); }
215+
216+
/// <summary>
217+
/// svuint16_t svbsl2n[_u16](svuint16_t op1, svuint16_t op2, svuint16_t op3)
218+
/// BSL2N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
219+
/// </summary>
220+
public static unsafe Vector<ushort> BitwiseSelectRightInverted(Vector<ushort> select, Vector<ushort> left, Vector<ushort> right) { throw new PlatformNotSupportedException(); }
221+
222+
/// <summary>
223+
/// svuint32_t svbsl2n[_u32](svuint32_t op1, svuint32_t op2, svuint32_t op3)
224+
/// BSL2N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
225+
/// </summary>
226+
public static unsafe Vector<uint> BitwiseSelectRightInverted(Vector<uint> select, Vector<uint> left, Vector<uint> right) { throw new PlatformNotSupportedException(); }
227+
228+
/// <summary>
229+
/// svuint64_t svbsl2n[_u64](svuint64_t op1, svuint64_t op2, svuint64_t op3)
230+
/// BSL2N Ztied1.D, Ztied1.D, Zop2.D, Zop3.D
231+
/// </summary>
232+
public static unsafe Vector<ulong> BitwiseSelectRightInverted(Vector<ulong> select, Vector<ulong> left, Vector<ulong> right) { throw new PlatformNotSupportedException(); }
233+
81234
// Shift left and insert
82235

83236
/// <summary>

0 commit comments

Comments
 (0)