[AMDGPU][GlobalISel] Fold redundant uniform G_AMDGPU_READANYLANE in RegBankCombiner

Author: gandhi56

llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp

@@ -133,6 +133,9 @@ class AMDGPURegBankLegalizeCombiner {
   std::pair<MachineInstr *, Register> tryMatch(Register Src, unsigned Opcode);
   Register tryMatchUnmergeDefs(SmallVectorImpl<Register> &DefRegs);
   SmallVector<Register> tryMatchMergeReadAnyLane(GMergeLikeInstr *Merge);
+  bool tryMatchMergeReadAnyLaneOrUniform(GMergeLikeInstr *Merge,
+                                         SmallVectorImpl<Register> &Srcs,
+                                         SmallVectorImpl<bool> &NeedsBroadcast);
   SmallVector<Register> getReadAnyLaneSrcs(Register Src);
   void replaceRegWithOrBuildCopy(Register Dst, Register Src);
 
@@ -187,6 +190,29 @@ SmallVector<Register> AMDGPURegBankLegalizeCombiner::tryMatchMergeReadAnyLane(
   return ReadAnyLaneSrcs;
 }
 
+bool AMDGPURegBankLegalizeCombiner::tryMatchMergeReadAnyLaneOrUniform(
+    GMergeLikeInstr *Merge, SmallVectorImpl<Register> &Srcs,
+    SmallVectorImpl<bool> &NeedsBroadcast) {
+  bool SawReadAnyLane = false;
+  for (unsigned I = 0; I < Merge->getNumSources(); ++I) {
+    Register Op = Merge->getSourceReg(I);
+    Register RALSrc;
+    if (mi_match(Op, MRI, m_GAMDGPUReadAnyLane(m_Reg(RALSrc)))) {
+      Srcs.push_back(RALSrc);
+      NeedsBroadcast.push_back(false);
+      SawReadAnyLane = true;
+      continue;
+    }
+    if (MRI.getRegBankOrNull(Op) == SgprRB) { // uniform: broadcast is legal
+      Srcs.push_back(Op);
+      NeedsBroadcast.push_back(true);
+      continue;
+    }
+    return false;
+  }
+  return SawReadAnyLane;
+}
+
 SmallVector<Register>
 AMDGPURegBankLegalizeCombiner::getReadAnyLaneSrcs(Register Src) {
   // Src = G_AMDGPU_READANYLANE RALSrc
@@ -280,8 +306,26 @@ bool AMDGPURegBankLegalizeCombiner::tryEliminateReadAnyLane(
 
   B.setInstrAndDebugLoc(Copy);
   SmallVector<Register> ReadAnyLaneSrcRegs = getReadAnyLaneSrcs(RALDst);
-  if (ReadAnyLaneSrcRegs.empty())
+  if (ReadAnyLaneSrcRegs.empty()) {
+    if (SrcMI.getOpcode() != AMDGPU::G_BITCAST) {
+      if (auto *Merge = getOpcodeDef<GMergeLikeInstr>(RALDst, MRI)) {
+        SmallVector<Register> Srcs;
+        SmallVector<bool> NeedsBroadcast;
+        if (tryMatchMergeReadAnyLaneOrUniform(Merge, Srcs, NeedsBroadcast)) {
+          for (unsigned I = 0; I < Srcs.size(); ++I)
+            if (NeedsBroadcast[I])
+              Srcs[I] = B.buildCopy({VgprRB, MRI.getType(Srcs[I])}, Srcs[I])
+                            .getReg(0);
+          auto VgprMerge =
+              B.buildMergeLikeInstr({VgprRB, MRI.getType(RALDst)}, Srcs);
+          replaceRegWithOrBuildCopy(Dst, VgprMerge.getReg(0));
+          eraseInstr(Copy, MRI);
+          return true;
+        }
+      }
+    }
     return false;
+  }
 
   Register ReadAnyLaneSrc;
   if (ReadAnyLaneSrcRegs.size() == 1) {

llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll

@@ -3851,15 +3851,13 @@ define amdgpu_kernel void @s_mul_u64_zext_with_sregs(ptr addrspace(1) %out, ptr
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7-NEXT:    v_mov_b32_e32 v0, 0x50
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s3, s[2:3], 0x0
-; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mul_hi_u32 v0, s3, v0
-; GFX7-NEXT:    s_mul_i32 s4, s3, 0x50
+; GFX7-NEXT:    v_mul_hi_u32 v1, s2, v0
+; GFX7-NEXT:    s_mul_i32 s3, s2, 0x50
+; GFX7-NEXT:    v_mov_b32_e32 v0, s3
+; GFX7-NEXT:    s_mov_b32 s2, -1
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    v_readfirstlane_b32 s5, v0
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -3872,11 +3870,9 @@ define amdgpu_kernel void @s_mul_u64_zext_with_sregs(ptr addrspace(1) %out, ptr
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT:    v_mul_hi_u32 v1, s2, v0
 ; GFX8-NEXT:    s_mulk_i32 s2, 0x50
-; GFX8-NEXT:    v_readfirstlane_b32 s3, v0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;

llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll

@@ -501,3 +501,61 @@ body: |
     SI_RETURN_TO_EPILOG implicit $vgpr0_vgpr1
 ...
 
+---
+name: op_readanylane_and_uniform_merge_to_virtual_vgpr
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
+
+    ; CHECK-LABEL: name: op_readanylane_and_uniform_merge_to_virtual_vgpr
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:sgpr(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY3]](s32)
+    ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:vgpr(s32) = G_FCANONICALIZE [[COPY4]]
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[COPY5]](s32), [[FCANONICALIZE]](s32)
+    ; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<2 x s32>), [[MV]](p1) :: (store (<2 x s32>), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s32) = COPY $sgpr1
+    %2:sgpr(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:sgpr(s32) = COPY $sgpr2
+    %4:sgpr(s32) = COPY $sgpr3
+    %5:sgpr(s32) = G_FCANONICALIZE %4
+    %6:sgpr(<2 x s32>) = G_BUILD_VECTOR %3(s32), %5(s32)
+    G_STORE %6(<2 x s32>), %2(p1) :: (store (<2 x s32>), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: op_readanylane_and_uniform_merge_to_physical_vgpr
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+
+    ; CHECK-LABEL: name: op_readanylane_and_uniform_merge_to_physical_vgpr
+    ; CHECK: liveins: $sgpr0, $sgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: [[FCANONICALIZE:%[0-9]+]]:vgpr(s32) = G_FCANONICALIZE [[COPY2]]
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[COPY3]](s32), [[FCANONICALIZE]](s32)
+    ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_vgpr1
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s32) = COPY $sgpr1
+    %2:sgpr(s32) = G_FCANONICALIZE %1
+    %3:sgpr(<2 x s32>) = G_BUILD_VECTOR %0(s32), %2(s32)
+    $vgpr0_vgpr1 = COPY %3(<2 x s32>)
+    SI_RETURN_TO_EPILOG implicit $vgpr0_vgpr1
+...
+

llvm/test/CodeGen/AMDGPU/GlobalISel/readanylane-combines.mir

@@ -224,77 +224,40 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 inreg %node_ptr, f
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-SDAG-NEXT:    ; return to shader part epilog
 ;
-; GFX1013-GISEL-LABEL: image_bvh_intersect_ray_a16:
-; GFX1013-GISEL:       ; %bb.0: ; %main_body
-; GFX1013-GISEL-NEXT:    s_and_b32 s8, s8, 0xffff
-; GFX1013-GISEL-NEXT:    s_mov_b32 s16, s9
-; GFX1013-GISEL-NEXT:    v_alignbit_b32 v0, s8, s7, 16
-; GFX1013-GISEL-NEXT:    s_lshr_b32 s9, s5, 16
-; GFX1013-GISEL-NEXT:    s_and_b32 s5, s5, 0xffff
-; GFX1013-GISEL-NEXT:    s_lshl_b32 s8, s9, 16
-; GFX1013-GISEL-NEXT:    s_and_b32 s9, s7, 0xffff
-; GFX1013-GISEL-NEXT:    v_readfirstlane_b32 s7, v0
-; GFX1013-GISEL-NEXT:    s_and_b32 s6, s6, 0xffff
-; GFX1013-GISEL-NEXT:    s_lshl_b32 s9, s9, 16
-; GFX1013-GISEL-NEXT:    s_or_b32 s5, s5, s8
-; GFX1013-GISEL-NEXT:    s_or_b32 s6, s6, s9
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v5, s5
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v6, s6
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v7, s7
-; GFX1013-GISEL-NEXT:    s_mov_b32 s17, s10
-; GFX1013-GISEL-NEXT:    s_mov_b32 s18, s11
-; GFX1013-GISEL-NEXT:    s_mov_b32 s19, s12
-; GFX1013-GISEL-NEXT:    image_bvh_intersect_ray v[0:3], v[0:7], s[16:19] a16
-; GFX1013-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GFX1013-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX1013-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX1013-GISEL-NEXT:    v_readfirstlane_b32 s2, v2
-; GFX1013-GISEL-NEXT:    v_readfirstlane_b32 s3, v3
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX1013-GISEL-NEXT:    ; return to shader part epilog
-;
-; GFX1030-GISEL-LABEL: image_bvh_intersect_ray_a16:
-; GFX1030-GISEL:       ; %bb.0: ; %main_body
-; GFX1030-GISEL-NEXT:    s_mov_b32 s16, s9
-; GFX1030-GISEL-NEXT:    s_lshr_b32 s9, s5, 16
-; GFX1030-GISEL-NEXT:    s_and_b32 s5, s5, 0xffff
-; GFX1030-GISEL-NEXT:    s_lshl_b32 s9, s9, 16
-; GFX1030-GISEL-NEXT:    s_and_b32 s6, s6, 0xffff
-; GFX1030-GISEL-NEXT:    s_or_b32 s5, s5, s9
-; GFX1030-GISEL-NEXT:    s_and_b32 s9, s7, 0xffff
-; GFX1030-GISEL-NEXT:    s_and_b32 s8, s8, 0xffff
-; GFX1030-GISEL-NEXT:    s_lshl_b32 s9, s9, 16
-; GFX1030-GISEL-NEXT:    v_alignbit_b32 v7, s8, s7, 16
-; GFX1030-GISEL-NEXT:    s_or_b32 s6, s6, s9
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v5, s5
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v6, s6
-; GFX1030-GISEL-NEXT:    s_mov_b32 s17, s10
-; GFX1030-GISEL-NEXT:    s_mov_b32 s18, s11
-; GFX1030-GISEL-NEXT:    s_mov_b32 s19, s12
-; GFX1030-GISEL-NEXT:    image_bvh_intersect_ray v[0:3], v[0:7], s[16:19] a16
-; GFX1030-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GFX1030-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX1030-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX1030-GISEL-NEXT:    v_readfirstlane_b32 s2, v2
-; GFX1030-GISEL-NEXT:    v_readfirstlane_b32 s3, v3
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX1030-GISEL-NEXT:    ; return to shader part epilog
+; GFX10-GISEL-LABEL: image_bvh_intersect_ray_a16:
+; GFX10-GISEL:       ; %bb.0: ; %main_body
+; GFX10-GISEL-NEXT:    s_mov_b32 s16, s9
+; GFX10-GISEL-NEXT:    s_lshr_b32 s9, s5, 16
+; GFX10-GISEL-NEXT:    s_and_b32 s5, s5, 0xffff
+; GFX10-GISEL-NEXT:    s_lshl_b32 s9, s9, 16
+; GFX10-GISEL-NEXT:    s_and_b32 s6, s6, 0xffff
+; GFX10-GISEL-NEXT:    s_or_b32 s5, s5, s9
+; GFX10-GISEL-NEXT:    s_and_b32 s9, s7, 0xffff
+; GFX10-GISEL-NEXT:    s_and_b32 s8, s8, 0xffff
+; GFX10-GISEL-NEXT:    s_lshl_b32 s9, s9, 16
+; GFX10-GISEL-NEXT:    v_alignbit_b32 v7, s8, s7, 16
+; GFX10-GISEL-NEXT:    s_or_b32 s6, s6, s9
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v6, s6
+; GFX10-GISEL-NEXT:    s_mov_b32 s17, s10
+; GFX10-GISEL-NEXT:    s_mov_b32 s18, s11
+; GFX10-GISEL-NEXT:    s_mov_b32 s19, s12
+; GFX10-GISEL-NEXT:    image_bvh_intersect_ray v[0:3], v[0:7], s[16:19] a16
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-SDAG-LABEL: image_bvh_intersect_ray_a16:
 ; GFX11-SDAG:       ; %bb.0: ; %main_body
@@ -526,79 +489,41 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 inreg %node_ptr,
 ; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-SDAG-NEXT:    ; return to shader part epilog
 ;
-; GFX1013-GISEL-LABEL: image_bvh64_intersect_ray_a16:
-; GFX1013-GISEL:       ; %bb.0: ; %main_body
-; GFX1013-GISEL-NEXT:    s_and_b32 s9, s9, 0xffff
-; GFX1013-GISEL-NEXT:    s_mov_b32 s16, s10
-; GFX1013-GISEL-NEXT:    v_alignbit_b32 v0, s9, s8, 16
-; GFX1013-GISEL-NEXT:    s_lshr_b32 s10, s6, 16
-; GFX1013-GISEL-NEXT:    s_and_b32 s6, s6, 0xffff
-; GFX1013-GISEL-NEXT:    s_lshl_b32 s9, s10, 16
-; GFX1013-GISEL-NEXT:    s_and_b32 s10, s8, 0xffff
-; GFX1013-GISEL-NEXT:    v_readfirstlane_b32 s8, v0
-; GFX1013-GISEL-NEXT:    s_and_b32 s7, s7, 0xffff
-; GFX1013-GISEL-NEXT:    s_lshl_b32 s10, s10, 16
-; GFX1013-GISEL-NEXT:    s_or_b32 s6, s6, s9
-; GFX1013-GISEL-NEXT:    s_or_b32 s7, s7, s10
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v5, s5
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v6, s6
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v7, s7
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v8, s8
-; GFX1013-GISEL-NEXT:    s_mov_b32 s17, s11
-; GFX1013-GISEL-NEXT:    s_mov_b32 s18, s12
-; GFX1013-GISEL-NEXT:    s_mov_b32 s19, s13
-; GFX1013-GISEL-NEXT:    image_bvh64_intersect_ray v[0:3], v[0:8], s[16:19] a16
-; GFX1013-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GFX1013-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX1013-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX1013-GISEL-NEXT:    v_readfirstlane_b32 s2, v2
-; GFX1013-GISEL-NEXT:    v_readfirstlane_b32 s3, v3
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX1013-GISEL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX1013-GISEL-NEXT:    ; return to shader part epilog
-;
-; GFX1030-GISEL-LABEL: image_bvh64_intersect_ray_a16:
-; GFX1030-GISEL:       ; %bb.0: ; %main_body
-; GFX1030-GISEL-NEXT:    s_mov_b32 s16, s10
-; GFX1030-GISEL-NEXT:    s_lshr_b32 s10, s6, 16
-; GFX1030-GISEL-NEXT:    s_and_b32 s6, s6, 0xffff
-; GFX1030-GISEL-NEXT:    s_lshl_b32 s10, s10, 16
-; GFX1030-GISEL-NEXT:    s_and_b32 s7, s7, 0xffff
-; GFX1030-GISEL-NEXT:    s_or_b32 s6, s6, s10
-; GFX1030-GISEL-NEXT:    s_and_b32 s10, s8, 0xffff
-; GFX1030-GISEL-NEXT:    s_and_b32 s9, s9, 0xffff
-; GFX1030-GISEL-NEXT:    s_lshl_b32 s10, s10, 16
-; GFX1030-GISEL-NEXT:    v_alignbit_b32 v8, s9, s8, 16
-; GFX1030-GISEL-NEXT:    s_or_b32 s7, s7, s10
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v5, s5
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v6, s6
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v7, s7
-; GFX1030-GISEL-NEXT:    s_mov_b32 s17, s11
-; GFX1030-GISEL-NEXT:    s_mov_b32 s18, s12
-; GFX1030-GISEL-NEXT:    s_mov_b32 s19, s13
-; GFX1030-GISEL-NEXT:    image_bvh64_intersect_ray v[0:3], v[0:8], s[16:19] a16
-; GFX1030-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GFX1030-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX1030-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX1030-GISEL-NEXT:    v_readfirstlane_b32 s2, v2
-; GFX1030-GISEL-NEXT:    v_readfirstlane_b32 s3, v3
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v2, s2
-; GFX1030-GISEL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX1030-GISEL-NEXT:    ; return to shader part epilog
+; GFX10-GISEL-LABEL: image_bvh64_intersect_ray_a16:
+; GFX10-GISEL:       ; %bb.0: ; %main_body
+; GFX10-GISEL-NEXT:    s_mov_b32 s16, s10
+; GFX10-GISEL-NEXT:    s_lshr_b32 s10, s6, 16
+; GFX10-GISEL-NEXT:    s_and_b32 s6, s6, 0xffff
+; GFX10-GISEL-NEXT:    s_lshl_b32 s10, s10, 16
+; GFX10-GISEL-NEXT:    s_and_b32 s7, s7, 0xffff
+; GFX10-GISEL-NEXT:    s_or_b32 s6, s6, s10
+; GFX10-GISEL-NEXT:    s_and_b32 s10, s8, 0xffff
+; GFX10-GISEL-NEXT:    s_and_b32 s9, s9, 0xffff
+; GFX10-GISEL-NEXT:    s_lshl_b32 s10, s10, 16
+; GFX10-GISEL-NEXT:    v_alignbit_b32 v8, s9, s8, 16
+; GFX10-GISEL-NEXT:    s_or_b32 s7, s7, s10
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v6, s6
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v7, s7
+; GFX10-GISEL-NEXT:    s_mov_b32 s17, s11
+; GFX10-GISEL-NEXT:    s_mov_b32 s18, s12
+; GFX10-GISEL-NEXT:    s_mov_b32 s19, s13
+; GFX10-GISEL-NEXT:    image_bvh64_intersect_ray v[0:3], v[0:8], s[16:19] a16
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-GISEL-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-GISEL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-GISEL-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-SDAG-LABEL: image_bvh64_intersect_ray_a16:
 ; GFX11-SDAG:       ; %bb.0: ; %main_body