[AMDGPU][GlobalISel] Fold trivial uniform G_AMDGPU_READANYLANE

Eliminate redundant G_AMDGPU_READANYLANE (selected as v_readfirstlane_b32)
that only round-trips a uniform value through the VGPR/SGPR banks.

Two complementary changes:

- AMDGPURegBankCombiner: add a combine that replaces a
  G_AMDGPU_READANYLANE whose source is already uniform (an sgpr value
  merely copied into a vgpr) with the original sgpr value.

- AMDGPURegBankLegalize: extend tryMatchMergeReadAnyLane to handle
  merges with mixed sources. When at least one source is a readanylane,
  the merge is rebuilt entirely in the vgpr bank, using each
  readanylane's vgpr operand directly and copying the plain uniform
  sources into vgpr. This removes the sgpr -> vgpr -> sgpr -> vgpr
  round trip for patterns like build_vector(readanylane(x), uniform_y)
  feeding a vector ALU op.

Regenerate affected check lines in packed-fp32.ll and mul.ll and add a
MIR test for the new combine.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: gandhi56

llvm/lib/Target/AMDGPU/AMDGPUCombine.td

@@ -173,6 +173,14 @@ class minmax_to_minmax3_opcodes<Instruction minmaxOpcode> : GICombineRule<
          [{ return matchMinMaxToMinMax3(*${min_or_max}, ${matchinfo}); }]),
   (apply [{ applyMinMaxToMinMax3(*${min_or_max}, ${matchinfo}); }])>;
 
+// Fold a redundant G_AMDGPU_READANYLANE (v_readfirstlane_b32) whose source is
+// already uniform, i.e. an sgpr -> vgpr -> sgpr round trip.
+def redundant_readanylane : GICombineRule<
+  (defs root:$readanylane, register_matchinfo:$matchinfo),
+  (match (wip_match_opcode G_AMDGPU_READANYLANE):$readanylane,
+         [{ return matchRedundantReadAnyLane(*${readanylane}, ${matchinfo}); }]),
+  (apply [{ Helper.replaceSingleDefInstWithReg(*${readanylane}, ${matchinfo}); }])>;
+
 def smax_to_minmax3 : minmax_to_minmax3_opcodes<G_SMAX>;
 def smin_to_minmax3 : minmax_to_minmax3_opcodes<G_SMIN>;
 def umax_to_minmax3 : minmax_to_minmax3_opcodes<G_UMAX>;
@@ -270,5 +278,6 @@ def AMDGPURegBankCombiner : GICombiner<
    cast_of_cast_combines, sext_trunc, zext_of_shift_amount_combines,
    d16_load, smax_to_minmax3, smin_to_minmax3, umax_to_minmax3,
    umin_to_minmax3, fmax_to_minmax3, fmin_to_minmax3, fmaximum_to_minmax3,
-   fminimum_to_minmax3, fmax_ieee_to_minmax3, fmin_ieee_to_minmax3]> {
+   fminimum_to_minmax3, fmax_ieee_to_minmax3, fmin_ieee_to_minmax3,
+   redundant_readanylane]> {
 }

llvm/lib/Target/AMDGPU/AMDGPURegBankCombiner.cpp

@@ -23,6 +23,7 @@
 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
 #include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
+#include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/Target/TargetMachine.h"
@@ -103,6 +104,8 @@ class AMDGPURegBankCombinerImpl : public Combiner {
   void applyMinMaxToMinMax3(MachineInstr &MI,
                             MinMaxToMinMax3MatchInfo &MatchInfo) const;
 
+  bool matchRedundantReadAnyLane(MachineInstr &MI, Register &Match) const;
+
 private:
   SIModeRegisterDefaults getMode() const;
   bool getIEEE() const;
@@ -563,6 +566,36 @@ bool AMDGPURegBankCombinerImpl::matchMinMaxToMinMax3(
   return true;
 }
 
+// Reading any lane of a value that is uniform across all lanes returns that
+// same value. G_AMDGPU_READANYLANE (selected as v_readfirstlane_b32) is emitted
+// by RegBankLegalize to move a uniform value from a vgpr into an sgpr. When the
+// source is itself already uniform (an sgpr value that was merely copied into a
+// vgpr), the read is a redundant sgpr -> vgpr -> sgpr round trip and the result
+// can be replaced with the original sgpr value.
+bool AMDGPURegBankCombinerImpl::matchRedundantReadAnyLane(
+    MachineInstr &MI, Register &Match) const {
+  assert(MI.getOpcode() == AMDGPU::G_AMDGPU_READANYLANE);
+  Register Dst = MI.getOperand(0).getReg();
+  Register Src = MI.getOperand(1).getReg();
+
+  // Look through copies to the value actually being read.
+  Register SrcNoCopy = getSrcRegIgnoringCopies(Src, MRI);
+  if (!SrcNoCopy.isValid())
+    return false;
+
+  // Only fold when the read value is uniform (lives in the sgpr bank).
+  const RegisterBank *RB = MRI.getRegBankOrNull(SrcNoCopy);
+  if (!RB || RB->getID() != AMDGPU::SGPRRegBankID)
+    return false;
+
+  // Copies preserve the type, but be defensive so we never change it.
+  if (MRI.getType(SrcNoCopy) != MRI.getType(Dst))
+    return false;
+
+  Match = SrcNoCopy;
+  return true;
+}
+
 bool AMDGPURegBankCombinerImpl::applyD16Load(
     unsigned D16Opc, MachineInstr &DstMI, MachineInstr *SmallLoad,
     Register SrcReg32ToOverwriteD16) const {

llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp

@@ -172,19 +172,40 @@ Register AMDGPURegBankLegalizeCombiner::tryMatchUnmergeDefs(
   return UnMerge->getSourceReg();
 }
 
-// Check if all merge sources are readanylanes and return the readanylane
-// sources if they are.
+// Return the vgpr sources for a merge that can be rebuilt entirely in the vgpr
+// bank. A source is either the vgpr operand of a readanylane, or, for a plain
+// uniform (sgpr) source, a fresh vgpr copy of it. This is only worthwhile (and
+// only returns a non-empty result) when at least one source is a readanylane,
+// so rebuilding the merge in vgpr removes that vgpr -> sgpr read.
 SmallVector<Register> AMDGPURegBankLegalizeCombiner::tryMatchMergeReadAnyLane(
     GMergeLikeInstr *Merge) {
-  SmallVector<Register> ReadAnyLaneSrcs;
-  for (unsigned i = 0; i < Merge->getNumSources(); ++i) {
+  SmallVector<Register> Srcs;
+  SmallVector<bool> NeedsCopyToVgpr;
+  bool FoundReadAnyLane = false;
+  for (unsigned I = 0; I < Merge->getNumSources(); ++I) {
     Register Src;
-    if (!mi_match(Merge->getSourceReg(i), MRI,
-                  m_GAMDGPUReadAnyLane(m_Reg(Src))))
-      return {};
-    ReadAnyLaneSrcs.push_back(Src);
+    if (mi_match(Merge->getSourceReg(I), MRI,
+                 m_GAMDGPUReadAnyLane(m_Reg(Src)))) {
+      Srcs.push_back(Src);
+      NeedsCopyToVgpr.push_back(false);
+      FoundReadAnyLane = true;
+      continue;
+    }
+    Srcs.push_back(Merge->getSourceReg(I));
+    NeedsCopyToVgpr.push_back(true);
+  }
+
+  if (!FoundReadAnyLane)
+    return {};
+
+  // Materialize vgpr copies for the uniform (non-readanylane) sources so the
+  // whole merge can live in the vgpr bank.
+  for (unsigned I = 0; I < Srcs.size(); ++I) {
+    if (NeedsCopyToVgpr[I])
+      Srcs[I] = B.buildCopy({VgprRB, MRI.getType(Srcs[I])}, Srcs[I]).getReg(0);
   }
-  return ReadAnyLaneSrcs;
+
+  return Srcs;
 }
 
 SmallVector<Register>
@@ -402,8 +423,8 @@ void AMDGPURegBankLegalizeCombiner::tryCombineS1AnyExt(MachineInstr &MI) {
 // Search through MRI for virtual registers with sgpr register bank and S1 LLT.
 [[maybe_unused]] static Register getAnySgprS1(const MachineRegisterInfo &MRI) {
   const LLT S1 = LLT::scalar(1);
-  for (unsigned i = 0; i < MRI.getNumVirtRegs(); ++i) {
-    Register Reg = Register::index2VirtReg(i);
+  for (unsigned I = 0; I < MRI.getNumVirtRegs(); ++I) {
+    Register Reg = Register::index2VirtReg(I);
     if (MRI.def_empty(Reg) || MRI.getType(Reg) != S1)
       continue;
 

llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll

@@ -3752,15 +3752,13 @@ define amdgpu_kernel void @s_mul_u64_zext_with_sregs(ptr addrspace(1) %out, ptr
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7-NEXT:    v_mov_b32_e32 v0, 0x50
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s3, s[2:3], 0x0
-; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_load_dword s2, s[2:3], 0x0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mul_hi_u32 v0, s3, v0
-; GFX7-NEXT:    s_mul_i32 s4, s3, 0x50
+; GFX7-NEXT:    v_mul_hi_u32 v1, s2, v0
+; GFX7-NEXT:    s_mul_i32 s3, s2, 0x50
+; GFX7-NEXT:    v_mov_b32_e32 v0, s3
+; GFX7-NEXT:    s_mov_b32 s2, -1
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    v_readfirstlane_b32 s5, v0
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -3773,11 +3771,9 @@ define amdgpu_kernel void @s_mul_u64_zext_with_sregs(ptr addrspace(1) %out, ptr
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s1
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT:    v_mul_hi_u32 v1, s2, v0
 ; GFX8-NEXT:    s_mulk_i32 s2, 0x50
-; GFX8-NEXT:    v_readfirstlane_b32 s3, v0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;

llvm/test/CodeGen/AMDGPU/GlobalISel/regbankcombiner-redundant-readanylane.mir

@@ -0,0 +1,48 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass=amdgpu-regbank-combiner %s -o - | FileCheck %s
+
+# A G_AMDGPU_READANYLANE whose source is a uniform (sgpr) value copied into a
+# vgpr is a redundant sgpr -> vgpr -> sgpr round trip and should be folded away.
+---
+name: readanylane_of_copy_from_sgpr
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: readanylane_of_copy_from_sgpr
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK-NEXT: $sgpr0 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:vgpr(s32) = COPY %0
+    %2:sgpr(s32) = G_AMDGPU_READANYLANE %1
+    $sgpr0 = COPY %2(s32)
+    S_ENDPGM 0
+...
+
+# Negative test: the source is a genuine vgpr (divergent) value, so the read is
+# required and must not be folded.
+---
+name: readanylane_of_divergent
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: readanylane_of_divergent
+    ; CHECK: liveins: $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:vgpr(s32) = G_ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[READ:%[0-9]+]]:sgpr(s32) = G_AMDGPU_READANYLANE [[ADD]]
+    ; CHECK-NEXT: $sgpr0 = COPY [[READ]](s32)
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s32) = G_ADD %0, %1
+    %3:sgpr(s32) = G_AMDGPU_READANYLANE %2
+    $sgpr0 = COPY %3(s32)
+    S_ENDPGM 0
+...