Promotion from dev branch inclusive to ded07b8

qiaojbao · qiaojbao · commit cf4271cbb7c6 · 2025-03-13T11:56:10.000+08:00
diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -476,8 +476,8 @@ static void assignCalleeSavedSpillSlots(MachineFunction &F,
     // Now that we know which registers need to be saved and restored, allocate
     // stack slots for them.
     for (auto &CS : CSI) {
-      // If the target has spilled this register to another register, we don't
-      // need to allocate a stack slot.
+      // If the target has spilled this register to another register or already
+      // handled it , we don't need to allocate a stack slot.
       if (CS.isSpilledToReg() || CS.isHandledByTarget())
         continue;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -1246,15 +1246,15 @@ bool AMDGPUCallLowering::lowerTailCall(
       // On GFX12, we can only change the VGPR allocation for wave32.
       if (!ST.isWave32()) {
         F.getContext().diagnose(DiagnosticInfoUnsupported(
-            F, "Dynamic VGPR mode is only supported for wave32\n"));
+            F, "dynamic VGPR mode is only supported for wave32"));
         return false;
       }
 
       ArgInfo FallbackExecArg = Info.OrigArgs[ChainCallArgIdx::FallbackExec];
       assert(FallbackExecArg.Regs.size() == 1 &&
              "Expected single register for fallback EXEC");
       if (!FallbackExecArg.Ty->isIntegerTy(ST.getWavefrontSize())) {
-        LLVM_DEBUG(dbgs() << "Bad type for fallback EXEC");
+        LLVM_DEBUG(dbgs() << "Bad type for fallback EXEC\n");
         return false;
       }
     }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -5475,6 +5475,7 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(TC_RETURN)
   NODE_NAME_CASE(TC_RETURN_GFX)
   NODE_NAME_CASE(TC_RETURN_CHAIN)
+  NODE_NAME_CASE(TC_RETURN_CHAIN_DVGPR)
   NODE_NAME_CASE(TRAP)
   NODE_NAME_CASE(RET_GLUE)
   NODE_NAME_CASE(WAVE_ADDRESS)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -402,6 +402,7 @@ enum NodeType : unsigned {
   TC_RETURN,
   TC_RETURN_GFX,
   TC_RETURN_CHAIN,
+  TC_RETURN_CHAIN_DVGPR,
   TRAP,
 
   // Masked control flow nodes.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td
@@ -99,6 +99,12 @@ def AMDGPUtc_return_chain: SDNode<"AMDGPUISD::TC_RETURN_CHAIN",
   [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]
 >;
 
+// With dynamic VGPRs.
+def AMDGPUtc_return_chain_dvgpr: SDNode<"AMDGPUISD::TC_RETURN_CHAIN_DVGPR",
+  SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
+  [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]
+>;
+
 def AMDGPUtrap : SDNode<"AMDGPUISD::TRAP",
   SDTypeProfile<0, 1, [SDTCisVT<0, i16>]>,
     [SDNPHasChain, SDNPVariadic, SDNPSideEffect, SDNPOptInGlue]
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -182,28 +182,26 @@ const MCExpr *AMDGPUAsmPrinter::lowerConstant(const Constant *CV) {
   return AsmPrinter::lowerConstant(CV);
 }
 
-static void emitVGPRBlockComment(const MachineInstr *MI, MCStreamer &OS) {
+static void emitVGPRBlockComment(const MachineInstr *MI, const SIInstrInfo *TII,
+                                 const TargetRegisterInfo *TRI,
+                                 const SIMachineFunctionInfo *MFI,
+                                 MCStreamer &OS) {
   // The instruction will only transfer a subset of the registers in the block,
   // based on the mask that is stored in m0. We could search for the instruction
   // that sets m0, but most of the time we'll already have the mask stored in
   // the machine function info. Try to use that. This assumes that we only use
   // block loads/stores for CSR spills.
-  const MachineFunction *MF = MI->getParent()->getParent();
-  const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
-  const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
-  const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
-
   Register RegBlock =
       TII->getNamedOperand(*MI, MI->mayLoad() ? AMDGPU::OpName::vdst
                                               : AMDGPU::OpName::vdata)
           ->getReg();
-  Register FirstRegInBlock = TRI.getSubReg(RegBlock, AMDGPU::sub0);
+  Register FirstRegInBlock = TRI->getSubReg(RegBlock, AMDGPU::sub0);
   uint32_t Mask = MFI->getMaskForVGPRBlockOps(RegBlock);
 
   SmallString<512> TransferredRegs;
   for (unsigned I = 0; I < 32; ++I) {
     if (Mask & (1 << I)) {
-      (llvm::Twine(" ") + TRI.getName(FirstRegInBlock + I))
+      (llvm::Twine(" ") + TRI->getRegAsmName(FirstRegInBlock + I))
           .toVector(TransferredRegs);
     }
   }
@@ -300,9 +298,11 @@ void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) {
       return;
     }
 
-    if (STI.getInstrInfo()->isBlockLoadStore(MI->getOpcode()))
-      if (isVerbose())
-        emitVGPRBlockComment(MI, *OutStreamer);
+    if (isVerbose())
+      if (STI.getInstrInfo()->isBlockLoadStore(MI->getOpcode()))
+        emitVGPRBlockComment(MI, STI.getInstrInfo(), STI.getRegisterInfo(),
+                             MF->getInfo<SIMachineFunctionInfo>(),
+                             *OutStreamer);
 
     MCInst TmpInst;
     MCInstLowering.lower(MI, TmpInst);
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1894,7 +1894,6 @@ void SIFrameLowering::determineCalleeSavesSGPR(MachineFunction &MF,
 
 static void assignSlotsUsingVGPRBlocks(MachineFunction &MF,
                                        const GCNSubtarget &ST,
-                                       const TargetRegisterInfo *TRI,
                                        std::vector<CalleeSavedInfo> &CSI,
                                        unsigned &MinCSFrameIndex,
                                        unsigned &MaxCSFrameIndex) {
@@ -1921,8 +1920,8 @@ static void assignSlotsUsingVGPRBlocks(MachineFunction &MF,
     if (!CanUseBlockOps(*CSIt))
       continue;
 
-    // Find all the regs that will fit in a 32-bit block starting at the current
-    // reg and build the mask. It should have 1 for every register that's
+    // Find all the regs that will fit in a 32-bit mask starting at the current
+    // reg and build said mask. It should have 1 for every register that's
     // included, with the current register as the least significant bit.
     uint32_t Mask = 1;
     CSEnd = std::remove_if(
@@ -1935,8 +1934,7 @@ static void assignSlotsUsingVGPRBlocks(MachineFunction &MF,
           }
         });
 
-    const TargetRegisterClass *BlockRegClass =
-        TII->getRegClassForBlockOp(TRI, MF);
+    const TargetRegisterClass *BlockRegClass = &AMDGPU::VReg_1024RegClass;
     Register RegBlock =
         MRI->getMatchingSuperReg(Reg, AMDGPU::sub0, BlockRegClass);
     if (!RegBlock) {
@@ -1990,8 +1988,7 @@ bool SIFrameLowering::assignCalleeSavedSpillSlots(
   bool UseVGPRBlocks = ST.useVGPRBlockOpsForCSR();
 
   if (UseVGPRBlocks)
-    assignSlotsUsingVGPRBlocks(MF, ST, TRI, CSI, MinCSFrameIndex,
-                               MaxCSFrameIndex);
+    assignSlotsUsingVGPRBlocks(MF, ST, CSI, MinCSFrameIndex, MaxCSFrameIndex);
 
   return assignCalleeSavedSpillSlots(MF, TRI, CSI);
 }
@@ -2152,10 +2149,10 @@ bool SIFrameLowering::restoreCalleeSavedRegisters(
     // VGPRs in the register block is reserved (e.g. if it's a WWM register),
     // then the whole block will be marked as reserved and `updateLiveness` will
     // skip it.
-    if (!MBB.isLiveIn(Reg))
-      MBB.addLiveIn(Reg);
+    MBB.addLiveIn(Reg);
   }
 
+  MBB.sortUniqueLiveIns();
   return false;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3710,6 +3710,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
   SDValue Callee = CLI.Callee;
 
   llvm::SmallVector<SDValue, 6> ChainCallSpecialArgs;
+  bool UsesDynamicVGPRs = false;
   if (IsChainCallConv) {
     // The last arguments should be the value that we need to put in EXEC,
     // followed by the flags and any other arguments with special meanings.
@@ -3758,6 +3759,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
         return lowerUnhandledCall(CLI, InVals, "Expected 3 additional args");
       }
 
+      UsesDynamicVGPRs = true;
       std::for_each(CLI.Args.begin() + ChainCallArgIdx::NumVGPRs,
                     CLI.Args.end(), PushNodeOrTargetConstant);
     }
@@ -4091,7 +4093,8 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
       break;
     case CallingConv::AMDGPU_CS_Chain:
     case CallingConv::AMDGPU_CS_ChainPreserve:
-      OPC = AMDGPUISD::TC_RETURN_CHAIN;
+      OPC = UsesDynamicVGPRs ? AMDGPUISD::TC_RETURN_CHAIN_DVGPR
+                             : AMDGPUISD::TC_RETURN_CHAIN;
       break;
     }
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5769,16 +5769,6 @@ const TargetRegisterClass *SIInstrInfo::getRegClass(const MCInstrDesc &TID,
                                    IsAllocatable);
 }
 
-const TargetRegisterClass *
-SIInstrInfo::getRegClassForBlockOp(const TargetRegisterInfo *TRI,
-                                   const MachineFunction &MF) const {
-  const MCInstrDesc &ScratchStoreBlockOp =
-      get(AMDGPU::SCRATCH_STORE_BLOCK_SADDR);
-  int VDataIdx = AMDGPU::getNamedOperandIdx(ScratchStoreBlockOp.getOpcode(),
-                                            AMDGPU::OpName::vdata);
-  return getRegClass(ScratchStoreBlockOp, VDataIdx, TRI, MF);
-}
-
 const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
                                                       unsigned OpNo) const {
   const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1441,10 +1441,6 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
                                          const MachineFunction &MF)
     const override;
 
-  const TargetRegisterClass *
-  getRegClassForBlockOp(const TargetRegisterInfo *TRI,
-                        const MachineFunction &MF) const;
-
   void fixImplicitOperands(MachineInstr &MI) const;
 
   MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -704,16 +704,16 @@ multiclass SI_CS_CHAIN_TC<
       SchedRW = [WriteBranch],
       isConvergent = 1,
       WaveSizePredicate = wavesizepred in {
-  // This is essentially a tail call, but it also takes a mask to put in EXEC
-  // right before jumping to the callee.
-  def NAME: SPseudoInstSI <(outs),
-      (ins CCR_SGPR_64:$src0, unknown:$callee, i32imm:$fpdiff, execrc:$exec)>;
-
-  // Same as above, but it will first try to reallocate the VGPRs, and choose an
-  // EXEC mask and a callee depending on the success of the reallocation attempt.
-  def _DVGPR : SPseudoInstSI <(outs),
-      (ins CCR_SGPR_64:$src0, i64imm:$callee, i32imm:$fpdiff, execrc:$exec,
-           SSrc_b32:$numvgprs, execrc:$fbexec, CCR_SGPR_64:$fbcallee)>;
+    // This is essentially a tail call, but it also takes a mask to put in EXEC
+    // right before jumping to the callee.
+    def NAME: SPseudoInstSI <(outs),
+        (ins CCR_SGPR_64:$src0, unknown:$callee, i32imm:$fpdiff, execrc:$exec)>;
+
+    // Same as above, but it will first try to reallocate the VGPRs, and choose an
+    // EXEC mask and a callee depending on the success of the reallocation attempt.
+    def _DVGPR : SPseudoInstSI <(outs),
+        (ins CCR_SGPR_64:$src0, i64imm:$callee, i32imm:$fpdiff, execrc:$exec,
+             SSrc_b32:$numvgprs, execrc:$fbexec, CCR_SGPR_64:$fbcallee)>;
   } // End FixedSize = 0 etc
 }
 
@@ -747,16 +747,15 @@ defm : si_cs_chain_tc_patterns<i64>;
 multiclass si_cs_chain_tc_dvgpr_patterns<
   ValueType execvt, RegisterOperand execrc = getSOPSrcForVT<execvt>.ret,
   Instruction tc = SI_CS_CHAIN_TC_W32_DVGPR> {
-  let AddedComplexity = 90 in {
   foreach callee = [ (i64 0), (tglobaladdr) ] in {
   def : GCNPat<
-    (AMDGPUtc_return_chain i64:$src0, callee, (i32 timm:$fpdiff), execvt:$exec,
-                           i32:$numvgprs, execvt:$fbexec, i64:$fbcallee),
+    (AMDGPUtc_return_chain_dvgpr i64:$src0, callee, (i32 timm:$fpdiff),
+                                 execvt:$exec, i32:$numvgprs,
+                                 execvt:$fbexec, i64:$fbcallee),
     (tc CCR_SGPR_64:$src0, (i64 0), i32imm:$fpdiff, execrc:$exec,
         SSrc_b32:$numvgprs, execrc:$fbexec, CCR_SGPR_64:$fbcallee)
   >;
   }
-  } // AddedComplexity
 }
 
 defm : si_cs_chain_tc_dvgpr_patterns<i32>; // On GFX12, dVGPR mode is wave32-only.
diff --git a/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp b/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp
@@ -117,11 +117,12 @@ static void splitBlock(MachineBasicBlock &MBB, MachineInstr &MI,
   MDT->applyUpdates(DTUpdates);
 }
 
-static void addRegOrCopyOp(MachineInstrBuilder &MIB, MachineOperand &Op) {
+static void copyOpWithoutRegFlags(MachineInstrBuilder &MIB,
+                                  MachineOperand &Op) {
   if (Op.isReg())
     MIB.addReg(Op.getReg());
   else
-    MIB->addOperand(Op);
+    MIB.add(Op);
 }
 
 void SILateBranchLowering::expandChainCall(MachineInstr &MI,
@@ -138,30 +139,35 @@ void SILateBranchLowering::expandChainCall(MachineInstr &MI,
     // * Try to change the VGPR allocation
     // * Select the callee based on the result of the reallocation attempt
     // * Select the EXEC mask based on the result of the reallocation attempt
+    // If any of the register operands of the chain pseudo is used in more than
+    // one of these instructions, we need to make sure that the kill flags
+    // aren't copied along.
     auto AllocMI =
         BuildMI(*MI.getParent(), MI, DL, TII->get(AMDGPU::S_ALLOC_VGPR));
-    addRegOrCopyOp(AllocMI,
-                   *TII->getNamedOperand(MI, AMDGPU::OpName::numvgprs));
+    copyOpWithoutRegFlags(AllocMI,
+                          *TII->getNamedOperand(MI, AMDGPU::OpName::numvgprs));
 
     auto SelectCallee =
         BuildMI(*MI.getParent(), MI, DL, TII->get(AMDGPU::S_CSELECT_B64))
             .addDef(TII->getNamedOperand(MI, AMDGPU::OpName::src0)->getReg());
-    addRegOrCopyOp(SelectCallee,
-                   *TII->getNamedOperand(MI, AMDGPU::OpName::src0));
-    addRegOrCopyOp(SelectCallee,
-                   *TII->getNamedOperand(MI, AMDGPU::OpName::fbcallee));
+    copyOpWithoutRegFlags(SelectCallee,
+                          *TII->getNamedOperand(MI, AMDGPU::OpName::src0));
+    copyOpWithoutRegFlags(SelectCallee,
+                          *TII->getNamedOperand(MI, AMDGPU::OpName::fbcallee));
 
     auto SelectExec = BuildMI(*MI.getParent(), MI, DL,
                               TII->get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
                                                      : AMDGPU::S_CSELECT_B64))
                           .addDef(ExecReg);
 
-    addRegOrCopyOp(SelectExec, *TII->getNamedOperand(MI, AMDGPU::OpName::exec));
-    addRegOrCopyOp(SelectExec,
-                   *TII->getNamedOperand(MI, AMDGPU::OpName::fbexec));
+    copyOpWithoutRegFlags(SelectExec,
+                          *TII->getNamedOperand(MI, AMDGPU::OpName::exec));
+    copyOpWithoutRegFlags(SelectExec,
+                          *TII->getNamedOperand(MI, AMDGPU::OpName::fbexec));
   } else {
     auto SetExec = BuildMI(*MI.getParent(), MI, DL, TII->get(MovOpc), ExecReg);
-    addRegOrCopyOp(SetExec, *TII->getNamedOperand(MI, AMDGPU::OpName::exec));
+    copyOpWithoutRegFlags(SetExec,
+                          *TII->getNamedOperand(MI, AMDGPU::OpName::exec));
   }
 
   for (unsigned OpIdx = MI.getNumExplicitOperands() - 1; OpIdx >= ExecIdx;
@@ -215,12 +221,12 @@ bool SILateBranchLowering::runOnMachineFunction(MachineFunction &MF) {
 
       case AMDGPU::SI_CS_CHAIN_TC_W32:
       case AMDGPU::SI_CS_CHAIN_TC_W64:
-        expandChainCall(MI, ST, /*DynamicVGPR=*/ false);
+        expandChainCall(MI, ST, /*DynamicVGPR=*/false);
         MadeChange = true;
         break;
       case AMDGPU::SI_CS_CHAIN_TC_W32_DVGPR:
       case AMDGPU::SI_CS_CHAIN_TC_W64_DVGPR:
-        expandChainCall(MI, ST, /*DynamicVGPR=*/ true);
+        expandChainCall(MI, ST, /*DynamicVGPR=*/true);
         MadeChange = true;
         break;
 
diff --git a/llvm/test/CodeGen/AMDGPU/pei-vgpr-block-spill-csr.mir b/llvm/test/CodeGen/AMDGPU/pei-vgpr-block-spill-csr.mir
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vgpr-block.ll b/llvm/test/CodeGen/AMDGPU/spill-vgpr-block.ll

Original file line number	Diff line number	Diff line change
`@@ -1246,15 +1246,15 @@ bool AMDGPUCallLowering::lowerTailCall(`
`1246`	`1246`	`// On GFX12, we can only change the VGPR allocation for wave32.`
`1247`	`1247`	`if (!ST.isWave32()) {`
`1248`	`1248`	`F.getContext().diagnose(DiagnosticInfoUnsupported(`
`1249`		`- F, "Dynamic VGPR mode is only supported for wave32\n"));`
	`1249`	`+ F, "dynamic VGPR mode is only supported for wave32"));`
`1250`	`1250`	`return false;`
`1251`	`1251`	`}`
`1252`	`1252`
`1253`	`1253`	`ArgInfo FallbackExecArg = Info.OrigArgs[ChainCallArgIdx::FallbackExec];`
`1254`	`1254`	`assert(FallbackExecArg.Regs.size() == 1 &&`
`1255`	`1255`	`"Expected single register for fallback EXEC");`
`1256`	`1256`	`if (!FallbackExecArg.Ty->isIntegerTy(ST.getWavefrontSize())) {`
`1257`		`- LLVM_DEBUG(dbgs() << "Bad type for fallback EXEC");`
	`1257`	`+ LLVM_DEBUG(dbgs() << "Bad type for fallback EXEC\n");`
`1258`	`1258`	`return false;`
`1259`	`1259`	`}`
`1260`	`1260`	`}`