-
Notifications
You must be signed in to change notification settings - Fork 15.2k
[GlobalISel] Add support for value/constants as inline asm memory operand #161501
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Conversation
This stack of pull requests is managed by Graphite. Learn more about stacking. |
@llvm/pr-subscribers-backend-amdgpu @llvm/pr-subscribers-backend-aarch64 Author: Pierre van Houtryve (Pierre-vh) ChangesInlineAsmLowering rejected inline assembly with memory reference inputs if the values passed to the inline asm weren't pointers. The DAG lowering however handled them just fine. This patch updates InlineAsmLowering to match the DAG functionality in that regard:
Full diff: https://github.com/llvm/llvm-project/pull/161501.diff 7 Files Affected:
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 0b6033b4ba60a..2dab265614531 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -502,6 +502,19 @@ class LLVM_ABI MachineIRBuilder {
/// \return a MachineInstrBuilder for the newly created instruction.
MachineInstrBuilder buildConstantPool(const DstOp &Res, unsigned Idx);
+ /// Build and insert \p Res = G_CONSTANT_POOL \p Idx
+ ///
+ /// G_CONSTANT_POOL materializes the address of an object in the constant
+ /// pool.
+ ///
+ /// \pre setBasicBlock or setMI must have been called.
+ /// \pre \p Res must be a generic virtual register with pointer type.
+ ///
+ /// \param Idx LLVM Constant that will be inserted into the constant pool.
+ ///
+ /// \return a MachineInstrBuilder for the newly created instruction.
+ MachineInstrBuilder buildConstantPool(const DstOp &Res, const Constant *V);
+
/// Build and insert \p Res = G_PTR_ADD \p Op0, \p Op1
///
/// G_PTR_ADD adds \p Op1 addressible units to the pointer specified by \p Op0,
diff --git a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
index b4e64d7416d86..0bc5b38ea3250 100644
--- a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
@@ -13,6 +13,8 @@
#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
@@ -454,26 +456,53 @@ bool InlineAsmLowering::lowerInlineAsm(
}
if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
-
- if (!OpInfo.isIndirect) {
- LLVM_DEBUG(dbgs()
- << "Cannot indirectify memory input operands yet\n");
- return false;
- }
-
- assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!");
-
const InlineAsm::ConstraintCode ConstraintID =
TLI->getInlineAsmMemConstraint(OpInfo.ConstraintCode);
InlineAsm::Flag OpFlags(InlineAsm::Kind::Mem, 1);
OpFlags.setMemConstraint(ConstraintID);
Inst.addImm(OpFlags);
- ArrayRef<Register> SourceRegs =
- GetOrCreateVRegs(*OpInfo.CallOperandVal);
- assert(
- SourceRegs.size() == 1 &&
- "Expected the memory input to fit into a single virtual register");
- Inst.addReg(SourceRegs[0]);
+
+ if (OpInfo.isIndirect) {
+ // already indirect
+ ArrayRef<Register> SourceRegs =
+ GetOrCreateVRegs(*OpInfo.CallOperandVal);
+ assert(SourceRegs.size() == 1 && "Expected the memory input to fit "
+ "into a single virtual register");
+ Inst.addReg(SourceRegs[0]);
+ break;
+ }
+
+ // Needs to be made indirect.
+ Value *OpVal = OpInfo.CallOperandVal;
+ // Constants go into the constant pool.
+ if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
+ isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
+ unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
+ LLT AddrPtrTy =
+ LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace));
+ auto Addr =
+ MIRBuilder.buildConstantPool(AddrPtrTy, cast<Constant>(OpVal));
+ Inst.addReg(Addr.getReg(0));
+ } else {
+ // TODO: Is using the DL instead of OpInfo fine here?
+ unsigned Bytes = DL.getTypeStoreSize(OpVal->getType());
+ Align Alignment = DL.getPrefTypeAlign(OpVal->getType());
+ int FrameIdx =
+ MF.getFrameInfo().CreateStackObject(Bytes, Alignment, false);
+
+ unsigned AddrSpace = DL.getAllocaAddrSpace();
+ LLT FramePtrTy =
+ LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace));
+ auto Ptr = MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx).getReg(0);
+ ArrayRef<Register> SourceRegs =
+ GetOrCreateVRegs(*OpInfo.CallOperandVal);
+ assert(SourceRegs.size() == 1 && "Expected the memory input to fit "
+ "into a single virtual register");
+ MIRBuilder.buildStore(SourceRegs[0], Ptr,
+ MachinePointerInfo::getFixedStack(MF, FrameIdx),
+ Alignment);
+ Inst.addReg(Ptr);
+ }
break;
}
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 03dfa6f3f243f..3a52f3c04579c 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -3519,9 +3519,7 @@ static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal,
Align Alignment(DL.getABITypeAlign(ConstVal->getType()));
- auto Addr = MIRBuilder.buildConstantPool(
- AddrPtrTy,
- MF.getConstantPool()->getConstantPoolIndex(ConstVal, Alignment));
+ auto Addr = MIRBuilder.buildConstantPool(AddrPtrTy, ConstVal);
MachineMemOperand *MMO =
MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF),
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 27df7e369436a..0bff391b58363 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -9,6 +9,7 @@
/// This file implements the MachineIRBuidler class.
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -175,6 +176,13 @@ MachineInstrBuilder MachineIRBuilder::buildConstantPool(const DstOp &Res,
return MIB;
}
+MachineInstrBuilder MachineIRBuilder::buildConstantPool(const DstOp &Res,
+ const Constant *V) {
+ Align Alignment(getDataLayout().getABITypeAlign(V->getType()));
+ return buildConstantPool(
+ Res, getMF().getConstantPool()->getConstantPoolIndex(V, Alignment));
+}
+
MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy,
unsigned JTI) {
return buildInstr(TargetOpcode::G_JUMP_TABLE, {PtrTy}, {})
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
index 29c320da6c0a7..f8cd868a4c755 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
@@ -37,15 +37,6 @@ define i64 @strict_align_feature(ptr %p) #0 {
attributes #0 = { "target-features"="+strict-align" }
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to translate instruction: call
-; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for direct_mem
-; FALLBACK-WITH-REPORT-OUT-LABEL: direct_mem
-define void @direct_mem(i32 %x, i32 %y) {
-entry:
- tail call void asm sideeffect "", "imr,imr,~{memory}"(i32 %x, i32 %y)
- ret void
-}
-
; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to lower function{{.*}}scalable_arg
; FALLBACK-WITH-REPORT-OUT-LABEL: scalable_arg
define <vscale x 16 x i8> @scalable_arg(<vscale x 16 x i1> %pred, ptr %addr) #1 {
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll
index 42f6570047fc7..c5e0d1ac4b33a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-inline-asm.ll
@@ -258,3 +258,88 @@ define i64 @test_input_with_matching_constraint_to_physical_register() {
%1 = tail call i64 asm "", "={x2},0"(i64 0)
ret i64 %1
}
+
+define void @test_indirectify_i32_value(i32 %x, i32 %y) {
+ ; CHECK-LABEL: name: test_indirectify_i32_value
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $w0, $w1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+ ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+ ; CHECK-NEXT: G_STORE [[COPY]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %stack.0)
+ ; CHECK-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1
+ ; CHECK-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX1]](p0) :: (store (s32) into %stack.1)
+ ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, {{[0-9]+}} /* mem:m */, [[FRAME_INDEX]](p0), 262158 /* mem:m */, [[FRAME_INDEX1]](p0)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ tail call void asm sideeffect "", "imr,imr,~{memory}"(i32 %x, i32 %y)
+ ret void
+}
+
+define void @test_indirectify_i32_constant() {
+ ; CHECK-LABEL: name: test_indirectify_i32_constant
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: [[CONSTANT_POOL:%[0-9]+]]:_(p0) = G_CONSTANT_POOL %const.0
+ ; CHECK-NEXT: [[CONSTANT_POOL1:%[0-9]+]]:_(p0) = G_CONSTANT_POOL %const.1
+ ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, {{[0-9]+}} /* mem:m */, [[CONSTANT_POOL]](p0), 262158 /* mem:m */, [[CONSTANT_POOL1]](p0)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ tail call void asm sideeffect "", "imr,imr,~{memory}"(i32 42, i32 0)
+ ret void
+}
+
+define void @test_indirectify_i16_value(i16 %val) {
+ ; CHECK-LABEL: name: test_indirectify_i16_value
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $w0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+ ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+ ; CHECK-NEXT: G_STORE [[TRUNC]](s16), [[FRAME_INDEX]](p0) :: (store (s16) into %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, {{[0-9]+}} /* mem:m */, [[FRAME_INDEX]](p0)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ tail call void asm sideeffect "", "imr,~{memory}"(i16 %val)
+ ret void
+}
+
+define void @test_indirectify_i16_constant() {
+ ; CHECK-LABEL: name: test_indirectify_i16_constant
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: [[CONSTANT_POOL:%[0-9]+]]:_(p0) = G_CONSTANT_POOL %const.0
+ ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, {{[0-9]+}} /* mem:m */, [[CONSTANT_POOL]](p0)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ tail call void asm sideeffect "", "imr,~{memory}"(i16 42)
+ ret void
+}
+
+define void @test_indirectify_i64_value(i64 %val) {
+ ; CHECK-LABEL: name: test_indirectify_i64_value
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+ ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0
+ ; CHECK-NEXT: G_STORE [[COPY]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, {{[0-9]+}} /* mem:m */, [[FRAME_INDEX]](p0)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ tail call void asm sideeffect "", "imr,~{memory}"(i64 %val)
+ ret void
+}
+
+define void @test_indirectify_i64_constant() {
+ ; CHECK-LABEL: name: test_indirectify_i64_constant
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: [[CONSTANT_POOL:%[0-9]+]]:_(p0) = G_CONSTANT_POOL %const.0
+ ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, {{[0-9]+}} /* mem:m */, [[CONSTANT_POOL]](p0)
+ ; CHECK-NEXT: RET_ReallyLR
+entry:
+ tail call void asm sideeffect "", "imr,~{memory}"(i64 42)
+ ret void
+}
+
+; TODO: add more types
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
index 2cde060529bec..80379ada0390b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-inline-asm.ll
@@ -331,6 +331,92 @@ define amdgpu_kernel void @asm_constraint_n_n() {
ret void
}
+define void @test_indirectify_i32_value(i32 %x, i32 %y) {
+ ; CHECK-LABEL: name: test_indirectify_i32_value
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.0
+ ; CHECK-NEXT: G_STORE [[COPY]](s32), [[FRAME_INDEX]](p5) :: (store (s32) into %stack.0, addrspace 5)
+ ; CHECK-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.1
+ ; CHECK-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX1]](p5) :: (store (s32) into %stack.1, addrspace 5)
+ ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, 262158 /* mem:m */, [[FRAME_INDEX]](p5), 262158 /* mem:m */, [[FRAME_INDEX1]](p5)
+ ; CHECK-NEXT: SI_RETURN
+entry:
+ tail call void asm sideeffect "", "imr,imr,~{memory}"(i32 %x, i32 %y)
+ ret void
+}
+
+define void @test_indirectify_i32_constant() {
+ ; CHECK-LABEL: name: test_indirectify_i32_constant
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: [[CONSTANT_POOL:%[0-9]+]]:_(p1) = G_CONSTANT_POOL %const.0
+ ; CHECK-NEXT: [[CONSTANT_POOL1:%[0-9]+]]:_(p1) = G_CONSTANT_POOL %const.1
+ ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, 262158 /* mem:m */, [[CONSTANT_POOL]](p1), 262158 /* mem:m */, [[CONSTANT_POOL1]](p1)
+ ; CHECK-NEXT: SI_RETURN
+entry:
+ tail call void asm sideeffect "", "imr,imr,~{memory}"(i32 42, i32 0)
+ ret void
+}
+
+
+define void @test_indirectify_i16_value(i16 %val) {
+ ; CHECK-LABEL: name: test_indirectify_i16_value
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+ ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.0
+ ; CHECK-NEXT: G_STORE [[TRUNC]](s16), [[FRAME_INDEX]](p5) :: (store (s16) into %stack.0, addrspace 5)
+ ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, 262158 /* mem:m */, [[FRAME_INDEX]](p5)
+ ; CHECK-NEXT: SI_RETURN
+entry:
+ tail call void asm sideeffect "", "imr,~{memory}"(i16 %val)
+ ret void
+}
+
+define void @test_indirectify_i16_constant() {
+ ; CHECK-LABEL: name: test_indirectify_i16_constant
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: [[CONSTANT_POOL:%[0-9]+]]:_(p1) = G_CONSTANT_POOL %const.0
+ ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, 262158 /* mem:m */, [[CONSTANT_POOL]](p1)
+ ; CHECK-NEXT: SI_RETURN
+entry:
+ tail call void asm sideeffect "", "imr,~{memory}"(i16 42)
+ ret void
+}
+
+define void @test_indirectify_i64_value(i64 %val) {
+ ; CHECK-LABEL: name: test_indirectify_i64_value
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: liveins: $vgpr0, $vgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+ ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.0
+ ; CHECK-NEXT: G_STORE [[MV]](s64), [[FRAME_INDEX]](p5) :: (store (s64) into %stack.0, addrspace 5)
+ ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, 262158 /* mem:m */, [[FRAME_INDEX]](p5)
+ ; CHECK-NEXT: SI_RETURN
+entry:
+ tail call void asm sideeffect "", "imr,~{memory}"(i64 %val)
+ ret void
+}
+
+define void @test_indirectify_i64_constant() {
+ ; CHECK-LABEL: name: test_indirectify_i64_constant
+ ; CHECK: bb.1.entry:
+ ; CHECK-NEXT: [[CONSTANT_POOL:%[0-9]+]]:_(p1) = G_CONSTANT_POOL %const.0
+ ; CHECK-NEXT: INLINEASM &"", 25 /* sideeffect mayload maystore attdialect */, 262158 /* mem:m */, [[CONSTANT_POOL]](p1)
+ ; CHECK-NEXT: SI_RETURN
+entry:
+ tail call void asm sideeffect "", "imr,~{memory}"(i64 42)
+ ret void
+}
+
!llvm.module.flags = !{!1}
!0 = !{i32 70}
!1 = !{i32 1, !"amdhsa_code_object_version", i32 500}
|
I'd prefer not to introduce G_CONSTANT_POOL just for this reason if we can avoid it, unless we canonicalize on G_CONSTANT_POOL for everything and avoid the ambiguity of when to use it or not. |
InlineAsmLowering rejected inline assembly with memory reference inputs if the values passed to the inline asm weren't pointers. The DAG lowering however handled them just fine.
This patch updates InlineAsmLowering to match the DAG functionality in that regard: