Skip to content

[CoroutineAccessors] Sign de/allocation functions. #84831

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 14, 2025
Merged

[CoroutineAccessors] Sign de/allocation functions. #84831

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cd8144a
[NFC] IRGen: Move these functions.
nate-chandler Oct 9, 2025
d00b73c
[NFC] IRGen: Refactor and deduplicate this code.
nate-chandler Oct 9, 2025
5a46d84
[NFC] IRGen: Move these functions.
nate-chandler Oct 10, 2025
603dc2e
[CoroutineAccessors] Sign de/allocation functions.
nate-chandler Oct 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions include/swift/ABI/MetadataValues.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1793,6 +1793,10 @@ namespace SpecialPointerAuthDiscriminators {
/// IsCurrentGlobalActor function used between the Swift runtime and
/// concurrency runtime.
const uint16_t IsCurrentGlobalActorFunction = 0xd1b8; // = 53688

/// Function pointers stored in the coro allocator struct.
const uint16_t CoroAllocationFunction = 0x5f95; // = 24469
const uint16_t CoroDeallocationFunction = 0x9faf; // = 40879
}

/// The number of arguments that will be passed directly to a generic
Expand Down
6 changes: 6 additions & 0 deletions include/swift/AST/IRGenOptions.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,12 @@ struct PointerAuthOptions : clang::PointerAuthOptions {

/// Like PartialApplyCapture but for use with CoroFunctionPointer values.
PointerAuthSchema CoroPartialApplyCapture;

/// Stored in a coro allocator struct, the function used to allocate memory.
PointerAuthSchema CoroAllocationFunction;

/// Stored in a coro allocator struct, the function used to deallocate memory.
PointerAuthSchema CoroDeallocationFunction;
};

enum class JITDebugArtifact : unsigned {
Expand Down
215 changes: 1 addition & 214 deletions lib/IRGen/GenCall.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
#include "EntryPointArgumentEmission.h"
#include "Explosion.h"
#include "GenCall.h"
#include "GenCoro.h"
#include "GenFunc.h"
#include "GenHeap.h"
#include "GenKeyPath.h"
Expand Down Expand Up @@ -5131,141 +5132,6 @@ void irgen::emitYieldManyCoroutineEntry(
allocFn, deallocFn, {});
}

static llvm::Constant *getCoroAllocFn(IRGenModule &IGM) {
auto isSwiftCoroCCAvailable = IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
return IGM.getOrCreateHelperFunction(
"_swift_coro_alloc", IGM.Int8PtrTy, {IGM.CoroAllocatorPtrTy, IGM.SizeTy},
[isSwiftCoroCCAvailable](IRGenFunction &IGF) {
auto parameters = IGF.collectParameters();
auto *allocator = parameters.claimNext();
auto *size = parameters.claimNext();
if (isSwiftCoroCCAvailable) {
// swiftcorocc is available, so if there's no allocator pointer,
// allocate storage on the stack and return a pointer to it without
// popping the stack.
auto *nullAllocator = IGF.Builder.CreateCmp(
llvm::CmpInst::Predicate::ICMP_EQ, allocator,
llvm::ConstantPointerNull::get(
cast<llvm::PointerType>(allocator->getType())));
auto *poplessReturn = IGF.createBasicBlock("popless");
auto *normalReturn = IGF.createBasicBlock("normal");
IGF.Builder.CreateCondBr(nullAllocator, poplessReturn, normalReturn);
IGF.Builder.emitBlock(poplessReturn);
// Emit the dynamic alloca.
auto *alloca =
IGF.Builder.IRBuilderBase::CreateAlloca(IGF.IGM.Int8Ty, size);
alloca->setAlignment(llvm::Align(MaximumAlignment));
auto *retPopless = IGF.Builder.CreateIntrinsic(
IGF.IGM.VoidTy, llvm::Intrinsic::ret_popless, {});
retPopless->setTailCallKind(
llvm::CallInst::TailCallKind::TCK_MustTail);
IGF.Builder.CreateRet(alloca);
// Start emitting the "normal" block.
IGF.Builder.emitBlock(normalReturn);
}
auto *calleePtr = IGF.Builder.CreateInBoundsGEP(
IGF.IGM.CoroAllocatorTy, allocator,
{llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0),
llvm::ConstantInt::get(IGF.IGM.Int32Ty, 1)});
auto *callee = IGF.Builder.CreateLoad(
Address(calleePtr, IGF.IGM.PtrTy, IGF.IGM.getPointerAlignment()),
"allocate_fn");
auto fnPtr = FunctionPointer::createUnsigned(
FunctionPointer::Kind::Function, callee,
Signature(cast<llvm::FunctionType>(IGF.IGM.CoroAllocateFnTy), {},
IGF.IGM.SwiftCC));
auto *call = IGF.Builder.CreateCall(fnPtr, {size});
call->setDoesNotThrow();
call->setCallingConv(IGF.IGM.SwiftCC);
IGF.Builder.CreateRet(call);
},
/*setIsNoInline=*/true,
/*forPrologue=*/false,
/*isPerformanceConstraint=*/false,
/*optionalLinkageOverride=*/nullptr, IGM.SwiftCoroCC,
/*transformAttributes=*/
[&IGM](llvm::AttributeList &attrs) {
IGM.addSwiftCoroAttributes(attrs, 0);
});
}

static llvm::Constant *getCoroDeallocFn(IRGenModule &IGM) {
auto isSwiftCoroCCAvailable = IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
return IGM.getOrCreateHelperFunction(
"_swift_coro_dealloc", IGM.VoidTy,
{IGM.CoroAllocatorPtrTy, IGM.Int8PtrTy},
[isSwiftCoroCCAvailable](IRGenFunction &IGF) {
auto parameters = IGF.collectParameters();
auto *allocator = parameters.claimNext();
auto *ptr = parameters.claimNext();
if (isSwiftCoroCCAvailable) {
// swiftcorocc is available, so if there's no allocator pointer,
// storage was allocated on the stack which will be naturally cleaned
// up when the coroutine's frame is "freed".
auto *nullAllocator = IGF.Builder.CreateCmp(
llvm::CmpInst::Predicate::ICMP_EQ, allocator,
llvm::ConstantPointerNull::get(
cast<llvm::PointerType>(allocator->getType())));
auto *bailBlock = IGF.createBasicBlock("null_allocator");
auto *normalBlock = IGF.createBasicBlock("nonnull_allocator");
IGF.Builder.CreateCondBr(nullAllocator, bailBlock, normalBlock);
IGF.Builder.emitBlock(bailBlock);
// Nothing to do here.
IGF.Builder.CreateRetVoid();
// Start emitting the "normal" block.
IGF.Builder.emitBlock(normalBlock);
}
auto shouldDeallocateImmediatelyFlag = CoroAllocatorFlags(0);
shouldDeallocateImmediatelyFlag.setShouldDeallocateImmediately(true);
auto *flagsPtr = IGF.Builder.CreateInBoundsGEP(
IGF.IGM.CoroAllocatorTy, allocator,
{llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0),
llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0)});
auto *flags = IGF.Builder.CreateLoad(
Address(flagsPtr, IGF.IGM.Int32Ty, Alignment(4)), "");
auto *deallocDeferringAllocator = IGF.Builder.CreateAnd(
flags,
llvm::APInt(IGF.IGM.Int32Ty->getBitWidth(),
shouldDeallocateImmediatelyFlag.getOpaqueValue()));
auto *isDeallocDeferringAllocator = IGF.Builder.CreateICmpNE(
deallocDeferringAllocator,
llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0));
auto *deferringAllocatorBlock =
IGF.createBasicBlock("deferring_allocator");
auto *normalBlock = IGF.createBasicBlock("normal");
IGF.Builder.CreateCondBr(isDeallocDeferringAllocator,
deferringAllocatorBlock, normalBlock);
IGF.Builder.emitBlock(deferringAllocatorBlock);
// Nothing to do here.
IGF.Builder.CreateRetVoid();
// Start emitting the "normal" block.
IGF.Builder.emitBlock(normalBlock);
auto *calleePtr = IGF.Builder.CreateInBoundsGEP(
IGF.IGM.CoroAllocatorTy, allocator,
{llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0),
llvm::ConstantInt::get(IGF.IGM.Int32Ty, 2)});
auto *callee = IGF.Builder.CreateLoad(
Address(calleePtr, IGF.IGM.PtrTy, IGF.IGM.getPointerAlignment()),
"deallocate_fn");
auto fnPtr = FunctionPointer::createUnsigned(
FunctionPointer::Kind::Function, callee,
Signature(cast<llvm::FunctionType>(IGF.IGM.CoroDeallocateFnTy), {},
IGF.IGM.SwiftCC));
auto *call = IGF.Builder.CreateCall(fnPtr, {ptr});
call->setDoesNotThrow();
call->setCallingConv(IGF.IGM.SwiftCC);
IGF.Builder.CreateRetVoid();
},
/*setIsNoInline=*/true,
/*forPrologue=*/false,
/*isPerformanceConstraint=*/false,
/*optionalLinkageOverride=*/nullptr, IGM.SwiftCoroCC,
/*transformAttributes=*/
[&IGM](llvm::AttributeList &attrs) {
IGM.addSwiftCoroAttributes(attrs, 0);
});
}

void irgen::emitYieldOnce2CoroutineEntry(IRGenFunction &IGF,
CanSILFunctionType fnType,
llvm::Value *buffer,
Expand Down Expand Up @@ -5308,85 +5174,6 @@ Address irgen::emitAllocYieldManyCoroutineBuffer(IRGenFunction &IGF) {
getYieldManyCoroutineBufferAlignment(IGF.IGM));
}

static llvm::Constant *getAddrOfSwiftCCMalloc(IRGenModule &IGM) {
auto mallocFnPtr = IGM.getMallocFunctionPointer();
auto sig = mallocFnPtr.getSignature();
if (sig.getCallingConv() == IGM.SwiftCC) {
return IGM.getMallocFn();
}
return IGM.getOrCreateHelperFunction(
"_swift_malloc", sig.getType()->getReturnType(), sig.getType()->params(),
[](IRGenFunction &IGF) {
auto parameters = IGF.collectParameters();
auto *size = parameters.claimNext();
auto malloc = IGF.IGM.getMallocFunctionPointer();
auto *call = IGF.Builder.CreateCall(malloc, {size});
IGF.Builder.CreateRet(call);
});
}

static llvm::Constant *getAddrOfSwiftCCFree(IRGenModule &IGM) {
auto freeFnPtr = IGM.getFreeFunctionPointer();
auto sig = freeFnPtr.getSignature();
if (sig.getCallingConv() == IGM.SwiftCC) {
return IGM.getFreeFn();
}
return IGM.getOrCreateHelperFunction(
"_swift_free", sig.getType()->getReturnType(), sig.getType()->params(),
[](IRGenFunction &IGF) {
auto parameters = IGF.collectParameters();
auto *ptr = parameters.claimNext();
auto free = IGF.IGM.getFreeFunctionPointer();
IGF.Builder.CreateCall(free, {ptr});
IGF.Builder.CreateRetVoid();
});
}

static llvm::Constant *getAddrOfGlobalCoroAllocator(
IRGenModule &IGM, CoroAllocatorKind kind, bool shouldDeallocateImmediately,
llvm::Constant *allocFn, llvm::Constant *deallocFn) {
auto entity = LinkEntity::forCoroAllocator(kind);
auto taskAllocator = IGM.getOrCreateLazyGlobalVariable(
entity,
[&](ConstantInitBuilder &builder) -> ConstantInitFuture {
auto allocator = builder.beginStruct(IGM.CoroAllocatorTy);
auto flags = CoroAllocatorFlags(kind);
flags.setShouldDeallocateImmediately(shouldDeallocateImmediately);
allocator.addInt32(flags.getOpaqueValue());
allocator.add(allocFn);
allocator.add(deallocFn);
return allocator.finishAndCreateFuture();
},
[&](llvm::GlobalVariable *var) { var->setConstant(true); });
return taskAllocator;
}
llvm::Constant *IRGenModule::getAddrOfGlobalCoroMallocAllocator() {
return getAddrOfGlobalCoroAllocator(*this, CoroAllocatorKind::Malloc,
/*shouldDeallocateImmediately=*/true,
getAddrOfSwiftCCMalloc(*this),
getAddrOfSwiftCCFree(*this));
}
llvm::Constant *IRGenModule::getAddrOfGlobalCoroAsyncTaskAllocator() {
return getAddrOfGlobalCoroAllocator(*this, CoroAllocatorKind::Async,
/*shouldDeallocateImmediately=*/false,
getTaskAllocFn(), getTaskDeallocFn());
}
llvm::Value *
irgen::emitYieldOnce2CoroutineAllocator(IRGenFunction &IGF,
std::optional<CoroAllocatorKind> kind) {
if (!kind) {
return IGF.getCoroutineAllocator();
}
switch (*kind) {
case CoroAllocatorKind::Stack:
return llvm::ConstantPointerNull::get(IGF.IGM.CoroAllocatorPtrTy);
case CoroAllocatorKind::Async:
return IGF.IGM.getAddrOfGlobalCoroAsyncTaskAllocator();
case CoroAllocatorKind::Malloc:
return IGF.IGM.getAddrOfGlobalCoroMallocAllocator();
}
llvm_unreachable("unhandled case");
}
StackAddress irgen::emitAllocYieldOnce2CoroutineFrame(IRGenFunction &IGF,
llvm::Value *size) {
return emitAllocCoroStaticFrame(IGF, size);
Expand Down
3 changes: 0 additions & 3 deletions lib/IRGen/GenCall.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,9 +220,6 @@ namespace irgen {
CanSILFunctionType coroutineType,
NativeCCEntryPointArgumentEmission &emission);

llvm::Value *
emitYieldOnce2CoroutineAllocator(IRGenFunction &IGF,
std::optional<CoroAllocatorKind> kind);
StackAddress emitAllocYieldOnce2CoroutineFrame(IRGenFunction &IGF,
llvm::Value *size);
void emitDeallocYieldOnce2CoroutineFrame(IRGenFunction &IGF,
Expand Down
Loading