Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/workflows/pull_request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,10 @@ jobs:
name: Test
uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main
with:
linux_exclude_swift_versions: '[{"swift_version": "5.8"}]'
linux_os_versions: '["jammy", "focal"]'
enable_macos_checks: false
macos_xcode_versions: '["16.3"]'

soundness:
name: Soundness
uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main
Expand Down
172 changes: 101 additions & 71 deletions Sources/CSystem/include/io_uring.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,112 @@
#include <sys/uio.h>
#include <signal.h>

#define __SWIFT_IORING_SQE_FALLBACK_STRUCT { \
__u8 opcode; \
__u8 flags; \
__u16 ioprio; \
__s32 fd; \
union { \
__u64 off; \
__u64 addr2; \
struct { \
__u32 cmd_op; \
__u32 __pad1; \
}; \
}; \
union { \
__u64 addr; \
__u64 splice_off_in; \
struct { \
__u32 level; \
__u32 optname; \
}; \
}; \
__u32 len; \
union { \
__kernel_rwf_t rw_flags; \
__u32 fsync_flags; \
__u16 poll_events; \
__u32 poll32_events; \
__u32 sync_range_flags; \
__u32 msg_flags; \
__u32 timeout_flags; \
__u32 accept_flags; \
__u32 cancel_flags; \
__u32 open_flags; \
__u32 statx_flags; \
__u32 fadvise_advice; \
__u32 splice_flags; \
__u32 rename_flags; \
__u32 unlink_flags; \
__u32 hardlink_flags; \
__u32 xattr_flags; \
__u32 msg_ring_flags; \
__u32 uring_cmd_flags; \
__u32 waitid_flags; \
__u32 futex_flags; \
__u32 install_fd_flags; \
__u32 nop_flags; \
}; \
__u64 user_data; \
union { \
__u16 buf_index; \
__u16 buf_group; \
} __attribute__((packed)); \
__u16 personality; \
union { \
__s32 splice_fd_in; \
__u32 file_index; \
__u32 optlen; \
struct { \
__u16 addr_len; \
__u16 __pad3[1]; \
}; \
}; \
union { \
struct { \
__u64 addr3; \
__u64 __pad2[1]; \
}; \
__u64 optval; \
__u8 cmd[0]; \
}; \
}

#if __has_include(<linux/io_uring.h>)
#include <linux/io_uring.h>

#ifdef IORING_TIMEOUT_BOOTTIME
// Kernel version >= 5.15, io_uring_sqe has file_index
// and all current Swift operations are supported.
#define __SWIFT_IORING_SUPPORTED true
typedef struct io_uring_sqe swift_io_uring_sqe;
#else
// io_uring_sqe is missing properties that IORequest expects.
// This configuration is not supported for now.
//
// Define a fallback struct to avoid build errors, but IORing
// will throw ENOTSUP on initialization.
#define __SWIFT_IORING_SUPPORTED false
typedef struct __SWIFT_IORING_SQE_FALLBACK_STRUCT swift_io_uring_sqe;
#endif

// We can define more specific availability later

#ifdef IORING_FEAT_RW_CUR_POS
// Kernel version >= 5.6, io_uring_sqe has open_flags
#endif

#ifdef IORING_FEAT_NODROP
// Kernel version >= 5.5, io_uring_sqe has cancel_flags
#endif

#else
// Minimal fallback definitions when linux/io_uring.h is not available (e.g. static SDK)
#include <stdint.h>

#define __SWIFT_IORING_SUPPORTED false

#define IORING_OFF_SQ_RING 0ULL
#define IORING_OFF_CQ_RING 0x8000000ULL
#define IORING_OFF_SQES 0x10000000ULL
Expand Down Expand Up @@ -44,77 +144,7 @@ typedef int32_t __s32;
typedef int __kernel_rwf_t;
#endif

struct io_uring_sqe {
__u8 opcode;
__u8 flags;
__u16 ioprio;
__s32 fd;
union {
__u64 off;
__u64 addr2;
struct {
__u32 cmd_op;
__u32 __pad1;
};
};
union {
__u64 addr;
__u64 splice_off_in;
struct {
__u32 level;
__u32 optname;
};
};
__u32 len;
union {
__kernel_rwf_t rw_flags;
__u32 fsync_flags;
__u16 poll_events;
__u32 poll32_events;
__u32 sync_range_flags;
__u32 msg_flags;
__u32 timeout_flags;
__u32 accept_flags;
__u32 cancel_flags;
__u32 open_flags;
__u32 statx_flags;
__u32 fadvise_advice;
__u32 splice_flags;
__u32 rename_flags;
__u32 unlink_flags;
__u32 hardlink_flags;
__u32 xattr_flags;
__u32 msg_ring_flags;
__u32 uring_cmd_flags;
__u32 waitid_flags;
__u32 futex_flags;
__u32 install_fd_flags;
__u32 nop_flags;
};
__u64 user_data;
union {
__u16 buf_index;
__u16 buf_group;
} __attribute__((packed));
__u16 personality;
union {
__s32 splice_fd_in;
__u32 file_index;
__u32 optlen;
struct {
__u16 addr_len;
__u16 __pad3[1];
};
};
union {
struct {
__u64 addr3;
__u64 __pad2[1];
};
__u64 optval;
__u8 cmd[0];
};
};
typedef struct __SWIFT_IORING_SQE_FALLBACK_STRUCT swift_io_uring_sqe;

struct io_uring_cqe {
__u64 user_data;
Expand Down
24 changes: 16 additions & 8 deletions Sources/System/IORing/IORing.swift
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ import Musl
#endif
import Synchronization

private var ioringSupported: Bool {
__SWIFT_IORING_SUPPORTED != 0
}

//This was #defines in older headers, so we redeclare it to get a consistent import
internal enum RegistrationOps: UInt32 {
case registerBuffers = 0
Expand Down Expand Up @@ -72,7 +76,7 @@ extension UnsafeMutableRawBufferPointer {
@inline(__always) @inlinable
internal func _tryWriteRequest(
_ request: __owned RawIORequest, ring: inout SQRing,
submissionQueueEntries: UnsafeMutableBufferPointer<io_uring_sqe>
submissionQueueEntries: UnsafeMutableBufferPointer<swift_io_uring_sqe>
)
-> Bool
{
Expand Down Expand Up @@ -151,9 +155,9 @@ internal func _flushQueue(ring: borrowing SQRing) -> UInt32 {

@inlinable
internal func _getSubmissionEntry(
ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer<io_uring_sqe>
ring: inout SQRing, submissionQueueEntries: UnsafeMutableBufferPointer<swift_io_uring_sqe>
) -> UnsafeMutablePointer<
io_uring_sqe
swift_io_uring_sqe
>? {
let next = ring.userTail &+ 1 //this is expected to wrap

Expand Down Expand Up @@ -196,7 +200,7 @@ private func setUpRing(
throw err
}

if params.features & IORING_FEAT_NODROP == 0
if params.features & IORing.Features.nonDroppingCompletions.rawValue == 0
{
close(ringDescriptor)
throw Errno.invalidArgument
Expand Down Expand Up @@ -266,7 +270,7 @@ private func setUpRing(
// map the submission queue
let sqes = mmap(
/* addr: */ nil,
/* len: */ Int(params.sq_entries) * MemoryLayout<io_uring_sqe>.size,
/* len: */ Int(params.sq_entries) * MemoryLayout<swift_io_uring_sqe>.size,
/* prot: */ PROT_READ | PROT_WRITE,
/* flags: */ MAP_SHARED | MAP_POPULATE,
/* fd: */ ringDescriptor,
Expand Down Expand Up @@ -307,7 +311,7 @@ public struct IORing: ~Copyable {

@usableFromInline let completionRing: CQRing

@usableFromInline let submissionQueueEntries: UnsafeMutableBufferPointer<io_uring_sqe>
@usableFromInline let submissionQueueEntries: UnsafeMutableBufferPointer<swift_io_uring_sqe>

// kept around for unmap / cleanup. TODO: we can save a few words of memory by figuring out how to handle cleanup for non-IORING_FEAT_SINGLE_MMAP better
let ringSize: Int
Expand Down Expand Up @@ -366,6 +370,10 @@ public struct IORing: ~Copyable {

/// Initializes an IORing with enough space for `queueDepth` prepared requests and completed operations
public init(queueDepth: UInt32, flags: SetupFlags = []) throws(Errno) {
guard ioringSupported else {
throw Errno.notSupported
}

let (params, tmpRingDescriptor, tmpRingPtr, tmpRingSize, tmpSQPtr, tmpSQSize, tmpCQPtr, tmpCQSize, sqes) = try setUpRing(queueDepth: queueDepth, flags: flags)
// All throws need to be before initializing ivars here to avoid
// "error: conditional initialization or destruction of noncopyable types is not supported;
Expand Down Expand Up @@ -421,7 +429,7 @@ public struct IORing: ~Copyable {
)

let submissionQueueEntries = UnsafeMutableBufferPointer(
start: sqes.assumingMemoryBound(to: io_uring_sqe.self),
start: sqes.assumingMemoryBound(to: swift_io_uring_sqe.self),
count: Int(params.sq_entries)
)

Expand Down Expand Up @@ -868,7 +876,7 @@ public struct IORing: ~Copyable {
}
munmap(
UnsafeMutableRawPointer(submissionQueueEntries.baseAddress!),
submissionQueueEntries.count * MemoryLayout<io_uring_sqe>.size
submissionQueueEntries.count * MemoryLayout<swift_io_uring_sqe>.size
)
close(ringDescriptor)
}
Expand Down
10 changes: 6 additions & 4 deletions Sources/System/IORing/RawIORequest.swift
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@ import CSystem

@usableFromInline
internal struct RawIORequest: ~Copyable {
@usableFromInline var rawValue: io_uring_sqe
// swift_io_uring_sqe is a typedef of io_uring_sqe on platforms where
// IORing is supported (currently requires kernel version >= 5.15).
@usableFromInline var rawValue: swift_io_uring_sqe
@usableFromInline var path: FilePath? //buffer owner for the path pointer that the sqe may have

@inlinable public init() {
self.rawValue = io_uring_sqe()
self.rawValue = swift_io_uring_sqe()
}
}

Expand Down Expand Up @@ -176,8 +178,8 @@ extension RawIORequest {

@inlinable
static func withTimeoutRequest<R>(
linkedTo opEntry: UnsafeMutablePointer<io_uring_sqe>,
in timeoutEntry: UnsafeMutablePointer<io_uring_sqe>,
linkedTo opEntry: UnsafeMutablePointer<swift_io_uring_sqe>,
in timeoutEntry: UnsafeMutablePointer<swift_io_uring_sqe>,
duration: Duration,
flags: TimeOutFlags,
work: () throws -> R) rethrows -> R {
Expand Down