Skip to content

feat(linux/xdgportal): use accurate frame timestamps from PipeWire metadata #4780

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
psyke83 wants to merge 3 commits into
base: master
Choose a base branch
from
Open

feat(linux/xdgportal): use accurate frame timestamps from PipeWire metadata #4780

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ab631d6
feat(linux/xdgportal): implement event-driven capture
psyke83 Feb 24, 2026
40ed1ca
fix(linux/xdgportal): fix pipewire teardown logic, GNOME XDG stop icon
psyke83 Feb 23, 2026
e2781dc
feat(linux/xdgportal): use accurate frame timestamps from PipeWire me…
psyke83 Feb 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/platform/linux/graphics.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,8 @@ namespace egl {
// PipeWire metadata
std::optional<uint64_t> pts;
std::optional<uint64_t> seq;
std::optional<bool> pw_damage;
std::optional<uint32_t> pw_flags;
};

class sws_t {
Expand Down
205 changes: 163 additions & 42 deletions src/platform/linux/portalgrab.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -723,6 +723,24 @@ namespace portal {

~pipewire_t() {
cleanup_stream();

pw_thread_loop_lock(loop);

if (core) {
pw_core_disconnect(core);
core = nullptr;
}
if (context) {
pw_context_destroy(context);
context = nullptr;
}

pw_thread_loop_unlock(loop);

pw_thread_loop_stop(loop);
if (fd >= 0) {
close(fd);
}
pw_thread_loop_destroy(loop);
}

Expand All @@ -734,6 +752,18 @@ namespace portal {
return stream_data.frame_cv;
}

uint64_t stream_get_nsec() {
return pw_stream_get_nsec(stream_data.stream);
}

bool is_frame_ready() const {
return stream_data.frame_ready;
}

void set_frame_ready(bool ready) {
stream_data.frame_ready = ready;
}

void init(int stream_fd, int stream_node, std::shared_ptr<shared_state_t> shared_state) {
fd = stream_fd;
node = stream_node;
Expand Down Expand Up @@ -767,21 +797,8 @@ namespace portal {
pw_stream_destroy(stream_data.stream);
stream_data.stream = nullptr;
}
if (core) {
pw_core_disconnect(core);
core = nullptr;
}
if (context) {
pw_context_destroy(context);
context = nullptr;
}

pw_thread_loop_unlock(loop);

pw_thread_loop_stop(loop);
if (fd >= 0) {
close(fd);
}
}
session_cache_t::instance().invalidate();
}
Expand Down Expand Up @@ -843,14 +860,13 @@ namespace portal {
}

// 2. Validate we have a buffer and a signal that it's "new"
if (stream_data.current_buffer && stream_data.frame_ready) {
if (stream_data.current_buffer) {
struct spa_buffer *buf = stream_data.current_buffer->buffer;

if (buf->datas[0].chunk->size != 0) {
const auto img_descriptor = static_cast<egl::img_descriptor_t *>(img);
img_descriptor->frame_timestamp = std::chrono::steady_clock::now();

// Passthrough PipeWire metadata
// PipeWire header metadata
struct spa_meta_header *h = static_cast<struct spa_meta_header *>(
spa_buffer_find_meta_data(buf, SPA_META_Header, sizeof(*h))
);
Expand All @@ -859,6 +875,23 @@ namespace portal {
img_descriptor->pts = h->pts;
}

// PipeWire flags
if (buf->n_datas > 0) {
img_descriptor->pw_flags = buf->datas[0].chunk->flags;
}

// PipeWire damage metadata
struct spa_meta_region *damage = (struct spa_meta_region *) spa_buffer_find_meta_data(
stream_data.current_buffer->buffer,
SPA_META_VideoDamage,
sizeof(*damage)
);
if (damage) {
img_descriptor->pw_damage = (damage->region.size.width > 0 && damage->region.size.height > 0);
} else {
img_descriptor->pw_damage = std::nullopt;
}

if (buf->datas[0].type == SPA_DATA_DmaBuf) {
img_descriptor->sd.width = stream_data.format.info.raw.size.width;
img_descriptor->sd.height = stream_data.format.info.raw.size.height;
Expand All @@ -874,10 +907,6 @@ namespace portal {
// Point the encoder to the front buffer
img->data = stream_data.front_buffer->data();
img->row_pitch = stream_data.local_stride;

// Reset flags
stream_data.frame_ready = false;
stream_data.current_buffer = nullptr;
}
}
} else {
Expand Down Expand Up @@ -1088,7 +1117,7 @@ namespace portal {

// Ack the buffer type and metadata
std::array<uint8_t, SPA_POD_BUFFER_SIZE> buffer;
std::array<const struct spa_pod *, 2> params;
std::array<const struct spa_pod *, 3> params;
int n_params = 0;
struct spa_pod_builder pod_builder = SPA_POD_BUILDER_INIT(buffer.data(), buffer.size());
auto buffer_param = static_cast<const struct spa_pod *>(spa_pod_builder_add_object(&pod_builder, SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers, SPA_PARAM_BUFFERS_dataType, SPA_POD_Int(buffer_types)));
Expand All @@ -1097,6 +1126,10 @@ namespace portal {
auto meta_param = static_cast<const struct spa_pod *>(spa_pod_builder_add_object(&pod_builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, SPA_POD_Id(SPA_META_Header), SPA_PARAM_META_size, SPA_POD_Int(sizeof(struct spa_meta_header))));
params[n_params] = meta_param;
n_params++;
int videoDamageRegionCount = 16;
auto damage_param = static_cast<const struct spa_pod *>(spa_pod_builder_add_object(&pod_builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, SPA_POD_Id(SPA_META_VideoDamage), SPA_PARAM_META_size, SPA_POD_CHOICE_RANGE_Int(sizeof(struct spa_meta_region) * videoDamageRegionCount, sizeof(struct spa_meta_region) * 1, sizeof(struct spa_meta_region) * videoDamageRegionCount)));
params[n_params] = damage_param;
n_params++;

pw_stream_update_params(d->stream, params.data(), n_params);
}
Expand Down Expand Up @@ -1185,9 +1218,17 @@ namespace portal {

platf::capture_e snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool show_cursor) {
// FIXME: show_cursor is ignored
auto start_time = std::chrono::steady_clock::now();
auto deadline = std::chrono::steady_clock::now() + timeout;
int retries = 0;

while (true) {
if (!wait_for_frame(deadline)) {
if (stream_stopped.load()) {
return platf::capture_e::interrupted;
}
return platf::capture_e::timeout;
}

if (!pull_free_image_cb(img_out)) {
return platf::capture_e::interrupted;
}
Expand All @@ -1197,35 +1238,27 @@ namespace portal {
pipewire.fill_img(img_egl);

// Check if we got valid data (either DMA-BUF fd or memory pointer)
bool is_valid_data = (img_egl->sd.fds[0] >= 0 || img_egl->data != nullptr);

// Duplicate detection: PipeWire seq increments on each new frame,
// pts advances with each buffer update. Both must advance to accept frame.
bool is_duplicate = (img_egl->seq.has_value() && img_egl->pts.has_value() && last_pts.has_value() && last_seq.has_value() && img_egl->pts.value() == last_pts.value() && img_egl->seq.value() == last_seq.value());

if (is_valid_data && !is_duplicate) {
// Frame found; check deadline
auto end_time = std::chrono::steady_clock::now();
if (end_time - start_time > timeout) {
bool is_valid = (img_egl->sd.fds[0] >= 0 || img_egl->data != nullptr);
// Check for duplicates
bool is_dup = last_pts.has_value() && is_buffer_redundant(img_egl, last_pts);

// Valid & non-duplicate frame
if (is_valid && !is_dup) {
// Check deadline
if (std::chrono::steady_clock::now() >= deadline) {
return platf::capture_e::timeout;
}

if (img_egl->seq.has_value() && img_egl->pts.has_value()) {
last_seq = img_egl->seq.value();
last_pts = img_egl->pts.value();
}
img_egl->sequence = ++sequence;
// Update frame metadata
update_frame_metadata(img_egl, retries);
return platf::capture_e::ok;
}

// No valid frame yet, or it was a duplicate
auto now = std::chrono::steady_clock::now();
if (now - start_time >= timeout) {
retries++;
if (std::chrono::steady_clock::now() >= deadline) {
return platf::capture_e::timeout;
}

std::unique_lock lock(pipewire.frame_mutex());
pipewire.frame_cv().wait_until(lock, start_time + timeout);
}
}

Expand Down Expand Up @@ -1267,6 +1300,8 @@ namespace portal {
stream_stopped.store(false);
previous_height.store(0);
previous_width.store(0);
// Delay interrupt signal to give Portal time to detect change
std::this_thread::sleep_for(std::chrono::milliseconds(1500));
return platf::capture_e::interrupted;
} else {
BOOST_LOG(warning) << "PipeWire stream disconnected. Forcing session reset."sv;
Expand Down Expand Up @@ -1342,6 +1377,92 @@ namespace portal {
}

private:
static bool is_buffer_redundant(const egl::img_descriptor_t *img, const std::optional<uint64_t> &last_pts) {
// Check for corrupted frame
if (img->pw_flags.has_value()) {
if (img->pw_flags.value() & SPA_CHUNK_FLAG_CORRUPTED) {
return true;
}
}

// Damage & PTS checks
if (img->pts.has_value() && last_pts.has_value()) {
int64_t delta = (int64_t) img->pts.value() - (int64_t) last_pts.value();

// PTS hasn't advanced:
if (delta <= 0) {
// Case A: Compositor says "No Damage"
if (img->pw_damage.has_value() && !img->pw_damage.value()) {
BOOST_LOG(debug) << "Dropping frame: PTS delta 0 and Damage metadata confirms NO pixels changed."sv;
return true;
}
// Case B: No damage metadata available, but delta is 0
if (!img->pw_damage.has_value()) {
return true;
}
}
}

return false;
}

void update_frame_metadata(egl::img_descriptor_t *img_egl, int retries) {
if (img_egl->seq.has_value() && img_egl->pts.has_value()) {
last_seq = img_egl->seq.value();
last_pts = img_egl->pts.value();

// Snapshot both clock domains simultaneously
Copy link
Contributor

@andygrundman andygrundman Feb 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is all this time code doing? If the frame is a duplicate, frame_timestamp should be left null. It gets filled in by code around stream.cpp:1471.

Copy link
Contributor Author

@psyke83 psyke83 Feb 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

update_frame_metadata() is only called on frames that pass a duplicate and snapshot timeout check. Previously, portalgrab's snapshot timeout was a no-op, but reactive capture (and the updates in this PR) make it actually block for new data, so it was only timing out for extreme cases where no data was being received.

The timeout is 1000ms, so only frames that are extremely late will hit the timeout.

const uint64_t pw_now_ns = pipewire.stream_get_nsec();
const auto steady_now = std::chrono::steady_clock::now();

// Calculate age (positive = past, negative = future)
const int64_t age_ns = static_cast<int64_t>(pw_now_ns) - static_cast<int64_t>(last_pts.value());

// Sanity check: max age is (largest of) 5 frames or 100ms
const int64_t max_age_ns = std::max<int64_t>(
std::chrono::nanoseconds {100ms}.count(),
std::chrono::duration_cast<std::chrono::nanoseconds>(delay * 5).count()
);
// Adaptive future tolerance: half of the target frame interval
const int64_t future_tolerance_ns = -std::chrono::duration_cast<std::chrono::nanoseconds>(delay / 2).count();

// Anchor the timestamp
if (age_ns >= future_tolerance_ns && age_ns < max_age_ns) {
// FRESH FRAME: Apply hardware-measured age to the steady_clock anchor
img_egl->frame_timestamp = steady_now - std::chrono::nanoseconds(age_ns);
} else {
// STALE/FUTURE: Fallback to current time
img_egl->frame_timestamp = steady_now;

if (age_ns < future_tolerance_ns) {
BOOST_LOG(debug) << "Frame age was future dated: " << (age_ns / 1000) << "us";
Copy link
Contributor

@andygrundman andygrundman Feb 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know this code will be removed but just FYI, use verbose log level for anything that logs very frequently or in a loop, etc.

Copy link
Contributor Author

@psyke83 psyke83 Feb 27, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good advice, but I suspect any logging level might be too chatty to be included in a final PR state. Right now, the change is not appropriate as there is just way too much compositor variance to be confident that this is working as expected, so either it needs some changes to the logic, or might just be better to drop it entirely. More eyes on the code might help figure out if the former is possible.

In the case of KWin (6.6.1), stale PTS data explodes the host latency when returning from idle - that necessitates the positive (past) age check. In the case of mutter, I don't think that frames returning from idle have stale pts timestamps, but it has the opposite issue where timestamps end up consistently future dated sometimes by as high as -2ms for an active stream running at 60fps.

Copy link
Contributor

@andygrundman andygrundman Feb 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, if we can't trust pipewire it's probably fine to just use now().

} else if (age_ns > max_age_ns && age_ns < 1'000'000'000) {
BOOST_LOG(debug) << "Frame age exceeded threshold: " << (age_ns / 1000) << "us";
}
}
}

img_egl->sequence = ++sequence;

if (retries > 0) {
BOOST_LOG(debug) << "Processed frame after " << retries << " redundant events."sv;
}
}

bool wait_for_frame(std::chrono::steady_clock::time_point deadline) {
std::unique_lock<std::mutex> lock(pipewire.frame_mutex());

bool success = pipewire.frame_cv().wait_until(lock, deadline, [&] {
return pipewire.is_frame_ready() || stream_stopped.load();
});

if (success && !stream_stopped.load()) {
pipewire.set_frame_ready(false);
return true;
}
return false;
}

static uint32_t lookup_pw_format(uint64_t fourcc) {
for (const auto &fmt : format_map) {
if (fmt.fourcc == 0) {
Expand Down