Try to get a better timestamp match via libobs composition timestamps

Author: palana

src/transcription-filter-callbacks.cpp

@@ -255,14 +255,24 @@ void send_caption_to_webvtt(uint64_t possible_end_ts_ms, DetectionResultWithText
 			if (!muxer)
 				continue;
 
+			auto &anchor = output.time_anchors[i];
+			if (!anchor.anchor)
+				return;
+
+			auto output_start_ts = anchor.anchor->composition_timestamp / 1'000'000;
+			if (possible_end_ts_ms < output_start_ts)
+				return;
+
 			auto duration = result.end_timestamp_ms - result.start_timestamp_ms;
-			auto segment_start_ts = possible_end_ts_ms - duration;
-			if (segment_start_ts < output.start_timestamp_ms) {
-				duration -= output.start_timestamp_ms - segment_start_ts;
-				segment_start_ts = output.start_timestamp_ms;
+			auto segment_start_ts = possible_end_ts_ms / 1'000'000 - duration;
+
+			if (segment_start_ts < output_start_ts) {
+				duration -= output_start_ts - segment_start_ts;
+				segment_start_ts = output_start_ts;
 			}
+
 			webvtt_muxer_add_cue(muxer.get(), lang_to_track->second,
-					     segment_start_ts - output.start_timestamp_ms, duration,
+					     (segment_start_ts - output_start_ts), duration,
 					     str_copy.c_str());
 		}
 	}
@@ -500,8 +510,50 @@ void output_packet_added_callback(obs_output_t *output, struct encoder_packet *p
 	if (!muxer)
 		return;
 
+	auto &time_anchor = it->time_anchors[pkt->track_idx];
+	if (!time_anchor.anchor) {
+		// CTS can repeat if there are
+		// 1) lagged frames (composition thread wasn't fast enough)
+		// 2) duplicated frames (encoder couldn't keep up)
+		// this is trying to find a frame that is neither lagged nor duplicated, to ensure a stable mapping from composition time to PTS
+		auto new_end = std::remove_if(
+			time_anchor.last_two_if_not_initialized.begin(),
+			time_anchor.last_two_if_not_initialized.end(),
+			[&](auto &val) { return val.composition_timestamp == pkt_time->cts; });
+		if (new_end != time_anchor.last_two_if_not_initialized.end())
+			time_anchor.last_two_if_not_initialized.erase(
+				new_end, time_anchor.last_two_if_not_initialized.end());
+
+		if (time_anchor.last_two_if_not_initialized.size() == 2) {
+			time_anchor.anchor = time_anchor.last_two_if_not_initialized.back();
+			time_anchor.last_two_if_not_initialized.clear();
+		} else {
+			time_anchor.last_two_if_not_initialized.push_back({
+				pkt->pts,
+				pkt_time->cts,
+			});
+		}
+	}
+
+	auto encoder = obs_output_get_video_encoder2(output, pkt->track_idx);
+	if (!encoder)
+		return;
+
+	auto video = obs_encoder_video(encoder);
+	auto voi = video_output_get_info(video);
+	if (!voi)
+		return;
+
+	uint64_t packet_absolute_timestamp = 0;
+	// time for subtitles only starts progressing once we have an anchor point
+	if (time_anchor.anchor && time_anchor.anchor->pts <= pkt->pts) {
+		packet_absolute_timestamp =
+			util_mul_div64(1000000000ULL, voi->fps_den, voi->fps_num) *
+			(pkt->pts - time_anchor.anchor->pts);
+	}
+
 	std::unique_ptr<WebvttBuffer, webvtt_buffer_deleter> buffer{
-		webvtt_muxer_try_mux_into_bytestream(muxer.get(), pkt_time->cts, pkt->keyframe,
+		webvtt_muxer_try_mux_into_bytestream(muxer.get(), packet_absolute_timestamp, pkt->keyframe,
 						     it->codec_flavor[pkt->track_idx])};
 
 	if (!buffer)
@@ -547,7 +599,6 @@ void add_webvtt_output(transcription_filter_data &gf, obs_output_t *output,
 	auto &entry = gf.active_outputs.back();
 	entry.output = obs_output_get_weak_output(output);
 	entry.output_type = output_type;
-	entry.start_timestamp_ms = start_ms;
 	obs_output_add_packet_callback_(output, output_packet_added_callback, &gf);
 }
 

src/transcription-filter-data.h

@@ -212,15 +212,25 @@ struct transcription_filter_data {
 		Recording,
 	};
 
+	struct webvtt_to_video_timestamp {
+		int64_t pts;
+		uint64_t composition_timestamp;
+	};
+
+	struct webvtt_video_time_anchor {
+		std::deque<webvtt_to_video_timestamp> last_two_if_not_initialized;
+		std::optional<webvtt_to_video_timestamp> anchor;
+	};
+
 	struct webvtt_output {
 		OBSWeakOutputAutoRelease output;
 		webvtt_output_type output_type;
-		uint64_t start_timestamp_ms;
 
 		bool initialized = false;
 		std::map<std::string, uint8_t> language_to_track;
 		std::unique_ptr<WebvttMuxer, webvtt_muxer_deleter>
 			webvtt_muxer[MAX_OUTPUT_VIDEO_ENCODERS];
+		webvtt_video_time_anchor time_anchors[MAX_OUTPUT_VIDEO_ENCODERS];
 		CodecFlavor codec_flavor[MAX_OUTPUT_VIDEO_ENCODERS] = {};
 	};
 

src/whisper-utils/whisper-processing.cpp

@@ -2,6 +2,7 @@
 
 #include <obs-module.h>
 
+#include <util/platform.h>
 #include <util/profiler.hpp>
 
 #include "plugin-support.h"
@@ -341,7 +342,7 @@ void run_inference_and_callbacks(transcription_filter_data *gf, uint64_t start_o
 				    pcm32f_size * sizeof(float));
 	}
 
-	auto inference_start_ts = now_ms();
+	auto inference_start_ts = os_gettime_ns();
 
 	struct DetectionResultWithText inference_result =
 		run_whisper_inference(gf, pcm32f_data, pcm32f_size_with_silence, start_offset_ms,