fix: non-monotonic operation order in AdLib execution context

packages/corelib/src/dataModel/PieceInstance.ts

@@ -61,15 +61,10 @@ export interface PieceInstance {
 	dynamicallyInserted?: Time
 
 	/** This is set when the duration needs to be overriden from some user action */
-	userDuration?:
-		| {
-				/** The time relative to the part (milliseconds since start of part) */
-				endRelativeToPart: number
-		  }
-		| {
-				/** The time relative to 'now' (ms since 'now') */
-				endRelativeToNow: number
-		  }
+	userDuration?: {
+		/** The time relative to the part (milliseconds since start of part) */
+		endRelativeToPart: number
+	}
 
 	/** The time the system started playback of this part, undefined if not yet played back (milliseconds since epoch) */
 	reportedStartedPlayback?: Time

packages/corelib/src/playout/__tests__/processAndPrune.test.ts

@@ -600,34 +600,6 @@ describe('resolvePrunedPieceInstances', () => {
 		} satisfies ResolvedPieceInstance)
 	})
 
-	test('numeric start, with userDuration.endRelativeToNow', async () => {
-		const nowInPart = 123
-		const piece = createPieceInstance({ start: 500 }, undefined, {
-			endRelativeToNow: 4000,
-		})
-
-		expect(resolvePrunedPieceInstance(nowInPart, clone(piece))).toStrictEqual({
-			instance: clone(piece),
-			timelinePriority: piece.priority,
-			resolvedStart: 500,
-			resolvedDuration: 4000 - 500 + nowInPart,
-		} satisfies ResolvedPieceInstance)
-	})
-
-	test('now start, with userDuration.endRelativeToNow', async () => {
-		const nowInPart = 123
-		const piece = createPieceInstance({ start: 'now' }, undefined, {
-			endRelativeToNow: 4000,
-		})
-
-		expect(resolvePrunedPieceInstance(nowInPart, clone(piece))).toStrictEqual({
-			instance: clone(piece),
-			timelinePriority: piece.priority,
-			resolvedStart: nowInPart,
-			resolvedDuration: 4000,
-		} satisfies ResolvedPieceInstance)
-	})
-
 	test('now start, with end cap, planned duration and userDuration.endRelativeToPart', async () => {
 		const nowInPart = 123
 		const piece = createPieceInstance({ start: 'now', duration: 3000 }, 5000, { endRelativeToPart: 2800 })
@@ -639,16 +611,4 @@ describe('resolvePrunedPieceInstances', () => {
 			resolvedDuration: 2800 - nowInPart,
 		} satisfies ResolvedPieceInstance)
 	})
-
-	test('now start, with end cap, planned duration and userDuration.endRelativeToNow', async () => {
-		const nowInPart = 123
-		const piece = createPieceInstance({ start: 'now', duration: 3000 }, 5000, { endRelativeToNow: 2800 })
-
-		expect(resolvePrunedPieceInstance(nowInPart, clone(piece))).toStrictEqual({
-			instance: clone(piece),
-			timelinePriority: piece.priority,
-			resolvedStart: nowInPart,
-			resolvedDuration: 2800,
-		} satisfies ResolvedPieceInstance)
-	})
 })

packages/corelib/src/playout/processAndPrune.ts

@@ -247,11 +247,7 @@ export function resolvePrunedPieceInstance(
 
 	// Consider the playout userDuration
 	if (pieceInstance.userDuration) {
-		if ('endRelativeToPart' in pieceInstance.userDuration) {
-			caps.push(pieceInstance.userDuration.endRelativeToPart - resolvedStart)
-		} else if ('endRelativeToNow' in pieceInstance.userDuration) {
-			caps.push(nowInPart + pieceInstance.userDuration.endRelativeToNow - resolvedStart)
-		}
+		caps.push(pieceInstance.userDuration.endRelativeToPart - resolvedStart)
 	}
 
 	return {

packages/job-worker/src/playout/__tests__/resolvedPieces.test.ts

@@ -295,32 +295,6 @@ describe('Resolved Pieces', () => {
 			] satisfies StrippedResult)
 		})
 
-		test('userDuration.endRelativeToNow', async () => {
-			const sourceLayerId = Object.keys(sourceLayers)[0]
-			expect(sourceLayerId).toBeTruthy()
-
-			const piece0 = createPieceInstance(
-				sourceLayerId,
-				{ start: 1000 },
-				{},
-				{
-					userDuration: {
-						endRelativeToNow: 2000,
-					},
-				}
-			)
-
-			const resolvedPieces = getResolvedPiecesInner(sourceLayers, 2500, [piece0])
-
-			expect(stripResult(resolvedPieces)).toEqual([
-				{
-					_id: piece0._id,
-					resolvedStart: 1000,
-					resolvedDuration: 3500,
-				},
-			] satisfies StrippedResult)
-		})
-
 		test('preroll has no effect', async () => {
 			const sourceLayerId = Object.keys(sourceLayers)[0]
 			expect(sourceLayerId).toBeTruthy()
@@ -645,41 +619,6 @@ describe('Resolved Pieces', () => {
 			] satisfies StrippedResult)
 		})
 
-		test('userDuration.endRelativeToNow', async () => {
-			const sourceLayerId = Object.keys(sourceLayers)[0]
-			expect(sourceLayerId).toBeTruthy()
-
-			const piece001 = createPieceInstance(
-				sourceLayerId,
-				{ start: 4000 },
-				{},
-				{
-					userDuration: {
-						endRelativeToNow: 1300,
-					},
-				}
-			)
-
-			const now = 990000
-			const nowInPart = 7000
-			const partStarted = now - nowInPart
-
-			const currentPartInfo = createPartInstanceInfo(partStarted, nowInPart, createPartInstance(), [piece001])
-
-			const simpleResolvedPieces = getResolvedPiecesForPartInstancesOnTimeline(
-				context,
-				{ current: currentPartInfo },
-				now
-			)
-			expect(stripResult(simpleResolvedPieces)).toEqual([
-				{
-					_id: piece001._id,
-					resolvedStart: partStarted + 4000,
-					resolvedDuration: -4000 + 7000 + 1300,
-				},
-			] satisfies StrippedResult)
-		})
-
 		test('basic previousPart', async () => {
 			const sourceLayerId = Object.keys(sourceLayers)[0]
 			expect(sourceLayerId).toBeTruthy()
@@ -809,7 +748,7 @@ describe('Resolved Pieces', () => {
 				},
 				{
 					userDuration: {
-						endRelativeToNow: 3400,
+						endRelativeToPart: 5400,
 					},
 				}
 			)

packages/job-worker/src/playout/adlibUtils.ts

@@ -21,7 +21,6 @@ import { PieceLifespan } from '@sofie-automation/blueprints-integration'
 import { SourceLayers } from '@sofie-automation/corelib/dist/dataModel/ShowStyleBase'
 import { updatePartInstanceRanksAfterAdlib } from '../updatePartInstanceRanksAndOrphanedState.js'
 import { setNextPart } from './setNext.js'
-import { calculateNowOffsetLatency } from './timeline/multi-gateway.js'
 import { logger } from '../logging.js'
 import { ReadonlyDeep } from 'type-fest'
 import { PlayoutRundownModel } from './model/PlayoutRundownModel.js'
@@ -279,8 +278,7 @@ export function innerStopPieces(
 	}
 
 	const resolvedPieces = getResolvedPiecesForCurrentPartInstance(context, sourceLayers, currentPartInstance)
-	const offsetRelativeToNow = (timeOffset || 0) + (calculateNowOffsetLatency(context, playoutModel) || 0)
-	const stopAt = getCurrentTime() + offsetRelativeToNow
+	const stopAt = playoutModel.getNowInPlayout() + (timeOffset ?? 0)
 	const relativeStopAt = stopAt - lastStartedPlayback
 
 	for (const resolvedPieceInstance of resolvedPieces) {
@@ -310,15 +308,9 @@ export function innerStopPieces(
 
 				const pieceInstanceModel = playoutModel.findPieceInstance(pieceInstance._id)
 				if (pieceInstanceModel) {
-					const newDuration: Required<PieceInstance>['userDuration'] = playoutModel.isMultiGatewayMode
-						? {
-								endRelativeToNow: offsetRelativeToNow,
-							}
-						: {
-								endRelativeToPart: relativeStopAt,
-							}
-
-					pieceInstanceModel.pieceInstance.setDuration(newDuration)
+					pieceInstanceModel.pieceInstance.setDuration({
+						endRelativeToPart: relativeStopAt,
+					})
 
 					stoppedInstances.push(pieceInstance._id)
 				} else {

packages/job-worker/src/playout/model/PlayoutModel.ts

@@ -32,6 +32,7 @@ import { PlayoutPieceInstanceModel } from './PlayoutPieceInstanceModel.js'
 import { PieceInstanceWithTimings } from '@sofie-automation/corelib/dist/playout/processAndPrune'
 import { PartCalculatedTimings } from '@sofie-automation/corelib/dist/playout/timings'
 import type { INotificationsModel } from '../../notifications/NotificationsModel.js'
+import { Time } from '@sofie-automation/blueprints-integration'
 
 export type DeferredFunction = (playoutModel: PlayoutModel) => void | Promise<void>
 export type DeferredAfterSaveFunction = (playoutModel: PlayoutModelReadonly) => void | Promise<void>
@@ -386,6 +387,13 @@ export interface PlayoutModel extends PlayoutModelReadonly, StudioPlayoutModelBa
 		toPieceInstances: PieceInstanceWithTimings[]
 	): PartCalculatedTimings
 
+	/**
+	 * Return an expected "now" value (i.e. the closest moment in time that can be safely addressed),
+	 * considering any playout latency. Every call will return a value greater or equal than previous,
+	 * meaning that this function is monotonic.
+	 */
+	getNowInPlayout(): Time
+
 	/** Lifecycle */
 
 	/**

packages/job-worker/src/playout/model/implementation/PlayoutModelImpl.ts

@@ -62,6 +62,7 @@ import { calculatePartTimings, PartCalculatedTimings } from '@sofie-automation/c
 import { PieceInstanceWithTimings } from '@sofie-automation/corelib/dist/playout/processAndPrune'
 import { stringifyError } from '@sofie-automation/shared-lib/dist/lib/stringifyError'
 import { NotificationsModelHelper } from '../../../notifications/NotificationsModelHelper.js'
+import { getExpectedLatency } from '@sofie-automation/corelib/dist/studio/playout'
 
 export class PlayoutModelReadonlyImpl implements PlayoutModelReadonly {
 	public readonly playlistId: RundownPlaylistId
@@ -264,6 +265,31 @@ export class PlayoutModelReadonlyImpl implements PlayoutModelReadonly {
 		}
 		return this.#isMultiGatewayMode
 	}
+
+	public get multiGatewayNowSafeLatency(): number | undefined {
+		return this.context.studio.settings.multiGatewayNowSafeLatency
+	}
+
+	/**
+	 * Calculate an offset to apply to the 'now' value, to compensate for delay in playout-gateway
+	 * The intention is that any concrete value used instead of 'now' should still be just in the future for playout-gateway
+	 */
+	protected getNowOffsetLatency(): number | undefined {
+		/** The timestamp that "now" was set to */
+		let nowOffsetLatency: number | undefined
+
+		if (this.isMultiGatewayMode) {
+			const playoutDevices = this.peripheralDevices.filter(
+				(device) => device.type === PeripheralDeviceType.PLAYOUT
+			)
+			const worstLatency = Math.max(0, ...playoutDevices.map((device) => getExpectedLatency(device).safe))
+			/** Add a little more latency, to account for network latency variability */
+			const ADD_SAFE_LATENCY = this.multiGatewayNowSafeLatency || 30
+			nowOffsetLatency = worstLatency + ADD_SAFE_LATENCY
+		}
+
+		return nowOffsetLatency
+	}
 }
 
 /**
@@ -859,6 +885,15 @@ export class PlayoutModelImpl extends PlayoutModelReadonlyImpl implements Playou
 		this.#playlistHasChanged = true
 	}
 
+	#lastMonotonicNowInPlayout = getCurrentTime()
+	getNowInPlayout(): number {
+		const nowOffsetLatency = this.getNowOffsetLatency() ?? 0
+		const targetNowTime = getCurrentTime() + nowOffsetLatency
+		const result = Math.max(this.#lastMonotonicNowInPlayout, targetNowTime)
+		this.#lastMonotonicNowInPlayout = result
+		return result
+	}
+
 	/** Notifications */
 
 	async getAllNotifications(