feat: add support for multi-step convsion of media files, allowing a user to specify an external application to use for converting a media file, using arbitrary arguments.

Author: nytamin

apps/package-manager/packages/generic/src/generateExpectations/nrk/expectations-lib.ts

@@ -34,7 +34,7 @@ export function generateMediaFileCopy(
 	managerId: ExpectationManagerId,
 	expWrap: ExpectedPackageWrap,
 	settings: PackageManagerSettings
-): Expectation.FileCopy {
+): Expectation.FileCopy | Expectation.MediaFileConvert {
 	const expWrapMediaFile = expWrap as ExpectedPackageWrapMediaFile
 
 	const endRequirement: Expectation.FileCopy['endRequirement'] = {
@@ -78,7 +78,38 @@ export function generateMediaFileCopy(
 		},
 	}
 
-	return exp
+	if (
+		expWrapMediaFile.expectedPackage.version.conversions &&
+		expWrapMediaFile.expectedPackage.version.conversions.length > 0
+	) {
+		// Return a MediaFileConvert expectation:
+
+		const convertExp: Expectation.MediaFileConvert = {
+			...exp,
+			type: Expectation.Type.MEDIA_FILE_CONVERT,
+			statusReport: {
+				...exp.statusReport,
+				label: exp.statusReport.label.replace('Copying media', 'Copy and converting media'),
+				description: exp.statusReport.description.replace('Copy media', 'Copy and convert media'),
+			},
+			startRequirement: {
+				sources: expWrapMediaFile.sources,
+			},
+			endRequirement: {
+				targets: endRequirement.targets,
+				content: endRequirement.content,
+				version: {
+					type: Expectation.Version.Type.MEDIA_FILE_CONVERT,
+					conversions: expWrapMediaFile.expectedPackage.version.conversions,
+				},
+			},
+		}
+
+		return convertExp
+	} else {
+		// Just return the copy expectation:
+		return exp
+	}
 }
 export function generateMediaFileVerify(
 	managerId: ExpectationManagerId,

shared/packages/api/src/expectationApi.ts

@@ -20,6 +20,7 @@ export namespace Expectation {
 		| PackageIframesScan
 		| MediaFileThumbnail
 		| MediaFilePreview
+		| MediaFileConvert
 		| QuantelClipCopy
 		// | QuantelClipScan
 		// | QuantelClipDeepScan
@@ -35,6 +36,7 @@ export namespace Expectation {
 		FILE_COPY_PROXY = 'file_copy_proxy',
 		MEDIA_FILE_THUMBNAIL = 'media_file_thumbnail',
 		MEDIA_FILE_PREVIEW = 'media_file_preview',
+		MEDIA_FILE_CONVERT = 'media_file_convert',
 		FILE_VERIFY = 'file_verify',
 		RENDER_HTML = 'render_html',
 
@@ -270,6 +272,22 @@ export namespace Expectation {
 		}
 		workOptions: WorkOptions.Base & WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath
 	}
+	/** Defines a Conversion of a Media file. One of the the sources are piped through an external executable and the resulting file is to be stored on the target. */
+	export interface MediaFileConvert extends Base {
+		type: Type.MEDIA_FILE_CONVERT
+
+		startRequirement: {
+			sources: SpecificPackageContainerOnPackage.FileSource[]
+		}
+		endRequirement: {
+			targets: SpecificPackageContainerOnPackage.FileTarget[]
+			content: {
+				filePath: string
+			}
+			version: Version.ExpectedMediaFileConvert
+		}
+		workOptions: WorkOptions.Base & WorkOptions.RemoveDelay & WorkOptions.UseTemporaryFilePath
+	}
 
 	/** Defines a Quantel clip. A Quantel clip is to be copied from one of the Sources, to the Target. */
 	export interface QuantelClipCopy extends Base {
@@ -508,6 +526,7 @@ export namespace Expectation {
 
 	// eslint-disable-next-line @typescript-eslint/no-namespace
 	export namespace WorkOptions {
+		export type All = Base & RemoveDelay & UseTemporaryFilePath
 		export interface Base {
 			/** If set, a worker might decide to wait with this expectation until the CPU load is lower. */
 			allowWaitForCPU?: boolean
@@ -534,6 +553,7 @@ export namespace Expectation {
 		export type ExpectAny =
 			| ExpectedFileOnDisk
 			| MediaFileThumbnail
+			| MediaFileConvert
 			| ExpectedCorePackageInfo
 			| ExpectedHTTPFile
 			| ExpectedQuantelClip
@@ -543,6 +563,7 @@ export namespace Expectation {
 		export type Any =
 			| FileOnDisk
 			| MediaFileThumbnail
+			| MediaFileConvert
 			| CorePackageInfo
 			| HTTPFile
 			| QuantelClip
@@ -554,6 +575,7 @@ export namespace Expectation {
 		}
 		export enum Type {
 			FILE_ON_DISK = 'file_on_disk',
+			MEDIA_FILE_CONVERT = 'media_file_convert',
 			MEDIA_FILE_THUMBNAIL = 'media_file_thumbnail',
 			MEDIA_FILE_PREVIEW = 'media_file_preview',
 			CORE_PACKAGE_INFO = 'core_package_info',
@@ -590,6 +612,48 @@ export namespace Expectation {
 		}
 		export type ExpectedMediaFileThumbnail = ExpectedType<MediaFileThumbnail>
 
+		export interface MediaFileConvert extends Base {
+			type: Type.MEDIA_FILE_CONVERT
+
+			/**
+			 * List of conversion steps to perform, in order.
+			 */
+			conversions: ConversionStep[]
+		}
+		export interface ConversionStep {
+			/**
+			 * Path to the executable.
+			 * Note: If this ends with '.exe', but runs on a non-Windows system, the '.exe' will be removed.
+			 */
+			executable: string
+			/**
+			 * Arguments to the executable.
+			 * Supported placeholders:
+			 * - {SOURCE} - replaced with the full path of the source file
+			 * - {TARGET} - replaced with the full path of the target file
+			 */
+			args: string[]
+
+			/**
+			 * Set to true if the executable needs the source to be locally available
+			 * (So PM will copy the source to a local temp folder before running the executable)
+			 */
+			needsLocalSource?: boolean
+			/**
+			 * Set to true if the executable needs the target to be locally available
+			 * (So PM will create the target in a local temp folder, and then copy it to the actual target when done)
+			 */
+			needsLocalTarget?: boolean
+
+			/**
+			 * Force the output filename from this step.
+			 * This can be useful in multi-step scenarios where you want to specify the inter-step filename.
+			 * This property is ignored in the final step.
+			 */
+			outputFileName?: string
+		}
+		export type ExpectedMediaFileConvert = MediaFileConvert // ExpectedType<MediaFileConvert>
+
 		export interface MediaFilePreview extends Base {
 			type: Type.MEDIA_FILE_PREVIEW
 			bitrate: string // default: '40k'

shared/packages/api/src/inputApi.ts

@@ -148,6 +148,38 @@ export namespace ExpectedPackage {
 			modifiedDate?: number // timestamp (ms)
 			checksum?: string
 			checkSumType?: 'sha' | 'md5' | 'whatever'
+			conversions?: {
+				/**
+				 * Path to the executable.
+				 * Note: If this ends with '.exe', but runs on a non-Windows system, the '.exe' will be removed.
+				 */
+				executable: string
+				/**
+				 * Arguments to the executable.
+				 * Supported placeholders:
+				 * - {SOURCE} - replaced with the full path of the source file
+				 * - {TARGET} - replaced with the full path of the target file
+				 */
+				args: string[]
+
+				/**
+				 * Set to true if the executable needs the source to be locally available
+				 * (So PM will copy the source to a local temp folder before running the executable)
+				 */
+				needsLocalSource?: boolean
+				/**
+				 * Set to true if the executable needs the target to be locally available
+				 * (So PM will create the target in a local temp folder, and then copy it to the actual target when done)
+				 */
+				needsLocalTarget?: boolean
+
+				/**
+				 * Force the output filename from this step.
+				 * This can be useful in multi-step scenarios where you want to specify the inter-step filename.
+				 * This property is ignored in the final step.
+				 */
+				outputFileName?: string
+			}[]
 		}
 		sources: {
 			containerId: PackageContainerId

shared/packages/worker/src/worker/workers/genericWorker/expectationHandlers/lib.ts

@@ -153,7 +153,7 @@ export async function lookupAccessorHandles<Metadata>(
 	combineWithPackageContainers: PackageContainerOnPackage[],
 	accessorContext: AccessorContext,
 	expectationContent: unknown,
-	expectationWorkOptions: unknown,
+	expectationWorkOptions: Expectation.WorkOptions.All,
 	checks: LookupChecks
 ): Promise<LookupPackageContainer<Metadata>> {
 	const prioritizedAccessors = prioritizeAccessors(packageContainers)

shared/packages/worker/src/worker/workers/genericWorker/expectationHandlers/lib/ffmpeg.ts

@@ -117,55 +117,20 @@ export async function spawnFFMpeg<Metadata>(
 	}
 	const lastFewLines: string[] = []
 
-	let fileDuration: number | undefined = undefined
 	ffMpegProcess.stderr.on('data', (data) => {
 		const str = data.toString()
 
 		log?.('ffmpeg:' + str)
 
-		// Duration is reported by FFMpeg at the beginning, this means that it successfully opened the source
-		// stream and has begun processing
-		const m = str.match(/Duration:\s?(\d+):(\d+):([\d.]+)/)
-		if (m) {
-			const hh = m[1]
-			const mm = m[2]
-			const ss = m[3]
-
-			fileDuration = parseInt(hh, 10) * 3600 + parseInt(mm, 10) * 60 + parseFloat(ss)
-
-			// Report back an initial status, because it looks nice:
-			// workInProgress._reportProgress(actualSourceVersionHash, 0)
-			onProgress?.(0).catch((err) => log?.(`spawnFFMpeg onProgress update failed: ${stringifyError(err)}`))
-
-			return
-		}
-		if (fileDuration) {
-			// Time position in source is reported periodically, but we need fileDuration to convert that into
-			// percentages
-			const m2 = str.match(/time=\s?(\d+):(\d+):([\d.]+)/)
-			if (m2) {
-				const hh = m2[1]
-				const mm = m2[2]
-				const ss = m2[3]
-
-				const progress = parseInt(hh, 10) * 3600 + parseInt(mm, 10) * 60 + parseFloat(ss)
-				// workInProgress._reportProgress(
-				// 	actualSourceVersionHash,
-				// 	((uploadIsDone ? 1 : 0.9) * progress) / fileDuration
-				// )
-				onProgress?.(((uploadIsDone ? 1 : 0.9) * progress) / fileDuration).catch((err) =>
-					log?.(`spawnFFMpeg onProgress update failed: ${stringifyError(err)}`)
-				)
-				return
-			}
-		}
-
 		lastFewLines.push(str)
 
 		if (lastFewLines.length > 10) {
 			lastFewLines.shift()
 		}
 	})
+	ffmpegInterpretProgress(ffMpegProcess, (progress) => {
+		onProgress?.(progress).catch((err) => log?.(`spawnFFMpeg onProgress update failed: ${stringifyError(err)}`))
+	})
 	const onClose = (code: number | null) => {
 		if (ffMpegProcess) {
 			log?.('ffmpeg: close ' + code)
@@ -199,3 +164,41 @@ export async function spawnFFMpeg<Metadata>(
 		},
 	}
 }
+
+export function ffmpegInterpretProgress(
+	ffMpegProcess: ChildProcessWithoutNullStreams,
+	cbProgress: (progress: number) => void
+): void {
+	let fileDuration: number | undefined = undefined
+	ffMpegProcess.stderr.on('data', (data) => {
+		const str = data.toString()
+
+		// Duration is reported by FFMpeg at the beginning, this means that it successfully opened the source
+		// stream and has begun processing
+		const m = str.match(/Duration:\s?(\d+):(\d+):([\d.]+)/)
+		if (m) {
+			const hh = m[1]
+			const mm = m[2]
+			const ss = m[3]
+
+			fileDuration = parseInt(hh, 10) * 3600 + parseInt(mm, 10) * 60 + parseFloat(ss)
+
+			return
+		}
+		if (fileDuration) {
+			// Time position in source is reported periodically, but we need fileDuration to convert that into
+			// percentages
+			const m2 = str.match(/time=\s?(\d+):(\d+):([\d.]+)/)
+			if (m2) {
+				const hh = m2[1]
+				const mm = m2[2]
+				const ss = m2[3]
+
+				const progress = parseInt(hh, 10) * 3600 + parseInt(mm, 10) * 60 + parseFloat(ss)
+
+				cbProgress(progress / fileDuration)
+				return
+			}
+		}
+	})
+}

shared/packages/worker/src/worker/workers/genericWorker/expectationHandlers/lib/file.ts

@@ -103,7 +103,8 @@ export async function isFileReadyToStartWorkingOn(
 export async function isFileFulfilled(
 	_worker: BaseWorker,
 	lookupSource: LookupPackageContainer<UniversalVersion>,
-	lookupTarget: LookupPackageContainer<UniversalVersion>
+	lookupTarget: LookupPackageContainer<UniversalVersion>,
+	modifySourceUVersion?: (version: UniversalVersion) => UniversalVersion
 ): Promise<ReturnTypeIsExpectationFulfilled> {
 	if (!lookupTarget.ready)
 		return {
@@ -140,8 +141,11 @@ export async function isFileFulfilled(
 		return { fulfilled: false, knownReason: lookupSource.knownReason, reason: lookupSource.reason }
 
 	const actualSourceVersion = await lookupSource.handle.getPackageActualVersion()
+	let actualSourceUVersion = makeUniversalVersion(actualSourceVersion)
+
+	if (modifySourceUVersion) actualSourceUVersion = modifySourceUVersion(actualSourceUVersion)
 
-	const issueVersions = compareUniversalVersions(makeUniversalVersion(actualSourceVersion), actualTargetVersion)
+	const issueVersions = compareUniversalVersions(actualSourceUVersion, actualTargetVersion)
 	if (!issueVersions.success) {
 		return { fulfilled: false, knownReason: issueVersions.knownReason, reason: issueVersions.reason }
 	}
@@ -152,7 +156,7 @@ export async function isFileFulfilled(
 	}
 }
 export async function doFileCopyExpectation(
-	exp: Expectation.FileCopy | Expectation.FileCopyProxy,
+	exp: Expectation.FileCopy | Expectation.FileCopyProxy | Expectation.MediaFileConvert,
 	lookupSource: LookupPackageContainer<UniversalVersion>,
 	lookupTarget: LookupPackageContainer<UniversalVersion>
 ): Promise<WorkInProgress | null> {